Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
19eefb66c6 | ||
|
|
081d9244d6 | ||
|
|
b55a38ca9f | ||
|
|
9006c82393 |
@@ -12,7 +12,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -704,97 +703,3 @@ func TestNothingIsAskedWhileTheBusLacksTheControllersGrant(t *testing.T) {
|
||||
t.Errorf("the undelivered condition was not cleared: %+v", last)
|
||||
}
|
||||
}
|
||||
|
||||
// hq issue 353, the review: the bus holds the controller's grant to ask only when both facts hold — the mesh
|
||||
// composes the controller's own grant, and the bus's module says it is in force on the running server — and
|
||||
// each fact not known is not held, said with why.
|
||||
func TestTheGrantIsHeldOnlyWhenComposedAndInForceOnTheBus(t *testing.T) {
|
||||
yes := func(context.Context) (bool, string, error) { return true, "", nil }
|
||||
no := func(why string) grantJudge {
|
||||
return func(context.Context) (bool, string, error) { return false, why, nil }
|
||||
}
|
||||
broken := func(context.Context) (bool, string, error) { return false, "", errors.New("the store is away") }
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
composed, inForce grantJudge
|
||||
held bool
|
||||
says string
|
||||
}{
|
||||
{"composed and in force", yes, yes, true, ""},
|
||||
{"not composed", no("no router is assigned"), yes, false, "no router is assigned"},
|
||||
{"composed, written, not reloaded", yes, no("the server has not reloaded"), false, "has not reloaded"},
|
||||
{"the composition unknown", broken, yes, false, "could not be worked out: the store is away"},
|
||||
{"the bus unknown", yes, broken, false, "could not be worked out: the store is away"},
|
||||
{"nothing judges the bus", yes, nil, false, "not judged"},
|
||||
} {
|
||||
held, why, err := grantHeldBy(c.composed, c.inForce, time.Now)(context.Background())
|
||||
if err != nil || held != c.held || !strings.Contains(why, c.says) {
|
||||
t.Errorf("%s: held %v (%q, %v)", c.name, held, why, err)
|
||||
}
|
||||
}
|
||||
// Remembered for grantLookEvery, then judged again.
|
||||
now := time.Date(2026, 10, 9, 18, 0, 0, 0, time.UTC)
|
||||
inForce := false
|
||||
judge := grantHeldBy(yes, func(context.Context) (bool, string, error) { return inForce, "not yet", nil },
|
||||
func() time.Time { return now })
|
||||
if held, _, _ := judge(context.Background()); held {
|
||||
t.Fatal("held before the bus enforces it")
|
||||
}
|
||||
inForce = true
|
||||
if held, _, _ := judge(context.Background()); held {
|
||||
t.Error("judged again inside grantLookEvery")
|
||||
}
|
||||
now = now.Add(grantLookEvery)
|
||||
if held, _, _ := judge(context.Background()); !held {
|
||||
t.Error("not judged again after grantLookEvery")
|
||||
}
|
||||
}
|
||||
|
||||
// The controller's own grant, as composed: present while a router that takes asks is assigned, absent
|
||||
// otherwise — whatever else in the user list changes.
|
||||
func TestTheControllersOwnGrantToAskIsReadFromTheComposition(t *testing.T) {
|
||||
router := broker.Declared{Module: "messenger", Holds: []broker.Seat{{Name: broker.AsksSeat, Scope: "mesh",
|
||||
Accepts: []string{"ask", "cancel"}, Emits: []string{"decided"}, ByCaller: []string{"ask", "cancel", "decided"}}}}
|
||||
other := broker.Declared{Module: "dunst"}
|
||||
with := broker.Records{Nodes: []string{"anchor"}, Assigned: map[string][]broker.Declared{"anchor": {router, other}}}
|
||||
without := broker.Records{Nodes: []string{"anchor"}, Assigned: map[string][]broker.Declared{"anchor": {other}}}
|
||||
if may, err := controllerMayAsk(with); err != nil || !may {
|
||||
t.Errorf("with a router assigned: %v %v", may, err)
|
||||
}
|
||||
if may, err := controllerMayAsk(without); err != nil || may {
|
||||
t.Errorf("with no router assigned: %v %v", may, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The bus module's answer: in force only when it says so in so many words.
|
||||
func TestTheBusModulesAnswerIsInForceOnlyWhenItSaysSo(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
answer string
|
||||
want bool
|
||||
says string
|
||||
}{
|
||||
{`{"allowed":true,"in_force":true}`, true, ""},
|
||||
{`{"allowed":true,"in_force":false,"why":"the server has not reloaded since it was written"}`, false, "not reloaded"},
|
||||
{`{"allowed":false,"in_force":false}`, false, "not in force"},
|
||||
{`{"allowed":true}`, false, "older than that answer"},
|
||||
{`{"allowed":false,"in_force":true}`, false, "not allowed"},
|
||||
{`not json`, false, "could not be read"},
|
||||
} {
|
||||
if got, why := grantInForce(json.RawMessage(c.answer)); got != c.want || !strings.Contains(why, c.says) {
|
||||
t.Errorf("%s: %v %q", c.answer, got, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The serving asker is wired to the real judgement: with records and a bus it cannot read, the grant is not
|
||||
// held, and so nothing is asked (a stub saying yes, or no judgement at all, would ask).
|
||||
func TestTheServingAskerJudgesTheGrantForReal(t *testing.T) {
|
||||
a := servingAsker(&stores{}, nil, nil, nil)
|
||||
if a.grantHeld == nil {
|
||||
t.Fatal("the serving asker does not judge the bus's grant")
|
||||
}
|
||||
held, why, err := a.grantHeld(context.Background())
|
||||
if err != nil || held || !strings.Contains(why, "could not be worked out") {
|
||||
t.Errorf("held %v (%q, %v) with nothing to read", held, why, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -235,27 +235,12 @@ func routerHereIn(inv *inventory.Inventory) func(ctx context.Context) (bool, err
|
||||
}
|
||||
}
|
||||
|
||||
// grantHeldIn says whether the bus holds the controller's grant to ask (novox/hq issue 353), judged on the two
|
||||
// facts that make it so, and failing closed on each:
|
||||
//
|
||||
// 1. **composed**: the user list the mesh composes now grants the controller itself to publish its ask
|
||||
// (controllerMayAsk) — the controller's own grant, not whether any user of the list changed;
|
||||
// 2. **in force**: the bus's own module, on the machine holding the bus, says the user list there allows it
|
||||
// and the server reloaded after that list was written (the nats module's `nats_user_can`, `in_force`) — a
|
||||
// list sent is not a list the bus enforces until it reloads.
|
||||
//
|
||||
// Anything that cannot be worked out is not held, with why, so the conditions that need the operator are said
|
||||
// as undelivered rather than asked into a refusal.
|
||||
func grantHeldIn(open *stores, conn *nats.Conn) func(ctx context.Context) (bool, string, error) {
|
||||
return grantHeldBy(composedGrantIn(open), busGrantIn(open, conn), time.Now)
|
||||
}
|
||||
|
||||
// grantJudge is one of the two facts: whether it holds, and why not.
|
||||
type grantJudge func(ctx context.Context) (bool, string, error)
|
||||
|
||||
// grantHeldBy is the judgement over the two facts, remembered for grantLookEvery: composing the list resolves
|
||||
// the bus's machine whole, and asking the bus's module is a call.
|
||||
func grantHeldBy(composed, inForce grantJudge, now func() time.Time) func(ctx context.Context) (bool, string, error) {
|
||||
// grantHeldIn says whether the bus holds the controller's grant to ask (novox/hq issue 353): the user list the
|
||||
// machine holding the bus was last sent is the one the mesh composes now (brokerBehind, the same judgement a
|
||||
// push makes to send that machine first). While it is behind, the controller's ask is refused by the bus,
|
||||
// whatever the record says of the router, so nothing is asked and the operator is told to push that machine.
|
||||
// Judged at most every grantLookEvery: composing the list resolves the bus's machine whole.
|
||||
func grantHeldIn(open *stores) func(ctx context.Context) (bool, string, error) {
|
||||
var mu sync.Mutex
|
||||
var at time.Time
|
||||
var held bool
|
||||
@@ -263,132 +248,22 @@ func grantHeldBy(composed, inForce grantJudge, now func() time.Time) func(ctx co
|
||||
return func(ctx context.Context) (bool, string, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if !at.IsZero() && now().Sub(at) < grantLookEvery {
|
||||
if !at.IsZero() && time.Since(at) < grantLookEvery {
|
||||
return held, why, nil
|
||||
}
|
||||
held, why = judgeGrant(ctx, composed, inForce)
|
||||
at = now()
|
||||
machine, behind, err := brokerBehind(ctx, open, nil)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
at, held, why = time.Now(), !behind, ""
|
||||
if behind {
|
||||
why = fmt.Sprintf("the bus's user list on %s is behind what the mesh composes, so the bus has not been "+
|
||||
"given the controller's grant to ask; `push %s` carries it", machine, machine)
|
||||
}
|
||||
return held, why, nil
|
||||
}
|
||||
}
|
||||
|
||||
// judgeGrant is held only when both facts hold; an error is a fact not known, never a yes.
|
||||
func judgeGrant(ctx context.Context, composed, inForce grantJudge) (bool, string) {
|
||||
for _, fact := range []struct {
|
||||
judge grantJudge
|
||||
what string
|
||||
}{{composed, "whether the mesh composes the controller a grant to ask"}, {inForce, "whether the bus enforces it"}} {
|
||||
if fact.judge == nil {
|
||||
return false, fact.what + " is not judged here"
|
||||
}
|
||||
ok, why, err := fact.judge(ctx)
|
||||
if err != nil {
|
||||
return false, fact.what + " could not be worked out: " + err.Error()
|
||||
}
|
||||
if !ok {
|
||||
return false, why
|
||||
}
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
|
||||
// controllerMayAsk says whether the controller's own grant, as composed from these records, lets it publish its
|
||||
// ask (ADR 0259 §3: composed while a router that takes asks under its asker's name is assigned).
|
||||
func controllerMayAsk(records broker.Records) (bool, error) {
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, u := range users {
|
||||
if u.Kind != broker.KindController {
|
||||
continue
|
||||
}
|
||||
perms, err := broker.PermissionsFor(u)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return broker.MayPublish(perms, asks.AskSubject(askerName)), nil
|
||||
}
|
||||
return false, errors.New("the composition holds no controller")
|
||||
}
|
||||
|
||||
// composedGrantIn is the first fact, from the mesh's records.
|
||||
func composedGrantIn(open *stores) grantJudge {
|
||||
return func(ctx context.Context) (bool, string, error) {
|
||||
if open == nil || open.inventory == nil {
|
||||
return false, "", errors.New("the mesh's records are not open")
|
||||
}
|
||||
records, err := open.inventory.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
may, err := controllerMayAsk(records)
|
||||
if err != nil || may {
|
||||
return may, "", err
|
||||
}
|
||||
return false, "the mesh composes the controller no grant to ask: no router that takes asks under its " +
|
||||
"asker's name is assigned", nil
|
||||
}
|
||||
}
|
||||
|
||||
// busGrantIn is the second fact, asked of the bus's own module on the machine holding the bus.
|
||||
func busGrantIn(open *stores, conn *nats.Conn) grantJudge {
|
||||
return func(ctx context.Context) (bool, string, error) {
|
||||
if open == nil || open.inventory == nil || conn == nil {
|
||||
return false, "", errors.New("the mesh's records or the bus are not open")
|
||||
}
|
||||
holders, err := seatHolders(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
h, held := holders[theBrokerSeat]
|
||||
if !held || h.Node == "" || h.Module == "" {
|
||||
return false, "no module holds the bus, so nothing says what it enforces", nil
|
||||
}
|
||||
a, err := link.AskModuleToolOn(ctx, conn, h.Module, busUserCanTool, h.Node, map[string]any{
|
||||
"user": broker.ControllerName, "action": "publish", "subject": asks.AskSubject(askerName)}, 15*time.Second)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
if a.Error != "" {
|
||||
return false, "", errors.New(a.Error)
|
||||
}
|
||||
inForce, why := grantInForce(a.Result)
|
||||
if !inForce {
|
||||
why = fmt.Sprintf("the bus on %s does not enforce the controller's grant to ask yet: %s; `push %s` "+
|
||||
"carries it, and the bus reloads it in place", h.Node, why, h.Node)
|
||||
}
|
||||
return inForce, why, nil
|
||||
}
|
||||
}
|
||||
|
||||
// busUserCanTool is the bus module's answer to whether a user may do something, and whether it is in force.
|
||||
const busUserCanTool = "nats_user_can"
|
||||
|
||||
// grantInForce reads the bus module's answer: in force only when it says so, in so many words. An answer
|
||||
// without `in_force` is a bus module older than the question, and is not a yes.
|
||||
func grantInForce(raw json.RawMessage) (bool, string) {
|
||||
var answer struct {
|
||||
Allowed bool `json:"allowed"`
|
||||
InForce *bool `json:"in_force"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &answer); err != nil {
|
||||
return false, "the bus module's answer could not be read"
|
||||
}
|
||||
switch {
|
||||
case answer.InForce == nil:
|
||||
return false, "the bus module does not say whether a grant is in force (it is older than that answer)"
|
||||
case !*answer.InForce && answer.Why != "":
|
||||
return false, answer.Why
|
||||
case !*answer.InForce:
|
||||
return false, "the bus module says it is not in force"
|
||||
case !answer.Allowed:
|
||||
return false, "the bus module says it is in force and not allowed"
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
|
||||
// grantLookEvery is how often the bus's user list is judged against the one its machine was last sent.
|
||||
const grantLookEvery = 30 * time.Second
|
||||
|
||||
@@ -427,19 +302,7 @@ func startAsking(ctx context.Context, open *stores, server *link.Server, conn *n
|
||||
fmt.Printf("the operator cannot be asked: %v\n", err)
|
||||
return
|
||||
}
|
||||
a := servingAsker(open, conn, js, keeper)
|
||||
if err := server.Decides(a); err != nil {
|
||||
fmt.Printf("the operator's answers cannot be heard, so nothing is asked: %v\n", err)
|
||||
return
|
||||
}
|
||||
askerFrom = a
|
||||
go a.keep(ctx)
|
||||
}
|
||||
|
||||
// servingAsker is the serving controller's asker, wired to the mesh: what it reads, publishes, performs and
|
||||
// records, and the judgements it asks by — the bus's grant to ask among them (novox/hq issue 353).
|
||||
func servingAsker(open *stores, conn *nats.Conn, js jetstream.JetStream, keeper *conditions.Keeper) *asker {
|
||||
return &asker{
|
||||
a := &asker{
|
||||
open: keeper.Open,
|
||||
silence: func(ctx context.Context, key string, d time.Duration, by, why string) error {
|
||||
_, err := keeper.Silence(ctx, key, d, by, why)
|
||||
@@ -457,7 +320,7 @@ func servingAsker(open *stores, conn *nats.Conn, js jetstream.JetStream, keeper
|
||||
},
|
||||
routerRecord: routerRecordOf(conn, open.inventory),
|
||||
routerHere: routerHereIn(open.inventory),
|
||||
grantHeld: grantHeldIn(open, conn),
|
||||
grantHeld: grantHeldIn(open),
|
||||
channels: channelsIn(open.inventory),
|
||||
raise: func(ctx context.Context, obs []conditions.Observation) error {
|
||||
return keeper.Reconcile(ctx, sourceAsker, obs)
|
||||
@@ -465,4 +328,10 @@ func servingAsker(open *stores, conn *nats.Conn, js jetstream.JetStream, keeper
|
||||
now: time.Now,
|
||||
logf: func(format string, args ...any) { fmt.Printf(format+"\n", args...) },
|
||||
}
|
||||
if err := server.Decides(a); err != nil {
|
||||
fmt.Printf("the operator's answers cannot be heard, so nothing is asked: %v\n", err)
|
||||
return
|
||||
}
|
||||
askerFrom = a
|
||||
go a.keep(ctx)
|
||||
}
|
||||
|
||||
@@ -119,13 +119,6 @@ func TestTheBusMachineIsBehindByItsUserListAlone(t *testing.T) {
|
||||
if userListBehind("", "") {
|
||||
t.Fatal("a machine sent no list reads as behind")
|
||||
}
|
||||
// A list that could not be composed is behind: not known is never "not behind" (the review of hq issue 353).
|
||||
if !listBehind("", errors.New("the plan cannot be worked out"), digestOf([]byte(list))) {
|
||||
t.Fatal("a list nobody could compose reads as current")
|
||||
}
|
||||
if listBehind(list, nil, digestOf([]byte(list))) || !listBehind(list, nil, "") {
|
||||
t.Fatal("listBehind does not read the digest as userListBehind does")
|
||||
}
|
||||
}
|
||||
|
||||
// The machine holding the bus goes first: its declaration carries the user list the new grants are
|
||||
|
||||
@@ -18,7 +18,7 @@ func foundDirectory(module string, since time.Time) inventory.ResourceHealth {
|
||||
return inventory.ResourceHealth{Module: module, Resource: module + ".data", Kind: link.KindDirectory,
|
||||
Target: "/srv/" + module, State: link.StateUnhealthy, Since: since,
|
||||
Reason: link.ReasonUsedAsFound + " owned by 1000:1000, mode 700, as found; root, mode 755 was declared and " +
|
||||
"not given it — `mesh-host hand-over` at the machine hands it to the mesh"}
|
||||
"not given it — `nox node hand-over laptop <directory>` on the control-node, with its path, hands it to the mesh"}
|
||||
}
|
||||
|
||||
func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A hand-over's line is judged before anything is asked (novox/hq issue 356): a node's name and the directory's
|
||||
// absolute path exactly as the engine states it — no `..`, no doubled or trailing separator, nothing relative.
|
||||
func TestAHandOverLineIsJudgedBeforeItIsAsked(t *testing.T) {
|
||||
for _, args := range [][]string{
|
||||
{},
|
||||
{"laptop"},
|
||||
{"laptop", "/srv/notes", "extra"},
|
||||
{"", "/srv/notes"},
|
||||
{"--node", "/srv/notes"},
|
||||
{"laptop", "srv/notes"},
|
||||
{"laptop", "/srv/../etc"},
|
||||
{"laptop", "/srv//notes"},
|
||||
{"laptop", "/srv/notes/"},
|
||||
{"laptop", "/srv/notes/."},
|
||||
} {
|
||||
if _, _, err := handOverLine(args); err == nil {
|
||||
t.Errorf("%q was taken", args)
|
||||
}
|
||||
}
|
||||
node, path, err := handOverLine([]string{"laptop", "/srv/notes"})
|
||||
if err != nil || node != "laptop" || path != "/srv/notes" {
|
||||
t.Fatalf("read as %q %q %v", node, path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Who hands over is the line's caller in the words every verb's caller is recorded in — the operator through
|
||||
// mesh-cli — or the controller's terminal when nobody is named.
|
||||
func TestAHandOverNamesWhoAsked(t *testing.T) {
|
||||
t.Setenv(link.CallerVar, " jo through mesh-cli on anchor ")
|
||||
if by := handOverBy(); by != "jo through mesh-cli on anchor" {
|
||||
t.Fatalf("by %q", by)
|
||||
}
|
||||
t.Setenv(link.CallerVar, "")
|
||||
if by := handOverBy(); by != "the controller's terminal" {
|
||||
t.Fatalf("by %q", by)
|
||||
}
|
||||
}
|
||||
|
||||
// **A hand-over is the controller's terminal's alone** (novox/hq issue 356, ADR 0266): through any verb, and
|
||||
// through mesh-cli outside the terminal, `node hand-over` is refused and nothing runs — at the next apply root
|
||||
// gives the directory to the account the module declares, and whoever may call a verb includes agents.
|
||||
func TestAHandOverIsRefusedThroughEveryVerb(t *testing.T) {
|
||||
line := []string{"node", "hand-over", "laptop", "/srv/notes"}
|
||||
if err := terminalOnly(line); err == nil {
|
||||
t.Fatal("node hand-over passed as a verb's line")
|
||||
}
|
||||
if _, err := argvFor("command", map[string]any{"command": "node hand-over laptop /srv/notes"}); err == nil {
|
||||
t.Fatal("the command verb composed node hand-over")
|
||||
}
|
||||
if _, err := ordinaryLine(line); err == nil {
|
||||
t.Fatal("node hand-over composed as an ordinary mesh-cli line")
|
||||
}
|
||||
if _, err := argvFor("node", map[string]any{"node": "laptop", "hand-over": "/srv/notes"}); err == nil {
|
||||
t.Fatal("the node verb composed a hand-over")
|
||||
}
|
||||
}
|
||||
|
||||
// The condition's words are plain and name no machine's binary; the operator's line, with the node and the path
|
||||
// (novox/hq ADR 0272), is in the summary the module's health gives (moduleHealthWord) and in the evidence.
|
||||
func TestTheUsedAsFoundConditionNamesTheOperatorsLine(t *testing.T) {
|
||||
rs := []inventory.ResourceHealth{foundDirectory("notes", time.Now().Add(-24*time.Hour))}
|
||||
o := usedAsFoundObservation("notes", "laptop", "notes on laptop uses notes.data as found", rs)
|
||||
if strings.Contains(o.Needs, "mesh-host") || strings.Contains(o.Explanation, "mesh-host") ||
|
||||
!strings.Contains(o.Needs, "control-node") {
|
||||
t.Fatalf("the condition's words: %q %q", o.Needs, o.Explanation)
|
||||
}
|
||||
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Needs: o.Needs,
|
||||
Resolved: o.Resolved}, "laptop"); !ok {
|
||||
t.Fatalf("not plain: %s", why)
|
||||
}
|
||||
f := gateFacts{now: time.Now(), health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: time.Now(),
|
||||
Resources: rs}}}
|
||||
h, why := moduleHealthWord("notes", "laptop", time.Now().Add(-time.Hour), f)
|
||||
if h != healthPerson || !strings.Contains(why, "`nox node hand-over laptop <directory>` on the control-node") ||
|
||||
strings.Contains(why, "mesh-host") || strings.Contains(why, "/srv/") {
|
||||
t.Fatalf("the module's health reads %v %q; want the operator's line", h, why)
|
||||
}
|
||||
}
|
||||
|
||||
// **Nothing is asked of a node the mesh does not know, and the engine's refusal is the command's failure**
|
||||
// (review of issue 356): a refused hand-over never exits as a success.
|
||||
func TestAHandOverAsksOnlyAKnownNodeAndFailsOnARefusal(t *testing.T) {
|
||||
t.Setenv(link.CallerVar, "jo through mesh-cli on anchor")
|
||||
asked := 0
|
||||
ask := func(answer link.HandOverAnswer) func(node, path, by string) (link.HandOverAnswer, error) {
|
||||
return func(node, path, by string) (link.HandOverAnswer, error) {
|
||||
asked++
|
||||
if node != "laptop" || path != "/srv/notes" || by != "jo through mesh-cli on anchor" {
|
||||
t.Fatalf("asked %q %q %q", node, path, by)
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
}
|
||||
unknown := func(string) error { return errors.New("no node called laptop") }
|
||||
known := func(string) error { return nil }
|
||||
var out bytes.Buffer
|
||||
err := handOverAsked([]string{"laptop", "/srv/notes"}, unknown, ask(link.HandOverAnswer{Said: "x"}), &out)
|
||||
if err == nil || asked != 0 || !strings.Contains(err.Error(), "nothing was asked") {
|
||||
t.Fatalf("an unknown node: %v, asked %d", err, asked)
|
||||
}
|
||||
err = handOverAsked([]string{"laptop", "/srv/../etc"}, known, ask(link.HandOverAnswer{Said: "x"}), &out)
|
||||
if err == nil || asked != 0 {
|
||||
t.Fatalf("a refused line was asked: %v, asked %d", err, asked)
|
||||
}
|
||||
err = handOverAsked([]string{"laptop", "/srv/notes"}, known,
|
||||
ask(link.HandOverAnswer{Refused: "/srv/notes is not used as found; nothing was handed over"}), &out)
|
||||
if err == nil || !strings.Contains(err.Error(), "laptop refused: /srv/notes is not used as found") || out.Len() != 0 {
|
||||
t.Fatalf("a refusal: %v, printed %q", err, out.String())
|
||||
}
|
||||
err = handOverAsked([]string{"laptop", "/srv/notes"}, known, ask(link.HandOverAnswer{Said: "handed over"}), &out)
|
||||
if err != nil || !strings.HasPrefix(out.String(), "handed over\n") || !strings.Contains(out.String(), "`nox push laptop`") {
|
||||
t.Fatalf("a record: %v, printed %q", err, out.String())
|
||||
}
|
||||
failing := func(string, string, string) (link.HandOverAnswer, error) {
|
||||
return link.HandOverAnswer{}, errors.New("no engine")
|
||||
}
|
||||
if err := handOverAsked([]string{"laptop", "/srv/notes"}, known, failing, &out); err == nil {
|
||||
t.Fatal("an ask that failed was a success")
|
||||
}
|
||||
}
|
||||
@@ -556,8 +556,8 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
||||
said = append(said, wait)
|
||||
}
|
||||
if len(found) > 0 {
|
||||
said = append(said, fmt.Sprintf("on %s, %s uses %s as found and waits for a person to hand it over "+
|
||||
"(`mesh-host hand-over <directory>` at the machine)", machine, module, strings.Join(found, ", ")))
|
||||
said = append(said, fmt.Sprintf("on %s, %s uses %s as found and waits for the operator to hand it over "+
|
||||
"(`nox node hand-over %s <directory>` on the control-node)", machine, module, strings.Join(found, ", "), machine))
|
||||
}
|
||||
return healthPerson, strings.Join(said, "; ")
|
||||
}
|
||||
@@ -678,7 +678,9 @@ func usedAsFoundObservation(module, node, said string, rs []inventory.ResourceHe
|
||||
o.Explanation = fmt.Sprintf("A directory of %s was already on %s, with another owner or mode than %s declares. "+
|
||||
"The mesh left it as it was rather than hand it to an account, so %s may not be able to use it.",
|
||||
module, node, module, module)
|
||||
o.Needs = fmt.Sprintf("on %s, run mesh-host hand-over with the directory's path as root.", node)
|
||||
// Plain words (ADR 0253): the line itself — `nox node hand-over <node> <path>` on the control-node (ADR 0272,
|
||||
// issue 356) — is in the summary and the evidence, which name the directory; a path is never in these.
|
||||
o.Needs = "hand the directory over from the control-node, as the operator; the details name it and the line to type."
|
||||
o.Resolved = fmt.Sprintf("%s's directory on %s is the mesh's", module, node)
|
||||
o.Actions = nil
|
||||
return o
|
||||
|
||||
@@ -426,10 +426,10 @@ func overlayShow(ctx context.Context, open *stores) error {
|
||||
tunnel.Interface, tunnel.Range, tunnel.Port)
|
||||
case n.Hub && hubName == n.Name && tunnel.Interface != "":
|
||||
fmt.Printf(" hub — found a tunnel on %s and did NOT take it over: its key is not the tunnel's; "+
|
||||
"`mesh-host overlay take --tunnel %s` on the machine takes it", tunnel.Interface, tunnel.Interface)
|
||||
"`nox-mesh-host overlay take --tunnel %s` on the machine takes it", tunnel.Interface, tunnel.Interface)
|
||||
case n.Hub:
|
||||
fmt.Print(" hub — found no tunnel; if the machine runs the predecessor's, " +
|
||||
"`mesh-host overlay take --tunnel <iface>` there adopts it (novox/hq ADR 0105)")
|
||||
"`nox-mesh-host overlay take --tunnel <iface>` there adopts it (novox/hq ADR 0105)")
|
||||
case !n.Reachable():
|
||||
fmt.Print(" not dialable")
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -17,6 +18,7 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/token"
|
||||
)
|
||||
|
||||
@@ -28,7 +30,7 @@ import (
|
||||
|
||||
func nodeCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New("node add <name>, node list, node show <name>, or " + publicDomainUsage)
|
||||
return errors.New("node add <name>, node list, node show <name>, " + publicDomainUsage + ", or " + handOverUsage)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
@@ -112,11 +114,102 @@ func nodeCommand(ctx context.Context, args []string) error {
|
||||
// an optional second argument is the home when it is not /home/<account>.
|
||||
return nodeAccount(ctx, inv, args[1:])
|
||||
|
||||
case "hand-over":
|
||||
// A directory the node-engine uses as found, handed to the mesh (novox/hq issue 356, issue 339). Here, at
|
||||
// the controller's terminal, and nowhere else: at the next apply root gives the directory to the account
|
||||
// the module declares, and whoever may call a verb includes agents.
|
||||
return nodeHandOver(ctx, open, args[1:])
|
||||
|
||||
default:
|
||||
return fmt.Errorf("node has no %q; it has add, list, show, public-domain, account and agent-account", args[0])
|
||||
return fmt.Errorf("node has no %q; it has add, list, show, public-domain, account, agent-account and hand-over", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
const handOverUsage = "node hand-over <node> <directory> — hand a directory the node-engine on <node> uses as found " +
|
||||
"to the mesh: its next apply gives it the declared owner and mode. The directory's absolute path, as the module's " +
|
||||
"condition names it"
|
||||
|
||||
// handOverLine reads a hand-over's line: the node and the directory's absolute path, exactly as the engine states
|
||||
// it. Judged before anything is asked, and judged again by the engine, which is the one that acts.
|
||||
func handOverLine(args []string) (node, path string, err error) {
|
||||
if len(args) != 2 {
|
||||
return "", "", errors.New(handOverUsage)
|
||||
}
|
||||
node, path = args[0], args[1]
|
||||
if node == "" || strings.HasPrefix(node, "-") {
|
||||
return "", "", fmt.Errorf("%q is not a node's name; %s", node, handOverUsage)
|
||||
}
|
||||
if !filepath.IsAbs(path) {
|
||||
return "", "", fmt.Errorf("%q is not an absolute path; %s", path, handOverUsage)
|
||||
}
|
||||
if filepath.Clean(path) != path {
|
||||
return "", "", fmt.Errorf("%q is not the directory's path as the engine states it (no `..`, no doubled or "+
|
||||
"trailing separator); %s", path, handOverUsage)
|
||||
}
|
||||
return node, path, nil
|
||||
}
|
||||
|
||||
// handOverBy is who hands the directory over, in the words a verb's caller is recorded in: the operator through
|
||||
// mesh-cli on the control-node, or whoever runs this controller's binary at its terminal.
|
||||
func handOverBy() string {
|
||||
if by := strings.TrimSpace(os.Getenv(link.CallerVar)); by != "" {
|
||||
return by
|
||||
}
|
||||
return "the controller's terminal"
|
||||
}
|
||||
|
||||
// nodeHandOver asks the node's engine to take a directory it uses as found as the mesh's, and says what came of
|
||||
// it. The engine records the hand-over or refuses; nothing is recorded here, because the directory is the
|
||||
// machine's and the engine is the one that reads it. The ask is signed with the mesh's key (issue 356's review).
|
||||
func nodeHandOver(ctx context.Context, open *stores, args []string) error {
|
||||
known := func(node string) error {
|
||||
_, err := open.inventory.NodeByName(ctx, node)
|
||||
return err
|
||||
}
|
||||
ask := func(node, path, by string) (link.HandOverAnswer, error) {
|
||||
ident, err := open.Identity(ctx)
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, fmt.Errorf("the mesh's signing key cannot be read, so nothing was asked of %s: %w",
|
||||
node, err)
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, fmt.Errorf("cannot reach the bus, so nothing was asked of %s: %w", node, err)
|
||||
}
|
||||
defer js.Close()
|
||||
return link.AskHandOver(ctx, js.Conn(), ident, node, path, by, link.HandOverWithin)
|
||||
}
|
||||
return handOverAsked(args, known, ask, os.Stdout)
|
||||
}
|
||||
|
||||
// handOverAsked is the hand-over's line with its two acts given: whether the mesh knows the node, and the ask.
|
||||
// Nothing is asked of a line or a node that is refused, and the engine's refusal is this command's failure —
|
||||
// never a success with the refusal printed.
|
||||
func handOverAsked(args []string, known func(node string) error,
|
||||
ask func(node, path, by string) (link.HandOverAnswer, error), out io.Writer) error {
|
||||
node, path, err := handOverLine(args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := known(node); err != nil {
|
||||
return fmt.Errorf("nothing was asked: %w", err)
|
||||
}
|
||||
answer, err := ask(node, path, handOverBy())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Refused != "" {
|
||||
return fmt.Errorf("%s refused: %s", node, answer.Refused)
|
||||
}
|
||||
fmt.Fprintln(out, answer.Said)
|
||||
fmt.Fprintf(out, " the module's condition clears once %s applies; `nox push %s` applies it now\n", node, node)
|
||||
return nil
|
||||
}
|
||||
|
||||
// addNode creates a node record, adopted when the operator says so (novox/hq ADR 0100).
|
||||
func addNode(ctx context.Context, inv *inventory.Inventory, args []string) error {
|
||||
set := flag.NewFlagSet("node add", flag.ContinueOnError)
|
||||
|
||||
@@ -1096,29 +1096,20 @@ func brokerBehind(ctx context.Context, open *stores, names []string) (string, bo
|
||||
return h.Node, false, nil
|
||||
}
|
||||
}
|
||||
// **What cannot be worked out is behind** (the review of hq issue 353). The list it would be sent cannot be
|
||||
// composed, so whether it carries the grants a send relies on is not known; it is taken as behind, so a push
|
||||
// sends that machine first and says why it cannot (`plan` says it too), and a rollout that needs the list
|
||||
// refuses rather than sending code whose grants the bus may not hold. It said "not behind" until then: a
|
||||
// send went ahead as if the bus held a list nobody could compose.
|
||||
list, composeErr := "", error(nil)
|
||||
plan, _, err := planFor(ctx, open, h.Node)
|
||||
if err != nil {
|
||||
composeErr = err
|
||||
} else if list, _, err = busUserList(ctx, inv, plan.Modules); err != nil {
|
||||
composeErr = err
|
||||
// It cannot be worked out: sending it would refuse the whole send, and `plan` says why.
|
||||
return h.Node, false, nil
|
||||
}
|
||||
list, _, err := busUserList(ctx, inv, plan.Modules)
|
||||
if err != nil {
|
||||
return h.Node, false, nil
|
||||
}
|
||||
sent, err := inv.SentBusUsers(ctx, h.Node)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return h.Node, listBehind(list, composeErr, sent), nil
|
||||
}
|
||||
|
||||
// listBehind is whether the bus's machine is behind: the list composed now is not the one last sent, or it
|
||||
// could not be composed at all — not known is behind, never "not behind".
|
||||
func listBehind(list string, composeErr error, sentDigest string) bool {
|
||||
return composeErr != nil || userListBehind(list, sentDigest)
|
||||
return h.Node, userListBehind(list, sent), nil
|
||||
}
|
||||
|
||||
// userListBehind is whether the user list composed now is not the one last sent, by its digest. An
|
||||
|
||||
+18
-4
@@ -289,6 +289,14 @@ func (p Principal) Username() string {
|
||||
// (novox/hq to-be 45 §6): its answer is an ordinary report, on its own report subject.
|
||||
func AskReportSubject(node string) string { return "mesh.node." + node + ".ask.report" }
|
||||
|
||||
// AskHandOverSubject is where the controller's terminal asks one machine's node-engine to hand a directory it
|
||||
// uses as found to the mesh (novox/hq issue 356, issue 339): a request on core NATS, answered once on the reply
|
||||
// it carries. Only the controller is granted a publish here (the writers table holds it), but **that is not who
|
||||
// the engine hears**: the bus lets any principal allowed to answer reply to a message it received, on the reply
|
||||
// subject that message named, so a message can arrive here from any responder. The ask is therefore signed with
|
||||
// the mesh's key (link.SignedHandOver), and the engine verifies it before reading anything out of it.
|
||||
func AskHandOverSubject(node string) string { return "mesh.node." + node + ".ask.hand-over" }
|
||||
|
||||
// inbox is a principal's own reply space. No user is ever granted a bare `_INBOX.>` (design 25
|
||||
// §4): with one account, inbox privacy is the permission list or it is nothing, so each user's
|
||||
// inbox is derived from its own identity and its permissions name that prefix and no other.
|
||||
@@ -562,7 +570,10 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// And the mesh asking it to say again what it last applied (novox/hq to-be 45 §6, the
|
||||
// `report` verb healer H1 asks): its own machine's, on core NATS and off any stream. It
|
||||
// answers through its report, the one thing it already says — no reply to anybody's inbox.
|
||||
AskReportSubject(p.Node)}
|
||||
AskReportSubject(p.Node),
|
||||
// And the controller's terminal asking it to hand a directory used as found to the mesh (novox/hq
|
||||
// issue 356), which it answers on the request's reply: the one request a node is asked.
|
||||
AskHandOverSubject(p.Node)}
|
||||
// The node-engine witnesses the core builds it places (novox/hq to-be 45 §8, ADR 0236; the
|
||||
// contract is lease/witness.go): it asks its own machine's node tools PING, and, where the
|
||||
// machine runs the controller, reads the lease's one key — read, never written.
|
||||
@@ -863,9 +874,12 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
PublishDeny: deny,
|
||||
Subscribe: sub,
|
||||
// A module answers what it was asked — a tool call reaches it on its own namespace, so the
|
||||
// authority is bounded by having been asked — and so does the controller. A node and a
|
||||
// person are never asked anything, and are granted nothing here.
|
||||
AllowResponses: p.Kind == KindModule || p.Kind == KindController || p.Kind == KindNodeTools,
|
||||
// authority is bounded by having been asked — and so does the controller. A node is asked one
|
||||
// thing, a hand-over on its own subject (novox/hq issue 356), and answers that: the node is its
|
||||
// machine's engine, root there already, and it is delivered only its own subjects. What it answers is
|
||||
// never trusted for being an answer — the controller reads the engine's words and records nothing. A
|
||||
// person is never asked anything, and is granted nothing here.
|
||||
AllowResponses: p.Kind == KindModule || p.Kind == KindController || p.Kind == KindNodeTools || p.Kind == KindNode,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -103,7 +103,8 @@ func TestAnInboxIsScopedToItsOwner(t *testing.T) {
|
||||
|
||||
// A responder answers on the caller's inbox, which it has no permission for. allow_responses is
|
||||
// what makes a scoped inbox workable at all — the authority is bounded by having been asked. A
|
||||
// module is asked on its own namespace and may answer; a node and a person are never asked.
|
||||
// module is asked on its own namespace and may answer; a node is asked one thing, a hand-over on its own
|
||||
// subject (novox/hq issue 356), and may answer that; a person is never asked.
|
||||
func TestOnlyWhatCanBeAskedMayAnswer(t *testing.T) {
|
||||
module, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "audit",
|
||||
Consumes: []string{"shop.order.placed"}, PasswordHash: "x"})
|
||||
@@ -111,8 +112,12 @@ func TestOnlyWhatCanBeAskedMayAnswer(t *testing.T) {
|
||||
t.Fatal("a module cannot answer a tool call on its own namespace")
|
||||
}
|
||||
node, _ := PermissionsFor(Principal{Kind: KindNode, Node: "one", PasswordHash: "x"})
|
||||
if node.AllowResponses {
|
||||
t.Fatal("a node was granted the right to answer, and nothing asks a node anything")
|
||||
if !node.AllowResponses || !slices.Contains(node.Subscribe, AskHandOverSubject("one")) {
|
||||
t.Fatalf("a node cannot answer the hand-over it is asked: %v %v", node.AllowResponses, node.Subscribe)
|
||||
}
|
||||
person, _ := PermissionsFor(Principal{Kind: KindPerson, Node: "one", Module: "jo", PasswordHash: "x"})
|
||||
if person.AllowResponses {
|
||||
t.Fatal("a person was granted the right to answer, and nothing asks a person anything")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+2
-1
@@ -34,7 +34,8 @@ accounts {
|
||||
} }
|
||||
{ user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: {
|
||||
publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.INFO.NODES.one", "$SRV.PING.node-tools.one", "mesh.control.one.>"] }
|
||||
subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.ask.report", "mesh.node.one.declare"] }
|
||||
subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.ask.hand-over", "mesh.node.one.ask.report", "mesh.node.one.declare"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "one.nats", password: "$2a$11$bbbbbbbbbbbbbbbbbbbbbb", permissions: {
|
||||
publish: { allow: ["$JS.API.STREAM.INFO.*", "$JS.API.STREAM.NAMES", "$JS.API.STREAM.SNAPSHOT.*", "$JS.SNAPSHOT.ACK.>"] }
|
||||
|
||||
@@ -84,6 +84,13 @@ func kvOf(bucket string) []string { return []string{"$KV." + bucket + ".>"} }
|
||||
var WritersTable = []WriterRow{
|
||||
{State: "a machine's declaration", Writer: "controller (lease holder)", KeptIn: "the bus, last per subject",
|
||||
Others: "read", Subjects: []string{"mesh.node.*.declare"}, Writes: isController},
|
||||
// The operator's hand-over of a directory used as found, asked of the machine's engine at the controller's
|
||||
// terminal (novox/hq issue 356). One publisher; and because a responder can still reach the subject through a
|
||||
// reply, the ask is signed with the mesh's key and the engine verifies it — the row bounds who is granted the
|
||||
// publish, the signature who is believed.
|
||||
{State: "a hand-over asked of a machine", Writer: "controller, at its terminal", KeptIn: "the machine, beside its state",
|
||||
Others: "the engine verifies the mesh's signature and records it, or refuses",
|
||||
Subjects: []string{"mesh.node.*.ask.hand-over"}, Writes: isController},
|
||||
{State: "a machine's applied state and its report", Writer: "the node-engine's apply queue",
|
||||
KeptIn: "the machine; the report on the bus", Others: "the reconcile and a delivery enqueue, never apply",
|
||||
// And its health statement between reports (novox/hq ADR 0240): the same writer stating the same
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
// to the table; one dropped from either fails.
|
||||
var designRows = []string{
|
||||
"a machine's declaration",
|
||||
"a hand-over asked of a machine",
|
||||
"a machine's applied state and its report",
|
||||
"the controller lease",
|
||||
"plans and their tiers",
|
||||
@@ -150,3 +151,22 @@ func TestSubjectsOverlap(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A hand-over asked of a machine has one publisher, the controller** (novox/hq issue 356): a grant that lets any
|
||||
// other principal publish it — a node, the node tools, a module — is refused at composition, naming the state.
|
||||
// (Who the engine believes is the signature's; this bounds who is granted the publish.)
|
||||
func TestAHandOverAskHasOnePublisher(t *testing.T) {
|
||||
for _, p := range []Principal{
|
||||
{Kind: KindNode, Node: "laptop"},
|
||||
{Kind: KindNodeTools, Node: "laptop", Module: RuntimeModule},
|
||||
{Kind: KindModule, Node: "laptop", Module: "notes"},
|
||||
} {
|
||||
err := CheckWriters(p, []string{"mesh.node.laptop.ask.hand-over"})
|
||||
if err == nil || !strings.Contains(err.Error(), "a hand-over asked of a machine") {
|
||||
t.Errorf("%s may publish a hand-over: %v", p.Username(), err)
|
||||
}
|
||||
}
|
||||
if err := CheckWriters(Principal{Kind: KindController}, []string{"mesh.node.>"}); err != nil {
|
||||
t.Fatalf("the controller may not ask a hand-over: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// A hand-over asked of a machine's node-engine (novox/hq issue 356, issue 339).
|
||||
//
|
||||
// The operator hands a directory the node-engine uses as found to the mesh at the controller's terminal:
|
||||
// `nox node hand-over <node> <path>` on the control-node (ADR 0272). The controller asks that machine's
|
||||
// engine on its own subject, a request on core NATS the engine answers once; the engine judges every
|
||||
// value and records the hand-over, or refuses and records nothing. The engine holds the same two shapes
|
||||
// in its own link code (mesh-host internal/link HandOverAsk, HandOverAnswer); a test on each side holds
|
||||
// the field names.
|
||||
|
||||
// HandOverAsk is what the controller asks: the node it is for, the directory's absolute path as the engine states
|
||||
// it, who asked in the controller's words, when the ask stops being good, and a nonce the engine takes once.
|
||||
type HandOverAsk struct {
|
||||
Node string `json:"node"`
|
||||
Path string `json:"path"`
|
||||
By string `json:"by"`
|
||||
Expires time.Time `json:"expires"`
|
||||
Nonce string `json:"nonce"`
|
||||
}
|
||||
|
||||
// SignedHandOver is the ask as it travels: its bytes exactly as signed, and the signature.
|
||||
//
|
||||
// **Signed, because the subject proves nothing** (review of issue 356). Only the controller may publish
|
||||
// `mesh.node.<node>.ask.hand-over`, but the bus lets any principal allowed to answer reply to a message it
|
||||
// received, on whatever reply subject that message named — so a tool server asked on its own subject with that
|
||||
// reply could hand the engine an ask the controller never made. The engine verifies this signature, with the
|
||||
// key it verifies declarations with, before it reads anything out of the ask.
|
||||
type SignedHandOver struct {
|
||||
Ask []byte `json:"ask"`
|
||||
Signature []byte `json:"signature"`
|
||||
}
|
||||
|
||||
// HandOverContext is prefixed to an ask's bytes before signing, so a hand-over's signature is never a
|
||||
// declaration's: the same key signs both, and a declaration is signed over its bytes alone.
|
||||
const HandOverContext = "novox-mesh hand-over v1\n"
|
||||
|
||||
// HandOverGood is how long a signed ask is good for: the engine refuses one past it, and one further ahead.
|
||||
const HandOverGood = time.Minute
|
||||
|
||||
// HandOverAnswer is the engine's answer: what it recorded, or why it refused.
|
||||
type HandOverAnswer struct {
|
||||
Said string `json:"said,omitempty"`
|
||||
Refused string `json:"refused,omitempty"`
|
||||
}
|
||||
|
||||
// HandOverWithin is how long the controller waits for the engine's answer: a file write, on a machine that is
|
||||
// up; a machine that is down is said as not answering.
|
||||
const HandOverWithin = 30 * time.Second
|
||||
|
||||
// AskHandOver asks one machine's node-engine to hand a directory used as found to the mesh, and reads its
|
||||
// answer. An error is the ask not reaching an engine, or an answer that is not one; a refusal is the engine's
|
||||
// and comes back in the answer.
|
||||
func AskHandOver(ctx context.Context, conn *nats.Conn, signer Signer, node, path, by string,
|
||||
timeout time.Duration) (HandOverAnswer, error) {
|
||||
if conn == nil {
|
||||
return HandOverAnswer{}, errors.New("this controller is not on the bus")
|
||||
}
|
||||
body, err := SignHandOver(ctx, signer, HandOverAsk{Node: node, Path: path, By: by})
|
||||
if err != nil {
|
||||
return HandOverAnswer{}, err
|
||||
}
|
||||
asking, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
subject := broker.AskHandOverSubject(node)
|
||||
refused, stop := refusalsOf(conn, subject)
|
||||
defer stop()
|
||||
type replied struct {
|
||||
msg *nats.Msg
|
||||
err error
|
||||
}
|
||||
done := make(chan replied, 1)
|
||||
go func() {
|
||||
msg, err := conn.RequestWithContext(asking, subject, body)
|
||||
done <- replied{msg, err}
|
||||
}()
|
||||
var reply *nats.Msg
|
||||
select {
|
||||
case r := <-done:
|
||||
reply, err = r.msg, r.err
|
||||
case why := <-refused:
|
||||
cancel()
|
||||
return HandOverAnswer{}, fmt.Errorf("the bus refused the controller asking %s for a hand-over: %v", node, why)
|
||||
}
|
||||
switch {
|
||||
case errors.Is(err, nats.ErrNoResponders):
|
||||
return HandOverAnswer{}, fmt.Errorf("nothing on %s answers a hand-over: its node-engine is not running, is not "+
|
||||
"on the bus, or is older than this ask (novox/hq issue 356); nothing was handed over", node)
|
||||
case errors.Is(err, context.DeadlineExceeded), errors.Is(err, nats.ErrTimeout):
|
||||
return HandOverAnswer{}, fmt.Errorf("%s did not answer the hand-over within %s; whether it was recorded is not "+
|
||||
"known — the module's condition says whether the directory is still used as found", node, timeout)
|
||||
case err != nil:
|
||||
return HandOverAnswer{}, err
|
||||
}
|
||||
var answer HandOverAnswer
|
||||
if err := json.Unmarshal(reply.Data, &answer); err != nil {
|
||||
return HandOverAnswer{}, fmt.Errorf("%s answered the hand-over with something unreadable: %w", node, err)
|
||||
}
|
||||
if answer.Said == "" && answer.Refused == "" {
|
||||
return HandOverAnswer{}, fmt.Errorf("%s answered the hand-over with neither a record nor a refusal", node)
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// SignHandOver fills the ask's expiry and nonce and signs it with the mesh's key, over HandOverContext and the
|
||||
// ask's bytes exactly as they travel.
|
||||
func SignHandOver(ctx context.Context, signer Signer, ask HandOverAsk) ([]byte, error) {
|
||||
if signer == nil {
|
||||
return nil, errors.New("no signing key, so no hand-over can be asked")
|
||||
}
|
||||
nonce := make([]byte, 16)
|
||||
if _, err := rand.Read(nonce); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ask.Nonce = hex.EncodeToString(nonce)
|
||||
ask.Expires = time.Now().UTC().Add(HandOverGood)
|
||||
raw, err := json.Marshal(ask)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
signature, err := signer.Sign(ctx, append([]byte(HandOverContext), raw...))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot sign the hand-over: %w", err)
|
||||
}
|
||||
return json.Marshal(SignedHandOver{Ask: raw, Signature: signature})
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The hand-over's ask and answer hold these field names; the engine's side holds the same list (mesh-host
|
||||
// internal/link, TestTheHandOverAskAndAnswerKeepTheirFieldNames).
|
||||
func TestTheHandOverAskAndAnswerKeepTheirFieldNames(t *testing.T) {
|
||||
body, _ := json.Marshal(HandOverAsk{Node: "laptop", Path: "/srv/notes", By: "jo through mesh-cli on anchor",
|
||||
Expires: time.Now(), Nonce: "n"})
|
||||
if got := keysIn(t, body); got != "by expires node nonce path" {
|
||||
t.Fatalf("the ask's fields are %q", got)
|
||||
}
|
||||
body, _ = json.Marshal(SignedHandOver{Ask: []byte("{}"), Signature: []byte("s")})
|
||||
if got := keysIn(t, body); got != "ask signature" {
|
||||
t.Fatalf("the signed ask's fields are %q", got)
|
||||
}
|
||||
body, _ = json.Marshal(HandOverAnswer{Said: "s", Refused: "r"})
|
||||
if got := keysIn(t, body); got != "refused said" {
|
||||
t.Fatalf("the answer's fields are %q", got)
|
||||
}
|
||||
if broker.AskHandOverSubject("laptop") != "mesh.node.laptop.ask.hand-over" {
|
||||
t.Fatalf("the subject is %q", broker.AskHandOverSubject("laptop"))
|
||||
}
|
||||
if HandOverContext != "novox-mesh hand-over v1\n" {
|
||||
t.Fatalf("the signing context is %q; the engine holds the same words", HandOverContext)
|
||||
}
|
||||
}
|
||||
|
||||
// keySigner signs with one key, as the controller's identity does.
|
||||
type keySigner struct{ key ed25519.PrivateKey }
|
||||
|
||||
func (k keySigner) Sign(_ context.Context, message []byte) ([]byte, error) {
|
||||
return ed25519.Sign(k.key, message), nil
|
||||
}
|
||||
|
||||
func testSigner(t *testing.T) (keySigner, ed25519.PublicKey) {
|
||||
t.Helper()
|
||||
public, private, err := ed25519.GenerateKey(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return keySigner{private}, public
|
||||
}
|
||||
|
||||
// **The ask is signed over the context and its bytes, with a fresh nonce and a short expiry** (review of issue
|
||||
// 356): the signature verifies over HandOverContext and the ask's bytes, and not over the bytes alone — so it is
|
||||
// never a declaration's — and two asks never share a nonce.
|
||||
func TestAHandOverIsSignedWithAContextANonceAndAnExpiry(t *testing.T) {
|
||||
signer, public := testSigner(t)
|
||||
body, err := SignHandOver(context.Background(), signer, HandOverAsk{Node: "laptop", Path: "/srv/notes", By: "jo"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var signed SignedHandOver
|
||||
if err := json.Unmarshal(body, &signed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !ed25519.Verify(public, append([]byte(HandOverContext), signed.Ask...), signed.Signature) {
|
||||
t.Fatal("the signature does not verify over the context and the ask")
|
||||
}
|
||||
if ed25519.Verify(public, signed.Ask, signed.Signature) {
|
||||
t.Fatal("the signature verifies over the ask's bytes alone, as a declaration's would")
|
||||
}
|
||||
var ask HandOverAsk
|
||||
if err := json.Unmarshal(signed.Ask, &ask); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ask.Node != "laptop" || ask.Path != "/srv/notes" || ask.By != "jo" || len(ask.Nonce) != 32 {
|
||||
t.Fatalf("signed %+v", ask)
|
||||
}
|
||||
if left := time.Until(ask.Expires); left <= 0 || left > HandOverGood {
|
||||
t.Fatalf("the ask is good for %s", left)
|
||||
}
|
||||
again, _ := SignHandOver(context.Background(), signer, HandOverAsk{Node: "laptop", Path: "/srv/notes", By: "jo"})
|
||||
var other SignedHandOver
|
||||
_ = json.Unmarshal(again, &other)
|
||||
var second HandOverAsk
|
||||
_ = json.Unmarshal(other.Ask, &second)
|
||||
if second.Nonce == ask.Nonce {
|
||||
t.Fatal("two asks share a nonce")
|
||||
}
|
||||
if _, err := SignHandOver(context.Background(), nil, HandOverAsk{}); err == nil {
|
||||
t.Fatal("an ask was made with no key")
|
||||
}
|
||||
}
|
||||
|
||||
// The ask reaches the machine's engine on its own subject and its answer comes back whole: what it recorded, or
|
||||
// its refusal as the engine worded it. A machine with no engine listening is said as not answering, and an
|
||||
// answer that is neither is refused rather than read as a record.
|
||||
func TestAHandOverIsAskedOfTheMachineAndItsAnswerComesBack(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
|
||||
signer, public := testSigner(t)
|
||||
var heard HandOverAsk
|
||||
engine, err := conn.Subscribe(broker.AskHandOverSubject("laptop"), func(msg *nats.Msg) {
|
||||
var signed SignedHandOver
|
||||
_ = json.Unmarshal(msg.Data, &signed)
|
||||
if !ed25519.Verify(public, append([]byte(HandOverContext), signed.Ask...), signed.Signature) {
|
||||
_ = msg.Respond([]byte(`{"refused":"not the mesh's signature"}`))
|
||||
return
|
||||
}
|
||||
_ = json.Unmarshal(signed.Ask, &heard)
|
||||
switch heard.Path {
|
||||
case "/srv/notes":
|
||||
body, _ := json.Marshal(HandOverAnswer{Said: "/srv/notes (notes.data) is handed to the mesh by " + heard.By})
|
||||
_ = msg.Respond(body)
|
||||
case "/srv/empty":
|
||||
_ = msg.Respond([]byte(`{}`))
|
||||
default:
|
||||
body, _ := json.Marshal(HandOverAnswer{Refused: heard.Path + " is not a directory this machine uses as found; nothing was handed over"})
|
||||
_ = msg.Respond(body)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = engine.Unsubscribe() })
|
||||
|
||||
ctx := context.Background()
|
||||
a, err := AskHandOver(ctx, conn, signer, "laptop", "/srv/notes", "jo through mesh-cli on anchor", 5*time.Second)
|
||||
if err != nil || a.Refused != "" || !strings.Contains(a.Said, "handed to the mesh by jo through mesh-cli on anchor") {
|
||||
t.Fatalf("answered %+v, %v", a, err)
|
||||
}
|
||||
if heard.Node != "laptop" || heard.Path != "/srv/notes" || heard.By != "jo through mesh-cli on anchor" {
|
||||
t.Fatalf("the engine heard %+v", heard)
|
||||
}
|
||||
a, err = AskHandOver(ctx, conn, signer, "laptop", "/srv/other", "jo", 5*time.Second)
|
||||
if err != nil || a.Said != "" || !strings.Contains(a.Refused, "nothing was handed over") {
|
||||
t.Fatalf("a refusal came back as %+v, %v", a, err)
|
||||
}
|
||||
if _, err := AskHandOver(ctx, conn, signer, "laptop", "/srv/empty", "jo", 5*time.Second); err == nil ||
|
||||
!strings.Contains(err.Error(), "neither") {
|
||||
t.Fatalf("an empty answer was taken: %v", err)
|
||||
}
|
||||
if _, err := AskHandOver(ctx, conn, signer, "anchor", "/srv/notes", "jo", 5*time.Second); err == nil ||
|
||||
!strings.Contains(err.Error(), "node-engine") {
|
||||
t.Fatalf("a machine with no engine listening: %v", err)
|
||||
}
|
||||
if _, err := AskHandOver(ctx, nil, signer, "anchor", "/srv/notes", "jo", time.Second); err == nil {
|
||||
t.Fatal("asked with no bus")
|
||||
}
|
||||
}
|
||||
|
||||
// A machine's grant hears its own hand-over ask and nobody else's, and may answer it: the one request a node is
|
||||
// asked (novox/hq issue 356).
|
||||
func TestAMachineHearsItsOwnHandOverAskAndMayAnswerIt(t *testing.T) {
|
||||
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindNode, Node: "laptop", PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var hears, other bool
|
||||
for _, s := range perms.Subscribe {
|
||||
hears = hears || s == "mesh.node.laptop.ask.hand-over"
|
||||
other = other || s == "mesh.node.anchor.ask.hand-over"
|
||||
}
|
||||
if !hears || other || !perms.AllowResponses {
|
||||
t.Fatalf("a machine's grant: hears its own %v, another's %v, answers %v (%v)", hears, other, perms.AllowResponses,
|
||||
perms.Subscribe)
|
||||
}
|
||||
}
|
||||
@@ -471,8 +471,9 @@ const KindUnit = "unit"
|
||||
const KindAccount = "account"
|
||||
|
||||
// KindDirectory is a directory of a module that the node-engine uses as found (novox/hq issue 339): there before
|
||||
// the mesh, with another owner or mode than declared, and left so until a person hands it over at the machine
|
||||
// (`mesh-host hand-over`). Stated unhealthy with a reason that starts ReasonUsedAsFound.
|
||||
// the mesh, with another owner or mode than declared, and left so until the operator hands it over at the
|
||||
// controller's terminal (`nox node hand-over <node> <path>`, issue 356). Stated unhealthy with a reason that starts
|
||||
// ReasonUsedAsFound.
|
||||
const KindDirectory = "directory"
|
||||
|
||||
// ReasonUsedAsFound starts the reason of a directory used as found.
|
||||
|
||||
@@ -46,7 +46,7 @@ bed asserts genesis **says** it found and took the tunnel.
|
||||
**Review changes (2026-09-24).** A spoke's `wg0.conf` names one peer — the hub — routed the
|
||||
whole range; the controller skips range-routed peers, so T2's enrolment carries no peer from the
|
||||
spoke. A hub that enrolled *before* this feature (a generated key) takes the tunnel over without
|
||||
re-enrolling: `mesh-host overlay take --tunnel wg0` on the machine rekeys the overlay key only and
|
||||
re-enrolling: `nox-mesh-host overlay take --tunnel wg0` on the machine rekeys the overlay key only and
|
||||
sends a signed rekey; the bed adds **R0** for it below. A takeover is composed only for a hub
|
||||
placed at the tunnel's address on the tunnel's port, and the host stops nothing until the declared
|
||||
interface matches the found one and the key file holds the found key; a mesh interface that fails
|
||||
@@ -58,7 +58,7 @@ to start gives the found unit back. The host's account has three states: `not-ta
|
||||
- **R0 — a hub enrolled with its own key takes the tunnel over by rekeying.** Genesis is run
|
||||
adopted *without* the tunnel being found (the bed stops `wg-quick@wg0` for the run, so the
|
||||
installer sees no tunnel, then starts it again — the pre-feature shape). Then on the anchor:
|
||||
`mesh-host overlay take --tunnel wg0`; `node show anchor` says "tunnel found wg0 …"; `overlay
|
||||
`nox-mesh-host overlay take --tunnel wg0`; `node show anchor` says "tunnel found wg0 …"; `overlay
|
||||
place anchor --hub --endpoint 192.0.2.10:51900 --site hosting` (with `:51820` first, which must
|
||||
be refused naming 51900); `plan anchor --json` names `Address = 10.10.0.1/32`, `ListenPort =
|
||||
51900`, two `/32` peers, `takes-over` wg0 and nothing in 10.42.0.0/16; then `push anchor --wait
|
||||
@@ -82,7 +82,7 @@ to start gives the found unit back. The host's account has three states: `not-ta
|
||||
- `overlay show` lists `anchor` as the hub over the tunnel it took over, and both peers under
|
||||
"peers of the tunnel … not nodes of the mesh", not yet enrolled.
|
||||
- **T2 — a peer enrols and keeps its address.** On `peer-a`: `node add peer-a --adopted`,
|
||||
token issued, `mesh-host enrol --token …` **with the broker reached over the tunnel** (the
|
||||
token issued, `nox-mesh-host enrol --token …` **with the broker reached over the tunnel** (the
|
||||
broker address in the token is `10.10.0.1:<bus>`, which only the tunnel routes); then `overlay
|
||||
place peer-a --site house` and a push. Assert: `node show peer-a` says a tunnel `wg0` was
|
||||
found and `overlay show` puts `peer-a` at **10.10.0.2**; the carried-peers list now says
|
||||
|
||||
Reference in New Issue
Block a user