diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index 9e48b6a..e871f34 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -1133,4 +1133,50 @@ func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) { if failed.Verdict == nil || !strings.Contains(strings.Join(failed.Verdict.Fails, "; "), "laptop (offline) did not answer") { t.Errorf("the silent node's verdict = %+v", failed.Verdict) } + + // 774/S59 (ADR 0587 §7): a joining node is written on the enrolment's warrant with no node to answer and no ask raised, + // and placement-changed carries that warrant. The assigner reads no lease (ADR 0567 §8): the enrolment's own issue waits + // for the lease write (ADR 0587 §4). + for len(heard) > 0 { + <-heard + } + joined, err := verbs.doPropose(map[string]any{"why": "enrolment of ace", "warrant": `{"node":"ace","address":"10.9.0.2","ask":"enrol-ace-1","warrant":"enrol-ace-1@2026-10-28T20:00:00Z"}`, + "changes": `[{"op":"add-node","node":"ace","decided":{"address":"10.9.0.2"},"keys":{"overlay":"dHVubmVs"}}]`}, "novox.mesh-enrolment") + if err != nil { + t.Fatal(err) + } + select { + case e := <-heard: + if e.Proposal.ID != joined.(proposed).Proposal.ID || e.Warrant == nil || e.Warrant.Warrant != "enrol-ace-1@2026-10-28T20:00:00Z" || len(e.Nodes) != 1 || e.Nodes[0].Node != "ace" { + t.Errorf("placement-changed for the joining node = %+v", e) + } + case <-time.After(15 * time.Second): + t.Fatal("no placement-changed heard for the joining node") + } + if ace, err := b.buckets[delivery.PlacementBucket].Get(b.ctx, "ace.node"); err != nil || !strings.Contains(string(ace.Value()), "10.9.0.2") { + t.Errorf("the joining node's key = %v, %v", ace, err) + } + if akey, _ := delivery.PlacementAskKey(joined.(proposed).Proposal.ID); b.hasKey(delivery.AssignerAsksBucket, akey) || b.hasKey(delivery.PlacementProposalsBucket, joined.(proposed).Proposal.ID) { + t.Error("an ask was raised for the joining node, or its proposal stays open") + } + // Its enrolment ends before it confirms: a remove-node on the same warrant deletes its key at once, with no ask, and + // placement-changed carries the warrant (the operator's decision of 2026-11-02 01:05). + for len(heard) > 0 { + <-heard + } + if _, err := verbs.doPropose(map[string]any{"why": "enrolment of ace revoked", "warrant": `{"node":"ace","address":"10.9.0.2","ask":"enrol-ace-1","warrant":"enrol-ace-1@2026-10-28T20:00:00Z"}`, + "changes": `[{"op":"remove-node","node":"ace"}]`}, "novox.mesh-enrolment"); err != nil { + t.Fatal(err) + } + select { + case e := <-heard: + if e.Warrant == nil || len(e.Nodes) != 1 || e.Nodes[0].Node != "ace" { + t.Errorf("placement-changed for the removed node = %+v", e) + } + case <-time.After(15 * time.Second): + t.Fatal("no placement-changed heard for the removed node") + } + if _, err := b.buckets[delivery.PlacementBucket].Get(b.ctx, "ace.node"); err == nil { + t.Error("the removed node's key stays") + } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go index c8ef722..aba87c5 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go @@ -268,6 +268,10 @@ func (p *passer) stepProposal(ctx context.Context, prop delivery.PlacementPropos p.endProposal(ctx, prop, "adopt and hand-back are gone since the lease write: placement is the mesh's alone") return } + if prop.Warrant != nil { + p.addNode(ctx, prop, revs) + return + } akey, _ := delivery.PlacementAskKey(prop.ID) again := !prop.Again.IsZero() && (prop.Verdict == nil || prop.Again.After(prop.Verdict.At)) if prop.State == delivery.ProposalAsked { @@ -544,6 +548,71 @@ func (p *passer) warranted(ctx context.Context, prop delivery.PlacementProposal, p.followWrite(prop, answer, changes) } +// addNode writes a joining node on the warrant of the enrolment that issued its token (ADR 0587 §7): the operator's Approve +// was the issue's, so no node is waited on and no ask is raised. The key is created at the revision it was laid over (none: a +// node with a key is no joining node); the enrolment recorded the warrant in the hand-act log when it was given, so this does +// not record it again; placement-changed carries the warrant. A key written meanwhile ends the proposal, saying so. +func (p *passer) addNode(ctx context.Context, prop delivery.PlacementProposal, revs map[string]uint64) { + current, _ := p.seatPlacements() + if moved(prop, revs) { + p.endProposal(ctx, prop, "a key it was laid over was written since: the enrolment proposes the node again") + return + } + laid, findings := delivery.LayChanges(current, prop.Changes) + node := prop.Changes[0].Node + if prop.Changes[0].Op == delivery.OpRemoveNode { + p.removeNode(ctx, prop, current, findings) + return + } + value, ok := laid[node] + if len(findings) > 0 || !ok { + p.endProposal(ctx, prop, "the node cannot be added: "+strings.Join(findings, "; ")) + return + } + rev, err := p.writeNode(ctx, node, &value) + if err != nil { + p.endProposal(ctx, prop, "the node's key was not written: "+err.Error()) + return + } + c := delivery.ChangeOf(delivery.Placement{Node: node}, value) + c.Before, c.After = prop.Base[node], rev + p.deleteCandidates(ctx, prop.ID) + p.deletePulses(ctx, prop.ID) + _ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID) + p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil) + w := prop.Warrant + p.sayChanged(prop, delivery.Answer{Option: assign.OptionYes, Warrant: w.Warrant, + Words: fmt.Sprintf("the operator approved the machine %s at %s through the enrolment's ask %s", w.Node, w.Address, w.Ask)}, + []delivery.PlacementChange{c}) +} + +// removeNode deletes the key of a joining node whose enrolment ended unconfirmed, on that enrolment's warrant (the operator's +// decision of 2026-11-02 01:05): no node answers and no ask is raised, as for its add-node; placement-changed carries the warrant. +// A node that runs modules or holds seats is no joining node and stays (LayChanges' finding ends the proposal). +func (p *passer) removeNode(ctx context.Context, prop delivery.PlacementProposal, current map[string]delivery.Placement, findings []string) { + node := prop.Changes[0].Node + before, ok := current[node] + if len(findings) > 0 || !ok { + p.endProposal(ctx, prop, "the node cannot be taken off: "+strings.Join(findings, "; ")) + return + } + rev, err := p.writeNode(ctx, node, nil) + if err != nil { + p.endProposal(ctx, prop, "the node's key was not deleted: "+err.Error()) + return + } + c := delivery.ChangeOf(before, delivery.Placement{Node: node}) + c.Before, c.After = prop.Base[node], rev + p.deleteCandidates(ctx, prop.ID) + p.deletePulses(ctx, prop.ID) + _ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID) + p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil) + w := prop.Warrant + p.sayChanged(prop, delivery.Answer{Option: assign.OptionYes, Warrant: w.Warrant, + Words: fmt.Sprintf("the machine %s left placement: its enrolment (ask %s) ended before it confirmed", w.Node, w.Ask)}, + []delivery.PlacementChange{c}) +} + // followWrite follows the keys a warrant wrote: the warrant recorded and placement-changed said. No verb of the // controller is called: the placement is the one source of where modules run (ADR 0567 §8, issue 774). func (p *passer) followWrite(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange) { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go index 15a565e..8f944dd 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go @@ -217,9 +217,16 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) { if err != nil { return nil, err } + warrant, err := warrantOf(args["warrant"]) + if err != nil { + return nil, err + } now := s.now() p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now, - Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded}) + Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Warrant: warrant}) + if refusal := delivery.AddNodeRefusal(p, moduleOfCaller(by)); refusal != "" { + findings = append(findings, refusal) + } if len(findings) > 0 { return nil, fmt.Errorf("the proposal is refused: %s", strings.Join(findings, "; ")) } @@ -229,6 +236,42 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) { return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil } +// warrantOf reads propose's warrant, an enrolment's for an add-node: an object, given as one or as its text; none is none. +func warrantOf(v any) (*delivery.AddNodeWarrant, error) { + var raw []byte + switch w := v.(type) { + case nil: + return nil, nil + case string: + if strings.TrimSpace(w) == "" { + return nil, nil + } + raw = []byte(w) + default: + var err error + if raw, err = json.Marshal(w); err != nil { + return nil, err + } + } + var out delivery.AddNodeWarrant + dec := json.NewDecoder(strings.NewReader(string(raw))) + dec.DisallowUnknownFields() + if err := dec.Decode(&out); err != nil { + return nil, fmt.Errorf("the warrant is no enrolment's warrant {node, address, ask, warrant}: %w", err) + } + return &out, nil +} + +// moduleOfCaller is the module a caller is, as the runtime proves it, for AddNodeRefusal: mesh-enrolment when the proven +// caller is that module's (its bus user `.mesh-enrolment`, never a person's `person.mesh-enrolment`), else the caller +// as proven, which the refusal names. +func moduleOfCaller(by string) string { + if (stdio.Callers{delivery.EnrolmentModule}).Allows(by) { + return delivery.EnrolmentModule + } + return by +} + // doWithdraw is the verb withdraw (§2): the proposal is marked, and the assigner deletes it and its candidates and says // placement-withdrawn carrying it whole. func (s *seatTools) doWithdraw(args map[string]any, by string) (any, error) { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go index 6e1d736..2636592 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go @@ -150,6 +150,50 @@ func TestTheSeatsVerbs(t *testing.T) { if _, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op"); err != nil { t.Errorf("a hold: %v", err) } + // 774/S59/AC1 (ADR 0587 §7): an add-node from mesh-enrolment whose warrant names this node and this address is proposed, with + // the node a joining one with no record; from another caller, naming another node or address, or with no warrant it is refused. + addNode := func(by, warrant string) error { + _, err := s.doPropose(map[string]any{"why": "enrolment of ace", "warrant": warrant, "changes": []any{map[string]any{"op": "add-node", "node": "ace", + "decided": map[string]any{"address": "10.9.0.2"}, "keys": map[string]any{"overlay": "dHVubmVs"}}}}, by) + return err + } + good := `{"node":"ace","address":"10.9.0.2","ask":"enrol-ace-1","warrant":"enrol-ace-1@2026-10-28T20:00:00Z"}` + for _, c := range []struct{ by, warrant, want string }{ + {"novox.mesh-grants", good, "mesh-enrolment's alone"}, + {"person.mesh-enrolment", good, "mesh-enrolment's alone"}, + {"novox.mesh-enrolment", `{"node":"g14","address":"10.9.0.2","ask":"a","warrant":"a@t"}`, "carries the warrant of g14"}, + {"novox.mesh-enrolment", `{"node":"ace","address":"10.9.0.9","ask":"a","warrant":"a@t"}`, "carries the warrant of ace at 10.9.0.9"}, + {"novox.mesh-enrolment", "", "carries no warrant"}, + } { + if err := addNode(c.by, c.warrant); err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("an add-node by %s with warrant %q: %v, want a refusal saying %q", c.by, c.warrant, err, c.want) + } + } + if err := addNode("novox.mesh-enrolment", good); err != nil { + t.Errorf("an add-node from mesh-enrolment: %v", err) + } + // A remove-node (the operator's decision of 2026-11-02 01:05) takes a joining node that never confirmed off placement: from + // mesh-enrolment, on the warrant naming that node, of a node running nothing, it is proposed with the node's key to delete; + // of a node running modules, or with another node's warrant, it is refused. + joiner, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "joiner", Decided: delivery.NodeDecided{Address: "10.9.0.7"}, Modules: []delivery.PlacedModule{}}) + st[delivery.PlacementState].values["placement.joiner"] = joiner + removeNode := func(node, warrant string) (any, error) { + return s.doPropose(map[string]any{"why": "enrolment ended", "warrant": warrant, + "changes": []any{map[string]any{"op": "remove-node", "node": node}}}, "novox.mesh-enrolment") + } + for _, c := range []struct{ node, warrant, want string }{ + {"anchor", `{"node":"anchor","address":"10.9.0.1","ask":"a","warrant":"a@t"}`, "runs modules or holds seats"}, + {"joiner", `{"node":"g14","address":"10.9.0.7","ask":"a","warrant":"a@t"}`, "carries the warrant of g14"}, + } { + if _, err := removeNode(c.node, c.warrant); err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("a remove-node of %s: %v, want a refusal saying %q", c.node, err, c.want) + } + } + if got, err := removeNode("joiner", `{"node":"joiner","address":"10.9.0.7","ask":"a","warrant":"a@t"}`); err != nil { + t.Errorf("a remove-node from mesh-enrolment: %v", err) + } else if rm := got.(proposed).Proposal; !slices.Equal(rm.Nodes(), []string{"joiner"}) || !slices.Equal(rm.RemovedNodes(), []string{"joiner"}) { + t.Errorf("the remove-node proposed %+v", rm) + } if _, err := s.doHandBack(map[string]any{"why": "back"}, "op"); err == nil || !strings.Contains(err.Error(), "since the lease write") { t.Errorf("a hand-back: %v", err) } diff --git a/modules/mesh-delivery-assigner/go.mod b/modules/mesh-delivery-assigner/go.mod index def263e..4a1f553 100644 --- a/modules/mesh-delivery-assigner/go.mod +++ b/modules/mesh-delivery-assigner/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-assigner go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-assigner/go.sum b/modules/mesh-delivery-assigner/go.sum index 60e9ab1..633697d 100644 --- a/modules/mesh-delivery-assigner/go.sum +++ b/modules/mesh-delivery-assigner/go.sum @@ -2,6 +2,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027 h1:a33dsavo git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c h1:/YCxh+NYfqry3PxZYY1nQXaccupobxpsFt8c9BYuGBg= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 h1:4I4ayo0fZYP4PBfEX1sGDDd6arnw5IQJySJ3jcFobPM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-assigner/internal/assign/compose_test.go b/modules/mesh-delivery-assigner/internal/assign/compose_test.go index c8921b4..1e202f5 100644 --- a/modules/mesh-delivery-assigner/internal/assign/compose_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/compose_test.go @@ -68,33 +68,33 @@ func TestAChangeOfOneModuleComposesItsMachine(t *testing.T) { } } -// 1297/AC2: a change that removes a module a machine runs fails, naming the machine and the module. -func TestRemovingAModuleAMachineRunsFails(t *testing.T) { - c := Change{Group: "g", Heads: "0123456789ab", Removed: []string{"web"}} - v := Compose(meshOf(), c, "sdk", 0) - if v.State != delivery.CompositionFail || v.Machine != "g14" || !strings.Contains(v.Words, "g14 runs web") { - t.Fatalf("composition = %+v, want a fail naming g14 and web", v) - } -} - -// 1297/B3c: a machine that composes on the trunk and not with the change fails it, naming the machine and why; the -// consumer's machine is touched when its provider changes. -func TestAMachineTheChangeBreaksFails(t *testing.T) { - c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"store": {Module: "store", Version: "2"}}} - v := Compose(meshOf(), c, "sdk", 0) - if v.State != delivery.CompositionFail || v.Machine != "ace" || !strings.Contains(v.Words, "database") { - t.Fatalf("composition = %+v, want a fail on ace naming the provision nothing provides", v) - } -} - -// 1297/B3c: what was already broken on the trunk is said and fails nothing. -func TestWhatWasBrokenBeforeFailsNothing(t *testing.T) { - m := meshOf() - m.Manifests["store"] = catalogue.Manifest{Module: "store", Version: "1"} - c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"mail": {Module: "mail", Version: "2", Requires: []string{"database"}}}} - v := Compose(m, c, "sdk", 0) - if v.State != delivery.CompositionPass || !hasWarning(v, "on the trunk either") { - t.Fatalf("composition = %+v, want a pass saying ace was already broken", v) +// 1297/AC2 and B3c: a change that removes a module a machine runs fails, naming the machine and the module; a machine that +// composes on the trunk and not with the change fails it, naming the machine and why (the consumer's machine is touched when +// its provider changes); what was already broken on the trunk is said and fails nothing. One table (three tests merged). +func TestAChangeThatBreaksAMachineFailsAndWhatWasBrokenFailsNothing(t *testing.T) { + broken := meshOf() + broken.Manifests["store"] = catalogue.Manifest{Module: "store", Version: "1"} + for _, c := range []struct { + name string + mesh Mesh + c Change + state string + must func(delivery.CompositionVerdict) bool + }{ + {"removing a module a machine runs", meshOf(), Change{Group: "g", Heads: "0123456789ab", Removed: []string{"web"}}, delivery.CompositionFail, + func(v delivery.CompositionVerdict) bool { + return v.Machine == "g14" && strings.Contains(v.Words, "g14 runs web") + }}, + {"a machine the change breaks", meshOf(), Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"store": {Module: "store", Version: "2"}}}, + delivery.CompositionFail, func(v delivery.CompositionVerdict) bool { + return v.Machine == "ace" && strings.Contains(v.Words, "database") + }}, + {"what was broken before", broken, Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"mail": {Module: "mail", Version: "2", Requires: []string{"database"}}}}, + delivery.CompositionPass, func(v delivery.CompositionVerdict) bool { return hasWarning(v, "on the trunk either") }}, + } { + if v := Compose(c.mesh, c.c, "sdk", 0); v.State != c.state || !c.must(v) { + t.Errorf("%s: composition = %+v", c.name, v) + } } } diff --git a/modules/mesh-delivery-assigner/internal/assign/seat.go b/modules/mesh-delivery-assigner/internal/assign/seat.go index 435ac4e..e9f9990 100644 --- a/modules/mesh-delivery-assigner/internal/assign/seat.go +++ b/modules/mesh-delivery-assigner/internal/assign/seat.go @@ -24,6 +24,8 @@ type ProposeIn struct { Revisions map[string]uint64 Held func(module string) bool Recorded func(node string) bool + // Warrant is the enrolment's warrant an add-node carries (ADR 0587 §7); only that change has one. + Warrant *delivery.AddNodeWarrant } // Propose lays a proposal over the current keys (§2): the proposal (state answering, each changed node's base revision), @@ -33,7 +35,7 @@ type ProposeIn struct { // record). A key whose value the proposal leaves as it is changes nothing and is not laid. func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Placement, []string) { p := delivery.PlacementProposal{ID: in.ID, By: strings.TrimSpace(in.By), Why: strings.TrimSpace(in.Why), At: in.At.UTC(), - Changes: in.Changes, Base: map[string]uint64{}, State: delivery.ProposalAnswering} + Changes: in.Changes, Warrant: in.Warrant, Base: map[string]uint64{}, State: delivery.ProposalAnswering} var findings []string if p.By == "" { findings = append(findings, "the runtime names no caller: a proposal is made by someone") @@ -65,6 +67,16 @@ func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Pla out[node] = &pl p.Base[node] = in.Revisions[node] } + // A remove-node deletes the key of a joining node that never confirmed (the operator's decision of 2026-11-02 01:05): its + // candidate is no placement, and the checks across nodes read the mesh without it. LayChanges said why one cannot go. + for _, node := range p.RemovedNodes() { + if _, ok := in.Current[node]; !ok { + continue + } + delete(all, node) + out[node] = nil + p.Base[node] = in.Revisions[node] + } if len(findings) == 0 && len(out) == 0 { findings = append(findings, "the changes leave every placement as it is") } @@ -75,6 +87,18 @@ func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Pla if recorded == nil { recorded = func(string) bool { return false } } + // A joining node writes its record only once it holds its account, which the add-node's write gives it: this one change by + // this one caller is the exception to a node with no record (ADR 0587 §7, changing ADR 0567 §2). + joining := map[string]bool{} + for _, c := range in.Changes { + if c.Op == delivery.OpAddNode { + joining[c.Node] = true + } + } + if len(joining) > 0 { + was := recorded + recorded = func(node string) bool { return joining[node] || was(node) } + } for _, r := range delivery.CheckPlacements(all, held, recorded) { findings = append(findings, r) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index bae4fd6..6c231fb 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -467,15 +467,20 @@ func TestTwoBuilderInstancesStartingTogetherEndWithOneDoneKey(t *testing.T) { // } } -// TestAnArchiveOnlyModuleNamingNoToolchainIsBuilt: the indexer gives an archive no language, so modules.ca-trust has no -// toolchains; the builder builds it on the build module's nodes all the same (review round 4). -func TestAnArchiveOnlyModuleNamingNoToolchainIsBuilt(t *testing.T) { - b := startBed(t) - b.archive(t, "ca-trust", "modules/ca-trust", commit) - b.start(t, "n1", nil) - v := b.await(t, "ca-trust", isState(delivery.BuildDone)) - if len(v.Toolchains) != 0 || len(v.Artifacts) != 1 || strings.HasPrefix(v.Step, "one node (") { - t.Errorf("build = %+v, want done naming no toolchain", v) +// TestAnArchiveOnlyModuleAndARootModuleAreBuilt: the indexer gives an archive no language, so modules.ca-trust has no +// toolchains; the builder builds it on the build module's nodes all the same (review round 4). A module whose path is "." +// is given to the build tools as it is (1615/AC4). One table: the two tests were merged to pay for the enrolment's. +func TestAnArchiveOnlyModuleAndARootModuleAreBuilt(t *testing.T) { + for _, c := range []struct{ name, path string }{{"no toolchain", "modules/ca-trust"}, {"root", delivery.RootPath}} { + t.Run(c.name, func(t *testing.T) { + b := startBed(t) + b.archive(t, "ca-trust", c.path, commit) + b.start(t, "n1", nil) + v := b.await(t, "ca-trust", isState(delivery.BuildDone)) + if c.name == "no toolchain" && (len(v.Toolchains) != 0 || len(v.Artifacts) != 1 || strings.HasPrefix(v.Step, "one node (")) { + t.Errorf("build = %+v, want done naming no toolchain", v) + } + }) } } @@ -530,14 +535,6 @@ func TestAToolchainChangeDoesNotBuildTheOldFingerprint(t *testing.T) { // 1615/A } } -// TestARootModuleIsBuilt: a module whose path is "." is given to the build tools as it is. -func TestARootModuleIsBuilt(t *testing.T) { // 1615/AC4 - b := startBed(t) - b.archive(t, "ca-trust", delivery.RootPath, commit) - b.start(t, "n1", nil) - b.await(t, "ca-trust", isState(delivery.BuildDone)) -} - func mustKey(t *testing.T, module string) string { k, err := delivery.BuilderBuildKey(module, fp) if err != nil { diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index 95009ea..236cb80 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index 8a22cf0..cf2ce55 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -2,6 +2,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a h1:AChSJUmH git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c h1:/YCxh+NYfqry3PxZYY1nQXaccupobxpsFt8c9BYuGBg= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 h1:4I4ayo0fZYP4PBfEX1sGDDd6arnw5IQJySJ3jcFobPM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/judge/rules.go b/modules/mesh-delivery-indexer/judge/rules.go index 37dd9f2..fa3ef9e 100644 --- a/modules/mesh-delivery-indexer/judge/rules.go +++ b/modules/mesh-delivery-indexer/judge/rules.go @@ -2450,19 +2450,24 @@ func parseManifest(raw []byte, predefined catalogue.PredefinedList) (catalogue.M // holding mesh-broker is granted the snapshot API and nothing else, so anything it declared to say // or hear on the bus would be granted nothing — refused here rather than silently dropped. // - // **But its tools bundle may say what it declares** (novox/hq ADR 0357): the node's runtime carries - // the bundle and is granted the events the module emits, under the module's name, as for any module it - // carries — the bus's account is not who publishes them. So emits are refused only for a bus module - // with no bundle to say them. + // **But its tools bundle may say and ask what it declares** (novox/hq ADR 0357): the node's runtime carries + // the bundle, is granted the events the module emits under the module's name, as for any module it carries, + // and asks every tool and seat verb on the bundle's behalf (mesh-sdk go/buscompose, the runtime's invoke of + // `*`) — the bus's account is not who publishes or asks. So emits and invokes are refused only for a bus + // module with no bundle to say or ask them (issue 774: nats's bundle records its hand act). if _, account := m.OwnSecrets["broker"]; account && m.ClaimsSeat(catalogue.BrokerSeat) { var said []string - if len(m.EmitsAll()) > 0 && !manifestShipsABundle(m) { + bundled := manifestShipsABundle(m) + if len(m.EmitsAll()) > 0 && !bundled { said = append(said, "emits") } + if len(m.Invokes) > 0 && !bundled { + said = append(said, "invokes") + } for _, f := range []struct { name string n int - }{{"consumes", len(m.Consumes)}, {"uses", len(m.Uses)}, {"invokes", len(m.Invokes)}, + }{{"consumes", len(m.Consumes)}, {"uses", len(m.Uses)}, {"state", len(m.State)}, {"logs", len(m.Logs)}, {"reads", len(m.Reads)}, {"streams", len(m.Streams)}} { if f.n > 0 { diff --git a/modules/mesh-delivery-indexer/judge/rules_test.go b/modules/mesh-delivery-indexer/judge/rules_test.go index 5445bd7..75bd736 100644 --- a/modules/mesh-delivery-indexer/judge/rules_test.go +++ b/modules/mesh-delivery-indexer/judge/rules_test.go @@ -4337,6 +4337,41 @@ func TestTheBussAccountSaysNothingOnTheBus(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "granted the bus's snapshot API and nothing else") { t.Fatalf("a bus module declaring what it emits was not refused, or not for the reason: %v", err) } + // Nor what it invokes, with no bundle for the runtime to ask on its behalf. + unbundledInvoke := []byte(`{"module":"bus","version":"1","provides":[{"name":"mesh-bus","scope":"mesh"}], + "claims":[{"name":"mesh-broker","scope":"mesh"}],"own-secrets":{"broker":"/run/broker"}, + "invokes":["seat:mesh-hand-acts.hand-act"]}`) + if _, err := ParseManifest(unbundledInvoke); err == nil || !strings.Contains(err.Error(), "its invokes would be granted nothing") { + t.Fatalf("a bus module with no bundle declaring what it invokes was not refused, or not for the reason: %v", err) + } + + // Its tools bundle asks what the module invokes (novox/hq ADR 0357; issue 774): the bundle asks through the + // node's runtime, which is granted every tool and seat verb (mesh-sdk go/buscompose), so a bus module that + // ships one may declare them. The shape of mesh-catalog's modules/nats/module.json at fc5b849d, cut to the + // parts the rule reads, was refused and held every bus user list from 2026-11-02 01:35. + nats := []byte(`{"module":"nats","version":"1","provides":[{"name":"mesh-bus","scope":"mesh","credential":{"form":"none"}}], + "claims":[{"name":"mesh-broker","scope":"mesh"}], + "own-secrets":{"broker":"/run/broker","broker-nats-tools":"${dir:tools-secret}/broker-nats-tools"}, + "resources":[{"id":"tools-secret","type":"directory","path":"/var/lib/mesh/nats-tools","mode":"0711","unknown":"program"}], + "emits":["consumer-removed","bucket-keys-removed","stream-removed","bucket-expiry-set"], + "invokes":["seat:mesh-hand-acts.hand-act"],"consumes":[], + "tools":["nats_server","nats_bucket_expiry"], + "build":{"artifacts":[{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/nats-tools", + "binary":"nats-tools","loads":["nats-tools"],"bus":{"removes":["DEAD_LETTER_NOTICES.controller","ASSIGNMENTS"]}}]}}`) + nm, err := ParseManifest(nats) + if err != nil { + t.Fatalf("the bus module whose tools bundle invokes a seat verb was refused: %v", err) + } + nbuilt, err := nm.Resolve([]catalogue.Built{{Name: "tools", Kind: catalogue.ArtifactBundle, Reference: "store/tools:1", Digest: "sha256:1"}}) + if err != nil { + t.Fatal(err) + } + if raw, err = json.Marshal(nbuilt); err != nil { + t.Fatal(err) + } + if _, err := ParseManifestForInputs(raw); err != nil { + t.Fatalf("the built manifest of the bus module whose tools bundle invokes a seat verb was refused: %v", err) + } // Its tools bundle says what it emits (novox/hq ADR 0357): the node's runtime carries the bundle and is // granted the module's events, so a bus module that ships one may declare them. @@ -5014,9 +5049,9 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) { // Forty-eight with mesh-delivery-checker (novox/hq ADR 0547 §10, issue 774); forty-nine with node-engine (novox/hq ADR 0563 §1); // fifty-three with the four health seats (novox/hq ADR 0564 clause 1); fifty-five with mesh-wants and mesh-hand-acts // (novox/hq ADR 0565 clause 1); fifty-six with mesh-artifact-sweep (novox/hq ADR 0571 clause 2); - // fifty-seven with mesh-grants (ADR 0582 clause 1). - if len(catalogue.Seats()) != 57 { - t.Errorf("the mesh defines %d seats rather than 57; the set is closed, so a change here is "+ + // fifty-seven with mesh-grants (ADR 0582 clause 1); fifty-eight with mesh-enrolment (novox/hq ADR 0587 §1). + if len(catalogue.Seats()) != 58 { + t.Errorf("the mesh defines %d seats rather than 58; the set is closed, so a change here is "+ "a decision (novox/hq ADR 0110): %s", len(catalogue.Seats()), seatNames()) } } diff --git a/modules/mesh-delivery-watchman/go.mod b/modules/mesh-delivery-watchman/go.mod index a74ba03..7d4da6f 100644 --- a/modules/mesh-delivery-watchman/go.mod +++ b/modules/mesh-delivery-watchman/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-watchman go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-watchman/go.sum b/modules/mesh-delivery-watchman/go.sum index e59eb50..b494e51 100644 --- a/modules/mesh-delivery-watchman/go.sum +++ b/modules/mesh-delivery-watchman/go.sum @@ -2,6 +2,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMi git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c h1:/YCxh+NYfqry3PxZYY1nQXaccupobxpsFt8c9BYuGBg= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 h1:4I4ayo0fZYP4PBfEX1sGDDd6arnw5IQJySJ3jcFobPM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index d25eec9..e1f83c2 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -246,7 +246,7 @@ func TestTheWatchmansFindings(t *testing.T) { w.Provision("ace.web.oidc", enc(provisions.Waits, "x"), Put) w.Provision("ace.web.oidc", nil, Delete) }, map[string]string{}}, - {"774: a node's own report: its network, the resolver file rewritten, failed units, a relogin, a directory used as found", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + {"774: a node's own report: its network, the resolver file rewritten, failed units, a relogin, a directory used as found, a login fallen back", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { report := func(h delivery.NodeHealth) []byte { b, err := delivery.EncodeNodeHealth(h) if err != nil { @@ -259,11 +259,13 @@ func TestTheWatchmansFindings(t *testing.T) { {Part: "resolv-conf", State: "unhealthy", Writer: "NetworkManager", Owner: "systemd-resolved", Said: "rewritten"}, {Part: "names", State: "unhealthy", Reason: "names do not resolve.", Said: "x"}, {Part: "route", State: "unhealthy", Reason: "no default route", Said: "y"}}}, - Units: &delivery.UnitsHealth{State: "degraded", Failed: []delivery.FailedUnit{{Unit: "cups.service", Scope: "system", Result: "exit-code"}}}, - Relogin: []delivery.NodeAccount{{Account: "jochen", Module: "docker", Reason: "a group"}}, - Found: []delivery.NodeDirectory{{Module: "photos", Resource: "data", Path: "/srv/photos", Reason: "owner differs"}}}), Put) + Units: &delivery.UnitsHealth{State: "degraded", Failed: []delivery.FailedUnit{{Unit: "cups.service", Scope: "system", Result: "exit-code"}}}, + Relogin: []delivery.NodeAccount{{Account: "jochen", Module: "docker", Reason: "a group"}}, + Found: []delivery.NodeDirectory{{Module: "photos", Resource: "data", Path: "/srv/photos", Reason: "owner differs"}}, + Fallback: &delivery.LoginFallback{Node: "novox", From: "node.novox--b", Why: "refused twice", At: time.Date(2026, 11, 2, 12, 0, 0, 0, time.UTC)}}), Put) for key, sev := range map[string]string{"machine.novox.network": Urgent, "machine.novox.systemd-resolved.rewritten": Urgent, - "machine.novox.units": Warning, "module.docker.novox.relogin-needed": Warning, "module.photos.novox.directory-used-as-found": Warning} { + "machine.novox.units": Warning, "module.docker.novox.relogin-needed": Warning, "module.photos.novox.directory-used-as-found": Warning, + "machine.novox.node-account-fallback": Warning} { if o.findings[key].Severity != sev { t.Errorf("%s = %+v", key, o.findings[key]) } diff --git a/modules/mesh-delivery-watchman/internal/watch/nodehealth.go b/modules/mesh-delivery-watchman/internal/watch/nodehealth.go index d3b6dd3..eda9df4 100644 --- a/modules/mesh-delivery-watchman/internal/watch/nodehealth.go +++ b/modules/mesh-delivery-watchman/internal/watch/nodehealth.go @@ -21,7 +21,9 @@ import ( // said once there, never while its own network or its silence says it; // - machine-units `machine..units`: failed units no module places; // - relogin-needed `module...relogin-needed` and directory-used-as-found -// `module...directory-used-as-found`. +// `module...directory-used-as-found`; +// - node-account-fallback `machine..node-account-fallback`: the engine fell back to its node's old login during a +// rotation (novox/hq ADR 0583 §6), raised while its report carries the fall back and cleared once it no longer does. // // A failing part that points at other mesh machines (NetworkPart.Targets: the hub's node, each resolver's node) is held // there, as the controller holds it: when every machine it points at is down on the record (silent, its own network @@ -36,6 +38,7 @@ const ( KindMachineUnits = "machine-units" KindReloginNeeded = "relogin-needed" KindUsedAsFound = "directory-used-as-found" + KindAccountFallback = delivery.NodeAccountFallbackFinding ) // The parts and states a report names (mesh-control internal/link). @@ -101,6 +104,10 @@ func (w *Watchman) nodeFindings(node string) { want[f.Key()] = true w.find(f) } + if f, raise := fallbackOf(node, h.Fallback); raise { + want[f.Key()] = true + w.find(f) + } } for key := range w.findings { if !want[key] && w.ownsHealthKey(key, node) { @@ -249,6 +256,19 @@ func foundOf(node string, dirs []delivery.NodeDirectory) []Finding { return out } +// fallbackOf is a node's fall back to its old login during a rotation of its account (ADR 0583 §6), one finding. +func fallbackOf(node string, f *delivery.LoginFallback) (Finding, bool) { + if f == nil || f.Node != node { + return Finding{}, false + } + said := fmt.Sprintf("%s fell back from its new login %s to its old one during a rotation of its bus account: %s", node, f.From, f.Why) + return Finding{Scope: ScopeMachine, ID: node, Token: KindAccountFallback, Kind: KindAccountFallback, Machine: node, Severity: Warning, + Source: "health-report", Since: f.At, Summary: said, Said: said, + Headline: fmt.Sprintf("%s fell back to its old bus login", node), + Needs: fmt.Sprintf("abandon the rotation of %s's account, or rotate it again once its new login authenticates.", node), + Resolved: fmt.Sprintf("%s no longer falls back to its old bus login", node)}, true +} + func orModule(m string) string { if m == "" { return "_" diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/confirm.go b/modules/mesh-enrolment/cmd/mesh-enrolment/confirm.go new file mode 100644 index 0000000..9324bcd --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/confirm.go @@ -0,0 +1,47 @@ +package main + +import ( + "fmt" + "strings" +) + +// Confirm ends an enrolment that came through (ADR 0587 §6): the node, dialled as its own login node. after it +// redeemed, calls it, and the runtime's proof of that caller is the confirmation (as ADR 0583 §3). Only then does the holder +// say node-enrolled, with the key's whole content, and delete the key and its address key. The node's name is the caller's +// own; an argument naming another node is refused. A confirm called again after a stop between the event and the deletes +// finishes the deletes and says nothing twice (the bus keeps one message of one id). +func (h *Holder) Confirm(caller, node string) (map[string]any, error) { + own, ok := strings.CutPrefix(caller, nodePrefix) + if !ok || CheckNode(own) != nil || strings.Contains(own, "--") { + return nil, fmt.Errorf("only the node itself, dialled as node., confirms its enrolment, and the runtime proved the caller to be %q", caller) + } + if node != "" && node != own { + return nil, fmt.Errorf("%s confirms its own enrolment only, not that of %s", caller, node) + } + row, found, err := h.Store.Get(NodeKey(own)) + if err != nil { + return nil, fmt.Errorf("the enrolment of %s cannot be read: %w", own, err) + } + if !found { + return nil, fmt.Errorf("no enrolment of %s is open: nothing to confirm", own) + } + e, err := decodeEnrolment(row.Value) + if err != nil { + return nil, err + } + if e.Stage != StageRedeeming { + return nil, fmt.Errorf("the enrolment of %s is %s: it is confirmed after redeem", own, e.Stage) + } + // The address key goes first (ADR 0589 §6): a confirm stopped midway never leaves one the catch-up would take for an + // expiry and unplace a node that joined. + if err := h.dropAddress(e); err != nil { + return nil, fmt.Errorf("%w (call confirm again)", err) + } + if err := h.sayEvent(EventEnrolled, e, ""); err != nil { + return nil, fmt.Errorf("node-enrolled of %s cannot be said, so nothing is deleted (call confirm again): %w", own, err) + } + if err := h.release(e); err != nil { + return nil, fmt.Errorf("the enrolment of %s is confirmed and not closed (call confirm again): %w", own, err) + } + return map[string]any{"node": own, "address": e.Address, "says": "the node is enrolled; its enrolment is closed and the vault is asked to drop the token"}, nil +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/confirm_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/confirm_test.go new file mode 100644 index 0000000..89a00b2 --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/confirm_test.go @@ -0,0 +1,44 @@ +package main + +import "testing" + +// 774/S57/AC1: a confirm from any caller but node. is refused; from node. node-enrolled is said and both keys +// are deleted; a stop between the event and the deletes is finished by calling confirm again. +func confirmIsTheNodesOwnAndEndsTheEnrolment(t *testing.T) { + for _, c := range []struct{ name, caller, node, want string }{ + {"the enrolment user", "enrol.ace", "", "dialled as node."}, + {"another node", "node.g14", "ace", "its own enrolment only"}, + {"the second login", "node.ace--b", "", "dialled as node."}, + {"no one", "unproven", "ace", "dialled as node."}, + } { + r := newRig(t) + r.seed("ace", "10.9.0.2", StageRedeeming) + if _, err := r.h.Confirm(c.caller, c.node); err == nil { + t.Errorf("%s: confirmed", c.name) + } else { + mustContain(t, err.Error(), c.want) + } + if len(r.said) != 0 || len(r.keys()) != 2 { + t.Errorf("%s: said %v, keys %v", c.name, r.events(), r.keys()) + } + } + r := newRig(t) + r.seed("ace", "10.9.0.2", StageLive) + if _, err := r.h.Confirm("node.ace", ""); err == nil { + t.Error("confirmed before redeem") + } + r.seed("ace", "10.9.0.2", StageRedeeming) + r.st.failDelete[NodeKey("ace")] = true + if _, err := r.h.Confirm("node.ace", ""); err == nil { + t.Fatal("a delete that failed was taken for done") + } + if r.has(AddressKey("10.9.0.2")) || !r.has(NodeKey("ace")) || len(r.said) != 1 { + t.Fatalf("keys %v, said %v", r.keys(), r.events()) + } + if _, err := r.h.Confirm("node.ace", ""); err != nil || len(r.keys()) != 0 { + t.Fatalf("%v, keys %v", err, r.keys()) + } + if r.events()[0] != EventEnrolled || r.said[0].ID != r.said[1].ID { + t.Errorf("events %v ids %s %s", r.events(), r.said[0].ID, r.said[1].ID) + } +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/end.go b/modules/mesh-enrolment/cmd/mesh-enrolment/end.go new file mode 100644 index 0000000..47f045a --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/end.go @@ -0,0 +1,210 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Revoke ends an enrolment at any stage (ADR 0587 §5), by a person's call: at asked it withdraws the ask; at redeeming it also +// undoes what redeem had not finished, the vault retiring a node account it already minted, and a node the yes added to +// placement is taken off it again. It only takes access away, so it needs no Approve; +// it is recorded in the hand-act log. enrolment-revoked carries the key's content and the why; the token is then dropped by +// the durable consumer of that event, never before. A revoke that stopped is finished by calling it again. +func (h *Holder) Revoke(caller, node, why string) (map[string]any, error) { + if caller == "" || caller == stdio.Unproven { + return nil, fmt.Errorf("the caller is not proven by the runtime, and a revoke is recorded with who made it") + } + if strings.TrimSpace(why) == "" { + return nil, fmt.Errorf("a revoke says why: it is recorded as an act done by hand") + } + row, found, err := h.Store.Get(NodeKey(node)) + if err != nil { + return nil, fmt.Errorf("the enrolment of %s cannot be read: %w", node, err) + } + if !found { + return nil, fmt.Errorf("no enrolment of %s is open: mesh-enrolment.tokens lists those that are", node) + } + e, err := decodeEnrolment(row.Value) + if err != nil { + return nil, err + } + if e.Stage == StageAsked && e.Ask != "" && h.Asker != nil { + if err := h.Asker.Cancel(e.Ask); err != nil { + h.logf("the ask %s of %s was not taken back at the router: %v", e.Ask, node, err) + } + } + if e.Stage == StageRedeeming { + if err := h.Vault.Retire(node); err != nil { + return nil, fmt.Errorf("the vault did not retire the account minted for %s, so nothing is ended (call revoke again): %w", node, err) + } + } + // A node that entered placement on the yes and never confirmed leaves it again (the operator's decision of 2026-11-02 01:05). + if err := h.unplace(e, "revoked: "+why); err != nil { + return nil, fmt.Errorf("%s is not taken off placement, so nothing is ended (call revoke again): %w", node, err) + } + out := map[string]any{"node": node, "was": e.Stage} + if h.Act != nil { + if err := h.Act("mesh-enrolment revoke of "+node+" at "+string(e.Stage), why+" (by "+caller+")", "enrolment-revoke"); err != nil { + out["recorded"] = "NOT recorded in the hand-act log: " + err.Error() + h.logf("the revoke of %s is not recorded in the hand-act log: %v", node, err) + } + } + if err := h.sayEvent(EventRevoked, e, why); err != nil { + return nil, fmt.Errorf("enrolment-revoked of %s cannot be said, so its keys stay (call revoke again): %w", node, err) + } + if err := h.release(e); err != nil { + return nil, fmt.Errorf("the enrolment of %s is revoked and its keys are not all deleted (call revoke again): %w", node, err) + } + out["says"] = "the enrolment is ended; the vault is asked to drop the token and, for a redeem under way, the account was retired" + return out, nil +} + +// Heard is the holder's watch of its own bucket (stdio.State.Follow): it keeps each node key's last content, and the bus's +// expiry marker of a key at stage live says enrolment-expired, naming the node and the enrolment-issued event that carried its +// content (ADR 0587 §5). redeem checks the expiry itself, so a marker heard late admits no machine. +func (h *Holder) Heard(op, key string, value []byte) error { + node, ok := strings.CutPrefix(key, "node.") + if !ok { + return nil + } + h.mu.Lock() + defer h.mu.Unlock() + if h.seen == nil { + h.seen = map[string]Enrolment{} + } + switch op { + case stdio.OpPut: + if e, err := decodeEnrolment(value); err == nil { + h.seen[node] = e + } + case stdio.OpDelete: + delete(h.seen, node) + case stdio.OpExpire: + e, known := h.seen[node] + delete(h.seen, node) + if known { + return h.endExpired(e) + } + // The key expired before the watch handed its content over: its address key still names its ask and issue (ADR 0589 §6). + return h.reconcileLocked() + } + return nil +} + +// Reconcile ends every enrolment whose key is gone while its address key still names its ask: one that expired while the +// holder was down, or before its watch caught up, or whose end stopped midway (ADR 0589 §6). Run at start, after each +// catch-up and on every expiry; a step that fails is tried again on the next pass. +func (h *Holder) Reconcile() error { + h.mu.Lock() + defer h.mu.Unlock() + return h.reconcileLocked() +} + +func (h *Holder) reconcileLocked() error { + keys, err := h.Store.Keys() + if err != nil { + return fmt.Errorf("the enrolments cannot be listed: %w", err) + } + open := map[string]bool{} + for _, k := range keys { + if n, ok := strings.CutPrefix(k, "node."); ok { + open[n] = true + } + } + var errs []error + for _, k := range keys { + address, ok := strings.CutPrefix(k, "address.") + if !ok { + continue + } + row, found, err := h.Store.Get(k) + if err != nil || !found { + continue + } + r, err := delivery.DecodeEnrolmentAddress(row.Value) + // An address with no ask is reserved by an issue under way: its enrolment key is written next, then the ask. + if err != nil || r.Ask == "" || open[r.Node] { + continue + } + // Only an ask answered yes made a token and placed the node (ADR 0589 §6): one declined, or not answered yet, is no + // expiry, and one not answered stays until its ask ends. + if yes, err := h.answeredYes(r.Ask); err != nil { + errs = append(errs, fmt.Errorf("the answer to %s cannot be read, so the address %s waits for the next pass: %w", r.Ask, address, err)) + continue + } else if !yes { + continue + } + h.logf("the enrolment of %s (ask %s) is gone and its address %s is still reserved: it expired unseen, and is ended now", r.Node, r.Ask, address) + if err := h.endExpired(Enrolment{Node: r.Node, Address: address, Ask: r.Ask, Issued: r.Issued}); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +// answeredYes reads the router's record of ask: whether the operator answered it yes. +func (h *Holder) answeredYes(ask string) (bool, error) { + _, w, err := h.whereIs(ask) + if err != nil { + return false, err + } + return w != nil && w.Option == "yes", nil +} + +// endExpired ends an enrolment whose key expired, in this order, each step safe to do again: the node, placed on the yes and +// never confirmed, is taken out of placement on the warrant of its ask (only when placement holds it); enrolment-expired names +// its issued event (said under one id, so said again is said once); the vault drops the token; the address key goes last, so +// a step that failed is found and done on the next pass (ADR 0589 §6). +func (h *Holder) endExpired(e Enrolment) error { + if err := h.unplace(e, "expired"); err != nil { + return fmt.Errorf("%s, whose enrolment expired, is not taken out of placement (tried again on the next pass): %w", e.Node, err) + } + body, err := delivery.EncodeEnrolmentExpiry(delivery.EnrolmentExpiry{Node: e.Node, IssuedID: delivery.EnrolmentIssuedID(e.Node, e.Issued)}) + if err != nil { + return err + } + if err := h.say(EventExpired, json.RawMessage(body), eventID(EventExpired, e)); err != nil { + return fmt.Errorf("enrolment-expired of %s cannot be said (tried again on the next pass): %w", e.Node, err) + } + if err := h.Vault.Drop(e.Node); err != nil { + return fmt.Errorf("the token of %s, whose enrolment expired, is not dropped (tried again on the next pass): %w", e.Node, err) + } + if row, found, err := h.Store.Get(AddressKey(e.Address)); err != nil { + return fmt.Errorf("the address key %s cannot be read (tried again on the next pass): %w", e.Address, err) + } else if found { + if r, err := delivery.DecodeEnrolmentAddress(row.Value); err == nil && r.Node == e.Node && r.Ask == e.Ask { + if err := h.Store.Delete(AddressKey(e.Address)); err != nil { + return fmt.Errorf("the address key %s is not deleted (tried again on the next pass): %w", e.Address, err) + } + } + } + return nil +} + +// Ended is the durable consumer's act on one of the holder's own ending events (ADR 0587 §5): the vault is asked to drop the +// node's token and its hash, and the event is acknowledged only when the vault answers that both are gone, so a drop that +// fails is delivered again by the bus. It never runs before an end: the consumer hears only the end events. +func (h *Holder) Ended(event string, body []byte) error { + var e struct { + Node string `json:"node"` + } + if err := json.Unmarshal(body, &e); err != nil || CheckNode(e.Node) != nil { + h.logf("the event %s names no node and is not acted on: %v", event, err) + return nil + } + if err := h.Vault.Drop(e.Node); err != nil { + return fmt.Errorf("the vault did not drop the token of %s (the bus delivers %s again): %w", e.Node, event, err) + } + h.mu.Lock() + h.ended = append(h.ended, json.RawMessage(`{"event":`+fmt.Sprintf("%q", event)+`,"content":`+string(body)+`}`)) + if len(h.ended) > 200 { + h.ended = h.ended[len(h.ended)-200:] + } + h.mu.Unlock() + return nil +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go new file mode 100644 index 0000000..a341180 --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/end_test.go @@ -0,0 +1,209 @@ +package main + +import ( + "encoding/json" + "errors" + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/delivery" + "reflect" + "slices" + "strings" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// 774/S58/AC1: a revoke at redeeming, whether the vault minted the account yet or not, takes both keys away, the vault retires +// the account, the revoke is recorded and enrolment-revoked is said; one the vault failed to retire ends nothing and is +// finished by calling it again. At asked it withdraws the ask; an expiry marker says enrolment-expired with the issued event. +func revokeEndsAtAnyStageAndAnExpiryMarkerSaysExpired(t *testing.T) { + r := newRig(t) + r.seed("ace", "10.9.0.2", StageRedeeming) + r.vault.fail["retire ace"] = true + if _, err := r.h.Revoke("person.operator", "ace", "never confirmed"); err == nil || len(r.keys()) != 2 || len(r.said) != 0 { + t.Fatalf("a failed retire ended it: %v, keys %v, events %v", err, r.keys(), r.events()) + } + if _, err := r.h.Revoke("person.operator", "ace", "never confirmed"); err != nil { + t.Fatal(err) + } + // The node the yes placed, never confirmed, leaves placement on the enrolment's warrant before the end is said. + if _, placed := r.rd["mesh-delivery-assigner.placement"]["ace.node"]; placed || len(r.added) == 0 || + !reflect.DeepEqual(r.added[len(r.added)-1].Change, delivery.PlacementProposalChange{Op: delivery.OpRemoveNode, Node: "ace"}) || r.added[len(r.added)-1].Warrant.Ask != "ask-ace" { + t.Errorf("revoked and still placed: %+v", r.added) + } + if len(r.keys()) != 0 || !slices.Equal(r.events(), []string{EventRevoked}) || len(r.acts) != 1 || !slices.Contains(r.vault.calls, "retire ace") { + t.Errorf("keys %v events %v acts %v vault %v", r.keys(), r.events(), r.acts, r.vault.calls) + } + mustContain(t, string(r.said[0].Body), "never confirmed") + // At asked it withdraws the ask and the vault is not asked to retire anything. + r.seed("g14", "10.9.0.3", StageAsked) + if _, err := r.h.Revoke("person.operator", "g14", "changed our mind"); err != nil || !slices.Equal(r.asker.cancelled, []string{"ask-g14"}) || slices.Contains(r.vault.calls, "retire g14") { + t.Errorf("%v, cancelled %v, vault %v", err, r.asker.cancelled, r.vault.calls) + } + if _, err := r.h.Revoke(stdio.Unproven, "ace", "x"); err == nil { + t.Error("revoked for no one") + } + // The bus's marker of a key at live: the content the watch handed over is said with the issued event. + r.seed("novox", "10.9.0.4", StageLive) + row, _, _ := r.st.Get(NodeKey("novox")) + _ = r.h.Heard(stdio.OpPut, NodeKey("novox"), row.Value) + if err := r.h.Heard(stdio.OpExpire, NodeKey("novox"), nil); err != nil { + t.Fatal(err) + } + if _, placed := r.rd["mesh-delivery-assigner.placement"]["novox.node"]; placed { + t.Error("expired and still placed") + } + last := r.said[len(r.said)-1] + x, err := delivery.DecodeEnrolmentExpiry(last.Body) + if err != nil || last.Event != EventExpired || x.Node != "novox" || !strings.HasPrefix(x.IssuedID, "enrolment-issued.novox.") || + !slices.Contains(r.vault.calls, "drop novox") || r.has(AddressKey("10.9.0.4")) { + t.Errorf("said %s %s: %+v, %v; vault %v, keys %v", last.Event, last.Body, x, err, r.vault.calls, r.keys()) + } + // ADR 0589 §6: an enrolment whose key expired unseen (before the watch handed it over, or while the holder was down) is + // found by its address key, which names its ask and issue: it leaves placement on that ask's warrant, enrolment-expired + // names its rebuilt issued event, the token is dropped and the address key goes last. A step that fails is done again on + // the next pass, the address key kept until then. + for _, c := range []struct { + name string + node string + fail string + passS func(r *rig) error + }{ + {"expired unseen", "unseen", "", func(r *rig) error { return r.h.Heard(stdio.OpExpire, NodeKey("unseen"), nil) }}, + {"the holder was down, and its drop failed once", "down", "drop down", func(r *rig) error { return r.h.Reconcile() }}, + } { + r.seed(c.node, "10.9.0.9", StageLive) + r.asker.answer("ask-"+c.node, "yes") + issued := r.now.Add(-time.Minute) + _ = r.st.Delete(NodeKey(c.node)) // the bus took the key; the watch never handed its content over + if c.fail != "" { + r.vault.fail[c.fail] = true + if err := c.passS(r); err == nil || !r.has(AddressKey("10.9.0.9")) { + t.Errorf("%s: a failed drop ended the pass (%v) or took the address key", c.name, err) + } + } + if err := c.passS(r); err != nil { + t.Fatalf("%s: %v", c.name, err) + } + last := r.said[len(r.said)-1] + x, err := delivery.DecodeEnrolmentExpiry(last.Body) + _, placed := r.rd["mesh-delivery-assigner.placement"][c.node+".node"] + removed := r.added[len(r.added)-1] + if err != nil || x.IssuedID != delivery.EnrolmentIssuedID(c.node, issued) || placed || removed.Change.Op != delivery.OpRemoveNode || + removed.Change.Node != c.node || removed.Warrant.Ask != "ask-"+c.node || !slices.Contains(r.vault.calls, "drop "+c.node) || r.has(AddressKey("10.9.0.9")) { + t.Errorf("%s: said %s (%v), placed %v, last placement call %+v, vault %v, keys %v", c.name, last.Body, err, placed, removed, r.vault.calls, r.keys()) + } + } + // ADR 0589 §6, the negative rows: an address key the catch-up finds with its enrolment key gone is no expiry when its ask + // was declined or is not answered yet, and none is left behind by a confirm, a decline or a refused add, which delete it + // first. In none of them is the node unplaced. + for _, c := range []struct { + name string + end func(r *rig, node string) + key bool // the address key stays (an ask not answered yet stays until its ask ends) + }{ + {"after a confirm stopped before the enrolment key went", func(r *rig, node string) { + e, _ := r.enrolment(node) + e.Stage, e.KeysDigest = StageRedeeming, delivery.KeysDigest("pk", "ov", "se", "sv") + _, _ = r.st.Put(NodeKey(node), e, 0) + r.asker.answer("ask-"+node, "yes") + r.st.failDelete[NodeKey(node)] = true // the confirm stops after the address key, before the enrolment key + _, _ = r.h.Confirm(NodeUser(node), "") + _ = r.st.Delete(NodeKey(node)) // and the key is gone by the next pass + }, false}, + {"after a decline", func(r *rig, node string) { + _ = r.h.Warranted(asks.Option{ID: "no"}, asks.Warrant{Ask: "ask-" + node}) + r.asker.answer("ask-"+node, "no") + }, false}, + {"after a refused add", func(r *rig, node string) { + delete(r.rd["mesh-delivery-assigner.placement"], node+".node") // the add is tried, and refused + r.placeErr = errors.New("mesh-placement did not take the node: the proposal is refused: change 1: an address taken") + _ = r.h.Warranted(yes, asks.Warrant{Ask: "ask-" + node}) + r.asker.answer("ask-"+node, "yes") + r.placeErr = nil + if !slices.Contains(r.events(), EventRefused) { + r.t.Errorf("a refused add did not end the enrolment: %v", r.events()) + } + }, false}, + {"a declined ask whose enrolment key is gone", func(r *rig, node string) { + r.asker.answer("ask-"+node, "no") + _ = r.st.Delete(NodeKey(node)) + }, true}, + {"an ask not answered yet", func(r *rig, node string) { _ = r.st.Delete(NodeKey(node)) }, true}, + } { + r := newRig(t) + stage := StageAsked + if strings.HasPrefix(c.name, "after a confirm") { + stage = StageLive + } + r.seed("joiner", "10.9.0.11", stage) + r.rd["mesh-delivery-assigner.placement"]["joiner.node"] = `{"v":1,"node":"joiner","decided":{"address":"10.9.0.11"},"modules":[]}` + c.end(r, "joiner") + removes := 0 + for _, a := range r.added { + if a.Change.Op == delivery.OpRemoveNode { + removes++ + } + } + if err := r.h.Reconcile(); err != nil { + t.Errorf("%s: %v", c.name, err) + } + after := 0 + for _, a := range r.added { + if a.Change.Op == delivery.OpRemoveNode { + after++ + } + } + if after != removes || r.has(AddressKey("10.9.0.11")) != c.key || slices.Contains(r.events(), EventExpired) { + t.Errorf("%s: the catch-up unplaced (%d to %d), address key %v, events %v", c.name, removes, after, r.has(AddressKey("10.9.0.11")), r.events()) + } + } + // An address reserved by an issue under way (no ask yet) is never taken for an expired enrolment. + _, _ = r.st.Put(AddressKey("10.9.0.10"), Reservation{Node: "busy"}, 0) + if err := r.h.Reconcile(); err != nil || !r.has(AddressKey("10.9.0.10")) { + t.Errorf("a reservation under way was ended: %v", err) + } +} + +// 774/S58/AC2 and AC3: the drop is asked of the vault only from an ending event, acknowledged only once the vault answers, and +// asked again when its first answer fails; and neither an event nor the bucket ever holds the token, sealed or not. +func theDropFollowsOnlyAnEndAndNothingHoldsTheToken(t *testing.T) { + r := newRig(t) + a, _ := r.h.Issue("person.operator", "ace", "a new machine", tunnelKey, 0) + _ = r.h.Warranted(yes, askWarrant(a.Ask)) + if _, err := r.h.Tokens("ace", false); err != nil { // relays the sealed token + t.Fatal(err) + } + if slices.ContainsFunc(r.vault.calls, func(c string) bool { return strings.HasPrefix(c, "drop") }) { + t.Fatalf("the vault was asked to drop a token before an end: %v", r.vault.calls) + } + if _, err := r.h.Revoke("person.operator", "ace", "lost the machine"); err != nil { + t.Fatal(err) + } + end := r.said[len(r.said)-1] + r.vault.fail["drop ace"] = true + if err := r.h.Ended(end.Event, end.Body); err == nil { + t.Fatal("an event was acknowledged though the vault did not answer") + } + if err := r.h.Ended(end.Event, end.Body); err != nil { + t.Fatal(err) + } + if n := strings.Count(strings.Join(r.vault.calls, ","), "drop ace"); n != 2 { + t.Errorf("the drop was asked %d times", n) + } + for _, s := range r.said { + if strings.Contains(string(s.Body), "sealed") || strings.Contains(string(s.Body), "token") { + t.Errorf("the event %s holds a token: %s", s.Event, s.Body) + } + } + for k, row := range r.st.rows { + if strings.Contains(string(row.Value), "sealed") { + t.Errorf("the key %s holds a token", k) + } + } + tk, _ := r.h.Tokens("", true) + if raw, _ := json.Marshal(tk); !strings.Contains(string(raw), "enrolment-revoked") { + t.Errorf("history %s", raw) + } +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment.go b/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment.go new file mode 100644 index 0000000..9994ad5 --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment.go @@ -0,0 +1,123 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "regexp" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// mesh-enrolment: the holder of the seat mesh-enrolment (novox/hq ADR 0587 §1), held once for the mesh. A Go bundle served +// by a runtime of its own, on its own bus account (`runs-as`). It mints and keeps no secret and holds no signing key: the +// vault mints the token and the node's account and signs (ADR 0587 §3). What it keeps is one key per open enrolment in its +// state `enrolments` (§2), and what it says is its five events. + +// Module and Seat are the module's name and the seat it holds. +const ( + Module = catalogue.EnrolmentSeat + Seat = catalogue.EnrolmentSeat + // State is the holder's bucket of open enrolments; its keys are node. and address.
(ADR 0587 §2). + State = delivery.EnrolmentsState +) + +// The seat's six verbs and five events (ADR 0587 §1). +var Verbs = []string{catalogue.EnrolmentVerbIssue, catalogue.EnrolmentVerbRedeem, catalogue.EnrolmentVerbConfirm, catalogue.EnrolmentVerbRevoke, + catalogue.EnrolmentVerbTokens, catalogue.EnrolmentVerbAgain} + +const ( + EventIssued = catalogue.EnrolmentIssuedEvent + EventEnrolled = catalogue.NodeEnrolledEvent + EventRefused = catalogue.EnrolmentRefusedEvent + EventRevoked = catalogue.EnrolmentRevokedEvent + EventExpired = catalogue.EnrolmentExpiredEvent +) + +// EndEvents are the events that end an enrolment; the holder's durable consumer hears each (ADR 0587 §5) and asks the vault +// to drop the token. A seat's event is heard as .. +var EndEvents = []string{EventEnrolled, EventRefused, EventRevoked, EventExpired} + +// The stages of an open enrolment (ADR 0587 §2), as the seat's contract names them (delivery.Enrolment). +const ( + // StageAsked: the operator's Approve is asked, no token exists. + StageAsked = delivery.EnrolmentAsked + // StageLive: the Approve is given, the token exists and expires on the bus with the key. + StageLive = delivery.EnrolmentLive + // StageRedeeming: the token is spent; the key has no expiry and ends only by confirm or revoke. + StageRedeeming = delivery.EnrolmentRedeeming +) + +// Enrolment and Reservation are the seat's state contract (delivery.Enrolment, delivery.EnrolmentAddress). +type ( + Enrolment = delivery.Enrolment + Reservation = delivery.EnrolmentAddress +) + +const ( + nodePrefix = "node." +) + +// NodeKey and AddressKey are the two keys of an enrolment. +func NodeKey(node string) string { return delivery.EnrolmentNodeKey(node) } +func AddressKey(address string) string { return delivery.EnrolmentAddressKey(address) } + +// EnrolUser and NodeUser are the two bus users of a node: the one it joins with, and its own. +func EnrolUser(node string) string { return delivery.EnrolmentUser(node) } +func NodeUser(node string) string { return nodePrefix + node } + +var nodeName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`) + +// CheckNode says node is a node name a bus user and a key can carry. +func CheckNode(node string) error { + if !nodeName.MatchString(node) { + return fmt.Errorf("%q is not a valid node name: lowercase letters, digits and hyphens, not starting with a hyphen", node) + } + return nil +} + +func decodeEnrolment(raw []byte) (Enrolment, error) { return delivery.DecodeEnrolment(raw) } + +// Encode is a value of the bucket as the seat's contract writes it: an Enrolment or a Reservation checked and versioned. +func Encode(v any) (json.RawMessage, error) { + switch x := v.(type) { + case Enrolment: + return delivery.EncodeEnrolment(x) + case Reservation: + return delivery.EncodeEnrolmentAddress(x.Node, x.Ask, x.Issued) + } + return json.Marshal(v) +} + +// Row is a key as the store holds it. +type Row struct { + Value []byte + Revision uint64 +} + +// ErrExists is a create refused because the key has a value; ErrChanged a write refused because the key moved. +var ( + ErrExists = errors.New("the key has a value already") + ErrChanged = errors.New("the key was written by another writer first") +) + +// Store is the holder's own state: stdio.State in the process, a map in a test. A ttl of zero is no expiry of the key's own. +type Store interface { + Get(key string) (Row, bool, error) + // Create writes a key only when it has none (ErrExists otherwise). + Create(key string, v any, ttl time.Duration) error + // Update writes a key only when it is still at revision (ErrChanged otherwise), with no expiry of its own. + Update(key string, v any, revision uint64) (uint64, error) + // Put writes a key whole, with the expiry ttl when it is not zero. + Put(key string, v any, ttl time.Duration) (uint64, error) + Delete(key string) error + Keys() ([]string, error) +} + +// Reader reads another module's state, `.`. +type Reader interface { + Keys(state string) ([]string, error) + Get(state, key string) ([]byte, bool, error) +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment_test.go new file mode 100644 index 0000000..bb157f8 --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment_test.go @@ -0,0 +1,61 @@ +package main + +import ( + "os" + "testing" +) + +// The module's unit tests, as two tables (the repository's test budget): the operator's side (issue, the tokens verb, the +// manifest, the read of the bus node's loaded list) and the joining node's side (redeem, confirm and every end). + +// TestIssueTokensAndTheManifest is 774/S54 and S55, and the read of whether the bus node loaded a user. +func TestIssueTokensAndTheManifest(t *testing.T) { + for _, c := range []struct { + name string + run func(*testing.T) + }{ + {"S55/AC1 issue refuses naming every finding", issueRefusesNamingEveryFinding}, + {"S55/AC2 two issues at once hold two addresses and a yes makes the token", twoIssuesAtOnceHoldTwoAddressesAndAYesMakesTheToken}, + {"S55/AC3 a declined or expired ask takes both keys; a vault failure keeps the enrolment for again", aDeclinedOrExpiredAskTakesBothKeysAndAVaultFailureKeepsTheEnrolmentForAgain}, + {"S54/AC1 tokens answers each stage and relays the sealed token of one node", tokensAnswersEachStageAndRelaysTheSealedTokenOfOneNode}, + {"S54/AC2 the manifest says what the code does", theManifestSaysWhatTheCodeDoes}, + {"S56 the bus node's loaded list is read from the grants part's pass", theLoadedListIsReadFromTheGrantsPass}, + } { + t.Run(c.name, c.run) + } +} + +// TestRedeemConfirmAndEveryEnd is 774/S56, S57 and S58. +func TestRedeemConfirmAndEveryEnd(t *testing.T) { + for _, c := range []struct { + name string + run func(*testing.T) + }{ + {"S56/AC1 redeem refuses what it must and spends the token once", redeemRefusesWhatItMustAndSpendsTheTokenOnce}, + {"S56/AC2 a redeem stopped midway is finished by the same keys again", aRedeemStoppedMidwayIsFinishedByTheSameKeysAgain}, + {"S57/AC1 confirm is the node's own and ends the enrolment", confirmIsTheNodesOwnAndEndsTheEnrolment}, + {"S58/AC1-AC2 revoke ends at any stage and an expiry marker says expired", revokeEndsAtAnyStageAndAnExpiryMarkerSaysExpired}, + {"S58/AC3 the drop follows only an end and nothing holds the token", theDropFollowsOnlyAnEndAndNothingHoldsTheToken}, + } { + t.Run(c.name, c.run) + } +} + +// The key pass of mesh-grants.user-list holds the part's whole pass result; testdata/grants-pass.json was written by +// mesh-grants' own Result (json.Marshal of grants.Result with CredentialsOf), not written by hand: node.ace is in the shadow +// list and loaded, node.g14 in the shadow list and not loaded yet. +func theLoadedListIsReadFromTheGrantsPass(t *testing.T) { + raw, err := os.ReadFile("testdata/grants-pass.json") + if err != nil { + t.Fatal(err) + } + loaded := loadedVia(memRead{"mesh-grants.user-list": {"pass": string(raw)}}) + for user, want := range map[string]bool{"node.ace": true, "node.g14": false, "node.none": false} { + if got, err := loaded(user); err != nil || got != want { + t.Errorf("%s loaded = %v, %v; want %v", user, got, err, want) + } + } + if _, err := loadedVia(memRead{"mesh-grants.user-list": {}})("node.ace"); err == nil { + t.Error("no pass composed read as an answer") + } +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go b/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go new file mode 100644 index 0000000..47b763e --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go @@ -0,0 +1,349 @@ +package main + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net/netip" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Vault is the vault's side of enrolment (ADR 0587 §3, §5): the token and the node's account, and the signatures. The +// holder keeps nothing of what it answers. +type Vault interface { + // Token asks the vault to make the token and its hash for node, expiring at expires, and answers what the joining + // machine must trust (the bus's address and fingerprint, the vault's signing key). No token is in the answer. + // The token carries tunnel, the node's first tunnel built at issue (ADR 0587 §3, insight of 2026-11-02). + Token(node string, expires time.Time, tunnel delivery.TokenTunnel) (delivery.EnrolmentTokenAnswer, error) + // Sealed asks the vault for node's token sealed to the operator key, as the vault sealed it. + Sealed(node string) (string, error) + // Drop asks the vault to drop node's token and its hash; it answers only once both are gone. + Drop(node string) error + // Account asks for node's account sealed to the sealing key; the same node and keys again answer the same account. + Account(node, sealingKey, keysDigest string) (delivery.NodeAccountAnswer, error) + // Retire asks the vault to retire the account it minted for node. + Retire(node string) error + // SignCertificate and SignReply sign through the vault's `sign`; the certificate's call carries every public key the node + // presented, which the vault checks against the digest node-account was minted for. + SignCertificate(node string, keys delivery.JoiningKeys) (delivery.SignAnswer, error) + SignReply(node string, reply []byte) (delivery.SignAnswer, error) + // Signer is the vault's public signing key in base64, which the redeem answer names for the node to check it against its token. + Signer() (string, error) +} + +// Asker is the operator's channel as the holder asks it (asker.Client is one). +type Asker interface { + Ask(asks.Ask) error + Cancel(id string) error +} + +// Holder is the seat's holder: the state it keeps, and every other part it reaches through a port. +type Holder struct { + Store Store + Read Reader + Vault Vault + Asker Asker + // Say publishes a seat event under a message id the bus keeps one of. + Say func(event string, body any, id string) error + // AddNode asks mesh-placement to add the node, or to take a node that never confirmed off again (ADR 0587 §7). + AddNode func(AddNode) error + // Warrant and Act record in the hand-act log, through the seat mesh-hand-acts: the operator's warrant for an ask, and an + // act done by hand (a revoke). + Warrant func(ask string) error + // WarrantOf is the warrant's id as the teller wrote it (the ask's id and when), read from the router's record of the ask. + WarrantOf func(ask string) (string, error) + Act func(what, why, cause string) error + // Loaded says whether the bus node loaded the bus user (ADR 0587 §3: redeem answers once node. is in its list). + Loaded func(user string) (bool, error) + Now func() time.Time + Logf func(string, ...any) + // AskLasts is how long the operator has to answer an issue's ask. + AskLasts time.Duration + + mu sync.Mutex + // seen is each node key's last content the watch handed over, so an expiry, which hands over no value, is said with it. + seen map[string]Enrolment + // ended is the end events this process heard, newest last: the history `tokens` answers (limit: EVENTS is not readable + // from a bundle, so a restart forgets them). + ended []json.RawMessage +} + +func (h *Holder) now() time.Time { + if h.Now != nil { + return h.Now() + } + return time.Now() +} + +func (h *Holder) logf(f string, a ...any) { + if h.Logf != nil { + h.Logf(f, a...) + } +} + +func (h *Holder) say(event string, body any, id string) error { + if h.Say == nil { + return errors.New("the seat's events cannot be said: no publisher") + } + return h.Say(event, body, id) +} + +// AddNode is what the holder asks mesh-placement to add (ADR 0587 §7): the proposal's one change, a joining node with the +// address the operator approved and the public halves of its keys, and the issue's warrant, which names this node and this +// address (delivery.AddNodeWarrant). +type AddNode struct { + Why string + Change delivery.PlacementProposalChange + Warrant delivery.AddNodeWarrant +} + +// place adds the node into placement, with the address the operator approved and its tunnel key, on the issue's warrant +// (ADR 0587 §7; the operator's decision of 2026-11-02 01:05), unless placement holds it at that address already. +func (h *Holder) place(e Enrolment, placements map[string]delivery.Placement) error { + if p, ok := placements[e.Node]; ok && p.Decided.Address == e.Address { + return nil + } + w, err := h.warrantOf(e) + if err != nil { + return err + } + return h.AddNode(AddNode{Why: "enrolment of " + e.Node + ": " + e.Why, Warrant: w, + Change: delivery.PlacementProposalChange{Op: delivery.OpAddNode, Node: e.Node, Decided: &delivery.NodeDecided{Address: e.Address}, + Keys: &delivery.JoiningKeys{Overlay: e.OverlayKey}}}) +} + +// unplace takes a node that never confirmed off placement again, on the same warrant, when its enrolment ends revoked, refused +// or expired: the hub stops peering a machine that will not join. A node placement does not hold is left as it is. +func (h *Holder) unplace(e Enrolment, why string) error { + placements, err := h.placements() + if err != nil { + return fmt.Errorf("placement cannot be read: %w", err) + } + if _, ok := placements[e.Node]; !ok { + return nil + } + w, err := h.warrantOf(e) + if err != nil { + return err + } + return h.AddNode(AddNode{Why: "enrolment of " + e.Node + " ended unconfirmed: " + why, Warrant: w, + Change: delivery.PlacementProposalChange{Op: delivery.OpRemoveNode, Node: e.Node}}) +} + +func (h *Holder) warrantOf(e Enrolment) (delivery.AddNodeWarrant, error) { + if h.AddNode == nil || h.WarrantOf == nil { + return delivery.AddNodeWarrant{}, errors.New("no route to mesh-placement") + } + warrant, err := h.WarrantOf(e.Ask) + if err != nil { + return delivery.AddNodeWarrant{}, fmt.Errorf("the operator's warrant for %s cannot be read from the router: %w", e.Ask, err) + } + return delivery.AddNodeWarrant{Node: e.Node, Address: e.Address, Ask: e.Ask, Warrant: warrant}, nil +} + +// checkTunnelKey says a tunnel key is a WireGuard public key, as `nox-mesh-host key` prints it: 32 bytes in standard base64 +// (as delivery.Enrolment requires it too); said here so issue names it among its findings. +func checkTunnelKey(k string) error { + raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(k)) + if err != nil || len(raw) != 32 { + return errors.New("no tunnel key is given (overlay_key: the line `nox-mesh-host key` prints on the new machine)") + } + return nil +} + +// issuing says whether issue is on: only while the controller's lease is handed to a holder that is no controller (ADR 0587 +// §4), and why either way. +func (h *Holder) issuing() (bool, string) { + raw, found, err := h.Read.Get("mesh-controller.lease", buscompose.ControllerLeaseKey) + return buscompose.LeaseHandedAway(raw, found, err) +} + +// openEnrolments reads every node key of the bucket. +func (h *Holder) openEnrolments() (map[string]Row, map[string]Enrolment, error) { + keys, err := h.Store.Keys() + if err != nil { + return nil, nil, fmt.Errorf("the enrolments cannot be listed: %w", err) + } + sort.Strings(keys) + rows, out := map[string]Row{}, map[string]Enrolment{} + for _, k := range keys { + node, ok := strings.CutPrefix(k, "node.") + if !ok { + continue + } + row, found, err := h.Store.Get(k) + if err != nil { + return nil, nil, fmt.Errorf("the enrolment of %s cannot be read: %w", node, err) + } + if !found { + continue + } + e, err := decodeEnrolment(row.Value) + if err != nil { + return nil, nil, fmt.Errorf("%s: %w", k, err) + } + rows[node], out[node] = row, e + } + return rows, out, nil +} + +// placements reads every node's placement, assembled from its keys. +func (h *Holder) placements() (map[string]delivery.Placement, error) { + keys, err := h.Read.Keys("mesh-delivery-assigner.placement") + if err != nil { + return nil, fmt.Errorf("placement cannot be listed: %w", err) + } + held := map[string][]byte{} + for _, k := range keys { + if raw, found, err := h.Read.Get("mesh-delivery-assigner.placement", k); err != nil { + return nil, fmt.Errorf("placement key %s cannot be read: %w", k, err) + } else if found { + held[k] = raw + } + } + p, err := delivery.PlacementsOf(held) + if len(p) == 0 && err != nil { + return nil, err + } + return p, nil +} + +// firstTunnel is the joining node's first tunnel as the token carries it (ADR 0587 §3, insight of 2026-11-02; as the +// controller's `token issue` builds it): the machine's own tunnel key given at issue, its address, the private network's +// range (privateRange), the hub's tunnel key from its record (the key its tunnel holds; on an adopted hub +// that has not written one, the found tunnel's key kept in its placement), the hub's endpoint from its placement, and the +// node's address on the private network. A mesh with no hub, or a hub with no key or no endpoint, has no tunnel to join +// through, and that is said. +func (h *Holder) firstTunnel(placements map[string]delivery.Placement, address, overlayKey string) (delivery.TokenTunnel, error) { + var hub *delivery.Placement + for _, p := range placements { + if p.Decided.Hub { + p := p + hub = &p + } + } + if hub == nil { + return delivery.TokenTunnel{}, errors.New("placement names no hub, so there is no tunnel to join through") + } + key := "" + if k, err := delivery.HaveNodeKey(hub.Node); err == nil { + if raw, found, err := h.Read.Get("node-engine.have", k); err == nil && found { + if hn, err := delivery.DecodeHaveNode(raw); err == nil && hn.Record != nil { + key = hn.Record.TunnelKey + } + } + } + if key == "" && hub.Decided.Tunnel != nil { + key = hub.Decided.Tunnel.PublicKey + } + var missing []string + if key == "" { + missing = append(missing, "no tunnel key in its record") + } + if hub.Decided.Endpoint == "" { + missing = append(missing, "no endpoint in its placement") + } + prefix, rerr := privateRange(placements) + if len(missing) > 0 { + return delivery.TokenTunnel{}, fmt.Errorf("the hub %s has %s, so there is no tunnel to join through", hub.Node, strings.Join(missing, " and ")) + } + if rerr != nil { + return delivery.TokenTunnel{}, rerr + } + return delivery.TokenTunnel{Key: overlayKey, Address: address + "/32", Range: prefix.String(), HubKey: key, HubEndpoint: hub.Decided.Endpoint}, nil +} + +// hasRecord says the node already writes a record (a node of the mesh, ADR 0587 §4: "a node record ... holds it"). +func (h *Holder) hasRecord(node string) bool { + keys, err := h.Read.Keys("node-engine.have") + if err != nil { + return false + } + for _, k := range keys { + if n, _, isNode, err := delivery.ParseHaveKey(k); err == nil && isNode && n == node { + return true + } + } + return false +} + +// privateRange is the private network's range: the hub's tunnel range in its placement, as the controller's `token issue` +// writes it into the token today (the adopted tunnel's range). A mesh whose hub's placement names none has no range a token +// can carry, and that is said rather than guessed. +func privateRange(placements map[string]delivery.Placement) (netip.Prefix, error) { + for _, p := range placements { + if t := p.Decided.Tunnel; p.Decided.Hub && t != nil && t.Range != "" { + r, err := netip.ParsePrefix(t.Range) + if err != nil { + return netip.Prefix{}, fmt.Errorf("the hub %s's tunnel range %q is no range: %w", p.Node, t.Range, err) + } + return r.Masked(), nil + } + } + return netip.Prefix{}, errors.New("no private network is known: the hub's placement names no tunnel range") +} + +// nextAddress is the lowest address of the private network that no placement key and no reservation holds (ADR 0587 §4), +// after skip. The network is the hub's range; the hub's own address and the tunnel's are never given. +func nextAddress(placements map[string]delivery.Placement, reserved map[string]bool, skip map[string]bool) (string, error) { + taken := map[string]bool{} + for _, p := range placements { + if a := p.Decided.Address; a != "" { + taken[a] = true + } + if t := p.Decided.Tunnel; t != nil { + if t.Address != "" { + if a, err := netip.ParsePrefix(t.Address); err == nil { + taken[a.Addr().String()] = true + } else { + taken[t.Address] = true + } + } + } + } + prefix, err := privateRange(placements) + if err != nil { + return "", err + } + for a := prefix.Addr().Next(); prefix.Contains(a); a = a.Next() { + s := a.String() + if prefix.Addr().Is4() && !prefix.Contains(a.Next()) { + break // the broadcast address + } + if taken[s] || reserved[s] || skip[s] { + continue + } + return s, nil + } + return "", fmt.Errorf("the private network %s has no free address", prefix) +} + +func (h *Holder) reservedAddresses() (map[string]bool, error) { + keys, err := h.Store.Keys() + if err != nil { + return nil, fmt.Errorf("the enrolments cannot be listed: %w", err) + } + out := map[string]bool{} + for _, k := range keys { + if a, ok := strings.CutPrefix(k, "address."); ok { + out[a] = true + } + } + return out, nil +} + +// lifetimeOf is the time a key lives on the bus for an enrolment that expires at expires. +func (h *Holder) lifetimeOf(expires time.Time) time.Duration { + if d := expires.Sub(h.now()); d > time.Second { + return d + } + return time.Second +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go new file mode 100644 index 0000000..fe757eb --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/integration_test.go @@ -0,0 +1,453 @@ +//go:build integration + +package main + +// mesh-enrolment whole on a real bus (novox/hq ADR 0587 §2-§8; issue 774): the holder's own code over a real nats-server's +// key-value bucket `enrolments` with a time to live per key (the keys written as the runtime writes them: a create with the +// expiry, a put with a message TTL, an update by revision), its events said on their real subjects, the bus's own expiry marker +// heard, and the other parts stood in for — the vault (which writes the new node's key of node-logins at node-account, as the +// vault does), the operator's channel, placement, and the grants part's rule that a key at live makes the user enrol. and +// a key gone takes it away. + +import ( + "context" + "encoding/json" + "errors" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +// busStore is the holder's bucket as the runtime writes it. +type busStore struct { + ctx context.Context + js jetstream.JetStream + kv jetstream.KeyValue +} + +func (b *busStore) Get(key string) (Row, bool, error) { + e, err := b.kv.Get(b.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return Row{}, false, nil + } + if err != nil { + return Row{}, false, err + } + return Row{Value: e.Value(), Revision: e.Revision()}, true, nil +} + +func (b *busStore) Create(key string, v any, ttl time.Duration) error { + raw, err := Encode(v) + if err != nil { + return err + } + var opts []jetstream.KVCreateOpt + if ttl > 0 { + opts = append(opts, jetstream.KeyTTL(ttl)) + } + _, err = b.kv.Create(b.ctx, key, raw, opts...) + if errors.Is(err, jetstream.ErrKeyExists) { + return ErrExists + } + return err +} + +func (b *busStore) Update(key string, v any, revision uint64) (uint64, error) { + raw, err := Encode(v) + if err != nil { + return 0, err + } + rev, err := b.kv.Update(b.ctx, key, raw, revision) + if err != nil && strings.Contains(err.Error(), "wrong last sequence") { + return 0, ErrChanged + } + return rev, err +} + +func (b *busStore) Put(key string, v any, ttl time.Duration) (uint64, error) { + raw, err := Encode(v) + if err != nil { + return 0, err + } + if ttl <= 0 { + return b.kv.Put(b.ctx, key, raw) + } + ack, err := b.js.Publish(b.ctx, "$KV."+b.kv.Bucket()+"."+key, raw, jetstream.WithMsgTTL(max(time.Second, ttl.Truncate(time.Second)))) + if err != nil { + return 0, err + } + return ack.Sequence, nil +} + +func (b *busStore) Delete(key string) error { + return b.kv.Delete(b.ctx, key) +} + +func (b *busStore) Keys() ([]string, error) { + keys, err := b.kv.Keys(b.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return keys, err +} + +// busReader reads other parts' buckets, `.`. +type busReader struct { + ctx context.Context + js jetstream.JetStream +} + +func (r busReader) bucket(state string) (jetstream.KeyValue, error) { + return r.js.KeyValue(r.ctx, strings.Replace(state, ".", "_", 1)) +} + +func (r busReader) Keys(state string) ([]string, error) { + kv, err := r.bucket(state) + if err != nil { + return nil, err + } + keys, err := kv.Keys(r.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return keys, err +} + +func (r busReader) Get(state, key string) ([]byte, bool, error) { + kv, err := r.bucket(state) + if err != nil { + return nil, false, err + } + e, err := kv.Get(r.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + return e.Value(), true, nil +} + +// standInVault holds the tokens it made, and writes a joining node's key of node-logins when it mints the account. +type standInVault struct { + ctx context.Context + logins jetstream.KeyValue + mu sync.Mutex + tokens map[string]delivery.Token + calls []string +} + +func (v *standInVault) note(c string) { + v.mu.Lock() + v.calls = append(v.calls, c) + v.mu.Unlock() +} + +// Token makes the token as the vault does: the bus it states, its signing key, a secret, and the first tunnel it is handed. +func (v *standInVault) Token(node string, expires time.Time, tunnel delivery.TokenTunnel) (delivery.EnrolmentTokenAnswer, error) { + v.mu.Lock() + v.tokens[node] = delivery.Token{Version: 1, Node: node, Broker: "bus:4222", Fingerprint: "f", Signer: make([]byte, 32), Secret: "secret-" + node, Tunnel: &tunnel} + v.mu.Unlock() + v.note("token " + node) + return delivery.EnrolmentTokenAnswer{Node: node, Broker: "bus:4222", Fingerprint: "f", Signer: "s"}, nil +} +func (v *standInVault) Sealed(node string) (string, error) { return "sealed-" + node, nil } +func (v *standInVault) Drop(node string) error { + v.mu.Lock() + delete(v.tokens, node) + v.mu.Unlock() + v.note("drop " + node) + return nil +} +func (v *standInVault) Account(node, sealing, digest string) (delivery.NodeAccountAnswer, error) { + raw, err := delivery.EncodeNodeLogin(delivery.NodeLogin{Node: node, Login: delivery.FirstLogin(node), Since: time.Now().UTC()}) + if err != nil { + return delivery.NodeAccountAnswer{}, err + } + if _, err := v.logins.Put(v.ctx, node, raw); err != nil { + return delivery.NodeAccountAnswer{}, err + } + return delivery.NodeAccountAnswer{Node: node, Login: delivery.FirstLogin(node), Account: "account-" + node}, nil +} +func (v *standInVault) Retire(node string) error { return nil } +func (v *standInVault) SignCertificate(node string, _ delivery.JoiningKeys) (delivery.SignAnswer, error) { + return delivery.SignAnswer{Kind: delivery.KindNodeCertificate, Node: node, Certificate: "PEM"}, nil +} +func (v *standInVault) SignReply(node string, reply []byte) (delivery.SignAnswer, error) { + return delivery.SignAnswer{Kind: delivery.KindEnrolment, Node: node, Signature: "sig", Signer: "fp"}, nil +} +func (v *standInVault) Signer() (string, error) { return "vault-key", nil } + +// 774/S60/AC1: issue, a warrant, redeem as enrol. and confirm, whole on a real bus. The user enrol. is composed +// while the key is live and leaves with it; the node confirms as the login its key of node-logins names; node-enrolled and +// placement-changed are said; the vault is asked to drop the token only after the end; and a token never redeemed is ended by +// the bus's own expiry, heard, saying enrolment-expired and bringing the drop. +func TestMeshEnrolmentWholeOnARealBus(t *testing.T) { + srv := testbus.Start(t) + nc, err := nats.Connect(srv.ClientURL()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + t.Cleanup(cancel) + mk := func(name string, ttl bool) jetstream.KeyValue { + cfg := jetstream.KeyValueConfig{Bucket: name, History: 5} + if ttl { + // A bucket with a time to live per key keeps one revision per key: the bus (nats-server 2.11.17) expires no key of + // one that keeps more, which this test found out (a state declared key-ttl takes no history). + cfg.History, cfg.LimitMarkerTTL = 1, time.Minute + } + kv, err := js.CreateKeyValue(ctx, cfg) + if err != nil { + t.Fatal(err) + } + return kv + } + enrolments := mk(delivery.EnrolmentsBucket, true) + logins, lease, placement := mk(delivery.NodeLoginsBucket, false), mk("mesh-controller_lease", false), mk("mesh-delivery-assigner_placement", false) + have := mk("node-engine_have", false) + if _, err := lease.Put(ctx, "holder", []byte(`{"handed-to":"mesh-grants","handed-by":"operator"}`)); err != nil { + t.Fatal(err) + } + if _, err := placement.Put(ctx, "hub.node", []byte(hubPlacement)); err != nil { + t.Fatal(err) + } + if _, err := have.Put(ctx, "hub.node", []byte(hubRecord)); err != nil { + t.Fatal(err) + } + // The seats' events are kept on a stream of the events, as the bus keeps them, and heard on their real subjects. + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "SEAT_EVENTS", + Subjects: []string{"mesh.seat.mesh-enrolment.event.>", "mesh.seat.mesh-placement.event.>"}}); err != nil { + t.Fatal(err) + } + var heardMu sync.Mutex + var heard []string + for _, subject := range []string{"mesh.seat.mesh-enrolment.event.*", "mesh.seat.mesh-placement.event.*"} { + if _, err := nc.Subscribe(subject, func(m *nats.Msg) { + heardMu.Lock() + heard = append(heard, m.Subject[strings.LastIndex(m.Subject, ".")+1:]) + heardMu.Unlock() + }); err != nil { + t.Fatal(err) + } + } + events := func() []string { heardMu.Lock(); defer heardMu.Unlock(); return slices.Clone(heard) } + + store := &busStore{ctx: ctx, js: js, kv: enrolments} + vault := &standInVault{ctx: ctx, logins: logins, tokens: map[string]delivery.Token{}} + asker := &fakeAsker{} + h := &Holder{Store: store, Read: busReader{ctx, js}, Vault: vault, Asker: asker, AskLasts: time.Hour, + Say: func(event string, body any, id string) error { + raw, _ := json.Marshal(body) + _, err := js.Publish(ctx, "mesh.seat.mesh-enrolment.event."+event, raw, jetstream.WithMsgID(id)) + return err + }, + // mesh-placement, stood in: an add-node writes the node's key, a remove-node deletes it, each saying placement-changed. + AddNode: func(a AddNode) error { + if a.Change.Op == delivery.OpRemoveNode { + if err := placement.Delete(ctx, a.Change.Node+".node"); err != nil { + return err + } + } else { + value := `{"v":1,"node":"` + a.Change.Node + `","decided":{"address":"` + a.Change.Decided.Address + `"},"modules":[]}` + if _, err := placement.Put(ctx, a.Change.Node+".node", []byte(value)); err != nil { + return err + } + } + _, err := js.Publish(ctx, "mesh.seat.mesh-placement.event.placement-changed", []byte(`{}`)) + return err + }, + WarrantOf: func(ask string) (string, error) { return ask + "@2026-11-02T10:00:00Z", nil }, + Warrant: func(string) error { return nil }, + Act: func(string, string, string) error { return nil }, + Loaded: func(string) (bool, error) { return true, nil }, + Logf: t.Logf} + + // The grants part's rule, stood in: a key at live is the user enrol.; a key gone takes it away. The holder's watch + // of its own bucket hears the bus's expiry marker. + var usersMu sync.Mutex + users := map[string]bool{} + hasUser := func(u string) bool { usersMu.Lock(); defer usersMu.Unlock(); return users[u] } + var down atomic.Bool // the holder is down: its watch hands nothing over + w, err := enrolments.WatchAll(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = w.Stop() }) + go func() { + for e := range w.Updates() { + if e == nil { + continue + } + node, ok := strings.CutPrefix(e.Key(), "node.") + if !ok || down.Load() { + continue + } + if e.Operation() == jetstream.KeyValuePut { + en, err := decodeEnrolment(e.Value()) + usersMu.Lock() + users[EnrolUser(node)] = err == nil && en.Stage == StageLive + usersMu.Unlock() + _ = h.Heard(stdio.OpPut, e.Key(), e.Value()) + continue + } + usersMu.Lock() + delete(users, EnrolUser(node)) + usersMu.Unlock() + // A delete is the holder's own; the marker the bus leaves when a key's own time to live ends is a purge. + if e.Operation() == jetstream.KeyValuePurge { + _ = h.Heard(stdio.OpExpire, e.Key(), nil) + } else { + _ = h.Heard(stdio.OpDelete, e.Key(), nil) + } + } + }() + // The durable consumer of the holder's own end events, stood in by a core subscription that asks the drop. + for _, e := range EndEvents { + key := Seat + "." + e + if _, err := nc.Subscribe("mesh.seat.mesh-enrolment.event."+e, func(m *nats.Msg) { + if err := h.Ended(key, m.Data); err != nil { + t.Errorf("%v", err) + } + }); err != nil { + t.Fatal(err) + } + } + waitFor := func(what string, ok func() bool) { + t.Helper() + for deadline := time.Now().Add(20 * time.Second); !ok(); time.Sleep(25 * time.Millisecond) { + if time.Now().After(deadline) { + t.Fatalf("never: %s (events %v, vault %v)", what, events(), vault.calls) + } + } + } + + // Issue, the operator's yes, redeem as enrol.ace, confirm as node.ace. + a, err := h.Issue("person.operator", "ace", "a new machine", tunnelKey, 0) + if err != nil { + t.Fatal(err) + } + if err := h.Warranted(asks.Option{ID: "yes"}, asks.Warrant{Ask: a.Ask}); err != nil { + t.Fatal(err) + } + waitFor("the user enrol.ace is composed", func() bool { return hasUser("enrol.ace") }) + // The token is whole (ADR 0587 §3, insight of 2026-11-02): it carries the first tunnel built at issue, the machine's tunnel + // key given then, the private network's range and the hub's key and endpoint. + vault.mu.Lock() + tok := vault.tokens["ace"] + vault.mu.Unlock() + if missing := tok.Missing(); len(missing) != 0 || tok.Tunnel == nil || tok.Tunnel.Key != tunnelKey || tok.Tunnel.Address != a.Address+"/32" { + t.Fatalf("the token misses %v: tunnel %+v", missing, tok.Tunnel) + } + // The node is in placement before redeem: the operator's yes placed it, so the hub peers it before the machine dials. + if ace, err := placement.Get(ctx, "ace.node"); err != nil || !strings.Contains(string(ace.Value()), a.Address) { + t.Fatalf("ace is not in placement before redeem: %v", err) + } + args := redeemArgsWith("ace", 1, "serving-ace", tunnelKey) + if _, err := h.Redeem("enrol.somebody", args); err == nil { + t.Fatal("redeemed by another caller") + } + ans, err := h.Redeem("enrol.ace", args) + if err != nil { + t.Fatal(err) + } + // The node dials as the login its key of node-logins names, and confirms as it. + e, err := logins.Get(ctx, "ace") + if err != nil { + t.Fatal(err) + } + nl, err := delivery.DecodeNodeLogin(e.Value()) + if err != nil || nl.Login != ans.Login { + t.Fatalf("node-logins says %+v, %v, and the answer %s", nl, err, ans.Login) + } + if _, err := h.Confirm(nl.Login, ""); err != nil { + t.Fatal(err) + } + waitFor("node-enrolled and placement-changed are said and the user leaves the list", func() bool { + ev := events() + return slices.Contains(ev, EventEnrolled) && slices.Contains(ev, "placement-changed") && !hasUser("enrol.ace") + }) + waitFor("the vault is asked to drop the token after the end", func() bool { + vault.mu.Lock() + defer vault.mu.Unlock() + _, held := vault.tokens["ace"] + return slices.Contains(vault.calls, "drop ace") && !held + }) + if keys, _ := store.Keys(); len(keys) != 0 { + t.Errorf("an ended enrolment leaves keys %v", keys) + } + + // A token never redeemed expires on the bus: the holder hears the marker, says enrolment-expired and the token is dropped. + if _, err := h.Issue("person.operator", "g14", "a second machine", tunnelKey, 2*time.Second); err != nil { + t.Fatal(err) + } + if err := h.Warranted(asks.Option{ID: "yes"}, asks.Warrant{Ask: asker.asked[len(asker.asked)-1]}); err != nil { + t.Fatal(err) + } + waitFor("enrolment-expired is said for g14", func() bool { return slices.Contains(events(), EventExpired) }) + waitFor("the expired token is dropped", func() bool { + vault.mu.Lock() + defer vault.mu.Unlock() + return slices.Contains(vault.calls, "drop g14") + }) + if hasUser("enrol.g14") { + t.Error("the user enrol.g14 stays after its key expired") + } + // It never confirmed, so it left placement again before its end was said. + if _, err := placement.Get(ctx, "g14.node"); err == nil { + t.Error("g14 stays in placement after its enrolment expired") + } + // Its address key had no expiry of its own: the holder took it last, after the drop (ADR 0589 §6). + waitFor("the expired enrolment's address key is deleted", func() bool { keys, _ := store.Keys(); return len(keys) == 0 }) + + // The holder was down when an enrolment expired: a holder started later finds its address key, which names its ask, and + // ends it the same way (here: the key deleted under the holder, then a pass as at start). + b, err := h.Issue("person.operator", "novox", "a third machine", tunnelKey, 0) + if err != nil { + t.Fatal(err) + } + if err := h.Warranted(asks.Option{ID: "yes"}, asks.Warrant{Ask: b.Ask}); err != nil { + t.Fatal(err) + } + asker.answer(b.Ask, "yes") // the router's record of the answer, which the catch-up reads (ADR 0589 §6) + if _, err := placement.Get(ctx, "novox.node"); err != nil { + t.Fatalf("novox was not placed on the yes: %v", err) + } + down.Store(true) + if err := enrolments.Purge(ctx, NodeKey("novox")); err != nil { // the bus took the key while no holder watched + t.Fatal(err) + } + waitFor("the key is gone", func() bool { _, found, _ := store.Get(NodeKey("novox")); return !found }) + if _, err := placement.Get(ctx, "novox.node"); err != nil { + t.Fatalf("novox left placement while no holder ran: %v", err) + } + if err := h.Reconcile(); err != nil { + t.Fatal(err) + } + if _, err := placement.Get(ctx, "novox.node"); err == nil { + t.Error("novox, expired while the holder was down, stays in placement") + } + waitFor("the token of novox is dropped and its address key goes", func() bool { + vault.mu.Lock() + dropped := slices.Contains(vault.calls, "drop novox") + vault.mu.Unlock() + keys, _ := store.Keys() + return dropped && len(keys) == 0 + }) +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go b/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go new file mode 100644 index 0000000..597f542 --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/issue.go @@ -0,0 +1,385 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asker" + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Event is what each of the seat's events carries (ADR 0587 §2): the key's whole content, which holds no secret and nothing +// derived from the token, and for an end the reason. Node is repeated at the top so a consumer reads it without the rest. +type Event struct { + Node string `json:"node"` + Enrolment Enrolment `json:"enrolment"` + Reason string `json:"reason,omitempty"` +} + +// eventID is the message id an event of e is said under: the bus keeps one message of it, so an event said again after a +// crash is not said twice. +func eventID(event string, e Enrolment) string { + return event + "." + e.Node + "." + strconv.FormatInt(e.Issued.UnixNano(), 10) +} + +func (h *Holder) sayEvent(event string, e Enrolment, reason string) error { + return h.say(event, Event{Node: e.Node, Enrolment: e, Reason: reason}, eventID(event, e)) +} + +// IssueAnswer is what `issue` answers: the ask is the operator's to answer, and nothing else happens until they do. +type IssueAnswer struct { + Node string `json:"node"` + Address string `json:"address"` + Ask string `json:"ask"` + Expires time.Time `json:"expires"` + Says string `json:"says"` +} + +// Issue opens an enrolment (ADR 0587 §4): it refuses at once, naming every finding, a name that is no node name or that a +// node record, a placement key or an open enrolment holds, and every call before the lease write; chooses the lowest free +// address and reserves it by creating its key (the bus refuses a key that exists, so two issues never get one address); +// writes the key at stage asked; and asks the operator, on a channel that proves who answered. The token is made only on +// the operator's yes (Warranted). +func (h *Holder) Issue(caller, node, why, overlayKey string, lasts time.Duration) (IssueAnswer, error) { + var findings []string + if caller == "" || caller == stdio.Unproven { + findings = append(findings, "the caller is not proven by the runtime, and an enrolment records who asked") + } + if err := CheckNode(node); err != nil { + findings = append(findings, err.Error()) + } + if strings.TrimSpace(why) == "" { + findings = append(findings, "no why is given: the operator is asked why this machine joins") + } + // The machine's tunnel key, made there by `nox-mesh-host key`: on the operator's yes the node enters placement with it, so + // the hub peers it before the machine redeems (the operator's decision of 2026-11-02 01:05). + if err := checkTunnelKey(overlayKey); err != nil { + findings = append(findings, err.Error()) + } + if lasts == 0 { + lasts = delivery.DefaultTokenLife + } + if lasts < 0 || lasts > delivery.MostTokenLife { + findings = append(findings, fmt.Sprintf("the expiry %s is outside zero to %s", lasts, delivery.MostTokenLife)) + } + if on, reason := h.issuing(); !on { + findings = append(findings, "issuing is off until the controller's lease is written to a holder that is no controller: "+reason+ + "; until then the controller's `token issue` is the only issuer (ADR 0587 §4)") + } + placements, perr := h.placements() + if perr != nil { + findings = append(findings, "placement cannot be read: "+perr.Error()) + } + if CheckNode(node) == nil { + if _, ok := placements[node]; ok { + findings = append(findings, node+" has a placement key already") + } + if h.hasRecord(node) { + findings = append(findings, node+" has a node record already") + } + if _, found, err := h.Store.Get(NodeKey(node)); err != nil { + findings = append(findings, "the enrolments cannot be read: "+err.Error()) + } else if found { + findings = append(findings, "an enrolment of "+node+" is open already: revoke it first") + } + } + if perr == nil { + if _, err := h.firstTunnel(placements, "0.0.0.0", overlayKey); err != nil { + findings = append(findings, err.Error()) + } + } + if len(findings) > 0 { + return IssueAnswer{}, fmt.Errorf("no enrolment of %q is issued: %s", node, strings.Join(findings, "; ")) + } + now := h.now() + e := Enrolment{Node: node, Why: strings.TrimSpace(why), Caller: caller, Stage: StageAsked, Issued: now, Expires: now.Add(lasts), + OverlayKey: overlayKey} + e.Ask = "enrol-" + node + "-" + now.UTC().Format("20060102t150405") + skip := map[string]bool{} + for { + reserved, err := h.reservedAddresses() + if err != nil { + return IssueAnswer{}, err + } + addr, err := nextAddress(placements, reserved, skip) + if err != nil { + return IssueAnswer{}, fmt.Errorf("no enrolment of %q is issued: %w", node, err) + } + switch err := h.Store.Create(AddressKey(addr), Reservation{Node: node}, 0); { + case err == nil: + e.Address = addr + case errors.Is(err, ErrExists): + skip[addr] = true // another issue reserved it first: the next + continue + default: + return IssueAnswer{}, fmt.Errorf("the address %s cannot be reserved: %w", addr, err) + } + break + } + if err := h.Store.Create(NodeKey(node), e, 0); err != nil { + _ = h.Store.Delete(AddressKey(e.Address)) + if errors.Is(err, ErrExists) { + return IssueAnswer{}, fmt.Errorf("no enrolment of %q is issued: an enrolment of it was opened meanwhile", node) + } + return IssueAnswer{}, fmt.Errorf("the enrolment of %s cannot be written: %w", node, err) + } + if err := h.raise(e); err != nil { + _ = h.release(e) + return IssueAnswer{}, err + } + return IssueAnswer{Node: node, Address: e.Address, Ask: e.Ask, Expires: e.Expires, + Says: "the operator is asked on their channel; only their yes makes the token. mesh-enrolment.tokens shows where it stands"}, nil +} + +// release takes an enrolment's two keys away, the address key first and only while it still names this node (a key some +// other enrolment now holds is never deleted), then the node's key, whose presence is what lets a call made again finish a +// release that stopped. It answers what could not be deleted. +func (h *Holder) release(e Enrolment) error { + var errs []error + if row, found, err := h.Store.Get(AddressKey(e.Address)); err != nil { + errs = append(errs, fmt.Errorf("the address key %s cannot be read: %w", e.Address, err)) + } else if found { + var r Reservation + if json.Unmarshal(row.Value, &r) == nil && r.Node == e.Node { + if err := h.Store.Delete(AddressKey(e.Address)); err != nil { + errs = append(errs, fmt.Errorf("the address key %s could not be deleted: %w", e.Address, err)) + } + } + } + if len(errs) == 0 { + if err := h.Store.Delete(NodeKey(e.Node)); err != nil { + errs = append(errs, fmt.Errorf("the key of %s could not be deleted: %w", e.Node, err)) + } + } + if err := errors.Join(errs...); err != nil { + h.logf("%v", err) + return err + } + return nil +} + +// AskOf is the ask of an enrolment, in the words the operator reads: that a machine of this name may join until the +// expiry at this address. Its yes binds the act (name, address, expiry). +func (h *Holder) AskOf(e Enrolment) (asks.Ask, error) { + act := asks.Act{"verb": Seat + ".issue", "module": Module, "node": e.Node, "address": e.Address, + "expires": e.Expires.UTC().Format(time.RFC3339)} + bind, err := asks.ActDigest(act) + if err != nil { + return asks.Ask{}, err + } + until := e.Expires.UTC().Format("2006-01-02 15:04 MST") + closes := h.now().Add(h.AskLasts) + if closes.After(e.Expires) { + closes = e.Expires + } + return asks.Ask{ID: e.Ask, Headline: "Let the machine " + e.Node + " join the mesh?", + Explanation: "A machine named " + e.Node + " may join the mesh until " + until + ", at the private address " + e.Address + ".\n" + + "Your yes lets the vault make a one-time token for it; you open the token at your own desk and type it on the new machine.\n" + + "What the machine runs is placed later, as for any machine.", + Who: asks.Operator, + Options: []asks.Option{{ID: "yes", Label: "Yes, let it join", Does: "The vault makes a one-time token for " + e.Node + ", valid until " + until + ".", Level: asks.Approve, Binds: bind}, + {ID: "no", Label: "No", Does: "Nothing is made; the address is free again.", Level: asks.Acknowledge}}, + Expires: closes, OnExpiry: "No token is made and the address is free again.", About: "mesh-enrolment-" + e.Node, + Details: "Asked by " + e.Caller + ": " + e.Why}, nil +} + +// raise asks the operator; a question that cannot be asked is taken back at the router, since a publish that timed out may +// have reached it, so a yes cannot land on an enrolment the holder has forgotten. +func (h *Holder) raise(e Enrolment) error { + ask, err := h.AskOf(e) + if err != nil { + return err + } + // The address key names the ask and the issue's time from now, with no expiry of its own: it outlives the enrolment key, + // so an enrolment that expires unseen can still be taken out of placement on its ask's warrant (ADR 0589 §6). + if _, err := h.Store.Put(AddressKey(e.Address), Reservation{Node: e.Node, Ask: e.Ask, Issued: e.Issued}, 0); err != nil { + return fmt.Errorf("the address key %s cannot name the ask %s, so the operator is not asked: %w", e.Address, e.Ask, err) + } + if err := h.Asker.Ask(ask); err != nil { + if cerr := h.Asker.Cancel(ask.ID); cerr != nil { + h.logf("the ask %s that could not be published was not taken back at the router: %v", ask.ID, cerr) + } + return fmt.Errorf("the operator could not be asked about %s: %w", e.Node, err) + } + return nil +} + +// byAsk finds the open enrolment an ask belongs to. +func (h *Holder) byAsk(ask string) (Enrolment, Row, bool, error) { + rows, open, err := h.openEnrolments() + if err != nil { + return Enrolment{}, Row{}, false, err + } + for n, e := range open { + if e.Ask == ask { + return e, rows[n], true, nil + } + } + return Enrolment{}, Row{}, false, nil +} + +// Warranted is the operator's answer, handed once by the asker (asker.Client.Listen): a no ends the enrolment; a yes records +// the warrant, adds the node to placement with its tunnel key, asks the vault for the token, moves the key to live with the bus's expiry at the token's, and says +// enrolment-issued (ADR 0587 §4, §5). +func (h *Holder) Warranted(o asks.Option, w asks.Warrant) error { + e, _, found, err := h.byAsk(w.Ask) + if err != nil { + return err + } + if !found { + return fmt.Errorf("the warrant answers the ask %s, which no open enrolment holds: nothing is made", w.Ask) + } + if e.Stage != StageAsked { + return nil // answered already + } + if o.ID != "yes" { + return h.refuse(e, "the operator declined") + } + if !h.now().Before(e.Expires) { + return h.refuse(e, "the enrolment's expiry passed before the operator answered") + } + if h.Warrant != nil { + if err := h.Warrant(w.Ask); err != nil { + return fmt.Errorf("the warrant %s is not recorded in the hand-act log, so no token is made (mesh-enrolment.again asks again): %w", w.Ask, err) + } + } + placements, err := h.placements() + if err != nil { + return fmt.Errorf("placement cannot be read, so the token's tunnel is not built (mesh-enrolment.again asks again): %w", err) + } + tunnel, err := h.firstTunnel(placements, e.Address, e.OverlayKey) + if err != nil { + return fmt.Errorf("the token's tunnel is not built (mesh-enrolment.again asks again): %w", err) + } + // The node enters placement now, with its tunnel key, on this warrant: the hub peers it before the machine redeems over + // the tunnel (the operator's decision of 2026-11-02 01:05). Only then is the token made. + if err := h.place(e, placements); err != nil { + if addRefused(err) { + // mesh-placement refused the node itself, not the call to it: the enrolment ends (ADR 0589 §6). + return h.refuse(e, "mesh-placement refused the node: "+err.Error()) + } + return fmt.Errorf("the node %s is not added to placement, so no token is made (mesh-enrolment.again asks again): %w", e.Node, err) + } + if _, err := h.Vault.Token(e.Node, e.Expires, tunnel); err != nil { + return fmt.Errorf("the vault did not make the token of %s (mesh-enrolment.again asks the operator again): %w", e.Node, err) + } + e.Stage = StageLive + ttl := h.lifetimeOf(e.Expires) + if _, err := h.Store.Put(NodeKey(e.Node), e, ttl); err != nil { + return fmt.Errorf("the enrolment of %s cannot be moved to live: %w", e.Node, err) + } + return h.sayEvent(EventIssued, e, "") +} + +// addRefused says mesh-placement answered the add-node with a refusal of the proposal (the assigner's words, "the proposal is +// refused: …"), as against a call that did not reach it or was not answered, which `again` asks once more. +func addRefused(err error) bool { + return err != nil && strings.Contains(err.Error(), "the proposal is refused") +} + +// dropAddress deletes the enrolment's address key while it still names this node: the first step of a confirm, a decline and a +// refused add (ADR 0589 §6), so the catch-up never takes a key left by one of them for an expiry never heard. +func (h *Holder) dropAddress(e Enrolment) error { + row, found, err := h.Store.Get(AddressKey(e.Address)) + if err != nil { + return fmt.Errorf("the address key %s cannot be read, so the end of %s waits: %w", e.Address, e.Node, err) + } + if !found { + return nil + } + if r, err := delivery.DecodeEnrolmentAddress(row.Value); err == nil && r.Node != e.Node { + return nil // another enrolment holds the address now + } + if err := h.Store.Delete(AddressKey(e.Address)); err != nil { + return fmt.Errorf("the address key %s is not deleted, so the end of %s waits: %w", e.Address, e.Node, err) + } + return nil +} + +// refuse ends an enrolment at stage asked: both keys go and enrolment-refused carries the key's content (ADR 0587 §4). +func (h *Holder) refuse(e Enrolment, reason string) error { + // The address key goes first (ADR 0589 §6): a refusal stopped midway never leaves one that looks like an expiry. + if err := h.dropAddress(e); err != nil { + return err + } + // A yes whose token failed added the node already; a refusal takes it off again. + if err := h.unplace(e, reason); err != nil { + return fmt.Errorf("%s is not taken off placement, so its enrolment stays: %w", e.Node, err) + } + if err := h.sayEvent(EventRefused, e, reason); err != nil { + return fmt.Errorf("enrolment-refused of %s cannot be said, so its keys stay: %w", e.Node, err) + } + return h.release(e) +} + +// Said is the asker's note of an ask that ended with no yes or no (expired, or refused by the router): the enrolment ends. +func (h *Holder) Said(hd asker.Handled, w asks.Warrant, err error) { + if err != nil { + h.logf("the answer to %s (%s) ended in an error: %v", w.Ask, hd, err) + } + if hd != asker.Recorded { + return + } + e, _, found, ferr := h.byAsk(w.Ask) + if ferr != nil || !found || e.Stage != StageAsked { + return + } + if rerr := h.refuse(e, "the ask ended without an answer ("+string(w.Outcome)+")"); rerr != nil { + h.logf("%v", rerr) + } +} + +// Again asks the operator once more, when the router no longer holds the ask of an enrolment that is still at stage asked +// (ADR 0587 §8): a new ask under a new id, for the same node, address and expiry. +func (h *Holder) Again(node string) (IssueAnswer, error) { + row, found, err := h.Store.Get(NodeKey(node)) + if err != nil { + return IssueAnswer{}, err + } + if !found { + return IssueAnswer{}, fmt.Errorf("no enrolment of %s is open", node) + } + e, err := decodeEnrolment(row.Value) + if err != nil { + return IssueAnswer{}, err + } + if e.Stage != StageAsked { + return IssueAnswer{}, fmt.Errorf("the enrolment of %s is %s, not asked: there is nothing to ask again (a redeem stopped midway is finished by the machine calling redeem again)", node, e.Stage) + } + if !h.now().Before(e.Expires) { + return IssueAnswer{}, fmt.Errorf("the enrolment of %s has expired; revoke it and issue it again", node) + } + if h.Asker != nil { + if state, _, werr := h.whereIs(e.Ask); werr == nil && (state == "open" || state == "held") { + return IssueAnswer{}, fmt.Errorf("the ask %s is still %s at the router: the operator can answer it", e.Ask, state) + } + } + old := e.Ask + e.Ask = "enrol-" + node + "-" + h.now().UTC().Format("20060102t150405") + if e.Ask == old { + e.Ask += "b" + } + if _, err := h.Store.Update(NodeKey(node), e, row.Revision); err != nil { + return IssueAnswer{}, fmt.Errorf("the enrolment of %s moved meanwhile: %w", node, err) + } + if err := h.raise(e); err != nil { + return IssueAnswer{}, err + } + return IssueAnswer{Node: node, Address: e.Address, Ask: e.Ask, Expires: e.Expires, Says: "the operator is asked again"}, nil +} + +// whereIs is the router's state of an ask, when the asker can read it. +func (h *Holder) whereIs(id string) (string, *asks.Warrant, error) { + if w, ok := h.Asker.(interface { + Where(string) (string, *asks.Warrant, error) + }); ok { + return w.Where(id) + } + return "", nil, errors.New("the asker cannot read the router's record") +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/issue_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/issue_test.go new file mode 100644 index 0000000..7c59eaf --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/issue_test.go @@ -0,0 +1,173 @@ +package main + +import ( + "slices" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/asker" + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +var yes = asks.Option{ID: "yes"} + +// 774/S55/AC1: issue refuses, naming every finding, what it must: the lease with the controller, a bad name, a name a node +// record, a placement key or an open enrolment holds, and an expiry past a day. +func issueRefusesNamingEveryFinding(t *testing.T) { + cases := []struct { + name string + setup func(r *rig) + node string + lasts time.Duration + want []string + }{ + {"the lease names the controller", func(r *rig) { r.rd["mesh-controller.lease"]["holder"] = `{"instance":"novox.mesh-controller"}` }, "ace", 0, + []string{"issuing is off until the controller's lease is written", "token issue"}}, + {"a bad name and a long expiry", nil, "Ace.1", 48 * time.Hour, []string{"not a valid node name", "outside zero to 24h0m0s"}}, + {"a placement key and a node record", func(r *rig) { + r.rd["mesh-delivery-assigner.placement"]["ace.node"] = `{"v":1,"node":"ace","modules":[]}` + r.rd["node-engine.have"]["ace"] = `{}` + }, "ace", 0, []string{"ace has a placement key already"}}, + {"an open enrolment", func(r *rig) { r.seed("ace", "10.9.0.2", StageLive) }, "ace", 0, []string{"an enrolment of ace is open already"}}, + {"no tunnel key and no range", func(r *rig) { + r.rd["mesh-delivery-assigner.placement"]["hub.node"] = strings.Replace(hubPlacement, `,"range":"10.9.0.0/24"`, "", 1) + }, "ace", -1, []string{"no tunnel key is given", "names no tunnel range"}}, + // The token carries the first tunnel (ADR 0587 §3, insight of 2026-11-02): no hub, or a hub with no key or endpoint. + {"no hub", func(r *rig) { + r.rd["mesh-delivery-assigner.placement"]["hub.node"] = strings.Replace(hubPlacement, `"hub":true,`, "", 1) + }, "ace", 0, []string{"placement names no hub"}}, + {"a hub with no key and no endpoint", func(r *rig) { + r.rd["mesh-delivery-assigner.placement"]["hub.node"] = strings.Replace(hubPlacement, `"endpoint":"192.0.2.1:51820",`, "", 1) + delete(r.rd["node-engine.have"], "hub.node") + }, "ace", 0, []string{"the hub hub has no tunnel key in its record and no endpoint in its placement"}}, + } + for _, c := range cases { + r := newRig(t) + if c.setup != nil { + c.setup(r) + } + key, lasts := tunnelKey, c.lasts + if lasts == -1 { + key, lasts = "", 0 + } + _, err := r.h.Issue("person.operator", c.node, "a new machine", key, lasts) + if err == nil { + t.Fatalf("%s: issued", c.name) + } + for _, w := range c.want { + mustContain(t, err.Error(), w) + } + if len(r.asker.asked) != 0 || strings.Contains(strings.Join(r.keys(), " "), "address.10.9.0.3") { + t.Errorf("%s: asked %v, keys %v", c.name, r.asker.asked, r.keys()) + } + } +} + +// 774/S55/AC2: two issues at once for two nodes hold two addresses; a yes makes the token, moves both keys to live with the +// token's expiry, records the warrant, says enrolment-issued, and a second answer changes nothing. +func twoIssuesAtOnceHoldTwoAddressesAndAYesMakesTheToken(t *testing.T) { + r := newRig(t) + var wg sync.WaitGroup + addrs := make([]string, 2) + for i, n := range []string{"ace", "g14"} { + wg.Add(1) + go func() { + defer wg.Done() + a, err := r.h.Issue("person.operator", n, "a new machine", tunnelKey, 0) + if err != nil { + t.Error(err) + } + addrs[i] = a.Address + }() + } + wg.Wait() + if addrs[0] == "" || addrs[0] == addrs[1] || addrs[0] == "10.9.0.1" || addrs[1] == "10.9.0.1" { + t.Fatalf("addresses %v", addrs) + } + e, _ := r.enrolment("ace") + if e.Stage != StageAsked || e.Caller != "person.operator" || e.Ask == "" || len(r.asker.asked) != 2 { + t.Fatalf("%+v, asked %v", e, r.asker.asked) + } + ask, _ := r.h.AskOf(e) + if ask.Options[0].Level != asks.Approve || ask.Options[0].Binds == "" || ask.Check(r.now) != nil { + t.Errorf("the ask does not authorise the act: %+v, %v", ask.Options[0], ask.Check(r.now)) + } + if err := r.h.Warranted(yes, asks.Warrant{Ask: e.Ask}); err != nil { + t.Fatal(err) + } + live, _ := r.enrolment("ace") + if live.Stage != StageLive || r.st.ttl[NodeKey("ace")] != time.Hour || r.st.ttl[AddressKey(live.Address)] != 0 || + len(r.warr) != 1 || r.warr[0] != e.Ask || r.events()[0] != EventIssued { + t.Errorf("live %+v, ttl %v/%v, warrants %v, events %v", live, r.st.ttl[NodeKey("ace")], r.st.ttl[AddressKey(live.Address)], r.warr, r.events()) + } + // The address key names the ask and the issue, with no expiry of its own, so it outlives the enrolment key (ADR 0589 §6). + if row, _, _ := r.st.Get(AddressKey(live.Address)); true { + if res, err := delivery.DecodeEnrolmentAddress(row.Value); err != nil || res.Ask != e.Ask || !res.Issued.Equal(live.Issued) { + t.Errorf("the address key %s: %+v, %v", row.Value, res, err) + } + } + if got, want := r.vault.tunnels["ace"], (delivery.TokenTunnel{Key: tunnelKey, Address: live.Address + "/32", Range: "10.9.0.0/24", + HubKey: "hub-tunnel-key", HubEndpoint: "192.0.2.1:51820"}); got != want { + t.Errorf("the token's tunnel %+v, want %+v", got, want) + } + // The yes added the node to placement with its tunnel key alone, on the warrant, before the token was made. + if p, tok := slices.Index(r.vault.calls, "add-node ace"), slices.Index(r.vault.calls, "token ace"); p < 0 || tok < p { + t.Errorf("the token was made before placement held the node: %v", r.vault.calls) + } + for _, a := range r.added { + if a.Change.Node == "ace" && (*a.Change.Keys != delivery.JoiningKeys{Overlay: tunnelKey} || a.Warrant.Ask != e.Ask || a.Change.Decided.Address != live.Address) { + t.Errorf("the add-node at the yes %+v", a) + } + } + if strings.Contains(string(r.said[0].Body), "secret") || r.said[0].ID != eventID(EventIssued, live) { + t.Errorf("the event %s carries a secret-like word or no stable id", r.said[0].Body) + } + if err := r.h.Warranted(yes, asks.Warrant{Ask: e.Ask}); err != nil || len(r.said) != 1 { + t.Errorf("a second answer said %v: %v", r.events(), err) + } +} + +// 774/S55/AC3: a declined or expired ask, heard, takes both keys away, and enrolment-refused carries the key's content; a +// vault that fails leaves the enrolment at asked, for `again` to ask once more. +func aDeclinedOrExpiredAskTakesBothKeysAndAVaultFailureKeepsTheEnrolmentForAgain(t *testing.T) { + r := newRig(t) + a, _ := r.h.Issue("person.operator", "ace", "a new machine", tunnelKey, 0) + if err := r.h.Warranted(asks.Option{ID: "no"}, asks.Warrant{Ask: a.Ask}); err != nil { + t.Fatal(err) + } + if len(r.keys()) != 0 || r.events()[0] != EventRefused || len(r.added) != 0 { + t.Fatalf("keys %v events %v placement %v", r.keys(), r.events(), r.added) + } + mustContain(t, string(r.said[0].Body), `"address":"`+a.Address+`"`) + b, _ := r.h.Issue("person.operator", "g14", "a new machine", tunnelKey, 0) + r.h.Said(asker.Recorded, asks.Warrant{Ask: b.Ask, Outcome: asks.OutcomeExpired}, nil) + if len(r.keys()) != 0 || r.events()[1] != EventRefused { + t.Fatalf("keys %v events %v", r.keys(), r.events()) + } + c, _ := r.h.Issue("person.operator", "novox", "a new machine", tunnelKey, 0) + r.vault.fail["token novox"] = true + if err := r.h.Warranted(yes, asks.Warrant{Ask: c.Ask}); err == nil { + t.Fatal("a vault that failed was taken for a token") + } + if e, _ := r.enrolment("novox"); e.Stage != StageAsked { + t.Fatalf("stage %s", e.Stage) + } + // The yes placed the node before the vault failed: a refusal now would take it off again (decline, below for ace). + if _, placed := r.rd["mesh-delivery-assigner.placement"]["novox.node"]; !placed { + t.Fatal("the yes did not place novox before the vault was asked") + } + r.now = r.now.Add(time.Minute) + again, err := r.h.Again("novox") + if err != nil || again.Ask == c.Ask || len(r.asker.asked) != 4 { + t.Fatalf("again %+v, %v, asked %v", again, err, r.asker.asked) + } + if err := r.h.Warranted(yes, asks.Warrant{Ask: again.Ask}); err != nil { + t.Fatal(err) + } + if e, _ := r.enrolment("novox"); e.Stage != StageLive { + t.Errorf("stage %s", e.Stage) + } +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go new file mode 100644 index 0000000..4dbad2b --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/kit_test.go @@ -0,0 +1,305 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// memStore is the holder's bucket in memory: revisions, and the expiry each key was written with. +type memStore struct { + mu sync.Mutex + rows map[string]Row + ttl map[string]time.Duration + rev uint64 + // failDelete names keys whose delete fails once. + failDelete map[string]bool +} + +func newMem() *memStore { + return &memStore{rows: map[string]Row{}, ttl: map[string]time.Duration{}, failDelete: map[string]bool{}} +} + +func (m *memStore) write(key string, v any, ttl time.Duration) uint64 { + raw, err := Encode(v) + if err != nil { + panic(err) // a test wrote a value the contract refuses + } + m.rev++ + m.rows[key] = Row{Value: raw, Revision: m.rev} + m.ttl[key] = ttl + return m.rev +} + +func (m *memStore) Get(key string) (Row, bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + r, ok := m.rows[key] + return r, ok, nil +} + +func (m *memStore) Create(key string, v any, ttl time.Duration) error { + m.mu.Lock() + defer m.mu.Unlock() + if _, ok := m.rows[key]; ok { + return ErrExists + } + m.write(key, v, ttl) + return nil +} + +func (m *memStore) Update(key string, v any, revision uint64) (uint64, error) { + m.mu.Lock() + defer m.mu.Unlock() + if r, ok := m.rows[key]; !ok || r.Revision != revision { + return 0, ErrChanged + } + return m.write(key, v, 0), nil +} + +func (m *memStore) Put(key string, v any, ttl time.Duration) (uint64, error) { + m.mu.Lock() + defer m.mu.Unlock() + return m.write(key, v, ttl), nil +} + +func (m *memStore) Delete(key string) error { + m.mu.Lock() + defer m.mu.Unlock() + if m.failDelete[key] { + delete(m.failDelete, key) + return errors.New("the bus did not answer") + } + delete(m.rows, key) + delete(m.ttl, key) + return nil +} + +func (m *memStore) Keys() ([]string, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out []string + for k := range m.rows { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +// memRead is other modules' states, `.` then key. +type memRead map[string]map[string]string + +func (r memRead) Keys(state string) ([]string, error) { + var out []string + for k := range r[state] { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +func (r memRead) Get(state, key string) ([]byte, bool, error) { + v, ok := r[state][key] + return []byte(v), ok, nil +} + +// fakeVault records its calls in order and fails on the verbs named in fail, once each. +type fakeVault struct { + calls []string + fail map[string]bool + tunnels map[string]delivery.TokenTunnel + cert delivery.JoiningKeys // the keys the certificate's call carried +} + +func (v *fakeVault) do(what string) error { + v.calls = append(v.calls, what) + if v.fail[what] { + delete(v.fail, what) + return errors.New("the vault did not answer " + what) + } + return nil +} + +func (v *fakeVault) Token(node string, expires time.Time, tunnel delivery.TokenTunnel) (delivery.EnrolmentTokenAnswer, error) { + if v.tunnels == nil { + v.tunnels = map[string]delivery.TokenTunnel{} + } + v.tunnels[node] = tunnel + return delivery.EnrolmentTokenAnswer{Node: node, Broker: "b", Fingerprint: "f", Signer: "s"}, v.do("token " + node) +} +func (v *fakeVault) Sealed(node string) (string, error) { + return "sealed-for-" + node, v.do("sealed " + node) +} +func (v *fakeVault) Drop(node string) error { return v.do("drop " + node) } +func (v *fakeVault) Account(node, sealing, digest string) (delivery.NodeAccountAnswer, error) { + return delivery.NodeAccountAnswer{Node: node, Login: "node." + node, Account: "account-" + node}, v.do("account " + node) +} +func (v *fakeVault) Retire(node string) error { return v.do("retire " + node) } +func (v *fakeVault) SignCertificate(node string, k delivery.JoiningKeys) (delivery.SignAnswer, error) { + serving := k.Serving + v.cert = k + return delivery.SignAnswer{Kind: delivery.KindNodeCertificate, Node: node, Certificate: "PEM-" + serving}, v.do("certificate " + node) +} +func (v *fakeVault) SignReply(node string, reply []byte) (delivery.SignAnswer, error) { + return delivery.SignAnswer{Kind: delivery.KindEnrolment, Node: node, Signature: "sig-" + delivery.DigestOf(reply)[7:15], Signer: "fp"}, v.do("sign " + node) +} +func (v *fakeVault) Signer() (string, error) { return "vault-signer", nil } + +type fakeAsker struct { + mu sync.Mutex + asked, cancelled []string + err error + answers map[string]string // the router's record: the option each ask was answered with +} + +// answer records that the operator answered ask with option, as the router keeps it. +func (a *fakeAsker) answer(ask, option string) { + a.mu.Lock() + defer a.mu.Unlock() + if a.answers == nil { + a.answers = map[string]string{} + } + a.answers[ask] = option +} + +func (a *fakeAsker) Where(id string) (string, *asks.Warrant, error) { + a.mu.Lock() + defer a.mu.Unlock() + if o, ok := a.answers[id]; ok { + return "answered", &asks.Warrant{Ask: id, Option: o}, nil + } + return "", nil, nil // no record read: as a router whose record of the ask is gone +} + +func (a *fakeAsker) Ask(k asks.Ask) error { + a.mu.Lock() + defer a.mu.Unlock() + if a.err != nil { + return a.err + } + a.asked = append(a.asked, k.ID) + return nil +} +func (a *fakeAsker) Cancel(id string) error { + a.mu.Lock() + defer a.mu.Unlock() + a.cancelled = append(a.cancelled, id) + return nil +} + +type said struct { + Event, ID string + Body json.RawMessage +} + +// rig is a holder over fakes, with the lease handed to mesh-grants, one hub in placement and a clock the test moves. +type rig struct { + mu sync.Mutex + t *testing.T + h *Holder + st *memStore + vault *fakeVault + asker *fakeAsker + rd memRead + now time.Time + said []said + added []AddNode + acts []string + warr []string + loaded bool + placeErr error +} + +// hubRecord is the hub's own record in node-engine.have: its tunnel key, which the token's tunnel names. +const hubRecord = `{"v":1,"record":{"v":1,"node":"hub","tunnel-key":"hub-tunnel-key"}}` + +// tunnelKey is a machine's tunnel public key as `nox-mesh-host key` prints it. +const tunnelKey = "dHVubmVsLWtleS1vZi10aGUtbmV3LW1hY2hpbmUtMDE=" + +const hubPlacement = `{"v":1,"node":"hub","decided":{"hub":true,"endpoint":"192.0.2.1:51820","address":"10.9.0.1","tunnel":{"interface":"wg0","address":"10.9.0.1/24","range":"10.9.0.0/24"}},"modules":[]}` + +func newRig(t *testing.T) *rig { + r := &rig{t: t, st: newMem(), vault: &fakeVault{fail: map[string]bool{}}, asker: &fakeAsker{}, now: time.Date(2026, 11, 2, 10, 0, 0, 0, time.UTC), loaded: true} + r.rd = memRead{ + "mesh-controller.lease": {"holder": `{"handed-to":"mesh-grants","handed-by":"operator"}`}, + "mesh-delivery-assigner.placement": {"hub.node": hubPlacement}, + "node-engine.have": {"hub.node": hubRecord}, + } + r.h = &Holder{Store: r.st, Read: r.rd, Vault: r.vault, Asker: r.asker, AskLasts: time.Hour, + Say: func(event string, body any, id string) error { + raw, _ := json.Marshal(body) + r.mu.Lock() + defer r.mu.Unlock() + r.said = append(r.said, said{event, id, raw}) + return nil + }, + AddNode: func(a AddNode) error { + r.added = append(r.added, a) + r.vault.calls = append(r.vault.calls, a.Change.Op+" "+a.Change.Node) + if r.placeErr == nil { // placement writes or deletes the node as the assigner does on the warrant + n := a.Change.Node + if a.Change.Op == delivery.OpRemoveNode { + delete(r.rd["mesh-delivery-assigner.placement"], n+".node") + } else { + r.rd["mesh-delivery-assigner.placement"][n+".node"] = `{"v":1,"node":"` + n + `","decided":{"address":"` + a.Change.Decided.Address + `"},"modules":[]}` + } + } + return r.placeErr + }, + Warrant: func(ask string) error { r.warr = append(r.warr, ask); return nil }, + WarrantOf: func(ask string) (string, error) { return ask + "@2026-11-02T10:00:00Z", nil }, + Act: func(what, why, cause string) error { r.acts = append(r.acts, what); return nil }, + Loaded: func(string) (bool, error) { return r.loaded, nil }, + Now: func() time.Time { return r.now }, + Logf: func(f string, a ...any) { t.Logf(f, a...) }} + return r +} + +func (r *rig) events() []string { + var out []string + for _, s := range r.said { + out = append(out, s.Event) + } + return out +} + +func (r *rig) enrolment(node string) (Enrolment, bool) { + row, ok, _ := r.st.Get(NodeKey(node)) + if !ok { + return Enrolment{}, false + } + e, err := decodeEnrolment(row.Value) + if err != nil { + r.t.Fatal(err) + } + return e, true +} + +func (r *rig) keys() []string { k, _ := r.st.Keys(); return k } + +func (r *rig) has(key string) bool { return slices.Contains(r.keys(), key) } + +func mustContain(t *testing.T, got, want string) { + t.Helper() + if !strings.Contains(got, want) { + t.Fatalf("%q does not contain %q", got, want) + } +} + +func errText(err error) string { + if err == nil { + return "" + } + return fmt.Sprint(err) +} + +func askWarrant(ask string) asks.Warrant { return asks.Warrant{Ask: ask} } diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/main.go b/modules/mesh-enrolment/cmd/mesh-enrolment/main.go new file mode 100644 index 0000000..5016954 --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/main.go @@ -0,0 +1,179 @@ +// mesh-enrolment: the holder of the seat mesh-enrolment (novox/hq ADR 0587). A Go bundle served by a runtime of its own, +// on its own bus account (`runs-as`), so that its account alone writes its bucket `enrolments` with per-key expiry. It +// asks the operator on their channel, the vault to mint and sign, mesh-placement to add the node and mesh-hand-acts to +// record the acts. stdout is the MCP channel; what this module says, it says on stderr. +package main + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asker" + "git.novox.be/novox/mesh-sdk/go/handacts" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +// runtimeStore is the holder's bucket, through the runtime that launched this bundle. +type runtimeStore struct{} + +func (runtimeStore) Get(key string) (Row, bool, error) { + e, err := stdio.State(State).Get(key) + if err != nil || e == nil { + return Row{}, false, err + } + return Row{Value: e.Value, Revision: e.Revision}, true, nil +} + +func (runtimeStore) Create(key string, v any, ttl time.Duration) error { + raw, err := Encode(v) + if err != nil { + return err + } + if ttl > 0 { + _, err = stdio.State(State).CreateTTL(key, raw, ttl) + } else { + _, err = stdio.State(State).Create(key, raw) + } + if errors.Is(err, stdio.ErrChanged) { + return ErrExists + } + return err +} + +func (runtimeStore) Update(key string, v any, revision uint64) (uint64, error) { + raw, err := Encode(v) + if err != nil { + return 0, err + } + rev, err := stdio.State(State).Update(key, raw, revision) + if errors.Is(err, stdio.ErrChanged) { + return 0, ErrChanged + } + return rev, err +} + +func (runtimeStore) Put(key string, v any, ttl time.Duration) (uint64, error) { + raw, err := Encode(v) + if err != nil { + return 0, err + } + if ttl > 0 { + return stdio.State(State).PutTTL(key, raw, ttl) + } + return stdio.State(State).Put(key, raw) +} + +func (runtimeStore) Delete(key string) error { return stdio.State(State).Delete(key) } +func (runtimeStore) Keys() ([]string, error) { return stdio.State(State).Keys() } + +// runtimeReader reads another module's state through the runtime. +type runtimeReader struct{} + +func (runtimeReader) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +func (runtimeReader) Get(state, key string) ([]byte, bool, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +// SeatEventSubject is where the seat says an event: mesh.seat..event.. +func SeatEventSubject(event string) string { return "mesh.seat." + Seat + ".event." + event } + +// newHolder is the holder over the runtime. +func newHolder() *Holder { + call := Call(stdio.Ask) + rec := handacts.Recorder{Ask: stdio.Ask} + rd := runtimeReader{} + client := asker.New("asked") + return &Holder{ + Store: runtimeStore{}, Read: rd, Vault: vaultOver{call: call, read: rd}, Asker: client, + Say: func(event string, body any, id string) error { + return stdio.SeatPublish(SeatEventSubject(event), body, id) + }, + AddNode: addNodeOver(call), + WarrantOf: warrantOf(client), + Warrant: func(ask string) error { _, err := rec.RecordWarrant(Module, ask); return err }, + Act: func(what, why, cause string) error { + _, err := rec.Record(handacts.Request{What: what, Why: why, Cause: cause}) + return err + }, + Loaded: loadedVia(rd), + Now: time.Now, + Logf: func(f string, a ...any) { logf("["+Module+"] "+f, a...) }, + AskLasts: 55 * time.Minute, + } +} + +// warrantOf is the warrant id the teller writes, `@`, read from the router's record of the ask. +func warrantOf(c *asker.Client) func(string) (string, error) { + return func(ask string) (string, error) { + state, w, err := c.Where(ask) + if err != nil { + return "", err + } + if w == nil { + return "", fmt.Errorf("the router records the ask %s as %q, with no warrant", ask, state) + } + return ask + "@" + w.At.UTC().Format(time.RFC3339Nano), nil + } +} + +// loadedVia says a bus user is loaded when the grants part's last pass lists it among the composed users and the bus node's +// loaded list holds it (its credentials row says loaded). +func loadedVia(rd Reader) func(string) (bool, error) { + return func(user string) (bool, error) { + raw, found, err := rd.Get("mesh-grants.user-list", "pass") + if err != nil { + return false, err + } + if !found { + return false, errors.New("the grants part has composed no list yet") + } + // The key holds the part's whole pass result (mesh-grants Result): the list state sits under user_list. + var pass struct { + UserList struct { + Compare struct { + Unread string `json:"unread"` + } `json:"compare"` + Credentials struct { + Users []struct { + User string `json:"user"` + InShadow bool `json:"in_shadow"` + Loaded bool `json:"loaded"` + } `json:"users"` + } `json:"credentials"` + } `json:"user_list"` + } + if err := json.Unmarshal(raw, &pass); err != nil { + return false, fmt.Errorf("the grants part's list state does not read: %w", err) + } + st := pass.UserList + if st.Compare.Unread != "" { + return false, errors.New("the bus node's loaded list cannot be read: " + st.Compare.Unread) + } + for _, u := range st.Credentials.Users { + if u.User == user { + return u.InShadow && u.Loaded, nil + } + } + return false, nil + } +} + +func main() { + h := newHolder() + client := h.Asker.(*asker.Client) + h.Run(func() error { return client.Listen(h.Warranted, h.Said) }) + if err := stdio.Serve("", Tools(h)); err != nil { + logf("%v", err) + os.Exit(1) + } +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go new file mode 100644 index 0000000..7ee0b09 --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem.go @@ -0,0 +1,124 @@ +package main + +import ( + "crypto/ed25519" + "encoding/base64" + "errors" + "fmt" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Redeem is the joining node-engine's one call (ADR 0587 §5, §6). It answers only a caller the runtime proves to be +// enrol., for a key at stage live (or at redeeming with the same keys' digest), checks the proof against the identity key +// presented, and then, each step read back and none repeated that is done: spends the token (the key to redeeming, by +// compare-and-set, with no expiry) once placement holds the node (written on the operator's yes; the vault reads the node's +// address there) and the tunnel key presented is the one issued for, asks the vault for the node's account and the certificate +// of the keys presented, +// waits for the bus node to load node., and answers the whole, signed by the vault. Called again with the same keys it +// finishes what is not done and answers the same. +func (h *Holder) Redeem(caller string, a delivery.Redeem) (delivery.RedeemAnswer, error) { + if err := CheckNode(a.Node); err != nil { + return delivery.RedeemAnswer{}, err + } + if caller != EnrolUser(a.Node) { + return delivery.RedeemAnswer{}, fmt.Errorf("only %s may redeem the enrolment of %s, and the runtime proved the caller to be %q", EnrolUser(a.Node), a.Node, caller) + } + pub, err := base64.StdEncoding.DecodeString(a.PublicKey) + proof, perr := a.ProofBytes() + if err != nil || perr != nil || len(pub) != ed25519.PublicKeySize || + !ed25519.Verify(pub, delivery.RedeemProof(a.Node, a.PublicKey, a.OverlayKey, a.SealingKey, a.ServingKey), proof) { + return delivery.RedeemAnswer{}, errors.New("the proof does not verify against the identity key presented: the enrolment is not spent") + } + if a.SealingKey == "" { + return delivery.RedeemAnswer{}, errors.New("no sealing key is presented, so the account cannot be sealed to the node: the enrolment is not spent") + } + digest := a.KeysDigest() + e, err := h.spend(a.Node, digest, a.OverlayKey) + if err != nil { + return delivery.RedeemAnswer{}, err + } + account, err := h.Vault.Account(e.Node, a.SealingKey, digest) + if err != nil { + return delivery.RedeemAnswer{}, fmt.Errorf("the vault did not give the account of %s (call redeem again with the same keys): %w", e.Node, err) + } + cert, err := h.Vault.SignCertificate(e.Node, delivery.JoiningKeys{Identity: a.PublicKey, Overlay: a.OverlayKey, Sealing: a.SealingKey, Serving: a.ServingKey}) + if err != nil { + return delivery.RedeemAnswer{}, fmt.Errorf("the vault did not sign the certificate of %s (call redeem again with the same keys): %w", e.Node, err) + } + if h.Loaded != nil { + loaded, err := h.Loaded(NodeUser(e.Node)) + if err != nil { + return delivery.RedeemAnswer{}, fmt.Errorf("whether the bus node loaded %s cannot be read (call redeem again with the same keys): %w", NodeUser(e.Node), err) + } + if !loaded { + return delivery.RedeemAnswer{}, fmt.Errorf("the bus node has not loaded %s yet (call redeem again with the same keys)", NodeUser(e.Node)) + } + } + signer, err := h.Vault.Signer() + if err != nil { + return delivery.RedeemAnswer{}, fmt.Errorf("the vault's signing key cannot be named (call redeem again with the same keys): %w", err) + } + out := delivery.RedeemAnswer{Node: e.Node, Login: account.Login, Account: account.Account, Certificate: cert.Certificate, Signer: signer} + if out.Login == "" { + out.Login = NodeUser(e.Node) + } + signed, err := h.Vault.SignReply(e.Node, out.SignedBytes()) + if err != nil { + return delivery.RedeemAnswer{}, fmt.Errorf("the vault did not sign the answer (call redeem again with the same keys): %w", err) + } + out.Signature = signed.Signature + return out, nil +} + +// spend moves the key from live to redeeming with the presented keys' digest, by compare-and-set, before its expiry; at +// redeeming it only checks the digest. A key that moved meanwhile is read again once. +func (h *Holder) spend(node, digest, overlay string) (Enrolment, error) { + for attempt := 0; attempt < 3; attempt++ { + row, found, err := h.Store.Get(NodeKey(node)) + if err != nil { + return Enrolment{}, fmt.Errorf("the enrolment of %s cannot be read: %w", node, err) + } + if !found { + return Enrolment{}, fmt.Errorf("no enrolment of %s is open: it was never issued, has ended or has expired", node) + } + e, err := decodeEnrolment(row.Value) + if err != nil { + return Enrolment{}, err + } + // The tunnel key presented is the one the operator approved at issue, which placement and the hub already hold. + if overlay != e.OverlayKey { + return Enrolment{}, fmt.Errorf("the tunnel key presented is not the one %s's enrolment was issued for: the enrolment is not spent", node) + } + // The node entered placement on the operator's yes; the vault reads its address there. + if placements, err := h.placements(); err != nil { + return Enrolment{}, fmt.Errorf("placement cannot be read (call redeem again with the same keys): %w", err) + } else if p, ok := placements[node]; !ok || p.Decided.Address != e.Address { + return Enrolment{}, fmt.Errorf("placement does not hold %s at %s yet (call redeem again with the same keys)", node, e.Address) + } + switch e.Stage { + case StageRedeeming: + if e.KeysDigest != digest { + return Enrolment{}, fmt.Errorf("the enrolment of %s was redeemed with other keys: a token is spent once", node) + } + return e, nil + case StageAsked: + return Enrolment{}, fmt.Errorf("the enrolment of %s waits for the operator's yes: no token exists yet", node) + } + if !h.now().Before(e.Expires) { + return Enrolment{}, fmt.Errorf("the enrolment of %s expired at %s", node, e.Expires.UTC().Format(time.RFC3339)) + } + e.Stage, e.KeysDigest = StageRedeeming, digest + if _, err := h.Store.Update(NodeKey(node), e, row.Revision); err != nil { + if errors.Is(err, ErrChanged) { + continue + } + return Enrolment{}, fmt.Errorf("the enrolment of %s cannot be spent: %w", node, err) + } + // The move to redeeming rewrites the key without an expiry, so a redeem under way never loses it (§5); the address key + // never had one. + return e, nil + } + return Enrolment{}, fmt.Errorf("the enrolment of %s kept changing: call redeem again", node) +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go new file mode 100644 index 0000000..6edebd4 --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/redeem_test.go @@ -0,0 +1,142 @@ +package main + +import ( + "bytes" + "crypto/ed25519" + "encoding/base64" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// redeemArgs are a node's redeem call with a proof signed by the key seeded, over the keys given. +func redeemArgs(node string, seed byte, serving string) delivery.Redeem { + return redeemArgsWith(node, seed, serving, tunnelKey) +} + +// redeemArgsWith is redeemArgs presenting the tunnel key given. +func redeemArgsWith(node string, seed byte, serving, overlay string) delivery.Redeem { + priv := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{seed}, ed25519.SeedSize)) + a := delivery.Redeem{Node: node, PublicKey: base64.StdEncoding.EncodeToString(priv.Public().(ed25519.PublicKey)), + OverlayKey: overlay, SealingKey: "sealing-" + node, ServingKey: serving} + a.Proof = base64.StdEncoding.EncodeToString(ed25519.Sign(priv, delivery.RedeemProof(node, a.PublicKey, a.OverlayKey, a.SealingKey, a.ServingKey))) + return a +} + +// 774/S56/AC1: a redeem from another caller, with a bad proof, after the expiry, or with other keys than the first, is +// refused (one row each), and a good one spends the token, writes the node into placement on the issue's warrant and +// answers once the bus node loaded the account. +func redeemRefusesWhatItMustAndSpendsTheTokenOnce(t *testing.T) { + good := redeemArgs("ace", 1, "serving-a") + bad := good + raw, _ := good.ProofBytes() + raw[0] ^= 1 + bad.Proof = base64.StdEncoding.EncodeToString(raw) + cases := []struct { + name string + caller string + args delivery.Redeem + setup func(r *rig) + want string + }{ + {"another caller", "enrol.g14", good, nil, "only enrol.ace may redeem"}, + {"a bad proof", "enrol.ace", bad, nil, "proof does not verify"}, + {"after the expiry", "enrol.ace", good, func(r *rig) { r.now = r.now.Add(2 * time.Hour) }, "expired at"}, + {"other keys than the first", "enrol.ace", redeemArgs("ace", 2, "serving-b"), func(r *rig) { + e, _ := r.enrolment("ace") + e.Stage, e.KeysDigest = StageRedeeming, good.KeysDigest() + _, _ = r.st.Put(NodeKey("ace"), e, 0) + }, "spent once"}, + {"before the operator's yes", "enrol.ace", good, func(r *rig) { + e, _ := r.enrolment("ace") + e.Stage = StageAsked + _, _ = r.st.Put(NodeKey("ace"), e, 0) + }, "waits for the operator's yes"}, + {"no enrolment", "enrol.novox", redeemArgs("novox", 3, "s"), nil, "no enrolment of novox is open"}, + {"another tunnel key than issued", "enrol.ace", func() delivery.Redeem { + priv := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{1}, ed25519.SeedSize)) + a := good + a.OverlayKey = "another-tunnel-key" + a.Proof = base64.StdEncoding.EncodeToString(ed25519.Sign(priv, delivery.RedeemProof("ace", a.PublicKey, a.OverlayKey, a.SealingKey, a.ServingKey))) + return a + }(), nil, "not the one ace's enrolment was issued for"}, + {"not in placement yet", "enrol.ace", good, func(r *rig) { delete(r.rd["mesh-delivery-assigner.placement"], "ace.node") }, "placement does not hold ace"}, + } + for _, c := range cases { + r := newRig(t) + r.seed("ace", "10.9.0.2", StageLive) + if c.setup != nil { + c.setup(r) + } + _, err := r.h.Redeem(c.caller, c.args) + if err == nil { + t.Errorf("%s: redeemed", c.name) + continue + } + mustContain(t, err.Error(), c.want) + if len(r.vault.calls) != 0 || len(r.added) != 0 { + t.Errorf("%s: the vault %v or placement %v was called", c.name, r.vault.calls, r.added) + } + } + r := newRig(t) + r.seed("ace", "10.9.0.2", StageLive) + r.st.ttl[NodeKey("ace")] = time.Hour + ans, err := r.h.Redeem("enrol.ace", good) + if err != nil { + t.Fatal(err) + } + e, _ := r.enrolment("ace") + if e.Stage != StageRedeeming || e.KeysDigest != good.KeysDigest() || r.st.ttl[NodeKey("ace")] != 0 { + t.Errorf("spent %+v, ttl %v", e, r.st.ttl[NodeKey("ace")]) + } + // Redeem writes no placement: the yes did. It binds the identity keys, which the vault checks against the account's digest. + if len(r.added) != 0 || ans.Signature == "" || ans.Account != "account-ace" || ans.Login != "node.ace" || ans.Signer != "vault-signer" || ans.Certificate != "PEM-serving-a" { + t.Errorf("placement %+v, answer %+v", r.added, ans) + } + // The certificate's call carries every key presented, which the vault checks against the account's digest. + if r.vault.cert.Digest() != good.KeysDigest() { + t.Errorf("the certificate was asked with %+v", r.vault.cert) + } + r.loaded = false + if _, err := r.h.Redeem("enrol.ace", good); err == nil { + t.Error("answered before the bus node loaded the account") + } +} + +// 774/S56/AC2: a redeem stopped after the vault's call, or after the placement call, called again with the same keys, +// finishes and answers the same, and the vault minted once (it is asked for the same account). +func aRedeemStoppedMidwayIsFinishedByTheSameKeysAgain(t *testing.T) { + good := redeemArgs("ace", 1, "serving-a") + whole := newRig(t) + whole.seed("ace", "10.9.0.2", StageLive) + want, err := whole.h.Redeem("enrol.ace", good) + if err != nil { + t.Fatal(err) + } + for _, stop := range []string{"certificate ace", "sign ace"} { + r := newRig(t) + r.seed("ace", "10.9.0.2", StageLive) + r.vault.fail[stop] = true + if _, err := r.h.Redeem("enrol.ace", good); err == nil { + t.Fatalf("%s: no error", stop) + } + got, err := r.h.Redeem("enrol.ace", good) + if err != nil || got != want { + t.Errorf("%s: %v, answer %+v, want %+v", stop, err, got, want) + } + if len(r.added) != 0 { + t.Errorf("redeem wrote placement %d times", len(r.added)) + } + } + // Placement that already holds the node at the approved address is not written again. + r := newRig(t) + r.seed("ace", "10.9.0.2", StageRedeeming) + e, _ := r.enrolment("ace") + e.KeysDigest = good.KeysDigest() + _, _ = r.st.Put(NodeKey("ace"), e, 0) + r.rd["mesh-delivery-assigner.placement"]["ace.node"] = `{"v":1,"node":"ace","decided":{"address":"10.9.0.2"},"modules":[]}` + if _, err := r.h.Redeem("enrol.ace", good); err != nil || len(r.added) != 0 { + t.Errorf("%v, placement written %d times", err, len(r.added)) + } +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/run.go b/modules/mesh-enrolment/cmd/mesh-enrolment/run.go new file mode 100644 index 0000000..b23efc9 --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/run.go @@ -0,0 +1,50 @@ +package main + +import ( + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Run starts what the holder does beside its tools, none of it on a timer: the watch of its own bucket (the bus's expiry +// markers), the durable consumer of its own ending events (the drop), and the operator's answers. Each is asked of the +// runtime as soon as it serves, and again with a growing wait until the runtime takes it. +func (h *Holder) Run(listen func() error) { + go func() { + stdio.State(State).Follow("", func(c stdio.StateChange) error { + err := h.Heard(c.Op, c.Key, c.Value) + if err != nil && c.Op == stdio.OpExpire { + // An expiry whose end stopped is finished by a later pass (ADR 0589 §6). + go retry("finish the enrolments that expired", h.logf, h.Reconcile) + } + return err + }) + // Caught up: an enrolment that expired while the holder was down is ended now. + retry("end the enrolments that expired unseen", h.logf, h.Reconcile) + }() + for _, event := range EndEvents { + go func(key string) { + <-stdio.Served() + retry("hear "+key, h.logf, func() error { + return stdio.Subscribe(key, func(e stdio.Envelope) error { return h.Ended(key, e.Body) }) + }) + }(Seat + "." + event) + } + if listen != nil { + go func() { + <-stdio.Served() + retry("hear the operator's answers", h.logf, listen) + }() + } +} + +func retry(what string, logf func(string, ...any), do func() error) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + err := do() + if err == nil { + return + } + logf("does not %s yet (%v); asking again in %s", what, err, wait) + time.Sleep(wait) + } +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/testdata/grants-pass.json b/modules/mesh-enrolment/cmd/mesh-enrolment/testdata/grants-pass.json new file mode 100644 index 0000000..1b04918 --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/testdata/grants-pass.json @@ -0,0 +1,53 @@ +{ + "user_list": { + "v": 1, + "mode": "shadow", + "inputs": "", + "digest": "", + "users": 0, + "left_out": null, + "compare": { + "shadow": 0, + "loaded": 0, + "added": null, + "dropped": null, + "differ": null, + "same": 0 + }, + "credentials": { + "users": [ + { + "user": "node.ace", + "in_shadow": true, + "loaded": true, + "hash_held": true, + "generation": 1, + "would": "same" + }, + { + "user": "node.g14", + "in_shadow": true, + "loaded": false, + "hash_held": true, + "generation": 1, + "would": "adds" + } + ], + "hashes_held": 2, + "left_out": [], + "loses_login": [], + "adds": [ + "node.g14" + ] + }, + "gaps": null, + "at": "0001-01-01T00:00:00Z" + }, + "revision": 0, + "written": false, + "memberships": 0, + "memberships_written": 0, + "assertions": 0, + "assertions_written": 0, + "published_or_asserted": "" +} \ No newline at end of file diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/tokens.go b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens.go new file mode 100644 index 0000000..a153797 --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens.go @@ -0,0 +1,118 @@ +package main + +import ( + "encoding/json" + "fmt" + "sort" + "time" +) + +// TokenRow is one open enrolment as `tokens` answers it (ADR 0587 §8): its stage, expiry, ask and revision, and for a node +// at stage live the token sealed to the operator key as the vault sealed it (the holder never reads it and keeps it nowhere). +type TokenRow struct { + Node string `json:"node"` + Address string `json:"address"` + Why string `json:"why"` + Caller string `json:"caller"` + Stage string `json:"stage"` + Ask string `json:"ask,omitempty"` + Issued time.Time `json:"issued"` + Expires time.Time `json:"expires"` + Age string `json:"age"` + Revision uint64 `json:"revision"` + Keys string `json:"keys-digest,omitempty"` + Sealed string `json:"sealed,omitempty"` + // Unsealed says why no sealed token is given for a node that should have one. + Unsealed string `json:"unsealed,omitempty"` +} + +// Issuing is whether issue is on, and why. +type Issuing struct { + On bool `json:"on"` + Why string `json:"why"` +} + +// Fill is how full the bucket is: its keys and the bytes of their values (the state's cap is the bus's, which no bundle +// can read, so the fill is stated without it). +type Fill struct { + Keys int `json:"keys"` + Bytes int `json:"bytes"` +} + +// TokensAnswer is what `tokens` answers. +type TokensAnswer struct { + Open []TokenRow `json:"open"` + Issuing Issuing `json:"issuing"` + Fill Fill `json:"fill"` + History []json.RawMessage `json:"history,omitempty"` + // HistoryFrom says where the history comes from, when one was asked for. + HistoryFrom string `json:"history-from,omitempty"` +} + +// Tokens answers one open enrolment or all, with the bucket's fill, whether issuing is on and why, and with history the +// ended enrolments the holder's own events say (ADR 0587 §8). +func (h *Holder) Tokens(node string, history bool) (TokensAnswer, error) { + rows, open, err := h.openEnrolments() + if err != nil { + return TokensAnswer{}, err + } + var out TokensAnswer + on, why := h.issuing() + out.Issuing = Issuing{On: on, Why: why} + if keys, err := h.Store.Keys(); err == nil { + out.Fill.Keys = len(keys) + for _, k := range keys { + if r, found, err := h.Store.Get(k); err == nil && found { + out.Fill.Bytes += len(r.Value) + } + } + } + if node != "" { + if _, ok := open[node]; !ok && !history { + return TokensAnswer{}, fmt.Errorf("no enrolment of %s is open: tokens without a node lists those that are", node) + } + } + names := make([]string, 0, len(open)) + for n := range open { + if node == "" || n == node { + names = append(names, n) + } + } + sort.Strings(names) + out.Open = []TokenRow{} + for _, n := range names { + e := open[n] + row := TokenRow{Node: e.Node, Address: e.Address, Why: e.Why, Caller: e.Caller, Stage: e.Stage, Ask: e.Ask, + Issued: e.Issued, Expires: e.Expires, Age: h.now().Sub(e.Issued).Round(time.Second).String(), + Revision: rows[n].Revision, Keys: e.KeysDigest} + if e.Stage == StageLive && node != "" { + // Asked of the vault at this moment and relayed as the vault sealed it: kept nowhere here (ADR 0587 §8). + sealed, err := h.Vault.Sealed(n) + if err != nil { + row.Unsealed = "the vault did not give the token: " + err.Error() + } else { + row.Sealed = sealed + } + } + out.Open = append(out.Open, row) + } + if history { + h.mu.Lock() + for _, e := range h.ended { + if node == "" || endedNode(e) == node { + out.History = append(out.History, e) + } + } + h.mu.Unlock() + out.HistoryFrom = "the end events this holder heard since it started; the bus keeps them 7 days, and a bundle cannot read that stream" + } + return out, nil +} + +func endedNode(raw json.RawMessage) string { + var e struct { + Node string `json:"node"` + } + _ = json.Unmarshal(raw, &e) + return e.Node +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go new file mode 100644 index 0000000..f7e9c01 --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go @@ -0,0 +1,120 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// seed writes an open enrolment of node at stage, with its address key, as the holder would have. +func (r *rig) seed(node, address string, stage string) { + e := Enrolment{Node: node, Address: address, Why: "a new machine", Caller: "person.operator", Ask: "ask-" + node, + Stage: stage, Issued: r.now.Add(-time.Minute), Expires: r.now.Add(time.Hour), OverlayKey: tunnelKey} + if stage == StageRedeeming { + e.KeysDigest = delivery.KeysDigest("pk", "ov", "se", "sv") + } + _, _ = r.st.Put(NodeKey(node), e, 0) + _, _ = r.st.Put(AddressKey(address), Reservation{Node: node, Ask: e.Ask, Issued: e.Issued}, 0) + if stage != StageAsked { // the operator's yes placed it, with its tunnel key + r.rd["mesh-delivery-assigner.placement"][node+".node"] = `{"v":1,"node":"` + node + `","decided":{"address":"` + address + `"},"modules":[]}` + } +} + +// 774/S54/AC1 and AC2: tokens answers each stage with its expiry, ask and revision, the fill and whether issuing is on; +// for one node at live it relays the vault's sealed token and keeps nothing; and the manifest says what the code does. +func tokensAnswersEachStageAndRelaysTheSealedTokenOfOneNode(t *testing.T) { + r := newRig(t) + r.seed("ace", "10.9.0.2", StageAsked) + r.seed("g14", "10.9.0.3", StageLive) + r.seed("novox", "10.9.0.4", StageRedeeming) + all, err := r.h.Tokens("", false) + if err != nil { + t.Fatal(err) + } + var stages []string + for _, row := range all.Open { + stages = append(stages, row.Node+":"+string(row.Stage)) + if row.Revision == 0 || row.Expires.IsZero() || row.Ask == "" || row.Sealed != "" { + t.Errorf("row %+v lacks revision, expiry or ask, or relays a token for a list of all", row) + } + } + if !slices.Equal(stages, []string{"ace:asked", "g14:live", "novox:redeeming"}) || all.Fill.Keys != 6 || !all.Issuing.On { + t.Errorf("tokens %v, fill %+v, issuing %+v", stages, all.Fill, all.Issuing) + } + one, err := r.h.Tokens("g14", false) + if err != nil || len(one.Open) != 1 || one.Open[0].Sealed != "sealed-for-g14" { + t.Fatalf("one node: %+v, %v", one, err) + } + for k, row := range r.st.rows { + if slices.Contains(r.vault.calls, "sealed g14") && string(row.Value) != "" { + var any map[string]json.RawMessage + _ = json.Unmarshal(row.Value, &any) + if _, has := any["sealed"]; has { + t.Errorf("the key %s holds a sealed token", k) + } + } + } + // Before the lease write issuing is off, and says why. + r.rd["mesh-controller.lease"]["holder"] = `{"instance":"novox.mesh-controller"}` + off, _ := r.h.Tokens("", false) + mustContain(t, off.Issuing.Why, "controller holds the lease") + if off.Issuing.On { + t.Error("issuing is on while the controller holds the lease") + } +} + +func theManifestSaysWhatTheCodeDoes(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Module string `json:"module"` + Runs string `json:"runs-as"` + Consumes []string `json:"consumes"` + Claims []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` + } `json:"claims"` + Emits []string `json:"emits"` + State []json.RawMessage `json:"state"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m.Module != Module || m.Runs != Module || len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || + !slices.Equal(m.Claims[0].Serves, Verbs) || len(m.Emits) != 0 { + t.Errorf("module %q runs as %q, claims %+v, emits %v", m.Module, m.Runs, m.Claims, m.Emits) + } + var heard []string + for _, e := range EndEvents { + heard = append(heard, Seat+"."+e) + } + if !slices.Equal(m.Consumes, heard) { + t.Errorf("consumes %v, the end events are %v", m.Consumes, heard) + } + served := map[string]bool{} + for _, tool := range Tools(&Holder{}) { + served[tool.Name] = true + } + for _, v := range Verbs { + if !served[Seat+"."+v] { + t.Errorf("the verb %s has no tool", v) + } + } + if len(m.State) != 2 || string(m.State[0]) == "" || !slices.ContainsFunc(m.State, func(s json.RawMessage) bool { + var d struct { + Name string `json:"name"` + TTL bool `json:"key-ttl"` + } + return json.Unmarshal(s, &d) == nil && d.Name == State && d.TTL + }) { + t.Errorf("state %s: enrolments must allow a time to live per key", m.State) + } +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go b/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go new file mode 100644 index 0000000..3365c65 --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/tools.go @@ -0,0 +1,120 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Tool names: the seat's verbs, as the manifest claims them (ADR 0587 §1, §8). +const ( + IssueTool = Seat + ".issue" + AgainTool = Seat + ".again" + RedeemTool = Seat + ".redeem" + ConfirmTool = Seat + ".confirm" + RevokeTool = Seat + ".revoke" + TokensTool = Seat + ".tokens" +) + +func obj(props map[string]any, required ...string) map[string]any { + m := map[string]any{"type": "object", "properties": props} + if len(required) > 0 { + m["required"] = required + } + return m +} + +func str(d string) map[string]any { return map[string]any{"type": "string", "description": d} } + +func arg(a map[string]any, k string) string { + s, _ := a[k].(string) + return strings.TrimSpace(s) +} + +func flag(a map[string]any, k string) bool { + return a[k] == true || strings.EqualFold(arg(a, k), "true") +} + +// lastsOf reads an expiry as a duration (delivery.TokenLife); none is the default. +func lastsOf(s string) (time.Duration, error) { return delivery.TokenLife(s) } + +// Tools are what the runtime serves. +func Tools(h *Holder) []stdio.Tool { + return []stdio.Tool{ + {Name: IssueTool, + Description: "Open an enrolment: let a new machine join the mesh (novox/hq ADR 0587 §4). Takes the node's name, a why and an " + + "expiry (one hour unless given, at most 24 hours). Refused at once, naming every finding, for a name that is no node name " + + "or that a node record, a placement key or an open enrolment holds, and for every call before the controller's lease is " + + "written to a holder that is no controller. Otherwise it chooses the lowest free private-network address, reserves it, " + + "writes the enrolment at stage asked and asks the operator on a channel that proves who answered; only their yes adds the " + + "node to placement with the machine's tunnel key (overlay_key, as `nox-mesh-host key` prints it there), so the hub peers " + + "it, and then makes the token, which carries the first tunnel (the vault makes it, this seat never holds it). The caller is the one the runtime proves.", + Input: obj(map[string]any{"node": str("the new machine's name"), "why": str("why this machine joins"), + "expires": str("how long the token lasts, e.g. 1h (default) up to 24h"), + "overlay_key": str("the new machine's tunnel public key, as `nox-mesh-host key` prints it there")}, "node", "why", "overlay_key"), + Run: func(a map[string]any) (any, error) { + lasts, err := lastsOf(arg(a, "expires")) + if err != nil { + return nil, err + } + return h.Issue(stdio.CallerOf(a), arg(a, "node"), arg(a, "why"), arg(a, "overlay_key"), lasts) + }}, + {Name: RedeemTool, + Description: "The joining machine's one call (novox/hq ADR 0587 §5, §6), answered only for a caller the runtime proves to be " + + "enrol.: its node-engine presents the public halves of the keys it made where it runs and a proof signed with its " + + "identity key, and the tunnel key it was issued for. The token is spent (once, with those keys) once placement holds the " + + "node (placed on the operator's yes), the vault gives the node's account and signs its certificate, and the answer is given once the bus node loaded node., " + + "signed by the vault. Called again with the same keys it finishes what is not done and answers the same.", + Input: obj(map[string]any{"node": str("the machine's name"), "public_key": str("its identity key, base64"), + "overlay_key": str("its private-network key"), "sealing_key": str("its sealing key, base64"), + "serving_key": str("its serving key"), "proof": str("the proof, base64")}, "node", "public_key", "proof"), + Run: func(a map[string]any) (any, error) { + var args delivery.Redeem + raw, err := json.Marshal(a) + if err != nil { + return nil, err + } + if err := json.Unmarshal(raw, &args); err != nil { + return nil, fmt.Errorf("the redeem call does not read: %w", err) + } + return h.Redeem(stdio.CallerOf(a), args) + }}, + {Name: ConfirmTool, + Description: "The new machine's confirmation (novox/hq ADR 0587 §6), answered only for the node dialled as its own login " + + "node. after it redeemed: the runtime's proof of that caller is the confirmation. Says node-enrolled and closes " + + "the enrolment; the vault is then asked to drop the token.", + Input: obj(map[string]any{"node": str("the machine's own name (optional)")}), + Run: func(a map[string]any) (any, error) { return h.Confirm(stdio.CallerOf(a), arg(a, "node")) }}, + {Name: RevokeTool, + Description: "End an enrolment at any stage (novox/hq ADR 0587 §5): at asked it withdraws the ask; at live or redeeming it deletes " + + "the key and its address key, and at redeeming the vault retires an account it already minted; a node the yes placed " + + "and that never confirmed is taken off placement again. It only takes access away, so it needs no Approve; it is recorded as an act done by hand. Says " + + "enrolment-revoked; the vault then drops the token. A machine stuck at redeeming is ended by this. A revoke that stopped is " + + "finished by calling it again.", + Input: obj(map[string]any{"node": str("the machine"), "why": str("why, kept with the act")}, "node", "why"), + Run: func(a map[string]any) (any, error) { + return h.Revoke(stdio.CallerOf(a), arg(a, "node"), arg(a, "why")) + }}, + {Name: AgainTool, + Description: "Ask the operator again about an enrolment still at stage asked whose ask the router no longer holds open " + + "(novox/hq ADR 0587 §8): a new ask for the same node, address and expiry. Nothing more: a redeem stopped midway is finished " + + "by the machine calling redeem again with the same keys.", + Input: obj(map[string]any{"node": str("the machine")}, "node"), + Run: func(a map[string]any) (any, error) { return h.Again(arg(a, "node")) }}, + {Name: TokensTool, + Description: "What the seat holds (novox/hq ADR 0587 §8): every open enrolment, or one node's, with its stage (asked, live or " + + "redeeming), address, expiry, ask, revision and age; whether issuing is on and why (it is on only from the controller's " + + "lease write); the bucket's fill. With a node at stage live, the token sealed to the operator key, asked of the vault at " + + "this moment and relayed as the vault sealed it: the one route by which the operator gets a token, which this seat never " + + "reads and keeps nowhere. With history=true the ended enrolments its events say. A machine stuck at redeeming shows here " + + "with its age; revoke ends it. Read-only.", + Input: obj(map[string]any{"node": str("one node"), "history": str("\"true\": the ended enrolments")}), + Run: func(a map[string]any) (any, error) { + return h.Tokens(arg(a, "node"), flag(a, "history")) + }}, + } +} diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/vault.go b/modules/mesh-enrolment/cmd/mesh-enrolment/vault.go new file mode 100644 index 0000000..0ebbebb --- /dev/null +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/vault.go @@ -0,0 +1,108 @@ +package main + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Seats the holder calls (ADR 0587 §3, §5, §7). The vault's two verbs are the ones its manifest names this seat the only +// caller of; its `sign` names this seat for a node's certificate and a redeem answer. +const ( + vaultToken = "seat:mesh-vault." + delivery.EnrolmentTokenVerb + vaultAccount = "seat:mesh-vault." + delivery.NodeAccountVerb + vaultSign = "seat:mesh-vault." + delivery.SignVerb + placementAdd = "seat:mesh-placement.propose" +) + +// Call asks a seat verb by its address (stdio.Ask through the runtime, or a test's). +type Call func(key string, body any) (json.RawMessage, error) + +// vaultOver is the vault as the holder reaches it: the seat's verbs over a Call, and its public signing key from its state. +type vaultOver struct { + call Call + read Reader +} + +func answer[T any](raw json.RawMessage, err error) (T, error) { + var out T + if err != nil { + return out, err + } + if uerr := json.Unmarshal(raw, &out); uerr != nil { + return out, fmt.Errorf("the vault's answer does not read: %w", uerr) + } + return out, nil +} + +func (v vaultOver) Token(node string, expires time.Time, t delivery.TokenTunnel) (delivery.EnrolmentTokenAnswer, error) { + // The whole first tunnel the token carries (delivery.TokenTunnel, catalogue's enrolment-token). + tunnel := map[string]any{"hub_key": t.HubKey, "hub_endpoint": t.HubEndpoint, "address": t.Address, "key": t.Key, "range": t.Range} + return answer[delivery.EnrolmentTokenAnswer](v.call(vaultToken, map[string]any{"node": node, + "expires": expires.UTC().Format(time.RFC3339), "tunnel": tunnel})) +} + +func (v vaultOver) Sealed(node string) (string, error) { + a, err := answer[delivery.EnrolmentTokenAnswer](v.call(vaultToken, map[string]any{"node": node, "sealed": "true"})) + if err == nil && a.Sealed == "" { + err = errors.New("the vault gave no sealed token") + } + return a.Sealed, err +} + +func (v vaultOver) Drop(node string) error { + a, err := answer[delivery.EnrolmentTokenAnswer](v.call(vaultToken, map[string]any{"node": node, "drop": "true"})) + if err == nil && !a.Dropped { + err = errors.New("the vault did not say the token and its hash are gone") + } + return err +} + +func (v vaultOver) Account(node, sealingKey, keysDigest string) (delivery.NodeAccountAnswer, error) { + return answer[delivery.NodeAccountAnswer](v.call(vaultAccount, map[string]any{"node": node, "sealing_key": sealingKey, "keys_digest": keysDigest})) +} + +func (v vaultOver) Retire(node string) error { + a, err := answer[delivery.NodeAccountAnswer](v.call(vaultAccount, map[string]any{"node": node, "retire": "true"})) + if err == nil && !a.Retired { + err = errors.New("the vault did not say the account is retired") + } + return err +} + +func (v vaultOver) SignCertificate(node string, k delivery.JoiningKeys) (delivery.SignAnswer, error) { + return answer[delivery.SignAnswer](v.call(vaultSign, map[string]any{"kind": delivery.KindNodeCertificate, "node": node, + "public_key": k.Identity, "overlay_key": k.Overlay, "sealing_key": k.Sealing, "serving_key": k.Serving})) +} + +func (v vaultOver) SignReply(node string, reply []byte) (delivery.SignAnswer, error) { + return answer[delivery.SignAnswer](v.call(vaultSign, map[string]any{"kind": delivery.KindEnrolment, "node": node, + "reply": base64.StdEncoding.EncodeToString(reply)})) +} + +func (v vaultOver) Signer() (string, error) { + raw, found, err := v.read.Get(delivery.SignersRead, delivery.SignersKeyKey) + if err != nil || !found { + return "", fmt.Errorf("the vault's signing key is not readable in %s: %v", delivery.SignersRead, err) + } + k, err := delivery.DecodeSignerKey(raw) + if err != nil { + return "", err + } + return k.Key, nil +} + +// addNodeOver asks mesh-placement to add the node with the issue's warrant (ADR 0587 §7). +func addNodeOver(call Call) func(AddNode) error { + return func(a AddNode) error { + _, err := call(placementAdd, map[string]any{"why": a.Why, "changes": []any{a.Change}, "warrant": a.Warrant}) + if err != nil { + return fmt.Errorf("mesh-placement did not take the node %s: %w", a.Change.Node, err) + } + return nil + } +} diff --git a/modules/mesh-enrolment/go.mod b/modules/mesh-enrolment/go.mod new file mode 100644 index 0000000..52cdb8e --- /dev/null +++ b/modules/mesh-enrolment/go.mod @@ -0,0 +1,22 @@ +module mesh-enrolment + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 + github.com/nats-io/nats-server/v2 v2.11.17 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-enrolment/go.sum b/modules/mesh-enrolment/go.sum new file mode 100644 index 0000000..ce8dc67 --- /dev/null +++ b/modules/mesh-enrolment/go.sum @@ -0,0 +1,31 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027 h1:a33dsavoaqllh40eVICbhxi6Kero1EGSjt1V+2fa2Hk= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4 h1:xeuMBwOHNKrVUdGVg7Bdv5tkdueBvngwJFyk6n10rp0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 h1:4I4ayo0fZYP4PBfEX1sGDDd6arnw5IQJySJ3jcFobPM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-enrolment/module.json b/modules/mesh-enrolment/module.json new file mode 100644 index 0000000..29c1455 --- /dev/null +++ b/modules/mesh-enrolment/module.json @@ -0,0 +1,89 @@ +{ + "module": "mesh-enrolment", + "version": "1", + "runs-as": "mesh-enrolment", + "claims": [ + { + "name": "mesh-enrolment", + "scope": "mesh", + "serves": [ + "issue", + "redeem", + "confirm", + "revoke", + "tokens", + "again" + ] + } + ], + "consumes": [ + "mesh-enrolment.node-enrolled", + "mesh-enrolment.enrolment-refused", + "mesh-enrolment.enrolment-revoked", + "mesh-enrolment.enrolment-expired" + ], + "uses": [ + "operator-channel" + ], + "invokes": [ + "seat:mesh-hand-acts.hand-act", + "seat:mesh-hand-acts.warranted", + "seat:mesh-placement.propose", + "seat:mesh-vault.enrolment-token", + "seat:mesh-vault.node-account", + "seat:mesh-vault.sign" + ], + "reads": [ + "mesh-controller.lease", + "mesh-delivery-assigner.placement", + "mesh-grants.user-list", + "mesh-vault.signers", + "node-engine.have" + ], + "state": [ + { + "name": "enrolments", + "key-ttl": true + }, + { + "name": "asked", + "history": 5 + } + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-enrolment", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-enrolment", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-enrolment" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-enrolment", + "place": ".", + "unknown": "program" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-enrolment", + "binary": "mesh-enrolment", + "loads": [ + "mesh-enrolment" + ] + } + ] + } +} diff --git a/modules/mesh-grants/go.mod b/modules/mesh-grants/go.mod index c069fd3..3f7dd28 100644 --- a/modules/mesh-grants/go.mod +++ b/modules/mesh-grants/go.mod @@ -3,7 +3,7 @@ module mesh-grants go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-grants/go.sum b/modules/mesh-grants/go.sum index adb6e38..df8d76d 100644 --- a/modules/mesh-grants/go.sum +++ b/modules/mesh-grants/go.sum @@ -2,6 +2,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4 h1:xeuMBwOH git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c h1:/YCxh+NYfqry3PxZYY1nQXaccupobxpsFt8c9BYuGBg= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 h1:4I4ayo0fZYP4PBfEX1sGDDd6arnw5IQJySJ3jcFobPM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index 9ae4a82..09dc244 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -3,6 +3,7 @@ package manifest import ( "bytes" "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -358,6 +359,16 @@ const ( xManifest = `{"module":"mod-x","version":"1","own-secrets":{"broker":"/var/lib/x/broker"},"state":[{"name":"kept","history":3}],` + `"streams":[{"name":"things","events":["thing.added"],"max-age-days":2,"max-mib":5}],"logs":[{"name":"history","max-age-days":30}]}` yManifest = `{"module":"mod-y","version":"1","own-secrets":{"broker":"/var/lib/y/broker"},"reads":["mod-x.kept"]}` + // vaultManifest is the vault as the composition needs it for a rotation (ADR 0583 §2, §3): its claim serving the verbs. + vaultManifest = `{"module":"mesh-vault","version":"1","own-secrets":{"broker":"/var/lib/v/broker"},` + + `"provides":[{"name":"secret","scope":"mesh"}],` + + `"claims":[{"name":"mesh-vault","serves":["rotate-node","node-account-confirm","abandon"]}],` + + `"tools":["rotate-node","node-account-confirm","abandon"],"state":[{"name":"node-logins"}]}` + // enrolmentManifest is the holder of mesh-enrolment as the composition needs it (ADR 0587 §1): the claim, its six verbs + // and its bucket. + enrolmentManifest = `{"module":"mesh-enrolment","version":"1","own-secrets":{"broker":"/var/lib/e/broker"},` + + `"claims":[{"name":"mesh-enrolment","serves":["issue","redeem","confirm","revoke","tokens","again"]}],` + + `"tools":["issue","redeem","confirm","revoke","tokens","again"],"state":[{"name":"enrolments","key-ttl":true}]}` ) // brokerSeat is one verb of the bus's seat as a membership serves it: nats claims the seat in fixture and no membership @@ -2404,6 +2415,103 @@ func TestGrants(t *testing.T) { // 774/AC5 (novox/hq ADR 0561 §1): the vault's account-issued wakes a pass, so the next list carries the new user's hash // within one pass; an event of another kind does not. + // 774/S52/AC1 (novox/hq ADR 0587 §4, §5): a key of the seat mesh-enrolment at live, with the vault's hash, composes + // enrol., which from the lease write may publish the seat's redeem in its own name and subscribe its own inbox, + // nothing else; a key at asked composes none; when the key ends the user is gone. + t.Run("an enrolment key at live composes its enrolment user, and its end takes it out", func(t *testing.T) { + b := fixture() + b.manifestOf("mesh-enrolment", enrolmentManifest) + b.assign("a", "mesh-enrolment") + b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"handed-to": "the operator's hand-over"}) + issued := time.Date(2026, 11, 2, 1, 0, 0, 0, time.UTC) + for node, stage := range map[string]string{"lab": delivery.EnrolmentLive, "pi": delivery.EnrolmentAsked} { + raw, err := delivery.EncodeEnrolment(delivery.Enrolment{Node: node, Address: "10.0.0.7", Why: "joins", + Caller: "a.mesh-enrolment", Stage: stage, Issued: issued, Expires: issued.Add(time.Hour), + OverlayKey: base64.StdEncoding.EncodeToString(make([]byte, 32))}) + if err != nil { + t.Fatal(err) + } + b.put(grants.ReadEnrolments, delivery.EnrolmentNodeKey(node), json.RawMessage(raw)) + } + b.put(grants.ReadAccounts, "enrol.lab", map[string]any{"user": "enrol.lab", "hash": "$2a$10$x", "generation": 1}) + in := grants.ReadInputs(b, &grants.Manifests{}) + if !slices.Equal(in.Enrolling, []string{"lab"}) || gapNamed(in.Gaps, "enrolment tokens") { + t.Fatalf("enrolling %v, gaps %v", in.Enrolling, in.Gaps) + } + users := shadowOf(t, b) + var lab *grants.User + for i := range users { + if users[i].User == "enrol.lab" { + lab = &users[i] + } + if users[i].User == "enrol.pi" { + t.Error("a key at asked composed enrol.pi") + } + } + if lab == nil || !slices.Equal(lab.Publish, []string{"mesh.seat.mesh-enrolment.call.redeem.enrol~lab"}) || + !slices.Equal(lab.Subscribe, []string{"_INBOX.enrol.lab.>"}) || lab.AllowResponses { + t.Fatalf("enrol.lab: %+v", lab) + } + if c := grants.CredentialsOf(users, nil, in.Hashes); !slices.Contains(c.Adds, "enrol.lab") { + t.Errorf("enrol.lab's hash is not carried: %+v", c) + } + b.mu.Lock() + delete(b.states[grants.ReadEnrolments], delivery.EnrolmentNodeKey("lab")) + b.mu.Unlock() + if slices.Contains(names(shadowOf(t, b)), "enrol.lab") { + t.Error("enrol.lab stays after its key ended") + } + if !slices.Contains(grants.Followed, grants.ReadEnrolments) { + t.Error("a key's change composes nothing again") + } + }) + // 774/S53 (novox/hq ADR 0583 §1, §2): while node-logins says b rotates to node.b--b, with its hash, both logins are in + // the list and b's machine stays placed under either; after rotation-done (b on node.b--b, nothing rotating) the old + // login is not (AC1). Each node's engine may read its own node-logins key alone (AC2). + t.Run("a node rotating is composed under both its logins, then the new one alone", func(t *testing.T) { + b := fixture() + b.manifestOf("mesh-vault", vaultManifest) + b.assign("a", "mesh-vault") + since := time.Date(2026, 11, 2, 1, 0, 0, 0, time.UTC) + login := func(l delivery.NodeLogin) { + raw, err := delivery.EncodeNodeLogin(l) + if err != nil { + t.Fatal(err) + } + b.put(grants.ReadNodeLogins, l.Node, json.RawMessage(raw)) + } + login(delivery.NodeLogin{Node: "b", Login: "node.b", Rotating: "node.b--b", Stage: delivery.RotationBegun, Since: since}) + b.put(grants.ReadAccounts, "node.b--b", map[string]any{"user": "node.b--b", "hash": "$2a$10$x", "generation": 1}) + users := shadowOf(t, b) + in := grants.ReadInputs(b, &grants.Manifests{}) + if got := names(users); !slices.Contains(got, "node.b") || !slices.Contains(got, "node.b--b") || !slices.Contains(got, "node.a") { + t.Fatalf("the node users are %v", got) + } + if c := grants.CredentialsOf(users, nil, in.Hashes); !slices.Contains(c.Adds, "node.b--b") { + t.Errorf("node.b--b's hash is not carried: %+v", c) + } + if placed := grants.Placed(in); !placed("node.b--b") || placed("node.c--b") || !in.Rotating()["node.b--b"] { + t.Error("the second login is not judged its node's, or not rotating") + } + get := func(key string) string { + return "$JS.API.DIRECT.GET.KV_" + delivery.NodeLoginsBucket + ".$KV." + delivery.NodeLoginsBucket + "." + key + } + for _, u := range users { + if !strings.HasPrefix(u.User, "node.") { + continue + } + node, _ := delivery.LoginNode(u.User) + for _, of := range []string{"a", "b"} { + if slices.Contains(u.Publish, get(of)) != (of == node) { + t.Errorf("%s reads %s's node-logins key: %v", u.User, of, of != node) + } + } + } + login(delivery.NodeLogin{Node: "b", Login: "node.b--b", Since: since.Add(time.Hour)}) + if got := names(shadowOf(t, b)); slices.Contains(got, "node.b") || !slices.Contains(got, "node.b--b") { + t.Errorf("after rotation-done the node users are %v", got) + } + }) t.Run("the vault's account-issued composes the list again", func(t *testing.T) { b := fixture() var handle func(string, []byte) diff --git a/modules/mesh-grants/internal/grants/follow.go b/modules/mesh-grants/internal/grants/follow.go index 3831219..9770249 100644 --- a/modules/mesh-grants/internal/grants/follow.go +++ b/modules/mesh-grants/internal/grants/follow.go @@ -21,9 +21,11 @@ type Watcher interface { // config values, the node agents' haves (a machine's first have, and the bus node's loaded list) and the assigner's // wants (the manifest each machine is wanted to run), and the vault's hashes of the bus accounts. // -// And its own state lease, whose key removal-due's expiry marker starts the day-later removal (ADR 0582 clause 6). +// And its own state lease, whose key removal-due's expiry marker starts the day-later removal (ADR 0582 clause 6), and +// the seat mesh-enrolment's keys (ADR 0587 §4, §5): a key at live composes enrol., and its end, by the holder or by +// the bus's expiry, is a change of the key that takes the user out; no event is needed beside the watch. var Followed = []string{ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts, ReadPlacement, ReadLease, ReadTakeover, - OwnLease} + ReadEnrolments, ReadNodeLogins, OwnLease} // OwnLease is the part's own state lease as the follower watches it, `.`. const OwnLease = Module + "." + StateLease diff --git a/modules/mesh-grants/internal/grants/inputs.go b/modules/mesh-grants/internal/grants/inputs.go index afe87f7..ac8e8e1 100644 --- a/modules/mesh-grants/internal/grants/inputs.go +++ b/modules/mesh-grants/internal/grants/inputs.go @@ -35,12 +35,18 @@ const ( // ReadPlacement is the seat mesh-placement's placement (ADR 0567 §1, §9): each node's modules and the seats // they hold. ReadPlacement = delivery.PlacementRead + // ReadEnrolments is the seat mesh-enrolment's open enrolments, read as a named reader (ADR 0587 §2, §4): a key at + // live or redeeming composes the user enrol.. + ReadEnrolments = delivery.EnrolmentsRead + // ReadNodeLogins is the vault's node-logins, read as its named reader (ADR 0583 §1 insight): each node's login in use + // and the one rotating, both composed as users of the node's permissions. + ReadNodeLogins = delivery.NodeLoginsRead ) // Reads is every state the part reads, in the manifest's order. Not the controller's assignments: the seat's placement // alone places modules (novox/hq ADR 0549 §5, ADR 0571 §10; issue 774). var Reads = []string{ReadModules, ReadManifests, ReadResolved, ReadHave, ReadWants, ReadBuilds, - ReadAccounts, ReadPlacement, ReadLease, ReadTakeover} + ReadAccounts, ReadPlacement, ReadLease, ReadTakeover, ReadEnrolments, ReadNodeLogins} // The bus module's tools the part calls, as its manifest's invokes name them: read-only, each. const ( @@ -75,7 +81,8 @@ type Gap struct { // Standing gaps: what the controller composes from its own store, which no bus value carries in this milestone. var standingGaps = []Gap{ - {Input: "enrolment tokens", Why: "the live enrolment tokens are in the controller's store alone: no enrolment user is composed"}, + {Input: "enrolment tokens", Why: "while the controller holds the lease, its live enrolment tokens are in its store alone: " + + "an enrolment user is composed only from the seat mesh-enrolment's keys"}, {Input: "people", Why: "the people and the tools they may invoke are in the controller's store alone: no person user is composed"}, {Input: "seat holdings", Why: "who holds each seat held once for the mesh is in the controller's store alone: a mesh seat claimed on two machines is composed as held on both"}, {Input: "announced states", Why: "the controller does not announce its states-announced readable, so every read is composed from `reads`, as when announcements cannot be read"}, @@ -118,6 +125,15 @@ type Inputs struct { Loaded *delivery.UserListWritten // Hashes are the vault's hashes of the bus accounts, by bus user (ReadAccounts); kept in memory, never written. Hashes map[string]HeldHash + // Enrolling is every node whose enrolment key is at live or redeeming (ReadEnrolments, ADR 0587 §4, §5), sorted: one + // user enrol. each. A key at asked has no token yet; an ended or expired key is gone from the bucket. + Enrolling []string + // NodeLogins is each node's key of the vault's node-logins (ReadNodeLogins, ADR 0583 §1); a node absent logs in as + // node. alone. + NodeLogins map[string]delivery.NodeLogin + // LeaseAway says the controller's lease is handed away, as ReadInputs read it: an enrolment user then enrols through + // the seat alone (ADR 0587 §6). + LeaseAway bool // Refused is every manifest the indexer holds that the part cannot read or mesh-sdk's reader refuses, assigned or // not, each with why: its module counts as placed nowhere, so it is said loudly, never passed over (issue 1945). Refused []Gap @@ -264,6 +280,7 @@ func ReadInputs(r Reader, cache *Manifests) Inputs { in.Placed[node] = p } } + in.LeaseAway, _ = leaseHandedAway(r) for node, p := range in.Placed { nodes[node] = true for _, m := range p.Assigned() { @@ -431,6 +448,13 @@ func ReadInputs(r Reader, cache *Manifests) Inputs { in.Gaps = append(in.Gaps, Gap{Input: "wanted manifests", Why: "no build record names the manifest at the wanted fingerprint, so the trunk's is used: " + strings.Join(fromTrunk, ", ")}) } in.Hashes = readHashes(r, &in.Gaps) + // The controller's tokens stand as a gap while it holds the lease; from the write the seat's keys are every token. + var enrolmentsRead bool + in.Enrolling, enrolmentsRead = readEnrolling(r, &in.Gaps) + if enrolmentsRead && in.LeaseAway { + in.Gaps = slices.DeleteFunc(in.Gaps, func(g Gap) bool { return g.Input == "enrolment tokens" }) + } + in.NodeLogins = readNodeLogins(r, &in.Gaps) for _, raw := range each(r, ReadResolved, &in.Gaps) { var v resolved if json.Unmarshal(raw, &v) != nil || v.Node == "" || v.Module == "" || v.Key == "" || len(v.Value) == 0 { @@ -445,6 +469,86 @@ func ReadInputs(r Reader, cache *Manifests) Inputs { return in } +// readEnrolling is every node with an enrolment key at live or redeeming (ADR 0587 §4, §5), and whether the seat's state +// was read. Not on the bus yet is a standing gap, never a failed read: the list is still composed and sealed, with no +// enrolment user. A key that does not read is named and composes no user. +func readEnrolling(r Reader, gaps *[]Gap) ([]string, bool) { + keys, err := r.Keys(ReadEnrolments) + if err != nil { + *gaps = append(*gaps, Gap{Input: ReadEnrolments, Why: "not read yet: no enrolment user is composed (" + err.Error() + ")"}) + return nil, false + } + var out []string + for _, k := range keys { + node, isNode := delivery.EnrolmentKeyNode(k) + if !isNode { + continue + } + raw, found, err := r.Get(ReadEnrolments, k) + if err != nil { + *gaps = append(*gaps, Gap{Input: ReadEnrolments + " " + k, Why: err.Error(), Failed: true}) + continue + } + if !found { + continue + } + e, err := delivery.DecodeEnrolment(raw) + if err != nil || e.Node != node { + *gaps = append(*gaps, Gap{Input: ReadEnrolments + " " + k, Why: "does not read as the enrolment of " + node + + ", so it composes no user"}) + continue + } + if e.Stage == delivery.EnrolmentLive || e.Stage == delivery.EnrolmentRedeeming { + out = append(out, node) + } + } + sort.Strings(out) + return out, true +} + +// readNodeLogins is every node's key of the vault's node-logins (ADR 0583 §1). Not on the bus yet is a standing gap, +// never a failed read: every node then logs in as node. alone, as before any rotation. A key that does not read is +// named and leaves its node at node.; a read that fails is a failed read, since a rotating node's new login would be +// left out. +func readNodeLogins(r Reader, gaps *[]Gap) map[string]delivery.NodeLogin { + keys, err := r.Keys(ReadNodeLogins) + if err != nil { + *gaps = append(*gaps, Gap{Input: ReadNodeLogins, Why: "not read yet: every node logs in as node. (" + err.Error() + ")"}) + return nil + } + out := map[string]delivery.NodeLogin{} + for _, node := range keys { + raw, found, err := r.Get(ReadNodeLogins, node) + if err != nil { + *gaps = append(*gaps, Gap{Input: ReadNodeLogins + " " + node, Why: err.Error(), Failed: true}) + continue + } + if !found { + continue + } + l, err := delivery.DecodeNodeLogin(raw) + if err != nil || l.Node != node { + *gaps = append(*gaps, Gap{Input: ReadNodeLogins + " " + node, Why: "does not read as the logins of " + node + + ", so it logs in as node." + node}) + continue + } + out[node] = l + } + return out +} + +// Rotating are the logins the vault's node-logins names in a rotation under way, as a set: a hash of one changes by the +// vault's rotation (ADR 0583 §2), never counted as a writer changing several hashes at once. +func (in Inputs) Rotating() map[string]bool { + out := map[string]bool{} + for _, l := range in.NodeLogins { + if l.Rotates() { + out[l.Login], out[l.Rotating] = true, true + } + } + return out +} + func gapNamed(gaps []Gap, input string) bool { for _, g := range gaps { if g.Input == input { diff --git a/modules/mesh-grants/internal/grants/records.go b/modules/mesh-grants/internal/grants/records.go index cf2556e..eaba280 100644 --- a/modules/mesh-grants/internal/grants/records.go +++ b/modules/mesh-grants/internal/grants/records.go @@ -36,8 +36,12 @@ func records(in Inputs, union bool) (buscompose.Records, []Gap) { for _, own := range catalogue.SeatsWithAProtocol() { seats[own.Name] = catalogue.SeatDeclaration{Name: own.Name, Scope: own.Scope, Accepts: own.Accepts, Emits: own.Emits, Serves: own.Serves} } + // The enrolment users, from the seat mesh-enrolment's keys, and from the lease write through its redeem alone (ADR 0587 + // §4, §6). out := buscompose.Records{Assigned: map[string][]buscompose.Declared{}, People: map[string][]string{}, - Interchangeable: map[string]bool{}} + Interchangeable: map[string]bool{}, Enrolling: slices.Clone(in.Enrolling), LeaseHandedAway: in.LeaseAway, + // And each node's logins (ADR 0583 §1): both while it rotates. + NodeLogins: in.NodeLogins} values := func(provider, setting string) []any { var vs []any for _, node := range in.Nodes { diff --git a/modules/mesh-grants/internal/grants/sealed.go b/modules/mesh-grants/internal/grants/sealed.go index a20e816..dbe2c83 100644 --- a/modules/mesh-grants/internal/grants/sealed.go +++ b/modules/mesh-grants/internal/grants/sealed.go @@ -336,8 +336,9 @@ func (p *Part) keepWrittenUsers(res *Result, list []byte, hashes map[string]Held } // changedHashes are the users of the list in force whose hash the list composed now would change with no rotation by the -// vault naming them, sorted; none when the part kept no credentials of the list in force (a list written before). -func (p *Part) changedHashes(hashes map[string]HeldHash) ([]string, error) { +// vault naming them (its announcement on the hash, or a node's login rotating in node-logins, ADR 0583 §2), sorted; none +// when the part kept no credentials of the list in force (a list written before). +func (p *Part) changedHashes(hashes map[string]HeldHash, rotating map[string]bool) ([]string, error) { raw, _, found, err := p.Store.Get(StateUserList, KeyWrittenCredentials) if err != nil || !found { return nil, err @@ -349,7 +350,7 @@ func (p *Part) changedHashes(hashes map[string]HeldHash) ([]string, error) { var changed []string for user, was := range had { h, ok := hashes[user] - if !ok || credentialOf(h) == was || h.Rotating != nil { + if !ok || credentialOf(h) == was || h.Rotating != nil || rotating[user] { continue } changed = append(changed, user) @@ -419,7 +420,7 @@ func (p *Part) holdsBack(in Inputs, leftOut, dropped, bus, lost []string, digest } // More than one loaded user's hash changing at once, with no rotation by the vault naming each (ADR 0561 §7): a new // writer of hashes must not lock several holders out in one list, as nine seats were cut on 2026-10-27 at 09:21. - changed, err := p.changedHashes(in.Hashes) + changed, err := p.changedHashes(in.Hashes, in.Rotating()) if err != nil { return "held: the credentials of the list in force cannot be read: " + err.Error() } diff --git a/modules/mesh-grants/internal/grants/userlist.go b/modules/mesh-grants/internal/grants/userlist.go index b83a6a5..ae76b45 100644 --- a/modules/mesh-grants/internal/grants/userlist.go +++ b/modules/mesh-grants/internal/grants/userlist.go @@ -12,6 +12,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/outward" ) @@ -614,7 +615,8 @@ func Placed(in Inputs) func(user string) bool { case len(parts) < 2: return true case parts[0] == "node" || parts[0] == "provisioner": - return slices.Contains(in.Nodes, parts[1]) + // A node's second login, node.--b, is its node's (ADR 0583 §1). + return slices.Contains(in.Nodes, strings.TrimSuffix(parts[1], delivery.SecondLoginSuffix)) case parts[0] == "person" || parts[0] == "enrol": return true } diff --git a/modules/mesh-grants/module.json b/modules/mesh-grants/module.json index ba6515d..dc309fc 100644 --- a/modules/mesh-grants/module.json +++ b/modules/mesh-grants/module.json @@ -22,7 +22,9 @@ "node-engine.have", "mesh-vault.bus-accounts", "mesh-controller.lease", - "mesh-conditions.takeover" + "mesh-conditions.takeover", + "mesh-enrolment.enrolments", + "mesh-vault.node-logins" ], "consumes": [ "mesh-conditions.condition-raised",