Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a5b4b1fba6 | ||
|
|
ec78fafc82 |
@@ -137,12 +137,7 @@ func takeWorkFrom(credential Credential, on string) (link.BuildMachine, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// **The seat this machine serves is the one its credential claims** (novox/hq ADR 0190, the
|
||||
// handover): the mesh issues a build machine's credential naming the seat its module claims,
|
||||
// and one binary serves the old role as `builder` and the new as `build-agent` from that alone.
|
||||
seat := link.BuildSeatClaimed(credential.seatsClaimed())
|
||||
fmt.Fprintf(os.Stderr, "taking build work as a holder of %s\n", seat)
|
||||
return link.MachineOverNATSOn(js, on, seat), nil
|
||||
return link.MachineOverNATS(js, on), nil
|
||||
}
|
||||
|
||||
// answer does one build and says what happened, whichever way it went.
|
||||
@@ -458,21 +453,6 @@ type Credential struct {
|
||||
// as two fields and this machine joins them once, here, to dial.
|
||||
User string `json:"user,omitempty"`
|
||||
Password string `json:"password,omitempty"`
|
||||
// Claims are the seats the module this credential was issued for claims, as the mesh writes
|
||||
// them beside the credential (novox/hq ADR 0159). The first is the build role this machine
|
||||
// serves; a credential naming none is from before claims travelled in it.
|
||||
Claims []struct {
|
||||
Seat string `json:"seat"`
|
||||
} `json:"claims,omitempty"`
|
||||
}
|
||||
|
||||
// seatsClaimed is the seats the credential names, in order.
|
||||
func (c Credential) seatsClaimed() []string {
|
||||
out := make([]string, 0, len(c.Claims))
|
||||
for _, claim := range c.Claims {
|
||||
out = append(out, claim.Seat)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// onTheNewBus is whether a credential is for the bus being built: its address says so, and the
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The seat a build machine serves comes from its credential (novox/hq ADR 0190 handover).
|
||||
func TestTheCredentialSaysWhichBuildRoleThisMachineServes(t *testing.T) {
|
||||
var held Credential
|
||||
if err := json.Unmarshal([]byte(`{"url":"nats://bus:4222","user":"anchor.builder","password":"x",
|
||||
"claims":[{"seat":"mesh-build-machine","scope":"mesh","serves":[]}]}`), &held); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := link.BuildSeatClaimed(held.seatsClaimed()); got != "mesh-build-machine" {
|
||||
t.Errorf("the old builder's credential serves %q", got)
|
||||
}
|
||||
var bare Credential
|
||||
if err := json.Unmarshal([]byte(`{"url":"nats://bus:4222","user":"anchor.build-agent","password":"x"}`), &bare); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := link.BuildSeatClaimed(bare.seatsClaimed()); got != link.TheBuildMachine {
|
||||
t.Errorf("a credential without claims serves %q, want %s", got, link.TheBuildMachine)
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,6 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -68,13 +67,6 @@ func assign(ctx context.Context, open *stores, node, module string) (string, err
|
||||
for _, line := range settled {
|
||||
said += "\n " + line
|
||||
}
|
||||
// Its bus credential, in the same act (novox/hq issue 203): an assignment pushed before its
|
||||
// credential exists delivers a process that cannot authenticate and crash-loops until somebody
|
||||
// runs a second verb and a second push. Issued here when the module speaks on the bus and has
|
||||
// no credential yet; kept when it has one, so re-assigning rotates nothing.
|
||||
if line := issueOnAssign(ctx, open, node, module); line != "" {
|
||||
said += "\n " + line
|
||||
}
|
||||
plan, _, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
// Kept, and still refused. Both halves are the answer, and the rest of the mesh is still
|
||||
@@ -160,33 +152,3 @@ func blockedElsewhere(ctx context.Context, open *stores, except string) string {
|
||||
out.WriteString("\nThis may or may not be what just changed — it is what is true now.")
|
||||
return out.String()
|
||||
}
|
||||
|
||||
// issueOnAssign gives a newly assigned module its bus credential, the way `module issue` does, and
|
||||
// says what it did in one line. Nothing for a module that declares no broker secret; nothing for one
|
||||
// whose user is already minted (a credential is rotated on purpose, never by re-assigning); and when
|
||||
// the bus cannot be reached from here, the line names the verb and the push that would refuse the
|
||||
// module until it is run — never a silent placeholder (novox/hq issue 203).
|
||||
func issueOnAssign(ctx context.Context, open *stores, node, module string) string {
|
||||
inv := open.inventory
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
m, known := shelf[module]
|
||||
if !known || mayIssue(m) != nil {
|
||||
return ""
|
||||
}
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
|
||||
if _, minted, err := inv.BusUserHash(ctx, user); err != nil || minted {
|
||||
return ""
|
||||
}
|
||||
busAddress, err := broker.BusAddress()
|
||||
if err == nil {
|
||||
err = issueOnTheNewBus(ctx, inv, m, node, busAddress)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Sprintf("its bus credential is not issued (%v): `module issue %s --node %s` first — "+
|
||||
"`push %s` refuses to send %s until it is", err, module, node, node, module)
|
||||
}
|
||||
return fmt.Sprintf("its bus credential is issued and sealed to %s, and arrives with the push", node)
|
||||
}
|
||||
|
||||
@@ -175,12 +175,6 @@ func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T)
|
||||
Guards: []int{15672},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker",
|
||||
"ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}})
|
||||
// The control plane's own bus user is the installer's, seeded at genesis before the controller
|
||||
// runs (SeedBusUser); without it a push now refuses the credential nobody issued (issue 203).
|
||||
if err := open.inventory.SeedBusUser(ctx, inventory.BusUser{Username: "anchor.mesh-controller",
|
||||
Kind: inventory.BusController, Node: "anchor", Module: "mesh-controller"}, "bootstrap"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -95,7 +95,7 @@ func showFiltering(f inventory.Filtering, adopted bool) {
|
||||
case fw.Active:
|
||||
fmt.Printf(" found firewall %s is ACTIVE on this converged machine; the next apply retires it again\n", fw.Kind)
|
||||
case fw.RetiredBy == "removed":
|
||||
fmt.Printf(" found firewall %s, removed: the mesh's filter is what filters this machine (novox/hq ADR 0180)\n", fw.Kind)
|
||||
fmt.Printf(" found firewall %s, removed: the mesh's filter is what filters this machine (novox/hq ADR 0175)\n", fw.Kind)
|
||||
case fw.RetiredBy == inventory.FilterMesh || fw.RetiredBy == "mesh":
|
||||
fmt.Printf(" found firewall %s, retired by the mesh; its configuration stays on disk\n", fw.Kind)
|
||||
case fw.RetiredBy != "":
|
||||
|
||||
@@ -430,13 +430,11 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
seat := buildSeatHeld(ctx)
|
||||
ask, err := askOverOn(seat)
|
||||
ask, err := askOver(server)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ask.Close()
|
||||
fmt.Printf(" of %s\n", seat)
|
||||
|
||||
if wait == 0 {
|
||||
// Asked and not waited for (novox/hq issue 176): the outcome is the role's event, and the
|
||||
@@ -524,8 +522,6 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
recorded := inventory.Source{
|
||||
Repository: result.Repository, Path: result.Path, Ref: result.Ref,
|
||||
BuiltFrom: result.Commit, Head: result.Commit,
|
||||
// What it stood on, so registration can judge a built manifest's base (to-be 38 WP2.4).
|
||||
Against: kept.Against,
|
||||
}
|
||||
if result.Source != nil && result.Source.Seat != "" {
|
||||
recorded.Repository, recorded.Seat = result.Source.Repository, result.Source.Seat
|
||||
@@ -557,7 +553,7 @@ func buildAndShow(ctx context.Context, source buildSource, path, ref string, wai
|
||||
}
|
||||
defer server.Close()
|
||||
|
||||
ask, err := askOverOn(buildSeatHeld(ctx))
|
||||
ask, err := askOver(server)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -670,56 +666,12 @@ func heldBy(ctx context.Context) map[string]string {
|
||||
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5). On the bus
|
||||
// the mesh runs on today this needs the controller's own connection, so it is handed one; on the bus
|
||||
// being built it dials, because a build request is a one-shot and holds nothing else.
|
||||
func askOverOn(seat string) (link.Builders, error) {
|
||||
func askOver(_ *link.Server) (link.Builders, error) {
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return link.BuildsOverNATSOn(address, seat)
|
||||
}
|
||||
|
||||
// buildSeatHeld is the build role to ask: the one some assigned module claims (novox/hq ADR 0190,
|
||||
// the handover). Read from the catalogue at ask time, because the answer changes exactly once, the
|
||||
// moment the first build-agent is assigned — and a controller that asked the new role before then
|
||||
// would queue work nothing takes, while the outcome that registers build-agent itself has to come
|
||||
// from the old builder. When the catalogue cannot be read the current role is asked, said aloud.
|
||||
func buildSeatHeld(ctx context.Context) string {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read what is assigned, so the build is asked of %s: %v\n",
|
||||
link.TheBuildMachine, err)
|
||||
return link.TheBuildMachine
|
||||
}
|
||||
defer open.Close()
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read the catalogue, so the build is asked of %s: %v\n",
|
||||
link.TheBuildMachine, err)
|
||||
return link.TheBuildMachine
|
||||
}
|
||||
return buildSeatAmong(entries)
|
||||
}
|
||||
|
||||
// buildSeatAmong is the rule, over what the catalogue holds: the current build role when any
|
||||
// assigned module claims it; else the retired role while an assigned module still claims that; else
|
||||
// the current role, which is where every ask goes once the handover is done.
|
||||
func buildSeatAmong(entries []inventory.Entry) string {
|
||||
heldBefore := false
|
||||
for _, e := range entries {
|
||||
if len(e.On) == 0 {
|
||||
continue
|
||||
}
|
||||
if e.Manifest.ClaimsSeat(link.TheBuildMachine) {
|
||||
return link.TheBuildMachine
|
||||
}
|
||||
if e.Manifest.ClaimsSeat(link.TheBuildMachineBefore) {
|
||||
heldBefore = true
|
||||
}
|
||||
}
|
||||
if heldBefore {
|
||||
return link.TheBuildMachineBefore
|
||||
}
|
||||
return link.TheBuildMachine
|
||||
return link.BuildsOverNATS(address)
|
||||
}
|
||||
|
||||
// buildLog prints everything a build machine said about one build, read back from the bus.
|
||||
@@ -739,13 +691,10 @@ func buildLog(ctx context.Context, id string) error {
|
||||
}
|
||||
defer js.Close()
|
||||
|
||||
// Under whichever build role did it: a build asked of the retired role during the handover
|
||||
// (ADR 0190) said its lines as that role's events, and a reader should not have to know which.
|
||||
lines := link.BuildLogOf("*", id)
|
||||
sub, err := js.Context().PullSubscribe(lines, "",
|
||||
sub, err := js.Context().PullSubscribe(link.BuildLog(id), "",
|
||||
nats.BindStream(broker.EventsStream), nats.DeliverAll(), nats.AckNone())
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot read %s from the bus: %w", lines, err)
|
||||
return fmt.Errorf("cannot read %s from the bus: %w", link.BuildLog(id), err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func claiming(module, seat string, on ...string) inventory.Entry {
|
||||
return inventory.Entry{
|
||||
Manifest: catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: seat}}},
|
||||
On: on,
|
||||
}
|
||||
}
|
||||
|
||||
// The controller asks the build role that has a holder (novox/hq ADR 0190 handover): the retired
|
||||
// one while only the builder is assigned, the current one from the first build-agent on, and the
|
||||
// current one when nothing holds either — where every ask goes once the handover is done.
|
||||
func TestTheControllerAsksTheBuildRoleThatHasAHolder(t *testing.T) {
|
||||
onlyTheBuilder := []inventory.Entry{
|
||||
claiming("builder", link.TheBuildMachineBefore, "anchor"),
|
||||
claiming("build-agent", link.TheBuildMachine), // registered, assigned nowhere yet
|
||||
}
|
||||
if got := buildSeatAmong(onlyTheBuilder); got != link.TheBuildMachineBefore {
|
||||
t.Errorf("with only the builder assigned, asked %q", got)
|
||||
}
|
||||
bothHeld := []inventory.Entry{
|
||||
claiming("builder", link.TheBuildMachineBefore, "anchor"),
|
||||
claiming("build-agent", link.TheBuildMachine, "home-server"),
|
||||
}
|
||||
if got := buildSeatAmong(bothHeld); got != link.TheBuildMachine {
|
||||
t.Errorf("with a build-agent assigned anywhere, asked %q", got)
|
||||
}
|
||||
neither := []inventory.Entry{claiming("builder", link.TheBuildMachineBefore)}
|
||||
if got := buildSeatAmong(neither); got != link.TheBuildMachine {
|
||||
t.Errorf("with no holder of either, asked %q, want the current role", got)
|
||||
}
|
||||
if got := buildSeatAmong(nil); got != link.TheBuildMachine {
|
||||
t.Errorf("an empty catalogue asks %q", got)
|
||||
}
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A fresh assignment is pushed before its credential exists (novox/hq issue 203): `assign` recorded
|
||||
// the module, `push` sealed a random own secret where the bus credential belongs, and the process
|
||||
// crash-looped until a person ran `module issue` and pushed again. Now assigning a module that speaks
|
||||
// on the bus issues its credential in the same act — or, when the bus cannot be reached from here,
|
||||
// says which verb to run — and a push never seals a placeholder in a credential's place.
|
||||
|
||||
func aTalker() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "talker", Version: "1",
|
||||
OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh/talker/broker"}},
|
||||
Resources: []map[string]any{
|
||||
{"id": "state", "type": "directory", "path": "/var/lib/mesh/talker", "mode": "0700"},
|
||||
}}
|
||||
}
|
||||
|
||||
func TestAssigningAModuleThatSpeaksOnTheBusNamesItsCredential(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aTalker())
|
||||
|
||||
// No bus is known to this process, so the credential cannot be issued here: the assignment
|
||||
// stands and says exactly what must happen before a push — never silently.
|
||||
said, err := assign(ctx, open, "laptop", "talker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(said, "module issue talker --node laptop") {
|
||||
t.Fatalf("an assignment whose credential could not be issued does not name the verb:\n%s", said)
|
||||
}
|
||||
|
||||
// And the push refuses to send it, naming the same verb, rather than sealing a placeholder.
|
||||
plan, settings, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = declarationFor(ctx, open, "laptop", plan, settings)
|
||||
if err == nil {
|
||||
t.Fatal("a push sealed a placeholder where talker's bus credential belongs")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "module issue talker --node laptop") || !strings.Contains(err.Error(), "issue 203") {
|
||||
t.Fatalf("the refusal does not say what to run: %v", err)
|
||||
}
|
||||
|
||||
// Once the user is minted, the push goes on to the credential the mesh sealed, and re-assigning
|
||||
// does not mint again: a credential rotates on purpose, never by habit.
|
||||
if _, err := open.inventory.MintBusPassword(ctx, inventory.BusUser{
|
||||
Username: "laptop.talker", Kind: inventory.BusModule, Node: "laptop", Module: "talker"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hash, _, err := open.inventory.BusUserHash(ctx, "laptop.talker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said, err = assign(ctx, open, "laptop", "talker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(said, "module issue") {
|
||||
t.Fatalf("a module with a minted credential was told to issue one:\n%s", said)
|
||||
}
|
||||
again, _, err := open.inventory.BusUserHash(ctx, "laptop.talker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again != hash {
|
||||
t.Fatal("re-assigning rotated the credential")
|
||||
}
|
||||
}
|
||||
|
||||
// A module that declares no broker secret is left alone: nothing to issue, nothing said.
|
||||
func TestAssigningAModuleThatDoesNotSpeakSaysNothingOfCredentials(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
register(t, open, helloWeb())
|
||||
said, err := assign(t.Context(), open, "laptop", "hello-web")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(said, "credential") {
|
||||
t.Fatalf("a module without a broker secret was told about credentials:\n%s", said)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package main
|
||||
|
||||
// The broker opening belongs only on the node that listens on it (novox/hq: it leaked onto
|
||||
// every enrolled node's declaration, opening a from-anywhere hole for a port nothing there
|
||||
// serves). foundationPortsFor is the scope.
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
func TestTheBrokerHostGetsTheFoundationOpening(t *testing.T) {
|
||||
broker := catalogue.Manifest{Module: "lavinmq", Listens: []catalogue.Listening{
|
||||
{Port: 5671, Protocol: "tcp", From: "mesh"},
|
||||
{Port: 5672, Protocol: "tcp", From: "mesh"},
|
||||
}}
|
||||
got := foundationPortsFor(5671, []catalogue.Manifest{broker})
|
||||
if len(got) != 1 || got[0] != 5671 {
|
||||
t.Fatalf("the node that listens on the broker port keeps it; got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeThatOnlyDialsTheBrokerGetsNoOpening(t *testing.T) {
|
||||
// ace's set: things that reach the broker as a client, none listening on 5671.
|
||||
ace := []catalogue.Manifest{
|
||||
{Module: "plex", Listens: []catalogue.Listening{{Port: 32400, Protocol: "tcp", From: "anywhere"}}},
|
||||
{Module: "postgres", Listens: []catalogue.Listening{{Port: 5432, Protocol: "tcp", From: "mesh"}}},
|
||||
}
|
||||
if got := foundationPortsFor(5671, ace); got != nil {
|
||||
t.Fatalf("a node that only dials out opens nothing for the broker; got %v", got)
|
||||
}
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A declaration composed earlier is numbered lower than one composed later, whatever order the two
|
||||
// are sent in (novox/hq issue 204). The number used to be taken at send time, after composing, so a
|
||||
// declaration composed before an assignment changed and sent after a newer one carried the higher
|
||||
// number — and the machine, which refuses a lower number, took the older content as the mesh's
|
||||
// newest word. Taken before the composition reads anything, the order of numbers is the order of
|
||||
// compositions, and the host's refusal does what it is for.
|
||||
func TestADeclarationComposedEarlierIsNumberedLowerWhateverOrderItIsSent(t *testing.T) {
|
||||
allot := numbered()
|
||||
var composed []string
|
||||
compose := func(stamp string) func(string) (sendable, error) {
|
||||
return func(node string) (sendable, error) {
|
||||
composed = append(composed, stamp)
|
||||
return sendable{Resources: []map[string]any{{"id": node + "." + stamp}}}, nil
|
||||
}
|
||||
}
|
||||
// Composed first — before an assignment changed — and sent last.
|
||||
stale, _ := composeEach([]string{"anchor"}, allot, compose("before"))
|
||||
// Composed after the change, sent first.
|
||||
fresh, _ := composeEach([]string{"anchor"}, allot, compose("after"))
|
||||
|
||||
if stale[0].declared.Sequence != 1 || fresh[0].declared.Sequence != 2 {
|
||||
t.Fatalf("the numbers do not follow the compositions: before=%d after=%d",
|
||||
stale[0].declared.Sequence, fresh[0].declared.Sequence)
|
||||
}
|
||||
// Sent in the other order, the numbers do not change — so the machine that has applied the
|
||||
// fresh one (2) refuses the stale one (1) when it arrives late.
|
||||
if !(stale[0].declared.Sequence < fresh[0].declared.Sequence) {
|
||||
t.Fatal("a declaration composed earlier must carry the lower number, however late it is sent")
|
||||
}
|
||||
if len(composed) != 2 || composed[0] != "before" {
|
||||
t.Fatalf("compositions happened in an unexpected order: %v", composed)
|
||||
}
|
||||
}
|
||||
|
||||
// The number is taken before the first read of the composition, not after it: an allotter that
|
||||
// fails leaves nothing composed for that machine, and the others are still composed.
|
||||
func TestTheNumberIsTakenBeforeComposingAndItsFailureIsARefusal(t *testing.T) {
|
||||
calls := 0
|
||||
allot := func(node string) (int64, error) {
|
||||
if node == "anchor" {
|
||||
return 0, context.DeadlineExceeded
|
||||
}
|
||||
return 7, nil
|
||||
}
|
||||
sending, refusals := composeEach([]string{"anchor", "laptop"}, allot, func(node string) (sendable, error) {
|
||||
calls++
|
||||
if node == "anchor" {
|
||||
t.Fatal("anchor was composed although its number could not be taken")
|
||||
}
|
||||
return sendable{}, nil
|
||||
})
|
||||
if calls != 1 || len(sending) != 1 || sending[0].node != "laptop" || sending[0].declared.Sequence != 7 {
|
||||
t.Fatalf("laptop should be composed with its number and anchor refused: %v / %v", sending, refusals)
|
||||
}
|
||||
if len(refusals) != 1 {
|
||||
t.Fatalf("anchor's failed number should be a refusal naming it: %v", refusals)
|
||||
}
|
||||
}
|
||||
|
||||
// What was sent is written down even when the sender's context is already cancelled (issue 204): a
|
||||
// controller replaced mid-send had told the machine and never recorded it, so status read "applied,
|
||||
// current" over a machine that had just been sent something else.
|
||||
func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cancel() // the sender is going away: its context is cancelled between the send and the record
|
||||
body := []byte(`{"declaration":1,"resources":[]}`)
|
||||
digest, err := recordSent(ctx, inv, "anchor", body)
|
||||
if err != nil {
|
||||
// NodeByName on the cancelled context may itself refuse; the record must still be possible
|
||||
// through the detached context, so look the node up again on a live one.
|
||||
t.Fatalf("recording a send after cancellation failed: %v", err)
|
||||
}
|
||||
outstanding, err := inv.Outstanding(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outstanding != digest || digest != digestOf(body) {
|
||||
t.Fatalf("the send was not recorded: outstanding %q, sent %q", outstanding, digest)
|
||||
}
|
||||
}
|
||||
@@ -147,40 +147,6 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// **The same list, for something other than a person** (novox/hq ADR 0195): what each module
|
||||
// is, where it runs, whether it is current, and what it says of itself.
|
||||
if len(args) > 1 && args[1] == "--json" {
|
||||
type listed struct {
|
||||
Module string `json:"module"`
|
||||
Version string `json:"version"`
|
||||
Built string `json:"built,omitempty"`
|
||||
Head string `json:"head,omitempty"`
|
||||
Current bool `json:"current"`
|
||||
Provided bool `json:"provided,omitempty"`
|
||||
// Tools says whether the module answers tools anywhere it runs: a list of its own,
|
||||
// a bundle the runtime serves, or a seat's verbs it claims (novox/hq ADR 0197) —
|
||||
// what the console checks the bus's answers against.
|
||||
Tools bool `json:"tools"`
|
||||
On []string `json:"on"`
|
||||
Provides []string `json:"provides,omitempty"`
|
||||
Requires []string `json:"requires,omitempty"`
|
||||
Claims []string `json:"claims,omitempty"`
|
||||
Capabilities []string `json:"capabilities,omitempty"`
|
||||
}
|
||||
out := make([]listed, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
m := e.Manifest
|
||||
l := listed{Module: m.Module, Version: m.Version, Built: e.Source.BuiltFrom, Head: e.Source.Head,
|
||||
Current: e.Provided || e.Source.Repository == "" || e.Source.Current(), Provided: e.Provided,
|
||||
On: append([]string{}, e.On...), Provides: m.Offers(), Requires: m.Requires,
|
||||
Capabilities: m.Capabilities, Tools: declaresTools(m)}
|
||||
for _, c := range m.Claims {
|
||||
l.Claims = append(l.Claims, c.At()+"/"+c.Name)
|
||||
}
|
||||
out = append(out, l)
|
||||
}
|
||||
return printJSON(out)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
fmt.Println("this mesh knows about no modules yet")
|
||||
return nil
|
||||
@@ -591,7 +557,7 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
|
||||
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username()
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{
|
||||
Username: user, Kind: busKindOf(m.Module), Node: node, Module: m.Module,
|
||||
Username: user, Kind: inventory.BusModule, Node: node, Module: m.Module,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -611,16 +577,6 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
|
||||
return issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password)
|
||||
}
|
||||
|
||||
// busKindOf is what a module's bus user is recorded as: the node's tool runtime where the module is
|
||||
// the runtime (novox/hq ADR 0175), a module otherwise. The username is the same either way — the
|
||||
// runtime is issued through this same path — and the kind is what a reader of the records sees.
|
||||
func busKindOf(module string) string {
|
||||
if module == catalogue.RuntimeModule {
|
||||
return inventory.BusNodeTools
|
||||
}
|
||||
return inventory.BusModule
|
||||
}
|
||||
|
||||
// issueWith is the delivery half: the minted password sealed to the machine as the module's broker
|
||||
// secret, and the module's consumer created where the bus can be reached. Split from the minting
|
||||
// so the move can issue every module against a bus whose address it worked out itself
|
||||
@@ -767,23 +723,3 @@ func claimsFor(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// declaresTools is whether a module answers tools wherever it runs (novox/hq ADR 0197): it names
|
||||
// tools of its own, its build delivers a bundle the node's runtime serves, or it claims a seat
|
||||
// whose verbs it serves. A module with none is never expected to announce anything.
|
||||
func declaresTools(m catalogue.Manifest) bool {
|
||||
if len(m.Tools) > 0 {
|
||||
return true
|
||||
}
|
||||
for _, b := range m.Bundles {
|
||||
if len(b.Loads) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
if len(c.Serves) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -232,9 +232,22 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// **A machine joining is on the network before it is anything else** (novox/hq ADR 0169). Its
|
||||
// token was issued for its tunnel key and gave it an address, so while that token can still be
|
||||
// used the hub carries it as a peer: it brings its tunnel up from the token and enrols over it.
|
||||
// When the token is spent the machine is on the network by what it runs, as every other is; when
|
||||
// it expires unused, the peer goes with it at the hub's next composition.
|
||||
joining, err := inv.NodesWithALiveToken(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
isJoining := map[string]bool{}
|
||||
for _, name := range joining {
|
||||
isJoining[name] = true
|
||||
}
|
||||
nodes := make([]overlay.Node, 0, len(places))
|
||||
for _, p := range places {
|
||||
if !on[p.Name] {
|
||||
if !on[p.Name] && !(isJoining[p.Name] && p.Key != "" && p.Address != "") {
|
||||
continue
|
||||
}
|
||||
n := overlay.Node{
|
||||
|
||||
@@ -3,7 +3,6 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -304,28 +303,3 @@ func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *t
|
||||
t.Fatalf("a machine that left the network is still a wildcard, or the one that stayed is not:\n%s", after)
|
||||
}
|
||||
}
|
||||
|
||||
// The roster is the machines and nothing else (novox/hq ADR 0191): each node's internal domain covers
|
||||
// every route on it, and a node's public domains are public DNS's. A routed name in `.Names` was a
|
||||
// private answer for a public name, handed by a resolver serving a LAN to a phone that could not use it.
|
||||
func TestTheRosterNamesOnlyTheMachines(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Reading)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(with.Names, with.Machines) {
|
||||
t.Fatalf("%s's roster names more than the machines:\n names %v\n machines %v", node, with.Names, with.Machines)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,10 +2,11 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -45,21 +46,6 @@ func nodeCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// **The same list, for something other than a person** — the console's discovery reads it
|
||||
// (novox/hq ADR 0195), and a reader that parses a printed column breaks when it is reworded.
|
||||
if len(args) > 1 && args[1] == "--json" {
|
||||
type listed struct {
|
||||
Name string `json:"name"`
|
||||
Heard string `json:"heard"`
|
||||
Mode string `json:"mode"`
|
||||
ID string `json:"id"`
|
||||
}
|
||||
out := make([]listed, 0, len(nodes))
|
||||
for _, n := range nodes {
|
||||
out = append(out, listed{Name: n.Name, Heard: heardFrom(n), Mode: modeOf(n), ID: n.ID})
|
||||
}
|
||||
return printJSON(out)
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
// Said rather than printed as nothing: an empty list and a failed read must never
|
||||
// look the same, and this command answering "none" is only honest because getting
|
||||
@@ -246,6 +232,8 @@ func tokenCommand(ctx context.Context, args []string) error {
|
||||
validFor := set.Duration("for", time.Hour, "how long the token may be used")
|
||||
adopted := set.Bool("adopted", false,
|
||||
"the machine joining is in use: it is adopted, and keeps what is found on it")
|
||||
tunnelKey := set.String("overlay-key", "",
|
||||
"the public half of the tunnel key the machine made (`nox-mesh-host key`): it joins through the tunnel")
|
||||
if err := set.Parse(args[1:]); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -299,6 +287,14 @@ func tokenCommand(ctx context.Context, args []string) error {
|
||||
default:
|
||||
return err
|
||||
}
|
||||
// **Through the tunnel** (novox/hq ADR 0169): the machine's key recorded, its address given, the
|
||||
// hub sent it as a peer — all before the token is shown, so the tunnel answers the first time the
|
||||
// machine knocks. The bus is then reached at its address on the private network.
|
||||
if *tunnelKey != "" {
|
||||
if made.Tunnel, made.Broker, err = throughTheTunnel(ctx, open, issued.Node, *tunnelKey, made.Broker); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
encoded, err := made.Encode()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -323,6 +319,66 @@ func tokenCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// throughTheTunnel makes a machine a peer of the hub for its token, and says what the token carries
|
||||
// for it: its first tunnel, and the bus at its address on the private network (novox/hq ADR 0169).
|
||||
//
|
||||
// The hub is pushed here, before the token is shown. A token shown before the hub knew the key is a
|
||||
// tunnel that does not answer, and a machine that cannot tell that from a bus that is down.
|
||||
func throughTheTunnel(ctx context.Context, open *stores, node inventory.Node, key, busAt string) (
|
||||
*token.Tunnel, string, error) {
|
||||
inv := open.inventory
|
||||
key = strings.TrimSpace(key)
|
||||
if raw, err := base64.StdEncoding.DecodeString(key); err != nil || len(raw) != 32 {
|
||||
return nil, "", fmt.Errorf("%q is not a tunnel public key: it is 32 bytes in base64, as "+
|
||||
"`nox-mesh-host key` prints it", key)
|
||||
}
|
||||
// The bus on the private network is the hub's address at the bus's own port, so the port must be
|
||||
// known before anything is recorded.
|
||||
_, port, err := net.SplitHostPort(busAt)
|
||||
if err != nil || port == "" {
|
||||
return nil, "", fmt.Errorf("the bus's address %q has no port to reach it on", busAt)
|
||||
}
|
||||
places, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
var hub *inventory.Overlay
|
||||
for i := range places {
|
||||
if places[i].Hub {
|
||||
hub = &places[i]
|
||||
}
|
||||
}
|
||||
if hub == nil || hub.Key == "" || hub.Endpoint == "" || hub.Address == "" {
|
||||
return nil, "", errors.New("this mesh has no hub with a key, an address and an endpoint to " +
|
||||
"dial, so there is no tunnel to join through: place one (`overlay place <node> --hub " +
|
||||
"--endpoint <host>:<port>`), or issue the token without --overlay-key")
|
||||
}
|
||||
if err := inv.RecordOverlayKey(ctx, node.ID, key); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if err := inv.BindTokenToKey(ctx, node.ID, key); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
cidr, err := overlayRange(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
address, err := inv.AssignAddress(ctx, node.ID, cidr)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if err := sendTo(ctx, open, []string{hub.Name}); err != nil {
|
||||
return nil, "", fmt.Errorf("%s was made a peer of the hub, and the hub could not be sent "+
|
||||
"it, so the tunnel would not answer — the token is not shown; issue it again once %s "+
|
||||
"can be pushed: %w", node.Name, hub.Name, err)
|
||||
}
|
||||
// An address, not a name — nothing resolves before the machine has joined (novox/hq ADR 0004).
|
||||
return &token.Tunnel{
|
||||
Key: key, Address: address + "/32", Range: cidr,
|
||||
HubKey: hub.Key, HubEndpoint: hub.Endpoint,
|
||||
}, net.JoinHostPort(hub.Address, port), nil
|
||||
}
|
||||
|
||||
// issueFor is the inventory's half of issuing a token: the record, made when it is new, adopted
|
||||
// when the operator says so, and the one-time secret for it. The node in what it returns carries
|
||||
// its mode, which is what the token says.
|
||||
@@ -516,13 +572,3 @@ func orNotReported(s string) string {
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// printJSON prints a value as indented JSON, the shape every `--json` answers in.
|
||||
func printJSON(v any) error {
|
||||
body, err := json.MarshalIndent(v, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -192,7 +191,7 @@ func TestWhatAHandedOverModuleRecordsAboutItsSource(t *testing.T) {
|
||||
t.Fatalf("the source records as %+v", from)
|
||||
}
|
||||
// A manifest with no provenance at all is legitimate: fixing something in a hurry.
|
||||
if from, err := whereItComesFrom("", "", "", "", false); err != nil || !reflect.DeepEqual(from, inventory.Source{}) {
|
||||
if from, err := whereItComesFrom("", "", "", "", false); err != nil || from != (inventory.Source{}) {
|
||||
t.Fatalf("a manifest handed over with no provenance was refused: %+v, %v", from, err)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
|
||||
+120
-29
@@ -16,6 +16,8 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
"net"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// working out what one machine should be.
|
||||
@@ -533,23 +535,6 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
var sealed string
|
||||
var err error
|
||||
if choosing == Allocating {
|
||||
// **The broker credential is never invented here** (novox/hq issue 203). Every other
|
||||
// own secret is the mesh's to make — a password nobody else knows — but this one
|
||||
// is an account on the bus, minted by `module issue` and sealed by it; a push that
|
||||
// made a random one would deliver a file the process cannot read and report the
|
||||
// machine applied. Refused by name, with the verb.
|
||||
if name == "broker" {
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username()
|
||||
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
} else if !minted {
|
||||
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf(
|
||||
"%s on %s has no bus credential: nothing was issued for %s, and a push "+
|
||||
"would seal a placeholder its process cannot read (novox/hq issue 203). "+
|
||||
"`module issue %s --node %s`, then push again",
|
||||
m.Module, node, user, m.Module, node)
|
||||
}
|
||||
}
|
||||
sealed, err = inv.SecretForModule(ctx, node, m.Module, name)
|
||||
} else {
|
||||
var held bool
|
||||
@@ -645,24 +630,43 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
}
|
||||
}
|
||||
|
||||
// **The roster is the machines and nothing else** (novox/hq ADR 0191). Each node has one internal
|
||||
// domain, `<node>.internal`, and every route on it is a name under that domain (ADR 0151), which
|
||||
// the resolver answers with one wildcard per machine — so no route needs a line of its own. A
|
||||
// node's public domains are the operator's and public DNS answers them; the mesh gives no private
|
||||
// answer for any of them. The roster once carried every routed name, public ones included, and a
|
||||
// resolver that also serves a LAN handed a phone a tunnel address for the mail server.
|
||||
// `.Names` and `.Machines` stay two fields so a module's template keeps rendering (issue 111).
|
||||
// And every routed name → the node that serves it (novox/hq ADR 0066). Alongside the
|
||||
// `<node>.internal` names above, so a container — or an internal ACME validator — resolves a
|
||||
// routed name to the proxy that serves it, mesh-wide. The mesh publishes the names it was told
|
||||
// to serve and knows nothing about what they mean.
|
||||
// Kept apart from the machines, because a fact about the machines must not be handed the names
|
||||
// the mesh merely serves (novox/hq 04-ISSUES/111).
|
||||
machines := make(map[string]string, len(names))
|
||||
for name, at := range names {
|
||||
machines[name] = at
|
||||
}
|
||||
routes, err := routeNamesInTheMesh(ctx, open)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
for name, at := range routes {
|
||||
names[name] = at
|
||||
}
|
||||
|
||||
// **The bus is never public** (novox/hq ADR 0169). It was a foundation port — widened from the
|
||||
// broker's own `from: mesh` to from-anywhere on the broker's host, so a machine could enrol
|
||||
// before it had an address on the private network. A machine joins through the tunnel now, and
|
||||
// every link to the bus crosses it, so its reach is what the `nats` module declares: the mesh.
|
||||
// Nothing the mesh itself needs is opened beyond what a module declares.
|
||||
// The ports the mesh itself needs open, which no module declares. Read from the broker this
|
||||
// control plane was told about rather than written down twice: the address a node is handed in
|
||||
// its token and the port its machine must accept on are the same fact.
|
||||
//
|
||||
// **Only on the node that listens on it** (novox/hq issue: the broker opening leaked onto
|
||||
// every node). The opening exists to WIDEN the broker's port to from-anywhere — a machine
|
||||
// enrolling is not on the mesh yet, so the broker's own `from: mesh` listen would refuse its
|
||||
// first dial. That widening belongs on the broker's host and nowhere else: a node that only
|
||||
// dials out needs no incoming rule, and an opening for a port nothing here listens on is a
|
||||
// from-anywhere hole for a dead port. So the foundation port is kept only when a module
|
||||
// resolved onto THIS node actually listens on it.
|
||||
var foundation []int
|
||||
if b, err := broker.FromEnvironment(); err == nil {
|
||||
if _, port, err := net.SplitHostPort(b.Address); err == nil {
|
||||
if n, err := strconv.Atoi(port); err == nil {
|
||||
foundation = foundationPortsFor(n, plan.Modules)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// And, for a module that keeps them, every operator-sealed secret in the mesh — the vault's
|
||||
// copy, outside the store (novox/hq ADR 0085, amended). Read only; nothing here mints. The
|
||||
@@ -734,6 +738,76 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
}, record, nil
|
||||
}
|
||||
|
||||
// routeNamesInTheMesh is every routed name and the address of the node that serves it (novox/hq
|
||||
// ADR 0066).
|
||||
//
|
||||
// **Mesh-wide, so any container resolves any routed name to its proxy** — including an internal
|
||||
// ACME validator, which cannot complete a challenge for a name it cannot reach. A routed name is
|
||||
// composed on the consumer's node (from its label and that node's public domain) and served by the
|
||||
// node answering the consumer's route requirement; this gathers both.
|
||||
//
|
||||
// It reads route names off resolutions rather than a table because there is no table: a route is a
|
||||
// contribution, computed from what each node runs. Name-agnostic — a contribution counts as a
|
||||
// routed name only because it carried a label the mesh composed, never because the mesh knows what
|
||||
// "route" means. A node that does not resolve is skipped, so one machine's broken set does not cost
|
||||
// the rest their names.
|
||||
//
|
||||
// **A node that could not be READ is a different matter and is raised.** Skipping one states, to
|
||||
// every machine at once, that its names do not exist — and since the roster is part of every
|
||||
// container's identity, that withdraws them and replaces every container (novox/hq 04-ISSUES/152,
|
||||
// 151). So every failure here says which machine and which read, because the alternative is a
|
||||
// mesh-wide refusal with nothing named in it.
|
||||
func routeNamesInTheMesh(ctx context.Context, open *stores) (map[string]string, error) {
|
||||
inv := open.inventory
|
||||
places, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("where the machines are cannot be read: %w", err)
|
||||
}
|
||||
address := map[string]string{}
|
||||
for _, p := range places {
|
||||
if strings.TrimSpace(p.Address) != "" {
|
||||
address[p.Name] = p.Address
|
||||
}
|
||||
}
|
||||
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("which machines the mesh has cannot be read: %w", err)
|
||||
}
|
||||
|
||||
// Every machine's resolution first, then the names across them at once: which node serves a
|
||||
// name is a question about the graph — the consumer on one machine, the provider on another —
|
||||
// and answered wrongly by looking at one contribution at a time (novox/hq issue 178).
|
||||
plans := map[string]catalogue.Resolution{}
|
||||
settings := map[string]catalogue.SettingsBy{}
|
||||
for _, n := range nodes {
|
||||
plan, layers, err := planFor(ctx, open, n.Name)
|
||||
switch {
|
||||
case unresolvable(err):
|
||||
// Their set does not compose, so they serve no names. Passed over, so one machine's
|
||||
// broken set does not cost the rest theirs.
|
||||
continue
|
||||
case err != nil:
|
||||
// The mesh could not be asked. Returning the roster without this machine's names would
|
||||
// state that they do not exist — to every machine, and indistinguishably from the
|
||||
// operator having withdrawn them (novox/hq 04-ISSUES/152).
|
||||
return nil, fmt.Errorf("the names %s serves cannot be read: %w", n.Name, err)
|
||||
}
|
||||
plans[n.Name], settings[n.Name] = plan, layers
|
||||
}
|
||||
served, err := catalogue.NamesServed(plans, settings)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]string{}
|
||||
for name, node := range served {
|
||||
if at := address[node]; at != "" {
|
||||
out[name] = at
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// certificateFor is what the mesh certifies about one machine's internal name.
|
||||
//
|
||||
// It reaches across two contexts and reads neither one's store from the other: `inventory` knows
|
||||
@@ -1306,6 +1380,23 @@ func composeBusUsers(ctx context.Context, inv *inventory.Inventory,
|
||||
return broker.ComposeAccounts(filled)
|
||||
}
|
||||
|
||||
// foundationPortsFor is the broker port, kept only when a module resolved onto this node listens
|
||||
// on it (novox/hq issue: the broker opening leaked onto every node). The foundation opening
|
||||
// exists to WIDEN the broker's `from: mesh` port to from-anywhere, because a machine enrolling is
|
||||
// not on the mesh yet and its first dial would be refused. That widening belongs on the broker's
|
||||
// host alone: a node that only dials out needs no incoming rule, and an opening for a port
|
||||
// nothing here listens on is a from-anywhere hole for a dead port.
|
||||
func foundationPortsFor(brokerPort int, modules []catalogue.Manifest) []int {
|
||||
for _, m := range modules {
|
||||
for _, l := range m.Listens {
|
||||
if l.Port == brokerPort {
|
||||
return []int{brokerPort}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// providerModuleOf is which module answers a need on the providing node: the one in this node's
|
||||
// own set when the provider is here, else the one the catalogue says offers it.
|
||||
func providerModuleOf(resolved catalogue.Resolution, open *stores, ctx context.Context, n catalogue.Needed) string {
|
||||
|
||||
+38
-79
@@ -131,17 +131,10 @@ func serve(ctx context.Context) error {
|
||||
|
||||
// And the mesh's own verbs, as the seat this control plane holds (novox/hq ADR 0154). Served
|
||||
// from the store's row, so what the seat declares is what is answered.
|
||||
handlers, behind, err := seatToolHandlers()
|
||||
handlers, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(behind) > 0 {
|
||||
// Said once, loudly, and then served anyway (novox/hq ADR 0185): the mesh keeps answering
|
||||
// while whatever put an older control plane here is undone.
|
||||
fmt.Printf("this control plane is behind the %s row: it cannot run %s. "+
|
||||
"Those answer the reason when called; everything else is served as usual\n",
|
||||
catalogue.ControllerSeatName, strings.Join(behind, ", "))
|
||||
}
|
||||
bus, isNATS := server.Bus().(link.OverNATS)
|
||||
if !isNATS {
|
||||
return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it")
|
||||
@@ -151,12 +144,6 @@ func serve(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
defer stopServing()
|
||||
// And says so on the bus (novox/hq ADR 0197): what it serves, as the NATS services protocol asks.
|
||||
stopAnnouncing, err := bus.Announce(seatAnnouncement(handlers), log.New(os.Stdout, "", log.LstdFlags))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stopAnnouncing()
|
||||
|
||||
return server.Serve(ctx)
|
||||
}
|
||||
@@ -211,12 +198,6 @@ func declare(ctx context.Context, args []string) error {
|
||||
if err := link.Declare(ctx, server.Bus(), ident, node, raw, 15*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
// Written down like every other send (novox/hq issue 204): a declaration a person sent by hand
|
||||
// is still what the machine was last told, and status must not read it as current for the one
|
||||
// the mesh would compose.
|
||||
if _, err := recordSent(ctx, inv, node, raw); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
|
||||
return nil
|
||||
}
|
||||
@@ -360,7 +341,7 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sending, refusals := composeEach(asked, allotting(held, inv), func(node string) (sendable, error) {
|
||||
sending, refusals := composeEach(asked, func(node string) (sendable, error) {
|
||||
plan, settings, err := planFor(held, open, node)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
@@ -382,8 +363,12 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
sentDigest := map[string]string{}
|
||||
defer release()
|
||||
for _, s := range sending {
|
||||
// The number is inside the signed bytes, so a replayed older declaration cannot borrow a
|
||||
// newer one's (novox/hq 04-ISSUES/107); it was taken when the composition began (issue 204).
|
||||
// Numbered under the hold, one higher than the last, before the body exists — the number is
|
||||
// inside the signed bytes, so a replayed older declaration cannot borrow a newer one's
|
||||
// (novox/hq 04-ISSUES/107).
|
||||
if err := number(ctx, inv, &s); err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := s.declared.Body()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -393,10 +378,14 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
// After it is away, not before. A digest recorded for something that failed to send would
|
||||
// make the machine look current for a declaration it never received.
|
||||
digest, err := recordSent(ctx, inv, s.node, body)
|
||||
record, err := inv.NodeByName(ctx, s.node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
digest := digestOf(body)
|
||||
if err := inv.RecordSent(ctx, record.ID, digest); err != nil {
|
||||
return err
|
||||
}
|
||||
sentDigest[s.node] = digest
|
||||
fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.declared.Resources))
|
||||
}
|
||||
@@ -480,7 +469,11 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
15*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := recordSent(ctx, inv, s.node, body); err != nil {
|
||||
record, err := inv.NodeByName(ctx, s.node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.declared.Resources))
|
||||
@@ -569,31 +562,17 @@ type readyNode struct {
|
||||
//
|
||||
// The all-or-nothing rule is kept where it means something — sendTo, which rotates a credential
|
||||
// across two machines that must agree — and dropped here, where it never did.
|
||||
func composeEach(names []string, allot func(node string) (int64, error),
|
||||
func composeEach(names []string,
|
||||
compose func(node string) (sendable, error)) ([]readyNode, []string) {
|
||||
|
||||
var sending []readyNode
|
||||
var refusals []string
|
||||
for _, name := range names {
|
||||
// **Numbered before it is composed, not before it is sent** (novox/hq issue 204). The
|
||||
// number says where this declaration stands against every other the mesh composed for the
|
||||
// machine, and the host refuses one lower than the last it applied. Taken at send time, as
|
||||
// it was, a declaration composed a minute ago — before an assignment changed — went out with
|
||||
// a number higher than one composed after the change and sent before it, and the machine
|
||||
// took the older content as the newer word: on 2026-10-02 a runtime assigned and applied on
|
||||
// two machines was undone two seconds later by exactly that. Taken here, before the first
|
||||
// read, what was composed earlier is numbered lower whatever order the sends happen in.
|
||||
seq, err := allot(name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
declared, err := compose(name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
declared.Sequence = seq
|
||||
if len(declared.Resources) == 0 {
|
||||
// Sent, not skipped (novox/hq issue 127). A node whose declaration composes to
|
||||
// nothing may have HELD something before — the broker opening a placement gave it,
|
||||
@@ -621,10 +600,13 @@ func sendRound(ctx context.Context, open *stores, names []string,
|
||||
return nil, err
|
||||
}
|
||||
defer release()
|
||||
sending, refused := composeEach(names, allotting(held, open.inventory), func(node string) (sendable, error) {
|
||||
sending, refused := composeEach(names, func(node string) (sendable, error) {
|
||||
return compose(held, node)
|
||||
})
|
||||
for _, s := range sending {
|
||||
if err := number(ctx, open.inventory, &s); err != nil {
|
||||
return refused, err
|
||||
}
|
||||
body, err := s.declared.Body()
|
||||
if err != nil {
|
||||
return refused, err
|
||||
@@ -681,12 +663,6 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
|
||||
var sending []readyNode
|
||||
var refusals []string
|
||||
for _, name := range names {
|
||||
// Numbered before composing, for the reason composeEach gives (novox/hq issue 204).
|
||||
seq, err := allot(ctx, inv, name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
plan, settings, err := planFor(ctx, open, name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
@@ -698,7 +674,6 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
declared.Sequence = seq
|
||||
reportLeftOut(name, declared)
|
||||
sending = append(sending, readyNode{name, declared})
|
||||
}
|
||||
@@ -714,6 +689,9 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
|
||||
defer server.Close()
|
||||
|
||||
for _, s := range sending {
|
||||
if err := number(ctx, inv, &s); err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := s.declared.Body()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -721,7 +699,11 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
|
||||
if err := link.Declare(ctx, server.Bus(), ident, s.node, body, 15*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := recordSent(ctx, inv, s.node, body); err != nil {
|
||||
record, err := inv.NodeByName(ctx, s.node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf(" sent %s %d resource(s)\n", s.node, len(s.declared.Resources))
|
||||
@@ -962,38 +944,15 @@ func seatHolders(ctx context.Context, inv *inventory.Inventory) (map[string]brok
|
||||
}
|
||||
|
||||
// number gives one send the next sequence for its node (novox/hq 04-ISSUES/107).
|
||||
// allotting is allot over one inventory, in the shape composeEach takes.
|
||||
func allotting(ctx context.Context, inv *inventory.Inventory) func(node string) (int64, error) {
|
||||
return func(node string) (int64, error) { return allot(ctx, inv, node) }
|
||||
}
|
||||
|
||||
// allot takes the next sequence for a machine — the number its next declaration carries.
|
||||
func allot(ctx context.Context, inv *inventory.Inventory, node string) (int64, error) {
|
||||
record, err := inv.NodeByName(ctx, node)
|
||||
func number(ctx context.Context, inv *inventory.Inventory, s *readyNode) error {
|
||||
record, err := inv.NodeByName(ctx, s.node)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
return err
|
||||
}
|
||||
return inv.NextSequence(ctx, record.ID)
|
||||
}
|
||||
|
||||
// recordSent writes down what a machine was just sent, and returns the digest.
|
||||
//
|
||||
// **On a context that outlives the caller's** (novox/hq issue 204). The record is written after the
|
||||
// declaration is away, so a send that failed is never recorded as current — and a controller being
|
||||
// replaced mid-send had its context cancelled between the two, so the machine was told and the mesh
|
||||
// never wrote it down: status read "applied, current" over a machine that had just been sent
|
||||
// something else. What was sent was sent; the record of it must not depend on the sender living
|
||||
// another second. Bounded, so a store that is away does not hold a dying process open for ever.
|
||||
func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte) (string, error) {
|
||||
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
record, err := inv.NodeByName(kept, node)
|
||||
seq, err := inv.NextSequence(ctx, record.ID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return err
|
||||
}
|
||||
digest := digestOf(body)
|
||||
if err := inv.RecordSent(kept, record.ID, digest); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return digest, nil
|
||||
s.declared.Sequence = seq
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ import (
|
||||
// the wrong machine no longer refuses the whole node), applied one level up.
|
||||
func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) {
|
||||
sending, refusals := composeEach(
|
||||
[]string{"anchor", "home-server", "laptop"}, numbered(),
|
||||
[]string{"anchor", "home-server", "laptop"},
|
||||
func(node string) (sendable, error) {
|
||||
if node == "anchor" {
|
||||
return sendable{}, errors.New(`nothing provides "acme-ca", wanted by route-proxy`)
|
||||
@@ -43,7 +43,7 @@ func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) {
|
||||
// (novox/hq issue 127): it may have held something before, and only sending the empty
|
||||
// declaration tells it to drop what the mesh owned. It is never a refusal.
|
||||
func TestAnEmptyDeclarationIsSentSoTheNodeDropsWhatItHeld(t *testing.T) {
|
||||
sending, refusals := composeEach([]string{"spare"}, numbered(),
|
||||
sending, refusals := composeEach([]string{"spare"},
|
||||
func(string) (sendable, error) { return sendable{}, nil })
|
||||
if len(sending) != 1 || len(refusals) != 0 {
|
||||
t.Errorf("an empty declaration must be sent, not skipped or refused: %v / %v", sending, refusals)
|
||||
@@ -74,9 +74,3 @@ func TestASkippedMachineIsStillAnError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// numbered is an allotter for tests: one higher per call, as the inventory's is per machine.
|
||||
func numbered() func(string) (int64, error) {
|
||||
var n int64
|
||||
return func(string) (int64, error) { n++; return n, nil }
|
||||
}
|
||||
|
||||
@@ -36,29 +36,17 @@ func tiersOf(set []string, edges []inventory.Edge) [][]string {
|
||||
for _, m := range set {
|
||||
deps[m] = map[string]bool{}
|
||||
}
|
||||
// The build seat's holders follow the controller that defines their worker (EdgeWorkerOf,
|
||||
// novox/hq issue 206), so the built-by edge from that controller to such a holder yields: the
|
||||
// controller is built by whichever build machine is running, as the runtime image always was.
|
||||
worker := map[string]map[string]bool{}
|
||||
for _, e := range edges {
|
||||
if e.Kind == inventory.EdgeWorkerOf && in[e.From] && in[e.To] {
|
||||
if worker[e.To] == nil {
|
||||
worker[e.To] = map[string]bool{}
|
||||
}
|
||||
worker[e.To][e.From] = true
|
||||
}
|
||||
}
|
||||
for _, e := range edges {
|
||||
// A code dependency — B packages A's source — rebuilds B with A, in the same tier: B's
|
||||
// build needs nothing of A's first. The other kinds order: stands-on and declared after
|
||||
// the base is built, built-by after the build machine is built and running — except for
|
||||
// what the build machine itself stands on, and for the controller whose worker the build
|
||||
// machine binds. The runtime image is built by the builder and the builder is built on the
|
||||
// runtime image; the image comes first, built by the builder that is running.
|
||||
// what the build machine itself stands on. The runtime image is built by the builder and
|
||||
// the builder is built on the runtime image; the image comes first, built by the builder
|
||||
// that is running, which is the only one there could be.
|
||||
if !in[e.From] || !in[e.To] || e.From == e.To || e.Kind == inventory.EdgePackages {
|
||||
continue
|
||||
}
|
||||
if e.Kind == inventory.EdgeBuiltBy && (isBaseOf(e.From, e.To, edges, in) || worker[e.From][e.To]) {
|
||||
if e.Kind == inventory.EdgeBuiltBy && isBaseOf(e.From, e.To, edges, in) {
|
||||
continue
|
||||
}
|
||||
deps[e.From][e.To] = true
|
||||
@@ -134,10 +122,7 @@ func reachableFrom(moved []string, edges []inventory.Edge) []string {
|
||||
for grew := true; grew; {
|
||||
grew = false
|
||||
for _, e := range edges {
|
||||
// Built-by and worker-of order a plan; neither widens it. A new build machine changes
|
||||
// nothing it builds, and a new controller changes nothing about the holder it orders —
|
||||
// what packages the controller's source is already a code edge.
|
||||
if e.Kind == inventory.EdgeBuiltBy || e.Kind == inventory.EdgeWorkerOf {
|
||||
if e.Kind == inventory.EdgeBuiltBy {
|
||||
continue
|
||||
}
|
||||
if in[e.To] && !in[e.From] {
|
||||
|
||||
@@ -115,14 +115,3 @@ func TestACycleIsOneLastTierAndSaidSo(t *testing.T) {
|
||||
t.Fatalf("a cycle should be one tier of two, said: %v", tiers)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/211: a merge moving the toolchain and a bundle compiled in it builds the
|
||||
// bundle a tier after the toolchain, not beside it.
|
||||
func TestABundleIsPlannedAfterTheToolchainItIsCompiledIn(t *testing.T) {
|
||||
edges := []inventory.Edge{{From: "node-tools", To: "mesh-tools", Kind: inventory.EdgeStandsOn}}
|
||||
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"},
|
||||
[]string{"mesh-tools", "node-tools"}, edges)
|
||||
if len(p.Tiers) != 2 || p.Tiers[0][0] != "mesh-tools" || p.Tiers[1][0] != "node-tools" {
|
||||
t.Fatalf("the toolchain, then the bundle: %v", p.Tiers)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -346,9 +346,7 @@ func rolloutMint(ctx context.Context, again bool) error {
|
||||
}
|
||||
machines++
|
||||
|
||||
case broker.KindModule, broker.KindNodeTools:
|
||||
// The runtime is minted and delivered exactly as a module is (novox/hq ADR 0175): it is
|
||||
// issued as the module it stands for, to that module's `broker` secret.
|
||||
case broker.KindModule:
|
||||
if p.Module == "mesh-controller" {
|
||||
// The control plane is a module too, and its `broker` secret is the old bus's
|
||||
// credential it is still using while this runs. Writing the new bus's blob there
|
||||
@@ -367,7 +365,7 @@ func rolloutMint(ctx context.Context, again bool) error {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: busKindOf(p.Module), Node: p.Node, Module: p.Module})
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusModule, Node: p.Node, Module: p.Module})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -2,12 +2,13 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A node's own set failing to compose, and the mesh being unable to answer at all, are different
|
||||
// things, and only the first may be passed over when something is gathered across every machine
|
||||
// (novox/hq 04-ISSUES/152). These pin that distinction where the gatherers rely on it.
|
||||
// (novox/hq 04-ISSUES/152). These pin that distinction where the three gatherers rely on it.
|
||||
|
||||
func TestASetThatDoesNotComposeIsMarkedAsTheNodesOwnProblem(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
@@ -44,3 +45,55 @@ func TestAStoreThatCannotBeReadIsNotANodeThatDoesNotCompose(t *testing.T) {
|
||||
t.Fatalf("a question the mesh could not answer was read as a node that runs nothing: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOneIncoherentNodeDoesNotCostTheRestTheirNames(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
for _, m := range []string{one.Module, two.Module} {
|
||||
if _, err := open.inventory.Assign(t.Context(), "laptop", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// laptop cannot compose. That is laptop's problem and nobody else's: the roster is still
|
||||
// answerable, and anchor keeps whatever it serves.
|
||||
if _, err := routeNamesInTheMesh(t.Context(), open); err != nil {
|
||||
t.Fatalf("one node's broken set cost the whole mesh its roster: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARosterIsNeverReturnedWithNamesItCouldNotRead(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
|
||||
stopped, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
|
||||
names, err := routeNamesInTheMesh(stopped, open)
|
||||
if err == nil {
|
||||
t.Fatalf("a roster was composed from a store that could not be read: %v", names)
|
||||
}
|
||||
// The failure must be raised, not turned into an absence. A roster missing a machine's names
|
||||
// is indistinguishable, on every machine that receives it, from the operator withdrawing them —
|
||||
// and because the roster is part of every container's identity, it replaces all of them.
|
||||
if names != nil {
|
||||
t.Fatalf("a partial roster was returned beside the error: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
// Kept so the reason survives the next person reading it: the message the gatherer raises must say
|
||||
// which machine could not be read, or the operator is left with a mesh-wide failure and no name.
|
||||
func TestTheRaisedFailureNamesTheMachineItCouldNotRead(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
stopped, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
|
||||
_, err := routeNamesInTheMesh(stopped, open)
|
||||
if err == nil {
|
||||
t.Fatal("no failure was raised")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "cannot be read") {
|
||||
t.Fatalf("the failure does not say the mesh could not be read: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,10 +6,8 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
"os"
|
||||
"os/exec"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
@@ -68,14 +66,14 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
case "status":
|
||||
return []string{"status", "--json"}, nil
|
||||
case "nodes":
|
||||
return []string{"node", "list", "--json"}, nil
|
||||
return []string{"node", "list"}, nil
|
||||
case "node":
|
||||
if err := need("node"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{"node", "show", str("node")}, nil
|
||||
case "modules":
|
||||
return []string{"module", "list", "--json"}, nil
|
||||
return []string{"module", "list"}, nil
|
||||
case "seats":
|
||||
return []string{"seats", "--json"}, nil
|
||||
case "builds":
|
||||
@@ -146,6 +144,26 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
// Half of either shape: the command says its usage, which names both shapes, and that is
|
||||
// the answer the caller needs.
|
||||
return []string{"rotate"}, nil
|
||||
case "token":
|
||||
// `token issue` at a shell (novox/hq ADR 0169). Exactly one of node or new; the command
|
||||
// refuses both or neither in its own words.
|
||||
argv := []string{"token", "issue"}
|
||||
if n := str("node"); n != "" {
|
||||
argv = append(argv, "--node", n)
|
||||
}
|
||||
if n := str("new"); n != "" {
|
||||
argv = append(argv, "--new", n)
|
||||
}
|
||||
if k := str("overlay_key"); k != "" {
|
||||
argv = append(argv, "--overlay-key", k)
|
||||
}
|
||||
if d := str("for"); d != "" {
|
||||
argv = append(argv, "--for", d)
|
||||
}
|
||||
if str("adopted") == "true" {
|
||||
argv = append(argv, "--adopted")
|
||||
}
|
||||
return argv, nil
|
||||
case "settings":
|
||||
// `settings set|clear` at a shell (novox/hq issue 198). The values travel as an argument
|
||||
// because a tool has no file to hand the command; the command reads either.
|
||||
@@ -232,15 +250,13 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||
}
|
||||
|
||||
// seatToolHandlers are the handlers for every verb the mesh-controller seat declares, from the
|
||||
// store's row, so a verb the row does not carry is not served. A verb it carries that this binary
|
||||
// cannot run is named at start and answers the reason when called — never a refusal to serve, which
|
||||
// would take the whole control plane down for one word (novox/hq ADR 0185).
|
||||
func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
// store's row, so a verb the row does not carry is not served and a verb it carries that this binary
|
||||
// cannot run is said at start rather than at the first call.
|
||||
func seatToolHandlers() (map[string]link.ToolHandler, error) {
|
||||
seat, known := catalogue.SeatNamed(catalogue.ControllerSeatName)
|
||||
if !known {
|
||||
return nil, nil, fmt.Errorf("this mesh defines no %s seat", catalogue.ControllerSeatName)
|
||||
return nil, fmt.Errorf("this mesh defines no %s seat", catalogue.ControllerSeatName)
|
||||
}
|
||||
var behind []string
|
||||
handlers := map[string]link.ToolHandler{}
|
||||
for _, v := range seat.Serves {
|
||||
verb := v.Name
|
||||
@@ -251,27 +267,8 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
continue
|
||||
}
|
||||
if _, err := argvFor(verb, sampleArguments(v)); err != nil {
|
||||
// **A row ahead of this binary is not a reason to go silent.**
|
||||
//
|
||||
// The row is the store's and a control plane follows it (novox/hq ADR 0154), so a verb
|
||||
// this build does not know means the row was widened by a newer one — the ordinary
|
||||
// state of a roll-out, and of a push that put an older control plane back. Refusing to
|
||||
// serve at all made that transient fatal: on 2026-10-02 one unknown verb took the whole
|
||||
// mesh off the bus for ten minutes, and the way back was a human running the binary by
|
||||
// hand, because the thing that would have repaired it is the thing that was down
|
||||
// (novox/hq 04-ISSUES/201, ADR 0185).
|
||||
//
|
||||
// So the verbs this binary knows are served, and this one answers the reason instead of
|
||||
// nothing: a caller gets a sentence naming the fault, and everything else keeps working
|
||||
// — including the push that replaces this binary with the one whose verb it is.
|
||||
behind = append(behind, verb)
|
||||
reason := err
|
||||
handlers[verb] = func(context.Context, json.RawMessage) (any, error) {
|
||||
return nil, fmt.Errorf("%s is in this mesh's %s row and the control plane running "+
|
||||
"here cannot run it: %w. It is a verb of a newer build; this one is behind",
|
||||
verb, catalogue.ControllerSeatName, reason)
|
||||
}
|
||||
continue
|
||||
return nil, fmt.Errorf("the %s seat's row declares %q, which this control plane cannot run: %w",
|
||||
catalogue.ControllerSeatName, verb, err)
|
||||
}
|
||||
handlers[verb] = func(ctx context.Context, raw json.RawMessage) (any, error) {
|
||||
args := map[string]any{}
|
||||
@@ -287,7 +284,7 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
return runVerb(ctx, argv)
|
||||
}
|
||||
}
|
||||
return handlers, behind, nil
|
||||
return handlers, nil
|
||||
}
|
||||
|
||||
// seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the
|
||||
@@ -381,46 +378,3 @@ func splitCommandLine(line string) ([]string, error) {
|
||||
}
|
||||
return words, nil
|
||||
}
|
||||
|
||||
// seatAnnouncement is what the controller says it serves on the bus (novox/hq ADR 0197): the
|
||||
// mesh-controller seat, one endpoint per verb it answers, each with the seat's own description and
|
||||
// argument schema — the same facts `tools` answers from the records, as NATS's services format.
|
||||
func seatAnnouncement(handlers map[string]link.ToolHandler) micro.Info {
|
||||
about := map[string]catalogue.Verb{}
|
||||
for _, s := range catalogue.SeatsWithAProtocol() {
|
||||
if s.Name == catalogue.ControllerSeatName {
|
||||
for _, v := range s.Serves {
|
||||
about[v.Name] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
verbs := make([]string, 0, len(handlers))
|
||||
for verb := range handlers {
|
||||
verbs = append(verbs, verb)
|
||||
}
|
||||
sort.Strings(verbs)
|
||||
var endpoints []micro.EndpointInfo
|
||||
for _, verb := range verbs {
|
||||
schema, _ := json.Marshal(about[verb].Input)
|
||||
// The same shape every tool runtime announces in (node-tools' announce package): the name is
|
||||
// `<seat>__<verb>`, as the protocol's characters allow; the metadata is what identifies it.
|
||||
endpoints = append(endpoints, micro.EndpointInfo{
|
||||
Name: catalogue.ControllerSeatName + "__" + verb,
|
||||
Subject: link.SeatToolSubject(catalogue.ControllerSeatName, verb),
|
||||
QueueGroup: "seat." + catalogue.ControllerSeatName,
|
||||
Metadata: map[string]string{
|
||||
"kind": "seat", "module": catalogue.ControllerSeatName, "tool": verb,
|
||||
"seat": catalogue.ControllerSeatName, "scope": "mesh", "interchangeable": "false",
|
||||
"description": about[verb].Description, "schema": string(schema),
|
||||
},
|
||||
})
|
||||
}
|
||||
return micro.Info{
|
||||
ServiceIdentity: micro.ServiceIdentity{
|
||||
Name: catalogue.ControllerSeatName, ID: "controller", Version: "0.1.0",
|
||||
Metadata: map[string]string{"seat": catalogue.ControllerSeatName, "scope": "mesh"},
|
||||
},
|
||||
Description: "the mesh's own verbs, answered by the holder of the mesh-controller seat",
|
||||
Endpoints: endpoints,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -76,6 +73,14 @@ func TestRotateTakesAProvisionOrAnOwnSecret(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// `token` is `token issue` at a shell, with the machine's tunnel key (novox/hq ADR 0169).
|
||||
func TestTokenIssuesForAMachineAndItsTunnelKey(t *testing.T) {
|
||||
argv, err := argvFor("token", map[string]any{"new": "laptop", "overlay_key": "k", "for": "2h"})
|
||||
if err != nil || strings.Join(argv, " ") != "token issue --new laptop --overlay-key k --for 2h" {
|
||||
t.Fatalf("token: %v %v", argv, err)
|
||||
}
|
||||
}
|
||||
|
||||
// `settings` is `settings set|clear` at a shell, with the values passed inline (novox/hq issue 198).
|
||||
func TestSettingsSetsOrClearsALayer(t *testing.T) {
|
||||
argv, err := argvFor("settings", map[string]any{"module": "dnsmasq", "values": `{"a":1}`, "node": "ace"})
|
||||
@@ -133,13 +138,10 @@ func TestActsDoNotBlockTheCall(t *testing.T) {
|
||||
|
||||
// What `tools` answers is the seats' records, with each verb's schema.
|
||||
func TestToolsAnswersTheSeatsRecords(t *testing.T) {
|
||||
handlers, behind, err := seatToolHandlers()
|
||||
handlers, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(behind) != 0 {
|
||||
t.Fatalf("this build cannot run %v of its own seat's verbs", behind)
|
||||
}
|
||||
if len(handlers) != len(catalogue.ControllerVerbs) {
|
||||
t.Fatalf("%d handlers for %d verbs", len(handlers), len(catalogue.ControllerVerbs))
|
||||
}
|
||||
@@ -201,98 +203,3 @@ func TestCommandRunsTheLineAsGiven(t *testing.T) {
|
||||
t.Fatal("an unclosed quote was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// A verb in the row that this binary cannot run does not take the control plane off the bus: the
|
||||
// rest are served, the unknown one answers the reason, and the start-up names it (novox/hq ADR
|
||||
// 0185). One unknown word cost the mesh ten minutes of silence on 2026-10-02, recoverable only by
|
||||
// a person running the binary by hand — the push that would have repaired it needs the control
|
||||
// plane that was down.
|
||||
func TestARowAheadOfThisBuildIsServedAnyway(t *testing.T) {
|
||||
seat, known := catalogue.SeatNamed(catalogue.ControllerSeatName)
|
||||
if !known {
|
||||
t.Fatal("no controller seat")
|
||||
}
|
||||
// The row as a newer control plane would have written it: every verb this build knows, and one
|
||||
// it does not.
|
||||
widened := seat
|
||||
widened.Serves = append(append([]catalogue.Verb{}, seat.Serves...),
|
||||
catalogue.Verb{Name: "teleport", Description: "a verb from a build that does not exist yet"})
|
||||
rows := catalogue.DefaultSeats()
|
||||
for i := range rows {
|
||||
if rows[i].Name == catalogue.ControllerSeatName {
|
||||
rows[i] = widened
|
||||
}
|
||||
}
|
||||
catalogue.UseSeats(rows)
|
||||
t.Cleanup(func() { catalogue.UseSeats(catalogue.DefaultSeats()) })
|
||||
|
||||
handlers, behind, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatalf("a row with one unknown verb refused to serve at all: %v", err)
|
||||
}
|
||||
if len(behind) != 1 || behind[0] != "teleport" {
|
||||
t.Fatalf("the verbs this build cannot run were reported as %v", behind)
|
||||
}
|
||||
if len(handlers) != len(widened.Serves) {
|
||||
t.Fatalf("%d handlers for %d verbs in the row", len(handlers), len(widened.Serves))
|
||||
}
|
||||
for _, known := range []string{"status", "nodes", "push"} {
|
||||
if handlers[known] == nil {
|
||||
t.Errorf("%s is not served although this build knows it", known)
|
||||
}
|
||||
}
|
||||
_, err = handlers["teleport"](context.Background(), nil)
|
||||
if err == nil {
|
||||
t.Fatal("the unknown verb answered as though it had run")
|
||||
}
|
||||
for _, want := range []string{"teleport", "cannot run it", "behind"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the answer does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0195: the console's discovery reads the machines and the modules; they answer as JSON,
|
||||
// as status and seats do, so nothing parses a printed column.
|
||||
func TestTheNodesAndModulesVerbsAnswerAsJSON(t *testing.T) {
|
||||
for verb, want := range map[string]string{"nodes": "[node list --json]", "modules": "[module list --json]"} {
|
||||
argv, err := argvFor(verb, map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fmt.Sprint(argv) != want {
|
||||
t.Errorf("%s runs %v, want %s", verb, argv, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0197: the controller announces exactly the verbs it serves, each on the subject and
|
||||
// queue it serves it on, with the seat's own description and schema, in NATS's services format.
|
||||
func TestTheControllerAnnouncesTheVerbsItServes(t *testing.T) {
|
||||
handlers, _, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info := seatAnnouncement(handlers)
|
||||
if info.Name != catalogue.ControllerSeatName || info.ID == "" || info.Version == "" {
|
||||
t.Fatalf("the service is not named for the seat: %+v", info.ServiceIdentity)
|
||||
}
|
||||
if len(info.Endpoints) != len(handlers) {
|
||||
t.Fatalf("%d endpoints announced for %d verbs served", len(info.Endpoints), len(handlers))
|
||||
}
|
||||
for _, e := range info.Endpoints {
|
||||
verb := e.Metadata["tool"]
|
||||
if _, served := handlers[verb]; !served || e.Name != catalogue.ControllerSeatName+"__"+verb {
|
||||
t.Errorf("%s (%s) is announced and not served under that name", e.Name, verb)
|
||||
}
|
||||
if e.Metadata["kind"] != "seat" || e.Metadata["seat"] != catalogue.ControllerSeatName {
|
||||
t.Errorf("%s is not announced as the seat's verb: %v", e.Name, e.Metadata)
|
||||
}
|
||||
if e.Subject != link.SeatToolSubject(catalogue.ControllerSeatName, verb) || e.QueueGroup != "seat."+catalogue.ControllerSeatName {
|
||||
t.Errorf("%s is announced on %s/%s, not where it is served", e.Name, e.Subject, e.QueueGroup)
|
||||
}
|
||||
if e.Metadata["description"] == "" || e.Metadata["schema"] == "" || e.Metadata["scope"] != "mesh" {
|
||||
t.Errorf("%s is announced without its description, schema or scope: %v", e.Name, e.Metadata)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The holder of the build seat follows the controller that defines its worker (novox/hq issue 206).
|
||||
// On 2026-10-03 a plan put the build machine in tier 0 and the controller in tier 1; the new build
|
||||
// machine could not bind the worker the old controller had defined, and nothing could build the
|
||||
// controller that would have redefined it. The built-by edge from the controller to its build
|
||||
// machine yields to that order: the controller is built by whichever build machine is running.
|
||||
func TestTheBuildSeatsHolderFollowsTheControllerThatDefinesItsWorker(t *testing.T) {
|
||||
edges := []inventory.Edge{
|
||||
{From: "build-agent", To: "mesh-controller", Kind: inventory.EdgePackages},
|
||||
{From: "build-agent", To: "mesh-controller", Kind: inventory.EdgeWorkerOf},
|
||||
{From: "mesh-controller", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
||||
{From: "route-proxy", To: "mesh-controller", Kind: inventory.EdgePackages},
|
||||
{From: "route-proxy", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
||||
}
|
||||
set := reachableFrom([]string{"mesh-controller"}, edges)
|
||||
if len(set) != 3 {
|
||||
t.Fatalf("the controller, what packages it, and nothing more: %v", set)
|
||||
}
|
||||
tiers := tiersOf(set, edges)
|
||||
pos := map[string]int{}
|
||||
for i, tier := range tiers {
|
||||
for _, m := range tier {
|
||||
pos[m] = i
|
||||
}
|
||||
}
|
||||
if pos["mesh-controller"] != 0 {
|
||||
t.Fatalf("the controller first, built by the build machine that is running: %v", tiers)
|
||||
}
|
||||
if pos["build-agent"] <= pos["mesh-controller"] {
|
||||
t.Fatalf("the build machine after the controller that defines its worker: %v", tiers)
|
||||
}
|
||||
if pos["route-proxy"] <= pos["build-agent"] {
|
||||
t.Fatalf("what the build machine builds comes after it: %v", tiers)
|
||||
}
|
||||
if hasCycle(tiers, edges) {
|
||||
t.Fatalf("no cycle here: %v", tiers)
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A backend behind the proxy learns the client used TLS and which name it asked for, so the addresses
|
||||
// it writes into its own pages are the ones a client can use (2026-10-03: a forge's Go import tag
|
||||
// named an http clone URL, and Go refused the module path).
|
||||
func TestABackendIsToldTheRequestWasHTTPSAndForWhichName(t *testing.T) {
|
||||
var proto, host, fwdHost, fwdFor string
|
||||
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
proto, host, fwdHost, fwdFor = r.Header.Get("X-Forwarded-Proto"), r.Host, r.Header.Get("X-Forwarded-Host"), r.Header.Get("X-Forwarded-For")
|
||||
}))
|
||||
defer backend.Close()
|
||||
where, _ := url.Parse(backend.URL)
|
||||
req := httptest.NewRequest(http.MethodGet, "https://git.example.org/novox/mesh-sdk/go?go-get=1", nil)
|
||||
req.TLS = &tls.ConnectionState{}
|
||||
req.Host = "git.example.org"
|
||||
req.RemoteAddr = "192.0.2.7:51000"
|
||||
towards(where).ServeHTTP(httptest.NewRecorder(), req)
|
||||
if proto != "https" || fwdHost != "git.example.org" || host != "git.example.org" || fwdFor != "192.0.2.7" {
|
||||
t.Errorf("the backend was told proto=%q host=%q forwarded-host=%q for=%q", proto, host, fwdHost, fwdFor)
|
||||
}
|
||||
}
|
||||
@@ -273,7 +273,7 @@ func (t *table) set(routes map[string][]rule, public map[string]bool) {
|
||||
log.Printf("route %s points at %q, which is not a URL: %v", host, r.target, err)
|
||||
continue
|
||||
}
|
||||
r.to = towards(where)
|
||||
r.to = httputil.NewSingleHostReverseProxy(where)
|
||||
if r.insecure {
|
||||
r.to.Transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
|
||||
}
|
||||
@@ -1060,17 +1060,3 @@ func asPort(v any) (int, bool) {
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// towards proxies to one backend and tells it what the client asked: **X-Forwarded-Proto, -Host and
|
||||
// -For**, set from the request this proxy received. A backend that builds its own addresses — a forge
|
||||
// writing its clone URL into a page, a login redirect — otherwise sees the plain HTTP hop from this
|
||||
// proxy and writes `http://`, though every client reached it over TLS: Go refused the forge's module
|
||||
// path for exactly that on 2026-10-03, its import tag naming an http clone URL.
|
||||
// The standard library's NewSingleHostReverseProxy sets only X-Forwarded-For.
|
||||
func towards(where *url.URL) *httputil.ReverseProxy {
|
||||
return &httputil.ReverseProxy{Rewrite: func(pr *httputil.ProxyRequest) {
|
||||
pr.SetURL(where)
|
||||
pr.Out.Host = pr.In.Host
|
||||
pr.SetXForwarded()
|
||||
}}
|
||||
}
|
||||
|
||||
@@ -234,13 +234,3 @@ func admitsSubject(pattern, subject []string) bool {
|
||||
}
|
||||
return len(pattern) == len(subject)
|
||||
}
|
||||
|
||||
// The two packages name the runtime module separately — the broker's types stay free of the
|
||||
// catalogue's on purpose — so this is what holds them to one string. A rename that reached only one
|
||||
// side would compose a runtime principal for a module nobody assigns, silently, and leave the one
|
||||
// that is assigned with a module's own grants.
|
||||
func TestTheBrokerAndTheCatalogueAgreeOnTheRuntimeModule(t *testing.T) {
|
||||
if RuntimeModule != catalogue.RuntimeModule {
|
||||
t.Fatalf("the broker calls the runtime %q and the catalogue %q", RuntimeModule, catalogue.RuntimeModule)
|
||||
}
|
||||
}
|
||||
|
||||
+15
-16
@@ -144,18 +144,12 @@ func ConsumerFor(p Principal) (Consumer, bool) {
|
||||
}, true
|
||||
}
|
||||
|
||||
// HolderConsumerFor is the worker a seat's holders share on that seat's work queue.
|
||||
// HolderConsumerFor is the worker a seat's holder gets on that seat's work queue.
|
||||
//
|
||||
// **One worker for every holder, and each holder pulls one ask when it is idle** (novox/hq ADR
|
||||
// 0190). The seat is *authority* — who may be the telegram sender — and the worker is *delivery*,
|
||||
// kept separate so that relaxing one changes nothing about the other: a node-scoped seat has a
|
||||
// holder per machine, and all of them take from this one consumer, so the work is shared without
|
||||
// any holder knowing about the others. Pulled rather than pushed because a push consumer hands the
|
||||
// next ask to whichever subscriber the server picks, busy or not, and a pulled one is asked for by
|
||||
// a holder that has just become free. Which is also what ends the race issue 186 describes — asks
|
||||
// delivered behind the one being worked, expiring unacknowledged and dropped after the fifth
|
||||
// redelivery: nothing is delivered that nobody asked for. A long build keeps its own ask alive
|
||||
// (stillWorking); the ack wait is for a holder that died.
|
||||
// **A queue group even though the seat guarantees one holder.** The seat is *authority* — who may
|
||||
// be the telegram sender — and the queue group is *delivery*. Tie delivery to the seat and the
|
||||
// day somebody allows two holders for throughput, every message is processed twice with nothing
|
||||
// reporting it. Kept separate, relaxing one changes nothing about the other.
|
||||
func HolderConsumerFor(node, module string, seat DeclaredSeat) (Consumer, bool) {
|
||||
if len(seat.Accepts) == 0 {
|
||||
return Consumer{}, false
|
||||
@@ -164,13 +158,18 @@ func HolderConsumerFor(node, module string, seat DeclaredSeat) (Consumer, bool)
|
||||
Name: "SEAT_" + upperSnake(seat.Name) + "_worker",
|
||||
Stream: seatStreamName(seat.Name),
|
||||
Filters: []string{"mesh.seat." + seat.Name + ".accept.>"},
|
||||
Queue: "holders",
|
||||
AckWaitSeconds: 60,
|
||||
MaxDeliver: 5,
|
||||
// As many in flight as there are holders working, which pulling bounds by itself: a holder
|
||||
// fetches one and fetches again only after it acknowledged. The server's default stands.
|
||||
Why: fmt.Sprintf("%s on %s holds %s; every holder pulls one ask at a time from this worker "+
|
||||
"and acknowledges after the work is done, so a crash mid-work redelivers rather than "+
|
||||
"loses and an idle holder is the one that takes the next ask", module, node, seat.Name),
|
||||
// **One in flight.** A holder works one ask at a time, so the server hands it one at a
|
||||
// time: with the default of many, every ask behind the one being worked was delivered,
|
||||
// left unacknowledged for the length of the work, redelivered after the ack wait, and
|
||||
// after the fifth time dropped — on 2026-10-01 twenty-six of forty-three builds asked in
|
||||
// two minutes were never built, and the queue read as empty (novox/hq issue 186).
|
||||
MaxAckPending: 1,
|
||||
Why: fmt.Sprintf("%s on %s holds %s; it acknowledges after the work is done, so a "+
|
||||
"crash mid-work redelivers rather than loses; one in flight, so a queue of asks is a "+
|
||||
"queue and not a race against the ack wait", module, node, seat.Name),
|
||||
}, true
|
||||
}
|
||||
|
||||
|
||||
@@ -88,20 +88,15 @@ func TestAModuleThatConsumesNothingGetsNoConsumer(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The seat is authority and the worker is delivery (novox/hq ADR 0190): one worker per seat, shared
|
||||
// by every holder and pulled from, so a second holder takes the next ask rather than a copy of the
|
||||
// same one — which is what a queue group used to guard, and what pulling one durable gives outright.
|
||||
func TestAHoldersWorkerIsOneSharedByItsHolders(t *testing.T) {
|
||||
// The seat is authority and the queue group is delivery. Tie them together and the day somebody
|
||||
// allows two holders, every message is processed twice with nothing reporting it.
|
||||
func TestAHoldersWorkerUsesAQueueGroupAnyway(t *testing.T) {
|
||||
c, ok := HolderConsumerFor("one", "telegram", telegramSeat())
|
||||
if !ok {
|
||||
t.Fatal("the holder of a seat with inbound work got no worker")
|
||||
}
|
||||
two, _ := HolderConsumerFor("two", "telegram", telegramSeat())
|
||||
if c.Name != two.Name || c.Stream != two.Stream {
|
||||
t.Fatal("two holders got two workers, so each would process every ask")
|
||||
}
|
||||
if c.Push || c.Queue != "" {
|
||||
t.Fatal("the worker is pushed, so the server would hand an ask to a busy holder")
|
||||
if c.Queue == "" {
|
||||
t.Fatal("the worker is not in a queue group, so a second holder would double-process")
|
||||
}
|
||||
if c.Stream != "SEAT_TELEGRAM_SENDER" {
|
||||
t.Fatalf("the worker reads %q, not the seat's own stream", c.Stream)
|
||||
@@ -159,23 +154,15 @@ func TestANodesDeclarationConsumerIsWhatItsOwnGrantAllows(t *testing.T) {
|
||||
has(t, perms.Subscribe, c.Filters[0])
|
||||
}
|
||||
|
||||
// Every holder of a seat shares one worker and pulls from it (novox/hq ADR 0190): no queue group
|
||||
// and no delivery subject, because a push consumer hands the next ask to whichever subscriber the
|
||||
// server picks, busy or not; and no cap of one in flight, because pulling bounds the asks in flight
|
||||
// by the holders that are free — which is what ended the race of issue 186, where asks delivered
|
||||
// behind the one being worked expired and were dropped.
|
||||
func TestAHoldersWorkerIsPulledByEveryHolder(t *testing.T) {
|
||||
c, found := HolderConsumerFor("anchor", "build-agent", DeclaredSeat{Name: "node-build-agent", Accepts: []string{"build"}})
|
||||
// A holder works one ask at a time, so the server hands it one at a time (novox/hq issue 186):
|
||||
// asks queued behind the one being worked wait in the stream rather than being delivered,
|
||||
// left to expire and dropped after the fifth redelivery.
|
||||
func TestAHoldersWorkerTakesOneAskAtATime(t *testing.T) {
|
||||
c, found := HolderConsumerFor("anchor", "builder", DeclaredSeat{Name: "mesh-build-machine", Accepts: []string{"build"}})
|
||||
if !found {
|
||||
t.Fatal("a seat that accepts work has no worker")
|
||||
}
|
||||
if c.Queue != "" || c.Push {
|
||||
t.Fatalf("the worker is pushed (queue %q, push %v); a holder pulls when it is free", c.Queue, c.Push)
|
||||
}
|
||||
if c.MaxAckPending != 0 {
|
||||
t.Fatalf("the worker caps asks in flight at %d; pulling bounds them by the holders working", c.MaxAckPending)
|
||||
}
|
||||
if c.Name != "SEAT_NODE_BUILD_AGENT_worker" || c.Stream != "SEAT_NODE_BUILD_AGENT" {
|
||||
t.Fatalf("the worker is %s on %s; one per seat, shared by its holders", c.Name, c.Stream)
|
||||
if c.MaxAckPending != 1 {
|
||||
t.Fatalf("the worker may have %d asks in flight; one, so a queue is a queue", c.MaxAckPending)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -214,51 +214,6 @@ func (j *JetStream) EnsureConsumer(c Consumer) error {
|
||||
|
||||
switch have, err := j.js.ConsumerInfo(c.Stream, c.Name); {
|
||||
case err == nil:
|
||||
// **The controller owns the worker's shape, type included** (novox/hq issue 206). A holder
|
||||
// built for a pull worker cannot bind a push one — `cannot pull subscribe to push based
|
||||
// consumer` — and on 2026-10-03 the build machine rolled before the controller that would
|
||||
// have redefined its worker, restarted on that for an hour, and nothing could build the
|
||||
// controller that would have ended it. The server cannot change a consumer's type in place,
|
||||
// so one of the wrong type is re-made: on a work queue nothing is lost, because what was
|
||||
// acknowledged is gone from the stream and what was not is delivered again from the start.
|
||||
// On any other stream a re-made consumer would replay what this one acknowledged (issue
|
||||
// 156), so there it is said and left, and the person re-makes it knowing the cost.
|
||||
if havePush, wantPush := have.Config.DeliverSubject != "", want.DeliverSubject != ""; havePush != wantPush {
|
||||
shape := func(push bool) string {
|
||||
if push {
|
||||
return "push"
|
||||
}
|
||||
return "pull"
|
||||
}
|
||||
info, err := j.js.StreamInfo(c.Stream)
|
||||
if err != nil {
|
||||
return fmt.Errorf("asking about stream %s to re-make consumer %s: %w", c.Stream, c.Name, err)
|
||||
}
|
||||
if info.Config.Retention != nats.WorkQueuePolicy {
|
||||
// **A stream that keeps its history is re-made from now on, never from the start.**
|
||||
// Left for a hand, the hand re-makes it with the server's default — everything the
|
||||
// stream holds — which on 2026-10-03 replayed every build ask since 1 October and
|
||||
// re-registered nine modules from the past (novox/hq issue 207). What this consumer
|
||||
// had not yet acknowledged is lost with it, and said: on a history stream that is
|
||||
// the smaller cost, and the asks in flight are visible to whoever asked.
|
||||
j.note("consumer %s on %s changes from %s to %s delivery on a stream that keeps its history: "+
|
||||
"re-made to deliver from now on, so nothing this one acknowledged comes back (novox/hq issue "+
|
||||
"207); %d ask(s) it had not acknowledged are not carried over and must be asked again",
|
||||
c.Name, c.Stream, shape(havePush), shape(wantPush), have.NumPending+uint64(have.NumAckPending))
|
||||
want.DeliverPolicy = nats.DeliverNewPolicy
|
||||
} else {
|
||||
j.note("consumer %s on %s changes from %s to %s delivery: re-made where it left off, nothing "+
|
||||
"acknowledged comes back and nothing pending is lost (novox/hq issue 206); a holder bound to "+
|
||||
"the old shape binds again", c.Name, c.Stream, shape(havePush), shape(wantPush))
|
||||
}
|
||||
if err := j.js.DeleteConsumer(c.Stream, c.Name); err != nil {
|
||||
return fmt.Errorf("re-making consumer %s on %s as %s: %w", c.Name, c.Stream, shape(wantPush), err)
|
||||
}
|
||||
if _, err := j.js.AddConsumer(c.Stream, want); err != nil {
|
||||
return fmt.Errorf("re-making consumer %s on %s as %s: %w", c.Name, c.Stream, shape(wantPush), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
// Where an existing consumer starts is its history, not something an assertion may move:
|
||||
// the server refuses a changed deliver policy outright. Carried across, so asserting twice
|
||||
// is the no-op a restart depends on.
|
||||
|
||||
@@ -73,37 +73,3 @@ func TestAnAccountMayReadItsOwnMembershipAndNoOthers(t *testing.T) {
|
||||
has(t, perms.Publish, "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.anchor.postgres")
|
||||
hasNot(t, perms.Subscribe, "mesh.assignment.>")
|
||||
}
|
||||
|
||||
// The runtime arriving on a machine changes nothing about what each module is issued (to-be 38 WP2):
|
||||
// the memberships are composed as before and the runtime reads several of them. What the machine's
|
||||
// user list gains is one runtime principal, and loses nothing but the runtime module's own.
|
||||
func TestTheRuntimeArrivingLeavesEveryMembershipAsItWas(t *testing.T) {
|
||||
filter := Seat{Name: "node-packet-filter", Scope: "node", Serves: []string{"rules", "reload"}}
|
||||
three := []Declared{
|
||||
{Module: "nftables", Holds: []Seat{filter}, Serves: []string{"firewall_rules"}},
|
||||
{Module: "zsh", Serves: []string{"execute"}},
|
||||
{Module: "systemd", Serves: []string{"units"}},
|
||||
}
|
||||
before := Records{Nodes: []string{"anchor"}, Assigned: map[string][]Declared{"anchor": three}}
|
||||
after := Records{Nodes: []string{"anchor"}, Assigned: map[string][]Declared{
|
||||
"anchor": append(append([]Declared{}, three...), Declared{Module: RuntimeModule}),
|
||||
}}
|
||||
for _, d := range three {
|
||||
was := MembershipFor("anchor", d, PlacementsOf(before, nil))
|
||||
is := MembershipFor("anchor", d, PlacementsOf(after, nil))
|
||||
if !reflect.DeepEqual(was, is) {
|
||||
t.Errorf("%s's membership changed when the runtime arrived:\n%+v\n%+v", d.Module, was, is)
|
||||
}
|
||||
}
|
||||
users, err := Users(after)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kinds := map[Kind]int{}
|
||||
for _, p := range users {
|
||||
kinds[p.Kind]++
|
||||
}
|
||||
if kinds[KindNodeTools] != 1 || kinds[KindModule] != 3 || kinds[KindNode] != 1 || kinds[KindController] != 1 {
|
||||
t.Errorf("the machine's users are %v; one runtime, the three modules, the host and the controller", kinds)
|
||||
}
|
||||
}
|
||||
|
||||
+10
-165
@@ -34,20 +34,8 @@ const (
|
||||
// authority is a list of tools and nothing else — not control, not declarations, not builds,
|
||||
// and no ability to answer anything, because a person asks.
|
||||
KindPerson Kind = "person"
|
||||
// KindNodeTools is a machine's tool runtime (novox/hq ADR 0175, to-be 38): one process per
|
||||
// node, on the host side, serving every assigned module's tools and every held seat's verbs.
|
||||
// Its authority is the union of what the modules it carries would each have had for their
|
||||
// tools — and nothing of what they consume, because tools are what it runs, not reactions.
|
||||
KindNodeTools Kind = "node-tools"
|
||||
)
|
||||
|
||||
// RuntimeModule is the module that IS the node's tool runtime (novox/hq ADR 0175). Where it is
|
||||
// assigned, the mesh composes one runtime principal for the machine in place of that module's own,
|
||||
// and the per-module containers that served tools until then stop being the way tools reach a node.
|
||||
// Mirrored in the catalogue package, which the agreement test holds to the same string; one
|
||||
// constant, so a rename is one edit and the two packages cannot drift.
|
||||
const RuntimeModule = "node-tools"
|
||||
|
||||
// Seat is a role on the bus as a principal relates to it: the subjects it accepts, and those it
|
||||
// emits (novox/hq ADR 0118, design 29 §5).
|
||||
type Seat struct {
|
||||
@@ -86,13 +74,6 @@ type Principal struct {
|
||||
// a namespace no such module owns. Every service started and the graph stayed empty.
|
||||
Watches []Seat
|
||||
|
||||
// Carries are the modules whose tools this principal serves, for a KindNodeTools principal
|
||||
// (novox/hq ADR 0175): every module assigned to its node, as each declares itself. Its
|
||||
// serving authority is the union of theirs — each module's own tool namespace and each held
|
||||
// seat's verbs on this node — derived from the same declarations the modules' own principals
|
||||
// are, so the runtime can serve nothing a module could not have served for itself.
|
||||
Carries []Declared
|
||||
|
||||
// Invokes are the tools this principal may call, as `<module>.<tool>`; a single `*` is every
|
||||
// tool. A person's whole authority (design 25 §7), and a module's only if its manifest says so
|
||||
// (novox/hq ADR 0152) — the console's does, and nothing else's.
|
||||
@@ -110,13 +91,10 @@ type Principal struct {
|
||||
PasswordHash string
|
||||
}
|
||||
|
||||
// seatsTheControllerAsks are the roles the mesh's own flows submit work to. Named rather than
|
||||
// meshSeatsTheControllerUses are the roles the mesh's own flows submit work to. Named rather than
|
||||
// derived from the seat set: the controller is not a module and declares no `uses`, so its side of a
|
||||
// seat has to be stated, and a list is what makes "which roles does the mesh itself talk to" answerable.
|
||||
// Both build roles while the handover runs (novox/hq ADR 0190): the controller asks whichever has a
|
||||
// holder, and the retired one has one until build-agent replaces the builder. The second entry
|
||||
// goes with the retired seat row.
|
||||
var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"}
|
||||
var meshSeatsTheControllerUses = []string{"mesh-build-machine"}
|
||||
|
||||
// enrolmentPrefix is the space every enrolling node's user and inbox live under, so the one place the
|
||||
// controller may answer an enrolment is derived from the same constant the user is named from.
|
||||
@@ -134,10 +112,7 @@ func (p Principal) Username() string {
|
||||
switch p.Kind {
|
||||
case KindPerson:
|
||||
return "person." + p.Module
|
||||
case KindModule, KindNodeTools:
|
||||
// The runtime is named exactly as the module it stands for would have been: the mesh
|
||||
// issues its credential through the same path a module's takes (`module issue`), and
|
||||
// that path knows the node and the module, not the kind.
|
||||
case KindModule:
|
||||
return p.Node + "." + p.Module
|
||||
case KindNode:
|
||||
return "node." + p.Node
|
||||
@@ -211,9 +186,7 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// Work the mesh's own flows submit to a role, and the outcomes they wait on (ADR 0121). A
|
||||
// build is the one today: the controller asks, and reads the answer from the seat's event
|
||||
// like the catalogue does — which is why no holder needs to publish into anybody's inbox.
|
||||
// A node-scoped seat's work subject carries no node (novox/hq ADR 0190): the ask goes to
|
||||
// the role, and whichever machine holding it is idle takes it.
|
||||
for _, seat := range seatsTheControllerAsks {
|
||||
for _, seat := range meshSeatsTheControllerUses {
|
||||
pub = append(pub, "mesh.seat."+seat+".accept.>")
|
||||
}
|
||||
// **And what the mesh says it did** (novox/hq ADR 0134). The control plane states its own
|
||||
@@ -236,8 +209,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// which this package mirrors rather than reads, and a verb the seat does not declare is a
|
||||
// subject nothing publishes.
|
||||
sub = append(sub, "mesh.seat."+ControllerSeat+".tool.>")
|
||||
// And says so (novox/hq ADR 0197): it answers discovery for the seat it serves.
|
||||
sub = append(sub, announcing(ControllerSeat)...)
|
||||
|
||||
// The two events it reacts to, and its ack subject on the stream they arrive from
|
||||
// (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the
|
||||
@@ -273,9 +244,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
return Permissions{}, err
|
||||
}
|
||||
pub = append(pub, invoked...)
|
||||
// And may ask what answers (novox/hq ADR 0197): a question every service answers about
|
||||
// itself, its replies to the asker's own inbox.
|
||||
pub = append(pub, discovering()...)
|
||||
|
||||
case KindEnrolment:
|
||||
// A leaked token is useless for anything but enrolling: it cannot read a declaration, hear
|
||||
@@ -332,15 +300,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// away — no other principal may subscribe this namespace, and a caller's authority is
|
||||
// still granted per tool, by name, on the publish side.
|
||||
sub = append(sub, own+".tool.>")
|
||||
// It says what it serves (novox/hq ADR 0197): discovery for its own name and every seat it
|
||||
// holds a verb of, answered by the runtime that serves them.
|
||||
announced := []string{p.Module}
|
||||
for _, s := range p.Holds {
|
||||
if len(s.Serves) > 0 {
|
||||
announced = append(announced, s.Name)
|
||||
}
|
||||
}
|
||||
sub = append(sub, announcing(announced...)...)
|
||||
// Its own membership (ADR 0160): the one subject a runtime derives for itself, read
|
||||
// directly from the stream and followed live. Nothing else's.
|
||||
sub = append(sub, MembershipSubject(p.Node, p.Module))
|
||||
@@ -387,17 +346,13 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
|
||||
// 3. Seats it holds: full participation.
|
||||
for _, s := range p.Holds {
|
||||
// Taking work from the role's queue: the worker consumer every holder shares (asked
|
||||
// about, pulled from, acknowledged), on the seat's own stream (novox/hq ADR 0190). A
|
||||
// holder pulls — asks the consumer for its next message, answered on its own inbox —
|
||||
// so what it needs is MSG.NEXT on that worker and nothing delivered to it. The first
|
||||
// machine to take work over the new bus was refused the asking (2026-09-28).
|
||||
// Taking work from the role's queue: the worker consumer it binds (asked about,
|
||||
// delivered on, acknowledged), each on the seat's own stream. The first machine to
|
||||
// take work over the new bus was refused the asking (2026-09-28).
|
||||
worker := "SEAT_" + upperSnake(s.Name) + "_worker"
|
||||
stream := seatStreamName(s.Name)
|
||||
pub = append(pub,
|
||||
"$JS.API.CONSUMER.INFO."+stream+"."+worker,
|
||||
"$JS.API.CONSUMER.MSG.NEXT."+stream+"."+worker,
|
||||
"$JS.ACK."+stream+"."+worker+".>")
|
||||
sub = append(sub, "_DELIVER."+worker, "_DELIVER."+worker+".>")
|
||||
pub = append(pub, "$JS.API.CONSUMER.INFO."+stream+"."+worker, "$JS.ACK."+stream+"."+worker+".>")
|
||||
for _, a := range s.Accepts {
|
||||
sub = append(sub, seatSubject(s, "accept", a))
|
||||
}
|
||||
@@ -420,75 +375,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
pub = append(pub, seatToolSubject(s, t, "*"))
|
||||
}
|
||||
}
|
||||
|
||||
case KindNodeTools:
|
||||
// **One process serves what every module on the machine would have served for itself**
|
||||
// (novox/hq ADR 0175). Each carried module's whole tool namespace — the same grant that
|
||||
// module's own principal has, for the same reason: the tools a module serves are what its
|
||||
// code answers, and a list here would be a second copy of it. Each held seat's verbs on
|
||||
// this node, as the holder's own principal would be granted them.
|
||||
var serves []string
|
||||
for _, d := range p.Carries {
|
||||
if !safeSubject.MatchString(d.Module) {
|
||||
return Permissions{}, fmt.Errorf(
|
||||
"%q cannot be part of a subject: a permission is a subject pattern, and this would widen it", d.Module)
|
||||
}
|
||||
serves = append(serves, d.Module)
|
||||
for _, s := range d.Holds {
|
||||
serves = append(serves, s.Name)
|
||||
}
|
||||
own := "mesh.mod." + d.Module
|
||||
sub = append(sub, own+".tool.>")
|
||||
// A tool that emits an event is the module's code and emits under the module's name
|
||||
// (ADR 0042); the runtime carrying that code may publish what the module declared it
|
||||
// emits, and nothing it did not.
|
||||
for _, e := range d.Emits {
|
||||
pub = append(pub, own+".event."+e)
|
||||
}
|
||||
for _, s := range d.Holds {
|
||||
for _, t := range s.Serves {
|
||||
sub = append(sub, seatToolSubject(s, t, p.Node))
|
||||
}
|
||||
}
|
||||
}
|
||||
// Every assigned module's membership on this node (ADR 0160): one per module, read
|
||||
// directly from the stream and followed live. This node's and no other's — the one token
|
||||
// that varies is the module, so the pattern is the machine's own assignments.
|
||||
sub = append(sub, "mesh.assignment."+p.Node+".*")
|
||||
pub = append(pub, "$JS.API.DIRECT.GET."+AssignmentsStream+".mesh.assignment."+p.Node+".*")
|
||||
// And every tool on the mesh (ADR 0175, decision 5): any node may call any tool on any
|
||||
// node, as the console already could — the runtime is the console's serving mode.
|
||||
invoked, err := invokedSubjects([]string{"*"})
|
||||
if err != nil {
|
||||
return Permissions{}, err
|
||||
}
|
||||
pub = append(pub, invoked...)
|
||||
// It says what it serves and may ask what answers (novox/hq ADR 0197): the runtime answers
|
||||
// discovery for each module and seat it carries, and the console it is asks the bus.
|
||||
// One service per runtime process, named for the runtime: the bus lets a principal answer each
|
||||
// request once, so the runtime announces everything it carries under its own name.
|
||||
sub = append(sub, announcing(append([]string{RuntimeModule}, serves...)...)...)
|
||||
pub = append(pub, discovering()...)
|
||||
// **And it consumes for the modules it carries** (novox/hq ADR 0198, which changes ADR 0175's
|
||||
// "it consumes nothing"): a module's long-running code is a bundle this runtime launches, and
|
||||
// the runtime is its bus — it reads the module's own durable consumer and acknowledges what
|
||||
// the module's code took. Exactly the grants the module's own principal has for that consumer,
|
||||
// on its name and no other's: asking about it, pulling from it, acknowledging it. The
|
||||
// consumer is still the controller's to make, from the module's own principal.
|
||||
for _, d := range p.Carries {
|
||||
own := Principal{Kind: KindModule, Node: p.Node, Module: d.Module, Emits: d.Emits,
|
||||
Consumes: d.Consumes, Serves: d.Serves, Holds: d.Holds, Uses: d.Uses, Watches: d.Watches}
|
||||
if _, consumes := ConsumerFor(own); !consumes {
|
||||
continue
|
||||
}
|
||||
stream, durable := consumerStream(own), consumerDurable(own)
|
||||
pub = append(pub,
|
||||
"$JS.API.CONSUMER.INFO."+stream+"."+durable,
|
||||
"$JS.API.CONSUMER.MSG.NEXT."+stream+"."+durable,
|
||||
"$JS.ACK."+stream+"."+durable+".>")
|
||||
}
|
||||
sub = unique(sub)
|
||||
pub = unique(pub)
|
||||
}
|
||||
|
||||
if p.Kind == KindPerson {
|
||||
@@ -496,11 +382,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// consumer, because nothing is delivered to a person — they ask and are answered.
|
||||
sub = append(sub, p.inbox())
|
||||
}
|
||||
if p.Kind == KindNodeTools {
|
||||
// Its reply space, so the answers to what its tools call come back to it. No ack subject
|
||||
// for the same reason a person has none: nothing is delivered to it.
|
||||
sub = append(sub, p.inbox())
|
||||
}
|
||||
|
||||
if p.Kind == KindModule || p.Kind == KindNode || p.Kind == KindController {
|
||||
// Its own reply space, and nothing wider.
|
||||
@@ -522,7 +403,7 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// A module answers what it was asked — a tool call reaches it on its own namespace, so the
|
||||
// authority is bounded by having been asked — and so does the controller. A node and a
|
||||
// person are never asked anything, and are granted nothing here.
|
||||
AllowResponses: p.Kind == KindModule || p.Kind == KindController || p.Kind == KindNodeTools,
|
||||
AllowResponses: p.Kind == KindModule || p.Kind == KindController,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -744,20 +625,6 @@ func ComposeAccounts(principals []Principal) (string, error) {
|
||||
return b.String(), nil
|
||||
}
|
||||
|
||||
// unique is a sorted list with each subject once. Two carried modules holding seats with the same
|
||||
// verb, or the runtime module itself carried beside the others, would otherwise write a grant twice
|
||||
// — harmless to the server, and noise in a file that is read as the mesh's authority model.
|
||||
func unique(values []string) []string {
|
||||
sort.Strings(values)
|
||||
out := values[:0]
|
||||
for i, v := range values {
|
||||
if i == 0 || v != values[i-1] {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func quoted(values []string) string {
|
||||
if len(values) == 0 {
|
||||
return ""
|
||||
@@ -806,25 +673,3 @@ func invokedSubjects(invokes []string) ([]string, error) {
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// announcing is what a principal that serves tools subscribes to answer the NATS services
|
||||
// protocol's discovery (novox/hq ADR 0197): the questions asked of every service, and those asked of
|
||||
// each name it serves — its own and no other's, so it cannot answer for a service it is not.
|
||||
func announcing(names ...string) []string {
|
||||
out := []string{"$SRV.PING", "$SRV.INFO", "$SRV.STATS"}
|
||||
for _, n := range names {
|
||||
if !safeSubject.MatchString(n) {
|
||||
continue
|
||||
}
|
||||
for _, verb := range []string{"PING", "INFO", "STATS"} {
|
||||
out = append(out, "$SRV."+verb+"."+n, "$SRV."+verb+"."+n+".>")
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// discovering is what a principal publishes to ask what answers (novox/hq ADR 0197): the services
|
||||
// protocol's discovery requests, whose replies come to its own inbox.
|
||||
func discovering() []string {
|
||||
return []string{"$SRV.PING", "$SRV.PING.>", "$SRV.INFO", "$SRV.INFO.>"}
|
||||
}
|
||||
|
||||
@@ -233,9 +233,7 @@ func TestAPersonReachesNothingButTools(t *testing.T) {
|
||||
perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo",
|
||||
Invokes: []string{"*"}, PasswordHash: "x"})
|
||||
for _, p := range perms.Publish {
|
||||
// A tool call, or asking what answers (novox/hq ADR 0197) — a question every service
|
||||
// answers about itself, which claims nothing and controls nothing.
|
||||
if !strings.Contains(p, ".tool.") && !strings.HasPrefix(p, "$SRV.") {
|
||||
if !strings.Contains(p, ".tool.") {
|
||||
t.Errorf("a person may publish %q, which is not a tool call", p)
|
||||
}
|
||||
}
|
||||
@@ -373,109 +371,3 @@ func TestAModulePullsItsOwnConsumerAndNoOthers(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The runtime's authority is the union of what the modules it carries would have been granted for
|
||||
// their tools (novox/hq ADR 0175): every carried module's tool namespace, every held seat's verbs
|
||||
// on this node, every module's membership on this node, and a call to anything. Nothing it
|
||||
// consumes, because it reacts to nothing.
|
||||
func TestTheRuntimeServesTheUnionAndConsumesForItsModules(t *testing.T) {
|
||||
filter := Seat{Name: "node-packet-filter", Scope: "node", Serves: []string{"rules", "reload"}}
|
||||
p := Principal{Kind: KindNodeTools, Node: "anchor", Module: RuntimeModule, Carries: []Declared{
|
||||
{Module: "nftables", Holds: []Seat{filter}, Serves: []string{"firewall_rules"}},
|
||||
{Module: "zsh", Emits: []string{"shell.opened"}, Consumes: []string{"shop.order.placed"}},
|
||||
{Module: RuntimeModule},
|
||||
}}
|
||||
perms, err := PermissionsFor(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"mesh.mod.nftables.tool.>", "mesh.mod.zsh.tool.>", "mesh.mod." + RuntimeModule + ".tool.>",
|
||||
"mesh.seat.node-packet-filter.tool.rules.anchor", "mesh.seat.node-packet-filter.tool.reload.anchor",
|
||||
"mesh.assignment.anchor.*",
|
||||
"_INBOX.anchor." + RuntimeModule + ".>",
|
||||
} {
|
||||
if !contains(perms.Subscribe, want) {
|
||||
t.Errorf("the runtime may not subscribe %s: %v", want, perms.Subscribe)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{
|
||||
"mesh.mod.*.tool.>", "mesh.seat.*.tool.>",
|
||||
"$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.anchor.*",
|
||||
"mesh.mod.zsh.event.shell.opened",
|
||||
} {
|
||||
if !contains(perms.Publish, want) {
|
||||
t.Errorf("the runtime may not publish %s: %v", want, perms.Publish)
|
||||
}
|
||||
}
|
||||
// It reads the consumer of every carried module that consumes — that module's, by its name, as
|
||||
// the module's own principal could (novox/hq ADR 0198) — and of no module that consumes nothing.
|
||||
for _, want := range []string{
|
||||
"$JS.API.CONSUMER.INFO.EVENTS.anchor_zsh",
|
||||
"$JS.API.CONSUMER.MSG.NEXT.EVENTS.anchor_zsh",
|
||||
"$JS.ACK.EVENTS.anchor_zsh.>",
|
||||
} {
|
||||
if !contains(perms.Publish, want) {
|
||||
t.Errorf("the runtime may not read zsh's consumer: %s missing from %v", want, perms.Publish)
|
||||
}
|
||||
}
|
||||
for _, s := range perms.Publish {
|
||||
if (strings.HasPrefix(s, "$JS.ACK.") || strings.Contains(s, "CONSUMER")) && !strings.Contains(s, "anchor_zsh") {
|
||||
t.Errorf("the runtime was granted a consumer no carried module of it consumes on: %s", s)
|
||||
}
|
||||
}
|
||||
// It pulls; nothing is pushed to it, and it subscribes no event subject directly.
|
||||
for _, s := range perms.Subscribe {
|
||||
if strings.Contains(s, ".event.") || strings.HasPrefix(s, "_DELIVER.") {
|
||||
t.Errorf("the runtime was granted a delivery: %s", s)
|
||||
}
|
||||
}
|
||||
if !perms.AllowResponses {
|
||||
t.Error("the runtime answers what it is asked, and may not reply")
|
||||
}
|
||||
if _, needed := ConsumerFor(p); needed {
|
||||
t.Error("a consumer would be made for the runtime itself; it reads its modules' consumers, never one of its own")
|
||||
}
|
||||
// Each subject once in each list: the file is read as the mesh's authority model. One subject may
|
||||
// stand in both — the runtime answers discovery on `$SRV.INFO` and, as the console, asks it
|
||||
// (novox/hq ADR 0197) — because subscribing and publishing are two different grants.
|
||||
for _, list := range [][]string{perms.Subscribe, perms.Publish} {
|
||||
seen := map[string]bool{}
|
||||
for _, s := range list {
|
||||
if seen[s] {
|
||||
t.Errorf("%s is granted twice", s)
|
||||
}
|
||||
seen[s] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func contains(list []string, want string) bool {
|
||||
for _, s := range list {
|
||||
if s == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// A node-scoped seat's work is shared (novox/hq ADR 0190): its holder on any machine subscribes the
|
||||
// seat's one work subject, with no node in it, so holders on several machines read one queue. The
|
||||
// node token belongs to a seat's tools, which are asked of one machine (design 33 §4), not to its work.
|
||||
func TestANodeSeatsWorkSubjectCarriesNoNode(t *testing.T) {
|
||||
seat := Seat{Name: "node-build-agent", Scope: "node", Accepts: []string{"build"}, Serves: []string{"status"}}
|
||||
perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "anchor", Module: "build-agent", Holds: []Seat{seat}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
has(t, perms.Subscribe, "mesh.seat.node-build-agent.accept.build")
|
||||
hasNot(t, perms.Subscribe, "mesh.seat.node-build-agent.accept.build.anchor")
|
||||
// And its tools still carry the machine.
|
||||
has(t, perms.Subscribe, "mesh.seat.node-build-agent.tool.status.anchor")
|
||||
// The controller asks the role, not a machine.
|
||||
controller, err := PermissionsFor(Principal{Kind: KindController})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
has(t, controller.Publish, "mesh.seat.node-build-agent.accept.>")
|
||||
}
|
||||
|
||||
@@ -217,15 +217,10 @@ var ControllerFollows = []string{
|
||||
// A build's outcome, which is the build-machine role's own event now (ADR 0121) rather than a
|
||||
// message on the control branch. Same three audiences, one publish: whoever asked, this, and the
|
||||
// catalogue.
|
||||
seatEventSubject("node-build-agent", "built"),
|
||||
seatEventSubject("mesh-build-machine", "built"),
|
||||
// The forge's merges: what moved a source, so the mesh builds what that source produces
|
||||
// without anybody telling it (novox/hq 04-ISSUES/131). Appended, because the index is a name.
|
||||
moduleEventSubject("gitea", "pull.merged"),
|
||||
// The retired build role's outcome too, while the handover runs (novox/hq ADR 0190): the one
|
||||
// build machine keeps answering on its seat until build-agent replaces it, and the outcome that
|
||||
// registers build-agent itself comes from there. Appended, for the same reason as above; goes
|
||||
// with the retired seat row.
|
||||
seatEventSubject("mesh-build-machine", "built"),
|
||||
}
|
||||
|
||||
// moduleEventSubject is where one module's event lands. The same derivation PermissionsFor uses, so
|
||||
|
||||
+6
-6
@@ -24,8 +24,8 @@ accounts {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused"] }
|
||||
subscribe: { allow: ["$JS.API.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: {
|
||||
@@ -37,18 +37,18 @@ accounts {
|
||||
subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.declare"] }
|
||||
} }
|
||||
{ user: "one.telegram", password: "$2a$11$tttttttttttttttttttttt", permissions: {
|
||||
publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "$JS.ACK.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker.>", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram", "$JS.API.CONSUMER.INFO.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram", "$JS.API.CONSUMER.MSG.NEXT.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.one.telegram", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"] }
|
||||
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.telegram", "$SRV.INFO.telegram.>", "$SRV.PING", "$SRV.PING.telegram", "$SRV.PING.telegram.>", "$SRV.STATS", "$SRV.STATS.telegram", "$SRV.STATS.telegram.>", "_INBOX.one.telegram.>", "mesh.assignment.one.telegram", "mesh.mod.telegram.tool.>", "mesh.seat.telegram-sender.accept.send"] }
|
||||
publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "$JS.ACK.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker.>", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram", "$JS.API.CONSUMER.INFO.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.one.telegram", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"] }
|
||||
subscribe: { allow: ["_DELIVER.SEAT_TELEGRAM_SENDER_worker", "_DELIVER.SEAT_TELEGRAM_SENDER_worker.>", "_INBOX.one.telegram.>", "mesh.assignment.one.telegram", "mesh.mod.telegram.tool.>", "mesh.seat.telegram-sender.accept.send"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "two.audit", password: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa", permissions: {
|
||||
publish: { allow: ["$JS.ACK.EVENTS.two_audit.>", "$JS.API.CONSUMER.INFO.EVENTS.two_audit", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_audit", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.two.audit"] }
|
||||
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.audit", "$SRV.INFO.audit.>", "$SRV.PING", "$SRV.PING.audit", "$SRV.PING.audit.>", "$SRV.STATS", "$SRV.STATS.audit", "$SRV.STATS.audit.>", "_INBOX.two.audit.>", "mesh.assignment.two.audit", "mesh.mod.audit.tool.>", "mesh.mod.shop.event.order.placed"] }
|
||||
subscribe: { allow: ["_INBOX.two.audit.>", "mesh.assignment.two.audit", "mesh.mod.audit.tool.>", "mesh.mod.shop.event.order.placed"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "two.shop", password: "$2a$11$ssssssssssssssssssssss", permissions: {
|
||||
publish: { allow: ["$JS.ACK.EVENTS.two_shop.>", "$JS.API.CONSUMER.INFO.EVENTS.two_shop", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_shop", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.two.shop", "mesh.mod.shop.event.order.placed", "mesh.seat.telegram-sender.accept.send"] }
|
||||
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.shop", "$SRV.INFO.shop.>", "$SRV.PING", "$SRV.PING.shop", "$SRV.PING.shop.>", "$SRV.STATS", "$SRV.STATS.shop", "$SRV.STATS.shop.>", "_INBOX.two.shop.>", "mesh.assignment.two.shop", "mesh.mod.shop.tool.>"] }
|
||||
subscribe: { allow: ["_INBOX.two.shop.>", "mesh.assignment.two.shop", "mesh.mod.shop.tool.>"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
]
|
||||
|
||||
@@ -62,33 +62,13 @@ func Users(r Records) ([]Principal, error) {
|
||||
|
||||
for _, node := range sortedCopy(r.Nodes) {
|
||||
out = append(out, Principal{Kind: KindNode, Node: node})
|
||||
// **Where the runtime is assigned, the machine gets one runtime principal in place of the
|
||||
// runtime module's own** (novox/hq ADR 0175, to-be 38). It carries every module on the
|
||||
// node: its serving grants are the union of theirs. Every other module keeps its own
|
||||
// principal — a module still serving tools from its own container holds its own
|
||||
// credential until it moves, and the two serve side by side in the meantime.
|
||||
runtimeHere := false
|
||||
for _, d := range r.Assigned[node] {
|
||||
if d.Module == RuntimeModule {
|
||||
runtimeHere = true
|
||||
}
|
||||
}
|
||||
for _, d := range r.Assigned[node] {
|
||||
if runtimeHere && d.Module == RuntimeModule {
|
||||
continue
|
||||
}
|
||||
out = append(out, Principal{
|
||||
Kind: KindModule, Node: node, Module: d.Module,
|
||||
Emits: d.Emits, Consumes: d.Consumes, Serves: d.Serves,
|
||||
Holds: d.Holds, Uses: d.Uses, Watches: d.Watches, Invokes: d.Invokes,
|
||||
})
|
||||
}
|
||||
if runtimeHere {
|
||||
out = append(out, Principal{
|
||||
Kind: KindNodeTools, Node: node, Module: RuntimeModule,
|
||||
Carries: append([]Declared(nil), r.Assigned[node]...),
|
||||
})
|
||||
}
|
||||
}
|
||||
for _, node := range sortedCopy(r.Enrolling) {
|
||||
out = append(out, Principal{Kind: KindEnrolment, Node: node})
|
||||
|
||||
@@ -245,54 +245,3 @@ func TestAUserListIsComposedBeforeAnythingMovesOntoTheBus(t *testing.T) {
|
||||
t.Errorf("the composed list does not contain the machine running the bus")
|
||||
}
|
||||
}
|
||||
|
||||
// Where the runtime module is assigned, the machine gets one runtime principal in place of the
|
||||
// runtime module's own (novox/hq ADR 0175, to-be 38). Every other module keeps its own: a module
|
||||
// still serving tools from its own container holds its own credential until it moves.
|
||||
func TestTheRuntimeModuleBecomesTheMachinesRuntimePrincipal(t *testing.T) {
|
||||
r := someRecords()
|
||||
r.Assigned["one"] = append(r.Assigned["one"], Declared{Module: RuntimeModule})
|
||||
users, err := Users(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var runtime *Principal
|
||||
for i := range users {
|
||||
p := &users[i]
|
||||
if p.Node == "one" && p.Module == RuntimeModule {
|
||||
if p.Kind == KindModule {
|
||||
t.Fatalf("%s on one was composed as an ordinary module beside the runtime", RuntimeModule)
|
||||
}
|
||||
runtime = p
|
||||
}
|
||||
}
|
||||
if runtime == nil || runtime.Kind != KindNodeTools {
|
||||
t.Fatalf("one runs %s and got no runtime principal: %v", RuntimeModule, namesOf(t, r))
|
||||
}
|
||||
if runtime.Username() != "one."+RuntimeModule {
|
||||
t.Errorf("the runtime is named %q; `module issue` names it as the module it stands for", runtime.Username())
|
||||
}
|
||||
carried := map[string]bool{}
|
||||
for _, d := range runtime.Carries {
|
||||
carried[d.Module] = true
|
||||
}
|
||||
if !carried["telegram"] || !carried[RuntimeModule] {
|
||||
t.Errorf("the runtime carries %v; it carries every module on its node", carried)
|
||||
}
|
||||
// And the other node, where the runtime is not assigned, is exactly as before.
|
||||
for _, p := range users {
|
||||
if p.Node == "two" && p.Kind == KindNodeTools {
|
||||
t.Fatal("two runs no runtime and was given a runtime principal")
|
||||
}
|
||||
}
|
||||
// A module serving its own tools beside the runtime keeps its own principal.
|
||||
found := false
|
||||
for _, p := range users {
|
||||
if p.Kind == KindModule && p.Node == "one" && p.Module == "telegram" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Error("telegram lost its own principal when the runtime arrived on its node")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,167 +0,0 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// A seat's worker that changed from push to pull delivery strands a holder built for the new shape
|
||||
// (novox/hq issue 206): the server refuses a pull subscription on a push consumer, and the controller
|
||||
// that would redefine it was the build that nobody could take. The controller owns the worker's
|
||||
// shape, type included: on a work queue it re-makes one of the wrong type, losing nothing, and a
|
||||
// pull subscription then binds and takes what was pending.
|
||||
//
|
||||
// docker run -d --rm --name t -p 14231:4222 nats:2.10-alpine -js
|
||||
// MESH_TEST_NATS=nats://127.0.0.1:14231 go test ./internal/broker/ -run TestAWorker
|
||||
func TestAWorkerOfTheWrongTypeIsRemadeOnAWorkQueueAndAPullThenBinds(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
js, err := Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer js.Close()
|
||||
|
||||
const stream, worker, filter = "SEAT_T_SHELF", "SEAT_T_SHELF_worker", "mesh.seat.t-shelf.accept.>"
|
||||
_ = js.js.DeleteStream(stream)
|
||||
if _, err := js.js.AddStream(&nats.StreamConfig{
|
||||
Name: stream, Subjects: []string{filter}, Retention: nats.WorkQueuePolicy, Storage: nats.MemoryStorage,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = js.js.DeleteStream(stream) }()
|
||||
|
||||
// The worker as the previous controller defined it: push, in a queue group.
|
||||
if _, err := js.js.AddConsumer(stream, &nats.ConsumerConfig{
|
||||
Durable: worker, AckPolicy: nats.AckExplicitPolicy, AckWait: 60 * time.Second, MaxDeliver: 5,
|
||||
FilterSubject: filter, DeliverSubject: "_DELIVER." + worker, DeliverGroup: "holders",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, body := range []string{"one", "two", "three"} {
|
||||
if _, err := js.js.Publish("mesh.seat.t-shelf.accept.build", []byte(body)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The old holder took and acknowledged the first ask, then went away.
|
||||
old, err := js.js.QueueSubscribeSync(filter, "holders", nats.Bind(stream, worker))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := old.NextMsg(twoSeconds)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(m.Data) != "one" {
|
||||
t.Fatalf("the first ask is %q", m.Data)
|
||||
}
|
||||
if err := m.AckSync(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := old.Unsubscribe(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The new controller asserts the worker as the mesh derives it now: pull.
|
||||
if err := js.EnsureConsumer(Consumer{
|
||||
Name: worker, Stream: stream, Filters: []string{filter}, AckWaitSeconds: 60, MaxDeliver: 5,
|
||||
Why: "the test's worker",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
have, err := js.js.ConsumerInfo(stream, worker)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if have.Config.DeliverSubject != "" || have.Config.DeliverGroup != "" {
|
||||
t.Fatalf("the worker is still push: %+v", have.Config)
|
||||
}
|
||||
|
||||
// A holder built for the new shape binds, and takes exactly what the old one left.
|
||||
sub, err := js.js.PullSubscribe(filter, worker, nats.Bind(stream, worker), nats.ManualAck())
|
||||
if err != nil {
|
||||
t.Fatalf("a pull subscription does not bind the re-made worker: %v", err)
|
||||
}
|
||||
got, err := sub.Fetch(3, nats.MaxWait(twoSeconds))
|
||||
if err != nil && len(got) == 0 {
|
||||
t.Fatalf("nothing pending was delivered: %v", err)
|
||||
}
|
||||
var bodies []string
|
||||
for _, g := range got {
|
||||
bodies = append(bodies, string(g.Data))
|
||||
_ = g.Ack()
|
||||
}
|
||||
if len(bodies) != 2 || bodies[0] != "two" || bodies[1] != "three" {
|
||||
t.Fatalf("the pending asks after the acknowledged one, in order: %v", bodies)
|
||||
}
|
||||
|
||||
// Asserted again, the pull worker is the no-op a restart depends on.
|
||||
if err := js.EnsureConsumer(Consumer{
|
||||
Name: worker, Stream: stream, Filters: []string{filter}, AckWaitSeconds: 60, MaxDeliver: 5,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// On a stream that keeps its history, a worker of the wrong type is re-made to deliver from now on:
|
||||
// re-making it from the start would replay what it acknowledged (novox/hq issue 156), and leaving it
|
||||
// for a hand re-made it exactly that way on 2026-10-03 (issue 207).
|
||||
func TestAWorkerOfTheWrongTypeOnAHistoryStreamIsRemadeFromNowOn(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
js, err := Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer js.Close()
|
||||
const stream, worker, filter = "EVENTS_T", "EVENTS_T_reader", "mesh.t.event.>"
|
||||
_ = js.js.DeleteStream(stream)
|
||||
if _, err := js.js.AddStream(&nats.StreamConfig{Name: stream, Subjects: []string{filter}, Storage: nats.MemoryStorage}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = js.js.DeleteStream(stream) }()
|
||||
if _, err := js.js.AddConsumer(stream, &nats.ConsumerConfig{
|
||||
Durable: worker, AckPolicy: nats.AckExplicitPolicy, AckWait: 60 * time.Second,
|
||||
FilterSubject: filter, DeliverSubject: "_DELIVER." + worker,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// History the old consumer would have acknowledged long ago, and must not come back.
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, err := js.js.Publish("mesh.t.event.old", []byte("old")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := js.EnsureConsumer(Consumer{Name: worker, Stream: stream, Filters: []string{filter}, AckWaitSeconds: 60}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
have, err := js.js.ConsumerInfo(stream, worker)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if have.Config.DeliverSubject != "" {
|
||||
t.Fatal("a history stream's consumer of the wrong type was left as it was")
|
||||
}
|
||||
if have.Config.DeliverPolicy != nats.DeliverNewPolicy || have.NumPending != 0 {
|
||||
t.Fatalf("re-made consumer delivers %v with %d pending; it must deliver from now on with nothing of the past", have.Config.DeliverPolicy, have.NumPending)
|
||||
}
|
||||
// And what arrives from now on is delivered.
|
||||
if _, err := js.js.Publish("mesh.t.event.new", []byte("new")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sub, err := js.js.PullSubscribe(filter, worker, nats.Bind(stream, worker))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := sub.Fetch(1, nats.MaxWait(3*time.Second))
|
||||
if err != nil || len(got) != 1 || string(got[0].Data) != "new" {
|
||||
t.Fatalf("the re-made consumer delivered %v, %v; want the one new message", got, err)
|
||||
}
|
||||
}
|
||||
@@ -587,12 +587,6 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
if err != nil {
|
||||
return catalogue.Built{}, fmt.Errorf("%s: compiling %s failed: %w", module, a.Name, err)
|
||||
}
|
||||
// **Every entrypoint the runtime may serve is executable** (novox/hq ADR 0193). The runtime
|
||||
// knows no language; for one that runs through an interpreter the build writes the launcher.
|
||||
launchers, err := writeLaunchers(compiled, chain, a)
|
||||
if err != nil {
|
||||
return catalogue.Built{}, fmt.Errorf("%s: writing %s's launchers failed: %w", module, a.Name, err)
|
||||
}
|
||||
say("bundle", "compiled, packing")
|
||||
body, err := pack(compiled)
|
||||
if err != nil {
|
||||
@@ -604,7 +598,7 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
if err != nil {
|
||||
return catalogue.Built{}, err
|
||||
}
|
||||
return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: where, Digest: digest, Launchers: launchers}, nil
|
||||
return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: where, Digest: digest}, nil
|
||||
|
||||
case catalogue.ArtifactPackage:
|
||||
// Built and published on a public base, to the mesh's package registry, by version
|
||||
@@ -974,26 +968,6 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain,
|
||||
if _, err := run(ctx, tree, "docker", invocation...); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if chain.Dependencies != "" {
|
||||
// **What the bundle runs with, from the image it was compiled in** (Toolchain.Dependencies).
|
||||
// A second run in the same image rather than a shell wrapped around the compiler: the
|
||||
// compile line stays a plain command a reader can run by hand, and the copy is one more
|
||||
// plain command beside it. Refused by name when the image carries no such directory — an
|
||||
// older toolchain image — because a bundle packed without its dependencies starts nowhere
|
||||
// and says so three layers away from here.
|
||||
copying := []string{
|
||||
"run", "--rm",
|
||||
"--volume", tree + ":" + within,
|
||||
"--workdir", within,
|
||||
base,
|
||||
"sh", "-c",
|
||||
`test -d "$1" || { echo "the toolchain image carries no $1: it predates the mesh shipping a bundle's dependencies, rebuild $2 first" >&2; exit 1; }; cp -a "$1/." "$3/"`,
|
||||
"dependencies", chain.Dependencies, chain.Base, out,
|
||||
}
|
||||
if _, err := run(ctx, tree, "docker", copying...); err != nil {
|
||||
return "", fmt.Errorf("copying the %s dependencies a bundle runs with: %w", chain.Language, err)
|
||||
}
|
||||
}
|
||||
return filepath.Join(tree, out), nil
|
||||
}
|
||||
|
||||
@@ -1171,9 +1145,6 @@ func readBy(manifest catalogue.Manifest) []catalogue.ArtifactContext {
|
||||
// binaryName is what a compiled bundle's executable is called: what the artifact says, or the name of
|
||||
// the package it is built from, which is what a compiler would have chosen anyway.
|
||||
func binaryName(a catalogue.Artifact) string {
|
||||
if name := catalogue.BinaryOf(a); name != "" {
|
||||
return name
|
||||
}
|
||||
if name := strings.TrimSpace(a.Binary); name != "" {
|
||||
return name
|
||||
}
|
||||
@@ -1182,45 +1153,3 @@ func binaryName(a catalogue.Artifact) string {
|
||||
}
|
||||
return a.Name
|
||||
}
|
||||
|
||||
// launcherSuffix is what a TypeScript entrypoint's launcher is called beside it: index.js is
|
||||
// started as index.serve.mjs (novox/hq ADR 0193). An ES module by its own extension, whatever the
|
||||
// bundle's package.json says.
|
||||
const launcherSuffix = ".serve.mjs"
|
||||
|
||||
// writeLaunchers writes, beside every entrypoint of a TypeScript bundle, an executable that
|
||||
// imports the entrypoint and serves what it registered over MCP on stdio — through the bundle's
|
||||
// own copy of the SDK, so registering and serving meet in one registry (novox/hq ADR 0193). Its
|
||||
// answer is each entrypoint's launcher, by entrypoint, relative to the bundle's root; nothing for
|
||||
// a language whose build is already executable.
|
||||
func writeLaunchers(root string, chain Toolchain, a catalogue.Artifact) (map[string]string, error) {
|
||||
if chain.Language != "typescript" {
|
||||
return nil, nil
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, entry := range a.Entrypoints {
|
||||
if !strings.HasSuffix(entry, ".js") {
|
||||
continue
|
||||
}
|
||||
launcher := strings.TrimSuffix(entry, ".js") + launcherSuffix
|
||||
body := "#!/usr/bin/env node\n" +
|
||||
"// Written by the mesh's builder (novox/hq ADR 0193): serve what " + entry + " registers,\n" +
|
||||
"// over MCP on stdio, as the module the node's runtime names in MESH_SERVED_MODULE.\n" +
|
||||
"import { serveRegisteredOverStdio } from \"@novox/mesh-sdk/stdio\";\n" +
|
||||
"await import(\"./" + filepath.Base(entry) + "\");\n" +
|
||||
"await serveRegisteredOverStdio();\n"
|
||||
path := filepath.Join(root, filepath.FromSlash(launcher))
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(body), 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// WriteFile honours the umask; the mode a machine unpacks is the one packed, so it is set.
|
||||
if err := os.Chmod(path, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[entry] = launcher
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -82,41 +82,6 @@ func TestABundleIsCompiledAndPackedWithNoDockerfile(t *testing.T) {
|
||||
if !strings.HasPrefix(digest, "sha256:") {
|
||||
t.Fatalf("the bundle was not pinned: %v", got.Manifest.Resources[0])
|
||||
}
|
||||
|
||||
// **And what it runs with, from the image it was compiled in** (novox/hq to-be 38 WP3). A
|
||||
// second run in the same toolchain image copies the toolchain's runtime directory — the
|
||||
// `"type": "module"` package.json and the pruned node_modules — into the output's root, and
|
||||
// refuses by name when the image carries none rather than packing a bundle that starts nowhere.
|
||||
var copied string
|
||||
for _, line := range r.ran {
|
||||
if strings.HasPrefix(line, "docker run") && strings.Contains(line, "/app/runtime") {
|
||||
copied = line
|
||||
}
|
||||
}
|
||||
if copied == "" {
|
||||
t.Fatalf("the bundle's dependencies were not copied in after the compile:\n%s", strings.Join(r.ran, "\n"))
|
||||
}
|
||||
if !strings.Contains(copied, "mesh-tools/build@sha256:") || !strings.Contains(copied, "predates") ||
|
||||
!strings.Contains(copied, Out("code")) {
|
||||
t.Fatalf("the copy does not run in the same toolchain, refuse an older image by name, or land in the artifact's output: %s", copied)
|
||||
}
|
||||
if strings.Index(strings.Join(r.ran, "\n"), "--outDir") > strings.Index(strings.Join(r.ran, "\n"), "/app/runtime") {
|
||||
t.Fatal("the dependencies were copied before the compile wrote its output")
|
||||
}
|
||||
}
|
||||
|
||||
// A language whose bundle carries its own dependencies copies nothing in: a Go binary is static.
|
||||
func TestOnlyALanguageWithARuntimeDirectoryCopiesDependenciesIn(t *testing.T) {
|
||||
ts, _ := ToolchainFor("typescript")
|
||||
if ts.Dependencies != "/app/runtime" {
|
||||
t.Fatalf("typescript bundles run with %q", ts.Dependencies)
|
||||
}
|
||||
for _, language := range []string{"go", "python"} {
|
||||
chain, _ := ToolchainFor(language)
|
||||
if chain.Dependencies != "" {
|
||||
t.Fatalf("%s copies %q into every bundle, and its bundles carry their own", language, chain.Dependencies)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Refused before anything is built, naming what to build first.** A base the mesh has not built
|
||||
@@ -180,13 +145,9 @@ func TestTwoBundlesInOneModuleArePackedSeparately(t *testing.T) {
|
||||
t.Fatalf("a module with two bundles did not build: %v", err)
|
||||
}
|
||||
|
||||
// Compiled into two different places. Only the compile lines: the copy of each bundle's
|
||||
// dependencies names the same directory again, deliberately.
|
||||
// Compiled into two different places.
|
||||
var outputs []string
|
||||
for _, line := range r.ran {
|
||||
if !strings.Contains(line, "--outDir") {
|
||||
continue
|
||||
}
|
||||
for _, part := range strings.Fields(line) {
|
||||
if strings.HasPrefix(part, ".mesh-build/") {
|
||||
outputs = append(outputs, part)
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0193: every entrypoint the runtime may serve is executable, and the runtime knows no
|
||||
// language — so a TypeScript bundle carries a launcher beside each entrypoint.
|
||||
func TestATypeScriptBundleCarriesAnExecutableLauncherBesideEachEntrypoint(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
chain, err := ToolchainFor("typescript")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := writeLaunchers(root, chain, catalogue.Artifact{Name: "tools", Kind: catalogue.ArtifactBundle,
|
||||
Language: "typescript", Entrypoints: []string{"tools/index.js", "index.js"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got["tools/index.js"] != "tools/index.serve.mjs" || got["index.js"] != "index.serve.mjs" {
|
||||
t.Fatalf("launchers: %v", got)
|
||||
}
|
||||
path := filepath.Join(root, "tools", "index.serve.mjs")
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Mode().Perm() != 0o755 {
|
||||
t.Errorf("the launcher is %v, not executable 0755", info.Mode().Perm())
|
||||
}
|
||||
body, _ := os.ReadFile(path)
|
||||
for _, want := range []string{"#!/usr/bin/env node\n", `from "@novox/mesh-sdk/stdio"`, `await import("./index.js")`, "serveRegisteredOverStdio()"} {
|
||||
if !strings.Contains(string(body), want) {
|
||||
t.Errorf("the launcher lacks %q:\n%s", want, body)
|
||||
}
|
||||
}
|
||||
|
||||
// A compiled language's build is executable already: no launcher.
|
||||
goChain, _ := ToolchainFor("go")
|
||||
none, err := writeLaunchers(t.TempDir(), goChain, catalogue.Artifact{Name: "tools", Kind: catalogue.ArtifactBundle, Language: "go"})
|
||||
if err != nil || len(none) != 0 {
|
||||
t.Errorf("a Go bundle was given launchers: %v %v", none, err)
|
||||
}
|
||||
}
|
||||
@@ -57,23 +57,6 @@ type Toolchain struct {
|
||||
// carrying its debug info. The mistake was believing a comment rather than reading the file it
|
||||
// produced (novox/hq 04-ISSUES/161).
|
||||
LinkerFlags []string
|
||||
// Dependencies is a directory inside the toolchain image whose contents a bundle in this
|
||||
// language runs with, copied whole into the compiled output's root after the compile.
|
||||
//
|
||||
// **A bundle that compiles is not yet a bundle that runs.** The compiler resolves `import
|
||||
// "nats"` from the toolchain image's own node_modules and the pack takes only what the compiler
|
||||
// wrote, so what a machine unpacked could not find a single dependency — and no TypeScript bundle
|
||||
// had ever run live to show it (novox/hq to-be 38 WP3). For TypeScript the directory holds a
|
||||
// `package.json` saying `"type": "module"` — Node reads a bare `.js` as CommonJS otherwise, so a
|
||||
// bundle with its dependencies and without that line still fails to start — and the pruned,
|
||||
// production-only node_modules the runtime itself ships with: the SDK's and the runtime's
|
||||
// dependencies, and nothing module-specific yet (novox/hq ADR 0188 §5: a skeleton; a module's
|
||||
// own npm dependencies are a later step). Empty for a language whose bundle carries its own —
|
||||
// a Go binary is static, a Python bundle is installed with its dependencies.
|
||||
//
|
||||
// A toolchain image without the directory fails the build by name rather than packing a bundle
|
||||
// that starts nowhere: the image predates this and must be rebuilt first.
|
||||
Dependencies string
|
||||
// SystemStamp is the variable this language's linker fills with the artifact's declared system,
|
||||
// for a language whose binaries are pinned to one at link time (novox/hq ADR 0005).
|
||||
//
|
||||
@@ -124,21 +107,14 @@ var toolchains = []Toolchain{
|
||||
// symlinks to a launcher that requires its library relatively — and the base image's own
|
||||
// assembly resolves them away, leaving a launcher whose relative require points nowhere.
|
||||
// Every module's hand-written Dockerfile had to know this. Now none of them does.
|
||||
// **Rooted at the module, so an entrypoint lands where it is named.** Without a root the
|
||||
// compiler takes the common directory of the files it is given: a module compiling only
|
||||
// `tools/index.ts` had its output at `index.js`, and the entrypoint it declared —
|
||||
// `tools/index.js`, "named as it will be found" — named a file the bundle did not
|
||||
// contain. The runtime that loads bundles by their declared entrypoints (novox/hq ADR
|
||||
// 0175) is what made this visible.
|
||||
Compile: []string{
|
||||
"node", "/app/node_modules/typescript/bin/tsc",
|
||||
"--module", "NodeNext", "--moduleResolution", "NodeNext",
|
||||
"--target", "ES2022", "--rootDir", ".",
|
||||
"--target", "ES2022",
|
||||
},
|
||||
OutputFlag: "--outDir",
|
||||
Unit: UnitSources,
|
||||
SourceExt: ".ts",
|
||||
Dependencies: "/app/runtime",
|
||||
OutputFlag: "--outDir",
|
||||
Unit: UnitSources,
|
||||
SourceExt: ".ts",
|
||||
},
|
||||
{
|
||||
Language: "go",
|
||||
|
||||
+1
-122
@@ -2,7 +2,6 @@ package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
@@ -29,9 +28,6 @@ type Built struct {
|
||||
Reference string
|
||||
// Digest is "sha256:<hex>", for an archive. An image reference already ends in one.
|
||||
Digest string
|
||||
// Launchers are, for a bundle in an interpreted language, the executable the build wrote beside
|
||||
// each entrypoint, by entrypoint (novox/hq ADR 0193): what the node's runtime starts to serve it.
|
||||
Launchers map[string]string
|
||||
}
|
||||
|
||||
// Resolve fills a manifest's resources in from what was built.
|
||||
@@ -71,37 +67,6 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) {
|
||||
out := m
|
||||
out.Build = nil
|
||||
out.Resources = nil
|
||||
// What the build compiled, kept on the resolved manifest (novox/hq ADR 0175): a tools bundle is
|
||||
// named by no resource of the module's own — the node's runtime loads it — so this is the only
|
||||
// place the mesh would otherwise not have it. In artifact order, so two resolutions of one
|
||||
// build compare equal.
|
||||
out.Bundles = nil
|
||||
if m.Build != nil {
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Kind != ArtifactBundle {
|
||||
continue
|
||||
}
|
||||
made := by[a.Name]
|
||||
// What the runtime loads: what the artifact said, else every entrypoint of a module
|
||||
// that declares tools, else nothing (the field's own rule; see Artifact.Loads).
|
||||
loads := append([]string(nil), a.Loads...)
|
||||
if a.Loads == nil && len(m.Tools) > 0 {
|
||||
loads = append([]string(nil), a.Entrypoints...)
|
||||
}
|
||||
// **Kept, never routed** (ADR 0155): the builder publishes to the store at the address
|
||||
// it reached it by, and a manifest carrying that address names an installation —
|
||||
// registration refused node-tools for exactly this on 2026-10-02. The build record
|
||||
// already keeps the store-relative form; the resolved manifest keeps the same, and
|
||||
// composition routes it through the store a machine reaches (Routed).
|
||||
out.Bundles = append(out.Bundles, Bundle{
|
||||
Name: a.Name, Source: Recorded(made.Reference), Digest: made.Digest,
|
||||
Language: a.Language, Entrypoints: append([]string(nil), a.Entrypoints...),
|
||||
Loads: loads, Env: copyWords(a.Env), Launchers: copyWords(made.Launchers),
|
||||
Binary: BinaryOf(a),
|
||||
})
|
||||
}
|
||||
sort.Slice(out.Bundles, func(i, j int) bool { return out.Bundles[i].Name < out.Bundles[j].Name })
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
named, _ := r["artifact"].(string)
|
||||
if named == "" {
|
||||
@@ -145,8 +110,7 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) {
|
||||
// The same on the wire: both are bytes fetched by digest and unpacked. They differ in
|
||||
// how they were made — one packed as it stood, the other compiled first — and a
|
||||
// machine has no reason to care which.
|
||||
// Kept, not routed, for the reason the bundles above are (ADR 0155).
|
||||
filled["source"] = Recorded(artifact.Reference)
|
||||
filled["source"] = artifact.Reference
|
||||
filled["digest"] = artifact.Digest
|
||||
// **And `${version}`, so a resource can name a place that is this build's alone**
|
||||
// (novox/hq ADR 0141, 04-ISSUES/142). A component is unpacked into a directory named
|
||||
@@ -208,12 +172,6 @@ func (b *Build) problems(module string) []string {
|
||||
// A bundle's source is the module's own directory by definition, and what it needs to say
|
||||
// is which compiler — because the mesh chooses that, and cannot choose for a module that
|
||||
// has not said.
|
||||
if len(a.Env) > 0 && a.Kind != ArtifactBundle {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q is a %q and says what it is given (env). Only a bundle the node's runtime "+
|
||||
"serves is given words (novox/hq ADR 0192); a container says its own environment",
|
||||
module, a.Name, a.Kind))
|
||||
}
|
||||
if a.Kind == ArtifactBundle || a.Kind == ArtifactPackage {
|
||||
// **Except for a language that compiles to a binary, where it names which one**
|
||||
// (novox/hq 04-ISSUES/142). A bundle in an interpreted language is the module's own
|
||||
@@ -233,21 +191,6 @@ func (b *Build) problems(module string) []string {
|
||||
"%s: %q is a bundle and says no language, so nothing can choose a compiler "+
|
||||
"for it", module, a.Name))
|
||||
}
|
||||
problems = append(problems, bundleEnvProblems(module, a)...)
|
||||
// What the runtime loads is among what was compiled (ADR 0175): a name here that is
|
||||
// not an entrypoint is a file the bundle does not contain, and the runtime would
|
||||
// fail to import it on every machine rather than here.
|
||||
for _, load := range a.Loads {
|
||||
found := false
|
||||
for _, e := range a.Entrypoints {
|
||||
found = found || e == load
|
||||
}
|
||||
if !found {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q says the runtime loads %q, which is not among its entrypoints — "+
|
||||
"what is loaded is compiled, so it is named there too", module, a.Name, load))
|
||||
}
|
||||
}
|
||||
// **A system, for a language that compiles to a binary** (novox/hq ADR 0142). A binary
|
||||
// is pinned to one operating system at link time so a host refuses to touch a machine
|
||||
// it was not built for (novox/hq ADR 0005); an artifact that says nothing would be
|
||||
@@ -372,67 +315,3 @@ func versionOf(digest string) string {
|
||||
}
|
||||
return hex
|
||||
}
|
||||
|
||||
// bundleEnvWords are the words the runtime sets for itself; a bundle that named one would be
|
||||
// telling the runtime what it is, which is the mesh's to say (novox/hq ADR 0192).
|
||||
var bundleEnvWords = map[string]bool{
|
||||
RuntimeToolModules: true, RuntimeBrokerFile: true, RuntimeOperatorAccount: true,
|
||||
RuntimeOperatorHome: true, RuntimeToolEnv: true,
|
||||
}
|
||||
|
||||
// bundleEnvProblems says what is wrong with what a bundle says it is given (novox/hq ADR 0192):
|
||||
// a value is a path or a constant written with the references a container's environment may use
|
||||
// for a place or a port, and never a secret's content or another module's binding — a secret
|
||||
// reaches a tool as a file whose path is named.
|
||||
func bundleEnvProblems(module string, a Artifact) []string {
|
||||
if len(a.Env) == 0 {
|
||||
return nil
|
||||
}
|
||||
var problems []string
|
||||
for _, word := range sortedKeys(a.Env) {
|
||||
value := a.Env[word]
|
||||
if bundleEnvWords[word] {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q gives itself %s, which the node's runtime sets for itself; a bundle is "+
|
||||
"given its own words beside the runtime's, never in place of them (novox/hq ADR 0192)",
|
||||
module, a.Name, word))
|
||||
}
|
||||
rest := ofPort.ReplaceAllString(dirRef.ReplaceAllString(value, ""), "")
|
||||
if strings.Contains(rest, "${") {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q gives %s the value %q. A bundle's word is a path or a constant, written with "+
|
||||
"${dir:…} and ${port:…} only; a secret reaches a tool as a file the mesh places, "+
|
||||
"named by its path, never as its content (novox/hq ADR 0192)",
|
||||
module, a.Name, word, value))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
func copyWords(in map[string]string) map[string]string {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(in))
|
||||
for k, v := range in {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// BinaryOf is what a bundle compiled to a binary is called once built: what the artifact names, else
|
||||
// the package it is built from, else the artifact's own name (novox/hq 04-ISSUES/142). Empty for a
|
||||
// language that does not compile to one. The builder writes the binary under this name, and the
|
||||
// composer runs it by it, so both ask here.
|
||||
func BinaryOf(a Artifact) string {
|
||||
if !compilesToABinary(a.Language) {
|
||||
return ""
|
||||
}
|
||||
if name := strings.TrimSpace(a.Binary); name != "" {
|
||||
return name
|
||||
}
|
||||
if from := strings.Trim(a.From, "./"); from != "" {
|
||||
return path.Base(from)
|
||||
}
|
||||
return a.Name
|
||||
}
|
||||
|
||||
@@ -508,27 +508,10 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
return nil, fmt.Errorf(
|
||||
"%s needs a secret called %q and none was made for it", m.Module, name)
|
||||
}
|
||||
// The runtime's credential belongs to the account the runtime runs as (novox/hq ADR 0175,
|
||||
// to-be 38 WP3): its process is composed `user: <account>` where the node has one, and a
|
||||
// root-owned 0600 file is one that process cannot read. Composed here rather than said in
|
||||
// the manifest, because a manifest cannot say ${machine:account} safely — a node with no
|
||||
// account has nothing to resolve it to, and then the runtime runs as root and the file
|
||||
// stays root's.
|
||||
owner := m.SecretsOwner
|
||||
if m.Module == RuntimeModule && r.Account != "" {
|
||||
owner = r.Account
|
||||
}
|
||||
first = append(first, ownedBy(owner, map[string]any{
|
||||
first = append(first, ownedBy(m.SecretsOwner, map[string]any{
|
||||
"id": NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, "sealed": sealed,
|
||||
}))
|
||||
}
|
||||
// This module's tools bundles, where the machine runs the node's tool runtime (novox/hq
|
||||
// ADR 0175, to-be 38 WP2). Mesh-computed like everything above it, and before the module's
|
||||
// own resources for the same reason: the runtime's process names the files inside these
|
||||
// and is restarted when one changes, so they are on the machine before it is.
|
||||
if r.runtimeHere() {
|
||||
first = append(first, bundleArchives(m)...)
|
||||
}
|
||||
// Operator-owned paths this module is granted use of (novox/hq ADR 0051). Written before
|
||||
// the module's own resources, and so before the container that mounts them: the host must
|
||||
// find each present — refusing clearly if the operator has not provided it — before it
|
||||
@@ -872,15 +855,6 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
if renamed := reflectsRenamed(m.Module, resource["reload-on"]); renamed != nil {
|
||||
copied["reload-on"] = renamed
|
||||
}
|
||||
// **What reads one of this module's own secrets is restarted when it changes** (novox/hq
|
||||
// issue 203, issue 206). A credential is re-issued by the mesh, and a container that
|
||||
// mounted the old file keeps the old one open: the build machine ran for an hour on a
|
||||
// credential the mesh had replaced, because its manifest restarted it on its
|
||||
// environment file and nobody had thought to name the credential too. Composed here so
|
||||
// no manifest has to say it, for a container or a daemon that names the secret's path.
|
||||
if reads := secretsReadBy(copied, m); len(reads) > 0 {
|
||||
copied["restart-on"] = withRestartOn(copied["restart-on"], reads)
|
||||
}
|
||||
// **A version prepares its state before it runs** (novox/hq ADR 0135). Derived from the
|
||||
// module's own resource rather than declared beside it: what prepares the state is the
|
||||
// module's own code, so what it is given has to be what that code is given — and a
|
||||
@@ -911,41 +885,6 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
out = append(out, fact)
|
||||
}
|
||||
}
|
||||
// The node's tool runtime, last (novox/hq ADR 0175, to-be 38 WP2.3): one process loading every
|
||||
// bundle delivered above and holding the credential sealed above, so both exist before it starts
|
||||
// — the order written here is the order the machine applies.
|
||||
if r.runtimeHere() {
|
||||
process, err := r.runtimeProcess(with)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
owner[fmt.Sprint(process["id"])] = RuntimeModule
|
||||
out = append(out, process)
|
||||
// What each module's bundles are given is read as the account the runtime runs as.
|
||||
words := map[string]map[string]string{}
|
||||
for _, m := range r.Modules {
|
||||
w, err := bundleWords(m, with)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
words[m.Module] = w
|
||||
}
|
||||
// And a file a module's words name is one the runtime is restarted for when it changes.
|
||||
if named := givenTo(out, owner, words, r.Account); len(named) > 0 {
|
||||
restarts, _ := process["restart-on"].([]any)
|
||||
seen := map[string]bool{}
|
||||
for _, id := range restarts {
|
||||
seen[fmt.Sprint(id)] = true
|
||||
}
|
||||
for _, id := range named {
|
||||
if !seen[id] {
|
||||
restarts = append(restarts, id)
|
||||
seen[id] = true
|
||||
}
|
||||
}
|
||||
process["restart-on"] = restarts
|
||||
}
|
||||
}
|
||||
if with.Adopted {
|
||||
// First, before anything a module declares: what the mesh needs reachable, then its guard.
|
||||
// The order a machine applies is the order written here.
|
||||
@@ -2112,87 +2051,3 @@ func portOfEndpoint(values map[string]any, ports map[string]int) {
|
||||
values["port"] = port
|
||||
}
|
||||
}
|
||||
|
||||
// secretsReadBy is the file resources of this module's own secrets that a container or a daemon reads
|
||||
// — named in its volumes, its environment or its env-files by the secret's placed path — as
|
||||
// restart-on ids. Nothing for other shapes, and nothing for a scheduled or run-once process, which
|
||||
// the host refuses a restart-on for (it runs again anyway, and reads the file afresh).
|
||||
func secretsReadBy(resource map[string]any, m Manifest) []string {
|
||||
kind := fmt.Sprint(resource["type"])
|
||||
if kind != "container" && kind != "process" {
|
||||
return nil
|
||||
}
|
||||
if resource["schedule"] != nil || resource["run-once"] == true {
|
||||
return nil
|
||||
}
|
||||
var mentioned []string
|
||||
for _, key := range []string{"volumes", "env", "env-file"} {
|
||||
mentioned = append(mentioned, stringsIn(resource[key])...)
|
||||
}
|
||||
var out []string
|
||||
for _, name := range sortedKeys(m.OwnSecrets) {
|
||||
path := m.OwnSecrets[name].Path
|
||||
if path == "" {
|
||||
continue
|
||||
}
|
||||
for _, s := range mentioned {
|
||||
// A volume is `source:destination[:mode]`; an env value or an env-file is the path itself.
|
||||
if s == path || strings.HasPrefix(s, path+":") {
|
||||
out = append(out, m.Module+"."+NeedID(name))
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// stringsIn is every string in a list or a map's values; nothing for anything else.
|
||||
func stringsIn(v any) []string {
|
||||
switch x := v.(type) {
|
||||
case []any:
|
||||
var out []string
|
||||
for _, item := range x {
|
||||
if s, ok := item.(string); ok {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
case []string:
|
||||
return x
|
||||
case map[string]any:
|
||||
var out []string
|
||||
for _, k := range sortedKeys(x) {
|
||||
if s, ok := x[k].(string); ok {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
case map[string]string:
|
||||
var out []string
|
||||
for _, k := range sortedKeys(x) {
|
||||
out = append(out, x[k])
|
||||
}
|
||||
return out
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// withRestartOn is a resource's restart-on list with these ids added once each.
|
||||
func withRestartOn(have any, add []string) []any {
|
||||
var out []any
|
||||
seen := map[string]bool{}
|
||||
for _, id := range reflectsRenamed("", have) {
|
||||
s := fmt.Sprint(id)
|
||||
if !seen[s] {
|
||||
seen[s] = true
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
for _, id := range add {
|
||||
if !seen[id] {
|
||||
seen[id] = true
|
||||
out = append(out, id)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -45,16 +43,8 @@ func jailsInto(modules []Manifest, j *Jailing) []map[string]any {
|
||||
|
||||
out := make([]map[string]any, 0, len(jails)+1)
|
||||
for _, d := range jails {
|
||||
// **The filter's digest rides in the jail file.** fail2ban is restarted when this file
|
||||
// changes, and the filter is a file of its own: a module that changed only what a failure
|
||||
// looks like rewrote the filter on disk and left the running jail on the old pattern, with
|
||||
// nothing said (novox/hq issue 191's rollout found it on gitea's sshd). Naming the filter's
|
||||
// digest here makes a changed pattern a changed jail file, so the restart the service
|
||||
// already takes on it covers the filter too.
|
||||
sum := sha256.Sum256([]byte(d.jail.Failregex))
|
||||
fmt.Fprintf(&composed, "\n# from %s, filter %s\n[%s]\nenabled = true\nfilter = %s\n%s\n",
|
||||
d.module, hex.EncodeToString(sum[:])[:12], d.jail.Name, d.jail.Name,
|
||||
strings.TrimRight(d.jail.Jail, "\n"))
|
||||
fmt.Fprintf(&composed, "\n# from %s\n[%s]\nenabled = true\nfilter = %s\n%s\n",
|
||||
d.module, d.jail.Name, d.jail.Name, strings.TrimRight(d.jail.Jail, "\n"))
|
||||
// The filter is a file of its own, named as the jail's filter= references it.
|
||||
out = append(out, map[string]any{
|
||||
"id": "filter-" + d.jail.Name,
|
||||
|
||||
@@ -44,29 +44,3 @@ func TestTheComposedJailFileIsWrittenEvenWhenEmpty(t *testing.T) {
|
||||
t.Fatalf("the empty composed jail file was not written alone: %v", files)
|
||||
}
|
||||
}
|
||||
|
||||
// A changed pattern restarts fail2ban (novox/hq issue 191's rollout): the service restarts when the
|
||||
// composed jail file changes, and the filter is a file of its own, so the jail file names the
|
||||
// filter's digest. Changing only the failregex must change the jail file; the same pattern must not.
|
||||
func TestAChangedFilterChangesTheJailFile(t *testing.T) {
|
||||
jailFile := func(failregex string) string {
|
||||
modules := []Manifest{
|
||||
{Module: "fail2ban", Jailing: &Jailing{Into: "/etc/fail2ban/jail.d/mesh.conf", FilterInto: "/etc/fail2ban/filter.d"}},
|
||||
{Module: "gitea", Jails: []Jail{{Name: "gitea", Failregex: failregex, Jail: "port = 222"}}},
|
||||
}
|
||||
for _, f := range jailsInto(modules, modules[0].Jailing) {
|
||||
if f["id"] == ComposedJailsID() {
|
||||
return f["content"].(string)
|
||||
}
|
||||
}
|
||||
t.Fatal("no composed jail file")
|
||||
return ""
|
||||
}
|
||||
before := jailFile("web login failed from <HOST>")
|
||||
if again := jailFile("web login failed from <HOST>"); again != before {
|
||||
t.Errorf("the same pattern composed a different jail file, which would restart fail2ban for nothing")
|
||||
}
|
||||
if after := jailFile("web login failed from <HOST>\n Invalid user .* from <HOST>"); after == before {
|
||||
t.Errorf("a changed pattern left the jail file as it was, so fail2ban keeps the old filter:\n%s", after)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -572,44 +572,6 @@ type Manifest struct {
|
||||
// a module that could ask for it could read every credential on the bus — and the claim on
|
||||
// `mesh-broker` is what authorises it, checked from this manifest alone.
|
||||
BusUsers string `json:"bus-users,omitempty"`
|
||||
|
||||
// Bundles are this module's compiled bundles as the build produced them: what each is called,
|
||||
// where it is, what it hashes to, what language it is in and which files a tool runtime loads
|
||||
// from it (novox/hq ADR 0175, to-be 38).
|
||||
//
|
||||
// **Derived, never written.** The manifest in a repository says `build.artifacts`; the manifest
|
||||
// the mesh holds says what came out, the way a resource naming an artifact comes to name a
|
||||
// digest. Kept here because a tools bundle is referenced by no resource of the module's own —
|
||||
// the node's runtime loads it, and the runtime is composed by the mesh — so without this the
|
||||
// resolved manifest would carry no trace of the one artifact the runtime needs. A repository
|
||||
// manifest that writes this beside a build is refused: it would be stating the build's output
|
||||
// by hand.
|
||||
Bundles []Bundle `json:"bundles,omitempty"`
|
||||
}
|
||||
|
||||
// Bundle is one compiled bundle after it exists, as the resolved manifest carries it.
|
||||
type Bundle struct {
|
||||
Name string `json:"name"`
|
||||
// Source is where a machine fetches it, kept without the store's address like every reference
|
||||
// the mesh records (artifacts.go); Digest is what it must hash to.
|
||||
Source string `json:"source"`
|
||||
Digest string `json:"digest"`
|
||||
// Language is what it was compiled from, which is what says how it is run.
|
||||
Language string `json:"language,omitempty"`
|
||||
// Entrypoints are the compiled files it was built around, relative to its root.
|
||||
Entrypoints []string `json:"entrypoints,omitempty"`
|
||||
// Loads are the entrypoints a node's tool runtime imports from it: what the artifact said, or
|
||||
// every entrypoint for a module declaring tools that said nothing. Empty for a bundle that is
|
||||
// run rather than loaded.
|
||||
Loads []string `json:"loads,omitempty"`
|
||||
// Env is what the artifact said it is given (ADR 0192), as written; composed per machine.
|
||||
Env map[string]string `json:"env,omitempty"`
|
||||
// Launchers are the executables the build wrote beside its entrypoints, by entrypoint (novox/hq
|
||||
// ADR 0193). A bundle built before them has none, and is served as it was built.
|
||||
Launchers map[string]string `json:"launchers,omitempty"`
|
||||
// Binary is the executable a bundle compiled to a binary is, at its root (novox/hq ADR 0193):
|
||||
// what runs it, where an interpreted bundle names an interpreter and an entrypoint.
|
||||
Binary string `json:"binary,omitempty"`
|
||||
}
|
||||
|
||||
// Build says how to produce this module's artifacts from its source.
|
||||
@@ -746,21 +708,6 @@ type Artifact struct {
|
||||
// somebody adds a helper. An empty list is a bundle that is run rather than loaded — a
|
||||
// provisioner or a step, named by whatever runs it.
|
||||
Entrypoints []string `json:"entrypoints,omitempty"`
|
||||
|
||||
// Loads are the entrypoints of this bundle the node's tool runtime loads (novox/hq ADR 0175,
|
||||
// to-be 38): the module's tool code, each file registering its tools as it is imported. A
|
||||
// subset of Entrypoints, for a bundle that also carries things that are RUN — a daemon, a
|
||||
// step, a report — and must not have them imported into the runtime.
|
||||
//
|
||||
// Absent means every entrypoint, for a module that declares `tools`: a bundle holding the
|
||||
// module's tools and nothing else is the ordinary case and should not have to say the same
|
||||
// list twice. A module declaring no tools has nothing the runtime loads, whatever it compiles.
|
||||
Loads []string `json:"loads,omitempty"`
|
||||
|
||||
// Env is what a tools bundle is given on a machine (novox/hq ADR 0192): words and their values,
|
||||
// paths and constants composed with ${dir:…} and ${port:…} exactly as a container's environment
|
||||
// is, never a secret's content. The node's runtime hands it to this bundle and to no other.
|
||||
Env map[string]string `json:"env,omitempty"`
|
||||
}
|
||||
|
||||
// Kinds an artifact may be.
|
||||
@@ -1386,15 +1333,6 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
//
|
||||
// Refused here because the alternative is a build that never returns, on a mesh new enough
|
||||
// that nobody is watching it yet.
|
||||
if m.Build != nil && len(m.Bundles) > 0 {
|
||||
// The output of a build, written beside the build that produces it (ADR 0175). A resource
|
||||
// naming a digest beside an `artifact` would be the same mistake, and is caught the same way:
|
||||
// what the mesh derives, a repository does not state.
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s writes `bundles` beside its build. The mesh derives that from what the build "+
|
||||
"produced; a manifest states `build.artifacts` and nothing about what came out",
|
||||
m.Module))
|
||||
}
|
||||
if m.Build != nil && len(m.Build.Artifacts) > 0 {
|
||||
for _, o := range m.Offers() {
|
||||
if o != ArtifactStoreProvision {
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Which machine serves each routed name (novox/hq ADR 0066, issue 178).
|
||||
//
|
||||
// A routed name is a label the mesh composed for a consumer's endpoint, and it is *served* by the
|
||||
// provider that answers requests for it — the proxy the consumer's route reaches. The same name is
|
||||
// composed into every labelled contribution the consumer makes, because a provider that must know
|
||||
// the consumer's public name (an identity provider composing a redirect) is told it the same way
|
||||
// (04-ISSUES/122). Attributing the name to whichever of those providers a map happened to yield
|
||||
// last sent a public name to the identity provider's machine on one plan and to the proxy's on the
|
||||
// next (forge issue 227), and the whole names region flipped with it.
|
||||
//
|
||||
// **The terminus serves the name.** Among the providers a name reaches, the one that serves it is
|
||||
// the one that is not itself routed: a provider that contributes a labelled name of its own to some
|
||||
// requirement is published through another provider, and is a consumer of names, not their end.
|
||||
// Name-agnostic — nothing here knows what "route" means — and structural: it reads the graph the
|
||||
// modules declared. Deterministic: names, requirements and nodes are walked in order, so two
|
||||
// plans of one mesh yield one region.
|
||||
|
||||
// NamesServed is every routed name across the mesh and the node that serves it, from every node's
|
||||
// resolution and settings. A name several termini claim goes to the first node in name order, so
|
||||
// the answer is stable; a name nothing terminal claims is left out.
|
||||
func NamesServed(plans map[string]Resolution, settings map[string]SettingsBy) (map[string]string, error) {
|
||||
nodes := make([]string, 0, len(plans))
|
||||
for n := range plans {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
|
||||
out := map[string]string{}
|
||||
for _, node := range nodes {
|
||||
plan := plans[node]
|
||||
all, err := plan.contributions(settings[node], nil, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
requirements := make([]string, 0, len(all))
|
||||
for to := range all {
|
||||
requirements = append(requirements, to)
|
||||
}
|
||||
sort.Strings(requirements)
|
||||
for _, to := range requirements {
|
||||
for _, given := range all[to] {
|
||||
if given.Node != "" {
|
||||
// Said from another machine; that machine's own resolution carries it.
|
||||
continue
|
||||
}
|
||||
// A routed name, and only that: a contribution the mesh composed a name for from a
|
||||
// label it was given. A grant that happens to carry a `name` of its own — a database
|
||||
// name — carries no label and is left alone.
|
||||
if _, labelled := given.Values["label"]; !labelled {
|
||||
continue
|
||||
}
|
||||
name, _ := given.Values["name"].(string)
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
serving := servingNodeOf(plan, to, given.From, node)
|
||||
if !servesNames(plans[serving], to) {
|
||||
continue
|
||||
}
|
||||
name = strings.ToLower(name)
|
||||
if held, taken := out[name]; !taken || serving < held {
|
||||
out[name] = serving
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// servingNodeOf is the node answering one consumer's requirement: whoever the plan needs it from,
|
||||
// or this same node when the provider is beside the consumer.
|
||||
func servingNodeOf(plan Resolution, requirement, consumer, self string) string {
|
||||
for _, need := range plan.Needs {
|
||||
if need.Name == requirement && need.For == consumer && need.From != "" {
|
||||
return need.From
|
||||
}
|
||||
}
|
||||
return self
|
||||
}
|
||||
|
||||
// servesNames says whether the module providing a requirement on a node is a terminus: it is not
|
||||
// itself published under a labelled name through some other provider. A node whose plan is not
|
||||
// known (it did not resolve) serves nothing.
|
||||
func servesNames(plan Resolution, requirement string) bool {
|
||||
for _, m := range plan.Modules {
|
||||
if !offers(m, requirement) {
|
||||
continue
|
||||
}
|
||||
return !contributesALabel(m)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func offers(m Manifest, requirement string) bool {
|
||||
for _, o := range m.Offers() {
|
||||
if o == requirement {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func contributesALabel(m Manifest) bool {
|
||||
for _, values := range m.Contributes {
|
||||
if _, labelled := values["label"]; labelled {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, locals := range m.ContributesMany {
|
||||
for _, values := range locals {
|
||||
if _, labelled := values["label"]; labelled {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The mesh of forge issue 227 (novox/hq issue 178): a dashboard on the home server contributes its
|
||||
// label to the route its proxy serves AND to the identity provider on the control node, which must
|
||||
// know the dashboard's public name to compose a redirect. Both contributions carry the composed
|
||||
// name; only the proxy serves it.
|
||||
func twoNodesOneName(t *testing.T) (map[string]Resolution, map[string]SettingsBy) {
|
||||
t.Helper()
|
||||
catalogue := shelf(
|
||||
Manifest{Module: "route-adapter", Version: "1", Provides: Offers("route"),
|
||||
Serves: map[string]map[string]any{"route": {}}, Receives: map[string]string{"route": "/etc/adapter/mesh.json"}},
|
||||
Manifest{Module: "route-proxy", Version: "1", Provides: Offers("route"),
|
||||
Serves: map[string]map[string]any{"route": {}}, Receives: map[string]string{"route": "/etc/proxy/mesh.json"}},
|
||||
Manifest{Module: "keycloak", Version: "1", Provides: FromAnywhere("oidc-client"),
|
||||
Serves: map[string]map[string]any{"oidc-client": {"token-path": "/token"}},
|
||||
Receives: map[string]string{"oidc-client": "/etc/keycloak/clients.json"},
|
||||
Listens: []Listening{{Port: 8080, From: FromMesh, Why: "the login page"}},
|
||||
// Published through the proxy itself: the identity provider is routed, not a router.
|
||||
Contributes: map[string]map[string]any{"route": {"label": "login", "endpoint": "web", "port": 8080}}},
|
||||
Manifest{Module: "grafana", Version: "1",
|
||||
Listens: []Listening{{Port: 3000, From: FromMesh, Why: "dashboards"}},
|
||||
Contributes: map[string]map[string]any{
|
||||
"route": {"label": "grafana", "endpoint": "web", "port": 3000},
|
||||
"oidc-client": {"label": "grafana", "endpoint": "web", "port": 3000, "callback": "/login"},
|
||||
}},
|
||||
)
|
||||
home := withDomain("home.example")
|
||||
home.Name, home.At = "home-server", "home-server.internal"
|
||||
control := withDomain("control.example")
|
||||
control.Name, control.At = "anchor", "anchor.internal"
|
||||
|
||||
onHome, err := Resolve(catalogue, []string{"grafana", "route-adapter"}, home, World{
|
||||
Offered: map[string][]Provider{"oidc-client": {{Node: "anchor", At: "anchor.internal", Module: "keycloak"}}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
onControl, err := Resolve(catalogue, []string{"keycloak", "route-proxy"}, control, World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return map[string]Resolution{"home-server": onHome, "anchor": onControl},
|
||||
map[string]SettingsBy{"home-server": {}, "anchor": {}}
|
||||
}
|
||||
|
||||
func TestANameResolvesToTheNodeWhoseProxyServesIt(t *testing.T) {
|
||||
plans, settings := twoNodesOneName(t)
|
||||
// Many times, because the fault was map order: one plan said one node, the next the other.
|
||||
for i := 0; i < 25; i++ {
|
||||
served, err := NamesServed(plans, settings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if served["grafana.home.example"] != "home-server" {
|
||||
t.Fatalf("run %d: the dashboard's name is served by %q, and its proxy is on the home server: %v",
|
||||
i, served["grafana.home.example"], served)
|
||||
}
|
||||
if served["login.control.example"] != "anchor" {
|
||||
t.Fatalf("run %d: the identity provider's own name is served by its proxy on the control node: %v", i, served)
|
||||
}
|
||||
if _, leaked := served["grafana.control.example"]; leaked {
|
||||
t.Fatalf("a name composed for the identity provider's benefit is not one it serves: %v", served)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A module that is routed several times names each route (ADR 0094's sibling for contributes);
|
||||
// every one of them is a name the mesh must resolve, and none reached the names region before.
|
||||
func TestEveryRouteOfAModuleWithSeveralIsANameServed(t *testing.T) {
|
||||
catalogue := shelf(
|
||||
Manifest{Module: "route-proxy", Version: "1", Provides: Offers("route"),
|
||||
Serves: map[string]map[string]any{"route": {}}, Receives: map[string]string{"route": "/etc/proxy/mesh.json"}},
|
||||
Manifest{Module: "photos", Version: "1",
|
||||
Listens: []Listening{{Port: 8102, From: FromMesh, Why: "web"}, {Port: 9102, From: FromMesh, Why: "api"}},
|
||||
ContributesMany: map[string]map[string]map[string]any{"route": {
|
||||
"site": {"label": "photos", "endpoint": "web", "port": 8102},
|
||||
"api": {"label": "photos-api", "endpoint": "api", "port": 9102},
|
||||
}}},
|
||||
)
|
||||
node := withDomain("control.example")
|
||||
node.Name, node.At = "anchor", "anchor.internal"
|
||||
plan, err := Resolve(catalogue, []string{"photos", "route-proxy"}, node, World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
served, err := NamesServed(map[string]Resolution{"anchor": plan}, map[string]SettingsBy{"anchor": {}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range []string{"photos.control.example", "photos-api.control.example"} {
|
||||
if served[name] != "anchor" {
|
||||
t.Fatalf("%s is not served by its proxy: %v", name, served)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A resolved manifest keeps a built artifact's reference in the store-relative form, never the
|
||||
// address the builder reached the store by (novox/hq ADR 0155): on 2026-10-02 the first bundle
|
||||
// resolved on the mesh carried the store's host in bundles[0].source and registration refused it
|
||||
// as naming an installation. Archive resources are the same kind of reference and get the same.
|
||||
func TestAResolvedReferenceIsKeptNotRouted(t *testing.T) {
|
||||
m, err := ParseManifest([]byte(`{
|
||||
"module": "sample", "version": "1",
|
||||
"tools": ["one"],
|
||||
"build": {"artifacts": [
|
||||
{"name": "code", "kind": "bundle", "language": "typescript", "entrypoints": ["tools/index.js"]},
|
||||
{"name": "files", "kind": "archive", "from": "files"}
|
||||
]},
|
||||
"resources": [{"id": "packed", "type": "archive", "path": "/opt/sample", "artifact": "files"}]
|
||||
}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := "sha256:" + strings.Repeat("ab", 32)
|
||||
resolved, err := m.Resolve([]Built{
|
||||
{Name: "code", Kind: ArtifactBundle, Reference: "http://store.example:5100/v2/sample/code/blobs/" + digest, Digest: digest},
|
||||
{Name: "files", Kind: ArtifactArchive, Reference: "http://store.example:5100/v2/sample/files/blobs/" + digest, Digest: digest},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := resolved.Bundles[0].Source, ArtifactStoreScheme+"sample/code/blobs/"+digest; got != want {
|
||||
t.Errorf("bundle source %q, want the kept form %q", got, want)
|
||||
}
|
||||
if got, want := resolved.Resources[0]["source"], ArtifactStoreScheme+"sample/files/blobs/"+digest; got != want {
|
||||
t.Errorf("archive source %q, want the kept form %q", got, want)
|
||||
}
|
||||
if problems := InstallationProblems(resolved); len(problems) != 0 {
|
||||
t.Errorf("a resolved manifest names an installation: %v", problems)
|
||||
}
|
||||
}
|
||||
@@ -28,10 +28,10 @@ import (
|
||||
|
||||
// A RosterFile is a file the mesh renders from the roster of machines, in the format the module
|
||||
// gives as a Go text/template. The template sees a rosterView: `.Node` (this machine's bare name),
|
||||
// `.Suffix` (what its mesh name ends in), and two sets of `{Name, FQDN, Address}` — `.Names` and
|
||||
// `.Machines`. Both are the nodes of the mesh: routed names were once in `.Names` too, and are not
|
||||
// since every route became a name under its node's internal domain (novox/hq ADR 0191) — the hq
|
||||
// issue 111 distinction is kept as two fields so the templates that range either keep rendering.
|
||||
// `.Suffix` (what its mesh name ends in), and two sets of `{Name, FQDN, Address}` — `.Names`, every
|
||||
// name the mesh serves, and `.Machines`, only the nodes of the mesh. Which set a template ranges is
|
||||
// how the hq issue 111 distinction is drawn: a container's hosts wants every name; a resolver told
|
||||
// the suffix is its own wants only the machines.
|
||||
type RosterFile struct {
|
||||
// Path is where on the machine the rendered file goes. Absolute, or it is refused here rather
|
||||
// than discovered as a daemon that reads nothing.
|
||||
|
||||
@@ -1,365 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The node's tool runtime, as the catalogue knows it (novox/hq ADR 0175, to-be 38).
|
||||
//
|
||||
// **One module is the runtime.** Where it is assigned, one process per machine serves every assigned
|
||||
// module's tools and every held seat's verbs, on the host side, from the bundles each module's build
|
||||
// produced — and no module needs a container to reach the bus with its tools. The name is a constant
|
||||
// rather than a manifest field because a rule turns on it: the composer places the runtime's process
|
||||
// where this module is, and registration refuses the old pattern once this module exists.
|
||||
|
||||
// RuntimeModule is the module that is the node's tool runtime. Mirrored in the broker package,
|
||||
// which composes a principal of its own for it; the agreement test there holds the two to one string.
|
||||
const RuntimeModule = "node-tools"
|
||||
|
||||
// BundleRoot is where a machine keeps the tools bundles the mesh delivers to it: under the mesh's
|
||||
// own directory, beside the daemons the host unpacks there, and never where a package manager also
|
||||
// writes. One directory per module, one per bundle beneath it, at a path that does not move with
|
||||
// the version — so the runtime's process names each entrypoint once and is restarted, not
|
||||
// recomposed, when a bundle changes.
|
||||
const BundleRoot = "/var/lib/mesh/bundles"
|
||||
|
||||
// BundleID names the archive resource that delivers one of a module's bundles; prefixed with the
|
||||
// module like every resource of its own.
|
||||
func BundleID(bundle string) string { return "bundle-" + bundle }
|
||||
|
||||
// BundlePath is where one module's bundle is unpacked on a machine.
|
||||
func BundlePath(module, bundle string) string { return BundleRoot + "/" + module + "/" + bundle }
|
||||
|
||||
// runtimeHere says whether this node's set includes the runtime module, which is what decides
|
||||
// whether anything about tools changes on the machine (to-be 38 WP2): until the runtime is assigned,
|
||||
// a node is sent exactly what it was sent before, bundles included, because a bundle nothing loads
|
||||
// is bytes nobody reads.
|
||||
func (r Resolution) runtimeHere() bool {
|
||||
for _, m := range r.Modules {
|
||||
if m.Module == RuntimeModule {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// bundleArchives is one archive per tools bundle of a module — a bundle the runtime LOADS something
|
||||
// from — as the host fetches and unpacks any artifact (novox/hq ADR 0175 §3: a module brings its
|
||||
// tools as a bundle, delivered by the host like any artifact, never an image). A bundle it loads
|
||||
// nothing from is run rather than loaded: a daemon, a step, the runtime itself — delivered by the
|
||||
// process that runs it, and not again here.
|
||||
//
|
||||
// The source is the kept reference; the per-resource pass that follows routes it through the
|
||||
// artifact store as this network reaches it now, as it does every image and archive the mesh built.
|
||||
func bundleArchives(m Manifest) []map[string]any {
|
||||
var out []map[string]any
|
||||
for _, b := range m.Bundles {
|
||||
if len(b.Loads) == 0 {
|
||||
continue
|
||||
}
|
||||
out = append(out, map[string]any{
|
||||
"id": BundleID(b.Name), "type": "archive",
|
||||
"source": b.Source, "digest": b.Digest,
|
||||
"path": BundlePath(m.Module, b.Name),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// RuntimeProcessID names the one process the mesh composes for a machine's runtime; prefixed with
|
||||
// the runtime module like a resource of its own, because that module is what the host sees it as.
|
||||
func RuntimeProcessID() string { return "runtime" }
|
||||
|
||||
// RuntimeToolModules is the variable the runtime reads the modules it serves from: one
|
||||
// `<module>=<entrypoint>` per file it loads, comma-separated — several entries may name one module.
|
||||
// RuntimeBrokerFile is where it reads the node's credential; RuntimeOperatorAccount and
|
||||
// RuntimeOperatorHome are the machine's operator account and home, handed to every tool's
|
||||
// environment (to-be 38 WP1), and absent on a machine with no account.
|
||||
const (
|
||||
RuntimeToolModules = "MESH_TOOL_MODULES"
|
||||
RuntimeBrokerFile = "MESH_BROKER_FILE"
|
||||
RuntimeOperatorAccount = "MESH_OPERATOR_ACCOUNT"
|
||||
RuntimeOperatorHome = "MESH_OPERATOR_HOME"
|
||||
// RuntimeToolEnv is every served module's composed environment, as JSON (novox/hq ADR 0192):
|
||||
// {"<module>": {"<word>": "<value>"}}. The runtime takes it at start, removes it from its own
|
||||
// environment and hands each module's words to that module's bundles alone. In the unit, so a
|
||||
// change to any module's words changes the process and restarts it.
|
||||
RuntimeToolEnv = "MESH_TOOL_ENV"
|
||||
)
|
||||
|
||||
// interpreterFor is how a bundle in a language is run: the program the host's unit starts, with the
|
||||
// bundle's entrypoint after it. The one thing the composer takes from a language, and said here
|
||||
// rather than in a manifest because the runtime's process is the mesh's to compose (to-be 38 WP3).
|
||||
func interpreterFor(language string) (string, error) {
|
||||
switch language {
|
||||
case "typescript":
|
||||
return "node", nil
|
||||
}
|
||||
return "", fmt.Errorf(
|
||||
"%s is written in %q, and the mesh knows no interpreter to run a %q bundle with",
|
||||
RuntimeModule, language, language)
|
||||
}
|
||||
|
||||
// runtimeProcess is the one process a machine runs the node's tool runtime as (novox/hq ADR 0175,
|
||||
// to-be 38 WP2.3): the runtime module's own bundle, run by its language's interpreter, told which
|
||||
// modules it serves and from which files, where its credential is, and who the machine's operator
|
||||
// is — and restarted when any bundle it loads or the credential it holds changes.
|
||||
//
|
||||
// Composed from the placed manifests, so the credential's path is where this node puts it. The
|
||||
// runtime runs as the operator's account when the machine has one, which is what lets a tool that
|
||||
// needs root escalate as the operator would (ADR 0175 §4); on a machine with no account it runs as
|
||||
// root, and the two operator words are not set.
|
||||
func (r Resolution) runtimeProcess(with Rendering) (map[string]any, error) {
|
||||
var runtime *Manifest
|
||||
for i := range r.Modules {
|
||||
if r.Modules[i].Module == RuntimeModule {
|
||||
runtime = &r.Modules[i]
|
||||
}
|
||||
}
|
||||
if runtime == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if len(runtime.Bundles) != 1 {
|
||||
return nil, fmt.Errorf(
|
||||
"%s is assigned to %s and its build produced %d bundle(s); the runtime is one bundle "+
|
||||
"the mesh runs, so the module declares exactly one (novox/hq to-be 38)",
|
||||
RuntimeModule, r.Node, len(runtime.Bundles))
|
||||
}
|
||||
bundle := runtime.Bundles[0]
|
||||
// What runs it (novox/hq ADR 0193): a runtime compiled to a binary runs itself, from its own
|
||||
// unpacked bundle; an interpreted one is its language's interpreter and its one entrypoint.
|
||||
var run []any
|
||||
if bundle.Binary != "" {
|
||||
run = []any{"./" + bundle.Binary}
|
||||
} else {
|
||||
if len(bundle.Entrypoints) != 1 {
|
||||
return nil, fmt.Errorf(
|
||||
"%s's bundle %q names %d entrypoint(s); the runtime is run from one, so the module "+
|
||||
"declares exactly one (novox/hq to-be 38)", RuntimeModule, bundle.Name, len(bundle.Entrypoints))
|
||||
}
|
||||
interpreter, err := interpreterFor(bundle.Language)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
run = []any{interpreter, bundle.Entrypoints[0]}
|
||||
}
|
||||
credential, declared := runtime.OwnSecrets["broker"]
|
||||
if !declared {
|
||||
return nil, fmt.Errorf(
|
||||
"%s declares no own secret named broker, and the node's credential is delivered there: "+
|
||||
"a module that speaks on the bus declares \"own-secrets\": {\"broker\": <path>}",
|
||||
RuntimeModule)
|
||||
}
|
||||
|
||||
// What it serves, and from which files: every module on this machine that composes here, in
|
||||
// name order, each bundle it loads from in the order the manifest gave. A module left out of
|
||||
// the declaration — a filter on an adopted machine — is left out of this too, or the runtime
|
||||
// would be told to load files that were never delivered.
|
||||
var served []string
|
||||
var restartOn []string
|
||||
given := map[string]map[string]string{}
|
||||
for _, m := range r.Modules {
|
||||
if with.Adopted && m.Filtering != nil {
|
||||
continue
|
||||
}
|
||||
words, err := bundleWords(m, with)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(words) > 0 {
|
||||
given[m.Module] = words
|
||||
}
|
||||
for _, b := range m.Bundles {
|
||||
if len(b.Loads) == 0 {
|
||||
continue
|
||||
}
|
||||
for _, load := range b.Loads {
|
||||
// What the runtime starts: the launcher the build wrote beside the entrypoint, where
|
||||
// it wrote one (ADR 0193); the entrypoint itself for a build from before them.
|
||||
if launcher, has := b.Launchers[load]; has {
|
||||
load = launcher
|
||||
}
|
||||
served = append(served, m.Module+"="+BundlePath(m.Module, b.Name)+"/"+load)
|
||||
}
|
||||
restartOn = append(restartOn, m.Module+"."+BundleID(b.Name))
|
||||
}
|
||||
}
|
||||
sort.Strings(served)
|
||||
restartOn = append(restartOn, RuntimeModule+"."+NeedID("broker"))
|
||||
sort.Strings(restartOn)
|
||||
|
||||
env := map[string]string{
|
||||
RuntimeToolModules: strings.Join(served, ","),
|
||||
RuntimeBrokerFile: credential.Path,
|
||||
}
|
||||
if len(given) > 0 {
|
||||
// Marshalled from maps, whose keys encoding/json sorts: the same words, the same unit.
|
||||
body, err := json.Marshal(given)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
env[RuntimeToolEnv] = string(body)
|
||||
}
|
||||
process := map[string]any{
|
||||
"id": RuntimeModule + "." + RuntimeProcessID(), "type": "process", "name": RuntimeModule,
|
||||
"source": bundle.Source, "digest": bundle.Digest,
|
||||
"run": run,
|
||||
"env": env,
|
||||
"restart-on": toAny(restartOn),
|
||||
}
|
||||
if r.Account != "" {
|
||||
env[RuntimeOperatorAccount] = r.Account
|
||||
env[RuntimeOperatorHome] = accountHomeOf(r.Account, r.AccountHome)
|
||||
process["user"] = r.Account
|
||||
}
|
||||
// Routed through the artifact store as this network reaches it now, like everything the mesh
|
||||
// built; refused with the same words when there is no store to route through.
|
||||
if err := artifactsInto(process, RuntimeModule, with); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return process, nil
|
||||
}
|
||||
|
||||
func toAny(in []string) []any {
|
||||
out := make([]any, 0, len(in))
|
||||
for _, s := range in {
|
||||
out = append(out, s)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// RuntimeImageModule and RuntimeImageArtifact name the image every per-module tool container was
|
||||
// built on: the tool runtime's own runtime image. With the runtime a module of its own, that image
|
||||
// stays the way a module's SERVICE may be built and stops being the way tools reach a node (ADR 0175).
|
||||
const (
|
||||
RuntimeImageModule = "mesh-tools"
|
||||
RuntimeImageArtifact = "runtime"
|
||||
)
|
||||
|
||||
// ToolContainerOnTheRuntime says why a manifest is the pattern ADR 0175 retires — a module whose tools
|
||||
// are served from a container built on the tool runtime's image — or nothing when it is not. Judged
|
||||
// from the manifest's own `build.on` when it is a repository manifest, and from what its build stood
|
||||
// on when it is a built one, because a resolved manifest carries no build. The gate itself is
|
||||
// registration's (to-be 38 WP2.4): once the runtime module is in the catalogue, this is refused.
|
||||
//
|
||||
// Three things must hold, and each alone is fine: declaring tools (a bundle does that); a container
|
||||
// (a module's service may well be one); building on the runtime's image (a service written against
|
||||
// the SDK may). All three is a container whose purpose is tools, which the runtime now serves.
|
||||
func ToolContainerOnTheRuntime(m Manifest, against []string) string {
|
||||
if len(m.Tools) == 0 {
|
||||
return ""
|
||||
}
|
||||
container := false
|
||||
for _, r := range m.Resources {
|
||||
if fmt.Sprint(r["type"]) == "container" {
|
||||
container = true
|
||||
}
|
||||
}
|
||||
if !container {
|
||||
return ""
|
||||
}
|
||||
onTheRuntime := false
|
||||
if m.Build != nil {
|
||||
for _, on := range m.Build.On {
|
||||
if on.Module == RuntimeImageModule && on.Artifact == RuntimeImageArtifact {
|
||||
onTheRuntime = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, ref := range against {
|
||||
path, kept := InArtifactStore(Recorded(ref))
|
||||
if kept && strings.HasPrefix(path, RuntimeImageModule+"/"+RuntimeImageArtifact+"@") {
|
||||
onTheRuntime = true
|
||||
}
|
||||
}
|
||||
if !onTheRuntime {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"%s declares tools and a container built on %s's %s image — a container whose purpose is "+
|
||||
"serving tools. The node's tool runtime (%s) serves every module's tools from its bundle "+
|
||||
"now (novox/hq ADR 0175, to-be 38); declare the tools as a bundle and drop the container",
|
||||
m.Module, RuntimeImageModule, RuntimeImageArtifact, RuntimeModule)
|
||||
}
|
||||
|
||||
// bundleWords is what one module's tools bundles are given on this machine (novox/hq ADR 0192):
|
||||
// each loaded bundle's env, its ${dir:…} resolved to where this machine places the module's
|
||||
// directories and its ${port:…} to the port this machine gave it — the same resolution a
|
||||
// container's environment gets. Two bundles of one module naming one word differently is refused:
|
||||
// the runtime hands a module's words to all its bundles.
|
||||
func bundleWords(m Manifest, with Rendering) (map[string]string, error) {
|
||||
var out map[string]string
|
||||
dirs := dirsFor(m, with)
|
||||
for _, b := range m.Bundles {
|
||||
if len(b.Loads) == 0 || len(b.Env) == 0 {
|
||||
continue
|
||||
}
|
||||
for _, word := range sortedKeys(b.Env) {
|
||||
value, err := dirFill(b.Env[word], dirs, m.Module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if value, err = portsFilledInto(value, m.Module+"'s bundle "+b.Name+" ("+word+")", m.Module, m.Listens, with); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if out == nil {
|
||||
out = map[string]string{}
|
||||
}
|
||||
if was, had := out[word]; had && was != value {
|
||||
return nil, fmt.Errorf("%s's bundles give %s two values (%q, %q); a module's words are "+
|
||||
"handed to all its bundles, so they agree (novox/hq ADR 0192)", m.Module, word, was, value)
|
||||
}
|
||||
out[word] = value
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// givenTo makes what a bundle's words name readable by the account the runtime runs as (novox/hq
|
||||
// ADR 0192): every file and directory of the module whose path a word names, or that holds one,
|
||||
// is owned by the account — a tool reads its configuration and its secret as the account, and a
|
||||
// root-owned 0600 file or a 0700 directory is one it cannot. Only where it says no owner already:
|
||||
// a module that named one knew why. Nothing on a machine with no account, where the runtime is root.
|
||||
//
|
||||
// It answers the files a word names exactly: what a tool reads, whose change the runtime must be
|
||||
// restarted for, as the container the tools came from was restarted when its configuration changed.
|
||||
func givenTo(out []map[string]any, owner map[string]string, words map[string]map[string]string, account string) []string {
|
||||
var named []string
|
||||
if len(words) == 0 {
|
||||
return nil
|
||||
}
|
||||
for _, resource := range out {
|
||||
module := owner[fmt.Sprint(resource["id"])]
|
||||
mine := words[module]
|
||||
if len(mine) == 0 {
|
||||
continue
|
||||
}
|
||||
kind := fmt.Sprint(resource["type"])
|
||||
if kind != "file" && kind != "directory" {
|
||||
continue
|
||||
}
|
||||
path, _ := resource["path"].(string)
|
||||
if path == "" {
|
||||
continue
|
||||
}
|
||||
for _, value := range mine {
|
||||
if kind == "file" && value == path {
|
||||
named = append(named, fmt.Sprint(resource["id"]))
|
||||
}
|
||||
}
|
||||
if _, said := resource["owner"]; said || account == "" {
|
||||
continue
|
||||
}
|
||||
for _, value := range mine {
|
||||
if value == path || strings.HasPrefix(value, strings.TrimRight(path, "/")+"/") {
|
||||
resource["owner"] = account
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(named)
|
||||
return named
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The packet-filter manifest as it was the day the runtime was decided (novox/hq ADR 0175): tools,
|
||||
// served from a container built on the tool runtime's image, with NET_ADMIN so the container could
|
||||
// reach the filter. The exact pattern to-be 38 WP4 moves it off, and the one the gate refuses.
|
||||
const thePacketFilterAsItWas = `{
|
||||
"module": "nftables",
|
||||
"version": "1",
|
||||
"capabilities": ["firewall", "container-runtime"],
|
||||
"claims": [{"name": "node-packet-filter", "scope": "node", "serves": ["rules", "reload", "remove"]}],
|
||||
"filtering": {"into": "/etc/nftables.conf"},
|
||||
"resources": [
|
||||
{"id": "mesh-state", "type": "directory", "mode": "0700", "place": "mesh"},
|
||||
{"id": "package", "type": "package", "package": "nftables"},
|
||||
{"id": "unit", "type": "file", "path": "/etc/systemd/system/mesh-filter.service",
|
||||
"content": "[Unit]\nDescription=The mesh's packet filter\n[Service]\nType=oneshot\nExecStart=nft -f /etc/nftables.conf\n", "mode": "0644"},
|
||||
{"id": "load", "type": "service", "unit": "mesh-filter.service", "state": "running", "boot": "enabled",
|
||||
"restart-on": ["unit"], "reload-on": ["filtering"]},
|
||||
{"id": "runtime", "type": "container", "name": "mesh-nftables", "network": "host",
|
||||
"capabilities": ["NET_ADMIN"],
|
||||
"volumes": ["${dir:mesh-state}/broker:/run/secrets/broker:ro", "/etc/nftables.conf:/etc/nftables.conf:ro"],
|
||||
"env": {"MESH_BROKER_FILE": "/run/secrets/broker", "MESH_FILTER_FILE": "/etc/nftables.conf"},
|
||||
"artifact": "runtime"}
|
||||
],
|
||||
"tools": ["firewall_rules"],
|
||||
"own-secrets": {"broker": "${dir:mesh-state}/broker"},
|
||||
"build": {
|
||||
"on": [
|
||||
{"arg": "BUILD_BASE", "module": "mesh-tools", "artifact": "build"},
|
||||
{"arg": "RUNTIME_BASE", "module": "mesh-tools", "artifact": "runtime"}
|
||||
],
|
||||
"artifacts": [{"name": "runtime", "kind": "image", "from": "Dockerfile"}]
|
||||
}
|
||||
}`
|
||||
|
||||
func TestAToolContainerOnTheRuntimeImageIsNamedForWhatItIs(t *testing.T) {
|
||||
m, err := ParseManifest([]byte(thePacketFilterAsItWas))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// From the repository: the manifest says what it builds on.
|
||||
why := ToolContainerOnTheRuntime(m, nil)
|
||||
if why == "" {
|
||||
t.Fatal("the packet filter's tool container was not recognised from its build")
|
||||
}
|
||||
for _, word := range []string{"nftables", "mesh-tools", "runtime", "ADR 0175", "bundle"} {
|
||||
if !strings.Contains(why, word) {
|
||||
t.Errorf("the refusal does not say %q: %s", word, why)
|
||||
}
|
||||
}
|
||||
|
||||
// Built: the manifest carries no build, and what it stood on says the same.
|
||||
built, err := m.Resolve([]Built{{Name: "runtime", Kind: ArtifactImage,
|
||||
Reference: ArtifactStoreScheme + "nftables/runtime@" + digest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stoodOn := []string{"anchor.internal:5100/mesh-tools/build@" + digest, "anchor.internal:5100/mesh-tools/runtime@" + digest}
|
||||
if ToolContainerOnTheRuntime(built, stoodOn) == "" {
|
||||
t.Error("the packet filter's tool container was not recognised from what its build stood on")
|
||||
}
|
||||
if ToolContainerOnTheRuntime(built, nil) != "" {
|
||||
t.Error("a built manifest with no record of its base was judged to be on the runtime")
|
||||
}
|
||||
|
||||
// Each of the three alone is an ordinary module.
|
||||
bundle := m
|
||||
bundle.Resources = m.Resources[:len(m.Resources)-1]
|
||||
if ToolContainerOnTheRuntime(bundle, nil) != "" {
|
||||
t.Error("a module with tools and no container is the pattern the runtime serves, and was refused")
|
||||
}
|
||||
service := m
|
||||
service.Tools = nil
|
||||
if ToolContainerOnTheRuntime(service, nil) != "" {
|
||||
t.Error("a service built against the SDK, declaring no tools, was refused")
|
||||
}
|
||||
elsewhere := m
|
||||
elsewhere.Build = &Build{On: []BuildsOn{{Arg: "NODE_BASE", Image: "node@" + digest}},
|
||||
Artifacts: m.Build.Artifacts}
|
||||
if ToolContainerOnTheRuntime(elsewhere, nil) != "" {
|
||||
t.Error("a tool container on a public base was refused as though it were on the runtime's")
|
||||
}
|
||||
}
|
||||
@@ -1,391 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The node's tool runtime (novox/hq ADR 0175, to-be 38): where the runtime module is assigned, a
|
||||
// machine is sent every assigned module's tools bundle as an archive, and the runtime's own process
|
||||
// loading them. Where it is not, the machine is sent exactly what it was sent before.
|
||||
|
||||
var bundleDigest = "sha256:" + strings.Repeat("b", 64)
|
||||
|
||||
// aToolsModule is a module whose tools come as a compiled bundle and nothing else — the shape every
|
||||
// module takes once its tool container goes (to-be 38 WP4).
|
||||
func aToolsModule(t *testing.T, name string, entrypoints ...string) Manifest {
|
||||
t.Helper()
|
||||
m := Manifest{Module: name, Version: "1", Tools: []string{"status"},
|
||||
Build: &Build{Artifacts: []Artifact{
|
||||
{Name: "tools", Kind: ArtifactBundle, Language: "typescript", Entrypoints: entrypoints},
|
||||
}}}
|
||||
resolved, err := m.Resolve([]Built{{Name: "tools", Kind: ArtifactBundle,
|
||||
Reference: ArtifactStoreScheme + name + "/tools/blobs/" + bundleDigest, Digest: bundleDigest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return resolved
|
||||
}
|
||||
|
||||
// theRuntime is the runtime module as the catalogue holds it: its own bundle, run rather than
|
||||
// loaded, and its broker secret to receive the node's credential in.
|
||||
func theRuntime(t *testing.T) Manifest {
|
||||
t.Helper()
|
||||
m := Manifest{Module: RuntimeModule, Version: "1",
|
||||
OwnSecrets: OwnSecrets{"broker": {Path: "/var/lib/mesh/" + RuntimeModule + "/broker"}},
|
||||
Build: &Build{Artifacts: []Artifact{{Name: "runtime", Kind: ArtifactBundle, Language: "typescript",
|
||||
Entrypoints: []string{"src/main.js"}}}}}
|
||||
resolved, err := m.Resolve([]Built{{Name: "runtime", Kind: ArtifactBundle,
|
||||
Reference: ArtifactStoreScheme + RuntimeModule + "/runtime/blobs/" + bundleDigest, Digest: bundleDigest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return resolved
|
||||
}
|
||||
|
||||
func TestABuildsBundlesAreCarriedOnTheResolvedManifest(t *testing.T) {
|
||||
m := aToolsModule(t, "nftables", "tools/index.js")
|
||||
if len(m.Bundles) != 1 {
|
||||
t.Fatalf("the resolved manifest carries %d bundle(s), not the one the build made", len(m.Bundles))
|
||||
}
|
||||
b := m.Bundles[0]
|
||||
if b.Name != "tools" || b.Digest != bundleDigest || b.Language != "typescript" ||
|
||||
b.Source != ArtifactStoreScheme+"nftables/tools/blobs/"+bundleDigest ||
|
||||
len(b.Entrypoints) != 1 || b.Entrypoints[0] != "tools/index.js" {
|
||||
t.Errorf("the bundle is carried as %+v", b)
|
||||
}
|
||||
// A repository manifest may not write what the build derives.
|
||||
raw := `{"module":"x","version":"1","build":{"artifacts":[{"name":"t","kind":"bundle","language":"typescript"}]},` +
|
||||
`"bundles":[{"name":"t","source":"s","digest":"` + bundleDigest + `"}]}`
|
||||
if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), "bundles") {
|
||||
t.Errorf("a manifest stating its build's output by hand was accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEveryToolsBundleIsDeliveredWhereTheRuntimeRuns(t *testing.T) {
|
||||
store := Rendering{ArtifactStore: "anchor.internal:5101",
|
||||
Needed: map[string]map[string]string{RuntimeModule: {"broker": "sealed-credential"}}}
|
||||
nftables := aToolsModule(t, "nftables", "tools/index.js")
|
||||
zsh := aToolsModule(t, "zsh", "tools/index.js", "tools/more.js")
|
||||
|
||||
t.Run("with the runtime, one archive per tools bundle", func(t *testing.T) {
|
||||
r := Resolution{Node: "anchor", Modules: []Manifest{nftables, zsh, theRuntime(t)}}
|
||||
out, err := r.Declaration(store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
archive := fileNamed(out, "nftables."+BundleID("tools"))
|
||||
if archive == nil {
|
||||
t.Fatalf("nftables' tools bundle was not delivered: %v", ids(out))
|
||||
}
|
||||
if archive["type"] != "archive" || archive["digest"] != bundleDigest ||
|
||||
archive["path"] != BundleRoot+"/nftables/tools" {
|
||||
t.Errorf("delivered as %v", archive)
|
||||
}
|
||||
if archive["source"] != "http://anchor.internal:5101/v2/nftables/tools/blobs/"+bundleDigest {
|
||||
t.Errorf("fetched from %v, not through the store as this network reaches it", archive["source"])
|
||||
}
|
||||
if fileNamed(out, "zsh."+BundleID("tools")) == nil {
|
||||
t.Errorf("zsh's tools bundle was not delivered: %v", ids(out))
|
||||
}
|
||||
// The runtime's own bundle is run, not loaded: its process delivers it, not an archive.
|
||||
if fileNamed(out, RuntimeModule+"."+BundleID("runtime")) != nil {
|
||||
t.Error("the runtime's own bundle was delivered as an archive beside its process")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("without the runtime, nothing changes", func(t *testing.T) {
|
||||
r := Resolution{Node: "anchor", Modules: []Manifest{nftables, zsh}}
|
||||
out, err := r.Declaration(store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, id := range ids(out) {
|
||||
if strings.Contains(id, BundleID("")) {
|
||||
t.Errorf("%s was delivered to a machine running no runtime to load it", id)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func ids(out []map[string]any) []string {
|
||||
var names []string
|
||||
for _, r := range out {
|
||||
names = append(names, r["id"].(string))
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// One process per machine runs the runtime from its own bundle, told what it serves and from where,
|
||||
// where its credential is, and who the operator is — restarted when any of that changes.
|
||||
func TestTheMachineRunsOneRuntimeLoadingEveryDeliveredBundle(t *testing.T) {
|
||||
with := Rendering{ArtifactStore: "anchor.internal:5101",
|
||||
Needed: map[string]map[string]string{RuntimeModule: {"broker": "sealed-credential"}}}
|
||||
nftables := aToolsModule(t, "nftables", "tools/index.js")
|
||||
// A bundle carrying a daemon beside its tools says which files the runtime loads.
|
||||
showcase := Manifest{Module: "showcase", Version: "1", Tools: []string{"greet"},
|
||||
Build: &Build{Artifacts: []Artifact{{Name: "code", Kind: ArtifactBundle, Language: "typescript",
|
||||
Entrypoints: []string{"daemon/index.js", "tools/index.js"}, Loads: []string{"tools/index.js"}}}}}
|
||||
showcase, err := showcase.Resolve([]Built{{Name: "code", Kind: ArtifactBundle,
|
||||
Reference: ArtifactStoreScheme + "showcase/code/blobs/" + bundleDigest, Digest: bundleDigest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
r := Resolution{Node: "anchor", Account: "ops", Modules: []Manifest{nftables, showcase, theRuntime(t)}}
|
||||
out, err := r.Declaration(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
process := fileNamed(out, RuntimeModule+"."+RuntimeProcessID())
|
||||
if process == nil {
|
||||
t.Fatalf("no runtime process was composed: %v", ids(out))
|
||||
}
|
||||
if process["type"] != "process" || process["name"] != RuntimeModule || process["digest"] != bundleDigest ||
|
||||
process["source"] != "http://anchor.internal:5101/v2/"+RuntimeModule+"/runtime/blobs/"+bundleDigest {
|
||||
t.Errorf("the runtime's process is %v", process)
|
||||
}
|
||||
if fmt.Sprint(process["run"]) != "[node src/main.js]" {
|
||||
t.Errorf("the runtime is run as %v; its bundle's one entrypoint, by its language's interpreter", process["run"])
|
||||
}
|
||||
env := process["env"].(map[string]string)
|
||||
if env[RuntimeToolModules] != "nftables="+BundleRoot+"/nftables/tools/tools/index.js,"+
|
||||
"showcase="+BundleRoot+"/showcase/code/tools/index.js" {
|
||||
t.Errorf("the runtime is told to serve %q: every loaded file, by module, and nothing a bundle runs", env[RuntimeToolModules])
|
||||
}
|
||||
if env[RuntimeBrokerFile] != "/var/lib/mesh/"+RuntimeModule+"/broker" {
|
||||
t.Errorf("the runtime reads its credential at %q, not where the module's own secret is placed", env[RuntimeBrokerFile])
|
||||
}
|
||||
if env[RuntimeOperatorAccount] != "ops" || env[RuntimeOperatorHome] != "/home/ops" || process["user"] != "ops" {
|
||||
t.Errorf("the operator is not handed to the runtime: %v as %v", env, process["user"])
|
||||
}
|
||||
// The credential the process reads belongs to the account it runs as, or it could not read it
|
||||
// (to-be 38 WP3); other modules' secrets are left as their manifests say.
|
||||
if credential := fileNamed(out, RuntimeModule+"."+NeedID("broker")); credential == nil || credential["owner"] != "ops" {
|
||||
t.Errorf("the runtime's credential is not the account's to read: %v", credential)
|
||||
}
|
||||
restarts := fmt.Sprint(process["restart-on"])
|
||||
for _, want := range []string{"nftables." + BundleID("tools"), "showcase." + BundleID("code"), RuntimeModule + "." + NeedID("broker")} {
|
||||
if !strings.Contains(restarts, want) {
|
||||
t.Errorf("the runtime is not restarted when %s changes: %s", want, restarts)
|
||||
}
|
||||
}
|
||||
// After every bundle and the credential, so both exist before it starts.
|
||||
names := ids(out)
|
||||
if names[len(names)-1] != RuntimeModule+"."+RuntimeProcessID() {
|
||||
t.Errorf("the runtime's process is not last: %v", names)
|
||||
}
|
||||
|
||||
t.Run("a machine with no account runs it as root without the operator words", func(t *testing.T) {
|
||||
out, err := Resolution{Node: "anchor", Modules: []Manifest{nftables, theRuntime(t)}}.Declaration(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
process := fileNamed(out, RuntimeModule+"."+RuntimeProcessID())
|
||||
env := process["env"].(map[string]string)
|
||||
if _, set := env[RuntimeOperatorAccount]; set {
|
||||
t.Error("an operator account was named on a machine that has none")
|
||||
}
|
||||
if _, set := process["user"]; set {
|
||||
t.Error("a user was set on a machine with no account")
|
||||
}
|
||||
if credential := fileNamed(out, RuntimeModule+"."+NeedID("broker")); credential == nil || credential["owner"] != nil {
|
||||
t.Errorf("the runtime's credential was given an owner on a machine with no account: %v", credential)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a runtime module built wrong is refused by name", func(t *testing.T) {
|
||||
two := Manifest{Module: RuntimeModule, Version: "1", OwnSecrets: OwnSecrets{"broker": {Path: "/b"}},
|
||||
Build: &Build{Artifacts: []Artifact{{Name: "runtime", Kind: ArtifactBundle, Language: "typescript",
|
||||
Entrypoints: []string{"a.js", "b.js"}}}}}
|
||||
resolved, err := two.Resolve([]Built{{Name: "runtime", Kind: ArtifactBundle,
|
||||
Reference: ArtifactStoreScheme + "x/runtime/blobs/" + bundleDigest, Digest: bundleDigest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = Resolution{Node: "anchor", Modules: []Manifest{resolved}}.Declaration(with)
|
||||
if err == nil || !strings.Contains(err.Error(), "entrypoint") {
|
||||
t.Errorf("a runtime bundle with two entrypoints was composed: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// novox/hq ADR 0192: a tools bundle says what it is given; the composer resolves it per machine as
|
||||
// a container's environment, hands it to the runtime as the module's words, and makes what the
|
||||
// words name readable by the account the runtime runs as.
|
||||
func TestABundleIsGivenItsWordsResolvedForThisMachine(t *testing.T) {
|
||||
with := Rendering{ArtifactStore: "anchor.internal:5101",
|
||||
Needed: map[string]map[string]string{
|
||||
RuntimeModule: {"broker": "sealed-credential"},
|
||||
"dash": {"token": "sealed-token"},
|
||||
}}
|
||||
dash := Manifest{Module: "dash", Version: "1", Tools: []string{"status"},
|
||||
Listens: []Listening{{Name: "web", Port: 3000, Protocol: "tcp"}},
|
||||
OwnSecrets: OwnSecrets{"token": {Path: "${dir:mesh-state}/token"}},
|
||||
Resources: []map[string]any{
|
||||
{"id": "mesh-state", "type": "directory", "mode": "0700", "place": "mesh"},
|
||||
{"id": "config", "type": "file", "path": "${dir:mesh-state}/config.json", "mode": "0600", "content": "{}\n"},
|
||||
{"id": "unrelated", "type": "file", "path": "/etc/dash.conf", "content": "x\n"},
|
||||
},
|
||||
Build: &Build{Artifacts: []Artifact{{Name: "tools", Kind: ArtifactBundle, Language: "typescript",
|
||||
Entrypoints: []string{"tools/index.js"},
|
||||
Env: map[string]string{
|
||||
"DASH_CONFIG_FILE": "${dir:mesh-state}/config.json",
|
||||
"DASH_TOKEN_FILE": "${dir:mesh-state}/token",
|
||||
"DASH_URL": "http://127.0.0.1:${port:3000}",
|
||||
"DASH_ADMIN": "mesh-admin",
|
||||
}}}}}
|
||||
if problems := dash.Build.problems(dash.Module); len(problems) > 0 {
|
||||
t.Fatalf("a bundle's words written with ${dir:…} and ${port:…} were refused: %v", problems)
|
||||
}
|
||||
dash, err := dash.Resolve([]Built{{Name: "tools", Kind: ArtifactBundle,
|
||||
Reference: ArtifactStoreScheme + "dash/tools/blobs/" + bundleDigest, Digest: bundleDigest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other := aToolsModule(t, "nftables", "tools/index.js")
|
||||
|
||||
out, err := Resolution{Node: "anchor", Account: "ops", Modules: []Manifest{dash, other, theRuntime(t)}}.Declaration(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir := fileNamed(out, "dash.mesh-state")
|
||||
if dir == nil {
|
||||
t.Fatalf("no directory: %v", ids(out))
|
||||
}
|
||||
at := fmt.Sprint(dir["path"])
|
||||
process := fileNamed(out, RuntimeModule+"."+RuntimeProcessID())
|
||||
env := process["env"].(map[string]string)
|
||||
var given map[string]map[string]string
|
||||
if err := json.Unmarshal([]byte(env[RuntimeToolEnv]), &given); err != nil {
|
||||
t.Fatalf("the runtime's %s is not JSON: %q", RuntimeToolEnv, env[RuntimeToolEnv])
|
||||
}
|
||||
want := map[string]string{
|
||||
"DASH_CONFIG_FILE": at + "/config.json",
|
||||
"DASH_TOKEN_FILE": at + "/token",
|
||||
"DASH_URL": "http://127.0.0.1:3000",
|
||||
"DASH_ADMIN": "mesh-admin",
|
||||
}
|
||||
if fmt.Sprint(given["dash"]) != fmt.Sprint(want) {
|
||||
t.Errorf("dash is given %v, want %v", given["dash"], want)
|
||||
}
|
||||
if _, has := given["nftables"]; has {
|
||||
t.Errorf("a module that declares no words was given some: %v", given)
|
||||
}
|
||||
// What the words name is the account's to read; nothing else of the module's is touched.
|
||||
for _, id := range []string{"dash.mesh-state", "dash.config", "dash." + NeedID("token")} {
|
||||
if r := fileNamed(out, id); r == nil || r["owner"] != "ops" {
|
||||
t.Errorf("%s is not the account's to read: %v", id, r)
|
||||
}
|
||||
}
|
||||
if r := fileNamed(out, "dash.unrelated"); r == nil || r["owner"] != nil {
|
||||
t.Errorf("a file no word names was given an owner: %v", r)
|
||||
}
|
||||
// A file a word names restarts the runtime when it changes, as it restarted the tool container.
|
||||
restarts := fmt.Sprint(process["restart-on"])
|
||||
for _, want := range []string{"dash.config", "dash." + NeedID("token")} {
|
||||
if !strings.Contains(restarts, want) {
|
||||
t.Errorf("the runtime is not restarted when %s changes: %s", want, restarts)
|
||||
}
|
||||
}
|
||||
if strings.Contains(restarts, "dash.unrelated") || strings.Contains(restarts, "dash.mesh-state") {
|
||||
t.Errorf("the runtime restarts for something no word names as a file: %s", restarts)
|
||||
}
|
||||
|
||||
t.Run("on a machine with no account the runtime is root and nothing is re-owned", func(t *testing.T) {
|
||||
out, err := Resolution{Node: "anchor", Modules: []Manifest{dash, theRuntime(t)}}.Declaration(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r := fileNamed(out, "dash.config"); r["owner"] != nil {
|
||||
t.Errorf("re-owned with no account: %v", r)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a change to a module's words changes the runtime's process", func(t *testing.T) {
|
||||
changed := dash
|
||||
changed.Bundles = append([]Bundle(nil), dash.Bundles...)
|
||||
changed.Bundles[0].Env = map[string]string{"DASH_ADMIN": "somebody-else"}
|
||||
out2, err := Resolution{Node: "anchor", Account: "ops", Modules: []Manifest{changed, theRuntime(t)}}.Declaration(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fmt.Sprint(fileNamed(out2, RuntimeModule+"."+RuntimeProcessID())["env"]) == fmt.Sprint(env) {
|
||||
t.Error("the runtime's process is the same after a module's words changed, so it would not restart")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestABundlesWordsAreRefusedWhenTheyAreNotPathsOrConstants(t *testing.T) {
|
||||
m := Manifest{Module: "dash", Version: "1", Build: &Build{Artifacts: []Artifact{
|
||||
{Name: "tools", Kind: ArtifactBundle, Language: "typescript", Entrypoints: []string{"tools/index.js"},
|
||||
Env: map[string]string{"DASH_TOKEN": "${secret:token}", RuntimeBrokerFile: "/x", "DASH_PEER": "${bound:db:url}"}},
|
||||
{Name: "runtime", Kind: ArtifactImage, From: "Dockerfile", Env: map[string]string{"X": "y"}},
|
||||
}}}
|
||||
said := strings.Join(m.Build.problems(m.Module), "\n")
|
||||
for _, want := range []string{
|
||||
`"tools" gives DASH_TOKEN the value "${secret:token}"`,
|
||||
`"tools" gives DASH_PEER the value "${bound:db:url}"`,
|
||||
`"tools" gives itself ` + RuntimeBrokerFile,
|
||||
`"runtime" is a "image" and says what it is given`,
|
||||
} {
|
||||
if !strings.Contains(said, want) {
|
||||
t.Errorf("not refused: %s\nsaid:\n%s", want, said)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0193: the runtime is told the launcher a build wrote, and the entrypoint itself for a
|
||||
// build from before launchers — so the move needs no flag day.
|
||||
func TestTheRuntimeStartsTheLauncherWhereTheBuildWroteOne(t *testing.T) {
|
||||
with := Rendering{ArtifactStore: "anchor.internal:5101",
|
||||
Needed: map[string]map[string]string{RuntimeModule: {"broker": "sealed-credential"}}}
|
||||
launched := Manifest{Module: "dash", Version: "1", Tools: []string{"status"},
|
||||
Build: &Build{Artifacts: []Artifact{{Name: "tools", Kind: ArtifactBundle, Language: "typescript",
|
||||
Entrypoints: []string{"tools/index.js"}}}}}
|
||||
launched, err := launched.Resolve([]Built{{Name: "tools", Kind: ArtifactBundle,
|
||||
Reference: ArtifactStoreScheme + "dash/tools/blobs/" + bundleDigest, Digest: bundleDigest,
|
||||
Launchers: map[string]string{"tools/index.js": "tools/index.serve.mjs"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
older := aToolsModule(t, "nftables", "tools/index.js")
|
||||
out, err := Resolution{Node: "anchor", Account: "ops", Modules: []Manifest{launched, older, theRuntime(t)}}.Declaration(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := fileNamed(out, RuntimeModule+"."+RuntimeProcessID())["env"].(map[string]string)
|
||||
want := "dash=" + BundleRoot + "/dash/tools/tools/index.serve.mjs,nftables=" + BundleRoot + "/nftables/tools/tools/index.js"
|
||||
if env[RuntimeToolModules] != want {
|
||||
t.Errorf("the runtime is told %q, want %q", env[RuntimeToolModules], want)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0193: a runtime compiled to a binary runs itself from its own unpacked bundle.
|
||||
func TestARuntimeCompiledToABinaryRunsItself(t *testing.T) {
|
||||
with := Rendering{ArtifactStore: "anchor.internal:5101",
|
||||
Needed: map[string]map[string]string{RuntimeModule: {"broker": "sealed-credential"}}}
|
||||
goRuntime := Manifest{Module: RuntimeModule, Version: "1",
|
||||
OwnSecrets: OwnSecrets{"broker": {Path: "/var/lib/mesh/" + RuntimeModule + "/broker"}},
|
||||
Build: &Build{Artifacts: []Artifact{{Name: "runtime", Kind: ArtifactBundle, Language: "go",
|
||||
System: "arch", From: "cmd/node-tools"}}}}
|
||||
goRuntime, err := goRuntime.Resolve([]Built{{Name: "runtime", Kind: ArtifactBundle,
|
||||
Reference: ArtifactStoreScheme + RuntimeModule + "/runtime/blobs/" + bundleDigest, Digest: bundleDigest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := Resolution{Node: "anchor", Account: "ops", Modules: []Manifest{aToolsModule(t, "nftables", "tools/index.js"), goRuntime}}.Declaration(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
process := fileNamed(out, RuntimeModule+"."+RuntimeProcessID())
|
||||
if fmt.Sprint(process["run"]) != "[./node-tools]" {
|
||||
t.Errorf("a Go runtime is run as %v, want its own binary", process["run"])
|
||||
}
|
||||
env := process["env"].(map[string]string)
|
||||
if env[RuntimeToolModules] == "" || process["user"] != "ops" {
|
||||
t.Errorf("the Go runtime is not told what to serve or whose it is: %v %v", env, process["user"])
|
||||
}
|
||||
}
|
||||
@@ -96,17 +96,8 @@ var defaultSeats = []Seat{
|
||||
// A build says what it does as it does it (novox/hq ADR 0157): `started` when work is taken,
|
||||
// `log.<build id>` for every line, `built` for the outcome. The log's tail token is the build's
|
||||
// id, so a reader follows one build by subject alone.
|
||||
// **Node-scoped, and every holder takes from one queue** (novox/hq ADR 0190): a build is asked of
|
||||
// the role, and whichever machine holding the seat is idle pulls it. One holder per machine is
|
||||
// what the scope says; sharing the work is what a seat's queue has always done.
|
||||
{Name: "node-build-agent", Scope: ScopeNode,
|
||||
Accepts: []string{"build"}, Emits: []string{"started", "built", "log.*"}, Decision: "novox/hq ADR 0190"},
|
||||
// **Retired by ADR 0190, kept while a manifest still claims it.** The one build machine's seat.
|
||||
// A claim to a seat the mesh no longer defines is refused, and the module holding this one is
|
||||
// assigned on a live machine until build-agent replaces it — removing the row first would make
|
||||
// that machine unresolvable in the meantime. Deleted once no registered manifest claims it.
|
||||
{Name: "mesh-build-machine", Scope: ScopeMesh,
|
||||
Accepts: []string{"build"}, Emits: []string{"started", "built", "log.*"}, Decision: "novox/hq ADR 0190"},
|
||||
Accepts: []string{"build"}, Emits: []string{"started", "built", "log.*"}, Decision: "novox/hq ADR 0121"},
|
||||
{Name: "node-dns-resolver", Scope: ScopeNode, Decision: "novox/hq ADR 0121"},
|
||||
// The intrusion prevention's verbs (novox/hq ADR 0179): what a person asks a machine's ban list
|
||||
// whatever keeps it — who is banned and why, ban one address, let one go. Every holder serves all
|
||||
|
||||
@@ -44,10 +44,9 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
|
||||
delivered[s.Delivers] = s.Name
|
||||
}
|
||||
}
|
||||
// Seventeen since node-build-agent (novox/hq ADR 0190) — sixteen once the retired
|
||||
// mesh-build-machine row goes, when no registered manifest claims it any more.
|
||||
if len(Seats()) != 17 {
|
||||
t.Errorf("the mesh defines %d seats rather than 17; the set is closed, so a change here is "+
|
||||
// Sixteen since node-service-manager (novox/hq ADR 0177).
|
||||
if len(Seats()) != 16 {
|
||||
t.Errorf("the mesh defines %d seats rather than 16; the set is closed, so a change here is "+
|
||||
"a decision (novox/hq ADR 0110): %s", len(Seats()), seatNames())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// What reads one of a module's own secrets is restarted when the secret changes (novox/hq issue 203,
|
||||
// issue 206): the build machine kept an hour-old credential open because its manifest restarted it
|
||||
// on its environment file alone. Composed, so a manifest need not say it; a scheduled process is
|
||||
// left alone, because the host refuses a restart-on for one and it reads the file afresh each run.
|
||||
func TestAContainerReadingAnOwnSecretIsRestartedWhenItChanges(t *testing.T) {
|
||||
m := Manifest{Module: "agent", Version: "1",
|
||||
OwnSecrets: OwnSecrets{"broker": {Path: "/var/lib/mesh/agent/broker"}},
|
||||
Resources: []map[string]any{
|
||||
{"id": "mesh-state", "type": "directory", "path": "/var/lib/mesh/agent", "mode": "0700"},
|
||||
{"id": "settings", "type": "file", "path": "/var/lib/mesh/agent/agent.env", "mode": "0600", "content": "A=1\n"},
|
||||
{"id": "server", "type": "container", "name": "agent", "network": "host",
|
||||
"image": "registry.example/agent@sha256:" + strings.Repeat("a", 64),
|
||||
"volumes": []any{"/var/lib/mesh/agent:/run/mesh:ro", "/var/lib/mesh/agent/broker:/run/mesh/broker:ro"},
|
||||
"env-file": []any{"/var/lib/mesh/agent/agent.env"},
|
||||
"restart-on": []any{"settings"}},
|
||||
{"id": "nightly", "type": "container", "name": "agent-nightly", "schedule": "0 3 * * *",
|
||||
"image": "registry.example/agent@sha256:" + strings.Repeat("a", 64),
|
||||
"volumes": []any{"/var/lib/mesh/agent/broker:/run/mesh/broker:ro"}},
|
||||
{"id": "other", "type": "container", "name": "agent-other",
|
||||
"image": "registry.example/agent@sha256:" + strings.Repeat("a", 64)},
|
||||
}}
|
||||
got, err := Resolve(shelf(m), []string{m.Module},
|
||||
Node{Name: "anchor", At: "10.0.0.1", Capabilities: map[string]bool{"container-runtime": true}}, World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := got.Declaration(Rendering{Needed: map[string]map[string]string{"agent": {"broker": "SEALED"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
by := map[string]map[string]any{}
|
||||
for _, r := range out {
|
||||
by[r["id"].(string)] = r
|
||||
}
|
||||
if want := []any{"agent.settings", "agent.needs-broker"}; !reflect.DeepEqual(by["agent.server"]["restart-on"], want) {
|
||||
t.Fatalf("the server reads the credential and is not restarted on it: %v", by["agent.server"]["restart-on"])
|
||||
}
|
||||
if _, has := by["agent.nightly"]["restart-on"]; has {
|
||||
t.Fatalf("a scheduled container was given a restart-on, which the host refuses: %v", by["agent.nightly"]["restart-on"])
|
||||
}
|
||||
if _, has := by["agent.other"]["restart-on"]; has {
|
||||
t.Fatalf("a container that reads no secret was given one to restart on: %v", by["agent.other"]["restart-on"])
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The bus is never public (novox/hq ADR 0169). Its port is what the bus module declares, the mesh,
|
||||
// and the control plane adds no opening of its own: a machine joins through the tunnel, so the
|
||||
// broker's host is filtered like any other. Before this, the broker port was a foundation port and
|
||||
// rendered from anywhere beside its from-the-mesh rule.
|
||||
func TestTheBusPortIsReachedFromTheMeshAlone(t *testing.T) {
|
||||
rules := []Rule{{Port: 4222, Protocol: "tcp", From: FromMesh, Because: []string{"nats"},
|
||||
Why: []string{"the mesh bus"}}}
|
||||
out := AsNftables(rules, []string{"10.10.0.1", "10.10.0.2"}, true, nil, []string{"eth0"}, "mesh0")
|
||||
|
||||
if !regexp.MustCompile(`ip saddr \{ 10\.10\.0\.1, 10\.10\.0\.2 \} tcp dport 4222 accept`).MatchString(out) {
|
||||
t.Fatalf("the bus is not reachable from the mesh:\n%s", out)
|
||||
}
|
||||
if regexp.MustCompile(`(?m)^\s*tcp dport 4222 accept`).MatchString(out) {
|
||||
t.Fatalf("the bus is reachable from anywhere:\n%s", out)
|
||||
}
|
||||
}
|
||||
@@ -136,6 +136,16 @@ var ControllerVerbs = []Verb{
|
||||
"node": "the machine that runs the module",
|
||||
"module": "the module's name",
|
||||
}, []string{"node", "module"})},
|
||||
{Name: "token", Description: "Issue a one-time token for a machine to join with. Give the public half of the " +
|
||||
"tunnel key the machine made (`nox-mesh-host key`): the machine is given its address and made a peer of " +
|
||||
"the hub, and joins through the tunnel. The token is shown once, in the answer.",
|
||||
Input: schema(map[string]string{
|
||||
"node": "a machine the mesh already has a record for",
|
||||
"new": "or the name of a machine to create the record for",
|
||||
"overlay_key": "the public half of the machine's tunnel key",
|
||||
"for": "how long it may be used, as a duration (default 1h)",
|
||||
"adopted": "\"true\" when the machine is in use and joins adopted",
|
||||
}, nil)},
|
||||
{Name: "settings", Description: "Set what an assignment is configured with: a module's settings for the whole mesh, " +
|
||||
"or for one machine. Replaces that layer whole — what it does not name, it no longer sets — and takes effect " +
|
||||
"at the next push. With clear, removes the layer and the module is back to what its definition says.",
|
||||
|
||||
@@ -42,9 +42,6 @@ const (
|
||||
BusModule = "module"
|
||||
BusEnrolment = "enrolment"
|
||||
BusPerson = "person"
|
||||
// BusNodeTools is a machine's tool runtime (novox/hq ADR 0175): named like the module it
|
||||
// stands for, recorded as what it is.
|
||||
BusNodeTools = "node-tools"
|
||||
)
|
||||
|
||||
// MintBusPassword makes a bus password and records its hash under a username, replacing whatever was
|
||||
|
||||
@@ -42,11 +42,6 @@ type Source struct {
|
||||
// Seen is when the source was last looked at — by a build, by hand, or by the forge saying it
|
||||
// moved. What a late report of an older move is judged against.
|
||||
Seen time.Time
|
||||
// Against is every artifact the build this manifest came from stood on, as recorded. Part of a
|
||||
// module's provenance like the commit is, and what tells a built manifest's base when the manifest
|
||||
// itself no longer carries its build (novox/hq to-be 38 WP2.4). Empty for a manifest handed over
|
||||
// by hand, which carries its `build.on` itself.
|
||||
Against []string
|
||||
}
|
||||
|
||||
// Current reports whether what the mesh holds is what the source last had.
|
||||
@@ -66,32 +61,6 @@ func (s Source) Current() bool {
|
||||
// gains a requirement, a claim, a resource. What matters is that the change is visible the next
|
||||
// time a node is resolved, which it is.
|
||||
func (i *Inventory) RegisterModule(ctx context.Context, m catalogue.Manifest, from Source) error {
|
||||
// **Once the node's tool runtime is in the catalogue, the pattern it retires may not spread**
|
||||
// (novox/hq ADR 0175, to-be 38 WP2.4): a module serving its tools from a container built on the
|
||||
// runtime's image. Refused at registration, by name, for a module that is new to the catalogue
|
||||
// or that was registered in another shape — the mechanism that keeps the old pattern from
|
||||
// returning by habit. **Not refused for a module already registered in that shape**: the
|
||||
// catalogue holds some thirty of them the day the runtime arrives, each moves to a bundle in
|
||||
// its own change (to-be 38 WP4 onward), and a gate that refused every rebuild of every unmoved
|
||||
// module in the meantime would stop the whole pipeline to make a point the record already makes.
|
||||
// Before the runtime exists the pattern is accepted as it always was.
|
||||
if m.Module != catalogue.RuntimeModule {
|
||||
if why := catalogue.ToolContainerOnTheRuntime(m, from.Against); why != "" {
|
||||
runtime, err := i.hasModule(ctx, catalogue.RuntimeModule)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if runtime {
|
||||
already, err := i.registeredInThatShape(ctx, m.Module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !already {
|
||||
return fmt.Errorf("%s is not registered: %s", m.Module, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
raw, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -120,36 +89,6 @@ func (i *Inventory) RegisterModule(ctx context.Context, m catalogue.Manifest, fr
|
||||
return err
|
||||
}
|
||||
|
||||
// registeredInThatShape is whether the catalogue already holds this module as a tools container on
|
||||
// the runtime's image — judged from the manifest it holds and what that module's newest build stood
|
||||
// on, the same two things the gate judges a new registration by. False for a module the catalogue
|
||||
// does not hold.
|
||||
func (i *Inventory) registeredInThatShape(ctx context.Context, name string) (bool, error) {
|
||||
held, err := i.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
stored, has := held[name]
|
||||
if !has {
|
||||
return false, nil
|
||||
}
|
||||
against, err := i.BuiltAgainst(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return catalogue.ToolContainerOnTheRuntime(stored, against[name]) != "", nil
|
||||
}
|
||||
|
||||
// hasModule is whether the catalogue holds a module of that name.
|
||||
func (i *Inventory) hasModule(ctx context.Context, name string) (bool, error) {
|
||||
var one int
|
||||
err := i.store.Pool().QueryRow(ctx, `select 1 from module where name = $1`, name).Scan(&one)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, nil
|
||||
}
|
||||
return err == nil, err
|
||||
}
|
||||
|
||||
// SourceMoved records that a module's source has a newer commit than the mesh has built.
|
||||
//
|
||||
// This is the whole of noticing. Nothing here builds anything — it writes down that the two
|
||||
|
||||
@@ -685,61 +685,3 @@ func TestRegisteringWithoutProvenanceKeepsTheSeat(t *testing.T) {
|
||||
t.Fatalf("a hand-registered manifest erased where the module comes from: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Once the node's tool runtime is in the catalogue, a module serving its tools from a container
|
||||
// built on the runtime's image is refused at registration, naming the record (novox/hq ADR 0175,
|
||||
// to-be 38 WP2.4) — for a module new to the catalogue or one that had moved away from it; a module
|
||||
// already standing in that shape is rebuilt as before, so the catalogue's pipeline keeps running
|
||||
// while each moves (WP3's amendment). Before the runtime, it is accepted as it always was — so a
|
||||
// mesh converts in the order the design says and nothing is refused before there is anything to
|
||||
// move to.
|
||||
func TestAToolContainerIsRefusedOnceTheRuntimeIsRegistered(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
ctx := t.Context()
|
||||
filter := catalogue.Manifest{Module: "nftables", Version: "1", Tools: []string{"firewall_rules"},
|
||||
Resources: []map[string]any{{"id": "runtime", "type": "container", "name": "mesh-nftables"}}}
|
||||
stoodOn := []string{catalogue.ArtifactStoreScheme + "mesh-tools/runtime@sha256:" + strings.Repeat("d", 64)}
|
||||
|
||||
// Before the runtime exists the old pattern is accepted as it always was — and built, which is
|
||||
// how the catalogue comes to know what the module stood on.
|
||||
if err := inv.RegisterModule(ctx, filter, Source{Repository: "/r", Against: stoodOn}); err != nil {
|
||||
t.Fatalf("before the runtime exists the old pattern is accepted: %v", err)
|
||||
}
|
||||
built := aBuild("nf1", "nftables", "")
|
||||
built.Against = stoodOn
|
||||
if err := inv.RecordBuild(ctx, built); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime := catalogue.Manifest{Module: catalogue.RuntimeModule, Version: "1"}
|
||||
if err := inv.RegisterModule(ctx, runtime, Source{Repository: "/r"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// **A module already registered in that shape is rebuilt without complaint** (to-be 38 WP2.4 as
|
||||
// amended by WP3): some thirty of them stand the day the runtime arrives, and each moves in its
|
||||
// own change. The gate is against the pattern spreading, not against the pipeline running.
|
||||
if err := inv.RegisterModule(ctx, filter, Source{Repository: "/r", Against: stoodOn}); err != nil {
|
||||
t.Fatalf("a rebuild of a module that already had the pattern was refused: %v", err)
|
||||
}
|
||||
// A module new to the catalogue in that shape is refused, naming the record.
|
||||
newcomer := filter
|
||||
newcomer.Module = "lamp"
|
||||
err := inv.RegisterModule(ctx, newcomer, Source{Repository: "/r", Against: stoodOn})
|
||||
if err == nil || !strings.Contains(err.Error(), "ADR 0175") {
|
||||
t.Fatalf("a new module in the old pattern was registered beside the runtime: %v", err)
|
||||
}
|
||||
// And a module that had moved its tools to a bundle may not come back to a container.
|
||||
moved := filter
|
||||
moved.Resources = nil
|
||||
if err := inv.RegisterModule(ctx, moved, Source{Repository: "/r", Against: stoodOn}); err != nil {
|
||||
t.Fatalf("a module whose tools are a bundle was refused: %v", err)
|
||||
}
|
||||
unbuilt := aBuild("nf2", "nftables", "")
|
||||
if err := inv.RecordBuild(ctx, unbuilt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = inv.RegisterModule(ctx, filter, Source{Repository: "/r", Against: stoodOn})
|
||||
if err == nil || !strings.Contains(err.Error(), "ADR 0175") {
|
||||
t.Fatalf("a module that had moved returned to the old pattern unrefused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
@@ -19,17 +18,8 @@ const (
|
||||
EdgeBuiltBy = "built-by"
|
||||
// EdgeDeclared: the manifest's own `build.on`.
|
||||
EdgeDeclared = "declared"
|
||||
// EdgeWorkerOf: the module holds the build seat, whose worker the control plane defines
|
||||
// (novox/hq issue 206). The one place a *running* order enters the graph: a build machine rolled
|
||||
// before the controller that redefines its worker cannot bind it, and nothing can then build the
|
||||
// controller that would end that — so the holder of the build seat follows the controller, and
|
||||
// the controller is built by whichever build machine is running, as it always was.
|
||||
EdgeWorkerOf = "worker-of"
|
||||
)
|
||||
|
||||
// TheControlPlane is the module that defines every seat's worker on the bus.
|
||||
const TheControlPlane = "mesh-controller"
|
||||
|
||||
// Edge is one dependency: From depends on To, in the way Kind says.
|
||||
type Edge struct {
|
||||
From string `json:"from"`
|
||||
@@ -73,7 +63,7 @@ func dependenciesOf(entries []Entry, against map[string][]string, read map[strin
|
||||
if r := repositoryKey(e.Source.Repository); r != "" {
|
||||
byRepository[r] = append(byRepository[r], name)
|
||||
}
|
||||
if e.Manifest.ClaimsSeat("node-build-agent") || e.Manifest.ClaimsSeat("mesh-build-machine") {
|
||||
if e.Manifest.ClaimsSeat("mesh-build-machine") {
|
||||
builders = append(builders, name)
|
||||
}
|
||||
}
|
||||
@@ -97,21 +87,6 @@ func dependenciesOf(entries []Entry, against map[string][]string, read map[strin
|
||||
add(name, on.Module, EdgeDeclared)
|
||||
}
|
||||
}
|
||||
// **A bundle stands on the toolchain it is compiled in** (novox/hq 04-ISSUES/211). A
|
||||
// manifest names its toolchain by language, not in `build.on`, so the edge was implicit
|
||||
// and a merge that moved the toolchain and a bundle together built both in one tier —
|
||||
// the bundle against the toolchain as it was, recorded as built from the new commit. Read
|
||||
// from the manifest, so it holds before any build has recorded what it stood on; and a
|
||||
// toolchain that moves rebuilds every bundle compiled in it, which is what a toolchain
|
||||
// carrying a bundle's dependencies requires.
|
||||
for _, a := range e.Manifest.Build.Artifacts {
|
||||
if a.Kind != catalogue.ArtifactBundle {
|
||||
continue
|
||||
}
|
||||
if chain, err := builder.ToolchainFor(a.Language); err == nil {
|
||||
add(name, chain.Base, EdgeStandsOn)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, ref := range against[name] {
|
||||
if rest, ok := strings.CutPrefix(ref, catalogue.ArtifactStoreScheme); ok {
|
||||
@@ -131,13 +106,6 @@ func dependenciesOf(entries []Entry, against map[string][]string, read map[strin
|
||||
}
|
||||
}
|
||||
}
|
||||
if known[TheControlPlane] {
|
||||
for _, b := range builders {
|
||||
if b != TheControlPlane {
|
||||
add(b, TheControlPlane, EdgeWorkerOf)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(a, b int) bool {
|
||||
if out[a].From != out[b].From {
|
||||
return out[a].From < out[b].From
|
||||
|
||||
@@ -12,7 +12,7 @@ func TestDependenciesAreOneRelationWithTheirKinds(t *testing.T) {
|
||||
return Entry{Manifest: catalogue.Manifest{Module: name}, Source: Source{Repository: repository}}
|
||||
}
|
||||
builder := entry("builder", "http://forge/novox/mesh-catalog.git")
|
||||
builder.Manifest.Claims = []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}
|
||||
builder.Manifest.Claims = []catalogue.Claim{{Name: "mesh-build-machine", Scope: catalogue.ScopeMesh}}
|
||||
plugin := entry("shop-plugin", "http://forge/novox/mesh-catalog.git")
|
||||
plugin.Manifest.Build = &catalogue.Build{On: []catalogue.BuildsOn{{Arg: "BASE", Module: "shop"}}}
|
||||
entries := []Entry{
|
||||
@@ -62,41 +62,3 @@ func TestDependenciesAreOneRelationWithTheirKinds(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/211: a bundle stands on the toolchain it is compiled in, so a merge moving both
|
||||
// builds the toolchain first — read from the manifest, before any build recorded it.
|
||||
func TestABundleStandsOnTheToolchainItIsCompiledIn(t *testing.T) {
|
||||
entries := []Entry{
|
||||
{Manifest: catalogue.Manifest{Module: "mesh-tools"}, Source: Source{Repository: "novox/mesh-tools"}},
|
||||
{Manifest: catalogue.Manifest{Module: "mesh-tools-go"}, Source: Source{Repository: "novox/mesh-tools-go"}},
|
||||
{Manifest: catalogue.Manifest{Module: "node-tools", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{
|
||||
{Name: "runtime", Kind: catalogue.ArtifactBundle, Language: "go", System: "arch", From: "cmd/node-tools"}}}},
|
||||
Source: Source{Repository: "novox/mesh-tools"}},
|
||||
{Manifest: catalogue.Manifest{Module: "nftables", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{
|
||||
{Name: "tools", Kind: catalogue.ArtifactBundle, Language: "typescript", Entrypoints: []string{"tools/index.js"}}}}},
|
||||
Source: Source{Repository: "novox/mesh-catalog"}},
|
||||
{Manifest: catalogue.Manifest{Module: "photos", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{
|
||||
{Name: "server", Kind: catalogue.ArtifactImage, From: "Dockerfile"}}}},
|
||||
Source: Source{Repository: "novox/photos"}},
|
||||
}
|
||||
edges := dependenciesOf(entries, nil, nil)
|
||||
has := func(from, to string) bool {
|
||||
for _, e := range edges {
|
||||
if e.From == from && e.To == to && e.Kind == EdgeStandsOn {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
if !has("nftables", "mesh-tools") {
|
||||
t.Errorf("a TypeScript bundle does not stand on the TypeScript toolchain: %v", edges)
|
||||
}
|
||||
if !has("node-tools", "mesh-tools-go") {
|
||||
t.Errorf("a Go bundle does not stand on the Go toolchain: %v", edges)
|
||||
}
|
||||
for _, e := range edges {
|
||||
if e.From == "photos" && e.Kind == EdgeStandsOn {
|
||||
t.Errorf("an image stands on a toolchain it is not compiled in: %v", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
-- A token issued for a tunnel key (novox/hq ADR 0169).
|
||||
--
|
||||
-- A machine that joins through the tunnel makes its key first, and the token is issued for it: the
|
||||
-- hub is told the key before the token is shown. So enrolment must take that key and no other — a
|
||||
-- different one is a machine the hub does not know, offering a tunnel that would never answer. Null
|
||||
-- for a token issued without one, which enrols as before.
|
||||
alter table enrolment_token add column overlay_key text;
|
||||
@@ -356,6 +356,33 @@ func (i *Inventory) Claim(ctx context.Context, secret, by string, again bool) (N
|
||||
return scanNode(i.store.Pool().QueryRow(ctx, `select `+nodeColumns+` from node where id = $1`, id))
|
||||
}
|
||||
|
||||
// BindTokenToKey records the tunnel key a node's live token was issued for (novox/hq ADR 0169), so
|
||||
// enrolment takes that key and no other. Refused when the node has no live token to bind: a key
|
||||
// recorded against nothing would be a promise nothing keeps.
|
||||
func (i *Inventory) BindTokenToKey(ctx context.Context, node, key string) error {
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`update enrolment_token set overlay_key = $2
|
||||
where node = $1 and redeemed is null and expires > now()`, node, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("no live token to issue for the tunnel key")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TokenKey is the tunnel key a token was issued for, or empty when it was issued without one.
|
||||
func (i *Inventory) TokenKey(ctx context.Context, secret string) (string, error) {
|
||||
var key *string
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select overlay_key from enrolment_token where secret = $1`, hashSecret(secret)).Scan(&key)
|
||||
if err != nil || key == nil {
|
||||
return "", err
|
||||
}
|
||||
return *key, nil
|
||||
}
|
||||
|
||||
// Spend makes a claimed token used, only for the presenter holding the claim. The last write to the
|
||||
// store in an enrolment, so a token is spent exactly when the node it enrolled is complete. Spent
|
||||
// again by the same presenter is not an error: an answer lost after the first spend.
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
)
|
||||
|
||||
// What answers announces itself (novox/hq ADR 0197). A holder that serves a seat's verbs answers the
|
||||
// NATS services protocol's discovery — `$SRV.PING` and `$SRV.INFO`, and each by its service's name
|
||||
// and instance — with exactly what it serves, in NATS's own format, so the console and the standard
|
||||
// `nats micro` commands learn what exists from what answers rather than from a roster.
|
||||
|
||||
// DiscoverySubjects are where one service instance is asked to say what it is.
|
||||
func DiscoverySubjects(name, id string) []string {
|
||||
var out []string
|
||||
for _, verb := range []string{"PING", "INFO", "STATS"} {
|
||||
out = append(out, "$SRV."+verb, "$SRV."+verb+"."+name, "$SRV."+verb+"."+name+"."+id)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Announce answers discovery for one service until stopped. The answer is fixed at the call: a holder
|
||||
// whose verbs change announces again. Every instance answers, so there is no queue group.
|
||||
func (b OverNATS) Announce(info micro.Info, logger *log.Logger) (func(), error) {
|
||||
info.Type = micro.InfoResponseType
|
||||
infoBody, err := json.Marshal(info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pingBody, err := json.Marshal(micro.Ping{ServiceIdentity: info.ServiceIdentity, Type: micro.PingResponseType})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Statistics the protocol asks for; the controller keeps none per verb, so it answers its
|
||||
// identity and its endpoints with nothing counted — an honest zero, not a refusal.
|
||||
stats := micro.Stats{ServiceIdentity: info.ServiceIdentity, Type: micro.StatsResponseType}
|
||||
for _, e := range info.Endpoints {
|
||||
stats.Endpoints = append(stats.Endpoints, µ.EndpointStats{Name: e.Name, Subject: e.Subject, QueueGroup: e.QueueGroup})
|
||||
}
|
||||
statsBody, err := json.Marshal(stats)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var subs []*nats.Subscription
|
||||
done := make(chan struct{})
|
||||
stop := func() {
|
||||
close(done)
|
||||
for _, s := range subs {
|
||||
_ = s.Unsubscribe()
|
||||
}
|
||||
}
|
||||
for _, subject := range DiscoverySubjects(info.Name, info.ID) {
|
||||
subject := subject
|
||||
body := infoBody
|
||||
switch {
|
||||
case len(subject) >= 9 && subject[:9] == "$SRV.PING":
|
||||
body = pingBody
|
||||
case len(subject) >= 10 && subject[:10] == "$SRV.STATS":
|
||||
body = statsBody
|
||||
}
|
||||
bind := func() (*nats.Subscription, error) {
|
||||
return b.Conn.Subscribe(subject, func(msg *nats.Msg) {
|
||||
if err := msg.Respond(body); err != nil && logger != nil {
|
||||
logger.Printf("%s: could not answer: %v", subject, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
sub, err := bind()
|
||||
if err != nil {
|
||||
stop()
|
||||
return nil, fmt.Errorf("announcing %s on %s: %w", info.Name, subject, err)
|
||||
}
|
||||
subs = append(subs, sub)
|
||||
go keepBound(sub, bind, subject, done, logger)
|
||||
}
|
||||
return stop, nil
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package link
|
||||
|
||||
import "testing"
|
||||
|
||||
// A build machine serves the seat its credential claims (novox/hq ADR 0190 handover): the old
|
||||
// `builder` keeps the old role, a `build-agent` takes the new, from one binary and no flag.
|
||||
func TestABuildMachineServesTheSeatItsCredentialClaims(t *testing.T) {
|
||||
if got := BuildSeatClaimed([]string{"mesh-build-machine"}); got != "mesh-build-machine" {
|
||||
t.Errorf("a credential claiming the old role serves %q", got)
|
||||
}
|
||||
if got := BuildSeatClaimed([]string{"node-build-agent"}); got != TheBuildMachine {
|
||||
t.Errorf("a credential claiming the new role serves %q", got)
|
||||
}
|
||||
if got := BuildSeatClaimed(nil); got != TheBuildMachine {
|
||||
t.Errorf("a credential claiming nothing serves %q, want the current role", got)
|
||||
}
|
||||
if got := BuildSeatClaimed([]string{"", "node-build-agent"}); got != TheBuildMachine {
|
||||
t.Errorf("an empty claim is skipped; got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// What a machine says about a build is the event of the seat it took the build from, so an outcome
|
||||
// is heard where the asker of that seat listens.
|
||||
func TestABuildsEventsAreItsSeats(t *testing.T) {
|
||||
if BuildOutcomeOf(TheBuildMachineBefore) != "mesh.seat.mesh-build-machine.event.built" {
|
||||
t.Error(BuildOutcomeOf(TheBuildMachineBefore))
|
||||
}
|
||||
if BuildWorkOf(TheBuildMachine) != BuildWork() || BuildOutcomeOf(TheBuildMachine) != BuildOutcome() ||
|
||||
BuildStartedOf(TheBuildMachine) != BuildStarted() || BuildLogOf(TheBuildMachine, "b1") != BuildLog("b1") {
|
||||
t.Error("the no-argument forms must name the current role")
|
||||
}
|
||||
}
|
||||
+7
-53
@@ -19,57 +19,13 @@ import (
|
||||
// act on or a declaration a node reconciles toward; a build is a request that takes minutes and has
|
||||
// exactly one answer. Too long for request/reply, too particular to be an event.
|
||||
|
||||
// TheBuildMachine is the role a build is submitted to: node-scoped, held on every machine that
|
||||
// builds, and the work shared among them (novox/hq ADR 0190). The name stays for every caller; what
|
||||
// it names moved from the mesh's one build machine to whichever build agent is idle.
|
||||
//
|
||||
// **Switching a live mesh over, in order** — and why no step strands a build. The old seat's
|
||||
// stream and worker (SEAT_MESH_BUILD_MACHINE, SEAT_MESH_BUILD_MACHINE_worker) stay on the bus until
|
||||
// removed by hand, and the builder keeps draining them while it is assigned, because a machine
|
||||
// serves the seat its credential claims (BuildSeatClaimed) and the controller asks the seat that
|
||||
// has a holder (buildSeatAmong in the command) and hears both seats' outcomes:
|
||||
//
|
||||
// 1. Merge the controller and the host's first user list together; the new controller rolls and,
|
||||
// seeing only the builder assigned, still asks mesh-build-machine — which the builder holds.
|
||||
// 2. Merge the catalogue's build-agent; the builder builds it and the controller registers it.
|
||||
// 3. On each machine that builds: `module issue build-agent --node <n>`, then `assign`, then
|
||||
// `push`. The first holder appears, and from then on asks go to node-build-agent.
|
||||
// 4. Unassign builder everywhere and `module forget` it.
|
||||
// 5. By hand: delete SEAT_MESH_BUILD_MACHINE and its worker, drop the retired seat row and
|
||||
// TheBuildMachineBefore with it, and the second entries in seatsTheControllerAsks and
|
||||
// ControllerFollows.
|
||||
const TheBuildMachine = "node-build-agent"
|
||||
|
||||
// TheBuildMachineBefore is the role a build was submitted to until ADR 0190: the mesh's one build
|
||||
// machine, mesh-scoped. Kept named while the handover runs — a machine whose credential claims it
|
||||
// still serves it, and the controller still hears its outcomes — and dropped with the retired seat
|
||||
// row once nothing claims it.
|
||||
const TheBuildMachineBefore = "mesh-build-machine"
|
||||
|
||||
// BuildSeatClaimed is the build role a machine serves: the first seat its credential claims, or the
|
||||
// current role when the credential names none (a credential from before claims travelled in it, or
|
||||
// one written by hand). **The credential decides, not the binary** (ADR 0190 handover): one build
|
||||
// machine binary runs as the old `builder` on the old seat and as a `build-agent` on the new one,
|
||||
// each taking the work the mesh issued it a credential for, so neither drains the other's queue
|
||||
// and the switch needs no flag day.
|
||||
func BuildSeatClaimed(claimed []string) string {
|
||||
for _, seat := range claimed {
|
||||
if seat != "" {
|
||||
return seat
|
||||
}
|
||||
}
|
||||
return TheBuildMachine
|
||||
}
|
||||
// TheBuildMachine is the role a build is submitted to.
|
||||
const TheBuildMachine = "mesh-build-machine"
|
||||
|
||||
// BuildWork is where a build request lands, and BuildOutcome is where its result does. Derived from
|
||||
// the seat, so both sides name the role and neither names the other. The no-argument forms name the
|
||||
// current role; the `Of` forms take the seat, for the handover during which two roles exist.
|
||||
func BuildWork() string { return BuildWorkOf(TheBuildMachine) }
|
||||
func BuildOutcome() string { return BuildOutcomeOf(TheBuildMachine) }
|
||||
func BuildWorkOf(seat string) string { return "mesh.seat." + seat + ".accept.build" }
|
||||
func BuildOutcomeOf(seat string) string {
|
||||
return "mesh.seat." + seat + ".event.built"
|
||||
}
|
||||
// the seat, so both sides name the role and neither names the other.
|
||||
func BuildWork() string { return "mesh.seat." + TheBuildMachine + ".accept.build" }
|
||||
func BuildOutcome() string { return "mesh.seat." + TheBuildMachine + ".event.built" }
|
||||
|
||||
// BuildStarted is where a build machine says it has taken a build, and BuildLog is where it says
|
||||
// what it is doing, one line per message, under the build's own id (novox/hq ADR 0157).
|
||||
@@ -79,10 +35,8 @@ func BuildOutcomeOf(seat string) string {
|
||||
// lived in one container's stderr on one machine. Every line is now an event of the role, retained
|
||||
// with the rest of the mesh's events, so a reader follows a build live by subscribing its subject,
|
||||
// or reads it back afterwards from the stream, and a viewer is a subscriber and nothing more.
|
||||
func BuildStarted() string { return BuildStartedOf(TheBuildMachine) }
|
||||
func BuildLog(id string) string { return BuildLogOf(TheBuildMachine, id) }
|
||||
func BuildStartedOf(seat string) string { return "mesh.seat." + seat + ".event.started" }
|
||||
func BuildLogOf(seat, id string) string { return "mesh.seat." + seat + ".event.log." + id }
|
||||
func BuildStarted() string { return "mesh.seat." + TheBuildMachine + ".event.started" }
|
||||
func BuildLog(id string) string { return "mesh.seat." + TheBuildMachine + ".event.log." + id }
|
||||
|
||||
// BuildStart is what a build machine says the moment it takes a build.
|
||||
type BuildStart struct {
|
||||
|
||||
@@ -26,7 +26,7 @@ func TestTheOldBusAnnouncesABuildUnderBothNames(t *testing.T) {
|
||||
if KeyRoleBuilt != "built" {
|
||||
t.Fatalf("the role's event is %q, and a holder emits its verbs bare", KeyRoleBuilt)
|
||||
}
|
||||
if TheBuildMachine != "node-build-agent" {
|
||||
if TheBuildMachine != "mesh-build-machine" {
|
||||
t.Fatalf("the role is %q", TheBuildMachine)
|
||||
}
|
||||
// The two must differ, or one publish would serve both and this doubling would be pointless.
|
||||
|
||||
@@ -25,34 +25,16 @@ import (
|
||||
type natsBuilds struct {
|
||||
js *broker.JetStream
|
||||
owned bool
|
||||
// seat is the build role asked: the one that has a holder (ADR 0190 handover), chosen by the
|
||||
// controller from what is assigned, so an ask lands where a machine is pulling.
|
||||
seat string
|
||||
}
|
||||
|
||||
// BuildsOverNATS is the asking side on the bus being built, asking the current build role. It dials,
|
||||
// because the command that asks for a build is a one-shot and holds nothing else.
|
||||
// BuildsOverNATS is the asking side on the bus being built. It dials, because the command that asks
|
||||
// for a build is a one-shot and holds nothing else.
|
||||
func BuildsOverNATS(address string) (Builders, error) {
|
||||
return BuildsOverNATSOn(address, TheBuildMachine)
|
||||
}
|
||||
|
||||
// BuildsOverNATSOn is the asking side for one named build role — during the handover from the one
|
||||
// build machine to build agents, the role that has a holder (ADR 0190).
|
||||
func BuildsOverNATSOn(address, seat string) (Builders, error) {
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot reach the bus at %s to ask for a build: %w", address, err)
|
||||
}
|
||||
return &natsBuilds{js: js, owned: true, seat: seat}, nil
|
||||
}
|
||||
|
||||
// role is the seat asked: what the asker was made for, or the current build role for one made
|
||||
// without saying (a test building the struct by hand).
|
||||
func (b *natsBuilds) role() string {
|
||||
if b.seat == "" {
|
||||
return TheBuildMachine
|
||||
}
|
||||
return b.seat
|
||||
return &natsBuilds{js: js, owned: true}, nil
|
||||
}
|
||||
|
||||
func (b *natsBuilds) Close() {
|
||||
@@ -69,7 +51,7 @@ func (b *natsBuilds) Ask(ctx context.Context, request BuildRequest) error {
|
||||
}
|
||||
publish, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
if _, err := b.js.Context().Publish(BuildWorkOf(b.role()), body, nats.Context(publish)); err != nil {
|
||||
if _, err := b.js.Context().Publish(BuildWork(), body, nats.Context(publish)); err != nil {
|
||||
return fmt.Errorf("cannot submit a build: %w", err)
|
||||
}
|
||||
return nil
|
||||
@@ -81,7 +63,7 @@ func (b *natsBuilds) Submit(ctx context.Context, request BuildRequest,
|
||||
// Subscribed before the ask, so an outcome cannot arrive before there is anywhere for it to
|
||||
// land. Core, not the stream: the asker is waiting now, and the durable copy of this outcome is
|
||||
// the same event on EVENTS, which the controller records.
|
||||
outcomes, err := b.js.Conn().SubscribeSync(BuildOutcomeOf(b.role()))
|
||||
outcomes, err := b.js.Conn().SubscribeSync(BuildOutcome())
|
||||
if err != nil {
|
||||
return BuildResult{}, fmt.Errorf("cannot listen for a build's outcome: %w", err)
|
||||
}
|
||||
@@ -98,7 +80,7 @@ func (b *natsBuilds) Submit(ctx context.Context, request BuildRequest,
|
||||
// be assumed, because nothing else will ever say so.
|
||||
publish, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
if _, err := b.js.Context().Publish(BuildWorkOf(b.role()), body, nats.Context(publish)); err != nil {
|
||||
if _, err := b.js.Context().Publish(BuildWork(), body, nats.Context(publish)); err != nil {
|
||||
return BuildResult{}, fmt.Errorf("cannot submit a build: %w", err)
|
||||
}
|
||||
|
||||
@@ -133,16 +115,9 @@ type natsMachine struct {
|
||||
sub *nats.Subscription
|
||||
}
|
||||
|
||||
// MachineOverNATS takes build work from the current build role.
|
||||
// MachineOverNATS takes build work from the role this machine holds.
|
||||
func MachineOverNATS(js *broker.JetStream, on string) BuildMachine {
|
||||
return MachineOverNATSOn(js, on, TheBuildMachine)
|
||||
}
|
||||
|
||||
// MachineOverNATSOn takes build work from the role named — the one this machine's credential claims
|
||||
// (ADR 0190 handover): its asks come from that seat's worker, and what it says about a build goes
|
||||
// out as that seat's events, so an outcome is heard where the asker listens.
|
||||
func MachineOverNATSOn(js *broker.JetStream, on, seat string) BuildMachine {
|
||||
return &natsMachine{js: js, on: on, seat: seat}
|
||||
return &natsMachine{js: js, on: on, seat: TheBuildMachine}
|
||||
}
|
||||
|
||||
func (m *natsMachine) Close() {
|
||||
@@ -151,31 +126,29 @@ func (m *natsMachine) Close() {
|
||||
}
|
||||
}
|
||||
|
||||
// Take binds to the role's worker and pulls one request at a time, handing each over.
|
||||
// Take binds to the role's worker and hands each request over, one at a time.
|
||||
//
|
||||
// **Bound, never created.** The work queue and the worker on it are the controller's to define
|
||||
// (design 25 §3), and a build machine reaches no part of the JetStream API — so a missing one is said
|
||||
// as the mesh's to answer rather than quietly created with whatever this client defaults to.
|
||||
//
|
||||
// **Pulled, one at a time, by whichever holder is free** (novox/hq ADR 0190). Every machine holding
|
||||
// the role binds this same worker; a machine asks for the next request only when it has finished
|
||||
// the last, so a slow machine never holds an ask an idle one could take, and a machine that took
|
||||
// five at once would run five container builds against one runtime and finish all of them slower
|
||||
// than the first.
|
||||
func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build)) error {
|
||||
worker, found := broker.HolderConsumerFor(m.on, "build-agent",
|
||||
worker, found := broker.HolderConsumerFor(m.on, "builder",
|
||||
broker.DeclaredSeat{Name: m.seat, Accepts: []string{"build"}})
|
||||
if !found {
|
||||
return fmt.Errorf("%s accepts no work, so there is nothing for this machine to take", m.seat)
|
||||
}
|
||||
|
||||
// One at a time, which the consumer's own ack-pending limit enforces rather than a prefetch
|
||||
// setting: a machine that took five requests at once would run five container builds against one
|
||||
// runtime and finish all of them slower than the first.
|
||||
work := make(chan *nats.Msg, 1)
|
||||
// **The consumer's own filter, not the one subject this machine cares about.** The client checks
|
||||
// what is asked for against the consumer's filter and refuses anything that is not the same —
|
||||
// "subject does not match consumer" — so subscribing `…accept.build` against a consumer filtered
|
||||
// on `…accept.>` is rejected even though it is narrower. Learned twice now, on two different
|
||||
// consumers, which is why it is written down here.
|
||||
filter := worker.Filters[0]
|
||||
sub, err := m.js.Context().PullSubscribe(filter, worker.Name,
|
||||
sub, err := m.js.Context().ChanQueueSubscribe(filter, worker.Queue, work,
|
||||
nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
@@ -186,33 +159,13 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
m.sub = sub
|
||||
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
}
|
||||
// One, and wait a while for it; an empty queue is a timeout, which is the normal state of a
|
||||
// machine with nothing to build, and is asked again.
|
||||
fetched, err := sub.Fetch(1, nats.Context(ctx))
|
||||
switch {
|
||||
case ctx.Err() != nil:
|
||||
// Ours ended: the machine is being stopped.
|
||||
return nil
|
||||
case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded), errors.Is(err, nats.ErrTimeout):
|
||||
// **An empty queue, not the end.** A fetch on a context without a deadline waits the
|
||||
// client's own while and then says the deadline passed — the client's, not ours. Read
|
||||
// as "stop", every idle build machine exited clean every half minute and was started
|
||||
// again by its supervisor, which looked like a crash loop with nothing in the log to
|
||||
// say why (2026-10-03, the first build agents). Asked again.
|
||||
continue
|
||||
case err != nil:
|
||||
if sub.IsValid() {
|
||||
// A transient fault in asking — a reconnect, a slow server — is asked past rather
|
||||
// than ending the machine; one that outlasts the ack wait redelivers nothing lost.
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
case msg, ok := <-work:
|
||||
if !ok {
|
||||
return errors.New("the bus stopped delivering build work")
|
||||
}
|
||||
return fmt.Errorf("the bus stopped delivering build work: %w", err)
|
||||
}
|
||||
for _, msg := range fetched {
|
||||
var request BuildRequest
|
||||
if err := json.Unmarshal(msg.Data, &request); err != nil {
|
||||
// Unreadable: terminated rather than retried, because the next attempt reads the same
|
||||
@@ -225,7 +178,7 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
// the ask to a second machine nor counts the wait against its deliveries.
|
||||
working := make(chan struct{})
|
||||
go stillWorking(msg, working)
|
||||
do(ctx, &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat})
|
||||
do(ctx, &natsBuild{request: request, msg: msg, on: m.on, js: m.js})
|
||||
close(working)
|
||||
}
|
||||
}
|
||||
@@ -236,9 +189,7 @@ type natsBuild struct {
|
||||
msg *nats.Msg
|
||||
on string
|
||||
js *broker.JetStream
|
||||
// seat is the role this build was taken from; what the machine says about it is that role's.
|
||||
seat string
|
||||
seq int
|
||||
seq int
|
||||
}
|
||||
|
||||
func (b *natsBuild) Request() BuildRequest { return b.request }
|
||||
@@ -265,7 +216,7 @@ func (b *natsBuild) Announce(ctx context.Context, result BuildResult) error {
|
||||
}
|
||||
publish, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
if _, err := b.js.Context().Publish(BuildOutcomeOf(b.seat), body, nats.Context(publish)); err != nil {
|
||||
if _, err := b.js.Context().Publish(BuildOutcome(), body, nats.Context(publish)); err != nil {
|
||||
return fmt.Errorf("cannot announce a build's outcome: %w", err)
|
||||
}
|
||||
return nil
|
||||
@@ -285,7 +236,7 @@ func (b *natsBuild) Began(ctx context.Context) error {
|
||||
}
|
||||
publish, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
if _, err := b.js.Context().Publish(BuildStartedOf(b.seat), body, nats.Context(publish)); err != nil {
|
||||
if _, err := b.js.Context().Publish(BuildStarted(), body, nats.Context(publish)); err != nil {
|
||||
return fmt.Errorf("cannot say a build started: %w", err)
|
||||
}
|
||||
return nil
|
||||
@@ -303,7 +254,7 @@ func (b *natsBuild) Say(step, message string) {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = b.js.Conn().Publish(BuildLogOf(b.seat, b.request.ID), body)
|
||||
_ = b.js.Conn().Publish(BuildLog(b.request.ID), body)
|
||||
}
|
||||
|
||||
func (b *natsBuild) Hold(after time.Duration) error { return b.msg.NakWithDelay(after) }
|
||||
|
||||
@@ -48,7 +48,7 @@ func aBusWithTheBuildRole(t *testing.T) *broker.JetStream {
|
||||
t.Fatal(err)
|
||||
}
|
||||
clean := func() {
|
||||
_ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT")
|
||||
_ = js.Context().DeleteStream("SEAT_MESH_BUILD_MACHINE")
|
||||
for _, s := range broker.MeshStreams() {
|
||||
_ = js.Context().PurgeStream(s.Name)
|
||||
}
|
||||
@@ -158,7 +158,7 @@ func TestNatsABuildIsTakenAndItsOutcomeReachesEverybody(t *testing.T) {
|
||||
// And the work left the queue: a request a machine took and settled must not be given to another.
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
info, err := js.Context().StreamInfo("SEAT_NODE_BUILD_AGENT")
|
||||
info, err := js.Context().StreamInfo("SEAT_MESH_BUILD_MACHINE")
|
||||
if err == nil && info.State.Msgs == 0 {
|
||||
return
|
||||
}
|
||||
@@ -177,7 +177,7 @@ func TestNatsABuildWaitsForAMachineRatherThanFailing(t *testing.T) {
|
||||
if _, err := js.Context().Publish(BuildWork(), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := js.Context().StreamInfo("SEAT_NODE_BUILD_AGENT")
|
||||
info, err := js.Context().StreamInfo("SEAT_MESH_BUILD_MACHINE")
|
||||
if err != nil || info.State.Msgs != 1 {
|
||||
t.Fatalf("the work did not queue: %+v %v", info, err)
|
||||
}
|
||||
@@ -245,119 +245,3 @@ func TestNatsWorkAMachineDidNotAnswerGoesBackToTheQueue(t *testing.T) {
|
||||
func quietLog() *log.Logger { return log.New(io.Discard, "", 0) }
|
||||
|
||||
var _ = quietLog
|
||||
|
||||
// Two machines holding the role share one queue (novox/hq ADR 0190): three asks, each machine takes
|
||||
// one and the third waits until one of them is done; an ask is never handed to a machine that is
|
||||
// busy; and a machine that stops mid-ask leaves its ask to the other.
|
||||
func TestNatsTwoMachinesShareTheWorkAndNeitherIsHandedMoreThanItCanTake(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
|
||||
for _, id := range []string{"w-1", "w-2", "w-3"} {
|
||||
body, _ := json.Marshal(BuildRequest{ID: id, Repository: "/r"})
|
||||
if _, err := js.Context().Publish(BuildWork(), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
type taken struct{ machine, id string }
|
||||
took := make(chan taken, 8)
|
||||
release := map[string]chan struct{}{"anchor": make(chan struct{}), "laptop": make(chan struct{})}
|
||||
machines := map[string]BuildMachine{}
|
||||
for _, name := range []string{"anchor", "laptop"} {
|
||||
name := name
|
||||
m := MachineOverNATS(js, name)
|
||||
machines[name] = m
|
||||
defer m.Close()
|
||||
go func() {
|
||||
_ = m.Take(ctx, func(ctx context.Context, work Build) {
|
||||
took <- taken{name, work.Request().ID}
|
||||
<-release[name]
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: name})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
}
|
||||
|
||||
// Each machine took exactly one, and they are different asks.
|
||||
first := map[string]string{}
|
||||
for i := 0; i < 2; i++ {
|
||||
select {
|
||||
case got := <-took:
|
||||
if _, twice := first[got.machine]; twice {
|
||||
t.Fatalf("%s was handed a second ask while busy with its first", got.machine)
|
||||
}
|
||||
first[got.machine] = got.id
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatalf("only %d machine(s) took work; two idle holders should both have", len(first))
|
||||
}
|
||||
}
|
||||
if first["anchor"] == first["laptop"] {
|
||||
t.Fatalf("both machines took %q: the queue is not shared, it is copied", first["anchor"])
|
||||
}
|
||||
// The third waits: nobody is free.
|
||||
select {
|
||||
case got := <-took:
|
||||
t.Fatalf("%s was handed %s while both machines were busy", got.machine, got.id)
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
// One finishes, and only then is the third taken — by that machine, the one that is free.
|
||||
close(release["anchor"])
|
||||
release["anchor"] = make(chan struct{})
|
||||
select {
|
||||
case got := <-took:
|
||||
if got.machine != "anchor" {
|
||||
t.Fatalf("the third ask went to %s, which is still busy", got.machine)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("the third ask was never taken after a machine became free")
|
||||
}
|
||||
// A machine that stops mid-ask leaves its ask unacknowledged, and the ack wait brings it round
|
||||
// to whoever is left — the path TestNatsWorkAMachineDidNotAnswerGoesBackToTheQueue proves with
|
||||
// an explicit hand-back, because the real wait is a minute. Here: the laptop goes, anchor
|
||||
// finishes, and with nothing queued nothing more is taken by the machine that is left.
|
||||
machines["laptop"].Close()
|
||||
close(release["anchor"])
|
||||
select {
|
||||
case got := <-took:
|
||||
t.Fatalf("%s took %s; the queue should be empty", got.machine, got.id)
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
}
|
||||
|
||||
// During the handover (ADR 0190) two build roles exist. A machine whose credential claims the retired
|
||||
// one takes an ask published to that seat and answers as that seat; the asker of that seat hears it.
|
||||
func TestNatsAMachineOnTheRetiredBuildRoleTakesThatRolesAsks(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
seats := []broker.DeclaredSeat{{Name: TheBuildMachineBefore, Accepts: []string{"build"}, Emits: []string{"built"}}}
|
||||
if err := broker.RaiseSeats(js, seats, map[string]broker.Holder{TheBuildMachineBefore: {Node: "anchor", Module: "builder"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = js.Context().DeleteStream("SEAT_MESH_BUILD_MACHINE") })
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
|
||||
machine := MachineOverNATSOn(js, "anchor", TheBuildMachineBefore)
|
||||
defer machine.Close()
|
||||
go func() {
|
||||
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
||||
_ = work.Began(ctx)
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, Repository: work.Request().Repository, On: "anchor", Commit: "abc"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
|
||||
asker, err := BuildsOverNATSOn(os.Getenv("MESH_TEST_NATS"), TheBuildMachineBefore)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer asker.Close()
|
||||
result, err := asker.Submit(ctx, BuildRequest{ID: "build-old-seat", Repository: "r"}, 20*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.On != "anchor" || result.ID != "build-old-seat" {
|
||||
t.Errorf("the retired role's holder did not answer: %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,12 +9,13 @@ import (
|
||||
// the streams and the controller's consumers are asserted by Raise, before anything is served.
|
||||
func ConnectNats(js *broker.JetStream, enroller Enroller, listener Listener) *Server {
|
||||
return &Server{
|
||||
inbound: Nats(js),
|
||||
bus: OverNATS{JS: js.Context(), Conn: js.Conn()},
|
||||
js: js,
|
||||
enroller: enroller,
|
||||
listener: listener,
|
||||
log: newLog(),
|
||||
inbound: Nats(js),
|
||||
bus: OverNATS{JS: js.Context(), Conn: js.Conn()},
|
||||
js: js,
|
||||
consumers: js,
|
||||
enroller: enroller,
|
||||
listener: listener,
|
||||
log: newLog(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
type ensured struct{ consumers []broker.Consumer }
|
||||
|
||||
func (e *ensured) EnsureConsumer(c broker.Consumer) error {
|
||||
e.consumers = append(e.consumers, c)
|
||||
return nil
|
||||
}
|
||||
|
||||
type acceptsAs string
|
||||
|
||||
func (n acceptsAs) Enrol(context.Context, EnrolRequest) (EnrolReply, error) {
|
||||
return EnrolReply{Accepted: true, Node: string(n)}, nil
|
||||
}
|
||||
|
||||
type anEnrolment struct{ body []byte }
|
||||
|
||||
func (m anEnrolment) Kind() string { return "enrol" }
|
||||
func (m anEnrolment) Body() []byte { return m.body }
|
||||
func (m anEnrolment) Redelivered() bool { return false }
|
||||
func (m anEnrolment) HeldFor() time.Duration { return 0 }
|
||||
func (m anEnrolment) Answer(context.Context, []byte) error { return nil }
|
||||
func (m anEnrolment) Took() error { return nil }
|
||||
func (m anEnrolment) Hold(time.Duration) error { return nil }
|
||||
func (m anEnrolment) Drop() error { return nil }
|
||||
|
||||
// **A node that enrols can hear its declarations at once** (novox/hq 04-ISSUES/146): its consumer is
|
||||
// made as it enrols, not only when the control plane next starts — the first machine of a mesh
|
||||
// enrols after the control plane is up, and heard nothing.
|
||||
func TestAnEnrolledNodeIsGivenHowItHearsItsDeclarations(t *testing.T) {
|
||||
made := &ensured{}
|
||||
s := &Server{enroller: acceptsAs("anchor"), consumers: made, log: log.New(io.Discard, "", 0)}
|
||||
body, _ := json.Marshal(EnrolRequest{Node: "anchor"})
|
||||
s.enrolling(context.Background(), anEnrolment{body: body})
|
||||
|
||||
want := broker.NodeConsumer("anchor")
|
||||
if len(made.consumers) != 1 || made.consumers[0].Name != want.Name || made.consumers[0].Stream != want.Stream {
|
||||
t.Fatalf("the enrolled node was given %v, want its own declaration consumer %v", made.consumers, want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package link_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A token issued for a tunnel key takes that key and no other** (novox/hq ADR 0169). The hub was
|
||||
// sent the key before the token was shown, so another key is a machine it does not know.
|
||||
func TestATokenIssuedForATunnelKeyTakesThatKeyAndNoOther(t *testing.T) {
|
||||
inv, ident := aMeshReadyToEnrol(t)
|
||||
ctx := context.Background()
|
||||
secret, public := aTokenFor(t, inv, "joiner")
|
||||
node, err := inv.NodeByName(ctx, "joiner")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const issuedFor = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
|
||||
if err := inv.BindTokenToKey(ctx, node.ID, issuedFor); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e := link.Enrolment{Inventory: inv, Identity: ident}
|
||||
|
||||
_, err = e.Enrol(ctx, link.EnrolRequest{Node: "joiner", Secret: secret, PublicKey: public,
|
||||
OverlayKey: "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="})
|
||||
if err == nil || !strings.Contains(err.Error(), "issued for the tunnel key") {
|
||||
t.Fatalf("a token issued for one key took another: %v", err)
|
||||
}
|
||||
|
||||
if _, err := e.Enrol(ctx, link.EnrolRequest{Node: "joiner", Secret: secret, PublicKey: public,
|
||||
OverlayKey: issuedFor}); err != nil {
|
||||
t.Fatalf("the key the token was issued for was refused: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -86,6 +86,18 @@ func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (reply Enrol
|
||||
if err != nil {
|
||||
return EnrolReply{}, err
|
||||
}
|
||||
// **A token issued for a tunnel key takes that key and no other** (novox/hq ADR 0169). The hub
|
||||
// was told it before the token was shown; another key is a machine the hub does not know.
|
||||
// Checked before anything is recorded, so a refusal changes nothing.
|
||||
bound, err := e.Inventory.TokenKey(ctx, secret)
|
||||
if err != nil {
|
||||
return EnrolReply{}, err
|
||||
}
|
||||
if bound != "" && request.OverlayKey != bound {
|
||||
return EnrolReply{}, fmt.Errorf("%s's token was issued for the tunnel key %s and the machine "+
|
||||
"offered %q — the key it made with `nox-mesh-host key` is the one to issue for",
|
||||
node.Name, bound, request.OverlayKey)
|
||||
}
|
||||
|
||||
if _, err := e.Identity.RecordNodeKey(ctx, node.ID, public); err != nil {
|
||||
return EnrolReply{}, fmt.Errorf("%s's key could not be recorded: %w", node.Name, err)
|
||||
|
||||
@@ -223,11 +223,10 @@ func kindOfSubject(subject string) (string, bool) {
|
||||
return KindCatchUp, true
|
||||
case broker.ControllerFollows[3]:
|
||||
return KindSourceMoved, true
|
||||
case BuildOutcome(), BuildOutcomeOf(TheBuildMachineBefore):
|
||||
case BuildOutcome():
|
||||
// A build's outcome is the role's event now, so it arrives on the events stream rather than
|
||||
// the control branch — and is acted on by the same handler, because what the controller does
|
||||
// with it did not change (novox/hq ADR 0121). From either build role while the handover
|
||||
// runs (ADR 0190): the old builder still answers on the retired seat until it is unassigned.
|
||||
// with it did not change (novox/hq ADR 0121).
|
||||
return KindBuilt, true
|
||||
}
|
||||
return "", false
|
||||
|
||||
@@ -73,6 +73,8 @@ type Server struct {
|
||||
inbound Inbound
|
||||
bus Bus
|
||||
js *broker.JetStream
|
||||
// consumers makes a node's declaration consumer as it enrols; the bus connection, or a stand-in.
|
||||
consumers interface{ EnsureConsumer(broker.Consumer) error }
|
||||
|
||||
enroller Enroller
|
||||
listener Listener
|
||||
@@ -383,6 +385,7 @@ func (s *Server) enrolling(ctx context.Context, m Control) {
|
||||
default:
|
||||
reply = accepted
|
||||
s.log.Printf("enrolled %s", accepted.Node)
|
||||
s.hearsItsDeclarations(accepted.Node)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -402,6 +405,24 @@ func (s *Server) enrolling(ctx context.Context, m Control) {
|
||||
_ = m.Took()
|
||||
}
|
||||
|
||||
// hearsItsDeclarations makes the consumer a node reads its declarations through, as it enrols and
|
||||
// before it is answered.
|
||||
//
|
||||
// **Created at enrolment, as the consumer's own doc has always said** (novox/hq 04-ISSUES/146). It
|
||||
// was asserted only when the control plane started, so the first machine of a mesh — which enrols
|
||||
// after the control plane is already up — joined and then heard nothing, its host retrying "consumer
|
||||
// not found" for ever. Failing here is said and does not unspend the token: the next start of the
|
||||
// control plane asserts it again.
|
||||
func (s *Server) hearsItsDeclarations(node string) {
|
||||
if s.consumers == nil {
|
||||
return
|
||||
}
|
||||
if err := s.consumers.EnsureConsumer(broker.NodeConsumer(node)); err != nil {
|
||||
s.log.Printf("%s enrolled, and how it hears its declarations could not be made — it will hear "+
|
||||
"nothing until the control plane next starts: %v", node, err)
|
||||
}
|
||||
}
|
||||
|
||||
// wasBuilt keeps what a builder said, whichever way it went.
|
||||
//
|
||||
// This is for results nobody was waiting for. A build asked for with `build` is answered directly
|
||||
|
||||
@@ -169,9 +169,10 @@ func (g *Generator) Graph() Graph { return g.graph }
|
||||
//
|
||||
// - No floor: no header, no localhost, no `127.0.1.1` — those are the machine's, above the region.
|
||||
// - A machine's own line is marked, and its mesh name resolves to its mesh address, not loopback.
|
||||
// - `.Names` is the machines (novox/hq ADR 0191): a route's internal name is under its node's
|
||||
// internal domain and the resolver answers it by wildcard, and a public name is public DNS's.
|
||||
// Machines with no address yet are already left out of the set.
|
||||
// - `.Names` is every name the mesh serves (issue 111), so anything on the machine reaching a
|
||||
// routed name through its resolver finds the machine serving it; machines with no address yet
|
||||
// are already left out of the set. A routed name is one alias, itself — a machine has a bare
|
||||
// name beside its full one, a routed name has nothing beside it (issue 157).
|
||||
const hostsTemplate = "# The mesh's names. This region is replaced whenever a machine joins or leaves.\n" +
|
||||
"{{range .Names}}{{.Address}}\t{{.FQDN}}{{if ne .Name .FQDN}}\t{{.Name}}{{end}}{{if eq .Name $.Node}}\t# this machine{{end}}\n{{end}}"
|
||||
|
||||
|
||||
@@ -55,6 +55,26 @@ type Token struct {
|
||||
// that it speaks the firewall found on the machine, because an adopted node keeps that firewall
|
||||
// in force. Absent for a converged node, so a converged token is byte for byte what it was.
|
||||
Adopted bool `json:"adopted,omitempty"`
|
||||
|
||||
// Tunnel is the one peer a joining machine needs, when the token was issued for its tunnel key
|
||||
// (novox/hq ADR 0169). The machine brings its tunnel up from this alone and reaches the bus over
|
||||
// it, at an address on the private network — so the bus is never open to the internet. Absent
|
||||
// on a token issued without a key, which then reads byte for byte as before.
|
||||
Tunnel *Tunnel `json:"tunnel,omitempty"`
|
||||
}
|
||||
|
||||
// Tunnel is the joining machine's side of its first tunnel: its own address and the hub to reach.
|
||||
type Tunnel struct {
|
||||
// Key is the public half of the key the machine made itself, which this token was issued for.
|
||||
// The private half never left the machine (novox/hq ADR 0004).
|
||||
Key string `json:"key"`
|
||||
// Address is the machine's own address on the private network, with its prefix.
|
||||
Address string `json:"address"`
|
||||
// Range is the private network, routed through the hub until the machine is told more.
|
||||
Range string `json:"range"`
|
||||
// HubKey and HubEndpoint are the hub's tunnel key and where it is dialled.
|
||||
HubKey string `json:"hub_key"`
|
||||
HubEndpoint string `json:"hub_endpoint"`
|
||||
}
|
||||
|
||||
// Missing names the parts that are not filled in.
|
||||
@@ -83,6 +103,19 @@ func (t Token) Missing() []string {
|
||||
if strings.TrimSpace(t.Secret) == "" {
|
||||
missing = append(missing, "the one-time secret — nothing to present")
|
||||
}
|
||||
if t.Tunnel != nil {
|
||||
for _, part := range []struct{ value, says string }{
|
||||
{t.Tunnel.Key, "the machine's own tunnel key — the hub would not know it"},
|
||||
{t.Tunnel.Address, "the machine's address on the private network"},
|
||||
{t.Tunnel.Range, "the private network's range — nothing to route through the hub"},
|
||||
{t.Tunnel.HubKey, "the hub's tunnel key — nothing to dial"},
|
||||
{t.Tunnel.HubEndpoint, "where the hub's tunnel is dialled"},
|
||||
} {
|
||||
if strings.TrimSpace(part.value) == "" {
|
||||
missing = append(missing, part.says)
|
||||
}
|
||||
}
|
||||
}
|
||||
return missing
|
||||
}
|
||||
|
||||
|
||||
@@ -172,3 +172,38 @@ func TestATokenWithNoNameIsRefused(t *testing.T) {
|
||||
t.Fatalf("the refusal does not say what is missing: %v", without.Missing())
|
||||
}
|
||||
}
|
||||
|
||||
// A token issued for a tunnel key carries the one peer a joining machine needs (novox/hq ADR 0169),
|
||||
// and says which part is missing rather than producing a tunnel that never answers.
|
||||
func TestATokenThroughTheTunnelCarriesThePeerOrSaysWhatIsMissing(t *testing.T) {
|
||||
whole := Token{Node: "n", Broker: "10.42.0.1:4222", Fingerprint: "sha256:x", Signer: make([]byte, 32), Secret: "s",
|
||||
Tunnel: &Tunnel{Key: "k", Address: "10.42.0.9/32", Range: "10.42.0.0/16", HubKey: "h", HubEndpoint: "198.51.100.1:51820"}}
|
||||
if !whole.Complete() {
|
||||
t.Fatalf("a whole token through the tunnel reads as missing %v", whole.Missing())
|
||||
}
|
||||
encoded, err := whole.Encode()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
back, err := Decode(encoded)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if back.Tunnel == nil || *back.Tunnel != *whole.Tunnel {
|
||||
t.Fatalf("the tunnel did not survive the round trip: %+v", back.Tunnel)
|
||||
}
|
||||
|
||||
part := whole
|
||||
part.Tunnel = &Tunnel{Key: "k", Address: "10.42.0.9/32"}
|
||||
if len(part.Missing()) != 3 {
|
||||
t.Errorf("a tunnel without the hub and the range should name three missing parts: %v", part.Missing())
|
||||
}
|
||||
|
||||
// And a token issued without a key carries no tunnel at all, byte for byte as before.
|
||||
plain := whole
|
||||
plain.Tunnel = nil
|
||||
raw, _ := plain.Encode()
|
||||
if strings.Contains(raw, "tunnel") {
|
||||
t.Error("a token without a key mentions a tunnel")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user