diff --git a/cmd/mesh-controller/assignment_accounts_test.go b/cmd/mesh-controller/assignment_accounts_test.go index 0dd9cbc2..646da68f 100644 --- a/cmd/mesh-controller/assignment_accounts_test.go +++ b/cmd/mesh-controller/assignment_accounts_test.go @@ -372,7 +372,7 @@ func TestTheAccountsAModuleSpeaksWithOnANode(t *testing.T) { t.Error("sealed no password") } return c.sealErr - }) + }, func() error { acts.did("after"); return nil }) if (err != nil) != (c.sealErr != nil) { t.Fatalf("error %v, want %v", err, c.sealErr) } @@ -385,6 +385,52 @@ func TestTheAccountsAModuleSpeaksWithOnANode(t *testing.T) { }) } + // Issue 774: where a vault is installed the controller mints nothing; `module issue` and an assignment's issue are + // forwarded to the vault's issue-account, and what the account needs beside its value still runs. A refusal fails + // `module issue` and only says itself on an assignment's issue: the vault mints on the want when it hears it. + for _, c := range []struct { + name string + only bool + answer string + refuse error + want string + fails bool + }{ + {"module issue: forwarded, minted by the vault", false, `{"minted":true,"account":{"secret":"own/ace/mesh-build-go/broker","generation":1}}`, nil, "after", false}, + {"an assignment's issue: forwarded, held already", true, `{"minted":false,"why":"held already"}`, nil, "after", false}, + {"an assignment's issue: refused, said", true, "", errors.New("nobody wants it yet"), "after", false}, + {"module issue: refused, failed", false, "", errors.New("nobody wants it"), "", true}, + } { + t.Run("forwarded to the vault: "+c.name, func(t *testing.T) { + was := vaultIssuer + t.Cleanup(func() { vaultIssuer = was }) + var asked []string + vaultIssuer = func(context.Context, accountIssuer) (vaultAsker, bool, error) { + return func(verb string, args map[string]any) (json.RawMessage, error) { + asked = append(asked, verb+" "+fmt.Sprint(args["user"])) + return json.RawMessage(c.answer), c.refuse + }, true, nil + } + acts := &anAccountStore{} + ctx, got := withIssued(t.Context()) + if c.only { + ctx = onlyIfAbsent(ctx) + } + err := issueAccount(ctx, acts, inventory.BusUser{Username: "ace.mesh-build-go", Kind: inventory.BusModule}, + func(string) error { acts.did("seal"); return nil }, func() error { acts.did("after"); return nil }) + if (err != nil) != c.fails || strings.Join(acts.acts, " ") != c.want { + t.Fatalf("error %v, acts %q; want failed %v, acts %q (nothing minted, locked or sealed here)", err, + strings.Join(acts.acts, " "), c.fails, c.want) + } + if strings.Join(asked, ",") != "issue-account ace.mesh-build-go" { + t.Errorf("the vault was asked %v", asked) + } + if minted := len(got.list()) == 1; minted != strings.Contains(c.answer, `"minted":true`) { + t.Errorf("said issued %v", got.list()) + } + }) + } + // The accounts of a module that no longer resolves on its node are stale, its provisioner client's among them; a // module still resolved (assigned or required), an account of another kind, and a node whose resolution is unknown // are left alone (issue 1945: ace's mesh-build-go rows from 17:19, marked sealed by migration 0108). diff --git a/cmd/mesh-controller/issued_to_vault.go b/cmd/mesh-controller/issued_to_vault.go index 41703413..e0c223bd 100644 --- a/cmd/mesh-controller/issued_to_vault.go +++ b/cmd/mesh-controller/issued_to_vault.go @@ -2,9 +2,15 @@ package main import ( "context" + "errors" "fmt" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/inventory" ) @@ -15,17 +21,19 @@ import ( // controller mints such an account and writes its hash to the bus's user list, as before, and then hands the whole // value at once to the vault's `accept` with origin `issued`: sealed here to the key the vault made for that call, so // the vault alone opens it (ADR 0349 clause 8). The vault keeps it as the module's own secret on that machine and -// raises `secret-changed`; a re-issue is the same hand with a new value, which replaces the old in place. An account -// not covered is minted and kept as it was and handed to nobody. +// raises `secret-changed`; a re-issue is the same hand with a new value, which replaces the old in place. Until issue 774 an +// account not covered was minted and kept as it was and handed to nobody. +// +// **Since issue 774 every account the controller mints is handed, covered or not**, once a vault is installed: until +// the grants part's take-over the controller still mints (the bus loads its list), and a vault that holds every value +// at once needs no node to report a hash for it to adopt (ADR 0547 clause 9). From the take-over the vault mints every +// account itself (issueAccount forwards to its issue-account), so this hand is reached only before it, and by +// `rollout mint`'s re-issue of a module's account onto a new bus. The vault never lets such a hand replace an account it +// minted. // vaultAskerForIssue is the vault as `module issue` asks it. A test replaces it. var vaultAskerForIssue = func(ctx context.Context) vaultAsker { return theVault(ctx) } -// accountIsCovered says whether the bus accounts minted for m are the vault's to hold (ADR 0374 clause 1). -func accountIsCovered(m catalogue.Manifest) bool { - return catalogue.CarriesVaultSealed(m.Module) || m.Moved -} - // vaultInstalled says whether the vault is assigned to any machine. Until it is, as in genesis (ADR 0113 §15 to §18, // mesh-host's bootstrap, which issues the bus's and the provisioner's accounts before the vault step), there is no // vault to hand to: the value is kept in the controller's store as before, and the vault takes it afterwards by the @@ -44,13 +52,45 @@ func vaultInstalled(ctx context.Context, inv *inventory.Inventory) (bool, error) return false, nil } -// handIssuedToVault hands the value just minted for module's own secret name on node to the vault, when the module's -// accounts are covered, and does nothing otherwise. Loud when the vault does not take it: the bus's user list -// already holds the new hash, so a vault that kept the old value would serve a password that no longer logs in. -func handIssuedToVault(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node, name, held string) error { - if !accountIsCovered(m) { +// grantsTookOver says whether the grants part took over the bus's user list: its key `user-list.sealed` says +// FollowGrants (novox/hq ADR 0547 clauses 7 and 8), which the bus node's engine follows. No bucket or no key yet is no +// take-over; a bus that cannot be read is an error, never a guess. A test replaces it. +var grantsTookOver = func(ctx context.Context) (bool, error) { + over := false + err := onTheBus(func(conn *nats.Conn) error { + js, err := jetstream.New(conn) + if err != nil { + return err + } + kv, err := js.KeyValue(ctx, broker.GrantsUserListBucket) + if errors.Is(err, jetstream.ErrBucketNotFound) { + return nil + } else if err != nil { + return err + } + e, err := kv.Get(ctx, broker.GrantsUserListKey) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil + } else if err != nil { + return err + } + g, err := sdkdelivery.DecodeGrantsUserList(e.Value()) + if err != nil { + return err + } + over = g.Follow == sdkdelivery.FollowGrants return nil + }) + if err != nil { + return false, fmt.Errorf("whether the grants part took over the bus's user list cannot be read: %w", err) } + return over, nil +} + +// handIssuedToVault hands the value just minted for module's own secret name on node to the vault, once one is +// installed, covered or not (issue 774), and does nothing before. Loud when the vault does not take it: the bus's user +// list already holds the new hash, so a vault that kept the old value would serve a password that no longer logs in. +func handIssuedToVault(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node, name, held string) error { if installed, err := vaultInstalled(ctx, inv); err != nil { return fmt.Errorf("whether the vault is installed cannot be read, so %s of %s was not handed to it: %w", name, m.Module, err) diff --git a/cmd/mesh-controller/issued_to_vault_test.go b/cmd/mesh-controller/issued_to_vault_test.go index 50068d32..13e51c5f 100644 --- a/cmd/mesh-controller/issued_to_vault_test.go +++ b/cmd/mesh-controller/issued_to_vault_test.go @@ -8,21 +8,50 @@ import ( "strings" "testing" + "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/testbus" ) -// Every other test mints without a vault to hand to: a stand-in that takes anything, replaced by the tests below. +// grantsTookOverOnTheBus is the reader of the grants part's take-over a test reads on its own bus. +var grantsTookOverOnTheBus = grantsTookOver + +// Every other test mints without a vault to hand to: a stand-in that takes anything, replaced by the tests below; and +// before the grants part's take-over, which no test bus says. func init() { vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask } + grantsTookOver = func(context.Context) (bool, error) { return false, nil } } -// A covered account is handed to the vault when it is minted (novox/hq ADR 0374, issue 787): the whole value, sealed -// to the key the vault made for the call, with origin issued. A module that is neither core nor moved hands nothing. -func TestModuleIssueHandsACoveredAccountToTheVaultAndOnlyThat(t *testing.T) { +// theGrantsPartTookOver says the grants part's take-over is on, or off, for the rest of the test. +func theGrantsPartTookOver(t *testing.T, over bool) { + t.Helper() + was := grantsTookOver + grantsTookOver = func(context.Context) (bool, error) { return over, nil } + t.Cleanup(func() { grantsTookOver = was }) +} + +// An account is handed to the vault when it is minted (novox/hq ADR 0374, issue 787): the whole value, sealed to the +// key the vault made for the call, with origin issued. Since issue 774 every account is, covered or not, until the +// grants part's take-over, read on the bus as the bus node reads it; from the take-over nothing is minted here and the +// issue is forwarded to the vault's issue-account. +func TestModuleIssueHandsEveryAccountToTheVaultUntilTheTakeOver(t *testing.T) { open := aMesh(t) ctx := t.Context() theVaultIsInstalled(t, open) + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + servingBus.Store(js) + t.Cleanup(func() { servingBus.Store(nil) }) + was := grantsTookOver + grantsTookOver = grantsTookOverOnTheBus + t.Cleanup(func() { grantsTookOver = was }) inv := open.inventory known := broker.Broker{Fingerprint: "ab"} @@ -66,11 +95,11 @@ func TestModuleIssueHandsACoveredAccountToTheVaultAndOnlyThat(t *testing.T) { t.Errorf("the vault was told %s, without %s", said, want) } } - if n := strings.Count(said, `"origin":"issued"`); n != 3 { - t.Errorf("%d accounts were handed with origin issued, want 3 (the core's two and the moved module's): %s", n, said) + if n := strings.Count(said, `"origin":"issued"`); n != 4 { + t.Errorf("%d accounts were handed with origin issued, want 4 (the core's two, the moved and the plain module's): %s", n, said) } - if strings.Contains(said, "kept-bot") { - t.Errorf("an account that is neither core nor moved was handed to the vault: %s", said) + if !strings.Contains(said, "kept-bot") { + t.Errorf("an account that is neither core nor moved was not handed to the vault: %s", said) } if strings.Contains(said, "first-core-password") || strings.Contains(said, "first-moved-password") { t.Errorf("a value crossed the bus in the clear: %s", said) @@ -83,6 +112,38 @@ func TestModuleIssueHandsACoveredAccountToTheVaultAndOnlyThat(t *testing.T) { !strings.Contains(got, `"url":"nats://10.0.0.1:4222"`) || !strings.Contains(got, `"fingerprint":"ab"`) { t.Errorf("the vault was handed %q after a re-issue, not the new account whole", got) } + + // The take-over: the grants part's key says the bus node follows its list. From then the vault mints. + api, err := jetstream.New(js.Conn()) + if err != nil { + t.Fatal(err) + } + kv, err := api.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.GrantsUserListBucket}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = api.DeleteKeyValue(context.Background(), broker.GrantsUserListBucket) }) + client := broker.Principal{Kind: broker.KindProvisioner, Node: "laptop", Module: "nats"}.Username() + for _, follow := range []string{sdkdelivery.FollowController, sdkdelivery.FollowGrants} { + value, err := sdkdelivery.EncodeGrantsUserList(sdkdelivery.GrantsUserList{Follow: follow}) + if err != nil { + t.Fatal(err) + } + if _, err := kv.Put(ctx, broker.GrantsUserListKey, value); err != nil { + t.Fatal(err) + } + before, _, _ := inv.BusUserHash(ctx, client) + if err := issueProvisionerClient(ctx, inv, core, "laptop", known, "10.0.0.1:4222"); err != nil { + t.Fatal(err) + } + after, _, _ := inv.BusUserHash(ctx, client) + if minted := before != after; minted != (follow == sdkdelivery.FollowController) { + t.Errorf("following %s's list, the controller minted %s: %v", follow, client, minted) + } + } + if said := vault.said(); strings.Count(said, `"verb":"issue-account"`) != 1 || !strings.Contains(said, `"user":"`+client+`"`) { + t.Errorf("the vault was not asked once, after the take-over, to issue %s: %s", client, said) + } } // A vault that does not take the value fails the issue loudly: the bus's user list holds the new hash already. diff --git a/cmd/mesh-controller/modules.go b/cmd/mesh-controller/modules.go index 1b2a5140..c355e753 100644 --- a/cmd/mesh-controller/modules.go +++ b/cmd/mesh-controller/modules.go @@ -1018,8 +1018,21 @@ const issueLockedFor = 2 * time.Minute // left ace's mesh-build-go with no credential), under the store's lock on the account, so two issuers never interleave // their look, mint and seal: the hash the bus holds is always the password sealed. A seal that fails leaves the // account unsealed, issued again by the next issue. -func issueAccount(ctx context.Context, accounts accountIssuer, u inventory.BusUser, seal func(password string) error) error { +// +// **From the grants part's take-over the controller mints nothing** (issue 774, research 072's gap "vault mints after +// the lease"; the controller is deprecated and hands its duties over): the bus then loads the grants part's list, whose +// hashes are the vault's, so the issue is forwarded to the vault's issue-account and only after runs (what the account +// needs beside its value: its durable consumers). **Before it** the bus loads the controller's own list, whose hashes +// come from its store alone, so an account the vault minted would be refused: the controller mints as before and hands +// every account it mints to an installed vault at once (handIssuedToVault), so none waits on a node reporting its hash. +func issueAccount(ctx context.Context, accounts accountIssuer, u inventory.BusUser, seal func(password string) error, + after func() error) error { only, _ := ctx.Value(mintOnlyIfAbsent{}).(bool) + if ask, installed, err := vaultIssuer(ctx, accounts); err != nil { + return fmt.Errorf("whether the vault issues bus accounts yet cannot be read, so %s is not issued: %w", u.Username, err) + } else if installed { + return forwardToTheVault(ctx, ask, u.Username, only, after) + } if !only { password, err := accounts.MintBusPassword(ctx, u) if err != nil { @@ -1062,6 +1075,67 @@ func issueAccount(ctx context.Context, accounts accountIssuer, u inventory.BusUs return nil } +// vaultIssuer answers the vault to forward an issue to, and whether the issue is the vault's: one is installed +// (vaultInstalled) and the grants part took over the bus's user list (grantsTookOver). A store that is not the +// controller's inventory (a test's) has none. A test replaces it. +var vaultIssuer = func(ctx context.Context, accounts accountIssuer) (vaultAsker, bool, error) { + inv, ok := accounts.(*inventory.Inventory) + if !ok { + return nil, false, nil + } + installed, err := vaultInstalled(ctx, inv) + if err != nil || !installed { + return nil, false, err + } + if over, err := grantsTookOver(ctx); err != nil || !over { + return nil, false, err + } + return vaultAskerForIssue(ctx), true, nil +} + +// issuedByTheVault is the vault's answer to issue-account (mesh-vault heldBackOrMinted). +type issuedByTheVault struct { + Minted bool `json:"minted"` + Why string `json:"why"` + Account struct { + Secret string `json:"secret"` + Generation int `json:"generation"` + } `json:"account"` +} + +// forwardToTheVault asks the vault to issue user (its verb issue-account) and then runs after. The vault mints only +// what the grants part wants and never anew a value it holds, so asking twice, or with the want's own mint, is harmless. +// An issue on an assignment (only) is not failed by a refusal: the vault mints on the want itself when it is heard, and +// the refusal is said; `module issue` fails with it. +func forwardToTheVault(ctx context.Context, ask vaultAsker, user string, only bool, after func() error) error { + raw, err := ask("issue-account", map[string]any{"user": user}) + if err != nil { + if !only { + return fmt.Errorf("the vault is the one issuer of bus accounts and did not issue %s: %w", user, err) + } + fmt.Printf(" %s is the vault's to mint, on the grants part's want of it; issue-account said: %v\n", user, err) + } else { + var got issuedByTheVault + if err := json.Unmarshal(raw, &got); err != nil { + return fmt.Errorf("the vault's answer to issue-account %s does not read: %w", user, err) + } + if got.Minted { + fmt.Printf("bus user %s minted by the vault as %s (generation %d)\n", user, got.Account.Secret, got.Account.Generation) + if g, _ := ctx.Value(issuedNowKey{}).(*issuedNow); g != nil { + g.mu.Lock() + g.users = append(g.users, user) + g.mu.Unlock() + } + } else { + fmt.Printf("bus user %s not minted anew by the vault: %s\n", user, got.Why) + } + } + if after == nil { + return nil + } + return after() +} + // issueOnTheNewBus gives an assigned module its credential on the bus being built. // // **Three things differ from a management call, and each is the point of the move.** The credential @@ -1090,7 +1164,7 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue if err := issueAccount(ctx, inv, inventory.BusUser{Username: user, Kind: busKindOf(m.Module), Node: node, Module: m.Module}, func(password string) error { return issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password) - }); err != nil { + }, func() error { return ensureModuleConsumer(m, node, busAddress) }); err != nil { return err } } @@ -1118,7 +1192,7 @@ func issueProcess(ctx context.Context, inv *inventory.Inventory, m catalogue.Man return issueAccount(ctx, inv, inventory.BusUser{Username: user, Kind: inventory.BusProcess, Node: node, Module: m.Module}, func(password string) error { return sealProcess(ctx, inv, m, pr, node, busAddress, known, reachable, user, password) - }) + }, func() error { return ensureProcessQueues(m, pr, node, busAddress) }) } // sealProcess seals a process's minted account to its node, hands it to the vault and makes its work-queue consumers. @@ -1135,30 +1209,40 @@ func sealProcess(ctx context.Context, inv *inventory.Inventory, m catalogue.Mani if err := handIssuedToVault(ctx, inv, m, node, name, string(held)); err != nil { return err } - if len(pr.Queues) > 0 { - if busAddress == "" { - fmt.Printf(" %s's process %s consumes its work queue; its consumer is created when the bus is "+ - "reachable (`push --behind`, then `rollout mint` again is harmless)\n", m.Module, pr.Process) - } else { - js, err := broker.Dial(busAddress) - if err != nil { - return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+ - "%s's process %s's work-queue consumer: %w", m.Module, pr.Process, err) - } - defer js.Close() - who := broker.Principal{Kind: broker.KindProcess, Node: node, Module: m.Module, Process: pr.Process} - for _, q := range pr.Queues { - if err := js.EnsureConsumer(broker.QueueConsumer(who, q)); err != nil { - return err - } - } - } + if err := ensureProcessQueues(m, pr, node, busAddress); err != nil { + return err } fmt.Printf("bus user %s minted for %s's process %s, scoped to its bus declaration\n", user, m.Module, pr.Process) fmt.Printf(" sealed to %s as %s. It arrives with the next push — `push %s` to send it\n", node, name, node) return nil } +// ensureProcessQueues makes a process's work-queue consumers where the bus can be reached: what its account needs beside +// its value, whoever minted it. +func ensureProcessQueues(m catalogue.Manifest, pr broker.ProcessPrincipal, node, busAddress string) error { + if len(pr.Queues) == 0 { + return nil + } + if busAddress == "" { + fmt.Printf(" %s's process %s consumes its work queue; its consumer is created when the bus is "+ + "reachable (`push --behind`, then `rollout mint` again is harmless)\n", m.Module, pr.Process) + return nil + } + js, err := broker.Dial(busAddress) + if err != nil { + return fmt.Errorf("the mesh cannot reach the bus to create %s's process %s's work-queue consumer: %w", + m.Module, pr.Process, err) + } + defer js.Close() + who := broker.Principal{Kind: broker.KindProcess, Node: node, Module: m.Module, Process: pr.Process} + for _, q := range pr.Queues { + if err := js.EnsureConsumer(broker.QueueConsumer(who, q)); err != nil { + return err + } + } + return nil +} + // heldProcessAccount is a process's account, in the provisioner client's form (novox/hq ADR 0436 §2): // {url, fingerprint, node, module, user, password} — never claims, since a process holds no seat (§3). func heldProcessAccount(known broker.Broker, reachable, node, module, user, password string) ([]byte, error) { @@ -1191,7 +1275,7 @@ func issueProvisionerClient(ctx context.Context, inv *inventory.Inventory, m cat return issueAccount(ctx, inv, inventory.BusUser{Username: user, Kind: inventory.BusProvisioner, Node: node, Module: m.Module}, func(password string) error { return sealProvisionerClient(ctx, inv, m, node, known, reachable, user, password) - }) + }, nil) } // sealProvisionerClient seals the provisioner client's minted account to its machine and hands it to the vault. @@ -1263,28 +1347,8 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife if err := handIssuedToVault(ctx, inv, m, node, "broker", string(held)); err != nil { return err } - - // And how it hears what it consumes. Derived from its declaration, and only when it declared - // something: a module that consumes nothing needs no consumer, and creating one would be a - // durable subscription nobody reads. - if consumer, needed := broker.ConsumerFor(broker.Principal{ - Kind: broker.KindModule, Node: node, Module: m.Module, - Emits: m.EmitsAll(), Consumes: m.Consumes, Serves: m.Tools, - }); needed { - if busAddress == "" { - fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push --behind`, then "+ - "`rollout mint` again is harmless)\n", m.Module) - } else { - js, err := broker.Dial(busAddress) - if err != nil { - return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+ - "how %s hears what it consumes: %w", m.Module, err) - } - defer js.Close() - if err := js.EnsureConsumer(consumer); err != nil { - return err - } - } + if err := ensureModuleConsumer(m, node, busAddress); err != nil { + return err } fmt.Printf("bus user %s minted for %s, scoped to what it emits and consumes\n", user, m.Module) @@ -1294,6 +1358,30 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife return nil } +// ensureModuleConsumer makes how a module hears what it consumes, where the bus can be reached: what its account needs +// beside its value, whoever minted it. Derived from its declaration, and only when it declared something: a module that +// consumes nothing needs no consumer, and creating one would be a durable subscription nobody reads. +func ensureModuleConsumer(m catalogue.Manifest, node, busAddress string) error { + consumer, needed := broker.ConsumerFor(broker.Principal{ + Kind: broker.KindModule, Node: node, Module: m.Module, + Emits: m.EmitsAll(), Consumes: m.Consumes, Serves: m.Tools, + }) + if !needed { + return nil + } + if busAddress == "" { + fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push --behind`, then "+ + "`rollout mint` again is harmless)\n", m.Module) + return nil + } + js, err := broker.Dial(busAddress) + if err != nil { + return fmt.Errorf("the mesh cannot reach the bus to create how %s hears what it consumes: %w", m.Module, err) + } + defer js.Close() + return js.EnsureConsumer(consumer) +} + // whereItComesFrom is the provenance a module handed over by hand records, and what a record must // say to be worth anything later. // diff --git a/cmd/mesh-controller/process_issue_integration_test.go b/cmd/mesh-controller/process_issue_integration_test.go index be013ced..5650ab37 100644 --- a/cmd/mesh-controller/process_issue_integration_test.go +++ b/cmd/mesh-controller/process_issue_integration_test.go @@ -21,7 +21,9 @@ import ( // account against the store — no bus to dial, busAddress "" — for mesh-mail's real manifest shape (review // round 1 finding 1: no own secret `broker` at all, its tools served by the tool runner), and issuing it // again is the path every bus account takes (`rollout mint`'s KindProcess case, and a re-run of `module -// issue`): it rotates the password, and the old hash no longer matches. +// issue`): it rotates the password, and the old hash no longer matches. Issue 774: every account minted is handed to the +// vault, covered or not, until the grants part's take-over; from it the issue is forwarded to the vault's issue-account, +// and the controller's hash stays as it was. func TestModuleIssueMintsAndDeliversAProcesssAccount(t *testing.T) { open := aMesh(t) ctx := t.Context() @@ -29,7 +31,6 @@ func TestModuleIssueMintsAndDeliversAProcesssAccount(t *testing.T) { theVaultIsInstalled(t, open) m := mailSenderWithProcess() - m.Moved = true // so handIssuedToVault's accountIsCovered is true, and the hand-off is exercised whole register(t, open, m) vault := newStandInVault(t) @@ -84,4 +85,13 @@ func TestModuleIssueMintsAndDeliversAProcesssAccount(t *testing.T) { if hash1 == hash2 { t.Fatal("issuing the process's account again did not rotate its password") } + + theGrantsPartTookOver(t, true) + issue() + if hash3, held, err := inv.BusUserHash(ctx, user); err != nil || !held || hash3 != hash2 { + t.Fatalf("the controller minted again after the grants part's take-over (held %v, %v)", held, err) + } + if said := vault.said(); !strings.Contains(said, `"verb":"issue-account"`) || !strings.Contains(said, `"user":"`+user+`"`) { + t.Errorf("the vault was not asked to issue %s after the take-over: %s", user, said) + } } diff --git a/cmd/mesh-controller/reissue_verified_sender.go b/cmd/mesh-controller/reissue_verified_sender.go index 0ad71848..daeec95b 100644 --- a/cmd/mesh-controller/reissue_verified_sender.go +++ b/cmd/mesh-controller/reissue_verified_sender.go @@ -207,8 +207,13 @@ func giveVerifiedSendersBack(ctx context.Context, d *doctor) { } // reissueVerifiedSendersOn is reissueVerifiedSenders on the bus: memberships read from the AssignmentsStream and -// published to it. +// published to it. Once the stream is gone (the bus module's tools delete it at the lease write, research 072 row 33) +// the memberships are the grants part's alone, and there is nothing here to issue again. func reissueVerifiedSendersOn(ctx context.Context, bus link.OverNATS, records broker.Records, where broker.Placements) int { + if _, err := bus.JS.StreamInfo(broker.AssignmentsStream, nats.Context(ctx)); errors.Is(err, nats.ErrStreamNotFound) { + fmt.Printf(" the assignments stream is gone: the grants part issues the memberships, none is given a verified sender here\n") + return 0 + } return reissueVerifiedSenders(records, where, func(node, module string) ([]byte, uint64, error) { m, err := bus.JS.GetLastMsg(broker.AssignmentsStream, broker.MembershipSubject(node, module)) diff --git a/cmd/mesh-controller/reissue_verified_sender_integration_test.go b/cmd/mesh-controller/reissue_verified_sender_integration_test.go index 3dcc45d4..823fa87e 100644 --- a/cmd/mesh-controller/reissue_verified_sender_integration_test.go +++ b/cmd/mesh-controller/reissue_verified_sender_integration_test.go @@ -6,6 +6,8 @@ import ( "encoding/json" "testing" + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/link" "github.com/novox/mesh-controller/internal/testbus" @@ -17,7 +19,10 @@ func TestNatsAMembershipIssuedWithoutItsVerifiedSenderIsIssuedAgainWhenTheJudgem t.Fatal(err) } t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { + // The assignments stream is no longer the mesh's (research 072 row 33): it is asserted by nothing, and stays on a + // bus only until the lease write deletes it. Made here as it is left there. + if _, err := js.Context().AddStream(&nats.StreamConfig{Name: broker.AssignmentsStream, + Subjects: []string{"mesh.assignment.*.*"}, MaxMsgsPerSubject: 1, AllowDirect: true}); err != nil { t.Fatal(err) } bus := link.OverNATS{Conn: js.Conn(), JS: js.Context()} diff --git a/cmd/mesh-controller/vault_secrets_test.go b/cmd/mesh-controller/vault_secrets_test.go index e89183b1..1daa2a4d 100644 --- a/cmd/mesh-controller/vault_secrets_test.go +++ b/cmd/mesh-controller/vault_secrets_test.go @@ -49,6 +49,12 @@ func (v *standInVault) ask(verb string, args map[string]any) (json.RawMessage, e // The vault asks the desk's prompt and the provider's tool itself, in its own name (issue 723): one step from // the controller, which carries no key and no sealed value. switch verb { + case "issue-account": + // The vault is the one issuer once installed (issue 774): the controller forwards, and keeps nothing. + if v.refuse != nil { + return nil, v.refuse + } + return json.Marshal(map[string]any{"minted": true, "account": map[string]any{"user": args["user"], "generation": 1}}) case "secret-ask": if v.refuse != nil { return nil, v.refuse diff --git a/internal/broker/modstreams_live_test.go b/internal/broker/modstreams_live_test.go index 92e6f996..3478d88c 100644 --- a/internal/broker/modstreams_live_test.go +++ b/internal/broker/modstreams_live_test.go @@ -21,7 +21,13 @@ func TestAModuleStreamOnARealBus(t *testing.T) { t.Fatal(err) } defer js.Close() - for _, s := range []Stream{MeshStreams()[3], mailStream.Stream()} { + var events Stream + for _, s := range MeshStreams() { + if s.Name == EventsStream { + events = s + } + } + for _, s := range []Stream{events, mailStream.Stream()} { _ = js.js.DeleteStream(s.Name) if err := js.EnsureStream(s); err != nil { t.Fatalf("%s: %v", s.Name, err) diff --git a/internal/broker/stream_names_agreement_test.go b/internal/broker/stream_names_agreement_test.go index 9990f2d0..92876a25 100644 --- a/internal/broker/stream_names_agreement_test.go +++ b/internal/broker/stream_names_agreement_test.go @@ -14,7 +14,9 @@ import ( // queues are left on the bus until a cleanup removes them, and no module may take their names meanwhile. So the catalogue // reserves every name of the broker's own, and may reserve these retired ones beside them. func TestTheCataloguesReservedStreamNamesAreTheMeshsOwn(t *testing.T) { - retired := []string{"CHECKS_OTHER", "CHECKS_READY"} + // And ASSIGNMENTS (research 072 row 33): the bus module's tools delete it at the lease write, and the controller + // asserts it no more, so it is never made again. + retired := []string{"CHECKS_OTHER", "CHECKS_READY", "ASSIGNMENTS"} var own []string for _, s := range MeshStreams() { own = append(own, s.Name) diff --git a/internal/broker/streams.go b/internal/broker/streams.go index 052d47a8..14d6d49b 100644 --- a/internal/broker/streams.go +++ b/internal/broker/streams.go @@ -171,14 +171,9 @@ func MeshStreams() []Stream { Why: "one declaration per node, always the newest; a node that sees sequence n refuses " + "n-1 by construction (issue 107)", }, - { - Name: AssignmentsStream, - Subjects: []string{"mesh.assignment.*.*"}, - Retention: RetentionLastPerSubject, - Direct: true, - Why: "one membership per assignment, always the newest: what the mesh issued this module " + - "on this machine to serve and to reach (ADR 0160); read directly by the runtime it is for", - }, + // The assignments stream (ADR 0160) is no longer the mesh's: a runtime reads its membership from the grants + // part's state memberships (ADR 0571 clause 7), and the bus module's tools delete the stream at the lease write + // (research 072 row 33), so it is not declared here, which would make it again at the controller's next start. { Name: EventsStream, // A seat's own events ride here too: they are 1:many like any event, and the diff --git a/internal/broker/streams_test.go b/internal/broker/streams_test.go index 7a52d422..42b57243 100644 --- a/internal/broker/streams_test.go +++ b/internal/broker/streams_test.go @@ -123,10 +123,11 @@ func subjectMatches(filter, subject string) bool { // Each relationship's retention is the thing that makes it what it is (design 29 §4). func TestEachStreamCarriesTheRetentionItsShapeNeeds(t *testing.T) { want := map[string]Retention{ - "CONTROL": RetentionWorkQueue, - "NODES": RetentionLastPerSubject, - "EVENTS": RetentionLimits, - "ASSIGNMENTS": RetentionLastPerSubject, + "CONTROL": RetentionWorkQueue, + "NODES": RetentionLastPerSubject, + "EVENTS": RetentionLimits, + // No ASSIGNMENTS: the bus module's tools delete it at the lease write and nothing makes it again (research 072 + // row 33); a runtime reads its membership from the grants part's state. // What a consumer gave up on (novox/hq issue 330): the server's notice taken once, the message // kept until somebody acts. "DEAD_LETTER_NOTICES": RetentionWorkQueue, diff --git a/internal/ownmanifest/public_issuer_test.go b/internal/ownmanifest/public_issuer_test.go index b49e74d5..5cecb50c 100644 --- a/internal/ownmanifest/public_issuer_test.go +++ b/internal/ownmanifest/public_issuer_test.go @@ -1,6 +1,8 @@ package ownmanifest import ( + "path/filepath" + "slices" "testing" . "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -94,8 +96,21 @@ func TestRouteProxyKeepsItsPublicAccountDirectory(t *testing.T) { // 2 and 4): a Go bundle of the module's own Go module stamped for alpine, unpacked by an archive resource, mounted // read-only at /opt/mesh, called through args, and named under restart-on so a new program never leaves the old // one running. The container's name, network and logging are what the jail and the routes rely on. +// +// And its routes are the seat mesh-provisioner's plans alone (issue 774, research 072 row 34): it reads them, and +// receives no file of contributions; no module of this repository does. func TestRouteProxyRunsItsMountedProgramOnTheTrustImage(t *testing.T) { m := catalogueManifest(t, "route-proxy") + if !slices.Contains(m.Reads, "mesh-provisioner.plans") { + t.Errorf("route-proxy does not read mesh-provisioner.plans, where its routes are: %v", m.Reads) + } + dirs, _ := filepath.Glob(filepath.Join("..", "..", "modules", "*")) + for _, dir := range dirs { + if r := catalogueManifest(t, filepath.Base(dir)); len(r.Receives) > 0 { + t.Errorf("%s receives %v: a route, like every provision here, is served from the seat's plans, never a "+ + "received file", r.Module, r.Receives) + } + } var bundle *Artifact for i, a := range m.Build.Artifacts { if a.Name == "server" { diff --git a/modules/route-proxy/README.md b/modules/route-proxy/README.md index 2767eca7..f5ddb810 100644 --- a/modules/route-proxy/README.md +++ b/modules/route-proxy/README.md @@ -5,26 +5,50 @@ The shipping form of the reference reverse proxy (novox/hq ADR 0007, novox/hq 08 ## What it is -**A route is a grant.** A module that must be reachable declares `requires: ["route"]` and -contributes the name it wants and the port it listens on; this module provides `route`, receives -every consumer that asked as a file, and forwards by the `Host` header of each request to -`http://:` — where `at` is where the mesh says that consumer's machine is (empty means -this machine, reached over loopback). +**A route is a provision** (issue 774, research 072 row 34; the operator, 2026-10-31: "there is only 1 path, the other +is simply deprecated/invalid"). A module that must be reachable declares `requires: ["route"]` and contributes the +name it wants and the port it listens on; this module provides `route`. The controller composes each consumer's plan, +and the seat `mesh-provisioner` keeps it in its state `plans`: the plan's entry for the route names this proxy as +its provider and carries what the consumer contributes, composed (its names joined to the node's domains, its +endpoint's port, its policy), and where the consumer's machine is on the private network, as one opaque payload +(`contributed`) the seat never reads. The proxy follows the plans on the bus, reads each contribution with its own +contract (below), and forwards by the `Host` header of each request to `http://:`, or over loopback for a +consumer on its own machine. -The control plane does not write the proxy's configuration. It delivers the facts — the -contributions file at `receives.route` — and the proxy turns them into whatever it runs. That -boundary is why swapping this proxy for another costs nothing to the modules that publish through -it. +## Its contract: a package of its own + +**route-proxy defines its own contract, and other modules use it** (the operator, 2026-10-31). It is the Go module +`git.novox.be/novox/mesh-controller/modules/route-proxy/contract`, in `contract/` here with its own `go.mod`, standard +library only, released by tags `modules/route-proxy/contract/vX.Y.Z`, the way mesh-sdk is released (`go/vX.Y.Z`); this +module depends on it by its version like any other. It holds: + +- **the route a consumer asks** (`Request`, read strictly by `DecodeRequest`: no field it does not name, a name of + either kind, a port where the rule forwards, http or https, a positive body limit, a secret's file never a + credential) and its JSON schema (`route.schema.json`, `Schema`), the form of a manifest's `contributes.route`; +- **the events this proxy says** on `mesh.mod.route-proxy.event.`: `route.served` and `route.withdrawn` as the + names it serves change, `route.refused` with why for a route a plan gives it and it does not serve (`Event`); +- **what it keeps on the bus**, its states `routes` and `unrouted`, their keys and records (`Route`, `Unrouted`, + `Other`, `Key`, `Split`); +- **its tools' names and arguments** (`ToolRoutes`, `ToolRoute`, `ToolUnrouted`, `RoutesArgs`, `RouteArgs`). + +The controller does not write the proxy's configuration. It composes the facts into the plans and the proxy turns them +into whatever it runs. That boundary is why swapping this proxy for another costs nothing to the modules that publish +through it. **There is no received file and no second source:** a route no plan names is not served, and a module +declaring `receives.route` is refused by the catalogue check (mesh-catalog `checks/credentialform`), but for the +migration scaffolding still listed there. | field | value | why | |---|---|---| -| `provides` | `route` (scope `mesh`) | a module on any node may require it | +| `provides` | `route` (scope `mesh`, answered by the helper) | a module on any node may require it | | `serves.route` | `{}` | a route hands back a name, not a credential — the consumer already knows the name it asked for | -| `receives.route` | `…/routes/mesh.json` | where the mesh writes every contribution; the proxy reads it as `$ROUTES` and re-reads on change | +| `reads` | `mesh-provisioner.plans`, `mesh-provisioner.provisions`, `mesh-delivery-assigner.placement` | the plans are where the routes are; the helper reads both; the placement says who is inside | +| `emits` | `route.served`, `route.withdrawn`, `route.refused` | what it serves, said as it changes (the contract's `Event`) | | `listens` | `80` and `443`, both `from: anywhere` | the one machine with a public opening; the firewall opens these for free (ADR 0045) | -No provisioner: the proxy neither mints a credential for anybody nor answers a provision other -than `route`. It reads the file the mesh writes, and its own broker credential is for its tools. +Until the bus answers, the proxy routes nothing and says so on its log; it refuses to start without its broker +credential (`MESH_BROKER_FILE`). Who is inside the private network, and so may be served an internal-only name, is +every node's address on it as the seat mesh-placement decides it (mesh-sdk `delivery.InsideAddresses`); until that +is read, an internal-only name is served to this machine alone. ## Which issuer: Let's Encrypt, stated here (novox/hq ADR 0226) @@ -77,7 +101,7 @@ traffic. | env | default | meaning | |---|---|---| -| `ROUTES` | `/routes/mesh.json` | the contributions file the mesh writes (`receives.route`) | +| `MESH_BROKER_FILE` | *(required)* | the proxy's broker credential: the plans it routes from, and its membership | | `LISTEN` | `:80` | HTTP, and the ACME HTTP-01 challenge | | `TLS_LISTEN` | `:443` | HTTPS; unset to serve plain HTTP only | | `ACME_CACHE` | `/acme` | where issued certificates persist; required when `TLS_LISTEN` is set, so a restart does not re-order | diff --git a/modules/route-proxy/cmd/route-provider/main.go b/modules/route-proxy/cmd/route-provider/main.go index a00a7090..b1eecda7 100644 --- a/modules/route-proxy/cmd/route-provider/main.go +++ b/modules/route-proxy/cmd/route-provider/main.go @@ -14,7 +14,7 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/provider" - "route-proxy/internal/routestate" + "git.novox.be/novox/mesh-controller/modules/route-proxy/contract" ) func say(format string, args ...any) { @@ -36,7 +36,7 @@ func main() { } }() } - tools := front{routes: stdio.State(routestate.RoutesState), unrouted: stdio.State(routestate.UnroutedState), + tools := front{routes: stdio.State(contract.RoutesState), unrouted: stdio.State(contract.UnroutedState), now: time.Now}.tools() if err := stdio.Serve("", tools); err != nil { say("%v", err) diff --git a/modules/route-proxy/cmd/route-provider/provider.go b/modules/route-proxy/cmd/route-provider/provider.go index 7eee671b..39973d4f 100644 --- a/modules/route-proxy/cmd/route-provider/provider.go +++ b/modules/route-proxy/cmd/route-provider/provider.go @@ -7,18 +7,20 @@ package main // **It composes nothing.** A route's public name is the label its consumer chose joined to the node's public domain, // which only the controller knows (the domain is placement), so the controller composes it and carries it in the // consumer's plan entry (`names`); the helper copies it into the entry's values unchanged. There is no credential for -// a route and nothing to make: the traefik files stay the TypeScript adapter's, written from the contributions the -// mesh still gives it. +// a route and nothing to make. The proxy itself serves the same plans' entries (cmd/route-proxy plans.go), the one +// source of routes (issue 774, research 072 row 34). import ( "git.novox.be/novox/mesh-sdk/go/provider" + + "git.novox.be/novox/mesh-controller/modules/route-proxy/contract" ) -// ProvisionName is what route-proxy provides, as its manifest says. -const ProvisionName = "route" - -// ModuleName is this module. -const ModuleName = "route-proxy" +// ProvisionName is what route-proxy provides, as its manifest says, and ModuleName this module. +const ( + ProvisionName = contract.Provision + ModuleName = contract.Module +) // newProvider is route-proxy answering its consumers over bus on node: the route, served as the manifest serves it // (nothing of its own), with no credential, and nothing to make or retire for a consumer. diff --git a/modules/route-proxy/cmd/route-provider/provider_test.go b/modules/route-proxy/cmd/route-provider/provider_test.go index 1c36b220..34b143b4 100644 --- a/modules/route-proxy/cmd/route-provider/provider_test.go +++ b/modules/route-proxy/cmd/route-provider/provider_test.go @@ -108,6 +108,8 @@ func TestAConsumerOfARouteIsToldItsNamesFromItsPlanEntry(t *testing.T) { // the public ingress does not change. Everything of the manifest but its build, what it provides and the state it // keeps is pinned by this digest; changing the ingress means changing it here, on purpose. Re-pinned in R4d (novox/hq // ADR 0480): the program is an archive mounted into the trust image, the container and its files are otherwise as before. +// Re-pinned for issue 774 (research 072 row 34): the received routes file, its directory, mount and ROUTES went, since +// the proxy serves from the seat's plans alone; the ingress, its certificates and its files are otherwise as before. func TestRouteProxyIsNotMovedAndKeepsItsOwnFiles(t *testing.T) { raw, err := os.ReadFile("../../module.json") if err != nil { @@ -128,12 +130,13 @@ func TestRouteProxyIsNotMovedAndKeepsItsOwnFiles(t *testing.T) { for _, k := range []string{"build", "provides", "state", "reads", "tools", "replaces"} { delete(m, k) } + // Refreshed on purpose for the events its contract names (`emits`: route.served, route.withdrawn, route.refused). pinned, err := json.Marshal(m) // sorted keys, compact if err != nil { t.Fatal(err) } sum := sha256.Sum256(pinned) - if got := hex.EncodeToString(sum[:]); got != "85cb955e5a4b2d8826f50aa76433064a576e8b15b71537032eb93c7388c0ee28" { + if got := hex.EncodeToString(sum[:]); got != "3a44c56cc021be4008f521d16fa02e1c0f0d2eedab2f4d0b60f7d3177269e542" { t.Fatalf("route-proxy's own manifest changed: digest %s", got) } // The two acme templates are pinned the same way (issue 849): they are what the proxy's certificates are asked diff --git a/modules/route-proxy/cmd/route-provider/tools.go b/modules/route-proxy/cmd/route-provider/tools.go index c01a0ba4..6129a4bc 100644 --- a/modules/route-proxy/cmd/route-provider/tools.go +++ b/modules/route-proxy/cmd/route-provider/tools.go @@ -14,14 +14,14 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" - "route-proxy/internal/routestate" + "git.novox.be/novox/mesh-controller/modules/route-proxy/contract" ) // The tools' names, as the manifest declares them. const ( - RoutesTool = "route_proxy_routes" - RouteTool = "route_proxy_route" - UnroutedTool = "route_proxy_unrouted" + RoutesTool = contract.ToolRoutes + RouteTool = contract.ToolRoute + UnroutedTool = contract.ToolUnrouted ) // reader is one of the module's states, as stdio.KeptState reads it. @@ -46,9 +46,9 @@ type RouteLine struct { // unknown before any request. Reachable string `json:"reachable"` // Certificate is its outcome, issuer and expiry in words; ExpiresInDays is the whole days until not-after. - Certificate string `json:"certificate"` - ExpiresInDays *int `json:"expires_in_days,omitempty"` - LastAnswer *routestate.Answer `json:"last_answer,omitempty"` + Certificate string `json:"certificate"` + ExpiresInDays *int `json:"expires_in_days,omitempty"` + LastAnswer *contract.Answer `json:"last_answer,omitempty"` } func (f front) tools() []stdio.Tool { @@ -99,7 +99,7 @@ func read[T any](state reader, node string) ([]T, error) { } var out []T for _, key := range keys { - at, _ := routestate.Split(key) + at, _ := contract.Split(key) if node != "" && at != node { continue } @@ -120,7 +120,7 @@ func read[T any](state reader, node string) ([]T, error) { } func (f front) list(node string) (map[string]any, error) { - routes, err := read[routestate.Route](f.routes, node) + routes, err := read[contract.Route](f.routes, node) if err != nil { return nil, err } @@ -131,7 +131,7 @@ func (f front) list(node string) (map[string]any, error) { if line.Reachable == "no" { unreachable++ } - if line.ExpiresInDays != nil && *line.ExpiresInDays < 14 || r.Certificate.Outcome == routestate.Failed { + if line.ExpiresInDays != nil && *line.ExpiresInDays < 14 || r.Certificate.Outcome == contract.Failed { expiring++ } lines = append(lines, line) @@ -146,7 +146,7 @@ func (f front) list(node string) (map[string]any, error) { "certificates_failing_or_within_14_days": expiring}, nil } -func (f front) line(r routestate.Route) RouteLine { +func (f front) line(r contract.Route) RouteLine { line := RouteLine{Name: r.Name, Proxy: r.Proxy, Reach: r.Reach, LastAnswer: r.LastAnswer, Reachable: "unknown", Targets: []string{}} for _, t := range r.Targets { @@ -182,7 +182,7 @@ func (f front) line(r routestate.Route) RouteLine { } // describe is one rule in a few words: path → module@node at target, or what it answers with. -func describe(t routestate.Target) string { +func describe(t contract.Target) string { path := t.Path if path == "" { path = "/" @@ -209,11 +209,11 @@ func (f front) one(name, node string) (map[string]any, error) { if name == "" { return nil, fmt.Errorf("name is the site to read, such as app.example.org") } - routes, err := read[routestate.Route](f.routes, node) + routes, err := read[contract.Route](f.routes, node) if err != nil { return nil, err } - var found []routestate.Route + var found []contract.Route for _, r := range routes { if r.Name == name { found = append(found, r) @@ -248,32 +248,32 @@ func (f front) refusals(node string) (map[string]any, error) { } sort.Slice(asked, func(i, j int) bool { return asked[i].AskedAt.After(asked[j].AskedAt) }) if asked == nil { - asked = []routestate.Unrouted{} + asked = []contract.Unrouted{} } return map[string]any{"unrouted": asked, "names": len(asked), "other_refusals": others}, nil } // asked is the names refused and kept by name, and each proxy's count of the refusals not kept by name. -func (f front) asked(node string) ([]routestate.Unrouted, []routestate.Other, error) { +func (f front) asked(node string) ([]contract.Unrouted, []contract.Other, error) { keys, err := f.unrouted.Keys() if err != nil { return nil, nil, err } names, others := keySet{}, keySet{} for _, key := range keys { - if _, name := routestate.Split(key); name == routestate.OtherName { + if _, name := contract.Split(key); name == contract.OtherName { others[key] = true } else { names[key] = true } } - asked, err := read[routestate.Unrouted](only{f.unrouted, names}, node) + asked, err := read[contract.Unrouted](only{f.unrouted, names}, node) if err != nil { return nil, nil, err } - counted, err := read[routestate.Other](only{f.unrouted, others}, node) + counted, err := read[contract.Other](only{f.unrouted, others}, node) if counted == nil { - counted = []routestate.Other{} + counted = []contract.Other{} } return asked, counted, err } diff --git a/modules/route-proxy/cmd/route-provider/tools_test.go b/modules/route-proxy/cmd/route-provider/tools_test.go index 3194e755..da91dfe8 100644 --- a/modules/route-proxy/cmd/route-provider/tools_test.go +++ b/modules/route-proxy/cmd/route-provider/tools_test.go @@ -9,7 +9,7 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" - "route-proxy/internal/routestate" + "git.novox.be/novox/mesh-controller/modules/route-proxy/contract" ) // held is a state as the bus holds it, read-only. @@ -40,19 +40,19 @@ func TestTheWebFrontsToolsAnswerFromTheBus(t *testing.T) { now := time.Date(2026, 10, 30, 21, 0, 0, 0, time.UTC) expires := now.Add(30 * 24 * time.Hour) routes := held{ - "anchor.grafana.example": routestate.Route{Name: "grafana.example", Proxy: "anchor", Reach: routestate.ReachPublic, - Targets: []routestate.Target{{Module: "grafana", Node: "home-server", Target: "http://10.0.0.2:3000"}}, - Certificate: routestate.Certificate{Authority: routestate.ReachPublic, Issuer: "Let's Encrypt R11", NotAfter: &expires, Outcome: routestate.Issued}, - LastAnswer: &routestate.Answer{Status: 502, At: now, Error: "dial tcp 10.0.0.2:3000: connect: connection refused"}}, - "anchor.drive.example": routestate.Route{Name: "drive.example", Proxy: "anchor", Reach: routestate.ReachPublic, - Targets: []routestate.Target{{Module: "nextcloud", Node: "home-server", Target: "http://10.0.0.2:8080"}}, - Certificate: routestate.Certificate{Authority: routestate.ReachPublic}, - LastAnswer: &routestate.Answer{Status: 200, At: now}}, + "anchor.grafana.example": contract.Route{Name: "grafana.example", Proxy: "anchor", Reach: contract.ReachPublic, + Targets: []contract.Target{{Module: "grafana", Node: "home-server", Target: "http://10.0.0.2:3000"}}, + Certificate: contract.Certificate{Authority: contract.ReachPublic, Issuer: "Let's Encrypt R11", NotAfter: &expires, Outcome: contract.Issued}, + LastAnswer: &contract.Answer{Status: 502, At: now, Error: "dial tcp 10.0.0.2:3000: connect: connection refused"}}, + "anchor.drive.example": contract.Route{Name: "drive.example", Proxy: "anchor", Reach: contract.ReachPublic, + Targets: []contract.Target{{Module: "nextcloud", Node: "home-server", Target: "http://10.0.0.2:8080"}}, + Certificate: contract.Certificate{Authority: contract.ReachPublic}, + LastAnswer: &contract.Answer{Status: 200, At: now}}, } unrouted := held{ - "anchor.old.example": routestate.Unrouted{Name: "old.example", Proxy: "anchor", AskedAt: now.Add(-time.Hour), Times: 3}, - "anchor.wiki.example": routestate.Unrouted{Name: "wiki.example", Proxy: "anchor", AskedAt: now, Times: 1}, - "anchor._other": routestate.Other{Proxy: "anchor", Minute: now, LastMinute: 40, SinceStart: 900}, + "anchor.old.example": contract.Unrouted{Name: "old.example", Proxy: "anchor", AskedAt: now.Add(-time.Hour), Times: 3}, + "anchor.wiki.example": contract.Unrouted{Name: "wiki.example", Proxy: "anchor", AskedAt: now, Times: 1}, + "anchor._other": contract.Other{Proxy: "anchor", Minute: now, LastMinute: 40, SinceStart: 900}, } tools := map[string]stdio.Tool{} for _, tool := range (front{routes: routes, unrouted: unrouted, now: func() time.Time { return now }}).tools() { diff --git a/modules/route-proxy/cmd/route-proxy/bus.go b/modules/route-proxy/cmd/route-proxy/bus.go index 9dbcc274..ff34c075 100644 --- a/modules/route-proxy/cmd/route-proxy/bus.go +++ b/modules/route-proxy/cmd/route-proxy/bus.go @@ -1,54 +1,58 @@ package main import ( - "encoding/json" "fmt" "log" + "sort" "strings" "sync/atomic" "time" "github.com/nats-io/nats.go" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/announce" + "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/membership" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/provisions" - "route-proxy/internal/routestate" + "git.novox.be/novox/mesh-controller/modules/route-proxy/contract" ) // keptStates is every state route-proxy keeps, as both its halves write them: the routes and the refusals this // proxy writes, readable by others, and the bound entries the provider helper writes. func keptStates() []announce.State { return []announce.State{ - {Name: routestate.BoundState}, - {Name: routestate.RoutesState, Readable: true}, - {Name: routestate.UnroutedState, Readable: true}, + {Name: contract.BoundState}, + {Name: contract.RoutesState, Readable: true}, + {Name: contract.UnroutedState, Readable: true}, } } -// What the mesh issued this proxy, read on the bus (novox/hq ADR 0160, ADR 0167). +// What the mesh says this proxy routes, and who is inside the mesh, read on the bus. // -// **The proxy is told, not left to work it out.** Its membership carries the routes it is given — -// the same contributions its file is written from — and every machine's address on the private -// network, which is who may be served an internal name. Read once at connect and followed, so a -// route added or a machine joining reaches a running proxy without a restart. +// **The routes are the seat mesh-provisioner's plans, alone** (issue 774, research 072 row 34): every plan's entry +// naming this proxy as a route's provider, followed, so a route added or withdrawn reaches a running proxy without a +// restart, and said on the bus as it changes (contract.EventServed, EventWithdrawn, EventRefused). **Who is inside** +// is every node's address on the private network as the seat mesh-placement decides it (mesh-sdk +// delivery.InsideAddresses), which is who may be served an internal name; until it is read, an internal name is +// served to this machine alone. -// followMembership connects with the credential in path and applies every membership the mesh -// issues this proxy. It retries the first connection for as long as it takes: a proxy that started -// before the bus keeps serving the file, and takes the bus when it answers. -func followMembership(path string, held *table, fromBus *atomic.Bool) { +// follow connects with the credential in path, follows the plans and the placement, and retries the connection for +// as long as it takes: until the bus answers, the proxy routes nothing, and says so. +func follow(path string, held *table) { for { - err := followOnce(path, held, fromBus) + err := followOnce(path, held) if err == nil { return } - log.Printf("cannot follow this proxy's membership, serving the file meanwhile: %v", err) + log.Printf("cannot follow the plans that say what this proxy routes, so it routes what it last read: %v", err) time.Sleep(30 * time.Second) } } -func followOnce(path string, held *table, fromBus *atomic.Bool) error { +func followOnce(path string, held *table) error { account, err := process.ReadAccount(path) if err != nil { return err @@ -63,44 +67,128 @@ func followOnce(path string, held *table, fromBus *atomic.Bool) error { if err := conn.Keeps(keptStates()...); err != nil { log.Printf("cannot say on the bus what this proxy keeps, so it keeps nothing there: %v", err) } else { - held.ledger.attach(account.Node, conn.State(routestate.RoutesState), conn.State(routestate.UnroutedState)) + held.ledger.attach(account.Node, conn.State(contract.RoutesState), conn.State(contract.UnroutedState)) } - subject := membership.Subject(account.Node, account.Module) - err = conn.FollowMembership(func(issued membership.Membership) { - if took := applyMembership(issued, held); took && !fromBus.Swap(true) { - log.Printf("routes now come from this proxy's membership on %s", subject) + say := func(eventType string, e contract.Event) { + if err := conn.Emit(eventType, e); err != nil { + log.Printf("cannot say %s of %q on the bus: %v", eventType, e.Name, err) } - }, func(err error) { log.Print(err) }) - if err != nil { + } + // Who is inside first, so the first table is served to the machines it is meant for. + if err := followInside(conn.Read(delivery.PlacementRead), held); err != nil { + log.Printf("cannot follow %s, so an internal name is served to this machine alone: %v", delivery.PlacementRead, err) + } + if err := followPlans(conn.Read(provisions.PlansRead), newPlanned(account.Node, say), held, say); err != nil { conn.Close() return err } return nil } -// applyMembership serves what a membership says, and says whether it said anything about routes. -// -// A membership with no routes in it is one from a controller older than ADR 0167, and the file stays -// the source rather than every route being withdrawn because a field was absent. -func applyMembership(issued membership.Membership, held *table) bool { - raw, carries := issued.Receives["route"] - if !carries { - return false +// planWatch is a state this proxy watches: every current key, then every change. +type planWatch interface { + Watch(key string, handle func(stdio.StateChange) error) error +} + +// followPlans serves what the plans give this proxy: the whole table once every current plan is read, then again on +// every change that alters it. A table built from part of the plans would withdraw every route not yet read. +func followPlans(plans planWatch, given *planned, held *table, say func(string, contract.Event)) error { + var read atomic.Bool // every current plan is read: from here, each change is served as it comes + err := plans.Watch("", func(c stdio.StateChange) error { + var value []byte + if c.Op != "delete" && c.Op != "purge" { + value = c.Value + } + if changed := given.take(c.Key, value); changed && read.Load() { + serve(given, held, say) + } + return nil + }) + if err != nil { + return fmt.Errorf("watching %s: %w", provisions.PlansRead, err) + } + read.Store(true) + serve(given, held, say) + return nil +} + +// serve replaces what the proxy serves with what the plans give it, and says each name it now serves or serves no +// more. +func serve(given *planned, held *table, say func(string, contract.Event)) { + before := held.served() + routes, public := routesOf(given.contributions()) + held.set(routes, public) + after := held.served() + for _, e := range servedChanges(given.node, before, after) { + say(e.kind, e.Event) + } + log.Printf("serving %d route(s) from the seat's plans: %s", len(routes), strings.Join(held.names(), ", ")) +} + +// change is one event serve says. +type change struct { + kind string + contract.Event +} + +// servedChanges is what changed between two tables' names, each name once, in order: a name now served, or now +// served at another reach or by another module, is served; a name no longer served is withdrawn. +func servedChanges(proxy string, before, after map[string]contract.Event) []change { + var out []change + for _, name := range sortedNames(after) { + if was, ok := before[name]; !ok || was != after[name] { + e := after[name] + e.Proxy = proxy + out = append(out, change{contract.EventServed, e}) + } } - var contributions []contribution - if err := json.Unmarshal(raw, &contributions); err != nil { - log.Printf("the routes in this proxy's membership are not contributions, keeping what is served: %v", err) - return false + for _, name := range sortedNames(before) { + if _, ok := after[name]; !ok { + out = append(out, change{contract.EventWithdrawn, contract.Event{Proxy: proxy, Name: name}}) + } } - inside, err := sourcesOf(issued.Mesh) + return out +} + +func sortedNames(m map[string]contract.Event) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// followInside follows the seat mesh-placement's state and serves an internal name to every node's address on the +// private network it decides, each time it changes (mesh-sdk delivery.InsideAddresses; research 072 rows 33 and 34: +// the membership that carried them went with the assignments stream). A key that does not read keeps who is inside +// as it was. +func followInside(placement planWatch, held *table) error { + keys := delivery.NewPlacementKeys() + var read atomic.Bool + apply := func() { + inside, err := sourcesOf(delivery.InsideAddresses(keys.All())) + if err != nil { + log.Printf("the placement says an address that is not one, so who is inside stays as it was: %v", err) + return + } + held.setInside(inside) + log.Printf("internal names are served to %d machine(s)", len(inside)) + } + err := placement.Watch("", func(c stdio.StateChange) error { + if _, err := keys.Put(c.Key, c.Value, c.Revision, c.Op == "delete" || c.Op == "purge"); err != nil { + log.Printf("the placement's key %s does not read, so who is inside stays as it was: %v", c.Key, err) + return nil + } + if read.Load() { + apply() + } + return nil + }) if err != nil { - log.Printf("the mesh in this proxy's membership is unreadable, keeping what is served: %v", err) - return false + return err } - routes, public := routesOf(contributions) - held.set(routes, public) - held.setInside(inside) - log.Printf("serving %d route(s) from the membership, internal names to %d machine(s): %s", - len(routes), len(inside), strings.Join(held.names(), ", ")) - return true + read.Store(true) + apply() + return nil } diff --git a/modules/route-proxy/cmd/route-proxy/ledger.go b/modules/route-proxy/cmd/route-proxy/ledger.go index fce37132..30aecb19 100644 --- a/modules/route-proxy/cmd/route-proxy/ledger.go +++ b/modules/route-proxy/cmd/route-proxy/ledger.go @@ -17,7 +17,7 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" - "route-proxy/internal/routestate" + "git.novox.be/novox/mesh-controller/modules/route-proxy/contract" ) // What this proxy says on the bus about the names it serves and the names it refuses (issue 774). @@ -55,9 +55,9 @@ type ledger struct { node string routes, unrouted kvStore - served map[string]*routestate.Route - previous map[string]routestate.Route // what the bus held for this machine when it was reached - asked map[string]*routestate.Unrouted + served map[string]*contract.Route + previous map[string]contract.Route // what the bus held for this machine when it was reached + asked map[string]*contract.Unrouted answerWritten map[string]time.Time askedWritten map[string]time.Time @@ -69,7 +69,7 @@ type ledger struct { domains map[string]bool newMinute time.Time newInMinute int - other routestate.Other + other contract.Other otherMinute time.Time otherInMinute int otherDirty bool @@ -81,9 +81,9 @@ type ledger struct { func newLedger() *ledger { return &ledger{ now: time.Now, - served: map[string]*routestate.Route{}, - previous: map[string]routestate.Route{}, - asked: map[string]*routestate.Unrouted{}, + served: map[string]*contract.Route{}, + previous: map[string]contract.Route{}, + asked: map[string]*contract.Unrouted{}, answerWritten: map[string]time.Time{}, askedWritten: map[string]time.Time{}, dirtyRoutes: map[string]bool{}, @@ -113,12 +113,12 @@ func (l *ledger) attach(node string, routes, unrouted kvStore) { if l == nil { return } - before := map[string]routestate.Route{} + before := map[string]contract.Route{} if keys, err := routes.Keys(); err != nil { log.Printf("cannot read back what this proxy said it serves: %v", err) } else { for _, key := range keys { - at, name := routestate.Split(key) + at, name := contract.Split(key) if at != node { continue } @@ -126,7 +126,7 @@ func (l *ledger) attach(node string, routes, unrouted kvStore) { if err != nil || entry == nil { continue } - var r routestate.Route + var r contract.Route if json.Unmarshal(entry.Value, &r) == nil { before[name] = r } @@ -152,7 +152,7 @@ func (l *ledger) attach(node string, routes, unrouted kvStore) { } // keepWhatWasKnown carries a certificate and a last answer read back from the bus onto a route that knows none yet. -func keepWhatWasKnown(r *routestate.Route, before routestate.Route) { +func keepWhatWasKnown(r *contract.Route, before contract.Route) { if r.Certificate.Outcome == "" && r.Certificate.NotAfter == nil && before.Certificate.Authority == r.Certificate.Authority { r.Certificate = before.Certificate } @@ -170,14 +170,14 @@ func (l *ledger) serving(routes map[string][]rule, public map[string]bool) { l.mu.Lock() defer l.mu.Unlock() for name, rules := range routes { - reach := routestate.ReachInternal + reach := contract.ReachInternal if public[name] { - reach = routestate.ReachPublic + reach = contract.ReachPublic } targets := targetsOf(rules) r := l.served[name] if r == nil { - r = &routestate.Route{Name: name} + r = &contract.Route{Name: name} if before, ok := l.previous[name]; ok { r.Certificate.Authority = reach keepWhatWasKnown(r, before) @@ -190,7 +190,7 @@ func (l *ledger) serving(routes map[string][]rule, public map[string]bool) { l.dirtyRoutes[name] = true } if r.Certificate.Authority != reach { - r.Certificate = routestate.Certificate{Authority: reach} + r.Certificate = contract.Certificate{Authority: reach} l.dirtyRoutes[name] = true } if _, was := l.asked[name]; was { @@ -200,7 +200,7 @@ func (l *ledger) serving(routes map[string][]rule, public map[string]bool) { } l.domains = map[string]bool{} for name := range routes { - if d := routestate.Registrable(name); d != "" { + if d := contract.Registrable(name); d != "" { l.domains[d] = true } } @@ -214,10 +214,10 @@ func (l *ledger) serving(routes map[string][]rule, public map[string]bool) { l.wake() } -func targetsOf(rules []rule) []routestate.Target { - out := make([]routestate.Target, 0, len(rules)) +func targetsOf(rules []rule) []contract.Target { + out := make([]contract.Target, 0, len(rules)) for _, r := range rules { - out = append(out, routestate.Target{Path: r.path, Module: r.from, Node: r.node, Target: r.target, + out = append(out, contract.Target{Path: r.path, Module: r.from, Node: r.node, Target: r.target, Deny: r.policy.deny, Redirect: r.policy.redirectTo, Auth: len(r.policy.users) > 0 || r.policy.sealed != "", Sealed: r.policy.sealed}) } @@ -241,7 +241,7 @@ func (l *ledger) answered(name, path string, status int, failed error) { why = failed.Error() } before := r.LastAnswer - r.LastAnswer = &routestate.Answer{Status: status, At: now, Error: why, Path: path} + r.LastAnswer = &contract.Answer{Status: status, At: now, Error: why, Path: path} if before != nil && before.Status == status && before.Error == why && now.Sub(l.answerWritten[name]) < refreshAfter { return } @@ -268,11 +268,11 @@ func (l *ledger) refused(name string, internalOnly, handshake bool) { now := l.now() a := l.asked[name] if a == nil { - if !routestate.Keyable(name) || !l.domains[routestate.Registrable(name)] || !l.mayAddName(now) { + if !contract.Keyable(name) || !l.domains[contract.Registrable(name)] || !l.mayAddName(now) { l.countOther(now) return } - a = &routestate.Unrouted{Name: name} + a = &contract.Unrouted{Name: name} l.asked[name] = a } a.AskedAt, a.InternalOnly, a.Handshake = now, internalOnly, handshake @@ -345,7 +345,7 @@ func (l *ledger) certified(name string, leaf *x509.Certificate) { notAfter := leaf.NotAfter.UTC() r.Certificate.Issuer, r.Certificate.NotAfter = issuerOf(leaf), ¬After r.Certificate.RenewedAt, r.Certificate.At = &now, &now - r.Certificate.Outcome, r.Certificate.Error = routestate.Issued, "" + r.Certificate.Outcome, r.Certificate.Error = contract.Issued, "" l.dirtyRoutes[name] = true l.wake() } @@ -362,13 +362,13 @@ func (l *ledger) handedOut(name string, leaf *x509.Certificate) { return } notAfter := leaf.NotAfter.UTC() - if r.Certificate.NotAfter != nil && r.Certificate.NotAfter.Equal(notAfter) && r.Certificate.Outcome != routestate.Failed { + if r.Certificate.NotAfter != nil && r.Certificate.NotAfter.Equal(notAfter) && r.Certificate.Outcome != contract.Failed { return } now := l.now() r.Certificate.Issuer, r.Certificate.NotAfter, r.Certificate.At = issuerOf(leaf), ¬After, &now - if r.Certificate.Outcome != routestate.Issued || r.Certificate.Error != "" { - r.Certificate.Outcome, r.Certificate.Error = routestate.Cached, "" + if r.Certificate.Outcome != contract.Issued || r.Certificate.Error != "" { + r.Certificate.Outcome, r.Certificate.Error = contract.Cached, "" } l.dirtyRoutes[name] = true l.wake() @@ -386,11 +386,11 @@ func (l *ledger) certificateFailed(name string, failed error) { return } now := l.now() - same := r.Certificate.Outcome == routestate.Failed && r.Certificate.Error == failed.Error() + same := r.Certificate.Outcome == contract.Failed && r.Certificate.Error == failed.Error() if same && r.Certificate.At != nil && now.Sub(*r.Certificate.At) < refreshAfter { return } - r.Certificate.Outcome, r.Certificate.Error, r.Certificate.At = routestate.Failed, failed.Error(), &now + r.Certificate.Outcome, r.Certificate.Error, r.Certificate.At = contract.Failed, failed.Error(), &now l.dirtyRoutes[name] = true l.wake() } @@ -423,7 +423,7 @@ func (l *ledger) flush() { l.forgetOldestAsked() var puts []put for name := range l.dirtyRoutes { - p := put{store: l.routes, key: routestate.Key(l.node, name), route: true} + p := put{store: l.routes, key: contract.Key(l.node, name), route: true} if r := l.served[name]; r != nil { c := *r c.Proxy = l.node @@ -432,7 +432,7 @@ func (l *ledger) flush() { puts = append(puts, p) } for name := range l.dirtyAsked { - p := put{store: l.unrouted, key: routestate.Key(l.node, name)} + p := put{store: l.unrouted, key: contract.Key(l.node, name)} if a := l.asked[name]; a != nil { c := *a c.Proxy = l.node @@ -443,7 +443,7 @@ func (l *ledger) flush() { if l.otherDirty { o := l.other o.Proxy = l.node - puts = append(puts, put{store: l.unrouted, key: routestate.Key(l.node, routestate.OtherName), value: o}) + puts = append(puts, put{store: l.unrouted, key: contract.Key(l.node, contract.OtherName), value: o}) } l.dirtyRoutes, l.dirtyAsked, l.otherDirty = map[string]bool{}, map[string]bool{}, false l.mu.Unlock() @@ -466,7 +466,7 @@ func (l *ledger) flush() { log.Printf("cannot write %s to the bus, kept for the next change: %v", p.key, err) } l.routesFailing = true - _, name := routestate.Split(p.key) + _, name := contract.Split(p.key) l.dirtyRoutes[name] = true default: if !l.askFailed { diff --git a/modules/route-proxy/cmd/route-proxy/ledger_test.go b/modules/route-proxy/cmd/route-proxy/ledger_test.go index fba0dd6c..91c01113 100644 --- a/modules/route-proxy/cmd/route-proxy/ledger_test.go +++ b/modules/route-proxy/cmd/route-proxy/ledger_test.go @@ -25,7 +25,7 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" - "route-proxy/internal/routestate" + "git.novox.be/novox/mesh-controller/modules/route-proxy/contract" ) // fakeState is a state on the bus, in memory, counting what was written to it. @@ -68,13 +68,13 @@ func (f *fakeState) Get(key string) (*stdio.StateEntry, error) { return &stdio.StateEntry{Key: key, Value: v}, nil } -func (f *fakeState) route(t *testing.T, key string) *routestate.Route { +func (f *fakeState) route(t *testing.T, key string) *contract.Route { t.Helper() raw, ok := f.values[key] if !ok { return nil } - var r routestate.Route + var r contract.Route if err := json.Unmarshal(raw, &r); err != nil { t.Fatal(err) } @@ -117,8 +117,8 @@ func TestTheBusIsToldWhatIsServedAndRefused(t *testing.T) { }() _, port, _ := net.SplitHostPort(gone) contributions := []contribution{ - {From: "grafana", Node: "anchor", At: "127.0.0.1", Values: map[string]any{"name": "grafana.example", "port": mustFloat(t, port)}}, - {From: "drive", Node: "home-server", At: "10.0.0.2", Values: map[string]any{"internal-name": "drive.internal", "port": 8080.0}}, + {From: "grafana", Node: "anchor", At: "127.0.0.1", Route: contract.Request{Name: "grafana.example", Port: int(mustFloat(t, port))}}, + {From: "drive", Node: "home-server", At: "10.0.0.2", Route: contract.Request{InternalName: "drive.internal", Port: 8080}}, } for _, row := range []struct { @@ -130,12 +130,12 @@ func TestTheBusIsToldWhatIsServedAndRefused(t *testing.T) { func(t *testing.T, held *table) {}, func(t *testing.T, routes, _ *fakeState) { r := routes.route(t, "anchor.grafana.example") - if r == nil || r.Proxy != "anchor" || r.Reach != routestate.ReachPublic || len(r.Targets) != 1 || + if r == nil || r.Proxy != "anchor" || r.Reach != contract.ReachPublic || len(r.Targets) != 1 || r.Targets[0].Module != "grafana" || r.Targets[0].Node != "anchor" || r.Targets[0].Target != "http://127.0.0.1:"+port { t.Fatalf("grafana's route is not on the bus as served: %+v", r) } - if d := routes.route(t, "anchor.drive.internal"); d == nil || d.Reach != routestate.ReachInternal || - d.Certificate.Authority != routestate.ReachInternal { + if d := routes.route(t, "anchor.drive.internal"); d == nil || d.Reach != contract.ReachInternal || + d.Certificate.Authority != contract.ReachInternal { t.Fatalf("an internal-only name is not on the bus as internal: %+v", d) } }}, @@ -172,7 +172,7 @@ func TestTheBusIsToldWhatIsServedAndRefused(t *testing.T) { req.RemoteAddr = "192.0.2.7:51000" handler(held).ServeHTTP(httptest.NewRecorder(), req) held.ledger.flush() - var a routestate.Unrouted + var a contract.Unrouted raw := held.ledger.unrouted.(*fakeState).values["anchor.wiki.grafana.example"] if err := json.Unmarshal(raw, &a); err != nil || a.Times != 1 || a.AskedAt.IsZero() { t.Fatalf("the refused name is not on the bus as asked: %s", raw) @@ -181,7 +181,7 @@ func TestTheBusIsToldWhatIsServedAndRefused(t *testing.T) { t.Fatalf("who asked is on the bus: %s", raw) } routes, public := routesOf(append(contributions, contribution{From: "grafana", Node: "anchor", - Values: map[string]any{"name": "wiki.grafana.example", "port": 3000.0}})) + Route: contract.Request{Name: "wiki.grafana.example", Port: 3000}})) held.set(routes, public) }, func(t *testing.T, _, unrouted *fakeState) { @@ -197,7 +197,7 @@ func TestTheBusIsToldWhatIsServedAndRefused(t *testing.T) { _, _ = pick(&tls.ClientHelloInfo{ServerName: "forgotten.grafana.example"}) }, func(t *testing.T, _, unrouted *fakeState) { - var a routestate.Unrouted + var a contract.Unrouted if err := json.Unmarshal(unrouted.values["anchor.forgotten.grafana.example"], &a); err != nil || !a.Handshake { t.Fatalf("a handshake refused for its name is not on the bus: %v", unrouted.values) } @@ -232,7 +232,7 @@ func TestTheBusIsToldWhatIsServedAndRefused(t *testing.T) { held.ledger.refused("w.scan.test", false, false) }, func(t *testing.T, _, unrouted *fakeState) { - var o routestate.Other + var o contract.Other if err := json.Unmarshal(unrouted.values["anchor._other"], &o); err != nil || o.LastMinute != 2 || o.SinceStart != 78 { t.Fatalf("the refusals not kept by name are not counted on the bus: %s", unrouted.values["anchor._other"]) } @@ -247,7 +247,7 @@ func TestTheBusIsToldWhatIsServedAndRefused(t *testing.T) { }, func(t *testing.T, routes, _ *fakeState) { c := routes.route(t, "anchor.grafana.example").Certificate - if c.Outcome != routestate.Issued || c.Issuer != "Let's Encrypt R11" || c.NotAfter == nil || + if c.Outcome != contract.Issued || c.Issuer != "Let's Encrypt R11" || c.NotAfter == nil || !c.NotAfter.Equal(notAfter) || c.RenewedAt == nil { t.Fatalf("the issued certificate is not on the bus: %+v", c) } @@ -261,7 +261,7 @@ func TestTheBusIsToldWhatIsServedAndRefused(t *testing.T) { }, func(t *testing.T, routes, _ *fakeState) { c := routes.route(t, "anchor.grafana.example").Certificate - if c.Outcome != routestate.Failed || c.Error != "acme: urn:ietf:params:acme:error:rateLimited" { + if c.Outcome != contract.Failed || c.Error != "acme: urn:ietf:params:acme:error:rateLimited" { t.Fatalf("the failed certificate is not on the bus: %+v", c) } }}, diff --git a/modules/route-proxy/cmd/route-proxy/main.go b/modules/route-proxy/cmd/route-proxy/main.go index 5288f55f..9203ba46 100644 --- a/modules/route-proxy/cmd/route-proxy/main.go +++ b/modules/route-proxy/cmd/route-proxy/main.go @@ -1,13 +1,14 @@ // A reverse proxy, in the form the mesh expects one. // -// **A route is a grant** (novox/hq ADR 0007, 08-connectivity §3). A module that must be reachable -// declares it requires `route` and contributes the name it wants; the proxy provides `route` and -// receives every consumer that asked. It is the mirror of a database grant: there the consumer +// **A route is a provision** (novox/hq ADR 0007, 08-connectivity §3; issue 774, research 072 row 34). A +// module that must be reachable declares it requires `route` and contributes the name it wants; the +// proxy provides `route` and serves every consumer whose plan, in the seat mesh-provisioner's state +// `plans`, names it as the route's provider. It is the mirror of a database grant: there the consumer // supplies a name and receives credentials, here it supplies a target and receives a name. // -// **It is an example, not part of the control plane.** The control plane decides and never touches -// a machine. A real deployment runs whatever proxy it likes — the contract is the file, not this -// program. What lives here is that contract, written as something that runs so it can be read +// **It is an example, not part of the controller.** The controller decides and never touches a +// machine. A real deployment runs whatever proxy it likes — the contract is the plan's entry, not +// this program. What lives here is that contract, written as something that runs so it can be read // rather than described. // // **A route also carries what a request arriving at it may do** (novox/hq ADR 0108). The grant used @@ -16,11 +17,9 @@ // an open middleware surface recreates the thing being replaced and is far harder to narrow later // than a closed one is to widen. // -// What it is given, written by the host from an ordinary declaration: -// -// $ROUTES every consumer, the name it asked for, and where the mesh says that machine is -// -// Each contribution's values carry the name and port as before, and optionally: +// What it is given, in each plan's entry for the route: every contribution the consumer makes, +// composed by the controller, and where the consumer's machine is on the private network (plans.go). +// Each contribution's values carry the name and port, and optionally: // // path the path prefix this rule is scoped to; absent means every path // priority which rule wins where two match; higher first, and the order is total @@ -35,8 +34,8 @@ // outright rather than served unprotected, and a secret that cannot be read makes the route refuse // rather than open — a gate that cannot check is not a gate that opens. // -// It re-reads on change rather than being restarted, for the same reason the provisioner does: -// a route arriving or leaving is an ordinary event and must not drop the connections of every +// It follows the plans on the bus rather than being restarted, for the same reason the provisioner +// does: a route arriving or leaving is an ordinary event and must not drop the connections of every // other workload. package main @@ -48,7 +47,6 @@ import ( "crypto/tls" "crypto/x509" "encoding/hex" - "encoding/json" "fmt" "log" "net" @@ -61,12 +59,13 @@ import ( "sort" "strings" "sync" - "sync/atomic" "time" "golang.org/x/crypto/acme" "golang.org/x/crypto/acme/autocert" "golang.org/x/crypto/bcrypt" + + "git.novox.be/novox/mesh-controller/modules/route-proxy/contract" ) // Where public certificates come from when nothing says otherwise. @@ -119,18 +118,16 @@ func onlyInternalNamesTheMeshSaid(held *table) autocert.HostPolicy { } } -// what the mesh writes: the contributions file, one entry per consumer. -type given struct { - Given []contribution `json:"given"` -} - +// contribution is one route a consumer asked for: the module, its machine, where that machine is, the local name of +// its contribution, and the route, composed by the mesh and read with route-proxy's contract (plans.go). type contribution struct { From string `json:"from"` Node string `json:"node"` // At is where that machine is on the private network. The mesh knows it; a proxy that had to // build it from the node name would be a naming convention copied into every provider. - At string `json:"at"` - Values map[string]any `json:"values"` + At string `json:"at"` + Local string `json:"local,omitempty"` + Route contract.Request `json:"route"` } // policy is what a rule does with a request that matched it. @@ -183,9 +180,9 @@ type rule struct { from, node string } -// table is what the proxy is currently serving, replaced whole whenever the file changes. +// table is what the proxy is currently serving, replaced whole whenever the plans change what they give it. // -// Replaced rather than merged: the file is the whole truth about who has a route, so merging +// Replaced rather than merged: the plans are the whole truth about who has a route, so merging // would keep serving a name whose module was unassigned — which is the stale-route fault // 08-connectivity lists as open, reintroduced one level down. // @@ -356,7 +353,7 @@ func (t *table) routed(host string) bool { // // **The mesh has two name spaces and two authorities** (novox/hq 03-DESIGN/01-to-be/08-connectivity // §2): a public name is certified by a public CA, an internal one by the mesh's own. This is -// composed only from `to` and `public`, which routesFrom already builds correctly — a host never +// composed only from `to` and `public`, which routesOf already builds correctly — a host never // lands in both a route's own `name` and only its `internal-name`, so nothing new has to be // tracked to tell the two apart. func (t *table) eligibleForInternalACME(host string) bool { @@ -413,6 +410,24 @@ func (t *table) namesSeenFrom(remote string) []string { return out } +// served is every name served, as an event says it: how far it reaches and the module whose rule serves it first. +func (t *table) served() map[string]contract.Event { + t.mu.RLock() + defer t.mu.RUnlock() + out := make(map[string]contract.Event, len(t.to)) + for name, rules := range t.to { + e := contract.Event{Name: name, Reach: contract.ReachInternal} + if t.public[name] { + e.Reach = contract.ReachPublic + } + if len(rules) > 0 { + e.Module, e.Node = rules[0].from, rules[0].node + } + out[name] = e + } + return out +} + func (t *table) names() []string { t.mu.RLock() defer t.mu.RUnlock() @@ -432,50 +447,21 @@ func main() { } func run() error { - path := strings.TrimSpace(os.Getenv("ROUTES")) - if path == "" { - return fmt.Errorf("ROUTES is not set, so this proxy does not know what it is routing") - } listen := strings.TrimSpace(os.Getenv("LISTEN")) if listen == "" { listen = ":80" } + // **The seat's plans are the one source of routes** (issue 774, research 072 row 34). Without the bus this proxy + // knows no route, so it refuses to start rather than serving an empty table and saying nothing. + credential := strings.TrimSpace(os.Getenv("MESH_BROKER_FILE")) + if credential == "" { + return fmt.Errorf("MESH_BROKER_FILE is not set, so this proxy cannot read the plans that say what it routes") + } held := newTable() held.ledger = newLedger() go held.ledger.loop() - // **The bus first, the file until it has spoken** (novox/hq ADR 0167). The membership carries - // the routes and who the mesh is; the file carries the routes alone, so while the proxy reads - // it an internal name is served to this machine and to nobody else — refused, never opened. - fromBus := &atomic.Bool{} - if credential := strings.TrimSpace(os.Getenv("MESH_BROKER_FILE")); credential != "" { - go followMembership(credential, held, fromBus) - } else { - log.Printf("MESH_BROKER_FILE is not set: routes come from %s alone, and a name that is only "+ - "internal is served to this machine alone", path) - } - read := func() { - if fromBus.Load() { - return - } - routes, public, err := routesFrom(path) - if err != nil { - // Kept serving what it had. A file being rewritten is momentarily unreadable, and - // dropping every route because one read landed mid-write would turn an ordinary - // event into an outage. - log.Printf("cannot read %s, keeping what is already served: %v", path, err) - return - } - held.set(routes, public) - log.Printf("serving %d route(s): %s", len(routes), strings.Join(held.names(), ", ")) - } - read() - - go func() { - for range time.Tick(2 * time.Second) { - read() - } - }() + go follow(credential, held) // TLS is opt-in. A proxy with no `TLS_LISTEN` serves plain HTTP exactly as before — which is // what an internal-only mesh wants, and what the mesh's own certificate authority already @@ -881,66 +867,26 @@ func boolByte(b bool) byte { return 0 } -// routesFrom reads what the mesh wrote and turns it into host → the rules for that host, and -// which of those hosts is a public name — the second is `name`, ACME-eligible; a host reached -// only through `internal-name` never appears there. +// routesOf turns the routes the plans give into host → the rules for that host, and which of those hosts is a public +// name — the second is `name`, ACME-eligible; a host reached only through `internal-name` never appears there. // -// **A route may carry either name, or both** (novox/hq ADR 0138). How far an endpoint reaches -// decides which names the mesh composes, so an endpoint that reaches only the private network -// arrives with an `internal-name` and no `name`. That is a whole route, not a malformed one: it is -// served under its internal name and certified by the internal authority. Only a route with -// neither name has nothing to be served under (novox/hq issue 191). -func routesFrom(path string) (map[string][]rule, map[string]bool, error) { - raw, err := os.ReadFile(path) - if err != nil { - return nil, nil, err - } - var said given - if err := json.Unmarshal(raw, &said); err != nil { - return nil, nil, err - } - routes, public := routesOf(said.Given) - return routes, public, nil -} - -// routesOf turns what the mesh gave into host → the rules for that host, and which hosts are public -// names — the same whether the contributions came in the file or in the membership. +// **A route may carry either name, or both** (novox/hq ADR 0138). How far an endpoint reaches decides which names the +// mesh composes, so an endpoint that reaches only the private network arrives with an `internal-name` and no `name`. +// That is a whole route, not a malformed one: it is served under its internal name and certified by the internal +// authority. Only a route with neither name has nothing to be served under (novox/hq issue 191). func routesOf(contributions []contribution) (map[string][]rule, map[string]bool) { out := map[string][]rule{} public := map[string]bool{} for _, c := range contributions { - name, _ := c.Values["name"].(string) - name = strings.TrimSpace(name) - internal, _ := c.Values["internal-name"].(string) - internal = strings.TrimSpace(internal) - if name == "" && internal == "" { - log.Printf("%s on %s asked for a route and named nothing; skipped", c.From, c.Node) - continue - } - // What the route is called in a log line: its public name when it has one. - called := name - if called == "" { - called = internal - } - - made := rule{path: asPath(c.Values["path"])} - if p, ok := asWhole(c.Values["priority"]); ok { - made.priority = p - } - made.policy.deny, _ = c.Values["deny"].(bool) - made.policy.redirectTo, _ = c.Values["redirect"].(string) - - if named, carried := c.Values["auth"].(string); carried && strings.TrimSpace(named) != "" { - // **A declaration names a secret; it never holds one** (ADR 0108). Refused rather than - // tolerated, and the whole rule is dropped rather than served unprotected — the - // rejected option cannot come back by accident, which is the failure this check exists - // to make impossible. - if looksLikeACredential(named) { - log.Printf("%s on %s declared route %q with a credential in the declaration rather "+ - "than the name of a secret; the whole route is refused (novox/hq ADR 0108)", - c.From, c.Node, called) - continue - } + // The contract admitted the route whole (contract.Request.Validate): it names a host, a rule that forwards + // gives a port, its scheme is http or https, its body limit is whole and positive, and its auth names a + // secret's file rather than holding a credential (novox/hq ADR 0108). + r := c.Route + name := strings.TrimSpace(r.Name) + internal := strings.TrimSpace(r.InternalName) + made := rule{path: r.Prefix(), priority: r.Priority} + made.policy.deny, made.policy.redirectTo = r.Deny, r.Redirect + if named := strings.TrimSpace(r.Auth); named != "" { users, err := usersFrom(named) if err != nil { // Fail closed: the rule is kept so the name stays routed and answers, and it @@ -952,13 +898,7 @@ func routesOf(contributions []contribution) (map[string][]rule, map[string]bool) } // Only a rule that actually proxies needs somewhere to send the request. - if !made.policy.deny && made.policy.redirectTo == "" { - port, ok := asPort(c.Values["port"]) - if !ok { - log.Printf("%s on %s asked for route %q and gave no usable port; skipped", - c.From, c.Node, called) - continue - } + if r.Forwards() { // Where the mesh says that machine is. Empty means it is this one — a workload beside // the proxy is ordinary, and reaching it over loopback is both correct and the only // thing that works when there is no private network. @@ -968,32 +908,12 @@ func routesOf(contributions []contribution) (map[string][]rule, map[string]bool) } // http unless the contribution says otherwise. A backend that terminates its own TLS // with a certificate this proxy has no reason to trust — Mailu's webmail front is the - // first of these — is the reason `insecure` exists, and it stays the exception: every - // other target the mesh hands this proxy is a plain workload on the private network. - scheme, _ := c.Values["scheme"].(string) - scheme = strings.ToLower(strings.TrimSpace(scheme)) - if scheme == "" { - scheme = "http" + // first of these — is the reason `insecure` exists, and it stays the exception. + made.insecure = r.Insecure + if r.MaxRequestBody != nil { + made.maxRequestBody = *r.MaxRequestBody } - if scheme != "http" && scheme != "https" { - log.Printf("%s on %s asked for route %q with scheme %q, which is neither http "+ - "nor https; skipped", c.From, c.Node, called, scheme) - continue - } - made.insecure, _ = c.Values["insecure"].(bool) - // A limit this proxy cannot read is a route it does not serve, named like a port that - // is not a port. Serving it without the limit would carry exactly what the module said - // not to carry, and report success doing it. - if asked, said := c.Values["max-request-body"]; said { - bytes, whole := asWhole(asked) - if !whole || bytes <= 0 { - log.Printf("%s on %s asked for route %q with a max-request-body of %v, which is "+ - "not a whole positive number of bytes; skipped", c.From, c.Node, called, asked) - continue - } - made.maxRequestBody = int64(bytes) - } - made.target = fmt.Sprintf("%s://%s:%d", scheme, at, port) + made.target = fmt.Sprintf("%s://%s:%d", r.SchemeOrDefault(), at, r.Port) } made.from, made.node = c.From, c.Node @@ -1007,9 +927,8 @@ func routesOf(contributions []contribution) (map[string][]rule, map[string]bool) // answered both for one route, as a convenience (reaching a service over the VPN without a // public TLS round trip), not as an access boundary; composing it here restores exactly // that, nothing more. Absent whenever the node composed no internal name (novox/hq ADR - // 0056's internalDomain half) — the same "nothing to join a label to" case the public name - // already has. And the only name, when the endpoint reaches no further than the private - // network. + // 0056's internalDomain half). And the only name, when the endpoint reaches no further than + // the private network. if internal != "" { out[strings.ToLower(internal)] = append(out[strings.ToLower(internal)], made) } @@ -1017,50 +936,6 @@ func routesOf(contributions []contribution) (map[string][]rule, map[string]bool) return out, public } -// asWhole is any whole number the mesh wrote, whatever its magnitude. -// -// **Not asPort.** Priority was read with the port reader first, which caps at 65535 — so a rule -// declared at a priority above that silently became priority 0 and stopped shadowing the route it -// exists to shadow. The one real rule this has to reproduce is declared at 100000, so the bug was -// exactly load-bearing. A priority is an ordering, not a port: it has no range. -func asWhole(v any) (int, bool) { - switch n := v.(type) { - case float64: - // JSON makes a float of every number, so a non-integral one was not meant as a priority. - if n != float64(int(n)) { - return 0, false - } - return int(n), true - case int: - return n, true - } - return 0, false -} - -// asPath is the path prefix a rule is scoped to, or "" for every path. -func asPath(v any) string { - p, _ := v.(string) - p = strings.TrimSpace(p) - if p == "" { - return "" - } - if !strings.HasPrefix(p, "/") { - p = "/" + p - } - return p -} - -// looksLikeACredential is the check that keeps a secret out of a declaration. -// -// It errs towards refusing: a value holding a `:` (the htpasswd separator) or opening with a bcrypt -// identifier is a credential, not a path, and no filesystem path the mesh writes needs either. A -// false refusal is a loud log and a route that does not serve; a false accept is a credential -// committed to a declaration, which is the thing being prevented. -func looksLikeACredential(v string) bool { - v = strings.TrimSpace(v) - return strings.Contains(v, ":") || strings.HasPrefix(v, "$2") -} - // usersFrom reads the credentials the mesh mounted, in the one format every htpasswd already is. func usersFrom(path string) (map[string]string, error) { raw, err := os.ReadFile(path) @@ -1085,23 +960,6 @@ func usersFrom(path string) (map[string]string, error) { return users, nil } -// asPort accepts what JSON makes of a number, which is a float even when it was written 8080. -func asPort(v any) (int, bool) { - switch n := v.(type) { - case float64: - if n < 1 || n > 65535 { - return 0, false - } - return int(n), true - case int: - if n < 1 || n > 65535 { - return 0, false - } - return n, true - } - return 0, false -} - // towards proxies to one backend and tells it what the client asked: **X-Forwarded-Proto, -Host and // -For**, set from the request this proxy received. A backend that builds its own addresses — a forge // writing its clone URL into a page, a login redirect — otherwise sees the plain HTTP hop from this diff --git a/modules/route-proxy/cmd/route-proxy/plans.go b/modules/route-proxy/cmd/route-proxy/plans.go new file mode 100644 index 00000000..d2623b87 --- /dev/null +++ b/modules/route-proxy/cmd/route-proxy/plans.go @@ -0,0 +1,147 @@ +package main + +// The routes this proxy serves, read from the seat mesh-provisioner's plans and nothing else (issue 774; research 072 +// row 34, the operator on 2026-10-31: "there is only 1 path, the other is simply deprecated/invalid"). +// +// **A route is a provision, in route-proxy's own contract.** A module that must be reachable requires `route`; the +// controller composes its plan on its node, and the plan's entry for the route names this proxy as its provider and +// carries what the module contributes, composed (its names joined to the node's domains, its endpoint's port), and +// where the module's machine is on the private network, as one opaque payload the seat never reads (mesh-sdk +// catalogue.PlanProvision.Contributed). The proxy decodes each contribution with its contract (contract.DecodeRequest) +// and serves every entry naming it. There is no file and no second source: a route the plans do not name is not +// served, and a route the contract refuses is said refused (contract.EventRefused), never served half. + +import ( + "fmt" + "log" + "sort" + "sync" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/provisions" + + "git.novox.be/novox/mesh-controller/modules/route-proxy/contract" +) + +// planned is the routes the seat's plans give this proxy on its node, by plan key, kept so a plan that cannot be read +// leaves what it gave served (a bind missed would read as a route withdrawn). +type planned struct { + node string + // said is where a refusal is told: the bus's events, nothing in a test that does not ask. + said func(eventType string, e contract.Event) + mu sync.Mutex + by map[string][]contribution +} + +func newPlanned(node string, said func(string, contract.Event)) *planned { + if said == nil { + said = func(string, contract.Event) {} + } + return &planned{node: node, said: said, by: map[string][]contribution{}} +} + +// take is one change of a plan's key: its value, or nil when the key was deleted. It answers whether what is served +// changed, and says on the log and the bus what it could not serve. +func (p *planned) take(key string, value []byte) bool { + p.mu.Lock() + defer p.mu.Unlock() + if value == nil { + _, had := p.by[key] + delete(p.by, key) + return had + } + gave, refused, err := routesOfPlan(p.node, key, value) + if err != nil { + log.Printf("the plan %s is not readable, so what it routes is kept as it was: %v", key, err) + return false + } + for _, r := range refused { + log.Printf("the plan %s gives %s on %s a route this proxy does not serve: %s", key, r.Module, r.Node, r.Why) + p.said(contract.EventRefused, r) + } + if len(gave) == 0 { + _, had := p.by[key] + delete(p.by, key) + return had + } + p.by[key] = gave + return true +} + +// contributions is every route the plans give this proxy, in the order of their plans' keys. +func (p *planned) contributions() []contribution { + p.mu.Lock() + defer p.mu.Unlock() + keys := make([]string, 0, len(p.by)) + for k := range p.by { + keys = append(keys, k) + } + sort.Strings(keys) + var out []contribution + for _, k := range keys { + out = append(out, p.by[k]...) + } + return out +} + +// routesOfPlan is the routes one plan gives the proxy on node: each contribution the plan's entries naming this proxy +// on node as the route's provider carry, decoded with the contract, reached on its own machine over loopback and on +// another at the address the plan says. A plan of another module, or one routing elsewhere, gives none. Refused is +// what the plan names and the proxy does not serve, each with why: an entry carrying nothing to serve, a payload or a +// route the contract does not read, or a consumer elsewhere with no address on the private network. +func routesOfPlan(node, key string, value []byte) ([]contribution, []contract.Event, error) { + plan, err := provisions.DecodePlan(value) + if err != nil { + return nil, nil, err + } + if plan.Node+"."+plan.Module != key { + return nil, nil, fmt.Errorf("it is the plan of %s.%s, under the key %s", plan.Node, plan.Module, key) + } + var out []contribution + var refused []contract.Event + refuse := func(local, why string) { + refused = append(refused, contract.Event{Proxy: node, Module: plan.Module, Node: plan.Node, Local: local, Why: why}) + } + named, carried := false, false + for _, e := range plan.Plan.Provisions { + if !routesHere(node, e) { + continue + } + named = true + // What the module contributes is on the requirement's first entry alone, however many local names it has. + if len(e.Contributed) == 0 { + continue + } + carried = true + given, err := catalogue.DecodeContributed(e.Contributed) + if err != nil { + refuse("", err.Error()) + continue + } + at := "" + if plan.Node != node { + if given.At == "" { + refuse("", fmt.Sprintf("it runs on %s and its plan says no address on the private network for it", plan.Node)) + continue + } + at = given.At + } + for _, one := range given.Each { + route, err := contract.DecodeRequest(one.Values) + if err != nil { + refuse(one.Local, err.Error()) + continue + } + out = append(out, contribution{From: plan.Module, Node: plan.Node, At: at, Local: one.Local, Route: route}) + } + } + if named && !carried { + refuse("", "its plan names this proxy for its route and carries nothing to serve") + } + return out, refused, nil +} + +// routesHere is whether a plan's entry is a route this proxy on node provides. +func routesHere(node string, e catalogue.PlanProvision) bool { + return e.Provision == contract.Provision && e.Provider == node && e.ProviderModule == contract.Module +} diff --git a/modules/route-proxy/cmd/route-proxy/policy_test.go b/modules/route-proxy/cmd/route-proxy/policy_test.go index 192af8d6..0f2b3646 100644 --- a/modules/route-proxy/cmd/route-proxy/policy_test.go +++ b/modules/route-proxy/cmd/route-proxy/policy_test.go @@ -57,11 +57,7 @@ func ask(t *testing.T, proxy, name, path string, auth [2]string) *http.Response func proxyFor(t *testing.T, routesJSON string) string { t.Helper() - path := filepath.Join(t.TempDir(), "routes.json") - if err := os.WriteFile(path, []byte(routesJSON), 0o644); err != nil { - t.Fatal(err) - } - routes, public, err := routesFrom(path) + routes, public, err := routesFrom(routesJSON) if err != nil { t.Fatal(err) } @@ -153,11 +149,7 @@ func TestACredentialInTheDeclarationIsRefusedRatherThanServed(t *testing.T) { `{"given":[{"from":"c","node":"n","at":"127.0.0.1","values":{"name":"c.example","port":8080,"auth":"$2a$10$abcdefghijklmnopqrstuv"}}]}`, } for _, body := range inline { - path := filepath.Join(t.TempDir(), "routes.json") - if err := os.WriteFile(path, []byte(body), 0o644); err != nil { - t.Fatal(err) - } - routes, _, err := routesFrom(path) + routes, _, err := routesFrom(body) if err != nil { t.Fatal(err) } diff --git a/modules/route-proxy/cmd/route-proxy/reach_test.go b/modules/route-proxy/cmd/route-proxy/reach_test.go index d008d649..25372c14 100644 --- a/modules/route-proxy/cmd/route-proxy/reach_test.go +++ b/modules/route-proxy/cmd/route-proxy/reach_test.go @@ -9,10 +9,16 @@ import ( "net/http" "net/http/httptest" "net/url" + "reflect" + "slices" + "sort" "strings" "testing" - "git.novox.be/novox/mesh-sdk/go/membership" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + + "git.novox.be/novox/mesh-controller/modules/route-proxy/contract" ) // behind is a workload the proxy can send to, and a table routing one public name and one @@ -26,12 +32,12 @@ func behind(t *testing.T, mesh ...string) *table { at, _ := url.Parse(workload.URL) host, port, _ := net.SplitHostPort(at.Host) - routes, public, err := routesFrom(write(t, fmt.Sprintf(`{"given":[ + routes, public, err := routesFrom(fmt.Sprintf(`{"given":[ {"from":"app","node":"anchor","at":%q, "values":{"name":"app.example","internal-name":"app.anchor.internal","port":%s}}, {"from":"admin","node":"anchor","at":%q, "values":{"internal-name":"admin.anchor.internal","port":%s}} - ]}`, host, port, host, port))) + ]}`, host, port, host, port)) if err != nil { t.Fatal(err) } @@ -165,42 +171,161 @@ func TestTheMeshIsReadAsAddressesAndRanges(t *testing.T) { } } -// What the mesh issues is what is served: the routes in the membership, internal names to the -// machines it names (novox/hq ADR 0167). -func TestAMembershipIsServedAsIssued(t *testing.T) { - held := newTable() - took := applyMembership(membership.Membership{ - Receives: map[string]json.RawMessage{"route": json.RawMessage(`[ - {"from":"admin","node":"anchor","at":"anchor.internal", - "values":{"internal-name":"admin.anchor.internal","port":8080}}]`)}, - Mesh: []string{"10.10.0.7"}, - }, held) - if !took { - t.Fatal("a membership carrying routes was not applied") - } - if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:1"); code == http.StatusNotFound { - t.Error("a machine the membership names was refused the internal-only route") - } - if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.9:1"); code != http.StatusNotFound { - t.Errorf("a machine the membership does not name was served the internal-only route: %d", code) - } +// fakePlans is the seat's plans as a watch hands them: every current key, then each change after Watch returned. +type fakePlans struct { + current map[string]string + handle func(stdio.StateChange) error } -// A membership that says nothing about routes is one from a controller that does not issue them, -// and changes nothing: the file stays the source rather than every route being withdrawn. -func TestAMembershipWithoutRoutesLeavesTheFileServing(t *testing.T) { - held := behind(t, "10.10.0.7") - before := held.names() - if applyMembership(membership.Membership{Mesh: []string{"10.10.0.7"}}, held) { - t.Error("a membership without routes was taken as the source of routes") +func (f *fakePlans) Watch(_ string, handle func(stdio.StateChange) error) error { + f.handle = handle + for k, v := range f.current { + if err := handle(stdio.StateChange{Key: k, Op: "put", Value: json.RawMessage(v), Current: true}); err != nil { + return err + } } - if got := held.names(); strings.Join(got, ",") != strings.Join(before, ",") { - t.Errorf("a membership without routes changed what is served: %v, was %v", got, before) + return nil +} + +// plan is a plan of the seat mesh-provisioner for module on node, its entries as given. +func plan(node, module, entries string) string { + return fmt.Sprintf(`{"v":1,"node":%q,"module":%q,"plan":{"templates":{"source":"","digest":""},"files":null,`+ + `"provisions":[%s]}}`, node, module, entries) +} + +// contributed is a plan entry's opaque payload: the consumer's machine at at, and each contribution by local name. +func contributed(at string, each ...string) string { + var parts []string + for i := 0; i+1 < len(each); i += 2 { + parts = append(parts, fmt.Sprintf(`{"local":%q,"values":%s}`, each[i], each[i+1])) } - if applyMembership(membership.Membership{ - Receives: map[string]json.RawMessage{"route": json.RawMessage(`[]`)}, - Mesh: []string{"not-an-address"}, - }, held) { - t.Error("a membership whose mesh cannot be read was applied") + return fmt.Sprintf(`"contributed":{"v":1,"at":%q,"each":[%s]}`, at, strings.Join(parts, ",")) +} + +// What the plans give is what is served, and nothing else (issue 774, research 072 row 34): each entry naming this +// proxy on its node as the route's provider, every contribution of it read with route-proxy's contract, reached over +// loopback on this machine and at the plan's address on another; an entry routing elsewhere, carrying nothing, naming +// no address or a route the contract refuses is not served, and is said refused. A change is served as it comes and +// said served or withdrawn, a plan that cannot be read keeps what it gave, and a deleted one withdraws it. The +// placement says only who is inside: an internal-only name is served to the machines it decides addresses for. +func TestThePlansAreServedAndThePlacementSaysWhoIsInside(t *testing.T) { + const route = `{"provision":"route","provider":"anchor","provider-module":"route-proxy",` + gitea := route + contributed("10.10.0.7", + "web", `{"name":"git.example","internal-name":"git.anchor.internal","port":3000}`, + "api", `{"internal-name":"git-api.anchor.internal","port":3001}`, + "refused", `{"name":"bad.example","port":3002,"host":"x"}`) + `}` + plans := &fakePlans{current: map[string]string{ + "laptop.gitea": plan("laptop", "gitea", gitea+`,{"provision":"postgres-database","provider":"anchor",`+ + `"provider-module":"postgres",`+contributed("10.10.0.7", "", `{"name":"gitea"}`)+`}`), + "anchor.blog": plan("anchor", "blog", route+contributed("10.10.0.1", "", `{"name":"blog.example","port":2368}`)+`}`), + "anchor.other": plan("anchor", "other", `{"provision":"route","provider":"edge","provider-module":"route-proxy",`+ + contributed("", "", `{"name":"other.example","port":80}`)+`}`), + "anchor.older": plan("anchor", "older", route+`"local":""}`), + "laptop.lost": plan("laptop", "lost", route+contributed("", "", `{"name":"lost.example","port":80}`)+`}`), + }} + held := newTable() + var said []string + say := func(kind string, e contract.Event) { + said = append(said, kind+" "+e.Proxy+" "+e.Name+" "+e.Module+" "+e.Local+" "+e.Why) + } + if err := followPlans(plans, newPlanned("anchor", say), held, say); err != nil { + t.Fatal(err) + } + served := func() map[string]string { + out := map[string]string{} + held.mu.RLock() + defer held.mu.RUnlock() + for host, rules := range held.to { + out[host] = rules[0].target + } + return out + } + want := map[string]string{"git.example": "http://10.10.0.7:3000", "git.anchor.internal": "http://10.10.0.7:3000", + "git-api.anchor.internal": "http://10.10.0.7:3001", "blog.example": "http://127.0.0.1:2368"} + if got := served(); !reflect.DeepEqual(got, want) { + t.Fatalf("serving %v, want %v", got, want) + } + if !held.eligibleForACME("git.example") || held.eligibleForACME("git-api.anchor.internal") { + t.Error("a public name is not certified publicly, or an internal-only one is") + } + // Each refusal said with why, and each name served said once. + sort.Strings(said) + for _, w := range []string{ + "route.refused anchor gitea refused not a route this contract reads: json: unknown field \"host\"", + "route.refused anchor lost it runs on laptop and its plan says no address on the private network for it", + "route.refused anchor older its plan names this proxy for its route and carries nothing to serve", + "route.served anchor blog.example blog ", + "route.served anchor git-api.anchor.internal gitea ", + } { + if !slices.Contains(said, w) { + t.Errorf("not said %q, said:\n%s", w, strings.Join(said, "\n")) + } + } + if len(said) != 7 { + t.Errorf("said %d events, want 3 refusals and 4 names served:\n%s", len(said), strings.Join(said, "\n")) + } + for _, row := range []struct { + name, key, value string + served []string + withdrawn []string + say []string + }{ + {"a route arrives", "anchor.new", plan("anchor", "new", route+contributed("", "", `{"name":"new.example","port":81}`)+`}`), + []string{"new.example"}, nil, []string{"route.served anchor new.example new "}}, + {"an unreadable plan keeps what it gave", "laptop.gitea", `{"v":1,`, []string{"git.example"}, nil, nil}, + {"a deleted plan withdraws its routes", "laptop.gitea", "", nil, []string{"git.example", "git-api.anchor.internal"}, + []string{"route.withdrawn anchor git-api.anchor.internal ", "route.withdrawn anchor git.anchor.internal ", + "route.withdrawn anchor git.example "}}, + } { + said = nil + change := stdio.StateChange{Key: row.key, Op: "put", Value: json.RawMessage(row.value)} + if row.value == "" { + change = stdio.StateChange{Key: row.key, Op: "delete"} + } + if err := plans.handle(change); err != nil { + t.Fatal(err) + } + got := served() + for _, host := range row.served { + if _, ok := got[host]; !ok { + t.Errorf("%s: %s is not served (%v)", row.name, host, got) + } + } + for _, host := range row.withdrawn { + if _, ok := got[host]; ok { + t.Errorf("%s: %s is still served", row.name, host) + } + } + if !slices.Equal(said, row.say) { + t.Errorf("%s: said %q, want %q", row.name, said, row.say) + } + } + + held = behind(t) + if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:1"); code != http.StatusNotFound { + t.Errorf("before the placement is read, a machine of the mesh was served an internal-only name: %d", code) + } + laptop, err := delivery.PlacementValues(delivery.Placement{Node: "laptop", Decided: delivery.NodeDecided{Address: "10.10.0.7"}}) + if err != nil { + t.Fatal(err) + } + placement := &fakePlans{current: map[string]string{}} + for k, v := range laptop { + placement.current[k] = string(v) + } + if err := followInside(placement, held); err != nil { + t.Fatal(err) + } + if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:1"); code == http.StatusNotFound { + t.Error("a machine the placement decides an address for was refused the internal-only route") + } + if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.8:1"); code != http.StatusNotFound { + t.Error("a machine the placement does not name was served the internal-only route") + } + if err := placement.handle(stdio.StateChange{Key: "laptop.node", Op: "put", Value: json.RawMessage(`{`)}); err != nil { + t.Fatal(err) + } + if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:1"); code == http.StatusNotFound { + t.Error("an unreadable placement key changed who is inside") } } diff --git a/modules/route-proxy/cmd/route-proxy/routes_test.go b/modules/route-proxy/cmd/route-proxy/routes_test.go index a8447f45..836dbb56 100644 --- a/modules/route-proxy/cmd/route-proxy/routes_test.go +++ b/modules/route-proxy/cmd/route-proxy/routes_test.go @@ -2,23 +2,40 @@ package main import ( "context" + "encoding/json" "fmt" "io" "net/http" "net/http/httptest" - "os" - "path/filepath" "strings" "testing" + + "git.novox.be/novox/mesh-controller/modules/route-proxy/contract" ) -func write(t *testing.T, body string) string { - t.Helper() - path := filepath.Join(t.TempDir(), "routes.json") - if err := os.WriteFile(path, []byte(body), 0o644); err != nil { - t.Fatal(err) +// routesFrom is the table a test's routes make, written as `{"given":[{"from","node","at","values"}…]}`: the +// contributions the plans give the proxy, each read with route-proxy's contract as routesOfPlan reads it, and one the +// contract refuses left out, as routesOfPlan leaves it out. +func routesFrom(body string) (map[string][]rule, map[string]bool, error) { + var said struct { + Given []struct { + From, Node, At string + Values json.RawMessage + } `json:"given"` } - return path + if err := json.Unmarshal([]byte(body), &said); err != nil { + return nil, nil, err + } + var given []contribution + for _, g := range said.Given { + route, err := contract.DecodeRequest(g.Values) + if err != nil { + continue + } + given = append(given, contribution{From: g.From, Node: g.Node, At: g.At, Route: route}) + } + routes, public := routesOf(given) + return routes, public, nil } // plain is the table an ordinary set of routes makes: one host, one target, no policy. @@ -51,9 +68,9 @@ func targetOf(routes map[string][]rule, host string) string { // A route is a grant: the consumer supplies a target, and where that machine is comes from the // mesh rather than from a naming convention the proxy has to know. func TestARouteGoesToWhereTheMeshSaysTheConsumerIs(t *testing.T) { - routes, _, err := routesFrom(write(t, `{"contributions":1,"requirement":"route","given":[ + routes, _, err := routesFrom(`{"contributions":1,"requirement":"route","given":[ {"from":"app","node":"laptop","at":"laptop.internal","values":{"name":"App.Example","port":8080}} - ]}`)) + ]}`) if err != nil { t.Fatal(err) } @@ -68,10 +85,10 @@ func TestARouteGoesToWhereTheMeshSaysTheConsumerIs(t *testing.T) { // target — the same convenience a predecessor proxy gave for reaching a service over the VPN // without a public TLS round trip. func TestARouteWithAnInternalNameIsReachableUnderBoth(t *testing.T) { - routes, public, err := routesFrom(write(t, `{"given":[ + routes, public, err := routesFrom(`{"given":[ {"from":"app","node":"anchor","at":"anchor.internal", "values":{"name":"app.example","internal-name":"app.anchor.internal","port":8080}} - ]}`)) + ]}`) if err != nil { t.Fatal(err) } @@ -94,10 +111,10 @@ func TestARouteWithAnInternalNameIsReachableUnderBoth(t *testing.T) { // one (novox/hq ADR 0138), and is served under that name rather than skipped as naming nothing — // skipping it left every internal-only module unreachable by name (novox/hq issue 191). func TestARouteWithOnlyAnInternalNameIsServed(t *testing.T) { - routes, public, err := routesFrom(write(t, `{"given":[ + routes, public, err := routesFrom(`{"given":[ {"from":"app","node":"anchor","at":"anchor.internal", "values":{"internal-name":"App.Anchor.Internal","port":8443,"scheme":"https","insecure":true}} - ]}`)) + ]}`) if err != nil { t.Fatal(err) } @@ -123,9 +140,9 @@ func TestARouteWithOnlyAnInternalNameIsServed(t *testing.T) { // A route with no internal-name composed gets no second host — the ordinary case, unchanged. func TestARouteWithNoInternalNameGetsNoAlias(t *testing.T) { - routes, _, err := routesFrom(write(t, `{"given":[ + routes, _, err := routesFrom(`{"given":[ {"from":"app","node":"anchor","values":{"name":"app.example","port":8080}} - ]}`)) + ]}`) if err != nil { t.Fatal(err) } @@ -137,9 +154,9 @@ func TestARouteWithNoInternalNameGetsNoAlias(t *testing.T) { // A workload beside the proxy is ordinary, and reaching it over loopback is both correct and the // only thing that works when there is no private network. func TestAConsumerOnTheProxysOwnMachineIsReachedOverLoopback(t *testing.T) { - routes, _, err := routesFrom(write(t, `{"given":[ + routes, _, err := routesFrom(`{"given":[ {"from":"app","node":"anchor","values":{"name":"app.example","port":9000}} - ]}`)) + ]}`) if err != nil { t.Fatal(err) } @@ -151,10 +168,10 @@ func TestAConsumerOnTheProxysOwnMachineIsReachedOverLoopback(t *testing.T) { // A route may name a target reached over https, for a backend that terminates its own TLS — the // shape Mailu's webmail front needs, which this proxy reaches as a plain workload otherwise. func TestARouteMayTargetHttps(t *testing.T) { - routes, _, err := routesFrom(write(t, `{"given":[ + routes, _, err := routesFrom(`{"given":[ {"from":"mail","node":"anchor","at":"anchor.internal", "values":{"name":"mail.example","port":7443,"scheme":"https","insecure":true}} - ]}`)) + ]}`) if err != nil { t.Fatal(err) } @@ -319,10 +336,10 @@ func TestNoCertificateIsAskedForOnAnUnroutedName(t *testing.T) { // public CA can validate a private name, and asking anyway would only spend the account's rate // limit on an order that can never succeed. func TestNoCertificateIsAskedForOnAnInternalAlias(t *testing.T) { - routes, public, err := routesFrom(write(t, `{"given":[ + routes, public, err := routesFrom(`{"given":[ {"from":"app","node":"anchor","at":"anchor.internal", "values":{"name":"app.example","internal-name":"app.anchor.internal","port":8080}} - ]}`)) + ]}`) if err != nil { t.Fatal(err) } @@ -341,10 +358,10 @@ func TestNoCertificateIsAskedForOnAnInternalAlias(t *testing.T) { // A certificate is asked of the *internal* authority only for a name that is routed here and is // not a route's own public name — the internal-network alias, never the route it accompanies. func TestTheInternalAuthorityOnlyCertifiesInternalOnlyAliases(t *testing.T) { - routes, public, err := routesFrom(write(t, `{"given":[ + routes, public, err := routesFrom(`{"given":[ {"from":"app","node":"anchor","at":"anchor.internal", "values":{"name":"app.example","internal-name":"app.anchor.internal","port":8080}} - ]}`)) + ]}`) if err != nil { t.Fatal(err) } @@ -385,10 +402,10 @@ func TestWithdrawingARouteWithdrawsItsCertificate(t *testing.T) { // default refuses them long before the workload is reached. It is configuration beside `insecure`, // not a fifth policy — novox/hq ADR 0108 closed that set at four. func TestARouteMayLimitTheBodyItCarries(t *testing.T) { - routes, _, err := routesFrom(write(t, `{"given":[ + routes, _, err := routesFrom(`{"given":[ {"from":"registry","node":"anchor","at":"anchor.internal", "values":{"name":"images.example","port":5000,"max-request-body":21474836480}} - ]}`)) + ]}`) if err != nil { t.Fatal(err) } @@ -403,9 +420,9 @@ func TestARouteMayLimitTheBodyItCarries(t *testing.T) { // Saying nothing leaves the route unlimited, which is what every route already got. func TestARouteThatSaysNothingCarriesAnySize(t *testing.T) { - routes, _, err := routesFrom(write(t, `{"given":[ + routes, _, err := routesFrom(`{"given":[ {"from":"app","node":"anchor","at":"anchor.internal","values":{"name":"app.example","port":8080}} - ]}`)) + ]}`) if err != nil { t.Fatal(err) } @@ -427,10 +444,10 @@ func TestABodyOverTheLimitIsRefusedAndOneUnderItIsCarried(t *testing.T) { at := strings.TrimPrefix(workload.URL, "http://") host, port, _ := strings.Cut(at, ":") - routes, _, err := routesFrom(write(t, `{"given":[ - {"from":"registry","node":"anchor","at":"`+host+`", - "values":{"name":"images.example","port":`+port+`,"max-request-body":8}} - ]}`)) + routes, _, err := routesFrom(`{"given":[ + {"from":"registry","node":"anchor","at":"` + host + `", + "values":{"name":"images.example","port":` + port + `,"max-request-body":8}} + ]}`) if err != nil { t.Fatal(err) } @@ -491,7 +508,7 @@ func TestAContributionThatCannotBeServedIsSkipped(t *testing.T) { {"a zero limit", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":0}}`, ""}, {"a fractional limit", `{"from":"r","node":"a","at":"a.internal","values":{"name":"images.example","port":5000,"max-request-body":1.5}}`, ""}, } { - routes, public, err := routesFrom(write(t, `{"given":[`+row.given+`]}`)) + routes, public, err := routesFrom(`{"given":[` + row.given + `]}`) if err != nil { t.Fatal(err) } diff --git a/modules/route-proxy/contract/contract.go b/modules/route-proxy/contract/contract.go new file mode 100644 index 00000000..99b8d46b --- /dev/null +++ b/modules/route-proxy/contract/contract.go @@ -0,0 +1,77 @@ +// Package contract is route-proxy's contract for the provision `route`: what a consumer asks a route with, what the +// proxy keeps on the bus about what it serves, the events it says, and its tools' arguments (issue 774; research 072 +// row 34; the operator, 2026-10-31: "the route-proxy should define its own contract for the seat (with the message +// contracts of course) which other modules can use"). +// +// **Published by the repository that holds route-proxy, as its own Go module**, versioned by tags of the form +// `modules/route-proxy/contract/vX.Y.Z`, the subdirectory form Go's module system expects, the same way mesh-sdk is +// published (`go/vX.Y.Z`). A module that asks a route, reads what the proxy serves or hears its events imports this +// package; nothing imports the repository. Standard library only, so a consumer takes nothing else with it. +// +// **How a route travels.** A module that must be reachable requires `route` and says, in its manifest's +// `contributes.route`, one Request, or several by local name; Schema is that object's JSON schema. The controller +// composes each (its label joined to the node's domains into `name` and `internal-name`, its endpoint's port into +// `port`) and the seat mesh-provisioner carries the result in the consumer's plan entry as an opaque payload +// (mesh-sdk catalogue.PlanProvision.Contributed, in the generic catalogue.ContributedForm). Neither the seat nor its +// client knows a field of a route: route-proxy decodes each contribution with DecodeRequest, and refuses, saying why +// (EventRefused), what this contract does not admit. +// +// **Two name spaces, two authorities** (novox/hq ADR 0056, ADR 0138). `name` is public: served to anyone and +// certified by the public ACME directory (Let's Encrypt). `internal-name` is served only inside the private network +// and certified by the mesh's own authority. A route carries either, or both. +package contract + +// Module is the module that provides a route, and Provision what it provides: what a plan's entry names when +// route-proxy is a route's provider. +const ( + Module = "route-proxy" + Provision = "route" +) + +// Version is the version of the Request form this package reads and writes. +const Version = 1 + +// The events route-proxy says (its manifest's `emits`), on mesh.mod.route-proxy.event., each with an Event. +const ( + // EventServed: a name is now served by this proxy (it was not, or it is now served from other rules). + EventServed = "route.served" + // EventWithdrawn: a name this proxy served is served no more, because no plan gives it any longer. + EventWithdrawn = "route.withdrawn" + // EventRefused: a plan gives this proxy a route it does not serve, and Why says why. + EventRefused = "route.refused" +) + +// Event is the body of every event route-proxy says: the proxy's machine, the name (empty for a refusal that named +// none), how far it reaches, and who asked it: the consumer module, its machine and the local name of its +// contribution. Why is set on a refusal alone. +type Event struct { + Proxy string `json:"proxy"` + Name string `json:"name,omitempty"` + Reach string `json:"reach,omitempty"` + Module string `json:"module,omitempty"` + Node string `json:"node,omitempty"` + Local string `json:"local,omitempty"` + Why string `json:"why,omitempty"` +} + +// The tools route-proxy serves (its manifest's `tools`), each answering from its states RoutesState and UnroutedState. +const ( + // ToolRoutes lists every name served, with RoutesArgs. + ToolRoutes = "route_proxy_routes" + // ToolRoute is one name in full, with RouteArgs. + ToolRoute = "route_proxy_route" + // ToolUnrouted lists the names asked for and not served, with RoutesArgs. + ToolUnrouted = "route_proxy_unrouted" +) + +// RoutesArgs are ToolRoutes' and ToolUnrouted's arguments: Node narrows to the proxy on one machine, every machine +// when empty. +type RoutesArgs struct { + Node string `json:"node,omitempty"` +} + +// RouteArgs are ToolRoute's arguments: the name, such as app.example.org, and optionally the machine whose proxy. +type RouteArgs struct { + Name string `json:"name"` + Node string `json:"node,omitempty"` +} diff --git a/modules/route-proxy/contract/contract_test.go b/modules/route-proxy/contract/contract_test.go new file mode 100644 index 00000000..5095e0ea --- /dev/null +++ b/modules/route-proxy/contract/contract_test.go @@ -0,0 +1,85 @@ +package contract + +import ( + "encoding/json" + "reflect" + "slices" + "strings" + "testing" +) + +func limit(n int64) *int64 { return &n } + +// A route is read strictly and whole (issue 774): what the contract names is admitted, what it does not, or a route +// it cannot serve, is refused naming why; and the published schema names exactly the fields the decoder reads. +func TestARouteIsReadStrictly(t *testing.T) { + for _, c := range []struct { + raw, refused string + want Request + }{ + {raw: `{"name":"app.example.org","internal-name":"app.anchor.internal","label":"app","endpoint":"web","port":8080}`, + want: Request{Name: "app.example.org", InternalName: "app.anchor.internal", Label: "app", Endpoint: "web", Port: 8080}}, + {raw: `{"internal-name":"app.anchor.internal","port":443,"scheme":"https","insecure":true,"max-request-body":1048576}`, + want: Request{InternalName: "app.anchor.internal", Port: 443, Scheme: "https", Insecure: true, MaxRequestBody: limit(1048576)}}, + {raw: `{"name":"git.example.org","path":"/api/internal","deny":true,"priority":100000}`, + want: Request{Name: "git.example.org", Path: "/api/internal", Deny: true, Priority: 100000}}, + {raw: `{"name":"old.example.org","redirect":"https://new.example.org"}`, + want: Request{Name: "old.example.org", Redirect: "https://new.example.org"}}, + {raw: `{"name":"admin.example.org","port":80,"auth":"/run/secrets/admin"}`, + want: Request{Name: "admin.example.org", Port: 80, Auth: "/run/secrets/admin"}}, + {raw: `{"label":"app","port":8080}`, refused: "names nothing"}, + {raw: `{"name":"app.example.org"}`, refused: "gives no port"}, + {raw: `{"name":"app.example.org","port":70000}`, refused: "not a port"}, + {raw: `{"name":"app.example.org","port":80.5}`, refused: "not a route"}, + {raw: `{"name":"app.example.org","port":"80"}`, refused: "not a route"}, + {raw: `{"name":"app.example.org","port":80,"scheme":"ftp"}`, refused: "neither http nor https"}, + {raw: `{"name":"app.example.org","port":80,"max-request-body":-1}`, refused: "max-request-body"}, + {raw: `{"name":"app.example.org","port":80,"max-request-body":0}`, refused: "max-request-body"}, + {raw: `{"name":"app.example.org","port":80,"auth":"admin:$2y$05$abc"}`, refused: "credential"}, + {raw: `{"name":"app.example.org","port":80,"host":"x"}`, refused: "unknown field \"host\""}, + {raw: `{"name":"app.example.org","port":80} {}`, refused: "more than one"}, + } { + got, err := DecodeRequest([]byte(c.raw)) + if c.refused != "" { + if err == nil || !strings.Contains(err.Error(), c.refused) { + t.Errorf("DecodeRequest(%s) = %+v, %v; want refused for %q", c.raw, got, err, c.refused) + } + continue + } + if err != nil || !reflect.DeepEqual(got, c.want) { + t.Errorf("DecodeRequest(%s) = %+v, %v; want %+v", c.raw, got, err, c.want) + continue + } + // What it encodes, it reads back the same. + if raw, err := got.Encode(); err != nil { + t.Errorf("Encode(%+v): %v", got, err) + } else if again, err := DecodeRequest(raw); err != nil || !reflect.DeepEqual(again, got) { + t.Errorf("Encode then DecodeRequest of %+v = %+v, %v", got, again, err) + } + } + if (Request{Path: "api"}).Prefix() != "/api" || (Request{}).Prefix() != "" || (Request{Scheme: " HTTPS"}).SchemeOrDefault() != "https" || + (Request{}).SchemeOrDefault() != "http" || (Request{Redirect: "x"}).Forwards() { + t.Error("Prefix, SchemeOrDefault or Forwards says otherwise than the contract") + } + + var schema struct { + Properties map[string]json.RawMessage `json:"properties"` + AdditionalProperties bool `json:"additionalProperties"` + } + if err := json.Unmarshal(Schema, &schema); err != nil || schema.AdditionalProperties { + t.Fatalf("the schema is not a closed object: %v", err) + } + var tags []string + for f := range reflect.TypeFor[Request]().Fields() { + tags = append(tags, strings.Split(f.Tag.Get("json"), ",")[0]) + } + var said []string + for name := range schema.Properties { + said = append(said, name) + } + slices.Sort(tags) + slices.Sort(said) + if !slices.Equal(tags, said) { + t.Fatalf("the schema names %v, the decoder reads %v", said, tags) + } +} diff --git a/modules/route-proxy/contract/go.mod b/modules/route-proxy/contract/go.mod new file mode 100644 index 00000000..91110457 --- /dev/null +++ b/modules/route-proxy/contract/go.mod @@ -0,0 +1,3 @@ +module git.novox.be/novox/mesh-controller/modules/route-proxy/contract + +go 1.26.0 diff --git a/modules/route-proxy/contract/request.go b/modules/route-proxy/contract/request.go new file mode 100644 index 00000000..1a10c98b --- /dev/null +++ b/modules/route-proxy/contract/request.go @@ -0,0 +1,128 @@ +package contract + +import ( + "bytes" + _ "embed" + "encoding/json" + "errors" + "fmt" + "strings" +) + +// Request is one route a consumer asks for, as its manifest's `contributes.route` says it and the controller composes +// it: the object Schema describes. Every field is optional to the decoder; Validate says which a route needs. +type Request struct { + // Name is the public name, served to anyone and certified by the public ACME directory; InternalName the name on + // the private network, served inside it alone and certified by the mesh's own authority. The controller composes + // both from Label and the node's domains, as far as the endpoint reaches (novox/hq ADR 0138); a manifest writes a + // label, never a domain. + Name string `json:"name,omitempty"` + InternalName string `json:"internal-name,omitempty"` + Label string `json:"label,omitempty"` + // Endpoint names the manifest's endpoint the route serves; the controller fills Port from it. + Endpoint string `json:"endpoint,omitempty"` + Port int `json:"port,omitempty"` + // Path is the path prefix the rule is scoped to, every path when empty; Priority which rule wins where two match, + // higher first. + Path string `json:"path,omitempty"` + Priority int `json:"priority,omitempty"` + // Scheme is how the consumer is reached, http (the default) or https; Insecure skips verifying its certificate, + // for a backend that terminates TLS with its own. + Scheme string `json:"scheme,omitempty"` + Insecure bool `json:"insecure,omitempty"` + // MaxRequestBody is the largest body, in bytes, the route carries; no limit when absent. Said, it is whole and + // positive: a limit the proxy cannot keep is a route it does not serve. + MaxRequestBody *int64 `json:"max-request-body,omitempty"` + // The policy of the rule, a closed set (novox/hq ADR 0108): Deny refuses every request, Redirect answers with a + // permanent redirect to that address, and Auth names the file of the secret a request must present, never the + // secret itself. + Deny bool `json:"deny,omitempty"` + Redirect string `json:"redirect,omitempty"` + Auth string `json:"auth,omitempty"` +} + +// Schema is Request's JSON schema, published with this package for what checks a manifest's `contributes.route`. +// +//go:embed route.schema.json +var Schema []byte + +// DecodeRequest reads one contribution's object strictly: no field this contract does not name, one document, and +// a route Validate admits. What it refuses, it refuses whole, naming why: a route is never served half. +func DecodeRequest(raw []byte) (Request, error) { + var r Request + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&r); err != nil { + return Request{}, fmt.Errorf("not a route this contract reads: %w", err) + } + if dec.More() { + return Request{}, errors.New("not a route this contract reads: more than one document") + } + if err := r.Validate(); err != nil { + return Request{}, err + } + return r, nil +} + +// Encode is r as a contribution's object, refused where DecodeRequest would refuse it. +func (r Request) Encode() ([]byte, error) { + if err := r.Validate(); err != nil { + return nil, err + } + return json.Marshal(r) +} + +// Validate says what makes r no route: no name of either kind; a port outside 1-65535, or none where the rule +// forwards (neither Deny nor Redirect); a scheme other than http or https; a body limit that is not positive; and a +// credential written where a secret's file is to be named (novox/hq ADR 0108: refused rather than tolerated). +func (r Request) Validate() error { + if strings.TrimSpace(r.Name) == "" && strings.TrimSpace(r.InternalName) == "" { + return errors.New("the route names nothing to be served under: neither name nor internal-name") + } + if r.Port < 0 || r.Port > 65535 { + return fmt.Errorf("the route's port %d is not a port", r.Port) + } + if r.Forwards() && r.Port == 0 { + return errors.New("the route forwards and gives no port") + } + switch strings.ToLower(strings.TrimSpace(r.Scheme)) { + case "", "http", "https": + default: + return fmt.Errorf("the route's scheme %q is neither http nor https", r.Scheme) + } + if r.MaxRequestBody != nil && *r.MaxRequestBody <= 0 { + return fmt.Errorf("the route's max-request-body %d is not a positive number of bytes", *r.MaxRequestBody) + } + if LooksLikeACredential(r.Auth) { + return errors.New("the route's auth holds a credential rather than the name of a secret's file (novox/hq ADR 0108)") + } + return nil +} + +// Forwards is whether the rule sends a request on to the consumer, rather than answering it itself. +func (r Request) Forwards() bool { return !r.Deny && r.Redirect == "" } + +// Prefix is the path prefix the rule is scoped to, beginning with a slash, or "" for every path. +func (r Request) Prefix() string { + p := strings.TrimSpace(r.Path) + if p != "" && !strings.HasPrefix(p, "/") { + p = "/" + p + } + return p +} + +// SchemeOrDefault is how the consumer is reached: the scheme, lower-cased, http when none is said. +func (r Request) SchemeOrDefault() string { + if s := strings.ToLower(strings.TrimSpace(r.Scheme)); s != "" { + return s + } + return "http" +} + +// LooksLikeACredential is the check that keeps a secret out of a declaration: a value holding a `:` (the htpasswd +// separator) or opening with a bcrypt identifier is a credential, not a path, and no path the mesh writes needs +// either. It errs towards refusing. +func LooksLikeACredential(v string) bool { + v = strings.TrimSpace(v) + return strings.Contains(v, ":") || strings.HasPrefix(v, "$2") +} diff --git a/modules/route-proxy/contract/route.schema.json b/modules/route-proxy/contract/route.schema.json new file mode 100644 index 00000000..af55a78d --- /dev/null +++ b/modules/route-proxy/contract/route.schema.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://git.novox.be/novox/mesh-controller/modules/route-proxy/contract/route.schema.json", + "title": "route", + "description": "One route a consumer asks route-proxy for, as a manifest's contributes.route says it (one, or several by local name) and the controller composes it. Version 1 of route-proxy's contract.", + "type": "object", + "additionalProperties": false, + "anyOf": [ + {"required": ["name"]}, + {"required": ["internal-name"]}, + {"required": ["label"]} + ], + "properties": { + "name": {"type": "string", "description": "the public name, served to anyone and certified by the public ACME directory; composed from label"}, + "internal-name": {"type": "string", "description": "the name on the private network, served inside it alone and certified by the mesh's own authority; composed from label"}, + "label": {"type": "string", "description": "the subdomain the module chooses; joined to the node's domains by the controller"}, + "endpoint": {"type": "string", "description": "the manifest's endpoint the route serves; the controller fills port from it"}, + "port": {"type": "integer", "minimum": 1, "maximum": 65535}, + "path": {"type": "string", "description": "the path prefix the rule is scoped to; every path when absent"}, + "priority": {"type": "integer", "description": "which rule wins where two match, higher first"}, + "scheme": {"enum": ["http", "https"], "description": "how the consumer is reached; http when absent"}, + "insecure": {"type": "boolean", "description": "skip verifying the consumer's certificate over https"}, + "max-request-body": {"type": "integer", "minimum": 1, "description": "the largest body, in bytes, the route carries"}, + "deny": {"type": "boolean", "description": "refuse every request matching the rule"}, + "redirect": {"type": "string", "description": "answer with a permanent redirect to this address"}, + "auth": {"type": "string", "description": "the file of the secret a request must present, never the secret itself"} + } +} diff --git a/modules/route-proxy/internal/routestate/routestate.go b/modules/route-proxy/contract/state.go similarity index 92% rename from modules/route-proxy/internal/routestate/routestate.go rename to modules/route-proxy/contract/state.go index 44dd82d8..e1fce955 100644 --- a/modules/route-proxy/internal/routestate/routestate.go +++ b/modules/route-proxy/contract/state.go @@ -1,11 +1,12 @@ -// Package routestate is what the web front keeps on the bus about the names it serves, and about the names it was asked -// for and does not serve (issue 774): the shape both halves of route-proxy agree on. The proxy writes it, on change, -// in its states `routes` and `unrouted`, announced readable; the module's tools read it from there, so the question -// "is this site reachable, and is its certificate good" is answered from the bus rather than with curl. +package contract + +// What the web front keeps on the bus about the names it serves, and about the names it was asked for and does not +// serve (issue 774). The proxy writes it, on change, in its states `routes` and `unrouted`, announced readable; the +// module's tools, and any module that reads them, read it from there, so the question "is this site reachable, and +// is its certificate good" is answered from the bus rather than with curl. // // A key is `.`: the machine whose proxy answers, then the name as asked, lower-cased. A node's name holds // no dot, so the first dot splits them. -package routestate import ( "net" diff --git a/modules/route-proxy/go.mod b/modules/route-proxy/go.mod index c826c270..8be93094 100644 --- a/modules/route-proxy/go.mod +++ b/modules/route-proxy/go.mod @@ -3,12 +3,13 @@ module route-proxy go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030231858-b9ef1e4fcb68 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 ) require ( + git.novox.be/novox/mesh-controller/modules/route-proxy/contract v0.1.0 github.com/klauspost/compress v1.20.0 // indirect github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect diff --git a/modules/route-proxy/go.sum b/modules/route-proxy/go.sum index 286ed6eb..6fbcb8c2 100644 --- a/modules/route-proxy/go.sum +++ b/modules/route-proxy/go.sum @@ -1,5 +1,7 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030231858-b9ef1e4fcb68 h1:2rMJX0RMIBaN5NDIaEZVwbj5MFism1O3bRayEz+WlOs= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030231858-b9ef1e4fcb68/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-controller/modules/route-proxy/contract v0.1.0 h1:89jE1oBK+l7gFJjz6wx8mUD3KxrqSDa7pUTlQkzC1do= +git.novox.be/novox/mesh-controller/modules/route-proxy/contract v0.1.0/go.mod h1:Yr+Xp89kpMiKH6kA06EU+d9I2CyzrK9yMX7J3pSv0hE= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a h1:DcSvfEmdPBKqCg5OmXRNb269EkkPe7BsEp+RykEhkgE= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/route-proxy/module.json b/modules/route-proxy/module.json index 1a42825f..67e20525 100644 --- a/modules/route-proxy/module.json +++ b/modules/route-proxy/module.json @@ -16,9 +16,6 @@ "serves": { "route": {} }, - "receives": { - "route": "${dir:routes-dir}/mesh.json" - }, "requires": [ "internal-acme-ca" ], @@ -73,12 +70,6 @@ "mode": "0700", "place": "mesh" }, - { - "id": "routes-dir", - "type": "directory", - "path": "/var/lib/route-proxy/routes", - "mode": "0700" - }, { "id": "acme-cache", "type": "directory", @@ -167,14 +158,12 @@ "${dir:state}/internal-acme.env" ], "volumes": [ - "${dir:routes-dir}:/routes:ro", "${dir:acme-cache}:/acme", "${dir:ca-dir}:/ca:ro", "${dir:mesh-state}/broker:/run/secrets/broker:ro", "${dir:state}/route-proxy:/opt/mesh:ro" ], "env": { - "ROUTES": "/routes/mesh.json", "LISTEN": ":80", "TLS_LISTEN": ":443", "ACME_CACHE": "/acme", @@ -256,6 +245,11 @@ "route_proxy_route", "route_proxy_unrouted" ], + "emits": [ + "route.served", + "route.withdrawn", + "route.refused" + ], "replaces": { "route_proxy_routes": [ "curl -I" @@ -268,6 +262,7 @@ ] }, "reads": [ + "mesh-delivery-assigner.placement", "mesh-provisioner.plans", "mesh-provisioner.provisions" ]