Merge pull request 'A module's process may hold a bus account of its own, and a module stream may be a work queue (ADR 0436, issues 1170, 1177-1181)' (#390) from feat/mail-sender-controller into main

This commit was merged in pull request #390.
This commit is contained in:
2026-10-19 02:07:51 +00:00
30 changed files with 2352 additions and 48 deletions
+20 -1
View File
@@ -680,7 +680,13 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
if !ok || family == "" || with.Needed[m.Module][name] == "" {
continue
}
first = append(first, ownedBy(m.SecretsOwner, map[string]any{
// A per-secret owner (novox/hq ADR 0436 §1) stands in for the module's secrets-owner, when
// the manifest named one for this secret (or the family it belongs to).
owner := m.SecretsOwner
if own.Owner != "" {
owner = own.Owner
}
first = append(first, ownedBy(owner, map[string]any{
"id": NeedID(name), "type": "file", "path": own.Path, "sealed": with.Needed[m.Module][name],
}))
}
@@ -694,6 +700,9 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
if m.Module == RuntimeModule && r.Account != "" {
owner = r.Account
}
if o := m.OwnSecrets[name].Owner; o != "" {
owner = o
}
first = append(first, ownedBy(owner, map[string]any{
"id": NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, HandedSecretField: name,
}))
@@ -709,6 +718,9 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
if m.Module == RuntimeModule && r.Account != "" {
owner = r.Account
}
if o := m.OwnSecrets[name].Owner; o != "" {
owner = o
}
first = append(first, ownedBy(owner, map[string]any{
"id": NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, VaultSealedField: box,
}))
@@ -742,6 +754,9 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
if m.Module == RuntimeModule && r.Account != "" {
owner = r.Account
}
if o := m.OwnSecrets[name].Owner; o != "" {
owner = o
}
first = append(first, ownedBy(owner, map[string]any{
"id": NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, "sealed": sealed,
}))
@@ -1169,6 +1184,10 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
delete(copied, SecretsInEnvironment)
delete(copied, NamesOnPurpose)
delete(copied, TrustedField)
// A process's bus stays controller-side (novox/hq ADR 0436 §1): never composed onto the
// node-engine's process resource, whose wire declaration is mesh-host's own, vendored, with a
// strict decoder that refuses an unknown key.
delete(copied, ProcessBusField)
// **An operator's value, from the assignment** (novox/hq ADR 0112, ADR 0155): what a
// definition may not carry because it is true of one installation only. Filled from
// the same layers a mergeable file takes, and refused when no layer set it.
+1 -1
View File
@@ -168,7 +168,7 @@ func placedManifest(m Manifest, with Rendering) (Manifest, error) {
if err != nil {
return m, err
}
own[name] = OwnSecret{Path: filled, Taken: s.Taken, IssuedBy: s.IssuedBy}
own[name] = OwnSecret{Path: filled, Taken: s.Taken, IssuedBy: s.IssuedBy, Prompt: s.Prompt, Owner: s.Owner}
}
m.OwnSecrets = own
}
+45 -10
View File
@@ -1953,6 +1953,7 @@ func parseManifest(raw []byte, predefined PredefinedList) (Manifest, error) {
problems = append(problems, LogProblems(m)...)
// And the streams it owns, and who may hear them (streams.go, novox/hq ADR 0363).
problems = append(problems, StreamProblems(m)...)
problems = append(problems, WorkQueueStreamProblems(m)...)
// And the defaults it gives its settings (setting_defaults.go, novox/hq ADR 0262), or, written in the
// config form, the whole declaration of each value (config.go, novox/hq ADR 0348).
if m.DeclaresConfig() {
@@ -1990,6 +1991,7 @@ func parseManifest(raw []byte, predefined PredefinedList) (Manifest, error) {
// facts about the catalogue and are checked at registration (CatalogueProblems).
problems = append(problems, declaredSeatProblems(m)...)
problems = append(problems, RunsAsProblems(m)...)
problems = append(problems, ProcessBusProblems(m)...)
problems = append(problems, ReadsHolderProblems(m)...)
problems = append(problems, FilesHolderProblems(m)...)
if m.Computed != "" && len(m.Resources) > 0 {
@@ -2348,6 +2350,7 @@ func parseManifest(raw []byte, predefined PredefinedList) (Manifest, error) {
m.Module, name, own.IssuedBy, IssuedOutside))
}
problems = append(problems, ownSecretPromptProblems(m, name, own.Prompt, IsFamily(name))...)
problems = append(problems, ownSecretOwnerProblems(m, name, own.Owner)...)
}
localOf := map[string]string{}
for _, to := range m.SecretRequirements() {
@@ -2854,12 +2857,7 @@ func replacesProblems(m Manifest) []string {
func invokeProblems(m Manifest) []string {
var problems []string
for _, t := range m.Invokes {
if t == "*" {
continue
}
t = strings.TrimPrefix(t, "seat:")
module, tool, named := strings.Cut(t, ".")
if !named || !name.MatchString(module) || !toolName.MatchString(tool) {
if !invokeNameValid(t) {
problems = append(problems, fmt.Sprintf(
"%s invokes %q, which does not name a tool: a module invokes <module>.<tool>, or "+
"* for every tool on the mesh (novox/hq ADR 0152)", m.Module, t))
@@ -2930,6 +2928,11 @@ type OwnSecret struct {
// family's member suffix, filled where the secret is asked for. Optional; reviewed in the
// module's pull request like the rest of the manifest.
Prompt string
// Owner is the account this secret's files belong to on the machine, in place of the module's
// secrets-owner (novox/hq ADR 0436 §1): an account the module makes (a `user` resource of that
// name), never root and never the operator account. Empty means the module's secrets-owner, or
// root where that is absent too.
Owner string
}
// MeshMayMake says the mesh may put a value it makes in place of the one the secret holds: the
@@ -2984,6 +2987,34 @@ func ownSecretPromptProblems(m Manifest, name, prompt string, family bool) []str
return problems
}
// ownSecretOwnerProblems is what is wrong with an own secret's declared owner (novox/hq ADR 0436 §1): naming
// an account the module makes — a `user` resource of that name — never root, which a secret file owned by it
// would make readable by anything, and never the operator account's placeholder (`${machine:account}`), which
// the mesh fills per machine and a manifest cannot say safely (novox/hq ADR 0436, settled by the lead).
func ownSecretOwnerProblems(m Manifest, name, owner string) []string {
if owner == "" {
return nil
}
if owner == "root" {
return []string{fmt.Sprintf(
"%s names %q as the owner of its secret %q; an own secret's owner is an account the module "+
"makes, never root (novox/hq ADR 0436 §1)", m.Module, owner, name)}
}
if owner == "${machine:account}" {
return []string{fmt.Sprintf(
"%s names %q as the owner of its secret %q; an own secret's owner is an account the module "+
"makes, never the operator account (novox/hq ADR 0436 §1)", m.Module, owner, name)}
}
for _, r := range m.Resources {
if fmt.Sprint(r["type"]) == "user" && fmt.Sprint(r["name"]) == owner {
return nil
}
}
return []string{fmt.Sprintf(
"%s names %q as the owner of its secret %q, which it does not make: a user resource named %q "+
"(novox/hq ADR 0436 §1)", m.Module, owner, name, owner)}
}
// OwnSecrets is a module's own secrets by name. On the wire each is a path, or an object naming
// the path and how it is taken; written back the way it was read, so a manifest the mesh holds
// keeps its bytes.
@@ -3006,13 +3037,14 @@ func (o *OwnSecrets) UnmarshalJSON(raw []byte) error {
Taken string `json:"taken,omitempty"`
IssuedBy string `json:"issued-by,omitempty"`
Prompt string `json:"prompt,omitempty"`
Owner string `json:"owner,omitempty"`
}
dec := json.NewDecoder(bytes.NewReader(body))
dec.DisallowUnknownFields()
if err := dec.Decode(&long); err != nil {
return fmt.Errorf("own-secrets.%s: a path, or {\"path\", \"taken\", \"issued-by\", \"prompt\"}: %w", name, typedUnknown(err))
return fmt.Errorf("own-secrets.%s: a path, or {\"path\", \"taken\", \"issued-by\", \"prompt\", \"owner\"}: %w", name, typedUnknown(err))
}
out[name] = OwnSecret{Path: long.Path, Taken: long.Taken, IssuedBy: long.IssuedBy, Prompt: long.Prompt}
out[name] = OwnSecret{Path: long.Path, Taken: long.Taken, IssuedBy: long.IssuedBy, Prompt: long.Prompt, Owner: long.Owner}
}
*o = out
return nil
@@ -3021,7 +3053,7 @@ func (o *OwnSecrets) UnmarshalJSON(raw []byte) error {
func (o OwnSecrets) MarshalJSON() ([]byte, error) {
entries := make(map[string]any, len(o))
for name, s := range o {
if s.Taken == "" && s.IssuedBy == "" && s.Prompt == "" {
if s.Taken == "" && s.IssuedBy == "" && s.Prompt == "" && s.Owner == "" {
entries[name] = s.Path
continue
}
@@ -3035,6 +3067,9 @@ func (o OwnSecrets) MarshalJSON() ([]byte, error) {
if s.Prompt != "" {
long["prompt"] = s.Prompt
}
if s.Owner != "" {
long["owner"] = s.Owner
}
entries[name] = long
}
return json.Marshal(entries)
@@ -3074,7 +3109,7 @@ func (o OwnSecrets) Lookup(name string) (s OwnSecret, family string, ok bool) {
}
if family == "" || len(declared) > len(family) {
s, family, ok = OwnSecret{Path: strings.Replace(f.Path, "*", rest, 1), Taken: f.Taken, IssuedBy: f.IssuedBy,
Prompt: f.Prompt}, declared, true
Prompt: f.Prompt, Owner: f.Owner}, declared, true
}
}
return s, family, ok
+116
View File
@@ -0,0 +1,116 @@
package catalogue
import (
"strings"
"testing"
)
// An own secret (or a family) may declare `owner` (novox/hq ADR 0436 §1): the account the files holding it
// belong to on the machine, when that account is not the module's secrets-owner default. The fixture is
// mesh-mail's future manifest shape (ADR 0436 §7): a process account `mesh-mail-send` the module makes, and its
// own-secret family `mail-credential-*` owned by it.
// mailSenderManifest is the fixture every mail-sender issue builds on: the real parts of mesh-mail's manifest
// taken from mesh-catalog/modules/mesh-mail/module.json, extended as ADR 0436 §7-§8 describe, with `own` naming
// the account that owns the family's files.
func mailSenderManifest(t *testing.T, own string) Manifest {
t.Helper()
ownSecrets := `"mail-credential-*":{"path":"/var/lib/mesh-mail/mail-credential-*.secret","issued-by":"outside"`
if own != "" {
ownSecrets += `,"owner":"` + own + `"`
}
ownSecrets += `}`
m, err := ParseManifest([]byte(`{"module":"mesh-mail","version":"1","secrets-owner":"mesh-mail","own-secrets":{` +
ownSecrets + `},"resources":[{"id":"account","type":"user","name":"mesh-mail-send"}]}`))
if err != nil {
t.Fatalf("a manifest with own=%q was refused: %v", own, err)
}
return m
}
// 1170/AC1: a family owned by the module's own account lands at 0600, owned by that account — and a module's
// other own secrets (no per-secret owner declared) keep secrets-owner, unchanged.
func TestAnOwnSecretsOwnerIsGivenToItsFiles(t *testing.T) {
m := mailSenderManifest(t, "mesh-mail-send")
m.OwnSecrets["smtp-relay-key"] = OwnSecret{Path: "/var/lib/mesh-mail/smtp-relay-key", Taken: TakenAtStart}
r := Resolution{Node: "novox", Modules: []Manifest{m}}
out, err := r.Declaration(Rendering{
Needed: map[string]map[string]string{"mesh-mail": {"mail-credential-smtp": "sealed-credential", "smtp-relay-key": "sealed-relay"}},
})
if err != nil {
t.Fatalf("composing the declaration: %v", err)
}
var family, plain map[string]any
for _, res := range out {
if res["type"] != "file" {
continue
}
switch res["path"] {
case "/var/lib/mesh-mail/mail-credential-smtp.secret":
family = res
case "/var/lib/mesh-mail/smtp-relay-key":
plain = res
}
}
if family == nil {
t.Fatalf("the family member was not composed at all: %v", out)
}
if got := family["owner"]; got != "mesh-mail-send" {
t.Fatalf("the family member's owner: %v; want mesh-mail-send", got)
}
// review round 1, finding 7: asserted, not merely logged. ownedBy never sets "mode", so a sealed secret
// file carries none here either — the host applies 0600 to every one regardless (novox/hq ADR 0086) —
// and the owner clause must not have added one of its own that could override that.
if _, has := family["mode"]; has {
t.Fatalf("the family member carries its own mode %v; the owner clause must not set one, so the "+
"host's 0600 for every sealed secret file stands", family["mode"])
}
if plain == nil {
t.Fatalf("the module's other own secret was not composed at all: %v", out)
}
if got := plain["owner"]; got != "mesh-mail" {
t.Fatalf("a secret with no per-secret owner kept %v; want the module's secrets-owner %q", got, "mesh-mail")
}
}
// 1170/AC2: an owner naming an account the module does not make, root, or the operator account's placeholder is
// refused, naming the secret and the account.
func TestAnOwnSecretsOwnerIsRefusedWhenItIsNotAnAccountTheModuleMakes(t *testing.T) {
for name, c := range map[string]struct{ owner, says string }{
"root": {"root", "root"},
"the operator account": {"${machine:account}", "operator account"},
"an account not declared": {"not-made", "not-made"},
} {
_, err := ParseManifest([]byte(`{"module":"mesh-mail","version":"1","own-secrets":{` +
`"mail-credential-*":{"path":"/var/lib/mesh-mail/mail-credential-*.secret","issued-by":"outside","owner":"` +
c.owner + `"}},"resources":[{"id":"account","type":"user","name":"mesh-mail-send"}]}`))
if err == nil || !strings.Contains(err.Error(), "mail-credential-*") || !strings.Contains(err.Error(), c.says) {
t.Errorf("%s: %v; want a refusal naming the secret and %q", name, err, c.says)
}
}
}
// 1170/AC3: a manifest with no owner on any own secret composes exactly as it did before this clause existed:
// every secret file belongs to the module's secrets-owner (or nothing, when that is absent too).
func TestNoOwnerOnAnySecretComposesAsMainDoes(t *testing.T) {
m := mailSenderManifest(t, "")
r := Resolution{Node: "novox", Modules: []Manifest{m}}
out, err := r.Declaration(Rendering{
Needed: map[string]map[string]string{"mesh-mail": {"mail-credential-smtp": "sealed-credential"}},
})
if err != nil {
t.Fatalf("composing the declaration: %v", err)
}
var found bool
for _, res := range out {
if res["type"] == "file" && res["path"] == "/var/lib/mesh-mail/mail-credential-smtp.secret" {
found = true
if got := res["owner"]; got != "mesh-mail" {
t.Fatalf("with no per-secret owner declared, the file's owner: %v; want secrets-owner %q", got, "mesh-mail")
}
}
}
if !found {
t.Fatalf("the family member was not composed at all: %v", out)
}
}
+189
View File
@@ -0,0 +1,189 @@
package catalogue
import (
"bytes"
"encoding/json"
"fmt"
"strings"
)
// ProcessBusField is the key a process resource declares its bus on. Controller-side only: never composed
// onto the node-engine's process resource (novox/hq ADR 0436 §1) — the wire declaration is mesh-host's own,
// vendored, with a strict decoder that refuses an unknown key.
const ProcessBusField = "bus"
// ProcessBus is what one process of a module does on the bus (novox/hq ADR 0436 §1, §2, §3, §5): its own bus
// account, granted exactly this and nothing of the module's other grants or the tool runner's — never a seat
// to hold or serve, never a tool call or seat verb answered, never a restricted stream heard, never a
// warrant published (§3, §5).
type ProcessBus struct {
// Emits are the events this process emits, named locally, as the module's own Emits names them
// (EmitsAll): a process emits only what its module may emit as a whole.
Emits []string `json:"emits,omitempty"`
// Queues are the module's own work-queue streams (ADR 0436 §6) this process consumes.
Queues []string `json:"queues,omitempty"`
// State is the module's own state, by local name, this process reads and writes.
State []string `json:"state,omitempty"`
// Asks says whether this process reads the router's records of this module's own asks.
Asks bool `json:"asks,omitempty"`
// Invokes are the tools and seat verbs this process calls, each <module>.<tool> or a role's
// seat:<seat>.<verb> (as Manifest.Invokes), or * for every tool on the mesh.
Invokes []string `json:"invokes,omitempty"`
}
// UnmarshalJSON reads a process's bus strictly: an unknown key — serves, a seat to hold or claim, or
// anything else that would answer a tool call or a seat's verb — is refused here, by name (novox/hq ADR
// 0436 §3).
func (b *ProcessBus) UnmarshalJSON(raw []byte) error {
type plain ProcessBus
var p plain
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&p); err != nil {
return fmt.Errorf("a process's bus is {emits, queues, state, asks, invokes}: %w", typedUnknown(err))
}
*b = ProcessBus(p)
return nil
}
// processBusOf is the bus a process resource declares, and whether it declared one at all.
func processBusOf(r map[string]any) (bus ProcessBus, present bool, err error) {
raw, present := r["bus"]
if !present {
return ProcessBus{}, false, nil
}
body, err := json.Marshal(raw)
if err != nil {
return ProcessBus{}, true, err
}
if err := json.Unmarshal(body, &bus); err != nil {
return ProcessBus{}, true, err
}
return bus, true, nil
}
// ProcessBuses is every process of this module that declares bus, by its id (the manifest's own resource
// id, novox/hq ADR 0436, settled by the lead) — what the grant composition reads to grant a process
// account exactly what it emits, consumes and calls (novox/hq ADR 0436 §2).
func (m Manifest) ProcessBuses() map[string]ProcessBus {
var out map[string]ProcessBus
for _, r := range m.Resources {
if fmt.Sprint(r["type"]) != "process" {
continue
}
b, present, err := processBusOf(r)
if !present || err != nil {
continue
}
if out == nil {
out = map[string]ProcessBus{}
}
out[fmt.Sprint(r["id"])] = b
}
return out
}
// ProcessBusProblems is what is wrong with a manifest's processes' bus declarations (novox/hq ADR 0436 §1,
// §3): the account rule is modelled on RunsAsProblems (seats_declared.go:522-550), and an invoked tool or
// seat verb's shape on invokeProblems.
func ProcessBusProblems(m Manifest) []string {
var problems []string
streams := map[string]bool{}
for _, s := range m.Streams {
streams[s.Name] = true
}
state := map[string]bool{}
for _, s := range m.State {
state[s.Name] = true
}
emits := map[string]bool{}
for _, e := range m.EmitsAll() {
emits[e] = true
}
for _, r := range m.Resources {
if fmt.Sprint(r["type"]) != "process" {
continue
}
id := fmt.Sprint(r["id"])
if _, present := r["bus"]; !present {
continue
}
b, _, err := processBusOf(r)
if err != nil {
problems = append(problems, fmt.Sprintf("%s's process %s declares bus: %v", m.Module, id, err))
continue
}
user, _ := r["user"].(string)
problems = append(problems, processAccountProblems(m, id, user)...)
for _, e := range b.Emits {
if !emits[e] {
problems = append(problems, fmt.Sprintf(
"%s's process %s's bus emits %q, which %s does not declare as its own (novox/hq ADR 0436 §1)",
m.Module, id, e, m.Module))
}
}
for _, q := range b.Queues {
if !streams[q] {
problems = append(problems, fmt.Sprintf(
"%s's process %s's bus queues %q, which %s does not declare as its own stream (novox/hq "+
"ADR 0436 §1)", m.Module, id, q, m.Module))
}
}
for _, s := range b.State {
if !state[s] {
problems = append(problems, fmt.Sprintf(
"%s's process %s's bus reads and writes state %q, which %s does not declare as its own "+
"(novox/hq ADR 0436 §1)", m.Module, id, s, m.Module))
}
}
for _, t := range b.Invokes {
if !invokeNameValid(t) {
problems = append(problems, fmt.Sprintf(
"%s's process %s's bus invokes %q, which does not name a tool: a process invokes "+
"<module>.<tool>, or a role's seat:<seat>.<verb> (novox/hq ADR 0436 §1)", m.Module, id, t))
}
}
}
return problems
}
// processAccountProblems is what is wrong with one process's bus account (novox/hq ADR 0436 §1, §2): an
// account the module makes — a `user` resource of that name — never root and never the operator account,
// which a manifest cannot say safely (a node with no account has nothing to resolve it to, novox/hq ADR
// 0436, settled by the lead).
func processAccountProblems(m Manifest, processID, user string) []string {
switch {
case user == "":
return []string{fmt.Sprintf(
"%s's process %s declares bus and names no account; a process account is a user resource the "+
"module makes, never root and never the operator account (novox/hq ADR 0436 §1, §2)",
m.Module, processID)}
case user == "root":
return []string{fmt.Sprintf(
"%s's process %s runs as root; a process account is the module's own, never root (novox/hq ADR "+
"0436 §1, §2)", m.Module, processID)}
case user == "${machine:account}":
return []string{fmt.Sprintf(
"%s's process %s runs as the operator account; a process account is the module's own, never the "+
"operator's (novox/hq ADR 0436 §1, §2)", m.Module, processID)}
}
for _, r := range m.Resources {
if fmt.Sprint(r["type"]) == "user" && fmt.Sprint(r["name"]) == user {
return nil
}
}
return []string{fmt.Sprintf(
"%s's process %s runs as %s, which it does not make: a user resource named %s (novox/hq ADR 0436 §1, "+
"§2)", m.Module, processID, user, user)}
}
// invokeNameValid says a module or process's invoked entry names a tool: <module>.<tool>, a role's
// seat:<seat>.<verb>, or * for every tool on the mesh (novox/hq ADR 0152, ADR 0436 §1).
func invokeNameValid(t string) bool {
if t == "*" {
return true
}
t = strings.TrimPrefix(t, "seat:")
first, second, named := strings.Cut(t, ".")
return named && name.MatchString(first) && toolName.MatchString(second)
}
+191
View File
@@ -0,0 +1,191 @@
package catalogue
import (
"strings"
"testing"
)
// A process resource may declare `bus` (novox/hq ADR 0436 §1): what it emits, the module's own work-queue
// streams it consumes, the module's own state it reads and writes, whether it reads the router's records of
// this module's own asks, and the tools and seat verbs it calls. The fixture is mesh-mail's future manifest
// shape (ADR 0436 §7, §8), copied from mesh-catalog/modules/mesh-mail/module.json and extended as settled by
// the lead: process id "send" is the manifest's own `id` (internal/catalogue/runtime.go:72-74 is the model for
// a process id); the account file uses the controller's real JSON tags (modules.go:1041-1056 issueWith).
func mailSenderProcessManifest(t *testing.T, bus string) string {
t.Helper()
process := `{"id":"send","type":"process","run":["./mesh-mail","send"],"user":"mesh-mail-send"`
if bus != "" {
process += `,"bus":` + bus
}
process += `}`
// The stream itself is plain here, not retention work-queue: 1179 checks only that a process names a
// stream the module owns, never its retention — that rule is 1178's (WorkQueueStreamProblems), and is
// tested against its own fixture there (work_queue_consumer_test.go) so this one stays accepted whether
// or not a process currently names "mail-sends" in bus.queues.
return `{"module":"mesh-mail","version":"1","emits":["mail.send.sent","mail.send.failed"],` +
`"state":["sends"],"streams":[{"name":"mail-sends","events":["mail.send.approved"],"max-age-days":1,` +
`"max-mib":64}],"resources":[{"id":"account","type":"user","name":"mesh-mail-send"},` + process + `]}`
}
const mailSenderBus = `{"emits":["mail.send.sent","mail.send.failed"],"queues":["mail-sends"],` +
`"state":["sends"],"asks":true,"invokes":["seat:operator-channel.tell"]}`
// 1179/AC1: a well-formed bus on a process whose account the module makes is accepted, and the catalogue
// gives it whole to the grant composition (ProcessBuses, which 1180 reads).
func TestAProcessDeclaresItsBusAndIsAccepted(t *testing.T) {
m, err := ParseManifest([]byte(mailSenderProcessManifest(t, mailSenderBus)))
if err != nil {
t.Fatalf("a well-formed process bus was refused: %v", err)
}
buses := m.ProcessBuses()
b, ok := buses["send"]
if !ok {
t.Fatalf("the process's bus was not given to the grant composition: %+v", buses)
}
if !strSliceEqual(b.Emits, []string{"mail.send.sent", "mail.send.failed"}) ||
!strSliceEqual(b.Queues, []string{"mail-sends"}) || !strSliceEqual(b.State, []string{"sends"}) ||
!b.Asks || !strSliceEqual(b.Invokes, []string{"seat:operator-channel.tell"}) {
t.Fatalf("the bus read as %+v", b)
}
}
func strSliceEqual(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
// 1179/AC2: a bus on a process with no user, user root, or the operator account is refused naming the
// process and the account rule.
func TestAProcessBusAccountMustBeTheModulesOwn(t *testing.T) {
for name, c := range map[string]struct{ user, says string }{
"no user": {"", "send"},
"user root": {"root", "root"},
"the operator account": {"${machine:account}", "operator"},
} {
raw := `{"module":"mesh-mail","version":"1","emits":["mail.send.sent"],"state":["sends"],` +
`"streams":[{"name":"mail-sends","events":["mail.send.approved"],"retention":"work-queue",` +
`"max-deliveries":5}],"resources":[{"id":"account","type":"user","name":"mesh-mail-send"},` +
`{"id":"send","type":"process","run":["./mesh-mail","send"]`
if c.user != "" {
raw += `,"user":"` + c.user + `"`
}
raw += `,"bus":{"emits":["mail.send.sent"]}}]}`
_, err := ParseManifest([]byte(raw))
if err == nil || !strings.Contains(err.Error(), "send") || !strings.Contains(err.Error(), c.says) {
t.Errorf("%s: %v; want a refusal naming the process and %q", name, err, c.says)
}
}
}
// 1179/AC3: a bus naming a stream, state or event the module does not declare as its own is refused naming
// it.
func TestAProcessBusNamesOnlyTheModulesOwn(t *testing.T) {
for name, c := range map[string]struct{ bus, says string }{
"another module's event": {`{"emits":["billing.paid"]}`, "billing.paid"},
"another module's queue": {`{"queues":["other-queue"]}`, "other-queue"},
"another module's state": {`{"state":["other-state"]}`, "other-state"},
} {
_, err := ParseManifest([]byte(mailSenderProcessManifest(t, c.bus)))
if err == nil || !strings.Contains(err.Error(), c.says) {
t.Errorf("%s: %v; want a refusal naming %q", name, err, c.says)
}
}
}
// 1179/AC4: a bus declaring serves, holds, claims or a seat is refused naming the key; a bus with invokes of
// a tool or a seat verb is accepted.
func TestAProcessBusNeverServesOrHoldsASeat(t *testing.T) {
for name, c := range map[string]struct{ bus, says string }{
"serves": {`{"emits":["mail.send.sent"],"serves":["send"]}`, "serves"},
"holds": {`{"emits":["mail.send.sent"],"holds":["mail-sender"]}`, "holds"},
"claims": {`{"emits":["mail.send.sent"],"claims":["mail-sender"]}`, "claims"},
"seat": {`{"emits":["mail.send.sent"],"seat":"mail-sender"}`, "seat"},
} {
_, err := ParseManifest([]byte(mailSenderProcessManifest(t, c.bus)))
if err == nil || !strings.Contains(err.Error(), c.says) {
t.Errorf("%s: %v; want a refusal naming %q", name, err, c.says)
}
}
accepted := `{"emits":["mail.send.sent","mail.send.failed"],"invokes":["seat:operator-channel.tell","mail.status"]}`
if _, err := ParseManifest([]byte(mailSenderProcessManifest(t, accepted))); err != nil {
t.Fatalf("a bus invoking a tool and a seat verb was refused: %v", err)
}
}
// 1179/AC5: a process declaring bus is composed with no bus key on the wire, and is otherwise byte for byte
// what main composes for the same process without bus.
func TestAProcessesBusNeverReachesTheWire(t *testing.T) {
withBus, err := ParseManifest([]byte(mailSenderProcessManifest(t, mailSenderBus)))
if err != nil {
t.Fatal(err)
}
withoutBus, err := ParseManifest([]byte(mailSenderProcessManifest(t, "")))
if err != nil {
t.Fatal(err)
}
with := Rendering{Needed: map[string]map[string]string{}}
outWith, err := Resolution{Node: "novox", Modules: []Manifest{withBus}}.Declaration(with)
if err != nil {
t.Fatalf("composing with bus: %v", err)
}
outWithout, err := Resolution{Node: "novox", Modules: []Manifest{withoutBus}}.Declaration(with)
if err != nil {
t.Fatalf("composing without bus: %v", err)
}
var withProcess, withoutProcess map[string]any
for _, r := range outWith {
if r["id"] == "mesh-mail.send" {
withProcess = r
}
}
for _, r := range outWithout {
if r["id"] == "mesh-mail.send" {
withoutProcess = r
}
}
if withProcess == nil || withoutProcess == nil {
t.Fatalf("the process was not composed at all: with=%v without=%v", withProcess, withoutProcess)
}
if _, has := withProcess["bus"]; has {
t.Fatalf("the process resource carried a bus key onto the wire: %v", withProcess)
}
delete(withProcess, "bus")
if !mapsEqual(withProcess, withoutProcess) {
t.Fatalf("with bus declared, the process composed differently: %+v\nwant: %+v", withProcess, withoutProcess)
}
}
func mapsEqual(a, b map[string]any) bool {
if len(a) != len(b) {
return false
}
for k, v := range a {
bv, ok := b[k]
if !ok {
return false
}
if av, ok := v.([]any); ok {
bbv, ok := bv.([]any)
if !ok || len(av) != len(bbv) {
return false
}
for i := range av {
if av[i] != bbv[i] {
return false
}
}
continue
}
if v != bv {
return false
}
}
return true
}
+2
View File
@@ -280,6 +280,8 @@ func CatalogueProblems(shelf Shelf) []string {
problems = append(problems, StreamNameProblems(shelf)...)
// A module that hears a restricted stream runs as an account of its own (novox/hq ADR 0376 §2).
problems = append(problems, RestrictedHearingProblems(shelf)...)
// A work-queue stream is never asked for by another module's consumes (novox/hq ADR 0436 §6).
problems = append(problems, WorkQueueConsumedElsewhereProblems(shelf)...)
// Who declares what, and who declared it first.
declaredBy := map[string]string{}
@@ -0,0 +1,71 @@
package catalogue
import (
"strings"
"testing"
)
// A module stream may declare retention: work-queue (novox/hq ADR 0436 §6): a message is removed once a
// consumer acknowledges it, the bus refusing a new one rather than dropping what it holds. The fixture is
// mesh-mail's future `mail-sends` stream (ADR 0436 §7, §8), copied from the module.json shape at
// /home/jochen/projects/novox/mesh-catalog/modules/mesh-mail/module.json and extended with retention,
// max-deliveries and duplicates as the ADR's clause 6 describes.
const mailSendsStream = `[{"name":"mail-sends","events":["mail.send.approved"],"max-age-days":1,"max-mib":64,` +
`"retention":"work-queue","max-deliveries":5,"duplicates":24}]`
// 1177/AC1: a well-formed work-queue stream is accepted and read back whole. Named by one process in
// bus.queues (novox/hq ADR 0436 §6, checked by 1178's WorkQueueStreamProblems), so this stays accepted once
// that rule exists.
func TestAWorkQueueStreamIsDeclaredAndAccepted(t *testing.T) {
m, err := ParseManifest([]byte(`{"module":"mesh-mail","version":"1","streams":` + mailSendsStream +
`,"resources":[{"id":"account","type":"user","name":"mesh-mail-send"},` +
`{"id":"send","type":"process","run":["./mesh-mail","send"],"user":"mesh-mail-send",` +
`"bus":{"queues":["mail-sends"]}}]}`))
if err != nil {
t.Fatalf("a well-formed work-queue stream was refused: %v", err)
}
s := m.Streams[0]
if s.Retention != StreamRetentionWorkQueue || s.MaxDeliveries != 5 || s.Duplicates != 24 ||
s.Days() != 1 || s.Cap() != 64 {
t.Fatalf("the stream read as %+v", s)
}
}
// 1177/AC2: a stream whose work-queue values are out of range, contradictory, or carries a grant is refused
// naming the stream and the rule.
func TestAWorkQueueStreamsValuesAreBounded(t *testing.T) {
for name, c := range map[string]struct{ streams, says string }{
"max-deliveries 0": {`[{"name":"mail-sends","events":["mail.send.approved"],"retention":"work-queue",` +
`"max-deliveries":0,"duplicates":24}]`, "between 1 and 10"},
"max-deliveries 11": {`[{"name":"mail-sends","events":["mail.send.approved"],"retention":"work-queue",` +
`"max-deliveries":11,"duplicates":24}]`, "between 1 and 10"},
"duplicates longer than its maximum age": {`[{"name":"mail-sends","events":["mail.send.approved"],` +
`"max-age-days":1,"retention":"work-queue","max-deliveries":5,"duplicates":25}]`, "longer than its maximum age"},
"an unknown retention": {`[{"name":"mail-sends","events":["mail.send.approved"],"retention":"sometimes"}]`,
"retention"},
"max-deliveries without retention work-queue": {`[{"name":"mail-sends","events":["mail.send.approved"],` +
`"max-deliveries":5}]`, "without retention"},
"duplicates without retention work-queue": {`[{"name":"mail-sends","events":["mail.send.approved"],` +
`"duplicates":24}]`, "without retention"},
"a work-queue stream carrying a grant": {`[{"name":"mail-sends","events":["mail.send.approved"],` +
`"retention":"work-queue","max-deliveries":5,` +
`"grant":{"config":"grants","events":"mail-sends.{key}.**"}}]`, "carrying a grant"},
} {
_, err := ParseManifest([]byte(mailManifest(t, c.streams, "")))
if err == nil || !strings.Contains(err.Error(), "mail-sends") || !strings.Contains(err.Error(), c.says) {
t.Errorf("%s: %v; want a refusal naming the stream and %q", name, err, c.says)
}
}
}
// 1177/AC3: a stream with no retention composes as main does — limits retention, dropping its oldest — proven
// at the broker, where the bus's retention policy is decided (internal/broker/modstreams_test.go).
func TestAStreamWithNoRetentionReadsAsLimits(t *testing.T) {
m, err := ParseManifest([]byte(mailManifest(t, mailStream, "")))
if err != nil {
t.Fatal(err)
}
if got := m.Streams[0].Retention; got != "" {
t.Fatalf("a stream declaring no retention read %q", got)
}
}
+119 -1
View File
@@ -30,6 +30,18 @@ const (
// GrantKey is the name in a grant's pattern that each key of the config value fills.
const GrantKey = "{key}"
// StreamRetentionWorkQueue is a stream's declared retention (novox/hq ADR 0436 §6): a message is removed
// once a consumer acknowledges it, and the bus refuses a new one rather than dropping what it holds. Empty
// is the mesh's own limits retention — kept until it ages or the stream fills, dropping its oldest.
const StreamRetentionWorkQueue = "work-queue"
// StreamLeastDeliveries and StreamMostDeliveries bound a work-queue stream's max-deliveries (novox/hq ADR
// 0436 §6): how many times a message is handed over before it is given up as a dead letter.
const (
StreamLeastDeliveries = 1
StreamMostDeliveries = 10
)
// ReservedStreamPrefixes are the starts of the stream names the mesh derives, which no module's stream
// may take: a seat's work queue, a log, a bucket, an object store, and the dead letters.
var ReservedStreamPrefixes = []string{"SEAT_", "LOG_", "KV_", "OBJ_", "DEAD_"}
@@ -48,6 +60,15 @@ type StreamDeclaration struct {
MaxMiB int `json:"max-mib,omitempty"`
// Grant restricts it, when said.
Grant *StreamGrant `json:"grant,omitempty"`
// Retention is how the stream decides what to keep when it fills (novox/hq ADR 0436 §6): empty for the
// mesh's own limits retention (dropping its oldest), or StreamRetentionWorkQueue.
Retention string `json:"retention,omitempty"`
// MaxDeliveries bounds how many times a work-queue message is handed over, 1 to 10; refused without
// retention work-queue (novox/hq ADR 0436 §6).
MaxDeliveries int `json:"max-deliveries,omitempty"`
// Duplicates is a work-queue stream's duplicate window in hours, at most its maximum age; refused
// without retention work-queue (novox/hq ADR 0436 §6).
Duplicates int `json:"duplicates,omitempty"`
}
// StreamGrant says which trusted config value of the module grants a consumer a part of the stream, and
@@ -67,7 +88,8 @@ func (s *StreamDeclaration) UnmarshalJSON(raw []byte) error {
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&p); err != nil {
return fmt.Errorf("a stream is {name, events, max-age-days, max-mib, grant}: %w", typedUnknown(err))
return fmt.Errorf("a stream is {name, events, max-age-days, max-mib, grant, retention, max-deliveries, "+
"duplicates}: %w", typedUnknown(err))
}
*s = StreamDeclaration(p)
return nil
@@ -163,6 +185,44 @@ func StreamProblems(m Manifest) []string {
problems = append(problems, fmt.Sprintf("%s caps stream %q at %d MiB; a stream holds between %d and %d "+
"(novox/hq ADR 0363)", m.Module, s.Name, s.MaxMiB, StreamLeastMiB, StreamMostMiB))
}
// A stream may declare retention: work-queue (novox/hq ADR 0436 §6): a message removed once a
// consumer acknowledges it, the bus refusing a new one when full rather than dropping its oldest, with
// max-deliveries (1 to 10) and a duplicate window (duplicates, hours, at most its maximum age) -- both
// meaningless, and refused, without it. A work-queue stream is read by its one consumer, never granted.
switch s.Retention {
case "", StreamRetentionWorkQueue:
default:
problems = append(problems, fmt.Sprintf(
"%s's stream %q declares retention %q; a stream's retention is %q or nothing (novox/hq ADR 0436 §6)",
m.Module, s.Name, s.Retention, StreamRetentionWorkQueue))
}
switch {
case s.Retention == StreamRetentionWorkQueue &&
(s.MaxDeliveries < StreamLeastDeliveries || s.MaxDeliveries > StreamMostDeliveries):
problems = append(problems, fmt.Sprintf(
"%s's stream %q declares max-deliveries %d; a work-queue stream's max-deliveries is between %d "+
"and %d (novox/hq ADR 0436 §6)", m.Module, s.Name, s.MaxDeliveries, StreamLeastDeliveries,
StreamMostDeliveries))
case s.Retention != StreamRetentionWorkQueue && s.MaxDeliveries != 0:
problems = append(problems, fmt.Sprintf(
"%s's stream %q declares max-deliveries %d without retention %q (novox/hq ADR 0436 §6)",
m.Module, s.Name, s.MaxDeliveries, StreamRetentionWorkQueue))
}
switch {
case s.Duplicates != 0 && s.Retention != StreamRetentionWorkQueue:
problems = append(problems, fmt.Sprintf(
"%s's stream %q declares duplicates %d hours without retention %q (novox/hq ADR 0436 §6)",
m.Module, s.Name, s.Duplicates, StreamRetentionWorkQueue))
case s.Duplicates != 0 && s.Duplicates > s.Days()*24:
problems = append(problems, fmt.Sprintf(
"%s's stream %q keeps duplicates %d hours, longer than its maximum age of %d days (novox/hq "+
"ADR 0436 §6)", m.Module, s.Name, s.Duplicates, s.Days()))
}
if s.Retention == StreamRetentionWorkQueue && s.Grant != nil {
problems = append(problems, fmt.Sprintf(
"%s's stream %q declares retention %q carrying a grant; a work-queue stream is read by its "+
"module's one consumer, never granted (novox/hq ADR 0436 §6)", m.Module, s.Name, s.Retention))
}
if len(s.Events) == 0 {
problems = append(problems, fmt.Sprintf("%s declares stream %q with no events", m.Module, s.Name))
}
@@ -291,6 +351,64 @@ func patternsOverlap(a, b string) bool {
}
}
// WorkQueueStreamProblems refuses a work-queue stream (novox/hq ADR 0436 §6) that is not read by exactly
// one process of its own module: named by none, it queues forever unconsumed; named by two, each would take
// half of what the other needed, silently.
func WorkQueueStreamProblems(m Manifest) []string {
var problems []string
named := map[string]int{}
for _, b := range m.ProcessBuses() {
for _, q := range b.Queues {
named[q]++
}
}
for _, s := range m.Streams {
if s.Retention != StreamRetentionWorkQueue {
continue
}
switch named[s.Name] {
case 0:
problems = append(problems, fmt.Sprintf(
"%s's work-queue stream %q: no process names it in bus.queues; a work-queue stream is read by "+
"exactly one process (novox/hq ADR 0436 §6)", m.Module, s.Name))
case 1:
default:
problems = append(problems, fmt.Sprintf(
"%s's work-queue stream %q: %d processes name it in bus.queues; a work-queue stream is read by "+
"exactly one (novox/hq ADR 0436 §6)", m.Module, s.Name, named[s.Name]))
}
}
return problems
}
// WorkQueueConsumedElsewhereProblems refuses another module's `consumes` naming a work-queue stream's events
// (novox/hq ADR 0436 §6): read by its module's one process alone, never broadcast to a consumer that only
// subscribes. Modelled on RestrictedHearingProblems.
func WorkQueueConsumedElsewhereProblems(shelf Shelf) []string {
var problems []string
for _, module := range shelfOrder(shelf) {
m := shelf[module]
for _, c := range m.Consumes {
emitter, rest, named := strings.Cut(c, ".")
provider, known := shelf[emitter]
if !named || emitter == OneName || emitter == TheRest || !known || emitter == module {
continue
}
hearing, _ := provider.StreamsHearing(rest)
for _, s := range hearing {
if s.Retention != StreamRetentionWorkQueue {
continue
}
problems = append(problems, fmt.Sprintf(
"%s consumes %s, which %s's work-queue stream %s carries; a work-queue stream is read by "+
"its module's one process alone, never by another module's consumes (novox/hq ADR 0436 "+
"§6)", module, c, emitter, s.BusName()))
}
}
}
return problems
}
// StreamNameProblems are the rules about stream names no single manifest can be judged against: two
// modules declaring one name.
func StreamNameProblems(shelf Shelf) []string {
@@ -0,0 +1,75 @@
package catalogue
import (
"strings"
"testing"
)
// A work-queue stream is read by exactly one process of its own module (novox/hq ADR 0436 §6): named once
// in a process's bus.queues, and never asked for by another module's consumes.
// mailSenderManifestWithQueue is the mesh-mail fixture (ADR 0436 §7, §8) with its process's bus.queues set
// as given, so each 1178/AC2 case can vary just that.
func mailSenderManifestWithQueue(queues string) Manifest {
return Manifest{
Module: "mesh-mail", Version: "1", Emits: []string{"mail.send.sent", "mail.send.failed"},
State: []StateDeclaration{{Name: "sends"}},
Streams: []StreamDeclaration{{Name: "mail-sends", Events: []string{"mail.send.approved"},
Retention: StreamRetentionWorkQueue, MaxDeliveries: 5}},
Resources: []map[string]any{
{"id": "account", "type": "user", "name": "mesh-mail-send"},
{"id": "send", "type": "process", "run": []any{"./mesh-mail", "send"}, "user": "mesh-mail-send",
"bus": map[string]any{"emits": []any{"mail.send.sent", "mail.send.failed"}, "queues": queueList(queues)}},
},
}
}
func queueList(q string) []any {
if q == "" {
return nil
}
out := []any{}
for _, s := range strings.Split(q, ",") {
out = append(out, s)
}
return out
}
// 1178/AC2 (named): a work-queue stream no process names is refused.
func TestAWorkQueueStreamNoProcessNamesIsRefused(t *testing.T) {
m := mailSenderManifestWithQueue("")
got := strings.Join(WorkQueueStreamProblems(m), "; ")
if !strings.Contains(got, "mail-sends") || !strings.Contains(got, "no process names it") {
t.Fatalf("an unnamed work-queue stream: %q", got)
}
}
// 1178/AC2 (named twice): a work-queue stream two processes name is refused.
func TestAWorkQueueStreamTwoProcessesNameIsRefused(t *testing.T) {
m := mailSenderManifestWithQueue("mail-sends")
m.Resources = append(m.Resources, map[string]any{"id": "resend", "type": "process",
"run": []any{"./mesh-mail", "send"}, "user": "mesh-mail-send",
"bus": map[string]any{"emits": []any{"mail.send.sent"}, "queues": []any{"mail-sends"}}})
got := strings.Join(WorkQueueStreamProblems(m), "; ")
if !strings.Contains(got, "mail-sends") || !strings.Contains(got, "2 processes name") {
t.Fatalf("a work-queue stream two processes name: %q", got)
}
}
// 1178/AC2 (named once): a work-queue stream exactly one process names is accepted.
func TestAWorkQueueStreamOneProcessNamesIsAccepted(t *testing.T) {
m := mailSenderManifestWithQueue("mail-sends")
if got := WorkQueueStreamProblems(m); len(got) != 0 {
t.Fatalf("a work-queue stream one process names was refused: %v", got)
}
}
// 1178/AC2 (another module's consumes): a work-queue stream another module's consumes asks for is refused.
func TestAWorkQueueStreamAnotherModulesConsumesIsRefused(t *testing.T) {
sender := mailSenderManifestWithQueue("mail-sends")
reader := Manifest{Module: "reader", Version: "1", Consumes: []string{"mesh-mail.mail.send.approved"}}
got := strings.Join(WorkQueueConsumedElsewhereProblems(Shelf{"mesh-mail": sender, "reader": reader}), "; ")
if !strings.Contains(got, "reader") || !strings.Contains(got, "MAIL_SENDS") {
t.Fatalf("another module's consumes of a work-queue stream: %q", got)
}
}