diff --git a/modules/mesh-grants/cmd/mesh-grants/composed_integration_test.go b/modules/mesh-grants/cmd/mesh-grants/composed_integration_test.go index a2617bf2..8e78640e 100644 --- a/modules/mesh-grants/cmd/mesh-grants/composed_integration_test.go +++ b/modules/mesh-grants/cmd/mesh-grants/composed_integration_test.go @@ -13,13 +13,16 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/seal" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" "mesh-grants/internal/buscompose" + grantsPart "mesh-grants/internal/grants" ) // theBusNodeFollowsTheSealedListUnderTheComposedGrants, run by TestTheGrantsPartWhole, defends novox/hq ADR 0547 clause 7 on a real server (issue 1945): with the list this composer writes, the grants part @@ -30,13 +33,22 @@ func theBusNodeFollowsTheSealedListUnderTheComposedGrants(t *testing.T) { records := buscompose.Records{ Nodes: []string{"novox", "ace"}, Assigned: map[string][]buscompose.Declared{ - "novox": {{Module: "nats", SnapshotsTheBus: true}, - {Module: buscompose.GrantsModule, State: []buscompose.Bucket{{Module: buscompose.GrantsModule, Name: "user-list"}}}}, + "novox": {{Module: buscompose.GrantsModule, State: []buscompose.Bucket{{Module: buscompose.GrantsModule, Name: "user-list"}}}}, "ace": {{Module: "plex", Reads: []string{delivery.ComposedRead}}, {Module: "mesh-delivery-view", Reads: []string{delivery.ComposedRead}, ReadsObjects: []string{"mesh-knowledge_sections"}}, {Module: "mesh-knowledge", Objects: []buscompose.ObjectBucket{{Module: "mesh-knowledge", Name: "sections", MaxMiB: 1, Restricted: true}}}}, }, } + // The bus module and the router as the part composes them from their own manifests (testdata: the catalogue's + // nats and messenger at the registry milestone's join): the bus module's keeper process with its own account. + fromManifests, gaps := grantsPart.Records(grantsPart.Inputs{Nodes: []string{"novox"}, + Assigned: map[string][]string{"novox": {"messenger", "nats"}}, + Manifests: map[string]catalogue.Manifest{"nats": manifestIn(t, "nats"), "messenger": manifestIn(t, "messenger")}}) + if len(gaps) != 0 { + t.Fatalf("composing nats and messenger from their manifests: %+v", gaps) + } + records.Assigned["novox"] = append(records.Assigned["novox"], fromManifests.Assigned["novox"]...) + records.BusState = fromManifests.BusState // The node-engine's state as the part's records carry it: every node's compositions (ADR 0593 §6). records.Engine = &buscompose.Declared{Module: buscompose.EngineModule, State: buscompose.EngineComposedBuckets(records.Nodes)} users, err := buscompose.Users(records) @@ -217,6 +229,51 @@ func theBusNodeFollowsTheSealedListUnderTheComposedGrants(t *testing.T) { if _, err := other.GetLastMsg("KV_"+buscompose.GrantsUserListBucket, "$KV."+buscompose.GrantsUserListBucket+"."+buscompose.GrantsUserListKey, nats.DirectGet()); err == nil { t.Errorf("another node read the key: %v", err) } + // Review round 1 (running F4; design 25 § 12.7): the bus module's keeper connects on its own process account as the + // part composed it from nats' manifest, reads the operator's warrant in the router's record of its ask, and is + // refused a write outside its grants. The router's state is made as the part makes a declared state, and written by + // the router's own account, keyed `.`. + { + asksStream := jetstream.StreamConfig{Name: "KV_messenger_asks", Subjects: []string{"$KV.messenger_asks.>"}, MaxMsgsPerSubject: 5, + AllowDirect: true, AllowRollup: true, DenyDelete: true, Discard: jetstream.DiscardNew} + if _, err := grants.CreateStream(ctx, asksStream); err != nil { + t.Fatalf("the grants part creates the router's asks: %v", err) + } + router, err := jetstream.New(dial("novox.messenger")) + if err != nil { + t.Fatal(err) + } + asks, err := router.KeyValue(ctx, "messenger_asks") + if err != nil { + t.Fatalf("the router binds its asks: %v", err) + } + if _, err := asks.Put(ctx, "nats.bus-1", []byte(`{"asker":"nats","ask":{"id":"bus-1"},"state":"answered","warrant":{"ask":"bus-1","asker":"nats","outcome":"chosen"}}`)); err != nil { + t.Fatalf("the router writes its record: %v", err) + } + keeper, err := process.Connect(process.Account{URL: s.ClientURL(), Node: "novox", Module: "nats", User: "novox.nats.keeper", + Password: "pw-novox.nats.keeper"}) + if err != nil { + t.Fatalf("the keeper connects on its composed account: %v", err) + } + t.Cleanup(keeper.Close) + e, err := keeper.Ask("messenger_asks", "bus-1") + if err != nil || e == nil || !bytes.Contains(e.Value, []byte(`"outcome":"chosen"`)) { + t.Fatalf("the keeper reads its ask's warrant under its own account: %+v, %v", e, err) + } + if e, err := keeper.Ask("messenger_asks", "other"); err != nil || e != nil { + t.Errorf("the keeper reads %+v (%v) for an ask it has no record of", e, err) + } + raw := dial("novox.nats.keeper") + for _, subject := range []string{"$KV.messenger_asks.nats.bus-1", "$KV." + buscompose.GrantsUserListBucket + ".list"} { + reply, err := raw.Request(subject, []byte(`{}`), 500*time.Millisecond) + if err == nil && reply.Header.Get("Status") == "" { + t.Errorf("the keeper wrote %s, outside its grants: %s", subject, reply.Data) + } + } + if _, err := raw.Request("$JS.API.DIRECT.GET.KV_messenger_asks.$KV.messenger_asks.mesh-config.x", nil, 500*time.Millisecond); err == nil { + t.Error("the keeper read another asker's record") + } + } // A context of its own: the refusals above each wait out their bound. ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -298,3 +355,17 @@ func theBusNodeFollowsTheSealedListUnderTheComposedGrants(t *testing.T) { t.Error("a module the grant does not name read the bucket") } } + +// manifestIn is a module's manifest as testdata keeps it. +func manifestIn(t *testing.T, module string) catalogue.Manifest { + t.Helper() + raw, err := os.ReadFile(filepath.Join("testdata", module+".module.json")) + if err != nil { + t.Fatal(err) + } + m, err := catalogue.Decode(raw) + if err != nil { + t.Fatalf("%s's manifest: %v", module, err) + } + return m +} diff --git a/modules/mesh-grants/cmd/mesh-grants/testdata/messenger.module.json b/modules/mesh-grants/cmd/mesh-grants/testdata/messenger.module.json new file mode 100644 index 00000000..f831d641 --- /dev/null +++ b/modules/mesh-grants/cmd/mesh-grants/testdata/messenger.module.json @@ -0,0 +1,229 @@ +{ + "module": "messenger", + "version": "1", + "slug": "msgr", + "runs-as": "messenger", + "seats": [ + { + "name": "operator-channel", + "scope": "mesh", + "serves": [ + "open", + "history", + "notify", + "tell", + "asks" + ], + "accepts": [ + "ask", + "cancel", + "condition" + ], + "emits": [ + "decided" + ], + "by-caller": [ + "ask", + "cancel", + "condition", + "decided" + ], + "records": [ + "asks" + ] + }, + { + "name": "channel", + "scope": "mesh", + "kinded": true, + "accepts": [ + "show", + "edit", + "send" + ] + }, + { + "name": "intake", + "scope": "mesh", + "kinded": true, + "emits": [ + "choice", + "link", + "failed", + "standing" + ], + "proofs": [ + "code" + ] + } + ], + "claims": [ + { + "name": "operator-channel", + "scope": "mesh", + "serves": [ + "open", + "history", + "notify", + "tell", + "asks" + ] + } + ], + "uses": [ + "channel" + ], + "consumes": [ + "mesh-conditions.condition-raised", + "mesh-conditions.condition-changed", + "mesh-conditions.condition-cleared", + "mesh-delivery-view.moving", + "mesh-delivery-view.settled", + "mesh-delivery-view.stuck", + "intake.choice", + "intake.link", + "intake.failed", + "intake.standing" + ], + "emits": [ + "refused", + "offered", + "closed" + ], + "streams": [ + { + "name": "asks", + "events": [ + "offered", + "closed" + ], + "max-age-days": 7, + "max-mib": 64, + "grant": { + "config": "readers", + "events": "{key}" + } + } + ], + "invokes": [ + "seat:mesh-conditions.conditions", + "seat:node-engine.root-free", + "seat:issue-tracker.tracking" + ], + "state": [ + "open", + "sent", + { + "name": "asks", + "ttl-seconds": 2592000 + }, + "identities", + "walks" + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "messenger", + "tools": [ + "messenger_status", + "messenger_recent", + "messenger_test", + "messenger_preview", + "messenger_check", + "messenger_identities", + "messenger_unlink" + ], + "config": { + "readers": { + "type": "object", + "description": "Who may hear an ask offered or closed (the stream of the asks, kept seven days), per event: offered and closed, each a list of module names, on every machine. The knowledge base alone by default, for its overview page; an asker's own answers (decided) are never on it (ADR 0616 §7).", + "trusted": true, + "default": {"offered": ["mesh-knowledge"], "closed": ["mesh-knowledge"]} + }, + "time-zone": { + "type": "string", + "description": "The time zone every time a message says is said in, an IANA name such as Europe/Brussels; empty is this machine's own zone.", + "trusted": true, + "default": "" + }, + "desktop-machines": { + "type": "list", + "description": "The machines whose desktop showed the operator's messages, by node name. Nothing in this module reads it since the desk channel took the desktop over; it is taken so the file stays as a layer that still sets it makes it, until that layer drops it.", + "trusted": true, + "default": [] + }, + "only-what-needs-me": { + "type": "list", + "description": "The kinds of channel set to say only what needs the operator: a walk that delivered with nothing held and nothing open is not told there (ADR 0308 §4); empty is none.", + "trusted": true, + "default": [] + }, + "messages-per-hour": { + "type": "number", + "description": "messages per channel per hour before the rest are held as one; 0 turns the pause off. The tell verb has its own cap of 10 and is not counted here.", + "minimum": 0, + "maximum": 1000, + "trusted": false, + "default": 100 + }, + "settle-seconds": { + "type": "number", + "description": "seconds a delivery that ended well still holds the messages about what it changed, so a problem that passes as the delivery settles is not sent (ADR 0505 §3); 0 holds nothing after the end.", + "minimum": 0, + "maximum": 3600, + "trusted": false, + "default": 120 + } + }, + "resources": [ + { + "id": "account", + "type": "user", + "name": "messenger", + "shell": "/usr/bin/nologin", + "home": "/var/lib/messenger" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "place": ".", + "owner": "messenger", + "unknown": "program" + }, + { + "id": "settings", + "type": "file", + "path": "${dir:state}/settings.json", + "mode": "0600", + "takes-config": ["time-zone", "desktop-machines", "only-what-needs-me", "messages-per-hour", "settle-seconds", "readers"], + "merge": "json", + "template": "files/settings.json", + "owner": "messenger" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/messenger", + "binary": "messenger", + "loads": [ + "messenger" + ], + "env": { + "MESH_MESSENGER_SETTINGS": "${dir:state}/settings.json", + "MESH_MESSENGER_STATE": "${dir:state}" + } + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } +} diff --git a/modules/mesh-grants/cmd/mesh-grants/testdata/nats.module.json b/modules/mesh-grants/cmd/mesh-grants/testdata/nats.module.json new file mode 100644 index 00000000..02b1320c --- /dev/null +++ b/modules/mesh-grants/cmd/mesh-grants/testdata/nats.module.json @@ -0,0 +1,353 @@ +{ + "module": "nats", + "version": "1", + "upgrade": { + "policy": "record", + "why": "the bus keeps the streams of the whole mesh: each move waits for a person's yes, which the operator's Approve of the release that carries its build gives, and its node backs its streams up before it applies (hq ADR 0236, ADR 0601)" + }, + "provides": [ + { + "name": "mesh-bus", + "scope": "mesh", + "credential": { + "form": "none" + } + }, + { + "name": "bus-state", + "scope": "mesh", + "credential": { + "form": "none" + } + } + ], + "claims": [ + { + "name": "mesh-broker", + "scope": "mesh", + "serves": [ + "list", + "show", + "check", + "orphans" + ] + } + ], + "bus-users": "/var/lib/nats-module/conf/accounts.conf", + "core-credentials": true, + "own-secrets": { + "broker": "${dir:mesh-state}/broker", + "broker-nats-tools": "${dir:tools-secret}/broker-nats-tools", + "broker-keeper": { + "path": "${dir:keeper-state}/broker-keeper", + "owner": "nats-keeper" + } + }, + "capabilities": [ + "container-runtime" + ], + "emits": [ + "consumer-removed", + "bucket-keys-removed", + "stream-removed", + "bucket-expiry-set" + ], + "invokes": [ + "seat:mesh-hand-acts.hand-act", + "mesh-delivery-indexer.indexer_bus_state" + ], + "consumes": [ + "mesh-delivery-indexer.manifest-accepted" + ], + "uses": [ + "operator-channel" + ], + "streams": [ + { + "name": "broker-events", + "events": [ + "created", + "refused", + "drifted", + "drift-cleared", + "full", + "full-cleared", + "orphaned", + "recreated", + "dropped" + ], + "max-age-days": 30, + "max-mib": 256 + } + ], + "state": [ + { + "name": "asked", + "history": 5 + }, + { + "name": "requests", + "history": 5, + "ttl-seconds": 86400 + } + ], + "listens": [ + { + "name": "bus", + "port": 4222, + "protocol": "tcp", + "from": "mesh", + "why": "the mesh bus \u2014 every link the mesh has, over TLS, reached across the overlay" + } + ], + "tools": [ + "nats_server", + "nats_connections", + "nats_closed_connections", + "nats_subscriptions", + "nats_streams", + "nats_backlog", + "nats_buckets", + "nats_users", + "nats_user_can", + "nats_snapshot_verify", + "nats_consumer_remove", + "nats_bucket_remove", + "nats_stream_remove", + "nats_bucket_expiry", + "nats_history", + "nats_create", + "nats_apply", + "nats_recreate", + "nats_drop", + "nats_keeper_health", + "nats_first_view_again", + "nats_requests" + ], + "guards": [ + 8222 + ], + "data": { + "own": [ + { + "id": "jetstream", + "path": "${dir:jetstream-data}", + "class": "valuable", + "active": "1d", + "backup": { + "dump": "p=${dir:snapshots}/bus.tar.partial; s=0; docker exec -i mesh-broker-nats /opt/mesh/mesh-nats-snapshot snapshot < ${dir:mesh-state}/broker > \"$p\" || s=$?; if [ $s -ne 0 ]; then echo \"mesh-nats-snapshot snapshot exited $s$([ $s -eq 137 -o $s -eq 143 ] && echo ' (killed by a signal)')\" >&2; docker inspect -f 'container {{.State.Status}} since {{.State.StartedAt}}' mesh-broker-nats >&2; rm -f \"$p\"; exit $s; fi; mv \"$p\" ${dir:snapshots}/bus.tar || exit 1", + "into": "snapshots" + }, + "why": "the bus's streams and key-value buckets: the hand-act log, conditions, every module's state; written all the time, so copied by the server's own snapshot of each stream (novox/hq ADR 0235), never as live files" + }, + { + "id": "snapshots", + "path": "${dir:snapshots}", + "class": "rebuildable", + "why": "last night's snapshot of every stream with its manifest, made again every night" + } + ] + }, + "resources": [ + { + "id": "keeper-account", + "type": "user", + "name": "nats-keeper", + "shell": "/usr/bin/nologin", + "home": "/var/lib/nats-keeper" + }, + { + "id": "jetstream-data", + "type": "directory", + "path": "/var/lib/mesh-broker-nats", + "mode": "0700", + "unknown": "program" + }, + { + "id": "conf-dir", + "type": "directory", + "path": "/var/lib/nats-module/conf", + "mode": "0700", + "unknown": "program" + }, + { + "id": "tools-secret", + "type": "directory", + "path": "/var/lib/mesh/nats-tools", + "mode": "0711", + "unknown": "program" + }, + { + "id": "keeper-state", + "type": "directory", + "path": "/var/lib/nats-keeper", + "mode": "0700", + "owner": "nats-keeper", + "unknown": "program" + }, + { + "id": "snapshots", + "type": "directory", + "mode": "0700", + "unknown": "program" + }, + { + "id": "mesh-state", + "type": "directory", + "mode": "0700", + "place": "mesh", + "unknown": "program" + }, + { + "id": "server-conf", + "type": "file", + "path": "/var/lib/nats-module/conf/nats.conf", + "template": "files/nats.conf", + "mode": "0644" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "place": ".", + "unknown": "program" + }, + { + "id": "program-snapshot", + "type": "archive", + "path": "${dir:state}/snapshot", + "artifact": "snapshot", + "unknown": "program" + }, + { + "id": "program-entrypoint", + "type": "archive", + "path": "${dir:state}/entrypoint", + "artifact": "entrypoint", + "unknown": "program" + }, + { + "id": "keeper", + "type": "process", + "name": "nats-keeper", + "artifact": "tools", + "run": [ + "./nats-tools", + "keep" + ], + "user": "nats-keeper", + "env": { + "MESH_BROKER_FILE": "${dir:keeper-state}/broker-keeper", + "MESH_NATS_FULL_RAISE": "75", + "MESH_NATS_FULL_CLEAR": "70", + "MESH_NATS_SNAPSHOTS_DIR": "${dir:keeper-state}" + }, + "bus": { + "state": [ + "asked", + "requests" + ], + "asks": true, + "keeps": true, + "invokes": [ + "mesh-delivery-indexer.indexer_bus_state", + "seat:mesh-hand-acts.hand-act" + ] + }, + "health": { + "kind": "tool", + "tool": "nats_keeper_health", + "interval": "60s", + "timeout": "10s", + "looks": 2, + "grace": "180s" + } + }, + { + "id": "server", + "type": "container", + "name": "mesh-broker-nats", + "ports": [ + "4222:4222", + "127.0.0.1:8222:8222" + ], + "volumes": [ + "/var/lib/mesh-broker-nats:/data", + "/var/lib/nats-module/conf:/etc/nats:ro", + "${access:tls}:/tls:ro", + "${dir:state}/snapshot:/opt/mesh:ro", + "${dir:state}/entrypoint:/opt/mesh-entrypoint:ro" + ], + "args": [ + "sh", + "/opt/mesh-entrypoint/entrypoint.sh" + ], + "artifact": "server", + "restart-on": [ + "program-snapshot", + "program-entrypoint" + ], + "health": { + "kind": "tcp", + "endpoint": "bus" + } + } + ], + "accesses": [ + { + "id": "tls", + "path": "/var/lib/mesh-broker-tls", + "mode": "read" + } + ], + "build": { + "artifacts": [ + { + "name": "server", + "kind": "upstream", + "from": "nats:2.11.17-alpine@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00" + }, + { + "name": "snapshot", + "kind": "bundle", + "language": "go", + "system": "alpine", + "from": "cmd/mesh-nats-snapshot", + "binary": "mesh-nats-snapshot" + }, + { + "name": "entrypoint", + "kind": "archive", + "from": "program" + }, + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/nats-tools", + "binary": "nats-tools", + "loads": [ + "nats-tools" + ], + "bus": { + "removes": [ + "controller-leftovers" + ] + }, + "env": { + "MESH_NATS_MONITOR": "http://127.0.0.1:8222", + "MESH_NATS_CONTAINER": "mesh-broker-nats", + "MESH_NATS_USERS_FILE": "/etc/nats/accounts.conf", + "MESH_NATS_SNAPSHOTS_DIR": "${dir:snapshots}", + "NATS_TOOLS_CREDENTIAL": "${dir:tools-secret}/broker-nats-tools" + } + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } +}