57/F2b: A settings layer no longer replaces what a provider serves (issue 696) #321

Merged
mesh-admin merged 1 commits from pkg/57-f2b-served-keys into feat/57-config-and-provisioning 2026-10-14 05:16:03 +00:00
7 changed files with 313 additions and 46 deletions
Showing only changes of commit c5fe7d4795 - Show all commits
+12 -3
View File
@@ -51,6 +51,15 @@ func stepCA() Manifest {
}
}
// stepCAWithItsRoot is the authority whose root the operator supplies: the served key asks for it as
// `${setting:root}` (ADR 0155), the only way a setting reaches what a provider serves since a layer
// stopped replacing a served key (novox/hq issue 696; before it, the empty `root` was replaced whole).
func stepCAWithItsRoot() Manifest {
m := stepCA()
m.Serves["acme-ca"]["root"] = "${setting:root}"
return m
}
// routeProxy issues from that authority, so it must be told the root to verify it with.
func routeProxy() Manifest {
return Manifest{
@@ -78,7 +87,7 @@ func routeProxy() Manifest {
// The consequence is silent and total: route-proxy wrote an empty CA bundle, fell back to the
// system trust store, could not verify the internal authority, and no certificate was ever issued.
func TestACoLocatedProvidersServedValuesReachItsConsumer(t *testing.T) {
r, err := Resolve(shelf(stepCA(), routeProxy()),
r, err := Resolve(shelf(stepCAWithItsRoot(), routeProxy()),
[]string{"step-ca", "route-proxy"}, reachable(), World{})
if err != nil {
t.Fatal(err)
@@ -106,7 +115,7 @@ func TestACoLocatedProvidersServedValuesReachItsConsumer(t *testing.T) {
// And the binding file says the same thing, for a consumer that reads the binding rather than a
// substituted placeholder. The two are one fact and must not be able to disagree.
func TestACoLocatedConsumersBindingCarriesTheSettledValues(t *testing.T) {
r, err := Resolve(shelf(stepCA(), routeProxy()),
r, err := Resolve(shelf(stepCAWithItsRoot(), routeProxy()),
[]string{"step-ca", "route-proxy"}, reachable(), World{})
if err != nil {
t.Fatal(err)
@@ -152,7 +161,7 @@ func TestACoLocatedProviderIsSettledWhicheverWayItWasPulledIn(t *testing.T) {
{"step-ca", "route-proxy"},
{"route-proxy", "step-ca"},
} {
r, err := Resolve(shelf(stepCA(), routeProxy()), assigned, reachable(), World{})
r, err := Resolve(shelf(stepCAWithItsRoot(), routeProxy()), assigned, reachable(), World{})
if err != nil {
t.Fatal(err)
}
@@ -165,16 +165,19 @@ func TestTwoConsumersOfOneProviderGetTheirOwnDerivation(t *testing.T) {
}
}
// An operator may still set what the provider serves, and the mesh still derives the rest: the
// setting is laid on first, then the consumer's half is filled.
// An operator may still set what the provider serves where its definition asks (${setting:…}), and
// the mesh still derives the rest: the setting is filled first, then the consumer's half. (Until
// novox/hq issue 696 the layer replaced the served bucket whole; a layer replaces no served key now.)
func TestASettingComposesWithADerivedValue(t *testing.T) {
r, err := Resolve(shelf(store(), files()), []string{"store", "files"}, reachable(), World{})
prefixed := store()
prefixed.Serves["s3-bucket"]["bucket"] = "${setting:bucket-prefix}${consumer:as:dns}"
r, err := Resolve(shelf(prefixed, files()), []string{"store", "files"}, reachable(), World{})
if err != nil {
t.Fatal(err)
}
out, err := r.Declaration(Rendering{
Settings: SettingsBy{"store": {{From: "the operator",
Values: map[string]any{"bucket": "team-${consumer:as:dns}"}}}},
Values: map[string]any{"bucket-prefix": "team-"}}}},
Grants: []Grant{{Provision: "s3-bucket", Consumer: "workstation", From: "files", Slug: "files",
Values: map[string]any{}, Sealed: "c2VhbGVk"}},
})
@@ -0,0 +1,194 @@
package catalogue
import (
"bytes"
"encoding/json"
"os"
"strings"
"testing"
)
// influxdb and grafana as the catalogue declares them, cut to what composes the data source on a
// machine like ace (novox/hq issue 696): what influxdb serves for influxdb-api and its mergeable
// runtime configuration, and grafana's binding, data source file and credential.
const grafanaLikeTheCatalogue = `{
"module": "grafana", "version": "1",
"requires": [{"name": "influxdb-api", "optional": true}],
"contributes": {"influxdb-api": {"access": "read"}},
"binds": {"influxdb-api": "${dir:state}/influxdb.json"},
"secrets": {"influxdb-api": "${dir:mesh-state}/influxdb-api"},
"resources": [
{"id": "mesh-state", "type": "directory", "mode": "0700", "place": "mesh"},
{"id": "state", "type": "directory", "mode": "0700", "place": "."},
{"id": "influxdb-datasources", "type": "directory", "mode": "0755", "owner": "472:0"},
{"id": "influxdb-datasource", "type": "file", "path": "${dir:influxdb-datasources}/mesh-influxdb.yaml", "mode": "0644",
"content": "apiVersion: 1\n# Written by the mesh from grafana's influxdb-api binding, only while it is bound (novox/hq ADR 0330):\n# without influxdb, grafana runs with no mesh data source. grafana reads it at start. Its own name and\n# uid, so a data source somebody made in the UI is never overwritten, and read-only in the UI because\n# the mesh resets it. The password is read from the file the mesh delivers, never written here.\ndatasources:\n - name: InfluxDB (mesh)\n uid: mesh-influxdb-api\n type: influxdb\n access: proxy\n url: ${bound:influxdb-api:scheme}://${bound:influxdb-api:at}:${bound:influxdb-api:port}\n user: ${bound:influxdb-api:as}\n isDefault: false\n editable: false\n jsonData:\n dbName: ${bound:influxdb-api:bucket}\n httpMode: POST\n secureJsonData:\n password: $__file{/run/secrets/influxdb/influxdb-api}\n",
"only-when-bound": "influxdb-api"}
]
}`
// influxdbServing is influxdb as the catalogue declared it before issue 696 (literal served org and
// bucket) or after (served from its settings, today's literals as their defaults).
func influxdbServing(after bool) string {
served := `"org": "mesh", "bucket": "default"`
settings, takes := ``, ``
if after {
served = `"org": "${setting:org}", "bucket": "${setting:bucket}"`
settings = `"settings": {
"org": {"kind": "preference", "default": "mesh", "why": "the org influxdb makes at its first start"},
"bucket": {"kind": "preference", "default": "default", "why": "the bucket influxdb makes at its first start"}},`
takes = `, "takes-settings": ["bucket", "org", "url"], "trusted": true`
}
return `{
"module": "influxdb", "version": "1",
"provides": [{"name": "influxdb-api", "scope": "mesh", "identity": {"in": "an InfluxDB v1 authorization"}}],
"listens": [{"name": "api", "port": 8086, "protocol": "tcp", "from": "mesh", "why": "queries and writes"}],
` + settings + `
"serves": {"influxdb-api": {"scheme": "http", "port": 8086, ` + served + `}},
"resources": [
{"id": "mesh-state", "type": "directory", "mode": "0700", "place": "mesh"},
{"id": "runtime-config", "type": "file", "path": "${dir:mesh-state}/config.json", "mode": "0600",
"content": "{}\n", "merge": "json"` + takes + `}
]
}`
}
// composedLikeAce is every resource of grafana and influxdb as one machine composes them, by id, with
// influxdb's settings layer on it.
func composedLikeAce(t *testing.T, influxdb string, layers []Layer) map[string]string {
t.Helper()
var ms []Manifest
for _, raw := range []string{influxdb, grafanaLikeTheCatalogue} {
m, err := ParseManifest([]byte(raw))
if err != nil {
t.Fatal(err)
}
ms = append(ms, m)
}
r, err := Resolve(shelf(ms...), []string{"influxdb", "grafana"}, reachable(), World{})
if err != nil {
t.Fatal(err)
}
out, err := r.Declaration(Rendering{Settings: SettingsBy{"influxdb": layers},
Needed: map[string]map[string]string{}})
if err != nil {
t.Fatal(err)
}
got := map[string]string{}
for _, res := range out {
id, _ := res["id"].(string)
if !strings.HasPrefix(id, "grafana.") && !strings.HasPrefix(id, "influxdb.") {
continue
}
b, err := json.Marshal(res)
if err != nil {
t.Fatal(err)
}
got[id] = string(b)
}
return got
}
// aceLikeLayer is a layer like influxdb's on ace: its own org and bucket (made up here), and the
// endpoint block the mesh reads itself.
func aceLikeLayer() []Layer {
return []Layer{{From: "ace", Values: map[string]any{
"org": "made-up-org", "bucket": "made-up-bucket",
"endpoints": map[string]any{"api": map[string]any{"label": "influx", "port": float64(8086), "reach": "internal"}},
}}}
}
// grafana composes byte for byte as it did before issue 696, on a machine like ace and on one
// without a layer, now that influxdb serves its org and bucket from its settings and no layer
// replaces a served key. The golden files are grafana's resources as the integration branch's
// controller composed them at 71362c1 (Settle still overriding), with influxdb serving the literals.
func TestGrafanaComposesTheSameOnceInfluxdbServesItsSettings(t *testing.T) {
for name, layers := range map[string][]Layer{"ace": aceLikeLayer(), "unlayered": nil} {
raw, err := os.ReadFile("testdata/influxdb-served-by-setting/grafana-" + name + ".json")
if err != nil {
t.Fatal(err)
}
var want map[string]json.RawMessage
if err := json.Unmarshal(raw, &want); err != nil {
t.Fatal(err)
}
got := composedLikeAce(t, influxdbServing(true), layers)
seen := 0
for id, v := range got {
if !strings.HasPrefix(id, "grafana.") {
continue
}
seen++
w, ok := want[id]
if !ok {
t.Errorf("%s: %s composes and did not before", name, id)
continue
}
var compact bytes.Buffer
if err := json.Compact(&compact, w); err != nil {
t.Fatal(err)
}
if compact.String() != v {
t.Errorf("%s: %s changed:\n got %s\nwant %s", name, id, v, compact.String())
}
}
if seen != len(want) {
t.Errorf("%s: %d of grafana's %d resources compose", name, seen, len(want))
}
}
}
// What a provider serves is its definition's, filled from settings only where it asks with
// ${setting:<key>} (novox/hq issue 696, ADR 0348 clause 11): a layer that sets a key the provider
// serves as a literal replaces nothing, and a key it does not serve is never added.
func TestALayerNoLongerReplacesAServedKey(t *testing.T) {
served, err := Settle(map[string]any{"host": "mail", "port": 25},
[]Layer{{From: "the mesh", Values: map[string]any{"host": "post", "sitename": "Mail"}}})
if err != nil {
t.Fatal(err)
}
if served["host"] != "mail" {
t.Errorf("a layer replaced the served host: %v", served)
}
if _, leaked := served["sitename"]; leaked {
t.Errorf("a setting for a file reached the consumers: %v", served)
}
// And a layer setting it is said to reach nothing, rather than taken for a change.
m := Manifest{Module: "mail", Serves: map[string]map[string]any{"smtp": {"host": "mail"}}}
if unused := UnusedSettings(m, []Layer{{From: "the mesh", Values: map[string]any{"host": "post"}}}); len(unused) != 1 {
t.Errorf("a layer setting a literal served key was not said to reach nothing: %v", unused)
}
// influxdb as it was before issue 696, serving literals, under a layer like ace's: the layer no
// longer reaches grafana — which is why the catalogue moves influxdb first.
got := composedLikeAce(t, influxdbServing(false), aceLikeLayer())
for _, id := range []string{"grafana.influxdb-datasource", "grafana.bound-influxdb-api"} {
if strings.Contains(got[id], "made-up") {
t.Errorf("%s still carries the layer's value over a literal served key: %s", id, got[id])
}
}
if !strings.Contains(got["grafana.influxdb-datasource"], `dbName: default\n`) {
t.Errorf("the data source does not name the served literal: %s", got["grafana.influxdb-datasource"])
}
}
// influxdb's own runtime configuration takes the keys it names: org and bucket from a layer like
// ace's, and not the endpoint block the mesh reads itself, which today's form wrote there too.
func TestInfluxdbsConfigTakesItsOrgAndBucket(t *testing.T) {
got := composedLikeAce(t, influxdbServing(true), aceLikeLayer())["influxdb.runtime-config"]
var r map[string]any
if err := json.Unmarshal([]byte(got), &r); err != nil {
t.Fatal(err)
}
if want := "{\n \"bucket\": \"made-up-bucket\",\n \"org\": \"made-up-org\"\n}\n"; r["content"] != want {
t.Errorf("config.json: got %q, want %q", r["content"], want)
}
got = composedLikeAce(t, influxdbServing(true), nil)["influxdb.runtime-config"]
if err := json.Unmarshal([]byte(got), &r); err != nil {
t.Fatal(err)
}
if r["content"] != "{}\n" {
t.Errorf("a default reached the mergeable file: %q", r["content"])
}
}
+30 -23
View File
@@ -117,25 +117,27 @@ func ApplySettings(resource map[string]any, layers []Layer) (map[string]any, err
return out, nil
}
// Settle lays settings over what a provider serves. Exported because a served fact is settled where
// the mesh is walked rather than where a node is declared.
// Settle fills what a provider serves from its settings. Exported because a served fact is settled
// where the mesh is walked rather than where a node is declared.
//
// **A setting overrides a served key; it never adds one** (novox/hq 04-ISSUES/173). What a consumer
// is told is the provider's contract, and a setting made for one of the provider's files — a site
// name, a public address — is not part of it. Before this, every setting of a module reached every
// consumer of every provision it served.
// **What a provider serves is its definition's; a layer replaces none of it** (novox/hq ADR 0348
// clause 11, issue 696). A served value is filled from the layers only where the definition asks,
// with `${setting:<key>}` (ADR 0155) — a mail provider's domain, an identity provider's issuer, a
// time-series store's org and bucket — and a layer that sets a key the provider serves as a literal
// changes nothing a consumer is told. Nor does a setting add a key (novox/hq 04-ISSUES/173): a
// setting made for one of the provider's files — a site name, a public address — is not part of the
// contract. Until issue 696, a layer replaced any served key it named; influxdb on ace was the one
// layer relying on it, and serves `${setting:org}` and `${setting:bucket}` since.
func Settle(base map[string]any, layers []Layer) (map[string]any, error) {
return overridden(base, layers, "what is served")
return filledFromSettings(deepCopy(base), layers, "what is served")
}
// overridden lays settings over a map whose keys are its contract: a contribution, a served fact.
// Only the keys the map already declares are touched; the rest of a layer is somebody else's
// business (a file's, another destination's) and is left to reach it there.
// overridden lays settings over a contribution: a map whose keys are its contract. Only the keys the
// map already declares are touched; the rest of a layer is somebody else's business (a file's,
// another destination's) and is left to reach it there.
//
// A declared value may itself be the operator's, `${setting:<key>}` (ADR 0155): a mail provider
// serves its domain, an identity provider its issuer, and neither is the definition's to state.
// Filled from the layers after the overrides, and refused by name when nothing sets it — a literal
// placeholder handed to a consumer is a service configured against a string nobody meant.
// A declared value may itself be the operator's, `${setting:<key>}` (ADR 0155), filled after the
// overrides (filledFromSettings).
func overridden(base map[string]any, layers []Layer, what string) (map[string]any, error) {
kept := make([]Layer, 0, len(layers))
for _, layer := range layers {
@@ -151,6 +153,14 @@ func overridden(base map[string]any, layers []Layer, what string) (map[string]an
if err != nil {
return nil, err
}
return filledFromSettings(merged, layers, what)
}
// filledFromSettings fills each `${setting:<key>}` in a map's top-level strings from the layers,
// later beating earlier and a module's defaults lowest, and refuses one nothing sets by name — a
// literal placeholder handed to a consumer is a service configured against a string nobody meant.
// It fills merged in place and returns it.
func filledFromSettings(merged map[string]any, layers []Layer, what string) (map[string]any, error) {
for key, value := range merged {
s, ok := value.(string)
if !ok {
@@ -177,7 +187,8 @@ func overridden(base map[string]any, layers []Layer, what string) (map[string]an
// module says what it means by default, and somebody says what it means here. A contribution that
// could not be settled would have to be edited to be reused anywhere else. The two differ in one
// respect, and the caller decides it: a file takes keys it did not declare (a setting may add to a
// configuration), a contribution or served fact does not (overridden).
// configuration), a contribution does not (overridden). A served fact takes no layer at all; only
// its ${setting:<key>} is filled (Settle).
func settle(base map[string]any, layers []Layer, protected map[string]bool, what string) (
map[string]any, error) {
merged := deepCopy(base)
@@ -353,8 +364,9 @@ func sortedKeysAny(m map[string]any) []string {
// way there is. This is what makes that visible at the moment they set it.
//
// Where a key can land: any mergeable file takes any key; a file asking for `${setting:<key>}`
// takes that key (ADR 0155); a contribution or a served fact takes a key it declares, and no other
// (novox/hq 04-ISSUES/173); and the mesh's own words — `expose`, `ports`, `reach`, `endpoints` —
// takes that key (ADR 0155); a contribution takes a key it declares, and no other (novox/hq
// 04-ISSUES/173); a served fact takes only the keys it asks for as `${setting:<key>}`, since a layer
// replaces no served key (novox/hq issue 696); and the mesh's own words — `expose`, `ports`, `reach`, `endpoints` —
// are read by the mesh. A key none of those takes is stray, and is said so rather than dropped.
func UnusedSettings(m Manifest, layers []Layer) []string {
named := namesItsKeys(m)
@@ -397,11 +409,6 @@ func UnusedSettings(m Manifest, layers []Layer) []string {
}
}
}
for _, served := range m.Serves {
for key := range served {
lands[key] = true
}
}
var unused []string
for _, layer := range layers {
for key := range layer.Whole() {
@@ -444,7 +451,7 @@ func UnusedSettings(m Manifest, layers []Layer) []string {
}
unused = append(unused, fmt.Sprintf(
"%s sets %q, and %s has no file that merges it, asks for no ${setting:%s}, and "+
"declares no %q in what it contributes or serves%s",
"declares no %q in what it contributes%s",
layer.From, key, m.Module, key, key, takenSaid(m)))
}
}
-16
View File
@@ -190,22 +190,6 @@ func TestASettingLandsOnlyWhereSomethingDeclaresIt(t *testing.T) {
}
}
func TestASettingOverridesAServedKeyAndAddsNone(t *testing.T) {
// What a consumer is told is the provider's contract. A setting made for one of the
// provider's files — its site name, its public address — is not part of it.
served, err := Settle(map[string]any{"host": "mail", "port": 25},
[]Layer{{From: "the mesh", Values: map[string]any{"host": "post", "sitename": "Mail"}}})
if err != nil {
t.Fatal(err)
}
if served["host"] != "post" {
t.Errorf("the setting did not override the served host: %v", served)
}
if _, leaked := served["sitename"]; leaked {
t.Errorf("a setting for a file reached the consumers: %v", served)
}
}
func TestContentThatIsNotJSONIsRefusedWhereSomebodyIsLooking(t *testing.T) {
// Rather than on the machine, at apply time, as a file the program cannot read.
_, err := ApplySettings(file(`this is not json`), nil)
@@ -0,0 +1,35 @@
{
"grafana.bound-influxdb-api": {
"content": "{\n \"as\": \"mesh_workstation_grafana\",\n \"at\": \"workstation.internal\",\n \"binding\": 1,\n \"from\": \"workstation\",\n \"generated\": \"by the mesh — do not edit; replaced whenever this changes. The credential is not here: it is sealed, in the file this module's manifest names under `secrets`\",\n \"provision\": \"influxdb-api\",\n \"serves\": {\n \"bucket\": \"made-up-bucket\",\n \"org\": \"made-up-org\",\n \"port\": 8086,\n \"scheme\": \"http\"\n }\n}\n",
"id": "grafana.bound-influxdb-api",
"mode": "0644",
"path": "/var/lib/grafana/influxdb.json",
"type": "file"
},
"grafana.influxdb-datasource": {
"content": "apiVersion: 1\n# Written by the mesh from grafana's influxdb-api binding, only while it is bound (novox/hq ADR 0330):\n# without influxdb, grafana runs with no mesh data source. grafana reads it at start. Its own name and\n# uid, so a data source somebody made in the UI is never overwritten, and read-only in the UI because\n# the mesh resets it. The password is read from the file the mesh delivers, never written here.\ndatasources:\n - name: InfluxDB (mesh)\n uid: mesh-influxdb-api\n type: influxdb\n access: proxy\n url: http://workstation.internal:8086\n user: mesh_workstation_grafana\n isDefault: false\n editable: false\n jsonData:\n dbName: made-up-bucket\n httpMode: POST\n secureJsonData:\n password: $__file{/run/secrets/influxdb/influxdb-api}\n",
"id": "grafana.influxdb-datasource",
"mode": "0644",
"path": "/var/lib/grafana/influxdb-datasources/mesh-influxdb.yaml",
"type": "file"
},
"grafana.influxdb-datasources": {
"id": "grafana.influxdb-datasources",
"mode": "0755",
"owner": "472:0",
"path": "/var/lib/grafana/influxdb-datasources",
"type": "directory"
},
"grafana.mesh-state": {
"id": "grafana.mesh-state",
"mode": "0700",
"path": "/var/lib/mesh/grafana",
"type": "directory"
},
"grafana.state": {
"id": "grafana.state",
"mode": "0700",
"path": "/var/lib/grafana",
"type": "directory"
}
}
@@ -0,0 +1,35 @@
{
"grafana.bound-influxdb-api": {
"content": "{\n \"as\": \"mesh_workstation_grafana\",\n \"at\": \"workstation.internal\",\n \"binding\": 1,\n \"from\": \"workstation\",\n \"generated\": \"by the mesh — do not edit; replaced whenever this changes. The credential is not here: it is sealed, in the file this module's manifest names under `secrets`\",\n \"provision\": \"influxdb-api\",\n \"serves\": {\n \"bucket\": \"default\",\n \"org\": \"mesh\",\n \"port\": 8086,\n \"scheme\": \"http\"\n }\n}\n",
"id": "grafana.bound-influxdb-api",
"mode": "0644",
"path": "/var/lib/grafana/influxdb.json",
"type": "file"
},
"grafana.influxdb-datasource": {
"content": "apiVersion: 1\n# Written by the mesh from grafana's influxdb-api binding, only while it is bound (novox/hq ADR 0330):\n# without influxdb, grafana runs with no mesh data source. grafana reads it at start. Its own name and\n# uid, so a data source somebody made in the UI is never overwritten, and read-only in the UI because\n# the mesh resets it. The password is read from the file the mesh delivers, never written here.\ndatasources:\n - name: InfluxDB (mesh)\n uid: mesh-influxdb-api\n type: influxdb\n access: proxy\n url: http://workstation.internal:8086\n user: mesh_workstation_grafana\n isDefault: false\n editable: false\n jsonData:\n dbName: default\n httpMode: POST\n secureJsonData:\n password: $__file{/run/secrets/influxdb/influxdb-api}\n",
"id": "grafana.influxdb-datasource",
"mode": "0644",
"path": "/var/lib/grafana/influxdb-datasources/mesh-influxdb.yaml",
"type": "file"
},
"grafana.influxdb-datasources": {
"id": "grafana.influxdb-datasources",
"mode": "0755",
"owner": "472:0",
"path": "/var/lib/grafana/influxdb-datasources",
"type": "directory"
},
"grafana.mesh-state": {
"id": "grafana.mesh-state",
"mode": "0700",
"path": "/var/lib/mesh/grafana",
"type": "directory"
},
"grafana.state": {
"id": "grafana.state",
"mode": "0700",
"path": "/var/lib/grafana",
"type": "directory"
}
}