Feature 57 (config and provisioning): release mesh-controller, the reviewed release branch merged with main, on top of main

This commit is contained in:
2026-10-16 08:28:38 +00:00
parent d70bb1f388
commit d984816844
259 changed files with 34899 additions and 353 deletions
+162
View File
@@ -0,0 +1,162 @@
package broker
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"time"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"golang.org/x/crypto/bcrypt"
)
// **Only mesh-config's account writes mesh-config's buckets** (novox/hq ADR 0348 §9 and How it is checked, design
// 57 K3, issue 662): on a real server whose users are composed as the controller composes them. mesh-config runs on
// an account of its own (`runs-as`, ADR 0259 §8), so the machine's runtime does not carry it and its own principal
// alone is granted the writes of its four buckets. A put from any other account is refused by the server: a
// module reading the resolved values through `reads`, the runtime of the machine mesh-config runs on, and the
// controller.
func TestNatsOnlyMeshConfigsAccountPutsToItsBuckets(t *testing.T) {
hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
if err != nil {
t.Fatal(err)
}
states := []string{"values", "resolved", "predefined", "proposals"}
var buckets []Bucket
for _, s := range states {
buckets = append(buckets, Bucket{Module: "mesh-config", Name: s})
}
seat := Seat{Name: "mesh-config", Scope: "mesh", Serves: []string{"config"}}
users, err := Users(Records{Nodes: []string{"novox", "g14"}, Assigned: map[string][]Declared{
"novox": {
{Module: RuntimeModule},
{Module: "mesh-config", RunsAs: "mesh-config", Holds: []Seat{seat}, State: buckets,
Emits: []string{"config-changed"},
Reads: []string{"mesh-controller.assignments", "mesh-controller.declared-config", "mesh-controller.predefined"}},
{Module: "mesh-issues", State: []Bucket{{Module: "mesh-issues", Name: "issues"}}},
},
"g14": {{Module: "dunst", Reads: []string{"mesh-config.resolved"}}},
}})
if err != nil {
t.Fatal(err)
}
for i := range users {
users[i].PasswordHash = string(hash)
}
accounts, err := ComposeAccounts(users)
if err != nil {
t.Fatal(err)
}
conf := filepath.Join(t.TempDir(), "bus.conf")
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
t.Fatal(err)
}
opts, err := server.ProcessConfigFile(conf)
if err != nil {
t.Fatalf("the composed accounts do not parse: %v", err)
}
opts.NoLog, opts.NoSigs = true, true
s, err := server.NewServer(opts)
if err != nil {
t.Fatal(err)
}
go s.Start()
if !s.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(s.Shutdown)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
connect := func(user string) (*nats.Conn, chan error) {
t.Helper()
refused := make(chan error, 8)
nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user),
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refused <- err }))
if err != nil {
t.Fatalf("%s cannot connect: %v", user, err)
}
t.Cleanup(nc.Close)
return nc, refused
}
// The controller makes the buckets, as it does from a manifest's `state`.
controller, controllerRefused := connect(ControllerName)
cjs, err := jetstream.New(controller)
if err != nil {
t.Fatal(err)
}
for _, b := range buckets {
if _, err := cjs.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b.Bucket()}); err != nil {
t.Fatalf("the controller cannot make %s: %v", b.Bucket(), err)
}
}
owner, _ := connect("novox.mesh-config")
ojs, err := jetstream.New(owner)
if err != nil {
t.Fatal(err)
}
for _, b := range buckets {
kv, err := ojs.KeyValue(ctx, b.Bucket())
if err != nil {
t.Fatalf("mesh-config cannot bind its own %s: %v", b.Bucket(), err)
}
if _, err := kv.Put(ctx, "mesh.font-size", []byte(`{"value":12}`)); err != nil {
t.Errorf("mesh-config cannot write its own %s: %v", b.Bucket(), err)
}
}
// Every other account: a publish on the bucket's subject is refused by the server, and the bucket holds
// what mesh-config wrote.
reader, readerRefused := connect("g14.dunst")
runtime, runtimeRefused := connect("novox." + RuntimeModule)
for _, other := range []struct {
name string
nc *nats.Conn
refused chan error
}{{"a module reading the resolved values", reader, readerRefused},
{"the runtime of mesh-config's machine", runtime, runtimeRefused},
{"the controller", controller, controllerRefused}} {
for _, b := range buckets {
if err := other.nc.Publish("$KV."+b.Bucket()+".mesh.font-size", []byte(`{"value":99}`)); err != nil {
t.Fatal(err)
}
_ = other.nc.Flush()
select {
case err := <-other.refused:
if !errors.Is(err, nats.ErrPermissionViolation) {
t.Errorf("%s putting to %s: the server said %v, want a permissions violation", other.name, b.Bucket(), err)
}
case <-time.After(3 * time.Second):
t.Errorf("the server did not refuse %s a put to %s", other.name, b.Bucket())
}
}
}
// The reader still reads what it was granted.
rjs, err := jetstream.New(reader)
if err != nil {
t.Fatal(err)
}
resolved, err := rjs.KeyValue(ctx, "mesh-config_resolved")
if err != nil {
t.Fatalf("a module reading mesh-config.resolved cannot bind it: %v", err)
}
e, err := resolved.Get(ctx, "mesh.font-size")
if err != nil || string(e.Value()) != `{"value":12}` {
t.Errorf("the resolved value reads %v (%v), want mesh-config's own write", e, err)
}
for _, b := range buckets {
kv, err := ojs.KeyValue(ctx, b.Bucket())
if err != nil {
t.Fatal(err)
}
if e, err := kv.Get(ctx, "mesh.font-size"); err != nil || string(e.Value()) != `{"value":12}` {
t.Errorf("%s holds %v (%v) after the refused puts", b.Bucket(), e, err)
}
}
}
+67 -2
View File
@@ -40,6 +40,45 @@ var (
AskedBucket = BucketName(ControllerSeat, "asked")
)
// The controller's record mirrors (novox/hq ADR 0289, design 54, design 57 package F3): a bucket per kind of
// record, one key per record, written from the store by the lease holder only when the record changed, with
// one event on the controller's seat per write. Read by whatever names them in `reads` as
// `mesh-controller.<mirror>` (ADR 0201 §1), written by nobody else. **No secret in either** (ADR 0201): an
// assignment's value names its module and node; a module's declared config is what its manifest, which is
// public, declares.
//
// AssignmentsBucket holds which module is assigned to which node, keyed `<node>.<module>`; and
// DeclaredConfigBucket each registered module's config declarations as its manifest holds them, keyed by
// the module: what mesh-config resolves a config value from (ADR 0348 clauses 2 and 4); and PredefinedBucket the
// list of predefined keys the mesh published (the newest trunk build's of the catalogue, ADR 0348 §7), one key
// `list`: what mesh-config publishes as mesh-config_predefined (design 57 K3, issue 662).
var (
AssignmentsBucket = BucketName(ControllerSeat, MirrorAssignments)
DeclaredConfigBucket = BucketName(ControllerSeat, MirrorDeclaredConfig)
PredefinedBucket = BucketName(ControllerSeat, MirrorPredefined)
)
// The mirrors' local names, as a reader names them in `reads` after `mesh-controller.`.
const (
MirrorAssignments = "assignments"
MirrorDeclaredConfig = "declared-config"
MirrorPredefined = "predefined"
)
// ControllerMirrors are the local names of the controller's record mirrors, which a module may read.
func ControllerMirrors() []string {
return []string{MirrorAssignments, MirrorDeclaredConfig, MirrorPredefined}
}
// The mirrors' caps (design 54 §5): one value per key, no age, 64 KiB a value, 64 MiB a bucket.
const (
MirrorValueBytes = 64 << 10
MirrorBytes = 64 << 20
// MirrorMarkerTTL is the longest a purge's marker is kept (design 54 §2): the bucket allows a
// per-message age for it, and the writer gives each marker a day.
MirrorMarkerTTL = 24 * time.Hour
)
// AskedKeptFor is how long an ask is kept after it was made: a month, as the router keeps its own.
const AskedKeptFor = 30 * 24 * time.Hour
@@ -66,12 +105,14 @@ const (
// IsControllerBucket says a bucket is the controller's own, not a module's state nothing declares.
func IsControllerBucket(bucket string) bool {
return bucket == CallsBucket || bucket == HandActsBucket || bucket == ConditionsBucket ||
bucket == ConditionHistoryBucket || bucket == LeaseBucket || bucket == AskedBucket
bucket == ConditionHistoryBucket || bucket == LeaseBucket || bucket == AskedBucket ||
bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket
}
// ControllerBuckets are the controller's own buckets, in the order they are asserted.
func ControllerBuckets() []string {
return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket}
return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket,
AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket}
}
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
@@ -168,6 +209,30 @@ func (j *JetStream) EnsureControllerBuckets() error {
}); err != nil {
return fmt.Errorf("asserting bucket %s: %w", AskedBucket, err)
}
// **The record mirrors: no age** (design 54 §5). A key is removed when its record is, by a purge whose
// marker lives a day; a key that expired would be a record the bus forgot while the store still holds it.
for _, m := range []struct{ bucket, why string }{
{AssignmentsBucket, "which module is assigned to which node, keyed <node>.<module> (novox/hq ADR 0289, " +
"design 54): written by the controller alone from its store when an assignment changes, read through `reads`"},
{DeclaredConfigBucket, "each registered module's config declarations as its manifest holds them, keyed by " +
"the module (novox/hq ADR 0348, design 57 F3): written by the controller alone from its store when a " +
"registration changes them, read through `reads`"},
{PredefinedBucket, "the list of predefined keys the mesh published, the newest trunk build's of the " +
"catalogue, one key `list` (novox/hq ADR 0348 §7, design 57 K3): written by the controller alone from " +
"its store when a newer list is kept, read through `reads`"},
} {
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
Bucket: m.bucket,
Description: m.why,
History: 1,
MaxValueSize: MirrorValueBytes,
MaxBytes: MirrorBytes,
Storage: jetstream.FileStorage,
LimitMarkerTTL: MirrorMarkerTTL,
}); err != nil {
return fmt.Errorf("asserting bucket %s: %w", m.bucket, err)
}
}
return nil
}
+131 -1
View File
@@ -21,7 +21,9 @@ func TestTheControllerMayWriteEveryBucketItWrites(t *testing.T) {
if err != nil {
t.Fatal(err)
}
want := []string{"$KV." + CallsBucket + ".>", "$KV." + HandActsBucket + ".>"}
want := []string{"$KV." + CallsBucket + ".>", "$KV." + HandActsBucket + ".>",
"$KV." + AssignmentsBucket + ".>", "$KV." + DeclaredConfigBucket + ".>",
"$KV." + PredefinedBucket + ".>"}
for _, seat := range seatsTheControllerAsks {
if hasCancelledSet(seat) {
want = append(want, "$KV."+CancelledSetName(seat)+".>")
@@ -37,6 +39,35 @@ func TestTheControllerMayWriteEveryBucketItWrites(t *testing.T) {
}
}
// **A module reads a record mirror through `reads`, and never writes it** (ADR 0201 §1, design 54 §5,
// design 57 F3): granted the direct reads and watches of the bucket, and refused at composition a grant to
// write it, whose writer is the controller.
func TestAModuleReadsTheControllersMirrorsAndWritesNone(t *testing.T) {
reader := Principal{Kind: KindModule, Node: "one", Module: "mesh-config", PasswordHash: "x",
Reads: []string{"mesh-controller." + MirrorAssignments, "mesh-controller." + MirrorDeclaredConfig,
"mesh-controller." + MirrorPredefined}}
p, err := PermissionsFor(reader)
if err != nil {
t.Fatal(err)
}
for _, bucket := range []string{AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket} {
for _, subject := range []string{"$JS.API.STREAM.INFO.KV_" + bucket, "$JS.API.DIRECT.GET.KV_" + bucket + ".>",
"$JS.API.CONSUMER.CREATE.KV_" + bucket + ".>"} {
if !slices.Contains(p.Publish, subject) {
t.Errorf("a module reading %s may not publish %s", bucket, subject)
}
}
for _, subject := range p.Publish {
if SubjectsOverlap(subject, "$KV."+bucket+".>") {
t.Errorf("a module reading %s may write it through %s", bucket, subject)
}
}
if err := CheckWriters(reader, []string{"$KV." + bucket + ".>"}); err == nil {
t.Errorf("a module granted a write of %s would compose", bucket)
}
}
}
// **A machine's node tools may say they are there, as that machine and no other** (novox/hq to-be 45
// S11), and the controller may hear the bus's advisories and ask who answers — read-only, named.
func TestTheWatchedSignalsMayBeSaidAndHeard(t *testing.T) {
@@ -96,3 +127,102 @@ func TestWhatTheControllerAskedIsBounded(t *testing.T) {
t.Errorf("history %d, age %s, bytes %d", status.History(), status.TTL(), info.Config.MaxBytes)
}
}
// **A listed mirror is no undeclared bucket** (ADR 0356 §3, ADR 0201 §16): the controller's list of its mirrors
// stands for their declaration, so the report of state nothing declares never names one.
func TestAListedMirrorIsNotReportedAsUndeclared(t *testing.T) {
names := []string{"gone_state"}
for _, m := range ControllerMirrors() {
names = append(names, BucketName(ControllerSeat, m))
}
undeclared, err := RaiseBuckets(fakeBuckets{names: names}, nil)
if err != nil {
t.Fatal(err)
}
if !slices.Equal(undeclared, []string{"gone_state"}) {
t.Fatalf("the report of undeclared state names %v", undeclared)
}
}
// **The mirrors' list and the writers table agree** (ADR 0356 §3, to-be 45 §1): every listed mirror is a row of
// the table whose one writer is the controller, on its bucket alone; and every row that writes a bucket of the
// controller's named as a mirror is on the list.
func TestTheMirrorsListAndTheWritersTableAgree(t *testing.T) {
controller := Principal{Kind: KindController}
module := Principal{Kind: KindModule, Module: ControllerSeat, Node: "one"}
rowed := map[string]bool{}
for _, row := range WritersTable {
if !strings.HasSuffix(row.State, " mirror") {
continue
}
for _, subject := range row.Subjects {
bucket := strings.TrimSuffix(strings.TrimPrefix(subject, "$KV."), ".>")
rowed[bucket] = true
if row.Writes == nil || !row.Writes(controller, subject) || row.Writes(module, subject) {
t.Errorf("%q is not written by the controller alone", row.State)
}
}
}
listed := map[string]bool{}
for _, m := range ControllerMirrors() {
bucket := BucketName(ControllerSeat, m)
listed[bucket] = true
if !rowed[bucket] {
t.Errorf("the mirror %s is listed and no row of the writers table writes it", bucket)
}
if !IsControllerBucket(bucket) || !slices.Contains(ControllerBuckets(), bucket) {
t.Errorf("the mirror %s is listed and not asserted as the controller's own", bucket)
}
}
for bucket := range rowed {
if !listed[bucket] {
t.Errorf("the writers table writes the mirror %s, which the list does not name", bucket)
}
}
}
// **The controller may ask the vault's intake verbs in its own name** (design 57's V2, issue 675): the vault
// takes a value only from the controller, which it knows from the subject that names its caller — so the
// controller is granted that subject for the verbs the vault's holder serves, and no more of the vault's seat.
// `get` among them since issue 755 (ADR 0349 §16): the vault answers the controller a core credential sealed to its
// machine alone, which the controller carries and cannot open.
// Composed from the records, as the issued tools are: the installer's first user list carries none.
func TestTheControllerMayAskTheVaultInItsOwnName(t *testing.T) {
vault := Declared{Module: "mesh-vault", Holds: []Seat{{Name: "mesh-vault",
Serves: []string{"get", "accept", "secret-ask", "issued", "rotate"}}}}
users, err := Users(Records{Nodes: []string{"novox"}, Assigned: map[string][]Declared{"novox": {vault}}})
if err != nil {
t.Fatal(err)
}
controller := users[0]
if strings.Join(controller.AsksTheVault, ",") != "accept,get,issued,rotate,secret-ask" {
t.Fatalf("composed %v", controller.AsksTheVault)
}
p, err := PermissionsFor(controller)
if err != nil {
t.Fatal(err)
}
for _, verb := range []string{"accept", "secret-ask", "issued", "rotate", "get"} {
if !MayPublish(p, CalledSubject("mesh.seat.mesh-vault.tool."+verb, ControllerName)) {
t.Errorf("the controller may not ask the vault's %s in its own name", verb)
}
if MayPublish(p, CalledSubject("mesh.seat.mesh-vault.tool."+verb, "anchor.node-tools")) {
t.Errorf("the controller may ask the vault's %s in another's name", verb)
}
}
if MayPublish(p, CalledSubject("mesh.seat.mesh-vault.tool.move-in", ControllerName)) {
t.Error("the controller may move a value in: that is a node-engine's alone")
}
// A vault that serves none of them yet composes no grant, nor does genesis.
older := Declared{Module: "mesh-vault", Holds: []Seat{{Name: "mesh-vault", Serves: []string{"move-in"}}}}
users, _ = Users(Records{Nodes: []string{"novox"}, Assigned: map[string][]Declared{"novox": {older}}})
if len(users[0].AsksTheVault) != 0 {
t.Errorf("a vault serving none composed %v", users[0].AsksTheVault)
}
genesis, _ := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
for _, s := range genesis.Publish {
if strings.Contains(s, "mesh-vault") {
t.Errorf("the installer's controller may publish %s", s)
}
}
}
+6 -2
View File
@@ -172,7 +172,9 @@ func TestRaisingLogsReportsWhatNothingDeclares(t *testing.T) {
// **No path of the mesh deletes or purges a log or a bucket, or recreates one** (novox/hq ADR 0297 §2,
// ADR 0201 §15–16): no code of this repository outside its tests calls the client's stream or bucket
// delete, or purges a stream, but for the controller lease's own stream, which purges its own history
// below a sequence (internal/lease). A new call is a decision, not a refactor.
// below a sequence (internal/lease), and the controller's record mirrors, which purge the key of a record
// the store no longer holds (internal/recordmirror; novox/hq ADR 0289, design 54 §2). A new call is a decision,
// not a refactor.
func TestNoPathDeletesALogOrABucket(t *testing.T) {
removers := map[string]bool{"DeleteStream": true, "DeleteKeyValue": true, "PurgeStream": true,
"DeleteObjectStore": true}
@@ -197,7 +199,9 @@ func TestNoPathDeletesALogOrABucket(t *testing.T) {
return true
}
name := sel.Sel.Name
if removers[name] || (name == "Purge" && !strings.Contains(filepath.ToSlash(path), "internal/lease/")) {
decided := strings.Contains(filepath.ToSlash(path), "internal/lease/") ||
strings.Contains(filepath.ToSlash(path), "internal/recordmirror/")
if removers[name] || (name == "Purge" && !decided) {
found = append(found, path+": "+name)
}
return true
+88
View File
@@ -0,0 +1,88 @@
package broker
import (
"strings"
"testing"
)
// The move of the secrets into the vault (design 57's V3, issue 676): each machine's node-engine may ask the
// vault's move-in in its own name — the subject that names it, and no other machine's — once the vault's holder
// serves it; it hears the controller's ask to move a value on its own subject; and the controller may copy its
// record of asks with the vault's move-asks. Composed from the records: a vault that serves neither grants nothing.
func TestANodeEngineMayMoveItsSecretsIntoTheVaultInItsOwnName(t *testing.T) {
vault := Declared{Module: "mesh-vault", Holds: []Seat{{Name: "mesh-vault",
Serves: []string{"get", "accept", "secret-ask", "issued", "rotate", "move-in", "move-asks"}}}}
users, err := Users(Records{Nodes: []string{"anchor", "shanks"}, Assigned: map[string][]Declared{"anchor": {vault}}})
if err != nil {
t.Fatal(err)
}
var anchor, controller Principal
for _, u := range users {
switch {
case u.Kind == KindNode && u.Node == "anchor":
anchor = u
case u.Kind == KindController:
controller = u
}
}
p, err := PermissionsFor(anchor)
if err != nil {
t.Fatal(err)
}
if !MayPublish(p, CalledSubject("mesh.seat.mesh-vault.tool.move-in", "node.anchor")) {
t.Error("anchor's node-engine may not ask the vault's move-in in its own name")
}
for _, s := range []string{CalledSubject("mesh.seat.mesh-vault.tool.move-in", "node.shanks"),
CalledSubject("mesh.seat.mesh-vault.tool.move-in", ControllerName),
CalledSubject("mesh.seat.mesh-vault.tool.get", "node.anchor"),
CalledSubject("mesh.seat.mesh-vault.tool.accept", "node.anchor"),
"mesh.seat.mesh-vault.tool.move-in"} {
if MayPublish(p, s) {
t.Errorf("anchor's node-engine may publish %s", s)
}
}
if !MaySubscribe(p, AskMoveSecretSubject("anchor")) || MaySubscribe(p, AskMoveSecretSubject("shanks")) {
t.Error("anchor's node-engine does not hear its own move asks alone")
}
// The vault's holder hears the call naming the node-engine.
held := false
for _, u := range users {
if u.Kind == KindModule && u.Module == "mesh-vault" {
held = true
h, err := PermissionsFor(u)
if err != nil {
t.Fatal(err)
}
if !MaySubscribe(h, CalledSubject("mesh.seat.mesh-vault.tool.move-in", "node.anchor")) {
t.Error("the vault's holder does not hear move-in asked by a node-engine")
}
}
}
if !held {
t.Fatal("no principal holds the vault's seat")
}
c, err := PermissionsFor(controller)
if err != nil {
t.Fatal(err)
}
if !MayPublish(c, CalledSubject("mesh.seat.mesh-vault.tool.move-asks", ControllerName)) {
t.Error("the controller may not copy its record of asks into the vault")
}
if MayPublish(c, CalledSubject("mesh.seat.mesh-vault.tool.move-in", ControllerName)) {
t.Error("the controller may move a value in itself: it opens none")
}
if !MayPublish(c, AskMoveSecretSubject("anchor")) {
t.Error("the controller may not ask a machine's node-engine to move a value")
}
// A vault that serves neither, the running one: no grant for either.
older := Declared{Module: "mesh-vault", Holds: []Seat{{Name: "mesh-vault", Serves: []string{"get"}}}}
users, _ = Users(Records{Nodes: []string{"anchor"}, Assigned: map[string][]Declared{"anchor": {older}}})
for _, u := range users {
p, _ := PermissionsFor(u)
for _, s := range p.Publish {
if strings.Contains(s, "move-in") || strings.Contains(s, "move-asks") {
t.Errorf("%s may publish %s while the vault serves neither", u.Username(), s)
}
}
}
}
+131 -4
View File
@@ -16,6 +16,7 @@ import (
"errors"
"fmt"
"regexp"
"slices"
"sort"
"strings"
"time"
@@ -42,6 +43,10 @@ const (
// Its authority is the union of what the modules it carries would each have had for their
// tools — and nothing of what they consume, because tools are what it runs, not reactions.
KindNodeTools Kind = "node-tools"
// KindProvisioner is a machine's provisioner client (novox/hq ADR 0349 clauses 6 and 11, design 57 P3): a
// service of the module holding NodeProvisionerSeat, outside the tool runner, under an account of its own so
// the vault can tell it from an agent. Its authority is provisionerGrants and nothing else.
KindProvisioner Kind = "provisioner"
)
// RuntimeModule is the module that IS the node's tool runtime (novox/hq ADR 0175). Where it is
@@ -125,11 +130,24 @@ type Principal struct {
// AsksIssued are the tools the controller asks in its own name for a credential a provider's software
// issued, each `<module>.<tool>` (novox/hq ADR 0305): on the subject that names its caller, so the tool
// knows the controller asks. Composed from the records, so the installer's first user list — a controller
// knows the controller asks. The vault's holder is granted the same tools in its own name (issue 723,
// vaultCalls); the controller's grant goes with its path in R1 (issue 682). Composed from the records, so the installer's first user list — a controller
// with no records — carries none, and moves with issue 464 like the rest of the controller's caller-named
// grants.
AsksIssued []string
// AsksTheVault are the vault's verbs the controller asks in its own name (VerbsTheControllerAsksTheVault,
// issue 675), those the vault's holder serves: on the subject that names its caller, so the vault knows the
// controller asks. Composed from the records like AsksIssued, so the installer's first user list carries none.
AsksTheVault []string
// AsksTheDesk says this principal is the vault's holder, which asks the desks' secret prompt in its own name
// (novox/hq ADR 0349 clause 9, issue 723): `node-launcher.secret` on any machine, on the subject that names
// its caller, and nothing else of that seat. With AsksIssued on the same principal, the vault's whole reach
// outside its own seat: it asks each issuing provider's tool in its own name (clause 10). The controller
// keeps its own grant to both until R1 (issue 682), for the modules still filled the old way (clause 17).
AsksTheDesk bool
// State is the local names of the state this principal's module keeps, and Reads the state of
// others it reads as `<module>.<name>` (novox/hq ADR 0201): a bucket each, kept by the owner's
// instances and read by whoever declares it.
@@ -160,6 +178,16 @@ type Principal struct {
// `<module>.<tool>` (novox/hq ADR 0240, to-be 48 §3): asked of the instance on its own machine and
// nowhere else.
Checks []string
// MovesIntoTheVault is a machine principal whose node-engine may ask the vault's `move-in` in its own name (design
// 57's V3, issue 676): set while the vault's holder serves it, so the engine can hand in the values sealed to its
// machine, which nothing else can open.
MovesIntoTheVault bool
// StatesItsKeyToTheVault is a machine principal whose node-engine may ask the vault's `node-key` in its own name
// (issue 786, novox/hq ADR 0372): set while the vault's holder serves it. That one verb, and nothing else new.
StatesItsKeyToTheVault bool
// BoundBuckets are the providers' buckets of entries, `<module>_bound`, a provisioner client reads its own
// machine's keys of (mesh-sdk go/provider, issue 679): every bucket named bound of a module the mesh places.
BoundBuckets []string
// PasswordHash is the bcrypt hash the mesh minted. The plaintext is sealed to the principal
// and never appears here: this file is written to a node's disk and read by a server, and a
@@ -273,6 +301,29 @@ var VerbsTheControllerAsksTheDeliveryOwner = []SeatVerb{{Seat: "mesh-delivery",
{Seat: "mesh-delivery", Verb: "close"}, {Seat: "mesh-delivery", Verb: "release"}, {Seat: "mesh-delivery", Verb: "stop"},
{Seat: "mesh-delivery", Verb: "times"}, {Seat: "mesh-delivery", Verb: "groups"}}
// VerbsTheControllerAsksTheVault are the mesh-vault seat's verbs the controller calls in its own name (novox/hq
// ADR 0349 clauses 8 to 10, design 57's V2, issue 675): it carries a value given at its terminal, typed at a desk
// or issued by a provider's software into the vault, sealed to a key the vault made for the call, and asks a
// rotation. The vault takes these from the controller alone, which it knows from the subject that names its
// caller; granted as Principal.AsksTheVault says.
//
// And `move-asks` (design 57's V3, issue 676): the controller copies its record of asks at a desk, which holds no
// value, into the vault's.
//
// And `get` (novox/hq ADR 0349 §16, issue 755): the controller asks for a core module's own credential sealed to its
// machine's key, which it carries in that machine's declaration and cannot open (mesh-vault's core.go).
var VerbsTheControllerAsksTheVault = []SeatVerb{{Seat: "mesh-vault", Verb: "accept"},
{Seat: "mesh-vault", Verb: "secret-ask"}, {Seat: "mesh-vault", Verb: "issued"}, {Seat: "mesh-vault", Verb: "rotate"},
{Seat: "mesh-vault", Verb: "move-asks"}, {Seat: "mesh-vault", Verb: "get"}}
// VaultMoveIn is the vault's verb a machine's node-engine asks in its own name to hand in a value sealed to its
// machine (design 57's V3, issue 676); granted as Principal.MovesIntoTheVault says.
var VaultMoveIn = SeatVerb{Seat: "mesh-vault", Verb: "move-in"}
// VaultNodeKey is the vault's verb a machine's node-engine asks in its own name to state its sealing key (issue 786,
// novox/hq ADR 0372); granted as Principal.StatesItsKeyToTheVault says.
var VaultNodeKey = SeatVerb{Seat: "mesh-vault", Verb: "node-key"}
// VerbsTheControllerActsOnAWarrant are the other seat verbs the controller calls when the operator's warrant
// chooses them (novox/hq ADR 0259): a machine's service restarted, and a walk started or stopped through the
// controller's own `plans`. Named one by one; a node seat's on any machine, a mesh seat's flat.
@@ -336,6 +387,10 @@ func (p Principal) Username() string {
return "node." + p.Node
case KindController:
return "controller"
case KindProvisioner:
// Named for the machine, as a node is: one client per machine, whichever module runs it, and never
// `<node>.<module>`, which is the module's own account or the tool runner's.
return "provisioner." + p.Node
case KindEnrolment:
// Per token, not one shared user. **The inbox is the reason**: with a single `enrolment`
// user every machine enrolling at once could read every other's answer, and an answer
@@ -375,6 +430,11 @@ func AskJudgeDirectoriesSubject(node string) string {
return "mesh.node." + node + ".ask.judge-directories"
}
// AskMoveSecretSubject is where the controller asks one machine's node-engine to move one value sealed to it into
// the vault (design 57's V3, issue 676): a request answered once, signed as a hand-over is (link.MoveSecretContext),
// for the same reason.
func AskMoveSecretSubject(node string) string { return "mesh.node." + node + ".ask.move-secret" }
// inbox is a principal's own reply space. No user is ever granted a bare `_INBOX.>` (design 25
// §4): with one account, inbox privacy is the permission list or it is nothing, so each user's
// inbox is derived from its own identity and its permissions name that prefix and no other.
@@ -598,6 +658,12 @@ func PermissionsFor(p Principal) (Permissions, error) {
pub = append(pub, "$JS.ACK."+DeadLetterNoticesStream+"."+ControllerName+".>", deadLetterPrefix+">",
againPrefix+">")
case KindProvisioner:
var err error
if pub, sub, err = provisionerGrants(p); err != nil {
return Permissions{}, err
}
case KindPerson:
// Tools, and nothing else. Every subject a person may publish is a tool call; a person
// who could publish an event would be able to claim a module said something.
@@ -659,11 +725,24 @@ func PermissionsFor(p Principal) (Permissions, error) {
// And asking it for a fresh search for setuid programs (novox/hq issue 361), answered the same way.
AskSetuidSearchSubject(p.Node),
// And asking it to judge its declared directories, read-only (novox/hq issue 358), answered the same way.
AskJudgeDirectoriesSubject(p.Node)}
AskJudgeDirectoriesSubject(p.Node),
// And asking it to move a value sealed to it into the vault (design 57's V3, issue 676), answered the
// same way.
AskMoveSecretSubject(p.Node)}
// The node-engine witnesses the core builds it places (novox/hq to-be 45 §8, ADR 0236; the
// contract is lease/witness.go): it asks its own machine's node tools PING, and, where the
// machine runs the controller, reads the lease's one key — read, never written.
pub = append(pub, WitnessSubjects(p)...)
// And, while the vault serves it, the vault's move-in in its own name and nobody else's (issue 676): the
// values sealed to this machine, which only it opens, handed in sealed to the vault's key for the call.
if p.MovesIntoTheVault {
pub = append(pub, "mesh.seat."+VaultMoveIn.Seat+"."+CallKind+"."+VaultMoveIn.Verb+"."+CallerToken(p.Username()))
}
// And, while the vault serves it, its node-key in its own name (issue 786): the machine's sealing key, stated
// by the engine that holds it, which the vault takes from no other caller.
if p.StatesItsKeyToTheVault {
pub = append(pub, "mesh.seat."+VaultNodeKey.Seat+"."+CallKind+"."+VaultNodeKey.Verb+"."+CallerToken(p.Username()))
}
case KindModule:
// 1. Its own namespace: it publishes its events there and serves its tools there. Nothing
@@ -874,6 +953,11 @@ func PermissionsFor(p Principal) (Permissions, error) {
return Permissions{}, err
}
pub = append(pub, invoked...)
// And its own machine's provisioner client, for a carried module's config and non-secret bound values,
// naming the module from the bundle that asks (novox/hq ADR 0370 §3, §4): the ask on its own machine's
// subject alone, and what the client says moved, so a module's code's watches are asked again.
pub = append(pub, ProvisionerAskSubject(p.Node))
sub = append(sub, "prov.changed."+p.Node+".>")
// It says what it serves and may ask what answers (novox/hq ADR 0197): the runtime answers
// discovery for each module and seat it carries, and the console it is asks the bus.
// One service per runtime process, named for the runtime: the bus lets a principal answer each
@@ -979,8 +1063,14 @@ func PermissionsFor(p Principal) (Permissions, error) {
// runs, so a change to them waits on a node-engine delivery. It calls and serves on the subjects that name
// no caller meanwhile — a caller-named call to its seat reaches nobody and is asked again on those at once —
// and moves with issue 464.
// The vault's calls in its own name (issue 723), kept apart so the desk's grant is not denied below.
var vaultsOwn []string
if p.Kind != KindController {
pub, sub = callerNamed(p.Username(), pub, sub)
if p.Kind == KindModule {
vaultsOwn = vaultCalls(p)
pub = append(pub, vaultsOwn...)
}
} else {
// **But the tools that answer a software-issued credential** (novox/hq ADR 0305 §2): those answer the
// controller alone, and know it only from the subject's last token, so it asks them in its own name —
@@ -992,6 +1082,13 @@ func PermissionsFor(p Principal) (Permissions, error) {
}
pub = append(pub, "mesh.mod."+module+"."+CallKind+"."+tool+".*."+CallerToken(p.Username()))
}
// **And the vault's intake verbs** (issue 675): the vault takes a value from the controller alone, known
// from the subject's last token — the verbs its holder serves, flat, and nothing else of its seat.
for _, v := range VerbsTheControllerAsksTheVault {
if slices.Contains(p.AsksTheVault, v.Verb) {
pub = append(pub, "mesh.seat."+v.Seat+"."+CallKind+"."+v.Verb+"."+CallerToken(p.Username()))
}
}
}
sort.Strings(pub)
@@ -1006,6 +1103,9 @@ func PermissionsFor(p Principal) (Permissions, error) {
if p.Kind != KindController {
for _, only := range ControllerOnly() {
for _, a := range pub {
if slices.Contains(vaultsOwn, a) {
continue
}
if SubjectsOverlap(a, only) {
deny = append(deny, only)
break
@@ -1022,11 +1122,38 @@ func PermissionsFor(p Principal) (Permissions, error) {
// thing, a hand-over on its own subject (novox/hq issue 356), and answers that: the node is its
// machine's engine, root there already, and it is delivered only its own subjects. What it answers is
// never trusted for being an answer — the controller reads the engine's words and records nothing. A
// person is never asked anything, and is granted nothing here.
AllowResponses: p.Kind == KindModule || p.Kind == KindController || p.Kind == KindNodeTools || p.Kind == KindNode,
// provisioner client is asked by its own machine's tool runner alone, on its own machine's subject (novox/hq
// ADR 0370 §3), and answers that. A person is never asked anything, and is granted nothing here.
AllowResponses: p.Kind == KindModule || p.Kind == KindController || p.Kind == KindNodeTools || p.Kind == KindNode ||
p.Kind == KindProvisioner,
}, nil
}
// vaultCalls are the calls the vault's holder makes in its own name (novox/hq ADR 0349 clauses 9 and 10, issue
// 723): the desks' secret prompt on any machine, and each tool that answers a credential its provider's software
// issued, on any machine — each on the subject whose last token is the vault's own name, which the bus lets it
// alone publish, so the prompt and the tool know the vault asks. Nothing else outside its seat.
func vaultCalls(p Principal) []string {
token := CallerToken(p.Username())
if token == "" {
return nil
}
var out []string
if p.AsksTheDesk {
for _, v := range VerbsTheControllerAsksForASecret {
out = append(out, "mesh.seat."+v.Seat+"."+CallKind+"."+v.Verb+".*."+token)
}
}
for _, key := range p.AsksIssued {
module, tool, ok := strings.Cut(key, ".")
if !ok || module == "" || tool == "" || strings.ContainsAny(key, "*> ") {
continue
}
out = append(out, "mesh.mod."+module+"."+CallKind+"."+tool+".*."+token)
}
return out
}
// seatSubject places a seat's verb under the kind of traffic it is.
//
// **The kind token is load-bearing, not decoration.** A stream is defined by a subject filter, so
+58
View File
@@ -0,0 +1,58 @@
package broker
import "testing"
// Issue 786 (novox/hq ADR 0372): each machine's node-engine may ask the vault's node-key in its own name, the
// subject that names it and no other machine's, once the vault's holder serves it; and nothing else of the vault's
// is granted by it. A vault that does not serve it grants nothing.
func TestANodeEngineMayStateItsSealingKeyToTheVaultInItsOwnNameAlone(t *testing.T) {
vault := Declared{Module: "mesh-vault", Holds: []Seat{{Name: "mesh-vault", Serves: []string{"get", "node-key"}}}}
users, err := Users(Records{Nodes: []string{"anchor", "shanks"}, Assigned: map[string][]Declared{"anchor": {vault}}})
if err != nil {
t.Fatal(err)
}
var anchor Principal
for _, u := range users {
if u.Kind == KindNode && u.Node == "anchor" {
anchor = u
}
}
p, err := PermissionsFor(anchor)
if err != nil {
t.Fatal(err)
}
if !MayPublish(p, CalledSubject("mesh.seat.mesh-vault.tool.node-key", "node.anchor")) {
t.Error("anchor's node-engine may not state its key to the vault in its own name")
}
for _, s := range []string{CalledSubject("mesh.seat.mesh-vault.tool.node-key", "node.shanks"),
CalledSubject("mesh.seat.mesh-vault.tool.node-key", ControllerName),
CalledSubject("mesh.seat.mesh-vault.tool.get", "node.anchor"),
CalledSubject("mesh.seat.mesh-vault.tool.move-in", "node.anchor"),
"mesh.seat.mesh-vault.tool.node-key"} {
if MayPublish(p, s) {
t.Errorf("anchor's node-engine may publish %s", s)
}
}
for _, u := range users {
if u.Kind == KindModule && u.Module == "mesh-vault" {
h, err := PermissionsFor(u)
if err != nil {
t.Fatal(err)
}
if !MaySubscribe(h, CalledSubject("mesh.seat.mesh-vault.tool.node-key", "node.anchor")) {
t.Error("the vault's holder does not hear node-key asked by a node-engine")
}
}
}
older := Declared{Module: "mesh-vault", Holds: []Seat{{Name: "mesh-vault", Serves: []string{"get", "move-in"}}}}
users, _ = Users(Records{Nodes: []string{"anchor"}, Assigned: map[string][]Declared{"anchor": {older}}})
for _, u := range users {
if u.Kind != KindNode {
continue
}
q, _ := PermissionsFor(u)
if MayPublish(q, CalledSubject("mesh.seat.mesh-vault.tool.node-key", "node.anchor")) {
t.Error("a node-engine may state its key while the vault does not serve node-key")
}
}
}
+135
View File
@@ -0,0 +1,135 @@
package broker
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
"golang.org/x/crypto/bcrypt"
)
// **On a real server, as composed** (design 57 P6, issue 672, ADR 0362): a machine's provisioner client's word on a
// module's files is kept by the mesh's own stream, newest per module, and the controller reads it by subject; the
// client cannot say it of another machine, and nothing but a machine's own client is granted to say it.
func TestTheProvisionersReportIsKeptAndReadByTheController(t *testing.T) {
hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
if err != nil {
t.Fatal(err)
}
accounts, err := ComposeAccounts([]Principal{
{Kind: KindController, PasswordHash: string(hash)},
{Kind: KindProvisioner, Node: "g14", Module: "mesh-provisioner-client", PasswordHash: string(hash)},
{Kind: KindProvisioner, Node: "shanks", Module: "mesh-provisioner-client", PasswordHash: string(hash)},
})
if err != nil {
t.Fatal(err)
}
conf := filepath.Join(t.TempDir(), "bus.conf")
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
t.Fatal(err)
}
opts, err := server.ProcessConfigFile(conf)
if err != nil {
t.Fatalf("the composed accounts do not parse: %v", err)
}
opts.NoLog, opts.NoSigs = true, true
s, err := server.NewServer(opts)
if err != nil {
t.Fatal(err)
}
go s.Start()
if !s.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
defer s.Shutdown()
refusals := make(chan error, 16)
connect := func(user string) *nats.Conn {
t.Helper()
nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user),
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refusals <- err }))
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
return nc
}
control := OnConn(connect("controller"))
if err := AssertMeshStreams(control); err != nil {
t.Fatalf("the mesh's streams cannot be asserted: %v", err)
}
g14 := connect("provisioner.g14")
for _, said := range []string{`{"status":"waiting"}`, `{"status":"applied","templates":"sha256:new"}`} {
if err := g14.Publish(ProvisionerAppliedSubject("g14", "dunst"), []byte(said)); err != nil {
t.Fatal(err)
}
}
if err := g14.Flush(); err != nil {
t.Fatal(err)
}
// Another machine's client says nothing of g14: refused by the server, and nothing kept.
shanks := connect("provisioner.shanks")
if err := shanks.Publish(ProvisionerAppliedSubject("g14", "dunst"), []byte(`{"status":"failed"}`)); err != nil {
t.Fatal(err)
}
_ = shanks.Flush()
select {
case err := <-refusals:
if !errors.Is(err, nats.ErrPermissionViolation) {
t.Fatalf("the server said %v, want a permissions violation", err)
}
case <-time.After(2 * time.Second):
t.Fatal("the server did not refuse shanks's client a report on g14")
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
js, err := control.Conn().JetStream(nats.Context(ctx))
if err != nil {
t.Fatal(err)
}
// A core publish is kept by the stream once the server has stored it: read until it has, a few seconds at most.
var got []byte
var said bool
for deadline := time.Now().Add(5 * time.Second); ; {
got, said, err = LastProvApplied(js, "g14", "dunst")
if (err == nil && said && strings.Contains(string(got), "applied")) || time.Now().After(deadline) {
break
}
time.Sleep(20 * time.Millisecond)
}
if err != nil || !said || string(got) != `{"status":"applied","templates":"sha256:new"}` {
t.Fatalf("the controller read %q %v %v, want g14's newest report", got, said, err)
}
if _, said, err := LastProvApplied(js, "shanks", "dunst"); err != nil || said {
t.Fatalf("a module nothing was said of reads as said: %v %v", said, err)
}
all, err := ProvAppliedReports(js)
if err != nil || len(all) != 1 || string(all["prov.applied.g14.dunst"]) != `{"status":"applied","templates":"sha256:new"}` {
t.Fatalf("every report, newest per module: %q %v", all, err)
}
info, err := js.StreamInfo(ProvAppliedStream)
if err != nil || info.State.Msgs != 1 {
t.Fatalf("the stream keeps the newest report per module only: %+v %v", info, err)
}
// The writers table: a machine's own client alone is granted to say it.
if err := CheckWriters(Principal{Kind: KindProvisioner, Node: "g14"}, []string{"prov.applied.g14.>"}); err != nil {
t.Fatalf("g14's client may not say its own: %v", err)
}
for _, p := range []Principal{{Kind: KindProvisioner, Node: "shanks"}, {Kind: KindNode, Node: "g14"},
{Kind: KindModule, Node: "g14", Module: "dunst"}} {
err := CheckWriters(p, []string{"prov.applied.g14.>"})
if err == nil || !strings.Contains(err.Error(), "a module's files placed on a machine") {
t.Errorf("%s may say g14's provisioner's report: %v", p.Username(), err)
}
}
}
+212
View File
@@ -0,0 +1,212 @@
package broker
import (
"errors"
"fmt"
"strings"
"github.com/nats-io/nats.go"
)
// The provisioner client's own bus account (novox/hq ADR 0349 clauses 6 and 11, design 57 package P3).
//
// **Not the tool runner's account.** The client fills a machine's files and asks the vault for the secrets that
// machine's declaration names; the vault hands a secret only to an account of its own, because a call under the
// tool runner's account cannot be told from an agent's on that machine (ADR 0305 §3, §12). So each machine that
// runs the client has a principal of its own, KindProvisioner, named for the machine, and granted what the client
// does and nothing else: read its own machine's keys in each domain's bucket, write its own machine's keys of the
// provisioner server's needs, applied and collections (design 57 P4), read the server's registry, say what it
// applied on its own machine, and ask the vault in its own name.
// NodeProvisionerSeat is the seat the provisioner client holds on every machine (ADR 0349 clause 3).
const NodeProvisionerSeat = "node-provisioner"
// ProvisionerSecret is the own secret of the module running the client that its bus account is delivered in, as
// `broker` is a module's own (novox/hq issue 195): a module that claims NodeProvisionerSeat and declares no own
// secret of this name has nowhere to read an account, and is given none.
const ProvisionerSecret = "provisioner-broker"
// ConfigResolvedBucket is the config domain's resolved values, keyed `<node>.<module>.<key>` (ADR 0348, research
// 051): mesh-config's state `resolved`, written by mesh-config alone.
var ConfigResolvedBucket = BucketName("mesh-config", "resolved")
// ProvisionerReads are the domains' buckets a provisioner client reads its own machine's keys of, each keyed
// `<node>.…` (research 051): placement, the controller's assignments mirror (`<node>.<module>`), whose value carries
// the provisioner's plan of a module handed to it (inventory.Provisioning, design 57 P5); and config,
// mesh-config's resolved values. **Named one by one**, as the controller's asks are: a domain that moves to the
// provisioner adds its bucket here, and a bucket not named is one the client cannot read.
var ProvisionerReads = []string{ConfigResolvedBucket}
// ProvisionerReadsWhole are the buckets a provisioner client reads every key of: the controller's assignments mirror,
// where a provider's consumers are bound to it in their own machines' plans, which the provider machine's client
// reads to fill ${consumers:<provision>} with each one's credential (novox/hq ADR 0373, issue 679). It holds placement
// and plans, never a secret's value; the client writes none of it.
var ProvisionerReadsWhole = []string{AssignmentsBucket}
// BoundState is a providing module's state of entries, one per consumer (mesh-sdk go/provider, issue 678): a
// consumer's machine's client reads its own keys of it, `<node>.<module>.<provision>[.<local>]`, to fill
// ${bound:<provision>:<key>} (issue 679).
const BoundState = "bound"
// ProvisionerServerModule is the module holding the seat mesh-provisioner once for the mesh (ADR 0349 clause 3,
// design 57 P4), whose buckets the clients write their own machine's keys of.
const ProvisionerServerModule = "mesh-provisioner"
// The provisioner server's buckets (ADR 0349 clause 3): its state, created by the controller from its manifest
// like any module's, and written by its own account whole.
var (
// ProvisionerRegistryBucket is which domain answers which kind of placeholder, keyed by the kind; the
// server's alone, read whole by every client.
ProvisionerRegistryBucket = BucketName(ProvisionerServerModule, "registry")
// ProvisionerNeedsBucket is what each module on each machine needs, `<node>.<module>.<kind>.<name>`.
ProvisionerNeedsBucket = BucketName(ProvisionerServerModule, "needs")
// ProvisionerAppliedBucket is what is applied where, `<node>.<module>`.
ProvisionerAppliedBucket = BucketName(ProvisionerServerModule, "applied")
// ProvisionerCollectBucket is the items of every collection, `<node>.<collection>.<contributor>`.
ProvisionerCollectBucket = BucketName(ProvisionerServerModule, "collect")
)
// ProvisionerWrites are the server's buckets a client writes, and reads, its own machine's keys of (`<node>.>`)
// and no other's: the needs it records, what it applied, and the items of the collections on its machine. A
// client cannot write another machine's needs, or say another machine applied a file.
var ProvisionerWrites = []string{ProvisionerNeedsBucket, ProvisionerAppliedBucket, ProvisionerCollectBucket}
// VerbsTheProvisionerAsksTheVault are the vault's verbs a provisioner client asks in its own name (ADR 0349
// clause 11): `get`, a secret its machine's declaration names, answered sealed to the asker's key. On the subject
// that names its caller, so the vault knows which machine's client asks from the bus's word.
var VerbsTheProvisionerAsksTheVault = []SeatVerb{{Seat: "mesh-vault", Verb: "get"}}
// VaultSecretChangedSubject is where the vault says a secret it holds changed (`secret-changed`: which, by whom,
// when, its generation; never a value, ADR 0349 clause 11), as link.EventSubject addresses mesh-vault's event. A
// provisioner client listens to it so a rotated secret reaches its file without a new declaration (design 57 V4):
// it asks the vault again, in its own name, for a secret its machine's declaration names.
const VaultSecretChangedSubject = "mesh.mod.mesh-vault.event.secret-changed"
// ProvisionerAppliedSubject is where a machine's client says what it applied for one module, applied or failed
// with the reason (ADR 0349 clauses 4 and 15): `prov.applied.<node>.<module>`.
func ProvisionerAppliedSubject(node, module string) string {
return "prov.applied." + node + "." + module
}
// ProvisionerAskSubject is where a machine's tool runner asks that machine's provisioner client for a config value
// or a non-secret bound value of a module it carries, naming the module from the bundle that asks (novox/hq ADR
// 0370 §1, §3, §4; issue 685): `prov.ask.<node>`, which the client subscribes and the tool runner's account of the
// same machine alone may publish. The bus proves only that the tool runner asks; the module is the runtime's word,
// so the client answers no secret here (ADR 0370 §2).
func ProvisionerAskSubject(node string) string { return "prov.ask." + node }
// ProvisionerChangedSubject is where a machine's client says that a value of one module it answers moved, so the
// tool runner asks again for the module's code's watches (ADR 0349 §14, ADR 0370 §1): `prov.changed.<node>.<module>`,
// carrying which value and never the value itself, heard by that machine's tool runner alone.
func ProvisionerChangedSubject(node, module string) string {
return "prov.changed." + node + "." + module
}
// provisionerGrants is a provisioner client's whole authority: to publish, and to subscribe.
func provisionerGrants(p Principal) (pub, sub []string, err error) {
if p.Node == "" {
// Refused rather than composed into `$KV.<bucket>..>`, a pattern no machine's keys are under, and a
// name every nameless client would share.
return nil, nil, errors.New("a provisioner client names no machine, so it would have no keys of its own " +
"to read and an account it shared with every other")
}
for _, bucket := range ProvisionerReads {
pub = append(pub, machineKeysRead(bucket, p.Node)...)
}
for _, bucket := range ProvisionerReadsWhole {
pub = append(pub, wholeBucketRead(bucket)...)
}
for _, bucket := range p.BoundBuckets {
module, name, ok := strings.Cut(bucket, "_")
if !ok || name != BoundState || !safeSubject.MatchString(module) || strings.Contains(module, "_") {
return nil, nil, fmt.Errorf("%q is not a provider's bucket of entries, <module>_%s", bucket, BoundState)
}
pub = append(pub, machineKeysRead(bucket, p.Node)...)
}
// The server's buckets (design 57 P4): its own machine's keys of the needs, the applied and the
// collections, read and written, `$KV.<bucket>.<node>.>` and nothing wider; and the registry, read whole,
// which says which domain answers which kind and is the same for every machine.
for _, bucket := range ProvisionerWrites {
pub = append(pub, machineKeysRead(bucket, p.Node)...)
pub = append(pub, "$KV."+bucket+"."+p.Node+".>")
}
pub = append(pub, wholeBucketRead(ProvisionerRegistryBucket)...)
pub = append(pub, "prov.applied."+p.Node+".>")
token := CallerToken(p.Username())
if token == "" {
return nil, nil, fmt.Errorf("%q cannot name a caller", p.Username())
}
for _, v := range VerbsTheProvisionerAsksTheVault {
pub = append(pub, "mesh.seat."+v.Seat+"."+CallKind+"."+v.Verb+"."+token)
}
// What moved of a value it answers its own machine's tool runner (ADR 0370 §1), under its own machine's name.
pub = append(pub, "prov.changed."+p.Node+".>")
// Its own reply space, for the vault's answers and its reads' and watches' deliveries; what the vault says
// changed; and its own machine's tool runner asking it for a module's config and non-secret bound values (ADR
// 0370 §3), which it answers on the ask's reply and nowhere else.
return pub, []string{p.inbox(), VaultSecretChangedSubject, ProvisionerAskSubject(p.Node)}, nil
}
// machineKeysRead is reading every key of a bucket under one machine's name, `<node>.>`, and no other key: a
// direct get of a key's own subject, and a consumer whose filter is under it — the server's create-with-filter
// form, which it holds to the filter the subject names (as stateGrants does for one key, novox/hq ADR 0260).
// Binding, deleting a consumer and flow control cannot be narrowed by key and read no value.
func machineKeysRead(bucket, node string) []string {
stream := "KV_" + bucket
keys := "$KV." + bucket + "." + node + ".>"
return []string{
"$JS.API.STREAM.INFO." + stream,
"$JS.API.DIRECT.GET." + stream + "." + keys,
"$JS.API.CONSUMER.CREATE." + stream + ".*." + keys,
"$JS.API.CONSUMER.DELETE." + stream + ".>",
"$JS.FC." + stream + ".>",
}
}
// wholeBucketRead is reading every key of a bucket and writing none: what stateGrants grants a module for a
// bucket it `reads`.
func wholeBucketRead(bucket string) []string {
stream := "KV_" + bucket
return []string{
"$JS.API.STREAM.INFO." + stream,
"$JS.API.DIRECT.GET." + stream + ".>",
"$JS.API.CONSUMER.CREATE." + stream + ".>",
"$JS.API.CONSUMER.DELETE." + stream + ".>",
"$JS.FC." + stream + ".>",
}
}
// LastProvApplied is what a machine's provisioner client last said of one module's files, as it published it on
// ProvisionerAppliedSubject and ProvAppliedStream keeps it (design 57 P6): false when it has said nothing of it in
// the stream's week.
func LastProvApplied(js nats.JetStreamContext, node, module string) ([]byte, bool, error) {
msg, err := js.GetLastMsg(ProvAppliedStream, ProvisionerAppliedSubject(node, module))
if errors.Is(err, nats.ErrMsgNotFound) {
return nil, false, nil
}
if err != nil {
return nil, false, fmt.Errorf("what %s's provisioner client said of %s cannot be read: %w", node, module, err)
}
return msg.Data, true, nil
}
// ProvAppliedReports is the newest word of every machine's provisioner client on every module it said one of, kept by
// ProvAppliedStream, by subject (`prov.applied.<node>.<module>`).
func ProvAppliedReports(js nats.JetStreamContext) (map[string][]byte, error) {
info, err := js.StreamInfo(ProvAppliedStream, &nats.StreamInfoRequest{SubjectsFilter: ProvAppliedSubjects})
if err != nil {
return nil, fmt.Errorf("the provisioner clients' reports cannot be listed: %w", err)
}
out := map[string][]byte{}
for subject := range info.State.Subjects {
msg, err := js.GetLastMsg(ProvAppliedStream, subject)
if errors.Is(err, nats.ErrMsgNotFound) {
continue // aged out between the two reads
}
if err != nil {
return nil, fmt.Errorf("the provisioner client's report on %s cannot be read: %w", subject, err)
}
out[subject] = msg.Data
}
return out, nil
}
+393
View File
@@ -0,0 +1,393 @@
package broker
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"time"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"golang.org/x/crypto/bcrypt"
)
// **On a real server, as composed** (design 57 package P3, issue 669): a machine's provisioner client reads and
// watches its own machine's keys in a domain's bucket, and the server refuses it another machine's keys, a write,
// and a report or a vault ask in another's name.
func TestAServerComposedFromTheGrantsRefusesTheProvisionerClientAnotherMachinesKeys(t *testing.T) {
hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
if err != nil {
t.Fatal(err)
}
vault := Seat{Name: "mesh-vault", Scope: "mesh", Serves: []string{"get"}}
accounts, err := ComposeAccounts([]Principal{
{Kind: KindController, PasswordHash: string(hash)},
{Kind: KindModule, Node: "novox", Module: "mesh-config", State: []string{"resolved"}, PasswordHash: string(hash)},
{Kind: KindModule, Node: "novox", Module: "mesh-vault", Holds: []Seat{vault}, PasswordHash: string(hash)},
{Kind: KindModule, Node: "novox", Module: "postgres", State: []string{"bound"}, PasswordHash: string(hash)},
{Kind: KindProvisioner, Node: "g14", Module: "mesh-provisioner", BoundBuckets: []string{"postgres_bound"},
PasswordHash: string(hash)},
})
if err != nil {
t.Fatal(err)
}
conf := filepath.Join(t.TempDir(), "bus.conf")
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
t.Fatal(err)
}
opts, err := server.ProcessConfigFile(conf)
if err != nil {
t.Fatalf("the composed accounts do not parse: %v", err)
}
opts.NoLog, opts.NoSigs = true, true
s, err := server.NewServer(opts)
if err != nil {
t.Fatal(err)
}
go s.Start()
if !s.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
defer s.Shutdown()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
connect := func(user string, refusals chan error) *nats.Conn {
t.Helper()
nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user),
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {
if refusals != nil {
refusals <- err
}
}))
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
return nc
}
// The controller makes the buckets, as it does from registration; the controller writes its mirror and
// mesh-config writes its own bucket.
control, err := jetstream.New(connect("controller", nil))
if err != nil {
t.Fatal(err)
}
put := func(js jetstream.JetStream, bucket string, values map[string]string) {
t.Helper()
kv, err := js.KeyValue(ctx, bucket)
if err != nil {
t.Fatal(err)
}
for k, v := range values {
if _, err := kv.PutString(ctx, k, v); err != nil {
t.Fatalf("writing %s in %s: %v", k, bucket, err)
}
}
}
for _, bucket := range append(append([]string{"postgres_bound"}, ProvisionerReads...), ProvisionerReadsWhole...) {
if _, err := control.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket}); err != nil {
t.Fatal(err)
}
}
put(control, AssignmentsBucket, map[string]string{"g14.dunst": "{}", "shanks.dunst": "{}"})
provider, err := jetstream.New(connect("novox.postgres", nil))
if err != nil {
t.Fatal(err)
}
put(provider, "postgres_bound", map[string]string{"g14.gitea.postgres-database": "{}",
"shanks.photos.postgres-database": "{}"})
config, err := jetstream.New(connect("novox.mesh-config", nil))
if err != nil {
t.Fatal(err)
}
put(config, ConfigResolvedBucket, map[string]string{"g14.dunst.font-size": "13", "shanks.dunst.font-size": "11"})
refusals := make(chan error, 16)
clientConn := connect("provisioner.g14", refusals)
client, err := jetstream.New(clientConn)
if err != nil {
t.Fatal(err)
}
resolved, err := client.KeyValue(ctx, ConfigResolvedBucket)
if err != nil {
t.Fatalf("the client cannot bind to the config domain's bucket: %v", err)
}
// Its own machine's key, read and watched; the watch delivers the current value first.
if e, err := resolved.Get(ctx, "g14.dunst.font-size"); err != nil || string(e.Value()) != "13" {
t.Fatalf("its own machine's key: %v %v", e, err)
}
w, err := resolved.Watch(ctx, "g14.>")
if err != nil {
t.Fatalf("watching its own machine's keys: %v", err)
}
select {
case e := <-w.Updates():
if e == nil || e.Key() != "g14.dunst.font-size" {
t.Fatalf("the watch delivered %v first", e)
}
case <-time.After(5 * time.Second):
t.Fatal("the watch of its own machine's keys delivered nothing")
}
_ = w.Stop()
assignments, err := client.KeyValue(ctx, AssignmentsBucket)
if err != nil {
t.Fatal(err)
}
if _, err := assignments.Get(ctx, "g14.dunst"); err != nil {
t.Fatalf("its own machine's placement: %v", err)
}
// Another machine's key: refused by the server, read or watched, and every key at once too; and a write of
// a domain's bucket, which is the domain's alone. Each refusal is the server's word, not a missing key.
drain := func() {
for {
select {
case <-refusals:
default:
return
}
}
}
serverRefuses := func(what string, act func(context.Context) error) {
t.Helper()
drain()
short, done := context.WithTimeout(ctx, 2*time.Second)
defer done()
if err := act(short); err == nil {
t.Errorf("the client may %s", what)
}
select {
case err := <-refusals:
if !errors.Is(err, nats.ErrPermissionViolation) {
t.Errorf("%s: the server said %v, want a permissions violation", what, err)
}
case <-time.After(time.Second):
t.Errorf("%s: the server did not refuse it", what)
}
}
serverRefuses("read another machine's config", func(c context.Context) error {
_, err := resolved.Get(c, "shanks.dunst.font-size")
return err
})
// Another machine's placement is read, since a provider's consumers are bound to it in their own machines'
// plans (ADR 0373, issue 679), and never written.
if _, err := assignments.Get(ctx, "shanks.dunst"); err != nil {
t.Fatalf("another machine's plan, which binds its consumers to a provider here: %v", err)
}
serverRefuses("write the assignments mirror", func(c context.Context) error {
_, err := assignments.PutString(c, "shanks.dunst", "{}")
return err
})
// A provider's entries: its own machine's consumers' keys alone (issue 679).
entries, err := client.KeyValue(ctx, "postgres_bound")
if err != nil {
t.Fatalf("the client cannot bind to a provider's bucket of entries: %v", err)
}
if _, err := entries.Get(ctx, "g14.gitea.postgres-database"); err != nil {
t.Fatalf("its own machine's consumer's entry: %v", err)
}
serverRefuses("read another machine's consumer's entry", func(c context.Context) error {
_, err := entries.Get(c, "shanks.photos.postgres-database")
return err
})
serverRefuses("write a provider's entry", func(c context.Context) error {
_, err := entries.PutString(c, "g14.gitea.postgres-database", "{}")
return err
})
for _, keys := range []string{"shanks.>", ">"} {
serverRefuses("watch "+keys, func(c context.Context) error {
w, err := resolved.Watch(c, keys)
if err != nil {
return err
}
defer func() { _ = w.Stop() }()
select {
case e := <-w.Updates():
if e != nil {
return nil
}
return errors.New("nothing delivered")
case <-c.Done():
return c.Err()
}
})
}
serverRefuses("write the config domain's bucket", func(c context.Context) error {
_, err := resolved.PutString(c, "g14.dunst.font-size", "40")
return err
})
// What it applied, on its own machine and no other's; the vault asked in its own name and no other's.
refused := func(subject string, want bool) {
t.Helper()
drain()
if err := clientConn.Publish(subject, []byte(`{}`)); err != nil {
t.Fatal(err)
}
_ = clientConn.Flush()
select {
case err := <-refusals:
if !want {
t.Errorf("%s was refused: %v", subject, err)
} else if !errors.Is(err, nats.ErrPermissionViolation) {
t.Errorf("%s: the server said %v, want a permissions violation", subject, err)
}
case <-time.After(500 * time.Millisecond):
if want {
t.Errorf("the server did not refuse %s", subject)
}
}
}
refused("prov.applied.g14.dunst", false)
refused("prov.applied.shanks.dunst", true)
refused("mesh.seat.mesh-vault.call.get.provisioner~g14", false)
refused("mesh.seat.mesh-vault.call.get.g14~node-tools", true)
refused("mesh.seat.mesh-vault.call.accept.provisioner~g14", true)
refused("mesh.mod.dunst.tool.anything", true)
}
// **On a real server, as composed** (novox/hq ADR 0370 §3, issue 685): a machine's tool runner asks its own
// machine's provisioner client and is answered; the server refuses the ask from another machine's tool runner, from
// a module's own account and from the client of another machine, and refuses the client what another machine's
// tool runner would hear.
func TestAServerComposedFromTheGrantsLetsOnlyItsOwnToolRunnerAskTheProvisionerClient(t *testing.T) {
hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
if err != nil {
t.Fatal(err)
}
accounts, err := ComposeAccounts([]Principal{
{Kind: KindController, PasswordHash: string(hash)},
{Kind: KindNodeTools, Node: "g14", Module: RuntimeModule, PasswordHash: string(hash),
Carries: []Declared{{Module: "claude-code"}}},
{Kind: KindNodeTools, Node: "shanks", Module: RuntimeModule, PasswordHash: string(hash),
Carries: []Declared{{Module: "claude-code"}}},
{Kind: KindModule, Node: "g14", Module: "dunst", PasswordHash: string(hash)},
{Kind: KindProvisioner, Node: "g14", Module: "mesh-provisioner", BoundBuckets: []string{"postgres_bound"},
PasswordHash: string(hash)},
{Kind: KindProvisioner, Node: "shanks", Module: "mesh-provisioner", PasswordHash: string(hash)},
})
if err != nil {
t.Fatal(err)
}
conf := filepath.Join(t.TempDir(), "bus.conf")
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\n"+accounts), 0o600); err != nil {
t.Fatal(err)
}
opts, err := server.ProcessConfigFile(conf)
if err != nil {
t.Fatalf("the composed accounts do not parse: %v", err)
}
opts.NoLog, opts.NoSigs = true, true
s, err := server.NewServer(opts)
if err != nil {
t.Fatal(err)
}
go s.Start()
if !s.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
defer s.Shutdown()
connect := func(user string, refusals chan error) *nats.Conn {
t.Helper()
nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user),
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {
if refusals != nil {
refusals <- err
}
}))
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
return nc
}
clientRefusals := make(chan error, 8)
client := connect("provisioner.g14", clientRefusals)
asked := make(chan string, 8)
if _, err := client.Subscribe(ProvisionerAskSubject("g14"), func(m *nats.Msg) {
asked <- string(m.Data)
_ = m.Respond([]byte(`{"value":"answered"}`))
}); err != nil {
t.Fatal(err)
}
if err := client.Flush(); err != nil {
t.Fatal(err)
}
select {
case err := <-clientRefusals:
t.Fatalf("the client may not hear its own machine's asks: %v", err)
case <-time.After(300 * time.Millisecond):
}
// Its own machine's tool runner asks and is answered.
runner := connect("g14.node-tools", nil)
reply, err := runner.Request(ProvisionerAskSubject("g14"), []byte(`{"module":"claude-code"}`), 5*time.Second)
if err != nil {
t.Fatalf("g14's tool runner cannot ask g14's client: %v", err)
}
if string(reply.Data) != `{"value":"answered"}` {
t.Fatalf("answered %s", reply.Data)
}
<-asked
// Anybody else's ask is refused by the server, and never reaches the client.
for _, user := range []string{"shanks.node-tools", "g14.dunst", "provisioner.shanks", "controller"} {
refusals := make(chan error, 4)
nc := connect(user, refusals)
if err := nc.Publish(ProvisionerAskSubject("g14"), []byte(`{"module":"claude-code"}`)); err != nil {
t.Fatal(err)
}
_ = nc.Flush()
select {
case err := <-refusals:
if !errors.Is(err, nats.ErrPermissionViolation) {
t.Errorf("%s: the server said %v, want a permissions violation", user, err)
}
case <-time.After(time.Second):
t.Errorf("the server did not refuse %s an ask of g14's client", user)
}
select {
case body := <-asked:
t.Errorf("g14's client heard %s from %s", body, user)
case <-time.After(200 * time.Millisecond):
}
}
// What moved is said under its own machine's name, heard by that machine's tool runner; another machine's name
// is refused it.
told := make(chan string, 4)
if _, err := runner.Subscribe("prov.changed.g14.>", func(m *nats.Msg) { told <- m.Subject }); err != nil {
t.Fatal(err)
}
_ = runner.Flush()
if err := client.Publish(ProvisionerChangedSubject("g14", "claude-code"), []byte(`{"kind":"config"}`)); err != nil {
t.Fatal(err)
}
select {
case subject := <-told:
if subject != "prov.changed.g14.claude-code" {
t.Errorf("heard %s", subject)
}
case <-time.After(5 * time.Second):
t.Error("g14's tool runner did not hear what g14's client said moved")
}
if err := client.Publish(ProvisionerChangedSubject("shanks", "claude-code"), []byte(`{}`)); err != nil {
t.Fatal(err)
}
_ = client.Flush()
select {
case err := <-clientRefusals:
if !errors.Is(err, nats.ErrPermissionViolation) {
t.Errorf("the server said %v, want a permissions violation", err)
}
case <-time.After(time.Second):
t.Error("the server did not refuse g14's client a word in shanks's name")
}
}
@@ -0,0 +1,217 @@
package broker
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"time"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"golang.org/x/crypto/bcrypt"
)
// **On a real server, as composed** (design 57 package P4, issue 670): the provisioner server's buckets are
// written by the server's own account whole, and by each machine's provisioner client for its own machine's keys
// alone (ADR 0349 clause 3). The server refuses a client another machine's needs, applied or items, and the
// registry; and refuses every other account a write, a provider reading the needs included.
func TestOnlyTheServerAndEachClientForItsOwnMachineWriteTheProvisionersBuckets(t *testing.T) {
hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
if err != nil {
t.Fatal(err)
}
pw := string(hash)
accounts, err := ComposeAccounts([]Principal{
{Kind: KindController, PasswordHash: pw},
{Kind: KindModule, Node: "novox", Module: ProvisionerServerModule,
State: []string{"registry", "needs", "applied", "collect"}, PasswordHash: pw},
// A provider learns its consumers from the needs (ADR 0349 clause 12): it reads them, and writes none.
{Kind: KindModule, Node: "novox", Module: "postgres", Reads: []string{"mesh-provisioner.needs"}, PasswordHash: pw},
{Kind: KindModule, Node: "g14", Module: "dunst", PasswordHash: pw},
{Kind: KindProvisioner, Node: "g14", Module: ProvisionerServerModule, PasswordHash: pw},
{Kind: KindProvisioner, Node: "shanks", Module: ProvisionerServerModule, PasswordHash: pw},
})
if err != nil {
t.Fatal(err)
}
conf := filepath.Join(t.TempDir(), "bus.conf")
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
t.Fatal(err)
}
opts, err := server.ProcessConfigFile(conf)
if err != nil {
t.Fatalf("the composed accounts do not parse: %v", err)
}
opts.NoLog, opts.NoSigs = true, true
s, err := server.NewServer(opts)
if err != nil {
t.Fatal(err)
}
go s.Start()
if !s.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
defer s.Shutdown()
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
type account struct {
js jetstream.JetStream
refusals chan error
}
connect := func(user string) account {
t.Helper()
refusals := make(chan error, 16)
nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user),
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refusals <- err }))
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
return account{js: js, refusals: refusals}
}
bucket := func(a account, name string) jetstream.KeyValue {
t.Helper()
kv, err := a.js.KeyValue(ctx, name)
if err != nil {
t.Fatalf("binding to %s: %v", name, err)
}
return kv
}
// The controller makes the buckets from the server's manifest, as it does for every module's state.
control := connect("controller")
for _, name := range append([]string{ProvisionerRegistryBucket}, ProvisionerWrites...) {
if _, err := control.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name}); err != nil {
t.Fatal(err)
}
}
// The server writes all four, every machine's keys.
srv := connect("novox." + ProvisionerServerModule)
for name, key := range map[string]string{ProvisionerRegistryBucket: "config", ProvisionerNeedsBucket: "shanks.dunst.config.font-size",
ProvisionerAppliedBucket: "shanks.dunst", ProvisionerCollectBucket: "shanks.zsh-snippets.git"} {
if _, err := bucket(srv, name).PutString(ctx, key, `{}`); err != nil {
t.Fatalf("the server writing %s in %s: %v", key, name, err)
}
}
// A client writes, reads and watches its own machine's keys, and reads the registry whole.
g14 := connect("provisioner.g14")
own := map[string]string{ProvisionerNeedsBucket: "g14.dunst.config.font-size", ProvisionerAppliedBucket: "g14.dunst",
ProvisionerCollectBucket: "g14.zsh-snippets.git"}
for name, key := range own {
kv := bucket(g14, name)
if _, err := kv.PutString(ctx, key, `{}`); err != nil {
t.Fatalf("the client writing its own machine's %s in %s: %v", key, name, err)
}
if e, err := kv.Get(ctx, key); err != nil || string(e.Value()) != `{}` {
t.Fatalf("the client reading back its own %s in %s: %v %v", key, name, e, err)
}
w, err := kv.Watch(ctx, "g14.>")
if err != nil {
t.Fatalf("the client watching its own machine's keys in %s: %v", name, err)
}
select {
case e := <-w.Updates():
if e == nil || e.Key() != key {
t.Fatalf("the watch of %s delivered %v first", name, e)
}
case <-time.After(5 * time.Second):
t.Fatalf("the watch of its own machine's keys in %s delivered nothing", name)
}
_ = w.Stop()
if err := kv.Delete(ctx, key); err != nil {
t.Fatalf("the client deleting its own %s in %s (a module taken off its machine): %v", key, name, err)
}
}
registry := bucket(g14, ProvisionerRegistryBucket)
if e, err := registry.Get(ctx, "config"); err != nil || e == nil {
t.Fatalf("the client reading the registry: %v %v", e, err)
}
// Each refusal is the server's word, not a missing key.
serverRefuses := func(a account, what string, act func(context.Context) error) {
t.Helper()
for drained := false; !drained; {
select {
case <-a.refusals:
default:
drained = true
}
}
short, done := context.WithTimeout(ctx, 2*time.Second)
defer done()
if err := act(short); err == nil {
t.Errorf("%s was let through", what)
}
select {
case err := <-a.refusals:
if !errors.Is(err, nats.ErrPermissionViolation) {
t.Errorf("%s: the server said %v, want a permissions violation", what, err)
}
case <-time.After(time.Second):
t.Errorf("%s: the server did not refuse it", what)
}
}
put := func(kv jetstream.KeyValue, key string) func(context.Context) error {
return func(c context.Context) error { _, err := kv.PutString(c, key, `{}`); return err }
}
for name, key := range map[string]string{ProvisionerNeedsBucket: "shanks.dunst.config.font-size",
ProvisionerAppliedBucket: "shanks.dunst", ProvisionerCollectBucket: "shanks.zsh-snippets.git"} {
kv := bucket(g14, name)
serverRefuses(g14, "g14's client writing shanks's "+key+" in "+name, put(kv, key))
serverRefuses(g14, "g14's client reading shanks's "+key+" in "+name, func(c context.Context) error {
_, err := kv.Get(c, key)
return err
})
}
serverRefuses(g14, "g14's client watching every machine's needs", func(c context.Context) error {
w, err := bucket(g14, ProvisionerNeedsBucket).Watch(c, ">")
if err != nil {
return err
}
defer func() { _ = w.Stop() }()
select {
case e := <-w.Updates():
if e != nil {
return nil
}
return errors.New("nothing delivered")
case <-c.Done():
return c.Err()
}
})
serverRefuses(g14, "a client writing the registry", put(registry, "config"))
// Shanks's client is held to shanks's keys the same way: its write of g14's applied is refused.
shanks := connect("provisioner.shanks")
if _, err := bucket(shanks, ProvisionerAppliedBucket).PutString(ctx, "shanks.dunst", `{}`); err != nil {
t.Fatalf("shanks's client writing its own applied: %v", err)
}
serverRefuses(shanks, "shanks's client writing g14's applied", put(bucket(shanks, ProvisionerAppliedBucket), "g14.dunst"))
// A provider reads every machine's needs and writes none; a module on a client's machine writes none.
provider := connect("novox.postgres")
needs := bucket(provider, ProvisionerNeedsBucket)
if e, err := needs.Get(ctx, "shanks.dunst.config.font-size"); err != nil || e == nil {
t.Fatalf("the provider reading the needs: %v %v", e, err)
}
serverRefuses(provider, "a provider writing the needs", put(needs, "shanks.postgres.bound.db"))
dunst := connect("g14.dunst")
for _, name := range append([]string{ProvisionerRegistryBucket}, ProvisionerWrites...) {
serverRefuses(dunst, "a module on g14 writing "+name, func(c context.Context) error {
// Bound by the stream's own subject: the module is granted no bind, so its put is published raw.
_, err := dunst.js.Publish(c, "$KV."+name+".g14.dunst", []byte(`{}`))
return err
})
}
}
+229
View File
@@ -0,0 +1,229 @@
package broker
import (
"slices"
"testing"
)
// The provisioner client's own bus account (novox/hq ADR 0349 clauses 6 and 11, design 57 package P3, issue
// 669): one per machine running it, apart from the machine's tool runner, granted to read its own machine's keys
// in each domain's bucket, to say what it applied on that machine, and to ask the vault in its own name; nothing
// else.
func provisionerRecords() Records {
client := Declared{Module: "mesh-provisioner", ProvisionsNode: true}
return Records{
Nodes: []string{"g14", "shanks", "ace"},
Assigned: map[string][]Declared{
"g14": {client, {Module: RuntimeModule}},
"shanks": {client},
"ace": {{Module: "dunst"}},
},
}
}
// One account per machine that runs the client, named for the machine and apart from its tool runner's: the
// vault tells the client from an agent on that machine by the name the bus proves.
func TestEachMachineRunningTheProvisionerClientHasAnAccountOfItsOwn(t *testing.T) {
users, err := Users(provisionerRecords())
if err != nil {
t.Fatal(err)
}
var got []string
for _, u := range users {
if u.Kind == KindProvisioner {
got = append(got, u.Username())
if u.Module != "mesh-provisioner" {
t.Errorf("%s is delivered to %q, want the module that runs it", u.Username(), u.Module)
}
}
}
if want := []string{"provisioner.g14", "provisioner.shanks"}; !slices.Equal(got, want) {
t.Fatalf("the provisioner clients' accounts are %v, want %v (none for a machine that does not run it)", got, want)
}
for _, u := range users {
if u.Kind == KindNodeTools && u.Username() == "provisioner.g14" {
t.Fatal("the client shares the tool runner's account")
}
}
}
// Its whole authority, subject by subject: a read of its own machine's keys in each domain's bucket, a read and a
// write of its own machine's keys in the provisioner server's needs, applied and collections, a read of the
// server's registry (design 57 P4), a publish of what it applied on its own machine, the vault's `get` in its own
// name, its own inbox and the vault's secret-changed.
func TestTheProvisionerClientIsGrantedItsMachinesKeysAndNothingElse(t *testing.T) {
perms, err := PermissionsFor(Principal{Kind: KindProvisioner, Node: "g14", Module: "mesh-provisioner",
BoundBuckets: []string{"postgres_bound"}})
if err != nil {
t.Fatal(err)
}
var wantPub []string
for _, bucket := range []string{"mesh-config_resolved", "postgres_bound"} {
stream := "KV_" + bucket
wantPub = append(wantPub,
"$JS.API.STREAM.INFO."+stream,
"$JS.API.DIRECT.GET."+stream+".$KV."+bucket+".g14.>",
"$JS.API.CONSUMER.CREATE."+stream+".*.$KV."+bucket+".g14.>",
"$JS.API.CONSUMER.DELETE."+stream+".>",
"$JS.FC."+stream+".>")
}
// The assignments mirror whole, read and never written: a provider's consumers are bound to it in their own
// machines' plans, which its machine's client reads to fill ${consumers:<provision>} (novox/hq ADR 0373, issue
// 679). The mirror holds placement and plans, never a value of a secret.
wantPub = append(wantPub,
"$JS.API.STREAM.INFO.KV_mesh-controller_assignments",
"$JS.API.DIRECT.GET.KV_mesh-controller_assignments.>",
"$JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments.>",
"$JS.API.CONSUMER.DELETE.KV_mesh-controller_assignments.>",
"$JS.FC.KV_mesh-controller_assignments.>")
for _, bucket := range []string{"mesh-provisioner_needs", "mesh-provisioner_applied", "mesh-provisioner_collect"} {
stream := "KV_" + bucket
wantPub = append(wantPub,
"$JS.API.STREAM.INFO."+stream,
"$JS.API.DIRECT.GET."+stream+".$KV."+bucket+".g14.>",
"$JS.API.CONSUMER.CREATE."+stream+".*.$KV."+bucket+".g14.>",
"$JS.API.CONSUMER.DELETE."+stream+".>",
"$JS.FC."+stream+".>",
"$KV."+bucket+".g14.>")
}
wantPub = append(wantPub,
"$JS.API.STREAM.INFO.KV_mesh-provisioner_registry",
"$JS.API.DIRECT.GET.KV_mesh-provisioner_registry.>",
"$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_registry.>",
"$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_registry.>",
"$JS.FC.KV_mesh-provisioner_registry.>")
wantPub = append(wantPub, "prov.applied.g14.>", "mesh.seat.mesh-vault.call.get.provisioner~g14",
"prov.changed.g14.>")
slices.Sort(wantPub)
if !slices.Equal(perms.Publish, wantPub) {
t.Errorf("publish:\n have %q\n want %q", perms.Publish, wantPub)
}
// Its own inbox, what the vault says changed (`secret-changed`: which, by whom, never a value), so a rotated
// secret reaches its file without a new declaration (design 57 V4), and its own machine's tool runner asking
// it for a carried module's config (ADR 0370 §3), which it answers on the ask's reply.
if want := []string{"_INBOX.provisioner.g14.>", "mesh.mod.mesh-vault.event.secret-changed", "prov.ask.g14"}; !slices.Equal(perms.Subscribe, want) {
t.Errorf("subscribe:\n have %q\n want %q", perms.Subscribe, want)
}
if !perms.AllowResponses || len(perms.PublishDeny) > 0 {
t.Errorf("it answers what it is asked and needs no deny: %+v", perms)
}
// No write of any bucket but its own machine's keys of the server's needs, applied and collections: a domain
// writes its own (ADR 0349 clause 3, research 051), and the registry is the server's alone.
for _, p := range perms.Publish {
if !SubjectsOverlap(p, "$KV.>") {
continue
}
if !slices.Contains([]string{"$KV.mesh-provisioner_needs.g14.>", "$KV.mesh-provisioner_applied.g14.>",
"$KV.mesh-provisioner_collect.g14.>"}, p) {
t.Errorf("it may write a bucket: %s", p)
}
}
}
// Each provider's bucket of entries, `<module>_bound` (mesh-sdk go/provider, issue 678), is read by every machine's
// client for that machine's consumers' keys alone (`<node>.>`), to fill ${bound:<provision>:<key>} (issue 679): the
// buckets of every module the mesh places that keeps state named bound, wherever it runs, and no other.
func TestTheProvisionerClientReadsItsMachinesKeysOfEachProvidersBoundBucket(t *testing.T) {
records := provisionerRecords()
records.Assigned["ace"] = append(records.Assigned["ace"],
Declared{Module: "postgres", State: []Bucket{{Module: "postgres", Name: "bound"}, {Module: "postgres", Name: "kept"}}},
Declared{Module: "keycloak", State: []Bucket{{Module: "keycloak", Name: "bound"}}})
records.Assigned["shanks"] = append(records.Assigned["shanks"],
Declared{Module: "postgres", State: []Bucket{{Module: "postgres", Name: "bound"}}})
users, err := Users(records)
if err != nil {
t.Fatal(err)
}
for _, u := range users {
if u.Kind != KindProvisioner {
continue
}
if want := []string{"keycloak_bound", "postgres_bound"}; !slices.Equal(u.BoundBuckets, want) {
t.Errorf("%s reads the bound buckets %v, want %v", u.Username(), u.BoundBuckets, want)
}
perms, err := PermissionsFor(u)
if err != nil {
t.Fatal(err)
}
mine := "$JS.API.DIRECT.GET.KV_postgres_bound.$KV.postgres_bound." + u.Node + ".>"
if !slices.Contains(perms.Publish, mine) {
t.Errorf("%s may not read its machine's keys of postgres_bound", u.Username())
}
for _, p := range perms.Publish {
if SubjectsOverlap(p, "$JS.API.DIRECT.GET.KV_postgres_kept.>") || SubjectsOverlap(p, "$KV.postgres_bound.>") ||
SubjectsOverlap(p, "$JS.API.DIRECT.GET.KV_postgres_bound.$KV.postgres_bound.ace.>") {
t.Errorf("%s is granted %s: another bucket, a write, or another machine's consumers", u.Username(), p)
}
}
}
if _, err := PermissionsFor(Principal{Kind: KindProvisioner, Node: "g14", Module: "mesh-provisioner",
BoundBuckets: []string{"postgres_bound.>"}}); err == nil {
t.Error("a bound bucket that is a pattern composed")
}
}
// A machine with no usable name reaches nothing: its grants would otherwise be a pattern over every machine.
func TestAProvisionerClientNamingNoMachineIsRefused(t *testing.T) {
for _, node := range []string{"", "g14.>", "*"} {
if _, err := PermissionsFor(Principal{Kind: KindProvisioner, Node: node, Module: "mesh-provisioner"}); err == nil {
t.Errorf("a provisioner client on %q composed", node)
}
}
}
// A whole mesh composes with the clients beside every other kind: nobody is granted a second writer's subject.
func TestAMeshWithProvisionerClientsComposes(t *testing.T) {
users, err := Users(provisionerRecords())
if err != nil {
t.Fatal(err)
}
for i := range users {
users[i].PasswordHash = "x"
}
if _, err := ComposeAccounts(users); err != nil {
t.Fatal(err)
}
}
// The client is asked on a subject its own machine's tool runner alone may publish (novox/hq ADR 0370 §3, issue
// 685): across a whole mesh, no principal but g14's tool runner may ask g14's client, none but g14's client hears
// it, and the client's word of what moved is heard by g14's tool runner alone.
func TestTheProvisionerClientIsAskedByItsOwnMachinesToolRunnerAlone(t *testing.T) {
records := provisionerRecords()
records.Assigned["shanks"] = append(records.Assigned["shanks"], Declared{Module: RuntimeModule},
Declared{Module: "dunst"})
records.Assigned["g14"] = append(records.Assigned["g14"], Declared{Module: "claude-code"})
users, err := Users(records)
if err != nil {
t.Fatal(err)
}
users = append(users, Principal{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}})
var asks, hears, told []string
for _, u := range users {
perms, err := PermissionsFor(u)
if err != nil {
t.Fatalf("%s: %v", u.Username(), err)
}
if slices.ContainsFunc(perms.Publish, func(p string) bool { return SubjectsOverlap(p, ProvisionerAskSubject("g14")) }) {
asks = append(asks, u.Username())
}
if slices.ContainsFunc(perms.Subscribe, func(p string) bool { return SubjectsOverlap(p, ProvisionerAskSubject("g14")) }) {
hears = append(hears, u.Username())
}
if slices.ContainsFunc(perms.Subscribe, func(p string) bool {
return SubjectsOverlap(p, ProvisionerChangedSubject("g14", "claude-code"))
}) {
told = append(told, u.Username())
}
}
if want := []string{"g14.node-tools"}; !slices.Equal(asks, want) {
t.Errorf("g14's client may be asked by %v, want %v alone", asks, want)
}
if want := []string{"provisioner.g14"}; !slices.Equal(hears, want) {
t.Errorf("g14's client's asks are heard by %v, want %v alone", hears, want)
}
if want := []string{"g14.node-tools"}; !slices.Equal(told, want) {
t.Errorf("what g14's client says moved is heard by %v, want %v alone", told, want)
}
}
+7
View File
@@ -34,6 +34,8 @@ func TestTheFactsTheGrantPermitsAreTheFactsTheMeshStates(t *testing.T) {
states = append(states, link.KeyChecked)
// And a walk kept in a new state, for the delivery it walks (novox/hq ADR 0239).
states = append(states, link.KeyPlanMoved)
// And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3; design 57 F3).
states = append(states, link.KeyAssignmentChanged, link.KeyDeclaredConfigChanged, link.KeyPredefinedChanged)
for _, event := range states {
if !slices.Contains(broker.ControllerStates, event) {
t.Errorf("the mesh states %q and its account may not publish it", event)
@@ -42,6 +44,11 @@ func TestTheFactsTheGrantPermitsAreTheFactsTheMeshStates(t *testing.T) {
if len(broker.ControllerStates) != len(states) {
t.Errorf("the grant permits %v, which is more than the mesh states", broker.ControllerStates)
}
// The record mirrors a module may read are the ones the bus holds (design 54 §5, design 57 F3).
if !slices.Equal(catalogue.ControllerMirrors, broker.ControllerMirrors()) {
t.Errorf("the catalogue lets a module read the controller's mirrors %v and the bus holds %v",
catalogue.ControllerMirrors, broker.ControllerMirrors())
}
// **And the seat says it.** A seat carries the protocol of its role (novox/hq ADR 0129), so the
// facts the control plane states are the seat's `emits` — which is what lets anything else declare
// that it consumes them, and what the subject-agreement check reads to know they have an owner.
+23 -1
View File
@@ -211,9 +211,28 @@ func MeshStreams() []Stream {
"stream's message, which ages out with its stream (ADR 0376), and full it refuses the next copy " +
"rather than drop one it holds",
},
{
Name: ProvAppliedStream,
Subjects: []string{ProvAppliedSubjects},
Retention: RetentionLastPerSubject,
Direct: true,
MaxAge: 7 * 24 * 60 * 60,
Why: "each machine's provisioner client's newest word on each module's files, applied, waiting or " +
"failed with the reason (novox/hq ADR 0349 §4): what a walk's end and a gate wait for (ADR 0362, " +
"ADR 0349 §15), read by subject; a week, as the events are",
},
}
}
// ProvAppliedStream keeps what each machine's provisioner client last said of each module's files, newest per
// subject (novox/hq ADR 0349 §4, ADR 0362, design 57 P6): `prov.applied.<node>.<module>` lies outside `mesh.*`, so
// no other stream captures it, and a report said while no controller listened is still there to be read.
const ProvAppliedStream = "PROV_APPLIED"
// ProvAppliedSubjects is every machine's client's reports: `prov.applied.<node>.<module>`, a module's name dots and
// all (ProvisionerAppliedSubject).
const ProvAppliedSubjects = "prov.applied.>"
// DeliverSubjectFor is where a push consumer's messages land.
//
// **Per consumer, which means per stream as well as per name** (novox/hq 04-ISSUES/146). A push
@@ -326,7 +345,10 @@ var ControllerStates = []string{"applied", "refused", "built-before",
// the pull request's status, and anybody else may read.
"checked",
// And a walk kept in a new state (novox/hq ADR 0239): what the delivery it walks reads its steps from.
"plan-moved"}
"plan-moved",
// And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3): the assignments and
// each registered module's declared config (design 57 F3).
"assignment-changed", "declared-config-changed", "predefined-changed"}
// BusAdvisories are what the bus server says about the mesh's own account that the controller
// reads (novox/hq to-be 45 §3, S9): a durable consumer that handed a message over as often as it
+3
View File
@@ -131,6 +131,9 @@ func TestEachStreamCarriesTheRetentionItsShapeNeeds(t *testing.T) {
// kept until somebody acts.
"DEAD_LETTER_NOTICES": RetentionWorkQueue,
"DEAD_LETTERS": RetentionLimits,
// What each machine's provisioner client last said of each module's files (design 57 P6): the newest
// word per module, what a walk's end reads.
"PROV_APPLIED": RetentionLastPerSubject,
}
got := map[string]Retention{}
for _, s := range MeshStreams() {
+2 -2
View File
@@ -24,7 +24,7 @@ accounts {
jetstream: enabled
users = [
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_predefined.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
allow_responses: { max: 1, ttl: "1m" }
} }
@@ -34,7 +34,7 @@ accounts {
} }
{ user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: {
publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.INFO.NODES.one", "$SRV.PING.node-tools.one", "mesh.control.one.>"] }
subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.ask.hand-over", "mesh.node.one.ask.judge-directories", "mesh.node.one.ask.report", "mesh.node.one.ask.setuid-search", "mesh.node.one.declare"] }
subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.ask.hand-over", "mesh.node.one.ask.judge-directories", "mesh.node.one.ask.move-secret", "mesh.node.one.ask.report", "mesh.node.one.ask.setuid-search", "mesh.node.one.declare"] }
allow_responses: { max: 1, ttl: "1m" }
} }
{ user: "one.nats", password: "$2a$11$bbbbbbbbbbbbbbbbbbbbbb", permissions: {
+96 -2
View File
@@ -2,6 +2,7 @@ package broker
import (
"fmt"
"slices"
"sort"
)
@@ -68,6 +69,10 @@ type Declared struct {
// RunsAs is the account the module runs as in a runtime of its own (novox/hq ADR 0259 §8): it is never
// carried by the machine's runtime, and reaches the bus on its own account.
RunsAs string
// ProvisionsNode says the module runs the machine's provisioner client: it claims NodeProvisionerSeat and
// declares the own secret ProvisionerSecret its account is delivered in (novox/hq ADR 0349 clause 11, design 57
// P3). The client is given a bus account of its own, never the module's or the tool runner's.
ProvisionsNode bool
}
// Records is what composing a user list needs to know about the mesh, and nothing more.
@@ -97,7 +102,7 @@ type Records struct {
// is a mesh that cannot be told anything, and there is no state of the records in which that is
// correct.
func Users(r Records) ([]Principal, error) {
out := []Principal{{Kind: KindController, Uses: asksSeatOf(r), AsksIssued: issuedOf(r)}}
out := []Principal{{Kind: KindController, Uses: asksSeatOf(r), AsksIssued: issuedOf(r), AsksTheVault: vaultVerbsOf(r)}}
for _, node := range sortedCopy(r.Nodes) {
witness := false
@@ -108,7 +113,17 @@ func Users(r Records) ([]Principal, error) {
}
checks = append(checks, d.Checks...)
}
out = append(out, Principal{Kind: KindNode, Node: node, WitnessesController: witness, Checks: checks})
out = append(out, Principal{Kind: KindNode, Node: node, WitnessesController: witness, Checks: checks,
MovesIntoTheVault: servesMoveIn(r), StatesItsKeyToTheVault: servesVerb(r, VaultNodeKey)})
// **And the machine's provisioner client, on an account of its own** (novox/hq ADR 0349 clause 11): one
// per machine that runs it, delivered to the module running it.
for _, d := range r.Assigned[node] {
if d.ProvisionsNode {
out = append(out, Principal{Kind: KindProvisioner, Node: node, Module: d.Module,
BoundBuckets: boundBucketsOf(r)})
break
}
}
// **Where the runtime is assigned, the machine gets one runtime principal in place of the
// runtime module's own** (novox/hq ADR 0175, to-be 38). It carries every module on the
// node: its serving grants are the union of theirs. Every other module keeps its own
@@ -141,6 +156,12 @@ func Users(r Records) ([]Principal, error) {
PerMachine: perMachineNames(d.State), KeyedReads: d.KeyedReads,
Logs: logNames(d.Logs), Streams: d.Streams, Hears: ComposedHears(d),
})
// **The vault's holder asks the desks and the issuing providers itself** (issue 723), under the
// account its own runtime runs on (ADR 0259 §8's runs-as): the one module granted it.
if holdsSeat(d, VaultSeat) {
out[len(out)-1].AsksTheDesk = true
out[len(out)-1].AsksIssued = issuedOf(r)
}
}
if runtimeHere {
var carried []Declared
@@ -254,3 +275,76 @@ func asksSeatOf(r Records) []Seat {
}
return nil
}
// VaultSeat is the vault's seat: its holder alone asks the desks and the issuing providers in its own name.
const VaultSeat = "mesh-vault"
func holdsSeat(d Declared, seat string) bool {
for _, s := range d.Holds {
if s.Name == seat {
return true
}
}
return false
}
// vaultVerbsOf are the vault's intake verbs its holder serves, as the records say (issue 675): what the controller
// is granted to ask it in its own name. None while no holder serves them.
func vaultVerbsOf(r Records) []string {
seen := map[string]bool{}
var out []string
for _, node := range sortedCopy(r.Nodes) {
for _, d := range r.Assigned[node] {
for _, s := range d.Holds {
for _, v := range VerbsTheControllerAsksTheVault {
if s.Name == v.Seat && slices.Contains(s.Serves, v.Verb) && !seen[v.Verb] {
seen[v.Verb] = true
out = append(out, v.Verb)
}
}
}
}
}
sort.Strings(out)
return out
}
// servesMoveIn says the vault's holder serves move-in, as the records say (issue 676): what lets each machine's
// node-engine ask it in its own name.
func servesMoveIn(r Records) bool { return servesVerb(r, VaultMoveIn) }
// servesVerb says a holder of verb's seat serves it, as the records say.
func servesVerb(r Records, verb SeatVerb) bool {
for _, node := range r.Nodes {
for _, d := range r.Assigned[node] {
for _, s := range d.Holds {
if s.Name == verb.Seat && slices.Contains(s.Serves, verb.Verb) {
return true
}
}
}
}
return false
}
// boundBucketsOf is every provider's bucket of entries, `<module>_bound` (mesh-sdk go/provider, issue 678): the state
// named bound of every module placed anywhere, once each, in name order. A consumer on any machine may be bound to
// it, so each machine's provisioner client reads its own keys of each (issue 679).
func boundBucketsOf(r Records) []string {
seen := map[string]bool{}
for _, mods := range r.Assigned {
for _, d := range mods {
for _, b := range d.State {
if b.Name == BoundState {
seen[b.Bucket()] = true
}
}
}
}
out := make([]string, 0, len(seen))
for b := range seen {
out = append(out, b)
}
sort.Strings(out)
return out
}
+227
View File
@@ -0,0 +1,227 @@
package broker
import (
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
"golang.org/x/crypto/bcrypt"
)
// vaultRecords is a mesh of two machines: anchor, a desk whose runtime carries the desk's prompt (rofi holding
// node-launcher) beside an issuing provider (plex) and a module with a tool of its own; and novox, where the vault
// runs in a runtime of its own on its own account (ADR 0259 §8's runs-as).
func vaultRecords() Records {
launcher := Seat{Name: "node-launcher", Scope: "node", Serves: []string{"menu", "secret"}}
vault := Seat{Name: VaultSeat, Scope: "mesh", Serves: []string{"get", "accept", "secret-ask", "issued", "rotate"}}
return Records{Nodes: []string{"anchor", "novox"}, Assigned: map[string][]Declared{
"anchor": {{Module: RuntimeModule}, {Module: "rofi", Holds: []Seat{launcher}, NoAccount: true},
{Module: "plex", Serves: []string{"plex_issued_credential", "plex_status"},
Issued: []string{"plex_issued_credential"}, NoAccount: true},
{Module: "other", Serves: []string{"tool"}, NoAccount: true}},
"novox": {{Module: RuntimeModule}, {Module: "mesh-vault", Holds: []Seat{vault}, RunsAs: "mesh-vault"}},
}}
}
// The vault's holder may ask the desk's prompt and each issuing tool in its own name, and nothing else outside its
// seat; no other module and no runtime is granted it (issue 723).
func TestTheVaultAsksTheDeskAndTheIssuingToolsInItsOwnNameAlone(t *testing.T) {
users, err := Users(vaultRecords())
if err != nil {
t.Fatal(err)
}
perms := map[string]Permissions{}
for _, u := range users {
p, err := PermissionsFor(u)
if err != nil {
t.Fatal(err)
}
perms[u.Username()] = p
}
vault, ok := perms["novox.mesh-vault"]
if !ok {
t.Fatal("the vault has no account of its own")
}
token := CallerToken("novox.mesh-vault")
for _, s := range []string{
"mesh.seat.node-launcher.call.secret.anchor." + token,
"mesh.mod.plex.call.plex_issued_credential.anchor." + token,
} {
if !MayPublish(vault, s) {
t.Errorf("the vault may not ask %s", s)
}
}
for _, s := range []string{
"mesh.seat.node-launcher.tool.secret.anchor",
"mesh.seat.node-launcher.call.secret.anchor.controller",
"mesh.seat.node-launcher.call.menu.anchor." + token,
"mesh.mod.rofi.call.node-launcher.secret.anchor." + token,
"mesh.mod.plex.call.plex_status.anchor." + token,
"mesh.mod.plex.tool.plex_issued_credential.anchor",
"mesh.mod.other.call.tool.anchor." + token,
"mesh.mod.other.tool.tool.anchor",
"mesh.seat.mesh-controller.call.status." + token,
} {
if MayPublish(vault, s) {
t.Errorf("the vault may publish %s", s)
}
}
// Outside its own namespace and its own seat, the vault publishes nothing but those two calls, its inbox's
// answers and the stream API's reads of what it keeps.
for _, s := range vault.Publish {
switch {
case strings.HasPrefix(s, "mesh.mod.mesh-vault."), strings.HasPrefix(s, "mesh.seat."+VaultSeat+"."),
s == "mesh.seat.node-launcher.call.secret.*."+token,
s == "mesh.mod.plex.call.plex_issued_credential.*."+token,
strings.HasPrefix(s, "$JS."), strings.HasPrefix(s, "$KV."), strings.HasPrefix(s, "_INBOX."),
strings.HasPrefix(s, "mesh.discover."), strings.HasPrefix(s, "$SRV."):
default:
t.Errorf("the vault is granted %s, outside its seat", s)
}
}
// Nobody else is granted the vault's calls; the controller keeps today's path until R1 (issue 682).
for name, p := range perms {
if name == "novox.mesh-vault" {
continue
}
for _, s := range []string{"mesh.seat.node-launcher.call.secret.anchor." + token,
"mesh.mod.plex.call.plex_issued_credential.anchor." + token} {
if MayPublish(p, s) {
t.Errorf("%s may call in the vault's name: %s", name, s)
}
}
}
controller := perms["controller"]
for _, s := range []string{"mesh.seat.node-launcher.tool.secret.anchor",
"mesh.mod.plex.call.plex_issued_credential.anchor.controller"} {
if !MayPublish(controller, s) {
t.Errorf("the controller lost today's path %s before R1", s)
}
}
if MayPublish(perms["anchor.node-tools"], "mesh.seat.node-launcher.call.secret.anchor."+CallerToken("anchor.node-tools")) {
t.Error("a machine's runtime may ask the desk's prompt")
}
}
// **On a real server, as composed** (issue 723): the vault's account asks the desk's prompt and an issuing tool
// and is answered, naming itself; the server refuses it any other call; and the controller is still answered on
// today's path. That only sealed values cross is the vault's test (mesh-catalog, mesh-vault intake_test.go).
func TestOnARealBusTheVaultsAccountIsAnsweredByTheDeskAndTheProviderAndNothingElse(t *testing.T) {
users, err := Users(vaultRecords())
if err != nil {
t.Fatal(err)
}
hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
if err != nil {
t.Fatal(err)
}
for i := range users {
users[i].PasswordHash = string(hash)
}
accounts, err := ComposeAccounts(users)
if err != nil {
t.Fatal(err)
}
conf := filepath.Join(t.TempDir(), "bus.conf")
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
t.Fatal(err)
}
opts, err := server.ProcessConfigFile(conf)
if err != nil {
t.Fatalf("the composed accounts do not parse: %v", err)
}
opts.NoLog, opts.NoSigs = true, true
s, err := server.NewServer(opts)
if err != nil {
t.Fatal(err)
}
go s.Start()
if !s.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
defer s.Shutdown()
// anchor's runtime serves the desk's prompt and plex's issuing tool, on every subject it is granted, and says
// which subject it heard.
holder, err := nats.Connect(s.ClientURL(), nats.UserInfo("anchor.node-tools", "pw"))
if err != nil {
t.Fatal(err)
}
defer holder.Close()
heard := make(chan string, 8)
answer := func(m *nats.Msg) {
heard <- m.Subject
_ = m.Respond([]byte(`{"result":{"sealed":"c2VhbGVkLXRvLXRoZS1jYWxs"}}`))
}
for _, subject := range []string{"mesh.seat.node-launcher.call.secret.anchor.*",
"mesh.seat.node-launcher.tool.secret.anchor", "mesh.mod.plex.call.plex_issued_credential.anchor.*",
"mesh.mod.other.call.tool.anchor.*", "mesh.seat.node-launcher.call.menu.anchor.*"} {
sub, err := holder.Subscribe(subject, answer)
if err != nil {
t.Fatal(err)
}
_ = holder.Flush()
if !sub.IsValid() {
t.Fatalf("the desk's runtime may not hear %s", subject)
}
}
refusals := make(chan error, 16)
vault, err := nats.Connect(s.ClientURL(), nats.UserInfo("novox.mesh-vault", "pw"),
nats.CustomInboxPrefix("_INBOX.novox.mesh-vault"),
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refusals <- err }))
if err != nil {
t.Fatalf("the vault's own account cannot connect: %v", err)
}
defer vault.Close()
token := CallerToken("novox.mesh-vault")
for _, subject := range []string{"mesh.seat.node-launcher.call.secret.anchor." + token,
"mesh.mod.plex.call.plex_issued_credential.anchor." + token} {
if _, err := vault.Request(subject, []byte(`{"seal_to":"a2V5LW9mLXRoZS1jYWxs"}`), 3*time.Second); err != nil {
t.Fatalf("the vault's call %s was not answered: %v", subject, err)
}
if got := <-heard; got != subject {
t.Fatalf("the holder heard %s, want %s", got, subject)
}
}
for _, subject := range []string{"mesh.seat.node-launcher.tool.secret.anchor",
"mesh.seat.node-launcher.call.secret.anchor.controller",
"mesh.seat.node-launcher.call.menu.anchor." + token,
"mesh.mod.other.call.tool.anchor." + token} {
if err := vault.Publish(subject, []byte(`{}`)); err != nil {
t.Fatal(err)
}
_ = vault.Flush()
select {
case err := <-refusals:
if !errors.Is(err, nats.ErrPermissionViolation) {
t.Errorf("%s: the server said %v, want a permissions violation", subject, err)
}
case <-time.After(3 * time.Second):
t.Errorf("the server did not refuse the vault %s", subject)
}
}
select {
case got := <-heard:
t.Errorf("a call the vault may not make reached the holder: %s", got)
case <-time.After(200 * time.Millisecond):
}
// The controller is still answered on today's path, for the modules filled the old way (until R1).
controller, err := nats.Connect(s.ClientURL(), nats.UserInfo("controller", "pw"),
nats.CustomInboxPrefix("_INBOX.controller"))
if err != nil {
t.Fatal(err)
}
defer controller.Close()
if _, err := controller.Request("mesh.seat.node-launcher.tool.secret.anchor", []byte(`{}`), 3*time.Second); err != nil {
t.Errorf("the controller's call to the desk on today's path: %v", err)
}
<-heard
}
+27
View File
@@ -77,6 +77,13 @@ func ownModule(p Principal, pattern string) bool {
return false
}
// ownProvisioner is a machine's provisioner client saying a report about its own machine: the third token of
// `prov.applied.<node>.…` is its machine (novox/hq ADR 0349 §4, design 57 P3).
func ownProvisioner(p Principal, pattern string) bool {
tokens := strings.Split(pattern, ".")
return p.Kind == KindProvisioner && len(tokens) > 3 && tokens[2] == p.Node
}
// kvOf is a bucket's write subjects.
func kvOf(bucket string) []string { return []string{"$KV." + bucket + ".>"} }
@@ -158,6 +165,26 @@ var WritersTable = []WriterRow{
{State: "a message a consumer gave up on", Writer: "controller", KeptIn: "the bus, the stream " + DeadLettersStream,
Others: "read, delivered again or dropped through the controller's dead-letters verb",
Subjects: []string{deadLetterPrefix + ">", againPrefix + ">"}, Writes: isController},
// The record mirrors (novox/hq ADR 0289 §9, design 54 §2): rows of this table, so a grant that lets
// anything else write them is refused at composition. Built so far: the assignments and each registered
// module's declared config (design 57 F3), and the list of predefined keys kept (design 57 K3).
{State: "the assignments mirror", Writer: "controller (lease holder), from its store, only on a change",
KeptIn: "key-value " + AssignmentsBucket, Others: "read through `reads` as mesh-controller." + MirrorAssignments,
Subjects: kvOf(AssignmentsBucket), Writes: isController},
{State: "the declared-config mirror", Writer: "controller (lease holder), from its store, only on a change",
KeptIn: "key-value " + DeclaredConfigBucket, Others: "read through `reads` as mesh-controller." + MirrorDeclaredConfig,
Subjects: kvOf(DeclaredConfigBucket), Writes: isController},
{State: "the predefined-keys mirror", Writer: "controller (lease holder), from its store, only on a change",
KeptIn: "key-value " + PredefinedBucket, Others: "read through `reads` as mesh-controller." + MirrorPredefined,
Subjects: kvOf(PredefinedBucket), Writes: isController},
// What a machine's provisioner client placed of a module's files (novox/hq ADR 0349 §4, design 57 P6): its own
// machine's word alone, which a walk's end and a gate wait for (ADR 0362, ADR 0349 §15). Kept newest per module
// in the stream ProvAppliedStream, and by the client in the provisioner server's `applied` bucket.
{State: "a module's files placed on a machine", Writer: "that machine's provisioner client",
KeptIn: "the bus, the stream " + ProvAppliedStream + ", newest per module; and key-value " +
ProvisionerAppliedBucket,
Others: "the controller reads it at a walk's end and a gate; a failure is that machine's failed apply",
Subjects: []string{"prov.applied.*.>"}, Writes: ownProvisioner},
}
// CheckWriters refuses a grant that lets a principal publish on a subject the writers table gives
+6
View File
@@ -34,6 +34,12 @@ var designRows = []string{
"the operator-channel's open messages",
"the facts snapshot",
"a message a consumer gave up on",
// The record mirrors, rows of to-be 45 §1's table by design 54 §2 (ADR 0289 §9).
"the assignments mirror",
"the declared-config mirror",
"the predefined-keys mirror",
// The provisioner client's word on what it placed (novox/hq ADR 0349 §4, design 57 P6).
"a module's files placed on a machine",
}
func TestTheWritersTableIsTheDesigns(t *testing.T) {
+64 -3
View File
@@ -108,6 +108,17 @@ type Result struct {
// recipe and the package it compiles; for any other, nothing is said and the whole of what the build
// sees stays its source, as before.
Sources []BuildSource
// Predefined is the list of predefined keys at the root of the repository at the commit built (novox/hq ADR
// 0348 §7), which the manifest was judged against; nil for a repository that holds none.
Predefined *Predefined
}
// Predefined is the list of predefined keys a build read: its text, canonical (catalogue.PredefinedList), and
// the generation of the commit built — how many commits it stands on, which orders two builds of one trunk.
type Predefined struct {
Keys []byte
Generation int
}
// BuildSource is the build source a build read in one repository (novox/hq ADR 0267).
@@ -265,14 +276,38 @@ func build(ctx context.Context, run Runner, publish Publisher,
"%s has no %s at %s, so there is nothing saying what it is: %w",
repository, ManifestName, describe(path), err)
}
manifest, err := catalogue.ParseManifest(raw)
// **The list of predefined keys at the commit built** (novox/hq ADR 0348 §7): predefined.json at the
// repository's root, read strictly; a list that is wrong fails the build, and the manifest is judged against
// the list of its own tree, never against one the mesh published from another commit.
predefined, err := catalogue.ReadPredefined(tree)
if err != nil {
say("predefined", "INVALID: %v", err)
return Result{}, err
}
manifest, err := catalogue.ParseManifestWith(raw, predefined)
if err != nil {
say("manifest", "INVALID: %v", err)
return Result{}, err
}
var listed *Predefined
if predefined != nil {
text, err := predefined.Canonical()
if err != nil {
return Result{}, err
}
listed = &Predefined{Keys: text, Generation: generationOf(ctx, run, tree, commit)}
say("predefined", "%d key(s) at %s, generation %d", len(predefined), short(commit), listed.Generation)
}
// What is judged only where a definition is written (novox/hq ADR 0326 §3, issue 574): a build is
// where a changed definition is first read, so it is refused here before anything is made of it.
if problems := catalogue.WrittenSettingsProblems(manifest); len(problems) > 0 {
// Judged with each template's text in its file, as written in the tree (novox/hq ADR 0349 §2): a template the
// module's directory lacks is refused here, before anything is built.
judged, err := catalogue.WithTemplatesFrom(manifest, within)
if err != nil {
say("manifest", "INVALID: %v", err)
return Result{}, err
}
if problems := catalogue.WrittenSettingsProblems(judged); len(problems) > 0 {
err := fmt.Errorf("%s", strings.Join(problems, "; "))
say("manifest", "INVALID: %v", err)
return Result{}, err
@@ -288,6 +323,11 @@ func build(ctx context.Context, run Runner, publish Publisher,
if src.tree, err = gitTree(ctx, run, tree, path); err != nil {
src.notPinned("its tree could not be named: " + err.Error())
}
// A module falling back to a predefined key was judged against the list, so a change to the list is a change
// to it: the planner rebuilds it, and its build publishes the list (ADR 0348 §7).
if predefined != nil && manifest.NamesPredefined() {
src.rootHas(catalogue.PredefinedFile)
}
// A build-time credential, written into the build context as .npmrc, but ONLY for a module that
// asks for it: a `package` artifact (which publishes), or an image whose Dockerfile COPYs .npmrc.
@@ -419,6 +459,11 @@ func build(ctx context.Context, run Runner, publish Publisher,
}
}
// Each template's text, read back from the archive the build made or took (novox/hq ADR 0349 §2).
if built, err = withTemplateTexts(manifest, within, built); err != nil {
say("templates", "FAILED: %v", err)
return Result{}, err
}
resolved, err := manifest.Resolve(built)
if err != nil {
say("resolve", "FAILED: %v", err)
@@ -448,7 +493,23 @@ func build(ctx context.Context, run Runner, publish Publisher,
return Result{Manifest: resolved, Commit: commit, Built: built,
Against: against(within, manifest, stoodOn), Read: readBy(manifest), Source: fingerprint,
Trunk: trunk, OnTrunk: onTrunk, Branches: branches, Sources: sources,
ReuseKey: key.Key, ReuseParts: key.Parts, NoKey: key.Why, Reused: reused, Why: builtWhy}, nil
ReuseKey: key.Key, ReuseParts: key.Parts, NoKey: key.Why, Reused: reused, Why: builtWhy,
Predefined: listed}, nil
}
// generationOf is how many commits the commit built stands on, itself included: of two commits on one trunk the
// later has more, so the controller keeps the list of the newer one whatever order it hears them in (ADR 0348
// §7). 0 when git could not say.
func generationOf(ctx context.Context, run Runner, tree, commit string) int {
out, err := run(ctx, tree, "git", "rev-list", "--count", commit)
if err != nil {
return 0
}
n, err := strconv.Atoi(strings.TrimSpace(out))
if err != nil || n < 0 {
return 0
}
return n
}
// takeACheckBuild is the walk's look for a check build of its key (novox/hq ADR 0350 §2, §3): the key taken by
+2
View File
@@ -83,6 +83,8 @@ func (r *recorded) run(_ context.Context, dir, name string, args ...string) (str
return r.tree + "\n", nil
case name == "git" && len(args) > 0 && args[0] == "rev-parse":
return "c0ffeec0ffeec0ffeec0ffeec0ffeec0ffeec0ff\n", nil
case name == "git" && len(args) > 1 && args[0] == "rev-list" && args[1] == "--count":
return "42\n", nil
}
return "", nil
}
+106
View File
@@ -0,0 +1,106 @@
package builder
import (
"context"
"slices"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// A build reads the list of predefined keys at the root of the repository at the commit it builds (novox/hq ADR
// 0348 §7, design 57 package K2, issue 661): it refuses a list that is wrong, judges the manifest against it, and
// carries it with the commit's generation for the controller to publish.
const theFirstKeys = `{
"font-size": {"type": "number", "description": "the size text is shown in, in points", "minimum": 6, "maximum": 72},
"agent-managed-settings": {"type": "object", "description": "the managed settings of every agent session, a JSON object", "trusted": true}
}`
// fallingBack is a module of the catalogue whose font size falls back to the predefined key named.
func fallingBack(key, kind string) string {
def := "10"
if kind == "string" {
def = `"10"`
}
return `{"module":"dunst","version":"1",
"build":{"artifacts":[{"name":"look","kind":"archive","from":"files"}]},
"config":{"font-size":{"type":"` + kind + `","description":"the notification's font size, in points",
"default":` + def + `,"predefined":"` + key + `"}},
"resources":[
{"id":"theme","type":"archive","path":"/opt/dunst","artifact":"look"},
{"id":"rc","type":"file","path":"/etc/xdg/dunst/dunstrc","content":"font = Inter ${config:font-size}\n"}]}`
}
func buildInTheCatalogue(t *testing.T, manifest string, list *string) (Result, error) {
t.Helper()
r := &recorded{contents: map[string]string{"modules/dunst/" + ManifestName: manifest,
"modules/dunst/files/theme.conf": "dark", "README.md": "the catalogue"}}
if list != nil {
r.contents[catalogue.PredefinedFile] = *list
}
return Build(context.Background(), onTrunk{recorded: r}.run, r, "https://forge.invalid/mesh-catalog.git",
"modules/dunst", "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil)
}
func TestABuildNamingAPredefinedKeyIsJudgedAgainstItsTreesListAndCarriesIt(t *testing.T) {
list := theFirstKeys
got, err := buildInTheCatalogue(t, fallingBack("font-size", "number"), &list)
if err != nil {
t.Fatalf("a module falling back to font-size did not build: %v", err)
}
if got.Predefined == nil || got.Predefined.Generation != 42 {
t.Fatalf("the list was not carried with its generation: %+v", got.Predefined)
}
carried, err := catalogue.ParsePredefined(got.Predefined.Keys)
if err != nil || len(carried) != 2 || carried["font-size"].Type != catalogue.ConfigNumber {
t.Fatalf("the list carried is %s (%v)", got.Predefined.Keys, err)
}
// A change to the list is a change to the module that was judged against it.
if len(got.Sources) == 0 || !slices.Contains(got.Sources[0].Paths, catalogue.PredefinedFile) {
t.Fatalf("its build source leaves the list out: %+v", got.Sources)
}
for _, c := range []struct{ manifest, says string }{
{fallingBack("font-scale", "number"), `"font-scale", which is not on the list`},
{fallingBack("font-size", "string"), "which is a number, and this value is a string"},
} {
if _, err := buildInTheCatalogue(t, c.manifest, &list); err == nil || !strings.Contains(err.Error(), c.says) {
t.Errorf("wanted the build refused saying %q, got %v", c.says, err)
}
}
}
func TestABuildRefusesAWrongList(t *testing.T) {
wrong := `{"font-size": {"type": "number", "description": "points", "unit": "pt"}}`
if _, err := buildInTheCatalogue(t, fallingBack("font-size", "number"), &wrong); err == nil ||
!strings.Contains(err.Error(), `unknown field "unit"`) {
t.Fatalf("a wrong list was built with: %v", err)
}
}
// A repository without the list builds as before, carrying none; a value naming a key there is refused.
func TestABuildWithoutAListCarriesNone(t *testing.T) {
plain := strings.Replace(fallingBack("font-size", "number"), `,"predefined":"font-size"`, "", 1)
got, err := buildInTheCatalogue(t, plain, nil)
if err != nil {
t.Fatal(err)
}
if got.Predefined != nil {
t.Fatalf("a list was carried from a tree holding none: %+v", got.Predefined)
}
if slices.Contains(got.Sources[0].Paths, catalogue.PredefinedFile) {
t.Fatalf("the build source names a list: %+v", got.Sources)
}
if _, err := buildInTheCatalogue(t, fallingBack("font-size", "number"), nil); err == nil ||
!strings.Contains(err.Error(), "is not present") {
t.Fatalf("a key named where no list is: %v", err)
}
// A list present and a module naming no key: carried, and not the module's build source.
list := theFirstKeys
got, err = buildInTheCatalogue(t, plain, &list)
if err != nil || got.Predefined == nil || slices.Contains(got.Sources[0].Paths, catalogue.PredefinedFile) {
t.Fatalf("a module naming no key: %+v %+v %v", got.Predefined, got.Sources, err)
}
}
+14 -2
View File
@@ -74,8 +74,14 @@ const reusedModule = `{"module":"meshboard","version":"1",
// aModuleAt is a repository holding the module under mod/, whose directory git names as tree, on its trunk.
func aModuleAt(t *testing.T, tree string) (*recorded, Runner, string) {
t.Helper()
r := &recorded{tree: tree, contents: map[string]string{"mod/" + ManifestName: reusedModule,
"mod/Dockerfile": "FROM scratch", "mod/files/theme.conf": "dark"}}
return aModuleWith(t, tree, map[string]string{"mod/" + ManifestName: reusedModule,
"mod/Dockerfile": "FROM scratch", "mod/files/theme.conf": "dark"})
}
// aModuleWith is aModuleAt with the repository's files given.
func aModuleWith(t *testing.T, tree string, contents map[string]string) (*recorded, Runner, string) {
t.Helper()
r := &recorded{tree: tree, contents: contents}
run := func(ctx context.Context, dir, name string, args ...string) (string, error) {
if name == "git" && len(args) > 0 && args[0] == "symbolic-ref" {
r.ran = append(r.ran, name+" "+strings.Join(args, " "))
@@ -101,6 +107,12 @@ var aPull = PullRef{Owner: "novox", Repo: "meshboard", Number: 7}
func aCheckBuilt(t *testing.T, scratch *aScratch, tree string) CheckBuilt {
t.Helper()
r, run, workspace := aModuleAt(t, tree)
return aCheckBuiltBy(t, scratch, r, run, workspace)
}
// aCheckBuiltBy is aCheckBuilt of the repository given.
func aCheckBuiltBy(t *testing.T, scratch *aScratch, r *recorded, run Runner, workspace string) CheckBuilt {
t.Helper()
spec := CheckSpec{ID: "build-1", Owner: aPull.Owner, Repo: aPull.Repo, Number: aPull.Number,
Builds: []CheckBuildSpec{{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod",
Ref: "main"}}}
+20 -1
View File
@@ -58,6 +58,9 @@ type sourceInputs struct {
// whole when an artifact's is not known (novox/hq ADR 0267).
own map[string]bool
ownIsAll bool
// root is the module's build source in its own repository outside its directory, relative to the
// repository's root: the list of predefined keys a manifest naming one was judged against (ADR 0348 §7).
root map[string]bool
// read is, per context (repository and ref), the build source read there; nil for one read whole.
read map[string]map[string]bool
readAs map[string]catalogue.ArtifactContext
@@ -86,6 +89,9 @@ func (s *sourceInputs) copy() *sourceInputs {
for k, v := range s.own {
c.own[k] = v
}
for k, v := range s.root {
c.root[k] = v
}
for k, v := range s.read {
if v == nil {
c.read[k] = nil
@@ -112,7 +118,7 @@ func (s *sourceInputs) contextOffHead() {
func newSourceInputs(module string) *sourceInputs {
return &sourceInputs{module: module, contexts: map[string]string{}, toolchains: map[string]string{},
own: map[string]bool{}, read: map[string]map[string]bool{}, readAs: map[string]catalogue.ArtifactContext{},
own: map[string]bool{}, root: map[string]bool{}, read: map[string]map[string]bool{}, readAs: map[string]catalogue.ArtifactContext{},
contextsAtHead: true}
}
@@ -126,6 +132,16 @@ func (s *sourceInputs) ownHas(entries ...string) {
}
}
// rootHas adds entries, relative to the repository's root, to the module's build source in its own repository.
func (s *sourceInputs) rootHas(entries ...string) {
if s == nil {
return
}
for _, e := range entries {
s.root[e] = true
}
}
// ownWhole says an artifact's build source in the module's own repository is not known: the module's
// whole directory is its source, as it was before.
func (s *sourceInputs) ownWhole() {
@@ -178,6 +194,9 @@ func (s *sourceInputs) buildSources() []BuildSource {
for e := range s.own {
own = append(own, withPrefix(s.prefix, e))
}
for e := range s.root {
own = append(own, e)
}
sort.Strings(own)
out = append(out, BuildSource{Paths: compactSorted(own)})
}
+95
View File
@@ -0,0 +1,95 @@
package builder
import (
"archive/tar"
"bytes"
"compress/gzip"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"path/filepath"
"github.com/novox/mesh-controller/internal/catalogue"
)
// withTemplateTexts is what was built with each template's text, for a module whose files name templates (novox/hq ADR
// 0349 §2): its templates archive — the one built from its templates directory, files/ unless its manifest names
// another (issue 731) — gains the text of every file in it, read back from the packed bytes, by its path inside the
// archive. Resolve reads them into the files' content.
//
// **Read from the archive, proven by its digest.** The module's templates directory is packed again, as the build packs
// it, and the bytes are read only when they hash to the digest the build made or took: a fresh build's own archive, or
// a check's build a walk copied by digest without building (novox/hq ADR 0350 §2). Packing is deterministic, so one
// source packs to one digest; a tree that packs to another is not the archive, and its text is refused.
func withTemplateTexts(manifest catalogue.Manifest, within string, built []catalogue.Built) ([]catalogue.Built, error) {
if !manifest.Templated() {
return built, nil
}
name := catalogue.TemplatesArtifact(manifest)
at := -1
for i, b := range built {
if b.Name == name {
at = i
}
}
if name == "" || at < 0 {
return nil, fmt.Errorf("%s names templates and built no archive of %s/ to read them from",
manifest.Module, manifest.TemplatesDir())
}
body, err := pack(filepath.Join(within, filepath.FromSlash(manifest.TemplatesDir())))
if err != nil {
return nil, fmt.Errorf("%s: packing %s/ to read its templates back failed: %w", manifest.Module,
manifest.TemplatesDir(), err)
}
sum := sha256.Sum256(body)
digest := "sha256:" + hex.EncodeToString(sum[:])
if built[at].Digest != digest {
return nil, fmt.Errorf("%s: %s/ packs to the digest %s, and the archive %q built or taken is %s: the "+
"templates are read only from the archive the mesh records", manifest.Module, manifest.TemplatesDir(),
digest, name, orNone(built[at].Digest))
}
texts, err := unpackTexts(body)
if err != nil {
return nil, fmt.Errorf("%s: reading the archive %q back failed: %w", manifest.Module, name, err)
}
out := append([]catalogue.Built(nil), built...)
out[at].Texts = texts
return out, nil
}
// unpackTexts is every file of a packed archive (pack), by its path inside it.
func unpackTexts(body []byte) (map[string]string, error) {
zipped, err := gzip.NewReader(bytes.NewReader(body))
if err != nil {
return nil, err
}
defer zipped.Close()
reader := tar.NewReader(zipped)
texts := map[string]string{}
for {
header, err := reader.Next()
if errors.Is(err, io.EOF) {
return texts, nil
}
if err != nil {
return nil, err
}
if header.Typeflag != tar.TypeReg {
continue
}
text, err := io.ReadAll(reader)
if err != nil {
return nil, err
}
texts[header.Name] = string(text)
}
}
func orNone(digest string) string {
if digest == "" {
return "no digest"
}
return digest
}
+181
View File
@@ -0,0 +1,181 @@
package builder
import (
"context"
"crypto/sha256"
"encoding/hex"
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// A module's files kept as template files in its repository (novox/hq ADR 0349 §2, design 57 package T1, issue
// 654): the build packs files/ into the module's templates archive and reads each template's text back from the
// packed bytes into the file's content, so the manifest the mesh holds is the one it would hold for the text inline.
const templatedModule = `{"module":"meshboard","version":"1",
"build":{"artifacts":[
{"name":"server","kind":"image","from":"Dockerfile"},
{"name":"files","kind":"archive","from":"files"}]},
"resources":[
{"id":"svc","type":"container","name":"meshboard","artifact":"server"},
{"id":"conf","type":"file","path":"/etc/meshboard/meshboard.conf","template":"files/meshboard.conf"},
{"id":"deeper","type":"file","path":"/etc/meshboard/deeper.conf","template":"files/a/deeper.conf"}]}`
var templatedFiles = map[string]string{
"Dockerfile": "FROM scratch", "files/meshboard.conf": "listen = 8080\n\"quoted\" é\n",
"files/a/deeper.conf": "deeper\n", "files/unused.conf": "packed, named by nothing",
}
func contentOf(t *testing.T, m catalogue.Manifest, id string) string {
t.Helper()
for _, r := range m.Resources {
if r["id"] == id {
if _, has := r[catalogue.TemplateField]; has {
t.Fatalf("%s still names its template in the built manifest: %v", id, r)
}
text, ok := r["content"].(string)
if !ok {
t.Fatalf("%s carries no content: %v", id, r)
}
return text
}
}
t.Fatalf("no resource %s in %v", id, m.Resources)
return ""
}
func TestABuildReadsEachTemplateBackFromItsArchive(t *testing.T) {
r, workspace := aRepository(t, templatedModule, templatedFiles)
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil,
Npmrc{}, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if text := contentOf(t, got.Manifest, "conf"); text != templatedFiles["files/meshboard.conf"] {
t.Fatalf("the template's text is %q, not the file's", text)
}
if text := contentOf(t, got.Manifest, "deeper"); text != templatedFiles["files/a/deeper.conf"] {
t.Fatalf("the template's text is %q, not the file's", text)
}
if r.archives["meshboard/files"] == "" {
t.Fatalf("the templates archive was not published: %v", r.archives)
}
if problem := catalogue.HeldTemplateProblem(got.Manifest); problem != "" {
t.Fatal(problem)
}
}
func TestABuildWhoseTemplateIsMissingFailsNamingIt(t *testing.T) {
files := map[string]string{}
for k, v := range templatedFiles {
files[k] = v
}
delete(files, "files/a/deeper.conf")
r, workspace := aRepository(t, templatedModule, files)
_, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil,
Npmrc{}, GitCredential{}, nil)
if err == nil || !strings.Contains(err.Error(), "files/a/deeper.conf") {
t.Fatalf("a build whose template is missing did not fail naming it: %v", err)
}
}
// A walk that takes its check's build copies the archive by digest and builds nothing: the templates are read back
// from the tree packed again, which is the archive only when it packs to the digest taken.
func TestAWalkTakingAChecksBuildReadsTheTemplatesBackToo(t *testing.T) {
withVersion(t, "builder-1")
contents := map[string]string{"mod/" + ManifestName: templatedModule}
for k, v := range templatedFiles {
contents["mod/"+k] = v
}
scratch := newScratch()
r, run, workspace := aModuleWith(t, "tree-one", contents)
checked := aCheckBuiltBy(t, scratch, r, run, workspace)
r, run, workspace = aModuleWith(t, "tree-one", contents)
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)},
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if got.Reused == nil || len(r.archives) > 0 {
t.Fatalf("the walk built again: reused %+v, why %q, published %v", got.Reused, got.Why, r.archives)
}
if text := contentOf(t, got.Manifest, "conf"); text != templatedFiles["files/meshboard.conf"] {
t.Fatalf("the template's text is %q, not the file's", text)
}
}
// The tree packed again must be the archive taken; text read from anything else is not the archive's.
func TestTemplateTextsAreRefusedWhenTheTreeDoesNotPackToTheDigestTaken(t *testing.T) {
m, err := catalogue.ParseManifest([]byte(templatedModule))
if err != nil {
t.Fatal(err)
}
within := writeTree(t, t.TempDir(), templatedFiles)
built := []catalogue.Built{{Name: "files", Kind: catalogue.ArtifactArchive, Digest: "sha256:" + strings.Repeat("0", 64)}}
if _, err := withTemplateTexts(m, within, built); err == nil || !strings.Contains(err.Error(), "digest") {
t.Fatalf("text was read from a tree that does not pack to the archive taken: %v", err)
}
}
// writeTree writes files under dir and answers dir.
func writeTree(t *testing.T, dir string, files map[string]string) string {
t.Helper()
for name, body := range files {
full := filepath.Join(dir, filepath.FromSlash(name))
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(full, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}
return dir
}
// A module whose files/ is an archive unpacked onto the machine keeps its templates in a directory it names (issue
// 731): the build reads each template back from the archive packed from that directory, and files/ is packed as it
// was, its templates nowhere in it.
const unpackingModule = `{"module":"meshboard","version":"1",
"templates":"templates",
"build":{"artifacts":[
{"name":"scripts","kind":"archive","from":"files"},
{"name":"templates","kind":"archive","from":"templates"}]},
"resources":[
{"id":"scripts","type":"archive","path":"/usr/local/lib/meshboard","artifact":"scripts"},
{"id":"conf","type":"file","path":"/etc/meshboard/meshboard.conf","template":"templates/meshboard.conf"}]}`
func TestABuildReadsEachTemplateBackFromTheTemplatesDirectoryItsManifestNames(t *testing.T) {
files := map[string]string{"files/bin/meshboard-hook": "#!/bin/sh\n", "templates/meshboard.conf": "listen = 8080\n"}
r, workspace := aRepository(t, unpackingModule, files)
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil,
Npmrc{}, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if text := contentOf(t, got.Manifest, "conf"); text != files["templates/meshboard.conf"] {
t.Fatalf("the template's text is %q, not the file's", text)
}
if got.Manifest.Templates != "" {
t.Fatalf("the built manifest still names its templates directory: %q", got.Manifest.Templates)
}
// files/ packs exactly as it would with no templates beside it.
alone, err := pack(writeTree(t, t.TempDir(), map[string]string{"bin/meshboard-hook": "#!/bin/sh\n"}))
if err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(alone)
packed := ""
for _, b := range got.Built {
if b.Name == "scripts" {
packed = b.Digest
}
}
if packed != "sha256:"+hex.EncodeToString(sum[:]) {
t.Fatalf("files/ packed to %q, not to what it packs to alone", packed)
}
}
+7 -7
View File
@@ -56,7 +56,7 @@ func anchorRendering(adopted bool) Rendering {
Generators: map[string]Generator{"overlay": hub{}},
Ports: map[string]map[int]int{"distribution": {5000: 5000}, "hello-web": {8080: 20001}},
Settings: SettingsBy{"distribution": {{From: "node anchor",
Values: map[string]any{ExposeSetting: map[string]any{"5000": FromEverywhere}}}}},
Placement: map[string]any{ExposeSetting: map[string]any{"5000": FromEverywhere}}}}},
Mesh: []string{"10.42.0.1"},
Foundation: []int{5671},
// What the machine reported faces outside, which every rule in the filter is written
@@ -310,14 +310,14 @@ func TestAGivenPortIsUsedEverywhereThePortIs(t *testing.T) {
func TestAGivenPortIsTheNodesAndReachesSomething(t *testing.T) {
store := anAdoptedAnchor().Modules[1]
node := func(v any) []Layer {
return []Layer{{From: "anchor", Values: map[string]any{PortsSetting: v}}}
return []Layer{{From: "anchor", Placement: map[string]any{PortsSetting: v}}}
}
if got, err := GivenPorts(store, node(map[string]any{"5432": float64(5433)})); err != nil ||
got[5432] != 5433 {
t.Fatalf("a node's given port was not read: %v %v", got, err)
}
if _, err := GivenPorts(store, []Layer{{From: MeshWideLayer,
Values: map[string]any{PortsSetting: map[string]any{"5432": float64(5433)}}}}); err == nil {
Placement: map[string]any{PortsSetting: map[string]any{"5432": float64(5433)}}}}); err == nil {
t.Fatal("a port given for the whole mesh was accepted")
}
if _, err := GivenPorts(store, node(map[string]any{"6000": float64(6001)})); err == nil {
@@ -409,7 +409,7 @@ func TestAGivenPortIsRefusedForAPortNoContainerPublishes(t *testing.T) {
onTheMachine := Manifest{Module: "daemon",
Listens: []Listening{{Port: 9000, From: FromMesh}}, Guards: []int{9000}}
layers := []Layer{{From: "node anchor",
Values: map[string]any{PortsSetting: map[string]any{"9000": float64(9100)}}}}
Placement: map[string]any{PortsSetting: map[string]any{"9000": float64(9100)}}}}
_, err := GivenPorts(onTheMachine, layers)
if err == nil || !strings.Contains(err.Error(), "does not publish") {
t.Fatalf("a port no container publishes was given: %v", err)
@@ -421,7 +421,7 @@ func TestAGivenPortIsRefusedForAPortNoContainerPublishes(t *testing.T) {
func TestAGuardedPortOpenedToEveryoneIsNotGuarded(t *testing.T) {
with := anchorRendering(true)
with.Settings["postgres"] = []Layer{{From: "node anchor",
Values: map[string]any{ExposeSetting: map[string]any{"5432": FromEverywhere}}}}
Placement: map[string]any{ExposeSetting: map[string]any{"5432": FromEverywhere}}}}
composed, err := anAdoptedAnchor().Compose(with)
if err != nil {
t.Fatal(err)
@@ -477,7 +477,7 @@ func portsAsThePlanWould(m Manifest, given map[int]int) map[int]int {
func TestAGivenPortNamesEitherEndOfWhatTheModulePublishes(t *testing.T) {
forge := aForge()
node := func(v any) []Layer {
return []Layer{{From: "node anchor", Values: map[string]any{PortsSetting: v}}}
return []Layer{{From: "node anchor", Placement: map[string]any{PortsSetting: v}}}
}
// The machine side — the number this module says it listens on, and the one the predecessor
@@ -573,7 +573,7 @@ func TestAGivenPortNamesEitherEndOfWhatTheModulePublishes(t *testing.T) {
func TestAGivenMachineSideReachesTheFilterTheOpeningAndTheConsumer(t *testing.T) {
forge := aForge()
given, err := GivenPorts(forge, []Layer{{From: "node anchor",
Values: map[string]any{PortsSetting: map[string]any{"2222": float64(222)}}}})
Placement: map[string]any{PortsSetting: map[string]any{"2222": float64(222)}}}})
if err != nil {
t.Fatalf("the machine side of a mapping cannot be given a port: %v", err)
}
+53
View File
@@ -0,0 +1,53 @@
package catalogue
import (
"fmt"
"regexp"
)
// What the machine's modules back up is placement (design 57 L3, issue 794). The holder of node-backup names it as
// `${backup:lines}` (what to run and which paths to keep) and `${backup:data}` (every item with its class and
// protection): the text `${contribution:node-backup:backup}` and `${contribution:node-backup:data}` render, from the
// same function, derived from each placed module's declared data (novox/hq ADR 0233). A handed holder has the text in
// its plan's placement for its client to fill; one not handed has the controller fill it where it stands.
// BackupNamespace is the placeholder kind of the machine's backup lines and data.
const BackupNamespace = "backup"
// ofBackup is where a holder places them: ${backup:<key>}. Loose inside the braces, so a misspelt key is found and
// refused rather than written out as text.
var ofBackup = regexp.MustCompile(`\$\{backup:([^}]*)\}`)
// backupKinds is the contribution kind each key of the backup placement renders.
var backupKinds = map[string]string{"lines": BackupKindBackup, "data": BackupKindData}
// backupPlacer is the machine's modules the backup placement is composed over, as the old contributions are.
type backupPlacer struct {
modules []Manifest
with Rendering
facts map[string]string
caps map[string]bool
unplaced *[]string
}
// placed is the text of one key for a holder, or what is wrong with naming it.
func (b backupPlacer) placed(holder Manifest, key string, item bool) (string, string) {
kind, known := backupKinds[key]
switch {
case item:
return "", "an item is kept per item; the machine's backup lines are placed only in a file"
case !known:
return "", fmt.Sprintf("the backup placement has `lines` and `data`, not %q", key)
case !holder.ClaimsSeat(BackupSeat):
return "", fmt.Sprintf("%s does not claim %s; the machine's backup lines are placed by its holder alone "+
"(novox/hq ADR 0212)", holder.Module, BackupSeat)
}
text, left, err := seatContributions(b.modules, holder, BackupSeat+":"+kind, b.with, b.facts, b.caps)
if b.unplaced != nil {
*b.unplaced = append(*b.unplaced, left...)
}
if err != nil {
return "", err.Error()
}
return text, ""
}
+126
View File
@@ -0,0 +1,126 @@
package catalogue
import (
"reflect"
"strings"
"testing"
)
// A machine's backup lines are placement (design 57 L3, issue 794): the holder of node-backup names them as
// `${backup:lines}` and `${backup:data}`, which a handed holder has its client fill from the plan and an unhanded
// one has the controller fill as it fills `${contribution:node-backup:backup}` and `${...:data}`: the same text.
const (
oldBackupText = "# lines\n${contribution:node-backup:backup}"
oldDataText = "# data\n${contribution:node-backup:data}"
placedBackupTxt = "# lines\n${backup:lines}"
placedDataTxt = "# data\n${backup:data}"
backupHolder = `{"module":"restic","version":"1","moved":true,"claims":[{"name":"node-backup","scope":"node"}],
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},
"resources":[{"id":"declared","type":"file","path":"/etc/backups.conf","template":"files/backups"},
{"id":"data-declared","type":"file","path":"/etc/data.conf","template":"files/data"}]}`
)
func dataModule(name string) string {
return `{"module":"` + name + `","version":"1","resources":[{"id":"d","type":"directory","mode":"0700"}],
"data":{"own":[{"id":"d","path":"${dir:d}","class":"irreplaceable","why":"the ` + name + ` data"}]}}`
}
// backupsOn is a machine of a holder whose files say these texts and the modules that declare data.
func backupsOn(t *testing.T, lines, data string, modules ...string) Resolution {
t.Helper()
all := []Manifest{held(t, backupHolder, builtWith("restic", map[string]string{"backups": lines, "data": data}))}
names := []string{"restic"}
for _, name := range modules {
all = append(all, mustParse(t, dataModule(name)))
names = append(names, name)
}
r, err := Resolve(shelf(all...), names, workstation(), World{})
if err != nil {
t.Fatal(err)
}
r.Account = "op"
return r
}
var backupModules = []string{"alpha", "beta", "gamma"}
// gammaLeftOut is a machine whose gamma does not place: its lines are kept, best effort.
func gammaLeftOut(with Rendering) Rendering {
with.NotPlaced = map[string]string{"gamma": "its placement setting does not read"}
return with
}
func TestTheBackupPlacementIsTheTextTheOldPlaceholdersRender(t *testing.T) {
// The old form is an unmoved holder: its placeholders are the controller's alone.
oldHolder := strings.Replace(backupHolder, `"moved":true,`, "", 1)
all := []Manifest{held(t, oldHolder, builtWith("restic", map[string]string{"backups": oldBackupText, "data": oldDataText}))}
for _, name := range backupModules {
all = append(all, mustParse(t, dataModule(name)))
}
old, err := Resolve(shelf(all...), append([]string{"restic"}, backupModules...), workstation(), World{})
if err != nil {
t.Fatal(err)
}
old.Account = "op"
wantLines, wantData := fileOf(t, old, gammaLeftOut(Rendering{}), "restic.declared"), fileOf(t, old, gammaLeftOut(Rendering{}), "restic.data-declared")
if !strings.Contains(wantLines, "# alpha\npath ") || !strings.Contains(wantLines, "# gamma\n") || !strings.Contains(wantData, "item d") {
t.Fatalf("the old form renders\n%s\n%s", wantLines, wantData)
}
moved := backupsOn(t, placedBackupTxt, placedDataTxt, backupModules...)
// Not handed: the controller fills the new placeholders with the old text.
if got := fileOf(t, moved, gammaLeftOut(Rendering{}), "restic.declared"); got != wantLines {
t.Errorf("not handed, the lines are\n%s\nwant\n%s", got, wantLines)
}
if got := fileOf(t, moved, gammaLeftOut(Rendering{}), "restic.data-declared"); got != wantData {
t.Errorf("not handed, the data is\n%s\nwant\n%s", got, wantData)
}
// Handed: the plan carries the same text as placement, and the files are the client's.
c, _ := composeWith(t, moved, gammaLeftOut(handingTo("store:5000")))
p := c.Plans["restic"].Placement
if got := "# lines\n" + p["backup:lines"]; got != wantLines {
t.Errorf("handed, the plan's lines are\n%s\nwant\n%s", got, wantLines)
}
if got := "# data\n" + p["backup:data"]; got != wantData {
t.Errorf("handed, the plan's data is\n%s\nwant\n%s", got, wantData)
}
if len(p) != 2 {
t.Errorf("the plan's placement is %v", p)
}
}
func TestAModuleAddedOrTakenAwayChangesTheBackupPlan(t *testing.T) {
plan := func(modules ...string) map[string]string {
c, _ := composeWith(t, backupsOn(t, placedBackupTxt, placedDataTxt, modules...), handingTo("store:5000"))
return c.Plans["restic"].Placement
}
two, three := plan("alpha", "beta"), plan("alpha", "beta", "gamma")
if reflect.DeepEqual(two, three) || strings.Contains(two["backup:lines"], "gamma") || !strings.Contains(three["backup:lines"], "# gamma\n") {
t.Errorf("two modules give %v, three give %v", two, three)
}
}
func TestAnUnknownBackupKeyIsRefused(t *testing.T) {
r := backupsOn(t, "${backup:rows}", placedDataTxt, backupModules...)
if _, err := r.Compose(handingTo("store:5000")); err == nil || !strings.Contains(err.Error(), "backup:rows") {
t.Errorf("handed, an unknown key is %v", err)
}
if _, err := r.Declaration(Rendering{}); err == nil || !strings.Contains(err.Error(), "backup:rows") {
t.Errorf("not handed, an unknown key is %v", err)
}
}
func fileOf(t *testing.T, r Resolution, with Rendering, id string) string {
t.Helper()
out, err := r.Declaration(with)
if err != nil {
t.Fatal(err)
}
for _, res := range out {
if res["id"] == id {
return res["content"].(string)
}
}
t.Fatalf("%s was not composed", id)
return ""
}
+2
View File
@@ -111,6 +111,7 @@ func boundNeed(n Needed, bound Chosen, catalogue map[string]Manifest, node Node,
}
held.From, held.At, held.Module = node.Name, at, m.Module
held.Serves, held.SharedOwn, held.Identity = servedByOne(m, n.Name), sharedByOne(m, n.Name), m.IdentityBoundOf(n.Name)
held.CredentialForm = m.CredentialFormOf(n.Name)
return held, true, ""
}
matching := bound.among(world.Offered[n.Name])
@@ -124,6 +125,7 @@ func boundNeed(n Needed, bound Chosen, catalogue map[string]Manifest, node Node,
identity := DefaultIdentityBound
if pm, known := catalogue[p.Module]; known {
held.SharedOwn, _ = pm.SharedCredentialOf(n.Name)
held.CredentialForm = pm.CredentialFormOf(n.Name)
identity = pm.IdentityBoundOf(n.Name)
}
held.From, held.At, held.Module, held.Serves, held.Identity = p.Node, p.At, p.Module, p.Serves, identity
+40
View File
@@ -35,6 +35,10 @@ type Built struct {
// Launchers are, for a bundle in an interpreted language, the executable the build wrote beside
// each entrypoint, by entrypoint (novox/hq ADR 0193): what the node's runtime starts to serve it.
Launchers map[string]string
// Texts are, for the archive a module packs its templates into (TemplatesArtifact), each file's text as the
// build read it back from the packed archive, by its path inside it (novox/hq ADR 0349 §2). Never carried on
// the wire or kept: Resolve reads them into the files' content, which is what the mesh holds.
Texts map[string]string `json:"-"`
}
// Resolve fills a manifest's resources in from what was built.
@@ -71,8 +75,44 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) {
m.Module, strings.Join(missing, " and "), oneOrOther(len(missing)))
}
// **Each template's text, as the build read it back from its templates archive** (novox/hq ADR 0349 §2): into
// the file's content, so what the mesh holds and composes is what the same text inline would be.
var handed *Handed
if m.Templated() {
name := TemplatesArtifact(m)
texts := by[name].Texts
if name == "" || texts == nil {
return Manifest{}, fmt.Errorf("%s names templates, and the build carried no text read back from the "+
"archive it packs %s/ into (%q): the builder that built it does not read templates",
m.Module, m.TemplatesDir(), name)
}
dir := m.TemplatesDir()
filled, err := fillTemplates(m.Module, dir, m.Resources, func(template string) (string, bool) {
t, ok := texts[strings.TrimPrefix(template, dir+"/")]
return t, ok
}, fmt.Sprintf("the archive %q", name))
if err != nil {
return Manifest{}, err
}
items, err := fillItemTemplates(m, m.Items, func(template string) (string, bool) {
t, ok := texts[strings.TrimPrefix(template, dir+"/")]
return t, ok
}, fmt.Sprintf("the archive %q", name))
if err != nil {
return Manifest{}, err
}
// And, for a moved module, where its templates are and which one each file and item is filled from
// (handed.go): read while its resources and items still name their templates.
handed = handedBy(m, by[name])
m.Resources = filled
m.Items = items
}
out := m
out.Handed = handed
out.Build = nil
// Read with the build, like Build: the manifest the mesh holds carries each file's text (issue 731).
out.Templates = ""
out.Resources = nil
// What the build compiled, kept on the resolved manifest (novox/hq ADR 0175): a tools bundle is
// named by no resource of the module's own — the node's runtime loads it — so this is the only
+12 -3
View File
@@ -51,6 +51,15 @@ func stepCA() Manifest {
}
}
// stepCAWithItsRoot is the authority whose root the operator supplies: the served key asks for it as
// `${setting:root}` (ADR 0155), the only way a setting reaches what a provider serves since a layer
// stopped replacing a served key (novox/hq issue 696; before it, the empty `root` was replaced whole).
func stepCAWithItsRoot() Manifest {
m := stepCA()
m.Serves["acme-ca"]["root"] = "${setting:root}"
return m
}
// routeProxy issues from that authority, so it must be told the root to verify it with.
func routeProxy() Manifest {
return Manifest{
@@ -78,7 +87,7 @@ func routeProxy() Manifest {
// The consequence is silent and total: route-proxy wrote an empty CA bundle, fell back to the
// system trust store, could not verify the internal authority, and no certificate was ever issued.
func TestACoLocatedProvidersServedValuesReachItsConsumer(t *testing.T) {
r, err := Resolve(shelf(stepCA(), routeProxy()),
r, err := Resolve(shelf(stepCAWithItsRoot(), routeProxy()),
[]string{"step-ca", "route-proxy"}, reachable(), World{})
if err != nil {
t.Fatal(err)
@@ -106,7 +115,7 @@ func TestACoLocatedProvidersServedValuesReachItsConsumer(t *testing.T) {
// And the binding file says the same thing, for a consumer that reads the binding rather than a
// substituted placeholder. The two are one fact and must not be able to disagree.
func TestACoLocatedConsumersBindingCarriesTheSettledValues(t *testing.T) {
r, err := Resolve(shelf(stepCA(), routeProxy()),
r, err := Resolve(shelf(stepCAWithItsRoot(), routeProxy()),
[]string{"step-ca", "route-proxy"}, reachable(), World{})
if err != nil {
t.Fatal(err)
@@ -152,7 +161,7 @@ func TestACoLocatedProviderIsSettledWhicheverWayItWasPulledIn(t *testing.T) {
{"step-ca", "route-proxy"},
{"route-proxy", "step-ca"},
} {
r, err := Resolve(shelf(stepCA(), routeProxy()), assigned, reachable(), World{})
r, err := Resolve(shelf(stepCAWithItsRoot(), routeProxy()), assigned, reachable(), World{})
if err != nil {
t.Fatal(err)
}
+635
View File
@@ -0,0 +1,635 @@
package catalogue
import (
"fmt"
"path"
"regexp"
"slices"
"sort"
"strconv"
"strings"
)
// Collections (novox/hq ADR 0349 clause 13, design 57 L1, issue 684): a provision with many providers.
//
// **A collector requires a collection; a contributor provides an item from a template.** A module taking additions
// from others names a collection in a file of its own, `${collected:<collection>}`, and declares it under
// `collections`, with the comment its tool writes a line in. Each contributor ships its item as a template of its
// own repository and declares it under `items`: `{"collection": …, "template": "files/<name>"}`. Its build packs the
// template with the module's others, and the mesh holds its text as it holds a file's (template.go).
//
// **The node's provisioner client fills both.** An item is filled on the node by the client of the node its
// contributor is placed on, from the same owners as a file, and recorded in mesh-provisioner_collect; a collector's
// file is filled by the same client from every item of its collection recorded for that node, so an item goes when
// its contributor leaves. So a module contributing an item is a moved module (handed.go), and the controller renders
// no item: the plan of a contributor names its items and their templates, and the plan of a collector names each
// collection it reads and its comment. Neither carries a contributor's text.
//
// **The three old namespaces read as collections.** `${shell:<shell>:<slot>}`, `${environment:<format>}` and
// `${contribution:<seat>:<kind>}` name, in a handed collector's file, the collection of the same name —
// `shell:zsh:normal` — with the comment the controller wrote before each module's piece. An item is contributed to
// one by that name, by a module the old form would have let contribute it. Each file is filled one way, never
// both (ADR 0349 clause 17): a handed collector is filled from items alone and an unhanded one from the old forms
// alone, and a piece of the other form on the same machine is said as not placed, never dropped quietly.
// Item is one item a module contributes to a collection: its template in the module's repository, or, in the
// manifest the mesh holds, the template's text, read back from the build's templates archive like a file's.
//
// **Attributes, a capability and a name** (design 57 L3, issue 777): an item may carry `attributes` (string keys and
// values; `order`, a whole number 0–99 defaulting to 50, orders the items of one placeholder) and `if-capability`,
// the capability a machine must report for the item to be placed on it. A module may give several items to one
// collection, each then with a `name` unique within the module and the collection. A collector takes only the items
// whose attributes match, `${collected:<collection>:<k>=<v>,…}`, and all of them without the selector.
type Item struct {
Collection string `json:"collection"`
Name string `json:"name,omitempty"`
Template string `json:"template,omitempty"`
Content string `json:"content,omitempty"`
Attributes map[string]string `json:"attributes,omitempty"`
IfCapability string `json:"if-capability,omitempty"`
}
// ItemKey is an item's key among its module's: the collection, and `/<name>` for a named item.
func ItemKey(collection, name string) string {
if name == "" {
return collection
}
return collection + "/" + name
}
func (it Item) key() string { return ItemKey(it.Collection, it.Name) }
// order is the item's place among the items of one placeholder: its `order` attribute, or OrderDefault.
func (it Item) order() int {
if n, err := strconv.Atoi(it.Attributes[OrderAttribute]); err == nil && n >= 0 && n <= 99 {
return n
}
return OrderDefault
}
// OrderAttribute is the attribute that orders items.
const OrderAttribute = "order"
// matches is whether the item's attributes carry every pair of the selector.
func (it Item) matches(where map[string]string) bool {
for k, v := range where {
if got, has := it.Attributes[k]; !has || got != v {
return false
}
}
return true
}
// itemsInOrder is the module's items by order, then name; the order the old form placed a module's pieces in.
func itemsInOrder(items []Item) []Item {
out := append([]Item(nil), items...)
sort.SliceStable(out, func(a, b int) bool {
if oa, ob := out[a].order(), out[b].order(); oa != ob {
return oa < ob
}
return out[a].Name < out[b].Name
})
return out
}
// Collection is one collection a module collects, and the comment its tool writes a line in: each contributor's item
// is introduced by `<comment> <contributor>`, and by nothing without one.
type Collection struct {
Name string `json:"name"`
Comment string `json:"comment,omitempty"`
}
// CollectedNamespace is a collection's placeholder namespace, `${collected:<collection>}`.
const CollectedNamespace = "collected"
// EnvironmentCollection is the one collection of the session environment (design 57 L3, issue 795): its items are
// typed by attributes, a variable (`kind=variable`, `name`) or a PATH entry (`kind=path`, `at=start|end`), and the
// template's text is the value or the entry. A collector names the rendering, `${collected:environment:posix}` or
// `${collected:environment:systemd}`, which is not a selector.
const EnvironmentCollection = "environment"
// oldCollectionNamespaces are the namespaces read as collections of their own name.
var oldCollectionNamespaces = []string{"shell", "environment", "contribution"}
var (
// ofCollected is where a collector places a collection. Loose inside the braces, so a misspelt name is refused.
ofCollected = regexp.MustCompile(`\$\{collected:([^}]*)\}`)
// collectionName is words joined by colons, so an old namespace's placeholder names its collection as it reads.
collectionName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*(:[a-z0-9][a-z0-9-]*)*$`)
// itemName, attributeKey and attributeValue are the words an item's name and attributes are made of; none holds
// a colon, a comma or an equals sign, which tell a selector from a collection's name.
itemName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
attributeKey = regexp.MustCompile(`^[a-z][a-z0-9-]*$`)
attributeValue = regexp.MustCompile(`^[a-z0-9][a-z0-9_.-]*$`)
)
// splitSelector is a placeholder's name without its selector, and the selector: `widgets:bar=top,place=left` is
// `widgets` and `bar=top,place=left`. A collection's name holds no `=`, so the selector begins at the last colon
// before the first one.
func splitSelector(name string) (collection, selector string) {
eq := strings.Index(name, "=")
if eq < 0 {
return name, ""
}
colon := strings.LastIndex(name[:eq], ":")
if colon < 0 {
return name, ""
}
return name[:colon], name[colon+1:]
}
// selectorOf is a selector's pairs, or why it is none.
func selectorOf(selector string) (map[string]string, error) {
out := map[string]string{}
if selector == "" {
return out, nil
}
for _, pair := range strings.Split(selector, ",") {
k, v, ok := strings.Cut(pair, "=")
switch {
case !ok || !attributeKey.MatchString(k) || !attributeValue.MatchString(v):
return nil, fmt.Errorf("%q in %q is not <attribute>=<value>", pair, selector)
case out[k] != "":
return nil, fmt.Errorf("%q names %s twice", selector, k)
}
out[k] = v
}
return out, nil
}
// collectionOfPlaceholder is the collection a placeholder's namespace and key name, and whether they name one.
func collectionOfPlaceholder(namespace, key string) (string, bool) {
key, _ = splitSelector(key)
switch {
case namespace == CollectedNamespace && strings.HasPrefix(key, EnvironmentCollection+":"):
// The environment is one collection, and the rest of the name picks the rendering (issue 795).
return EnvironmentCollection, true
case namespace == CollectedNamespace:
return key, true
case namespace == "environment":
return EnvironmentCollection, true
case slices.Contains(oldCollectionNamespaces, namespace):
return namespace + ":" + key, true
}
return "", false
}
// oldCollection is an old namespace's collection by its name: its comment, or why no item may be contributed to it.
// ok is false for a name that is not an old namespace's.
//
// forItem is whether an item is to be contributed to it: an item may be contributed to a kind received as data,
// which a collector may place as a collection only where its shape says its pieces are text (design 57 L3).
func oldCollection(name string, forItem bool) (comment string, old bool, wrong error) {
namespace, key, _ := strings.Cut(name, ":")
if !slices.Contains(oldCollectionNamespaces, namespace) {
return "", false, nil
}
switch namespace {
case "shell":
target, slot, two := strings.Cut(key, ":")
if !two || !oneOf(contributionTargets(), target) || !oneOf(knownSlots, slot) {
return "", true, fmt.Errorf("%s is no slot: shell code is shell:<shell>:<slot>, the shell one of %s, "+
"the session's file one of %s or the power moment one of %s, and the slot one of %s", name,
strings.Join(knownShells, ", "), strings.Join(sessionFiles, ", "), strings.Join(powerMoments, ", "),
strings.Join(knownSlots, ", "))
}
// shellCode's line before each module's code.
return "#", true, nil
case "environment":
if key == "" {
// The collection itself, which an item is contributed to; a placeholder names a rendering.
return "#", true, nil
}
if key != EnvironmentPOSIX && key != EnvironmentSystemd {
return "", true, fmt.Errorf("%s is no format: the environment is environment:%s or environment:%s",
name, EnvironmentPOSIX, EnvironmentSystemd)
}
return "#", true, nil
}
seat, kind, _ := strings.Cut(key, ":")
s, r, ok := receivable(seat, kind)
switch {
case !ok && s.Name == "":
return "", true, fmt.Errorf("%s names the seat %q, which the mesh does not define", name, seat)
case !ok:
return "", true, fmt.Errorf("%s: %s; %s", name, s.Name, kindsOf(s))
case r.Shape != nil && (forItem || collectableAsText(r)):
return r.Comment, true, nil
case r.Shape != nil:
return "", true, fmt.Errorf("%s is received by %s as data in its own shape, which no item of text is "+
"(novox/hq ADR 0255)", name, s.Name)
case r.Dirs:
return "", true, fmt.Errorf("%s names its contributors' directories, which the holder reads as paths "+
"the controller fills; it is not yet a collection (design 57 L1)", name)
case s.Name != canonicalSeat(seat):
return "", true, fmt.Errorf("%s names the seat by another name; a collection names it %s", name, s.Name)
}
return r.Comment, true, nil
}
// placerOfOld is the seat whose holder alone may collect an old namespace's collection (ADR 0203, 0204, 0208,
// 0211, 0212): the same seat that places it the old way.
func placerOfOld(name string) string {
namespace, key, _ := strings.Cut(name, ":")
switch namespace {
case "shell":
target, _, _ := strings.Cut(key, ":")
return placerOf(target)
case "environment":
return EnvironmentSeat
}
seat, _, _ := strings.Cut(key, ":")
return canonicalSeat(seat)
}
// collectionProblems is what is wrong with a module's collections and items, from the manifest alone.
func collectionProblems(m Manifest) []string {
var problems []string
declared := map[string]bool{}
for i, c := range m.Collections {
switch {
case !collectionName.MatchString(c.Name):
problems = append(problems, fmt.Sprintf("%s's collection %d is %q, which is not a collection's name: "+
"words of lower-case letters, digits and hyphens, joined by colons", m.Module, i+1, c.Name))
case declared[c.Name]:
problems = append(problems, fmt.Sprintf("%s collects %s twice", m.Module, c.Name))
}
if _, old, _ := oldCollection(c.Name, false); old {
problems = append(problems, fmt.Sprintf("%s collects %s, which is read from its placeholder: a module "+
"places it as ${%s} where it holds the seat that places it, and declares nothing", m.Module, c.Name,
c.Name))
}
if strings.ContainsAny(c.Comment, "\n\r") {
problems = append(problems, fmt.Sprintf("%s's collection %s has a comment holding a line break",
m.Module, c.Name))
}
declared[c.Name] = true
}
// A collection declared and placed nowhere is a word read by nothing; judged where every file's text is here.
if !m.Templated() {
named := map[string]bool{}
for _, r := range m.Resources {
if text, ok := r["content"].(string); ok {
for _, f := range ofCollected.FindAllStringSubmatch(text, -1) {
base, _ := splitSelector(f[1])
named[base] = true
}
}
}
for _, c := range m.Collections {
if !named[c.Name] && collectionName.MatchString(c.Name) {
problems = append(problems, fmt.Sprintf("%s collects %s and no file of its places it as "+
"${collected:%s}", m.Module, c.Name, c.Name))
}
}
}
contributed := map[string]bool{}
for i, it := range m.Items {
where := fmt.Sprintf("%s's item %d", m.Module, i+1)
if !collectionName.MatchString(it.Collection) {
problems = append(problems, fmt.Sprintf("%s is to %q, which is not a collection's name", where,
it.Collection))
} else if _, old, wrong := oldCollection(it.Collection, true); old && wrong != nil {
problems = append(problems, fmt.Sprintf("%s: %v", where, wrong))
}
switch {
case it.Name != "" && !itemName.MatchString(it.Name):
problems = append(problems, fmt.Sprintf("%s: the name %q is not a name: lower-case letters, digits and "+
"hyphens", where, it.Name))
case contributed[it.key()]:
problems = append(problems, fmt.Sprintf("%s contributes to %s twice; a module contributes one item to "+
"a collection, or several each with a name of its own", m.Module, it.key()))
case it.Name == "" && slices.ContainsFunc(m.Items, func(o Item) bool {
return o.Collection == it.Collection && o.Name != ""
}):
problems = append(problems, fmt.Sprintf("%s contributes several items to %s and %s has no name; each "+
"of several items has a name of its own", m.Module, it.Collection, where))
}
contributed[it.key()] = true
problems = append(problems, itemAttributeProblems(m.Module, where, it)...)
if it.Collection == EnvironmentCollection {
problems = append(problems, environmentItemProblems(where, it)...)
}
switch {
case it.Template == "" && it.Content == "":
problems = append(problems, fmt.Sprintf("%s names no template: an item is a template of the module's "+
"own, \"template\": \"%s/<name>\" (novox/hq ADR 0349 §13)", where, m.TemplatesDir()))
case it.Template != "" && it.Content != "":
problems = append(problems, fmt.Sprintf("%s names its template and says its content too", where))
case it.Template != "":
if wrong := itemTemplateProblem(m, it.Template); wrong != "" {
problems = append(problems, fmt.Sprintf("%s: %s", where, wrong))
}
// **Filled by the provisioner's client alone**: the controller renders no item.
if !m.Moved {
problems = append(problems, fmt.Sprintf("%s: an item is filled by the provisioner's client of the "+
"node its contributor is placed on, so a module contributing one is marked moved "+
"(\"moved\": true; novox/hq ADR 0349 §13, §17)", where))
}
}
if it.Content != "" {
problems = append(problems, itemTextProblems(m.Module, it)...)
}
}
return problems
}
// itemAttributeProblems is what is wrong with an item's attributes and capability. The attributes of an item of a kind
// received as data are the fields its shape places by (and `order`), each with a value the shape takes.
func itemAttributeProblems(module, where string, it Item) []string {
var problems []string
for _, k := range sortedKeys(it.Attributes) {
v := it.Attributes[k]
switch {
case !attributeKey.MatchString(k):
problems = append(problems, fmt.Sprintf("%s: the attribute %q is not a name: lower-case letters, digits "+
"and hyphens", where, k))
continue
case !attributeFits(it.Collection, k, v):
problems = append(problems, fmt.Sprintf("%s: %s=%q is not a value: lower-case letters, digits, hyphens, "+
"dots and underscores (an environment variable's name, letters, digits and underscores)", where, k, v))
continue
}
if n, err := strconv.Atoi(v); k == OrderAttribute && (err != nil || n < 0 || n > 99) {
problems = append(problems, fmt.Sprintf("%s: order=%q is not a whole number from 0 to 99", where, v))
}
}
if namespace, key, _ := strings.Cut(it.Collection, ":"); namespace == "contribution" {
seat, kind, _ := strings.Cut(key, ":")
if _, r, ok := receivable(seat, kind); ok && r.Shape != nil {
for _, k := range sortedKeys(it.Attributes) {
var place *PlaceField
for i := range r.Shape.Places {
if r.Shape.Places[i].Field == k {
place = &r.Shape.Places[i]
}
}
switch {
case k == OrderAttribute:
case place == nil:
problems = append(problems, fmt.Sprintf("%s: %s is placed by %s, and %q is none of them", where,
it.Collection, placeFields(r), k))
case !oneOf(place.Values, it.Attributes[k]):
problems = append(problems, fmt.Sprintf("%s: %s=%s; %s is one of %s", where, k, it.Attributes[k], k,
strings.Join(place.Values, ", ")))
}
}
}
}
switch {
case it.IfCapability == "":
case !capabilityName.MatchString(it.IfCapability):
problems = append(problems, fmt.Sprintf("%s: if-capability %q is not a capability's name", where, it.IfCapability))
case !oneOf(KnownCapabilities, it.IfCapability):
problems = append(problems, fmt.Sprintf("%s: if-capability %q is one no machine reports; the node-engine "+
"detects %s", where, it.IfCapability, strings.Join(KnownCapabilities, ", ")))
}
return problems
}
// AttributeFits is whether an item's attribute is of the shape its collection allows: words of lower case, except
// the environment's `name`, which is a variable's name (design 57 L3, issue 795). The same rule is mesh-catalog's
// provisioner client's, and the inventory's.
func attributeFits(collection, k, v string) bool {
if collection == EnvironmentCollection && k == "name" {
return variableName.MatchString(v)
}
return attributeKey.MatchString(k) && attributeValue.MatchString(v)
}
// environmentItemProblems is what is wrong with an item of the environment collection: typed by `kind`, a variable
// with its `name`, a PATH entry `at` the start or the end, and no attribute besides `order`.
func environmentItemProblems(where string, it Item) []string {
var problems []string
for _, k := range sortedKeys(it.Attributes) {
if !slices.Contains([]string{"kind", "name", "at", OrderAttribute}, k) {
problems = append(problems, fmt.Sprintf("%s: %s is typed by kind, name and at, and %q is none of them",
where, it.Collection, k))
}
}
switch kind := it.Attributes["kind"]; kind {
case "variable":
if it.Attributes["name"] == "" || it.Attributes["at"] != "" {
problems = append(problems, fmt.Sprintf("%s: a variable names itself with name=<NAME> and has no at", where))
} else if it.Attributes["name"] == "PATH" {
problems = append(problems, fmt.Sprintf("%s: PATH is not a variable; a PATH entry is kind=path", where))
}
case "path":
if it.Attributes["name"] != "" || (it.Attributes["at"] != PathAtStart && it.Attributes["at"] != PathAtEnd) {
problems = append(problems, fmt.Sprintf("%s: a PATH entry is at=%s or at=%s and has no name", where,
PathAtStart, PathAtEnd))
}
default:
problems = append(problems, fmt.Sprintf("%s: an item of %s is kind=variable or kind=path, not %q", where,
it.Collection, kind))
}
return problems
}
// itemTemplateProblem is what is wrong with an item's template path, or "".
func itemTemplateProblem(m Manifest, name string) string {
dir := m.TemplatesDir()
if strings.HasPrefix(name, "/") || path.Clean(name) != name || !strings.HasPrefix(name, dir+"/") ||
strings.Contains(name, "\\") {
return fmt.Sprintf("the template %q is not a file under %s/ of its own repository: a template is %q, a "+
"plain relative path, and the build packs only %s/", name, dir, dir+"/<name>", dir)
}
return ""
}
// itemTextProblems is what an item's text may not name: a secret or a bound value, since an item is kept in a
// bucket and no secret is (ADR 0349 clause 7), and a collection, since an item is not a collector. And every
// placeholder no pass fills, as in a file's content.
func itemTextProblems(module string, it Item) []string {
var problems []string
for _, f := range templatePlaceholder.FindAllStringSubmatch(it.Content, -1) {
namespace, key := f[1], f[2]
switch _, collection := collectionOfPlaceholder(namespace, key); {
case namespace == "secret" || namespace == "bound":
problems = append(problems, fmt.Sprintf("%s's item of %s names %s: an item is kept in a bucket, where "+
"no secret is (novox/hq ADR 0349 §7)", module, it.Collection, f[0]))
case collection:
problems = append(problems, fmt.Sprintf("%s's item of %s names %s: an item is not a collector",
module, it.Collection, f[0]))
}
}
return append(problems, unconsumedPlaceholders(module, map[string]any{"id": "item:" + it.key(),
"type": "file", "content": it.Content})...)
}
// isOld is whether a collection's name is an old namespace's.
func isOld(name string) bool {
_, old, _ := oldCollection(name, true)
return old
}
// collectedPlaceholderProblems is what is wrong with one resource's ${collected:…}: placed only in a file's content,
// and naming a collection the module declares it collects.
func collectedPlaceholderProblems(m Manifest, r map[string]any) []string {
var problems []string
for _, field := range sortedKeys(r) {
v, ok := r[field].(string)
if !ok {
continue
}
for _, f := range ofCollected.FindAllStringSubmatch(v, -1) {
base, selector := splitSelector(f[1])
if _, err := selectorOf(selector); err != nil {
problems = append(problems, fmt.Sprintf("%s's resource %v names %s: %v", m.Module, r["id"], f[0], err))
continue
}
switch {
case field != "content":
problems = append(problems, fmt.Sprintf("%s's resource %v names %s in its %s; a collection is placed "+
"only in a file's content", m.Module, r["id"], f[0], field))
case isOld(base):
if _, _, wrong := oldCollection(base, false); wrong != nil {
problems = append(problems, fmt.Sprintf("%s's resource %v names %s: %v", m.Module, r["id"], f[0], wrong))
}
case !slices.ContainsFunc(m.Collections, func(c Collection) bool { return c.Name == base }):
problems = append(problems, fmt.Sprintf("%s's resource %v names %s, and %s does not collect %q: "+
"declare it under collections (novox/hq ADR 0349 §13)", m.Module, r["id"], f[0], m.Module, base))
}
}
}
return problems
}
// collectionComment is the comment of a collection a module's file names: declared, or the old namespace's.
func collectionComment(m Manifest, name string) (string, error) {
if comment, old, wrong := oldCollection(name, false); old {
return comment, wrong
}
for _, c := range m.Collections {
if c.Name == name {
return c.Comment, nil
}
}
return "", fmt.Errorf("%s names ${collected:%s} and does not collect it", m.Module, name)
}
// itemsLeftOut says each item of a collection a file is filled with the old way that the controller could not render
// there (heldItemText): the collector is not handed to the provisioner on this machine (ADR 0349 clause 17). Said,
// never dropped quietly.
func itemsLeftOut(modules []Manifest, holder Manifest, collection string, facts map[string]string) []string {
return itemsLeftOutFor(modules, holder, collection, nil, facts, nil)
}
// itemsLeftOutFor is itemsLeftOut for a placeholder that selects by attributes (where), on a machine with these
// capabilities: an item the selector does not take, or whose capability the machine lacks, is not left out but not
// wanted here.
func itemsLeftOutFor(modules []Manifest, holder Manifest, collection string, where map[string]string,
facts map[string]string, caps map[string]bool) []string {
var out []string
for _, m := range inModuleOrder(modules) {
for _, it := range itemsInOrder(m.Items) {
if it.Collection != collection || !it.matches(where) || (it.IfCapability != "" && !caps[it.IfCapability]) {
continue
}
if _, rendered := heldText(it, facts); rendered && renderedOldWay(collection) {
continue
}
out = append(out, fmt.Sprintf("%s's item of %s is not placed: %s is filled the old way on this "+
"machine, from contributions, and an item only by the provisioner's client (design 57 L1)",
m.Module, it.key(), holder.Module))
}
}
return out
}
// oldFormsLeftOut says each piece of an old form a handed collector's collections would have held: the collector is
// filled from items alone, and the piece's module contributes none (ADR 0349 clause 17).
func oldFormsLeftOut(modules []Manifest, holder Manifest, collections []PlanCollection) []string {
var out []string
for _, c := range collections {
namespace, key, _ := strings.Cut(c.Name, ":")
for _, m := range inModuleOrder(modules) {
old := false
switch namespace {
case "shell":
target, slot, _ := strings.Cut(key, ":")
old = slices.ContainsFunc(m.Shell, func(s ShellCode) bool { return s.For == target && s.Slot == slot })
case "environment":
old = m.Environment != nil && (len(m.Environment.Variables) > 0 || len(m.Environment.Path) > 0)
case "contribution":
seat, kind, _ := strings.Cut(key, ":")
old = slices.ContainsFunc(m.allContributions(), func(s SeatContribution) bool {
return canonicalSeat(s.Seat) == canonicalSeat(seat) && s.Kind == kind
})
}
if old {
out = append(out, fmt.Sprintf("%s's %s is not placed: %s is filled by the provisioner's client on "+
"this machine, from items, and %s contributes it the old way (design 57 L1)", m.Module, c.Name,
holder.Module, m.Module))
}
}
}
return out
}
// heldItemText is a moved contributor's items of an old namespace's collection as the controller renders them into a
// collector not handed on this machine (design 57 L2, issue 681): the text the mesh holds, read back from the
// contributor's templates archive, with the machine's facts filled, each ending its line. As a moved module's file
// keeps its text for a machine that cannot take it handed (handed.go), an item keeps its place in a collector filled
// the old way, so the file stays what its old form made it. Only `shell:` and `contribution:` collections, which the
// controller renders as text; false for a module with no such item, or an item whose text the mesh does not hold or
// which names anything but a machine fact this machine has, which is then said as not placed.
func heldItemText(m Manifest, collection string, facts map[string]string) (string, bool) {
items := heldItems(m, collection, nil, facts, nil)
var b strings.Builder
for _, h := range items {
b.WriteString(h.text)
}
return b.String(), len(items) > 0
}
// renderedOldWay is whether the controller renders a collection's items into a collector not handed.
func renderedOldWay(collection string) bool {
namespace, _, _ := strings.Cut(collection, ":")
return namespace == "shell" || namespace == "contribution" || namespace == EnvironmentCollection
}
// heldItem is one item of a module rendered the old way.
type heldItem struct {
item Item
text string
}
// heldItems is a module's items of the collection that the selector takes and the machine's capabilities allow, by
// order then name, each rendered as heldText does; the ones it cannot render are left out (itemsLeftOutFor says so).
func heldItems(m Manifest, collection string, where map[string]string, facts map[string]string,
caps map[string]bool) []heldItem {
if !renderedOldWay(collection) {
return nil
}
var out []heldItem
for _, it := range itemsInOrder(m.Items) {
if it.Collection != collection || !it.matches(where) || (it.IfCapability != "" && !caps[it.IfCapability]) {
continue
}
if text, ok := heldText(it, facts); ok {
out = append(out, heldItem{item: it, text: text})
}
}
return out
}
// heldText is an item's text with the machine's facts filled, ending its line; false for an item whose text the mesh
// does not hold or which names anything but a machine fact this machine has.
func heldText(it Item, facts map[string]string) (string, bool) {
if it.Content == "" {
return "", false
}
text := it.Content
for _, f := range templatePlaceholder.FindAllStringSubmatch(text, -1) {
value, has := facts[f[2]]
if f[1] != "machine" || !has {
return "", false
}
text = strings.ReplaceAll(text, f[0], value)
}
if !strings.HasSuffix(text, "\n") {
text += "\n"
}
return text, true
}
+135
View File
@@ -0,0 +1,135 @@
package catalogue
import (
"os"
"path/filepath"
"reflect"
"testing"
)
// A bar's blocks are a collection of text (design 57 L3, issue 781): the holder of node-bar names the kind as
// `${collected:contribution:node-bar:block:bar=<b>,place=<p>}`, which a handed collector has its client fill from the
// items and an unhanded one has the controller fill as it fills the old placeholder.
const (
collectedBarsText = "top\n${collected:contribution:node-bar:block:bar=top,place=status}" +
"bottom\n${collected:contribution:node-bar:block:bar=bottom,place=status}"
collectedBars = `{"module":"bars","version":"1","moved":true,
"claims":[{"name":"node-bar","scope":"node","renders":{"block":"{{.shows}}"}}],
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},
"resources":[{"id":"conf","type":"file","path":"/etc/bars.conf","template":"files/bars"}]}`
itemsOfPower = `{"module":"power","version":"1","moved":true,
"items":[{"collection":"contribution:node-bar:block","name":"battery","if-capability":"battery","attributes":{"bar":"bottom","place":"status","order":"40"},"template":"files/battery"},
{"collection":"contribution:node-bar:block","name":"draw","attributes":{"bar":"bottom","place":"status","order":"45"},"template":"files/draw"}],
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}}`
)
func collectedBarsOn(t *testing.T, caps map[string]bool) Resolution {
t.Helper()
bars := held(t, collectedBars, builtWith("bars", map[string]string{"bars": collectedBarsText}))
power := held(t, itemsOfPower, builtWith("power", map[string]string{"battery": "B\n", "draw": "D ${machine:name}\n"}))
r, err := Resolve(shelf(bars, power), []string{"bars", "power"}, workstation(), World{})
if err != nil {
t.Fatal(err)
}
r.Account, r.Capabilities = "op", caps
return r
}
func TestACollectorMayNameTheBarsBlocksAsACollectionAndNoOtherDataKind(t *testing.T) {
bars := held(t, collectedBars, builtWith("bars", map[string]string{"bars": collectedBarsText}))
if _, err := collectedBarsOn(t, nil).Compose(handingTo("store:5000")); err != nil {
t.Fatalf("the handed collector is refused: %v", err)
}
if got := collectedPlaceholderProblems(bars, map[string]any{"id": "x",
"content": "${collected:contribution:node-bar:block:bar=top,place=status}"}); len(got) != 0 {
t.Fatalf("the check refuses it: %v", got)
}
// A data kind whose shape does not say its items are text stays refused.
_, plain, _ := receivable(BarSeat, BarKindBlock)
plain.Shape = &Shape{}
if collectableAsText(plain) {
t.Fatal("a shape not marked as text is collectable")
}
if _, _, err := oldCollection("contribution:node-bar:block", false); err != nil {
t.Fatal(err)
}
}
func TestTheBarsCollectionIsFilledTheOldWayWhereTheCollectorIsNotHanded(t *testing.T) {
for _, caps := range []map[string]bool{nil, {"battery": true}} {
got := composedFile(t, collectedBarsOn(t, caps), "bars.conf")
want := "top\nbottom\n# power\nD workstation\n"
if caps["battery"] {
want = "top\nbottom\n# power\nB\n# power\nD workstation\n"
}
if got != want {
t.Errorf("caps %v: the file is %q, want %q", caps, got, want)
}
}
}
func TestTheBarsCollectionIsNamedInTheHandedPlan(t *testing.T) {
c, _ := composeWith(t, collectedBarsOn(t, map[string]bool{"battery": true}), handingTo("store:5000"))
if !reflect.DeepEqual(c.Plans["bars"].Collections, []PlanCollection{{Name: "contribution:node-bar:block", Comment: "#"}}) {
t.Fatalf("the plan collects %+v", c.Plans["bars"].Collections)
}
if n := len(c.Plans["power"].Items); n != 2 {
t.Fatalf("the contributor's plan names %d items", n)
}
}
// The bars of the catalogue's i3status-rust, with the blocks of power's items, compose to the files the old shaped
// contribution made (issue 781). testdata/bars holds the catalogue's bars and block texts as of the move and the bars as
// composed before it, for a machine with neither capability, the battery, the power meter and both. Where the
// collector is not handed the controller fills the files; where it is, it names the items for the client, which
// fills the same files (mesh-catalog's provisioner client holds that half against the same golden files).
func TestTheBarsOfTheCatalogueAreTheSameHandedOrNot(t *testing.T) {
read := func(name string) string {
b, err := os.ReadFile(filepath.Join("testdata", "bars", name))
if err != nil {
t.Fatal(err)
}
return string(b)
}
collector := `{"module":"bars","version":"1","moved":true,"claims":[{"name":"node-bar","scope":"node"}],
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},
"resources":[{"id":"top","type":"file","path":"/t","template":"files/top"},
{"id":"bottom","type":"file","path":"/b","template":"files/bottom"}]}`
power := `{"module":"power","version":"1","moved":true,
"items":[{"collection":"contribution:node-bar:block","name":"battery","if-capability":"battery","attributes":{"bar":"bottom","place":"status","order":"40"},"template":"files/battery"},
{"collection":"contribution:node-bar:block","name":"power-draw","if-capability":"power-meter","attributes":{"bar":"bottom","place":"status","order":"45"},"template":"files/draw"}],
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}}`
for name, caps := range map[string]map[string]bool{"none": nil, "battery": {"battery": true},
"power-meter": {"power-meter": true}, "both": {"battery": true, "power-meter": true}} {
bars := held(t, collector, builtWith("bars", map[string]string{"top": read("top-bar.toml"), "bottom": read("bottom-bar.toml")}))
blocks := held(t, power, builtWith("power", map[string]string{"battery": read("bar-battery"), "draw": read("bar-power-draw")}))
r, err := Resolve(shelf(bars, blocks), []string{"bars", "power"}, workstation(), World{})
if err != nil {
t.Fatal(err)
}
r.Account, r.Capabilities = "op", caps
// Not handed: the controller fills them.
for id, which := range map[string]string{"bars.top": "top", "bars.bottom": "bottom"} {
if got := composedFile(t, r, id); got != read(name+"."+which+"-bar.toml") {
t.Errorf("%s, %s bar, not handed:\n%s\nwant\n%s", name, which, got, read(name+"."+which+"-bar.toml"))
}
}
// Handed: the plan names the items the machine's capabilities allow, for the client to fill.
c, _ := composeWith(t, r, handingTo("store:5000"))
var names []string
for _, it := range c.Plans["power"].Items {
names = append(names, it.Name)
}
want := []string{}
if caps["battery"] {
want = append(want, "battery")
}
if caps["power-meter"] {
want = append(want, "power-draw")
}
if !reflect.DeepEqual(append([]string{}, names...), want) {
t.Errorf("%s: the plan names %v, want %v", name, names, want)
}
}
}
+60
View File
@@ -0,0 +1,60 @@
package catalogue
import (
"os"
"path/filepath"
"strings"
"testing"
)
// The collectors design 57 L2 moved (issue 681), composed with nothing handed: zsh's rc, triggerhappy's triggers and
// power's after-wake moment, from the catalogue's real templates and its moved contributors' real items, are byte
// for byte what the controller composed from the old forms before the move (testdata/l2-moved: templates/ and items/
// copied from mesh-catalog at the move, want/ composed at the feature branch before it).
func l2Moved(t *testing.T, parts ...string) string {
t.Helper()
raw, err := os.ReadFile(filepath.Join(append([]string{"testdata", "l2-moved"}, parts...)...))
if err != nil {
t.Fatal(err)
}
return string(raw)
}
func TestTheCollectorsL2MovedComposeAsBeforeWhereNothingIsHanded(t *testing.T) {
// Each moved contributor, its items read back from its templates archive as the mesh holds them.
items := map[string][]string{
"asus-zephyrus-g14": {"shell:after-wake:normal", "contribution:node-hotkeys:trigger"},
"claude-code": {"shell:zsh:normal"},
"mesh-cli": {"shell:zsh:normal"},
"powerlevel10k": {"shell:zsh:normal"},
"zsh-autosuggestions": {"shell:zsh:normal"},
"zsh-syntax-highlighting": {"shell:zsh:last"},
}
var modules []Manifest
for _, module := range sortedKeys(items) {
m := Manifest{Module: module, Moved: true}
for _, collection := range items[module] {
m.Items = append(m.Items, Item{Collection: collection,
Content: l2Moved(t, "items", module+"."+strings.ReplaceAll(collection, ":", "-"))})
}
modules = append(modules, m)
}
for _, c := range []struct{ module, seat string }{{"zsh", LoginShellSeat}, {"triggerhappy", HotkeysSeat},
{"power", PowerSeat}} {
holder := Manifest{Module: c.module, Moved: true, Claims: []Claim{{Name: c.seat}}}
resource := map[string]any{"id": "file", "type": "file", "path": "/etc/" + c.module,
"content": l2Moved(t, "templates", c.module)}
var unplaced []string
if err := contributionsInto(resource, holder, append([]Manifest{holder}, modules...),
map[string]string{"name": "g14"}, Rendering{}, map[string]bool{}, &unplaced); err != nil {
t.Fatalf("%s: %v", c.module, err)
}
if got, want := resource["content"], l2Moved(t, "want", c.module); got != want {
t.Errorf("%s's file is not what the old forms made:\n%s\nwant:\n%s", c.module, got, want)
}
if len(unplaced) != 0 {
t.Errorf("%s: unplaced %v", c.module, unplaced)
}
}
}
+158
View File
@@ -0,0 +1,158 @@
package catalogue
import (
"reflect"
"strings"
"testing"
)
// An item may carry attributes and a capability, a module may give several items to one collection, and a
// placeholder may select by attributes (design 57 L3, issue 777). The controller leaves out of a plan an item whose
// capability the machine lacks, accepts a selecting placeholder where it accepts the collection, and renders a
// selected moved item into an old placeholder exactly as the contribution it replaces.
const (
barCollector = `{"module":"bars","version":"1","moved":true,
"collections":[{"name":"blocks","comment":"#"}],
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},
"resources":[{"id":"conf","type":"file","path":"/etc/bars.conf","template":"files/bars"}]}`
barsText = "top\n${collected:blocks:bar=top,place=status}bottom\n${collected:blocks:bar=bottom,place=status}${collected:blocks}"
barContribute = `{"module":"power","version":"1","moved":true,
"items":[{"collection":"blocks","name":"battery","if-capability":"battery","attributes":{"bar":"bottom","place":"status","order":"20"},"template":"files/battery"},
{"collection":"blocks","name":"clock","attributes":{"bar":"top","place":"status"},"template":"files/clock"}],
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}}`
)
func blocksOn(t *testing.T, caps map[string]bool) Composed {
t.Helper()
bars := held(t, barCollector, builtWith("bars", map[string]string{"bars": barsText}))
power := held(t, barContribute, builtWith("power", map[string]string{"battery": "B\n", "clock": "C\n"}))
r, err := Resolve(shelf(bars, power), []string{"bars", "power"}, workstation(), World{})
if err != nil {
t.Fatal(err)
}
r.Account, r.Capabilities = "op", caps
c, _ := composeWith(t, r, handingTo("store:5000"))
return c
}
func TestAnItemWhoseCapabilityTheMachineLacksIsLeftOutOfItsPlan(t *testing.T) {
with := blocksOn(t, map[string]bool{"battery": true}).Plans["power"]
want := []PlanItem{
{Collection: "blocks", Name: "battery", Template: "battery",
Attributes: map[string]string{"bar": "bottom", "place": "status", "order": "20"}},
{Collection: "blocks", Name: "clock", Template: "clock", Attributes: map[string]string{"bar": "top", "place": "status"}},
}
if !reflect.DeepEqual(with.Items, want) {
t.Fatalf("with the capability the plan carries %+v", with.Items)
}
without := blocksOn(t, map[string]bool{"seat": true}).Plans["power"]
if !reflect.DeepEqual(without.Items, want[1:]) {
t.Fatalf("without it the plan carries %+v", without.Items)
}
}
func TestACollectorSelectingByAttributesCollectsTheCollectionOnce(t *testing.T) {
bars := blocksOn(t, nil).Plans["bars"]
if !reflect.DeepEqual(bars.Collections, []PlanCollection{{Name: "blocks", Comment: "#"}}) {
t.Fatalf("the plan collects %+v", bars.Collections)
}
}
func TestASelectorIsRefusedWhenMalformedOrNamingACollectionNotCollected(t *testing.T) {
for placeholder, want := range map[string]string{
"${collected:blocks:bar}": "", // no equals sign: a collection named blocks:bar
"${collected:blocks:bar=top,bar=bottom}": "names bar twice",
"${collected:blocks:bar=}": "is not <attribute>=<value>",
"${collected:other:bar=top}": `does not collect "other"`,
} {
raw := strings.Replace(barCollector, `"template":"files/bars"`, `"content":"`+placeholder+`"`, 1)
raw = strings.Replace(raw, `"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},`, "", 1)
raw = strings.Replace(raw, `"moved":true,`, "", 1)
m, err := ParseManifest([]byte(raw))
if err == nil {
_, err = (Resolution{Node: "n", Modules: []Manifest{m}}).Declaration(Rendering{})
}
if want == "" {
continue
}
if err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("%s: %v, want %q", placeholder, err, want)
}
}
}
func TestItemNamesAttributesAndCapabilitiesAreJudged(t *testing.T) {
item := func(extra string) string {
return `{"module":"p","moved":true,"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},` +
`"items":[` + extra + `]}`
}
for items, want := range map[string]string{
`{"collection":"c","name":"a","template":"files/a"},{"collection":"c","template":"files/b"}`: "has no name",
`{"collection":"c","name":"a","template":"files/a"},{"collection":"c","name":"a","template":"files/b"}`: "twice",
`{"collection":"c","name":"Big","template":"files/a"}`: "is not a name",
`{"collection":"c","attributes":{"order":"100"},"template":"files/a"}`: "0 to 99",
`{"collection":"c","attributes":{"bar":"a b"},"template":"files/a"}`: "is not a value",
`{"collection":"c","if-capability":"batery","template":"files/a"}`: "no machine reports",
`{"collection":"contribution:node-bar:block","attributes":{"bar":"left"},"template":"files/a"}`: "bar=left",
`{"collection":"contribution:node-bar:block","attributes":{"colour":"red"},"template":"files/a"}`: `"colour" is none of them`,
`{"collection":"c","name":"a","template":"files/a","colour":"red"}`: "colour",
} {
_, err := ParseManifest([]byte(item(items)))
if err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("%s: %v, want %q", items, err, want)
}
}
ok := item(`{"collection":"contribution:node-bar:block","name":"a","attributes":{"bar":"top","place":"status","order":"5"},"if-capability":"battery","template":"files/a"}`)
if _, err := ParseManifest([]byte(ok)); err != nil {
t.Fatal(err)
}
}
// A moved contributor's items, selected by the old placeholder's pair, render in the old order (order, module, item
// name) and with the same lines as the contributions they replace.
func TestASelectedMovedItemRendersIntoTheOldBarPlaceholderAsTheContributionDid(t *testing.T) {
piece := func(order, command string) string {
return "[[block]]\nblock = \"custom\"\ncommand = \"" + command + "\"\ninterval = 5\n"
}
asOld := Manifest{Module: "power", Contributions: []SeatContribution{
block(`{"bar":"bottom","place":"status","order":20,"shows":"command","options":{"command":"two","interval":5}}`),
block(`{"bar":"bottom","place":"status","shows":"command","options":{"command":"one","interval":5}}`),
block(`{"bar":"top","place":"resources","shows":"command","options":{"command":"up","interval":5}}`),
}}
asItems := Manifest{Module: "power", Moved: true, Items: []Item{
{Collection: "contribution:node-bar:block", Name: "b", Content: piece("", "one"),
Attributes: map[string]string{"bar": "bottom", "place": "status"}},
{Collection: "contribution:node-bar:block", Name: "a", Content: piece("", "two"),
Attributes: map[string]string{"bar": "bottom", "place": "status", "order": "20"}},
{Collection: "contribution:node-bar:block", Name: "c", Content: piece("", "up"),
Attributes: map[string]string{"bar": "top", "place": "resources"}},
{Collection: "contribution:node-bar:block", Name: "d", Content: "x\n", IfCapability: "battery",
Attributes: map[string]string{"bar": "bottom", "place": "status"}},
}}
other := Manifest{Module: "another", Contributions: []SeatContribution{
block(`{"bar":"bottom","place":"status","order":20,"shows":"command","options":{"command":"mid","interval":5}}`)}}
for _, caps := range []map[string]bool{nil, {"battery": true}} {
was := composedFile(t, Resolution{Node: "n", Account: "op", Capabilities: caps,
Modules: []Manifest{barHolder(), asOld, other}}, "a-bar.bottom")
is := composedFile(t, Resolution{Node: "n", Account: "op", Capabilities: caps,
Modules: []Manifest{barHolder(), asItems, other}}, "a-bar.bottom")
// The old form puts the contributions of a module under one line naming it and the item form puts one under
// each item: the same pieces in the same order, otherwise the same bytes.
want := "own-first\nown-middle\n# another\n" + piece("", "mid") + "# power\n" + piece("", "two") +
"# power\n" + piece("", "one")
if caps["battery"] {
want += "# power\nx\n"
}
want += "own-last\n"
if is != want {
t.Errorf("caps %v: the items render\n%s\nnot\n%s", caps, is, want)
}
if !strings.Contains(was, piece("", "two")) {
t.Errorf("the contributions made\n%s", was)
}
}
if got := composedFile(t, Resolution{Node: "n", Account: "op", Modules: []Manifest{barHolder(), asItems}}, "a-bar.top"); !strings.Contains(got, "\"up\"") || strings.Contains(got, `"one"`) {
t.Errorf("the top bar renders\n%s", got)
}
}
+179
View File
@@ -0,0 +1,179 @@
package catalogue
import (
"reflect"
"strings"
"testing"
)
// A collection is declared and filled by the client (novox/hq ADR 0349 §13, design 57 L1, issue 684): a collector
// names a collection it collects, a contributor an item from a template of its own, and a handed composition plans
// both for the node's provisioner client and renders neither. The three old namespaces read as collections of their
// own name. Each file is filled one way: a piece of the other form on the machine is said as not placed.
const (
collectorZsh = `{"module":"zsh","version":"1","moved":true,
"claims":[{"name":"node-login-shell","scope":"node"}],
"collections":[{"name":"widgets","comment":"#"}],
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},
"resources":[{"id":"rc","type":"file","path":"${machine:account-home}/.zshrc","owner":"${machine:account}",
"mode":"0644","template":"files/zshrc"}]}`
zshrcText = "# zsh\n${shell:zsh:normal}${collected:widgets}"
contributorText = "source ${machine:account-home}/.p10k.zsh\n"
contributorP10k = `{"module":"prompt","version":"1","moved":true,
"items":[{"collection":"shell:zsh:normal","template":"files/zsh"}],
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}}`
oldFormAliases = `{"module":"aliases","version":"1",
"shell":[{"for":"zsh","slot":"normal","code":"alias ll='ls -l'\n"}]}`
)
func builtWith(module string, texts map[string]string) []Built {
return []Built{{Name: "files", Kind: ArtifactArchive,
Reference: ArtifactStoreScheme + module + "/files/blobs/sha256:" + strings.Repeat("e", 64),
Digest: "sha256:" + strings.Repeat("e", 64), Texts: texts}}
}
func collectionsOn(t *testing.T) Resolution {
t.Helper()
zsh := held(t, collectorZsh, builtWith("zsh", map[string]string{"zshrc": zshrcText}))
prompt := held(t, contributorP10k, builtWith("prompt", map[string]string{"zsh": contributorText}))
aliases, err := ParseManifest([]byte(oldFormAliases))
if err != nil {
t.Fatal(err)
}
r, err := Resolve(shelf(zsh, prompt, aliases), []string{"zsh", "prompt", "aliases"}, workstation(), World{})
if err != nil {
t.Fatal(err)
}
r.Account = "op"
return r
}
func TestAContributorsBuildKeepsItsItemsTemplateAndText(t *testing.T) {
m := held(t, contributorP10k, builtWith("prompt", map[string]string{"zsh": contributorText}))
if m.Handed == nil || !reflect.DeepEqual(m.Handed.Items, map[string]string{"shell:zsh:normal": "zsh"}) {
t.Fatalf("the build kept %+v", m.Handed)
}
if !reflect.DeepEqual(m.Items, []Item{{Collection: "shell:zsh:normal", Content: contributorText}}) {
t.Fatalf("the held items are %+v: the template's text, read back", m.Items)
}
}
func TestAHandedCollectorAndContributorArePlannedAndNeitherIsRendered(t *testing.T) {
c, got := composeWith(t, collectionsOn(t), handingTo("store:5000"))
if strings.Contains(got, ".p10k.zsh") || strings.Contains(got, "alias ll") {
t.Fatalf("the controller rendered a contribution into a handed collector: %s", got)
}
zsh, prompt := c.Plans["zsh"], c.Plans["prompt"]
wantCollections := []PlanCollection{{Name: "shell:zsh:normal", Comment: "#"}, {Name: "widgets", Comment: "#"}}
if !reflect.DeepEqual(zsh.Collections, wantCollections) || len(zsh.Items) != 0 {
t.Fatalf("zsh's plan collects %+v and contributes %+v", zsh.Collections, zsh.Items)
}
if !reflect.DeepEqual(prompt.Items, []PlanItem{{Collection: "shell:zsh:normal", Template: "zsh"}}) ||
len(prompt.Files) != 0 || prompt.Placement["machine:account-home"] != "/home/op" {
t.Fatalf("prompt's plan is %+v: its item, by template, and the placement its template names", prompt)
}
// The old form on the same machine is not placed, and said so.
if len(c.Unplaced) != 1 || !strings.Contains(c.Unplaced[0], "aliases's shell:zsh:normal is not placed") {
t.Fatalf("unplaced: %v", c.Unplaced)
}
}
// A collector filled the old way holds the old forms and each moved contributor's item, rendered from the text the
// mesh holds as its old form was (design 57 L2, issue 681), so the file loses no line where nothing is handed; a
// collection of the new form stays empty.
func TestACollectorFilledTheOldWayHoldsTheOldFormsAndEachHeldItem(t *testing.T) {
c, got := composeWith(t, collectionsOn(t), Rendering{ArtifactStore: "store:5000"})
if c.Plans != nil {
t.Fatalf("handed on a machine that cannot take it: %v", c.Plans)
}
if !strings.Contains(got, `"content":"# zsh\n# aliases\nalias ll='ls -l'\n# prompt\nsource /home/op/.p10k.zsh\n"`) {
t.Fatalf("the old way does not hold the old forms and the held item in module order: %s", got)
}
if len(c.Unplaced) != 0 {
t.Fatalf("unplaced: %v", c.Unplaced)
}
}
// An item the controller cannot render the old way (it names a value but a machine fact) is said as not placed.
func TestAHeldItemNamingMoreThanAFactIsSaidNotPlacedTheOldWay(t *testing.T) {
zsh := held(t, collectorZsh, builtWith("zsh", map[string]string{"zshrc": zshrcText}))
if _, ok := heldItemText(Manifest{Module: "p", Items: []Item{{Collection: "shell:zsh:normal",
Content: "export THEME=${config:theme}\n"}}}, "shell:zsh:normal", map[string]string{}); ok {
t.Fatal("an item naming a config value was rendered by the controller")
}
if _, ok := heldItemText(Manifest{Module: "p", Items: []Item{{Collection: "shell:zsh:normal",
Content: "cd ${machine:account-home}\n"}}}, "shell:zsh:normal", map[string]string{}); ok {
t.Fatal("an item naming a fact this machine lacks was rendered")
}
if _, ok := heldItemText(Manifest{Module: "p", Items: []Item{{Collection: "widgets", Content: "x\n"}}},
"widgets", nil); ok {
t.Fatal("an item of a collection of the new form was rendered the old way")
}
left := itemsLeftOut([]Manifest{zsh, {Module: "p", Items: []Item{{Collection: "shell:zsh:normal",
Content: "export THEME=${config:theme}\n"}}}}, zsh, "shell:zsh:normal", map[string]string{})
if len(left) != 1 || !strings.Contains(left[0], "p's item of shell:zsh:normal is not placed") {
t.Fatalf("left out: %v", left)
}
}
// Each old namespace L2 moves renders a moved contributor's item exactly as its old form did, where the collector is
// not handed: the same contributor written both ways composes the same bytes (design 57 L2, issue 681).
func TestAMovedContributorsItemRendersAsItsOldFormDid(t *testing.T) {
code := "# two lines\nalias a=b"
asOld := Manifest{Module: "laptop", Shell: []ShellCode{{For: "zsh", Slot: "normal", Code: code}},
Contributions: []SeatContribution{{Seat: HotkeysSeat, Kind: "trigger", Content: "KEY_F21 1 touchpad\n"}}}
asItem := Manifest{Module: "laptop", Moved: true, Items: []Item{{Collection: "shell:zsh:normal", Content: code},
{Collection: "contribution:node-hotkeys:trigger", Content: "KEY_F21 1 touchpad\n"}}}
other := Manifest{Module: "zz-other", Shell: []ShellCode{{For: "zsh", Slot: "normal", Code: "x=1\n"}},
Contributions: []SeatContribution{{Seat: HotkeysSeat, Kind: "trigger", Content: "KEY_MUTE 1 mute\n"}}}
first := Manifest{Module: "aa-first", Shell: []ShellCode{{For: "zsh", Slot: "normal", Code: "y=2\n"}}}
for _, mods := range [][]Manifest{{other, first}, {}} {
was := append(append([]Manifest(nil), mods...), asOld)
is := append(append([]Manifest(nil), mods...), asItem)
if a, b := shellCodeOn(was, "zsh", "normal", nil), shellCodeOn(is, "zsh", "normal", nil); a != b || a == "" {
t.Errorf("shell: the old form made %q, the item %q", a, b)
}
a, _, errA := seatContributions(was, hotkeysHolder(), "node-hotkeys:trigger", Rendering{}, nil, nil)
b, left, errB := seatContributions(is, hotkeysHolder(), "node-hotkeys:trigger", Rendering{}, nil, nil)
if errA != nil || errB != nil || a != b || a == "" || len(left) != 0 {
t.Errorf("hotkeys: the old form made %q, the item %q (%v %v %v)", a, b, errA, errB, left)
}
}
}
func TestCollectionsAndItemsAreJudged(t *testing.T) {
for name, c := range map[string]struct{ raw, says string }{
"an item of a module not moved": {`{"module":"m","version":"1",
"items":[{"collection":"shell:zsh:normal","template":"files/x"}],
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}}`, "is marked moved"},
"two items to one collection": {`{"module":"m","version":"1","moved":true,
"items":[{"collection":"a","template":"files/x"},{"collection":"a","template":"files/y"}],
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}}`, "contributes to a twice"},
"an item to no slot": {`{"module":"m","version":"1","moved":true,
"items":[{"collection":"shell:zsh:middle","template":"files/x"}],
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}}`, "is no slot"},
"an item outside the templates": {`{"module":"m","version":"1","moved":true,
"items":[{"collection":"a","template":"x"}],
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}}`, "is not a file under files/"},
"an item naming a secret": {`{"module":"m","version":"1",
"items":[{"collection":"a","content":"t=${secret:token}"}]}`, "where no secret is"},
"a collection not declared": {`{"module":"m","version":"1",
"resources":[{"id":"f","type":"file","path":"/f","content":"${collected:a}"}]}`, "does not collect"},
"an old namespace declared": {`{"module":"m","version":"1","collections":[{"name":"shell:zsh:normal"}],
"resources":[{"id":"f","type":"file","path":"/f","content":"x"}]}`, "read from its placeholder"},
"a collection placed nowhere": {`{"module":"m","version":"1","collections":[{"name":"a"}],
"resources":[{"id":"f","type":"file","path":"/f","content":"x"}]}`, "no file of its places it"},
"a collection in a path": {`{"module":"m","version":"1","collections":[{"name":"a"}],
"resources":[{"id":"f","type":"file","path":"/${collected:a}","content":"${collected:a}"}]}`,
"only in a file's content"},
} {
_, err := ParseManifest([]byte(c.raw))
if err == nil || !strings.Contains(err.Error(), c.says) {
t.Errorf("%s: %v, want it to say %q", name, err, c.says)
}
}
if _, err := ParseManifest([]byte(collectorZsh)); err != nil {
t.Errorf("a collector: %v", err)
}
}
+504
View File
@@ -0,0 +1,504 @@
package catalogue
import (
"encoding/json"
"fmt"
"regexp"
"slices"
"strings"
)
// A module declares its config (novox/hq ADR 0348 §1, §4, §5, §6, §8, §11; design 57 package K1, issue 660).
//
// **One concept, two spellings for a while.** "Config" replaces "settings": the manifest says `config` for
// `settings`, `takes-config` for `takes-settings` and `${config:<key>}` for `${setting:<key>}`. The old names stay
// accepted as aliases until every module has moved (ADR 0348 §1), and a module uses one form or the other, never
// both. The config form is read into the very fields the settings form fills — Settings, a mergeable file's
// takes-settings and `${setting:<key>}` — so composition has one path and `${config:<key>}` fills exactly as
// `${setting:<key>}` does: a module rewritten from settings to config composes byte-identically. It is written
// back in the form it was read in, so what the mesh holds is what its author wrote.
//
// **A moved module is held to the whole declaration.** Under `config` every value has a type (number, string or
// boolean; an object or a list only when trusted), a description including its unit, its validation (enum,
// minimum and maximum, pattern), whether it is trusted, optionally the predefined key it falls back to, and either
// a default or `required`; it lands in a file or is read at runtime (`runtime: true`); every mergeable file names
// the keys it takes; a `reload-on` or `restart-on` names a file the module declares. A module not yet moved keeps
// today's form and today's rules (SettingProblems, TakenSettingsProblems) until the catalogue's list of such
// modules is empty (ADR 0348 §11).
//
// **A required value nothing sets** is the operator's own: it composes as an operator's setting nobody gave does
// today, refused by name at composition (UnsetSettingError). Who reports it once mesh-config answers values is the
// provisioner's (ADR 0349), not this file's.
// ConfigField is the manifest key a module declares its config values under.
const ConfigField = "config"
// TakesConfigField is the key a mergeable file names the config values it takes by.
const TakesConfigField = "takes-config"
// SettingsField is the old name of ConfigField, read as its alias.
const SettingsField = "settings"
// configRef is how a file asks for a config value: ${config:<key>}, the same keys as settingRef.
var configRef = regexp.MustCompile(`\$\{config:([a-z0-9][a-z0-9_.-]*)\}`)
// takesConfigKey and takesSettingsKey are the two spellings as a key of a JSON object. A string value is never
// followed by a colon, and a quote inside a string is escaped, so these match only keys.
var (
takesConfigKey = regexp.MustCompile(`"` + TakesConfigField + `"(\s*):`)
takesSettingsKey = regexp.MustCompile(`"` + TakesSettingsField + `"(\s*):`)
)
// The types a config value may have (ADR 0348 §4).
const (
ConfigNumber = "number"
ConfigString = "string"
ConfigBoolean = "boolean"
ConfigObject = "object"
ConfigList = "list"
)
// accountFacts are what the mesh places for a module's account, never config values (ADR 0348 §11).
var accountFacts = map[string]bool{"puid": true, "pgid": true}
// configForms is which of the two forms a manifest was written in, by the words of each it uses.
type configForms struct {
config, settings []string
}
// DeclaresConfig is whether the manifest is written in the config form: `config`, `takes-config` or
// `${config:<key>}` (ADR 0348 §1, §11).
func (m Manifest) DeclaresConfig() bool { return len(m.forms.config) > 0 }
// toSettingsForm is raw with the config form's placeholders and takes-config read as the settings form's, and
// which words of each form it used. `config` itself is a top-level key, renamed where the keys are read
// (UnmarshalJSON).
func toSettingsForm(raw []byte) ([]byte, configForms) {
var f configForms
if configRef.Match(raw) {
f.config = append(f.config, "${config:…}")
}
if takesConfigKey.Match(raw) {
f.config = append(f.config, TakesConfigField)
}
if settingRef.Match(raw) {
f.settings = append(f.settings, "${setting:…}")
}
if takesSettingsKey.Match(raw) {
f.settings = append(f.settings, TakesSettingsField)
}
if len(f.config) == 0 {
return raw, f
}
raw = configRef.ReplaceAllFunc(raw, func(ref []byte) []byte {
return []byte("${setting:" + string(configRef.FindSubmatch(ref)[1]) + "}")
})
raw = takesConfigKey.ReplaceAllFunc(raw, func(key []byte) []byte {
return []byte(`"` + TakesSettingsField + `"` + string(takesConfigKey.FindSubmatch(key)[1]) + ":")
})
return raw, f
}
// toConfigForm is a manifest written in the settings form's words, as JSON, in the config form's: what
// toSettingsForm and UnmarshalJSON read, written back.
func toConfigForm(raw []byte) ([]byte, error) {
var keys map[string]json.RawMessage
if err := json.Unmarshal(raw, &keys); err != nil {
return nil, err
}
if settings, has := keys[SettingsField]; has {
keys[ConfigField] = settings
delete(keys, SettingsField)
var err error
if raw, err = json.Marshal(keys); err != nil {
return nil, err
}
}
raw = settingRef.ReplaceAllFunc(raw, func(ref []byte) []byte {
return []byte("${config:" + string(settingRef.FindSubmatch(ref)[1]) + "}")
})
raw = takesSettingsKey.ReplaceAllFunc(raw, func(key []byte) []byte {
return []byte(`"` + TakesConfigField + `"` + string(takesSettingsKey.FindSubmatch(key)[1]) + ":")
})
return raw, nil
}
// inConfigWords is a refusal said of a module in the config form in its words: `${config:<key>}` and
// takes-config, where the rules shared with the settings form say `${setting:<key>}` and takes-settings.
func inConfigWords(problem string) string {
problem = settingRef.ReplaceAllStringFunc(problem, func(ref string) string {
return "${config:" + settingRef.FindStringSubmatch(ref)[1] + "}"
})
return strings.ReplaceAll(problem, TakesSettingsField, TakesConfigField)
}
// MixedFormProblems is a manifest using both forms, said with the words of each (ADR 0348 §1).
func MixedFormProblems(m Manifest) []string {
if len(m.forms.config) == 0 || len(m.forms.settings) == 0 {
return nil
}
return []string{fmt.Sprintf("%s mixes the config form (%s) with the settings form (%s): a module uses one form "+
"or the other, never both; write %s, %s and ${config:<key>} throughout (novox/hq ADR 0348 §1)",
m.Module, strings.Join(m.forms.config, ", "), strings.Join(m.forms.settings, ", "),
ConfigField, TakesConfigField)}
}
// configDeclaration is the config form's declaration as written: nothing of the settings form's, and nothing
// unsaid.
type configDeclaration struct {
Type string `json:"type,omitempty"`
Description string `json:"description,omitempty"`
Enum []any `json:"enum,omitempty"`
Minimum *float64 `json:"minimum,omitempty"`
Maximum *float64 `json:"maximum,omitempty"`
Pattern string `json:"pattern,omitempty"`
Trusted bool `json:"trusted,omitempty"`
Predefined string `json:"predefined,omitempty"`
Default any `json:"default,omitempty"`
Required bool `json:"required,omitempty"`
Runtime bool `json:"runtime,omitempty"`
}
// MarshalJSON writes a settings declaration exactly as before, and a config declaration without the settings
// form's kind and why, and without a default it does not have.
func (d SettingDeclaration) MarshalJSON() ([]byte, error) {
type plain SettingDeclaration
if d.Kind != "" || d.Why != "" {
return json.Marshal(plain(d))
}
return json.Marshal(configDeclaration{Type: d.Type, Description: d.Description, Enum: d.Enum,
Minimum: d.Minimum, Maximum: d.Maximum, Pattern: d.Pattern, Trusted: d.Trusted, Predefined: d.Predefined,
Default: d.Default, Required: d.Required, Runtime: d.Runtime})
}
// configShaped is whether a declaration says anything of the config form's.
func configShaped(d SettingDeclaration) bool {
return d.Type != "" || d.Description != "" || len(d.Enum) > 0 || d.Minimum != nil || d.Maximum != nil ||
d.Pattern != "" || d.Trusted || d.Predefined != "" || d.Required || d.Runtime
}
// TypedSettingProblems is a declaration under `settings` that says what only `config` says: a value with a type,
// a description and its validation is declared in the config form (ADR 0348 §1, §4).
func TypedSettingProblems(m Manifest) []string {
var problems []string
for _, key := range sortedSettingKeys(m.Settings) {
if configShaped(m.Settings[key]) {
problems = append(problems, fmt.Sprintf("%s's setting %q says its type, description or validation "+
"under %s: a value declared with those is declared under config, with ${config:<key>} and "+
"takes-config, never under %s (novox/hq ADR 0348 §1, §4)", m.Module, key, SettingsField, SettingsField))
}
}
return problems
}
// ConfigProblems is every way a module in the config form declares its config wrongly, each said by name with
// what to do (ADR 0348 How it is checked, design 57 package K1).
func ConfigProblems(m Manifest) []string {
if !m.DeclaresConfig() {
return nil
}
var problems []string
placed := configPlaced(m)
fallsBack := fileFallsBack(m)
renamed := renamedTaken(m)
for _, key := range sortedSettingKeys(m.Settings) {
d := m.Settings[key]
say := func(format string, args ...any) {
problems = append(problems, fmt.Sprintf("%s's config value %q: ", m.Module, key)+fmt.Sprintf(format, args...))
}
if !settingRef.MatchString("${setting:" + key + "}") {
say("not a usable key: lower-case letters, digits, dots, dashes and underscores")
continue
}
if meshWords[key] || accountFacts[key] {
say("%q is placement, which the mesh decides, never a config value (novox/hq ADR 0348 §11)", key)
continue
}
if d.Kind != "" || d.Why != "" {
say("kind and why are the settings form's; under config a value says its type and its description " +
"(novox/hq ADR 0348 §1, §4)")
}
switch d.Type {
case ConfigNumber, ConfigString, ConfigBoolean:
case ConfigObject, ConfigList:
if !d.Trusted {
say("is %s and not trusted: an object or a list decides more than one value can say, so it is "+
"trusted, \"trusted\": true, or split into numbers, strings and booleans (novox/hq ADR 0348 §4)", d.Type)
}
case "":
say("no type: say \"type\": number, string or boolean, or object or list when trusted (novox/hq ADR 0348 §4)")
default:
say("type %q is none of number, string, boolean, object and list (novox/hq ADR 0348 §4)", d.Type)
}
if strings.TrimSpace(d.Description) == "" {
say("no description: say in a sentence what the value is, including its unit (novox/hq ADR 0348 §4)")
}
switch {
case d.Required && d.Default != nil:
say("both a default and required: a required value is the operator's own and has no default; give " +
"one or the other (novox/hq ADR 0348 §4)")
case !d.Required && d.Default == nil:
say("neither a default nor required: give a default, or say \"required\": true for a value that is " +
"the operator's own (novox/hq ADR 0348 §4)")
}
if as := renamed[key]; as != "" && d.Required {
say("is laid into its file as %q, and only when it is set, so the file and its code fall back when "+
"it is not: it is never required; give it the file's default, the empty value of its type "+
"(novox/hq issue 751)", as)
}
for _, wrong := range validationProblems(d) {
say("%s", wrong)
}
if word := operatorsWord(key); word != "" && d.Default != nil && !fallsBack[key] {
say("a key naming %q is the operator's value: declare it \"required\": true, with no default, "+
"unless a mergeable file takes it and no placeholder asks for it, where the file falls back "+
"(novox/hq ADR 0348 §4, ADR 0112, issue 735)", word)
}
if d.Predefined != "" {
p, listed := m.predefined[d.Predefined]
switch {
case m.predefined == nil:
say("falls back to the predefined key %q, and the list of predefined keys (%s at the root of "+
"the catalogue's repository) is not present where it is judged: a value falls back only to a "+
"key on that list; judge it in the catalogue, or leave \"predefined\" out (novox/hq ADR 0348 §7)",
d.Predefined, PredefinedFile)
case !listed:
say("falls back to %q, which is not on the list of predefined keys: name a key on it, or leave "+
"\"predefined\" out (novox/hq ADR 0348 §7)", d.Predefined)
default:
if p.Type != d.Type {
say("falls back to the predefined key %q, which is a %s, and this value is a %s: a value falls "+
"back only to a key of its own type (novox/hq ADR 0348 §4)", d.Predefined, p.Type, d.Type)
}
if p.Trusted && !d.Trusted {
say("falls back to the trusted predefined key %q and is not trusted itself: a value that falls "+
"back to a trusted key is trusted, \"trusted\": true (novox/hq ADR 0348 §8)", d.Predefined)
}
}
}
// A template's text is not here yet. Resolve fills it from the build, WithTemplatesFrom from the tree; neither
// judges again itself, but WithTemplatesFrom parses its result again, and a resolved build is parsed again
// where it is registered, and this is judged then.
if !placed[key] && !d.Runtime && !m.Templated() {
say("placed nowhere: no file says ${config:%s} or takes it under %s, and it is not marked "+
"\"runtime\": true for a value the module's code reads itself (novox/hq ADR 0348 §5)", key, TakesConfigField)
}
}
used := settingKeysUsedBy(m)
for _, key := range sortedKeys(used) {
if _, declared := m.Settings[key]; !declared {
problems = append(problems, fmt.Sprintf("%s says ${config:%s} and declares no such value under %s: "+
"declare %q with its type, description and a default or required (novox/hq ADR 0348 §4, §5)",
m.Module, key, ConfigField, key))
}
}
problems = append(problems, configTakesProblems(m)...)
problems = append(problems, reflectsProblems(m)...)
return problems
}
// configPlaced is every value a file, a unit, a directory's owner, a contribution or a served fact asks for, and
// every value a mergeable file takes.
func configPlaced(m Manifest) map[string]bool {
placed := settingKeysUsedBy(m)
for _, r := range m.Resources {
for _, k := range takenKeyList(r) {
placed[k] = true
}
}
return placed
}
// fileFallsBack is every value a mergeable file takes and no placeholder asks for. Unset, such a value is simply
// not laid over the file: the file's own content, or the module's code reading the file, falls back. So one named
// after the operator's value (a url, a user, an address) is not `required`, and its default is the file's: its
// value in the content, else the empty value of its type, never laid over the file (the operator's decision on
// issue 664, issue 735). A placeholder has no such fallback: its default would be filled in where the operator's
// value belongs (ADR 0112).
func fileFallsBack(m Manifest) map[string]bool {
asked := settingKeysUsedBy(m)
out := map[string]bool{}
for _, r := range m.Resources {
if how, _ := r["merge"].(string); how == "" {
continue
}
for _, k := range takenKeyList(r) {
if !asked[k] {
out[k] = true
}
}
}
return out
}
// renamedTaken is every value a mergeable file takes under a name of its own, with that name (issue 751).
func renamedTaken(m Manifest) map[string]string {
out := map[string]string{}
for _, r := range m.Resources {
if how, _ := r["merge"].(string); how == "" {
continue
}
for _, e := range takenEntries(r[TakesSettingsField]) {
if e.renamed {
out[e.key] = e.as
}
}
}
return out
}
// validationProblems is what is wrong with a declaration's validation, and its default against it.
func validationProblems(d SettingDeclaration) []string {
var problems []string
isNumber := d.Type == ConfigNumber
if len(d.Enum) > 0 && d.Type != ConfigNumber && d.Type != ConfigString {
problems = append(problems, fmt.Sprintf("an enum is for a number or a string, and this is a %s", d.Type))
}
if (d.Minimum != nil || d.Maximum != nil) && !isNumber {
problems = append(problems, fmt.Sprintf("a minimum or maximum is for a number, and this is a %s", d.Type))
}
if d.Minimum != nil && d.Maximum != nil && *d.Minimum > *d.Maximum {
problems = append(problems, fmt.Sprintf("its minimum %v is above its maximum %v", *d.Minimum, *d.Maximum))
}
var pattern *regexp.Regexp
if d.Pattern != "" {
if d.Type != ConfigString {
problems = append(problems, fmt.Sprintf("a pattern is for a string, and this is a %s", d.Type))
}
var err error
if pattern, err = regexp.Compile(`^(?:` + d.Pattern + `)$`); err != nil {
problems = append(problems, fmt.Sprintf("its pattern %q is not a regular expression: %v", d.Pattern, err))
}
}
for _, v := range d.Enum {
if !ofType(v, d.Type) {
problems = append(problems, fmt.Sprintf("its enum holds %v, which is not a %s", v, d.Type))
}
}
if d.Default == nil {
return problems
}
if !ofType(d.Default, d.Type) {
if d.Type != "" {
problems = append(problems, fmt.Sprintf("its default %v is not a %s", d.Default, d.Type))
}
return problems
}
if len(d.Enum) > 0 && !slices.ContainsFunc(d.Enum, func(v any) bool { return v == d.Default }) {
problems = append(problems, fmt.Sprintf("its default %v is not one of %v", d.Default, d.Enum))
}
if n, ok := d.Default.(float64); ok {
if d.Minimum != nil && n < *d.Minimum {
problems = append(problems, fmt.Sprintf("its default %v is below its minimum %v", n, *d.Minimum))
}
if d.Maximum != nil && n > *d.Maximum {
problems = append(problems, fmt.Sprintf("its default %v is above its maximum %v", n, *d.Maximum))
}
}
if s, ok := d.Default.(string); ok && pattern != nil && !pattern.MatchString(s) {
problems = append(problems, fmt.Sprintf("its default %q does not match its pattern %q", s, d.Pattern))
}
return problems
}
// ofType is whether a value read from JSON is of a config type.
func ofType(v any, kind string) bool {
switch v.(type) {
case float64:
return kind == ConfigNumber
case string:
return kind == ConfigString
case bool:
return kind == ConfigBoolean
case map[string]any:
return kind == ConfigObject
case []any:
return kind == ConfigList
}
return false
}
// configTakesProblems is what ADR 0348 §11 adds for a module in the config form: an account fact in
// takes-config (the mesh's own words are refused for both forms by TakenSettingsProblems), and a mergeable file
// naming no keys where none of the module's does (where one does, TakenSettingsProblems names every one that
// does not).
func configTakesProblems(m Manifest) []string {
var problems, silent []string
naming := false
for _, r := range m.Resources {
if how, _ := r["merge"].(string); how == "" {
continue
}
if _, said := r[TakesSettingsField]; !said {
silent = append(silent, fmt.Sprint(r["id"]))
continue
}
naming = true
for _, k := range takenKeyList(r) {
_, declared := m.Settings[k]
switch {
case accountFacts[k]:
problems = append(problems, fmt.Sprintf("%s's %v names %q in %s, and %q is an account fact the "+
"mesh places, never a config value (novox/hq ADR 0348 §11)", m.Module, r["id"], k,
TakesConfigField, k))
case meshWords[k]:
// Refused for both forms by TakenSettingsProblems, in its words.
case !declared:
problems = append(problems, fmt.Sprintf("%s's %v names %q in %s and declares no such value under "+
"%s: a value a file takes is declared like every other, with its type, description and a "+
"default or required, or the config verb could never set it (novox/hq ADR 0348 §4, §11)",
m.Module, r["id"], k, TakesConfigField, ConfigField))
}
}
}
if !naming {
for _, id := range silent {
problems = append(problems, fmt.Sprintf("%s's mergeable %s names no %s: a module that declares config "+
"names the keys each mergeable file takes, an empty list for none (novox/hq ADR 0348 §11)",
m.Module, id, TakesConfigField))
}
}
return problems
}
// meshMadeIDs are the resources the mesh composes into a module's own set, which its `restart-on` may name
// without declaring them; meshMadePrefixes are those named after what they are for.
var (
meshMadeIDs = []string{ComposedJailsID(), FilteringID(), CertificateID(), AuthorityID(), BusUsersID()}
meshMadePrefixes = []string{NeedID(""), SecretID(""), strings.TrimSuffix(GrantID("", ""), "-"), BoundID(""), ReceivedID(""), AccessID("/")}
)
// reflectsProblems is a `reload-on` or `restart-on` naming a file the module does not declare (ADR 0348 §6): not
// one of its own resources, not one the mesh composes for it, and not another module's, which is named
// `<module>.<id>`.
func reflectsProblems(m Manifest) []string {
own := map[string]bool{}
for _, r := range m.Resources {
own[fmt.Sprint(r["id"])] = true
}
meshMade := func(id string) bool {
if slices.Contains(meshMadeIDs, id) {
return true
}
for _, p := range meshMadePrefixes {
if strings.HasPrefix(id, p) && len(id) > len(p) {
return true
}
}
return false
}
var problems []string
for _, r := range m.Resources {
for _, field := range []string{"reload-on", "restart-on"} {
for _, id := range stringsOf(r[field]) {
if strings.Contains(id, ".") || own[id] || meshMade(id) {
continue
}
problems = append(problems, fmt.Sprintf("%s's %v names %q in its %s, and %s declares no such "+
"file: name a file of its own, or another module's as <module>.<id> (novox/hq ADR 0348 §6)",
m.Module, r["id"], id, field, m.Module))
}
}
}
return problems
}
+40
View File
@@ -0,0 +1,40 @@
package catalogue
import "strings"
// What the move of config reads of a module's declaration (novox/hq ADR 0348 §9, design 57 K5, issue 664): where a
// layer's key goes, how two layers' values become one, and whether a value is one its declaration takes.
// TakenAs is each name a mergeable file takes a config value under that is not the value's config key, with that
// key (issue 751): a layer set before the module moved may still say the file's name, and the move sets it under
// the config key.
func TakenAs(m Manifest) map[string]string {
out := map[string]string{}
for key, as := range renamedTaken(m) {
out[as] = key
}
return out
}
// AskedKeys is every value a placeholder of the module asks for: a file, a unit, a directory's owner, a
// contribution or a served fact. Such a value is taken whole from the last layer that sets it, where a mergeable
// file lays one object over another key by key.
func AskedKeys(m Manifest) map[string]bool { return settingKeysUsedBy(m) }
// LaidOver is a value laid over another as a later layer is laid over an earlier: two objects key by key, anything
// else replaced whole.
func LaidOver(under, over any) any { return mergeValue(under, over) }
// ValueProblems is what is wrong with a value for a declaration: its type, and its enum, bounds or pattern.
func ValueProblems(d SettingDeclaration, v any) []string {
if v == nil {
return []string{"no value"}
}
d.Default = v
d.Required = false
var problems []string
for _, p := range validationProblems(d) {
problems = append(problems, strings.ReplaceAll(p, "its default", "the value"))
}
return problems
}
+185
View File
@@ -0,0 +1,185 @@
package catalogue
import (
"encoding/json"
"strings"
"testing"
)
// A takes-config entry may lay a value under the name the file's software reads (issue 751): the key stays a
// config key, lower-case, and the file names its own variable, such as ATLASSIAN_URL.
// renaming is inConfigForm with its rules file taking `url` as NOTIFIER_URL, a value the code reading the file
// falls back on when the file lacks it.
func renaming() map[string]any {
m := inConfigForm()
m["config"].(map[string]any)["url"] = map[string]any{"type": "string",
"description": "the address the notifier posts to, a URL; unset, the notifier's own environment says it",
"default": ""}
resourceOf(m, "rules")["takes-config"] = []any{"position", "theme",
map[string]any{"key": "url", "as": "NOTIFIER_URL"}}
return m
}
// rulesComposed is the rules file's content as the module's defaults and these layers make it.
func rulesComposed(t *testing.T, m Manifest, layers ...Layer) string {
t.Helper()
all := layers
if d, ok := Defaults(m); ok {
all = append([]Layer{d}, layers...)
}
for _, r := range m.Resources {
if r["id"] != "rules" {
continue
}
out, err := ApplySettings(r, all)
if err != nil {
t.Fatal(err)
}
if _, left := out[TakesSettingsField]; left {
t.Fatalf("the composed file still says %s: %v", TakesSettingsField, out)
}
return out["content"].(string)
}
t.Fatal("no rules file")
return ""
}
func TestATakenValueIsLaidUnderTheFilesOwnName(t *testing.T) {
m, err := parseRaw(t, renaming())
if err != nil {
t.Fatalf("a renamed takes-config entry was refused: %v", err)
}
got := rulesComposed(t, m, Layer{From: "ace", Values: map[string]any{"url": "https://hooks.example.test/n"}})
var doc map[string]any
if err := json.Unmarshal([]byte(got), &doc); err != nil {
t.Fatal(err)
}
if doc["NOTIFIER_URL"] != "https://hooks.example.test/n" {
t.Fatalf("url was not laid as NOTIFIER_URL: %s", got)
}
if _, said := doc["url"]; said {
t.Fatalf("url was laid under the config key too: %s", got)
}
}
func TestAnUnsetRenamedValueLeavesTheFileAsItWas(t *testing.T) {
plain := inConfigForm()
base, err := parseRaw(t, plain)
if err != nil {
t.Fatal(err)
}
m, err := parseRaw(t, renaming())
if err != nil {
t.Fatal(err)
}
want := rulesComposed(t, base)
if got := rulesComposed(t, m); got != want {
t.Fatalf("with no layer the file is\n%s\nwanted, as without the entry,\n%s", got, want)
}
other := Layer{From: "ace", Values: map[string]any{"theme": "light"}}
if got, want := rulesComposed(t, m, other), rulesComposed(t, base, other); got != want {
t.Fatalf("with url unset the file is\n%s\nwanted\n%s", got, want)
}
}
// A layer said before its module moved names the file's own spelling, such as nodered's timeZone on one machine.
// Until K5 moves the layer and renames the key, that spelling still reaches the file, so the move changes no
// machine's file; the config key wins where one layer says both.
func TestALayerStillSayingTheFilesNameReachesItUntilItsMove(t *testing.T) {
m, err := parseRaw(t, renaming())
if err != nil {
t.Fatal(err)
}
for _, said := range []string{"url", "NOTIFIER_URL"} {
if stray := UnusedSettings(m, []Layer{{From: "ace", Values: map[string]any{said: "https://x.test"}}}); len(stray) > 0 {
t.Fatalf("%s, which the rules file takes, is said stray: %v", said, stray)
}
got := rulesComposed(t, m, Layer{From: "ace", Values: map[string]any{said: "https://x.test"}})
var doc map[string]any
if err := json.Unmarshal([]byte(got), &doc); err != nil {
t.Fatal(err)
}
if doc["NOTIFIER_URL"] != "https://x.test" || doc["url"] != nil {
t.Fatalf("a layer saying %s made the file %s", said, got)
}
}
both := rulesComposed(t, m, Layer{From: "ace", Values: map[string]any{"url": "https://key.test", "NOTIFIER_URL": "https://old.test"}})
if !strings.Contains(both, "https://key.test") || strings.Contains(both, "old.test") {
t.Fatalf("where one layer says both, the config key did not win: %s", both)
}
later := rulesComposed(t, m, Layer{From: "mesh", Values: map[string]any{"url": "https://mesh.test"}},
Layer{From: "ace", Values: map[string]any{"NOTIFIER_URL": "https://ace.test"}})
if !strings.Contains(later, "https://ace.test") {
t.Fatalf("a later layer's old spelling did not beat an earlier layer's key: %s", later)
}
// Who may set it is the same under either spelling: the rules file is trusted, so both are the terminal's.
terminal := TerminalKeys(m)
for _, k := range []string{"url", "NOTIFIER_URL"} {
if !listHolds(terminal, k) {
t.Fatalf("%s is not the terminal's in a trusted file: %v", k, terminal)
}
}
}
func listHolds(list []string, s string) bool {
for _, x := range list {
if x == s {
return true
}
}
return false
}
func TestARenamedEntryIsWrittenBackAsItWasSaid(t *testing.T) {
m, err := parseRaw(t, renaming())
if err != nil {
t.Fatal(err)
}
raw, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(raw), `"takes-config":["position","theme",{"as":"NOTIFIER_URL","key":"url"}]`) {
t.Fatalf("written back as %s", raw)
}
if _, err := ParseManifest(raw); err != nil {
t.Fatalf("read again: %v", err)
}
}
func TestARenamedEntryIsRefusedWhereItCouldNotHold(t *testing.T) {
entry := func(m map[string]any, e any) map[string]any {
resourceOf(m, "rules")["takes-config"] = []any{"position", "theme", e}
return m
}
refused(t, entry(renaming(), map[string]any{"key": "url"}), "notifier's rules", `"key" and "as"`)
refused(t, entry(renaming(), map[string]any{"key": "url", "as": "NOTIFIER_URL", "when": "set"}), `"key" and "as"`)
refused(t, entry(renaming(), map[string]any{"key": "URL", "as": "NOTIFIER_URL"}), "not a usable key")
refused(t, entry(renaming(), map[string]any{"key": "url", "as": "NOTIFIER URL"}), "not a usable name")
refused(t, entry(renaming(), map[string]any{"key": "url", "as": "endpoints"}), `"endpoints"`, "the mesh reads")
refused(t, entry(renaming(), map[string]any{"key": "url", "as": "theme"}), `"theme"`, "twice")
refused(t, entry(renaming(), map[string]any{"key": "theme", "as": "THEME"}), `"theme"`, "twice")
refused(t, entry(renaming(), map[string]any{"key": "nothing", "as": "NOTHING"}), `"nothing"`, "declares no such value")
kept := entry(renaming(), map[string]any{"key": "url", "as": "NOTIFIER_URL"})
resourceOf(kept, "rules")["protected"] = []any{"NOTIFIER_URL"}
refused(t, kept, `"NOTIFIER_URL"`, "protected")
// Laid only when set, so the code's own fallback holds: never required.
required := renaming()
declaration(required, "url")["required"] = true
delete(declaration(required, "url"), "default")
refused(t, required, `"url"`, "only when it is set", "never required")
}
func TestARenamedEntryIsTheConfigFormsOnly(t *testing.T) {
m := map[string]any{
"module": "notifier", "version": "1",
"resources": []any{
map[string]any{"id": "rules", "type": "file", "path": "/x/rules.json", "merge": "json", "content": "{}",
"takes-settings": []any{map[string]any{"key": "url", "as": "NOTIFIER_URL"}}},
},
}
refused(t, m, "notifier's rules", "config form")
}
+484
View File
@@ -0,0 +1,484 @@
package catalogue
import (
"bytes"
"encoding/json"
"errors"
"strings"
"testing"
)
// A module declares its config (novox/hq ADR 0348 §1, §4, §5, §6, §8, §11; design 57 package K1, issue 660):
// one registration refusal per rule of design 57's K1, the old names read as aliases, and a module rewritten
// from settings to config composing byte-identically.
// inConfigForm is a module in the config form that registers: every value typed, described, defaulted or
// required, placed in a file or read at runtime, and its mergeable file naming its keys.
func inConfigForm() map[string]any {
return map[string]any{
"module": "notifier",
"version": "1",
"config": map[string]any{
"font-size": map[string]any{"type": "number", "description": "the notification's font size, in points",
"minimum": 6, "maximum": 40, "default": 10},
"width": map[string]any{"type": "number", "description": "the notification's width, in pixels",
"default": 250},
"position": map[string]any{"type": "string", "description": "the corner it shows in, a word",
"enum": []any{"top-right", "top-left"}, "default": "top-right"},
"theme": map[string]any{"type": "string", "description": "the colour theme, a lower-case name",
"pattern": "[a-z]+", "default": "dark"},
"history": map[string]any{"type": "number", "description": "how many notifications it keeps, a count",
"default": 20, "runtime": true},
"sender": map[string]any{"type": "string", "description": "the address it mails a digest from",
"required": true},
},
"resources": []any{
map[string]any{"id": "configuration", "type": "file", "path": "/x/notifierrc",
"content": "font = Inter ${config:font-size}\nwidth = ${config:width}\nfrom = ${config:sender}\n"},
map[string]any{"id": "rules", "type": "file", "path": "/x/rules.json", "merge": "json",
"takes-config": []any{"position", "theme"}, "content": "{\"position\": \"top-right\"}\n"},
map[string]any{"id": "notifier", "type": "service", "unit": "notifier.service",
"reload-on": []any{"configuration"}, "restart-on": []any{"rules"}},
},
}
}
// declaration is one value of inConfigForm's config, to change in place.
func declaration(m map[string]any, key string) map[string]any {
return m["config"].(map[string]any)[key].(map[string]any)
}
func resourceOf(m map[string]any, id string) map[string]any {
for _, r := range m["resources"].([]any) {
if r.(map[string]any)["id"] == id {
return r.(map[string]any)
}
}
return nil
}
func parseRaw(t *testing.T, m map[string]any) (Manifest, error) {
t.Helper()
raw, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
return ParseManifest(raw)
}
// refused is that the manifest is refused with every one of these words.
func refused(t *testing.T, m map[string]any, words ...string) {
t.Helper()
_, err := parseRaw(t, m)
if err == nil {
t.Fatalf("registered; wanted a refusal saying %q", words)
}
for _, w := range words {
if !strings.Contains(err.Error(), w) {
t.Fatalf("refused without %q:\n%v", w, err)
}
}
}
func TestAModuleInTheConfigFormRegisters(t *testing.T) {
m, err := parseRaw(t, inConfigForm())
if err != nil {
t.Fatal(err)
}
if !m.DeclaresConfig() {
t.Fatal("a module declaring config is not read as in the config form")
}
d := m.Settings["font-size"]
if d.Type != "number" || d.Description == "" || d.Minimum == nil || *d.Minimum != 6 || d.Default != float64(10) {
t.Fatalf("font-size read as %+v", d)
}
if s := m.Settings["sender"]; !s.Required || s.Default != nil {
t.Fatalf("sender read as %+v", s)
}
}
// Rule 1: a value without type or description.
func TestConfigRefusesAValueWithoutTypeOrDescription(t *testing.T) {
m := inConfigForm()
delete(declaration(m, "width"), "type")
refused(t, m, `notifier's config value "width"`, "no type")
m = inConfigForm()
declaration(m, "width")["type"] = "colour"
refused(t, m, `"width"`, `type "colour"`)
m = inConfigForm()
delete(declaration(m, "width"), "description")
refused(t, m, `"width"`, "no description")
}
// Rule 2: a value with neither a default nor required.
func TestConfigRefusesAValueWithNeitherDefaultNorRequired(t *testing.T) {
m := inConfigForm()
delete(declaration(m, "width"), "default")
refused(t, m, `"width"`, "neither a default nor required")
m = inConfigForm()
declaration(m, "sender")["default"] = "noreply@example.org"
refused(t, m, `"sender"`, "both a default and required")
}
// Rule 3: a default failing its validation.
func TestConfigRefusesADefaultFailingItsValidation(t *testing.T) {
for _, c := range []struct {
key string
value any
said string
}{
{"font-size", 4, "below its minimum"},
{"font-size", 41, "above its maximum"},
{"font-size", "10", "not a number"},
{"position", "bottom", "not one of"},
{"theme", "Dark", "does not match"},
} {
m := inConfigForm()
declaration(m, c.key)["default"] = c.value
refused(t, m, `"`+c.key+`"`, c.said)
}
}
// Rule 4: an object or a list not trusted.
func TestConfigRefusesAnObjectOrListNotTrusted(t *testing.T) {
for _, kind := range []string{"object", "list"} {
m := inConfigForm()
def := any(map[string]any{"a": 1})
if kind == "list" {
def = []any{"a"}
}
m["config"].(map[string]any)["rules"] = map[string]any{"type": kind, "description": "the rules, as " + kind,
"default": def, "runtime": true}
refused(t, m, `"rules"`, "is "+kind+" and not trusted")
m["config"].(map[string]any)["rules"].(map[string]any)["trusted"] = true
if _, err := parseRaw(t, m); err != nil {
t.Fatalf("a trusted %s is refused: %v", kind, err)
}
}
}
// withPredefined is the list of predefined keys the mesh published, for the length of a test.
func withPredefined(t *testing.T, keys PredefinedList) {
t.Helper()
was := ReplacePublished(keys)
t.Cleanup(func() { ReplacePublished(was) })
}
// Rule 5: a predefined key not on the list, or of another type. Where no list is present, a value naming one is
// refused saying so.
func TestConfigRefusesAPredefinedKeyNotOnTheListOrOfAnotherType(t *testing.T) {
withPredefined(t, nil)
m := inConfigForm()
declaration(m, "font-size")["predefined"] = "font-size"
refused(t, m, `"font-size"`, "the list of predefined keys", "is not present")
withPredefined(t, map[string]PredefinedKey{"font-size": {Type: "number"}, "accent": {Type: "string"}})
if _, err := parseRaw(t, m); err != nil {
t.Fatalf("a value falling back to a listed key of its type is refused: %v", err)
}
m = inConfigForm()
declaration(m, "font-size")["predefined"] = "font-scale"
refused(t, m, `"font-size"`, `"font-scale"`, "not on the list")
m = inConfigForm()
declaration(m, "font-size")["predefined"] = "accent"
refused(t, m, `"font-size"`, `"accent"`, "is a string")
}
// Rule 6: an untrusted value falling back to a trusted predefined key.
func TestConfigRefusesAnUntrustedValueFallingBackToATrustedKey(t *testing.T) {
withPredefined(t, nil)
m := inConfigForm()
declaration(m, "theme")["predefined"] = "theme"
refused(t, m, `"theme"`, "is not present")
withPredefined(t, map[string]PredefinedKey{"theme": {Type: "string", Trusted: true}})
refused(t, m, `"theme"`, "trusted", "is not")
declaration(m, "theme")["trusted"] = true
if _, err := parseRaw(t, m); err != nil {
t.Fatalf("a trusted value falling back to a trusted key is refused: %v", err)
}
}
// Rule 7: a value neither placed in a template nor read at runtime.
func TestConfigRefusesAValueNeitherPlacedNorReadAtRuntime(t *testing.T) {
m := inConfigForm()
delete(declaration(m, "history"), "runtime")
refused(t, m, `"history"`, "${config:history}", `"runtime": true`)
// Taken by a mergeable file is placed.
m = inConfigForm()
if _, err := parseRaw(t, m); err != nil {
t.Fatal(err)
}
}
// Rule 8: a placeholder naming an undeclared value.
func TestConfigRefusesAPlaceholderNamingAnUndeclaredValue(t *testing.T) {
m := inConfigForm()
resourceOf(m, "configuration")["content"] = "font = ${config:font-size}\nwidth = ${config:width}\n" +
"from = ${config:sender}\ncolour = ${config:colour}\n"
refused(t, m, "${config:colour}", "declares no such value")
}
// Rule 9: a reload-on or restart-on naming a file the module does not declare.
func TestConfigRefusesAReloadOrRestartOnNamingAnUndeclaredFile(t *testing.T) {
m := inConfigForm()
resourceOf(m, "notifier")["reload-on"] = []any{"configuration", "dunstrc"}
refused(t, m, "reload-on", `"dunstrc"`, "declares no such")
m = inConfigForm()
resourceOf(m, "notifier")["restart-on"] = []any{"rule"}
refused(t, m, "restart-on", `"rule"`)
// Another module's file, and one the mesh makes for the module, are not the module's to declare.
m = inConfigForm()
resourceOf(m, "notifier")["restart-on"] = []any{"rules", "resolver.fact-zones", "secret-mail", "composed-jails"}
if _, err := parseRaw(t, m); err != nil {
t.Fatalf("refused: %v", err)
}
}
// Rule 10: a moved module's mergeable file not naming its keys under takes-config.
func TestConfigRefusesAMergeableFileNotNamingItsKeys(t *testing.T) {
m := inConfigForm()
delete(resourceOf(m, "rules"), "takes-config")
// position and theme are then placed nowhere; say they are read at runtime so only the file's rule speaks.
declaration(m, "position")["runtime"] = true
declaration(m, "theme")["runtime"] = true
refused(t, m, "rules", "names no takes-config")
}
// Rule 11: a mesh word, puid or pgid in takes-config.
func TestConfigRefusesAMeshWordOrAnAccountFactInTakesConfig(t *testing.T) {
for _, word := range []string{"ports", "expose", "reach", "endpoints", "places", "accesses", "networks", "puid", "pgid"} {
m := inConfigForm()
resourceOf(m, "rules")["takes-config"] = []any{"position", "theme", word}
refused(t, m, `"`+word+`"`, "takes-config")
}
}
// A module uses one form or the other, never both.
func TestAManifestMixingTheConfigAndSettingsFormsIsRefused(t *testing.T) {
m := inConfigForm()
resourceOf(m, "configuration")["content"] = "font = ${config:font-size}\nwidth = ${setting:width}\nfrom = ${config:sender}\n"
refused(t, m, "mixes", "${setting:", "config")
m = inConfigForm()
delete(resourceOf(m, "rules"), "takes-config")
resourceOf(m, "rules")["takes-settings"] = []any{"position", "theme"}
refused(t, m, "mixes", "takes-settings")
m = inConfigForm()
m["settings"] = map[string]any{"colour": map[string]any{"kind": "preference", "default": "red", "why": "red"}}
refused(t, m, "mixes", "settings")
}
// The settings form keeps its own declaration; the config form's has no kind or why.
func TestEachFormKeepsItsOwnDeclaration(t *testing.T) {
m := inConfigForm()
declaration(m, "width")["why"] = "fits a title"
refused(t, m, `"width"`, "why")
err := parsed(t, Manifest{Module: "notifier",
Settings: map[string]SettingDeclaration{"width": {Kind: KindPreference, Default: float64(250),
Why: "fits a title", Type: "number"}},
Resources: []map[string]any{{"id": "c", "type": "file", "path": "/x", "content": "${setting:width}"}}})
if err == nil || !strings.Contains(err.Error(), "under config") {
t.Fatalf("a typed value under settings: %v", err)
}
}
// What the mesh holds is what was written: a module in the config form is written back in it, and read again
// the same.
func TestAConfigManifestIsWrittenBackInItsOwnForm(t *testing.T) {
m, err := parseRaw(t, inConfigForm())
if err != nil {
t.Fatal(err)
}
raw, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
for _, said := range []string{`"config":`, `${config:font-size}`, `"takes-config":`, `"required":true`} {
if !strings.Contains(string(raw), said) {
t.Fatalf("written back without %s: %s", said, raw)
}
}
for _, unsaid := range []string{`"settings":`, `${setting:`, `"takes-settings"`, `"kind"`, `"why"`, `"default":null`} {
if strings.Contains(string(raw), unsaid) {
t.Fatalf("written back with %s: %s", unsaid, raw)
}
}
again, err := ParseManifest(raw)
if err != nil {
t.Fatal(err)
}
twice, _ := json.Marshal(again)
if !bytes.Equal(raw, twice) {
t.Fatalf("read again it is written\n%s\nwhere it was\n%s", twice, raw)
}
}
// A module in the settings form is written back byte for byte as before.
func TestASettingsManifestIsWrittenBackAsBefore(t *testing.T) {
d := SettingDeclaration{Kind: KindPreference, Default: float64(10), Why: "readable"}
raw, err := json.Marshal(d)
if err != nil {
t.Fatal(err)
}
if string(raw) != `{"kind":"preference","default":10,"why":"readable"}` {
t.Fatalf("written as %s", raw)
}
}
// settingsDunst is the dunst module as it is written today; configDunst the same module rewritten from settings to
// config.
const settingsDunst = `{"module": "dunst", "version": "1",
"settings": {
"font-size": {"kind": "preference", "default": 10, "why": "10 points of Inter reads well on a screen of about 100 DPI"},
"width": {"kind": "preference", "default": 250, "why": "250 pixels fits a title of about forty characters at 10 points"}
},
"resources": [
{"id": "package", "type": "package", "package": "dunst"},
{"id": "configuration", "type": "file", "path": "/home/me/.config/dunst/dunstrc", "mode": "0644",
"content": "[global]\n font = Inter ${setting:font-size}\n width = ${setting:width}\n"},
{"id": "rules", "type": "file", "path": "/home/me/.config/dunst/rules.json", "merge": "json",
"takes-settings": [], "content": "{\"urgency\": \"low\"}\n"},
{"id": "notifier", "type": "service", "unit": "dunst.service", "reload-on": ["configuration"]}
]}`
const configDunst = `{"module": "dunst", "version": "1",
"config": {
"font-size": {"type": "number", "description": "the font size of a notification, in points", "minimum": 6,
"maximum": 40, "default": 10},
"width": {"type": "number", "description": "the width of a notification, in pixels", "minimum": 100,
"default": 250}
},
"resources": [
{"id": "package", "type": "package", "package": "dunst"},
{"id": "configuration", "type": "file", "path": "/home/me/.config/dunst/dunstrc", "mode": "0644",
"content": "[global]\n font = Inter ${config:font-size}\n width = ${config:width}\n"},
{"id": "rules", "type": "file", "path": "/home/me/.config/dunst/rules.json", "merge": "json",
"takes-config": [], "content": "{\"urgency\": \"low\"}\n"},
{"id": "notifier", "type": "service", "unit": "dunst.service", "reload-on": ["configuration"]}
]}`
// ${config:<key>} fills exactly as ${setting:<key>} does: the module rewritten from settings to config composes
// to the same bytes, with no layer, with the node's, and with the mesh's under the node's.
func TestAModuleRewrittenFromSettingsToConfigComposesByteIdentically(t *testing.T) {
composed := func(manifest string, layers []Layer) []byte {
t.Helper()
m, err := ParseManifest([]byte(manifest))
if err != nil {
t.Fatal(err)
}
r := anAdoptedAnchor()
r.Modules = append(r.Modules, m)
with := anchorRendering(false)
if layers != nil {
with.Settings["dunst"] = layers
}
c, err := r.Compose(with)
if err != nil {
t.Fatal(err)
}
if why, left := c.LeftOut["dunst"]; left {
t.Fatalf("dunst was left out: %s", why)
}
raw, err := json.Marshal(c)
if err != nil {
t.Fatal(err)
}
return raw
}
for _, layers := range [][]Layer{
nil,
{{From: "anchor", Values: map[string]any{"font-size": float64(13)}}},
{{From: MeshWideLayer, Values: map[string]any{"width": float64(300)}},
{From: "anchor", Values: map[string]any{"font-size": float64(12), "width": float64(320)}}},
} {
was, is := composed(settingsDunst, layers), composed(configDunst, layers)
if !bytes.Equal(was, is) {
t.Fatalf("composed differently with %v:\nsettings: %s\nconfig: %s", layers, was, is)
}
if !strings.Contains(string(is), "font = Inter 1") || strings.Contains(string(is), "${config:") {
t.Fatalf("not filled: %s", is)
}
}
}
// A key a mergeable file takes is a config value like every other: one not declared under config is refused by
// name (ADR 0348 §4, review of mesh-controller#320).
func TestConfigRefusesATakesConfigKeyNotDeclared(t *testing.T) {
m := inConfigForm()
resourceOf(m, "rules")["takes-config"] = []any{"position", "theme", "urgency"}
refused(t, m, `"urgency"`, "takes-config", "declares no such value under config")
}
// A required value nothing sets is refused at composition in the config form's words: ${config:<key>} and the
// config verb, never `settings set` (review of mesh-controller#320). The settings form's words are unchanged.
func TestAnUnsetRequiredValueIsRefusedInTheConfigFormsWords(t *testing.T) {
m, err := parseRaw(t, inConfigForm())
if err != nil {
t.Fatal(err)
}
r := anAdoptedAnchor()
r.Modules = append(r.Modules, m)
composed, err := r.Compose(anchorRendering(false))
if err != nil {
t.Fatal(err)
}
why, left := composed.LeftOut["notifier"]
if !left {
t.Fatal("a module whose required value nothing sets was not left out")
}
for _, said := range []string{"${config:sender}", `required config value "sender"`, "config verb"} {
if !strings.Contains(why, said) {
t.Fatalf("left out without %q: %s", said, why)
}
}
for _, unsaid := range []string{"${setting:", "settings set"} {
if strings.Contains(why, unsaid) {
t.Fatalf("left out in the settings form's words (%q): %s", unsaid, why)
}
}
var unset *UnsetSettingError
if err := unsetInConfigWords(settingInto(map[string]any{"type": "file", "content": "${setting:sender}"}, nil,
"notifier"), m); !errors.As(err, &unset) || unset.Setting != "sender" {
t.Fatalf("not typed as a value nobody gave: %v", err)
}
settingsForm := notifier()
settingsForm.Resources[0]["content"] = "from = ${setting:domain}\n"
err = unsetInConfigWords(settingInto(settingsForm.Resources[0], nil, "notifier"), settingsForm)
if err == nil || !strings.Contains(err.Error(), "`settings set notifier <file>`") {
t.Fatalf("the settings form's refusal changed: %v", err)
}
}
// A value named after the operator's own (a url, a user, an address) is `required` where nothing falls back when
// it is unset; where a mergeable file takes it and no placeholder asks for it, the file's own content (or the
// module's code reading the file) falls back, so it keeps a default and stays optional, as it is today: the
// operator's decision on issue 664's second question (issue 735).
func TestAnOperatorNamedValueAFileFallsBackOnMayHaveADefault(t *testing.T) {
m := inConfigForm()
m["config"].(map[string]any)["url"] = map[string]any{"type": "string",
"description": "the address the notifier posts to, a URL; empty for the one the file's software picks",
"default": ""}
resourceOf(m, "rules")["takes-config"] = []any{"position", "theme", "url"}
if _, err := parseRaw(t, m); err != nil {
t.Fatalf("a taken value named after the operator's, with a default, was refused: %v", err)
}
// A placeholder asking for it has no fallback but the default, which would put a value in the definition
// where the operator's belongs (ADR 0112).
asked := inConfigForm()
asked["config"].(map[string]any)["url"] = map[string]any{"type": "string", "description": "a URL", "default": ""}
resourceOf(asked, "rules")["takes-config"] = []any{"position", "theme", "url"}
resourceOf(asked, "configuration")["content"] = "font = Inter ${config:font-size}\nwidth = ${config:width}\n" +
"from = ${config:sender}\npost = ${config:url}\n"
refused(t, asked, `"url"`, `a key naming "url" is the operator's value`)
// Read by the module's code alone, nothing in the definition says it falls back.
runtime := inConfigForm()
runtime["config"].(map[string]any)["url"] = map[string]any{"type": "string", "description": "a URL",
"default": "", "runtime": true}
refused(t, runtime, `"url"`, `a key naming "url" is the operator's value`)
}
+110
View File
@@ -0,0 +1,110 @@
package catalogue
import (
"fmt"
"regexp"
"slices"
)
// A provider's consumers' credentials, in a file of the provider's own (novox/hq ADR 0373, issue 679).
//
// A provider in the tool runner may not ask the vault (ADR 0349 clause 11, ADR 0370 clause 2), so its template names
// `${consumers:<provision>}` and its machine's provisioner client fills it: one entry per consumer bound to the
// provider for that provision, each carrying the credential the vault keeps for the pair. Only the client fills it,
// so the placeholder is a provider's alone, for a provision it provides, in a file only the provider reads (mode 0600
// or 0400, owned at runtime, so a change on disk is written back); and a module not handed to the client on a machine
// is refused there, never written the placeholder's text, nor an empty list the provider would read as no consumers.
// ConsumersNamespace is a provider's consumers' credentials' placeholder namespace, `${consumers:<provision>}`.
const ConsumersNamespace = "consumers"
// ofConsumers is where a provider places its consumers' credentials. Loose inside the braces, so a misspelt name is
// refused rather than passed over.
var ofConsumers = regexp.MustCompile(`\$\{consumers:([^}]*)\}`)
// consumersPlaceholderProblems is what is wrong with one resource's ${consumers:…}: placed only in a file's content,
// for a provision the module provides, in a file only the provider reads and the client writes back.
func consumersPlaceholderProblems(m Manifest, r map[string]any) []string {
var problems []string
for _, field := range sortedKeys(r) {
v, ok := r[field].(string)
if !ok {
continue
}
for _, f := range ofConsumers.FindAllStringSubmatch(v, -1) {
switch {
case field != "content" || fmt.Sprint(r["type"]) != "file":
problems = append(problems, fmt.Sprintf("%s's resource %v names %s in its %s; a provider's consumers "+
"are placed only in a file's content (novox/hq ADR 0373)", m.Module, r["id"], f[0], field))
continue
case !slices.ContainsFunc(m.Provides, func(o Offer) bool { return o.Name == f[1] }):
problems = append(problems, fmt.Sprintf("%s's resource %v names %s, and %s does not provide %q: only a "+
"provider's template holds its consumers' credentials (novox/hq ADR 0373)", m.Module, r["id"], f[0],
m.Module, f[1]))
}
if mode := fmt.Sprint(r["mode"]); mode != "0600" && mode != "0400" {
problems = append(problems, fmt.Sprintf("%s's resource %v holds its consumers' credentials (%s) with "+
"mode %q: the provider alone reads it, 0600 or 0400 (novox/hq ADR 0373)", m.Module, r["id"], f[0],
r["mode"]))
}
if owner := ownershipOf(r); owner != RuntimeOwned {
problems = append(problems, fmt.Sprintf("%s's resource %v holds its consumers' credentials (%s) and is "+
"%s at runtime: it is owned, written back whenever it differs (novox/hq ADR 0373)", m.Module,
r["id"], f[0], owner))
}
}
}
return problems
}
// consumersHandedOnly says what a file naming ${consumers:…} is where it would be written the old way: the provisioner's
// client alone fills it, so on a machine that does not hand the module it is **left out** of the declaration, never
// written as that text or as an empty list the provider would read as no consumers, and never the reason the module is
// refused: a provider not handed there keeps working as it did, from its grant files (novox/hq ADR 0373, issue 780).
// A moved module judged from its repository's tree (WithTemplatesFrom: `module check`), on a machine
// that would hand it, has no templates archive to hand and is judged as it composes; there the placeholder is judged
// and taken out of the text, since what is composed so is never sent (judged answers whether the file is so judged).
func consumersHandedOnly(m Manifest, r map[string]any, with Rendering) (judged, leftOut bool) {
content, _ := r["content"].(string)
switch {
case ofConsumers.FindString(content) == "":
return false, false
case m.Moved && m.Handed == nil && with.Hands && with.Provisioner != "":
return true, false
}
return false, true
}
// consumersLeftOutLine is the plan's one line for a file left out because the module is not handed.
func consumersLeftOutLine(m Manifest, r map[string]any) string {
return fmt.Sprintf("%s's %v names ${consumers:…}, which the provisioner's client alone fills: left out, handed only "+
"(novox/hq ADR 0373)", m.Module, r["id"])
}
// consumersNotHanded is consumersHandedOnly as a refusal, for a caller that has no way to leave the file out.
func consumersNotHanded(m Manifest, r map[string]any, with Rendering) (judged bool, err error) {
judged, left := consumersHandedOnly(m, r, with)
if left {
return false, fmt.Errorf("%s's resource %v names %s, which the provisioner's client alone fills, and %s is not "+
"handed to it on this machine: it would be written as that text, or empty, and the provider would read no "+
"consumers (novox/hq ADR 0373)", m.Module, r["id"], ofConsumers.FindString(fmt.Sprint(r["content"])), m.Module)
}
return judged, nil
}
// consumersOfTemplate is what is wrong with a handed file's template naming ${consumers:…}, judged on the file's
// resource as if its text were inline.
func consumersOfTemplate(m Manifest, id, text string) []string {
for _, r := range m.Resources {
if fmt.Sprint(r["id"]) != id {
continue
}
withText := map[string]any{}
for k, v := range r {
withText[k] = v
}
withText["content"] = text
return consumersPlaceholderProblems(m, withText)
}
return nil
}
+33
View File
@@ -0,0 +1,33 @@
package catalogue
import (
"encoding/json"
"slices"
)
// The core's own credentials (novox/hq ADR 0349 §16, issue 755). The bus, the node-engine, the controller and
// mesh-provisioner need their files before any provisioner client runs, so their files stay placement and their
// credentials travel in the declaration as values the vault sealed to the machine they run on, which the controller
// carries and cannot open; the node-engine opens them there (mesh-host's VaultBoxContract).
// CoreCredentialModules are the modules whose own secrets are the core's credentials: the bus, mesh-provisioner and
// its client. The node-engine and the controller hold no own secret of a module's. **Fixed here, in the controller's
// code, as the clause says** — and the vault keeps the same list (mesh-vault's CoreModules), answering the controller
// for these alone.
var CoreCredentialModules = []string{"mesh-provisioner", "mesh-provisioner-client", "nats"}
// CarriesVaultSealed says whether module's own secrets are the core's credentials.
func CarriesVaultSealed(module string) bool { return slices.Contains(CoreCredentialModules, module) }
// VaultSealedField is the field of a file the vault sealed to its machine (mesh-host's File.VaultSealed).
const VaultSealedField = "vault-sealed"
// ForeseenVaultBox stands for a core credential's box in a composition asked ahead of the send, when none is kept
// yet: the send asks the vault for it. Never sent.
var ForeseenVaultBox = json.RawMessage(`{"v":1,"eph":"","iv":"","tag":"","ct":"","foreseen":"asked of the vault by the next send"}`)
// vaultSealedOf is the vault's box for one of module's own secrets, if the rendering was given one.
func (with Rendering) vaultSealedOf(module, name string) (json.RawMessage, bool) {
box, ok := with.VaultSealed[module][name]
return box, ok && len(box) > 0
}
+107
View File
@@ -0,0 +1,107 @@
package catalogue
import (
"encoding/json"
"reflect"
"strings"
"testing"
)
// The core's own credentials travel sealed by the vault to their machine (novox/hq ADR 0349 §16, issue 755): the
// controller carries the vault's box and cannot open it. The list is fixed here, and the composition carries the box
// in place of the controller's own sealed value only where the rendering was given one.
func TestTheCoresOwnCredentialsAreTheBusTheProvisionerAndItsClient(t *testing.T) {
if want := []string{"mesh-provisioner", "mesh-provisioner-client", "nats"}; !reflect.DeepEqual(CoreCredentialModules, want) {
t.Fatalf("the core's modules are %v, want %v (mesh-vault's CoreModules is the same list)", CoreCredentialModules, want)
}
for _, m := range []string{"nats", "mesh-provisioner", "mesh-provisioner-client"} {
if !CarriesVaultSealed(m) {
t.Errorf("%s is not on the list", m)
}
}
for _, m := range []string{"postgres", "mesh-vault", "mesh-controller", ""} {
if CarriesVaultSealed(m) {
t.Errorf("%s is on the list", m)
}
}
}
const provisionerClient = `{"module":"mesh-provisioner-client","version":"1",
"own-secrets":{"provisioner-broker":"${dir:state}/broker"},
"resources":[{"id":"state","type":"directory","mode":"0700","place":"."}]}`
func clientOn(t *testing.T) Resolution {
t.Helper()
m, err := ParseManifest([]byte(provisionerClient))
if err != nil {
t.Fatal(err)
}
return notifierOn(t, m)
}
func brokerFile(t *testing.T, c Composed) map[string]any {
t.Helper()
for _, r := range c.Resources {
if r["id"] == "mesh-provisioner-client.needs-provisioner-broker" {
return r
}
}
return nil
}
var aVaultBox = json.RawMessage(`{"v":1,"eph":"ZQ==","iv":"aQ==","tag":"dA==","ct":"Yw=="}`)
func TestACoreCredentialCarriesTheVaultsBoxAndNoSealOfTheControllers(t *testing.T) {
r := clientOn(t)
with := Rendering{Needed: map[string]map[string]string{"mesh-provisioner-client": {"provisioner-broker": "nacl"}},
VaultSealed: map[string]map[string]json.RawMessage{"mesh-provisioner-client": {"provisioner-broker": aVaultBox}}}
c, got := composeWith(t, r, with)
f := brokerFile(t, c)
if f == nil {
t.Fatalf("the credential's file is not declared: %s", got)
}
if f["sealed"] != nil || !reflect.DeepEqual(f["vault-sealed"], aVaultBox) || strings.Contains(got, `"nacl"`) {
t.Fatalf("the file is %v: the vault's box alone, never the controller's seal", f)
}
// Without the controller's own value at all: the vault's box is enough.
with.Needed = nil
if c, _ := composeWith(t, r, with); brokerFile(t, c) == nil {
t.Fatal("a core credential with the vault's box and no seal of the controller's was refused or dropped")
}
}
func TestACoreModuleTheVaultGaveNothingForIsLeftOutWithWhy(t *testing.T) {
r := clientOn(t)
c, got := composeWith(t, r, Rendering{NotPlaced: map[string]string{
"mesh-provisioner-client": "the vault did not answer for its credential"}})
if brokerFile(t, c) != nil || strings.Contains(got, "mesh-provisioner-client.") {
t.Fatalf("a core module the vault gave nothing for was placed: %s", got)
}
if !strings.Contains(c.LeftOut["mesh-provisioner-client"], "the vault did not answer") {
t.Fatalf("left out %v", c.LeftOut)
}
}
// An engine below VaultBoxContract is rendered with neither field: its composition is the controller's sealed value,
// byte for byte what it was.
func TestWithoutTheVaultsBoxTheCompositionIsAsBefore(t *testing.T) {
r := clientOn(t)
with := Rendering{Needed: map[string]map[string]string{"mesh-provisioner-client": {"provisioner-broker": "nacl"}}}
before, _ := composeWith(t, r, with)
with.VaultSealed, with.NotPlaced = map[string]map[string]json.RawMessage{}, map[string]string{}
after, _ := composeWith(t, r, with)
if !reflect.DeepEqual(before, after) || brokerFile(t, before)["sealed"] != "nacl" {
t.Fatalf("the composition changed: %v", brokerFile(t, after))
}
}
// Asked ahead of a send that has no box kept yet: a stand-in, named as foreseen, never sent.
func TestAForeseenVaultBoxIsNamed(t *testing.T) {
r := clientOn(t)
c, _ := composeWith(t, r, Rendering{VaultSealed: map[string]map[string]json.RawMessage{
"mesh-provisioner-client": {"provisioner-broker": ForeseenVaultBox}}})
if !reflect.DeepEqual(c.Foreseen, []string{"mesh-provisioner-client/provisioner-broker"}) {
t.Fatalf("foreseen %v", c.Foreseen)
}
}
+1 -1
View File
@@ -173,7 +173,7 @@ func TestTheBackupHoldersLinesAreDerivedFromTheData(t *testing.T) {
{"id":"meta","path":"${dir:meta}","class":"rebuildable"}]},
"resources":[{"id":"meta","type":"directory","mode":"0700"}]}`)
facts := map[string]string{"account-home": "/home/op"}
with := Rendering{Settings: SettingsBy{"media": {{From: "node", Values: map[string]any{AccessesSetting: map[string]any{"films": "/tank/films"}}}}}}
with := Rendering{Settings: SettingsBy{"media": {LayerOf("node", map[string]any{AccessesSetting: map[string]any{"films": "/tank/films"}})}}}
backup, _, err := seatContributions([]Manifest{pg, agent, media}, Manifest{}, BackupSeat+":"+BackupKindBackup, with, facts, nil)
if err != nil {
t.Fatal(err)
+192 -7
View File
@@ -93,6 +93,15 @@ type Rendering struct {
// makes. Nil on every composition that is sent, and on every other question, which then refuse
// a secret not made with a *NotMadeError.
Foreseen map[string]map[string]bool
// VaultSealed is, for a module on the core's list (CoreCredentialModules), each own secret as the vault sealed it
// to this machine (the mesh's sealed box, as the vault answered it), keyed like Needed (novox/hq ADR 0349 §16,
// issue 755). Given only for a machine whose engine opens the box (VaultBoxContract): its file then carries the
// box and never the controller's own sealed value. Nil everywhere else, and the composition is as it was.
VaultSealed map[string]map[string]json.RawMessage
// NotPlaced is each core module left out of this machine's declaration because the vault gave nothing for its
// credential and no box was kept, with why (issue 755): the machine keeps what it holds of it, and a module not
// yet there is not placed. Nil everywhere else.
NotPlaced map[string]string
// Mesh is every node's address on the private network, which is what a rule saying "from the
// mesh" resolves to. Passed in for the same reason grants are: who else is on the network is
@@ -257,6 +266,13 @@ type Rendering struct {
// contract is link.BusyContract or later). To an older, strict engine the field is not sent, and the
// container is replaced at once, as before.
ReadsBusy bool
// Hands says this machine's node-engine writes a filled file on the provisioner client's word alone (its
// statement's contract is link.HandedContract or later, design 57 P7), and Provisioner is the account the
// provisioner's client runs as here, empty where none runs (novox/hq ADR 0349 §4). With both, a moved module's
// templated files are handed to the client (handed.go); without either, it composes as before.
Hands bool
Provisioner string
}
// RootField is a user resource's field saying the account must never become root without a person
@@ -331,6 +347,15 @@ type Composed struct {
// naming its module and why (novox/hq ADR 0255): that piece is left out and the machine is told
// everything else. Push and plan say it; the merge gate refuses a change that adds one.
Unplaced []string
// HandedOnly is each file left out because only the provisioner's client fills it and the module is not handed on
// this machine (a provider's consumers file, ADR 0373), said once in the plan. Not an unplaced contribution, which
// the merge gate refuses: a module not handed composes as it did.
HandedOnly []string
// Filled is the path of every file handed to the provisioner, in the order composed: the declaration's
// filled files (novox/hq ADR 0349 §4, §5). Plans is each handed module's plan on this machine, by module.
// Both empty where nothing is handed, and the declaration is then what it was.
Filled []string
Plans map[string]Plan
}
// ForeseenSealed is what stands for an own secret a send will make, in a composition asked ahead of
@@ -390,8 +415,9 @@ func (r Resolution) Compose(with Rendering) (Composed, error) {
owner := map[string]string{}
received := map[string]map[string][]Contribution{}
leftOut := map[string]string{}
var foreseen, unplaced []string
resources, err := r.compose(with, owner, received, leftOut, &foreseen, &unplaced)
var foreseen, unplaced, handedOnly []string
handed := &handedFiles{plans: map[string]Plan{}}
resources, err := r.compose(with, owner, received, leftOut, &foreseen, &unplaced, &handedOnly, handed)
if err != nil {
return Composed{}, err
}
@@ -405,8 +431,13 @@ func (r Resolution) Compose(with Rendering) (Composed, error) {
})
}
sort.Strings(unplaced)
return Composed{Resources: resources, Owner: owner, Received: received, LeftOut: leftOut,
Foreseen: foreseen, Unplaced: unplaced}, nil
sort.Strings(handedOnly)
out := Composed{Resources: resources, Owner: owner, Received: received, LeftOut: leftOut,
Foreseen: foreseen, Unplaced: unplaced, HandedOnly: handedOnly, Filled: handed.paths}
if plans := handed.finished(r.Needs); len(plans) > 0 {
out.Plans = plans
}
return out, nil
}
// BusMembershipID names the resource carrying a machine's membership for the new bus, and
@@ -417,7 +448,7 @@ const BusMembershipPath = "/var/lib/mesh/membership-next.json"
func (r Resolution) compose(with Rendering, owner map[string]string,
received map[string]map[string][]Contribution, leftOut map[string]string,
foreseen, unplaced *[]string) ([]map[string]any, error) {
foreseen, unplaced, handedOnly *[]string, handed *handedFiles) ([]map[string]any, error) {
// **A setting is judged where it is stored, and an impossible one costs a module, not a
// machine** (novox/hq ADR 0163, rule 6). A definition that moved under a stored setting makes
// this module uncomposable; it is left out of the declaration — its held things kept, its
@@ -426,6 +457,11 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
left := r.LeftOut(with.Settings, with.Adopted)
kept := make([]Manifest, 0, len(r.Modules))
var stillBackedUp []Manifest
for module, why := range with.NotPlaced {
if _, already := left[module]; !already {
left[module] = why
}
}
for _, m := range r.Modules {
if why, isLeft := left[m.Module]; isLeft {
if leftOut != nil {
@@ -648,7 +684,36 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
"id": NeedID(name), "type": "file", "path": own.Path, "sealed": with.Needed[m.Module][name],
}))
}
// **A handed module's own secrets are the vault's** (design 57 V4, issue 647): each file holding one is
// declared placed and without a value, filled by the machine's provisioner client from the vault; the
// controller makes and seals none of them here.
handsSecrets := with.Handing(m)
for _, name := range sortedKeys(m.OwnSecrets.Plain()) {
if handsSecrets {
owner := m.SecretsOwner
if m.Module == RuntimeModule && r.Account != "" {
owner = r.Account
}
first = append(first, ownedBy(owner, map[string]any{
"id": NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, HandedSecretField: name,
}))
continue
}
// **A core credential is the vault's box** (ADR 0349 §16, issue 755): carried as the vault sealed it to this
// machine, which the controller cannot open; the controller's own sealed value is not sent with it.
if box, ok := with.vaultSealedOf(m.Module, name); ok {
if string(box) == string(ForeseenVaultBox) {
*foreseen = append(*foreseen, m.Module+"/"+name)
}
owner := m.SecretsOwner
if m.Module == RuntimeModule && r.Account != "" {
owner = r.Account
}
first = append(first, ownedBy(owner, map[string]any{
"id": NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, VaultSealedField: box,
}))
continue
}
sealed := with.Needed[m.Module][name]
if sealed == "" && with.Foreseen[m.Module][name] {
// Not made, and the next send makes it: composed with a stand-in so that whatever
@@ -718,6 +783,16 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
found = &r.Needs[i]
}
}
// **A credential the vault keeps for this holder alone** (issue 757): where the module is handed, its
// file is declared placed and without a value, and the machine's client fills it from the vault.
if found != nil && found.IssuedToHolder && with.Handing(m) {
first = append(first, ownedBy(m.SecretsOwner, map[string]any{
"id": SecretID(SecretLocal(to, file.Local)), "type": "file", "path": file.Path,
HandedProvisionField: PlanProvision{Name: SecretLocal(to, file.Local), Provision: to,
Local: file.Local, Provider: found.From, ProviderModule: found.Module},
}))
continue
}
if found != nil && found.ByRecord && found.Sealed == "" && !found.Manager {
// Answered by a record whose key has not been supplied since this consumer was
// put on it. **Refused, not skipped.** The mesh discarded the plaintext when the
@@ -837,7 +912,8 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
if err != nil {
return nil, err
}
first = append(first, file)
// Read by the module as its secrets' owner when it names one: the vault, as its own account (issue 723).
first = append(first, ownedBy(m.SecretsOwner, file))
}
if m.Computed != "" {
mine, part := generated[m.Module]
@@ -973,7 +1049,100 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
// ADR 0163, rule 4); judged above, so an invalid one is not here.
keptNetworks, _ := KeptNetworks(m, with.Settings[m.Module], with.Adopted)
// **A moved module's templated files are the provisioner's** (novox/hq ADR 0349 §4, §6, design 57 P6): on a
// machine that can take them, each is declared placed and without its text, and no pass below fills it.
handing, err := handed.begin(r.Node, m, dirs, accessPaths, thisMachine, with, r.Capabilities, r.Needs,
backupPlacer{append(append([]Manifest(nil), r.Modules...), stillBackedUp...), with, thisMachine, r.Capabilities,
unplaced})
if err != nil {
return nil, err
}
// **What the mesh calls the module through each provision it requires** (issue 792): the names its binding
// file carries, composed by the same call, handed to the provider through the plan's entry.
if _, ok := handed.plans[m.Module]; ok && handing != nil {
for _, to := range requirementsContributedTo(m) {
own, err := r.ownNames(m, to, with.Settings[m.Module])
if err != nil {
return nil, err
}
handed.name(m.Module, to, own)
}
}
// A handed collector is filled from items alone: each old form its collections would have held is said
// (design 57 L1).
if plan, ok := handed.plans[m.Module]; ok && handing != nil && len(plan.Collections) > 0 {
*unplaced = append(*unplaced, oldFormsLeftOut(append(append([]Manifest(nil), r.Modules...),
stillBackedUp...), m, plan.Collections)...)
}
for _, unsettled := range resources {
if pr, ok := unsettled[HandedProvisionField].(PlanProvision); ok {
if handing == nil {
return nil, fmt.Errorf("%s: its credential of %s is handed to the provisioner and the module is "+
"not: nothing would fill it", m.Module, pr.Provision)
}
marked := map[string]any{}
for k, v := range unsettled {
marked[k] = v
}
delete(marked, HandedProvisionField)
marked[HandedSecretField] = pr.Name
file, sec, err := handedSecretFile(marked, m, dirs, accessPaths, thisMachine, placed)
if err != nil {
return nil, err
}
file["id"] = m.Module + "." + fmt.Sprint(unsettled["id"])
owner[fmt.Sprint(file["id"])] = m.Module
pr.Path = sec.Path
if err := handed.addProvisionFile(m.Module, pr); err != nil {
return nil, err
}
out = append(out, file)
continue
}
if unsettled[HandedSecretField] != nil {
if handing == nil {
return nil, fmt.Errorf("%s: its own secret %v is handed to the provisioner and the module is "+
"not: nothing would fill it", m.Module, unsettled[HandedSecretField])
}
file, sec, err := handedSecretFile(unsettled, m, dirs, accessPaths, thisMachine, placed)
if err != nil {
return nil, err
}
file["id"] = m.Module + "." + fmt.Sprint(unsettled["id"])
owner[fmt.Sprint(file["id"])] = m.Module
if err := handed.addSecretFile(m.Module, sec); err != nil {
return nil, err
}
out = append(out, file)
continue
}
if handing != nil && fmt.Sprint(unsettled["type"]) == "file" {
if text, _ := unsettled["content"].(string); len(secretsUsed(text)) > 0 &&
!handing[fmt.Sprint(unsettled["id"])] {
return nil, fmt.Errorf("%s: its file %v names a secret in its text, and a handed module's "+
"secrets reach only the files its provisioner client fills: keep it in a template "+
"(design 57 V4)", m.Module, unsettled["id"])
}
}
if handing[fmt.Sprint(unsettled["id"])] {
file, line, err := handedFile(unsettled, m, dirs, accessPaths, thisMachine, placed, with)
if err != nil {
return nil, err
}
file["id"] = m.Module + "." + fmt.Sprint(unsettled["id"])
owner[fmt.Sprint(file["id"])] = m.Module
if err := handed.add(m.Module, line); err != nil {
return nil, err
}
out = append(out, file)
continue
}
if fmt.Sprint(unsettled["type"]) == "file" {
if _, left := consumersHandedOnly(m, unsettled, with); left {
*handedOnly = append(*handedOnly, consumersLeftOutLine(m, unsettled))
continue
}
}
resource, err := ApplySettings(unsettled, with.Settings[m.Module])
if err != nil {
return nil, err
@@ -1003,7 +1172,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
// **An operator's value, from the assignment** (novox/hq ADR 0112, ADR 0155): what a
// definition may not carry because it is true of one installation only. Filled from
// the same layers a mergeable file takes, and refused when no layer set it.
if err := settingInto(copied, WithDefaults(m, with.Settings[m.Module]), m.Module); err != nil {
if err := unsetInConfigWords(settingInto(copied, WithDefaults(m, with.Settings[m.Module]), m.Module), m); err != nil {
return nil, err
}
// **Placed before anything reads a path.** A pathless directory receives the path
@@ -1202,6 +1371,18 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
}
process["restart-on"] = restarts
}
// **And what a module of its own account is given, its own account reads** (issue 723): the files and
// directories its words name that say no owner, given to the account it runs as, never the operator's.
for _, m := range r.Modules {
if m.RunsAs == "" {
continue
}
w, err := bundleWords(m, with)
if err != nil {
return nil, err
}
givenTo(out, owner, map[string]map[string]string{m.Module: w}, m.RunsAs)
}
}
if with.Adopted {
// First, before anything a module declares: what the mesh needs reachable, then its guard.
@@ -1449,6 +1630,10 @@ type Contribution struct {
// Empty is root, which is what it was and what a module with no bundle and no declared owner
// still wants.
func (r Resolution) provisionsAs(m Manifest) string {
// A module of its own account reads them as that account (ADR 0259 §8, issue 723).
if m.RunsAs != "" {
return m.RunsAs
}
if len(m.Bundles) > 0 && r.Account != "" {
return r.Account
}
@@ -165,16 +165,19 @@ func TestTwoConsumersOfOneProviderGetTheirOwnDerivation(t *testing.T) {
}
}
// An operator may still set what the provider serves, and the mesh still derives the rest: the
// setting is laid on first, then the consumer's half is filled.
// An operator may still set what the provider serves where its definition asks (${setting:…}), and
// the mesh still derives the rest: the setting is filled first, then the consumer's half. (Until
// novox/hq issue 696 the layer replaced the served bucket whole; a layer replaces no served key now.)
func TestASettingComposesWithADerivedValue(t *testing.T) {
r, err := Resolve(shelf(store(), files()), []string{"store", "files"}, reachable(), World{})
prefixed := store()
prefixed.Serves["s3-bucket"]["bucket"] = "${setting:bucket-prefix}${consumer:as:dns}"
r, err := Resolve(shelf(prefixed, files()), []string{"store", "files"}, reachable(), World{})
if err != nil {
t.Fatal(err)
}
out, err := r.Declaration(Rendering{
Settings: SettingsBy{"store": {{From: "the operator",
Values: map[string]any{"bucket": "team-${consumer:as:dns}"}}}},
Values: map[string]any{"bucket-prefix": "team-"}}}},
Grants: []Grant{{Provision: "s3-bucket", Consumer: "workstation", From: "files", Slug: "files",
Values: map[string]any{}, Sealed: "c2VhbGVk"}},
})
+3 -3
View File
@@ -7,7 +7,7 @@ import (
func configured(block map[string]any) SettingsBy {
return SettingsBy{"media": {{From: "node anchor",
Values: map[string]any{EndpointsSetting: block}}}}
Placement: map[string]any{EndpointsSetting: block}}}}
}
// **One block per endpoint, saying all three things.** The machine port, the subdomain and the reach
@@ -109,7 +109,7 @@ func TestAReachInABlockIsHeldToTheFourValues(t *testing.T) {
// **Two places giving one endpoint a machine port is the confusion this key exists to end.**
func TestAnEndpointGivenAPortTwiceIsRefused(t *testing.T) {
m := aMediaServer()
_, err := GivenPorts(m, []Layer{{From: "node anchor", Values: map[string]any{
_, err := GivenPorts(m, []Layer{{From: "node anchor", Placement: map[string]any{
EndpointsSetting: map[string]any{"web": map[string]any{"port": 20009}},
PortsSetting: map[string]any{"80": 30000},
}}})
@@ -120,7 +120,7 @@ func TestAnEndpointGivenAPortTwiceIsRefused(t *testing.T) {
// And the same for its reach, said once here and once through the older key.
func TestAnEndpointWhoseReachIsAlsoExposedIsRefused(t *testing.T) {
_, err := Endpoints(aMediaServer(), []Layer{{From: "node anchor", Values: map[string]any{
_, err := Endpoints(aMediaServer(), []Layer{{From: "node anchor", Placement: map[string]any{
EndpointsSetting: map[string]any{"web": map[string]any{"reach": ReachInternal}},
ExposeSetting: map[string]any{"80": FromEverywhere},
}}})
+149 -17
View File
@@ -111,7 +111,7 @@ var (
ofEnvironment = regexp.MustCompile(`\$\{environment:([^}]*)\}`)
ofShell = regexp.MustCompile(`\$\{shell:([^}]*)\}`)
// ofContributed is either kind of contributed text, matched together so both fill in one pass.
ofContributed = regexp.MustCompile(`\$\{(shell|contribution):([^}]*)\}`)
ofContributed = regexp.MustCompile(`\$\{(shell|contribution|backup):([^}]*)\}`)
)
// variableName is a POSIX shell variable name, which is also what environment.d accepts.
@@ -226,6 +226,8 @@ func (m Manifest) contributionPlaceholderProblems() []string {
for _, r := range m.Resources {
problems = append(problems, placeholderProblems(m, r)...)
problems = append(problems, seatPlaceholderProblems(m, r)...)
problems = append(problems, collectedPlaceholderProblems(m, r)...)
problems = append(problems, consumersPlaceholderProblems(m, r)...)
// And every placeholder no pass fills where it stands: a misspelt namespace, a key its
// namespace cannot take, or a field its pass does not read (novox/hq issue 231).
problems = append(problems, unconsumedPlaceholders(m.Module, r)...)
@@ -353,10 +355,11 @@ func inModuleOrder(modules []Manifest) []Manifest {
func variablesSetOnce(modules []Manifest) error {
setter := map[string]string{}
for _, m := range inModuleOrder(modules) {
if m.Environment == nil {
e := environmentOf(m)
if e == nil {
continue
}
for _, n := range sortedKeys(m.Environment.Variables) {
for _, n := range sortedKeys(e.Variables) {
if first, taken := setter[n]; taken {
return fmt.Errorf(
"%s and %s both set %s on this machine; the account has one environment, so one "+
@@ -379,13 +382,14 @@ func environmentOn(modules []Manifest, facts map[string]string) (contributedEnvi
}
placed := map[string]bool{}
for _, m := range inModuleOrder(modules) {
if m.Environment == nil {
e := environmentOf(m)
if e == nil {
continue
}
if len(m.Environment.Variables) > 0 {
if len(e.Variables) > 0 {
set := setBy{module: m.Module, values: map[string]string{}}
for _, n := range sortedKeys(m.Environment.Variables) {
v, err := factsIn(m.Environment.Variables[n], facts, m.Module, n)
for _, n := range sortedKeys(e.Variables) {
v, err := factsIn(e.Variables[n], facts, m.Module, n)
if err != nil {
return env, err
}
@@ -394,7 +398,7 @@ func environmentOn(modules []Manifest, facts map[string]string) (contributedEnvi
}
env.variables = append(env.variables, set)
}
for _, p := range m.Environment.Path {
for _, p := range e.Path {
entry, err := factsIn(p.Entry, facts, m.Module, "a PATH entry")
if err != nil {
return env, err
@@ -417,6 +421,37 @@ func environmentOn(modules []Manifest, facts map[string]string) (contributedEnvi
return env, nil
}
// environmentOf is a module's environment as the old form and the items of the environment collection say it: an
// item's template is the variable's value or the PATH entry, and its attributes the kind, the name and where on PATH
// (design 57 L3, issue 795). A moved contributor's items are rendered here into a collector not handed, as the
// provisioner's client renders them into one that is. An item whose text the mesh does not hold is not here; it is
// said as not placed (itemsLeftOutFor).
func environmentOf(m Manifest) *Environment {
var out *Environment
if m.Environment != nil {
out = &Environment{Variables: map[string]string{}, Path: append([]PathEntry(nil), m.Environment.Path...)}
for k, v := range m.Environment.Variables {
out.Variables[k] = v
}
}
for _, it := range itemsInOrder(m.Items) {
if it.Collection != EnvironmentCollection || it.Content == "" {
continue
}
if out == nil {
out = &Environment{Variables: map[string]string{}}
}
value := strings.TrimSuffix(it.Content, "\n")
switch it.Attributes["kind"] {
case "variable":
out.Variables[it.Attributes["name"]] = value
case "path":
out.Path = append(out.Path, PathEntry{Entry: value, At: it.Attributes["at"]})
}
}
return out
}
// factsIn resolves a contributed value's ${machine:…} facts with this machine's — first, before
// either format is written, so both say the same thing (novox/hq ADR 0203).
func factsIn(v string, facts map[string]string, module, what string) (string, error) {
@@ -521,22 +556,91 @@ func entriesOf(entries []placedOn) string {
// order, each module's pieces in the order it declared them, each preceded by a line naming the
// module, and empty when nothing is contributed.
func shellCode(modules []Manifest, shell, slot string) string {
var b strings.Builder
return shellCodeOn(modules, shell, slot, nil)
}
// shellCodeOn is shellCode on a machine with these facts: a moved contributor's item of the slot's collection is
// rendered in its module's place as its old form was (heldItemText), so a collector not handed keeps every line.
func shellCodeOn(modules []Manifest, shell, slot string, facts map[string]string) string {
return shellCodeFor(modules, shell, slot, facts, nil)
}
// oldFormOfCollected is a file's text with each `${collected:contribution:<seat>:<kind>:…}` of a kind received as data
// read as the old `${contribution:<seat>:<kind>:…}`: where the collector is not handed, the controller renders the
// kind's items as it rendered its contributions, so the two forms place the same text (design 57 L3, issue 781).
func oldFormOfCollected(content string) string {
return ofCollected.ReplaceAllStringFunc(content, func(placeholder string) string {
name := ofCollected.FindStringSubmatch(placeholder)[1]
base, _ := splitSelector(name)
if format, is := strings.CutPrefix(base, EnvironmentCollection+":"); is {
return "${environment:" + format + "}"
}
if !strings.HasPrefix(base, "contribution:") {
return placeholder
}
seat, kind, _ := strings.Cut(strings.TrimPrefix(base, "contribution:"), ":")
if _, r, ok := receivable(seat, kind); !ok || !collectableAsText(r) {
return placeholder
}
return "${" + name + "}"
})
}
// shellCodeFor is shellCodeOn on a machine with these capabilities, which an item's `if-capability` is judged by. The
// pieces are placed by order (an item's `order` attribute, the default for the old form), then module, as the
// provisioner's client places them.
func shellCodeFor(modules []Manifest, shell, slot string, facts map[string]string, caps map[string]bool) string {
var blocks []textBlock
for _, m := range inModuleOrder(modules) {
named := false
var old strings.Builder
for _, c := range m.Shell {
if c.For != shell || c.Slot != slot {
continue
}
if !named {
fmt.Fprintf(&b, "# %s\n", m.Module)
named = true
if old.Len() == 0 {
fmt.Fprintf(&old, "# %s\n", m.Module)
}
b.WriteString(c.Code)
old.WriteString(c.Code)
if !strings.HasSuffix(c.Code, "\n") {
b.WriteString("\n")
old.WriteString("\n")
}
}
blocks = append(blocks, moduleBlocks(m, "#", old.String(), heldItems(m, "shell:"+shell+":"+slot, nil, facts, caps))...)
}
return joinBlocks(blocks)
}
// textBlock is a stretch of a collector's text and the order it is placed by.
type textBlock struct {
order int
text string
}
// moduleBlocks is one module's pieces of a placeholder: what its old form made (placed by the default order), and
// each of its held items under a line naming the module, as the provisioner's client writes one per item. An item
// of the default order that follows the old form's text shares its line, as a module's item always did.
func moduleBlocks(m Manifest, comment, old string, items []heldItem) []textBlock {
var blocks []textBlock
if old != "" {
blocks = append(blocks, textBlock{OrderDefault, old})
}
for i, h := range items {
text := h.text
if i == 0 && old != "" && h.item.order() == OrderDefault {
blocks[0].text += text
continue
}
blocks = append(blocks, textBlock{h.item.order(), fmt.Sprintf("%s %s\n", comment, m.Module) + text})
}
return blocks
}
// joinBlocks is the blocks by order, those of one order in the order given (module order).
func joinBlocks(blocks []textBlock) string {
sort.SliceStable(blocks, func(a, b int) bool { return blocks[a].order < blocks[b].order })
var b strings.Builder
for _, blk := range blocks {
b.WriteString(blk.text)
}
return b.String()
}
@@ -551,13 +655,34 @@ func shellCode(modules []Manifest, shell, slot string) string {
// is filled before the shell's code is, and each is replaced in a single pass over what the holder
// wrote, so a contributed piece is never scanned again.
func contributionsInto(resource map[string]any, m Manifest, modules []Manifest, facts map[string]string, with Rendering, caps map[string]bool, unplaced *[]string) error {
if problems := append(placeholderProblems(m, resource), seatPlaceholderProblems(m, resource)...); len(problems) > 0 {
problems := append(placeholderProblems(m, resource), seatPlaceholderProblems(m, resource)...)
if problems = append(problems, collectedPlaceholderProblems(m, resource)...); len(problems) > 0 {
return fmt.Errorf("%s", problems[0])
}
judged, err := consumersNotHanded(m, resource, with)
if err != nil {
return err
}
if judged {
resource["content"] = ofConsumers.ReplaceAllString(fmt.Sprint(resource["content"]), "")
}
content, ok := resource["content"].(string)
if !ok {
return nil
}
content = oldFormOfCollected(content)
// **An item is the provisioner client's** (design 57 L1): a collector filled here, the old way, holds the old
// forms alone, and each item of its collections is said as not placed. A collection of the new form is empty.
if unplaced != nil {
for _, f := range templatePlaceholder.FindAllStringSubmatch(content, -1) {
if collection, is := collectionOfPlaceholder(f[1], f[2]); is {
_, selector := splitSelector(f[2])
where, _ := selectorOf(selector)
*unplaced = append(*unplaced, itemsLeftOutFor(modules, m, collection, where, facts, caps)...)
}
}
}
content = ofCollected.ReplaceAllString(content, "")
if ofEnvironment.MatchString(content) {
env, err := environmentOn(modules, facts)
if err != nil {
@@ -578,6 +703,13 @@ func contributionsInto(resource map[string]any, m Manifest, modules []Manifest,
content = ofContributed.ReplaceAllStringFunc(content, func(placeholder string) string {
found := ofContributed.FindStringSubmatch(placeholder)
first, second, _ := strings.Cut(found[2], ":")
if found[1] == BackupNamespace {
placed, problem := backupPlacer{modules, with, facts, caps, unplaced}.placed(m, found[2], false)
if problem != "" && failed == nil {
failed = fmt.Errorf("%s names %s: %s", m.Module, placeholder, problem)
}
return placed
}
if found[1] == "contribution" {
placed, left, err := seatContributions(modules, m, found[2], with, facts, caps)
if err != nil && failed == nil {
@@ -588,7 +720,7 @@ func contributionsInto(resource map[string]any, m Manifest, modules []Manifest,
}
return placed
}
return shellCode(modules, first, second)
return shellCodeFor(modules, first, second, facts, caps)
})
if failed != nil {
return failed
@@ -0,0 +1,165 @@
package catalogue
import (
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
)
// The session environment is a collection of typed items (design 57 L3, issue 795): a variable is an item with
// kind=variable and name=<NAME>, a PATH entry one with kind=path and at=start|end, each a template whose text is the
// value or the entry. Where node-env is not handed the controller renders the items as the old form was rendered;
// where it is, it names the collection for the client, which fills the same files (mesh-catalog's provisioner client
// holds that half against the same golden files). testdata/environment holds the environment of the catalogue's five
// contributors as the old form composed it, with all five and with zsh alone.
type envDecl struct{ module, item, name, value string }
func environmentOfTheCatalogue() []envDecl {
return []envDecl{
{"adwaita", "gtk-theme", "GTK_THEME", "Adwaita:dark"},
{"adwaita", "gtk2-rc-files", "GTK2_RC_FILES", "/usr/share/themes/Adwaita-dark/gtk-2.0/gtkrc"},
{"adwaita", "qt-qpa-platformtheme", "QT_QPA_PLATFORMTHEME", "qt6ct"},
{"adwaita", "qt-style-override", "QT_STYLE_OVERRIDE", "Fusion"},
{"adwaita", "qt-select", "QT_SELECT", "6"},
{"adwaita", "xcursor-theme", "XCURSOR_THEME", "Adwaita"},
{"adwaita", "xcursor-size", "XCURSOR_SIZE", "24"},
{"clipmenu", "cm-selections", "CM_SELECTIONS", "clipboard"},
{"clipmenu", "cm-max-clips", "CM_MAX_CLIPS", "500"},
{"clipmenu", "cm-histlength", "CM_HISTLENGTH", "15"},
{"i3", "xdg-current-desktop", "XDG_CURRENT_DESKTOP", "i3"},
{"i3", "xdg-session-desktop", "XDG_SESSION_DESKTOP", "i3"},
{"xterm", "terminal", "TERMINAL", "xterm"},
{"zsh", "editor", "EDITOR", "vim"},
{"zsh", "visual", "VISUAL", "vim"},
{"zsh", "xdg-config-home", "XDG_CONFIG_HOME", "${machine:account-home}/.config"},
{"zsh", "path-local-bin", "path:10", "${machine:account-home}/.local/bin"},
{"zsh", "path-scripts", "path:20", "${machine:account-home}/scripts"},
{"zsh", "path-scripts-bin", "path:30", "${machine:account-home}/scripts/bin"},
}
}
const environmentCollector = `{"module":"node-env","version":"1","moved":true,
"claims":[{"name":"node-environment","scope":"node"}],
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},
"resources":[{"id":"posix","type":"file","path":"/home/op/.config/mesh/environment.sh","template":"files/posix"},
{"id":"systemd","type":"file","path":"/home/op/.config/environment.d/50-mesh.conf","template":"files/systemd"}]}`
// nodeEnvWith is node-env, its file placing the environment as plan says, with the declared contributors' items.
func nodeEnvWith(t *testing.T, decls []envDecl, plan string) Resolution {
t.Helper()
collector := held(t, environmentCollector, builtWith("node-env", map[string]string{
"posix": "# header\n" + plan, "systemd": "${environment:systemd}"}))
mods := []Manifest{collector}
order := []string{"node-env"}
byModule := map[string][]envDecl{}
var modules []string
for _, d := range decls {
if byModule[d.module] == nil {
modules = append(modules, d.module)
}
byModule[d.module] = append(byModule[d.module], d)
}
for _, name := range modules {
var items []string
texts := map[string]string{}
for _, d := range byModule[name] {
attrs := fmt.Sprintf(`{"kind":"variable","name":%q}`, d.name)
if order, isPath := strings.CutPrefix(d.name, "path:"); isPath {
attrs = fmt.Sprintf(`{"kind":"path","at":"start","order":%q}`, order)
}
items = append(items, fmt.Sprintf(`{"collection":"environment","name":%q,"attributes":%s,"template":"files/env-%s"}`,
d.item, attrs, d.item))
texts["env-"+d.item] = d.value + "\n"
}
raw := fmt.Sprintf(`{"module":%q,"version":"1","moved":true,"items":[%s],
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}}`, name, strings.Join(items, ","))
mods = append(mods, held(t, raw, builtWith(name, texts)))
order = append(order, name)
}
r, err := Resolve(shelf(mods...), order, workstation(), World{})
if err != nil {
t.Fatal(err)
}
r.Account = "op"
return r
}
func TestTheEnvironmentOfTheCatalogueIsTheSameHandedOrNot(t *testing.T) {
read := func(name string) string {
b, err := os.ReadFile(filepath.Join("testdata", "environment", name))
if err != nil {
t.Fatal(err)
}
return string(b)
}
for name, only := range map[string]string{"all": "", "zsh": "zsh"} {
var decls []envDecl
for _, d := range environmentOfTheCatalogue() {
if only == "" || d.module == only {
decls = append(decls, d)
}
}
// Named as the collector names it, and as the old placeholder does.
for _, plan := range []string{"${collected:environment:posix}", "${environment:posix}"} {
r := nodeEnvWith(t, decls, plan)
// Not handed: the controller renders each moved contributor's items as the old form was rendered.
if got, want := composedFile(t, r, "node-env.posix"), "# header\n"+read(name+".posix"); got != want {
t.Errorf("%s, %s, not handed, the POSIX file is\n%s\nwant\n%s", name, plan, got, want)
}
if got, want := composedFile(t, r, "node-env.systemd"), read(name+".systemd"); got != want {
t.Errorf("%s, %s, not handed, the environment.d file is\n%s\nwant\n%s", name, plan, got, want)
}
// Handed: one collection, named for the client; each contributor's plan names its items.
c, _ := composeWith(t, r, handingTo("store:5000"))
if got := c.Plans["node-env"].Collections; !reflect.DeepEqual(got, []PlanCollection{{Name: "environment", Comment: "#"}}) {
t.Errorf("%s: the plan collects %+v", name, got)
}
items, got := len(decls), 0
for m := range c.Plans {
got += len(c.Plans[m].Items)
}
if got != items {
t.Errorf("%s: the plans name %d items, want %d", name, got, items)
}
}
}
}
func TestAnEnvironmentItemIsTypedAndNamesWhatItNeeds(t *testing.T) {
item := func(attrs map[string]string) []string {
return environmentItemProblems("x", Item{Collection: "environment", Attributes: attrs})
}
for _, ok := range []map[string]string{
{"kind": "variable", "name": "GTK_THEME"},
{"kind": "path", "at": "end", "order": "10"},
} {
if p := item(ok); len(p) != 0 {
t.Errorf("%v is refused: %v", ok, p)
}
}
for _, bad := range []map[string]string{
nil, {"kind": "other"}, {"kind": "variable"}, {"kind": "variable", "name": "PATH"},
{"kind": "path"}, {"kind": "path", "at": "middle"}, {"kind": "path", "at": "start", "name": "X"},
{"kind": "variable", "name": "A", "color": "red"},
} {
if p := item(bad); len(p) == 0 {
t.Errorf("%v is taken", bad)
}
}
if !attributeFits("environment", "name", "GTK_THEME") || attributeFits("shell:zsh:normal", "name", "GTK_THEME") ||
attributeFits("environment", "kind", "Variable") {
t.Error("an upper-case name is the environment's name alone")
}
// A module that sets a variable twice, in two modules, is refused as the old form refused it.
a := Manifest{Module: "a", Items: []Item{{Collection: "environment", Content: "1\n",
Attributes: map[string]string{"kind": "variable", "name": "X"}}}}
b := Manifest{Module: "b", Items: []Item{{Collection: "environment", Content: "2\n",
Attributes: map[string]string{"kind": "variable", "name": "X"}}}}
if err := variablesSetOnce([]Manifest{a, b}); err == nil || !strings.Contains(err.Error(), "both set X") {
t.Errorf("two modules setting X: %v", err)
}
}
+4 -4
View File
@@ -132,7 +132,7 @@ func Exposure(m Manifest, layers []Layer) (map[int]string, error) {
out := map[int]string{}
for _, layer := range layers {
raw, ok := layer.Values[ExposeSetting]
raw, ok := layer.Placement[ExposeSetting]
if !ok {
continue
}
@@ -651,7 +651,7 @@ func GivenPorts(m Manifest, layers []Layer) (map[int]int, error) {
out := map[int]int{} // the port the module names → the machine port it is on
by := map[int]int{} // and which of the setting's ports put it there, for the refusal
for _, layer := range layers {
raw, ok := layer.Values[PortsSetting]
raw, ok := layer.Placement[PortsSetting]
if !ok {
continue
}
@@ -951,7 +951,7 @@ func Reaches(m Manifest, layers []Layer) (map[int]string, error) {
out[declared[name]] = ep.Reach
}
for _, layer := range layers {
raw, ok := layer.Values[ReachSetting]
raw, ok := layer.Placement[ReachSetting]
if !ok {
continue
}
@@ -1069,7 +1069,7 @@ func Endpoints(m Manifest, layers []Layer) (map[string]Endpoint, error) {
out := map[string]Endpoint{}
for _, layer := range layers {
raw, ok := layer.Values[EndpointsSetting]
raw, ok := layer.Placement[EndpointsSetting]
if !ok {
continue
}
+1 -1
View File
@@ -752,7 +752,7 @@ func TestExposureSettingOverridesAListensSource(t *testing.T) {
func TestExposureRefusesAPortNotListenedOnAndABadSource(t *testing.T) {
postgres := Manifest{Module: "postgres", Listens: []Listening{{Port: 5432, From: FromMesh}}}
layer := func(port, source string) []Layer {
return []Layer{{From: "node", Values: map[string]any{ExposeSetting: map[string]any{port: source}}}}
return []Layer{{From: "node", Placement: map[string]any{ExposeSetting: map[string]any{port: source}}}}
}
if _, err := Exposure(postgres, layer("5432", FromEverywhere)); err != nil {
@@ -100,7 +100,7 @@ func TestTheForgesPortIsGivenLikeAnyOtherProvidersPort(t *testing.T) {
// The catalogue's number is a default and the node's setting moves it.
given, err := GivenPorts(forge, []Layer{{From: "anchor",
Values: map[string]any{PortsSetting: map[string]any{"3000": float64(3100)}}}})
Placement: map[string]any{PortsSetting: map[string]any{"3000": float64(3100)}}}})
if err != nil {
t.Fatalf("the forge's port cannot be given on a node: %v", err)
}
@@ -298,7 +298,7 @@ func TestTheForgesSshPortIsTheMeshsFixedConventionByDefault(t *testing.T) {
func TestANodeMayGiveTheForgesSshPortADifferentNumber(t *testing.T) {
forge := catalogueManifest(t, "gitea")
given, err := GivenPorts(forge, []Layer{{From: "anchor",
Values: map[string]any{PortsSetting: map[string]any{"22": float64(9022)}}}})
Placement: map[string]any{PortsSetting: map[string]any{"22": float64(9022)}}}})
if err != nil {
t.Fatalf("the forge's ssh port cannot be moved on a node: %v", err)
}
+1
View File
@@ -181,6 +181,7 @@ type barOption struct {
// - shows: battery or command; options: what that kind takes.
func barBlockShape() *Shape {
return &Shape{
Text: true,
Places: []PlaceField{{Field: "bar", Values: []string{"bottom", "top"}},
{Field: "place", Values: []string{"resources", "status"}}},
Order: "order",
+799
View File
@@ -0,0 +1,799 @@
package catalogue
import (
"errors"
"fmt"
"maps"
"regexp"
"slices"
"sort"
"strings"
)
// A module handed to the provisioner (novox/hq ADR 0349 clauses 2, 4, 6 and 17, design 57 P6, issue 672).
//
// **Marked in its manifest, one module at a time.** A module whose manifest says `"moved": true` has moved: each
// file of it that names a template is filled by the provisioner's client on the node, from the templates archive
// its build packed, and the controller fills nothing in it. Every other resource of the module, and every file of
// a module not marked, is composed exactly as before (clause 17: each file the old way or the new way, never both).
//
// **The build keeps what the client needs.** Resolving a moved module's build keeps, beside the files' text the
// mesh holds, where its templates archive is and what it hashes to, and which template each file is filled from
// (Handed). The text stays, so a machine whose node-engine cannot take a handed file is still sent the module the
// old way, byte for byte.
//
// **Handed only where the machine can take it.** A node's declaration hands a moved module's files over only when
// its node-engine writes a filled file on the client's word alone (Rendering.Hands) and the node runs a provisioner
// client (Rendering.Provisioner): then each such file is declared with its path, mode, owner and who owns it at
// runtime and without its text, the declaration names it among the filled files, and the module's plan says the
// client where its templates are and the value of every placement placeholder they name. Anywhere else the module
// composes as if it were not marked.
// MovedField is the manifest's key marking a module moved.
const MovedField = "moved"
// Handed is what a moved module's build kept for the provisioner (novox/hq ADR 0349 §2): its templates archive, as
// recorded (store-relative, routed at composition like any archive), and its digest; and, for each file resource
// filled from a template, that template's path inside the archive. Written by Resolve, never by hand.
type Handed struct {
Source string `json:"source"`
Digest string `json:"digest"`
Templates map[string]string `json:"templates"`
// Items is, for each item the module contributes, by ItemKey (its collection, and `/<name>` for a named one),
// that item's template inside the archive (collections.go, design 57 L1).
Items map[string]string `json:"items,omitempty"`
}
// Plan is a moved module's plan on one node: what the node's provisioner client fills its files from. The shape
// of the assignments mirror's `provisioning` (mesh-controller internal/inventory Provisioning).
type Plan struct {
Templates PlanTemplates `json:"templates"`
Files []PlanFile `json:"files"`
Placement map[string]string `json:"placement,omitempty"`
// Secrets are the module's own secrets this declaration names (design 57 V4, issue 647): each its templates
// name, and each it keeps in a file of its own, with that file's path. By name, never a value: the client asks
// the vault for each, and the vault hands one only when the plan names it.
Secrets []PlanSecret `json:"secrets,omitempty"`
// Items are the module's items of collections (design 57 L1): each filled by the client like a file and recorded
// in mesh-provisioner_collect for its node. Collections are the collections its files name, as their collector,
// each with its comment. By template and name, never a contributor's text.
Items []PlanItem `json:"items,omitempty"`
Collections []PlanCollection `json:"collections,omitempty"`
// Provisions are what it requires here, each with the provider resolution chose to serve it (issue 760), so the
// vault sees a consumer bind to a provider and unbind from it (novox/hq ADR 0349 clause 8); and, for a
// credential the vault keeps for it alone, the name its files call it and the file holding it (issue 757). By
// name, never a value.
Provisions []PlanProvision `json:"provisions,omitempty"`
}
// PlanItem is one item the module contributes: to which collection, and its template inside the archive.
//
// A named item has its Name and Attributes (design 57 L3, issue 777): the client records them with the item and
// places it under a placeholder selecting by them. An item whose `if-capability` the machine lacks is not in the plan.
type PlanItem struct {
Collection string `json:"collection"`
Name string `json:"name,omitempty"`
Attributes map[string]string `json:"attributes,omitempty"`
Template string `json:"template"`
}
// PlanCollection is one collection the module collects, and the comment introducing each contributor's item.
type PlanCollection struct {
Name string `json:"name"`
Comment string `json:"comment,omitempty"`
}
// PlanProvision is one provision the module requires, under one local name, and the provider serving it: the module
// (ProviderModule) on the machine (Provider) that resolution chose for this consumer. Where several providers offer
// a mesh-scoped provision, it is the one this consumer is bound to (a pin, the seat's holder, or where its data is),
// never the others (issue 760). Local is the name the consumer keeps the credential under where it keeps several
// from one requirement (ADR 0094), empty for the ordinary one; a consumer has one provider per provision and local
// name.
//
// Name and Path are set only for a credential the vault keeps for this holder alone, which its machine's client
// fills (issue 757): the value the provider's software issued for this module on this machine. Name is what the
// module's files call it (`${secret:<name>}`) and Path the file holding it alone.
//
// CredentialForm is the form the provider's offer declares for its credential (Manifest.CredentialFormOf, issue
// 768): the vault makes a first value in that form for a provider it holds no row of. Empty where the provider's
// manifest was not in hand, and the vault then makes nothing.
type PlanProvision struct {
Name string `json:"name,omitempty"`
Provision string `json:"provision"`
Local string `json:"local,omitempty"`
Provider string `json:"provider"`
ProviderModule string `json:"provider-module"`
CredentialForm string `json:"credential-form,omitempty"`
Path string `json:"path,omitempty"`
// Contributes is what the consumer's manifest tells its provider for this provision (Manifest.Contributes: a
// database's name, its extensions), so the provider's helper learns it from the bus (issue 776). Nil where the
// manifest contributes nothing for it, and the entry is then as it was.
Contributes map[string]any `json:"contributes,omitempty"`
// At, From and As are what today's binding file tells the consumer of this provision (issue 788): where the
// provider is on the private network, the machine, and who the consumer is at its end (its slug first), so the
// provider's helper writes them into the consumer's entry and the client fills `${bound:<provision>:at|from|as}`
// from there. Empty where the resolution knew none.
At string `json:"at,omitempty"`
From string `json:"from,omitempty"`
As string `json:"as,omitempty"`
// Names are what the mesh calls the consumer through this provision (issue 792), keyed as its binding file names
// them (`name`, `internal-name`, `name-<local>`, `internal-name-<local>`): composed by the controller, which
// alone knows the node's domains, and carried unchanged by the provider's helper into the consumer's entry. Nil
// where the consumer contributes nothing to the provision, and the entry is then as it was.
Names map[string]string `json:"names,omitempty"`
}
// provisionsOf is, from a resolution's needs, each provision module requires and its provider, by provision then
// local name. A need answered by a record (a licence) or by no module names no provider on a machine and is left
// out: the vault has no provider's value to make for it.
func provisionsOf(needs []Needed, module, as string, contributes map[string]map[string]any,
named map[string]bool) []PlanProvision {
var out []PlanProvision
for _, n := range needs {
if n.For != module || n.ByRecord || n.Module == "" || n.From == "" {
continue
}
p := PlanProvision{Provision: n.Name, Local: n.Local, Provider: n.From, ProviderModule: n.Module,
CredentialForm: n.CredentialForm, Contributes: maps.Clone(contributes[n.Name]),
At: n.At, From: n.From, As: as}
// **A credential only a template names** (issue 788): named like a file's, so the client fills it from the
// vault's row for this holder.
if (named[n.Name] || named[SecretLocal(n.Name, n.Local)]) && n.IssuedToHolder {
p.Name = SecretLocal(n.Name, n.Local)
}
out = append(out, p)
}
return out
}
// sortProvisions orders a plan's provisions by provision, then local name.
func sortProvisions(ps []PlanProvision) {
sort.SliceStable(ps, func(i, j int) bool {
if ps[i].Provision != ps[j].Provision {
return ps[i].Provision < ps[j].Provision
}
return ps[i].Local < ps[j].Local
})
}
// PlanSecret is one of the module's own secrets, by its name in the manifest; Path is the file holding it alone,
// when the manifest's own-secrets places one.
type PlanSecret struct {
Name string `json:"name"`
Path string `json:"path,omitempty"`
}
// PlanTemplates is where the templates archive is fetched from on this node, and what it hashes to.
type PlanTemplates struct {
Source string `json:"source"`
Digest string `json:"digest"`
}
// PlanFile is one file the client fills.
type PlanFile struct {
ID string `json:"id"`
Path string `json:"path"`
Template string `json:"template"`
OwnerAtRuntime string `json:"owner-at-runtime"`
Takes string `json:"takes,omitempty"`
}
// The words a placed file says who owns it at runtime with (novox/hq ADR 0349 §5).
const (
RuntimeOwned = "owned"
RuntimeSeed = "seed"
RuntimeManaged = "managed"
)
// OwnerAtRuntimeField is the key a declared file says who owns it at runtime by (the node-engine's
// `owner-at-runtime`).
const OwnerAtRuntimeField = "owner-at-runtime"
// ownershipOf is who owns a file at runtime, by its form, as the node-engine maps the old forms (mesh-host
// declaration.File.Ownership): written into is managed, created once is a seed, any other is owned.
func ownershipOf(r map[string]any) string {
switch {
case r["into"] != nil && fmt.Sprint(r["into"]) != "":
return RuntimeManaged
case r["create-once"] == true:
return RuntimeSeed
}
return RuntimeOwned
}
// takesOf is how the program reading a file takes a change, by what names the file among the module's resources:
// restart-on wins over reload-on; empty when nothing names it (the program reads it when it uses it).
func takesOf(m Manifest, id string) string {
takes := ""
for _, r := range m.Resources {
if slices.Contains(stringsIn(r["restart-on"]), id) {
return "restart-on"
}
if slices.Contains(stringsIn(r["reload-on"]), id) {
takes = "reload-on"
}
}
return takes
}
// movedProblems is what is wrong with a module's mark: a moved module must name a template (as written) or carry
// what its build kept (as held), and Handed is the build's word, never a repository's.
func movedProblems(m Manifest) []string {
switch {
case m.Handed != nil && !m.Moved:
return []string{fmt.Sprintf("%s carries what a moved module's build keeps (handed) and is not marked moved",
m.Module)}
case m.Handed != nil && m.Build != nil:
return []string{fmt.Sprintf("%s says handed in its repository: that is what its build keeps, never "+
"written by hand (novox/hq ADR 0349 §2)", m.Module)}
case m.Moved && m.Handed == nil && !m.Templated():
return []string{fmt.Sprintf("%s is marked moved and names no template: a moved module's files are filled "+
"from template files of its own, `\"template\": \"%s/<name>\"` (novox/hq ADR 0349 §2)", m.Module,
m.TemplatesDir())}
}
if !m.Moved {
return nil
}
// **A moved module's secrets are the vault's, handed to its machine's client** (design 57 V4): a secret the
// mesh makes, by name. A family's members and a value issued outside the mesh are given by a person and may be
// absent, which the client cannot tell from one the vault does not hold yet; they move with a package of their
// own.
var problems []string
if fams := m.OwnSecrets.Families(); len(fams) > 0 {
problems = append(problems, fmt.Sprintf("%s is marked moved and keeps a secret family (%s): a moved module's "+
"secrets are handed by the vault by name, and a family's members are not yet (design 57 V4)", m.Module,
strings.Join(fams, ", ")))
}
for _, name := range sortedKeys(m.OwnSecrets.Plain()) {
if m.OwnSecrets[name].IssuedBy == IssuedOutside {
problems = append(problems, fmt.Sprintf("%s is marked moved and its secret %s is issued outside the mesh: "+
"a value that may not be given is not yet handed by the vault (design 57 V4)", m.Module, name))
}
}
return problems
}
// handedBy is what a moved module's build keeps, from its templates archive as built, or nil for a module not
// moved.
func handedBy(m Manifest, archive Built) *Handed {
if !m.Moved {
return nil
}
dir := m.TemplatesDir()
h := &Handed{Source: Recorded(archive.Reference), Digest: archive.Digest, Templates: map[string]string{}}
for _, r := range m.Resources {
name, has, wrong := templateOf(m.Module, dir, r)
if !has || wrong != "" {
continue
}
h.Templates[fmt.Sprint(r["id"])] = strings.TrimPrefix(name, dir+"/")
}
for _, it := range m.Items {
if it.Template == "" || itemTemplateProblem(m, it.Template) != "" {
continue
}
if h.Items == nil {
h.Items = map[string]string{}
}
h.Items[it.key()] = strings.TrimPrefix(it.Template, dir+"/")
}
return h
}
// handing is whether this composition hands the module's templated files to the provisioner.
func (with Rendering) handing(m Manifest) bool {
return m.Moved && m.Handed != nil && with.Hands && with.Provisioner != ""
}
// Handing is whether a machine composed with this rendering hands the module to the provisioner, its files and
// its own secrets (design 57 P6, V4): a moved module there, assigned or only pulled in by another (issue 765), on a
// machine that can take it. The controller makes and seals none of such a module's secrets for that machine: its
// client asks the vault.
func (with Rendering) Handing(m Manifest) bool { return with.handing(m) }
// requiredByWords begins why a module only pulled in by another is on a machine (Resolution.Because): `required by
// <module>`.
const requiredByWords = "required by "
// RequiredBy is the module that requires module on this machine when nothing assigned it there, only pulled it in
// (issue 765): what its value in the assignments mirror names. Empty for a module assigned there, or not there.
func (r Resolution) RequiredBy(module string) string {
by, _ := strings.CutPrefix(r.Because[module], requiredByWords)
if by == r.Because[module] {
return ""
}
return by
}
// HandedProvisionField marks, between composing a handed module's credential file for something it requires and
// placing it, one the vault keeps for this holder alone (issue 757): its value is the PlanProvision without its path.
// Never on the wire.
const HandedProvisionField = "handed-provision"
// HandedSecretField marks, between composing a module's own secret files and placing them, the one a handing
// composition hands to the provisioner: its value is the secret's name. Never on the wire.
const HandedSecretField = "handed-secret"
// handedSecretFile is a handed module's own secret file as the declaration carries it: placed, the secret's
// alone (0600 unless the manifest says otherwise), owned at runtime, without a value; the client fills it from the
// vault.
func handedSecretFile(r map[string]any, m Manifest, dirs, accessPaths, facts map[string]string,
placed map[string]Placement) (map[string]any, PlanSecret, error) {
name := fmt.Sprint(r[HandedSecretField])
out := map[string]any{}
for k, v := range r {
out[k] = v
}
delete(out, HandedSecretField)
delete(out, "sealed")
if err := dirInto(out, dirs, m.Module); err != nil {
return nil, PlanSecret{}, err
}
if err := accessInto(out, accessPaths, m.Module); err != nil {
return nil, PlanSecret{}, err
}
ownerInto(out, placed)
if err := machineInto(out, facts, m.Module); err != nil {
return nil, PlanSecret{}, err
}
if out["mode"] == nil {
out["mode"] = "0600"
}
out[OwnerAtRuntimeField] = RuntimeOwned
return out, PlanSecret{Name: name, Path: fmt.Sprint(out["path"])}, nil
}
// templatePlaceholder is a placeholder as a template writes it: its namespace and its key.
var templatePlaceholder = regexp.MustCompile(`\$\{([a-z][a-z-]*):([^}\s$]+)\}`)
// configNamespaces are the namespaces the config domain answers: `config`, and `setting`, which the manifest's
// reading writes it as (toSettingsForm).
var configNamespaces = []string{"config", SettingField}
// SettingField is the settings form's word for a config placeholder's namespace.
const SettingField = "setting"
// mergesFile is whether a file resource says `merge` or `takes-settings`: the layers' settings are merged into it.
func mergesFile(r map[string]any) bool {
_, merges := r["merge"]
_, takes := r[TakesSettingsField]
return merges || takes
}
// mergingIn is whether the module's resource id is a merging file (mergesFile).
func mergingIn(m Manifest, id string) bool {
for _, r := range m.Resources {
if fmt.Sprint(r["id"]) == id {
return mergesFile(r)
}
}
return false
}
// handedFile is a moved module's templated file as a handing declaration carries it: placed (its path and owner
// filled by placement alone), saying who owns it at runtime, without its text. And its line of the plan.
func handedFile(r map[string]any, m Manifest, dirs, accessPaths, facts map[string]string, placed map[string]Placement,
with Rendering) (map[string]any, PlanFile, error) {
id := fmt.Sprint(r["id"])
out := map[string]any{}
for k, v := range r {
out[k] = v
}
// **No text, and nothing filled into it**: the client fills it from the template; the declaration says where
// it goes and how it is written.
delete(out, "content")
delete(out, TrustedField)
delete(out, NamesOnPurpose)
delete(out, SecretsInEnvironment)
if err := dirInto(out, dirs, m.Module); err != nil {
return nil, PlanFile{}, err
}
if err := accessInto(out, accessPaths, m.Module); err != nil {
return nil, PlanFile{}, err
}
ownerInto(out, placed)
if err := machineInto(out, facts, m.Module); err != nil {
return nil, PlanFile{}, err
}
out[OwnerAtRuntimeField] = ownershipOf(out)
path := fmt.Sprint(out["path"])
return out, PlanFile{ID: id, Path: path, Template: m.Handed.Templates[id],
OwnerAtRuntime: fmt.Sprint(out[OwnerAtRuntimeField]), Takes: takesOf(m, id)}, nil
}
// placementOf is the value of every placement placeholder the module's handed templates name, keyed as the
// placeholder names it (`machine:account`), each filled by the same pass that fills it in a declaration. A
// template naming a value only another domain answers — a secret, a binding, a contribution — is refused by
// name: the client fills config and placement, and the rest move with their own packages (design 57).
func placementOf(node string, m Manifest, texts, items map[string]string, dirs, accessPaths, facts map[string]string,
with Rendering, moved map[string]bool, backups backupPlacer) (map[string]string, []string, []string, []PlanCollection, error) {
out := map[string]string{}
var secrets, required []string
var problems []string
var collections []PlanCollection
all := map[string]string{}
templateOf := map[string]string{}
for id, text := range texts {
all[id], templateOf[id] = text, m.Handed.Templates[id]
}
for key, text := range items {
all["item:"+key], templateOf["item:"+key] = text, m.Handed.Items[key]
}
judgedConsumers := map[string]bool{}
for _, id := range sortedKeys(all) {
item := strings.HasPrefix(id, "item:")
for _, found := range templatePlaceholder.FindAllStringSubmatch(all[id], -1) {
kind, key := found[1], found[2]
if collection, is := collectionOfPlaceholder(kind, key); is {
// **A collection its file names, read by the client from the items there** (design 57 L1).
comment, err := collectionComment(m, collection)
_, selector := splitSelector(key)
_, selErr := selectorOf(selector)
switch {
case selErr != nil && !item:
problems = append(problems, fmt.Sprintf("%s: its template %s names %s: %v", m.Module,
templateOf[id], found[0], selErr))
case item:
problems = append(problems, fmt.Sprintf("%s: its item's template %s names %s: an item is not a "+
"collector", m.Module, templateOf[id], found[0]))
case err != nil:
problems = append(problems, fmt.Sprintf("%s: its template %s names %s: %v", m.Module,
templateOf[id], found[0], err))
case !slices.ContainsFunc(collections, func(c PlanCollection) bool { return c.Name == collection }):
collections = append(collections, PlanCollection{Name: collection, Comment: comment})
}
continue
}
switch {
case slices.Contains(configNamespaces, kind):
continue
case kind == ConsumersNamespace:
// **A provider's consumers' credentials** (ADR 0373): its machine's client fills them from the vault,
// for a provision the module provides, in a file only it reads (consumersPlaceholderProblems).
if item {
problems = append(problems, fmt.Sprintf("%s: its item's template %s names %s: an item is kept in a "+
"bucket, where no secret is (novox/hq ADR 0349 §7)", m.Module, templateOf[id], found[0]))
} else if !judgedConsumers[id] {
judgedConsumers[id] = true
problems = append(problems, consumersOfTemplate(m, id, all[id])...)
}
continue
case item && (kind == "secret" || kind == "bound"):
problems = append(problems, fmt.Sprintf("%s: its item's template %s names %s: an item is kept in a "+
"bucket, where no secret is (novox/hq ADR 0349 §7)", m.Module, templateOf[id], found[0]))
continue
case kind == "secret":
// **The module's own secret, by name** (design 57 V4): its machine's client asks the vault for it.
// A requirement's credential is a binding's, and moves with the providers (design 57 B1–B3).
if _, own := m.OwnSecrets.Plain()[key]; !own && moved[key] {
// **A credential it requires, from a provider that has moved** (issue 788): the client fills it
// from the vault's row for this holder, which the plan names.
if !slices.Contains(required, key) {
required = append(required, key)
}
} else if !own {
problems = append(problems, fmt.Sprintf("%s: its template %s names %s, which is no secret of its "+
"own; a credential it requires moves with its provider (design 57 B1–B3)", m.Module,
templateOf[id], found[0]))
} else if !slices.Contains(secrets, key) {
secrets = append(secrets, key)
}
continue
case kind == "bound" && moved[strings.SplitN(key, ":", 2)[0]]:
// **What its moved provider told it** (issue 788): the client fills it from the provider's entry.
continue
case !slices.Contains(PlacementKinds, kind):
if known(kind) {
problems = append(problems, fmt.Sprintf("%s: its template %s names %s, and the provisioner's "+
"client fills only config, secrets, collections and placement until that domain moves "+
"(design 57)", m.Module, templateOf[id], found[0]))
}
continue
}
name := kind + ":" + key
if _, done := out[name]; done {
continue
}
if kind == BackupNamespace {
// **What the machine's modules back up** (design 57 L3, issue 794): the text the old contribution
// placeholders render, from the same function, over the modules placed on this machine.
value, problem := backups.placed(m, key, item)
if problem != "" {
problems = append(problems, fmt.Sprintf("%s: its template %s names %s: %s", m.Module,
templateOf[id], found[0], problem))
continue
}
out[name] = value
continue
}
value, err := placed(node, found[0], m, dirs, accessPaths, facts, with)
if err != nil {
problems = append(problems, err.Error())
continue
}
out[name] = value
}
}
if len(problems) > 0 {
sort.Strings(problems)
return nil, nil, nil, nil, errors.New(strings.Join(problems, "; "))
}
sort.Slice(collections, func(a, b int) bool { return collections[a].Name < collections[b].Name })
return out, secrets, required, collections, nil
}
// PlacementKinds are the placeholder namespaces placement answers (ADR 0349 clause 2), as a plan's keys begin.
var PlacementKinds = []string{"dir", "access", "port", "seat", "machine", BackupNamespace}
// known is whether a namespace is one the mesh fills somewhere.
func known(kind string) bool {
for _, f := range fillers {
if f.namespace == kind {
return true
}
}
return false
}
// placed is one placement placeholder's value, filled by the passes that fill it in a file's content.
func placed(node, placeholder string, m Manifest, dirs, accessPaths, facts map[string]string, with Rendering) (string, error) {
r := map[string]any{"id": "placement", "type": "file", "path": "/placement", "content": placeholder}
if err := dirInto(r, dirs, m.Module); err != nil {
return "", err
}
if err := accessInto(r, accessPaths, m.Module); err != nil {
return "", err
}
if err := portInto(r, m.Module, m.Listens, with); err != nil {
return "", err
}
if err := seatInto(r, m.Module, with); err != nil {
return "", err
}
if err := machineInto(r, facts, m.Module); err != nil {
return "", err
}
value := fmt.Sprint(r["content"])
if value == placeholder {
return "", fmt.Errorf("%s: %s is not answered by placement on %s", m.Module, placeholder, node)
}
return value, nil
}
// routedTemplates is the templates archive as this node fetches it, through the artifact store it reaches.
func routedTemplates(m Manifest, with Rendering) (PlanTemplates, error) {
r := map[string]any{"id": "templates", "source": m.Handed.Source}
if err := artifactsInto(r, m.Module, with); err != nil {
return PlanTemplates{}, err
}
return PlanTemplates{Source: fmt.Sprint(r["source"]), Digest: m.Handed.Digest}, nil
}
// handedFiles gathers, over one composition, the files handed to the provisioner and each handed module's plan.
type handedFiles struct {
paths []string
plans map[string]Plan
// secrets is each handed module's own secrets its declaration names, by name: the path of the file holding one
// alone, or "" for one only its templates name.
secrets map[string]map[string]string
// provisions is each handed module's credentials of what it requires that the vault keeps for it (issue 757).
provisions map[string][]PlanProvision
// contributes is each handed module's manifest `contributes`, by the requirement it is about (issue 776).
contributes map[string]map[string]map[string]any
// identities is who each handed module is at its providers' ends (issue 788), and required the credentials of
// what it requires that its templates name.
identities map[string]string
required map[string]map[string]bool
// names is the names composed for each handed module through each requirement it contributes to (issue 792).
names map[string]map[string]map[string]string
}
// name records the names composed for a handed module through one requirement, as its binding file carries them.
func (h *handedFiles) name(module, provision string, own map[string]any) {
values := map[string]string{}
withOwnNames(values, own)
if len(values) == 0 {
return
}
if h.names == nil {
h.names = map[string]map[string]map[string]string{}
}
if h.names[module] == nil {
h.names[module] = map[string]map[string]string{}
}
h.names[module][provision] = values
}
// begin starts a module's plan where this composition hands it, and answers the ids of the files handed: none for
// a module that is not handed here. The plan's templates and placement values are composed now; each file is added
// as it is composed.
//
// **Assigned or only pulled in by another** (issue 765): the plan travels in the module's value of the assignments
// mirror, `<node>.<module>`, which the client reads (inventory.Provisioning); a module pulled in has a value there of
// its own, which says what pulled it in.
func (h *handedFiles) begin(node string, m Manifest, dirs, accessPaths, facts map[string]string,
with Rendering, caps map[string]bool, needs []Needed, backups backupPlacer) (map[string]bool, error) {
if !with.handing(m) {
return nil, nil
}
texts := map[string]string{}
ids := map[string]bool{}
for _, r := range m.Resources {
id := fmt.Sprint(r["id"])
if _, named := m.Handed.Templates[id]; !named {
continue
}
if mergesFile(r) {
// **A merging file is filled the old way** (issue 831, ADR 0349 clause 17): handing it needs the client
// to merge the layers, which nothing builds yet. Its text stays, the controller fills and merges it.
continue
}
if fmt.Sprint(r["type"]) != "file" {
return nil, fmt.Errorf("%s: %s is a %v and its build kept a template for it; only a file is filled "+
"from one", m.Module, id, r["type"])
}
text, _ := r["content"].(string)
texts[id] = text
ids[id] = true
}
for _, id := range sortedKeys(m.Handed.Templates) {
if !ids[id] && !mergingIn(m, id) {
return nil, fmt.Errorf("%s: its build kept a template for %s, which it does not declare", m.Module, id)
}
}
// **Its items, by template** (design 57 L1): the client fills each and records it for this node.
items := map[string]string{}
var planned []PlanItem
for _, it := range m.Items {
template, kept := m.Handed.Items[it.key()]
if !kept {
return nil, fmt.Errorf("%s: its build kept no template for its item of %s", m.Module, it.key())
}
// **A capability the machine lacks leaves the item out of its plan** (design 57 L3): the client judges none.
if it.IfCapability != "" && !caps[it.IfCapability] {
continue
}
items[it.key()] = it.Content
planned = append(planned, PlanItem{Collection: it.Collection, Name: it.Name,
Attributes: it.Attributes, Template: template})
}
if len(ids) == 0 && len(planned) == 0 {
return nil, nil
}
templates, err := routedTemplates(m, with)
if err != nil {
return nil, err
}
// What it requires from a provider that has moved (issue 788): its templates may name what that provider tells it.
moved := map[string]bool{}
for _, n := range needs {
if n.For == m.Module && n.ProviderMoved {
moved[n.Name] = true
// A credential kept per local name (issue 793) is named by its local in a template.
moved[SecretLocal(n.Name, n.Local)] = true
}
}
placement, secrets, required, collections, err := placementOf(node, m, texts, items, dirs, accessPaths, facts,
with, moved, backups)
if err != nil {
return nil, err
}
if h.identities == nil {
h.identities = map[string]string{}
h.required = map[string]map[string]bool{}
}
h.identities[m.Module] = ConsumerIdentity(node, IdentitySource(m.Slug, m.Module))
h.required[m.Module] = map[string]bool{}
for _, name := range required {
h.required[m.Module][name] = true
}
for _, name := range secrets {
h.secret(m.Module, PlanSecret{Name: name})
}
plan := Plan{Templates: templates, Items: planned, Collections: collections}
if len(placement) > 0 {
plan.Placement = placement
}
h.plans[m.Module] = plan
if len(m.Contributes) > 0 {
if h.contributes == nil {
h.contributes = map[string]map[string]map[string]any{}
}
h.contributes[m.Module] = m.Contributes
}
return ids, nil
}
// add is one handed file of a module, composed. A path handed twice is refused: the node-engine writes a filled
// file for exactly one resource, and the client's write names only the path (mesh-host internal/apply).
func (h *handedFiles) add(module string, f PlanFile) error {
if slices.Contains(h.paths, f.Path) {
return fmt.Errorf("%s: %s is handed to the provisioner twice on this machine; a filled file has one "+
"resource and one writer (novox/hq ADR 0349 §5)", module, f.Path)
}
plan := h.plans[module]
plan.Files = append(plan.Files, f)
h.plans[module] = plan
h.paths = append(h.paths, f.Path)
return nil
}
// secret is one of a handed module's own secrets its declaration names; a file holding it alone is a filled file.
func (h *handedFiles) secret(module string, sec PlanSecret) {
if h.secrets == nil {
h.secrets = map[string]map[string]string{}
}
if h.secrets[module] == nil {
h.secrets[module] = map[string]string{}
}
if sec.Path != "" || h.secrets[module][sec.Name] == "" {
h.secrets[module][sec.Name] = sec.Path
}
}
// addSecretFile is a handed module's own secret file, composed: a filled file, and its secret in the plan.
func (h *handedFiles) addSecretFile(module string, sec PlanSecret) error {
if slices.Contains(h.paths, sec.Path) {
return fmt.Errorf("%s: %s is handed to the provisioner twice on this machine; a filled file has one "+
"resource and one writer (novox/hq ADR 0349 §5)", module, sec.Path)
}
h.paths = append(h.paths, sec.Path)
h.secret(module, sec)
return nil
}
// addProvisionFile is a handed module's credential file for something it requires, which the vault keeps for it
// alone (issue 757): a filled file, and the credential in the plan.
func (h *handedFiles) addProvisionFile(module string, pr PlanProvision) error {
if slices.Contains(h.paths, pr.Path) {
return fmt.Errorf("%s: %s is handed to the provisioner twice on this machine; a filled file has one "+
"resource and one writer (novox/hq ADR 0349 §5)", module, pr.Path)
}
h.paths = append(h.paths, pr.Path)
if h.provisions == nil {
h.provisions = map[string][]PlanProvision{}
}
h.provisions[module] = append(h.provisions[module], pr)
return nil
}
// finished is every plan with its secrets, by name, and what its module requires, each with its provider.
func (h *handedFiles) finished(needs []Needed) map[string]Plan {
if len(h.plans) == 0 {
return nil
}
for module, plan := range h.plans {
// **Every provision it requires, with its provider** (issue 760); a credential the vault keeps for it alone
// is that provision's entry, with its name and file (issue 757).
provisions := provisionsOf(needs, module, h.identities[module], h.contributes[module], h.required[module])
for _, credential := range h.provisions[module] {
i := slices.IndexFunc(provisions, func(p PlanProvision) bool {
return p.Provision == credential.Provision && p.Local == credential.Local
})
if i < 0 {
credential.As = h.identities[module]
provisions = append(provisions, credential)
continue
}
provisions[i].Name, provisions[i].Path = credential.Name, credential.Path
}
for i := range provisions {
provisions[i].Names = maps.Clone(h.names[module][provisions[i].Provision])
}
sortProvisions(provisions)
plan.Provisions = provisions
h.plans[module] = plan
}
for module, named := range h.secrets {
plan, ok := h.plans[module]
if !ok {
continue
}
plan.Secrets = nil
for _, name := range sortedKeys(named) {
plan.Secrets = append(plan.Secrets, PlanSecret{Name: name, Path: named[name]})
}
h.plans[module] = plan
}
return h.plans
}
+125
View File
@@ -0,0 +1,125 @@
package catalogue
import (
"reflect"
"strings"
"testing"
)
// gitea's server.env, as the catalogue keeps it: its database as the provider told it and the consumer's own login.
const giteaServerEnv = "GITEA__security__INTERNAL_TOKEN=${secret:internal-token}\n" +
"GITEA__database__DB_TYPE=postgres\n" +
"GITEA__database__HOST=${bound:postgres-database:at}:${bound:postgres-database:port}\n" +
"GITEA__database__NAME=${bound:postgres-database:as}\n" +
"GITEA__database__USER=${bound:postgres-database:as}\n" +
"GITEA__database__PASSWD=${secret:postgres-database}\n"
// databaseConsumer is a handed module, with the template gitea's server.env is, requiring a database from a provider
// that has moved or not (issue 788); slug is the module's identity source when it declares one.
func databaseConsumer(t *testing.T, slug string, moved bool) Resolution {
t.Helper()
raw := strings.Replace(botMoved(), `"own-secrets":{"token":"${dir:state}/token"}`,
`"requires":["postgres-database"],"own-secrets":{"internal-token":"${dir:state}/internal-token"}`, 1)
if slug != "" {
raw = strings.Replace(raw, `"version":"1",`, `"version":"1","slug":"`+slug+`",`, 1)
}
return withDatabase(t, raw, giteaServerEnv, moved)
}
// withDatabase resolves the module in raw, whose template is text, against a provider of postgres-database that has
// moved or not.
func withDatabase(t *testing.T, raw, text string, moved bool) Resolution {
t.Helper()
r, err := Resolve(shelf(held(t, raw, builtBot(text)),
Manifest{Module: "postgres", Version: "1", Provides: []Offer{{Name: "postgres-database", Scope: ScopeNode}}}),
[]string{"bot"}, workstation(), World{})
if err != nil {
t.Fatal(err)
}
r.Account = "op"
// Answered on this machine, it states no need; one from the provider's machine is stated, as the plan does.
r.Needs = []Needed{{Name: "postgres-database", For: "bot", Module: "postgres", From: "novox", At: "10.7.0.1",
CredentialForm: CredentialFormPair, IssuedToHolder: moved, ProviderMoved: moved}}
return r
}
// The forge's server.env composes for a handed module when what it requires has moved, its plan entry carrying the
// provider's address, machine and the consumer's identity as today's binding file gives them; and is refused with the
// text it has always had when the provider has not.
func TestAHandedModuleMayNameWhatItRequiresWhereItsProviderHasMoved(t *testing.T) {
c, got := composeWith(t, databaseConsumer(t, "", true), handingTo("store:5000"))
if strings.Contains(got, "sealed") {
t.Fatalf("a handed module's declaration carries a sealed value: %s", got)
}
want := []PlanProvision{{Name: "postgres-database", Provision: "postgres-database", Provider: "novox",
ProviderModule: "postgres", CredentialForm: CredentialFormPair, At: "10.7.0.1", From: "novox",
As: ConsumerIdentity("workstation", "bot")}}
if !reflect.DeepEqual(c.Plans["bot"].Provisions, want) {
t.Fatalf("the plan names %+v, want %+v", c.Plans["bot"].Provisions, want)
}
for _, unmoved := range []Resolution{databaseConsumer(t, "", false)} {
_, err := unmoved.Compose(handingTo("store:5000"))
if err == nil || !strings.Contains(err.Error(), "which is no secret of its own; a credential it requires moves "+
"with its provider (design 57 B1–B3)") {
t.Fatalf("a provider not moved, the credential: %v", err)
}
bound := strings.Replace(giteaServerEnv, "${secret:postgres-database}", "x", 1)
r := withDatabase(t, strings.Replace(botMoved(), `"own-secrets":{"token":"${dir:state}/token"}`,
`"requires":["postgres-database"],"own-secrets":{"internal-token":"${dir:state}/internal-token"}`, 1),
bound, false)
_, err = r.Compose(handingTo("store:5000"))
if err == nil || !strings.Contains(err.Error(), "the provisioner's client fills only config, secrets, "+
"collections and placement until that domain moves (design 57)") {
t.Fatalf("a provider not moved, the binding: %v", err)
}
}
}
// A module with a slug is told who it is as its binding file says, the slug first (issue 788).
func TestAHandedModulesPlanEntryCarriesItsIdentityAsTheBindingFileGivesIt(t *testing.T) {
c, _ := composeWith(t, databaseConsumer(t, "agent", true), handingTo("store:5000"))
if got := c.Plans["bot"].Provisions[0].As; got != ConsumerIdentity("workstation", "agent") {
t.Fatalf("the plan entry says %q", got)
}
// Nothing known of where the provider is: the entry says none.
c, _ = composeWith(t, forgeReader(t, true, true), handingTo("store:5000"))
if p := c.Plans["bot"].Provisions[0]; p.At != "" || p.From != "novox" {
t.Fatalf("the entry says at %q from %q", p.At, p.From)
}
}
// A handed module's credentials the vault keeps as one login per local name (issue 793): gitea's internal token and
// admin password, which the `secret` provision hands as `${secret:<local>}`. Each is a plan entry naming its local,
// its file declared placed and without a value; the template naming the local is allowed.
func TestAHandedModulesLocalCredentialsOfTheVaultAreNamedByLocalInItsPlan(t *testing.T) {
raw := strings.Replace(botMoved(), `"own-secrets":{"token":"${dir:state}/token"}`,
`"requires":["secret"],"secrets":{"secret":{"internal-token":"${dir:state}/internal-token.secret",`+
`"admin":"${dir:state}/admin.secret"}},"own-secrets":{"token":"${dir:state}/token"}`, 1)
r, err := Resolve(shelf(held(t, raw, builtBot("T=${secret:internal-token}\n")),
Manifest{Module: "vault", Version: "1", Provides: []Offer{{Name: "secret", Scope: ScopeNode}}}),
[]string{"bot"}, workstation(), World{})
if err != nil {
t.Fatal(err)
}
r.Account = "op"
r.Needs = []Needed{
{Name: "secret", Local: "internal-token", For: "bot", Module: "vault", From: "novox",
CredentialForm: CredentialFormPair, IssuedToHolder: true, ProviderMoved: true},
{Name: "secret", Local: "admin", For: "bot", Module: "vault", From: "novox",
CredentialForm: CredentialFormPair, IssuedToHolder: true, ProviderMoved: true},
}
c, got := composeWith(t, r, handingTo("store:5000"))
if strings.Contains(got, "sealed") {
t.Fatalf("the declaration carries a sealed value: %s", got)
}
var names []string
for _, p := range c.Plans["bot"].Provisions {
if p.Provision != "secret" || p.Provider != "novox" || p.Path == "" || p.Name != p.Local {
t.Errorf("the plan entry is %+v", p)
}
names = append(names, p.Name)
}
if !reflect.DeepEqual(names, []string{"admin", "internal-token"}) {
t.Fatalf("the plan names %v, want admin and internal-token", names)
}
}
+134
View File
@@ -0,0 +1,134 @@
package catalogue
import (
"os"
"path/filepath"
"strings"
"testing"
)
// A provider's consumers' credentials reach it in a file its machine's provisioner client fills (novox/hq ADR 0373,
// issue 679): `${consumers:<provision>}`, which only a provider's template may hold, for a provision it provides, in a
// file of mode 0600 owned at runtime; filled by the client alone, so a module not handed to it on a machine is
// refused there rather than written the placeholder's text, or nothing.
const (
handedProvider = `{"module":"db","version":"1","moved":true,
"provides":[{"name":"postgres-database","scope":"mesh","identity":{"max":63,"in":"a PostgreSQL role"}}],
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},
"resources":[
{"id":"state","type":"directory","mode":"0700","place":"."},
{"id":"consumers","type":"file","path":"${dir:state}/consumers.json","mode":"0600","template":"files/consumers.json"}]}`
consumersText = "${consumers:postgres-database}"
)
func builtProvider(text string) []Built {
return []Built{{Name: "files", Kind: ArtifactArchive,
Reference: ArtifactStoreScheme + "db/files/blobs/sha256:" + strings.Repeat("f", 64),
Digest: "sha256:" + strings.Repeat("f", 64), Texts: map[string]string{"consumers.json": text}}}
}
func TestAHandedProvidersConsumersFileIsHandedToTheClient(t *testing.T) {
r := notifierOn(t, held(t, handedProvider, builtProvider(consumersText)))
c, got := composeWith(t, r, handingTo("store:5000"))
if strings.Contains(got, "${consumers:") {
t.Fatalf("the declaration carries the placeholder: %s", got)
}
plan := c.Plans["db"]
if len(plan.Files) != 1 || plan.Files[0].Template != "consumers.json" || plan.Files[0].OwnerAtRuntime != RuntimeOwned {
t.Fatalf("the plan fills %+v, want the consumers' file owned at runtime", plan.Files)
}
}
func TestWhatAConsumersPlaceholderMayNotBeIsRefused(t *testing.T) {
for why, c := range map[string]struct{ raw, text, want string }{
"a provision the module does not provide": {handedProvider, "${consumers:cache}", "does not provide"},
"a module providing nothing": {strings.Replace(handedProvider, `"provides":[{"name":"postgres-database","scope":"mesh","identity":{"max":63,"in":"a PostgreSQL role"}}],`,
"", 1), consumersText, "does not provide"},
"a file others may read": {strings.Replace(handedProvider, `"mode":"0600","template"`,
`"mode":"0640","template"`, 1), consumersText, "0600"},
"a file the program keeps": {strings.Replace(handedProvider, `"mode":"0600","template"`,
`"mode":"0600","create-once":true,"template"`, 1), consumersText, "owned"},
} {
m, err := ParseManifest([]byte(c.raw))
if err != nil {
t.Fatalf("%s: %v", why, err)
}
resolved, err := m.Resolve(builtProvider(c.text))
if err == nil {
r := notifierOn(t, resolved)
_, err = r.Compose(handingTo("store:5000"))
}
if err == nil || !strings.Contains(err.Error(), c.want) {
t.Errorf("%s: %v, want a refusal saying %q", why, err, c.want)
}
}
}
// Where the provider is not handed, nothing fills the placeholder, and it is never written as its text or as an empty
// list the provider would read as no consumers: the file is left out, said once, and the module composes exactly as
// it would without the file, so a provider below the handing contract keeps working from its grant files (issue 780).
func TestAProvidersConsumersFileIsLeftOutWhereItIsNotHanded(t *testing.T) {
withFile := notifierOn(t, held(t, handedProvider, builtProvider(consumersText)))
without := strings.Replace(handedProvider, `,
{"id":"consumers","type":"file","path":"${dir:state}/consumers.json","mode":"0600","template":"files/consumers.json"}`, "", 1)
if without == handedProvider {
t.Fatal("the manifest without the file was not made")
}
bare := notifierOn(t, held(t, strings.Replace(strings.Replace(without, `"moved":true,`, "", 1),
`"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},`, "", 1), nil))
for name, with := range map[string]Rendering{
"a node-engine that fills no file": {},
"a node-engine that fills, no client": {Hands: true, ArtifactStore: "store:5000"},
"a client, a node-engine that does not": {Provisioner: "mesh-provisioner-client", ArtifactStore: "store:5000"},
} {
c, got := composeWith(t, withFile, with)
_, want := composeWith(t, bare, with)
if got != want {
t.Errorf("%s: composes\n%s\nwithout its file, it composed\n%s", name, got, want)
}
if strings.Contains(got, "consumers") {
t.Errorf("%s: the declaration names the consumers' file: %s", name, got)
}
if len(c.HandedOnly) != 1 || !strings.Contains(c.HandedOnly[0], "left out, handed only") {
t.Errorf("%s: said %q, want the one line saying it is left out", name, c.HandedOnly)
}
if len(c.Unplaced) != 0 || len(c.Filled) != 0 || len(c.Plans) != 0 {
t.Errorf("%s: unplaced %v, filled %v, plans %v", name, c.Unplaced, c.Filled, c.Plans)
}
}
// Where it is handed, the file is there and nothing is said.
c, _ := composeWith(t, withFile, handingTo("store:5000"))
if len(c.HandedOnly) != 0 || len(c.Filled) != 1 {
t.Fatalf("handed: said %q, filled %v", c.HandedOnly, c.Filled)
}
}
// What `module check` and the merge gate judge: the provider read from its tree, its text inline. Judged by the same
// rules, a provider naming a provision it does not provide is refused there too, and one that does composes.
func TestAProviderReadFromItsTreeIsJudgedByTheSameRules(t *testing.T) {
read := func(text string) (Manifest, error) {
dir := t.TempDir()
if err := os.MkdirAll(filepath.Join(dir, "files"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "files", "consumers.json"), []byte(text), 0o644); err != nil {
t.Fatal(err)
}
m, err := ParseManifest([]byte(handedProvider))
if err != nil {
t.Fatal(err)
}
return WithTemplatesFrom(m, dir)
}
filled, err := read(consumersText)
if err != nil {
t.Fatalf("a provider read from its tree is refused: %v", err)
}
if _, err := notifierOn(t, filled).Compose(handingTo("store:5000")); err != nil {
t.Fatalf("a provider read from its tree does not compose where it would be handed: %v", err)
}
if _, err := read("${consumers:cache}"); err == nil || !strings.Contains(err.Error(), "does not provide") {
t.Fatalf("a provider read from its tree naming a provision it does not provide: %v", err)
}
}
+57
View File
@@ -0,0 +1,57 @@
package catalogue
import (
"reflect"
"strings"
"testing"
)
// issue 831: a moved module's merging file is not handed (ADR 0349 clause 17, the lead's limit): the controller
// fills and merges it as for a module not moved, and the module's other files stay handed.
func TestAMovedModulesMergingFileIsFilledTheOldWayAndItsPlainFileIsHanded(t *testing.T) {
raw := strings.Replace(handedNotifier, `{"id":"legacy"`,
`{"id":"settings","type":"file","path":"${machine:account-home}/.config/notify/settings.json","owner":"${machine:account}","mode":"0644","template":"files/settings.json","merge":"json","takes-config":["width"]},
{"id":"legacy"`, 1)
moved := strings.Replace(raw, `"version":"1",`, `"version":"1","moved":true,`, 1)
built := []Built{{Name: "files", Kind: ArtifactArchive,
Reference: ArtifactStoreScheme + "notify/files/blobs/sha256:" + strings.Repeat("d", 64),
Digest: "sha256:" + strings.Repeat("d", 64),
Texts: map[string]string{"rc": notifierText, "settings.json": "{\"theme\": \"dark\"}\n"}}}
cb, _ := composeWith(t, notifierOn(t, held(t, raw, built)), handingTo("store:5000"))
c, got := composeWith(t, notifierOn(t, held(t, moved, built)), handingTo("store:5000"))
var merged, plain map[string]any
for _, r := range c.Resources {
switch r["id"] {
case "notify.settings":
merged = r
case "notify.configuration":
plain = r
}
}
if merged == nil || plain == nil {
t.Fatalf("not composed: %s", got)
}
if text, _ := merged["content"].(string); !strings.Contains(text, `"theme"`) || merged["owner-at-runtime"] != nil {
t.Fatalf("the merging file is not filled the old way: %v", merged)
}
if plain["content"] != nil || plain["owner-at-runtime"] != "owned" {
t.Fatalf("the plain file is not handed: %v", plain)
}
for _, p := range c.Filled {
if strings.HasSuffix(p, "settings.json") {
t.Fatalf("the merging file is named among the filled files: %v", c.Filled)
}
}
if len(c.Plans["notify"].Files) != 1 || c.Plans["notify"].Files[0].ID != "configuration" {
t.Fatalf("the plan is %+v", c.Plans)
}
if len(c.Unplaced) > 0 {
t.Fatalf("unplaced: %v", c.Unplaced)
}
// byte-identical to the not-moved composition
for _, r := range cb.Resources {
if r["id"] == "notify.settings" && !reflect.DeepEqual(r, merged) {
t.Fatalf("the merging file differs from the not-moved composition:\nbefore %v\nnow %v", r, merged)
}
}
}
+244
View File
@@ -0,0 +1,244 @@
package catalogue
import (
"encoding/json"
"reflect"
"strings"
"testing"
)
// A handed module's plan names the provider serving each provision it requires (design 57 V5a, issue 760): the
// module on the machine resolution chose for this consumer, and the local name it keeps the credential under where
// it keeps several, so the vault, which reads the plan in the assignments mirror, sees a consumer bind and unbind
// (novox/hq ADR 0349 clause 8). One entry per provision and local name, the same entry that names a credential
// the vault keeps for it alone (issue 757).
// notifierConsumer is the notifier requiring a mesh-scoped database, marked moved when moved, with its credential
// kept under two local names (ADR 0094) when locals is set.
func notifierConsumer(t *testing.T, moved, locals bool) Manifest {
t.Helper()
raw := handedNotifier
if moved {
raw = notifierMoved()
}
add := `"requires":["postgres-database"],`
if locals {
add += `"secrets":{"postgres-database":{"hot":"/var/lib/notify/hot","cold":"/var/lib/notify/cold"}},`
}
raw = strings.Replace(raw, `"build":`, add+`"build":`, 1)
return held(t, raw, builtNotifier())
}
// twoStores is a mesh where two machines each run a store providing the database, the consumer pinned to one.
func twoStores(pin string) World {
return World{Offered: map[string][]Provider{"postgres-database": {
{Node: "anchor", At: "anchor.internal", Module: "store", Serves: map[string]any{"port": 5432}},
{Node: "home", At: "home.internal", Module: "store-two", Serves: map[string]any{"port": 5434}},
}}, Pinned: map[string]Chosen{"postgres-database": {Node: pin}}}
}
func consumerOn(t *testing.T, m Manifest, w World) Resolution {
t.Helper()
node := workstation()
node.At = "workstation.internal"
stores := shelf(m,
Manifest{Module: "store", Version: "1", Provides: []Offer{{Name: "postgres-database", Scope: ScopeMesh}},
Serves: map[string]map[string]any{"postgres-database": {"port": 5432}}},
Manifest{Module: "store-two", Version: "1", Provides: []Offer{{Name: "postgres-database", Scope: ScopeMesh}},
Serves: map[string]map[string]any{"postgres-database": {"port": 5434}}})
r, err := Resolve(stores, []string{m.Module}, node, w)
if err != nil {
t.Fatal(err)
}
r.Account = "op"
return r
}
func TestAHandedModulesPlanNamesTheProviderServingEachProvision(t *testing.T) {
c, _ := composeWith(t, consumerOn(t, notifierConsumer(t, true, false), twoStores("anchor")), handingTo("store:5000"))
want := []PlanProvision{{Provision: "postgres-database", Provider: "anchor", ProviderModule: "store",
CredentialForm: CredentialFormPair, At: "anchor.internal", From: "anchor", As: ConsumerIdentity("workstation", "notify")}}
if got := c.Plans["notify"].Provisions; !reflect.DeepEqual(got, want) {
t.Fatalf("the plan names %+v, want %+v", got, want)
}
// **A changed binding changes the plan**: pinned to the other store, the plan names it and not the first.
c, _ = composeWith(t, consumerOn(t, notifierConsumer(t, true, false), twoStores("home")), handingTo("store:5000"))
want = []PlanProvision{{Provision: "postgres-database", Provider: "home", ProviderModule: "store-two",
CredentialForm: CredentialFormPair, At: "home.internal", From: "home", As: ConsumerIdentity("workstation", "notify")}}
if got := c.Plans["notify"].Provisions; !reflect.DeepEqual(got, want) {
t.Fatalf("rebound, the plan names %+v, want %+v", got, want)
}
}
// A consumer keeping one provider's credential under several local names is bound once per local name.
func TestAHandedModulesPlanNamesEachLocalName(t *testing.T) {
c, _ := composeWith(t, consumerOn(t, notifierConsumer(t, true, true), twoStores("anchor")), handingTo("store:5000"))
want := []PlanProvision{
{Provision: "postgres-database", Local: "cold", Provider: "anchor", ProviderModule: "store",
CredentialForm: CredentialFormPair, At: "anchor.internal", From: "anchor", As: ConsumerIdentity("workstation", "notify")},
{Provision: "postgres-database", Local: "hot", Provider: "anchor", ProviderModule: "store",
CredentialForm: CredentialFormPair, At: "anchor.internal", From: "anchor", As: ConsumerIdentity("workstation", "notify")},
}
if got := c.Plans["notify"].Provisions; !reflect.DeepEqual(got, want) {
t.Fatalf("the plan names %+v, want %+v", got, want)
}
}
// A module not moved, or moved where it is not handed, composes as before and has no plan.
func TestAConsumerNotHandedComposesAsBefore(t *testing.T) {
_, before := composeWith(t, consumerOn(t, notifierConsumer(t, false, false), twoStores("anchor")),
handingTo("store:5000"))
c, got := composeWith(t, consumerOn(t, notifierConsumer(t, true, false), twoStores("anchor")),
Rendering{Provisioner: "mesh-provisioner-client", ArtifactStore: "store:5000"})
if got != before {
t.Fatalf("the moved consumer composes differently where it is not handed:\nbefore: %s\nnow: %s", before, got)
}
if c.Plans != nil {
t.Fatalf("a consumer not handed has plans %+v", c.Plans)
}
}
// A credential the vault keeps for the holder alone is its provision's one entry, with its provider's module too.
func TestAHandedModulesIssuedCredentialIsItsProvisionsOneEntry(t *testing.T) {
r := forgeReader(t, true, true)
r.Needs = append(r.Needs, Needed{Name: "s3-bucket", From: "ace", For: "bot", Module: "minio"},
Needed{Name: "openai", From: "personal", For: "bot", ByRecord: true})
c, _ := composeWith(t, r, handingTo("store:5000"))
got := c.Plans["bot"].Provisions
if len(got) != 2 || got[0].Provision != "forge-read" || got[0].Name != "forge-read" ||
got[0].ProviderModule != "forge" || got[0].Path == "" ||
!reflect.DeepEqual(got[1], PlanProvision{Provision: "s3-bucket", Provider: "ace", ProviderModule: "minio",
From: "ace", As: ConsumerIdentity("workstation", "bot")}) {
t.Fatalf("the plan names %+v: one entry per provision, the issued credential's with its name and file, "+
"none for a licence", got)
}
}
// Each entry names the credential form the provider's offer declares (issue 768), from the provider's manifest, so
// the vault makes a first value in that form for a provider it holds no row of: one of its own secrets shared by
// every consumer, a login per consumer when the offer says no credential, issued by its software, or given by a
// person (the offer's own secret issued outside).
func TestAHandedModulesPlanNamesTheCredentialFormOfItsProvidersOffer(t *testing.T) {
storeOffering := func(credential string) Manifest {
raw := `{"module":"store","version":"1","provides":[{"name":"postgres-database","scope":"mesh"` +
credential + `}],"serves":{"postgres-database":{"port":5432}}`
if strings.Contains(credential, `"own"`) {
raw += `,"own-secrets":{"admin":{"path":"/var/lib/store/admin","taken":"at-start"},` +
`"token":{"path":"/var/lib/store/token","taken":"at-start","issued-by":"outside"}}`
}
return mustParse(t, raw+`}`)
}
for credential, want := range map[string]string{
``: CredentialFormPair,
`,"credential":{"own":"admin"}`: CredentialFormShared,
`,"credential":{"own":"token"}`: CredentialFormGiven,
`,"credential":{"issued-by":"outside"}`: CredentialFormGiven,
`,"credential":{"issued-by":"software","tool":"store_issued_token"}`: CredentialFormIssued,
} {
node := workstation()
node.At = "workstation.internal"
consumer := notifierConsumer(t, true, false)
r, err := Resolve(shelf(consumer, storeOffering(credential)), []string{consumer.Module}, node, twoStores("anchor"))
if err != nil {
t.Fatal(err)
}
r.Account = "op"
c, _ := composeWith(t, r, handingTo("store:5000"))
got := c.Plans["notify"].Provisions
if len(got) != 1 || got[0].CredentialForm != want {
t.Errorf("an offer with %q: the plan names %+v, want the form %q", credential, got, want)
}
}
if got := (Manifest{Module: "x"}).CredentialFormOf("postgres-database"); got != "" {
t.Errorf("a module offering nothing has the credential form %q", got)
}
}
// A consumer's plan entry carries what its manifest contributes to that provision (issue 776), so the provider's
// helper learns a database's name from the bus; a consumer contributing nothing has the entry it had.
func TestAHandedModulesPlanEntryCarriesWhatItContributesToItsProvider(t *testing.T) {
m := notifierConsumer(t, true, false)
m.Contributes = map[string]map[string]any{"postgres-database": {"name": "nextcloud"}}
c, _ := composeWith(t, consumerOn(t, m, twoStores("anchor")), handingTo("store:5000"))
want := []PlanProvision{{Provision: "postgres-database", Provider: "anchor", ProviderModule: "store",
CredentialForm: CredentialFormPair, Contributes: map[string]any{"name": "nextcloud"}, At: "anchor.internal", From: "anchor", As: ConsumerIdentity("workstation", "notify"),
// What the binding file says as `${bound:postgres-database:name}` (issue 792).
Names: map[string]string{"name": "nextcloud"}}}
if got := c.Plans["notify"].Provisions; !reflect.DeepEqual(got, want) {
t.Fatalf("the plan names %+v, want %+v", got, want)
}
m = notifierConsumer(t, true, false)
c, _ = composeWith(t, consumerOn(t, m, twoStores("anchor")), handingTo("store:5000"))
raw, err := json.Marshal(c.Plans["notify"].Provisions)
if err != nil {
t.Fatal(err)
}
const before = `[{"provision":"postgres-database","provider":"anchor","provider-module":"store",` +
`"credential-form":"pair","at":"anchor.internal","from":"anchor","as":"mesh_workstation_notify"}]`
if string(raw) != before {
t.Fatalf("a consumer contributing nothing has the entry %s, want %s", raw, before)
}
}
// A consumer's plan entry carries the names the mesh composed for it through that provision (issue 792): what its
// binding file names as `${bound:<provision>:name|internal-name|name-<local>|internal-name-<local>}`, so a moved
// provider's helper copies them into the consumer's entry and the client fills the same text. A consumer
// contributing no label has the entry it had.
func TestAHandedModulesPlanEntryCarriesTheNamesComposedForIt(t *testing.T) {
m := notifierConsumer(t, true, false)
m.ContributesMany = map[string]map[string]map[string]any{
"postgres-database": {"web": {"label": "git"}, "api": {"label": "git-api"}}}
r := consumerOn(t, m, twoStores("anchor"))
r.PublicDomain = "example.org"
c, _ := composeWith(t, r, handingTo("store:5000"))
got := c.Plans["notify"].Provisions
if len(got) != 1 {
t.Fatalf("the plan names %+v", got)
}
want := map[string]string{"name-web": "git.example.org", "name-api": "git-api.example.org"}
own, err := r.ownNames(m, "postgres-database", nil)
if err != nil {
t.Fatal(err)
}
file := map[string]string{}
withOwnNames(file, own)
for k, v := range file {
if strings.HasPrefix(k, "internal-") {
want[k] = v
}
}
if !reflect.DeepEqual(got[0].Names, want) {
t.Fatalf("the entry carries %v, the binding file names %v", got[0].Names, want)
}
}
// The flag a provides entry carries, `"answered-by": "helper"`, is read and written back, and refused with any other
// word (issue 792). It is no part of moving the module: the flag alone hands none of the module's files, and the
// module composes byte for byte as it did without it.
func TestAnOfferAnsweredByTheHelperIsReadAndHandsNoFile(t *testing.T) {
with := strings.Replace(handedNotifier, `"version":"1",`,
`"version":"1","provides":[{"name":"route","scope":"mesh","answered-by":"helper"}],`, 1)
without := strings.Replace(handedNotifier, `"version":"1",`,
`"version":"1","provides":[{"name":"route","scope":"mesh"}],`, 1)
flagged := held(t, with, builtNotifier())
if flagged.Moved || len(flagged.Provides) != 1 || flagged.Provides[0].AnsweredBy != AnsweredByHelper {
t.Fatalf("the flag was read as %+v (moved %v)", flagged.Provides, flagged.Moved)
}
if flagged.Handed != nil {
t.Fatalf("the flag alone kept what a moved module's build keeps: %+v", flagged.Handed)
}
raw, err := json.Marshal(flagged.Provides[0])
if err != nil || !strings.Contains(string(raw), `"answered-by":"helper"`) {
t.Fatalf("written back as %s: %v", raw, err)
}
if _, err := ParseManifest([]byte(strings.Replace(with, `"helper"`, `"harness"`, 1))); err == nil {
t.Fatal("an unknown answerer was accepted")
}
c1, before := composeWith(t, notifierOn(t, held(t, without, builtNotifier())), handingTo("store:5000"))
c2, got := composeWith(t, notifierOn(t, flagged), handingTo("store:5000"))
if got != before || len(c2.Filled) > 0 || c2.Plans != nil || c1.Plans != nil {
t.Fatalf("the flag changed what is composed or handed:\nbefore: %s\nnow: %s\n%v %v", before, got, c2.Filled, c2.Plans)
}
}
+121
View File
@@ -0,0 +1,121 @@
package catalogue
import (
"encoding/json"
"reflect"
"strings"
"testing"
)
// A module only pulled in by another is handed exactly like an assigned one (issue 765): its templated files, its
// own secrets and its items go to the machine's provisioner client wherever the machine can take them. A module not
// moved composes as before, pulled in or not.
// pulledInBy is a module that requires what raw's module provides, so resolving it alone pulls that module in.
const pulledInBy = `{"module":"desk","version":"1","requires":["pulled-in-here"]}`
// providing is raw's module providing what desk requires.
func providing(raw string) string {
return strings.Replace(raw, `"version":"1",`, `"version":"1","provides":[{"name":"pulled-in-here"}],`, 1)
}
// onWorkstation resolves m on the workstation either assigned beside desk, or pulled in by desk alone.
func onWorkstation(t *testing.T, m Manifest, assigned bool) Resolution {
t.Helper()
desk, err := ParseManifest([]byte(pulledInBy))
if err != nil {
t.Fatal(err)
}
want := []string{"desk"}
if assigned {
want = append(want, m.Module)
}
r, err := Resolve(shelf(m, desk), want, workstation(), World{})
if err != nil {
t.Fatal(err)
}
why := map[bool]string{true: "assigned", false: "required by desk"}[assigned]
if r.Because[m.Module] != why {
t.Fatalf("%s is on the workstation because %q, want %q", m.Module, r.Because[m.Module], why)
}
r.Account = "op"
return r
}
// resourcesOf is the module's resources in a composition, as JSON.
func resourcesOf(t *testing.T, c Composed, module string) string {
t.Helper()
var out []map[string]any
for _, r := range c.Resources {
if strings.HasPrefix(r["id"].(string), module+".") {
out = append(out, r)
}
}
raw, err := json.Marshal(out)
if err != nil {
t.Fatal(err)
}
return string(raw)
}
func TestAModulePulledInIsHandedLikeAnAssignedOne(t *testing.T) {
for name, m := range map[string]struct {
module string
held Manifest
with Rendering
}{
"its templated file": {"notify", held(t, providing(notifierMoved()), builtNotifier()), handingTo("store:5000")},
"its own secret": {"bot", held(t, providing(botMoved()), builtBot(botText)),
sealedForBot(handingTo("store:5000"))},
"its item": {"prompt", held(t, providing(contributorP10k),
builtWith("prompt", map[string]string{"zsh": contributorText})), handingTo("store:5000")},
} {
assigned, _ := composeWith(t, onWorkstation(t, m.held, true), m.with)
pulled, got := composeWith(t, onWorkstation(t, m.held, false), m.with)
plan, handed := pulled.Plans[m.module]
if !handed {
t.Errorf("%s: a pulled-in moved module is not handed: plans %+v", name, pulled.Plans)
continue
}
if !reflect.DeepEqual(plan, assigned.Plans[m.module]) || !reflect.DeepEqual(pulled.Filled, assigned.Filled) {
t.Errorf("%s: pulled in, its plan is %+v and the filled files %v; assigned, %+v and %v", name, plan,
pulled.Filled, assigned.Plans[m.module], assigned.Filled)
}
if a, p := resourcesOf(t, assigned, m.module), resourcesOf(t, pulled, m.module); a != p {
t.Errorf("%s: pulled in, its resources are\n%s\nassigned,\n%s", name, p, a)
}
if strings.Contains(got, "sealed-to-the-node") || strings.Contains(got, "width = ") {
t.Errorf("%s: the controller filled or sealed something of a handed module: %s", name, got)
}
}
}
func TestAModulePulledInAndNotMovedComposesAsBefore(t *testing.T) {
for name, m := range map[string]Manifest{
"a file": held(t, providing(handedNotifier), builtNotifier()),
"a secret": held(t, providing(handedBot), builtBot(botText)),
} {
_, before := composeWith(t, onWorkstation(t, m, false), sealedForBot(Rendering{ArtifactStore: "store:5000"}))
c, got := composeWith(t, onWorkstation(t, m, false), sealedForBot(handingTo("store:5000")))
if got != before || c.Plans != nil || len(c.Filled) > 0 {
t.Errorf("%s: a pulled-in module not moved composes differently where the machine hands:\nbefore: %s\nnow: %s"+
"\nplans %v, filled %v", name, before, got, c.Plans, c.Filled)
}
}
}
// What requires a module pulled in is said by name, for its value in the assignments mirror; nothing for one
// assigned.
func TestAPulledInModuleSaysWhatRequiresIt(t *testing.T) {
m := held(t, providing(notifierMoved()), builtNotifier())
if by := onWorkstation(t, m, false).RequiredBy("notify"); by != "desk" {
t.Errorf("pulled in, notify is required by %q, want desk", by)
}
assigned := onWorkstation(t, m, true)
if by := assigned.RequiredBy("notify"); by != "" {
t.Errorf("assigned, notify is required by %q", by)
}
if by := assigned.RequiredBy("absent"); by != "" {
t.Errorf("a module not on the machine is required by %q", by)
}
}
+252
View File
@@ -0,0 +1,252 @@
package catalogue
import (
"reflect"
"slices"
"strings"
"testing"
)
// A moved module's own secrets come from the vault (novox/hq ADR 0349 clauses 7 and 11, design 57 V4, issue 647):
// where the module is handed, its declaration carries no sealed value — each file holding a secret alone is placed
// without one and filled by the client, and the plan names every secret its templates and those files need, by
// name. Where it is not handed, and for every module not moved, the composition is byte for byte what it was.
const (
handedBot = `{"module":"bot","version":"1",
"own-secrets":{"token":"${dir:state}/token"},
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},
"resources":[
{"id":"state","type":"directory","mode":"0700","place":"."},
{"id":"conf","type":"file","path":"/etc/bot/bot.conf","mode":"0640","template":"files/bot.conf"},
{"id":"daemon","type":"process","name":"bot","run":["bot"],"env":{"BOT_TOKEN_FILE":"${dir:state}/token"}}]}`
botText = "token = ${secret:token}\nhome = ${machine:account-home}\n"
)
func botMoved() string {
return strings.Replace(handedBot, `"version":"1",`, `"version":"1","moved":true,`, 1)
}
func builtBot(text string) []Built {
return []Built{{Name: "files", Kind: ArtifactArchive,
Reference: ArtifactStoreScheme + "bot/files/blobs/sha256:" + strings.Repeat("e", 64),
Digest: "sha256:" + strings.Repeat("e", 64), Texts: map[string]string{"bot.conf": text}}}
}
// What the controller seals for the bot when it fills it the old way: a stand-in sealed value.
func sealedForBot(with Rendering) Rendering {
with.Needed = map[string]map[string]string{"bot": {"token": "sealed-to-the-node"}}
return with
}
func TestAHandedModulesSecretsAreNamedAndNoValueIsSealed(t *testing.T) {
r := notifierOn(t, held(t, botMoved(), builtBot(botText)))
c, got := composeWith(t, r, handingTo("store:5000"))
if strings.Contains(got, "sealed") || strings.Contains(got, "${secret:") {
t.Fatalf("a handed module's declaration carries a sealed value or a secret placeholder: %s", got)
}
var token map[string]any
for _, res := range c.Resources {
if res["id"] == "bot.needs-token" {
token = res
}
}
if token == nil {
t.Fatalf("the token's file is not declared: %s", got)
}
if token["path"] == nil || strings.Contains(token["path"].(string), "${") || token["mode"] != "0600" ||
token["owner-at-runtime"] != "owned" || token[HandedSecretField] != nil {
t.Fatalf("the token's file is declared %v: placed, 0600, owned at runtime, no value", token)
}
path := token["path"].(string)
if !reflect.DeepEqual(c.Filled, []string{path, "/etc/bot/bot.conf"}) {
t.Fatalf("the filled files are %v", c.Filled)
}
if want := []PlanSecret{{Name: "token", Path: path}}; !reflect.DeepEqual(c.Plans["bot"].Secrets, want) {
t.Fatalf("the plan names secrets %+v, want %+v", c.Plans["bot"].Secrets, want)
}
// A process naming the token's file still restarts when its client writes it.
for _, res := range c.Resources {
if res["id"] == "bot.daemon" && !strings.Contains(strings.Join(stringsIn(res["restart-on"]), " "), "bot.needs-token") {
t.Errorf("the daemon does not restart on its token's file: %v", res["restart-on"])
}
}
}
// A secret only a template names is in the plan by name alone.
func TestASecretOnlyATemplateNamesIsPlannedByName(t *testing.T) {
raw := strings.Replace(botMoved(), `"own-secrets":{"token":"${dir:state}/token"},`,
`"own-secrets":{"token":"${dir:state}/token","api-key":"${dir:state}/api-key"},`, 1)
r := notifierOn(t, held(t, raw, builtBot(botText+"key = ${secret:api-key}\n")))
c, _ := composeWith(t, r, handingTo("store:5000"))
names := []string{}
for _, s := range c.Plans["bot"].Secrets {
names = append(names, s.Name)
}
if strings.Join(names, " ") != "api-key token" {
t.Fatalf("the plan names %v", c.Plans["bot"].Secrets)
}
}
// Nothing changes for a module that has not moved, nor for a moved one where it is not handed: its secrets are
// sealed into the declaration as before.
func TestAMovedModulesSecretsAreSealedAsBeforeWhereItIsNotHanded(t *testing.T) {
_, before := composeWith(t, notifierOn(t, held(t, handedBot, builtBot(botText))),
sealedForBot(handingTo("store:5000")))
if !strings.Contains(before, "sealed-to-the-node") {
t.Fatalf("the unmoved module is not sealed the old way: %s", before)
}
for name, with := range map[string]Rendering{
"an engine that does not take a filled file alone": {Provisioner: "mesh-provisioner-client", ArtifactStore: "store:5000"},
"no provisioner client on the machine": {Hands: true, ArtifactStore: "store:5000"},
} {
c, got := composeWith(t, notifierOn(t, held(t, botMoved(), builtBot(botText))), sealedForBot(with))
if got != before {
t.Errorf("%s: the moved module composes differently from its unmarked twin:\nbefore: %s\nnow: %s",
name, before, got)
}
if len(c.Filled) > 0 || c.Plans != nil {
t.Errorf("%s: something was handed: %v %v", name, c.Filled, c.Plans)
}
}
}
func TestWhatAHandedModuleCannotHaveFromTheVaultIsRefused(t *testing.T) {
// A template naming a credential the module requires: a binding's, which moves with its provider.
requires := strings.Replace(botMoved(), `"own-secrets":`, `"requires":["database"],"secrets":{"database":"/etc/bot/db"},"own-secrets":`, 1)
r, err := Resolve(shelf(held(t, requires, builtBot("db = ${secret:database}\n")),
Manifest{Module: "db", Version: "1", Provides: []Offer{{Name: "database", Scope: ScopeNode}}}),
[]string{"bot"}, workstation(), World{})
if err != nil {
t.Fatal(err)
}
r.Account = "op"
if _, err := r.Compose(handingTo("store:5000")); err == nil || !strings.Contains(err.Error(), "moves with its provider") {
t.Errorf("a template naming a required credential: %v", err)
}
// A file with its text inline naming a secret, in a handed module.
inline := strings.Replace(botMoved(), `{"id":"daemon"`,
`{"id":"legacy","type":"file","path":"/etc/bot/legacy","content":"t = ${secret:token}\n"},{"id":"daemon"`, 1)
r = notifierOn(t, held(t, inline, builtBot(botText)))
if _, err := r.Compose(handingTo("store:5000")); err == nil || !strings.Contains(err.Error(), "keep it in a template") {
t.Errorf("an inline file naming a secret in a handed module: %v", err)
}
// A family and a secret issued outside the mesh are not handed by name yet: the mark is refused.
for why, raw := range map[string]string{
"a family": strings.Replace(botMoved(), `"own-secrets":{"token":"${dir:state}/token"}`, `"own-secrets":{"token":"${dir:state}/token","key-*":"${dir:state}/keys"}`, 1),
"issued outside": strings.Replace(botMoved(), `"own-secrets":{"token":"${dir:state}/token"}`, `"own-secrets":{"token":{"path":"${dir:state}/token","issued-by":"outside"}}`, 1),
} {
if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), "design 57 V4") {
t.Errorf("a moved module with %s: %v", why, err)
}
}
}
// A credential of what a handed module requires that the vault keeps for it alone (issue 757): the forge's read
// token for one agent, issued by the forge's software per holder. Where the module is handed, its file is declared
// placed and without a value, and the plan names it with its provision and provider; anywhere else, and for a
// credential the mesh makes, the composition is what it was.
func forgeReader(t *testing.T, moved bool, issued bool) Resolution {
t.Helper()
raw := strings.Replace(handedBot, `"own-secrets":`,
`"requires":[{"name":"forge-read","optional":true}],"secrets":{"forge-read":"${dir:state}/forge.secret"},"own-secrets":`, 1)
if moved {
raw = strings.Replace(raw, `"version":"1",`, `"version":"1","moved":true,`, 1)
}
r := notifierOn(t, held(t, raw, builtBot(botText)))
r.Needs = append(r.Needs, Needed{Name: "forge-read", From: "novox", For: "bot", Module: "forge",
Sealed: "sealed-forge-read", IssuedToHolder: issued})
return r
}
func TestAHandedModulesCredentialIssuedForItIsNamedInItsPlanAndNoValueIsSealed(t *testing.T) {
c, got := composeWith(t, forgeReader(t, true, true), handingTo("store:5000"))
if strings.Contains(got, "sealed") || strings.Contains(got, HandedProvisionField) {
t.Fatalf("a handed module's declaration carries a sealed value or the marker: %s", got)
}
var file map[string]any
for _, res := range c.Resources {
if res["id"] == "bot."+SecretID("forge-read") {
file = res
}
}
if file == nil {
t.Fatalf("the credential's file is not declared: %s", got)
}
path, _ := file["path"].(string)
if path == "" || strings.Contains(path, "${") || file["mode"] != "0600" || file["owner-at-runtime"] != "owned" {
t.Fatalf("the credential's file is declared %v: placed, 0600, owned at runtime, no value", file)
}
if !strings.Contains(strings.Join(c.Filled, " "), path) {
t.Fatalf("the credential's file is not among the filled files %v", c.Filled)
}
want := []PlanProvision{{Name: "forge-read", Provision: "forge-read", Provider: "novox", ProviderModule: "forge",
Path: path, From: "novox", As: ConsumerIdentity("workstation", "bot")}}
if !reflect.DeepEqual(c.Plans["bot"].Provisions, want) {
t.Fatalf("the plan names %+v, want %+v", c.Plans["bot"].Provisions, want)
}
}
func TestACredentialIssuedForAHolderIsSealedAsBeforeWhereItsModuleIsNotHanded(t *testing.T) {
_, before := composeWith(t, forgeReader(t, false, false), sealedForBot(handingTo("store:5000")))
if !strings.Contains(before, "sealed-forge-read") {
t.Fatalf("the unmoved module's credential is not sealed the old way: %s", before)
}
for why, r := range map[string]Resolution{
"a module not moved": forgeReader(t, false, true),
"a moved module whose credential is made": forgeReader(t, true, false),
} {
c, got := composeWith(t, r, sealedForBot(handingTo("store:5000")))
if !strings.Contains(got, "sealed-forge-read") {
t.Errorf("%s: the credential is not sealed the old way: %s", why, got)
}
// Its provider is named (issue 760), and no credential for the client to fill.
if p, ok := c.Plans["bot"]; ok && slices.ContainsFunc(p.Provisions, func(pr PlanProvision) bool {
return pr.Name != "" || pr.Path != ""
}) {
t.Errorf("%s: the plan names a credential to fill: %+v", why, p.Provisions)
}
}
c, got := composeWith(t, forgeReader(t, true, true), sealedForBot(Rendering{Provisioner: "mesh-provisioner-client",
ArtifactStore: "store:5000"}))
if !strings.Contains(got, "sealed-forge-read") || c.Plans != nil {
t.Errorf("where the engine does not take a filled file alone, the credential is not sealed the old way: %s", got)
}
}
// A handed consumer's own login from a provider that gives each consumer one (issue 779): the same entry as for an
// issued credential, with the provider's form on it, and nothing sealed; where the consumer is not handed, the
// login is sealed as it was.
func TestAHandedConsumersOwnLoginFromAPerLoginProviderIsNamedInItsPlanAndNoValueIsSealed(t *testing.T) {
pair := func(handed bool) Resolution {
r := forgeReader(t, true, handed)
r.Needs[len(r.Needs)-1].CredentialForm = CredentialFormPair
return r
}
c, got := composeWith(t, pair(true), handingTo("store:5000"))
if strings.Contains(got, "sealed") || strings.Contains(got, HandedProvisionField) {
t.Fatalf("a handed consumer's declaration carries its login sealed or the marker: %s", got)
}
var path string
for _, res := range c.Resources {
if res["id"] == "bot."+SecretID("forge-read") {
path, _ = res["path"].(string)
if res["mode"] != "0600" || res["owner-at-runtime"] != "owned" || res["content"] != nil || res["sealed"] != nil {
t.Fatalf("the login's file is declared %v: placed, 0600, owned at runtime, no value", res)
}
}
}
if path == "" || !strings.Contains(strings.Join(c.Filled, " "), path) {
t.Fatalf("the login's file %q is not declared and filled: %v", path, c.Filled)
}
want := []PlanProvision{{Name: "forge-read", Provision: "forge-read", Provider: "novox", ProviderModule: "forge",
CredentialForm: CredentialFormPair, Path: path, From: "novox", As: ConsumerIdentity("workstation", "bot")}}
if !reflect.DeepEqual(c.Plans["bot"].Provisions, want) {
t.Fatalf("the plan names %+v, want %+v", c.Plans["bot"].Provisions, want)
}
// Not marked as the vault's, the login is sealed as before.
_, before := composeWith(t, pair(false), sealedForBot(handingTo("store:5000")))
if !strings.Contains(before, "sealed-forge-read") {
t.Fatalf("a consumer whose login the vault does not keep for it has it sealed no more: %s", before)
}
}
+188
View File
@@ -0,0 +1,188 @@
package catalogue
import (
"encoding/json"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
)
// A module marked moved hands its templated files to the provisioner (novox/hq ADR 0349 §4, §6, §17, design 57 P6,
// issue 672): on a machine whose node-engine takes a filled file on the client's word and that runs a client, each
// is declared placed and without its text, named among the filled files, and planned for the client; anywhere
// else, and for every module not marked, the composition is byte for byte what it was.
const (
handedNotifier = `{"module":"notify","version":"1",
"config":{"width":{"type":"number","description":"How wide a notification is, in pixels.","default":250}},
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},
"resources":[
{"id":"configuration-dir","type":"directory","path":"${machine:account-home}/.config/notify","owner":"${machine:account}","mode":"0755"},
{"id":"configuration","type":"file","path":"${machine:account-home}/.config/notify/rc","owner":"${machine:account}","mode":"0644","template":"files/rc"},
{"id":"legacy","type":"file","path":"/etc/notify.conf","content":"plain = yes\n"},
{"id":"notifier","type":"service","unit":"notify.service","scope":"user","user":"${machine:account}","reload-on":["configuration"]}]}`
notifierText = "width = ${config:width}\nhome = ${machine:account-home}\n"
)
func notifierMoved() string {
return strings.Replace(handedNotifier, `"version":"1",`, `"version":"1","moved":true,`, 1)
}
func builtNotifier() []Built {
return []Built{{Name: "files", Kind: ArtifactArchive,
Reference: ArtifactStoreScheme + "notify/files/blobs/sha256:" + strings.Repeat("d", 64),
Digest: "sha256:" + strings.Repeat("d", 64), Texts: map[string]string{"rc": notifierText}}}
}
func notifierOn(t *testing.T, m Manifest) Resolution {
t.Helper()
r, err := Resolve(shelf(m), []string{m.Module}, workstation(), World{})
if err != nil {
t.Fatal(err)
}
r.Account = "op"
return r
}
func composeWith(t *testing.T, r Resolution, with Rendering) (Composed, string) {
t.Helper()
c, err := r.Compose(with)
if err != nil {
t.Fatal(err)
}
raw, err := json.Marshal(c.Resources)
if err != nil {
t.Fatal(err)
}
return c, string(raw)
}
func handingTo(store string) Rendering {
return Rendering{Hands: true, Provisioner: "mesh-provisioner-client", ArtifactStore: store}
}
func TestAMovedModulesBuildKeepsWhatTheProvisionerNeeds(t *testing.T) {
m := held(t, notifierMoved(), builtNotifier())
want := &Handed{Source: ArtifactStoreScheme + "notify/files/blobs/sha256:" + strings.Repeat("d", 64),
Digest: "sha256:" + strings.Repeat("d", 64), Templates: map[string]string{"configuration": "rc"}}
if !reflect.DeepEqual(m.Handed, want) {
t.Fatalf("the build kept %+v, want %+v", m.Handed, want)
}
if notMoved := held(t, handedNotifier, builtNotifier()); notMoved.Handed != nil {
t.Fatalf("a module not marked moved kept %+v", notMoved.Handed)
}
}
// Nothing changes for a module that has not moved, nor for a moved one where it cannot be handed.
func TestAMovedModuleComposesAsBeforeWhereItIsNotHanded(t *testing.T) {
_, before := composeWith(t, notifierOn(t, held(t, handedNotifier, builtNotifier())), handingTo("store:5000"))
for name, with := range map[string]Rendering{
"an engine that does not take a filled file alone": {Provisioner: "mesh-provisioner-client", ArtifactStore: "store:5000"},
"no provisioner client on the machine": {Hands: true, ArtifactStore: "store:5000"},
} {
c, got := composeWith(t, notifierOn(t, held(t, notifierMoved(), builtNotifier())), with)
if got != before {
t.Errorf("%s: the moved module composes differently from its unmarked twin:\nbefore: %s\nnow: %s",
name, before, got)
}
if len(c.Filled) > 0 || c.Plans != nil {
t.Errorf("%s: something was handed: %v %v", name, c.Filled, c.Plans)
}
}
if !strings.Contains(before, `width = 250`) {
t.Fatalf("the unmarked module is not filled the old way: %s", before)
}
}
func TestAMovedModulesTemplatedFileIsHandedToTheProvisioner(t *testing.T) {
c, got := composeWith(t, notifierOn(t, held(t, notifierMoved(), builtNotifier())), handingTo("store:5000"))
var file, legacy, dir map[string]any
for _, r := range c.Resources {
switch r["id"] {
case "notify.configuration":
file = r
case "notify.legacy":
legacy = r
case "notify.configuration-dir":
dir = r
}
}
if file == nil || legacy == nil || dir == nil {
t.Fatalf("not composed: %s", got)
}
wantFile := map[string]any{"id": "notify.configuration", "type": "file", "path": "/home/op/.config/notify/rc",
"owner": "op", "mode": "0644", "owner-at-runtime": "owned"}
if !reflect.DeepEqual(file, wantFile) {
t.Fatalf("the handed file is declared %v, want %v (placed, no text, owned at runtime)", file, wantFile)
}
if legacy["content"] != "plain = yes\n" || legacy["owner-at-runtime"] != nil {
t.Fatalf("a file with its text inline is not composed the old way: %v", legacy)
}
if dir["path"] != "/home/op/.config/notify" {
t.Fatalf("the module's other resources are not placed as before: %v", dir)
}
if strings.Contains(got, "width = ") {
t.Fatalf("the controller filled the handed file: %s", got)
}
if !reflect.DeepEqual(c.Filled, []string{"/home/op/.config/notify/rc"}) {
t.Fatalf("filled files %v", c.Filled)
}
want := map[string]Plan{"notify": {
Templates: PlanTemplates{Source: "http://store:5000/v2/notify/files/blobs/sha256:" + strings.Repeat("d", 64),
Digest: "sha256:" + strings.Repeat("d", 64)},
Files: []PlanFile{{ID: "configuration", Path: "/home/op/.config/notify/rc", Template: "rc",
OwnerAtRuntime: "owned", Takes: "reload-on"}},
Placement: map[string]string{"machine:account-home": "/home/op"},
}}
if !reflect.DeepEqual(c.Plans, want) {
t.Fatalf("the plans are %+v, want %+v", c.Plans, want)
}
}
func TestAMovedModulesMarkIsJudged(t *testing.T) {
for name, raw := range map[string]string{
"marked moved and naming no template": `{"module":"bare","version":"1","moved":true,
"resources":[{"id":"conf","type":"file","path":"/etc/bare","content":"x"}]}`,
"handed written in a repository": `{"module":"bare","version":"1","moved":true,
"handed":{"source":"s","digest":"d","templates":{}},
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},
"resources":[{"id":"conf","type":"file","path":"/etc/bare","template":"files/bare"}]}`,
"handed and not moved": `{"module":"bare","version":"1",
"handed":{"source":"s","digest":"d","templates":{}},
"resources":[{"id":"conf","type":"file","path":"/etc/bare","content":"x"}]}`,
} {
if _, err := ParseManifest([]byte(raw)); err == nil {
t.Errorf("%s: accepted", name)
}
}
}
// What `module check` and the merge gate judge: a moved module read from its tree, its text inline. It is still
// marked moved, and composes as before, since nothing of its build's archive is there to hand.
func TestAMovedModuleReadFromItsTreeIsJudgedAsItComposesUnhanded(t *testing.T) {
dir := t.TempDir()
if err := os.MkdirAll(filepath.Join(dir, "files"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "files", "rc"), []byte(notifierText), 0o644); err != nil {
t.Fatal(err)
}
m, err := ParseManifest([]byte(notifierMoved()))
if err != nil {
t.Fatal(err)
}
filled, err := WithTemplatesFrom(m, dir)
if err != nil {
t.Fatalf("a moved module read from its tree is refused: %v", err)
}
if !filled.Moved || filled.Handed != nil {
t.Fatalf("read from its tree: moved %v, handed %+v", filled.Moved, filled.Handed)
}
_, got := composeWith(t, notifierOn(t, filled), handingTo("store:5000"))
_, want := composeWith(t, notifierOn(t, held(t, handedNotifier, builtNotifier())), handingTo("store:5000"))
if got != want {
t.Fatalf("read from its tree it composes differently:\nwant: %s\ngot: %s", want, got)
}
}
+38 -3
View File
@@ -196,13 +196,31 @@ func healthProblems(m Manifest) []string {
}
var toolChecks []string
otherKinds := 0
for _, r := range m.Resources {
resources := m.Resources
// A module's own runtime is judged as the process the controller writes for it (novox/hq ADR 0366 §1),
// and only a module that runs as its own account has one.
own, hasOwn := OwnRuntimeHealth(m)
if hasOwn {
if m.RunsAs == "" {
problems = append(problems, fmt.Sprintf("%s declares runtime-health and runs as no account of its own "+
"(runs-as): only a module served by a runtime of its own has that runtime's health to declare "+
"(novox/hq ADR 0366 §1)", m.Module))
hasOwn = false
} else {
resources = append(append([]map[string]any(nil), m.Resources...), own)
}
}
for i, r := range resources {
h, has, read := ReadHealth(r)
if !has {
continue
}
ownRuntime := hasOwn && i == len(resources)-1
id, typ := fmt.Sprint(r["id"]), fmt.Sprint(r["type"])
where := m.Module + ": " + id
if ownRuntime {
where = m.Module + ": its runtime-health"
}
for _, p := range read {
problems = append(problems, m.Module+": "+p)
}
@@ -279,7 +297,11 @@ func healthProblems(m Manifest) []string {
problems = append(problems, fmt.Sprintf("%s's tool check asks %q, which %s does not serve (its tools: %s)",
where, h.Tool, m.Module, orNoneWords(m.Tools)))
}
toolChecks = append(toolChecks, id)
// Its own runtime being alive, judged by the node-engine, is the check the module does not run
// itself that this tool check stands beside (novox/hq ADR 0366 §2).
if !ownRuntime {
toolChecks = append(toolChecks, id)
}
} else {
if h.Tool != "" {
problems = append(problems, fmt.Sprintf("%s's %s check names a tool, which only a tool check asks", where, h.Kind))
@@ -372,7 +394,11 @@ func Undeclared(m Manifest) []string {
// is granted to ask of its own node tools (to-be 48 §3).
func HealthChecks(m Manifest) []string {
var out []string
for _, r := range m.Resources {
resources := m.Resources
if own, has := OwnRuntimeHealth(m); has && m.RunsAs != "" {
resources = append(append([]map[string]any(nil), resources...), own)
}
for _, r := range resources {
h, has, _ := ReadHealth(r)
if has && h.Kind == HealthTool && h.Tool != "" && LongRunning(r) {
out = append(out, m.Module+"."+h.Tool)
@@ -382,6 +408,15 @@ func HealthChecks(m Manifest) []string {
return out
}
// OwnRuntimeHealth is a module's `runtime-health` as the resource it is put on (novox/hq ADR 0366 §1): the
// process `<module>.own-runtime`, carrying it as its `health`. False when the module declares none.
func OwnRuntimeHealth(m Manifest) (map[string]any, bool) {
if m.RuntimeHealth == nil {
return nil, false
}
return map[string]any{"id": m.Module + "." + OwnRuntimeID(), "type": "process", HealthField: m.RuntimeHealth}, true
}
// healthInto composes a resource's `health` into the node-engine's words, or takes it away (to-be 48 §2,
// §3): for an engine that reads it, the endpoint becomes the port this machine published it on and the
// defaults are written out; for one that does not — older and strict — the field is not sent, and the
+162 -6
View File
@@ -176,8 +176,16 @@ type Offer struct {
// bound to its data, and moves only by a person (novox/hq ADR 0232). `false` for a provision any
// provider answers alike, the mesh's resolver; unsaid, see KeepsConsumerData.
KeepsConsumerData *bool `json:"keeps-consumer-data,omitempty"`
// AnsweredBy says who answers this provision's consumers: AnsweredByHelper when the module runs the SDK's provider
// helper for it, so a handed consumer's template may name what the provider tells it (`${bound:<provision>:<key>}`)
// from the helper's entry on the bus (issue 792). It is separate from `moved`: it hands none of the module's own
// files. Unsaid, the controller answers the consumer.
AnsweredBy string `json:"answered-by,omitempty"`
}
// AnsweredByHelper is the only answerer an offer may name besides the controller.
const AnsweredByHelper = "helper"
// OfferIdentity is what an offer says about the names its backend keeps for its consumers.
type OfferIdentity struct {
// None is set by `"identity": false`: the provision keeps no name derived from its consumer.
@@ -334,6 +342,47 @@ func (m Manifest) SharedCredentialOf(provision string) (string, bool) {
return "", false
}
// The credential forms an offer declares (issue 768), as a consumer's plan names them for the vault, which makes
// a first value in that form for a provider it holds no row of.
const (
// CredentialFormShared is one of the provider's own secrets, shared by every consumer (novox/hq ADR 0158).
CredentialFormShared = "shared"
// CredentialFormPair is a login of its own for each consumer (novox/hq ADR 0048, ADR 0158 §11): the offer says
// no credential.
CredentialFormPair = "pair"
// CredentialFormIssued is issued by the provider's software (novox/hq ADR 0305); the mesh never makes it.
CredentialFormIssued = "issued"
// CredentialFormGiven is given by a person (novox/hq ADR 0337): the offer's credential, or the own secret it
// shares, is issued outside the mesh; the mesh never makes it.
CredentialFormGiven = "given"
)
// CredentialFormOf is the credential form this module's offer of a provision declares, read from the offer's
// `credential` alone (issue 768): no new manifest field, since that field already says whether the provider shares
// one of its own secrets, its software issues it, or a person gives it, and saying none is a login per consumer.
// "" when the module does not offer the provision.
func (m Manifest) CredentialFormOf(provision string) string {
for _, o := range m.Provides {
if o.Name != provision {
continue
}
switch c := o.Credential; {
case c == nil:
return CredentialFormPair
case c.IssuedBy == IssuedOutside:
return CredentialFormGiven
case c.IssuedBy == IssuedBySoftware:
return CredentialFormIssued
case c.Own != "" && m.OwnSecrets[c.Own].IssuedBy == IssuedOutside:
return CredentialFormGiven
case c.Own != "":
return CredentialFormShared
}
return CredentialFormPair
}
return ""
}
// ProvisionsSharing is every provision of this module whose credential is the named own secret.
func (m Manifest) ProvisionsSharing(own string) []string {
var out []string
@@ -368,13 +417,19 @@ func (o *Offer) UnmarshalJSON(raw []byte) error {
Reach string `json:"reach,omitempty"`
Identity *OfferIdentity `json:"identity,omitempty"`
Keeps *bool `json:"keeps-consumer-data,omitempty"`
AnsweredBy string `json:"answered-by,omitempty"`
}
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&full); err != nil {
return fmt.Errorf("a provided name is either a string or {name, scope, credential, reach, identity, "+
"keeps-consumer-data}: %w", typedUnknown(err))
"keeps-consumer-data, answered-by}: %w", typedUnknown(err))
}
if full.AnsweredBy != "" && full.AnsweredBy != AnsweredByHelper {
return fmt.Errorf("%s says answered-by %q: the only answerer an offer names is %q", full.Name, full.AnsweredBy,
AnsweredByHelper)
}
o.AnsweredBy = full.AnsweredBy
o.Name, o.Scope, o.Credential, o.Reach, o.Identity = full.Name, full.Scope, full.Credential, full.Reach, full.Identity
o.KeepsConsumerData = full.Keeps
return nil
@@ -383,7 +438,8 @@ func (o *Offer) UnmarshalJSON(raw []byte) error {
// MarshalJSON writes back the short form when there is nothing else to say, so a manifest that
// went through the mesh comes out looking like the one that went in.
func (o Offer) MarshalJSON() ([]byte, error) {
if o.Scope == "" && o.Credential == nil && o.Reach == "" && o.Identity == nil && o.KeepsConsumerData == nil {
if o.Scope == "" && o.Credential == nil && o.Reach == "" && o.Identity == nil && o.KeepsConsumerData == nil &&
o.AnsweredBy == "" {
return json.Marshal(o.Name)
}
return json.Marshal(struct {
@@ -393,7 +449,19 @@ func (o Offer) MarshalJSON() ([]byte, error) {
Reach string `json:"reach,omitempty"`
Identity *OfferIdentity `json:"identity,omitempty"`
Keeps *bool `json:"keeps-consumer-data,omitempty"`
}{o.Name, o.Scope, o.Credential, o.Reach, o.Identity, o.KeepsConsumerData})
AnsweredBy string `json:"answered-by,omitempty"`
}{o.Name, o.Scope, o.Credential, o.Reach, o.Identity, o.KeepsConsumerData, o.AnsweredBy})
}
// AnsweredByHelperFor is whether this module's offer of a provision says its consumers are answered by the provider
// helper (issue 792).
func (m Manifest) AnsweredByHelperFor(provision string) bool {
for _, o := range m.Provides {
if o.Name == provision && o.AnsweredBy == AnsweredByHelper {
return true
}
}
return false
}
// Manifest is everything a module says about itself.
@@ -544,12 +612,21 @@ type Manifest struct {
// stays refused by name until a layer sets it. The mesh's layer, then the node's, override it.
Settings map[string]SettingDeclaration `json:"settings,omitempty"`
// forms is which of the two forms its config is written in, `config` or its alias `settings` (config.go,
// novox/hq ADR 0348 §1): the config form is read into Settings, a mergeable file's takes-settings and
// `${setting:<key>}`, and written back as it was read.
forms configForms
// unknown is the first key this manifest has that this controller does not know, at any depth, when
// it was read from the store (novox/hq ADR 0262): a manifest a newer controller registered.
// ParseManifest refuses it; the store's catalogue still loads, and the module is left out of every
// machine's declaration by name until the controller is updated (LeftOut).
unknown string
// predefined is the list of predefined keys it was judged against (predefined.go, novox/hq ADR 0348 §7):
// read again — its templates filled from the tree — it is judged against the same list.
predefined PredefinedList
// bestEffort marks the view of a left-out module that only its backup lines are made from
// (backupView, novox/hq ADR 0262): a line of it that cannot be placed is said in the plan's list of
// what could not be placed, never an error that would cost the whole machine its declaration.
@@ -652,6 +729,19 @@ type Manifest struct {
// wrong the moment anybody edits anything.
Build *Build `json:"build,omitempty"`
// Templates is the module's directory its template files are kept in, and its build packs into its templates
// archive: files/ when it says none (DefaultTemplatesDir; novox/hq ADR 0349 §2, issue 731). A module whose files/
// is already an archive unpacked onto the machine names another, so its templates neither land beside what it
// unpacks nor change that archive's digest. A word of the repository's manifest only, like Build: the manifest
// the mesh holds carries each file's text and names no templates directory.
Templates string `json:"templates,omitempty"`
// Moved marks a module handed to the provisioner (novox/hq ADR 0349 §17, design 57 P6): each of its files that
// names a template is filled on the node by the provisioner's client, and the controller fills nothing in it.
// Handed is what its build kept for that (handed.go): written by Resolve, never in a repository.
Moved bool `json:"moved,omitempty"`
Handed *Handed `json:"handed,omitempty"`
// Binds is where this module wants to be told about something it requires, per requirement.
//
// Because "this machine needs a database from the anchor" is useless to the program that
@@ -730,6 +820,11 @@ type Manifest struct {
// that says a warrant, or speaks for a channel kind that proves its sender.
RunsAs string `json:"runs-as,omitempty"`
// RuntimeHealth is how the runtime of its own a module of its own account is served by is ready
// (novox/hq ADR 0366): one `health` in the form a resource's takes (ADR 0240), put by the controller on
// the process it writes for that runtime (`<module>.own-runtime`). Refused without runs-as.
RuntimeHealth any `json:"runtime-health,omitempty"`
// Keeps is where this module wants every operator-sealed secret in the mesh written — the
// vault's field, and so far nobody else's (novox/hq ADR 0085, amended).
//
@@ -783,6 +878,12 @@ type Manifest struct {
// holder places them; the module depends on the seat (ADR 0210 §3).
Contributions []SeatContribution `json:"contributions,omitempty"`
// Collections are the collections this module collects (novox/hq ADR 0349 §13, design 57 L1): each placed in a
// file of its own as ${collected:<name>}, filled on the node by the provisioner's client from every item of it
// there. Items are what it contributes to others' collections, each a template of its own (collections.go).
Collections []Collection `json:"collections,omitempty"`
Items []Item `json:"items,omitempty"`
// Guards are ports of this module's the mesh refuses on an adopted node except from the
// private network and from the machine itself (novox/hq ADR 0100) — the store's port and the
// broker's management port. The ports the software uses; the mesh guards where the machine
@@ -1339,10 +1440,25 @@ type manifestFields Manifest
// strict read there failed the whole catalogue, and with it every plan and every send, the moment a
// controller was rolled back. UnknownField says what was set aside.
func (m *Manifest) UnmarshalJSON(raw []byte) error {
// The config form read into the settings form's fields, so both compose by one path (config.go).
raw, forms := toSettingsForm(raw)
var keys map[string]json.RawMessage
if err := json.Unmarshal(raw, &keys); err != nil {
return err
}
if settings, has := keys[SettingsField]; has && string(settings) != "null" {
forms.settings = append(forms.settings, SettingsField)
}
if config, has := keys[ConfigField]; has {
if string(config) != "null" {
forms.config = append(forms.config, ConfigField)
}
if _, both := keys[SettingsField]; !both {
keys[SettingsField] = config
}
// Beside `settings` it is refused by name (MixedFormProblems), never merged into it.
delete(keys, ConfigField)
}
plain := map[string]string{}
many := map[string]map[string]string{}
if secrets, ok := keys["secrets"]; ok && string(secrets) != "null" {
@@ -1471,6 +1587,7 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
}
*m = Manifest(fields)
m.unknown = unknown
m.forms = forms
m.OptionalRequires = optionalRequires
if len(plain) > 0 {
m.Secrets = plain
@@ -1488,8 +1605,16 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
}
// MarshalJSON writes `secrets` and `contributes` back in the shape they were read: single values,
// and objects of local names.
// and objects of local names; and its config in the form it was read in (config.go).
func (m Manifest) MarshalJSON() ([]byte, error) {
raw, err := m.marshalFields()
if err != nil || !m.DeclaresConfig() {
return raw, err
}
return toConfigForm(raw)
}
func (m Manifest) marshalFields() ([]byte, error) {
raw, err := json.Marshal(manifestFields(m))
if err != nil {
return nil, err
@@ -1589,6 +1714,17 @@ func SecretLocal(to, local string) string {
func (m Manifest) UnknownField() string { return m.unknown }
func ParseManifest(raw []byte) (Manifest, error) {
return parseManifest(raw, PublishedPredefined())
}
// ParseManifestWith is ParseManifest judging every value that falls back to a predefined key against the list
// given — the one read from the tree the manifest is in (predefined.go, novox/hq ADR 0348 §7) — where
// ParseManifest judges against the list the mesh published. Nil is no list: a value naming a key is refused.
func ParseManifestWith(raw []byte, predefined PredefinedList) (Manifest, error) {
return parseManifest(raw, predefined)
}
func parseManifest(raw []byte, predefined PredefinedList) (Manifest, error) {
var m Manifest
// Strictly. **An unknown key is refused**, which is the discipline the host's declaration
// parser has and manifests lacked (novox/hq 04-ISSUES/003): a `scope:` key survived in five
@@ -1616,6 +1752,7 @@ func ParseManifest(raw []byte) (Manifest, error) {
}
return Manifest{}, fmt.Errorf("this is not a module manifest: %w", err)
}
m.predefined = predefined
var problems []string
if !name.MatchString(m.Module) {
@@ -1809,8 +1946,14 @@ func ParseManifest(raw []byte) (Manifest, error) {
problems = append(problems, LogProblems(m)...)
// And the streams it owns, and who may hear them (streams.go, novox/hq ADR 0363).
problems = append(problems, StreamProblems(m)...)
// And the defaults it gives its settings (setting_defaults.go, novox/hq ADR 0262).
problems = append(problems, SettingProblems(m)...)
// And the defaults it gives its settings (setting_defaults.go, novox/hq ADR 0262), or, written in the
// config form, the whole declaration of each value (config.go, novox/hq ADR 0348).
if m.DeclaresConfig() {
problems = append(problems, ConfigProblems(m)...)
} else {
problems = append(problems, SettingProblems(m)...)
problems = append(problems, TypedSettingProblems(m)...)
}
// And what it requires optionally, and the files it delivers only while bound (novox/hq ADR 0330).
problems = append(problems, OptionalProblems(m)...)
// And which setting keys each mergeable file takes (setting_takes.go, novox/hq ADR 0326).
@@ -1879,6 +2022,11 @@ func ParseManifest(raw []byte) (Manifest, error) {
}
}
problems = append(problems, m.Build.problems(m.Module)...)
// A file's text in a template file of the module's repository (novox/hq ADR 0349 §2): only a file, never beside
// content, only under files/, and packed by the module's build.
problems = append(problems, templateProblems(m)...)
problems = append(problems, movedProblems(m)...)
problems = append(problems, collectionProblems(m)...)
problems = append(problems, undeliveredBundles(m)...)
// **What provides the artifact store cannot be delivered through it** (novox/hq 04-ISSUES/029).
//
@@ -2367,6 +2515,14 @@ func ParseManifest(raw []byte) (Manifest, error) {
problems = append(problems, healthProblems(m)...)
// And whether a container is busy (novox/hq ADR 0335), likewise.
problems = append(problems, busyProblems(m)...)
if m.DeclaresConfig() {
// Said in the words it was written in: the rules shared with the settings form say ${setting:<key>}
// and takes-settings.
for i, p := range problems {
problems[i] = inConfigWords(p)
}
problems = append(problems, MixedFormProblems(m)...)
}
if len(problems) > 0 {
sort.Strings(problems)
return Manifest{}, fmt.Errorf("this manifest cannot be used:\n - %s",
+79 -2
View File
@@ -34,6 +34,83 @@ const PlacesSetting = "places"
// AccessesSetting is the settings key that says where a module's accesses are on this node, by id.
const AccessesSetting = "accesses"
// **Placement is not config** (novox/hq ADR 0348 clause 11, issues 651 and 736). `places`, `accesses` and
// `networks` — where an assignment's things are — and `endpoints`, `expose`, `reach` and `ports` — where it
// is reached and from how far — are facts of an assignment the controller decides, held beside the
// assignment and apart from its settings: a Layer carries them in Placement, never in Values, and only
// their readers (Places, AccessPlaces, KeptNetworks, Endpoints, Exposure, Reaches, GivenPorts) read them.
// A person still says them as keys of a layer through `settings`; LayerOf splits such a layer, so what
// they type is routed to the placement and nothing they do changes.
// placementKeys are the keys of a layer that are placement, not config.
var placementKeys = map[string]bool{PlacesSetting: true, AccessesSetting: true, NetworksSetting: true,
EndpointsSetting: true, ExposeSetting: true, ReachSetting: true, PortsSetting: true}
// PlacementKeys are the keys of a layer that are placement, not config, sorted: what a settings row must
// no longer hold once the move of placement has run (issue 736).
func PlacementKeys() []string {
keys := make([]string, 0, len(placementKeys))
for k := range placementKeys {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
// SplitPlacement divides a layer as a person says it into its config values and its placement. Each is
// nil when it holds nothing; the values themselves are not copied.
func SplitPlacement(values map[string]any) (config, placement map[string]any) {
for key, value := range values {
if placementKeys[key] {
if placement == nil {
placement = map[string]any{}
}
placement[key] = value
continue
}
if config == nil {
config = map[string]any{}
}
config[key] = value
}
return config, placement
}
// LayerOf is a layer as a person says it — config values and placement in one map — split into the two.
func LayerOf(from string, values map[string]any) Layer {
config, placement := SplitPlacement(values)
if config == nil {
config = map[string]any{}
}
return Layer{From: from, Values: config, Placement: placement}
}
// Whole is the layer as a person says it: its values and its placement in one map, a new map.
func (l Layer) Whole() map[string]any {
out := make(map[string]any, len(l.Values)+len(l.Placement))
for k, v := range l.Values {
out[k] = v
}
for k, v := range l.Placement {
out[k] = v
}
return out
}
// asSaid is the layers with placement folded back into the values, for what reads a layer whole as it
// did while placement was kept in it: today's mergeable file, which takes every key (ADR 0326 §3), and
// the judgement that no value holds a line end.
func asSaid(layers []Layer) []Layer {
out := make([]Layer, len(layers))
for i, l := range layers {
out[i] = l
if len(l.Placement) > 0 {
out[i] = Layer{From: l.From, Values: l.Whole(), Default: l.Default}
}
}
return out
}
// Placement is what an assignment says about one of a module's directories.
type Placement struct {
// Path is where the directory is on this machine. Absolute.
@@ -208,7 +285,7 @@ func Places(m Manifest, layers []Layer) (map[string]Placement, error) {
}
out := map[string]Placement{}
for _, layer := range layers {
raw, ok := layer.Values[PlacesSetting]
raw, ok := layer.Placement[PlacesSetting]
if !ok {
continue
}
@@ -278,7 +355,7 @@ func AccessPlaces(m Manifest, layers []Layer) (map[string]string, error) {
}
out := map[string]string{}
for _, layer := range layers {
raw, ok := layer.Values[AccessesSetting]
raw, ok := layer.Placement[AccessesSetting]
if !ok {
continue
}
+102 -1
View File
@@ -23,7 +23,7 @@ func placeable() Manifest {
}
func placedBy(values map[string]any) Rendering {
return Rendering{Settings: SettingsBy{"arr": {{From: "node anchor", Values: values}}}}
return Rendering{Settings: SettingsBy{"arr": {LayerOf("node anchor", values)}}}
}
func TestAnAssignmentPlacesDirectoriesAndAccesses(t *testing.T) {
@@ -179,3 +179,104 @@ func TestAnUnknownAccessReferenceIsRefusedAtParse(t *testing.T) {
t.Fatalf("an access with no id and no path was accepted: %v", err)
}
}
// **Placement is read from the placement, never from a layer's values** (novox/hq ADR 0348 clause 11, issue
// 651). A `places` or `accesses` among the config values places nothing; the same in Placement does. A
// mergeable file of today's form still takes every key of the layer as it is said, placement too, so it
// composes as it did while placement was kept in the layer.
func TestPlacementIsReadFromThePlacementOnly(t *testing.T) {
m := placeable()
said := map[string]any{
PlacesSetting: map[string]any{"config": "/services/arr/config"},
AccessesSetting: map[string]any{"series": "/storage/media/series", "spool": "/storage/downloads"},
"log": "warn",
}
among := []Layer{{From: "anchor", Values: said}}
if placed, err := Places(m, among); err != nil || placed != nil {
t.Fatalf("a place among the config values placed %v (%v)", placed, err)
}
if _, _, err := accessesFor(m, among); err == nil {
t.Fatal("an access among the config values placed it")
}
beside := []Layer{LayerOf("anchor", said)}
if beside[0].Values["log"] != "warn" || beside[0].Values[PlacesSetting] != nil || beside[0].Placement[PlacesSetting] == nil {
t.Fatalf("LayerOf split %+v", beside[0])
}
if placed, err := Places(m, beside); err != nil || placed["config"].Path != "/services/arr/config" {
t.Fatalf("the placement did not place: %v (%v)", placed, err)
}
file := map[string]any{"id": "runtime", "type": "file", "path": "/x", "content": `{"a":1}`, "merge": "json"}
whole, err := ApplySettings(file, among)
if err != nil {
t.Fatal(err)
}
split, err := ApplySettings(file, beside)
if err != nil {
t.Fatal(err)
}
if whole["content"] != split["content"] || !strings.Contains(split["content"].(string), `"places"`) {
t.Fatalf("today's mergeable file composes %q, was %q", split["content"], whole["content"])
}
}
// **The four words of where an assignment is reached are placement too** (novox/hq ADR 0348 clause 11, issue 736).
// `endpoints`, `expose`, `reach` and `ports` among a layer's config values say nothing to the filter, the ports or
// the names; the same in Placement do, and LayerOf puts them there. A mergeable file of today's form still takes
// every key of the layer as it is said, so it composes as it did while they were kept among the values.
func TestTheFourWordsAreReadFromThePlacementOnly(t *testing.T) {
m := Manifest{Module: "forge", Version: "1",
Listens: []Listening{{Name: "web", Port: 3000, Protocol: "tcp", From: FromMesh}, {Name: "ssh", Port: 22, Protocol: "tcp", From: FromMesh},
{Name: "api", Port: 8086, Protocol: "tcp", From: FromMesh}},
Resources: []map[string]any{{"id": "server", "type": "container", "image": "x", "ports": []any{"3000", "222:22", "8086"}}}}
said := map[string]any{
ExposeSetting: map[string]any{"22": FromEverywhere},
PortsSetting: map[string]any{"22": float64(2222)},
ReachSetting: map[string]any{"3000": "internal"},
EndpointsSetting: map[string]any{"api": map[string]any{"label": "influx", "port": float64(18086), "reach": "public"}},
"log": "warn",
}
among := []Layer{{From: "anchor", Values: said}}
if e, err := Exposure(m, among); err != nil || e != nil {
t.Fatalf("an expose among the config values exposed %v (%v)", e, err)
}
if g, err := GivenPorts(m, among); err != nil || g != nil {
t.Fatalf("a ports among the config values gave %v (%v)", g, err)
}
if r, err := Reaches(m, among); err != nil || r != nil {
t.Fatalf("a reach among the config values reached %v (%v)", r, err)
}
if b, err := Endpoints(m, among); err != nil || len(b) != 0 {
t.Fatalf("an endpoints among the config values configured %v (%v)", b, err)
}
beside := []Layer{LayerOf("anchor", said)}
if len(beside[0].Values) != 1 || beside[0].Values["log"] != "warn" || len(beside[0].Placement) != 4 {
t.Fatalf("LayerOf split %+v", beside[0])
}
if e, err := Exposure(m, beside); err != nil || e[22] != FromEverywhere {
t.Fatalf("the placement did not expose: %v (%v)", e, err)
}
if g, err := GivenPorts(m, beside); err != nil || g[22] != 2222 || g[8086] != 18086 {
t.Fatalf("the placement did not give the ports: %v (%v)", g, err)
}
if r, err := Reaches(m, beside); err != nil || r[3000] != "internal" || r[8086] != "public" {
t.Fatalf("the placement did not reach: %v (%v)", r, err)
}
if b, err := Endpoints(m, beside); err != nil || b["api"].Label != "influx" {
t.Fatalf("the placement did not configure the endpoint: %v (%v)", b, err)
}
file := map[string]any{"id": "runtime", "type": "file", "path": "/x", "content": `{"a":1}`, "merge": "json"}
whole, err := ApplySettings(file, among)
if err != nil {
t.Fatal(err)
}
split, err := ApplySettings(file, beside)
if err != nil {
t.Fatal(err)
}
if whole["content"] != split["content"] || !strings.Contains(split["content"].(string), `"endpoints"`) {
t.Fatalf("today's mergeable file composes %q, was %q", split["content"], whole["content"])
}
}
+236
View File
@@ -0,0 +1,236 @@
package catalogue
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"sort"
"strings"
"sync"
)
// The list of predefined keys (novox/hq ADR 0348 §7, design 57 package K2, issue 661).
//
// **A fixed, reviewable list, owned by the build.** Mesh config and node config hold values only for the keys
// in the catalogue's `predefined.json`, at the root of its repository: each key with its type, a description
// including its unit, its validation (enum, minimum and maximum, pattern) and whether it is trusted. A module's
// value may fall back to one of them, of the same type (ADR 0348 §2, §4).
//
// **Read where a manifest is judged, from the tree it is judged in.** A build reads the file at the root of the
// repository at the commit it builds, `module check` at the root of the repository above each manifest, the
// merge gate at the root of the tree under check; each refuses a file that is not exactly a list of keys, and
// judges every manifest it reads against it (ParseManifestWith). Everywhere else — registration, a stored build
// read again — a manifest is judged against the list the mesh published (PublishedPredefined): the newest trunk
// build's of the catalogue, which the controller keeps and loads with its store.
// PredefinedFile is the list's file at the root of the catalogue's repository.
const PredefinedFile = "predefined.json"
// CatalogueRepository is the name of the repository whose list the mesh publishes: the catalogue's, by the name
// a check finds it beside under (internal/builder's check, ADR 0121).
const CatalogueRepository = "mesh-catalog"
// PredefinedKey is one key of the list: its type, its description, its validation and whether it is trusted
// (ADR 0348 §7). It has no default: a module's value falling back to it has its own.
type PredefinedKey struct {
Type string `json:"type"`
Description string `json:"description"`
Enum []any `json:"enum,omitempty"`
Minimum *float64 `json:"minimum,omitempty"`
Maximum *float64 `json:"maximum,omitempty"`
Pattern string `json:"pattern,omitempty"`
Trusted bool `json:"trusted,omitempty"`
}
// PredefinedList is the list by key. Nil is no list at all, which is not the same as an empty one: a value
// naming a key is refused then saying the list is not present.
type PredefinedList map[string]PredefinedKey
// ParsePredefined reads predefined.json strictly: one JSON object of keys, each with exactly the fields of a
// PredefinedKey, and every key judged as a module's config value is (PredefinedProblems). Anything else is
// refused whole, naming each problem — a list half read would let a module fall back to a key nobody reviewed.
func ParsePredefined(raw []byte) (PredefinedList, error) {
var keys map[string]json.RawMessage
if err := json.Unmarshal(raw, &keys); err != nil {
return nil, fmt.Errorf("%s is not a JSON object of keys: %w", PredefinedFile, err)
}
if keys == nil {
return nil, fmt.Errorf("%s is not a JSON object of keys: it says null", PredefinedFile)
}
list := PredefinedList{}
var problems []string
for _, key := range sortedKeys(keys) {
decoder := json.NewDecoder(bytes.NewReader(keys[key]))
decoder.DisallowUnknownFields()
var k PredefinedKey
if err := decoder.Decode(&k); err != nil {
problems = append(problems, fmt.Sprintf("%q is not a predefined key's declaration (type, description, "+
"enum, minimum, maximum, pattern, trusted): %v", key, err))
continue
}
list[key] = k
}
problems = append(problems, PredefinedProblems(list)...)
if len(problems) > 0 {
return nil, fmt.Errorf("%s is refused: %s", PredefinedFile, strings.Join(problems, "; "))
}
return list, nil
}
// PredefinedProblems is every way a key of the list is declared wrongly, by the rules a module's config value
// is held to (ADR 0348 §4, §7, §11): a usable name that is not placement, a type, a description, validation
// that fits the type, and an object or a list only when trusted.
func PredefinedProblems(list PredefinedList) []string {
var problems []string
for _, key := range sortedKeys(list) {
k := list[key]
say := func(format string, args ...any) {
problems = append(problems, fmt.Sprintf("the predefined key %q: ", key)+fmt.Sprintf(format, args...))
}
if !settingRef.MatchString("${setting:" + key + "}") {
say("not a usable key: lower-case letters, digits, dots, dashes and underscores")
continue
}
if meshWords[key] || accountFacts[key] {
say("%q is placement, which the mesh decides, never a config value (novox/hq ADR 0348 §11)", key)
continue
}
switch k.Type {
case ConfigNumber, ConfigString, ConfigBoolean:
case ConfigObject, ConfigList:
if !k.Trusted {
say("is %s and not trusted: an object or a list is trusted, \"trusted\": true (novox/hq ADR 0348 §4)", k.Type)
}
case "":
say("no type: say \"type\": number, string or boolean, or object or list when trusted (novox/hq ADR 0348 §4)")
default:
say("type %q is none of number, string, boolean, object and list (novox/hq ADR 0348 §4)", k.Type)
}
if strings.TrimSpace(k.Description) == "" {
say("no description: say in a sentence what the value is, including its unit (novox/hq ADR 0348 §7)")
}
for _, wrong := range validationProblems(SettingDeclaration{Type: k.Type, Enum: k.Enum, Minimum: k.Minimum,
Maximum: k.Maximum, Pattern: k.Pattern}) {
say("%s", wrong)
}
}
return problems
}
// ReadPredefined is the list in predefined.json at a repository's root, and nil with no error when the root
// holds none: a repository other than the catalogue, or the catalogue before the file.
func ReadPredefined(root string) (PredefinedList, error) {
path := filepath.Join(root, PredefinedFile)
info, err := os.Lstat(path)
if errors.Is(err, fs.ErrNotExist) {
return nil, nil
}
if err != nil {
return nil, err
}
if !info.Mode().IsRegular() {
return nil, fmt.Errorf("%s is %s, not a plain file", path, kindOf(info.Mode()))
}
raw, err := os.ReadFile(path)
if err != nil {
return nil, err
}
return ParsePredefined(raw)
}
// RepositoryRoot is the nearest directory at or above dir holding `.git`: the root of the repository a manifest
// is in, where `module check` reads the list. Empty when none is: a manifest outside a repository is judged
// against no list.
func RepositoryRoot(dir string) string {
at, err := filepath.Abs(dir)
if err != nil {
return ""
}
for {
if _, err := os.Lstat(filepath.Join(at, ".git")); err == nil {
return at
}
up := filepath.Dir(at)
if up == at {
return ""
}
at = up
}
}
// PredefinedAbove is the list at the root of the repository above dir (RepositoryRoot), nil when there is no
// repository or it holds no list.
func PredefinedAbove(dir string) (PredefinedList, error) {
root := RepositoryRoot(dir)
if root == "" {
return nil, nil
}
return ReadPredefined(root)
}
// Canonical is the list as JSON with its keys sorted: what a build carries and the controller keeps, so one list
// is one text.
func (l PredefinedList) Canonical() ([]byte, error) {
if l == nil {
return nil, nil
}
return json.Marshal(map[string]PredefinedKey(l))
}
// Keys is the list's keys, sorted.
func (l PredefinedList) Keys() []string {
out := make([]string, 0, len(l))
for k := range l {
out = append(out, k)
}
sort.Strings(out)
return out
}
// NamesPredefined is whether any value of the manifest falls back to a predefined key.
func (m Manifest) NamesPredefined() bool {
for _, d := range m.Settings {
if d.Predefined != "" {
return true
}
}
return false
}
// publishedPredefined is the list the mesh published, which a manifest is judged against where no tree is read.
var publishedPredefined struct {
sync.RWMutex
list PredefinedList
}
// UsePredefined makes list the one a manifest is judged against where no tree is read: what the controller
// loads from its store, and what it keeps when a newer trunk build of the catalogue publishes one. Nil leaves
// the list it has: a store that holds none yet never empties one already loaded.
func UsePredefined(list PredefinedList) {
if list == nil {
return
}
publishedPredefined.Lock()
defer publishedPredefined.Unlock()
publishedPredefined.list = list
}
// PublishedPredefined is the list a manifest is judged against where no tree is read; nil when none is loaded.
func PublishedPredefined() PredefinedList {
publishedPredefined.RLock()
defer publishedPredefined.RUnlock()
return publishedPredefined.list
}
// ReplacePublished sets the published list, nil included, and answers the one it replaced: for a test that
// judges against a list of its own and puts the mesh's back.
func ReplacePublished(list PredefinedList) (was PredefinedList) {
publishedPredefined.Lock()
defer publishedPredefined.Unlock()
was, publishedPredefined.list = publishedPredefined.list, list
return was
}
+175
View File
@@ -0,0 +1,175 @@
package catalogue
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
)
// The list of predefined keys (novox/hq ADR 0348 §7, design 57 package K2, issue 661): read strictly, and every
// manifest naming a key judged against the list of the tree it is read in.
// theFirstKeys is the catalogue's first list: font-size and agent-managed-settings.
const theFirstKeys = `{
"font-size": {"type": "number", "description": "the size text is shown in, in points", "minimum": 6, "maximum": 72},
"agent-managed-settings": {"type": "object", "description": "the managed settings of every agent session, a JSON object", "trusted": true}
}`
func TestTheFirstListIsRead(t *testing.T) {
list, err := ParsePredefined([]byte(theFirstKeys))
if err != nil {
t.Fatal(err)
}
if got := strings.Join(list.Keys(), ","); got != "agent-managed-settings,font-size" {
t.Fatalf("keys %s", got)
}
if k := list["font-size"]; k.Type != ConfigNumber || *k.Minimum != 6 || *k.Maximum != 72 || k.Trusted {
t.Fatalf("font-size read as %+v", k)
}
if k := list["agent-managed-settings"]; k.Type != ConfigObject || !k.Trusted {
t.Fatalf("agent-managed-settings read as %+v", k)
}
// One list is one text, however it was written.
a, _ := list.Canonical()
again, _ := ParsePredefined(a)
b, _ := again.Canonical()
if string(a) != string(b) {
t.Fatalf("canonical text moved:\n%s\n%s", a, b)
}
}
func TestAListIsRefusedWholeForAnyWrongKey(t *testing.T) {
for _, c := range []struct{ file, says string }{
{`[]`, "not a JSON object of keys"},
{`null`, "not a JSON object of keys"},
{`{"font-size": {"type": "number", "description": "points", "default": 10}}`, `unknown field "default"`},
{`{"font-size": {"description": "points"}}`, "no type"},
{`{"font-size": {"type": "decimal", "description": "points"}}`, `type "decimal"`},
{`{"font-size": {"type": "number"}}`, "no description"},
{`{"accent": {"type": "object", "description": "colours"}}`, "not trusted"},
{`{"font-size": {"type": "number", "description": "points", "minimum": 9, "maximum": 6}}`, "above its maximum"},
{`{"accent": {"type": "string", "description": "a name", "pattern": "("}}`, "not a regular expression"},
{`{"accent": {"type": "boolean", "description": "on or off", "minimum": 1}}`, "minimum or maximum is for a number"},
{`{"Font Size": {"type": "number", "description": "points"}}`, "not a usable key"},
{`{"places": {"type": "string", "description": "where"}}`, "placement"},
{`{"puid": {"type": "number", "description": "an id"}}`, "placement"},
} {
if _, err := ParsePredefined([]byte(c.file)); err == nil || !strings.Contains(err.Error(), c.says) {
t.Errorf("%s: wanted a refusal saying %q, got %v", c.file, c.says, err)
}
}
}
// fallingBackTo is inConfigForm with its font size falling back to a predefined key.
func fallingBackTo(t *testing.T, key string) []byte {
t.Helper()
m := inConfigForm()
declaration(m, "font-size")["predefined"] = key
raw, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
return raw
}
// A manifest naming font-size registers against the list; one naming a key not on it, or of another type, is
// refused — judged against the list given, whatever the mesh published.
func TestAManifestIsJudgedAgainstTheListOfItsTree(t *testing.T) {
withPredefined(t, nil)
list, err := ParsePredefined([]byte(theFirstKeys))
if err != nil {
t.Fatal(err)
}
if _, err := ParseManifestWith(fallingBackTo(t, "font-size"), list); err != nil {
t.Fatalf("a value falling back to font-size is refused: %v", err)
}
if _, err := ParseManifestWith(fallingBackTo(t, "font-scale"), list); err == nil ||
!strings.Contains(err.Error(), `"font-scale", which is not on the list`) {
t.Fatalf("a key not on the list: %v", err)
}
m := inConfigForm()
declaration(m, "theme")["predefined"] = "font-size"
raw, _ := json.Marshal(m)
if _, err := ParseManifestWith(raw, list); err == nil || !strings.Contains(err.Error(), "which is a number") {
t.Fatalf("a key of another type: %v", err)
}
// Where no tree is read, the published list judges: none published refuses.
if _, err := ParseManifest(fallingBackTo(t, "font-size")); err == nil || !strings.Contains(err.Error(), "is not present") {
t.Fatalf("judged against no list: %v", err)
}
withPredefined(t, list)
if _, err := ParseManifest(fallingBackTo(t, "font-size")); err != nil {
t.Fatalf("judged against the published list: %v", err)
}
}
// A template's text filled from the tree keeps the list the manifest was judged against: read again, it is not
// judged against the published one.
func TestTemplatesFilledKeepTheListTheManifestWasJudgedAgainst(t *testing.T) {
withPredefined(t, nil)
list, _ := ParsePredefined([]byte(theFirstKeys))
dir := t.TempDir()
if err := os.MkdirAll(filepath.Join(dir, DefaultTemplatesDir), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, DefaultTemplatesDir, "rc"), []byte("font = ${config:font-size}\n"), 0o644); err != nil {
t.Fatal(err)
}
m := inConfigForm()
declaration(m, "font-size")["predefined"] = "font-size"
r := resourceOf(m, "configuration")
delete(r, "content")
r["template"] = DefaultTemplatesDir + "/rc"
m["build"] = map[string]any{"artifacts": []any{map[string]any{"name": "files", "kind": "archive", "from": "files"}}}
declaration(m, "width")["runtime"] = true
declaration(m, "sender")["runtime"] = true
raw, _ := json.Marshal(m)
parsed, err := ParseManifestWith(raw, list)
if err != nil {
t.Fatal(err)
}
if _, err := WithTemplatesFrom(parsed, dir); err != nil {
t.Fatalf("filled from the tree, it was judged against another list: %v", err)
}
}
// module check reads the list at the root of the repository above the manifest; no repository, no list.
func TestTheListIsFoundAtTheRepositoryRootAboveAManifest(t *testing.T) {
root := t.TempDir()
module := filepath.Join(root, "modules", "dunst")
if err := os.MkdirAll(module, 0o755); err != nil {
t.Fatal(err)
}
if list, err := PredefinedAbove(module); err != nil || list != nil {
t.Fatalf("outside a repository: %v %v", list, err)
}
if err := os.Mkdir(filepath.Join(root, ".git"), 0o755); err != nil {
t.Fatal(err)
}
if list, err := PredefinedAbove(module); err != nil || list != nil {
t.Fatalf("a repository without the file: %v %v", list, err)
}
if err := os.WriteFile(filepath.Join(root, PredefinedFile), []byte(theFirstKeys), 0o644); err != nil {
t.Fatal(err)
}
list, err := PredefinedAbove(module)
if err != nil || len(list) != 2 {
t.Fatalf("the list above: %v %v", list, err)
}
if err := os.WriteFile(filepath.Join(root, PredefinedFile), []byte(`{"font-size": {}}`), 0o644); err != nil {
t.Fatal(err)
}
if _, err := PredefinedAbove(module); err == nil {
t.Fatal("a wrong list above was read")
}
if err := os.Remove(filepath.Join(root, PredefinedFile)); err != nil {
t.Fatal(err)
}
if err := os.Symlink("/etc/hostname", filepath.Join(root, PredefinedFile)); err == nil {
if _, err := PredefinedAbove(module); err == nil || !strings.Contains(err.Error(), "not a plain file") {
t.Fatalf("a link read as the list: %v", err)
}
}
}
+4 -4
View File
@@ -18,7 +18,7 @@ func aRoutedWeb() Manifest {
func reachSet(reach string) SettingsBy {
return SettingsBy{"web": {{From: "node anchor",
Values: map[string]any{ReachSetting: map[string]any{"3000": reach}}}}}
Placement: map[string]any{ReachSetting: map[string]any{"3000": reach}}}}}
}
// namesFor renders the contribution a routed module makes and returns the two names it carries.
@@ -145,7 +145,7 @@ func TestAPublicNameLeavesARoutedPortAsTheManifestSaid(t *testing.T) {
// written rather than accepted and ignored.
func TestAReachForAPortTheModuleDoesNotListenOnIsRefused(t *testing.T) {
_, err := Reaches(aRoutedWeb(), []Layer{{From: "node anchor",
Values: map[string]any{ReachSetting: map[string]any{"9999": ReachInternal}}}})
Placement: map[string]any{ReachSetting: map[string]any{"9999": ReachInternal}}}})
if err == nil || !strings.Contains(err.Error(), "reaches nothing") {
t.Fatalf("a reach naming an undeclared port was accepted: %v", err)
}
@@ -157,7 +157,7 @@ func TestAReachForAPortTheModuleDoesNotListenOnIsRefused(t *testing.T) {
func TestAValueThatIsNotAReachIsRefused(t *testing.T) {
for _, wrong := range []string{"mesh", "anywhere", "private", "true"} {
_, err := Reaches(aRoutedWeb(), []Layer{{From: "node anchor",
Values: map[string]any{ReachSetting: map[string]any{"3000": wrong}}}})
Placement: map[string]any{ReachSetting: map[string]any{"3000": wrong}}}})
if err == nil || !strings.Contains(err.Error(), "a reach is") {
t.Fatalf("%q was accepted as a reach: %v", wrong, err)
}
@@ -168,7 +168,7 @@ func TestAValueThatIsNotAReachIsRefused(t *testing.T) {
// and accepting both would have the filter follow one while the names followed the other — the
// disagreement ADR 0138 exists to remove, reintroduced by the migration away from the older word.
func TestReachAndExposeForOnePortAreRefused(t *testing.T) {
_, err := Reaches(aRoutedWeb(), []Layer{{From: "node anchor", Values: map[string]any{
_, err := Reaches(aRoutedWeb(), []Layer{{From: "node anchor", Placement: map[string]any{
ReachSetting: map[string]any{"3000": ReachInternal},
ExposeSetting: map[string]any{"3000": FromEverywhere},
}}})
+22 -5
View File
@@ -227,6 +227,9 @@ type Needed struct {
// by every consumer (novox/hq ADR 0158): the name of that secret in the provider's definition.
// The plan mints the pair's copy from the provider's value rather than a value of its own.
SharedOwn string
// CredentialForm is the credential form the provider's offer declares (Manifest.CredentialFormOf, issue 768),
// named in a handed consumer's plan for the vault; empty where the provider's manifest is not in hand.
CredentialForm string
// Manager is set when this holder is a refreshable-grant licence's MANAGER, delivered the refresh
// token rather than an access token (novox/hq ADR 0050). It changes one thing downstream: an empty
// Sealed is tolerated — the manager has not adopted a refresh token yet, which is a real waiting
@@ -248,6 +251,16 @@ type Needed struct {
// consumer's side for `status` and on the provider's before it grants. No bound for a provision
// answered by a record, which keeps no name of anybody's.
Identity IdentityBound
// IssuedToHolder is set when the vault keeps this need's credential for this holder alone (issue 757): the
// providing module is marked moved and its software issues the provision's credential for each holder (issue
// 749), kept as provision/<From>/<Name>/<node>/<For>. Where the consumer is handed to the provisioner, its
// machine's client asks the vault for it and the declaration carries no value. Set by the plan, which reads the
// provider's manifest.
IssuedToHolder bool
// ProviderMoved is set when the providing module is marked moved (issue 788): a handed consumer's template may
// then name `${bound:<provision>:<key>}` and `${secret:<provision>}`, which its machine's client fills from the
// provider's entry and the vault. Set by the plan, which reads the provider's manifest.
ProviderMoved bool
}
// Refusal is why a set of assignments cannot become a declaration.
@@ -530,7 +543,8 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
needs = append(needs, Needed{
Name: want, From: node.Name, At: at, Module: by.Module,
Serves: servedByOne(by, want), For: because[want],
SharedOwn: sharedByOne(by, want), Identity: by.IdentityBoundOf(want)})
SharedOwn: sharedByOne(by, want), CredentialForm: by.CredentialFormOf(want),
Identity: by.IdentityBoundOf(want)})
} else if served := servedByOne(by, want); len(served) > 0 {
// Answered here with no credential to mint, but the provider serves facts the
// consumer cannot guess — a port, a model name — and so still needs a binding.
@@ -550,7 +564,8 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
at = "127.0.0.1"
}
needs = append(needs, Needed{
Name: want, From: node.Name, At: at, Serves: served, For: because[want], Module: by.Module})
Name: want, From: node.Name, At: at, Serves: served, For: because[want], Module: by.Module,
CredentialForm: by.CredentialFormOf(want)})
}
continue
}
@@ -579,16 +594,18 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
node.Name, want, p.Node, meshNetwork))
return
}
shared := ""
shared, form := "", ""
// A provider whose definition is not in hand is held to the tightest bound the
// mesh knows, not to none: what it keeps is not known here (ADR 0225).
bound := DefaultIdentityBound
if pm, known := catalogue[p.Module]; known {
shared, _ = pm.SharedCredentialOf(want)
form = pm.CredentialFormOf(want)
bound = pm.IdentityBoundOf(want)
}
needs = append(needs, Needed{Name: want, From: p.Node, At: p.At, Module: p.Module,
Serves: p.Serves, For: because[want], SharedOwn: shared, Identity: bound, LeftOut: p.LeftOut})
Serves: p.Serves, For: because[want], SharedOwn: shared, CredentialForm: form, Identity: bound,
LeftOut: p.LeftOut})
}
switch {
case world.Unchecked:
@@ -735,7 +752,7 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
satisfied[o] = true
}
if _, ok := because[m.Module]; !ok {
because[m.Module] = fmt.Sprintf("required by %s", because[want])
because[m.Module] = requiredByWords + because[want]
}
for _, r := range m.Wants() {
+5
View File
@@ -320,6 +320,11 @@ func (r Resolution) ownRuntimes(with Rendering) ([]map[string]any, error) {
"run": []any{"./" + program.Binary}, "env": env, "restart-on": toAny(restartOn),
"user": m.RunsAs,
}
// How it is ready, as the module declared it (novox/hq ADR 0366 §1), in the node-engine's words.
if _, has := OwnRuntimeHealth(m); has {
process[HealthField] = m.RuntimeHealth
healthInto(process, m, with)
}
if err := artifactsInto(process, RuntimeModule, with); err != nil {
return nil, err
}
+129
View File
@@ -0,0 +1,129 @@
package catalogue
import (
"encoding/json"
"strings"
"testing"
"github.com/novox/mesh-host/validate"
)
// novox/hq ADR 0366: a module that runs as its own account declares its runtime's health under one manifest
// key, `runtime-health`, in the form a resource's `health` takes; the controller puts it on the process it
// writes for that runtime, and a tool check there stands beside the node-engine's judgement that the runtime
// is alive. A module that runs as no account of its own is refused the key.
// ownAccountManifest is a module of its own account with nothing long-running but its own runtime, given
// runtime-health (or none, when health is nil), and running as no account when runsAs is false.
func ownAccountManifest(t *testing.T, runsAs bool, health any) []byte {
t.Helper()
m := map[string]any{"module": "keeper", "tools": []any{"keeper_status", "keeper_health"},
"own-secrets": map[string]any{"broker": map[string]any{"path": "/var/lib/keeper/broker"}},
"resources": []any{map[string]any{"id": "account", "type": "user", "name": "keeper"}}}
if runsAs {
m["runs-as"], m["secrets-owner"] = "keeper", "keeper"
}
if health != nil {
m["runtime-health"] = health
}
raw, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
return raw
}
func TestAModuleOfItsOwnAccountDeclaresItsRuntimesHealthByItsOwnTool(t *testing.T) {
m, err := ParseManifest(ownAccountManifest(t, true, map[string]any{"kind": "tool", "tool": "keeper_health"}))
if err != nil {
t.Fatalf("a tool check on its own runtime was refused: %v", err)
}
if got := HealthChecks(m); len(got) != 1 || got[0] != "keeper.keeper_health" {
t.Errorf("the engine is granted %v, want keeper.keeper_health", got)
}
if r, has := OwnRuntimeHealth(m); !has || r["id"] != "keeper."+OwnRuntimeID() {
t.Errorf("its runtime's health reads as %v %v", r, has)
}
}
func TestRuntimeHealthIsRefusedWithoutAnAccountOfItsOwn(t *testing.T) {
_, err := ParseManifest(ownAccountManifest(t, false, map[string]any{"kind": "tool", "tool": "keeper_health"}))
if err == nil || !strings.Contains(err.Error(), "runtime-health") || !strings.Contains(err.Error(), "runs-as") {
t.Errorf("runtime-health without runs-as: %v", err)
}
}
func TestARuntimeHealthIsHeldToTheHealthRules(t *testing.T) {
for _, c := range []struct {
health any
says string
}{
{map[string]any{"kind": "tool", "tool": "keeper_restart"}, "does not serve"},
{map[string]any{"kind": "tool", "tool": "keeper_health", "interval": "2s"}, "no more often"},
{map[string]any{"kind": "exec", "command": "true"}, "only a container"},
{map[string]any{"kind": "tool", "tool": "keeper_health", "port": 80}, "names a port"},
{"tool", "is an object"},
} {
_, err := ParseManifest(ownAccountManifest(t, true, c.health))
if err == nil || !strings.Contains(err.Error(), c.says) {
t.Errorf("%v: %v, want it refused saying %q", c.health, err, c.says)
}
}
}
// The runtime-health is composed onto <module>.own-runtime, in the words the node-engine takes; to an engine
// that does not read health it is not sent, as for every resource.
func TestARuntimeHealthIsComposedOntoTheModulesOwnRuntime(t *testing.T) {
goRuntime := Manifest{Module: RuntimeModule, Version: "1",
OwnSecrets: OwnSecrets{"broker": {Path: "/var/lib/mesh/" + RuntimeModule + "/broker"}},
Build: &Build{Artifacts: []Artifact{{Name: "runtime", Kind: ArtifactBundle, Language: "go",
System: "arch", From: "cmd/node-tools"}}}}
goRuntime, err := goRuntime.Resolve([]Built{{Name: "runtime", Kind: ArtifactBundle,
Reference: ArtifactStoreScheme + RuntimeModule + "/runtime/blobs/" + bundleDigest, Digest: bundleDigest}})
if err != nil {
t.Fatal(err)
}
keeper := aToolsModule(t, "keeper", "tools/index.js")
keeper.Tools = []string{"keeper_health"}
keeper.RunsAs, keeper.SecretsOwner = "keeper", "keeper"
keeper.OwnSecrets = OwnSecrets{"broker": {Path: "/var/lib/keeper/broker"}}
keeper.RuntimeHealth = map[string]any{"kind": "tool", "tool": "keeper_health", "interval": "60s", "timeout": "10s",
"looks": 2.0}
for _, reads := range []bool{true, false} {
with := Rendering{ArtifactStore: "anchor.internal:5101", ReadsHealth: reads,
Needed: map[string]map[string]string{RuntimeModule: {"broker": "sealed"}, "keeper": {"broker": "own"}}}
out, err := Resolution{Node: "anchor", Account: "ops", Modules: []Manifest{keeper, goRuntime}}.Declaration(with)
if err != nil {
t.Fatal(err)
}
own := fileNamed(out, "keeper."+OwnRuntimeID())
if own == nil {
t.Fatalf("no own runtime: %v", ids(out))
}
h, has := own[HealthField].(map[string]any)
if !reads {
if has {
t.Errorf("health sent to an engine that does not read it: %v", h)
}
continue
}
if !has || h["kind"] != "tool" || h["tool"] != "keeper_health" || h["interval"] != "1m0s" ||
h["timeout"] != "10s" || h["looks"] != 2 || h["grace"] != "1m0s" {
t.Errorf("its own runtime's health: %v", own[HealthField])
}
if machine := fileNamed(out, RuntimeModule+"."+RuntimeProcessID()); machine[HealthField] != nil {
t.Errorf("the machine's runtime was given the module's health: %v", machine[HealthField])
}
body, err := json.Marshal(map[string]any{"declaration": validate.Version, "resources": out})
if err != nil {
t.Fatal(err)
}
if problems := validate.Declaration(body); len(problems) > 0 {
t.Errorf("composed into something the node-engine refuses: %v", problems)
}
}
// The catalogue's own manifest is not written into.
if _, ok := keeper.RuntimeHealth.(map[string]any)["grace"]; ok {
t.Error("composing wrote the defaults into the catalogue's manifest")
}
}
+56
View File
@@ -529,3 +529,59 @@ func TestAModuleOfItsOwnAccountIsRefusedTheOperatorsAndTheAgentsAccount(t *testi
}
}
}
// Issue 723: the vault runs as its own account, so what the mesh writes for it — the export it keeps, the grant
// files it holds for its consumers, and what its words name — belongs to that account, never the operator's.
func TestWhatAModuleOfItsOwnAccountReadsIsItsAccounts(t *testing.T) {
with := Rendering{ArtifactStore: "anchor.internal:5101",
Needed: map[string]map[string]string{RuntimeModule: {"broker": "sealed-credential"}, "vault": {"broker": "own"}},
Kept: &KeptExport{Export: 1, OperatorKey: "k"},
Grants: []Grant{{Provision: "secret", Consumer: "anchor", From: "telegram",
Values: map[string]any{"name": "telegram"}}}}
goRuntime := Manifest{Module: RuntimeModule, Version: "1",
OwnSecrets: OwnSecrets{"broker": {Path: "/var/lib/mesh/" + RuntimeModule + "/broker"}},
Build: &Build{Artifacts: []Artifact{{Name: "runtime", Kind: ArtifactBundle, Language: "go",
System: "arch", From: "cmd/node-tools"}}}}
goRuntime, err := goRuntime.Resolve([]Built{{Name: "runtime", Kind: ArtifactBundle,
Reference: ArtifactStoreScheme + RuntimeModule + "/runtime/blobs/" + bundleDigest, Digest: bundleDigest}})
if err != nil {
t.Fatal(err)
}
vault := aToolsModule(t, "vault", "tools/index.js")
vault.RunsAs, vault.SecretsOwner = "vault", "vault"
vault.OwnSecrets = OwnSecrets{"broker": {Path: "/var/lib/vault/broker"}}
vault.Keeps = "/var/lib/vault/root"
vault.Grants = map[string]string{"secret": "/var/lib/vault/grants"}
vault.Receives = map[string]string{"secret": "/var/lib/vault/grants/mesh.json"}
vault.Bundles[0].Env = map[string]string{"MESH_RECEIVES": "/var/lib/vault/grants/mesh.json"}
out, err2 := Resolution{Node: "anchor", Account: "ops", Modules: []Manifest{vault, goRuntime}}.Declaration(with)
if err = err2; err != nil {
t.Fatal(err)
}
seen := 0
for _, r := range out {
path := fmt.Sprint(r["path"])
if !strings.HasPrefix(path, "/var/lib/vault/") || r["type"] != "file" {
continue
}
seen++
if r["owner"] != "vault" {
t.Errorf("%s (%v) belongs to %v, not the vault's own account", path, r["id"], r["owner"])
}
}
if seen < 4 {
t.Errorf("only %d of the vault's files were composed: %v", seen, ids(out))
}
// Carried by the machine's runtime instead, nothing of it changes: the operator's account reads them.
carried := vault
carried.RunsAs, carried.SecretsOwner = "", ""
out, err = Resolution{Node: "anchor", Account: "ops", Modules: []Manifest{carried, goRuntime}}.Declaration(with)
if err != nil {
t.Fatal(err)
}
for _, r := range out {
if path := fmt.Sprint(r["path"]); strings.HasPrefix(path, "/var/lib/vault/") && r["owner"] == "vault" {
t.Errorf("%s belongs to the vault's account though it runs as none", path)
}
}
}
+13 -9
View File
@@ -209,13 +209,13 @@ func seatContributions(modules []Manifest, holder Manifest, placeholder string,
if err != nil {
return "", nil, err
}
return shapedContributions(modules, holder, facts["name"], s, r, where, caps)
return shapedContributions(modules, holder, facts["name"], facts, s, r, where, caps)
}
var failed error
var unplacedLines []string
var b strings.Builder
var blocks []textBlock
for _, m := range inModuleOrder(modules) {
named := false
var old strings.Builder
// A left-out module's lines are best effort (novox/hq ADR 0262): what cannot be placed is said,
// and the machine is declared without it. A placement setting that does not read is not
// replaced by the definition's own path, which may not be where the data is.
@@ -273,15 +273,19 @@ func seatContributions(modules []Manifest, holder Manifest, placeholder string,
failed = lineFailed
}
}
if !named {
fmt.Fprintf(&b, "%s %s\n", r.Comment, m.Module)
named = true
if old.Len() == 0 {
fmt.Fprintf(&old, "%s %s\n", r.Comment, m.Module)
}
b.WriteString(content)
old.WriteString(content)
if !strings.HasSuffix(content, "\n") {
b.WriteString("\n")
old.WriteString("\n")
}
}
var held []heldItem
if !r.Dirs {
held = heldItems(m, "contribution:"+s.Name+":"+kind, nil, facts, caps)
}
blocks = append(blocks, moduleBlocks(m, r.Comment, old.String(), held)...)
}
return b.String(), unplacedLines, failed
return joinBlocks(blocks), unplacedLines, failed
}
+33 -4
View File
@@ -42,8 +42,14 @@ type Shape struct {
// (novox/hq ADR 0260): a contributor offers only state it keeps itself, and the holder on the same
// machine is granted to read that key, so the holder's manifest names no contributor.
Reads func(data map[string]any) []StateRead `json:"-"`
// Text says every piece of this kind is, rendered, text that a collector may place as a collection of items
// (design 57 L3): a collector then names the kind as `${collected:contribution:<seat>:<kind>:<field>=<value>,…}`.
Text bool
}
// collectableAsText is whether a collector may name a kind received as data as a collection.
func collectableAsText(r Receivable) bool { return r.Shape != nil && r.Shape.Text }
// PlaceField is one field a holder places by, and the values it takes.
type PlaceField struct {
Field string
@@ -289,6 +295,10 @@ type placedPiece struct {
module string
data map[string]any
order int
// text is a moved contributor's item, rendered (collections.go heldItems): placed as it is, not through the
// holder's template.
text string
item bool
}
// shapedContributions is every module's data of one kind to one seat, narrowed by where and by the
@@ -298,7 +308,7 @@ type placedPiece struct {
// A piece the template renders to nothing — a `shows` the holder does not know yet — or fails on is
// left out and answered as unplaced, naming its module: the other pieces and the rest of the machine's
// declaration go on (novox/hq ADR 0255).
func shapedContributions(modules []Manifest, holder Manifest, machine string, s Seat, r Receivable, where map[string]string, caps map[string]bool) (string, []string, error) {
func shapedContributions(modules []Manifest, holder Manifest, machine string, facts map[string]string, s Seat, r Receivable, where map[string]string, caps map[string]bool) (string, []string, error) {
var pieces []placedPiece
for _, m := range inModuleOrder(modules) {
for _, c := range m.allContributions() {
@@ -323,18 +333,37 @@ func shapedContributions(modules []Manifest, holder Manifest, machine string, s
}
pieces = append(pieces, placedPiece{module: m.Module, data: c.Data, order: order})
}
// A moved contributor's items of the kind, by their attributes (design 57 L3, issue 777): each in its
// module's place, by its order, then its name.
if !r.Dirs {
for _, h := range heldItems(m, "contribution:"+s.Name+":"+r.Kind, where, facts, caps) {
pieces = append(pieces, placedPiece{module: m.Module, order: h.item.order(), text: h.text, item: true})
}
}
}
if len(pieces) == 0 {
return "", nil, nil
}
t, err := holderTemplate(holder, s, r.Kind)
if err != nil {
return "", nil, err
var t *template.Template
for _, p := range pieces {
if p.item {
continue
}
var err error
if t, err = holderTemplate(holder, s, r.Kind); err != nil {
return "", nil, err
}
break
}
sort.SliceStable(pieces, func(a, b int) bool { return pieces[a].order < pieces[b].order })
var b strings.Builder
var unplaced []string
for _, p := range pieces {
if p.item {
fmt.Fprintf(&b, "%s %s\n", r.Comment, p.module)
b.WriteString(p.text)
continue
}
out, err := renderPiece(t, p.module, machine, p.data)
if err == nil && strings.TrimSpace(out) == "" {
err = fmt.Errorf("%s's template renders nothing for it", holder.Module)
+136 -2
View File
@@ -107,7 +107,9 @@ var defaultSeats = append([]Seat{
// A pull request's merge check, judged (novox/hq to-be 45 §9).
"checked",
// A walk kept in a new state, for the delivery it walks (novox/hq ADR 0239).
"plan-moved"},
"plan-moved",
// A key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3; design 57 F3).
"assignment-changed", "declared-config-changed", "predefined-changed"},
Serves: ControllerVerbs},
// The store's first verbs (novox/hq ADR 0159): the smallest set that makes the store askable,
// served by whichever module holds the seat with tools of these names.
@@ -128,7 +130,97 @@ var defaultSeats = append([]Seat{
// The vault: the controller seals every minted credential with what it provides, which is the
// test for a seat of the mesh's own (novox/hq ADR 0161) — a second provider of `secret` is a
// second claimant, refused by name, rather than a candidate for a pin.
{Name: "mesh-vault", Scope: ScopeMesh, Delivers: "secret", Decision: "novox/hq ADR 0161"},
//
// Its verb `get` hands a secret on request and reply, sealed to the key the asker sends and only to an
// asker the vault allows (novox/hq ADR 0349 §11, issue 674). `accept`, `secret-ask`, `issued` and `rotate`
// take a value in and make one anew (ADR 0349 clauses 8 to 10, issue 675): the first three in two steps from
// the controller alone, which carries the value sealed to a key the vault made for the call and cannot open
// it. All optional until the vault that runs serves them (design 33 §7).
{Name: "mesh-vault", Scope: ScopeMesh, Delivers: "secret", Decision: "novox/hq ADR 0161",
Serves: []Verb{
{Name: "get", Optional: true, Description: "One secret, sealed to the public key the asker sends (the " +
"sealed box of ADR 0206), with its generation; answered only to an asker the vault allows, as the " +
"bus proved it. Never the value in the clear, and never to a private key.",
Input: schema(map[string]string{"name": "the secret's name",
"public_key": "the asker's X25519 public key as SPKI PEM — never a private key"},
[]string{"name", "public_key"})},
{Name: "accept", Optional: true, Description: "Take a value given at the controller's terminal, from the " +
"controller alone: name the secret and get a call and a key made for it, then send the call and the " +
"value sealed to that key. The vault alone opens it, and never remakes an accepted credential.",
Input: schema(vaultIntake(map[string]string{"node": "the machine, for a module's own secret",
"module": "the module, for a module's own secret", "name": "the secret's name in the module",
"provider": "the provider's machine, for a provision's credential", "provision": "the provision",
"by": "who gave it at the terminal",
"origin": "issued for a bus account the controller minted for a core or moved module and hands " +
"at once (ADR 0374, issue 787); left out for a value given at the terminal"}), nil)},
{Name: "secret-ask", Optional: true, Description: "Ask the operator for a module's own secret at a desk, " +
"from the controller alone: the ask is recorded in the vault's store and bounded before the prompt " +
"opens, and what is typed is sealed to a key the vault made for the call and opens there alone.",
Input: schema(vaultIntake(map[string]string{"node": "the machine the secret is for",
"module": "the module", "name": "the secret's name in the module",
"desk": "the machine whose prompt asks", "asked_by": "who asked, as the bus named its caller",
"outcome": "instead of sealed: how the prompt ended"}), nil)},
{Name: "issued", Optional: true, Description: "Take the credential a provider's software issued for a " +
"provision, from the controller alone, which asks the provider's tool with a key the vault made for " +
"the call; kept as the provision's one credential, or, when the tool answers a value per holder named, " +
"each as that holder's own; the same value again writes nothing.",
Input: withHolders(schema(vaultIntake(map[string]string{"provision": "the provision",
"provider": "the provider's machine", "module": "the provider's module"}), nil))},
{Name: "rotate", Optional: true, Description: "Make a secret anew. Refused for a value a provider's " +
"software issued and for an accepted credential. Never answers the value.",
Input: schema(map[string]string{"node": "the machine, for a module's own secret", "module": "the module",
"name": "the secret's name in the module", "provider": "the provider's machine",
"provision": "the provision", "by": "who asked, when the controller carries the act"}, nil)},
// The move of the controller's secrets (design 57's V3, issue 676): each machine's node-engine hands in what
// was sealed to its machine, which the controller cannot open; the controller copies its record of asks.
{Name: "move-in", Optional: true, Description: "Take one value of the controller's store in, as it is, " +
"from the node-engine of a machine it was sealed to and no other caller: name the row, its origin " +
"and generation and get a call and a key made for it, then send the call, the value sealed to that " +
"key and a key to read back to. Never replaces a value the vault holds; answers what it kept, read " +
"back sealed to that key.",
Input: schema(vaultIntake(map[string]string{"node": "a module's own secret: its machine",
"module": "its module", "name": "its name in the module",
"provider": "a pair credential: the provider's machine", "provision": "the provision",
"consumer": "the consumer's machine", "consumer_module": "the consuming module",
"local": "the credential's local name, when it has one",
"origin": "made, accepted or issued, as the controller's store says",
"generation": "the controller's generation for the value", "made_at": "when it was made (RFC 3339)",
"read_back_to": "the second step: a raw X25519 public key the read-back is sealed to"}), nil)},
// A machine's node-engine states its own sealing key (novox/hq ADR 0372, issue 786).
{Name: "node-key", Optional: true, Description: "State this machine's sealing key to the vault, in its " +
"own name, from the node-engine of the machine alone: at its start and whenever the key changes. A " +
"first or changed key is recorded and the machine's core secrets are said changed; the same key " +
"again changes nothing.",
Input: schema(map[string]string{"key": "the machine's sealing key, a raw X25519 public key in base64"}, nil)},
{Name: "move-asks", Optional: true, Description: "Copy rows of the controller's record of asks at a desk " +
"into the vault's, as they are and once, from the controller alone. No value is in any of them.",
Input: map[string]any{"type": "object", "properties": map[string]any{"asks": map[string]any{
"type": "array", "items": map[string]any{"type": "object"},
"description": "the controller's rows: id, node, module, name, desk, asked_by, opened_at, ended_at, outcome"}}}},
}},
// The owner of every config value (novox/hq ADR 0348 §9, design 57 K3): its holder alone writes the buckets
// mesh-config_values, _resolved, _predefined and _proposals, and serves the one verb `config` (§1, §10).
// Optional until the mesh-config that runs serves it (design 33 §7).
{Name: "mesh-config", Scope: ScopeMesh, Decision: "novox/hq ADR 0348",
Serves: []Verb{
{Name: "config", Optional: true, Description: "Set, clear, list or read the history of config values: " +
"mesh config or node config for a predefined key, or a module's own value on one node, each set " +
"with why, validated, and naming each module on each node whose value moves. A trusted value only " +
"from the controller's terminal.",
Input: configInput()},
}},
// The provisioner's server (novox/hq ADR 0349 clause 3, design 57 P4): its holder keeps the registry of which
// domain answers which kind of placeholder, and the buckets mesh-provisioner_needs, _applied and _collect,
// which it and each machine's provisioner client (for its own machine's keys) alone write. Its one verb reads
// them. Optional until the mesh-provisioner that runs serves it (design 33 §7).
{Name: "mesh-provisioner", Scope: ScopeMesh, Decision: "novox/hq ADR 0349",
Serves: []Verb{
{Name: "provisioning", Optional: true, Description: "Read what the provisioner keeps: the registry " +
"(which domain answers which kind of placeholder), the needs each module on each node recorded, what " +
"is applied where, or the items of every collection, narrowed by node and module. Reads only.",
Input: schema(map[string]string{"what": "registry (when not given), needs, applied or collect",
"node": "a node", "module": "a module: for collect, a contributor", "collection": "for collect: a collection"}, nil)},
}},
// Named for its scope since 2026-09-30 (novox/hq ADR 0156); `the-artifact-store` resolves to it as
// an alias on a mesh that predates the rename. It serves artifacts of every kind a build makes —
// images and archives, by digest — which is why the provision is the artifact store and not an
@@ -261,6 +353,10 @@ var defaultSeats = append([]Seat{
// and their path is its protocol: a shell sources ~/.config/mesh/environment.sh without knowing
// which module wrote it.
{Name: EnvironmentSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0203"},
// The provisioner client on every machine (novox/hq ADR 0349 clause 3, design 57 P5): a service of its own on a
// bus account of its own (mesh-controller internal/broker NodeProvisionerSeat, design 57 P3), which fills the
// machine's templates. It serves no verb an agent calls.
{Name: "node-provisioner", Scope: ScopeNode, Decision: "novox/hq ADR 0349"},
// The login shell (novox/hq ADR 0204, replacing the module-declared `login-shell` of ADR 0176):
// the mesh's, so a second shell module claims the seat rather than declaring a second one, and
// the seat exists whether or not zsh's definition is registered. `execute` is the contract a caller
@@ -344,6 +440,21 @@ func isSystemSeatName(name string) bool {
// change to data, not to this code.
var seats = defaultSeats
// configInput is the schema of mesh-config's verb: a value of the key's own type (a number for a number, never its
// text), and a limit that is a number; every other argument a string.
func configInput() map[string]any {
in := schema(map[string]string{"action": "list (when not given), set, clear or history",
"key": "the config key", "value": "for set: the value, of the key's type",
"module": "a module: its own value, with node", "node": "a node",
"why": "for set and clear: why", "by": "who gave the value at the controller's terminal",
"limit": "for history: at most this many, newest"}, nil)
props := in["properties"].(map[string]any)
props["value"] = map[string]any{"description": "for set: the value, of the key's type: a number, a string, " +
"a boolean, or for a trusted key an object or a list"}
props["limit"] = map[string]any{"type": "number", "description": "for history: at most this many, newest"}
return in
}
// DefaultSeats is the set the mesh ships with, for seeding the store's seat table.
func DefaultSeats() []Seat { return append([]Seat(nil), defaultSeats...) }
@@ -810,3 +921,26 @@ func buildAgentVerbs() []Verb {
// BrokerSeat is the mesh's bus. Its holder is the bus, and its account — when it declares one — may
// snapshot the bus's streams and do nothing else (novox/hq ADR 0235).
const BrokerSeat = "mesh-broker"
// vaultIntake is a two-step vault verb's arguments: what its first step names, then the call and what was sealed.
// withHolders adds to `issued` the provision's holders from that provider (issue 749): a provider's tool may answer a
// value for each, which the vault keeps as that holder's own and hands to it alone.
func withHolders(input map[string]any) map[string]any {
props, _ := input["properties"].(map[string]any)
props["holders"] = map[string]any{"type": "array",
"description": "the provision's holders from that provider, each {node, module}: its tool may answer a value " +
"for each, kept as that holder's own and handed to it alone",
"items": map[string]any{"type": "object", "properties": map[string]any{
"node": map[string]any{"type": "string", "description": "the holder's machine"},
"module": map[string]any{"type": "string", "description": "the consuming module"}}}}
return input
}
func vaultIntake(first map[string]string) map[string]string {
out := map[string]string{"call": "the second step: the call the first step answered",
"sealed": "the second step: the value sealed to the call's key — never the value"}
for k, v := range first {
out[k] = v
}
return out
}
+3 -3
View File
@@ -48,7 +48,7 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
delivered[s.Delivers] = s.Name
}
}
// Forty-one with node-reads (novox/hq ADR 0310); forty with node-nfs-server and node-mounts (novox/hq ADR 0263); thirty-eight with node-resolver (novox/hq ADR 0247); thirty-seven with mesh-delivery (novox/hq ADR 0239); thirty-six since node-resolver-config retired
// Forty-four with node-provisioner (novox/hq ADR 0349 clause 3); forty-three with mesh-provisioner (novox/hq ADR 0349 clause 3); forty-two with mesh-config (novox/hq ADR 0348 §9); forty-one with node-reads (novox/hq ADR 0310); forty with node-nfs-server and node-mounts (novox/hq ADR 0263); thirty-eight with node-resolver (novox/hq ADR 0247); thirty-seven with mesh-delivery (novox/hq ADR 0239); thirty-six since node-resolver-config retired
// into node-uplink (novox/hq ADR 0223); thirty-seven
// since the retired node-dns-resolver went (novox/hq ADR 0220); thirty-eight with
// node-backup (novox/hq ADR 0214); thirty-seven with node-message-bus (novox/hq ADR 0215);
@@ -59,8 +59,8 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
// node-container-runtime (ADR 0207); nineteen with node-environment and node-login-shell (ADR 0203,
// ADR 0204); seventeen with node-build-agent (ADR 0190). One fewer once the retired
// mesh-build-machine row goes, when no registered manifest claims it.
if len(Seats()) != 41 {
t.Errorf("the mesh defines %d seats rather than 41; the set is closed, so a change here is "+
if len(Seats()) != 44 {
t.Errorf("the mesh defines %d seats rather than 44; the set is closed, so a change here is "+
"a decision (novox/hq ADR 0110): %s", len(Seats()), seatNames())
}
}
@@ -0,0 +1,194 @@
package catalogue
import (
"bytes"
"encoding/json"
"os"
"strings"
"testing"
)
// influxdb and grafana as the catalogue declares them, cut to what composes the data source on a
// machine like ace (novox/hq issue 696): what influxdb serves for influxdb-api and its mergeable
// runtime configuration, and grafana's binding, data source file and credential.
const grafanaLikeTheCatalogue = `{
"module": "grafana", "version": "1",
"requires": [{"name": "influxdb-api", "optional": true}],
"contributes": {"influxdb-api": {"access": "read"}},
"binds": {"influxdb-api": "${dir:state}/influxdb.json"},
"secrets": {"influxdb-api": "${dir:mesh-state}/influxdb-api"},
"resources": [
{"id": "mesh-state", "type": "directory", "mode": "0700", "place": "mesh"},
{"id": "state", "type": "directory", "mode": "0700", "place": "."},
{"id": "influxdb-datasources", "type": "directory", "mode": "0755", "owner": "472:0"},
{"id": "influxdb-datasource", "type": "file", "path": "${dir:influxdb-datasources}/mesh-influxdb.yaml", "mode": "0644",
"content": "apiVersion: 1\n# Written by the mesh from grafana's influxdb-api binding, only while it is bound (novox/hq ADR 0330):\n# without influxdb, grafana runs with no mesh data source. grafana reads it at start. Its own name and\n# uid, so a data source somebody made in the UI is never overwritten, and read-only in the UI because\n# the mesh resets it. The password is read from the file the mesh delivers, never written here.\ndatasources:\n - name: InfluxDB (mesh)\n uid: mesh-influxdb-api\n type: influxdb\n access: proxy\n url: ${bound:influxdb-api:scheme}://${bound:influxdb-api:at}:${bound:influxdb-api:port}\n user: ${bound:influxdb-api:as}\n isDefault: false\n editable: false\n jsonData:\n dbName: ${bound:influxdb-api:bucket}\n httpMode: POST\n secureJsonData:\n password: $__file{/run/secrets/influxdb/influxdb-api}\n",
"only-when-bound": "influxdb-api"}
]
}`
// influxdbServing is influxdb as the catalogue declared it before issue 696 (literal served org and
// bucket) or after (served from its settings, today's literals as their defaults).
func influxdbServing(after bool) string {
served := `"org": "mesh", "bucket": "default"`
settings, takes := ``, ``
if after {
served = `"org": "${setting:org}", "bucket": "${setting:bucket}"`
settings = `"settings": {
"org": {"kind": "preference", "default": "mesh", "why": "the org influxdb makes at its first start"},
"bucket": {"kind": "preference", "default": "default", "why": "the bucket influxdb makes at its first start"}},`
takes = `, "takes-settings": ["bucket", "org", "url"], "trusted": true`
}
return `{
"module": "influxdb", "version": "1",
"provides": [{"name": "influxdb-api", "scope": "mesh", "identity": {"in": "an InfluxDB v1 authorization"}}],
"listens": [{"name": "api", "port": 8086, "protocol": "tcp", "from": "mesh", "why": "queries and writes"}],
` + settings + `
"serves": {"influxdb-api": {"scheme": "http", "port": 8086, ` + served + `}},
"resources": [
{"id": "mesh-state", "type": "directory", "mode": "0700", "place": "mesh"},
{"id": "runtime-config", "type": "file", "path": "${dir:mesh-state}/config.json", "mode": "0600",
"content": "{}\n", "merge": "json"` + takes + `}
]
}`
}
// composedLikeAce is every resource of grafana and influxdb as one machine composes them, by id, with
// influxdb's settings layer on it.
func composedLikeAce(t *testing.T, influxdb string, layers []Layer) map[string]string {
t.Helper()
var ms []Manifest
for _, raw := range []string{influxdb, grafanaLikeTheCatalogue} {
m, err := ParseManifest([]byte(raw))
if err != nil {
t.Fatal(err)
}
ms = append(ms, m)
}
r, err := Resolve(shelf(ms...), []string{"influxdb", "grafana"}, reachable(), World{})
if err != nil {
t.Fatal(err)
}
out, err := r.Declaration(Rendering{Settings: SettingsBy{"influxdb": layers},
Needed: map[string]map[string]string{}})
if err != nil {
t.Fatal(err)
}
got := map[string]string{}
for _, res := range out {
id, _ := res["id"].(string)
if !strings.HasPrefix(id, "grafana.") && !strings.HasPrefix(id, "influxdb.") {
continue
}
b, err := json.Marshal(res)
if err != nil {
t.Fatal(err)
}
got[id] = string(b)
}
return got
}
// aceLikeLayer is a layer like influxdb's on ace: its own org and bucket (made up here), and the
// endpoint block the mesh reads itself.
func aceLikeLayer() []Layer {
return []Layer{{From: "ace", Values: map[string]any{
"org": "made-up-org", "bucket": "made-up-bucket",
"endpoints": map[string]any{"api": map[string]any{"label": "influx", "port": float64(8086), "reach": "internal"}},
}}}
}
// grafana composes byte for byte as it did before issue 696, on a machine like ace and on one
// without a layer, now that influxdb serves its org and bucket from its settings and no layer
// replaces a served key. The golden files are grafana's resources as the integration branch's
// controller composed them at 71362c1 (Settle still overriding), with influxdb serving the literals.
func TestGrafanaComposesTheSameOnceInfluxdbServesItsSettings(t *testing.T) {
for name, layers := range map[string][]Layer{"ace": aceLikeLayer(), "unlayered": nil} {
raw, err := os.ReadFile("testdata/influxdb-served-by-setting/grafana-" + name + ".json")
if err != nil {
t.Fatal(err)
}
var want map[string]json.RawMessage
if err := json.Unmarshal(raw, &want); err != nil {
t.Fatal(err)
}
got := composedLikeAce(t, influxdbServing(true), layers)
seen := 0
for id, v := range got {
if !strings.HasPrefix(id, "grafana.") {
continue
}
seen++
w, ok := want[id]
if !ok {
t.Errorf("%s: %s composes and did not before", name, id)
continue
}
var compact bytes.Buffer
if err := json.Compact(&compact, w); err != nil {
t.Fatal(err)
}
if compact.String() != v {
t.Errorf("%s: %s changed:\n got %s\nwant %s", name, id, v, compact.String())
}
}
if seen != len(want) {
t.Errorf("%s: %d of grafana's %d resources compose", name, seen, len(want))
}
}
}
// What a provider serves is its definition's, filled from settings only where it asks with
// ${setting:<key>} (novox/hq issue 696, ADR 0348 clause 11): a layer that sets a key the provider
// serves as a literal replaces nothing, and a key it does not serve is never added.
func TestALayerNoLongerReplacesAServedKey(t *testing.T) {
served, err := Settle(map[string]any{"host": "mail", "port": 25},
[]Layer{{From: "the mesh", Values: map[string]any{"host": "post", "sitename": "Mail"}}})
if err != nil {
t.Fatal(err)
}
if served["host"] != "mail" {
t.Errorf("a layer replaced the served host: %v", served)
}
if _, leaked := served["sitename"]; leaked {
t.Errorf("a setting for a file reached the consumers: %v", served)
}
// And a layer setting it is said to reach nothing, rather than taken for a change.
m := Manifest{Module: "mail", Serves: map[string]map[string]any{"smtp": {"host": "mail"}}}
if unused := UnusedSettings(m, []Layer{{From: "the mesh", Values: map[string]any{"host": "post"}}}); len(unused) != 1 {
t.Errorf("a layer setting a literal served key was not said to reach nothing: %v", unused)
}
// influxdb as it was before issue 696, serving literals, under a layer like ace's: the layer no
// longer reaches grafana — which is why the catalogue moves influxdb first.
got := composedLikeAce(t, influxdbServing(false), aceLikeLayer())
for _, id := range []string{"grafana.influxdb-datasource", "grafana.bound-influxdb-api"} {
if strings.Contains(got[id], "made-up") {
t.Errorf("%s still carries the layer's value over a literal served key: %s", id, got[id])
}
}
if !strings.Contains(got["grafana.influxdb-datasource"], `dbName: default\n`) {
t.Errorf("the data source does not name the served literal: %s", got["grafana.influxdb-datasource"])
}
}
// influxdb's own runtime configuration takes the keys it names: org and bucket from a layer like
// ace's, and not the endpoint block the mesh reads itself, which today's form wrote there too.
func TestInfluxdbsConfigTakesItsOrgAndBucket(t *testing.T) {
got := composedLikeAce(t, influxdbServing(true), aceLikeLayer())["influxdb.runtime-config"]
var r map[string]any
if err := json.Unmarshal([]byte(got), &r); err != nil {
t.Fatal(err)
}
if want := "{\n \"bucket\": \"made-up-bucket\",\n \"org\": \"made-up-org\"\n}\n"; r["content"] != want {
t.Errorf("config.json: got %q, want %q", r["content"], want)
}
got = composedLikeAce(t, influxdbServing(true), nil)["influxdb.runtime-config"]
if err := json.Unmarshal([]byte(got), &r); err != nil {
t.Fatal(err)
}
if r["content"] != "{}\n" {
t.Errorf("a default reached the mergeable file: %q", r["content"])
}
}
+28 -1
View File
@@ -31,6 +31,28 @@ type SettingDeclaration struct {
Default any `json:"default"`
// Why this default: one sentence for whoever wonders whether to change it.
Why string `json:"why"`
// The config form's declaration (novox/hq ADR 0348 §4, config.go): none of these is said under
// `settings`, and neither Kind nor Why under `config`.
// Type is number, string or boolean; object or list only when Trusted.
Type string `json:"type,omitempty"`
// Description says what the value is, including its unit.
Description string `json:"description,omitempty"`
// Enum, Minimum, Maximum and Pattern are its validation: the values it may take, the bounds of a
// number, and the whole of a string matching the pattern.
Enum []any `json:"enum,omitempty"`
Minimum *float64 `json:"minimum,omitempty"`
Maximum *float64 `json:"maximum,omitempty"`
Pattern string `json:"pattern,omitempty"`
// Trusted is a value that decides what runs or who can reach what (ADR 0348 §8).
Trusted bool `json:"trusted,omitempty"`
// Predefined is the predefined key the value falls back to, of the same type (ADR 0348 §2, §7).
Predefined string `json:"predefined,omitempty"`
// Required is the operator's own value: it has no default (ADR 0348 §4).
Required bool `json:"required,omitempty"`
// Runtime is a value the module's code reads at runtime rather than one placed in a file (ADR 0348 §5).
Runtime bool `json:"runtime,omitempty"`
}
// KindPreference is the kind of a setting that may have a default.
@@ -203,11 +225,16 @@ type SettingSource struct {
func Effective(m Manifest, layers []Layer) []SettingSource {
byKey := map[string]*SettingSource{}
for key, d := range m.Settings {
if d.Default == nil {
// A required config value has no default (ADR 0348 §4); a settings declaration always has one.
continue
}
byKey[key] = &SettingSource{Key: key, Value: d.Default, From: DefaultLayer, FromDefault: true,
Default: d.Default, HasDefault: true}
}
for _, layer := range layers {
for key, v := range layer.Values {
// As the layer is said, placement too: `settings` answers it as a person set it.
for key, v := range layer.Whole() {
s, ok := byKey[key]
if !ok {
s = &SettingSource{Key: key}
+1 -1
View File
@@ -427,7 +427,7 @@ func TestALeftOutModulesUnplaceableBackupLineCostsOnlyThatLine(t *testing.T) {
r := anAdoptedAnchor()
r.Modules = append(r.Modules, holder, media, placedBadly)
with := anchorRendering(false)
with.Settings["notes"] = []Layer{{From: "anchor", Values: map[string]any{PlacesSetting: "not a map"}}}
with.Settings["notes"] = []Layer{LayerOf("anchor", map[string]any{PlacesSetting: "not a map"})}
composed, err := r.Compose(with)
if err != nil {
t.Fatalf("an unplaceable line of a left-out module failed the machine: %v", err)
@@ -1,6 +1,8 @@
package catalogue
import (
"os"
"path/filepath"
"strings"
"testing"
)
@@ -179,3 +181,33 @@ func TestAnOwnersKeyIsTheTerminals(t *testing.T) {
}
}
}
// A key asked only in a template is not refused while the manifest still names its templates (issue 817), and
// is judged once they are filled: WithTemplatesFrom over a tree whose template does not ask for it refuses it.
func TestAnOwnerSettingAskedOnlyInATemplateIsJudgedOnceFilled(t *testing.T) {
raw := `{"module":"sonarr","version":"1",
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},
"resources":[{"id":"recycle-bin","type":"directory","path":"/srv/sonarr/recycle","mode":"0755","owner":"${setting:puid}:${setting:pgid}"},
{"id":"identity","type":"file","path":"/srv/sonarr/identity.env","mode":"0644","template":"files/identity.env"}]}`
m, err := ParseManifest([]byte(raw))
if err != nil {
t.Fatalf("a templated manifest was refused: %v", err)
}
write := func(text string) string {
dir := t.TempDir()
if err := os.MkdirAll(filepath.Join(dir, "files"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "files", "identity.env"), []byte(text), 0o644); err != nil {
t.Fatal(err)
}
return dir
}
if _, err := WithTemplatesFrom(m, write("PUID=${setting:puid}\nPGID=${setting:pgid}\n")); err != nil {
t.Errorf("a template asking for both keys was refused: %v", err)
}
_, err = WithTemplatesFrom(m, write("NOTHING=1\n"))
if err == nil || !strings.Contains(err.Error(), "asks for \"puid\" nowhere else") {
t.Errorf("a template not asking for puid was not refused: %v", err)
}
}
+47 -16
View File
@@ -1,6 +1,7 @@
package catalogue
import (
"errors"
"fmt"
"regexp"
"sort"
@@ -47,10 +48,30 @@ func settingsUsed(content string) []string {
type UnsetSettingError struct {
Module, Setting string
said string
// where says what asks for it — "a file", "a service whose unit", "a directory <id> whose owner" — and
// setToday which of the module's keys a layer sets, for saying it again in the config form's words.
where, setToday string
}
func (e *UnsetSettingError) Error() string { return e.said }
// unsetInConfigWords is the refusal of a required config value nothing sets, in the words of a module written in the
// config form (novox/hq ADR 0348 §1, §4): `${config:<key>}` and the config verb, never `settings set`. Any other
// error, and any error of a module in the settings form, is returned as it was.
func unsetInConfigWords(err error, m Manifest) error {
var unset *UnsetSettingError
if err == nil || !m.DeclaresConfig() || !errors.As(err, &unset) {
return err
}
return &UnsetSettingError{Module: unset.Module, Setting: unset.Setting, where: unset.where,
setToday: unset.setToday, said: fmt.Sprintf(
"%s has %s that says ${config:%s}, and nothing sets the required config value %q for it — a "+
"required value is the operator's own and has no default (novox/hq ADR 0348 §4): set it with "+
"the config verb, for %s on this node, key %q%s",
unset.Module, unset.where, unset.Setting, unset.Setting, unset.Module, unset.Setting,
strings.ReplaceAll(unset.setToday, "no setting is set", "no config value is set"))}
}
// settingInto fills a file's ${setting:…} placeholders from the layers over a module.
//
// The last layer setting a key wins, which is the node's over the mesh's over the module's own
@@ -74,12 +95,13 @@ func settingInto(resource map[string]any, layers []Layer, module string) error {
for _, key := range settingsUsed(content) {
value, set := settingValue(layers, key)
if !set {
return &UnsetSettingError{Module: module, Setting: key, said: fmt.Sprintf(
"%s has a file that says ${setting:%s}, and nothing sets %q for it — an operator's "+
"value is the assignment's, never the definition's (novox/hq ADR 0112), and only a "+
"preference has a default in the definition (ADR 0262): "+
"`settings set %s <file>` with {%q: …}%s",
module, key, key, module, key, orNoSettings(layers))}
return &UnsetSettingError{Module: module, Setting: key, where: "a file",
setToday: orNoSettings(layers), said: fmt.Sprintf(
"%s has a file that says ${setting:%s}, and nothing sets %q for it — an operator's "+
"value is the assignment's, never the definition's (novox/hq ADR 0112), and only a "+
"preference has a default in the definition (ADR 0262): "+
"`settings set %s <file>` with {%q: …}%s",
module, key, key, module, key, orNoSettings(layers))}
}
content = strings.ReplaceAll(content, "${setting:"+key+"}", plainly(value))
}
@@ -130,11 +152,12 @@ func settingIntoUnit(resource map[string]any, layers []Layer, module string) err
for _, key := range settingsUsed(unit) {
value, set := settingValue(layers, key)
if !set {
return &UnsetSettingError{Module: module, Setting: key, said: fmt.Sprintf(
"%s has a service whose unit says ${setting:%s}, and nothing sets %q for it — an operator's "+
"value is the assignment's, never the definition's (novox/hq ADR 0112): "+
"`settings set %s <file>` with {%q: …}%s",
module, key, key, module, key, orNoSettings(layers))}
return &UnsetSettingError{Module: module, Setting: key, where: "a service whose unit",
setToday: orNoSettings(layers), said: fmt.Sprintf(
"%s has a service whose unit says ${setting:%s}, and nothing sets %q for it — an operator's "+
"value is the assignment's, never the definition's (novox/hq ADR 0112): "+
"`settings set %s <file>` with {%q: …}%s",
module, key, key, module, key, orNoSettings(layers))}
}
v := plainly(value)
if !unitPart.MatchString(v) {
@@ -207,6 +230,12 @@ func OwnerSettingProblems(m Manifest) []string {
m.Module, r["id"], owner))
continue
}
// A template's text is not here yet, so what the module asks for elsewhere cannot be known: WithTemplatesFrom
// parses its result again with every template filled, and this half is judged then (as config.go skips its
// "placed nowhere" rule while m.Templated()).
if m.Templated() {
continue
}
for _, key := range settingsUsed(owner) {
if !asked[key] {
problems = append(problems, fmt.Sprintf("%s: the directory %v's owner says ${setting:%s}, and the "+
@@ -229,11 +258,13 @@ func settingIntoOwner(resource map[string]any, layers []Layer, module string) er
for _, key := range settingsUsed(owner) {
value, set := settingValue(layers, key)
if !set {
return &UnsetSettingError{Module: module, Setting: key, said: fmt.Sprintf(
"%s has a directory %v whose owner says ${setting:%s}, and nothing sets %q for it — an operator's "+
"value is the assignment's, never the definition's (novox/hq ADR 0112): "+
"`settings set %s <file>` with {%q: …}%s",
module, resource["id"], key, key, module, key, orNoSettings(layers))}
return &UnsetSettingError{Module: module, Setting: key,
where: fmt.Sprintf("a directory %v whose owner", resource["id"]), setToday: orNoSettings(layers),
said: fmt.Sprintf(
"%s has a directory %v whose owner says ${setting:%s}, and nothing sets %q for it — an operator's "+
"value is the assignment's, never the definition's (novox/hq ADR 0112): "+
"`settings set %s <file>` with {%q: …}%s",
module, resource["id"], key, key, module, key, orNoSettings(layers))}
}
v := plainly(value)
id, whole := wholeID(v)
+159 -28
View File
@@ -18,29 +18,104 @@ import (
// TakesSettingsField is the key a mergeable file carries to name the top-level setting keys it takes.
const TakesSettingsField = "takes-settings"
// takenKeys is the keys a resource names in takes-settings, and whether it names any list at all. An
// empty list is a list: that file takes no setting.
func takenKeys(resource map[string]any) (map[string]bool, bool) {
// **A taken value may be laid under the file's own name** (issue 751). An entry of takes-config is a key, laid
// under that key, or {"key": <the config key>, "as": <the file's name for it>}: the key stays a config key,
// lower-case, and the file is written in its software's spelling, `atlassian-url` as ATLASSIAN_URL. Like every
// taken value it is laid only when a layer sets it, never from a default, so an unset one leaves the file's bytes
// and the code's own fallback (its environment) as they were; for that reason it is never `required`
// (ConfigProblems). The settings form has no such entry: it is the config form's alone.
// Fields of a renamed takes-config entry.
const (
takenKeyField = "key"
takenAsField = "as"
)
// takenEntry is one value a mergeable file takes: its key and the name the file has for it.
type takenEntry struct {
key, as string
renamed bool
}
// takenEntries is the entries of a takes-settings list in their order, a usable entry only: a string is a key
// under its own name, an object with exactly key and as a key under another (TakenSettingsProblems says the rest).
func takenEntries(raw any) []takenEntry {
list, _ := raw.([]any)
var out []takenEntry
for _, item := range list {
if e, ok := asTakenEntry(item); ok {
out = append(out, e)
}
}
return out
}
// asTakenEntry reads one item of a takes-settings list, and whether it is shaped as an entry at all.
func asTakenEntry(item any) (takenEntry, bool) {
switch v := item.(type) {
case string:
return takenEntry{key: v, as: v}, true
case map[string]any:
key, keyOK := v[takenKeyField].(string)
as, asOK := v[takenAsField].(string)
if !keyOK || !asOK || len(v) != 2 {
return takenEntry{}, false
}
return takenEntry{key: key, as: as, renamed: true}, true
}
return takenEntry{}, false
}
// takenKeyList is the keys a resource takes, in order: the config keys, never the file's names for them.
func takenKeyList(resource map[string]any) []string {
var keys []string
for _, e := range takenEntries(resource[TakesSettingsField]) {
keys = append(keys, e.key)
}
return keys
}
// takenKeys is the keys a resource names in takes-settings, each with the name the file has for it, and whether
// it names any list at all. An empty list is a list: that file takes no setting.
func takenKeys(resource map[string]any) (map[string]string, bool) {
raw, said := resource[TakesSettingsField]
if !said {
return nil, false
}
keys := map[string]bool{}
for _, k := range stringsOf(raw) {
keys[k] = true
keys := map[string]string{}
for _, e := range takenEntries(raw) {
keys[e.key] = e.as
}
return keys, true
}
// onlyTaken is the layers with every key the file does not name left out. The file's own defaults and
// what the mesh reads itself are not touched here: settle already keeps both out of a file.
func onlyTaken(layers []Layer, taken map[string]bool) []Layer {
// onlyTaken is the layers with every key the file does not name left out, and each key it takes under the
// file's name for it. The file's own defaults and what the mesh reads itself are not touched here: settle
// already keeps both out of a file.
//
// **A layer said before its module moved still reaches the file** (issue 751): such a layer names a renamed value
// by the file's own spelling (nodered's timeZone on one machine), and until K5 moves the layer and renames the key
// that spelling is taken as the value's, so moving the module changes no machine's file. Where one layer says
// both, the config key wins.
func onlyTaken(layers []Layer, taken map[string]string) []Layer {
oldSpelling := map[string]string{}
for key, as := range taken {
if as != key {
oldSpelling[as] = key
}
}
kept := make([]Layer, 0, len(layers))
for _, layer := range layers {
values := map[string]any{}
for key, value := range layer.Values {
if taken[key] {
values[key] = value
if as, takes := taken[key]; takes {
values[as] = value
continue
}
if newKey, old := oldSpelling[key]; old {
if _, said := layer.Values[newKey]; !said {
values[key] = value
}
}
}
kept = append(kept, Layer{From: layer.From, Values: values, Default: layer.Default})
@@ -48,6 +123,19 @@ func onlyTaken(layers []Layer, taken map[string]bool) []Layer {
return kept
}
// takenNames is every key a layer may say for the values a resource takes: each config key and, for a renamed
// one, the file's own spelling a layer said before its module moved (onlyTaken).
func takenNames(resource map[string]any) []string {
var names []string
for _, e := range takenEntries(resource[TakesSettingsField]) {
names = append(names, e.key)
if e.renamed {
names = append(names, e.as)
}
}
return names
}
// namesItsKeys says whether a module's mergeable files name the keys they take (ADR 0326 §2): true
// only when it has one and every one carries takes-settings. A module with a silent mergeable file
// keeps today's form (§3), where every key reaches every mergeable file.
@@ -83,7 +171,8 @@ func TakenSettingsProblems(m Manifest) []string {
// Said only when none of the module's mergeable files names its keys: when one does, §2 already
// names every file that does not, and a file is not reported twice.
func WrittenSettingsProblems(m Manifest) []string {
if len(m.Settings) == 0 {
if len(m.Settings) == 0 || m.DeclaresConfig() {
// A module in the config form is held to this where it is read, in its own words (ConfigProblems).
return nil
}
var silent []string
@@ -137,26 +226,42 @@ func takenSettingsProblems(m Manifest) []string {
for _, k := range stringsOf(r["protected"]) {
protected[k] = true
}
seen := map[string]bool{}
seen, landed := map[string]bool{}, map[string]bool{}
for _, item := range list {
if _, isObject := item.(map[string]any); isObject {
e, shaped := asTakenEntry(item)
switch {
case !m.DeclaresConfig():
problems = append(problems, fmt.Sprintf("%s's %s: %v in %s lays a value under another name, "+
"which only the config form says: write the key alone (novox/hq issue 751)",
m.Module, id, item, TakesSettingsField))
continue
case !shaped:
problems = append(problems, fmt.Sprintf("%s's %s: %v in %s is a key alone, or an object "+
"saying exactly %q and %q: the config key and the file's name for it (novox/hq issue 751)",
m.Module, id, item, TakesSettingsField, takenKeyField, takenAsField))
continue
case !settingRef.MatchString("${setting:" + e.key + "}"):
problems = append(problems, fmt.Sprintf("%s's %s: %q in %s is not a usable key: a config "+
"key is lower-case letters, digits, dots, dashes and underscores; the file's own spelling "+
"is its %q", m.Module, id, e.key, TakesSettingsField, takenAsField))
continue
case !usableKey(e.as):
problems = append(problems, fmt.Sprintf("%s's %s: %q, the name %s takes %q as, is not a "+
"usable name in a file", m.Module, id, e.as, TakesSettingsField, e.key))
continue
}
problems = append(problems, takenEntryProblems(m.Module, id, e, seen, landed, protected)...)
continue
}
key, isString := item.(string)
switch {
case !isString || !usableKey(key):
if !isString || !usableKey(key) {
problems = append(problems, fmt.Sprintf("%s's %s: %v in %s is not a usable setting key",
m.Module, id, item, TakesSettingsField))
case seen[key]:
problems = append(problems, fmt.Sprintf("%s's %s names %q twice in %s",
m.Module, id, key, TakesSettingsField))
case meshWords[key]:
problems = append(problems, fmt.Sprintf("%s's %s names %q in %s, and the mesh reads %q itself "+
"(novox/hq ADR 0164 §14)", m.Module, id, key, TakesSettingsField, key))
case protected[key]:
problems = append(problems, fmt.Sprintf("%s's %s names %q in %s and keeps it as protected: a "+
"key is the file's or the setting's, never both", m.Module, id, key, TakesSettingsField))
}
if isString {
seen[key] = true
continue
}
problems = append(problems, takenEntryProblems(m.Module, id, takenEntry{key: key, as: key}, seen,
landed, protected)...)
}
}
if len(naming) > 0 {
@@ -169,6 +274,32 @@ func takenSettingsProblems(m Manifest) []string {
return problems
}
// takenEntryProblems is what is wrong with one usable entry of a file's takes-settings, against those before it:
// a key taken twice, or two values laid under one name in the file; a word the mesh reads itself, as the key or as
// the file's name; a name the file keeps as protected. seen and landed are the keys and the names so far, added to.
func takenEntryProblems(module, id string, e takenEntry, seen, landed, protected map[string]bool) []string {
var problems []string
switch {
case seen[e.key]:
problems = append(problems, fmt.Sprintf("%s's %s names %q twice in %s", module, id, e.key, TakesSettingsField))
case landed[e.as]:
problems = append(problems, fmt.Sprintf("%s's %s names %q twice in %s, as a key or as the name a key is "+
"laid under: one name in a file holds one value", module, id, e.as, TakesSettingsField))
case meshWords[e.key] || meshWords[e.as]:
word := e.key
if !meshWords[word] {
word = e.as
}
problems = append(problems, fmt.Sprintf("%s's %s names %q in %s, and the mesh reads %q itself "+
"(novox/hq ADR 0164 §14)", module, id, word, TakesSettingsField, word))
case protected[e.as]:
problems = append(problems, fmt.Sprintf("%s's %s names %q in %s and keeps it as protected: a "+
"key is the file's or the setting's, never both", module, id, e.as, TakesSettingsField))
}
seen[e.key], landed[e.as] = true, true
return problems
}
// usableKey is a key a file can be said to take: not empty, and no space, line end or control
// character. Not the lower-case rule of a `${setting:<key>}`: a file's own keys are its software's
// spelling, such as `timeZone`.
@@ -195,7 +326,7 @@ func takenSaid(m Manifest) string {
if how, _ := r["merge"].(string); how == "" {
continue
}
keys := stringsOf(r[TakesSettingsField])
keys := takenKeyList(r)
if len(keys) == 0 {
files = append(files, fmt.Sprintf("%v takes none", r["id"]))
continue
+45 -29
View File
@@ -37,6 +37,11 @@ type Layer struct {
// Default marks the layer a module's own defaults make (novox/hq ADR 0262). Marked rather than
// recognised by From, which is a node's name for a node's layer, and a node may be called anything.
Default bool
// Placement is the assignment's placement at this layer's scope — `places`, `accesses`, `networks`,
// `endpoints`, `expose`, `reach`, `ports` — which the controller holds apart from settings (novox/hq
// ADR 0348 clause 11, issues 651 and 736): read by their readers only, and never from Values. LayerOf
// splits a layer as a person says it into the two.
Placement map[string]any
}
// ApplySettings produces a resource's final content from the module's own and the layers over it.
@@ -45,6 +50,10 @@ type Layer struct {
// module first, then this node's. A node that differs from the rest is expressed by differing,
// rather than by repeating everything the rest already say.
func ApplySettings(resource map[string]any, layers []Layer) (map[string]any, error) {
// Today's form reaches a mergeable file with every key of the layer as it was said, placement too
// (novox/hq ADR 0326 §3, ADR 0348 clause 11: a module not yet moved keeps it), so a file composes
// byte for byte as it did while placement was kept in the layer.
layers = asSaid(layers)
how, _ := resource["merge"].(string)
taken, names := takenKeys(resource)
if how == "" {
@@ -108,25 +117,27 @@ func ApplySettings(resource map[string]any, layers []Layer) (map[string]any, err
return out, nil
}
// Settle lays settings over what a provider serves. Exported because a served fact is settled where
// the mesh is walked rather than where a node is declared.
// Settle fills what a provider serves from its settings. Exported because a served fact is settled
// where the mesh is walked rather than where a node is declared.
//
// **A setting overrides a served key; it never adds one** (novox/hq 04-ISSUES/173). What a consumer
// is told is the provider's contract, and a setting made for one of the provider's files — a site
// name, a public address — is not part of it. Before this, every setting of a module reached every
// consumer of every provision it served.
// **What a provider serves is its definition's; a layer replaces none of it** (novox/hq ADR 0348
// clause 11, issue 696). A served value is filled from the layers only where the definition asks,
// with `${setting:<key>}` (ADR 0155) — a mail provider's domain, an identity provider's issuer, a
// time-series store's org and bucket — and a layer that sets a key the provider serves as a literal
// changes nothing a consumer is told. Nor does a setting add a key (novox/hq 04-ISSUES/173): a
// setting made for one of the provider's files — a site name, a public address — is not part of the
// contract. Until issue 696, a layer replaced any served key it named; influxdb on ace was the one
// layer relying on it, and serves `${setting:org}` and `${setting:bucket}` since.
func Settle(base map[string]any, layers []Layer) (map[string]any, error) {
return overridden(base, layers, "what is served")
return filledFromSettings(deepCopy(base), layers, "what is served")
}
// overridden lays settings over a map whose keys are its contract: a contribution, a served fact.
// Only the keys the map already declares are touched; the rest of a layer is somebody else's
// business (a file's, another destination's) and is left to reach it there.
// overridden lays settings over a contribution: a map whose keys are its contract. Only the keys the
// map already declares are touched; the rest of a layer is somebody else's business (a file's,
// another destination's) and is left to reach it there.
//
// A declared value may itself be the operator's, `${setting:<key>}` (ADR 0155): a mail provider
// serves its domain, an identity provider its issuer, and neither is the definition's to state.
// Filled from the layers after the overrides, and refused by name when nothing sets it — a literal
// placeholder handed to a consumer is a service configured against a string nobody meant.
// A declared value may itself be the operator's, `${setting:<key>}` (ADR 0155), filled after the
// overrides (filledFromSettings).
func overridden(base map[string]any, layers []Layer, what string) (map[string]any, error) {
kept := make([]Layer, 0, len(layers))
for _, layer := range layers {
@@ -142,6 +153,14 @@ func overridden(base map[string]any, layers []Layer, what string) (map[string]an
if err != nil {
return nil, err
}
return filledFromSettings(merged, layers, what)
}
// filledFromSettings fills each `${setting:<key>}` in a map's top-level strings from the layers,
// later beating earlier and a module's defaults lowest, and refuses one nothing sets by name — a
// literal placeholder handed to a consumer is a service configured against a string nobody meant.
// It fills merged in place and returns it.
func filledFromSettings(merged map[string]any, layers []Layer, what string) (map[string]any, error) {
for key, value := range merged {
s, ok := value.(string)
if !ok {
@@ -168,7 +187,8 @@ func overridden(base map[string]any, layers []Layer, what string) (map[string]an
// module says what it means by default, and somebody says what it means here. A contribution that
// could not be settled would have to be edited to be reused anywhere else. The two differ in one
// respect, and the caller decides it: a file takes keys it did not declare (a setting may add to a
// configuration), a contribution or served fact does not (overridden).
// configuration), a contribution does not (overridden). A served fact takes no layer at all; only
// its ${setting:<key>} is filled (Settle).
func settle(base map[string]any, layers []Layer, protected map[string]bool, what string) (
map[string]any, error) {
merged := deepCopy(base)
@@ -344,8 +364,9 @@ func sortedKeysAny(m map[string]any) []string {
// way there is. This is what makes that visible at the moment they set it.
//
// Where a key can land: any mergeable file takes any key; a file asking for `${setting:<key>}`
// takes that key (ADR 0155); a contribution or a served fact takes a key it declares, and no other
// (novox/hq 04-ISSUES/173); and the mesh's own words — `expose`, `ports`, `reach`, `endpoints` —
// takes that key (ADR 0155); a contribution takes a key it declares, and no other (novox/hq
// 04-ISSUES/173); a served fact takes only the keys it asks for as `${setting:<key>}`, since a layer
// replaces no served key (novox/hq issue 696); and the mesh's own words — `expose`, `ports`, `reach`, `endpoints` —
// are read by the mesh. A key none of those takes is stray, and is said so rather than dropped.
func UnusedSettings(m Manifest, layers []Layer) []string {
named := namesItsKeys(m)
@@ -366,8 +387,7 @@ func UnusedSettings(m Manifest, layers []Layer) []string {
lands := settingKeysUsedBy(m)
// A mergeable file takes the keys it names (novox/hq ADR 0326 §5).
for _, r := range m.Resources {
taken, _ := takenKeys(r)
for key := range taken {
for _, key := range takenNames(r) {
lands[key] = true
}
}
@@ -388,14 +408,9 @@ func UnusedSettings(m Manifest, layers []Layer) []string {
}
}
}
for _, served := range m.Serves {
for key := range served {
lands[key] = true
}
}
var unused []string
for _, layer := range layers {
for key := range layer.Values {
for key := range layer.Whole() {
if lands[key] {
continue
}
@@ -435,7 +450,7 @@ func UnusedSettings(m Manifest, layers []Layer) []string {
}
unused = append(unused, fmt.Sprintf(
"%s sets %q, and %s has no file that merges it, asks for no ${setting:%s}, and "+
"declares no %q in what it contributes or serves%s",
"declares no %q in what it contributes%s",
layer.From, key, m.Module, key, key, takenSaid(m)))
}
}
@@ -483,7 +498,7 @@ func KeptNetworks(m Manifest, layers []Layer, adopted bool) (map[string][]string
}
out := map[string][]string{}
for _, layer := range layers {
raw, ok := layer.Values[NetworksSetting]
raw, ok := layer.Placement[NetworksSetting]
if !ok {
continue
}
@@ -543,7 +558,8 @@ var networkName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]*$`)
// refused where it is stored (SetSettings, with UnusedSettings) and said where a plan is read,
// and never costs a module its place.
func JudgeSettings(m Manifest, layers []Layer, adopted bool) error {
if err := settingsHoldOneLine(m.Module, layers); err != nil {
// Placement is judged for line ends as it always was: a path is written into files as well.
if err := settingsHoldOneLine(m.Module, asSaid(layers)); err != nil {
return err
}
// Which keys each mergeable file takes is the definition's to say right (novox/hq ADR 0326);
@@ -580,7 +596,7 @@ func JudgeSettings(m Manifest, layers []Layer, adopted bool) error {
for k, v := range settled {
copied[k] = v
}
if err := settingInto(copied, WithDefaults(m, layers), m.Module); err != nil {
if err := unsetInConfigWords(settingInto(copied, WithDefaults(m, layers), m.Module), m); err != nil {
return err
}
}
+9 -9
View File
@@ -27,12 +27,12 @@ func TestASettingThatCannotComposeIsRefusedByNameAndLeavesOnlyItsModuleOut(t *te
if c.name == "a mesh-wide port" {
from = MeshWideLayer
}
err := JudgeSettings(web, []Layer{{From: from, Values: c.layer}}, true)
err := JudgeSettings(web, []Layer{LayerOf(from, c.layer)}, true)
if err == nil || !strings.Contains(err.Error(), c.refuse) {
t.Errorf("%s: judged %v, want %q", c.name, err, c.refuse)
}
}
if err := JudgeSettings(web, []Layer{{From: "anchor", Values: map[string]any{PortsSetting: map[string]any{"8080": 10000}}}}, true); err != nil {
if err := JudgeSettings(web, []Layer{{From: "anchor", Placement: map[string]any{PortsSetting: map[string]any{"8080": 10000}}}}, true); err != nil {
t.Fatalf("a port the module publishes was refused: %v", err)
}
@@ -40,7 +40,7 @@ func TestASettingThatCannotComposeIsRefusedByNameAndLeavesOnlyItsModuleOut(t *te
// the machine is declared.
r := anAdoptedAnchor()
with := anchorRendering(false)
with.Settings = SettingsBy{"hello-web": {{From: "anchor", Values: map[string]any{PortsSetting: map[string]any{"9999": 10000}}}}}
with.Settings = SettingsBy{"hello-web": {{From: "anchor", Placement: map[string]any{PortsSetting: map[string]any{"9999": 10000}}}}}
composed, err := r.Compose(with)
if err != nil {
t.Fatal(err)
@@ -69,8 +69,8 @@ func TestASettingThatCannotComposeIsRefusedByNameAndLeavesOnlyItsModuleOut(t *te
// declaration as the networks it also joins.
func TestAKeptNetworkReachesTheContainerOnAnAdoptedMachineOnly(t *testing.T) {
r := anAdoptedAnchor()
keep := SettingsBy{"hello-web": {{From: "anchor",
Values: map[string]any{NetworksSetting: map[string]any{"server": []any{"predecessor_default"}}}}}}
keep := SettingsBy{"hello-web": {LayerOf("anchor",
map[string]any{NetworksSetting: map[string]any{"server": []any{"predecessor_default"}}})}}
with := anchorRendering(true)
with.Settings = keep
@@ -107,13 +107,13 @@ func TestAKeptNetworkReachesTheContainerOnAnAdoptedMachineOnly(t *testing.T) {
layer Layer
want string
}{
{"mesh-wide", Layer{From: MeshWideLayer, Values: map[string]any{NetworksSetting: map[string]any{"server": []any{"x"}}}},
{"mesh-wide", LayerOf(MeshWideLayer, map[string]any{NetworksSetting: map[string]any{"server": []any{"x"}}}),
"a found network is a fact about one machine"},
{"an unknown container", Layer{From: "anchor", Values: map[string]any{NetworksSetting: map[string]any{"db": []any{"x"}}}},
{"an unknown container", LayerOf("anchor", map[string]any{NetworksSetting: map[string]any{"db": []any{"x"}}}),
`names the container "db", which it does not declare`},
{"not a list", Layer{From: "anchor", Values: map[string]any{NetworksSetting: map[string]any{"server": "x"}}},
{"not a list", LayerOf("anchor", map[string]any{NetworksSetting: map[string]any{"server": "x"}}),
"is a list of network names"},
{"not a network name", Layer{From: "anchor", Values: map[string]any{NetworksSetting: map[string]any{"server": []any{"a/b"}}}},
{"not a network name", LayerOf("anchor", map[string]any{NetworksSetting: map[string]any{"server": []any{"a/b"}}}),
"which is not a network name"},
} {
_, err := KeptNetworks(web, []Layer{c.layer}, true)
-16
View File
@@ -190,22 +190,6 @@ func TestASettingLandsOnlyWhereSomethingDeclaresIt(t *testing.T) {
}
}
func TestASettingOverridesAServedKeyAndAddsNone(t *testing.T) {
// What a consumer is told is the provider's contract. A setting made for one of the
// provider's files — its site name, its public address — is not part of it.
served, err := Settle(map[string]any{"host": "mail", "port": 25},
[]Layer{{From: "the mesh", Values: map[string]any{"host": "post", "sitename": "Mail"}}})
if err != nil {
t.Fatal(err)
}
if served["host"] != "post" {
t.Errorf("the setting did not override the served host: %v", served)
}
if _, leaked := served["sitename"]; leaked {
t.Errorf("a setting for a file reached the consumers: %v", served)
}
}
func TestContentThatIsNotJSONIsRefusedWhereSomebodyIsLooking(t *testing.T) {
// Rather than on the machine, at apply time, as a file the program cannot read.
_, err := ApplySettings(file(`this is not json`), nil)
+13
View File
@@ -124,6 +124,14 @@ func StateProblems(m Manifest) []string {
return problems
}
// ControllerMirrors are the controller's record mirrors a module may read, each `mesh-controller.<name>`
// (novox/hq ADR 0289, design 54 §5, design 57 F3). **The controller keeps them, not its manifest's `state`**:
// they are asserted with the controller's own buckets, with options of their own, and written by the
// controller's account alone (the writers table); declared as a module's state, they would be asserted a
// second way and granted to the module's own principal as well. Named here so a read of one is a read of
// kept state. The bus's list is broker.ControllerMirrors; a test keeps the two agreeing.
var ControllerMirrors = []string{"assignments", "declared-config", "predefined"}
// StateReadsNothingDeclares is every read across a catalogue whose owner is present and declares no
// such state. An absent owner says nothing — a module may be installed long before the one whose
// state it reads, as a consumer may before its emitter (design 32 §1).
@@ -136,6 +144,11 @@ func StateReadsNothingDeclares(manifests []Manifest) []string {
}
declared[m.Module] = own
}
if own, present := declared[ControllerSeatName]; present {
for _, name := range ControllerMirrors {
own[name] = true
}
}
var problems []string
for _, m := range manifests {
for _, r := range m.Reads {
+15
View File
@@ -67,6 +67,21 @@ func TestAReadNamesStateItsOwnerKeeps(t *testing.T) {
}
}
// **The controller's record mirrors are kept state** (design 54 §5, design 57 F3): a module may read
// `mesh-controller.assignments` and `mesh-controller.declared-config` although the controller's manifest
// declares no `state`; a read of anything else of the controller's is still named.
func TestAReadOfTheControllersMirrorsIsAReadOfKeptState(t *testing.T) {
controller := Manifest{Module: ControllerSeatName}
reader := Manifest{Module: "mesh-config", Reads: []string{"mesh-controller.assignments", "mesh-controller.declared-config"}}
if p := StateReadsNothingDeclares([]Manifest{controller, reader}); len(p) != 0 {
t.Fatalf("a read of the controller's mirrors was refused: %v", p)
}
other := Manifest{Module: "other", Reads: []string{"mesh-controller.calls"}}
if p := StateReadsNothingDeclares([]Manifest{controller, other}); len(p) != 1 {
t.Fatalf("a read of the controller's own calls was not named: %v", p)
}
}
// **Across the whole catalogue**: every state name is local, and every read whose owner is present
// names state that owner keeps.
func TestEveryManifestsStateIsLocalAndEveryReadIsKept(t *testing.T) {
+1 -1
View File
@@ -36,7 +36,7 @@ var ReservedStreamPrefixes = []string{"SEAT_", "LOG_", "KV_", "OBJ_", "DEAD_"}
// MeshStreamNames are the mesh's own streams by name. Written here as well as in the broker, which this
// package cannot import; a test keeps them agreeing.
var MeshStreamNames = []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS", "DEAD_LETTER_NOTICES", "DEAD_LETTERS"}
var MeshStreamNames = []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS", "DEAD_LETTER_NOTICES", "DEAD_LETTERS", "PROV_APPLIED"}
// StreamDeclaration is one stream a module owns.
type StreamDeclaration struct {
+454
View File
@@ -0,0 +1,454 @@
package catalogue
import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path"
"path/filepath"
"sort"
"strings"
"unicode/utf8"
)
// A file's text kept as a template file in its module's repository (novox/hq ADR 0349 §2, design 57 package T1,
// issue 654).
//
// **Two forms, one composition.** A file resource says its text inline (`content`) or names a template file
// beside its manifest (`"template": "files/<name>"`), never both. The module's build packs `files/` into an
// archive artifact of its own — an `archive` artifact built `from: "files"` — and resolving the build reads each
// template's text back from that packed archive into `content` and drops `template`. So the manifest the mesh
// holds, composition and the node-engine are exactly as they were: a module with its text in `files/` composes
// byte-identically to the same module with it inline. Where there is no build — `module check`, the merge gate,
// `module add` — the text is read from the repository tree beside the manifest (WithTemplatesFrom).
//
// **files/ unless the manifest names another directory** (issue 731). A module whose files/ is already an archive
// unpacked onto the machine names its own (`"templates": "templates"`), packed into an archive no resource names, so
// its templates neither land beside what it unpacks nor change that archive's digest.
//
// **Read back by the builder, carried in the resolved manifest.** The builder reads each template from the very
// bytes it hashed into the archive's digest, so the text the mesh holds is the archive's. The controller
// registers the resolved manifest as the build result carries it and fetches nothing from the artifact store; a
// manifest that still names a template when it would be held — built by a builder that predates templates — is
// refused at registration (HeldTemplateProblem).
// TemplateField is the key a file resource names its template file by.
const TemplateField = "template"
// DefaultTemplatesDir is the directory a module's template files are kept in, and its templates archive is built
// from, when its manifest names none (Manifest.Templates, issue 731).
const DefaultTemplatesDir = "files"
// TemplatesDir is the module's directory its template files are kept in, and the directory its templates archive is
// built from: the one its manifest names, else files/.
func (m Manifest) TemplatesDir() string {
if m.Templates != "" {
return m.Templates
}
return DefaultTemplatesDir
}
// templatesDirProblem is what is wrong with the templates directory a manifest names, or "": one directory of the
// module's own, a plain relative path inside it — never absolute, never leaving it, never the module's directory
// itself — and named only by a module that names a template, since a word read by nothing is refused.
func templatesDirProblem(m Manifest, templated bool) string {
d := m.Templates
if d == "" {
return ""
}
if strings.HasPrefix(d, "/") || strings.Contains(d, "\\") || path.Clean(d) != d || d == "." || d == ".." ||
strings.HasPrefix(d, "../") {
return fmt.Sprintf("%s names the templates directory %q, which is not a directory of its own: a plain "+
"relative path inside the module's directory, such as \"templates\" (issue 731)", m.Module, d)
}
if !templated {
return fmt.Sprintf("%s names the templates directory %q and names no template: a file resource names its "+
"template as %q, or the manifest says no templates directory", m.Module, d, d+"/<name>")
}
return ""
}
// Templated is whether any of the manifest's resources, or any item it contributes (collections.go), names a
// template.
func (m Manifest) Templated() bool {
for _, r := range m.Resources {
if _, has := r[TemplateField]; has {
return true
}
}
for _, it := range m.Items {
if it.Template != "" {
return true
}
}
return false
}
// TemplatesArtifact is the name of the archive artifact the module packs its templates into — the one built from its
// templates directory (TemplatesDir) — or "" when it declares none, or more than one (both refused by templateProblems
// for a module naming a template).
func TemplatesArtifact(m Manifest) string {
found := templatesArtifacts(m)
if len(found) != 1 {
return ""
}
return found[0]
}
func templatesArtifacts(m Manifest) []string {
if m.Build == nil {
return nil
}
var found []string
for _, a := range m.Build.Artifacts {
if a.Kind == ArtifactArchive && a.URL == "" && cleanFrom(a.From) == m.TemplatesDir() {
found = append(found, a.Name)
}
}
return found
}
// cleanFrom is an artifact's directory as one relative path: "./files/" is "files".
func cleanFrom(from string) string {
c := path.Clean(strings.TrimSpace(from))
return strings.TrimPrefix(c, "./")
}
// templateOf is the template a resource names, whether it names one, and what is wrong with it ("" when nothing):
// dir is the module's templates directory.
func templateOf(module, dir string, r map[string]any) (string, bool, string) {
raw, has := r[TemplateField]
if !has {
return "", false, ""
}
id := fmt.Sprint(r["id"])
if t := fmt.Sprint(r["type"]); t != "file" {
return "", true, fmt.Sprintf("%s: %s is a %s and names a template; only a file takes its text from one "+
"(novox/hq ADR 0349 §2)", module, id, t)
}
name, ok := raw.(string)
if !ok || strings.TrimSpace(name) == "" {
return "", true, fmt.Sprintf("%s: %s names a template that is not a path: %v; a template is %q",
module, id, raw, dir+"/<name>")
}
if _, both := r["content"]; both {
return name, true, fmt.Sprintf("%s: %s names the template %q and says its content too; a file's text is "+
"in one place, the template or the content, never both (novox/hq ADR 0349 §2)", module, id, name)
}
if strings.HasPrefix(name, "/") || path.Clean(name) != name || !strings.HasPrefix(name, dir+"/") ||
strings.Contains(name, "\\") {
return name, true, fmt.Sprintf("%s: %s names the template %q, which is not a file under %s/ of its own "+
"repository: a template is %q, a plain relative path, and the build packs only %s/",
module, id, name, dir, dir+"/<name>", dir)
}
return name, true, ""
}
// templateProblems is what is wrong with the manifest's templates, as written: a template on something not a file,
// beside content, or outside the module's templates directory; a templates directory that is not one, or that no
// template is in; or a module naming templates whose build packs no archive from that directory. Whether each
// template exists is judged where a tree is (ReadTemplates) or the archive's text is (Resolve).
func templateProblems(m Manifest) []string {
var problems []string
templated := m.Templated()
if wrong := templatesDirProblem(m, templated); wrong != "" {
return append(problems, wrong)
}
dir := m.TemplatesDir()
for _, r := range m.Resources {
if _, _, wrong := templateOf(m.Module, dir, r); wrong != "" {
problems = append(problems, wrong)
}
}
if !templated {
return problems
}
switch found := templatesArtifacts(m); len(found) {
case 0:
problems = append(problems, fmt.Sprintf("%s names templates and its build packs no archive from %s/: "+
"declare an artifact {\"name\": %q, \"kind\": \"archive\", \"from\": %q} (novox/hq ADR 0349 §2)",
m.Module, dir, path.Base(dir), dir))
case 1:
default:
sort.Strings(found)
problems = append(problems, fmt.Sprintf("%s packs %s/ into %s; its templates are read from one archive",
m.Module, dir, strings.Join(found, " and ")))
}
return problems
}
// fillTemplates is resources with each template's text as its content and the template key dropped: text says the text
// of a template by its path ("<templates directory>/<name>"), and whether it has one. A template it has no text for, or
// text that is not UTF-8, is refused by name. Every resource is copied; the manifest's own are left as they were.
func fillTemplates(module, dir string, resources []map[string]any, text func(string) (string, bool),
where string) ([]map[string]any, error) {
out := make([]map[string]any, 0, len(resources))
var problems []string
for _, r := range resources {
name, has, wrong := templateOf(module, dir, r)
if !has {
out = append(out, r)
continue
}
if wrong != "" {
problems = append(problems, wrong)
continue
}
body, ok := text(name)
if !ok {
problems = append(problems, fmt.Sprintf("%s: %v names the template %q, and %s holds no such file",
module, r["id"], name, where))
continue
}
if !utf8.ValidString(body) {
// A manifest is JSON: text that is not UTF-8 would be changed on the way, silently.
problems = append(problems, fmt.Sprintf("%s: %v names the template %q, which is not UTF-8 text; a "+
"file's text is carried as JSON and would not reach the machine as written", module, r["id"], name))
continue
}
filled := make(map[string]any, len(r))
for k, v := range r {
filled[k] = v
}
delete(filled, TemplateField)
filled["content"] = body
out = append(out, filled)
}
if len(problems) > 0 {
return nil, errors.New(strings.Join(problems, "; "))
}
return out, nil
}
// fillItemTemplates is items with each template's text as its content and the template dropped, as fillTemplates
// does for files (collections.go).
func fillItemTemplates(m Manifest, items []Item, text func(string) (string, bool), where string) ([]Item, error) {
if len(items) == 0 {
return items, nil
}
out := make([]Item, 0, len(items))
var problems []string
for _, it := range items {
if it.Template == "" {
out = append(out, it)
continue
}
if wrong := itemTemplateProblem(m, it.Template); wrong != "" {
problems = append(problems, fmt.Sprintf("%s's item of %s: %s", m.Module, it.Collection, wrong))
continue
}
body, ok := text(it.Template)
switch {
case !ok:
problems = append(problems, fmt.Sprintf("%s: its item of %s names the template %q, and %s holds no "+
"such file", m.Module, it.Collection, it.Template, where))
continue
case !utf8.ValidString(body):
problems = append(problems, fmt.Sprintf("%s: its item of %s names the template %q, which is not UTF-8 "+
"text", m.Module, it.Collection, it.Template))
continue
}
it.Template, it.Content = "", body
out = append(out, it)
}
if len(problems) > 0 {
return nil, errors.New(strings.Join(problems, "; "))
}
return out, nil
}
// ReadTemplates is the text of each template the manifest names, read from the module's directory in a repository tree,
// by template path. Refused by name: a template the directory lacks, and anything under its templates directory
// (TemplatesDir) the build's packing refuses — that directory itself, or any directory or file under it, that is a
// symlink or not a plain file — so nothing judged here can be read from outside the tree, and no tree passes here that
// the build would refuse. Each template is read through an os.Root of the module's directory besides, which no path
// element can leave.
func ReadTemplates(m Manifest, dir string) (map[string]string, error) {
problems := templatesTreeProblems(m.Module, dir, m.TemplatesDir())
root, err := os.OpenRoot(dir)
if err != nil {
return nil, fmt.Errorf("%s: the module's directory cannot be opened to read its templates: %w", m.Module, err)
}
defer root.Close()
texts := map[string]string{}
type named struct {
id any
template string
wrong string
}
var templates []named
for _, r := range m.Resources {
name, has, wrong := templateOf(m.Module, m.TemplatesDir(), r)
if has {
templates = append(templates, named{r["id"], name, wrong})
}
}
for _, it := range m.Items {
if it.Template != "" {
templates = append(templates, named{"its item of " + it.key(), it.Template,
itemTemplateProblem(m, it.Template)})
}
}
for _, t := range templates {
r := map[string]any{"id": t.id}
name, wrong := t.template, t.wrong
if wrong != "" {
problems = append(problems, wrong)
continue
}
if _, read := texts[name]; read {
continue
}
info, err := root.Lstat(name)
switch {
case errors.Is(err, fs.ErrNotExist):
problems = append(problems, fmt.Sprintf("%s: %v names the template %q, and the module's directory "+
"has no such file", m.Module, r["id"], name))
continue
case err != nil:
problems = append(problems, fmt.Sprintf("%s: %v names the template %q, which cannot be read: %v",
m.Module, r["id"], name, err))
continue
case !info.Mode().IsRegular():
problems = append(problems, fmt.Sprintf("%s: %v names the template %q, which is not a plain file; "+
"the build packs only files", m.Module, r["id"], name))
continue
}
body, err := root.ReadFile(name)
if err != nil {
problems = append(problems, fmt.Sprintf("%s: %v names the template %q, which cannot be read: %v",
m.Module, r["id"], name, err))
continue
}
texts[name] = string(body)
}
if len(problems) > 0 {
return nil, errors.New(strings.Join(problems, "; "))
}
return texts, nil
}
// templatesTreeProblems is everything under the module's templates directory (templates, inside dir) that the
// build's packing refuses: that directory itself not a plain directory, and any entry under it that is neither a
// directory nor a plain file — a symlink, to a file or a directory, a device, a pipe. Each said by its path. A
// templates directory that does not exist is no problem here: each template it would have held is refused as missing.
func templatesTreeProblems(module, dir, templates string) []string {
top := filepath.Join(dir, filepath.FromSlash(templates))
info, err := os.Lstat(top)
if errors.Is(err, fs.ErrNotExist) {
return nil
}
if err != nil {
return []string{fmt.Sprintf("%s: %s/ cannot be read: %v", module, templates, err)}
}
if !info.IsDir() {
return []string{fmt.Sprintf("%s: %s/ is %s, not a directory of the module's own; its templates are read "+
"only from inside the repository, and the build packs only directories and files", module, templates,
kindOf(info.Mode()))}
}
var problems []string
err = filepath.WalkDir(top, func(at string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() || d.Type().IsRegular() {
return nil
}
rel, _ := filepath.Rel(dir, at)
problems = append(problems, fmt.Sprintf("%s: %s is %s; the build packs only directories and files, so "+
"%s/ holds nothing else", module, filepath.ToSlash(rel), kindOf(d.Type()), templates))
return nil
})
if err != nil {
problems = append(problems, fmt.Sprintf("%s: %s/ cannot be read: %v", module, templates, err))
}
return problems
}
// kindOf says what a file that is not a plain file is.
func kindOf(mode fs.FileMode) string {
switch {
case mode&fs.ModeSymlink != 0:
return "a symlink"
case mode&fs.ModeDevice != 0:
return "a device"
case mode&fs.ModeNamedPipe != 0:
return "a named pipe"
case mode&fs.ModeSocket != 0:
return "a socket"
case mode.IsDir():
return "a directory"
}
return "not a plain file"
}
// WithTemplatesFrom is the manifest with each template's text read from the module's directory in a repository
// tree as its content: what `module check`, the merge gate and `module add` judge, where nothing was built. Read
// again strictly afterwards, so every rule about a file's text judges the template's text exactly as it judges
// inline content. A manifest naming no template is returned as it is.
func WithTemplatesFrom(m Manifest, dir string) (Manifest, error) {
if !m.Templated() {
return m, nil
}
texts, err := ReadTemplates(m, dir)
if err != nil {
return Manifest{}, err
}
filled, err := fillTemplates(m.Module, m.TemplatesDir(), m.Resources, func(name string) (string, bool) {
t, ok := texts[name]
return t, ok
}, "the module's directory")
if err != nil {
return Manifest{}, err
}
items, err := fillItemTemplates(m, m.Items, func(name string) (string, bool) {
t, ok := texts[name]
return t, ok
}, "the module's directory")
if err != nil {
return Manifest{}, err
}
out := m
out.Resources = filled
out.Items = items
// Nothing names a template now, so nothing names its directory.
out.Templates = ""
// A moved module's mark was judged above, while it still named its templates; read again with its text inline,
// it is judged as it composes without its build's templates archive — as before, nothing handed (handed.go).
out.Moved = false
raw, err := json.Marshal(out)
if err != nil {
return Manifest{}, err
}
again, err := parseManifest(raw, m.predefined)
if err != nil {
return Manifest{}, err
}
again.Moved = m.Moved
return again, nil
}
// HeldTemplateProblem is why a manifest cannot be held by the mesh because it still names a template, or "": the
// manifest the mesh holds carries every file's text, which a build reads back from its templates archive. One
// naming a template was resolved by a builder that does not read templates, and would compose files with no text.
func HeldTemplateProblem(m Manifest) string {
var named []string
for _, r := range m.Resources {
if t, has := r[TemplateField]; has {
named = append(named, fmt.Sprintf("%v (%v)", r["id"], t))
}
}
for _, it := range m.Items {
if it.Template != "" {
named = append(named, fmt.Sprintf("its item of %s (%s)", it.Collection, it.Template))
}
}
if len(named) == 0 {
return ""
}
return fmt.Sprintf("%s still names the template(s) of %s: the manifest the mesh holds carries each file's "+
"text, read back from the build's templates archive — this one was resolved by a builder that does not "+
"read templates, or not built at all (novox/hq ADR 0349 §2)", m.Module, strings.Join(named, ", "))
}
+323
View File
@@ -0,0 +1,323 @@
package catalogue
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
)
// A file resource may name a template file in its module's repository (novox/hq ADR 0349 §2, design 57 package
// T1, issue 654): the build packs files/ into an archive, and resolving the build reads each template's text back
// into content, so the mesh holds, composes and sends exactly what it would for the same text inline.
// The two forms of one module: its text inline, and its text in files/.
const (
inlineTwin = `{"module":"tidy","version":"1",
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},
"resources":[
{"id":"conf","type":"directory","path":"${dir:conf}"},
{"id":"main","type":"file","path":"${dir:conf}/tidy.conf","mode":"0644","content":"` + mainText + `"},
{"id":"extra","type":"file","path":"${dir:conf}/extra.conf","content":"` + extraText + `"}]}`
templatedTwin = `{"module":"tidy","version":"1",
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},
"resources":[
{"id":"conf","type":"directory","path":"${dir:conf}"},
{"id":"main","type":"file","path":"${dir:conf}/tidy.conf","mode":"0644","template":"files/tidy.conf"},
{"id":"extra","type":"file","path":"${dir:conf}/extra.conf","template":"files/deeper/extra.conf"}]}`
mainText = `# tidy\nroot = ${dir:conf}\nquoted = \"yes\" é\n`
extraText = `one\ntwo\n`
)
// templateTexts is what a build reads back from the packed files/ archive: by path inside it.
func templateTexts(t *testing.T) map[string]string {
t.Helper()
var m struct{ Main, Extra string }
if err := json.Unmarshal([]byte(`{"Main":"`+mainText+`","Extra":"`+extraText+`"}`), &m); err != nil {
t.Fatal(err)
}
return map[string]string{"tidy.conf": m.Main, "deeper/extra.conf": m.Extra, "unused.conf": "not named"}
}
func builtFiles(texts map[string]string) []Built {
return []Built{{Name: "files", Kind: ArtifactArchive,
Reference: ArtifactStoreScheme + "tidy/files/blobs/sha256:" + strings.Repeat("c", 64),
Digest: "sha256:" + strings.Repeat("c", 64), Texts: texts}}
}
// held is a manifest as registration holds it: resolved with what was built, then read again strictly.
func held(t *testing.T, raw string, built []Built) Manifest {
t.Helper()
m, err := ParseManifest([]byte(raw))
if err != nil {
t.Fatal(err)
}
resolved, err := m.Resolve(built)
if err != nil {
t.Fatal(err)
}
again, err := json.Marshal(resolved)
if err != nil {
t.Fatal(err)
}
out, err := ParseManifest(again)
if err != nil {
t.Fatal(err)
}
if problem := HeldTemplateProblem(out); problem != "" {
t.Fatal(problem)
}
return out
}
func composed(t *testing.T, m Manifest) string {
t.Helper()
got, err := Resolve(shelf(m), []string{m.Module}, workstation(), World{})
if err != nil {
t.Fatal(err)
}
out, err := json.Marshal(mustDeclare(t, got))
if err != nil {
t.Fatal(err)
}
return string(out)
}
func TestATemplatedModuleComposesByteIdenticallyToItsInlineTwin(t *testing.T) {
inline := composed(t, held(t, inlineTwin, builtFiles(nil)))
templated := composed(t, held(t, templatedTwin, builtFiles(templateTexts(t))))
if inline != templated {
t.Fatalf("the two forms compose differently:\ninline: %s\ntemplated: %s", inline, templated)
}
if !strings.Contains(templated, "quoted = \\\"yes\\\"") || strings.Contains(templated, `"template"`) {
t.Fatalf("the composition does not carry the template's text as content: %s", templated)
}
}
// What `module check`, the merge gate and `module add` judge: the text read from the tree beside the manifest.
func TestATemplatedModuleReadFromItsTreeComposesByteIdenticallyToItsInlineTwin(t *testing.T) {
dir := t.TempDir()
for name, text := range templateTexts(t) {
full := filepath.Join(dir, "files", filepath.FromSlash(name))
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(full, []byte(text), 0o644); err != nil {
t.Fatal(err)
}
}
m, err := ParseManifest([]byte(templatedTwin))
if err != nil {
t.Fatal(err)
}
filled, err := WithTemplatesFrom(m, dir)
if err != nil {
t.Fatal(err)
}
if filled.Templated() {
t.Fatal("a manifest read from its tree still names a template")
}
resolved, err := filled.Resolve(builtFiles(nil))
if err != nil {
t.Fatal(err)
}
if got, want := composed(t, resolved), composed(t, held(t, inlineTwin, builtFiles(nil))); got != want {
t.Fatalf("read from the tree, the module composes differently:\ninline: %s\ntemplated: %s", want, got)
}
}
func TestAFileNamingBothATemplateAndContentIsRefused(t *testing.T) {
raw := strings.Replace(templatedTwin, `"template":"files/tidy.conf"`,
`"template":"files/tidy.conf","content":"x"`, 1)
_, err := ParseManifest([]byte(raw))
if err == nil || !strings.Contains(err.Error(), "main") || !strings.Contains(err.Error(), "never both") {
t.Fatalf("a file with both a template and content was not refused by name: %v", err)
}
}
func TestATemplateOutsideFilesIsRefusedByName(t *testing.T) {
for _, bad := range []string{"../tidy.conf", "/etc/tidy.conf", "files", "files/", "files/../tidy.conf",
"files/./tidy.conf", "other/tidy.conf", "tidy.conf", "files//tidy.conf", `files\\tidy.conf`} {
raw := strings.Replace(templatedTwin, `"files/tidy.conf"`, `"`+bad+`"`, 1)
_, err := ParseManifest([]byte(raw))
if err == nil || !strings.Contains(err.Error(), "main") || !strings.Contains(err.Error(), "not a file under files/") {
t.Errorf("the template %q was not refused by name: %v", bad, err)
}
}
}
func TestATemplateOnAResourceThatIsNotAFileIsRefused(t *testing.T) {
raw := strings.Replace(templatedTwin, `{"id":"conf","type":"directory","path":"${dir:conf}"}`,
`{"id":"conf","type":"directory","path":"${dir:conf}","template":"files/tidy.conf"}`, 1)
_, err := ParseManifest([]byte(raw))
if err == nil || !strings.Contains(err.Error(), "conf is a directory and names a template") {
t.Fatalf("a directory naming a template was not refused: %v", err)
}
}
func TestATemplatedModuleWhoseBuildPacksNoFilesIsRefused(t *testing.T) {
for _, build := range []string{
``,
`"build":{"artifacts":[{"name":"look","kind":"archive","from":"theme"}]},`,
`"build":{"artifacts":[{"name":"a","kind":"archive","from":"files"},{"name":"b","kind":"archive","from":"./files/"}]},`,
} {
raw := strings.Replace(templatedTwin,
`"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},`, build, 1)
_, err := ParseManifest([]byte(raw))
if err == nil || !strings.Contains(err.Error(), "files/") {
t.Errorf("a templated module with the build %q was not refused: %v", build, err)
}
}
}
func TestAMissingTemplateIsRefusedByName(t *testing.T) {
m, err := ParseManifest([]byte(templatedTwin))
if err != nil {
t.Fatal(err)
}
// In the tree beside the manifest.
dir := t.TempDir()
if err := os.MkdirAll(filepath.Join(dir, "files"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "files", "tidy.conf"), []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
_, err = WithTemplatesFrom(m, dir)
if err == nil || !strings.Contains(err.Error(), "extra") || !strings.Contains(err.Error(), "files/deeper/extra.conf") {
t.Fatalf("a template missing from the tree was not refused by name: %v", err)
}
// And in what the build read back from its archive.
texts := templateTexts(t)
delete(texts, "deeper/extra.conf")
_, err = m.Resolve(builtFiles(texts))
if err == nil || !strings.Contains(err.Error(), "extra") || !strings.Contains(err.Error(), "files/deeper/extra.conf") {
t.Fatalf("a template missing from the archive was not refused by name: %v", err)
}
}
func TestATemplateThatIsASymlinkIsRefused(t *testing.T) {
m, err := ParseManifest([]byte(templatedTwin))
if err != nil {
t.Fatal(err)
}
dir := t.TempDir()
if err := os.MkdirAll(filepath.Join(dir, "files", "deeper"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "elsewhere"), []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "files", "tidy.conf"), []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.Symlink(filepath.Join(dir, "elsewhere"), filepath.Join(dir, "files", "deeper", "extra.conf")); err != nil {
t.Fatal(err)
}
if _, err := WithTemplatesFrom(m, dir); err == nil || !strings.Contains(err.Error(), "not a plain file") {
t.Fatalf("a template that is a symlink was not refused: %v", err)
}
}
func TestAResolveWithoutTheArchivesTextIsRefused(t *testing.T) {
m, err := ParseManifest([]byte(templatedTwin))
if err != nil {
t.Fatal(err)
}
if _, err := m.Resolve(builtFiles(nil)); err == nil || !strings.Contains(err.Error(), "files") {
t.Fatalf("a templated module resolved with no text read back from its archive: %v", err)
}
}
func TestATemplateThatIsNotUTF8IsRefused(t *testing.T) {
m, err := ParseManifest([]byte(templatedTwin))
if err != nil {
t.Fatal(err)
}
texts := templateTexts(t)
texts["tidy.conf"] = "\xff\xfe"
if _, err := m.Resolve(builtFiles(texts)); err == nil || !strings.Contains(err.Error(), "UTF-8") {
t.Fatalf("a template that is not UTF-8 was resolved: %v", err)
}
}
// What registration refuses: a manifest to be held that still names a template — one resolved by a builder that
// predates templates would otherwise compose files with no text.
func TestAManifestStillNamingATemplateIsNotHeld(t *testing.T) {
m, err := ParseManifest([]byte(templatedTwin))
if err != nil {
t.Fatal(err)
}
if problem := HeldTemplateProblem(m); !strings.Contains(problem, "main") || !strings.Contains(problem, "files/tidy.conf") {
t.Fatalf("a manifest naming a template would be held: %q", problem)
}
if problem := HeldTemplateProblem(held(t, inlineTwin, builtFiles(nil))); problem != "" {
t.Fatalf("an inline manifest is refused: %s", problem)
}
}
// A symlinked files/, or a symlinked directory under it, would read a file outside the tree: refused by name,
// as is any symlink or special file under files/ that no template names, which the build's packing refuses.
func TestASymlinkAnywhereUnderFilesIsRefusedByName(t *testing.T) {
m, err := ParseManifest([]byte(templatedTwin))
if err != nil {
t.Fatal(err)
}
outside := t.TempDir()
if err := os.MkdirAll(filepath.Join(outside, "deeper"), 0o755); err != nil {
t.Fatal(err)
}
for _, name := range []string{"tidy.conf", "deeper/extra.conf", "secret"} {
if err := os.WriteFile(filepath.Join(outside, filepath.FromSlash(name)), []byte("outside"), 0o644); err != nil {
t.Fatal(err)
}
}
write := func(t *testing.T, dir, name string) {
t.Helper()
full := filepath.Join(dir, filepath.FromSlash(name))
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(full, []byte("inside"), 0o644); err != nil {
t.Fatal(err)
}
}
link := func(t *testing.T, target, at string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(at), 0o755); err != nil {
t.Fatal(err)
}
if err := os.Symlink(target, at); err != nil {
t.Fatal(err)
}
}
for _, c := range []struct {
name string
make func(t *testing.T, dir string)
names string
}{
{"files/ itself", func(t *testing.T, dir string) {
link(t, outside, filepath.Join(dir, "files"))
}, "files/ is a symlink"},
{"a directory under files/", func(t *testing.T, dir string) {
write(t, dir, "files/tidy.conf")
link(t, filepath.Join(outside, "deeper"), filepath.Join(dir, "files", "deeper"))
}, "files/deeper is a symlink"},
{"a file no template names", func(t *testing.T, dir string) {
write(t, dir, "files/tidy.conf")
write(t, dir, "files/deeper/extra.conf")
link(t, filepath.Join(outside, "secret"), filepath.Join(dir, "files", "unnamed"))
}, "files/unnamed is a symlink"},
} {
dir := t.TempDir()
c.make(t, dir)
_, err := WithTemplatesFrom(m, dir)
if err == nil || !strings.Contains(err.Error(), c.names) {
t.Errorf("a symlinked %s was not refused by name: %v", c.name, err)
}
if err != nil && strings.Contains(err.Error(), "outside") {
t.Errorf("a symlinked %s was read: %v", c.name, err)
}
}
}
+151
View File
@@ -0,0 +1,151 @@
package catalogue
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
)
// A module may name the directory its templates are kept in (`"templates": "<dir>"`, default files/; issue 731): a
// module whose files/ is already an archive unpacked onto the machine keeps its templates in a directory of their
// own, packed into an archive no resource names, so its unpacked files/ is exactly what it was.
// The two forms of one module whose files/ is unpacked onto the machine: its text inline, and in templates/.
const (
unpackingInline = `{"module":"tidy","version":"1",
"build":{"artifacts":[{"name":"scripts","kind":"archive","from":"files"}]},
"resources":[
{"id":"scripts","type":"archive","path":"/usr/local/lib/tidy","artifact":"scripts"},
{"id":"conf","type":"directory","path":"${dir:conf}"},
{"id":"main","type":"file","path":"/etc/tidy.conf","mode":"0644","content":"` + mainText + `"},
{"id":"extra","type":"file","path":"/etc/extra.conf","content":"` + extraText + `"}]}`
unpackingTemplated = `{"module":"tidy","version":"1",
"templates":"templates",
"build":{"artifacts":[{"name":"scripts","kind":"archive","from":"files"},
{"name":"templates","kind":"archive","from":"templates"}]},
"resources":[
{"id":"scripts","type":"archive","path":"/usr/local/lib/tidy","artifact":"scripts"},
{"id":"conf","type":"directory","path":"${dir:conf}"},
{"id":"main","type":"file","path":"/etc/tidy.conf","mode":"0644","template":"templates/tidy.conf"},
{"id":"extra","type":"file","path":"/etc/extra.conf","template":"templates/deeper/extra.conf"}]}`
)
func builtUnpacking(templates map[string]string) []Built {
built := []Built{{Name: "scripts", Kind: ArtifactArchive,
Reference: "https://store.invalid/scripts.tar.gz",
Digest: "sha256:" + strings.Repeat("a", 64)}}
if templates != nil {
built = append(built, Built{Name: "templates", Kind: ArtifactArchive,
Reference: ArtifactStoreScheme + "tidy/templates/blobs/sha256:" + strings.Repeat("b", 64),
Digest: "sha256:" + strings.Repeat("b", 64), Texts: templates})
}
return built
}
func TestAModuleWithItsOwnTemplatesDirectoryComposesByteIdenticallyToItsInlineTwin(t *testing.T) {
inline := composed(t, held(t, unpackingInline, builtUnpacking(nil)))
templated := composed(t, held(t, unpackingTemplated, builtUnpacking(templateTexts(t))))
if inline != templated {
t.Fatalf("the two forms compose differently:\ninline: %s\ntemplated: %s", inline, templated)
}
// The unpacked archive is the files/ one, by its digest; the templates archive reaches no machine.
if !strings.Contains(templated, strings.Repeat("a", 64)) || strings.Contains(templated, strings.Repeat("b", 64)) {
t.Fatalf("the composition does not unpack files/ alone: %s", templated)
}
}
func TestTheManifestTheMeshHoldsNamesNoTemplatesDirectory(t *testing.T) {
m := held(t, unpackingTemplated, builtUnpacking(templateTexts(t)))
raw, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(raw), `"templates"`) {
t.Fatalf("the held manifest still names its templates directory: %s", raw)
}
}
func TestAModuleWithItsOwnTemplatesDirectoryIsReadFromItsTree(t *testing.T) {
dir := t.TempDir()
for name, text := range templateTexts(t) {
full := filepath.Join(dir, "templates", filepath.FromSlash(name))
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(full, []byte(text), 0o644); err != nil {
t.Fatal(err)
}
}
// files/ holds what the machine unpacks, a symlink among it judged by nobody here: it is not the templates.
if err := os.MkdirAll(filepath.Join(dir, "files", "bin"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "files", "bin", "tidy"), []byte("#!/bin/sh\n"), 0o755); err != nil {
t.Fatal(err)
}
m, err := ParseManifest([]byte(unpackingTemplated))
if err != nil {
t.Fatal(err)
}
filled, err := WithTemplatesFrom(m, dir)
if err != nil {
t.Fatal(err)
}
resolved, err := filled.Resolve(builtUnpacking(map[string]string{}))
if err != nil {
t.Fatal(err)
}
if got, want := composed(t, resolved), composed(t, held(t, unpackingInline, builtUnpacking(nil))); got != want {
t.Fatalf("read from the tree, the module composes differently:\ninline: %s\ntemplated: %s", want, got)
}
// A symlink under the templates directory is refused by its path, as under files/.
if err := os.Symlink("/etc/hostname", filepath.Join(dir, "templates", "elsewhere")); err != nil {
t.Fatal(err)
}
if _, err := WithTemplatesFrom(m, dir); err == nil || !strings.Contains(err.Error(), "templates/elsewhere is a symlink") {
t.Fatalf("a symlink under templates/ was not refused by its path: %v", err)
}
}
func TestATemplateOutsideTheNamedTemplatesDirectoryIsRefused(t *testing.T) {
raw := strings.Replace(unpackingTemplated, `"template":"templates/tidy.conf"`, `"template":"files/tidy.conf"`, 1)
if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), "files/tidy.conf") ||
!strings.Contains(err.Error(), "templates/") {
t.Fatalf("a template outside the named templates directory was not refused by name: %v", err)
}
}
func TestATemplatesDirectoryNoArchiveIsBuiltFromIsRefused(t *testing.T) {
raw := strings.Replace(unpackingTemplated, `,
{"name":"templates","kind":"archive","from":"templates"}`, "", 1)
if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), "no archive from templates/") {
t.Fatalf("a module whose build packs no archive from its templates directory was not refused: %v", err)
}
}
func TestATemplatesDirectoryOnAModuleNamingNoTemplateIsRefused(t *testing.T) {
raw := strings.Replace(unpackingInline, `"version":"1",`, `"version":"1","templates":"templates",`, 1)
if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), "names no template") {
t.Fatalf("a templates directory read by nothing was not refused: %v", err)
}
}
func TestATemplatesDirectoryThatIsNotAPlainRelativeDirectoryIsRefused(t *testing.T) {
for _, dir := range []string{"/etc", "../other", "..", "templates/../files", "./templates", ".", "a\\b", "templates/"} {
raw := strings.Replace(unpackingTemplated, `"templates":"templates"`, `"templates":`+jsonString(dir), 1)
if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), "templates directory") {
t.Errorf("the templates directory %q was not refused: %v", dir, err)
}
}
}
func TestATemplatesDirectoryNamedFilesIsTheDefault(t *testing.T) {
raw := strings.Replace(templatedTwin, `"version":"1",`, `"version":"1","templates":"files",`, 1)
if got, want := composed(t, held(t, raw, builtFiles(templateTexts(t)))),
composed(t, held(t, inlineTwin, builtFiles(nil))); got != want {
t.Fatalf("naming files/ as the templates directory composes differently:\n%s\n%s", got, want)
}
}
+4 -5
View File
@@ -145,11 +145,10 @@ func asksFor(r map[string]any) []string {
content, _ := r["content"].(string)
asked := settingsUsed(content)
if how, _ := r["merge"].(string); how != "" {
if taken, names := takenKeys(r); names {
// The file takes exactly the keys it names (ADR 0326), whatever its content holds.
for key := range taken {
asked = append(asked, key)
}
if _, names := takenKeys(r); names {
// The file takes exactly the keys it names (ADR 0326), whatever its content holds, and a renamed
// one under the file's own spelling too, which a layer said before its module moved (issue 751).
asked = append(asked, takenNames(r)...)
sort.Strings(asked)
return asked
}
+7 -7
View File
@@ -13,22 +13,22 @@ func TestAPlacementOrAnAccessAtTheMachinesOwnIsRefused(t *testing.T) {
for _, path := range []string{"/", "/etc", "/etc/", "/etc/sudoers.d", "/usr/bin", "/root", "/var", "/var/lib",
"/home", "/home/", "/run", "/run/user/1000", "/var/lib/mesh/x", "/var/lib/mesh-host", "/var/lib/mesh-host/identity",
"/srv/../etc", "//etc", "/proc/1", "/sys", "/dev", "/boot/efi", "/bin", "/sbin", "/lib", "/lib64"} {
layers := []Layer{{From: "laptop", Values: map[string]any{PlacesSetting: map[string]any{"data": path}}}}
layers := []Layer{LayerOf("laptop", map[string]any{PlacesSetting: map[string]any{"data": path}})}
if _, err := Places(m, layers); err == nil || !strings.Contains(err.Error(), "issue 339") {
t.Errorf("a place at %s: %v", path, err)
}
layers = []Layer{{From: "laptop", Values: map[string]any{AccessesSetting: map[string]any{"media": path}}}}
layers = []Layer{LayerOf("laptop", map[string]any{AccessesSetting: map[string]any{"media": path}})}
if _, err := AccessPlaces(m, layers); err == nil || !strings.Contains(err.Error(), "issue 339") {
t.Errorf("an access at %s: %v", path, err)
}
}
for _, path := range []string{"/srv/notes", "/mnt/plex/data", "/storage/media", "/services/media/movies",
"/var/lib/notes/data", "/home/restic/repo", "/var/lib/mesh-store"} {
layers := []Layer{{From: "laptop", Values: map[string]any{PlacesSetting: map[string]any{"data": path}}}}
layers := []Layer{LayerOf("laptop", map[string]any{PlacesSetting: map[string]any{"data": path}})}
if got, err := Places(m, layers); err != nil || got["data"].Path != path {
t.Errorf("a place at %s: %v %v", path, got, err)
}
layers = []Layer{{From: "laptop", Values: map[string]any{AccessesSetting: map[string]any{"media": path}}}}
layers = []Layer{LayerOf("laptop", map[string]any{AccessesSetting: map[string]any{"media": path}})}
if got, err := AccessPlaces(m, layers); err != nil || got["media"] != path {
t.Errorf("an access at %s: %v %v", path, got, err)
}
@@ -111,17 +111,17 @@ func TestTheRuntimesDataAndAnyHomesSSHAreTheMachinesOwn(t *testing.T) {
for _, path := range []string{"/var/spool", "/var/spool/cron", "/var/lib/docker", "/var/lib/docker/volumes",
"/var/lib/containers/storage", "/var/lib/containerd", "/var/lib/containerd/io.containerd.snapshotter.v1", "/opt", "/opt/app", "/home/alice/.ssh", "/home/alice/.ssh/keys",
"/srv/backup/.ssh", "/root/.ssh"} {
layers := []Layer{{From: "laptop", Values: map[string]any{PlacesSetting: map[string]any{"data": path}}}}
layers := []Layer{LayerOf("laptop", map[string]any{PlacesSetting: map[string]any{"data": path}})}
if _, err := Places(m, layers); err == nil || !strings.Contains(err.Error(), "issue 339") {
t.Errorf("a place at %s: %v", path, err)
}
layers = []Layer{{From: "laptop", Values: map[string]any{AccessesSetting: map[string]any{"media": path}}}}
layers = []Layer{LayerOf("laptop", map[string]any{AccessesSetting: map[string]any{"media": path}})}
if _, err := AccessPlaces(m, layers); err == nil || !strings.Contains(err.Error(), "issue 339") {
t.Errorf("an access at %s: %v", path, err)
}
}
for _, path := range []string{"/var/lib/dockerish", "/home/alice/ssh", "/home/alice/.sshd-notes", "/storage/media"} {
layers := []Layer{{From: "laptop", Values: map[string]any{PlacesSetting: map[string]any{"data": path}}}}
layers := []Layer{LayerOf("laptop", map[string]any{PlacesSetting: map[string]any{"data": path}})}
if _, err := Places(m, layers); err != nil {
t.Errorf("a place at %s: %v", path, err)
}
+8
View File
@@ -0,0 +1,8 @@
[[block]]
block = "battery"
format = "$icon $percentage{ $time|}"
charging_format = "$icon $percentage{ $time|}"
not_charging_format = "$icon $percentage"
full_format = "$icon $percentage"
empty_format = "$icon $percentage"
missing_format = ""
+5
View File
@@ -0,0 +1,5 @@
[[block]]
block = "custom"
command = "~/.local/bin/i3status-value power.draw ${machine:name}"
interval = 5
hide_when_empty = true
+119
View File
@@ -0,0 +1,119 @@
# The bottom bar (module i3status-rust, novox/hq ADR 0208). Owned by the mesh: replaced at every
# push. Adopted from the workstations of 2026-10-04 with what follows a machine's hardware or one
# person's devices left out: a block that depends on the hardware is contributed by the module that
# knows it (novox/hq ADR 0255), and a headset's address is not the bar's to know. Every block written
# here works the same on any machine.
icons_format = "{icon}"
[theme]
theme = "plain"
[theme.overrides]
idle_bg = "#000000"
idle_fg = "#f1f1f1"
info_bg = "#000000"
info_fg = "#f1f1f1"
good_bg = "#000000"
good_fg = "#859900"
warning_bg = "#000000"
warning_fg = "#de5200"
critical_bg = "#000000"
critical_fg = "#ff0000"
separator = " | "
separator_fg = "#ffffff"
separator_bg = "#000000"
[icons]
icons = "custom-icons"
[[block]]
block = "tea_timer"
format = "$icon{ $minutes:$seconds|}"
done_cmd = "notify-send 'Timer finished'"
# Pending updates: official and, where an AUR helper is installed, AUR. The script keeps the last good
# count through a network gap and shows an error only after ten failed checks in a row. Click: the
# list, in the launcher's menu.
[[block]]
block = "custom"
command = "~/.local/bin/i3status-updates 10"
json = true
interval = 1800
[[block.click]]
button = "left"
cmd = "checkupdates | dmenu -l 20 -p Updates"
[[block]]
block = "cpu"
[[block]]
block = "disk_space"
path = "/"
info_type = "used"
alert_unit = "GB"
interval = 20
warning = 850
alert = 920
format = "$icon / $used.eng(w:2) ($percentage.eng(w:2))"
[[block]]
block = "disk_space"
path = "/home"
info_type = "used"
alert_unit = "GB"
interval = 20
warning = 850
alert = 920
format = "$icon /home $used.eng(w:2) ($percentage.eng(w:2))"
[[block]]
block = "memory"
# What programs hold, without the cache the kernel gives back when asked: the figure the memory-pressure
# module judges by. Counting the cache as used read near full on a machine with room to spare.
format = "$icon $mem_used.eng(w:2) ($mem_used_percents.eng(w:2))"
format_alt = "$icon_swap $swap_used.eng(w:2) ($swap_used_percents.eng(w:2))"
# Blocks other modules contribute beside the machine's resources (novox/hq ADR 0255), rendered from
# what they show by this module's template, and only on a machine that has what they show.
[[block]]
block = "docker"
interval = 2
format = "$icon $running/$total"
[[block]]
block = "sound"
[[block.click]]
button = "left"
cmd = "pavucontrol"
# Blocks other modules contribute beside the sound (novox/hq ADR 0255): the power module's battery and
# power draw, where the machine has them.
# power
[[block]]
block = "battery"
format = "$icon $percentage{ $time|}"
charging_format = "$icon $percentage{ $time|}"
not_charging_format = "$icon $percentage"
full_format = "$icon $percentage"
empty_format = "$icon $percentage"
missing_format = ""
# The weather from the Norwegian Meteorological Institute, which needs no key, where the machine is.
[[block]]
block = "weather"
format = "$icon $weather_verbose ($location) $temp"
autolocate = true
autolocate_interval = 600
[block.service]
name = "metno"
[[block]]
block = "notify"
driver = "dunst"
format = " $icon {($notification_count.eng(w:1))|}"
[[block]]
block = "time"
interval = 1
format = "$timestamp.datetime(f:'%a %d/%m %H:%M:%S')"
+36
View File
@@ -0,0 +1,36 @@
# The top bar (module i3status-rust, novox/hq ADR 0208): the focused window's title and the uptime.
# Owned by the mesh: replaced at every push. Adopted unchanged from the workstations of 2026-10-04.
icons_format = "{icon}"
[theme]
theme = "plain"
[theme.overrides]
idle_bg = "#000000"
idle_fg = "#f1f1f1"
info_bg = "#000000"
info_fg = "#f1f1f1"
good_bg = "#000000"
good_fg = "#859900"
warning_bg = "#000000"
warning_fg = "#de5200"
critical_bg = "#000000"
critical_fg = "#ff0000"
separator = " | "
separator_fg = "#ffffff"
separator_bg = "#000000"
[icons]
icons = "custom-icons"
[[block]]
block = "focused_window"
[block.format]
full = " $title.str(max_w:50) |"
short = " $title.str(max_w:100) |"
[[block]]
block = "uptime"
interval = 60
# Blocks other modules contribute to the top bar (novox/hq ADR 0255). None do yet; both places are kept
# so that one that does is never dropped.
+126
View File
@@ -0,0 +1,126 @@
# The bottom bar (module i3status-rust, novox/hq ADR 0208). Owned by the mesh: replaced at every
# push. Adopted from the workstations of 2026-10-04 with what follows a machine's hardware or one
# person's devices left out: a block that depends on the hardware is contributed by the module that
# knows it (novox/hq ADR 0255), and a headset's address is not the bar's to know. Every block written
# here works the same on any machine.
icons_format = "{icon}"
[theme]
theme = "plain"
[theme.overrides]
idle_bg = "#000000"
idle_fg = "#f1f1f1"
info_bg = "#000000"
info_fg = "#f1f1f1"
good_bg = "#000000"
good_fg = "#859900"
warning_bg = "#000000"
warning_fg = "#de5200"
critical_bg = "#000000"
critical_fg = "#ff0000"
separator = " | "
separator_fg = "#ffffff"
separator_bg = "#000000"
[icons]
icons = "custom-icons"
[[block]]
block = "tea_timer"
format = "$icon{ $minutes:$seconds|}"
done_cmd = "notify-send 'Timer finished'"
# Pending updates: official and, where an AUR helper is installed, AUR. The script keeps the last good
# count through a network gap and shows an error only after ten failed checks in a row. Click: the
# list, in the launcher's menu.
[[block]]
block = "custom"
command = "~/.local/bin/i3status-updates 10"
json = true
interval = 1800
[[block.click]]
button = "left"
cmd = "checkupdates | dmenu -l 20 -p Updates"
[[block]]
block = "cpu"
[[block]]
block = "disk_space"
path = "/"
info_type = "used"
alert_unit = "GB"
interval = 20
warning = 850
alert = 920
format = "$icon / $used.eng(w:2) ($percentage.eng(w:2))"
[[block]]
block = "disk_space"
path = "/home"
info_type = "used"
alert_unit = "GB"
interval = 20
warning = 850
alert = 920
format = "$icon /home $used.eng(w:2) ($percentage.eng(w:2))"
[[block]]
block = "memory"
# What programs hold, without the cache the kernel gives back when asked: the figure the memory-pressure
# module judges by. Counting the cache as used read near full on a machine with room to spare.
format = "$icon $mem_used.eng(w:2) ($mem_used_percents.eng(w:2))"
format_alt = "$icon_swap $swap_used.eng(w:2) ($swap_used_percents.eng(w:2))"
# Blocks other modules contribute beside the machine's resources (novox/hq ADR 0255), rendered from
# what they show by this module's template, and only on a machine that has what they show.
[[block]]
block = "docker"
interval = 2
format = "$icon $running/$total"
[[block]]
block = "sound"
[[block.click]]
button = "left"
cmd = "pavucontrol"
# Blocks other modules contribute beside the sound (novox/hq ADR 0255): the power module's battery and
# power draw, where the machine has them.
# power
[[block]]
block = "battery"
format = "$icon $percentage{ $time|}"
charging_format = "$icon $percentage{ $time|}"
not_charging_format = "$icon $percentage"
full_format = "$icon $percentage"
empty_format = "$icon $percentage"
missing_format = ""
# power
[[block]]
block = "custom"
command = "~/.local/bin/i3status-value power.draw workstation"
interval = 5
hide_when_empty = true
# The weather from the Norwegian Meteorological Institute, which needs no key, where the machine is.
[[block]]
block = "weather"
format = "$icon $weather_verbose ($location) $temp"
autolocate = true
autolocate_interval = 600
[block.service]
name = "metno"
[[block]]
block = "notify"
driver = "dunst"
format = " $icon {($notification_count.eng(w:1))|}"
[[block]]
block = "time"
interval = 1
format = "$timestamp.datetime(f:'%a %d/%m %H:%M:%S')"

Some files were not shown because too many files have changed in this diff Show More