novox/mesh-controller
13
0
Fork 0
Code Issues Pull Requests 5 Releases Wiki Activity

Feature 57 (config and provisioning): release mesh-controller, the reviewed release branch merged with main, on top of main

Browse Source
This commit is contained in:
mesh-tools
2026-10-16 08:28:38 +00:00
parent d70bb1f388
commit d984816844
259 changed files with 34899 additions and 353 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-builder/main.go
+5
View File
@@ -428,6 +428,11 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
for _, s := range built.Sources {
result.Sources = append(result.Sources, link.BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths})
}
// The list of predefined keys the manifest was judged against, for the controller to publish
// (novox/hq ADR 0348 §7).
if p := built.Predefined; p != nil {
result.Predefined = &link.PredefinedList{Keys: p.Keys, Generation: p.Generation}
}
say("built", built.Manifest.Module+" from "+short(built.Commit))
}
}
cmd/mesh-controller/adoption.go
+3 -2
View File
@@ -424,8 +424,9 @@ func comparisonOf(module string, c comparison, opts takeOptions) (preview string
}
// And its settings on this machine, composed against its definition (rule 1, rule 6).
for _, layer := range c.layers {
keys := make([]string, 0, len(layer.Values))
for k := range layer.Values {
said := layer.Whole()
keys := make([]string, 0, len(said))
for k := range said {
keys = append(keys, k)
}
sort.Strings(keys)
cmd/mesh-controller/apply_failing.go
+46
View File
@@ -38,15 +38,61 @@ func watchApplyFailing(f *signalFacts) []conditions.Observation {
}
}
var out []conditions.Observation
said := map[string]bool{}
for _, d := range f.doing {
if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] {
continue
}
out = append(out, applyFailing(d, f.now))
said[d.Node] = true
}
// **A module's files the provisioner failed to place are a failed apply too** (novox/hq ADR 0362 §3): its client's
// newest word, judged by the same bound — the client says a failure once, and fills again on every change —
// on a machine whose node-engine applied what it was sent.
failed := map[string][]inventory.ProvReport{}
for _, r := range f.provFailed {
if r.Status != inventory.ProvFailed || f.now.Sub(r.At) <= applyEvery || !heard[r.Node] || said[r.Node] {
continue
}
failed[r.Node] = append(failed[r.Node], r)
}
for _, node := range sortedNames(failed) {
out = append(out, provisioningFailing(node, failed[node], f.now))
}
return out
}
// provisioningFailing is one machine's S21 condition for the modules whose files its provisioner client failed to
// place.
func provisioningFailing(m string, failed []inventory.ProvReport, now time.Time) conditions.Observation {
sort.Slice(failed, func(a, b int) bool { return failed[a].Module < failed[b].Module })
var names, words []string
since := failed[0].At
for _, r := range failed {
names = append(names, r.Module)
words = append(words, r.Module+": "+oneLine(orNoReason(r.Reason)))
if r.At.Before(since) {
since = r.At
}
}
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m,
Severity: conditions.Warning, Said: strings.Join(words, "; "),
Resolved: m + " applies what it is sent again"}
o.Summary = fmt.Sprintf("%s's provisioner has failed to place the files of %s since %s (bound %s)", m,
strings.Join(names, ", "), since.UTC().Format("2006-01-02 15:04 MST"), ago(applyEvery))
o.Headline = m + " is not applying its update"
o.Explanation = fmt.Sprintf("The provisioner on %s has not been able to write the files of %s for %s. The "+
"details say why.", m, strings.Join(names, ", "), humanDuration(now.Sub(since)))
return o
}
func orNoReason(s string) string {
if s == "" {
return "no reason given"
}
return s
}
// applyFailing is one machine's S21 condition.
func applyFailing(d inventory.Doing, now time.Time) conditions.Observation {
m := d.Node
cmd/mesh-controller/build.go
+35 -3
View File
@@ -183,8 +183,10 @@ func buildFrom(result link.BuildResult) inventory.Build {
kept.Manifest = recordedManifest(result.Manifest, announced)
// The module name comes from the manifest, which only exists when the build got that far.
if len(kept.Manifest) > 0 {
if m, err := catalogue.ParseManifest(kept.Manifest); err == nil {
kept.Module = m.Module
if list, err := judgedWith(result); err == nil {
if m, err := catalogue.ParseManifestWith(kept.Manifest, list); err == nil {
kept.Module = m.Module
}
}
}
return kept
@@ -603,6 +605,11 @@ func saysWhenThePolicyActs(ctx context.Context, inv *inventory.Inventory, module
// written with the same values.
func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResult) (
catalogue.Manifest, inventory.Build, error) {
// The list of predefined keys as the store keeps it now: another controller process may have kept a newer one.
if err := loadPredefined(ctx, inv); err != nil {
return catalogue.Manifest{}, inventory.Build{}, fmt.Errorf("the list of predefined keys kept cannot be "+
"read: %w", err)
}
kept := buildFrom(result)
if err := inv.RecordBuild(ctx, kept); err != nil {
return catalogue.Manifest{}, kept, err
@@ -613,7 +620,15 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
return catalogue.Manifest{}, kept, fmt.Errorf("%s could not build %s:\n%s",
result.On, result.Repository, result.Failed)
}
manifest, err := catalogue.ParseManifest(kept.Manifest)
// Judged first against the list the build was judged against: the catalogue's at the commit built (novox/hq
// ADR 0348 §7), so a module naming a key its own commit adds is read; and again below against the list the
// mesh publishes, before it is registered.
list, err := judgedWith(result)
if err != nil {
return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
result.Repository, short(result.Commit), err)
}
manifest, err := catalogue.ParseManifestWith(kept.Manifest, list)
if err != nil {
return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s and what came back is not a manifest: %w",
result.On, result.Repository, err)
@@ -687,6 +702,23 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
"back: it is recorded and not registered again — a newer build is", result.On, manifest.Module,
short(result.Commit))
}
// **The newest trunk build's list of predefined keys is published** (novox/hq ADR 0348 §7): a build of the
// catalogue that may be registered keeps its list when its commit is newer than the one kept. Then the module
// is judged against the list the mesh publishes, as every manifest read again is: one naming a key that list
// does not hold, or of another type, is refused here and never enters the catalogue.
if carried, err := carriedList(result); err != nil {
return manifest, kept, err
} else if carried != nil {
if err := publishPredefined(ctx, inv, result, carried); err != nil {
fmt.Printf("%s: the list of predefined keys it carried is not published: %v\n", result.ID, err)
}
}
if manifest.NamesPredefined() {
if _, err := catalogue.ParseManifest(kept.Manifest); err != nil {
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: judged against the "+
"list of predefined keys the mesh publishes, %w", result.On, manifest.Module, short(result.Commit), err)
}
}
// **A build whose source is unchanged is never a move** (novox/hq issue 280): a rebuild made from
// what the build the mesh stands on was made from registers that build's artifacts at the new
// commit, so no machine is sent a new digest for a source nobody changed — an image is not
cmd/mesh-controller/check.go
+36 -1
View File
@@ -47,6 +47,24 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
shelf := catalogue.Shelf{}
faulted := map[string]bool{}
failed := 0
// The list of predefined keys at the root of the repository above each manifest (novox/hq ADR 0348 §7), read
// once per repository: a manifest naming a key is judged against the list its own tree holds.
lists := map[string]catalogue.PredefinedList{}
listOf := func(path string) (catalogue.PredefinedList, error) {
root := catalogue.RepositoryRoot(filepath.Dir(path))
if root == "" {
return nil, nil
}
if list, read := lists[root]; read {
return list, nil
}
list, err := catalogue.ReadPredefined(root)
if err != nil {
return nil, err
}
lists[root] = list
return list, nil
}
for _, path := range paths {
raw, err := os.ReadFile(path)
if err != nil {
@@ -54,7 +72,18 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
failed++
continue
}
m, err := catalogue.ParseManifest(raw)
list, err := listOf(path)
if err != nil {
fmt.Fprintf(out, "%s: %v\n", path, err)
failed++
continue
}
m, err := catalogue.ParseManifestWith(raw, list)
if err == nil {
// Each template's text read from the tree beside the manifest, and judged as inline content is
// (novox/hq ADR 0349 §2): a missing template is refused by name.
m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(path))
}
if err != nil {
fmt.Fprintf(out, "%s: %v\n", path, err)
failed++
@@ -215,6 +244,12 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
checks = append(checks, fmt.Sprintf("%v by %s", r["id"], catalogue.HealthWords(h)))
}
}
// And its own runtime's, where it runs as its own account (novox/hq ADR 0366).
if own, has := catalogue.OwnRuntimeHealth(m); has {
if h, _, _ := catalogue.ReadHealth(own); h.Kind != "" {
checks = append(checks, fmt.Sprintf("%v by %s", own["id"], catalogue.HealthWords(h)))
}
}
if len(checks) > 0 {
fmt.Fprintf(out, ", ready: %s", strings.Join(checks, "; "))
}
cmd/mesh-controller/config_move.go
+616
View File
@@ -0,0 +1,616 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"os"
"reflect"
"sort"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// **The move of config** (novox/hq ADR 0348 §9 and its consequences, design 57 K5, issue 664): every config value a
// settings layer holds is set in mesh-config, one of the moves of data a release runs once after its delivery, after
// the move of placement. A module's layer on a node becomes its own value on that node (`module.<node>.<module>.<key>`).
// A whole-mesh layer has no module value for the whole mesh (ADR 0348 §3): its value becomes mesh config for the
// predefined key it falls back to where that module alone falls back to the key (claude-code's managed settings
// become `agent-managed-settings`), and otherwise the module's own value on each node it is assigned to, laid under
// that node's layer as composition lays it. A key a layer still says by its file's name (nodered's `timeZone`) is
// set under its config key (`time-zone`, issue 751).
//
// **Every node composes byte for byte as before.** The move sets the values, reads every module's resolved values
// back through mesh-config and compares them with what the layers lay today; only when they agree does it empty the
// layers it moved (each kept whole in its history first) and turn on the switch (inventory.FinishConfigMove), after
// which the controller reads mesh-config where it read the layers. A module still in the settings form keeps its
// layers and is said so. Anything that cannot move as it composes refuses the whole move, by name, before anything
// is set. Asked again, it sets nothing that is set already.
// configLevel is where mesh-config keeps a value: mesh config, node config, or a module's own value on a node.
type configLevel struct {
Level string `json:"level"`
Node string `json:"node,omitempty"`
Module string `json:"module,omitempty"`
}
func (l configLevel) valueKey(key string) string {
switch l.Level {
case "node":
return "node." + l.Node + "." + key
case "module":
return "module." + l.Node + "." + l.Module + "." + key
}
return "mesh." + key
}
func (l configLevel) String() string {
switch l.Level {
case "node":
return "node config on " + l.Node
case "module":
return l.Module + "'s own value on " + l.Node
}
return "mesh config"
}
// configSet is one value the move sets in mesh-config, and the layers it comes from.
type configSet struct {
configLevel
Key string `json:"key"`
Value any `json:"value"`
Trusted bool `json:"trusted,omitempty"`
// From is the layers it comes from, as `<module> on <where>: <key as the layer says it>`.
From []string `json:"from"`
// Already is true when mesh-config holds this value at this level already: nothing is set.
Already bool `json:"already,omitempty"`
}
// configMoveAnswer is what the move of config says it did, or would do.
type configMoveAnswer struct {
// DryRun is true when nothing was set: asked without --confirm.
DryRun bool `json:"dry-run"`
Sets []configSet `json:"sets"`
// Moved is each layer whose values the move sets, emptied once they read back.
Moved []string `json:"moved"`
// Stays is each layer of a module still in the settings form: kept, and composed from as before.
Stays []string `json:"stays,omitempty"`
// Unread is each layer that composes nothing (its module is not assigned where it says): kept in the history,
// emptied, and set nowhere.
Unread []string `json:"unread,omitempty"`
// Problems is why the move refuses, each by name; nothing is set while there is one.
Problems []string `json:"problems,omitempty"`
// Switched is true once the controller reads mesh-config where it read the layers.
Switched bool `json:"switched"`
}
// configMovePlan is the move worked out from the layers, before anything is asked of mesh-config.
type configMovePlan struct {
answer configMoveAnswer
// moved is the layers the move empties once the values read back.
moved []inventory.LayerToMove
// expected is, for each module the move reaches, keyed `<node>.<module>`, every value composition lays over
// its files today: what mesh-config must resolve for it from mesh config, node config or its own value.
expected map[string]map[string]any
// modules is each module in the config form, by the nodes it is assigned to: whose resolved values are read back.
modules map[string][]string
}
func whereOf(node string) string {
if node == "" {
return "the whole mesh"
}
return node
}
// planConfigMove works out the move from every layer that holds a value, the catalogue, and where each module is
// assigned.
func planConfigMove(layers []inventory.LayerToMove, shelf map[string]catalogue.Manifest,
on map[string][]string) configMovePlan {
p := configMovePlan{answer: configMoveAnswer{Sets: []configSet{}, Moved: []string{}},
expected: map[string]map[string]any{}, modules: map[string][]string{}}
problem := func(format string, a ...any) {
p.answer.Problems = append(p.answer.Problems, fmt.Sprintf(format, a...))
}
for name, m := range shelf {
if m.DeclaresConfig() {
p.modules[name] = on[name]
}
}
byModule := map[string]*layersOfModule{}
var names []string
for _, l := range layers {
said := fmt.Sprintf("%s on %s", l.Module, whereOf(l.Node))
m, known := shelf[l.Module]
if !known || !m.DeclaresConfig() {
p.answer.Stays = append(p.answer.Stays, said+" (its module is in the settings form)")
continue
}
if _, placement := catalogue.SplitPlacement(l.Values); len(placement) > 0 {
problem("%s still holds placement (%s): run move-placement first", said, strings.Join(movedKeys(placement), ", "))
continue
}
assigned := on[l.Module]
if (l.Node == "" && len(assigned) == 0) || (l.Node != "" && !containsString(assigned, l.Node)) {
p.answer.Unread = append(p.answer.Unread, said+" (the module is not assigned there, so it composes nothing)")
p.moved = append(p.moved, l)
continue
}
// The layer's keys as config keys: a key the file names otherwise is set under its config key.
as := catalogue.TakenAs(m)
values := map[string]any{}
from := map[string]string{}
for _, k := range movedKeys(l.Values) {
key := k
if _, declared := m.Settings[k]; !declared {
if renamed, ok := as[k]; ok {
key = renamed
} else {
problem("%s says %q, which %s does not declare: declare it, or clear it from the layer", said, k, l.Module)
continue
}
}
if _, both := values[key]; both {
// The layer says the key and the file's name for it: the config key wins, as composition has it (issue 751).
if key != k {
continue
}
}
values[key] = l.Values[k]
from[key] = said + ": " + k
}
om := byModule[l.Module]
if om == nil {
om = &layersOfModule{nodes: map[string]map[string]any{}, saidAt: map[string]map[string]string{}}
byModule[l.Module] = om
names = append(names, l.Module)
}
om.saidAt[l.Node] = from
if l.Node == "" {
om.mesh = values
} else {
om.nodes[l.Node] = values
}
p.moved = append(p.moved, l)
p.answer.Moved = append(p.answer.Moved, said)
}
sort.Strings(names)
// Which predefined key each module falls back to, so a whole-mesh value goes to mesh config only where no other
// module's value moves with it.
fallingBack := map[string][]string{}
for name, m := range shelf {
for _, d := range m.Settings {
if d.Predefined != "" && !containsString(fallingBack[d.Predefined], name) {
fallingBack[d.Predefined] = append(fallingBack[d.Predefined], name)
}
}
}
for _, name := range names {
m, om := shelf[name], byModule[name]
asked := catalogue.AskedKeys(m)
keys := map[string]bool{}
for k := range om.mesh {
keys[k] = true
}
for _, values := range om.nodes {
for k := range values {
keys[k] = true
}
}
for _, key := range movedKeys(keys) {
d := m.Settings[key]
meshValue, meshSays := om.mesh[key]
alone := d.Predefined != "" && len(fallingBack[d.Predefined]) == 1
if meshSays && alone {
p.set(configLevel{Level: "mesh"}, d.Predefined, meshValue, d, om.said("", key))
}
for _, node := range on[name] {
nodeValue, nodeSays := om.nodes[node][key]
value := meshValue
switch {
case nodeSays && meshSays:
_, under := meshValue.(map[string]any)
_, over := nodeValue.(map[string]any)
if under && over && asked[key] {
problem("%s on %s lays an object for %q over the whole mesh's, and a placeholder takes it whole "+
"where its file merges the two: say the value on %s whole, then move", name, node, key, node)
continue
}
value = catalogue.LaidOver(meshValue, nodeValue)
case nodeSays:
value = nodeValue
case !meshSays:
continue
}
p.expect(node, name, key, value)
if nodeSays || !alone {
saidBy := om.said(node, key)
if meshSays {
saidBy = append(om.said("", key), saidBy...)
}
p.set(configLevel{Level: "module", Node: node, Module: name}, key, value, d, saidBy)
}
}
}
}
sort.Strings(p.answer.Problems)
return p
}
// layersOfModule is one module's layers as the move reads them: its whole-mesh layer's config values, each node's,
// and where each key was said, by node ("" for the whole mesh).
type layersOfModule struct {
mesh map[string]any
nodes map[string]map[string]any
saidAt map[string]map[string]string
}
func (om *layersOfModule) said(node, key string) []string {
if s, ok := om.saidAt[node][key]; ok {
return []string{s}
}
return nil
}
// set is one value the move sets, judged against its declaration as mesh-config will judge it.
func (p *configMovePlan) set(level configLevel, key string, value any, d catalogue.SettingDeclaration, from []string) {
p.answer.Sets = append(p.answer.Sets, configSet{configLevel: level, Key: key, Value: value, Trusted: d.Trusted,
From: from})
for _, wrong := range catalogue.ValueProblems(d, value) {
p.answer.Problems = append(p.answer.Problems, fmt.Sprintf("%s at %s: %s", key, level, wrong))
}
}
func (p *configMovePlan) expect(node, module, key string, value any) {
k := node + "." + module
if p.expected[k] == nil {
p.expected[k] = map[string]any{}
}
p.expected[k][key] = value
}
func movedKeys[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
// configService is mesh-config as the move asks it: what it holds at each level, what it resolved, and a value set
// as the controller carries it from its terminal.
type configService interface {
// Values is every value set, by its key in mesh-config_values (`mesh.<key>`, `node.<node>.<key>`,
// `module.<node>.<module>.<key>`).
Values(ctx context.Context) (map[string]json.RawMessage, error)
// Resolved is every resolved value.
Resolved(ctx context.Context) ([]resolvedEntry, error)
// Set sets one value at its level, by whom and why.
Set(ctx context.Context, level configLevel, key string, value any, by, why string) error
}
// runConfigMove is the move against a store and mesh-config: a dry run says what it would set.
func runConfigMove(ctx context.Context, inv *inventory.Inventory, config configService, apply bool, by,
why string) (configMoveAnswer, error) {
layers, holding, err := inv.LayersToMove(ctx)
if err != nil {
return configMoveAnswer{}, err
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return configMoveAnswer{}, err
}
shelf := map[string]catalogue.Manifest{}
on := map[string][]string{}
for _, e := range entries {
shelf[e.Manifest.Module] = e.Manifest
on[e.Manifest.Module] = e.On
}
p := planConfigMove(layers, shelf, on)
if len(holding) > 0 && len(p.answer.Problems) == 0 {
p.answer.Problems = append(p.answer.Problems, "a layer still holds placement ("+strings.Join(holding, "; ")+
"): run move-placement first")
}
moved, err := inv.ConfigMoved(ctx)
if err != nil {
return configMoveAnswer{}, err
}
p.answer.Switched = moved.Moved
if config == nil {
return p.answer, errors.New("mesh-config cannot be asked, so nothing was moved")
}
held, err := config.Values(ctx)
if err != nil {
return p.answer, fmt.Errorf("what mesh-config holds cannot be read, so nothing was moved: %w", err)
}
for i := range p.answer.Sets {
s := &p.answer.Sets[i]
if was, ok := held[s.valueKey(s.Key)]; ok && sameJSON(was, s.Value) {
s.Already = true
}
}
p.answer.DryRun = !apply
if !apply || len(p.answer.Problems) > 0 {
if apply {
return p.answer, fmt.Errorf("the move of config refuses, and nothing was set: %s",
strings.Join(p.answer.Problems, "; "))
}
return p.answer, nil
}
if len(p.moved) == 0 && moved.Moved {
return p.answer, nil
}
for _, s := range p.answer.Sets {
if s.Already {
continue
}
said := "moved from the settings layer of " + strings.Join(s.From, " and ") + " (issue 664): " + why
if err := config.Set(ctx, s.configLevel, s.Key, s.Value, by, said); err != nil {
return p.answer, fmt.Errorf("%s at %s was not set, and no layer was emptied; ask the move again: %w",
s.Key, s.configLevel, err)
}
}
// Read back through mesh-config before anything points at it (design 57): every module the move reaches, and
// every other in the config form, must resolve exactly what its layers lay today.
if differs, err := readsBack(ctx, config, p); err != nil {
return p.answer, err
} else if len(differs) > 0 {
return p.answer, fmt.Errorf("mesh-config does not resolve what the layers lay today, so no layer was emptied and "+
"the controller still composes from the layers: %s", strings.Join(differs, "; "))
}
if err := inv.FinishConfigMove(ctx, p.moved, by, why); err != nil {
return p.answer, err
}
p.answer.Switched = true
return p.answer, nil
}
// readsBack compares what mesh-config resolved with what the layers lay, for every module in the config form on
// every node it is assigned to, and says each difference. A value the layers do not lay and mesh-config resolves from
// a level would be laid once the switch is on: a difference too.
func readsBack(ctx context.Context, config configService, p configMovePlan) ([]string, error) {
var resolved []resolvedEntry
var err error
// mesh-config writes the resolved values with each set; a pass that missed one is put right by its next
// resolution, so a difference is read again for a little while before it is said.
for wait := 0; ; wait++ {
if resolved, err = config.Resolved(ctx); err != nil {
return nil, fmt.Errorf("mesh-config's resolved values cannot be read back, so no layer was emptied: %w", err)
}
differs, err := differences(resolved, p)
if err != nil || len(differs) == 0 || wait >= readBackTries {
return differs, err
}
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(readBackEvery):
}
}
}
// How often, and how many times, the move reads mesh-config's resolved values back before it says they differ.
var (
readBackEvery = 2 * time.Second
readBackTries = 15
)
func differences(resolved []resolvedEntry, p configMovePlan) ([]string, error) {
var differs []string
for _, module := range movedKeys(p.modules) {
for _, node := range p.modules[module] {
laid, err := configLaid(resolved, node, module)
if err != nil {
return nil, err
}
want := p.expected[node+"."+module]
if want == nil {
want = map[string]any{}
}
for _, k := range movedKeys(mergedKeys(laid, want)) {
got, has := laid[k]
w, wants := want[k]
switch {
case !wants:
differs = append(differs, fmt.Sprintf("%s on %s resolves %s from a level no layer says; clear it "+
"there, or set it in a layer first", module, node, k))
case !has:
differs = append(differs, fmt.Sprintf("%s on %s does not resolve %s from the value moved", module, node, k))
case !reflect.DeepEqual(normalJSON(got), normalJSON(w)):
differs = append(differs, fmt.Sprintf("%s on %s resolves %s as %s, and its layers lay %s", module,
node, k, compactJSON(got), compactJSON(w)))
}
}
}
}
return differs, nil
}
func mergedKeys(a, b map[string]any) map[string]bool {
out := map[string]bool{}
for k := range a {
out[k] = true
}
for k := range b {
out[k] = true
}
return out
}
func normalJSON(v any) any {
raw, _ := json.Marshal(v)
var out any
_ = json.Unmarshal(raw, &out)
return out
}
func compactJSON(v any) string {
raw, _ := json.Marshal(v)
return string(raw)
}
func sameJSON(raw json.RawMessage, v any) bool {
var held any
if err := json.Unmarshal(raw, &held); err != nil {
return false
}
return reflect.DeepEqual(held, normalJSON(v))
}
// busConfig is mesh-config on this controller's bus: its buckets read directly, a value set through its verb in
// the controller's own name, so mesh-config takes a trusted value as given at the controller's terminal (ADR 0348 §8).
type busConfig struct{}
func (busConfig) Values(ctx context.Context) (map[string]json.RawMessage, error) {
out := map[string]json.RawMessage{}
err := onTheBus(func(conn *nats.Conn) error {
js, err := jetstream.New(conn)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
defer cancel()
kv, err := js.KeyValue(ctx, broker.BucketName("mesh-config", "values"))
if err != nil {
return err
}
w, err := kv.WatchAll(ctx, jetstream.IgnoreDeletes())
if err != nil {
return err
}
defer func() { _ = w.Stop() }()
for e := range w.Updates() {
if e == nil {
return nil
}
var v struct {
Value json.RawMessage `json:"value"`
}
if err := json.Unmarshal(e.Value(), &v); err != nil {
return fmt.Errorf("mesh-config holds %q, which is not a value: %w", e.Key(), err)
}
out[e.Key()] = v.Value
}
return errors.New("mesh-config's values stopped answering")
})
return out, err
}
func (busConfig) Resolved(ctx context.Context) ([]resolvedEntry, error) { return readResolved(ctx) }
func (busConfig) Set(ctx context.Context, level configLevel, key string, value any, by, why string) error {
args := map[string]any{"action": "set", "key": key, "value": value, "why": why, "by": by}
if level.Node != "" {
args["node"] = level.Node
}
if level.Module != "" {
args["module"] = level.Module
}
return onTheBus(func(conn *nats.Conn) error {
answer, err := link.AskMeshSeatToolAs(ctx, conn, broker.ControllerName, "mesh-config", "config", args, 30*time.Second)
if err != nil {
return err
}
if answer.Error != "" {
return errors.New(answer.Error)
}
return nil
})
}
// moveConfigCommand is `move-config`: the move of config (issue 664).
func moveConfigCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("move-config", flag.ContinueOnError)
asJSON := set.Bool("json", false, "answer as JSON")
confirm := set.Bool("confirm", false, "set the values and switch, rather than only say what would move")
f := addHandActFlags(set)
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if len(positionals) != 0 {
return errors.New("move-config [--json] [--confirm --why <text>]")
}
if !*confirm && f.given() {
return errors.New("move-config takes --why only with --confirm: without it nothing is moved. Nothing was done")
}
if *confirm {
if err := f.require("move-config"); err != nil {
return err
}
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
if *confirm {
f.record(ctx, "move-config", nil)
}
why := ""
if *confirm {
why = strings.TrimSpace(*f.why)
}
answer, err := runConfigMove(ctx, open.inventory, busConfig{}, *confirm, "the move of config, run by "+link.Caller(), why)
if *asJSON {
encoder := json.NewEncoder(os.Stdout)
encoder.SetIndent("", " ")
if encErr := encoder.Encode(answer); encErr != nil {
return encErr
}
return err
}
fmt.Print(describeConfigMove(answer))
return err
}
// describeConfigMove says the answer as a person reads it.
func describeConfigMove(a configMoveAnswer) string {
var b strings.Builder
verb := "set"
if a.DryRun {
verb = "would set"
}
for _, s := range a.Sets {
state := verb
if s.Already {
state = "holds already"
}
fmt.Fprintf(&b, "%s %s = %s at %s (from %s)\n", state, s.Key, compactJSON(s.Value), s.configLevel,
strings.Join(s.From, " and "))
}
for _, s := range a.Unread {
fmt.Fprintf(&b, "emptied, set nowhere: %s\n", s)
}
for _, s := range a.Stays {
fmt.Fprintf(&b, "stays: %s\n", s)
}
for _, s := range a.Problems {
fmt.Fprintf(&b, "refused: %s\n", s)
}
switch {
case len(a.Problems) > 0:
b.WriteString("nothing was set: the move refuses until each is put right\n")
case a.DryRun:
fmt.Fprintf(&b, "%d layer(s), %d value(s); nothing was set — with --confirm --why <text> they are\n",
len(a.Moved), len(a.Sets))
case a.Switched:
fmt.Fprintf(&b, "%d layer(s) moved into mesh-config, each kept whole in its history first; the controller "+
"reads mesh-config where it read them\n", len(a.Moved))
}
return b.String()
}
cmd/mesh-controller/config_moved_test.go
+785
View File
@@ -0,0 +1,785 @@
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"reflect"
"sort"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
snapshot "github.com/novox/mesh-controller/internal/facts"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/overlay"
)
// configuredMesh is every live settings layer (issue 664) — read with the settings verb on 2026-10-15, on the four
// machines as anchor, laptop, workstation and desktop, the whole mesh's with no node; personal values stood in for,
// and the placement words left out, which move-placement moves first — with each module in the config form shaped
// as its definition on the feature branch: its config, its files (templates inlined) and directories, placeholders
// of kinds this test's mesh does not hold stood in for. A module still in the settings form (the media catalogue's,
// and kernel) stands in as a mergeable file that takes every key. `on` is where each is assigned, as live.
type configuredMesh struct {
Modules []json.RawMessage `json:"modules"`
Layers []configuredLayer `json:"layers"`
On map[string][]string `json:"on"`
// Predefined is the catalogue's list of predefined keys, which a module falls back to.
Predefined json.RawMessage `json:"predefined"`
}
type configuredLayer struct {
Node string `json:"node"`
Module string `json:"module"`
Values map[string]any `json:"values"`
}
var fourMachines = []string{"anchor", "laptop", "workstation", "desktop"}
// aConfiguredMesh is aMesh with two machines more, every module of the configured mesh registered and assigned, and
// its layers held as the store holds them once move-placement has run.
func aConfiguredMesh(t *testing.T) (*stores, configuredMesh, map[string]map[string]any) {
t.Helper()
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
for i, name := range []string{"workstation", "desktop"} {
record, err := inv.AddNode(ctx, name)
if err != nil {
t.Fatal(err)
}
if err := inv.SetPlace(ctx, name, name+".example:51820", "here", false, fmt.Sprintf("10.77.0.%d", i+3)); err != nil {
t.Fatal(err)
}
if err := inv.RecordProfile(ctx, record.ID, map[string]any{"capabilities": []any{
map[string]any{"name": "container-runtime", "present": true},
map[string]any{"name": "systemd", "present": true}}}); err != nil {
t.Fatal(err)
}
if err := inv.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
t.Fatal(err)
}
if err := inv.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil {
t.Fatal(err)
}
if _, err := inv.Assign(ctx, name, overlay.Name); err != nil {
t.Fatal(err)
}
}
raw, err := os.ReadFile(filepath.Join("testdata", "config-moved", "mesh.json"))
if err != nil {
t.Fatal(err)
}
var w configuredMesh
if err := json.Unmarshal(raw, &w); err != nil {
t.Fatal(err)
}
list, err := catalogue.ParsePredefined(w.Predefined)
if err != nil {
t.Fatal(err)
}
catalogue.UsePredefined(list)
served := map[string]map[string]any{}
for _, text := range w.Modules {
var shape map[string]json.RawMessage
if err := json.Unmarshal(text, &shape); err != nil {
t.Fatal(err)
}
var name string
_ = json.Unmarshal(shape["module"], &name)
if s, ok := shape["serves-checked"]; ok {
var serves map[string]any
if err := json.Unmarshal(s, &serves); err != nil {
t.Fatal(err)
}
served[name] = serves
delete(shape, "serves-checked")
}
text, err := json.Marshal(shape)
if err != nil {
t.Fatal(err)
}
m, err := catalogue.ParseManifest(text)
if err != nil {
t.Fatalf("%s: %v", name, err)
}
register(t, open, m)
}
for _, l := range w.Layers {
values, err := json.Marshal(l.Values)
if err != nil {
t.Fatal(err)
}
if l.Node == "" {
inv.ExecForTest(t, `insert into settings (node, module, values) values (null, $1, $2)`, l.Module, string(values))
continue
}
inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`,
l.Node, l.Module, string(values))
}
for _, module := range movedKeys(w.On) {
for _, node := range w.On[module] {
if _, err := assign(ctx, open, node, module); err != nil {
t.Fatalf("assign %s %s: %v", node, module, err)
}
}
}
return open, w, served
}
// standInMeshConfig is mesh-config as this test holds it: each value at its level, and resolved from the catalogue
// the store holds as mesh-config resolves (novox/hq ADR 0348 §2): the module's own value on the node, else node
// config then mesh config for the predefined key it falls back to, else its default; a required value nothing sets is
// unset.
type standInMeshConfig struct {
open *stores
values map[string]any
sets []string
// asked is each call of its verb: the action and the level's key.
asked []string
}
func (c *standInMeshConfig) Values(context.Context) (map[string]json.RawMessage, error) {
out := map[string]json.RawMessage{}
for k, v := range c.values {
raw, err := json.Marshal(v)
if err != nil {
return nil, err
}
out[k] = raw
}
return out, nil
}
func (c *standInMeshConfig) Set(_ context.Context, level configLevel, key string, value any, by, why string) error {
if by == "" || why == "" {
return fmt.Errorf("%s at %s is set without who or why", key, level)
}
c.values[level.valueKey(key)] = normalJSON(value)
c.sets = append(c.sets, level.valueKey(key))
return nil
}
func (c *standInMeshConfig) Resolved(ctx context.Context) ([]resolvedEntry, error) {
entries, err := c.open.inventory.Catalogued(ctx)
if err != nil {
return nil, err
}
var out []resolvedEntry
for _, e := range entries {
m := e.Manifest
if !m.DeclaresConfig() {
continue
}
for _, node := range e.On {
for _, key := range movedKeys(m.Settings) {
d := m.Settings[key]
r := resolvedEntry{Node: node, Module: m.Module, Key: key, Source: "default"}
value, has := c.values["module."+node+"."+m.Module+"."+key]
switch {
case has:
r.Source = "module"
case d.Predefined != "" && c.values["node."+node+"."+d.Predefined] != nil:
value, r.Source = c.values["node."+node+"."+d.Predefined], "node"
case d.Predefined != "" && c.values["mesh."+d.Predefined] != nil:
value, r.Source = c.values["mesh."+d.Predefined], "mesh"
case d.Required:
r.Source = "unset"
default:
value = d.Default
}
if r.Source != "unset" {
if r.Value, err = json.Marshal(value); err != nil {
return nil, err
}
}
out = append(out, r)
}
}
}
return out, nil
}
// laid is the stand-in as composition reads it once the switch is on (inventory.ConfigValues).
func (c *standInMeshConfig) laid(ctx context.Context, node, module string) (map[string]any, error) {
resolved, err := c.Resolved(ctx)
if err != nil {
return nil, err
}
return configLaid(resolved, node, module)
}
// snapshotFacts is the facts snapshot as gatherFacts gives it.
type snapshotFacts = snapshot.Facts
// toldEveryMachine is what each of the four machines is told — every resource and the filter's rules — and what
// each module that serves something serves, settled from its layers, as JSON a person can diff.
func toldEveryMachine(t *testing.T, open *stores, served map[string]map[string]any, on map[string][]string) map[string]string {
t.Helper()
ctx := t.Context()
gens, err := generators(ctx, open)
if err != nil {
t.Fatal(err)
}
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
t.Fatal(err)
}
told := map[string]string{}
for _, node := range fourMachines {
plan, settings, err := planFor(ctx, open, node)
if err != nil {
t.Fatalf("%s: %v", node, err)
}
with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Allocating)
if err != nil {
t.Fatalf("%s: %v", node, err)
}
composed, err := plan.Compose(with)
if err != nil {
t.Fatalf("%s: %v", node, err)
}
rules, err := plan.Rules(with)
if err != nil {
t.Fatalf("%s: %v", node, err)
}
settles := map[string]any{}
for module, serves := range served {
if !containsString(on[module], node) {
continue
}
layers, err := open.inventory.SettingsFor(ctx, node, module)
if err != nil {
t.Fatal(err)
}
for name, s := range serves {
fact, _ := s.(map[string]any)
settled, err := catalogue.Settle(fact, catalogue.WithDefaults(shelf[module], layers))
if err != nil {
settles[module+" serves "+name] = "error: " + err.Error()
continue
}
settles[module+" serves "+name] = settled
}
}
got, err := json.MarshalIndent(map[string]any{"resources": composed.Resources, "left-out": composed.LeftOut,
"rules": rules, "served": settles}, "", " ")
if err != nil {
t.Fatal(err)
}
told[node] = string(got)
}
return told
}
// effectiveEverywhere is each module's every value on each machine it is assigned to, as `settings show` and
// `settings preferences` say them (catalogue.Effective): the value, never where it came from, which the move
// changes by design; a key a layer says by its file's name is read as its config key.
func effectiveEverywhere(t *testing.T, open *stores, w configuredMesh) map[string]any {
t.Helper()
ctx := t.Context()
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
t.Fatal(err)
}
out := map[string]any{}
for _, module := range movedKeys(w.On) {
if !shelf[module].DeclaresConfig() {
continue
}
for _, node := range w.On[module] {
layers, err := open.inventory.SettingsFor(ctx, node, module)
if err != nil {
t.Fatal(err)
}
// A layer's old spelling of a key (nodered's timeZone) is listed beside the key it is the file's name
// for, and reaches the file as that key does (issue 751): read as that key, which the move sets.
as := catalogue.TakenAs(shelf[module])
for _, v := range catalogue.Effective(shelf[module], layers) {
if key, old := as[v.Key]; old {
out[node+"."+module+"."+key] = normalJSON(v.Value)
continue
}
if _, said := out[node+"."+module+"."+v.Key]; said && v.FromDefault {
continue
}
out[node+"."+module+"."+v.Key] = normalJSON(v.Value)
}
}
}
return out
}
func setAt(a configMoveAnswer, key string) []configSet {
var out []configSet
for _, s := range a.Sets {
if s.valueKey(s.Key) == key {
out = append(out, s)
}
}
return out
}
// **The settings layers move into mesh-config and no machine is told anything new** (novox/hq ADR 0348 §9, design
// 57 K5, issue 664). Every live layer is seeded as the store holds it once move-placement has run, on modules shaped
// as their definitions, and every machine composed. The move, asked without confirming, says each value it would set
// and where from, and sets nothing; confirmed, it sets them in mesh-config, reads them back, empties the layers it
// moved and switches. Each machine is then told byte for byte what it was told before, every module serves the same,
// and `settings` lists the same values `config` resolves. Asked again, it sets nothing.
func TestEveryMachineIsToldTheSameAfterTheMoveOfConfig(t *testing.T) {
open, w, served := aConfiguredMesh(t)
ctx := t.Context()
readBackTries = 0
t.Cleanup(func() { readBackTries = 15 })
inv := open.inventory
config := &standInMeshConfig{open: open, values: map[string]any{}}
inv.ReadsConfigFrom(config.laid)
before := toldEveryMachine(t, open, served, w.On)
listedBefore := effectiveEverywhere(t, open, w)
factsBefore, err := gatherFacts(ctx, open, "2.11.17")
if err != nil {
t.Fatal(err)
}
asked, err := runConfigMove(ctx, inv, config, false, "the move of config, run by a test", "")
if err != nil || !asked.DryRun || asked.Switched || len(asked.Problems) > 0 || len(config.sets) > 0 {
t.Fatalf("the move, not confirmed, says %+v (%v) and set %v", asked, err, config.sets)
}
// nodered's live layer says the file's name for its time zone: set under the config key (issue 751).
if s := setAt(asked, "module.anchor.nodered.time-zone"); len(s) != 1 || s[0].Value != "Europe/Brussels" ||
!strings.HasSuffix(s[0].From[0], ": timeZone") {
t.Errorf("nodered's time zone moves as %+v", s)
}
for _, s := range asked.Sets {
if s.Key == "timeZone" {
t.Errorf("a key is set under the file's name: %+v", s)
}
}
// claude-code's whole-mesh managed settings become mesh config for agent-managed-settings (ADR 0348's
// consequences), and its whole-mesh layer sets no module value.
if s := setAt(asked, "mesh.agent-managed-settings"); len(s) != 1 || s[0].From[0] != "claude-code on the whole mesh: managed_settings" {
t.Errorf("claude-code's managed settings move as %+v", s)
}
for _, node := range fourMachines {
if s := setAt(asked, "module."+node+".claude-code.managed_settings"); len(s) > 0 {
t.Errorf("claude-code's managed settings are also set on %s: %+v", node, s)
}
}
// A whole-mesh layer of a value no predefined key holds goes to each machine it is assigned to, under that
// machine's layer: power's on the workstation is the workstation's, its own elsewhere.
if s := setAt(asked, "module.workstation.power.handle-power-key"); len(s) != 1 || s[0].Value != "suspend" {
t.Errorf("power's key on the workstation moves as %+v", s)
}
if s := setAt(asked, "module.anchor.power.handle-power-key"); len(s) != 1 || s[0].Value != "poweroff" {
t.Errorf("power's key on the anchor moves as %+v", s)
}
for _, s := range asked.Sets {
if s.Level == "node" {
t.Errorf("a machine's layer became node config: %+v", s)
}
}
var stays []string
for _, s := range asked.Stays {
stays = append(stays, strings.Fields(s)[0])
}
sort.Strings(stays)
if want := []string{"bazarr", "bazarr", "bookshelf", "jackett", "kernel", "kometa", "lidarr", "nzbget", "ombi",
"plex", "qbittorrent", "radarr", "sonarr", "tautulli"}; len(stays) < 10 || !containsString(stays, "kernel") {
t.Errorf("the layers that stay are %v, want those of %v", stays, want)
}
if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) {
t.Fatal("a dry run changed what a machine is told")
}
moved, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "release 57 is delivered")
if err != nil || moved.DryRun || !moved.Switched {
t.Fatalf("the move says %+v (%v)", moved, err)
}
if len(config.sets) != len(asked.Sets) {
t.Fatalf("the move set %d values, and said it would set %d", len(config.sets), len(asked.Sets))
}
if held := inv.QueryForTest(t, `select module from settings s where values <> '{}'
and module in (select name from module where manifest ? 'config')`); len(held) != 0 {
t.Fatalf("a layer of a module in the config form still holds values: %v", held)
}
after := toldEveryMachine(t, open, served, w.On)
for _, node := range fourMachines {
if after[node] != before[node] {
t.Errorf("%s is told something else after the move:\n%s\nwas:\n%s", node, after[node], before[node])
}
}
if listedAfter := effectiveEverywhere(t, open, w); !reflect.DeepEqual(listedAfter, listedBefore) {
t.Errorf("settings lists other values after the move:\n%v\nwas:\n%v", listedAfter, listedBefore)
}
// And config lists what settings listed: each module's resolved value on each machine is the value settings
// said before the move.
resolved, err := config.Resolved(ctx)
if err != nil {
t.Fatal(err)
}
for _, r := range resolved {
said, listed := listedBefore[r.Node+"."+r.Module+"."+r.Key]
if r.Source == "unset" {
if listed && said != nil {
t.Errorf("%s of %s on %s is unset in config, and settings said %v", r.Key, r.Module, r.Node, said)
}
continue
}
var v any
_ = json.Unmarshal(r.Value, &v)
if !listed || !reflect.DeepEqual(v, said) {
t.Errorf("config resolves %s of %s on %s as %v, and settings said %v", r.Key, r.Module, r.Node, v, said)
}
}
// The facts snapshot carries mesh-config's values for the merge gate, as each machine's layer of each module, and
// each machine's declaration composes to the digest it did before the move.
factsAfter, err := gatherFacts(ctx, open, "2.11.17")
if err != nil {
t.Fatal(err)
}
// Machines are named in a snapshot by stand-ins of its own; each is found by what it composes.
digests := func(f snapshotFacts) ([]string, string) {
var out []string
zone := ""
for _, m := range f.Machines {
out = append(out, m.Declaration.Digest)
for _, l := range m.Settings {
if l.Module == "nodered" {
zone = compactJSON(l.Values["time-zone"])
}
}
}
sort.Strings(out)
return out, zone
}
was, _ := digests(factsBefore)
now, zone := digests(factsAfter)
if zone != `"Europe/Brussels"` {
t.Errorf("the facts snapshot does not carry nodered's time zone from mesh-config: %s", zone)
}
if len(now) != len(fourMachines) || now[0] == "" || !reflect.DeepEqual(now, was) {
t.Errorf("the declarations in the facts snapshot are %v after the move, %v before", now, was)
}
for _, s := range factsAfter.Settings {
if _, isConfig := map[string]bool{"betterbird": true, "claude-code": true, "power": true, "xdg": true}[s.Module]; isConfig &&
len(s.Values) > 0 {
t.Errorf("the facts snapshot still carries the whole mesh's layer of %s: %v", s.Module, s.Values)
}
}
config.sets = nil
again, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "again")
if err != nil || len(config.sets) != 0 || len(again.Moved) != 0 {
t.Fatalf("asked again, the move set %v and says %+v (%v)", config.sets, again, err)
}
if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) {
t.Fatal("the move asked again changed what a machine is told")
}
}
// **The move refuses what it cannot move as it composes, and sets nothing** (issue 664): a layer that still holds
// placement (move-placement has not run), a key its module does not declare, and a value its declaration refuses —
// each said by name; and a value mesh-config resolves from a level no layer says (it would be laid once the switch
// is on) leaves every layer and the switch as they were.
func TestTheMoveOfConfigRefusesWhatWouldChangeAMachine(t *testing.T) {
open, _, _ := aConfiguredMesh(t)
ctx := t.Context()
readBackTries = 0
t.Cleanup(func() { readBackTries = 15 })
inv := open.inventory
config := &standInMeshConfig{open: open, values: map[string]any{}}
inv.ReadsConfigFrom(config.laid)
inv.ExecForTest(t, `update settings set values = values || '{"endpoints": {"web": {"reach": "public"}}}'
where module = 'nodered'`)
inv.ExecForTest(t, `update settings set values = values || '{"colour": "blue"}' where module = 'unifi'`)
inv.ExecForTest(t, `update settings set values = '{"font-size": 900}' where module = 'dunst'`)
answer, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered")
if err == nil || answer.Switched || len(config.sets) > 0 {
t.Fatalf("the move went ahead: %+v (%v), set %v", answer, err, config.sets)
}
for _, want := range []string{"nodered on anchor still holds placement (endpoints): run move-placement first",
`unifi on anchor says "colour", which unifi does not declare`, "font-size at dunst's own value on workstation"} {
if !strings.Contains(err.Error(), want) {
t.Errorf("the refusal does not say %q: %v", want, err)
}
}
inv.ExecForTest(t, `update settings set values = values - 'endpoints' where module = 'nodered'`)
inv.ExecForTest(t, `update settings set values = values - 'colour' where module = 'unifi'`)
inv.ExecForTest(t, `update settings set values = '{"font-size": 13, "width": 500}' where module = 'dunst'`)
// Set in mesh-config before the move, on a module whose layers say nothing of it: the switch would lay it.
config.values["module.anchor.grafana."+movedKeys(func() map[string]catalogue.SettingDeclaration {
shelf, _ := inv.Catalogue(ctx)
return shelf["grafana"].Settings
}())[0]] = "set before the move"
answer, err = runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered")
if err == nil || !strings.Contains(err.Error(), "grafana on anchor resolves") || answer.Switched {
t.Fatalf("a value no layer says was laid by the switch: %+v (%v)", answer, err)
}
if moved, _ := inv.ConfigMoved(ctx); moved.Moved {
t.Fatal("the switch is on after a refused move")
}
if held := inv.QueryForTest(t, `select count(*) from settings where values <> '{}'`); held[0][0] == "0" {
t.Fatal("a refused move emptied the layers")
}
}
// The verb's command line: a dry run unless confirmed, and a confirmed move says why, recorded as a person's step of
// the release, never a repair (issue 664).
func TestTheMoveOfConfigIsADryRunUnlessConfirmed(t *testing.T) {
for _, c := range []struct {
args map[string]any
want string
}{
{map[string]any{}, "move-config --json"},
{map[string]any{"confirm": "true", "why": "release 57 is delivered"}, "move-config --json --confirm --why release 57 is delivered"},
} {
argv, err := argvFor("move-config", c.args)
if err != nil || strings.Join(argv, " ") != c.want {
t.Errorf("%v: %q %v, want %q", c.args, argv, err, c.want)
}
}
for _, args := range []map[string]any{{"confirm": "true"}, {"why": "tidy"}, {"node": "anchor"}} {
if argv, err := argvFor("move-config", args); err == nil {
t.Errorf("%v was composed as %q", args, argv)
}
}
if repairingCommand([]string{"move-config", "--json", "--confirm", "--why", "x"}) != "move-config" {
t.Error("a confirmed move through the generic verb would go unrecorded")
}
if !personsDecision(link.HandAct{Verb: "move-config"}) {
t.Error("a move of the release would count toward a healer the mesh lacks")
}
}
// ask is mesh-config's verb as the stand-in answers it: list, set, clear and propose.
func (c *standInMeshConfig) ask(ctx context.Context, args map[string]any) (json.RawMessage, error) {
str := func(k string) string { s, _ := args[k].(string); return s }
level := configLevel{Level: "mesh"}
switch {
case str("module") != "" && str("node") != "":
level = configLevel{Level: "module", Node: str("node"), Module: str("module")}
case str("node") != "":
level = configLevel{Level: "node", Node: str("node")}
}
c.asked = append(c.asked, str("action")+" "+level.valueKey(str("key")))
switch str("action") {
case "list":
resolved, err := c.Resolved(ctx)
if err != nil {
return nil, err
}
out := map[string]any{"resolved": []resolvedEntry{}, "set": []any{}}
var listed []resolvedEntry
for _, r := range resolved {
if r.Module == str("module") && (str("node") == "" || r.Node == str("node")) {
listed = append(listed, r)
}
}
out["resolved"] = listed
var set []any
for _, k := range movedKeys(c.values) {
parts := strings.SplitN(k, ".", 4)
if len(parts) == 4 && parts[0] == "module" && parts[2] == str("module") {
set = append(set, map[string]any{"level": configLevel{Level: "module", Node: parts[1], Module: parts[2]},
"key": parts[3], "value": c.values[k], "by": "a test"})
}
}
out["set"] = set
return json.Marshal(out)
case "set":
if str("why") == "" || str("by") == "" {
return nil, fmt.Errorf("a set without why or by: %v", args)
}
return nil, c.Set(ctx, level, str("key"), args["value"], str("by"), str("why"))
case "clear":
if _, had := c.values[level.valueKey(str("key"))]; !had {
return nil, fmt.Errorf("nothing is set for %s", str("key"))
}
delete(c.values, level.valueKey(str("key")))
return json.RawMessage(`{}`), nil
case "propose":
return json.RawMessage(`{"proposal":"p-1"}`), nil
}
return nil, fmt.Errorf("the stand-in does not %s", str("action"))
}
// **`settings` answers by forwarding to `config`** (novox/hq ADR 0348 §1, design 57 K5, issue 664): once the move
// has run, a module in the config form is shown from mesh-config, its values set and cleared there as its own on the
// machine (placement still beside the assignment), and proposed through it; a trusted value is refused through a
// verb, a module value for the whole mesh is refused, and a module still in the settings form keeps today's verb.
func TestTheSettingsVerbForwardsToConfigOnceTheValuesMoved(t *testing.T) {
open, w, served := aConfiguredMesh(t)
ctx := t.Context()
inv := open.inventory
readBackTries = 0
t.Cleanup(func() { readBackTries = 15 })
config := &standInMeshConfig{open: open, values: map[string]any{}}
inv.ReadsConfigFrom(config.laid)
savedRead, savedAsk := readConfigValues, configAsk
readConfigValues, configAsk = config.laid, config.ask
t.Cleanup(func() { readConfigValues, configAsk = savedRead, savedAsk })
if _, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil {
t.Fatal(err)
}
shown := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "nodered", "--node", "anchor"}) })
for _, want := range []string{`"time-zone": "Europe/Brussels"`, `time-zone = "Europe/Brussels" (module)`,
"mesh-config.config", `"mqtt": {`} {
if !strings.Contains(shown, want) {
t.Errorf("settings show does not say %q:\n%s", want, shown)
}
}
if strings.Contains(shown, "timeZone") {
t.Errorf("settings show still says the file's name:\n%s", shown)
}
config.asked = nil
said := stdoutOf(t, func() error {
return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 500}`, "--node", "workstation"})
})
if config.values["module.workstation.dunst.font-size"] != float64(12) || !strings.Contains(said, "~ font-size") ||
containsString(config.asked, "set module.workstation.dunst.width") {
t.Fatalf("settings set reached mesh-config as %v (%v):\n%s", config.values, config.asked, said)
}
told := toldEveryMachine(t, open, served, w.On)["workstation"]
if !strings.Contains(told, "12") {
t.Error("the workstation is not told the font size set through settings")
}
if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`, "--node", "workstation"}); err == nil ||
!strings.Contains(err.Error(), "would no longer set width") {
t.Fatalf("a set dropping a value without --replace: %v", err)
}
if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`}); err == nil ||
!strings.Contains(err.Error(), "no own value for the whole mesh") {
t.Fatalf("a module value for the whole mesh: %v", err)
}
t.Setenv(verbVar, "settings")
for _, c := range []struct{ module, node, values, key, held, why string }{
// A file root reads: refused as before the move (novox/hq issue 339).
{"hostname", "anchor", `{"hostname": "elsewhere"}`, "module.anchor.hostname.hostname", "Ace", "issue 339"},
{"messenger", "laptop", `{"desktop-machines": ["g14", "shanks"], "time-zone": "UTC"}`,
"module.laptop.messenger.time-zone", "Europe/Brussels", "issue 339"},
} {
err := settingsCommand(ctx, []string{"set", c.module, c.values, "--node", c.node})
if err == nil || !strings.Contains(err.Error(), c.why) || config.values[c.key] != c.held {
t.Fatalf("a trusted value of %s through a verb: %v, holds %v", c.module, err, config.values[c.key])
}
}
// And a trusted value no file makes the terminal's is refused as trusted, since mesh-config takes the controller's
// word for it (ADR 0348 §8).
if err := refuseTrustedThroughAVerb(ctx, inv, "messenger", "laptop", []string{"time-zone"}); err == nil ||
!strings.Contains(err.Error(), "ADR 0348 §8") {
t.Fatalf("a trusted value through a verb: %v", err)
}
if err := refuseTrustedThroughAVerb(ctx, inv, "dunst", "workstation", []string{"width"}); err != nil {
t.Fatalf("a value nobody trusts through a verb: %v", err)
}
stdoutOf(t, func() error {
return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 600}`, "--node", "workstation"})
})
if config.values["module.workstation.dunst.width"] != float64(600) {
t.Fatalf("a value nobody trusts was not set through a verb: %v", config.values["module.workstation.dunst.width"])
}
t.Setenv(verbVar, "")
stdoutOf(t, func() error { return settingsCommand(ctx, []string{"clear", "dunst", "--node", "workstation"}) })
for _, k := range []string{"font-size", "width"} {
if _, held := config.values["module.workstation.dunst."+k]; held {
t.Errorf("settings clear left dunst's %s on the workstation", k)
}
}
config.asked = nil
proposed := stdoutOf(t, func() error {
return settingsCommand(ctx, []string{"propose", "hostname", `{"hostname": "elsewhere"}`, "--node", "anchor"})
})
if !containsString(config.asked, "propose module.anchor.hostname.hostname") || !strings.Contains(proposed, "p-1") {
t.Fatalf("settings propose reached mesh-config as %v:\n%s", config.asked, proposed)
}
config.asked = nil
kept := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "kernel", "--node", "laptop"}) })
if !strings.Contains(kept, "transparent-huge-pages") || len(config.asked) > 0 {
t.Fatalf("a module in the settings form is shown as %q, asking mesh-config %v", kept, config.asked)
}
}
// **An approval for a module whose config has moved is refused, and writes nothing** (issue 830, novox/hq ADR 0348):
// a proposal raised while grafana read its layer (it had none, so the layer is as the operator was shown it) and
// approved after the move would put its values back into the layer, ReadsConfig would turn false, and the module
// would compose from emptied layers everywhere.
func TestAnApprovalRaisedBeforeTheMoveOfConfigIsRefusedAfterIt(t *testing.T) {
open, _, _ := aConfiguredMesh(t)
ctx := t.Context()
readBackTries = 0
t.Cleanup(func() { readBackTries = 15 })
inv := open.inventory
config := &standInMeshConfig{open: open, values: map[string]any{}}
inv.ReadsConfigFrom(config.laid)
shelf, err := inv.Catalogue(ctx)
if err != nil {
t.Fatal(err)
}
key := movedKeys(func() map[string]catalogue.SettingDeclaration { return shelf["grafana"].Settings }())[0]
values := map[string]any{key: "raised before the move"}
p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(),
Digest: layerDigest(values), BeforeDigest: layerDigest(nil)}
if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has {
t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err)
}
if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched {
t.Fatalf("the move says %+v (%v)", moved, err)
}
if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads {
t.Fatalf("grafana does not read config after the move: %v %v", reads, err)
}
_, err = setLayerIn(open)(ctx, p, "s30", "approved by the operator")
if err == nil || !strings.Contains(err.Error(), "moved to mesh-config since the operator was shown the change") ||
!strings.Contains(err.Error(), "propose it again") {
t.Fatalf("the stale approval: %v", err)
}
if layer, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has && len(layer) > 0 {
t.Errorf("the refused approval wrote the layer: %v %v %v", layer, has, err)
}
if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads {
t.Errorf("the refused approval turned ReadsConfig off: %v %v", reads, err)
}
}
// **A proposal of placement only is set after the move, as before** (issue 830): forwardPropose sends it down the
// old path on purpose, so refusing it would loop the operator (refused, proposed again, refused).
func TestAPlacementOnlyApprovalRaisedBeforeTheMoveIsSetAfterIt(t *testing.T) {
open, _, _ := aConfiguredMesh(t)
ctx := t.Context()
readBackTries = 0
t.Cleanup(func() { readBackTries = 15 })
inv := open.inventory
config := &standInMeshConfig{open: open, values: map[string]any{}}
inv.ReadsConfigFrom(config.laid)
values := map[string]any{catalogue.PlacesSetting: map[string]any{"data": "/srv/grafana-data"}}
p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(),
Digest: layerDigest(values), BeforeDigest: layerDigest(nil)}
if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has {
t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err)
}
if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched {
t.Fatalf("the move says %+v (%v)", moved, err)
}
if _, err := setLayerIn(open)(ctx, p, "s31", "approved by the operator"); err != nil {
t.Fatalf("the placement-only approval after the move: %v", err)
}
if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || !has {
t.Errorf("the approval did not set the layer: %v %v", has, err)
}
if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads {
t.Errorf("the approval turned ReadsConfig off: %v %v", reads, err)
}
}
cmd/mesh-controller/config_read.go
+128
View File
@@ -0,0 +1,128 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
)
// **Where the controller reads config values once the move has run** (novox/hq ADR 0348 §9, design 57 K5, issue 664):
// mesh-config_resolved, the bucket mesh-config alone writes, keyed `<node>.<module>.<key>`, each value with the
// source it came from. Read only for a module the switch reaches (inventory.ReadsConfig), so a controller before the
// move never asks the bus for config. The exception to ADR 0348 §9 ("the controller holds no config value") that
// ADR 0349 clause 17's old way needs, until design 57's R1 hands every module to the provisioner.
// resolvedEntry is one key of mesh-config_resolved as the controller reads it: the value and where it came from.
type resolvedEntry struct {
Node string `json:"node"`
Module string `json:"module"`
Key string `json:"key"`
Value json.RawMessage `json:"value,omitempty"`
Source string `json:"source"`
}
// The sources a resolved value is laid over a module's files from: mesh config, node config and the module's own
// value. Its manifest's default is composition's already, and a taken key's default is never laid over its file;
// a required value nothing sets (`unset`) is refused by composition by name, as it is today.
var laidSources = map[string]bool{"mesh": true, "node": true, "module": true}
// configLaid is the values of one module on one node that are laid over its files, from the resolved entries.
func configLaid(entries []resolvedEntry, node, module string) (map[string]any, error) {
out := map[string]any{}
for _, e := range entries {
if e.Node != node || e.Module != module || !laidSources[e.Source] {
continue
}
var v any
if err := json.Unmarshal(e.Value, &v); err != nil {
return nil, fmt.Errorf("mesh-config's value %s of %s on %s cannot be read: %w", e.Key, module, node, err)
}
out[e.Key] = v
}
return out, nil
}
// resolvedRead is what one read of the whole bucket held, and when it was read.
type resolvedRead struct {
mu sync.Mutex
at time.Time
entries []resolvedEntry
}
// resolvedFresh is how long one read of the bucket answers: a plan reads every module of a machine, and one read
// serves them all; a value set since is read by the next composition.
const resolvedFresh = 5 * time.Second
var theResolved resolvedRead
// readConfigValues is where an opened inventory reads config values once the switch is on; a test replaces it.
var readConfigValues inventory.ConfigValues = busConfigValues
// busConfigValues reads one module's config values on one node from mesh-config_resolved (inventory.ConfigValues).
func busConfigValues(ctx context.Context, node, module string) (map[string]any, error) {
theResolved.mu.Lock()
defer theResolved.mu.Unlock()
if theResolved.entries == nil || time.Since(theResolved.at) > resolvedFresh {
entries, err := readResolved(ctx)
if err != nil {
return nil, err
}
theResolved.entries, theResolved.at = entries, time.Now()
}
return configLaid(theResolved.entries, node, module)
}
// readResolved is every entry of mesh-config_resolved, read on the bus.
func readResolved(ctx context.Context) ([]resolvedEntry, error) {
entries := []resolvedEntry{}
err := onTheBus(func(conn *nats.Conn) error {
js, err := jetstream.New(conn)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
defer cancel()
kv, err := js.KeyValue(ctx, broker.ConfigResolvedBucket)
if err != nil {
return fmt.Errorf("%s cannot be opened: %w", broker.ConfigResolvedBucket, err)
}
w, err := kv.WatchAll(ctx, jetstream.IgnoreDeletes())
if err != nil {
return err
}
defer func() { _ = w.Stop() }()
for {
select {
case <-ctx.Done():
return fmt.Errorf("%s was not read whole: %w", broker.ConfigResolvedBucket, ctx.Err())
case e, open := <-w.Updates():
if !open {
return errors.New(broker.ConfigResolvedBucket + " stopped answering")
}
if e == nil {
return nil
}
var r resolvedEntry
if err := json.Unmarshal(e.Value(), &r); err != nil {
return fmt.Errorf("%s holds %q, which is not a resolved value: %w", broker.ConfigResolvedBucket,
e.Key(), err)
}
if r.Node == "" || r.Module == "" || r.Key == "" || !strings.HasPrefix(e.Key(), r.Node+"."+r.Module+".") {
return fmt.Errorf("%s holds %q, whose value names another key", broker.ConfigResolvedBucket, e.Key())
}
entries = append(entries, r)
}
}
})
return entries, err
}
cmd/mesh-controller/core_box.go
+297
View File
@@ -0,0 +1,297 @@
package main
import (
"context"
"crypto/ecdh"
"crypto/x509"
"encoding/base64"
"encoding/json"
"encoding/pem"
"errors"
"fmt"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The core's own credentials, sealed by the vault (novox/hq ADR 0349 §16, issue 755; the lead's decision on issue 755,
// notes 4 and 5).
//
// For a machine whose engine opens the vault's box (link.VaultBoxContract), each own secret of a module on the core's
// list (catalogue.CoreCredentialModules) travels as the vault sealed it to that machine's key, which the controller
// asks for on the send path (`get`, answered to the controller alone and sealed only to the key that machine's engine
// stated: mesh-vault's core.go) and carries without being able to open it. A machine below that contract is sent the
// controller's own sealed value, exactly as before.
//
// **The vault seals every answer afresh**, so the box last got is kept (inventory.CoreBox) and carried again while the
// vault's generation and the machine's key are the same: a declaration that changed with every answer would never be
// current. Every question but a send reads the kept box and asks nothing.
//
// **When the vault does not answer, the controller never falls back to its own seal.** A credential with a kept box
// keeps it — the machine keeps the file it has — and one with none leaves its module out of the declaration (the
// machine keeps what it holds of it, and a module not there yet is not placed). Either way the send raises a
// condition naming the vault and the machine, cleared by the first send the vault answers whole.
// engineOpensVaultBox says whether a machine's node-engine opens the vault's box, by its own newest statement.
func engineOpensVaultBox(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) {
stated, had, err := inv.HealthOf(ctx, node)
if err != nil {
return false, err
}
return had && stated.Contract >= link.VaultBoxContract, nil
}
// coreAnswer is the vault's `get` answer as the controller reads it: the name, the generation and the box, never
// opened.
type coreAnswer struct {
Name string `json:"name"`
Generation int `json:"generation"`
Sealed json.RawMessage `json:"sealed"`
}
// coreAsk asks the vault for one secret sealed to the given key.
type coreAsk func(name, publicKey string) (coreAnswer, error)
// coreBoxes is what the vault's boxes make of one machine's composition.
type coreBoxes struct {
// sealed is each core credential's box, by module and name (catalogue.Rendering.VaultSealed).
sealed map[string]map[string]json.RawMessage
// notPlaced is each core module left out, with why (catalogue.Rendering.NotPlaced).
notPlaced map[string]string
// keep is each box newly answered, to keep.
keep []inventory.CoreBox
// held is each credential the vault did not answer for whose kept box is carried again; silent is why the vault
// did not answer, per credential.
held, silent []string
}
// coreModulesOf is every module of the plan on the core's list whose own secrets this machine's declaration carries,
// with their plain own secret names, sorted: one handed to the provisioner has its secrets from the vault through the
// client, and one issued outside the mesh is never the vault's to seal here.
func coreModulesOf(modules []catalogue.Manifest, handed func(catalogue.Manifest) bool) map[string][]string {
out := map[string][]string{}
for _, m := range modules {
if !catalogue.CarriesVaultSealed(m.Module) || (handed != nil && handed(m)) {
continue
}
var names []string
for name, own := range m.OwnSecrets.Plain() {
if own.IssuedBy != catalogue.IssuedOutside {
names = append(names, name)
}
}
if len(names) > 0 {
sort.Strings(names)
out[m.Module] = names
}
}
return out
}
// sealingKeyPEM is a machine's sealing key, recorded raw in base64, as the SPKI PEM the vault seals to.
func sealingKeyPEM(raw string) (string, error) {
b, err := base64.StdEncoding.DecodeString(strings.TrimSpace(raw))
if err != nil || len(b) != 32 {
return "", errors.New("the machine's sealing key is not a raw X25519 key")
}
pub, err := ecdh.X25519().NewPublicKey(b)
if err != nil {
return "", err
}
der, err := x509.MarshalPKIXPublicKey(pub)
if err != nil {
return "", err
}
return string(pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: der})), nil
}
// coreBoxesOf decides each core credential of a machine: on a send, the vault's answer (the kept box while its
// generation and key are the same), the kept box where the vault did not answer, or the module left out where nothing
// was kept; on any other question, the kept box, or a stand-in the next send asks for. Pure: what to keep and what to
// say is answered, not done.
func coreBoxesOf(node, sealingKey string, core map[string][]string, kept map[string]map[string]inventory.CoreBox,
choosing Choosing, ask coreAsk) coreBoxes {
out := coreBoxes{sealed: map[string]map[string]json.RawMessage{}, notPlaced: map[string]string{}}
modules := make([]string, 0, len(core))
for m := range core {
modules = append(modules, m)
}
sort.Strings(modules)
for _, module := range modules {
if sealingKey == "" {
out.notPlaced[module] = fmt.Sprintf("%s has no sealing key, so the vault can seal %s's credentials to "+
"nothing there (novox/hq ADR 0349 §16)", node, module)
continue
}
key, err := sealingKeyPEM(sealingKey)
if err != nil {
out.notPlaced[module] = fmt.Sprintf("%s's credentials cannot be asked of the vault: %v", module, err)
continue
}
boxes := map[string]json.RawMessage{}
for _, name := range core[module] {
k, has := kept[module][name]
valid := has && k.SealedTo == sealingKey && len(k.Box) > 0
if choosing != Allocating {
if valid {
boxes[name] = k.Box
} else {
boxes[name] = catalogue.ForeseenVaultBox
}
continue
}
secret := "own/" + node + "/" + module + "/" + name
got, err := ask(secret, key)
if err == nil && (got.Generation <= 0 || canonicalBox(got.Sealed) == nil) {
err = errors.New("it answered no box")
}
if err != nil {
out.silent = append(out.silent, fmt.Sprintf("%s: %v", secret, err))
if valid {
boxes[name] = k.Box
out.held = append(out.held, module+"/"+name)
continue
}
out.notPlaced[module] = fmt.Sprintf("the vault did not give %s's credential %s for %s (%v), and the "+
"controller never seals it itself (novox/hq ADR 0349 §16): not placed until the vault answers",
module, name, node, err)
break
}
if valid && k.Generation == got.Generation {
boxes[name] = k.Box
continue
}
box := canonicalBox(got.Sealed)
boxes[name] = box
out.keep = append(out.keep, inventory.CoreBox{Module: module, Name: name, Generation: got.Generation,
SealedTo: sealingKey, Box: box})
}
if _, left := out.notPlaced[module]; !left {
out.sealed[module] = boxes
}
}
return out
}
// canonicalBox is the vault's box in the one form the declaration carries — the sealed box's fields, in their order,
// and nothing else — or nil when it is not one.
func canonicalBox(raw json.RawMessage) json.RawMessage {
var b struct {
V int `json:"v"`
Eph string `json:"eph"`
IV string `json:"iv"`
Tag string `json:"tag"`
Ct string `json:"ct"`
}
if err := json.Unmarshal(raw, &b); err != nil || b.V <= 0 || b.Eph == "" || b.Ct == "" {
return nil
}
out, _ := json.Marshal(b)
return out
}
// coreVault is how a send asks the vault for a core credential: its `get`, in the controller's name. A test stands
// in for it.
var coreVault = func(ctx context.Context) coreAsk {
vault := theVault(ctx)
return func(name, publicKey string) (coreAnswer, error) {
raw, err := vault("get", map[string]any{"name": name, "public_key": publicKey})
if err != nil {
return coreAnswer{}, err
}
var a coreAnswer
if err := json.Unmarshal(raw, &a); err != nil {
return coreAnswer{}, fmt.Errorf("the vault's answer is not one: %w", err)
}
return a, nil
}
}
// coreBoxesFor is coreBoxesOf for one machine, read from and kept in the store, and said on a send.
func coreBoxesFor(ctx context.Context, inv *inventory.Inventory, node string, modules []catalogue.Manifest,
handed func(catalogue.Manifest) bool, choosing Choosing) (coreBoxes, error) {
core := coreModulesOf(modules, handed)
if len(core) == 0 {
if choosing == Allocating {
settleVaultSilent(ctx, node)
}
return coreBoxes{}, nil
}
key, err := inv.SealingKeyOf(ctx, node)
if err != nil {
return coreBoxes{}, err
}
kept, err := inv.KeptCoreBoxes(ctx, node)
if err != nil {
return coreBoxes{}, err
}
var ask coreAsk
if choosing == Allocating {
ask = coreVault(ctx)
}
out := coreBoxesOf(node, key, core, kept, choosing, ask)
for _, b := range out.keep {
if err := inv.KeepCoreBox(ctx, node, b); err != nil {
return coreBoxes{}, err
}
}
if choosing == Allocating {
sayVaultSilent(ctx, node, out)
}
return out, nil
}
// kindVaultSilent is a send that did not have the vault's box for a core credential of a machine.
const kindVaultSilent = "vault-silent"
// vaultSilentObservation is that, as a condition: `machine.<machine>.vault-silent`.
func vaultSilentObservation(node string, b coreBoxes) conditions.Observation {
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: kindVaultSilent, Token: kindVaultSilent,
Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource}
var said []string
if len(b.notPlaced) > 0 {
o.Severity = conditions.Urgent
var left []string
for m := range b.notPlaced {
left = append(left, m)
}
sort.Strings(left)
said = append(said, "not placed: "+strings.Join(left, ", "))
}
if len(b.held) > 0 {
said = append(said, "kept as last sent: "+strings.Join(b.held, ", "))
}
o.Summary = fmt.Sprintf("the vault did not answer for the core's credentials on %s; %s", node, strings.Join(said, "; "))
o.Said = o.Summary + " — " + strings.Join(b.silent, "; ")
return o
}
// sayVaultSilent raises the condition for a send the vault did not answer whole, and clears it for one it did.
func sayVaultSilent(ctx context.Context, node string, b coreBoxes) {
if len(b.silent) == 0 && len(b.notPlaced) == 0 {
settleVaultSilent(ctx, node)
return
}
fmt.Printf("HELD: %s\n", vaultSilentObservation(node, b).Said)
if conditionsFrom == nil {
return
}
if _, err := conditionsFrom.Observe(ctx, vaultSilentObservation(node, b)); err != nil {
fmt.Printf("the condition for the vault's silence on %s could not be raised: %v\n", node, err)
}
}
// settleVaultSilent clears the condition once a send had every core credential of the machine from the vault.
func settleVaultSilent(ctx context.Context, node string) {
if conditionsFrom == nil {
return
}
key := vaultSilentObservation(node, coreBoxes{}).Key()
if _, err := conditionsFrom.Clear(ctx, key, "the vault answered for every core credential of "+node); err != nil {
fmt.Printf("the condition %s could not be cleared: %v\n", key, err)
}
}
cmd/mesh-controller/core_box_test.go
+220
View File
@@ -0,0 +1,220 @@
package main
import (
"context"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The core's own credentials travel sealed by the vault to their machine (novox/hq ADR 0349 §16, issue 755; the lead's
// decision, issue 755 notes 4 and 5): to an engine that opens the vault's box, the send carries the box the vault
// answered the controller and never the controller's own seal; when the vault does not answer, a kept box is carried
// again and a module with none is not placed, with a condition naming the vault and the machine. An engine below the
// contract composes exactly as before, and the vault is not asked.
// coreStandIn answers `get` with a box per name and generation, or its silence.
type coreStandIn struct {
asked []string
keys []string
generation int
silent error
}
func (v *coreStandIn) ask(name, key string) (coreAnswer, error) {
v.asked, v.keys = append(v.asked, name), append(v.keys, key)
if v.silent != nil {
return coreAnswer{}, v.silent
}
box := `{"v":1,"eph":"ZQ==","iv":"aQ==","tag":"dA==","ct":"` + strings.Repeat("c", len(v.asked)) + `"}`
return coreAnswer{Name: name, Generation: v.generation, Sealed: json.RawMessage(box)}, nil
}
func withVault(t *testing.T, v *coreStandIn) {
t.Helper()
before := coreVault
coreVault = func(context.Context) coreAsk { return v.ask }
t.Cleanup(func() { coreVault = before })
}
func aCoreMesh(t *testing.T, contract int) (*stores, *coreStandIn) {
t.Helper()
open := aMesh(t)
ctx := t.Context()
client := aClientModule()
register(t, open, client)
if _, err := assign(ctx, open, "laptop", client.Module); err != nil {
t.Fatal(err)
}
if err := issueProvisionerClient(ctx, open.inventory, client, "laptop", broker.Broker{Fingerprint: "ab"},
"10.0.0.1:4222"); err != nil {
t.Fatal(err)
}
if _, err := open.inventory.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: contract,
SaidAt: time.Now()}); err != nil {
t.Fatal(err)
}
v := &coreStandIn{generation: 1}
withVault(t, v)
return open, v
}
func clientsBroker(resources []map[string]any) map[string]any {
for _, r := range resources {
if r["id"] == "mesh-provisioner-client.needs-"+broker.ProvisionerSecret {
return r
}
}
return nil
}
func vaultSilence(t *testing.T, node string) (conditions.Condition, bool) {
t.Helper()
c, found, err := conditionsFrom.Get(t.Context(), vaultSilentObservation(node, coreBoxes{}).Key())
if err != nil {
t.Fatal(err)
}
return c, found
}
func TestTheSendCarriesTheVaultsBoxForACoreCredentialAndKeepsIt(t *testing.T) {
open, vault := aCoreMesh(t, link.VaultBoxContract)
key, err := open.inventory.SealingKeyOf(t.Context(), "laptop")
if err != nil {
t.Fatal(err)
}
f := clientsBroker(sentTo(t, open, "laptop"))
if f == nil || f["sealed"] != nil || f[catalogue.VaultSealedField] == nil {
t.Fatalf("the core credential is sent as %v: the vault's box, never the controller's seal", f)
}
if want := []string{"own/laptop/mesh-provisioner-client/" + broker.ProvisionerSecret}; !reflect.DeepEqual(vault.asked, want) {
t.Fatalf("the vault was asked %v", vault.asked)
}
if want, _ := sealingKeyPEM(key); vault.keys[0] != want {
t.Fatalf("the vault was asked to seal to %q, not the machine's key", vault.keys[0])
}
first, _ := json.Marshal(f[catalogue.VaultSealedField])
// The vault answers afresh with the same generation: the kept box is carried again, so the declaration is stable.
again, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField])
if string(again) != string(first) {
t.Fatalf("a fresh box of the same generation changed the declaration: %s, then %s", first, again)
}
// A rotation in the vault (a newer generation) is carried.
vault.generation = 2
rotated, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField])
if string(rotated) == string(first) {
t.Fatal("a newer generation was not carried")
}
if _, found := vaultSilence(t, "laptop"); found {
t.Fatal("a send the vault answered raised its silence")
}
}
func TestASilentVaultHoldsTheKeptBoxAndSaysSo(t *testing.T) {
open, vault := aCoreMesh(t, link.VaultBoxContract)
first, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField])
vault.silent = errors.New("no responders")
held := clientsBroker(sentTo(t, open, "laptop"))
got, _ := json.Marshal(held[catalogue.VaultSealedField])
if held == nil || held["sealed"] != nil || string(got) != string(first) {
t.Fatalf("a silent vault's credential is sent as %v: the box kept, never the controller's seal", held)
}
c, found := vaultSilence(t, "laptop")
if !found || !strings.Contains(c.Summary, "vault") || !strings.Contains(c.Summary, "laptop") {
t.Fatalf("no condition names the vault and the machine: %+v", c)
}
// The vault answers again: the condition clears.
vault.silent = nil
sentTo(t, open, "laptop")
if _, found := vaultSilence(t, "laptop"); found {
t.Fatal("the condition stayed after the vault answered")
}
}
func TestACoreModuleFirstPlacedWhileTheVaultIsSilentIsNotPlaced(t *testing.T) {
open, vault := aCoreMesh(t, link.VaultBoxContract)
vault.silent = errors.New("no responders")
plan, settings, err := planFor(t.Context(), open, "laptop")
if err != nil {
t.Fatal(err)
}
declared, err := declarationFor(t.Context(), open, "laptop", plan, settings)
if err != nil {
t.Fatal(err)
}
for _, r := range declared.Resources {
if strings.HasPrefix(r["id"].(string), "mesh-provisioner-client.") {
t.Fatalf("a core module was placed though the vault gave nothing for it: %v", r)
}
}
if why := declared.leftOutWhy["mesh-provisioner-client"]; !strings.Contains(why, "never seals it itself") {
t.Fatalf("left out %v", declared.leftOutWhy)
}
c, found := vaultSilence(t, "laptop")
if !found || c.Severity != conditions.Urgent || !strings.Contains(c.Summary, "not placed") {
t.Fatalf("the condition does not say why: %+v", c)
}
}
// An engine below VaultBoxContract is sent the controller's own seal, as before, and the vault is not asked.
func TestAnEngineBelowTheContractIsSentTheControllersSealAsBefore(t *testing.T) {
open, vault := aCoreMesh(t, link.HandedContract)
f := clientsBroker(sentTo(t, open, "laptop"))
if f == nil || f["sealed"] == nil || f[catalogue.VaultSealedField] != nil {
t.Fatalf("an older engine is sent %v", f)
}
if len(vault.asked) != 0 {
t.Fatalf("the vault was asked %v for an older engine", vault.asked)
}
}
// Every question but a send reads the kept box and asks nothing; with none kept, a stand-in the next send asks for.
func TestOnlyASendAsksTheVault(t *testing.T) {
core := map[string][]string{"nats": {"broker"}}
key := aPublicKey(t)
kept := map[string]map[string]inventory.CoreBox{"nats": {"broker": {Module: "nats", Name: "broker",
Generation: 1, SealedTo: key, Box: json.RawMessage(`{"v":1}`)}}}
never := func(string, string) (coreAnswer, error) {
t.Fatal("the vault was asked by a question that is not a send")
return coreAnswer{}, nil
}
for _, choosing := range []Choosing{Reading, Foreseeing} {
if got := coreBoxesOf("anchor", key, core, kept, choosing, never); string(got.sealed["nats"]["broker"]) != `{"v":1}` {
t.Errorf("read %s", got.sealed["nats"]["broker"])
}
if got := coreBoxesOf("anchor", key, core, nil, choosing, never); string(got.sealed["nats"]["broker"]) !=
string(catalogue.ForeseenVaultBox) {
t.Errorf("with none kept, read %s", got.sealed["nats"]["broker"])
}
}
// A box kept for another key of the machine is never carried.
other := coreBoxesOf("anchor", aPublicKey(t), core, kept, Allocating,
func(string, string) (coreAnswer, error) { return coreAnswer{}, errors.New("silent") })
if _, left := other.notPlaced["nats"]; !left {
t.Fatalf("a box sealed to the machine's old key was carried: %+v", other)
}
}
// The core's modules on a machine: on the list, not handed, with their own secrets but one issued outside.
func TestTheCoresModulesAreTheListsWithTheirOwnSecrets(t *testing.T) {
modules := []catalogue.Manifest{
{Module: "nats", OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/a"}, "given": {Path: "/b",
IssuedBy: catalogue.IssuedOutside}}},
{Module: "postgres", OwnSecrets: catalogue.OwnSecrets{"superuser": {Path: "/c"}}},
{Module: "mesh-provisioner", OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/d"}}},
}
got := coreModulesOf(modules, func(m catalogue.Manifest) bool { return m.Module == "mesh-provisioner" })
if want := map[string][]string{"nats": {"broker"}}; !reflect.DeepEqual(got, want) {
t.Fatalf("got %v, want %v", got, want)
}
}
cmd/mesh-controller/delivery.go
+3 -1
View File
@@ -1051,7 +1051,9 @@ func appliedFrom(ctx context.Context, inv *inventory.Inventory) inventory.Applie
fmt.Fprintf(os.Stderr, "the report of %s after %s could not be read: %v\n", node, walkSent.Format(time.RFC3339), err)
return inventory.AppliedReport{}, false
}
return r, ok
// And, for each module the send handed to the provisioner, its client's report on that build (novox/hq ADR
// 0362): applied only once both said it, at the later; failed if the client failed.
return inventory.ProvisionedSend(node, send, r, ok, provLookup(ctx))
}
}
cmd/mesh-controller/desk_secret.go
+15
View File
@@ -72,6 +72,9 @@ type deskGive struct {
// nil keeps no record (a test that does not look). end closes it with how it ended.
open func(node, module, name, desk string) (int64, error)
end func(id int64, outcome string) error
// vault, when set, is the path of a module whose secrets are filled the new way (vault_secrets.go): the vault
// records the ask, holds its bounds and opens what is typed; open, end and accept are not used.
vault vaultAsker
}
// askedByName is the caller as the prompt names it: the first clause of what the bus said, in the characters
@@ -138,6 +141,9 @@ func (d deskGive) give(node, module, name, desk string) (string, error) {
"bus, where an agent may answer first (novox/hq ADR 0259 §10)", module, name, node, module, name)
}
}
if d.vault != nil {
return d.giveToVault(node, module, name, desk)
}
// The bounds, read and kept before anybody is asked to type (novox/hq ADR 0277): one open prompt per secret,
// few an hour. How the ask ends is recorded whatever happens below.
outcome := "failed"
@@ -297,6 +303,15 @@ func askAtDesk(ctx context.Context, node, module, name, desk string) error {
return announceSecretGiven(ctx, node, module, name, how, askedBy)
},
}
// A module whose secrets are filled the new way is asked for by the vault (ADR 0349 clause 9); every other
// keeps today's path (clause 17).
ours, err := filledTheNewWay(module)
if err != nil {
return fmt.Errorf("nothing was asked: %w", err)
}
if ours {
d.vault = theVault(ctx)
}
words, err := d.give(node, module, name, desk)
if err != nil {
return err
cmd/mesh-controller/facts.go
+2 -2
View File
@@ -348,11 +348,11 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
if layer.From == catalogue.MeshWideLayer {
if !meshWide[module] {
meshWide[module] = true
f.Settings = append(f.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)})
f.Settings = append(f.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Whole())})
}
continue
}
m.Settings = append(m.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)})
m.Settings = append(m.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Whole())})
}
}
m.Declaration = declarationFacts(ctx, open, n.Name, gens, gensErr, scrub)
cmd/mesh-controller/gate.go
+37 -1
View File
@@ -235,6 +235,10 @@ type gateFacts struct {
sent map[string]inventory.SentDeclaration
sentBuilds map[string]map[string]string
commits map[string]string
// provisioned answers each machine's provisioner client's newest report of a module (novox/hq ADR 0349 §15):
// a module the gate's send handed to the provisioner is healthy only once it reported that build placed. Nil
// reads as nothing said.
provisioned inventory.ProvLookup
}
// heldReading is the provider a module's findings are held under, as "<module> on <machine>", and, when that
@@ -291,7 +295,7 @@ func (f gateFacts) supersededOn(module, machine string) (string, bool) {
var gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
inv := open.inventory
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
rolledBack: witnessed.all()}
rolledBack: witnessed.all(), provisioned: provLookup(ctx)}
reports, err := inv.LastReports(ctx)
if err != nil {
return f, err
@@ -368,6 +372,12 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
case r.Outcome != inventory.OutcomeApplied:
return healthNotYet, fmt.Sprintf("%s reported %q", machine, r.Outcome)
}
// **And its files placed** (novox/hq ADR 0349 §15): a module the send handed to the provisioner is judged only
// once that machine's client reported the build's templates placed, so a module whose files are not yet written
// is never judged; a failure there is the build's on that machine.
if h, why := judgeProvisioned(module, machine, r, f); h != healthGood {
return h, why
}
// **No new condition about it**: about the machine itself, or naming the module on that machine,
// raised since the judging began. The gate's own are not evidence about the build. A fault that was
// there at the send and reopened since is not new; one that had cleared before the send and came back
@@ -449,6 +459,32 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
return healthGood, ""
}
// judgeProvisioned is whether a module's files are placed on a machine, for a module the gate's send handed to the
// provisioner there: healthy for any other.
func judgeProvisioned(module, machine string, r inventory.Reported, f gateFacts) (health, string) {
sent, kept := f.sent[machine]
digest := sent.Provisioned[module]
if !kept || digest == "" {
return healthGood, ""
}
at := time.Time{}
if r.At != nil {
at = *r.At
}
one := inventory.SentDeclaration{Digest: sent.Digest, At: sent.At, Provisioned: map[string]string{module: digest}}
got, ok := inventory.ProvisionedSend(machine, one, inventory.AppliedReport{At: at,
Outcome: inventory.OutcomeApplied}, true, f.provisioned)
switch {
case !ok:
return healthNotYet, fmt.Sprintf("the provisioner on %s has not reported %s's files placed for the new "+
"build", machine, module)
case got.Outcome != inventory.OutcomeApplied:
return healthBroken, fmt.Sprintf("the provisioner on %s did not place %s's files: %s", machine, module,
got.Outcome)
}
return healthGood, ""
}
// machineWord is what one judging found wrong with a machine itself, apart from its modules: facts is
// the judging's facts without those conditions, on what each names among the modules the send moved,
// and whole what holds the machine back as a whole.
cmd/mesh-controller/generation.go
+20 -1
View File
@@ -2,6 +2,7 @@ package main
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
@@ -60,12 +61,30 @@ type sentRecord struct {
toldGeneration int64
sender string
modules []string
// plans is each module the send handed to the provisioner, its plan, as JSON (design 57 P6); nil for none.
plans []byte
}
// sentRecordOf is a composed send's record: its sender is the caller this process acts for.
func sentRecordOf(ctx context.Context, d sendable) sentRecord {
return sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom,
r := sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom,
toldGeneration: d.Generation, sender: senderOf(callerOf(ctx)), modules: d.modules}
if len(d.plans) > 0 {
kept := map[string]keptPlan{}
for module, p := range d.plans {
kept[module] = keptPlan{Plan: p, RequiredBy: d.requiredBy[module]}
}
// A map of plain strings and slices: it marshals.
r.plans, _ = json.Marshal(kept)
}
return r
}
// keptPlan is a handed module's plan as a send keeps it (migration 0100): the plan, and for a module only pulled in
// by another, the module that requires it (issue 765, inventory.SentProvisioningPlans).
type keptPlan struct {
catalogue.Plan
RequiredBy string `json:"required-by,omitempty"`
}
// callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the
cmd/mesh-controller/given_test.go
+6 -3
View File
@@ -8,7 +8,8 @@ import (
)
// A rotation asked through the seat carries why to the command, which records it in the hand-act
// log (novox/hq ADR 0228); why with a provision is refused as passed over, not dropped.
// log (novox/hq ADR 0228); and with a provision too, since the vault asks why of every rotation of a
// credential it holds (issue 767).
func TestARotationThroughTheSeatCarriesWhy(t *testing.T) {
argv, err := argvFor("rotate", map[string]any{"node": "anchor", "module": "letta",
"secret": "server-password", "why": "leaked into logs", "cause": "leaked"})
@@ -19,8 +20,10 @@ func TestARotationThroughTheSeatCarriesWhy(t *testing.T) {
if err != nil || strings.Join(argv, " ") != "secret rotate anchor letta server-password" {
t.Fatalf("without why: %v %v", argv, err)
}
if argv, err := argvFor("rotate", map[string]any{"provision": "postgres-database", "why": "leaked"}); err == nil {
t.Fatalf("why beside a provision was passed over: %v", argv)
argv, err = argvFor("rotate", map[string]any{"provision": "postgres-database", "consumer": "ace", "why": "leaked",
"cause": "leaked"})
if err != nil || strings.Join(argv, " ") != "rotate postgres-database --consumer ace --why leaked --cause leaked" {
t.Fatalf("why beside a provision: %v %v", argv, err)
}
}
cmd/mesh-controller/handacts.go
+9
View File
@@ -105,6 +105,12 @@ var handActVerbs = []handActVerb{
// (ADR 0257).
{Verb: "mirrors", Decision: "which copies in the store no record names are the mesh's is a person's " +
"word (ADR 0257)"},
// A move of data a release runs once after its delivery (design 57): a person's step of the release, never a
// repair.
{Verb: "move-placement", Decision: "a release's move of data runs once after its delivery, on a person's " +
"word (design 57, issue 736)"},
{Verb: "move-config", Decision: "a release's move of data runs once after its delivery, on a person's " +
"word (design 57, issue 664)"},
{Verb: "bus upgrade", Decision: "the bus is never rolled by the mesh: replacing it is a planned step a " +
"person starts (ADR 0236)"},
{Verb: "upgrade release-backlog", Decision: "after a release plan failed, the next opens only when a " +
@@ -118,6 +124,9 @@ var handActVerbs = []handActVerb{
{Verb: "secret accept", Decision: "a value an outside party issued is given by a person, at their desk"},
{Verb: "secret rotate", Decision: "a value a person judged disclosed is replaced on their word",
DecidedFor: []string{causeLeakedInLogs}},
// A provider's credential the vault holds, rotated there on a person's word (issue 767): the same judgement.
{Verb: "rotate", Decision: "a value a person judged disclosed is replaced on their word",
DecidedFor: []string{causeLeakedInLogs}},
}
// personsDecision is whether an act in the log is a person's decision by design, by the verb that
cmd/mesh-controller/issued.go
+60 -5
View File
@@ -43,6 +43,32 @@ const issuedAsk = 20 * time.Second
// issuedProvider is one assignment of a module whose offer says its software issues the provision's credential.
type issuedProvider struct {
Node, Module, Tool string
// Holders are the provision's holders from this assignment's machine, named to the vault, whose tool may answer
// a value for each (issue 749).
Holders []issuedHolder
}
// issuedHolder is one holder of a provision from a provider: a consuming module on one machine (issue 749).
type issuedHolder struct {
Node string `json:"node"`
Module string `json:"module"`
}
func (h issuedHolder) String() string { return h.Module + " on " + h.Node }
// issuedHoldersFrom are the provision's holders from provider, each consuming module on each machine once.
func issuedHoldersFrom(holders []inventory.Holder, provider string) []issuedHolder {
seen := map[issuedHolder]bool{}
var out []issuedHolder
for _, h := range holders {
one := issuedHolder{Node: h.Consumer, Module: h.ConsumerModule}
if h.Provider != provider || seen[one] {
continue
}
seen[one] = true
out = append(out, one)
}
return out
}
// issuedTaker is the line's work, with what it reaches handed in: the store, the bus and the sends.
@@ -53,6 +79,10 @@ type issuedTaker struct {
keep func(provision, provider, value string) (inventory.IssuedKept, error)
record func(act link.HandAct) error
send func(machines []string) error
// ofTheVault says whether a provider module's secrets are filled the new way; for one that is, the value goes
// into the vault (vault, vault_secrets.go) and nothing is kept, sealed or sent here. Nil is none.
ofTheVault vaultSwitch
vault vaultAsker
}
// take asks every provider of the provision for its software's value and keeps each; it answers what it did, in
@@ -90,6 +120,15 @@ func (t issuedTaker) take(provision string) (string, error) {
// takeFrom is one provider: asked, opened, kept, recorded and sent when it changed.
func (t issuedTaker) takeFrom(provision string, p issuedProvider) (string, error) {
if t.ofTheVault != nil {
ours, err := t.ofTheVault(p.Module)
if err != nil {
return "", err
}
if ours {
return t.takeIntoVault(provision, p)
}
}
public, private, err := secrets.Keypair()
if err != nil {
return "", fmt.Errorf("no key could be made to take the value: %w", err)
@@ -174,6 +213,16 @@ func issuedProvidersOf(ctx context.Context, inv *inventory.Inventory, provision
if err != nil {
return nil, true, err
}
out, err := issuedProvidersFrom(shelf, holders, func(node string) ([]string, error) {
return inv.Assigned(ctx, node)
}, provision)
return out, true, err
}
// issuedProvidersFrom are the assignments to ask, from the provision's holders: on every machine providing it, the
// assigned modules whose offer says their software issues it, each with that machine's holders (issue 749).
func issuedProvidersFrom(shelf map[string]catalogue.Manifest, holders []inventory.Holder,
assignedOn func(node string) ([]string, error), provision string) ([]issuedProvider, error) {
nodes := map[string]bool{}
for _, h := range holders {
nodes[h.Provider] = true
@@ -185,13 +234,17 @@ func issuedProvidersOf(ctx context.Context, inv *inventory.Inventory, provision
sort.Strings(names)
var out []issuedProvider
for _, node := range names {
assigned, err := inv.Assigned(ctx, node)
assigned, err := assignedOn(node)
if err != nil {
return nil, true, err
return nil, err
}
out = append(out, issuedOn(shelf, node, assigned, provision)...)
on := issuedOn(shelf, node, assigned, provision)
for i := range on {
on[i].Holders = issuedHoldersFrom(holders, node)
}
out = append(out, on...)
}
return out, true, nil
return out, nil
}
// offersIssued is whether any module offers the provision with a credential its software issues.
@@ -262,7 +315,9 @@ func secretIssued(ctx context.Context, args []string) error {
return err
})
},
send: func(machines []string) error { return sendTo(ctx, open, machines) },
send: func(machines []string) error { return sendTo(ctx, open, machines) },
ofTheVault: filledTheNewWay,
vault: theVault(ctx),
}
words, err := t.take(provision)
if words != "" {
cmd/mesh-controller/issued_test.go
+82 -1
View File
@@ -193,7 +193,7 @@ func TestTheProvidersAskedAreTheOffersThatSaySo(t *testing.T) {
"jellyfin": {Module: "jellyfin", Provides: []catalogue.Offer{{Name: "plex-api"}}},
}
got := issuedOn(shelf, "ace", []string{"jellyfin", "plex", "kometa"}, "plex-api")
if len(got) != 1 || got[0] != (issuedProvider{Node: "ace", Module: "plex", Tool: "plex_issued_credential"}) {
if len(got) != 1 || got[0].Node != "ace" || got[0].Module != "plex" || got[0].Tool != "plex_issued_credential" {
t.Fatalf("the providers asked: %+v", got)
}
if err := issuedRefusesRotation(shelf, "plex-api"); err == nil || !strings.Contains(err.Error(), "secret issued plex-api") {
@@ -216,3 +216,84 @@ func TestAnIssuedValueIsADecisionInTheHandActLog(t *testing.T) {
}
t.Fatal("secret issued is not in the hand-act table")
}
// **A need's credential is the vault's for its holder alone** (issue 757) exactly when its providing module is marked
// moved, so `secret issued` goes to the vault naming each holder, and its software issues the provision's credential.
func TestANeedIsIssuedToItsHolderOnlyWhereTheProviderIsMovedAndItsSoftwareIssuesIt(t *testing.T) {
forge := func(moved bool, issued bool) catalogue.Manifest {
offer := catalogue.Offer{Name: "gitea-read"}
if issued {
offer.Credential = &catalogue.OfferCredential{IssuedBy: catalogue.IssuedBySoftware, Tool: "gitea_agent_token"}
}
return catalogue.Manifest{Module: "gitea", Moved: moved, Provides: []catalogue.Offer{offer}}
}
shared := func(moved bool) catalogue.Manifest {
return catalogue.Manifest{Module: "gitea", Moved: moved, OwnSecrets: catalogue.OwnSecrets{"key": {Path: "/k"}},
Provides: []catalogue.Offer{{Name: "gitea-read", Credential: &catalogue.OfferCredential{Own: "key"}}}}
}
given := func() catalogue.Manifest {
return catalogue.Manifest{Module: "gitea", Moved: true, Provides: []catalogue.Offer{{Name: "gitea-read",
Credential: &catalogue.OfferCredential{IssuedBy: catalogue.IssuedOutside}}}}
}
need := catalogue.Needed{Name: "gitea-read", From: "novox", Module: "gitea", For: "claude-code"}
for why, c := range map[string]struct {
shelf map[string]catalogue.Manifest
need catalogue.Needed
want bool
}{
"moved and issued by its software": {map[string]catalogue.Manifest{"gitea": forge(true, true)}, need, true},
"not moved": {map[string]catalogue.Manifest{"gitea": forge(false, true)}, need, false},
"a login for each consumer": {map[string]catalogue.Manifest{"gitea": forge(true, false)}, need, true},
"a login for each, not moved": {map[string]catalogue.Manifest{"gitea": forge(false, false)}, need, false},
"a credential every holder shares": {map[string]catalogue.Manifest{"gitea": shared(true)}, need, false},
"a credential a person gives": {map[string]catalogue.Manifest{"gitea": given()}, need, false},
"a provider not on the shelf": {map[string]catalogue.Manifest{}, need, false},
"answered by a record": {map[string]catalogue.Manifest{"gitea": forge(true, true)},
catalogue.Needed{Name: "gitea-read", ByRecord: true, Module: "gitea", For: "claude-code"}, false},
} {
if got := issuedToHolder(c.shelf, c.need); got != c.want {
t.Errorf("%s: issued to its holder is %v, want %v", why, got, c.want)
}
}
}
// **The vault is a moved provider for its consumers' credentials** (issue 793): its pair rows are the vault's since
// V3, so a need of `secret` is issued to its holder and answered by a moved provider, whether or not the module is
// marked moved; another module is not.
func TestTheModuleHoldingTheVaultSeatCountsAsMovedForItsConsumers(t *testing.T) {
vault := catalogue.Manifest{Module: "mesh-vault", Claims: []catalogue.Claim{{Name: "mesh-vault"}},
Provides: []catalogue.Offer{{Name: "secret"}}}
other := catalogue.Manifest{Module: "mesh-vault", Provides: []catalogue.Offer{{Name: "secret"}}}
need := catalogue.Needed{Name: "secret", Local: "admin", From: "novox", Module: "mesh-vault", For: "gitea"}
if !issuedToHolder(map[string]catalogue.Manifest{"mesh-vault": vault}, need) ||
!providerMoved(map[string]catalogue.Manifest{"mesh-vault": vault}, need) {
t.Error("the vault's consumer is not answered by a moved provider")
}
if issuedToHolder(map[string]catalogue.Manifest{"mesh-vault": other}, need) ||
providerMoved(map[string]catalogue.Manifest{"mesh-vault": other}, need) {
t.Error("a module not holding the vault seat counts as moved")
}
}
// **A provision answered by the helper makes its provider a moved one for that provision alone** (issue 792): the
// provides entry says `"answered-by": "helper"`, the module need not be marked moved, and no credential is the vault's
// for it. Another provision of the module, or a module without the entry, does not count.
func TestAnOfferAnsweredByTheHelperCountsAsMovedForThatProvisionOnly(t *testing.T) {
proxy := catalogue.Manifest{Module: "route-proxy", Provides: []catalogue.Offer{
{Name: "route", AnsweredBy: catalogue.AnsweredByHelper}, {Name: "other"}}}
plain := catalogue.Manifest{Module: "route-proxy", Provides: []catalogue.Offer{{Name: "route"}}}
shelf := func(m catalogue.Manifest) map[string]catalogue.Manifest {
return map[string]catalogue.Manifest{m.Module: m}
}
route := catalogue.Needed{Name: "route", From: "novox", Module: "route-proxy", For: "gitea"}
other := catalogue.Needed{Name: "other", From: "novox", Module: "route-proxy", For: "gitea"}
if !providerMoved(shelf(proxy), route) {
t.Error("a provision answered by the helper is not answered by a moved provider")
}
if providerMoved(shelf(proxy), other) || providerMoved(shelf(plain), route) {
t.Error("a provision not answered by the helper counts as moved")
}
if issuedToHolder(shelf(proxy), route) {
t.Error("the flag made a credential the vault's")
}
}
cmd/mesh-controller/issued_to_vault.go
+67
View File
@@ -0,0 +1,67 @@
package main
import (
"context"
"fmt"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// A covered account is handed to the vault when it is minted (novox/hq ADR 0374, issue 787).
//
// **Covered** are the bus accounts `module issue` mints for a core module (catalogue.CoreCredentialModules) or for a
// module marked moved; no other account is. Until the vault makes bus accounts (ADR 0113 clause 6, issue 774) the
// controller mints such an account and writes its hash to the bus's user list, as before, and then hands the whole
// value at once to the vault's `accept` with origin `issued`: sealed here to the key the vault made for that call, so
// the vault alone opens it (ADR 0349 clause 8). The vault keeps it as the module's own secret on that machine and
// raises `secret-changed`; a re-issue is the same hand with a new value, which replaces the old in place. An account
// not covered is minted and kept as it was and handed to nobody.
// vaultAskerForIssue is the vault as `module issue` asks it. A test replaces it.
var vaultAskerForIssue = func(ctx context.Context) vaultAsker { return theVault(ctx) }
// accountIsCovered says whether the bus accounts minted for m are the vault's to hold (ADR 0374 clause 1).
func accountIsCovered(m catalogue.Manifest) bool {
return catalogue.CarriesVaultSealed(m.Module) || m.Moved
}
// vaultInstalled says whether the vault is assigned to any machine. Until it is, as in genesis (ADR 0113 §15 to §18,
// mesh-host's bootstrap, which issues the bus's and the provisioner's accounts before the vault step), there is no
// vault to hand to: the value is kept in the controller's store as before, and the vault takes it afterwards by the
// move (move-in takes a value of the controller's store with origin made, accepted or issued, as it is). Only a
// vault that is installed and does not answer fails the issue.
func vaultInstalled(ctx context.Context, inv *inventory.Inventory) (bool, error) {
entries, err := inv.Catalogued(ctx)
if err != nil {
return false, err
}
for _, e := range entries {
if e.Manifest.Module == "mesh-vault" && len(e.On) > 0 {
return true, nil
}
}
return false, nil
}
// handIssuedToVault hands the value just minted for module's own secret name on node to the vault, when the module's
// accounts are covered, and does nothing otherwise. Loud when the vault does not take it: the bus's user list
// already holds the new hash, so a vault that kept the old value would serve a password that no longer logs in.
func handIssuedToVault(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node, name, held string) error {
if !accountIsCovered(m) {
return nil
}
if installed, err := vaultInstalled(ctx, inv); err != nil {
return fmt.Errorf("whether the vault is installed cannot be read, so %s of %s was not handed to it: %w", name,
m.Module, err)
} else if !installed {
return nil
}
target := ownSecretArgs(node, m.Module, name)
target["origin"] = "issued"
if _, err := acceptIntoVault(vaultAskerForIssue(ctx), target, held, "module issue"); err != nil {
return fmt.Errorf("the credential is minted and kept here, and the bus's user list holds its hash, but the "+
"vault did not take %s of %s on %s: %w (`module issue` again hands a new one)", name, m.Module, node, err)
}
return nil
}
cmd/mesh-controller/issued_to_vault_test.go
+157
View File
@@ -0,0 +1,157 @@
package main
import (
"context"
"errors"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
)
// Every other test mints without a vault to hand to: a stand-in that takes anything, replaced by the tests below.
func init() {
vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask }
}
// A covered account is handed to the vault when it is minted (novox/hq ADR 0374, issue 787): the whole value, sealed
// to the key the vault made for the call, with origin issued. A module that is neither core nor moved hands nothing.
func TestModuleIssueHandsACoveredAccountToTheVaultAndOnlyThat(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
theVaultIsInstalled(t, open)
inv := open.inventory
known := broker.Broker{Fingerprint: "ab"}
core := aProvisioner()
core.Module = "nats"
moved := aBot("moved-bot", true)
moved.OwnSecrets["broker"] = catalogue.OwnSecret{Path: "/var/lib/moved-bot/broker"}
plain := aBot("kept-bot", false)
plain.OwnSecrets["broker"] = catalogue.OwnSecret{Path: "/var/lib/kept-bot/broker"}
vault := newStandInVault(t)
vaultAskerForIssue = func(context.Context) vaultAsker { return vault.ask }
t.Cleanup(func() {
vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask }
})
for _, m := range []catalogue.Manifest{core, moved, plain} {
register(t, open, m)
if _, err := assign(ctx, open, "laptop", m.Module); err != nil {
t.Fatal(err)
}
}
issue := func(m catalogue.Manifest, password string) {
t.Helper()
if err := issueWith(ctx, inv, m, "laptop", "", known, "10.0.0.1:4222", "module."+m.Module+".laptop", password); err != nil {
t.Fatal(err)
}
}
issue(core, "first-core-password")
issue(moved, "first-moved-password")
issue(plain, "kept-password")
if err := issueProvisionerClient(ctx, inv, core, "laptop", known, "10.0.0.1:4222"); err != nil {
t.Fatal(err)
}
said := vault.said()
for _, want := range []string{`"module":"nats"`, `"module":"moved-bot"`, `"name":"broker"`,
`"name":"` + broker.ProvisionerSecret + `"`, `"node":"laptop"`} {
if !strings.Contains(said, want) {
t.Errorf("the vault was told %s, without %s", said, want)
}
}
if n := strings.Count(said, `"origin":"issued"`); n != 3 {
t.Errorf("%d accounts were handed with origin issued, want 3 (the core's two and the moved module's): %s", n, said)
}
if strings.Contains(said, "kept-bot") {
t.Errorf("an account that is neither core nor moved was handed to the vault: %s", said)
}
if strings.Contains(said, "first-core-password") || strings.Contains(said, "first-moved-password") {
t.Errorf("a value crossed the bus in the clear: %s", said)
}
// A re-issue hands the new value, which the vault opens: the whole {url, fingerprint, node, user, password}.
issue(moved, "second-moved-password")
if got := vault.taken["accept"]; !strings.Contains(got, "second-moved-password") ||
!strings.Contains(got, `"user":"module.moved-bot.laptop"`) || !strings.Contains(got, `"node":"laptop"`) ||
!strings.Contains(got, `"url":"nats://10.0.0.1:4222"`) || !strings.Contains(got, `"fingerprint":"ab"`) {
t.Errorf("the vault was handed %q after a re-issue, not the new account whole", got)
}
}
// A vault that does not take the value fails the issue loudly: the bus's user list holds the new hash already.
func TestModuleIssueFailsWhenTheVaultDoesNotTakeACoveredAccount(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
moved := aBot("moved-bot", true)
moved.OwnSecrets["broker"] = catalogue.OwnSecret{Path: "/var/lib/moved-bot/broker"}
theVaultIsInstalled(t, open)
register(t, open, moved)
if _, err := assign(ctx, open, "laptop", moved.Module); err != nil {
t.Fatal(err)
}
vault := newStandInVault(t)
vault.refuse = errors.New("the vault is not there")
vaultAskerForIssue = func(context.Context) vaultAsker { return vault.ask }
t.Cleanup(func() {
vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask }
})
err := issueWith(ctx, open.inventory, moved, "laptop", "", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222",
"module.moved-bot.laptop", "pw")
if err == nil || !strings.Contains(err.Error(), "vault did not take") {
t.Fatalf("a vault that took nothing was reported as %v", err)
}
}
func theVaultIsInstalled(t *testing.T, open *stores) {
t.Helper()
vault := catalogue.Manifest{Module: "mesh-vault", Version: "1"}
register(t, open, vault)
if _, err := assign(t.Context(), open, "laptop", vault.Module); err != nil {
t.Fatal(err)
}
}
// In genesis the bus's and the provisioner's accounts are issued before the vault is installed (ADR 0113 §15 to §18):
// there is no vault to hand to, so nothing is asked and nothing fails, and the value stays in the controller's store
// as before, for the move to carry into the vault. Once the vault is installed, one that does not answer fails.
func TestModuleIssueBeforeTheVaultIsInstalledKeepsTheValueAndHandsNothing(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
known := broker.Broker{Fingerprint: "ab"}
core := aProvisioner()
core.Module = "nats"
register(t, open, core)
if _, err := assign(ctx, open, "laptop", core.Module); err != nil {
t.Fatal(err)
}
vault := newStandInVault(t)
vault.refuse = errors.New("there is no vault yet")
vaultAskerForIssue = func(context.Context) vaultAsker { return vault.ask }
t.Cleanup(func() {
vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask }
})
if err := issueWith(ctx, inv, core, "laptop", "", known, "10.0.0.1:4222", "module.nats.laptop", "pw"); err != nil {
t.Fatalf("genesis's issue of the bus's account failed with no vault installed: %v", err)
}
if err := issueProvisionerClient(ctx, inv, core, "laptop", known, "10.0.0.1:4222"); err != nil {
t.Fatalf("genesis's issue of the client's account failed with no vault installed: %v", err)
}
if said := vault.said(); said != "" {
t.Errorf("the vault was asked before it was installed: %s", said)
}
if _, held, err := inv.ModuleSecretIfIssued(ctx, "laptop", "nats", "broker"); err != nil || !held {
t.Errorf("the value is not kept in the controller's store (held %v, %v)", held, err)
}
theVaultIsInstalled(t, open)
if err := issueWith(ctx, inv, core, "laptop", "", known, "10.0.0.1:4222", "module.nats.laptop", "pw2"); err == nil {
t.Error("a vault that is installed and does not answer did not fail the issue")
}
}
cmd/mesh-controller/lean_walk_test.go
+56 -2
View File
@@ -3,6 +3,7 @@ package main
import (
"context"
"slices"
"strings"
"testing"
"time"
@@ -30,13 +31,13 @@ func TestAWalkEndsOnItsMachinesReports(t *testing.T) {
{Node: "novox", Outcome: inventory.OutcomeFailed, At: &at, Declared: "d-novox"},
{Node: "shanks", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-before"},
}
waiting, failed, silent := walkReports(p, reports, now)
waiting, failed, silent := walkReports(p, reports, now, nil)
if !slices.Equal(waiting, []string{"shanks"}) || !slices.Equal(failed, []string{"novox"}) ||
!slices.Equal(silent, []string{"laptop"}) {
t.Fatalf("waiting %v, failed %v, silent %v", waiting, failed, silent)
}
reports[2].Declared = "d-shanks"
if waiting, _, _ := walkReports(p, reports, now); len(waiting) != 0 {
if waiting, _, _ := walkReports(p, reports, now, nil); len(waiting) != 0 {
t.Fatalf("the walk still waits once every machine reported: %v", waiting)
}
}
@@ -186,3 +187,56 @@ func TestAMachineSilentPastItsBoundIsLeftAndSaidAtOnce(t *testing.T) {
t.Fatalf("the silence is not said: %+v", obs)
}
}
// Issue 840 (novox/hq ADR 0362 §1-5): a walk's end counts a machine's send of a provisioned module applied only once
// the engine and the machine's provisioner client both said so.
func TestAWalkEndWaitsForTheProvisioner(t *testing.T) {
now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)
at := now.Add(-10 * time.Second)
plan := func(sentAt time.Time) inventory.Plan {
return inventory.Plan{Modules: map[string]*inventory.PlanModule{"app": {Rest: map[string]inventory.SentDeclaration{
"ace": {Digest: "d-ace", At: sentAt, Provisioned: map[string]string{"app": "t1"}},
"g14": {Digest: "d-g14", At: sentAt, Provisioned: map[string]string{"app": "t1"}},
}}}}
}
reports := []inventory.Reported{
{Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"},
{Node: "g14", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-g14"},
}
says := map[string]inventory.ProvReport{}
prov := func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok }
applied := inventory.ProvReport{Status: inventory.ProvApplied, Templates: "t1", At: at}
// (a) the engine applied, the client has not said: the end waits on both.
waiting, failed, silent := walkReports(plan(now.Add(-time.Minute)), reports, now, prov)
if !slices.Equal(waiting, []string{"ace", "g14"}) || len(failed)+len(silent) != 0 {
t.Fatalf("a: waiting %v, failed %v, silent %v", waiting, failed, silent)
}
// ace's client applied, g14's failed on that build (b): ace done, g14 a failed apply naming the reason.
says["ace/app"] = applied
says["g14/app"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t1", Reason: "no value", At: at}
waiting, failed, silent = walkReports(plan(now.Add(-time.Minute)), reports, now, prov)
if len(waiting) != 0 || len(silent) != 0 || len(failed) != 1 || !strings.HasPrefix(failed[0], "g14 (") ||
!strings.Contains(failed[0], "no value") {
t.Fatalf("b: waiting %v, failed %v, silent %v", waiting, failed, silent)
}
// (c) no client report past ApplySilentAfter: named silent, never applied.
delete(says, "g14/app")
waiting, failed, silent = walkReports(plan(now.Add(-inventory.ApplySilentAfter-time.Minute)), reports, now, prov)
if len(waiting) != 0 || len(failed) != 0 || !slices.Equal(silent, []string{"g14"}) {
t.Fatalf("c: waiting %v, failed %v, silent %v", waiting, failed, silent)
}
// both said it: nothing waits.
says["g14/app"] = applied
if waiting, failed, silent = walkReports(plan(now.Add(-time.Minute)), reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 {
t.Fatalf("both said: waiting %v, failed %v, silent %v", waiting, failed, silent)
}
// (d) a send handing nothing to the provisioner is the engine's report alone, as before.
bare := plan(now.Add(-time.Minute))
bare.Modules["app"].Rest = map[string]inventory.SentDeclaration{"ace": {Digest: "d-ace", At: now.Add(-time.Minute)}}
if waiting, failed, silent = walkReports(bare, reports, now, func(string, string) (inventory.ProvReport, bool) {
return inventory.ProvReport{}, false
}); len(waiting)+len(failed)+len(silent) != 0 {
t.Fatalf("d: waiting %v, failed %v, silent %v", waiting, failed, silent)
}
}
cmd/mesh-controller/main.go
+9 -1
View File
@@ -121,6 +121,10 @@ func run() error {
return imagesCommand(ctx, args[1:])
case "mirrors":
return mirrorsCommand(ctx, args[1:])
case "move-placement":
return movePlacementCommand(ctx, args[1:])
case "move-config":
return moveConfigCommand(ctx, args[1:])
case "plans":
return plansCommand(ctx, args[1:])
case "delivery":
@@ -275,6 +279,10 @@ func usage() {
artifacts [--json] [--repository <r>] [--collected] every artifact the mesh made: kept and why, eligible, collected (ADR 0251)
collect [--json] [--most <n>] what the store's sweep would let go of, now; nothing is done
collect --confirm --why <text> [--most <n>] run the sweep now: hold every kept archive, let go of the eligible
move-placement [--json] which settings layers still hold placement keys (endpoints, expose, reach, ports); nothing is moved
move-placement --confirm --why <text> move them beside the assignment, once, after the delivery (issue 736)
move-config [--json] every config value the settings layers hold, and where in mesh-config it goes; nothing is set
move-config --confirm --why <text> set them in mesh-config, read them back, and read config from there (issue 664)
images <node> [--json] every container image the machine's declaration names, now and as last sent
seats [--json] every seat this mesh defines, what it delivers, and who holds it
seat rename <from> <to> rename a seat; its former name still resolves (ADR 0122)
@@ -331,7 +339,7 @@ func usage() {
licence add|list|use|key model access, under the name a person calls it
licence manager <name> <node> the node that holds a refreshable licence's refresh token
licence refresh <name> mint a new access token and seal it to every holder
rotate <provision> [--consumer <n>] [--module <m>] a new credential for every holder, both ends at once
rotate <provision> [--consumer <n>] [--module <m>] [--why <text>] a new credential for every holder, both ends at once (--why where the vault holds it)
ask <module> <tool> [json] call one of a module's tools over the broker, and print its answer
pin <node> <provision> <from-node> <module>
which provider this one gets a provision from: the module, and its node
cmd/mesh-controller/merge_gate.go
+48 -5
View File
@@ -550,8 +550,17 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve
out[k] = m
}
var failures []string
// The list of predefined keys at the tree's root (novox/hq ADR 0348 §7): what the build of this tree judges each
// manifest against and registration publishes, so what the gate judges each against too. A list that is
// wrong is refused by name, and every manifest is then judged against none.
predefined, err := catalogue.ReadPredefined(tree)
if err != nil {
failures = append(failures, fmt.Sprintf("%s: the controller judging this (%s) refuses it — %s",
catalogue.PredefinedFile, version, oneLine(err.Error())))
predefined = nil
}
found := map[string]string{} // module → its directory in the tree
err := filepath.WalkDir(tree, func(path string, d fs.DirEntry, err error) error {
err = filepath.WalkDir(tree, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
@@ -575,7 +584,19 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve
if err != nil {
return err
}
m, err := catalogue.ParseManifest(raw)
m, err := catalogue.ParseManifestWith(raw, predefined)
// The manifest as written, still naming its templates: what the stand-in build resolves, as a real build
// does, so a moved module is handed its templates (issue 876).
written := m
var texts map[string]string
if err == nil {
// Each template's text from the tree beside the manifest, as the build reads it back from the archive it
// packs (novox/hq ADR 0349 §2): so a change to a template alone is a change to its module.
m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(path))
}
if err == nil && written.Templated() {
texts, err = catalogue.ReadTemplates(written, filepath.Dir(path))
}
if err != nil {
if inScope(dir) {
failures = append(failures, fmt.Sprintf("%s: the controller judging this (%s) refuses it — %s",
@@ -600,12 +621,12 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve
"of one module name (issue 239)", orRoot(dir), m.Module, s.Repository))
return nil
}
resolved, err := standInBuild(m)
resolved, err := standInBuild(written, texts)
if err == nil {
// As registration reads a built manifest: strictly, again (takeIn).
var raw []byte
if raw, err = json.Marshal(resolved); err == nil {
resolved, err = catalogue.ParseManifest(raw)
resolved, err = catalogue.ParseManifestWith(raw, predefined)
}
}
if err == nil {
@@ -723,15 +744,24 @@ func builtAlike(m catalogue.Manifest) string {
// standInBuild resolves a manifest as a build would, with artifacts nobody built: a digest made from the
// module's and the artifact's names, in the store's own vocabulary. Composition and validation read the
// shape of a reference, never its bytes.
func standInBuild(m catalogue.Manifest) (catalogue.Manifest, error) {
func standInBuild(m catalogue.Manifest, texts map[string]string) (catalogue.Manifest, error) {
if m.Build == nil {
return m.Resolve(nil)
}
// The templates archive carries each template's text by its path inside the archive, as a build reads it back.
archive, inside := catalogue.TemplatesArtifact(m), m.TemplatesDir()+"/"
read := map[string]string{}
for template, text := range texts {
read[strings.TrimPrefix(template, inside)] = text
}
var built []catalogue.Built
for _, a := range m.Build.Artifacts {
sum := sha256.Sum256([]byte("gate\x00" + m.Module + "\x00" + a.Name))
digest := fmt.Sprintf("sha256:%x", sum)
b := catalogue.Built{Name: a.Name, Kind: a.Kind, Digest: digest}
if a.Name == archive && archive != "" {
b.Texts = read
}
switch a.Kind {
case catalogue.ArtifactImage, catalogue.ArtifactUpstream:
b.Reference = catalogue.ArtifactStoreScheme + m.Module + "/" + a.Name + "@" + digest
@@ -1083,6 +1113,19 @@ func raiseFromFacts(ctx context.Context, open *stores, f snapshot.Facts, shelf m
continue
}
}
// And the provisioner client's account, the same way (novox/hq ADR 0349 clause 11, design 57 P3).
if a.Provider == "" && a.Name == broker.ProvisionerSecret {
if pm, held := shelf[a.Module]; !held || !runsTheProvisionerClient(pm) {
continue
}
if _, err := inv.MintBusPassword(ctx, inventory.BusUser{Kind: inventory.BusProvisioner, Node: m.Name,
Module: a.Module, Username: broker.Principal{Kind: broker.KindProvisioner, Node: m.Name,
Module: a.Module}.Username()}); err != nil {
notes = append(notes, fmt.Sprintf("%s's provisioner client account for %s is not kept: %s",
m.Described(), a.Module, oneLine(err.Error())))
continue
}
}
var err error
if a.Provider == "" {
err = inv.AcceptSecretForModule(ctx, m.Name, a.Module, a.Name, standIn)
cmd/mesh-controller/mesh_for_test.go
+15 -1
View File
@@ -33,7 +33,21 @@ import (
func aMesh(t *testing.T) *stores {
t.Helper()
inventory.ForTest(t) // raises the store, migrates it, and points the environment at it
licences.ForTest(t) // planning reaches this one too, by name and never by connection
return aMeshOnTheStore(t)
}
// aMeshBefore is aMesh on a store migrated only below migration `number`, with the function that runs the
// rest: for a test of what a migration that moves data does to what the mesh composes (novox/hq issue 651).
func aMeshBefore(t *testing.T, number int) (*stores, func()) {
t.Helper()
_, migrate := inventory.ForTestBefore(t, number)
return aMeshOnTheStore(t), migrate
}
// aMeshOnTheStore is the mesh aMesh makes, on the store the environment points at.
func aMeshOnTheStore(t *testing.T) *stores {
t.Helper()
licences.ForTest(t) // planning reaches this one too, by name and never by connection
open, err := openStores(t.Context())
if err != nil {
cmd/mesh-controller/modules.go
+74 -1
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"net"
"os"
"path/filepath"
"sort"
"strings"
"time"
@@ -127,6 +128,10 @@ func moduleCommand(ctx context.Context, args []string) error {
if err != nil {
return err
}
// A manifest handed over by hand takes each template's text from the tree beside it (novox/hq ADR 0349 §2).
if m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(positionals[0])); err != nil {
return err
}
from, err := whereItComesFrom(*repo, *ref, *commit, *path, *self)
if err != nil {
return err
@@ -441,6 +446,10 @@ func settingsCommand(ctx context.Context, args []string) error {
if len(positionals) == 2 {
values = positionals[1]
}
// Once its config values are mesh-config's, they are proposed there (issue 664).
if forwarded, err := forwardPropose(ctx, inv, positionals[0], *node, values, *clear); forwarded || err != nil {
return err
}
return proposeCommand(ctx, positionals[0], *node, values, *clear, *replace)
case "proposals":
if len(positionals) > 1 || *node != "" || *clear || *replace || *history {
@@ -461,6 +470,15 @@ func settingsCommand(ctx context.Context, args []string) error {
where = *node
}
// Once the move of config has run, a module in the config form is read and set through mesh-config (issue 664).
if (args[0] == "set" && len(positionals) == 2) || ((args[0] == "show" && !*history) || args[0] == "clear") &&
len(positionals) == 1 {
if forwarded, err := forwardSettings(ctx, inv, args[0], positionals[0], *node, positionals, *replace); forwarded ||
err != nil {
return err
}
}
switch args[0] {
case "set":
if len(positionals) != 2 {
@@ -537,6 +555,10 @@ func settingsCommand(ctx context.Context, args []string) error {
if p.SetBy != "" {
by = "; " + p.SetBy
}
if p.Why != "" {
// A layer whose placement moved out of it (migration 0097): still the layer, as it stands.
by += "; " + p.Why
}
fmt.Printf("%s on %s, until %s (%s%s):\n %s\n", positionals[0], where,
p.ReplacedAt.Local().Format("2006-01-02 15:04:05"), p.ReplacedBy, by, shown)
}
@@ -940,7 +962,55 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
if err != nil {
return err
}
return issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password)
if err := issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password); err != nil {
return err
}
if !runsTheProvisionerClient(m) {
return nil
}
return issueProvisionerClient(ctx, inv, m, node, known, reachable)
}
// runsTheProvisionerClient says a module runs its machine's provisioner client, which reaches the bus on an
// account of its own (novox/hq ADR 0349 clause 11, design 57 P3): it claims the seat and declares the own secret
// that account is delivered in. The same test the bus's user list makes (inventory's declaredFor).
func runsTheProvisionerClient(m catalogue.Manifest) bool {
_, reads := m.OwnSecrets[broker.ProvisionerSecret]
return reads && m.ClaimsSeat(broker.NodeProvisionerSeat)
}
// issueProvisionerClient gives a machine's provisioner client its account: minted under the client's own name,
// never the module's or the tool runner's, and sealed to the machine as the module's own secret
// broker.ProvisionerSecret, which the client's service reads (novox/hq ADR 0349 clause 11, design 57 P3). No
// claims and no consumer: the client holds no seat on this account and is delivered nothing but its answers.
func issueProvisionerClient(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node string,
known broker.Broker, reachable string) error {
user := broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: m.Module}.Username()
password, err := inv.MintBusPassword(ctx, inventory.BusUser{
Username: user, Kind: inventory.BusProvisioner, Node: node, Module: m.Module,
})
if err != nil {
return err
}
held, err := json.Marshal(struct {
URL string `json:"url"`
Fingerprint string `json:"fingerprint,omitempty"`
Node string `json:"node"`
User string `json:"user"`
Password string `json:"password"`
}{URL: "nats://" + reachable, Fingerprint: known.Fingerprint, Node: node, User: user, Password: password})
if err != nil {
return err
}
if err := inv.AcceptSecretForModule(ctx, node, m.Module, broker.ProvisionerSecret, string(held)); err != nil {
return err
}
if err := handIssuedToVault(ctx, inv, m, node, broker.ProvisionerSecret, string(held)); err != nil {
return err
}
fmt.Printf("bus user %s minted for %s's provisioner client, scoped to %s's keys, its reports and the vault\n",
user, m.Module, node)
return nil
}
// busKindOf is what a module's bus user is recorded as: the node's tool runtime where the module is
@@ -985,6 +1055,9 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
if err := inv.AcceptSecretForModule(ctx, node, m.Module, "broker", string(held)); err != nil {
return err
}
if err := handIssuedToVault(ctx, inv, m, node, "broker", string(held)); err != nil {
return err
}
// And how it hears what it consumes. Derived from its declaration, and only when it declared
// something: a module that consumes nothing needs no consumer, and creating one would be a
cmd/mesh-controller/placement_move.go
+99
View File
@@ -0,0 +1,99 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"os"
"strings"
"github.com/novox/mesh-controller/internal/inventory"
)
// movePlacementAnswer is what the move of placement says it did, or would do (issue 736).
type movePlacementAnswer struct {
// DryRun is true when nothing was moved: asked without --confirm.
DryRun bool `json:"dry-run"`
// Moved is each layer that held a placement key among its settings, and which keys.
Moved []inventory.MovedLayer `json:"moved"`
}
// movePlacementCommand is `move-placement`: the move of placement out of the settings layers (novox/hq ADR 0348
// clause 11, issue 736), one of the moves of data a release runs once after its delivery (design 57).
func movePlacementCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("move-placement", flag.ContinueOnError)
asJSON := set.Bool("json", false, "answer as JSON")
confirm := set.Bool("confirm", false, "move the keys, rather than only say which layers hold them")
f := addHandActFlags(set)
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if len(positionals) != 0 {
return errors.New("move-placement [--json] [--confirm --why <text>]")
}
if !*confirm && f.given() {
return errors.New("move-placement takes --why only with --confirm: without it nothing is moved. Nothing was done")
}
if *confirm {
// A move a person asks for says why, before anything is asked of the store.
if err := f.require("move-placement"); err != nil {
return err
}
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
if *confirm {
f.record(ctx, "move-placement", nil)
}
answer, err := runMovePlacement(ctx, open.inventory, *confirm)
if err != nil {
return err
}
if *asJSON {
encoder := json.NewEncoder(os.Stdout)
encoder.SetIndent("", " ")
return encoder.Encode(answer)
}
fmt.Print(describeMovePlacement(answer))
return nil
}
// runMovePlacement is the move against a store: a dry run says what it would move.
func runMovePlacement(ctx context.Context, inv *inventory.Inventory, apply bool) (movePlacementAnswer, error) {
moved, err := inv.MovePlacement(ctx, apply)
if err != nil {
return movePlacementAnswer{}, err
}
return movePlacementAnswer{DryRun: !apply, Moved: moved}, nil
}
// describeMovePlacement says the answer as a person reads it.
func describeMovePlacement(a movePlacementAnswer) string {
if len(a.Moved) == 0 {
return "no settings layer holds a placement key: nothing to move\n"
}
var b strings.Builder
verb := "moved"
if a.DryRun {
verb = "would move"
}
for _, m := range a.Moved {
where := m.Node
if where == "" {
where = "the whole mesh"
}
fmt.Fprintf(&b, "%s on %s: %s %s\n", m.Module, where, verb, strings.Join(m.Keys, ", "))
}
if a.DryRun {
fmt.Fprintf(&b, "%d layer(s); nothing was moved — with --confirm --why <text> they are\n", len(a.Moved))
} else {
fmt.Fprintf(&b, "%d layer(s) moved beside their assignments, each kept whole in its history first\n", len(a.Moved))
}
return b.String()
}
cmd/mesh-controller/placement_moved_test.go
+113
View File
@@ -0,0 +1,113 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"sort"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// **Placement leaves the settings layers and no machine is told anything new** (novox/hq ADR 0348 clause
// 11, issue 651). A mesh is seeded as the store holds it before migration 0097 — `places`, `accesses` and a
// kept network among the settings, at the mesh's level and a machine's, one layer of placement only — then
// migrated, and every machine's declaration is composed. What each machine is told of the seeded modules is
// byte for byte what the controller before this change composed from the same seed: the golden files were
// written by the integration branch's controller at bfc79c6 running this same test (MESH_WRITE_GOLDEN=1),
// where there was no migration 0097 to run.
func TestADeclarationIsTheSameAfterPlacementMoves(t *testing.T) {
open, migrate := aMeshBefore(t, 97)
ctx := t.Context()
inv := open.inventory
image := "registry.invalid:5000/library@sha256:" + strings.Repeat("a", 64)
register(t, open, catalogue.Manifest{Module: "library", Version: "1",
Accesses: []catalogue.Access{{ID: "series"}, {ID: "downloads", Mode: catalogue.AccessRead}},
Resources: []map[string]any{
{"id": "config", "type": "directory", "mode": "0750"},
{"id": "server", "type": "container", "image": image,
"volumes": []any{"${dir:config}:/config", "${access:series}:/series", "${access:downloads}:/downloads:ro"}},
// Today's form: a mergeable file naming no keys takes every key of the layer (ADR 0326 §3).
{"id": "runtime-config", "type": "file", "path": "${dir:config}/runtime.json", "mode": "0640",
"content": `{"log":"info"}`, "merge": "json"},
}})
register(t, open, catalogue.Manifest{Module: "found", Version: "1",
Resources: []map[string]any{{"id": "server", "type": "container", "image": image}}})
if err := inv.SetAdopted(ctx, "anchor", true); err != nil {
t.Fatal(err)
}
for _, l := range []struct{ node, module, values string }{
{"", "library", `{"accesses": {"downloads": "/storage/downloads"}, "log": "warn"}`},
{"anchor", "library", `{"places": {"config": {"path": "/var/lib/library/config", "owner": "1001:2000"}},
"accesses": {"series": "/storage/media/series"}, "workers": 4}`},
{"laptop", "library", `{"accesses": {"series": "/home/media/series"}}`},
{"anchor", "found", `{"networks": {"server": ["predecessor_default"]}}`},
} {
if l.node == "" {
inv.ExecForTest(t, `insert into settings (node, module, values) values (null, $1, $2)`, l.module, l.values)
continue
}
inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`,
l.node, l.module, l.values)
}
migrate()
// Assigned once the store is migrated: an assignment reads the layers, as this controller holds them.
for _, a := range [][]string{{"anchor", "library", "found"}, {"laptop", "library"}} {
if _, err := assign(ctx, open, a[0], a[1:]...); err != nil {
t.Fatalf("assign %v: %v", a, err)
}
}
gens, err := generators(ctx, open)
if err != nil {
t.Fatal(err)
}
for _, node := range []string{"anchor", "laptop"} {
plan, settings, err := planFor(ctx, open, node)
if err != nil {
t.Fatal(err)
}
with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Allocating)
if err != nil {
t.Fatal(err)
}
composed, err := plan.Compose(with)
if err != nil {
t.Fatal(err)
}
if len(composed.LeftOut) > 0 {
t.Fatalf("%s leaves out %v", node, composed.LeftOut)
}
var told []map[string]any
for _, r := range composed.Resources {
if owner := composed.Owner[r["id"].(string)]; owner == "library" || owner == "found" {
told = append(told, r)
}
}
sort.Slice(told, func(i, j int) bool { return told[i]["id"].(string) < told[j]["id"].(string) })
got, err := json.MarshalIndent(told, "", " ")
if err != nil {
t.Fatal(err)
}
golden := filepath.Join("testdata", "placement-moved", node+".json")
if os.Getenv("MESH_WRITE_GOLDEN") != "" {
if err := os.MkdirAll(filepath.Dir(golden), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(golden, append(got, '\n'), 0o644); err != nil {
t.Fatal(err)
}
continue
}
want, err := os.ReadFile(golden)
if err != nil {
t.Fatal(err)
}
if string(want) != string(got)+"\n" {
t.Errorf("%s is told something else after placement moved:\n%s\nwas:\n%s", node, got, want)
}
}
}
cmd/mesh-controller/placement_words_moved_test.go
+243
View File
@@ -0,0 +1,243 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"reflect"
"sort"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/link"
)
// wordedMesh is every live layer that says `endpoints`, `expose`, `reach` or `ports` (issue 736) — 36, read with
// the settings verb on 2026-10-14, ace's as anchor's and novox's as laptop's, one stand-in for an address — split as
// the store holds them once migration 0097 has run: the four words still among the settings, `places` and
// `accesses` already beside them. With each module shaped as its definition in the catalogue (mesh-catalog's, and
// the media catalogue's) is where the four words reach: its listens, what its containers publish and on which
// network, its route, its directories and accesses, and its mergeable files — every key of the layer, or the keys
// they name. The shapes are the modules'; the files' text and the images are the test's.
type wordedMesh struct {
Modules []json.RawMessage `json:"modules"`
Layers []struct {
Node string `json:"node"`
Module string `json:"module"`
Settings map[string]any `json:"settings"`
Placement map[string]any `json:"placement"`
} `json:"layers"`
}
func theWordedMesh(t *testing.T) wordedMesh {
t.Helper()
raw, err := os.ReadFile(filepath.Join("testdata", "placement-words-moved", "mesh.json"))
if err != nil {
t.Fatal(err)
}
var w wordedMesh
if err := json.Unmarshal(raw, &w); err != nil {
t.Fatal(err)
}
return w
}
// aMeshWithTheWordsInItsLayers is aMesh with every module of the worded mesh registered, a route provider on both
// machines, and its layers held as the store holds them before the four words move.
func aMeshWithTheWordsInItsLayers(t *testing.T) (*stores, wordedMesh) {
t.Helper()
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
w := theWordedMesh(t)
image := "registry.invalid:5000/library@sha256:" + strings.Repeat("a", 64)
register(t, open, catalogue.Manifest{Module: "proxy", Version: "1", Provides: catalogue.FromAnywhere("route"),
Receives: map[string]string{"route": "/var/lib/proxy/routes.json"}})
for _, text := range w.Modules {
m, err := catalogue.ParseManifest([]byte(strings.ReplaceAll(string(text), "IMAGE", image)))
if err != nil {
t.Fatal(err)
}
register(t, open, m)
}
on := map[string][]string{"anchor": {"proxy"}, "laptop": {"proxy"}}
for _, l := range w.Layers {
settings, err := json.Marshal(l.Settings)
if err != nil {
t.Fatal(err)
}
inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`,
l.Node, l.Module, string(settings))
if l.Placement != nil {
placement, err := json.Marshal(l.Placement)
if err != nil {
t.Fatal(err)
}
inv.ExecForTest(t, `insert into placement (node, module, values) select id, $2, $3 from node where name = $1`,
l.Node, l.Module, string(placement))
}
on[l.Node] = append(on[l.Node], l.Module)
}
for _, node := range []string{"anchor", "laptop"} {
if err := inv.SetPublicDomain(ctx, node, node+".example.test"); err != nil {
t.Fatal(err)
}
if _, err := assign(ctx, open, node, on[node]...); err != nil {
t.Fatalf("assign %s %v: %v", node, on[node], err)
}
}
return open, w
}
// toldOfTheWords is what a machine is told of the modules whose layers say the four words, and of the route
// provider that serves their names: those modules' resources and the filter's rules, as JSON a person can diff.
func toldOfTheWords(t *testing.T, open *stores, node string) string {
t.Helper()
ctx := t.Context()
gens, err := generators(ctx, open)
if err != nil {
t.Fatal(err)
}
plan, settings, err := planFor(ctx, open, node)
if err != nil {
t.Fatal(err)
}
with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Allocating)
if err != nil {
t.Fatal(err)
}
composed, err := plan.Compose(with)
if err != nil {
t.Fatal(err)
}
if len(composed.LeftOut) > 0 {
t.Fatalf("%s leaves out %v", node, composed.LeftOut)
}
rules, err := plan.Rules(with)
if err != nil {
t.Fatal(err)
}
ours := map[string]bool{"proxy": true}
for _, l := range theWordedMesh(t).Layers {
ours[l.Module] = true
}
var told []map[string]any
for _, r := range composed.Resources {
if ours[composed.Owner[r["id"].(string)]] {
told = append(told, r)
}
}
sort.Slice(told, func(i, j int) bool { return told[i]["id"].(string) < told[j]["id"].(string) })
got, err := json.MarshalIndent(map[string]any{"resources": told, "rules": rules}, "", " ")
if err != nil {
t.Fatal(err)
}
return string(got) + "\n"
}
// sameAsTheGolden holds what a machine is told to the golden file the controller before issue 736 wrote from the
// same seed (MESH_WRITE_GOLDEN=1 writes it instead).
func sameAsTheGolden(t *testing.T, open *stores, when string) {
t.Helper()
for _, node := range []string{"anchor", "laptop"} {
got := toldOfTheWords(t, open, node)
golden := filepath.Join("testdata", "placement-words-moved", node+".json")
if os.Getenv("MESH_WRITE_GOLDEN") != "" {
if err := os.MkdirAll(filepath.Dir(golden), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(golden, []byte(got), 0o644); err != nil {
t.Fatal(err)
}
continue
}
want, err := os.ReadFile(golden)
if err != nil {
t.Fatal(err)
}
if string(want) != got {
t.Errorf("%s: %s is told something else:\n%s\nwas:\n%s", when, node, got, want)
}
}
}
// **The four words leave the settings layers and no machine is told anything new** (novox/hq ADR 0348 clause 11,
// issue 736). Every live layer that says `endpoints`, `expose`, `reach` or `ports` is seeded as the store holds it
// after delivery, on a module shaped as its definition is, and every machine's declaration is composed. What each
// is told is byte for byte what the controller before this change composed from the same seed.
func TestADeclarationIsTheSameWhileTheWordsAreStillInTheLayers(t *testing.T) {
open, _ := aMeshWithTheWordsInItsLayers(t)
sameAsTheGolden(t, open, "before the move")
}
// **And after the move, still nothing new** (issue 736). The move of placement, asked without confirming, says
// every layer that holds one of the four words and moves nothing; confirmed, it moves them all, no settings layer
// holds a placement key after, and each machine is told byte for byte what it was told before the move — and what
// the controller before this change told it. Asked again, it moves nothing.
func TestADeclarationIsTheSameAfterTheMoveOfPlacement(t *testing.T) {
open, w := aMeshWithTheWordsInItsLayers(t)
ctx := t.Context()
inv := open.inventory
asked, err := runMovePlacement(ctx, inv, false)
if err != nil || !asked.DryRun || len(asked.Moved) != len(w.Layers) {
t.Fatalf("the move, not confirmed, says %+v (%v); %d layers hold the words", asked, err, len(w.Layers))
}
if held := inv.QueryForTest(t, `select module from settings where values ?| $1`, catalogue.PlacementKeys()); len(held) != len(w.Layers) {
t.Fatalf("the move, not confirmed, moved something: %d layers still hold the words", len(held))
}
sameAsTheGolden(t, open, "after the move was asked without confirming")
moved, err := runMovePlacement(ctx, inv, true)
if err != nil || moved.DryRun || !reflect.DeepEqual(moved.Moved, asked.Moved) {
t.Fatalf("the move says %+v (%v), it said it would move %+v", moved, err, asked.Moved)
}
if held := inv.QueryForTest(t, `select module from settings where values ?| $1`, catalogue.PlacementKeys()); len(held) != 0 {
t.Fatalf("a settings layer still holds a placement key: %v", held)
}
sameAsTheGolden(t, open, "after the move")
if again, err := runMovePlacement(ctx, inv, true); err != nil || len(again.Moved) != 0 {
t.Fatalf("asked again, the move moved %+v (%v)", again, err)
}
if said := describeMovePlacement(movePlacementAnswer{}); !strings.Contains(said, "nothing to move") {
t.Fatalf("an empty move says %q", said)
}
}
// The verb's command line: a dry run unless confirmed, and a confirmed move says why, recorded as a person's step
// of the release, never a repair (issue 736).
func TestTheMoveOfPlacementIsADryRunUnlessConfirmed(t *testing.T) {
for _, c := range []struct {
args map[string]any
want string
}{
{map[string]any{}, "move-placement --json"},
{map[string]any{"confirm": "false"}, "move-placement --json"},
{map[string]any{"confirm": "true", "why": "release 57 is delivered"}, "move-placement --json --confirm --why release 57 is delivered"},
} {
argv, err := argvFor("move-placement", c.args)
if err != nil || strings.Join(argv, " ") != c.want {
t.Errorf("%v: %q %v, want %q", c.args, argv, err, c.want)
}
}
for _, args := range []map[string]any{
{"confirm": "true"}, // a move without a why
{"why": "tidy"}, // a why for a dry run, recorded nowhere
{"confirm": "yes", "why": "x"}, // a switch is true or false
{"node": "anchor"},
} {
if argv, err := argvFor("move-placement", args); err == nil {
t.Errorf("%v was composed as %q", args, argv)
}
}
if repairingCommand([]string{"move-placement", "--json", "--confirm", "--why", "x"}) != "move-placement" {
t.Error("a confirmed move through the generic verb would go unrecorded")
}
if repairingCommand([]string{"move-placement", "--json"}) != "" {
t.Error("a dry run was taken for an act by hand")
}
if !personsDecision(link.HandAct{Verb: "move-placement"}) {
t.Error("a move of the release would count toward a healer the mesh lacks")
}
}
cmd/mesh-controller/plain_words.go
+9
View File
@@ -186,6 +186,15 @@ var plainWordings = map[string]func(conditions.Observation) words{
}),
// The controller or the node-engine kept at the build a machine was last sent, over an older record (novox/hq
// issue 201).
kindVaultSilent: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: "The vault did not give " + m + " the core's credentials",
Needs: "check the vault answers (vault_status); the next send asks it again.",
Explanation: fmt.Sprintf("The bus's and the provisioner's credentials reach %s sealed by the vault, and "+
"the vault did not answer the last send for them. %s keeps the files it has; a core module not "+
"there yet is not placed. The mesh never seals them itself instead.", m, m),
Resolved: "Resolved: the vault gave " + m + " the core's credentials"}
}),
kindCoreDowngrade: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
component := idPart(o, 0)
cmd/mesh-controller/plan.go
+128 -4
View File
@@ -150,6 +150,10 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
// password a provider is told to create is the one its consumer was given — and sealed to
// this node before it was ever written down, so nothing between here and there can read it.
for i, n := range resolved.Needs {
// **Issued for this holder alone and kept by the vault** (issue 757): where this consumer is handed to the
// provisioner, its machine's client asks the vault for it, and the declaration carries no value.
resolved.Needs[i].IssuedToHolder = issuedToHolder(shelf, n)
resolved.Needs[i].ProviderMoved = providerMoved(shelf, n)
if n.ByRecord {
// Answered by something the mesh holds, so there is no pair-wise secret between two
// machines. Its key was supplied by a person and sealed to this node then; the mesh
@@ -476,9 +480,23 @@ func declarationWith(ctx context.Context, open *stores, node string,
out := sendable{Resources: composed.Resources, Adoption: adoption,
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld,
unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced,
modules: namedModules(plan, composed.LeftOut),
unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced, handedOnly: composed.HandedOnly,
modules: namedModules(plan, composed.LeftOut),
Filled: composed.Filled, plans: composed.Plans,
localNetwork: catalogue.LocalNetworkPorts(rules, with.LocalNetworks)}
if len(composed.Filled) > 0 {
out.Provisioner = with.Provisioner
}
// A handed module only pulled in by another keeps what requires it, for its value of its own in the assignments
// mirror (issue 765).
for module := range composed.Plans {
if by := plan.RequiredBy(module); by != "" {
if out.requiredBy == nil {
out.requiredBy = map[string]string{}
}
out.requiredBy[module] = by
}
}
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
// 0221). Read only on the send path: a question about what would be sent records nothing.
if choosing == Allocating {
@@ -591,6 +609,10 @@ func reportLeftOut(node string, declared sendable) {
for _, u := range declared.unplaced {
fmt.Printf("%s: %s\n", node, u)
}
// And each file only the provisioner's client fills, left out because the module is not handed here.
for _, u := range declared.handedOnly {
fmt.Printf("%s: %s\n", node, u)
}
}
// busCredentialIssued refuses an own secret called `broker` whose bus account nobody issued.
@@ -601,10 +623,17 @@ func reportLeftOut(node string, declared sendable) {
// cannot read and report the machine applied. Refused by name, with the verb — on the send, and on
// a question asked ahead of it (Foreseeing), so that one is never told the push will make it.
func busCredentialIssued(ctx context.Context, inv *inventory.Inventory, node, module, name string) error {
if name != "broker" {
var user string
switch name {
case "broker":
user = broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
case broker.ProvisionerSecret:
// And the machine's provisioner client's account, issued with its module (novox/hq ADR 0349 clause 11,
// design 57 P3): the same refusal, for the same reason.
user = broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: module}.Username()
default:
return nil
}
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
return err
} else if !minted {
@@ -728,7 +757,18 @@ func renderingFor(ctx context.Context, open *stores, node string,
// per node, so a module running on three machines has three.
needed := map[string]map[string]string{}
foreseen := map[string]map[string]bool{}
// **A module handed to the provisioner here has its secrets from the vault** (novox/hq ADR 0349 §7, §11,
// design 57 V4): its machine's client asks for each, so the controller makes, reads and seals none of them for
// this machine. Every other module's are made and sealed as before.
hands, err := engineTakesHanded(ctx, inv, node)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
handing := catalogue.Rendering{Hands: hands, Provisioner: provisionerAccount(plan)}
for _, m := range plan.Modules {
if handing.Handing(m) {
continue
}
// **A secret family is never made** (novox/hq ADR 0283): each member a person gave on this machine is
// placed, and one not given is nothing — the module says it waits for it.
for _, family := range m.OwnSecrets.Families() {
@@ -822,6 +862,22 @@ func renderingFor(ctx context.Context, open *stores, node string,
needed[m.Module][name] = sealed
}
}
// **The core's own credentials as the vault sealed them** (novox/hq ADR 0349 §16, issue 755), for an engine that
// opens the vault's box (core_box.go): the declaration carries the box, never the seal made above. Below that
// contract nothing here is asked and the composition is as it was.
var core coreBoxes
opensBox, err := engineOpensVaultBox(ctx, inv, node)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
if opensBox {
core, err = coreBoxesFor(ctx, inv, node, plan.Modules, func(m catalogue.Manifest) bool {
return handing.Handing(m)
}, choosing)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
}
// And a certificate for this machine's name inside the mesh, when anything on it asks. Issued
// rather than stored: the node's key does not change, so signing again produces an equally
// valid certificate and there is nothing to keep in step.
@@ -1016,8 +1072,11 @@ func renderingFor(ctx context.Context, open *stores, node string,
ReadsHealth: readsHealth,
JudgesRoot: judgesRoot,
ReadsBusy: readsBusy,
Hands: hands,
Provisioner: provisionerAccount(plan),
BusMembership: memberships[node],
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports,
VaultSealed: core.sealed, NotPlaced: core.notPlaced,
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
Machines: machines, Zones: zones, Holders: replicas,
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
@@ -1059,6 +1118,35 @@ func engineReadsBusy(ctx context.Context, inv *inventory.Inventory, node string)
return had && stated.Contract >= link.BusyContract, nil
}
// engineTakesHanded says whether a machine's node-engine writes a filled file on the provisioner client's word alone
// (link.HandedContract, design 57 P7), by its own newest statement: only there is a moved module handed to the
// provisioner (design 57 P6). Anywhere else a filled file would have two writers, and the module is sent as before.
func engineTakesHanded(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) {
stated, had, err := inv.HealthOf(ctx, node)
if err != nil {
return false, err
}
return had && stated.Contract >= link.HandedContract, nil
}
// provisionerAccount is the account the provisioner's client runs as on this machine (novox/hq ADR 0349 §4): the user
// the process of the module running it runs as, empty where no such module is on the machine.
func provisionerAccount(plan catalogue.Resolution) string {
for _, m := range plan.Modules {
if !runsTheProvisionerClient(m) {
continue
}
for _, r := range m.Resources {
if r["type"] == "process" {
if user, _ := r["user"].(string); user != "" && user != "root" {
return user
}
}
}
}
return ""
}
// busyNotSent is, for a node-engine that does not read a container's busy, each container whose module declares
// one: not sent it, so replaced at once while it may hold work (novox/hq ADR 0335). Nothing for one that reads it.
func busyNotSent(modules []catalogue.Manifest, reads bool) []string {
@@ -1846,6 +1934,42 @@ func issuedOutsideBy(shelf map[string]catalogue.Manifest, n catalogue.Needed) (s
return catalogue.OffersIssuedOutside(shelf, n.Name)
}
// issuedToHolder is whether the vault keeps a need's credential for its holder alone (issues 749, 757 and 779): its
// providing module is marked moved, so the vault holds the credential in a row of its own for each holder
// (filledTheNewWay), and its offer says either that the provider's software issues the provision's credential, or
// that the provider gives each consumer a login of its own (form pair, ADR 0158 §11). A credential every holder
// shares, or one a person gives, is neither.
func issuedToHolder(shelf map[string]catalogue.Manifest, n catalogue.Needed) bool {
if n.ByRecord || n.Module == "" {
return false
}
provider, known := shelf[n.Module]
if !known || !movedProvider(provider) {
return false
}
if _, issued := provider.IssuedCredentialOf(n.Name); issued {
return true
}
return provider.CredentialFormOf(n.Name) == catalogue.CredentialFormPair
}
// providerMoved is whether a need is answered by a module marked moved (issue 788), or by one whose offer of the
// provision says the provider helper answers it (issue 792).
func providerMoved(shelf map[string]catalogue.Manifest, n catalogue.Needed) bool {
if n.ByRecord || n.Module == "" {
return false
}
provider, known := shelf[n.Module]
return known && (movedProvider(provider) || provider.AnsweredByHelperFor(n.Name))
}
// movedProvider is whether a provider's credentials are the vault's: the module is marked moved, or it holds the
// mesh-vault seat, whose pair rows have been in the vault since V3 (issue 793), so its consumers' credentials
// (gitea's internal token and admin password) are filled from those very rows with the values unchanged.
func movedProvider(m catalogue.Manifest) bool {
return m.Moved || slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == "mesh-vault" })
}
func providerModuleOf(resolved catalogue.Resolution, open *stores, ctx context.Context, n catalogue.Needed) string {
for _, m := range resolved.Modules {
if _, shared := m.SharedCredentialOf(n.Name); shared {
cmd/mesh-controller/predefined.go
+93
View File
@@ -0,0 +1,93 @@
package main
import (
"context"
"fmt"
"path"
"strings"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The list of predefined keys the mesh publishes (novox/hq ADR 0348 §7, design 57 package K2, issue 661).
//
// **The newest trunk build of the catalogue wins.** A build of the catalogue reads predefined.json at the commit
// it builds and carries it with the commit's generation (link.PredefinedList). Registration keeps it in the
// controller's store when that generation is larger than the one kept (inventory.KeepPredefined), so an older
// build's list heard later never replaces a newer one, and every manifest the controller reads again is judged
// against the list kept (catalogue.UsePredefined). mesh-config publishes it on the bus from there (design 57
// package K3); the controller keeps no config value, only the keys.
// fromTheCatalogue is whether a build is of the catalogue's repository, whose list alone the mesh publishes: by
// its name, in the URL form or the seat form.
func fromTheCatalogue(result link.BuildResult) bool {
for _, repository := range []string{result.Repository, sourceRepository(result)} {
name := strings.TrimSuffix(path.Base(strings.TrimRight(repository, "/")), ".git")
if name == catalogue.CatalogueRepository {
return true
}
}
return false
}
func sourceRepository(result link.BuildResult) string {
if result.Source == nil {
return ""
}
return result.Source.Repository
}
// carriedList is the list a build of the catalogue carried, read strictly; nil for a build of another
// repository, or one that carried none.
func carriedList(result link.BuildResult) (catalogue.PredefinedList, error) {
if result.Predefined == nil || !fromTheCatalogue(result) {
return nil, nil
}
list, err := catalogue.ParsePredefined(result.Predefined.Keys)
if err != nil {
return nil, fmt.Errorf("%s carried a list of predefined keys that is refused: %w", result.ID, err)
}
return list, nil
}
// judgedWith is the list a build's manifest is first judged against at registration: the catalogue's own at the
// commit built, which its build judged it against, where it carried one; the list kept otherwise.
func judgedWith(result link.BuildResult) (catalogue.PredefinedList, error) {
list, err := carriedList(result)
if err != nil || list != nil {
return list, err
}
return catalogue.PublishedPredefined(), nil
}
// loadPredefined makes the list kept in the store the one this controller judges manifests against. A store
// keeping none leaves the list it has.
func loadPredefined(ctx context.Context, inv *inventory.Inventory) error {
kept, has, err := inv.PredefinedKept(ctx)
if err != nil || !has {
return err
}
catalogue.UsePredefined(kept.Keys)
return nil
}
// publishPredefined keeps a trunk build's list when it is newer than the one kept, says so, and loads what the
// store keeps then — this build's, or a newer one another controller process kept.
func publishPredefined(ctx context.Context, inv *inventory.Inventory, result link.BuildResult,
list catalogue.PredefinedList) error {
kept, err := inv.KeepPredefined(ctx, inventory.PredefinedRecord{Keys: list, Repository: result.Repository,
Commit: result.Commit, Generation: result.Predefined.Generation, Build: result.ID})
if err != nil {
return err
}
if kept {
fmt.Printf("%s: the list of predefined keys at %s (generation %d) is published: %s\n", result.ID,
short(result.Commit), result.Predefined.Generation, strings.Join(list.Keys(), ", "))
} else {
fmt.Printf("%s: the list of predefined keys at %s (generation %d) is not newer than the one kept, which "+
"stands\n", result.ID, short(result.Commit), result.Predefined.Generation)
}
return loadPredefined(ctx, inv)
}
cmd/mesh-controller/predefined_test.go
+197
View File
@@ -0,0 +1,197 @@
package main
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/link"
)
// The list of predefined keys (novox/hq ADR 0348 §7, design 57 package K2, issue 661): `module check` and the
// merge gate judge each manifest against the list at the root of its tree; registration publishes the newest
// trunk build's list of the catalogue and judges every module against the list the mesh publishes.
const theFirstKeys = `{
"font-size": {"type": "number", "description": "the size text is shown in, in points", "minimum": 6, "maximum": 72},
"agent-managed-settings": {"type": "object", "description": "the managed settings of every agent session, a JSON object", "trusted": true}
}`
// notifierFallingBackTo is the notifier in the config form, its font size falling back to the key named.
func notifierFallingBackTo(key, kind string) string {
def := "10"
if kind == "string" {
def = `"10"`
}
return `{"module":"notifier","version":"1",
"config":{"font-size":{"type":"` + kind + `","description":"the notification's font size, in points",
"default":` + def + `,"predefined":"` + key + `"}},
"resources":[{"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644",
"content":"font = ${config:font-size}\n"}]}`
}
// withoutAPublishedList judges the test against no list the mesh published, and puts the process's back after.
func withoutAPublishedList(t *testing.T) {
t.Helper()
was := catalogue.ReplacePublished(nil)
t.Cleanup(func() { catalogue.ReplacePublished(was) })
}
// aRepositoryTree is a repository's checkout holding one module at modules/notifier, and the list when given.
func aRepositoryTree(t *testing.T, manifest string, list *string) string {
t.Helper()
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, ".git"), 0o755); err != nil {
t.Fatal(err)
}
if list != nil {
if err := os.WriteFile(filepath.Join(root, catalogue.PredefinedFile), []byte(*list), 0o644); err != nil {
t.Fatal(err)
}
}
return aTemplatedModule(t, filepath.Join(root, "modules", "notifier"), manifest, nil)
}
func TestModuleCheckJudgesAManifestAgainstTheListAtItsRepositoryRoot(t *testing.T) {
withoutAPublishedList(t)
list := theFirstKeys
check := func(path string) (string, error) {
var out bytes.Buffer
err := moduleCheck([]string{path}, &out)
return out.String(), err
}
if out, err := check(aRepositoryTree(t, notifierFallingBackTo("font-size", "number"), &list)); err != nil ||
!strings.Contains(out, "notifier: ok") {
t.Fatalf("a module naming font-size was refused: %v\n%s", err, out)
}
for _, c := range []struct {
manifest string
list *string
says string
}{
{notifierFallingBackTo("font-scale", "number"), &list, `"font-scale", which is not on the list`},
{notifierFallingBackTo("font-size", "string"), &list, "which is a number, and this value is a string"},
{notifierFallingBackTo("font-size", "number"), nil, "is not present"},
{notifierFallingBackTo("font-size", "number"), ptr(`{"font-size": {"type": "number"}}`), "no description"},
} {
if out, err := check(aRepositoryTree(t, c.manifest, c.list)); err == nil || !strings.Contains(out, c.says) {
t.Errorf("wanted the check refused saying %q, got %v:\n%s", c.says, err, out)
}
}
}
func ptr(s string) *string { return &s }
// writeTheList puts the list at the root of a tree the gate judges.
func writeTheList(t *testing.T, tree, list string) string {
t.Helper()
if err := os.WriteFile(filepath.Join(tree, catalogue.PredefinedFile), []byte(list), 0o644); err != nil {
t.Fatal(err)
}
return tree
}
func TestTheMergeGateJudgesEachManifestAgainstTheListOfTheTree(t *testing.T) {
withoutAPublishedList(t)
f, manifests := templatedMesh(t)
v := gateJudged(t, f, writeTheList(t, aTree(t, withEdit(manifests, "notifier",
notifierFallingBackTo("font-size", "number"))), theFirstKeys), "modules/notifier/module.json")
if v.Verdict == "fail" || v.Modules["notifier"] != "changed" {
t.Fatalf("a module naming font-size on the tree's list failed the gate (%v):\n%s", v.Modules, v.Report())
}
for _, c := range []struct{ manifest, list, says string }{
{notifierFallingBackTo("font-scale", "number"), theFirstKeys, `"font-scale", which is not on the list`},
{notifierFallingBackTo("font-size", "string"), theFirstKeys, "which is a number"},
{inlineNotifier, `{"font-size": {"type": "number", "description": "points", "unit": "pt"}}`, `unknown field "unit"`},
} {
v := gateJudged(t, f, writeTheList(t, aTree(t, withEdit(manifests, "notifier", c.manifest)), c.list),
"modules/notifier/module.json", catalogue.PredefinedFile)
if v.Verdict != "fail" || !strings.Contains(v.Report(), c.says) {
t.Errorf("wanted the gate to fail saying %q:\n%s", c.says, v.Report())
}
}
}
// catalogueBuild is a trunk build of the catalogue carrying the list at a generation.
func catalogueBuild(id, manifest, list string, generation int) link.BuildResult {
r := link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
Path: "modules/notifier", Ref: "main", On: "anchor", Commit: strings.Repeat(id[len(id)-1:], 12),
Manifest: json.RawMessage(manifest), Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
if list != "" {
r.Predefined = &link.PredefinedList{Keys: json.RawMessage(list), Generation: generation}
}
return r
}
func TestRegistrationPublishesTheNewestTrunkListAndJudgesEveryModuleAgainstIt(t *testing.T) {
withoutAPublishedList(t)
open := aMesh(t)
ctx := t.Context()
// A manifest naming font-size registers, and its build's list is published.
if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory,
catalogueBuild("b-1", notifierFallingBackTo("font-size", "number"), theFirstKeys, 10))); err != nil {
t.Fatalf("a module naming font-size was not registered: %v", err)
}
kept, has, err := open.inventory.PredefinedKept(ctx)
if err != nil || !has || kept.Generation != 10 || kept.Build != "b-1" {
t.Fatalf("the list was not published: %+v %v %v", kept, has, err)
}
if got := catalogue.PublishedPredefined(); len(got) != 2 {
t.Fatalf("the controller judges against %v, not the list published", got)
}
// One naming a key not on the list, or of another type, is refused and never enters the catalogue.
for i, c := range []struct{ manifest, says string }{
{notifierFallingBackTo("font-scale", "number"), "not on the list"},
{notifierFallingBackTo("font-size", "string"), "which is a number"},
} {
id := []string{"b-2", "b-3"}[i]
if _, _, err := takeIn(ctx, open.inventory, catalogueBuild(id, c.manifest, theFirstKeys, 10)); err == nil ||
!strings.Contains(err.Error(), c.says) {
t.Errorf("wanted the registration refused saying %q, got %v", c.says, err)
}
}
src, err := open.inventory.SourceOf(ctx, "notifier")
if err != nil || src.BuiltFrom != strings.Repeat("1", 12) {
t.Fatalf("a refused build replaced the registered one: %+v %v", src, err)
}
// A newer build's list, which drops agent-managed-settings, replaces the published one.
smaller := `{"font-size": {"type": "number", "description": "points", "minimum": 6, "maximum": 72}}`
if _, _, err := takeIn(ctx, open.inventory, catalogueBuild("b-4",
notifierFallingBackTo("font-size", "number"), smaller, 12)); err != nil {
t.Fatal(err)
}
// The older build's outcome heard later registers nothing new about the list: the newer stands.
if _, _, err := takeIn(ctx, open.inventory, catalogueBuild("b-5",
notifierFallingBackTo("font-size", "number"), theFirstKeys, 11)); err != nil {
t.Fatal(err)
}
kept, _, _ = open.inventory.PredefinedKept(ctx)
if kept.Build != "b-4" || len(kept.Keys) != 1 || len(catalogue.PublishedPredefined()) != 1 {
t.Fatalf("an older build's list heard later replaced the newer one: %+v", kept)
}
// A build of another repository publishes nothing, and its manifest is judged against the published list.
other := catalogueBuild("b-6", notifierFallingBackTo("font-size", "number"), theFirstKeys, 99)
other.Repository, other.Source.Repository = "http://forge.internal:20000/novox/notifier.git", "novox/notifier"
other.Manifest = json.RawMessage(strings.Replace(notifierFallingBackTo("font-size", "number"), `"notifier"`,
`"chimes"`, 1))
if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, other)); err != nil {
t.Fatalf("a module of another repository naming a published key: %v", err)
}
if kept, _, _ = open.inventory.PredefinedKept(ctx); kept.Build != "b-4" {
t.Fatalf("a build of another repository published its list: %+v", kept)
}
other.ID, other.Manifest = "b-7", json.RawMessage(strings.Replace(notifierFallingBackTo("agent-managed-settings",
"number"), `"notifier"`, `"chimes"`, 1))
if _, _, err := takeIn(ctx, open.inventory, other); err == nil || !strings.Contains(err.Error(), "not on the list") {
t.Fatalf("a key only another repository's list holds was registered: %v", err)
}
}
cmd/mesh-controller/proposals.go
+16
View File
@@ -54,6 +54,7 @@ import (
"git.novox.be/novox/mesh-sdk/go/asks"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
@@ -906,6 +907,21 @@ func setLayerIn(open *stores) setOnWarrant {
return "", fmt.Errorf("the layer of %s on %s changed since the operator was shown the change: it is not set; "+
"propose it again", p.Module, p.where())
}
// A proposal that carries config values, raised before the module's config moved, is not applied after it: its
// values in the layer would turn ReadsConfig off and undo the move (issue 830, novox/hq ADR 0348). A proposal
// of placement only is set as before: forwardPropose sends it down this path on purpose.
carried := p.Values
if p.Clear {
carried = before
}
if config, _ := catalogue.SplitPlacement(carried); len(config) > 0 {
if reads, err := inv.ReadsConfig(ctx, p.Module); err != nil {
return "", err
} else if reads {
return "", fmt.Errorf("the config of %s moved to mesh-config since the operator was shown the change: it is "+
"not set; propose it again (it then goes to mesh-config)", p.Module)
}
}
var changed string
if p.Clear {
if err := inv.ClearSettingsBy(ctx, p.Node, p.Module, setBy); err != nil {
cmd/mesh-controller/provisioned.go
+100
View File
@@ -0,0 +1,100 @@
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
)
// What a machine's provisioner client said of a module's files (novox/hq ADR 0362, ADR 0349 §15, design 57 P6,
// issue 672), read where a walk's end and a gate count a machine's send applied: a module the send handed to the
// provisioner is applied there only once the client reported `prov.applied` on that build's templates as well.
// provAppliedOn is the client's newest report of a module on a machine, from the stream that keeps it
// (broker.ProvAppliedStream); false when it has said nothing of it. A variable so a test can say the client's words.
var provAppliedOn = func(ctx context.Context, node, module string) (inventory.ProvReport, bool, error) {
var raw []byte
var said bool
read := func(js nats.JetStreamContext) error {
var err error
raw, said, err = broker.LastProvApplied(js, node, module)
return err
}
var err error
if d := doctorFrom; d != nil && d.js != nil {
err = read(d.js.Context())
} else {
err = onTheBus(func(conn *nats.Conn) error {
js, err := conn.JetStream(nats.Context(ctx))
if err != nil {
return err
}
return read(js)
})
}
if err != nil || !said {
return inventory.ProvReport{}, false, err
}
var r inventory.ProvReport
if err := json.Unmarshal(raw, &r); err != nil {
return inventory.ProvReport{}, false, fmt.Errorf("what %s's provisioner client said of %s is not its report: %w",
node, module, err)
}
return r, true, nil
}
// provLookup answers the clients' reports for one reading; a report that cannot be read is not counted, which
// leaves the machine waiting, then silent, never applied — and is said.
func provLookup(ctx context.Context) inventory.ProvLookup {
return func(node, module string) (inventory.ProvReport, bool) {
r, said, err := provAppliedOn(ctx, node, module)
if err != nil {
fmt.Fprintf(os.Stderr, "what %s's provisioner client said of %s could not be read: %v\n", node, module, err)
return inventory.ProvReport{}, false
}
return r, said
}
}
// provisioningFailed is every module whose files a machine's provisioner client last said it failed to place, from
// the stream that keeps their reports.
func provisioningFailed(js nats.JetStreamContext) ([]inventory.ProvReport, error) {
all, err := broker.ProvAppliedReports(js)
if err != nil {
return nil, err
}
var out []inventory.ProvReport
for subject, raw := range all {
var r inventory.ProvReport
if err := json.Unmarshal(raw, &r); err != nil {
return nil, fmt.Errorf("what was said on %s is not a provisioner client's report: %w", subject, err)
}
if r.Status == inventory.ProvFailed {
out = append(out, r)
}
}
return out, nil
}
// provisionedOutcome is a machine's report of one send as a delivery's end counts it (novox/hq ADR 0362 §1-5, issue
// 840): the engine's own report, combined with its provisioner client's for a send that handed modules to it. The
// outcome of a client's failure begins inventory.OutcomeProvisioningFailed; a report not yet applied on both sides
// is none (false). A send handing nothing is the engine's report unchanged. onSend says the report is about the send.
func provisionedOutcome(node string, sent inventory.SentDeclaration, r inventory.Reported, onSend bool,
prov inventory.ProvLookup) (string, bool) {
if len(sent.Provisioned) == 0 {
return r.Outcome, onSend
}
engine := inventory.AppliedReport{Outcome: r.Outcome}
if r.At != nil {
engine.At = *r.At
}
got, ok := inventory.ProvisionedSend(node, sent, engine, onSend, prov)
return got.Outcome, ok
}
cmd/mesh-controller/provisioned_test.go
+114
View File
@@ -0,0 +1,114 @@
package main
import (
"encoding/json"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// The controller hands a moved module to the provisioner, and counts it applied only once its files are placed
// (design 57 P6, issue 672; novox/hq ADR 0349 §4, §15, ADR 0362).
// A declaration that hands files says which, and whose client fills them; one that hands none is byte for byte
// what it was.
func TestADeclarationNamesTheFilledFilesOnlyWhenItHandsAny(t *testing.T) {
plain := sendable{Resources: []map[string]any{{"id": "notify.configuration", "type": "file", "path": "/x",
"content": "a"}}, Sequence: 4}
before, err := plain.Body()
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(before), "filled_files") || strings.Contains(string(before), "provisioner_account") {
t.Fatalf("a declaration handing nothing names filled files: %s", before)
}
handing := plain
handing.Filled, handing.Provisioner = []string{"/x"}, "mesh-provisioner-client"
body, err := handing.Body()
if err != nil {
t.Fatal(err)
}
var envelope struct {
Filled []string `json:"filled_files"`
Provisioner string `json:"provisioner_account"`
}
if err := json.Unmarshal(body, &envelope); err != nil || len(envelope.Filled) != 1 || envelope.Filled[0] != "/x" ||
envelope.Provisioner != "mesh-provisioner-client" {
t.Fatalf("the handing declaration says %s", body)
}
}
func TestTheProvisionersAccountIsTheClientsProcessUser(t *testing.T) {
client := catalogue.Manifest{Module: "mesh-provisioner-client",
OwnSecrets: catalogue.OwnSecrets{"provisioner-broker": {Path: "/var/lib/x/broker"}},
Claims: []catalogue.Claim{{Name: "node-provisioner", Scope: "node"}},
Resources: []map[string]any{{"id": "account", "type": "user", "name": "mesh-provisioner-client"},
{"id": "client", "type": "process", "name": "mesh-provisioner-client", "user": "mesh-provisioner-client"}}}
if got := provisionerAccount(catalogue.Resolution{Modules: []catalogue.Manifest{client}}); got != "mesh-provisioner-client" {
t.Fatalf("the client's account is %q", got)
}
if got := provisionerAccount(catalogue.Resolution{}); got != "" {
t.Fatalf("a machine without a client names %q", got)
}
}
// The first-machine gate counts a handed module healthy only once its node's prov.applied for the new build has
// come (ADR 0349 §15, kept by ADR 0362 for a walk finishing at its gate); a failure there fails it.
func TestTheGateWaitsForTheProvisionersReport(t *testing.T) {
now := time.Date(2026, 10, 14, 23, 0, 0, 0, time.UTC)
since := now.Add(-time.Minute)
reported := now.Add(-50 * time.Second)
digest := "sha256:" + strings.Repeat("d", 64)
facts := func(said map[string]inventory.ProvReport) gateFacts {
return gateFacts{now: now,
reports: map[string]inventory.Reported{"g14": {Node: "g14", Outcome: inventory.OutcomeApplied,
At: &reported, Current: true}},
sent: map[string]inventory.SentDeclaration{"g14": {Digest: "sent", At: since,
Provisioned: map[string]string{"dunst": digest}}},
provisioned: func(node, module string) (inventory.ProvReport, bool) {
r, ok := said[node+"."+module]
return r, ok
}}
}
m := catalogue.Manifest{Module: "dunst"}
if h, why := judgeHealth("dunst", "", m, "g14", since, facts(nil)); h != healthNotYet ||
!strings.Contains(why, "has not reported dunst's files placed") {
t.Fatalf("judged without the provisioner's report: %v %q", h, why)
}
failed := map[string]inventory.ProvReport{"g14.dunst": {Status: inventory.ProvFailed, Templates: digest,
Reason: "a value is not there", At: now.Add(-10 * time.Second)}}
if h, why := judgeHealth("dunst", "", m, "g14", since, facts(failed)); h != healthBroken ||
!strings.Contains(why, "a value is not there") {
t.Fatalf("a failed placing is not the build's failure there: %v %q", h, why)
}
placed := map[string]inventory.ProvReport{"g14.dunst": {Status: inventory.ProvApplied, Templates: digest,
At: now.Add(-10 * time.Second)}}
if h, why := judgeHealth("dunst", "", m, "g14", since, facts(placed)); strings.Contains(why, "provisioner") {
t.Fatalf("placed, the module is still held on the provisioner: %v %q", h, why)
}
// A module the send did not hand is judged as before.
if h, why := judgeHealth("picom", "", catalogue.Manifest{Module: "picom"}, "g14", since, facts(nil)); strings.Contains(why, "provisioner") {
t.Fatalf("a module not handed waits on the provisioner: %v %q", h, why)
}
}
// A machine whose provisioner client failed to place a module's files has a failed apply, said as apply-failing
// once the failure outlasts an apply interval, with the client's reason (ADR 0362 §3).
func TestAFailedPlacingRaisesApplyFailing(t *testing.T) {
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
f := calm(now)
f.provFailed = []inventory.ProvReport{{Node: "anchor", Module: "dunst", Status: inventory.ProvFailed,
Reason: "the templates cannot be fetched", At: now.Add(-time.Minute)}}
if got := watchApplyFailing(f); len(got) != 0 {
t.Fatalf("raised within an apply interval: %+v", got)
}
f.provFailed[0].At = now.Add(-10 * time.Minute)
got := watchApplyFailing(f)
if len(got) != 1 || got[0].Key() != "machine.anchor.apply-failing" ||
!strings.Contains(got[0].Summary, "dunst") || !strings.Contains(got[0].Said, "the templates cannot be fetched") {
t.Fatalf("a failed placing is not raised as anchor's apply-failing: %+v", got)
}
}
cmd/mesh-controller/provisioner_account_test.go
+62
View File
@@ -0,0 +1,62 @@
package main
import (
"slices"
"testing"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
func aProvisioner() catalogue.Manifest {
return catalogue.Manifest{Module: "mesh-provisioner", Version: "1",
DefinesSeats: []catalogue.SeatDeclaration{{Name: broker.NodeProvisionerSeat, Scope: catalogue.ScopeNode,
Serves: []catalogue.Verb{{Name: "get"}}}},
Claims: []catalogue.Claim{{Name: broker.NodeProvisionerSeat, Scope: catalogue.ScopeNode}},
OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/mesh-provisioner/broker"},
broker.ProvisionerSecret: {Path: "/run/mesh-provisioner/client"}},
}
}
// A machine's provisioner client is issued an account of its own, under its own name, sealed to the machine as the
// module's own secret its service reads (novox/hq ADR 0349 clause 11, design 57 P3); until it is, a push refuses
// to invent one, as it does for a module's `broker`.
func TestTheProvisionerClientsAccountIsIssuedForItsServiceAndNeverInvented(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
m := aProvisioner()
register(t, open, m)
if _, err := assign(ctx, open, "laptop", m.Module); err != nil {
t.Fatal(err)
}
if err := busCredentialIssued(ctx, inv, "laptop", m.Module, broker.ProvisionerSecret); err == nil {
t.Fatal("a push would seal a placeholder for the client's account before it was issued")
}
if err := issueProvisionerClient(ctx, inv, m, "laptop", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222"); err != nil {
t.Fatal(err)
}
if _, minted, err := inv.BusUserHash(ctx, "provisioner.laptop"); err != nil || !minted {
t.Fatalf("the client's account is not minted: %v %v", minted, err)
}
states, err := inv.SecretsOf(ctx, "laptop", m.Module)
if err != nil {
t.Fatal(err)
}
if !slices.ContainsFunc(states, func(s inventory.SecretState) bool { return s.Name == broker.ProvisionerSecret }) {
t.Fatalf("the client's account is not sealed to the machine for its service: %+v", states)
}
if err := busCredentialIssued(ctx, inv, "laptop", m.Module, broker.ProvisionerSecret); err != nil {
t.Fatalf("an issued client account is refused: %v", err)
}
if !runsTheProvisionerClient(m) {
t.Fatal("the module is not read as running the client")
}
unread := m
unread.OwnSecrets = catalogue.OwnSecrets{"broker": {Path: "/run/broker"}}
if runsTheProvisionerClient(unread) {
t.Fatal("a module with nowhere to read the client's account is read as running it")
}
}
cmd/mesh-controller/push.go
+7 -1
View File
@@ -265,6 +265,12 @@ func serve(ctx context.Context) (err error) {
// The facts snapshot a merge check is fed (novox/hq to-be 45 §9): kept current while this
// controller holds the lease, read by the build seat from the artifact store.
go exportingFacts(ctx, open, bus.Conn.ConnectedServerVersion)
// The record mirrors (novox/hq ADR 0289, design 57 F3): the assignments and each registered module's
// declared config on the bus, written from the store only when a record changed, by the lease holder.
// The provisioner's plan of each handed module, as its machine was last sent it (design 57 P6), in the module's
// assignment.
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
go mirroringRecords(ctx, inv, bus.Conn, bus)
// Every call carries the lease's epoch, and its record is written only under the lease (novox/hq
// to-be 45 §6).
link.Calls.UnderLease(func() (uint64, error) { return theLease.epoch(ctx) })
@@ -1695,7 +1701,7 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body
// send that is away look failed: it is said, loudly, and the machine's own record stands.
err = inv.RecordSentWith(kept, record.ID, digest, builds, epoch, sent.toldGeneration, inventory.Send{
Sequence: sent.sequence, Epoch: int64(sent.epoch), Sender: sent.sender, Generation: sent.generation,
Digest: digest, Modules: sent.modules})
Digest: digest, Modules: sent.modules, Provisioning: sent.plans})
var unrecorded *inventory.SendNotRecordedError
switch {
case errors.As(err, &unrecorded):
cmd/mesh-controller/record_mirrors.go
+106
View File
@@ -0,0 +1,106 @@
package main
import (
"context"
"fmt"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/recordmirror"
)
// The controller's record mirrors on the bus (novox/hq ADR 0289, design 54, design 57 F3): the assignments,
// and each registered module's declared config, which mesh-config resolves a module's config values from.
//
// **Built from the store, on a tick of their own, by the lease holder alone.** A record changes through a
// verb that runs as a command of its own, a reconcile, a merge's registration or a pending assignment made,
// so no one call site sees every change; what every change has in common is that it is committed to the
// store. So the serving controller reads the store every mirrorEvery and writes only what differs from what
// the bucket holds (ADR 0289 §1): a quiet mesh writes nothing, and a change rolled back is never read. A
// mirror is therefore at most mirrorEvery behind its store.
// mirrorEvery is how often the mirrors are compared with the store: two small reads of the store.
const mirrorEvery = 10 * time.Second
// mirroringRecords keeps the mirrors current for as long as this controller serves: ctx ends when it stops.
func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, announce recordmirror.Announcer) {
api, err := jetstream.New(conn)
if err != nil {
fmt.Printf("the record mirrors are not kept: %v\n", err)
return
}
holds := func() bool {
epoch, err := theLease.epoch(ctx)
return err == nil && epoch > 0
}
epoch := func() uint64 {
e, _ := theLease.epoch(ctx)
return e
}
sources := map[string]func(context.Context) (map[string][]byte, error){
recordmirror.Assignments.Name: inv.AssignmentsMirror,
recordmirror.DeclaredConfig.Name: inv.DeclaredConfigMirror,
recordmirror.Predefined.Name: inv.PredefinedMirror,
}
writers := map[string]*recordmirror.Writer{}
failing := map[string]string{}
tick := time.NewTicker(mirrorEvery)
defer tick.Stop()
for {
if holds() {
for _, m := range recordmirror.Mirrors() {
why := mirrorOnce(ctx, api, m, writers, sources[m.Name], announce, epoch, holds)
if why != failing[m.Name] {
if why != "" {
fmt.Printf("the %s mirror cannot be kept: %s\n", m.Name, why)
} else {
fmt.Printf("the %s mirror is kept again\n", m.Name)
}
failing[m.Name] = why
}
}
}
select {
case <-ctx.Done():
return
case <-tick.C:
}
}
}
// mirrorOnce brings one mirror to what the store holds, and answers why it could not; empty when it could.
func mirrorOnce(ctx context.Context, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer,
source func(context.Context) (map[string][]byte, error), announce recordmirror.Announcer, epoch func() uint64,
holds func() bool) string {
reading, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
w := writers[m.Name]
if w == nil {
kv, err := api.KeyValue(reading, m.Bucket)
if err != nil {
return fmt.Sprintf("the bucket %s cannot be read: %v", m.Bucket, err)
}
w = &recordmirror.Writer{Mirror: m, KV: kv, Announce: announce, Epoch: epoch, Holds: holds}
writers[m.Name] = w
}
want, err := source(reading)
if err != nil {
return fmt.Sprintf("the store cannot be read: %v", err)
}
lost := w.LostEvents
r, err := w.Sync(reading, want)
if r.Written+r.Removed > 0 {
fmt.Printf("the %s mirror moved: %d written, %d removed\n", m.Name, r.Written, r.Removed)
}
if w.LostEvents > lost {
fmt.Printf("the %s mirror lost %d event(s): the bucket holds the change, its notice was refused twice\n",
m.Name, w.LostEvents-lost)
}
if err != nil {
return err.Error()
}
return ""
}
cmd/mesh-controller/release.go
+20 -4
View File
@@ -1013,7 +1013,16 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.
}
// **A machine that says nothing on its send is left silent, not failed** (ADR 0353 §4): it is not judged
// until it reports, and past ApplySilentAfter it is listed and left.
if !reportedOnSend(byNode[node], node, g) {
outcome, reported := byNode[node].Outcome, reportedOnSend(byNode[node], node, g)
if sent, kept := g.Sent[node]; kept && !gateFirstMachine {
// A module handed to the provisioner is applied only once its client said so too (ADR 0362, issue 840).
carried := make([]string, 0, len(g.Carried))
for _, c := range g.Carried {
carried = append(carried, c.Module)
}
outcome, reported = provisionedOutcome(node, sent.Only(carried...), byNode[node], reported, provLookup(ctx))
}
if !reported {
if now.Sub(*g.Since) > inventory.ApplySilentAfter {
g.Silent = true
r.Silent = append(r.Silent, node)
@@ -1028,18 +1037,25 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.
if !gateFirstMachine {
// **Judged by its report alone** (novox/hq ADR 0360 §7): applied passes; failed or refused is named and
// fails the walk, with nothing put back — apply-failing says it.
switch rep := byNode[node]; rep.Outcome {
switch outcome {
case inventory.OutcomeApplied:
decide(g, inventory.GatePassed, node+" reported it applied its send (novox/hq ADR 0360)", now)
passCarried(ctx, open, p, g, "")
r.Done = append(r.Done, node)
changed = true
case inventory.OutcomeFailed, inventory.OutcomeRefused:
decide(g, inventory.GateFailed, fmt.Sprintf("%s %s its send", node, rep.Outcome), now)
decide(g, inventory.GateFailed, fmt.Sprintf("%s %s its send", node, outcome), now)
r.Failed = append(r.Failed, node)
changed = true
notes = append(notes, fmt.Sprintf("%s %s its send: nothing put back, apply-failing says it", node, rep.Outcome))
notes = append(notes, fmt.Sprintf("%s %s its send: nothing put back, apply-failing says it", node, outcome))
default:
if strings.HasPrefix(outcome, inventory.OutcomeProvisioningFailed) {
decide(g, inventory.GateFailed, fmt.Sprintf("%s: %s", node, outcome), now)
r.Failed = append(r.Failed, node)
changed = true
notes = append(notes, fmt.Sprintf("%s: %s: nothing put back, apply-failing says it", node, outcome))
continue
}
judging = append(judging, node+": not yet reported on its send")
}
continue
cmd/mesh-controller/release_plan.go
+21 -4
View File
@@ -1153,7 +1153,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
if err != nil {
return false, err
}
waiting, failed, silent := walkReports(*p, reports, time.Now().UTC())
waiting, failed, silent := walkReports(*p, reports, time.Now().UTC(), provLookup(ctx))
if len(waiting) > 0 {
note := "sent; waiting for " + strings.Join(waiting, ", ") + " to report on their send"
changed := p.State != inventory.PlanRolling || p.Note != note
@@ -1189,17 +1189,29 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
// walkReports is, for every machine a walk sent its rest, whether it reported on that send (novox/hq ADR 0360 §4):
// those still waiting, those that reported a failed or refused apply, and those silent past ApplySilentAfter,
// each sorted. Pure.
func walkReports(p inventory.Plan, reports []inventory.Reported, now time.Time) (waiting, failed, silent []string) {
func walkReports(p inventory.Plan, reports []inventory.Reported, now time.Time, prov inventory.ProvLookup) (waiting, failed, silent []string) {
byNode := map[string]inventory.Reported{}
for _, r := range reports {
byNode[r.Node] = r
}
// The modules of this delivery sent to each machine: a machine is judged on those, not on every module its
// declaration hands to the provisioner (ADR 0362 §1-2, issue 840).
carried := map[string][]string{}
for m, s := range p.Modules {
if s == nil {
continue
}
for node := range s.Rest {
carried[node] = append(carried[node], m)
}
}
seen := map[string]bool{}
for _, s := range p.Modules {
if s == nil {
continue
}
for node, sent := range s.Rest {
sent = sent.Only(carried[node]...)
if seen[node] {
continue
}
@@ -1208,11 +1220,16 @@ func walkReports(p inventory.Plan, reports []inventory.Reported, now time.Time)
if at.IsZero() && s.SentAt != nil {
at = *s.SentAt
}
// A module handed to the provisioner is applied only once its client said so too (ADR 0362 §1-5).
outcome, reported := provisionedOutcome(node, sent, r, r.At != nil && sent.ReportsOn(r), prov)
switch {
case r.At != nil && sent.ReportsOn(r):
case reported:
seen[node] = true
if r.Outcome == inventory.OutcomeFailed || r.Outcome == inventory.OutcomeRefused {
switch {
case outcome == inventory.OutcomeFailed || outcome == inventory.OutcomeRefused:
failed = append(failed, node)
case strings.HasPrefix(outcome, inventory.OutcomeProvisioningFailed):
failed = append(failed, node+" ("+outcome+")")
}
case !at.IsZero() && now.Sub(at) > inventory.ApplySilentAfter:
seen[node] = true
cmd/mesh-controller/release_provisioned_test.go
+172
View File
@@ -0,0 +1,172 @@
package main
import (
"context"
"encoding/json"
"slices"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// handingAnchor makes every send to anchor hand the provisioner the given modules (module -> templates digest), and
// the clients' words the given reports (key `node/module`), as the stream would give them (issue 840, ADR 0362).
func handingAnchor(t *testing.T, b *backlogMesh, handed map[string]string) map[string]inventory.ProvReport {
t.Helper()
says := map[string]inventory.ProvReport{}
wasSays := provAppliedOn
provAppliedOn = func(_ context.Context, node, module string) (inventory.ProvReport, bool, error) {
r, ok := says[node+"/"+module]
return r, ok, nil
}
inner := sendRollout
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
got, err := inner(ctx, open, names)
if err != nil {
return got, err
}
for _, n := range names {
if n != "anchor" {
continue
}
inv := open.inventory
s, _, err := inv.SentTo(ctx, n)
if err != nil {
return nil, err
}
builds, _, err := inv.SentBuilds(ctx, n)
if err != nil {
return nil, err
}
plans := map[string]inventory.Provisioning{}
for m, d := range handed {
plans[m] = inventory.Provisioning{Templates: inventory.TemplatesArchive{Digest: d}}
}
raw, err := json.Marshal(plans)
if err != nil {
return nil, err
}
id := nodeID(t, open, n)
if err := inv.RecordSentWith(ctx, id, s.Digest, builds, 0, 0, inventory.Send{Node: id, Provisioning: raw}); err != nil {
return nil, err
}
}
return got, nil
}
t.Cleanup(func() { sendRollout, provAppliedOn = inner, wasSays })
return says
}
func stepRelease(t *testing.T, b *backlogMesh, n int) inventory.Plan {
t.Helper()
for i := 0; i < n; i++ {
advancePlans(t.Context(), b.open)
}
return b.release(t)
}
// Issue 840 (novox/hq ADR 0362 §1-5): the release's rest gate, which every delivery passes since ADR 0360, counts a
// machine's send of a provisioned module applied only once the engine and the machine's provisioner client both said
// so; a failed client is a failed apply named with its reason; none past ApplySilentAfter is silent, never done.
func TestTheRestGateWaitsForTheProvisioner(t *testing.T) {
judgeLeavesAsTheCore(t)
applied := inventory.ProvReport{Status: inventory.ProvApplied, Templates: "t1", At: time.Now()}
t.Run("a engine applied, the client has not said: no verdict, the release stays open", func(t *testing.T) {
b := aBacklog(t)
handingAnchor(t, b, map[string]string{"app": "t1"})
p := stepRelease(t, b, 8)
g := p.Release.Gates["anchor"]
if !p.Open() || g == nil || g.Verdict != "" || g.Silent || slices.Contains(p.Release.Done, "anchor") {
t.Fatalf("anchor is judged before its provisioner reported: %s %s %+v", p.State, p.Note, g)
}
if !slices.Contains(p.Release.Done, "laptop") {
t.Fatalf("laptop hands nothing and passes on the engine's report: %+v", p.Release)
}
})
t.Run("b the client failed: a failed apply named with the reason", func(t *testing.T) {
b := aBacklog(t)
says := handingAnchor(t, b, map[string]string{"app": "t1"})
says["anchor/app"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t1", Reason: "no value", At: time.Now()}
p := stepRelease(t, b, 8)
g := p.Release.Gates["anchor"]
if g == nil || g.Verdict != inventory.GateFailed || !slices.Contains(p.Release.Failed, "anchor") ||
!strings.Contains(p.Note, "provisioning failed (app: no value)") {
t.Fatalf("the failure is not named: %s %q %+v %+v", p.State, p.Note, g, p.Release)
}
})
t.Run("c no client report past ApplySilentAfter: silent, never done", func(t *testing.T) {
b := aBacklog(t)
handingAnchor(t, b, map[string]string{"app": "t1"})
p := stepRelease(t, b, 8)
g := p.Release.Gates["anchor"]
if g == nil {
t.Fatalf("no gate: %+v", p.Release)
}
since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute)
g.Since = &since
if err := b.open.inventory.SavePlan(t.Context(), &p); err != nil {
t.Fatal(err)
}
p = stepRelease(t, b, 1)
if !slices.Equal(p.Release.Silent, []string{"anchor"}) || slices.Contains(p.Release.Done, "anchor") {
t.Fatalf("anchor is not silent: %s %s %+v", p.State, p.Note, p.Release)
}
})
t.Run("d both applied: the gate passes", func(t *testing.T) {
b := aBacklog(t)
says := handingAnchor(t, b, map[string]string{"app": "t1"})
says["anchor/app"] = applied
p := stepRelease(t, b, 8)
if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") || len(p.Release.Failed) > 0 {
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
}
})
t.Run("e a send handing nothing passes on the engine's report", func(t *testing.T) {
b := aBacklog(t)
p := stepRelease(t, b, 8)
if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") {
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
}
})
t.Run("f a module the delivery does not carry does not touch its end", func(t *testing.T) {
// anchor's declaration also hands `same`, rebuilt with the very bytes and so not carried: its failed report
// is none of this delivery's.
b := aBacklog(t)
says := handingAnchor(t, b, map[string]string{"app": "t1", "same": "t9"})
says["anchor/app"] = applied
says["anchor/same"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t9", Reason: "stuck", At: time.Now()}
p := stepRelease(t, b, 8)
if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") || len(p.Release.Failed) > 0 {
t.Fatalf("another module's failure ended this delivery: %s %s %+v", p.State, p.Note, p.Release)
}
})
}
// Issue 840: a walk's end waits on the modules it sent to a machine, not on every module that machine's declaration
// hands to the provisioner.
func TestAWalkEndWaitsOnlyOnTheModulesItSent(t *testing.T) {
now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)
at := now.Add(-10 * time.Second)
handed := map[string]string{"app": "t1", "other": "t2"}
p := inventory.Plan{Modules: map[string]*inventory.PlanModule{"app": {Rest: map[string]inventory.SentDeclaration{
"ace": {Digest: "d-ace", At: now.Add(-time.Minute), Provisioned: handed}}}}}
reports := []inventory.Reported{{Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"}}
says := map[string]inventory.ProvReport{"ace/app": {Status: inventory.ProvApplied, Templates: "t1", At: at}}
prov := func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok }
// other has said nothing: not this walk's to wait for.
if waiting, failed, silent := walkReports(p, reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 {
t.Fatalf("missing: waiting %v, failed %v, silent %v", waiting, failed, silent)
}
says["ace/other"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t2", Reason: "stuck", At: at}
if waiting, failed, silent := walkReports(p, reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 {
t.Fatalf("failed: waiting %v, failed %v, silent %v", waiting, failed, silent)
}
// once the walk carries it too, it does.
p.Modules["other"] = &inventory.PlanModule{Rest: p.Modules["app"].Rest}
if _, failed, _ := walkReports(p, reports, now, prov); len(failed) != 1 || !strings.Contains(failed[0], "other: stuck") {
t.Fatalf("a carried module's failure is not named: %v", failed)
}
}
cmd/mesh-controller/rollout.go
+13
View File
@@ -379,6 +379,19 @@ func rolloutMint(ctx context.Context, again bool) error {
}
modules++
case broker.KindProvisioner:
// A machine's provisioner client, on an account of its own (novox/hq ADR 0349 clause 11), delivered
// to the module running it.
m, inShelf := shelf[p.Module]
if !inShelf || !runsTheProvisionerClient(m) {
skipped++
continue
}
if err := issueProvisionerClient(ctx, inv, m, p.Node, known, busAddress); err != nil {
return err
}
modules++
default:
skipped++
}
cmd/mesh-controller/rotate.go
+141 -11
View File
@@ -6,6 +6,7 @@ import (
"flag"
"fmt"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/inventory"
)
@@ -39,6 +40,9 @@ func rotateCommand(ctx context.Context, args []string) error {
// of one provision, each with its own credential; one module that leaked its credential (novox/hq
// issue 268) is no reason to restart every other one on the machine.
module := set.String("module", "", "only this consuming module's credential")
// Why, which the vault asks of every rotation (issue 767) and the hand-act log records; required only where a
// provider's credential is the vault's, so nothing the operator typed before changes meaning.
why := addHandActFlags(set)
positionals, err := parseAround(set, args)
if err != nil {
return err
@@ -89,17 +93,6 @@ func rotateCommand(ctx context.Context, args []string) error {
// Every machine at both ends, named before anything changes. A person about to rotate a
// production credential is entitled to know the blast radius before it is the past tense.
affected := map[string]bool{}
for _, h := range holders {
affected[h.Consumer] = true
affected[h.Provider] = true
}
machines := make([]string, 0, len(affected))
for name := range affected {
machines = append(machines, name)
}
sort.Strings(machines)
fmt.Printf("rotating %s for %d holder(s):\n", provision, len(holders))
for _, h := range holders {
// The module, because a machine may hold several credentials for one provision and
@@ -107,6 +100,35 @@ func rotateCommand(ctx context.Context, args []string) error {
fmt.Printf(" %s on %s, from %s%s\n", h.ConsumerModule, h.Consumer, h.Provider, asLocal(h.Local))
}
// **The vault's, where the provider is filled the new way** (issue 767, ADR 0349 clause 8): its credential is
// the vault's, so the vault remakes it and its holders take it from there; the controller writes no value of its
// own and sends nothing for it. Which holders go where is read before anything changes.
toVault, today, err := rotateRoutes(holders,
func(provider, provision string) (string, error) { return offeringModuleOn(ctx, provider, provision) },
filledTheNewWay)
if err != nil {
return fmt.Errorf("nothing was rotated: %w", err)
}
if len(toVault) > 0 && !why.given() {
return errors.New("not rotated: the vault holds this credential and rotates it only with a reason, " +
"`--why \"…\"`; nothing was asked")
}
if why.given() {
why.record(ctx, "rotate", append([]string{provision}, rotateNarrowArgs(*only, *module)...))
}
for _, g := range toVault {
words, err := rotateProvisionInVault(theVault(ctx), g, *only, *module, actingAs(), *why.why)
if err != nil {
return err
}
fmt.Println(words)
}
if len(today) == 0 {
return nil
}
holders = today
machines := machinesOf(holders)
for _, h := range holders {
if err := inv.RotateSecret(ctx, h.Provision, h.Consumer, h.ConsumerModule, h.Provider, h.Local); err != nil {
// Partly rotated, and said so plainly. What is gone is remade on the next push, so
@@ -137,6 +159,21 @@ func rotateCommand(ctx context.Context, args []string) error {
return nil
}
// machinesOf is every machine at both ends of these holders, in order.
func machinesOf(holders []inventory.Holder) []string {
affected := map[string]bool{}
for _, h := range holders {
affected[h.Consumer] = true
affected[h.Provider] = true
}
machines := make([]string, 0, len(affected))
for name := range affected {
machines = append(machines, name)
}
sort.Strings(machines)
return machines
}
// ofModule is the holders whose consuming module is this one; all of them when none is named.
func ofModule(holders []inventory.Holder, module string) []inventory.Holder {
if module == "" {
@@ -165,3 +202,96 @@ func asLocal(local string) string {
}
return " (as " + local + ")"
}
// rotateGroup is the holders of one provider's credential for one provision whose provider is filled the new way:
// the vault's to rotate (issue 767).
type rotateGroup struct {
Provider, Provision, Module string
Holders []inventory.Holder
}
// rotateRoutes says which holders the vault rotates, grouped by provider and provision, and which today's path
// does: the vault's are those whose provider's offering module on its machine is filled the new way (the switch
// `accept --provider` reads too). Anything that cannot be read is an error, never today's path by default: a value
// remade where its holders no longer look would be reported rotated and change nothing.
func rotateRoutes(holders []inventory.Holder, offering func(provider, provision string) (string, error),
ofTheVault vaultSwitch) ([]rotateGroup, []inventory.Holder, error) {
type key struct{ provider, provision string }
decided := map[key]int{} // index into groups, or -1 for today's path
var groups []rotateGroup
var today []inventory.Holder
for _, h := range holders {
k := key{h.Provider, h.Provision}
i, seen := decided[k]
if !seen {
module, err := offering(h.Provider, h.Provision)
if err != nil {
return nil, nil, fmt.Errorf("which module offers %s on %s cannot be read: %w", h.Provision, h.Provider, err)
}
ours := false
if module != "" {
if ours, err = ofTheVault(module); err != nil {
return nil, nil, err
}
}
i = -1
if ours {
i = len(groups)
groups = append(groups, rotateGroup{Provider: h.Provider, Provision: h.Provision, Module: module})
}
decided[k] = i
}
if i < 0 {
today = append(today, h)
continue
}
groups[i].Holders = append(groups[i].Holders, h)
}
return groups, today, nil
}
// rotateNarrowArgs is --consumer and --module as typed, for the hand-act log.
func rotateNarrowArgs(consumer, module string) []string {
var out []string
if consumer != "" {
out = append(out, "--consumer", consumer)
}
if module != "" {
out = append(out, "--module", module)
}
return out
}
// rotateProvisionInVault asks the vault to rotate one provider's credential for a provision (issue 767): a shared
// value is made anew for every holder, and the vault refuses it for one consumer alone; for a provider whose
// consumers keep a login each (ADR 0158 §11), consumer and module name whose own login is remade, and only theirs.
// The vault says secret-changed and each holder takes the new value from it; nothing is sealed or sent here.
func rotateProvisionInVault(ask vaultAsker, g rotateGroup, consumer, module, by, why string) (string, error) {
if strings.TrimSpace(why) == "" {
return "", errors.New("not rotated: the vault rotates a credential only with a reason, `--why \"…\"`; nothing was asked")
}
args := provisionArgs(g.Provider, g.Provision)
args["by"], args["why"] = by, strings.TrimSpace(why)
if consumer != "" {
args["consumer"] = consumer
}
if module != "" {
args["consumer_module"] = module
}
changed, err := endVaultCall(ask, "rotate", args)
if err != nil {
return "", fmt.Errorf("%s from %s not rotated, and nothing else was: %w", g.Provision, g.Provider, err)
}
if len(changed.Each) > 0 {
each := make([]string, 0, len(changed.Each))
for _, c := range changed.Each {
each = append(each, fmt.Sprintf("%s (generation %d)", c.Name, c.Generation))
}
return fmt.Sprintf("rotated in the vault, each consumer's own login of %s from %s, asked by %s: %s; the values "+
"are sealed and not shown, and each holder takes its own from the vault", g.Provision, g.Provider,
changed.By, strings.Join(each, ", ")), nil
}
return fmt.Sprintf("rotated %s in the vault (generation %d), one value for every holder of %s from %s, asked by "+
"%s; the value is sealed and not shown, and each holder takes it from the vault", changed.Name,
changed.Generation, g.Provision, g.Provider, changed.By), nil
}
cmd/mesh-controller/rotate_vault_test.go
+124
View File
@@ -0,0 +1,124 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/inventory"
)
// `rotate <provision>` reaches the vault for a provider filled the new way (issue 767), and the controller writes no
// value of its own for it: its holders go to the vault, named by provider and provision, and no other holder does.
func TestARotationOfAProvisionTheVaultOwnsGoesToTheVaultAndNotTheControllersStore(t *testing.T) {
holders := []inventory.Holder{
{Provision: "postgres-database", Consumer: "g14", ConsumerModule: "gitea", Provider: "novox"},
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "umami", Provider: "novox"},
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "n8n", Provider: "ace"},
}
offering := func(provider, provision string) (string, error) {
if provider == "novox" {
return "postgres", nil
}
return "postgres-old", nil
}
var asked []string
moved := func(module string) (bool, error) {
asked = append(asked, module)
return module == "postgres", nil
}
vault, today, err := rotateRoutes(holders, offering, moved)
if err != nil {
t.Fatal(err)
}
if len(vault) != 1 || vault[0].Provider != "novox" || vault[0].Provision != "postgres-database" ||
vault[0].Module != "postgres" || len(vault[0].Holders) != 2 {
t.Fatalf("to the vault: %+v", vault)
}
if len(today) != 1 || today[0].Provider != "ace" {
t.Fatalf("today's path rotates a holder the vault owns, or misses one it does not: %+v", today)
}
if fmt.Sprint(asked) != "[postgres postgres-old]" {
t.Fatalf("the switch was read for %v, not the offering module", asked)
}
}
// A switch that cannot be read, or a provider whose offering module cannot be found, routes nothing: an error
// before anything changes, never today's path by default.
func TestARotationWhoseOwnerCannotBeReadChangesNothing(t *testing.T) {
holders := []inventory.Holder{{Provision: "postgres-database", Consumer: "g14", ConsumerModule: "gitea",
Provider: "novox"}}
broken := func(string) (bool, error) { return false, errors.New("the catalogue is unreadable") }
if _, _, err := rotateRoutes(holders, func(string, string) (string, error) { return "postgres", nil }, broken); err == nil {
t.Fatal("an unreadable switch fell back to today's path")
}
fine := func(string) (bool, error) { return true, nil }
if _, _, err := rotateRoutes(holders, func(string, string) (string, error) { return "", errors.New("no store") },
fine); err == nil {
t.Fatal("an unreadable provider fell back to today's path")
}
}
// What the vault is told: the provider and provision, and --consumer and --module as the one consumer whose own
// login is remade; nothing named when the operator named nobody, so a shared value is rotated for every holder.
func TestTheVaultIsToldTheProvisionAndTheConsumerTheOperatorNamed(t *testing.T) {
group := rotateGroup{Provider: "novox", Provision: "postgres-database", Module: "postgres"}
for _, c := range []struct {
consumer, module string
want, absent []string
}{
{"", "", []string{`"provider":"novox"`, `"provision":"postgres-database"`, `"why":"yearly"`,
`"by":"g14/claude-code"`}, []string{`"consumer"`, `"consumer_module"`}},
{"g14", "", []string{`"consumer":"g14"`}, []string{`"consumer_module"`}},
{"", "gitea", []string{`"consumer_module":"gitea"`}, []string{`"consumer":`}},
{"g14", "gitea", []string{`"consumer":"g14"`, `"consumer_module":"gitea"`}, nil},
} {
var said string
ask := func(verb string, args map[string]any) (json.RawMessage, error) {
raw, _ := json.Marshal(map[string]any{"verb": verb, "args": args})
said = string(raw)
return json.Marshal(vaultChanged{Name: "provision/novox/postgres-database", By: "g14/claude-code",
Each: []vaultChanged{{Name: "pair/novox/postgres-database/g14/gitea", Generation: 3}}})
}
words, err := rotateProvisionInVault(ask, group, c.consumer, c.module, "g14/claude-code", "yearly")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(said, `"verb":"rotate"`) {
t.Fatalf("%+v: the vault was asked %s", c, said)
}
for _, w := range c.want {
if !strings.Contains(said, w) {
t.Errorf("%+v: the vault was not told %s: %s", c, w, said)
}
}
for _, a := range c.absent {
if strings.Contains(said, a) {
t.Errorf("%+v: the vault was told %s: %s", c, a, said)
}
}
if !strings.Contains(words, "pair/novox/postgres-database/g14/gitea (generation 3)") {
t.Errorf("%+v: the operator was not told whose login was remade: %s", c, words)
}
}
}
// The vault rotates only with a why: asked of the vault without one, nothing is sent; and the vault's refusal (a
// shared value named for one consumer) is the operator's answer, not a rotation in the controller's store.
func TestARotationInTheVaultSaysWhyAndPassesOnItsRefusal(t *testing.T) {
group := rotateGroup{Provider: "anchor", Provision: "stand-in-api", Module: "stand-in"}
sent := 0
ask := func(string, map[string]any) (json.RawMessage, error) {
sent++
return nil, errors.New("not rotated: provision/anchor/stand-in-api is one value every holder shares")
}
if _, err := rotateProvisionInVault(ask, group, "", "", "g14/claude-code", " "); err == nil || sent != 0 {
t.Fatalf("a rotation without a why reached the vault: %v", err)
}
if _, err := rotateProvisionInVault(ask, group, "g14", "", "g14/claude-code", "suspected"); err == nil ||
!strings.Contains(err.Error(), "every holder shares") {
t.Fatalf("the vault's refusal was not said: %v", err)
}
}
cmd/mesh-controller/runtime_health_test.go
+55
View File
@@ -0,0 +1,55 @@
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/link"
)
// novox/hq ADR 0366 §3: a module's own runtime stated unhealthy by its runtime-health raises the module's usual
// condition, module.<module>.<machine>.unhealthy, through ADR 0240's path, saying the tool's reason, and the first
// statement that says it healthy clears it.
func TestAnUnhealthyOwnRuntimeRaisesTheModulesUsualCondition(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv, k := open.inventory, conditionsFrom
const key = "module.mesh-vault.anchor.unhealthy"
said := func(at time.Time, state, reason string) link.Health {
return link.Health{Contract: link.LivenessContract, At: at, Resources: []link.ResourceHealth{{
Module: "mesh-vault", Resource: "mesh-vault.own-runtime", Kind: "process", Target: "mesh-vault-runtime.service",
State: state, Reason: reason, Since: at, Check: "tool"}}}
}
why := "re-sealing to the operator key failed on 3 passes in a row: the operator key is not readable"
for i := 0; i < 2; i++ {
at := h0.Add(time.Duration(i) * time.Minute)
if err := stateHealth(ctx, inv, k, "anchor", said(at, link.StateUnhealthy, why), at); err != nil {
t.Fatal(err)
}
}
c, had, err := k.Get(ctx, key)
if err != nil || !had {
t.Fatalf("two unhealthy statements of its own runtime raised no %s: %v", key, err)
}
evidence := ""
for _, e := range c.Evidence {
evidence += e.Said + "\n"
}
if !strings.Contains(c.Summary+evidence, "operator key is not readable") {
t.Errorf("the condition does not say the tool's reason: %s / %s", c.Summary, evidence)
}
at := h0.Add(3 * time.Minute)
if err := stateHealth(ctx, inv, k, "anchor", said(at, link.StateHealthy, ""), at); err != nil {
t.Fatal(err)
}
list, err := k.Open(ctx)
if err != nil {
t.Fatal(err)
}
for _, o := range list {
if o.Key == key {
t.Fatalf("a healthy statement did not clear %s", key)
}
}
}
cmd/mesh-controller/seatverbs.go
+66 -1
View File
@@ -730,6 +730,38 @@ func (a *verbArguments) commandLine() ([]string, error) {
return nil, fmt.Errorf("%w: letting go of artifacts is a hand act, which says why. Nothing was done", err)
}
return append(argv, "--confirm", "--why", why), nil
// The move of placement out of the settings layers (issue 736), a dry run unless confirmed.
case "move-placement":
argv := []string{"move-placement", "--json"}
why := str("why")
if !on("confirm") {
if why != "" {
return nil, errors.New("move-placement takes why only with confirm: without confirm nothing is " +
"moved, and a reason for nothing would be recorded nowhere. Nothing was done")
}
return argv, nil
}
if err := need("why"); err != nil {
return nil, fmt.Errorf("%w: moving the settings layers' placement is an act by hand, which says why. "+
"Nothing was done", err)
}
return append(argv, "--confirm", "--why", why), nil
// The move of config into mesh-config (issue 664), a dry run unless confirmed.
case "move-config":
argv := []string{"move-config", "--json"}
why := str("why")
if !on("confirm") {
if why != "" {
return nil, errors.New("move-config takes why only with confirm: without confirm nothing is set, " +
"and a reason for nothing would be recorded nowhere. Nothing was done")
}
return argv, nil
}
if err := need("why"); err != nil {
return nil, fmt.Errorf("%w: moving the settings layers into mesh-config is an act by hand, which says why. "+
"Nothing was done", err)
}
return append(argv, "--confirm", "--why", why), nil
case "images":
if err := need("node"); err != nil {
return nil, err
@@ -837,6 +869,13 @@ func (a *verbArguments) commandLine() ([]string, error) {
return nil, err
}
return []string{"secret", "ask", str("node"), str("module"), str("secret"), "--at", str("at")}, nil
case "secret-move":
// The move of the controller's secrets into the vault (design 57's V3, issue 676): no value in the arguments
// or the answer, a machine's name at most.
if n := str("node"); n != "" {
return []string{"secret", "move", n}, nil
}
return []string{"secret", "move"}, nil
case "issued":
// The credential a provider's software issued, taken from the provider (novox/hq ADR 0305): no value in
// the arguments, the provision alone.
@@ -866,6 +905,13 @@ func (a *verbArguments) commandLine() ([]string, error) {
if m := str("module"); m != "" {
argv = append(argv, "--module", m)
}
// Why, which the vault asks of a rotation of a credential it holds (issue 767); a cause only beside it.
if w := str("why"); w != "" {
argv = append(argv, "--why", w)
if c := str("cause"); c != "" {
argv = append(argv, "--cause", c)
}
}
return argv, nil
}
_, node := a.given["node"]
@@ -1036,6 +1082,10 @@ var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "co
"hand-acts": true, "durations": true, "conditions": true, "doctor": true, "retire": true, "cleanup": true, "data": true,
// What the records say the registries may keep (novox/hq ADR 0251).
"artifacts": true, "collect": true, "images": true, "mirrors": true,
// The move of placement says each layer it moved (issue 736).
"move-placement": true,
// The move of config says each value it set and where from (issue 664).
"move-config": true,
// The delivery's owner's verbs answer JSON where they read (plan, order, check, walks) — novox/hq ADR 0239.
"delivery": true,
// The build queue, each ask with its position among the waiting (novox/hq issue 455).
@@ -1081,6 +1131,10 @@ func repairingCommand(argv []string) string {
return "collect"
case argv[0] == "mirrors" && slices.Contains(argv, "--confirm"):
return "mirrors"
case argv[0] == "move-placement" && slices.Contains(argv, "--confirm"):
return "move-placement"
case argv[0] == "move-config" && slices.Contains(argv, "--confirm"):
return "move-config"
}
return ""
}
@@ -1732,6 +1786,16 @@ func takenFromTheProvider(argv []string) bool {
!strings.HasPrefix(argv[2], "-")
}
// movedByTheirMachines is exactly the line the `secret-move` verb composes: `secret move`, or `secret move <node>`
// (design 57's V3, issue 676). It carries no value and answers none: each machine opens what was sealed to it and
// hands it to the vault, which never replaces a value it holds.
func movedByTheirMachines(argv []string) bool {
if len(argv) < 2 || argv[0] != "secret" || argv[1] != "move" {
return false
}
return len(argv) == 2 || (len(argv) == 3 && argv[2] != "" && !strings.HasPrefix(argv[2], "-"))
}
// terminalOnly refuses, through any verb, a command that is the operator's at the controller's terminal
// alone (novox/hq ADR 0266). **Every `node` subcommand that is not a read**: `node account` and
// `node agent-account` above all. Whoever may call a verb includes agents, and an agent that named itself
@@ -1748,7 +1812,8 @@ func terminalOnly(argv []string) error {
// Of a secret's commands only rotation, which seals the new value to the machine that uses it, and the
// `give` verb's own line: an own secret typed by the operator into the desk's hidden prompt, sealed to this
// call and then to the module's machine, so no value travels in the verb or its answer (hq ADR 0259 §10).
if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") && !givenAtTheDesk(argv) && !takenFromTheProvider(argv) {
if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") && !givenAtTheDesk(argv) && !takenFromTheProvider(argv) &&
!movedByTheirMachines(argv) {
return terminalRefusal("secret %s is run at the controller's terminal only, never through a verb: accepting, "+
"recovering or exporting a secret hands it to whoever asks, and that includes agents (novox/hq ADR "+
"0266). Nothing was done", strings.Join(argv[1:], " "))
cmd/mesh-controller/seatverbs_schema_test.go
+2
View File
@@ -285,6 +285,8 @@ var accountedFlags = map[string]map[string]string{
"collect": {"json": "set by the verb: the answer is data"},
"images": {"json": "set by the verb: the answer is data"},
"mirrors": {"json": "set by the verb: the answer is data"},
"move-placement": {"json": "set by the verb: the answer is data"},
"move-config": {"json": "set by the verb: the answer is data"},
"conditions history": {"json": "set by the verb: the answer is data"},
"conditions show": {"json": "set by the verb: the answer is data"},
"healers": {"json": "set by the verb: the answer is data"},
cmd/mesh-controller/secret.go
+124 -6
View File
@@ -53,6 +53,9 @@ func secretCommand(ctx context.Context, args []string) error {
return secretRecover(ctx, args[1:])
case "export":
return secretExport(ctx, args[1:])
case "move":
// The controller's secrets moved into the vault, each by the machine it was sealed to (design 57's V3).
return secretMove(ctx, args[1:])
default:
return errors.New(secretUsage)
}
@@ -111,7 +114,12 @@ func secretCommand(ctx context.Context, args []string) error {
if value == "" {
return errors.New("there is nothing to seal")
}
return acceptForProvision(ctx, node, module, name, value)
// The provider's module decides the path: filled the new way, the vault keeps the provision's credential.
return bySwitch(module, filledTheNewWay,
func() error { return acceptForProvision(ctx, node, module, name, value) },
func() error {
return said(acceptIntoVault(theVault(ctx), provisionArgs(node, name), value, actingAs()))
})
}
if *desk != "" {
if *from != "" || *provider != "" {
@@ -128,23 +136,112 @@ func secretCommand(ctx context.Context, args []string) error {
if value == "" {
return errors.New("there is nothing to seal")
}
// The module whose secrets hold the value decides the path: with --provider, the value is the provider's one
// credential for the provision, so the provider's module decides, as for an outside offer above.
decides := module
if *provider != "" {
if decides, err = offeringModuleOn(ctx, *provider, name); err != nil {
return err
}
}
return bySwitch(decides, filledTheNewWay,
func() error { return acceptKept(ctx, node, module, name, *provider, *local, value) },
func() error {
// Sealed here to the vault's key for this one call; only the sealed box leaves this process.
target := ownSecretArgs(node, module, name)
if *provider != "" {
if *local != "" {
return errors.New("--local names a credential inside the module; the vault keeps one credential " +
"per provider and provision, so it takes no --local: nothing was sealed")
}
target = provisionArgs(*provider, name)
}
if *provider != "" {
return said(acceptIntoVault(theVault(ctx), target, value, actingAs()))
}
return giveIntoVault(ctx, node, module, name, target, value)
})
}
// announceTheGiven is the announcement of a secret given at the terminal (announceSecretGiven, the step acceptKept
// runs); a variable so a test can make it fail.
var announceTheGiven = announceSecretGiven
// giveIntoVault is `secret accept` of a module's own secret for a module filled the new way, under the same
// N1-give rule as acceptKept, through the same steps (keepGiven, announceSecretGiven): a module that runs as an
// account of its own is announced on the operator's channels before the vault is asked, and when the announcement
// fails nothing is sealed and the vault is never called; any other is sealed first and announced after.
func giveIntoVault(ctx context.Context, node, module, name string, target map[string]any, value string) error {
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
trusted, err := open.inventory.RunsAsItsOwnAccount(ctx, module)
if err != nil {
return err
}
var words string
askedBy := link.Caller()
_, unannounced, err := keepGiven(trusted,
func() error {
return announceTheGiven(ctx, node, module, name, "at the controller's terminal", askedBy)
},
func() (bool, error) {
var err error
words, err = acceptIntoVault(theVault(ctx), target, value, actingAs())
return false, err
})
if err != nil {
if trusted && unannounced != nil {
return fmt.Errorf("%s runs as an account of its own, and the change of its %s could not be announced on "+
"your channels first, so nothing was sealed and the vault was not asked: %w", module, name, err)
}
return err
}
fmt.Println(words)
if unannounced != nil {
fmt.Printf(" this change could NOT be announced on the operator's channels: %v\n", unannounced)
}
// The giving is recorded, never the value (novox/hq ADR 0380 §6), as acceptKept records its own.
act := terminalGiveHandAct(node, module, name, askedBy)
if err := onTheBus(func(conn *nats.Conn) error {
_, err := link.RecordHandAct(ctx, conn, act)
return err
}); err != nil {
fmt.Printf(" this act could NOT be recorded in the hand-act log, and is done anyway: %v\n", err)
}
return nil
}
// said prints what was done, or answers why not.
func said(words string, err error) error {
if err != nil {
return err
}
fmt.Println(words)
return nil
}
// acceptKept is today's `secret accept`: the value sealed into the controller's store, for a module whose secrets
// are filled the old way.
func acceptKept(ctx context.Context, node, module, name, provider, local, value string) error {
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
if *provider != "" {
if provider != "" {
// Into the pair, not into the module's own secrets: what the provider is asked to create
// and what the consumer reads are the same value, and neither end can be told a different
// one later without the other (novox/hq 04-ISSUES/070).
if err := open.inventory.AcceptSecretForPair(ctx, name, node, module, *provider, *local, value); err != nil {
if err := open.inventory.AcceptSecretForPair(ctx, name, node, module, provider, local, value); err != nil {
return err
}
fmt.Printf("%s on %s now holds %q from %s%s, sealed to both machines.\n", module, node, name, *provider, asLocal(*local))
fmt.Printf("%s on %s now holds %q from %s%s, sealed to both machines.\n", module, node, name, provider, asLocal(local))
fmt.Printf(" the mesh cannot read it back, will not replace it with one of its own, and will not rotate it\n")
fmt.Printf(" run `push %s` and `push %s` to send it\n", *provider, node)
fmt.Printf(" run `push %s` and `push %s` to send it\n", provider, node)
return nil
}
// A trusted party's secret is announced before it is kept (the confirmation review of 2026-10-09, N1-give):
@@ -221,7 +318,8 @@ const secretUsage = "secret rotate <node> <module> <name> [--why <text> [--cause
"secret accept <node> <module> <name> [--from <file> | --at-desk <machine>] [--provider <node> [--local <name>]]\n" +
"secret accept <provider's node> <provider module> <provision> [--from <file>] (a credential issued outside the mesh, for every consumer)\n" +
"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>] [--provider <node>]\n" +
"secret export [--out <file>]"
"secret export [--out <file>]\n" +
"secret move [<node>] (every secret the controller holds, moved into the vault by the machine it was sealed to)"
// secretRecover is break-glass: a secret opened with the operator's key, written to a file.
//
@@ -500,6 +598,26 @@ func secretRotate(ctx context.Context, args []string) error {
return errors.New(secretUsage)
}
node, module, name := rest[0], rest[1], rest[2]
return bySwitch(module, filledTheNewWay,
func() error { return rotateKept(ctx, node, module, name, why) },
func() error {
reason := ""
if why.why != nil {
reason = strings.TrimSpace(*why.why)
}
words, err := rotateInVault(theVault(ctx), ownSecretArgs(node, module, name), actingAs(), reason)
if err != nil {
return err
}
why.record(ctx, "secret rotate", []string{node, module, name})
fmt.Println(words)
return nil
})
}
// rotateKept is today's `secret rotate`: made anew in the controller's store and sent, for a module whose
// secrets are filled the old way.
func rotateKept(ctx context.Context, node, module, name string, why handActFlags) error {
open, err := openStores(ctx)
if err != nil {
return err
cmd/mesh-controller/secret_move.go
+336
View File
@@ -0,0 +1,336 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// `secret move` — the controller's secrets move into the vault (novox/hq ADR 0349 clause 7; design 57's V3, issue 676;
// the operator's decision of 2026-10-14 on that issue). Run once, after the feature's delivery.
//
// **The controller cannot open what it holds, so each machine moves its own.** For every machine — or the ones named —
// the controller reads every row sealed to it, as the store holds it, and asks that machine's node-engine to move each
// one, signed (link.AskMoveSecret): the engine opens it with its sealing key and hands it to the vault's move-in, sealed
// to the call's key, keeping its origin and generation. The vault takes it from that engine alone, never over a value
// it holds, and reads back what it kept to the engine, which compares; the engine answers only then. What moved is
// recorded here, never a value. Nothing is rotated, and nothing points at the vault because of this: a module's
// secrets are filled from the vault only once it moves (V4, issue 647).
//
// **A machine asleep is moved when it is next heard from**: once the move has started, a machine that reports is moved
// in the background (moveWhenHeard), and what is left of it is said on the next `secret move`.
//
// The controller's record of asks at a desk, which holds no value, is copied as it is, once (`move-asks`).
const secretMoveUsage = "secret move [<node>] — move every secret the controller holds into the vault, as it is: each " +
"machine's node-engine opens what was sealed to it and hands it to the vault, which reads it back; no value is " +
"rotated, and a machine not answering is moved when it is next heard from"
// moveAsk is the ask of one row of a machine's engine.
type moveAsk func(row inventory.SecretToMove, ask link.MoveSecretAsk) (link.MoveSecretAnswer, error)
// nodeMove is what the move did on one machine.
type nodeMove struct {
Node string
Moved int
Already int
// Refused are the rows not moved, each with why; Changed the rows changed in the store after they moved.
Refused []string
Changed []string
// Unreached is why the machine's engine could not be asked: it is moved when next heard from.
Unreached error
// Left is how many rows were not asked because the machine was not reached.
Left int
// Failed is why its rows could not be read.
Failed error
}
// done says nothing is left to move on the machine.
func (m nodeMove) done() bool {
return m.Failed == nil && m.Unreached == nil && len(m.Refused) == 0
}
func (m nodeMove) String() string {
var b strings.Builder
if m.Failed != nil {
fmt.Fprintf(&b, "%s: its rows could not be read, nothing was moved: %v\n", m.Node, m.Failed)
return b.String()
}
fmt.Fprintf(&b, "%s: %d moved into the vault and read back the same, %d there already", m.Node, m.Moved, m.Already)
if n := len(m.Refused); n > 0 {
fmt.Fprintf(&b, ", %d not moved", n)
}
b.WriteString("\n")
for _, r := range m.Refused {
fmt.Fprintf(&b, " not moved: %s\n", r)
}
for _, c := range m.Changed {
fmt.Fprintf(&b, " %s was changed in the controller's store after it moved; the vault keeps what it took, "+
"and the move never replaces it\n", c)
}
if m.Unreached != nil {
fmt.Fprintf(&b, " %d left: %s was not reached (%v); it is moved when it is next heard from\n", m.Left, m.Node,
m.Unreached)
}
return b.String()
}
// moveRowAsk is a row as its machine's engine is asked to move it.
func moveRowAsk(r inventory.SecretToMove) link.MoveSecretAsk {
a := link.MoveSecretAsk{Origin: r.Origin, Generation: r.Generation, MadeAt: r.MadeAt.UTC(), Sealed: r.Sealed,
SealedTo: r.SealedTo}
if r.Own() {
a.Module, a.Name = r.Module, r.Name
} else {
a.Provider, a.Provision, a.Consumer, a.ConsumerModule, a.Local = r.Provider, r.Provision, r.Consumer,
r.ConsumerModule, r.Local
}
return a
}
// moveNode moves every row sealed to one machine that has not moved, one ask each, and records what the vault kept.
// The first ask the engine does not answer ends the machine's turn: the rest wait for it to be heard from again.
func moveNode(node string, rows func(string) ([]inventory.SecretToMove, error), ask moveAsk,
record func(inventory.SecretToMove, int) error) nodeMove {
m := nodeMove{Node: node}
all, err := rows(node)
if err != nil {
m.Failed = err
return m
}
for n, r := range all {
switch {
case r.Moved != nil && r.Moved.Digest == r.Digest:
m.Already++
continue
case r.Moved != nil:
m.Changed = append(m.Changed, r.Secret)
continue
case !r.Current:
m.Refused = append(m.Refused, fmt.Sprintf("%s is sealed to an earlier sealing key of %s, which no longer "+
"opens it: nothing can move it; it moves once it is made or accepted again", r.Secret, node))
continue
}
answer, err := ask(r, moveRowAsk(r))
if err != nil {
m.Unreached, m.Left = err, len(all)-n
return m
}
if answer.Refused != "" {
m.Refused = append(m.Refused, fmt.Sprintf("%s: %s", r.Secret, answer.Refused))
continue
}
if answer.Secret != r.Secret {
m.Refused = append(m.Refused, fmt.Sprintf("%s: the vault kept it as %s, not under the name the move "+
"gives it", r.Secret, answer.Secret))
continue
}
if err := record(r, answer.Generation); err != nil {
m.Refused = append(m.Refused, fmt.Sprintf("%s is in the vault, but that it moved could not be recorded: %v",
r.Secret, err))
continue
}
if answer.Unchanged {
m.Already++
} else {
m.Moved++
}
}
return m
}
// movedAsksAtOnce is how many rows of the record of asks one call to the vault carries (mesh-vault MovedAsksAtOnce).
const movedAsksAtOnce = 500
// moveAsks copies the controller's record of asks into the vault, in parts, and answers how many it copied.
func moveAsks(asks []inventory.SecretAsk, vault vaultAsker) (int, error) {
copied := 0
for start := 0; start < len(asks); start += movedAsksAtOnce {
end := min(start+movedAsksAtOnce, len(asks))
var rows []map[string]any
for _, a := range asks[start:end] {
row := map[string]any{"id": a.ID, "node": a.Node, "module": a.Module, "name": a.Name, "desk": a.Desk,
"asked_by": a.AskedBy, "opened_at": a.OpenedAt.UTC().Format(time.RFC3339Nano)}
if a.EndedAt != nil {
row["ended_at"], row["outcome"] = a.EndedAt.UTC().Format(time.RFC3339Nano), a.Outcome
}
rows = append(rows, row)
}
raw, err := vault("move-asks", map[string]any{"asks": rows})
if err != nil {
return copied, fmt.Errorf("the vault did not take the record of asks after %d rows: %w", copied, err)
}
var said struct {
Copied int `json:"copied"`
Already int `json:"already"`
}
_ = json.Unmarshal(raw, &said)
copied += said.Copied + said.Already
}
return copied, nil
}
// moveOnTheBus is the move's ask over the bus, signed with the mesh's key.
func moveOnTheBus(ctx context.Context, open *stores, conn *nats.Conn, by string) (moveAsk, error) {
ident, err := open.Identity(ctx)
if err != nil {
return nil, fmt.Errorf("the mesh's signing key cannot be read, so nothing was asked: %w", err)
}
return func(r inventory.SecretToMove, row link.MoveSecretAsk) (link.MoveSecretAnswer, error) {
return link.AskMoveSecret(ctx, conn, ident, r.Node, by, row, link.MoveSecretWithin)
}, nil
}
// secretMove is `secret move [<node>]`: the record of asks copied once, then every machine's rows — or one's — and
// what moved said per machine. Any row not moved fails the command, said by name.
func secretMove(ctx context.Context, args []string) error {
if len(args) > 1 || (len(args) == 1 && (args[0] == "" || strings.HasPrefix(args[0], "-"))) {
return errors.New(secretMoveUsage)
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
by := actingAs()
if by == "" {
by = handOverBy()
}
move, err := inv.StartSecretMove(ctx, by)
if err != nil {
return err
}
var nodes []string
if len(args) == 1 {
if _, err := inv.NodeByName(ctx, args[0]); err != nil {
return fmt.Errorf("nothing was moved: %w", err)
}
nodes = args
} else {
all, err := inv.Nodes(ctx)
if err != nil {
return err
}
for _, n := range all {
nodes = append(nodes, n.Name)
}
}
fmt.Printf("the move of the secrets into the vault, started %s by %s\n", move.StartedAt.Local().Format("2006-01-02 15:04"),
move.StartedBy)
if move.AsksMovedAt == nil {
asks, err := inv.SecretAsks(ctx, time.Time{})
if err != nil {
return err
}
copied, err := moveAsks(asks, theVault(ctx))
if err != nil {
return fmt.Errorf("the record of asks was not copied, and no secret was moved: %w", err)
}
if err := inv.RecordAsksMoved(ctx); err != nil {
return err
}
fmt.Printf("the record of asks at a desk: %d rows copied into the vault as they are\n", copied)
}
failed := 0
err = onTheBus(func(conn *nats.Conn) error {
ask, err := moveOnTheBus(ctx, open, conn, "the move of the secrets, by "+by)
if err != nil {
return err
}
for _, node := range nodes {
m := moveNode(node, func(n string) ([]inventory.SecretToMove, error) { return inv.SecretsToMoveFrom(ctx, n) },
ask, func(r inventory.SecretToMove, g int) error { return inv.RecordSecretMoved(ctx, r, g) })
fmt.Print(m.String())
if m.Failed != nil || len(m.Refused) > 0 {
failed++
}
}
return nil
})
if err != nil {
return err
}
if failed > 0 {
return fmt.Errorf("on %d machine(s) a secret was not moved, said above by name", failed)
}
return nil
}
// moveHeard is the machines being moved in the background now, and when each was last moved, so a machine that reports
// often is not asked again before moveHeardEvery has passed.
var moveHeard = struct {
sync.Mutex
running map[string]bool
last map[string]time.Time
}{running: map[string]bool{}, last: map[string]time.Time{}}
// moveHeardEvery is how often a machine heard from is moved at most.
const moveHeardEvery = 10 * time.Minute
// moveWhenHeard moves what is left of a machine that was just heard from, once the move has started: a machine asleep
// when it started is moved when it next reports. What it did is said on the controller's output, and recorded for the
// next `secret move`.
func moveWhenHeard(ctx context.Context, open *stores, conn *nats.Conn, node string) {
if open == nil || conn == nil || node == "" {
return
}
moveHeard.Lock()
if moveHeard.running[node] || time.Since(moveHeard.last[node]) < moveHeardEvery {
moveHeard.Unlock()
return
}
moveHeard.running[node], moveHeard.last[node] = true, time.Now()
moveHeard.Unlock()
defer func() {
moveHeard.Lock()
delete(moveHeard.running, node)
moveHeard.Unlock()
}()
inv := open.inventory
move, started, err := inv.SecretMoveStarted(ctx)
if err != nil || !started {
return
}
rows, err := inv.SecretsToMoveFrom(ctx, node)
if err != nil {
return
}
left := false
for _, r := range rows {
left = left || (r.Moved == nil && r.Current)
}
if !left {
return
}
ask, err := moveOnTheBus(ctx, open, conn, "the move of the secrets, by "+move.StartedBy+", on hearing from "+node)
if err != nil {
fmt.Fprintf(os.Stderr, "mesh-controller: %s was heard from and its secrets not moved: %v\n", node, err)
return
}
m := moveNode(node, func(string) ([]inventory.SecretToMove, error) { return rows, nil }, ask,
func(r inventory.SecretToMove, g int) error { return inv.RecordSecretMoved(ctx, r, g) })
if m.done() {
fmt.Printf("mesh-controller: %s was heard from; the move of its secrets is done: %s", node, m.String())
return
}
fmt.Fprintf(os.Stderr, "mesh-controller: %s was heard from; the move of its secrets is not done: %s", node, m.String())
}
// moveHeardOn is the connection a machine heard from is moved over: the serving controller's bus.
func moveHeardOn() *nats.Conn {
if js := servingBus.Load(); js != nil {
return js.Conn()
}
return nil
}
cmd/mesh-controller/secret_move_test.go
+158
View File
@@ -0,0 +1,158 @@
package main
import (
"encoding/json"
"errors"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
func aRowToMove(secret string, current bool) inventory.SecretToMove {
return inventory.SecretToMove{Secret: secret, Node: "anchor", Module: "postgres", Name: strings.TrimPrefix(secret, "own/anchor/postgres/"),
Origin: "made", Generation: "g-1", MadeAt: time.Unix(1, 0), Sealed: "c2VhbGVk", SealedTo: "k", Current: current,
Digest: "sha256:" + secret}
}
// Each row sealed to a machine is asked of its engine once, as the store holds it; what the vault kept and read back
// the same is recorded; a row the vault held already counts as there; a row moved before is not asked again, and one
// changed in the store since it moved is said; a row sealed to a key the machine no longer has is said, not asked.
func TestAMachinesRowsAreMovedOnceEachAndWhatMovedIsSaid(t *testing.T) {
rows := []inventory.SecretToMove{
aRowToMove("own/anchor/postgres/superuser", true),
aRowToMove("own/anchor/postgres/replication", true),
aRowToMove("own/anchor/postgres/old-key", false),
aRowToMove("own/anchor/postgres/refused", true),
aRowToMove("own/anchor/postgres/moved-before", true),
aRowToMove("own/anchor/postgres/changed-since", true),
}
rows[4].Moved = &inventory.SecretMoved{Node: "anchor", Generation: 1, Digest: rows[4].Digest}
rows[5].Moved = &inventory.SecretMoved{Node: "anchor", Generation: 1, Digest: "sha256:something before"}
var asked []link.MoveSecretAsk
ask := func(r inventory.SecretToMove, row link.MoveSecretAsk) (link.MoveSecretAnswer, error) {
asked = append(asked, row)
switch r.Secret {
case "own/anchor/postgres/refused":
return link.MoveSecretAnswer{Refused: "the vault already holds another value; nothing was moved"}, nil
case "own/anchor/postgres/replication":
return link.MoveSecretAnswer{Secret: r.Secret, Generation: 3, Unchanged: true}, nil
}
return link.MoveSecretAnswer{Secret: r.Secret, Generation: 1}, nil
}
recorded := map[string]int{}
record := func(r inventory.SecretToMove, generation int) error { recorded[r.Secret] = generation; return nil }
m := moveNode("anchor", func(string) ([]inventory.SecretToMove, error) { return rows, nil }, ask, record)
if m.Moved != 1 || m.Already != 2 || len(m.Refused) != 2 || len(m.Changed) != 1 || m.Unreached != nil {
t.Fatalf("the move said %+v", m)
}
if len(asked) != 3 || asked[0].Module != "postgres" || asked[0].Name != "superuser" || asked[0].Sealed != "c2VhbGVk" ||
asked[0].Origin != "made" || asked[0].Generation != "g-1" || asked[0].SealedTo != "k" {
t.Fatalf("asked %+v", asked)
}
if recorded["own/anchor/postgres/superuser"] != 1 || recorded["own/anchor/postgres/replication"] != 3 || len(recorded) != 2 {
t.Errorf("recorded %v", recorded)
}
said := m.String()
for _, want := range []string{"anchor: 1 moved into the vault and read back the same, 2 there already",
"own/anchor/postgres/old-key", "earlier sealing key", "own/anchor/postgres/refused", "already holds another value",
"own/anchor/postgres/changed-since", "changed in the controller's store after it moved"} {
if !strings.Contains(said, want) {
t.Errorf("the move did not say %q:\n%s", want, said)
}
}
if strings.Contains(said, "c2VhbGVk") {
t.Error("the move said a sealed value")
}
}
// A machine that does not answer is left for when it is next heard from: nothing more is asked of it now, and that
// is said; a pair is asked with both its ends and never as an own secret.
func TestAMachineNotAnsweringIsMovedWhenNextHeardFrom(t *testing.T) {
pair := inventory.SecretToMove{Secret: "pair/anchor/database/shanks/gitea", Node: "shanks", Provider: "anchor",
Provision: "database", Consumer: "shanks", ConsumerModule: "gitea", Origin: "accepted", Sealed: "x",
SealedTo: "k", Current: true}
rows := []inventory.SecretToMove{pair, pair}
calls := 0
var asked link.MoveSecretAsk
ask := func(_ inventory.SecretToMove, row link.MoveSecretAsk) (link.MoveSecretAnswer, error) {
calls++
asked = row
return link.MoveSecretAnswer{}, errors.New("nothing on shanks answers a move")
}
m := moveNode("shanks", func(string) ([]inventory.SecretToMove, error) { return rows, nil }, ask,
func(inventory.SecretToMove, int) error { t.Fatal("recorded"); return nil })
if calls != 1 || m.Unreached == nil || !strings.Contains(m.String(), "moved when it is next heard from") {
t.Fatalf("calls %d, %+v: %s", calls, m, m.String())
}
if asked.Module != "" || asked.Provider != "anchor" || asked.Consumer != "shanks" || asked.ConsumerModule != "gitea" {
t.Errorf("the pair was asked as %+v", asked)
}
failing := moveNode("x", func(string) ([]inventory.SecretToMove, error) { return nil, errors.New("no store") }, nil, nil)
if failing.Failed == nil {
t.Error("a store that cannot be read was not said")
}
}
// The record of asks is copied into the vault in parts, as it is: every row's fields, nothing more.
func TestTheRecordOfAsksIsCopiedInParts(t *testing.T) {
ended := time.Unix(100, 0).UTC()
var asks []inventory.SecretAsk
for n := 1; n <= movedAsksAtOnce+3; n++ {
asks = append(asks, inventory.SecretAsk{ID: int64(n), Node: "anchor", Module: "telegram", Name: "token",
AskedBy: "g14/claude-code", Desk: "laptop", OpenedAt: time.Unix(int64(n), 0).UTC(), EndedAt: &ended,
Outcome: "given"})
}
var parts []int
vault := func(verb string, args map[string]any) (json.RawMessage, error) {
if verb != "move-asks" {
t.Fatalf("asked %s", verb)
}
rows := args["asks"].([]map[string]any)
parts = append(parts, len(rows))
if rows[0]["desk"] != "laptop" || rows[0]["asked_by"] != "g14/claude-code" || rows[0]["outcome"] != "given" ||
rows[0]["id"] == nil || rows[0]["opened_at"] == nil || rows[0]["ended_at"] == nil {
t.Fatalf("a row went as %v", rows[0])
}
return json.Marshal(map[string]int{"copied": len(rows)})
}
copied, err := moveAsks(asks, vault)
if err != nil || copied != movedAsksAtOnce+3 || len(parts) != 2 || parts[0] != movedAsksAtOnce {
t.Fatalf("copied %d in %v, %v", copied, parts, err)
}
if _, err := moveAsks(asks, func(string, map[string]any) (json.RawMessage, error) {
return nil, errors.New("the vault is away")
}); err == nil {
t.Error("a copy the vault refused was not said")
}
}
// The verb runs the move's one line through the seat and nothing else of the secret commands.
func TestTheMoveIsAVerbOfItsOwn(t *testing.T) {
argv, err := argvFor("secret-move", map[string]any{})
if err != nil || strings.Join(argv, " ") != "secret move" {
t.Fatalf("argv %v, %v", argv, err)
}
argv, err = argvFor("secret-move", map[string]any{"node": "anchor"})
if err != nil || strings.Join(argv, " ") != "secret move anchor" {
t.Fatalf("argv %v, %v", argv, err)
}
if err := terminalOnly([]string{"secret", "move", "anchor"}); err != nil {
t.Fatalf("the move's line was refused through its verb: %v", err)
}
for _, line := range [][]string{{"secret", "move", "--from", "x"}, {"secret", "move", "a", "b"}} {
if err := terminalOnly(line); err == nil {
t.Errorf("%q ran through a verb", line)
}
}
found := false
for _, v := range catalogue.ControllerVerbs {
found = found || v.Name == "secret-move"
}
if !found {
t.Fatal("the controller's seat does not declare the secret-move verb")
}
}
cmd/mesh-controller/sendable.go
+17
View File
@@ -77,6 +77,9 @@ type sendable struct {
// unplaced is every contribution its holder's template could not render, naming its module and
// why (novox/hq ADR 0255): left out of this declaration, for push and plan to say, never on the wire.
unplaced []string
// handedOnly is each file left out because only the provisioner's client fills it and the module is not handed
// here (a provider's consumers file, ADR 0373).
handedOnly []string
// Builds is the build of each module this declaration carries — module to the commit its build
// was made from — recorded with the send and never on the wire (novox/hq issue 259, ADR 0221).
// Composed only on the send path; nil records that it is not known.
@@ -84,6 +87,16 @@ type sendable struct {
// Bindings is where each consumer of a provision that keeps its data is bound in this declaration
// (novox/hq ADR 0232), recorded with the send and never on the wire. Composed only on the send path.
Bindings []inventory.Binding
// Filled is the path of every file this declaration hands to the provisioner, and Provisioner the account the
// client that fills them runs as (novox/hq ADR 0349 §4, §5, design 57 P6): on the wire as `filled_files` and
// `provisioner_account`, both absent when nothing is handed, so every other declaration is byte for byte what
// it was. plans is each handed module's plan, kept with the send for the assignments mirror; never on the wire.
Filled []string
Provisioner string
plans map[string]catalogue.Plan
// requiredBy is, for each handed module only pulled in by another, the module that requires it (issue 765): kept
// beside its plan, so the assignments mirror gives the plan a value of its own. Never on the wire.
requiredBy map[string]string
}
// adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on
@@ -113,6 +126,10 @@ func (s sendable) Body() ([]byte, error) {
if s.Generation > 0 {
envelope["generation"] = s.Generation
}
if len(s.Filled) > 0 {
envelope["filled_files"] = s.Filled
envelope["provisioner_account"] = s.Provisioner
}
// An empty declaration is deliberate here — the node owns nothing the mesh put there
// (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness
// is meant, so a truncated or mis-composed body is never mistaken for "own nothing".
cmd/mesh-controller/sent_required_by_test.go
+125
View File
@@ -0,0 +1,125 @@
package main
import (
"encoding/json"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A send keeps, beside each handed module's plan, what requires a module only pulled in (issue 765), so the
// assignments mirror gives that plan a value of its own; an assigned module's plan is kept as the bytes it was.
func TestASendKeepsWhatRequiresAPulledInModule(t *testing.T) {
plan := catalogue.Plan{Templates: catalogue.PlanTemplates{Source: "http://store:5000/v2/notify/files",
Digest: "sha256:" + strings.Repeat("d", 64)},
Files: []catalogue.PlanFile{{ID: "rc", Path: "/home/op/rc", Template: "rc", OwnerAtRuntime: "owned"}}}
r := sentRecordOf(t.Context(), sendable{plans: map[string]catalogue.Plan{"dunst": plan, "notify": plan},
requiredBy: map[string]string{"notify": "desk"}})
kept := string(r.plans)
want := `{"dunst":{"templates":{"source":"http://store:5000/v2/notify/files","digest":"sha256:` +
strings.Repeat("d", 64) + `"},"files":[{"id":"rc","path":"/home/op/rc","template":"rc","owner-at-runtime":"owned"}]},` +
`"notify":{"templates":{"source":"http://store:5000/v2/notify/files","digest":"sha256:` + strings.Repeat("d", 64) +
`"},"files":[{"id":"rc","path":"/home/op/rc","template":"rc","owner-at-runtime":"owned"}],"required-by":"desk"}}`
if kept != want {
t.Fatalf("the send keeps\n%s\nwant\n%s", kept, want)
}
}
// **A module only pulled in by another is handed on a machine that can take it** (issue 765): on a contract-11
// node running a client, a moved module that nothing assigned there, required by one that is, has its files and its
// own secret handed like an assigned one's, no secret of it made or sealed by the controller, and its plan mirrored
// under its own key, saying what requires it. A module not moved, pulled in the same way, is sent as before.
func TestAModulePulledInIsHandedOnAMachineThatCanTakeItAndMirrored(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
client := aClientModule()
moved, kept := aBot("moved-bot", true), aBot("kept-bot", false)
moved.Provides = []catalogue.Offer{{Name: "moved-bot-api"}}
kept.Provides = []catalogue.Offer{{Name: "kept-bot-api"}}
desk := catalogue.Manifest{Module: "desk", Version: "1", Requires: []string{"moved-bot-api", "kept-bot-api"}}
for _, m := range []catalogue.Manifest{client, moved, kept, desk} {
register(t, open, m)
}
for _, m := range []string{client.Module, desk.Module} {
if _, err := assign(ctx, open, "laptop", m); err != nil {
t.Fatal(err)
}
}
if err := issueProvisionerClient(ctx, inv, client, "laptop", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222"); err != nil {
t.Fatal(err)
}
if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: link.HandedContract,
SaidAt: time.Now()}); err != nil {
t.Fatal(err)
}
plan, settings, err := planFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
if plan.RequiredBy("moved-bot") != "desk" || plan.RequiredBy("kept-bot") != "desk" {
t.Fatalf("the bots are on laptop because %v", plan.Because)
}
declared, err := declarationFor(ctx, open, "laptop", plan, settings)
if err != nil {
t.Fatal(err)
}
var movedToken, keptToken, movedConf map[string]any
for _, r := range declared.Resources {
switch r["id"] {
case "moved-bot.needs-token":
movedToken = r
case "kept-bot.needs-token":
keptToken = r
case "moved-bot.conf":
movedConf = r
}
}
if movedToken == nil || movedToken["sealed"] != nil || movedToken["owner-at-runtime"] != "owned" {
t.Fatalf("the pulled-in moved module's secret file is sent as %v: placed, with no value", movedToken)
}
if movedConf == nil || movedConf["content"] != nil || movedConf["owner-at-runtime"] != "owned" {
t.Fatalf("the pulled-in moved module's templated file is sent as %v: placed, with no text", movedConf)
}
if keptToken == nil || keptToken["sealed"] == nil {
t.Fatalf("the pulled-in module not moved has its secret sent as %v: sealed, as before", keptToken)
}
if _, held, err := inv.ModuleSecretIfIssued(ctx, "laptop", "moved-bot", "token"); err != nil || held {
t.Fatalf("the send made the pulled-in handed module's secret in the controller's store (held %v, %v)", held, err)
}
if _, has := declared.plans["kept-bot"]; has || len(declared.plans) != 1 {
t.Fatalf("the send hands %v", declared.plans)
}
record, err := inv.NodeByName(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, inventory.Send{Digest: "first",
Provisioning: sentRecordOf(ctx, declared).plans}); err != nil {
t.Fatal(err)
}
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
mirror, err := inv.AssignmentsMirror(ctx)
if err != nil {
t.Fatal(err)
}
var value inventory.AssignmentMirrored
if err := json.Unmarshal(mirror["laptop.moved-bot"], &value); err != nil {
t.Fatalf("no value for the pulled-in module: %s (%v)", mirror["laptop.moved-bot"], err)
}
if value.RequiredBy != "desk" || !value.Assigned.IsZero() || value.Provisioning == nil ||
len(value.Provisioning.Files) != 1 || value.Provisioning.Files[0].Path != "/etc/moved-bot/bot.conf" ||
len(value.Provisioning.Secrets) != 1 || value.Provisioning.Secrets[0].Name != "token" {
t.Fatalf("the pulled-in module is mirrored as %s", mirror["laptop.moved-bot"])
}
if _, has := mirror["laptop.kept-bot"]; has {
t.Fatalf("a pulled-in module not handed is mirrored: %s", mirror["laptop.kept-bot"])
}
}
cmd/mesh-controller/settings_forward.go
+372
View File
@@ -0,0 +1,372 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// **`settings` answers by forwarding to `config`** (novox/hq ADR 0348 §1, design 57 K5, issue 664). Once the move of
// config has run, a module in the config form keeps its config values in mesh-config (inventory.ReadsConfig), so the
// settings verb reads and writes them there: `show` lists what `config list` lists, `set` sets each value as the
// module's own on the node and `clear` clears them, and `propose` proposes each through mesh-config. Placement stays
// the controller's and is read and written beside the assignment as before (ADR 0348 §11). A module still in the
// settings form, and every module before the move, keeps today's verb unchanged.
// configAsk asks mesh-config's verb in the controller's own name, so mesh-config takes a value as the controller
// carries it (ADR 0348 §8); a test replaces it.
var configAsk = func(ctx context.Context, args map[string]any) (json.RawMessage, error) {
var result json.RawMessage
err := onTheBus(func(conn *nats.Conn) error {
answer, err := link.AskMeshSeatToolAs(ctx, conn, broker.ControllerName, "mesh-config", "config", args, 30*time.Second)
if err != nil {
return err
}
if answer.Error != "" {
return errors.New(answer.Error)
}
result = answer.Result
return nil
})
return result, err
}
// configListed is what `config list` answers, as the settings verb reads it.
type configListed struct {
Resolved []resolvedEntry `json:"resolved"`
Set []struct {
Level configLevel `json:"level"`
Key string `json:"key"`
Value json.RawMessage `json:"value"`
By string `json:"by"`
} `json:"set"`
}
func listConfig(ctx context.Context, module, node string) (configListed, error) {
args := map[string]any{"action": "list", "module": module}
if node != "" {
args["node"] = node
}
raw, err := configAsk(ctx, args)
if err != nil {
return configListed{}, fmt.Errorf("mesh-config does not answer %s's config values: %w", module, err)
}
var l configListed
if err := json.Unmarshal(raw, &l); err != nil {
return configListed{}, fmt.Errorf("mesh-config's listing cannot be read: %w", err)
}
return l, nil
}
// ownValues is the module's own values on the node, as mesh-config holds them; none for the whole mesh, which has no
// module value (ADR 0348 §3).
func (l configListed) ownValues(module, node string) map[string]any {
out := map[string]any{}
if node == "" {
return out
}
for _, s := range l.Set {
if s.Level.Level == "module" && s.Level.Node == node && s.Level.Module == module {
var v any
if json.Unmarshal(s.Value, &v) == nil {
out[s.Key] = v
}
}
}
return out
}
// forwardSettings answers a settings action for a module whose config values are mesh-config's, and says whether it
// did: false leaves the action to today's verb.
func forwardSettings(ctx context.Context, inv *inventory.Inventory, action, module, node string, positionals []string,
replace bool) (bool, error) {
if module == "" {
return false, nil
}
reads, err := inv.ReadsConfig(ctx, module)
if err != nil || !reads {
return false, err
}
where := whereOf(node)
switch action {
case "show":
listed, err := listConfig(ctx, module, node)
if err != nil {
return true, err
}
layer, _, err := inv.Layer(ctx, node, module)
if err != nil {
return true, err
}
if layer == nil {
layer = map[string]any{}
}
for k, v := range listed.ownValues(module, node) {
layer[k] = v
}
if len(layer) == 0 {
fmt.Printf("%s on %s: no layer — the module's definition says\n", module, where)
} else {
shown, err := json.MarshalIndent(layer, "", " ")
if err != nil {
return true, err
}
fmt.Println(string(shown))
}
fmt.Printf(" its config values are mesh-config's since the move of config (issue 664): mesh-config.config\n")
fmt.Printf("%s on %s, every value and where it comes from:\n", module, where)
for _, r := range listed.Resolved {
value := string(r.Value)
if value == "" {
value = "unset"
}
on := ""
if node == "" {
on = " on " + r.Node
}
fmt.Printf(" %s%s = %s (%s)\n", r.Key, on, value, r.Source)
}
return true, nil
case "set":
var raw []byte
if strings.HasPrefix(strings.TrimSpace(positionals[1]), "{") {
raw = []byte(positionals[1])
} else {
return true, errors.New("the settings of a module whose config values are mesh-config's are given as " +
"{…}, never a file; or set each with mesh-config.config")
}
var values map[string]any
if err := json.Unmarshal(raw, &values); err != nil {
return true, fmt.Errorf("%s is not a settings object: %w", positionals[1], err)
}
return true, forwardSet(ctx, inv, module, node, values, replace)
case "clear":
listed, err := listConfig(ctx, module, node)
if err != nil {
return true, err
}
own := listed.ownValues(module, node)
layer, _, err := inv.Layer(ctx, node, module)
if err != nil {
return true, err
}
before := map[string]any{}
for k, v := range layer {
before[k] = v
}
for k, v := range own {
before[k] = v
}
if err := refuseTerminalSettingsThroughAVerb(ctx, inv, before, nil, module, where); err != nil {
return true, err
}
if err := refuseTrustedThroughAVerb(ctx, inv, module, where, movedKeys(own)); err != nil {
return true, err
}
for _, k := range movedKeys(own) {
if _, err := configAsk(ctx, map[string]any{"action": "clear", "module": module, "node": node, "key": k,
"why": "settings clear " + module + " on " + where, "by": setByWords()}); err != nil {
return true, fmt.Errorf("%s of %s on %s was not cleared: %w", k, module, where, err)
}
}
if len(layer) > 0 {
if err := inv.ClearSettingsBy(ctx, node, module, setByWords()); err != nil {
return true, err
}
}
fmt.Printf("%s on %s is back to what the module says\n", module, where)
return true, nil
}
return false, nil
}
// forwardSet sets a layer of a module whose config values are mesh-config's: its placement beside the assignment
// as before, each config value as the module's own on the node, and each it no longer names cleared.
func forwardSet(ctx context.Context, inv *inventory.Inventory, module, node string, values map[string]any,
replace bool) error {
where := whereOf(node)
config, placement := catalogue.SplitPlacement(values)
listed, err := listConfig(ctx, module, node)
if err != nil {
return err
}
own := listed.ownValues(module, node)
layer, _, err := inv.Layer(ctx, node, module)
if err != nil {
return err
}
before := map[string]any{}
for k, v := range layer {
before[k] = v
}
for k, v := range own {
before[k] = v
}
if err := refuseTerminalSettingsThroughAVerb(ctx, inv, before, values, module, where); err != nil {
return err
}
added, changed, removed := settingsChange(before, values)
if len(removed) > 0 && !replace {
return fmt.Errorf("%s on %s: this layer would no longer set %s. A layer is replaced whole; read it with "+
"`settings show %s%s` and include what should stay, or add --replace if the removal is meant (novox/hq "+
"ADR 0217). Nothing was changed", module, where, strings.Join(removed, ", "), module, nodeFlag(node))
}
var toSet, toClear []string
for _, k := range movedKeys(config) {
if was, had := own[k]; !had || !sameJSON(valueJSON(was), config[k]) {
toSet = append(toSet, k)
}
}
for _, k := range movedKeys(own) {
if _, kept := config[k]; !kept {
toClear = append(toClear, k)
}
}
if node == "" && len(toSet)+len(toClear) > 0 {
return fmt.Errorf("%s has no own value for the whole mesh (novox/hq ADR 0348 §3): name the node, or set mesh "+
"config for the predefined key a value falls back to with mesh-config.config. Nothing was changed", module)
}
if err := refuseTrustedThroughAVerb(ctx, inv, module, where, append(append([]string{}, toSet...), toClear...)); err != nil {
return err
}
if placement == nil {
placement = map[string]any{}
}
if _, placed := catalogue.SplitPlacement(layer); !sameLayer(placed, placement) {
if err := inv.SetSettingsBy(ctx, node, module, placement, setByWords()); err != nil {
return err
}
}
for _, k := range toSet {
if _, err := configAsk(ctx, map[string]any{"action": "set", "module": module, "node": node, "key": k,
"value": config[k], "why": "settings set " + module + " on " + where, "by": setByWords()}); err != nil {
return fmt.Errorf("%s of %s on %s was not set: %w", k, module, where, err)
}
}
for _, k := range toClear {
if _, err := configAsk(ctx, map[string]any{"action": "clear", "module": module, "node": node, "key": k,
"why": "settings set " + module + " on " + where + " --replace", "by": setByWords()}); err != nil {
return fmt.Errorf("%s of %s on %s was not cleared: %w", k, module, where, err)
}
}
fmt.Printf("%s on %s:\n", module, where)
if len(added)+len(changed)+len(removed) == 0 {
fmt.Println(" nothing changed")
}
for _, p := range added {
fmt.Printf(" + %s\n", p)
}
for _, p := range changed {
fmt.Printf(" ~ %s\n", p)
}
for _, p := range removed {
fmt.Printf(" - %s\n", p)
}
fmt.Println(" its config values are mesh-config's (issue 664): each was set there, with its history; the module " +
"takes them when its files are filled again")
return nil
}
func valueJSON(v any) json.RawMessage {
raw, _ := json.Marshal(v)
return raw
}
// refuseTrustedThroughAVerb refuses, through a verb, a trusted config value (ADR 0348 §8): mesh-config takes the
// controller's word for one as given at its terminal, and whoever may call a verb includes agents.
func refuseTrustedThroughAVerb(ctx context.Context, inv *inventory.Inventory, module, where string, keys []string) error {
verb, through := throughAVerb()
if !through || len(keys) == 0 {
return nil
}
shelf, err := inv.Catalogue(ctx)
if err != nil {
return fmt.Errorf("which config values of %s are trusted cannot be read, so nothing was changed: %w", module, err)
}
for _, k := range keys {
if shelf[module].Settings[k].Trusted {
return fmt.Errorf("%s of %s on %s is trusted (novox/hq ADR 0348 §8): it is set at the controller's terminal "+
"or on the operator's Approve, never through a verb alone (this line came through %q). Propose it with "+
"mesh-config.config, action propose. Nothing was changed", k, module, where, verb)
}
}
return nil
}
// forwardPropose proposes, through mesh-config, each config value of a layer proposed for a module whose config
// values are mesh-config's (ADR 0348 §8): mesh-config asks the operator under its own name, one ask per value. A
// proposal naming placement too is refused, so the operator is never asked half of it; placement alone is proposed
// as before. Says whether it answered.
func forwardPropose(ctx context.Context, inv *inventory.Inventory, module, node, valuesArg string, clear bool) (bool, error) {
reads, err := inv.ReadsConfig(ctx, module)
if err != nil || !reads {
return false, err
}
where := whereOf(node)
if node == "" {
return true, fmt.Errorf("%s has no own value for the whole mesh (novox/hq ADR 0348 §3): name the node, or "+
"propose mesh config for the predefined key a value falls back to with mesh-config.config. Nothing was "+
"proposed", module)
}
proposals := map[string]any{}
var clears []string
if clear {
listed, err := listConfig(ctx, module, node)
if err != nil {
return true, err
}
layer, _, err := inv.Layer(ctx, node, module)
if err != nil {
return true, err
}
if len(layer) > 0 {
return true, fmt.Errorf("%s on %s keeps placement beside its assignment (%s): propose clearing it apart "+
"from its config values. Nothing was proposed", module, where, strings.Join(movedKeys(layer), ", "))
}
clears = movedKeys(listed.ownValues(module, node))
} else {
var values map[string]any
if err := json.Unmarshal([]byte(valuesArg), &values); err != nil {
return true, fmt.Errorf("%s is not a settings object: %w", valuesArg, err)
}
config, placement := catalogue.SplitPlacement(values)
if len(placement) > 0 {
if len(config) > 0 {
return true, fmt.Errorf("%s names placement (%s) and config values: propose them apart, the config "+
"values through mesh-config. Nothing was proposed", module, strings.Join(movedKeys(placement), ", "))
}
return false, nil
}
proposals = config
}
for _, k := range movedKeys(proposals) {
answer, err := configAsk(ctx, map[string]any{"action": "propose", "module": module, "node": node, "key": k,
"value": proposals[k], "why": "settings propose " + module + " on " + where})
if err != nil {
return true, fmt.Errorf("%s of %s on %s was not proposed: %w", k, module, where, err)
}
fmt.Printf("%s of %s on %s, proposed through mesh-config: %s\n", k, module, where, answer)
}
for _, k := range clears {
answer, err := configAsk(ctx, map[string]any{"action": "propose", "module": module, "node": node, "key": k,
"clear": true, "why": "settings propose " + module + " on " + where + " --clear"})
if err != nil {
return true, fmt.Errorf("clearing %s of %s on %s was not proposed: %w", k, module, where, err)
}
fmt.Printf("clearing %s of %s on %s, proposed through mesh-config: %s\n", k, module, where, answer)
}
return true, nil
}
cmd/mesh-controller/status_summary.go
+5
View File
@@ -224,5 +224,10 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e
// nothing it caused (novox/hq ADR 0228).
go replaceGiven(context.WithoutCancel(ctx), l.open, report)
}
// And, once the move of the secrets into the vault has started, what is left of this machine's (design 57's V3,
// issue 676): a machine asleep when it started is moved when it is next heard from. Off the report's path.
if err == nil && l.open != nil && report.Superseded == "" && report.Rekey == nil && report.Node != "" {
go moveWhenHeard(context.WithoutCancel(ctx), l.open, moveHeardOn(), report.Node)
}
return news, err
}
cmd/mesh-controller/stores.go
+8
View File
@@ -3,6 +3,7 @@ package main
import (
"context"
"fmt"
"os"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
@@ -124,6 +125,13 @@ func openInventory(ctx context.Context) (*inventory.Inventory, error) {
if aliases, err := inv.Aliases(ctx); err == nil {
catalogue.UseAliases(aliases)
}
// And the list of predefined keys the mesh publishes (novox/hq ADR 0348 §7), which every manifest read again
// is judged against. A store a migration has not reached yet keeps none, and nothing names a key then.
if err := loadPredefined(ctx, inv); err != nil {
fmt.Fprintf(os.Stderr, "the list of predefined keys kept cannot be read: %v\n", err)
}
// And where config values are read once the move of config has run (issue 664): mesh-config_resolved.
inv.ReadsConfigFrom(readConfigValues)
return inv, nil
}
cmd/mesh-controller/template_test.go
+272
View File
@@ -0,0 +1,272 @@
package main
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
snapshot "github.com/novox/mesh-controller/internal/facts"
"github.com/novox/mesh-controller/internal/inventory"
)
// A file's text kept as a template file in its module's repository (novox/hq ADR 0349 §2, design 57 package T1,
// issue 654): `module check` and the merge gate read it from the tree beside the manifest, so a change to a
// template alone is judged as a change to its module.
const templatedNotifier = `{"module":"notifier","version":"1",
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},
"resources":[{"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644","template":"files/notifier.conf"}]}`
// inlineNotifier is the same module as the mesh holds it: built, its text inline.
const inlineNotifier = `{"module":"notifier","version":"1",
"resources":[{"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644","content":"volume = 3\n"}]}`
// aTemplatedModule writes a module's directory: its manifest and its files/.
func aTemplatedModule(t *testing.T, dir, manifest string, files map[string]string) string {
t.Helper()
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
path := filepath.Join(dir, "module.json")
if err := os.WriteFile(path, []byte(manifest), 0o644); err != nil {
t.Fatal(err)
}
for name, text := range files {
full := filepath.Join(dir, filepath.FromSlash(name))
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(full, []byte(text), 0o644); err != nil {
t.Fatal(err)
}
}
return path
}
func TestModuleCheckReadsTemplatesFromTheTreeBesideTheManifest(t *testing.T) {
good := aTemplatedModule(t, t.TempDir(), templatedNotifier, map[string]string{"files/notifier.conf": "volume = 3\n"})
var out bytes.Buffer
if err := moduleCheck([]string{good}, &out); err != nil || !strings.Contains(out.String(), "notifier: ok") {
t.Fatalf("a module with its text in files/ was refused: %v\n%s", err, out.String())
}
// A change to the template alone, to text the controller refuses inline, is refused the same.
bad := aTemplatedModule(t, t.TempDir(), templatedNotifier,
map[string]string{"files/notifier.conf": "volume = 3\nstore = ${dir:nowhere}\n"})
out.Reset()
if err := moduleCheck([]string{bad}, &out); err == nil || !strings.Contains(out.String(), "nowhere") {
t.Fatalf("a template whose text is refused inline passed:\n%s", out.String())
}
// A symlink under files/ that no template names: the build's packing refuses it, so the check does too.
linked := t.TempDir()
aTemplatedModule(t, linked, templatedNotifier, map[string]string{"files/notifier.conf": "volume = 3\n"})
if err := os.Symlink("/etc/hostname", filepath.Join(linked, "files", "elsewhere")); err != nil {
t.Fatal(err)
}
out.Reset()
if err := moduleCheck([]string{filepath.Join(linked, "module.json")}, &out); err == nil ||
!strings.Contains(out.String(), "files/elsewhere is a symlink") {
t.Fatalf("a symlink under files/ passed the check:\n%s", out.String())
}
// And a template the tree lacks, by name.
missing := aTemplatedModule(t, t.TempDir(), templatedNotifier, nil)
out.Reset()
if err := moduleCheck([]string{missing}, &out); err == nil || !strings.Contains(out.String(), "files/notifier.conf") {
t.Fatalf("a missing template was not refused by name:\n%s", out.String())
}
}
// templatedMesh is catalogueMesh with the notifier on the laptop, registered as built: its text inline.
func templatedMesh(t *testing.T) (snapshot.Facts, map[string]string) {
t.Helper()
open := aMesh(t)
ctx := t.Context()
manifests := map[string]string{
"lemurs": `{"module":"lemurs","version":"1","capabilities":["systemd"],
"resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`,
"notifier": inlineNotifier,
}
for name, raw := range manifests {
m, err := catalogue.ParseManifest([]byte(raw))
if err != nil {
t.Fatalf("%s: %v", name, err)
}
if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog",
Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil {
t.Fatal(err)
}
if _, err := assign(ctx, open, "laptop", name); err != nil {
t.Fatalf("assign %s: %v", name, err)
}
}
f, err := gatherFacts(ctx, open, "2.11.17")
if err != nil {
t.Fatal(err)
}
return f, manifests
}
// aTemplatedTree is the catalogue's checkout with the notifier's text in files/.
func aTemplatedTree(t *testing.T, manifests map[string]string, text string, has bool) string {
t.Helper()
tree := aTree(t, withEdit(manifests, "notifier", ""))
files := map[string]string{}
if has {
files["files/notifier.conf"] = text
}
aTemplatedModule(t, filepath.Join(tree, "modules", "notifier"), templatedNotifier, files)
return tree
}
const notifierTemplate = "modules/notifier/files/notifier.conf"
func TestTheMergeGateJudgesAChangeToATemplateAlone(t *testing.T) {
f, manifests := templatedMesh(t)
// The module moved to its template, the same text: nothing changes on any machine.
v := gateJudged(t, f, aTemplatedTree(t, manifests, "volume = 3\n", true), "modules/notifier/module.json",
notifierTemplate)
if v.Verdict == "fail" || v.Modules["notifier"] != "" {
t.Fatalf("the module moved to its template with the same text reads as a change (%v):\n%s", v.Modules, v.Report())
}
// The template alone changed: its module changed, and composes.
v = gateJudged(t, f, aTemplatedTree(t, manifests, "volume = 4\n", true), notifierTemplate)
if v.Verdict == "fail" || v.Modules["notifier"] != "changed" {
t.Fatalf("a change to the template alone is not a change to its module (%v):\n%s", v.Modules, v.Report())
}
if v.Width == nil || strings.Join(v.Width.Modules, ",") != "notifier" {
t.Fatalf("a change to the template rebuilds %+v, not its module", v.Width)
}
// The template alone changed to text the controller refuses: the pull request fails, naming the module.
v = gateJudged(t, f, aTemplatedTree(t, manifests, "store = ${dir:nowhere}\n", true), notifierTemplate)
if v.Verdict != "fail" || !strings.Contains(v.Report(), "notifier") {
t.Fatalf("a template the controller refuses passed the gate:\n%s", v.Report())
}
// A symlinked files/ fails: nothing is read from outside the tree.
tree := aTree(t, withEdit(manifests, "notifier", ""))
elsewhere := t.TempDir()
if err := os.WriteFile(filepath.Join(elsewhere, "notifier.conf"), []byte("volume = 3\n"), 0o644); err != nil {
t.Fatal(err)
}
aTemplatedModule(t, filepath.Join(tree, "modules", "notifier"), templatedNotifier, nil)
if err := os.Symlink(elsewhere, filepath.Join(tree, "modules", "notifier", "files")); err != nil {
t.Fatal(err)
}
v = gateJudged(t, f, tree, notifierTemplate)
if v.Verdict != "fail" || !strings.Contains(v.Report(), "files/ is a symlink") {
t.Fatalf("a symlinked files/ passed the gate:\n%s", v.Report())
}
// And the template removed from the tree fails, naming it.
v = gateJudged(t, f, aTemplatedTree(t, manifests, "", false), notifierTemplate)
if v.Verdict != "fail" || !strings.Contains(v.Report(), "files/notifier.conf") {
t.Fatalf("a missing template passed the gate:\n%s", v.Report())
}
}
// What the gate composes for the module in its template form is what the mesh composes for it inline.
func TestTheMergeGateResolvesATemplatedModuleToItsInlineTwin(t *testing.T) {
dir := t.TempDir()
path := aTemplatedModule(t, dir, templatedNotifier, map[string]string{"files/notifier.conf": "volume = 3\n"})
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
m, err := catalogue.ParseManifest(raw)
if err != nil {
t.Fatal(err)
}
texts, err := catalogue.ReadTemplates(m, dir)
if err != nil {
t.Fatal(err)
}
resolved, err := standInBuild(m, texts)
if err != nil {
t.Fatal(err)
}
inline, err := catalogue.ParseManifest([]byte(inlineNotifier))
if err != nil {
t.Fatal(err)
}
if builtAlike(resolved) != builtAlike(inline) {
got, _ := json.Marshal(resolved)
t.Fatalf("the templated module resolves to %s, not its inline twin", got)
}
}
// A module whose files/ is unpacked onto the machine names its own templates directory (issue 731): `module check`
// reads its templates from there and says of it exactly what it says of the same module with its text inline.
func TestModuleCheckReadsTemplatesFromTheTemplatesDirectoryTheManifestNames(t *testing.T) {
const inline = `{"module":"notifier","version":"1",
"build":{"artifacts":[{"name":"scripts","kind":"archive","from":"files"}]},
"resources":[{"id":"scripts","type":"archive","path":"/usr/local/lib/notifier","artifact":"scripts"},
{"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644","content":"volume = 3\n"}]}`
const templated = `{"module":"notifier","version":"1","templates":"templates",
"build":{"artifacts":[{"name":"scripts","kind":"archive","from":"files"},
{"name":"templates","kind":"archive","from":"templates"}]},
"resources":[{"id":"scripts","type":"archive","path":"/usr/local/lib/notifier","artifact":"scripts"},
{"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644","template":"templates/notifier.conf"}]}`
scripts := map[string]string{"files/bin/notify": "#!/bin/sh\n"}
var want bytes.Buffer
if err := moduleCheck([]string{aTemplatedModule(t, t.TempDir(), inline, scripts)}, &want); err != nil {
t.Fatalf("the inline module was refused: %v\n%s", err, want.String())
}
with := map[string]string{"files/bin/notify": "#!/bin/sh\n", "templates/notifier.conf": "volume = 3\n"}
var got bytes.Buffer
if err := moduleCheck([]string{aTemplatedModule(t, t.TempDir(), templated, with)}, &got); err != nil {
t.Fatalf("a module with its text in templates/ was refused: %v\n%s", err, got.String())
}
if got.String() != want.String() {
t.Fatalf("module check says otherwise of the templated module:\ninline: %s\ntemplated: %s", want.String(),
got.String())
}
// Its text in files/ instead, where the machine unpacks it, is not where its manifest says its templates are.
misplaced := strings.Replace(templated, `"templates/notifier.conf"`, `"files/notifier.conf"`, 1)
got.Reset()
if err := moduleCheck([]string{aTemplatedModule(t, t.TempDir(), misplaced,
map[string]string{"files/notifier.conf": "volume = 3\n"})}, &got); err == nil ||
!strings.Contains(got.String(), "files/notifier.conf") {
t.Fatalf("a template outside the templates directory the manifest names passed:\n%s", got.String())
}
}
// Issue 876: the gate judges a moved module that names its templates as registration reads it after a real build,
// handed its templates; it refused every such module as "marked moved and names no template". This calls the gate's
// own shelf (shelfWithChange), which needs no database.
func TestReplay876(t *testing.T) {
moved := strings.Replace(templatedNotifier, `"version":"1",`, `"version":"1","moved":true,`, 1)
judge := func(manifest string, files map[string]string) (map[string]catalogue.Manifest, []string) {
tree := t.TempDir()
aTemplatedModule(t, filepath.Join(tree, "modules", "notifier"), manifest, files)
out, failures, err := shelfWithChange(map[string]catalogue.Manifest{}, nil, snapshot.Facts{},
"novox/mesh-catalog", tree, map[string]string{}, nil)
if err != nil {
t.Fatal(err)
}
return out, failures
}
out, failures := judge(moved, map[string]string{"files/notifier.conf": "volume = 3\n"})
if len(failures) > 0 {
t.Fatalf("a moved module naming its templates was refused: %v", failures)
}
if h := out["notifier"].Handed; h == nil || h.Templates["conf"] != "notifier.conf" {
t.Fatalf("the gate did not hand the moved module its templates: %+v", h)
}
// A moved module naming no template is still refused.
none := strings.Replace(strings.Replace(moved, `"template":"files/notifier.conf"`,
`"content":"volume = 3\n"`, 1), `"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},`, "", 1)
if _, failures := judge(none, nil); len(failures) != 1 || !strings.Contains(failures[0], "names no template") {
t.Fatalf("a moved module naming no template was not refused: %v", failures)
}
}
cmd/mesh-controller/testdata/config-moved/mesh.json
Vendored
+3661
View File
File diff suppressed because one or more lines are too long
cmd/mesh-controller/testdata/placement-moved/anchor.json
Vendored
+46
View File
@@ -0,0 +1,46 @@
[
{
"id": "found.server",
"image": "registry.invalid:5000/library@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"networks": [
"predecessor_default"
],
"type": "container"
},
{
"id": "library.access-storage/downloads",
"mode": "read",
"path": "/storage/downloads",
"type": "access"
},
{
"id": "library.access-storage/media/series",
"mode": "read",
"path": "/storage/media/series",
"type": "access"
},
{
"id": "library.config",
"mode": "0750",
"owner": "1001:2000",
"path": "/var/lib/library/config",
"type": "directory"
},
{
"content": "{\n \"accesses\": {\n \"downloads\": \"/storage/downloads\",\n \"series\": \"/storage/media/series\"\n },\n \"log\": \"warn\",\n \"places\": {\n \"config\": {\n \"owner\": \"1001:2000\",\n \"path\": \"/var/lib/library/config\"\n }\n },\n \"workers\": 4\n}\n",
"id": "library.runtime-config",
"mode": "0640",
"path": "/var/lib/library/config/runtime.json",
"type": "file"
},
{
"id": "library.server",
"image": "registry.invalid:5000/library@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"type": "container",
"volumes": [
"/var/lib/library/config:/config",
"/storage/media/series:/series",
"/storage/downloads:/downloads:ro"
]
}
]
cmd/mesh-controller/testdata/placement-moved/laptop.json
Vendored
+37
View File
@@ -0,0 +1,37 @@
[
{
"id": "library.access-home/media/series",
"mode": "read",
"path": "/home/media/series",
"type": "access"
},
{
"id": "library.access-storage/downloads",
"mode": "read",
"path": "/storage/downloads",
"type": "access"
},
{
"id": "library.config",
"mode": "0750",
"path": "/var/lib/library/config",
"type": "directory"
},
{
"content": "{\n \"accesses\": {\n \"downloads\": \"/storage/downloads\",\n \"series\": \"/home/media/series\"\n },\n \"log\": \"warn\"\n}\n",
"id": "library.runtime-config",
"mode": "0640",
"path": "/var/lib/library/config/runtime.json",
"type": "file"
},
{
"id": "library.server",
"image": "registry.invalid:5000/library@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"type": "container",
"volumes": [
"/var/lib/library/config:/config",
"/home/media/series:/series",
"/storage/downloads:/downloads:ro"
]
}
]
cmd/mesh-controller/testdata/placement-words-moved/anchor.json
Vendored
+2219
View File
File diff suppressed because one or more lines are too long
cmd/mesh-controller/testdata/placement-words-moved/laptop.json
Vendored
+279
View File
@@ -0,0 +1,279 @@
{
"resources": [
{
"id": "distribution.registry-data",
"mode": "0700",
"path": "/var/lib/distribution/registry-data",
"type": "directory"
},
{
"id": "distribution.state",
"mode": "0700",
"path": "/var/lib/distribution/state",
"type": "directory"
},
{
"id": "distribution.store",
"image": "registry.invalid:5000/library@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"ports": [
"5100:5000"
],
"type": "container",
"volumes": [
"/var/lib/distribution/state:/state",
"/var/lib/distribution/registry-data:/registry-data"
]
},
{
"id": "gitea.data",
"mode": "0700",
"path": "/var/lib/gitea/data",
"type": "directory"
},
{
"id": "gitea.grants",
"mode": "0700",
"path": "/var/lib/gitea/grants",
"type": "directory"
},
{
"id": "gitea.mesh-state",
"mode": "0700",
"path": "/var/lib/gitea/mesh-state",
"type": "directory"
},
{
"content": "{\n \"expose\": {\n \"22\": \"anywhere\"\n }\n}\n",
"id": "gitea.runtime-config",
"mode": "0640",
"path": "/var/lib/gitea/runtime-config.json",
"type": "file"
},
{
"id": "gitea.runtime-state",
"mode": "0700",
"path": "/var/lib/gitea/runtime-state",
"type": "directory"
},
{
"id": "gitea.server",
"image": "registry.invalid:5000/library@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"ports": [
"20000:3000",
"222:22"
],
"type": "container",
"volumes": [
"/var/lib/gitea/mesh-state:/mesh-state",
"/var/lib/gitea/runtime-state:/runtime-state",
"/var/lib/gitea/state:/state",
"/var/lib/gitea/grants:/grants",
"/var/lib/gitea/data:/data"
]
},
{
"id": "gitea.state",
"mode": "0700",
"path": "/var/lib/gitea/state",
"type": "directory"
},
{
"id": "mssql.data",
"mode": "0700",
"path": "/var/lib/mssql/data",
"type": "directory"
},
{
"id": "mssql.dumps",
"mode": "0700",
"path": "/var/lib/mssql/dumps",
"type": "directory"
},
{
"id": "mssql.grants",
"mode": "0700",
"path": "/var/lib/mssql/grants",
"type": "directory"
},
{
"id": "mssql.net",
"name": "mssql",
"type": "network"
},
{
"id": "mssql.server",
"image": "registry.invalid:5000/library@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"network": "mssql",
"ports": [
"4848:1433"
],
"type": "container",
"volumes": [
"/var/lib/mssql/state:/state",
"/var/lib/mssql/grants:/grants",
"/var/lib/mssql/data:/data",
"/var/lib/mssql/dumps:/dumps"
]
},
{
"id": "mssql.state",
"mode": "0700",
"path": "/var/lib/mssql/state",
"type": "directory"
},
{
"id": "postgres.dumps",
"mode": "0700",
"path": "/var/lib/postgres/dumps",
"type": "directory"
},
{
"id": "postgres.grants",
"mode": "0700",
"path": "/var/lib/postgres/grants",
"type": "directory"
},
{
"id": "postgres.server",
"image": "registry.invalid:5000/library@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"ports": [
"6852:5432"
],
"type": "container",
"volumes": [
"/var/lib/postgres/state:/state",
"/var/lib/postgres/grants:/grants",
"/var/lib/postgres/store-data:/store-data",
"/var/lib/postgres/dumps:/dumps"
]
},
{
"id": "postgres.state",
"mode": "0700",
"path": "/var/lib/postgres/state",
"type": "directory"
},
{
"id": "postgres.store-data",
"mode": "0700",
"path": "/var/lib/postgres/store-data",
"type": "directory"
},
{
"content": "{\n \"contributions\": 1,\n \"generated\": \"by the mesh — do not edit; replaced whenever a module contributing to route arrives or leaves\",\n \"given\": [\n {\n \"from\": \"gitea\",\n \"node\": \"laptop\",\n \"at\": \"laptop.internal\",\n \"as\": \"mesh_laptop_gitea\",\n \"secret\": \"/laptop.gitea.secret\",\n \"values\": {}\n },\n {\n \"from\": \"umami\",\n \"node\": \"laptop\",\n \"at\": \"laptop.internal\",\n \"as\": \"mesh_laptop_umami\",\n \"secret\": \"/laptop.umami.secret\",\n \"values\": {\n \"endpoint\": \"web\",\n \"internal-name\": \"umami.laptop.internal\",\n \"label\": \"umami\",\n \"name\": \"umami.laptop.example.test\",\n \"port\": 1212\n }\n },\n {\n \"from\": \"gitea\",\n \"values\": {\n \"deny\": true,\n \"internal-name\": \"git.laptop.internal\",\n \"label\": \"git\",\n \"name\": \"git.laptop.example.test\",\n \"path\": \"/api/internal\",\n \"priority\": 100000\n }\n },\n {\n \"from\": \"gitea\",\n \"values\": {\n \"endpoint\": \"web\",\n \"internal-name\": \"git.laptop.internal\",\n \"label\": \"git\",\n \"name\": \"git.laptop.example.test\",\n \"port\": 20000\n }\n }\n ],\n \"requirement\": \"route\"\n}\n",
"id": "proxy.received-route",
"mode": "0644",
"path": "/var/lib/proxy/routes.json",
"type": "file"
},
{
"id": "umami.grants",
"mode": "0700",
"path": "/var/lib/umami/grants",
"type": "directory"
},
{
"id": "umami.net",
"name": "umami",
"type": "network"
},
{
"id": "umami.server",
"image": "registry.invalid:5000/library@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"network": "umami",
"ports": [
"1212:3000"
],
"type": "container",
"volumes": [
"/var/lib/umami/state:/state",
"/var/lib/umami/grants:/grants"
]
},
{
"id": "umami.state",
"mode": "0700",
"path": "/var/lib/umami/state",
"type": "directory"
}
],
"rules": [
{
"Port": 222,
"Protocol": "tcp",
"From": "anywhere",
"Because": [
"gitea"
],
"Why": [
""
]
},
{
"Port": 1212,
"Protocol": "tcp",
"From": "mesh",
"Because": [
"umami"
],
"Why": [
""
]
},
{
"Port": 4848,
"Protocol": "tcp",
"From": "mesh",
"Because": [
"mssql"
],
"Why": [
""
]
},
{
"Port": 5100,
"Protocol": "tcp",
"From": "mesh",
"Because": [
"distribution"
],
"Why": [
""
]
},
{
"Port": 6852,
"Protocol": "tcp",
"From": "mesh",
"Because": [
"postgres"
],
"Why": [
""
]
},
{
"Port": 20000,
"Protocol": "tcp",
"From": "mesh",
"Because": [
"gitea"
],
"Why": [
""
]
},
{
"Port": 51820,
"Protocol": "udp",
"From": "anywhere",
"Because": [
"mesh-wireguard"
],
"Why": [
"the private network — a node at another site has no other way in"
]
}
]
}
cmd/mesh-controller/testdata/placement-words-moved/mesh.json
Vendored
+3288
View File
File diff suppressed because it is too large Load Diff
cmd/mesh-controller/vault_handed_test.go
+83
View File
@@ -0,0 +1,83 @@
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A machine's send makes and seals none of the secrets of a module handed to the provisioner there (novox/hq ADR
// 0349 §7, §11, design 57 V4, issue 647): its client asks the vault for them. A module not moved, on the same
// machine, has its secret made and sealed as before.
func aClientModule() catalogue.Manifest {
return catalogue.Manifest{Module: "mesh-provisioner-client", Version: "1",
Claims: []catalogue.Claim{{Name: broker.NodeProvisionerSeat, Scope: catalogue.ScopeNode}},
OwnSecrets: catalogue.OwnSecrets{broker.ProvisionerSecret: {Path: "/var/lib/mesh-provisioner-client/broker"}},
Resources: []map[string]any{{"id": "client", "type": "process", "name": "mesh-provisioner-client",
"run": []any{"./mesh-provisioner-client"}, "user": "mesh-provisioner-client"}}}
}
func aBot(module string, moved bool) catalogue.Manifest {
m := catalogue.Manifest{Module: module, Version: "1",
OwnSecrets: catalogue.OwnSecrets{"token": {Path: "/var/lib/" + module + "/token"}},
Resources: []map[string]any{{"id": "conf", "type": "file", "path": "/etc/" + module + "/bot.conf",
"content": "token = ${secret:token}\n"}}}
if moved {
m.Moved = true
m.Handed = &catalogue.Handed{Source: "https://templates.example/" + module + ".tar.gz",
Digest: "sha256:" + strings.Repeat("e", 64),
Templates: map[string]string{"conf": "bot.conf"}}
}
return m
}
func TestTheSendSealsNoSecretOfAModuleHandedThere(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
client := aClientModule()
for _, m := range []catalogue.Manifest{client, aBot("moved-bot", true), aBot("kept-bot", false)} {
register(t, open, m)
if _, err := assign(ctx, open, "laptop", m.Module); err != nil {
t.Fatal(err)
}
}
if err := issueProvisionerClient(ctx, inv, client, "laptop", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222"); err != nil {
t.Fatal(err)
}
if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: link.HandedContract,
SaidAt: time.Now()}); err != nil {
t.Fatal(err)
}
sent := sentTo(t, open, "laptop")
var moved, kept map[string]any
for _, r := range sent {
switch r["id"] {
case "moved-bot.needs-token":
moved = r
case "kept-bot.needs-token":
kept = r
}
}
if moved == nil || moved["sealed"] != nil || moved["owner-at-runtime"] != "owned" {
t.Fatalf("the handed module's secret file is sent as %v: placed, with no value", moved)
}
if kept == nil || kept["sealed"] == nil {
t.Fatalf("the module not moved has its secret sent as %v: sealed, as before", kept)
}
if _, held, err := inv.ModuleSecretIfIssued(ctx, "laptop", "moved-bot", "token"); err != nil || held {
t.Fatalf("the send made the handed module's secret in the controller's store (held %v, %v)", held, err)
}
for _, r := range sent {
if r["id"] == "moved-bot.conf" && (r["content"] != nil || r["sealed"] != nil) {
t.Fatalf("the handed module's templated file carries text or a value: %v", r)
}
}
}
cmd/mesh-controller/vault_secrets.go
+403
View File
@@ -0,0 +1,403 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/secrets"
)
// The vault's verbs, reached from the controller's (novox/hq ADR 0349 clauses 8 to 10; design 57's V2, issue 675).
//
// **Each module is filled the old way or the new way, by its form** (ADR 0349 clause 17, the release lead's
// decision of 2026-10-14 on issue 675). `secret accept`, `secret ask` (and `accept --at-desk`), `secret issued`
// and `secret rotate` go to the vault only for a module whose secrets are filled the new way — its machine asks
// the vault for them — and keep today's path, the controller's store and the declaration it seals, for every other
// module. Forwarding a module still filled the old way would report success and change nothing on its machine:
// the vault hands it nothing, and its machine never asks.
//
// **What the controller carries, it cannot open; and it carries only what is given at its own terminal.** The vault
// takes a value given at this terminal in two steps of one verb: the first names the secret and answers a key the
// vault made for that one call; the second carries the value sealed here to it (clause 8). A value typed at a desk
// or issued by a provider's software the controller does not carry at all (issue 723): it asks the vault once, and
// the vault asks the desk's prompt or the provider's tool itself, in its own name on its own bus account (ADR 0259
// §8's runs-as), so the prompt and the tool know the vault asks and the value opens there alone (clauses 9 and 10).
// vaultSwitch says whether a module's secrets are filled the new way, or why that could not be read.
type vaultSwitch func(module string) (bool, error)
// secretsOfTheVault says whether a module's secrets are filled the new way: the switch the controller's secret
// verbs turn on. **A module marked moved** (design 57 V4, issue 647): where it is handed to the provisioner, its
// machine's client fills its `${secret:}` and its secret files from the vault, and the controller makes and seals
// none of them (catalogue handed.go), so a value given, asked for, issued or rotated goes to the vault. Every
// other module keeps today's path. A catalogue that cannot be read is an error, never today's path by default: a
// value kept where the module's machine no longer looks would be reported kept and change nothing.
func secretsOfTheVault(module string) (bool, error) {
ctx, cancel := context.WithTimeout(context.Background(), vaultWithin)
defer cancel()
open, err := openStores(ctx)
if err != nil {
return false, fmt.Errorf("whether %s's secrets are the vault's cannot be read: %w", module, err)
}
defer open.Close()
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
return false, fmt.Errorf("whether %s's secrets are the vault's cannot be read: %w", module, err)
}
return movedOn(shelf, module), nil
}
// movedOn is whether the catalogue marks the module moved: its secrets are the vault's.
func movedOn(shelf map[string]catalogue.Manifest, module string) bool { return shelf[module].Moved }
// filledTheNewWay is the switch as the four commands read it — accept, secret-ask (and accept --at-desk), issued and
// rotate, each at its own call site: secretsOfTheVault. A test replaces it to prove that each call site sends a
// module filled the old way down today's path and one filled the new way to the vault (the review of issue 675).
var filledTheNewWay vaultSwitch = secretsOfTheVault
// switchOf is a switch that cannot fail, as a test writes one.
func switchOf(f func(module string) bool) vaultSwitch {
return func(module string) (bool, error) { return f(module), nil }
}
// offeringModuleOn is the module on the provider's machine that offers the provision: the one whose secrets hold
// the provision's one credential, and so the one that decides the path of `accept --provider`. "" when no module,
// or more than one, on that machine offers it: today's path then, which says what is wrong.
func offeringModuleOn(ctx context.Context, provider, provision string) (string, error) {
open, err := openStores(ctx)
if err != nil {
return "", err
}
defer open.Close()
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
return "", err
}
assigned, err := open.inventory.Assigned(ctx, provider)
if err != nil {
return "", err
}
return offeredOn(shelf, assigned, provision), nil
}
// offeredOn is the one module of those assigned that offers the provision, or "" for none or several.
func offeredOn(shelf map[string]catalogue.Manifest, assigned []string, provision string) string {
found := ""
for _, module := range assigned {
if slices.Contains(shelf[module].Offers(), provision) {
if found != "" {
return ""
}
found = module
}
}
return found
}
// bySwitch runs the vault's path for a module whose secrets are filled the new way, and today's for every other.
func bySwitch(module string, ofTheVault vaultSwitch, today, vault func() error) error {
if ofTheVault == nil {
return today()
}
ours, err := ofTheVault(module)
if err != nil {
return fmt.Errorf("nothing was done: %w", err)
}
if ours {
return vault()
}
return today()
}
// vaultAsker asks one of the vault's verbs in the controller's name and answers its result, or its refusal.
type vaultAsker func(verb string, args map[string]any) (json.RawMessage, error)
// vaultWithin is how long the vault is given to answer one step.
const vaultWithin = 20 * time.Second
// vaultAskWithin is how long the vault is given to answer `secret-ask`, which waits for the operator at the desk:
// its prompt's time (mesh-vault's DeskPromptFor, twenty seconds) and the bus's, under the thirty seconds the caller
// of this verb waits.
const vaultAskWithin = 28 * time.Second
// theVault asks the vault's seat on this controller's bus, on the subject that names the controller.
func theVault(ctx context.Context) vaultAsker {
return func(verb string, args map[string]any) (json.RawMessage, error) {
var result json.RawMessage
err := onTheBus(func(conn *nats.Conn) error {
within := vaultWithin
if verb == "secret-ask" {
within = vaultAskWithin
}
answer, err := link.AskMeshSeatToolAs(ctx, conn, broker.ControllerName, "mesh-vault", verb, args, within)
if err != nil {
return err
}
if answer.Error != "" {
return errors.New(answer.Error)
}
result = answer.Result
return nil
})
return result, err
}
}
// vaultOpened is a first step's answer: the call, the key made for it, the secret's name in the vault, and who
// asked as the vault recorded it. Never a value.
type vaultOpened struct {
Call string `json:"call"`
SealTo string `json:"seal_to"`
Secret string `json:"secret"`
AskedBy string `json:"asked_by"`
}
// vaultChanged is what the vault kept: never the value.
type vaultChanged struct {
Name string `json:"name"`
Generation int `json:"generation"`
By string `json:"by"`
Unchanged bool `json:"unchanged"`
// Each is, for a provider whose consumers keep a login each, each consumer's own login remade (issue 767).
Each []vaultChanged `json:"each,omitempty"`
}
func openVaultCall(ask vaultAsker, verb string, args map[string]any) (vaultOpened, error) {
raw, err := ask(verb, args)
if err != nil {
return vaultOpened{}, fmt.Errorf("the vault did not open the call: %w", err)
}
var o vaultOpened
if err := json.Unmarshal(raw, &o); err != nil || o.Call == "" || o.SealTo == "" {
return vaultOpened{}, errors.New("the vault answered no call and no key to seal to")
}
return o, nil
}
func endVaultCall(ask vaultAsker, verb string, args map[string]any) (vaultChanged, error) {
raw, err := ask(verb, args)
if err != nil {
return vaultChanged{}, err
}
var c vaultChanged
if err := json.Unmarshal(raw, &c); err != nil {
return vaultChanged{}, errors.New("the vault answered something that is not what it kept")
}
return c, nil
}
// ownSecretArgs names a module's own secret on one machine, as the vault's verbs take it.
func ownSecretArgs(node, module, name string) map[string]any {
return map[string]any{"node": node, "module": module, "name": name}
}
// provisionArgs names the one credential of a provider assignment and provision.
func provisionArgs(provider, provision string) map[string]any {
return map[string]any{"provider": provider, "provision": provision}
}
// acceptIntoVault is `secret accept` for a module filled the new way: the value read at this terminal is sealed
// here to the key the vault made for this call, and only the sealed box crosses the bus (ADR 0349 clause 8).
func acceptIntoVault(ask vaultAsker, target map[string]any, value, by string) (string, error) {
first := map[string]any{"by": by}
for k, v := range target {
first[k] = v
}
opened, err := openVaultCall(ask, "accept", first)
if err != nil {
return "", fmt.Errorf("nothing was read or sealed: %w", err)
}
sealed, err := secrets.Seal(opened.SealTo, []byte(value))
value = ""
if err != nil {
return "", fmt.Errorf("the vault's key for this call is not a sealing key; nothing was sent: %w", err)
}
changed, err := endVaultCall(ask, "accept", map[string]any{"call": opened.Call, "sealed": sealed})
if err != nil {
return "", fmt.Errorf("the vault did not take it: %w", err)
}
if changed.Unchanged {
return fmt.Sprintf("the vault already holds this value as %s (generation %d); nothing changed", opened.Secret,
changed.Generation), nil
}
return fmt.Sprintf("the vault now holds %s (generation %d), accepted: sealed at this terminal to the vault's key "+
"for this call, opened in the vault alone and never remade by it; its machine takes it from the vault",
opened.Secret, changed.Generation), nil
}
// rotateInVault is `secret rotate` for a module filled the new way: the vault makes the value anew and says it
// changed; the module's machine takes it from the vault, with no declaration sent.
//
// **The vault rotates only with a why** (the review of issue 675): a rotation by hand is an act worth its reason,
// and the vault refuses one without it, so it is refused here before the vault is asked.
func rotateInVault(ask vaultAsker, target map[string]any, by, why string) (string, error) {
if strings.TrimSpace(why) == "" {
return "", errors.New("not rotated: the vault rotates a secret only with a reason, `--why \"…\"`; nothing was asked")
}
args := map[string]any{"by": by, "why": why}
for k, v := range target {
args[k] = v
}
changed, err := endVaultCall(ask, "rotate", args)
if err != nil {
return "", fmt.Errorf("not rotated: %w", err)
}
return fmt.Sprintf("rotated %s in the vault (generation %d), asked by %s; the value is sealed and not shown, and "+
"its machine takes it from the vault", changed.Name, changed.Generation, changed.By), nil
}
// vaultAsked is how the vault's ask at a desk ended: what it kept, or why nothing was given. Never the value.
type vaultAsked struct {
Secret string `json:"secret"`
Generation int `json:"generation"`
AskedBy string `json:"asked_by"`
Outcome string `json:"outcome"`
Unchanged bool `json:"unchanged"`
}
// giveToVault is the desk's path for a module filled the new way (ADR 0349 clause 9, ADR 0277 §22–§27 in the
// vault; issue 723): one ask of the vault, which records the ask and holds its bounds, asks the desk's prompt itself
// in its own name, the prompt naming the asker as the vault recorded it, and opens what is typed. The controller
// never reaches the desk, and nothing typed passes it, sealed or not.
func (d deskGive) giveToVault(node, module, name, desk string) (string, error) {
raw, err := d.vault("secret-ask", map[string]any{"node": node, "module": module, "name": name, "desk": desk,
"asked_by": askedByName(d.askedBy)})
if err != nil {
return "", fmt.Errorf("the vault did not ask at the desk on %s: %w", desk, err)
}
var asked vaultAsked
if err := json.Unmarshal(raw, &asked); err != nil || asked.Outcome == "" {
return "", fmt.Errorf("the vault answered something that is not how its ask at the desk on %s ended", desk)
}
switch asked.Outcome {
case "given":
case "timed-out":
return "", fmt.Errorf("%w: the prompt on %s was not answered in time (asked by the vault)", errNothingGiven, desk)
case "dismissed", "empty":
return "", fmt.Errorf("%w: the prompt on %s was %s (asked by the vault)", errNothingGiven, desk, asked.Outcome)
default:
return "", fmt.Errorf("the vault's ask at the desk on %s ended %s", desk, asked.Outcome)
}
act := link.HandAct{Verb: "secret accept", Args: []string{node, module, name, "--at-desk", desk},
Why: fmt.Sprintf("the operator gave %s for %s on %s at the desk on %s, asked by %s; kept in the vault", name,
module, node, desk, asked.AskedBy),
Cause: "given-at-the-desk"}
recorded := ""
if err := d.record(act); err != nil {
recorded = fmt.Sprintf("\n this act could NOT be recorded in the hand-act log, and is done anyway: %v", err)
}
if d.announce != nil {
if err := d.announce(node, module, name, "at the desk on "+desk); err != nil {
recorded += fmt.Sprintf("\n this change could NOT be announced on the operator's channels: %v", err)
}
}
return fmt.Sprintf("%s on %s now holds %q, given at the desk on %s and kept in the vault as %s (generation %d); "+
"the vault asked the desk itself and opened it alone, and %s takes it from there", module, node, name, desk,
asked.Secret, asked.Generation, node) + recorded, nil
}
// vaultIssued is what the vault kept of a value a provider's software issued. Never the value.
type vaultIssued struct {
Secret string `json:"secret"`
Generation int `json:"generation"`
Unchanged bool `json:"unchanged"`
// Holders are what the vault kept for each holder when the tool answered a value per holder, and NotAnswered the
// holders named that it gave nothing for (issue 749).
Holders []vaultIssuedHolder `json:"holders,omitempty"`
NotAnswered []issuedHolder `json:"not_answered,omitempty"`
}
// vaultIssuedHolder is what the vault kept for one holder. Never the value.
type vaultIssuedHolder struct {
Node string `json:"node"`
Module string `json:"module"`
Secret string `json:"secret"`
Generation int `json:"generation"`
Unchanged bool `json:"unchanged,omitempty"`
}
// takeIntoVault is `secret issued` for a provider filled the new way (ADR 0349 clause 10; issue 723): one ask of
// the vault, which asks the provider's tool itself, in its own name, with a key made for the call, and opens what
// the tool sealed. The controller never reaches the tool, and nothing is sealed, kept or sent here: each holder
// takes it from the vault.
func (t issuedTaker) takeIntoVault(provision string, p issuedProvider) (string, error) {
args := map[string]any{"provision": provision, "provider": p.Node, "module": p.Module, "tool": p.Tool}
if len(p.Holders) > 0 {
args["holders"] = p.Holders
}
raw, err := t.vault("issued", args)
if err != nil {
return "", fmt.Errorf("the vault did not take it from its tool %s: %w", p.Tool, err)
}
var changed vaultIssued
if err := json.Unmarshal(raw, &changed); err != nil || changed.Secret == "" {
return "", fmt.Errorf("the vault answered something that is not what it kept from its tool %s", p.Tool)
}
if len(changed.Holders) > 0 {
return t.takenPerHolder(provision, p, changed), nil
}
if changed.Unchanged {
return fmt.Sprintf("%s from %s on %s: the vault already holds the value its software issued (generation %d); "+
"nothing was written", provision, p.Module, p.Node, changed.Generation), nil
}
recorded := ""
if err := t.record(link.HandAct{Verb: handActIssued, Args: []string{provision},
Why: fmt.Sprintf("%s on %s gave the credential its software issued for %s to the vault, which asked it; kept "+
"in the vault as %s", p.Module, p.Node, provision, changed.Secret),
Cause: "issued-by-software"}); err != nil {
recorded = fmt.Sprintf("\n this was NOT recorded in the hand-act log, and is done anyway: %v", err)
}
return fmt.Sprintf("%s from %s on %s: the vault asked its tool itself, and the value its software issued is kept "+
"in the vault as %s (generation %d); each holder takes it from there%s", provision, p.Module, p.Node,
changed.Secret, changed.Generation, recorded), nil
}
// takenPerHolder says what the vault kept for each holder when the provider's tool answered a value per holder (issue
// 749), records the act when any holder's value changed, and names each holder the tool gave nothing for.
func (t issuedTaker) takenPerHolder(provision string, p issuedProvider, changed vaultIssued) string {
var kept, moved []string
for _, h := range changed.Holders {
holder := issuedHolder{Node: h.Node, Module: h.Module}
kept = append(kept, fmt.Sprintf("%s as %s (generation %d)", holder, h.Secret, h.Generation))
if !h.Unchanged {
moved = append(moved, holder.String())
}
}
missing := ""
if len(changed.NotAnswered) > 0 {
names := make([]string, 0, len(changed.NotAnswered))
for _, h := range changed.NotAnswered {
names = append(names, h.String())
}
missing = fmt.Sprintf("\n its tool gave nothing for %s, which holds no value from it", strings.Join(names, ", "))
}
if len(moved) == 0 {
return fmt.Sprintf("%s from %s on %s: the vault already holds the value its software issued for each holder (%s); "+
"nothing was written%s", provision, p.Module, p.Node, strings.Join(kept, ", "), missing)
}
recorded := ""
if err := t.record(link.HandAct{Verb: handActIssued, Args: []string{provision},
Why: fmt.Sprintf("%s on %s gave the vault, which asked it, a credential its software issued for each holder of %s; "+
"new for %s", p.Module, p.Node, provision, strings.Join(moved, ", ")),
Cause: "issued-by-software"}); err != nil {
recorded = fmt.Sprintf("\n this was NOT recorded in the hand-act log, and is done anyway: %v", err)
}
return fmt.Sprintf("%s from %s on %s: the vault asked its tool itself, and keeps the value its software issued for "+
"each holder, handed to that holder alone: %s%s%s", provision, p.Module, p.Node, strings.Join(kept, ", "), missing,
recorded)
}
// actingAs is who acts in this process, as the vault records it beside the controller that carries the act.
func actingAs() string {
return strings.TrimSpace(askedByName(link.Caller()))
}
cmd/mesh-controller/vault_secrets_test.go
+530
View File
@@ -0,0 +1,530 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"log"
"strings"
"sync"
"testing"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/secrets"
"github.com/novox/mesh-controller/internal/testbus"
)
// The controller's secret verbs reach the vault only for a module filled the new way (design 57's V2, issue 675,
// the release lead's decision on it; novox/hq ADR 0349 clauses 8 to 10 and 17).
// standInVault answers the vault's two-step verbs as mesh-vault does: a key made for each call, the value opened
// with it here and nowhere else. It keeps what it took and every argument it was sent, as the bus carried them.
type standInVault struct {
t *testing.T
mu sync.Mutex
keys map[string]string // call → private key
taken map[string]string // secret → value
sent []string // every argument, as JSON
refuse error // refuses every first step when set
outcome []string
again bool // the value is held already
// perHolder answers `issued` with a value per holder named, the last one not answered (issue 749).
perHolder bool
// asked is how the vault's own ask at a desk ends, as mesh-vault answers `secret-ask` (issue 723): the vault
// asks the desk itself, so the controller sends one step and is told how it ended.
asked string
}
func newStandInVault(t *testing.T) *standInVault {
return &standInVault{t: t, keys: map[string]string{}, taken: map[string]string{}}
}
func (v *standInVault) ask(verb string, args map[string]any) (json.RawMessage, error) {
v.mu.Lock()
defer v.mu.Unlock()
raw, _ := json.Marshal(map[string]any{"verb": verb, "args": args})
v.sent = append(v.sent, string(raw))
if verb == "rotate" {
return json.Marshal(vaultChanged{Name: "own/g14/dunst/token", Generation: 2, By: fmt.Sprint(args["by"])})
}
// The vault asks the desk's prompt and the provider's tool itself, in its own name (issue 723): one step from
// the controller, which carries no key and no sealed value.
switch verb {
case "secret-ask":
if v.refuse != nil {
return nil, v.refuse
}
outcome := v.asked
if outcome == "" {
outcome = "given"
}
return json.Marshal(vaultAsked{Secret: fmt.Sprintf("own/%v/%v/%v", args["node"], args["module"], args["name"]),
Generation: 1, AskedBy: "as-the-vault-recorded", Outcome: outcome})
case "issued":
if v.refuse != nil {
return nil, v.refuse
}
// A tool answering a value per holder (issue 749): the vault keeps one for each holder named, and says which
// it was not given.
if holders, ok := args["holders"].([]issuedHolder); ok && v.perHolder {
out := vaultIssued{Secret: fmt.Sprintf("provision/%v/%v", args["provider"], args["provision"]), Unchanged: v.again}
for i, h := range holders {
if i == len(holders)-1 && len(holders) > 1 {
out.NotAnswered = append(out.NotAnswered, h)
continue
}
out.Holders = append(out.Holders, vaultIssuedHolder{Node: h.Node, Module: h.Module,
Secret: fmt.Sprintf("%s/%s/%s", out.Secret, h.Node, h.Module), Generation: 1, Unchanged: v.again})
}
return json.Marshal(out)
}
return json.Marshal(vaultIssued{Secret: fmt.Sprintf("provision/%v/%v", args["provider"], args["provision"]),
Generation: 1, Unchanged: v.again})
}
call, _ := args["call"].(string)
if call == "" {
if v.refuse != nil {
return nil, v.refuse
}
public, private, err := secrets.Keypair()
if err != nil {
v.t.Fatal(err)
}
call = fmt.Sprintf("call-%d", len(v.keys)+1)
v.keys[call] = private
secret := fmt.Sprintf("%s/%v", verb, args["name"])
return json.Marshal(vaultOpened{Call: call, SealTo: public, Secret: secret, AskedBy: "as-the-vault-recorded"})
}
private, ok := v.keys[call]
if !ok {
return nil, errors.New("no such call")
}
delete(v.keys, call)
if o, _ := args["outcome"].(string); o != "" {
v.outcome = append(v.outcome, o)
return json.Marshal(vaultChanged{Unchanged: true})
}
value, err := secrets.Open(private, fmt.Sprint(args["sealed"]))
if err != nil {
return nil, errors.New("not sealed to this call")
}
// As mesh-vault takes a typed value: its line ending off (intake.go AskEnd).
v.taken[verb] = strings.TrimRight(string(value), "\r\n")
return json.Marshal(vaultChanged{Name: verb, Generation: 1, Unchanged: v.again})
}
func (v *standInVault) said() string {
v.mu.Lock()
defer v.mu.Unlock()
return strings.Join(v.sent, "\n")
}
// A module's secrets are the vault's once the catalogue marks it moved (design 57 V4, issue 647): its machine's
// client asks the vault for them. Every module not marked, and one the catalogue does not hold, keeps today's path.
func TestAModulesSecretsAreTheVaultsOnceItIsMarkedMoved(t *testing.T) {
open := aMesh(t)
register(t, open, aBot("moved-bot", true))
register(t, open, aBot("kept-bot", false))
for module, want := range map[string]bool{"moved-bot": true, "kept-bot": false, "nowhere": false} {
got, err := secretsOfTheVault(module)
if err != nil || got != want {
t.Errorf("%s: the vault's %v (%v), want %v", module, got, err, want)
}
}
}
// A switch that cannot be read does nothing, and says why: never today's path by default.
func TestASwitchThatCannotBeReadDoesNothing(t *testing.T) {
var ran []string
err := bySwitch("dunst", func(string) (bool, error) { return false, errors.New("the store is gone") },
func() error { ran = append(ran, "today"); return nil },
func() error { ran = append(ran, "vault"); return nil })
if err == nil || len(ran) > 0 || !strings.Contains(err.Error(), "the store is gone") {
t.Errorf("an unreadable switch ran %v (%v)", ran, err)
}
}
// The switch: today's path for a module filled the old way, the vault's for one filled the new way, and never
// both.
func TestTheSwitchRunsOnePathByTheModulesForm(t *testing.T) {
moved := switchOf(func(m string) bool { return m == "dunst" })
for module, want := range map[string]string{"telegram": "today", "dunst": "vault"} {
var ran []string
err := bySwitch(module, moved,
func() error { ran = append(ran, "today"); return nil },
func() error { ran = append(ran, "vault"); return nil })
if err != nil || strings.Join(ran, ",") != want {
t.Errorf("%s ran %v (%v), not %s", module, ran, err, want)
}
}
var ran []string
_ = bySwitch("dunst", nil, func() error { ran = append(ran, "today"); return nil },
func() error { ran = append(ran, "vault"); return nil })
if strings.Join(ran, ",") != "today" {
t.Errorf("no switch ran %v", ran)
}
}
// `secret accept` for a module filled the new way: the value read at the terminal is sealed there to the key the
// vault made for this one call; no clear value crosses the bus or reaches the controller's log. Over a real bus,
// with a tap on every subject.
func TestAnAcceptedValueCrossesTheBusOnlySealedToTheVault(t *testing.T) {
conn, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(conn.Close)
before := handActConn
handActConn = conn
t.Cleanup(func() { handActConn = before })
var tapped bytes.Buffer
var tapMu sync.Mutex
tap, err := conn.Subscribe(">", func(m *nats.Msg) {
tapMu.Lock()
defer tapMu.Unlock()
tapped.WriteString(m.Subject + " " + string(m.Data) + "\n")
})
if err != nil {
t.Fatal(err)
}
defer tap.Unsubscribe()
vault := newStandInVault(t)
var subjects []string
holder, err := conn.Subscribe(broker.CalledPattern(link.SeatToolSubject("mesh-vault", "accept")), func(m *nats.Msg) {
subjects = append(subjects, m.Subject)
var args map[string]any
_ = json.Unmarshal(m.Data, &args)
result, err := vault.ask("accept", args)
answer := link.Answer{Result: result}
if err != nil {
answer = link.Answer{Error: err.Error()}
}
raw, _ := json.Marshal(answer)
_ = m.Respond(raw)
})
if err != nil {
t.Fatal(err)
}
defer holder.Unsubscribe()
var logged bytes.Buffer
was := log.Writer()
log.SetOutput(&logged)
t.Cleanup(func() { log.SetOutput(was) })
const value = "pw-of-last-year-that-must-not-show"
words, err := acceptIntoVault(theVault(context.Background()), provisionArgs("novox", "postgres"), value, "jochen")
if err != nil {
t.Fatal(err)
}
if vault.taken["accept"] != value {
t.Fatalf("the vault opened %q", vault.taken["accept"])
}
if err := conn.Flush(); err != nil {
t.Fatal(err)
}
tapMu.Lock()
crossed := tapped.String()
tapMu.Unlock()
if !strings.Contains(crossed, "mesh.seat.mesh-vault.call.accept.controller") {
t.Fatalf("the tap heard no call to the vault: %s", crossed)
}
for _, s := range subjects {
if !strings.HasSuffix(s, ".controller") {
t.Errorf("the vault was asked on %s, not in the controller's name", s)
}
}
for where, text := range map[string]string{"the bus": crossed, "the log": logged.String(), "the answer": words} {
if strings.Contains(text, value) || strings.Contains(text, "must-not-show") {
t.Errorf("the value crossed %s in the clear", where)
}
}
if !strings.Contains(words, "sealed at this terminal") {
t.Errorf("the answer: %q", words)
}
// A vault that refuses the first step: nothing is sealed or sent.
vault.refuse = errors.New("the vault has no operator key")
if _, err := acceptIntoVault(vault.ask, ownSecretArgs("g14", "dunst", "token"), value, "jochen"); err == nil ||
!strings.Contains(err.Error(), "nothing was read or sealed") {
t.Errorf("a refused call said %v", err)
}
}
// The desk's path for a module filled the new way (issue 723): the controller asks the vault once; the vault
// records the ask, holds its bounds and asks the desk's prompt itself, so the controller never reaches the desk and
// carries no key and no sealed value; today's store is not touched. For a module filled the old way, the vault is
// never asked.
func TestAnAskForAModuleFilledTheNewWayIsTheVaults(t *testing.T) {
d, accepted, acts, asked := aDesk(t, sealedTo(t, typed))
d.askedBy = "g14/claude-code"
d.open = func(node, module, name, desk string) (int64, error) {
t.Fatal("today's record was asked")
return 0, nil
}
vault := newStandInVault(t)
d.vault = vault.ask
words, err := d.give("anchor", "telegram", "telegram-token", "laptop")
if err != nil {
t.Fatal(err)
}
if len(*accepted) != 0 {
t.Fatal("the controller opened and kept what was typed")
}
if len(*asked) != 0 {
t.Fatalf("the controller asked the desk's prompt itself: %v", *asked)
}
said := vault.said()
if !strings.Contains(said, `"verb":"secret-ask"`) || !strings.Contains(said, `"asked_by":"g14/claude-code"`) ||
!strings.Contains(said, `"desk":"laptop"`) || strings.Count(said, "\n") != 0 {
t.Errorf("the vault was not asked once, naming who asked and the desk: %s", said)
}
if strings.Contains(said, "seal_to") || strings.Contains(said, "sealed") || strings.Contains(said, `"call"`) {
t.Errorf("the controller carried a key or a sealed value: %s", said)
}
if len(*acts) != 1 || (*acts)[0].Cause != "given-at-the-desk" || !strings.Contains((*acts)[0].Why, "kept in the vault") ||
!strings.Contains((*acts)[0].Why, "as-the-vault-recorded") {
t.Errorf("the act: %+v", *acts)
}
if !strings.Contains(words, "own/anchor/telegram/telegram-token") || strings.Contains(words, typed) {
t.Errorf("said %q", words)
}
// Dismissed and timed out: nothing given, said so.
for _, outcome := range []string{"dismissed", "timed-out", "empty"} {
d2, _, _, _ := aDesk(t, sealedTo(t, typed))
v2 := newStandInVault(t)
v2.asked = outcome
d2.vault = v2.ask
if _, err := d2.give("anchor", "telegram", "telegram-token", "laptop"); !errors.Is(err, errNothingGiven) {
t.Errorf("%s: %v", outcome, err)
}
}
// Failed at the desk: an error, not "nothing given".
d5, _, acts5, _ := aDesk(t, sealedTo(t, typed))
v5 := newStandInVault(t)
v5.asked = "failed"
d5.vault = v5.ask
if _, err := d5.give("anchor", "telegram", "telegram-token", "laptop"); err == nil || errors.Is(err, errNothingGiven) ||
len(*acts5) != 0 {
t.Errorf("a failed ask: %v, acts %d", err, len(*acts5))
}
// The vault's bounds refuse: said so, nobody is asked by the controller.
d3, _, _, asked3 := aDesk(t, sealedTo(t, typed))
v3 := newStandInVault(t)
v3.refuse = errors.New("an ask for telegram-token of telegram on anchor is still open")
d3.vault = v3.ask
if _, err := d3.give("anchor", "telegram", "telegram-token", "laptop"); err == nil ||
!strings.Contains(err.Error(), "still open") || len(*asked3) != 0 {
t.Errorf("a refused ask: %v, prompts %d", err, len(*asked3))
}
// Filled the old way: today's path, the vault never asked.
d4, accepted4, _, _ := aDesk(t, sealedTo(t, typed))
if _, err := d4.give("anchor", "telegram", "telegram-token", "laptop"); err != nil {
t.Fatal(err)
}
if len(*accepted4) != 1 || d4.vault != nil {
t.Error("a module filled the old way did not take today's path")
}
}
// `secret issued` for a provider filled the new way (issue 723): the controller asks the vault once, naming the
// provider's machine, module and tool; the vault asks the tool itself; the controller never reaches the tool and
// nothing is kept, sealed or sent here. A provider filled the old way keeps today's path and the vault is never
// asked.
func TestAnIssuedValueForAProviderFilledTheNewWayGoesToTheVault(t *testing.T) {
var r issuedRecorder
vault := newStandInVault(t)
taker := r.taker(t, func(issuedProvider, map[string]any) (link.Answer, error) {
t.Fatal("the controller asked the provider's tool itself")
return link.Answer{}, nil
})
taker.keep = func(string, string, string) (inventory.IssuedKept, error) {
t.Fatal("today's store was asked to keep it")
return inventory.IssuedKept{}, nil
}
taker.ofTheVault = switchOf(func(m string) bool { return m == "plex" })
taker.vault = vault.ask
words, err := taker.take("plex-api")
if err != nil {
t.Fatal(err)
}
if len(r.sent) != 0 {
t.Errorf("machines were sent: %v", r.sent)
}
if len(r.acts) != 1 || r.acts[0].Verb != handActIssued || !strings.Contains(r.acts[0].Why, "kept in the vault") {
t.Errorf("the act: %+v", r.acts)
}
said := vault.said()
for _, want := range []string{`"verb":"issued"`, `"provider":"ace"`, `"module":"plex"`, `"tool":"plex_issued_credential"`} {
if !strings.Contains(said, want) {
t.Errorf("the vault was told %s, without %s", said, want)
}
}
if strings.Contains(said, "seal_to") || strings.Contains(said, "sealed") || strings.Contains(said, `"call"`) {
t.Errorf("the controller carried a key or a sealed value: %s", said)
}
if !strings.Contains(words, "provision/ace/plex-api") {
t.Errorf("said %q", words)
}
// The same value again: the vault writes nothing, and nothing is recorded.
again := newStandInVault(t)
again.again = true
r2 := issuedRecorder{}
t2 := r2.taker(t, issuedSealed(t, "the-servers-own-token"))
t2.ofTheVault, t2.vault = switchOf(func(string) bool { return true }), again.ask
if words, err := t2.take("plex-api"); err != nil || len(r2.acts) != 0 || !strings.Contains(words, "nothing was written") {
t.Errorf("%q %v %+v", words, err, r2.acts)
}
// The vault refuses (its tool refused it): said, nothing recorded.
refused := newStandInVault(t)
refused.refuse = errors.New("its tool plex_issued_credential refused: this tool answers the vault alone")
r4 := issuedRecorder{}
t4 := r4.taker(t, issuedSealed(t, "x"))
t4.ofTheVault, t4.vault = switchOf(func(string) bool { return true }), refused.ask
if _, err := t4.take("plex-api"); err == nil || !strings.Contains(err.Error(), "answers the vault alone") || len(r4.acts) != 0 {
t.Errorf("a refusal: %v, acts %d", err, len(r4.acts))
}
// Filled the old way: today's path.
var r3 issuedRecorder
t3 := r3.taker(t, issuedSealed(t, "the-servers-own-token"))
t3.ofTheVault = switchOf(func(string) bool { return false })
t3.vault = func(string, map[string]any) (json.RawMessage, error) {
t.Fatal("the vault was asked for a provider filled the old way")
return nil, nil
}
if _, err := t3.take("plex-api"); err != nil || len(r3.kept) != 1 {
t.Errorf("today's path: %v, kept %d", err, len(r3.kept))
}
}
// `secret issued` names to the vault the provision's holders from that provider (issue 749): a tool may answer a
// value for each, which the vault keeps as that holder's own; the controller says what was kept for whom and which
// holder the tool gave nothing for, never a value. A provider answering one value is told the holders all the same
// and keeps the provision's one credential.
func TestAnIssuedValuePerHolderNamesTheHoldersToTheVault(t *testing.T) {
var r issuedRecorder
vault := newStandInVault(t)
vault.perHolder = true
taker := r.taker(t, func(issuedProvider, map[string]any) (link.Answer, error) {
t.Fatal("the controller asked the provider's tool itself")
return link.Answer{}, nil
})
taker.providers = func(string) ([]issuedProvider, bool, error) {
return []issuedProvider{{Node: "novox", Module: "gitea", Tool: "gitea_agent_token", Holders: []issuedHolder{
{Node: "anchor", Module: "claude-code"}, {Node: "laptop", Module: "claude-code"},
{Node: "shanks", Module: "claude-code"}}}}, true, nil
}
taker.ofTheVault = switchOf(func(m string) bool { return m == "gitea" })
taker.vault = vault.ask
words, err := taker.take("gitea-read")
if err != nil {
t.Fatal(err)
}
said := vault.said()
if !strings.Contains(said, `"holders":[{"node":"anchor","module":"claude-code"},{"node":"laptop","module":"claude-code"},{"node":"shanks","module":"claude-code"}]`) {
t.Errorf("the vault was told %s", said)
}
for _, want := range []string{"provision/novox/gitea-read/anchor/claude-code", "provision/novox/gitea-read/laptop/claude-code",
"gave nothing for claude-code on shanks"} {
if !strings.Contains(words, want) {
t.Errorf("said %q, without %q", words, want)
}
}
if len(r.acts) != 1 || !strings.Contains(r.acts[0].Why, "claude-code on anchor") || !strings.Contains(r.acts[0].Why, "claude-code on laptop") {
t.Errorf("the act: %+v", r.acts)
}
if len(r.sent) != 0 || len(r.kept) != 0 {
t.Error("the controller kept or sent something")
}
// The same values again: nothing written, nothing recorded.
again := newStandInVault(t)
again.perHolder, again.again = true, true
r2 := issuedRecorder{}
t2 := r2.taker(t, nil)
t2.providers, t2.ofTheVault, t2.vault = taker.providers, taker.ofTheVault, again.ask
if words, err := t2.take("gitea-read"); err != nil || len(r2.acts) != 0 || !strings.Contains(words, "nothing was written") {
t.Errorf("again: %q %v %+v", words, err, r2.acts)
}
}
// The holders named are the provision's holders from that provider, each consuming module on each machine once.
func TestTheHoldersNamedAreThoseOfThatProvider(t *testing.T) {
holders := []inventory.Holder{
{Provision: "gitea-read", Consumer: "anchor", ConsumerModule: "claude-code", Provider: "novox"},
{Provision: "gitea-read", Consumer: "anchor", ConsumerModule: "claude-code", Local: "second", Provider: "novox"},
{Provision: "gitea-read", Consumer: "laptop", ConsumerModule: "claude-code", Provider: "novox"},
{Provision: "gitea-read", Consumer: "ace", ConsumerModule: "claude-code", Provider: "shanks"},
}
got := issuedHoldersFrom(holders, "novox")
if fmt.Sprint(got) != "[claude-code on anchor claude-code on laptop]" {
t.Errorf("the holders named: %v", got)
}
if got := issuedHoldersFrom(holders, "g14"); len(got) != 0 {
t.Errorf("a provider holding nothing named %v", got)
}
shelf := map[string]catalogue.Manifest{"gitea": {Module: "gitea", Provides: []catalogue.Offer{{Name: "gitea-read",
Credential: &catalogue.OfferCredential{IssuedBy: catalogue.IssuedBySoftware, Tool: "gitea_agent_token"}}}}}
providers, err := issuedProvidersFrom(shelf, holders, func(string) ([]string, error) { return []string{"gitea"}, nil },
"gitea-read")
if err != nil || len(providers) != 2 || providers[0].Node != "novox" || fmt.Sprint(providers[0].Holders) !=
"[claude-code on anchor claude-code on laptop]" || fmt.Sprint(providers[1].Holders) != "[claude-code on ace]" {
t.Errorf("the providers asked: %+v %v", providers, err)
}
}
// `secret rotate` for a module filled the new way asks the vault, naming who asked; nothing is sent.
func TestARotationForAModuleFilledTheNewWayIsTheVaults(t *testing.T) {
vault := newStandInVault(t)
if _, err := rotateInVault(vault.ask, ownSecretArgs("g14", "dunst", "token"), "g14/claude-code", " "); err == nil ||
vault.said() != "" {
t.Fatalf("a rotation without a why reached the vault: %v", err)
}
words, err := rotateInVault(vault.ask, ownSecretArgs("g14", "dunst", "token"), "g14/claude-code", "leaked")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(vault.said(), `"verb":"rotate"`) || !strings.Contains(vault.said(), `"by":"g14/claude-code"`) ||
!strings.Contains(vault.said(), `"module":"dunst"`) || !strings.Contains(vault.said(), `"why":"leaked"`) {
t.Errorf("the vault was told %s", vault.said())
}
if !strings.Contains(words, "generation 2") || !strings.Contains(words, "takes it from the vault") {
t.Errorf("%q", words)
}
}
// The module that decides `accept --provider` is the one on the provider's machine that offers the provision; none,
// or several, is none, and today's path then says what is wrong.
func TestTheProvidersModuleIsTheOneThatOffersTheProvision(t *testing.T) {
offer := func(module string, provisions ...string) catalogue.Manifest {
m := catalogue.Manifest{Module: module}
for _, p := range provisions {
m.Provides = append(m.Provides, catalogue.Offer{Name: p})
}
return m
}
shelf := map[string]catalogue.Manifest{"tv": offer("tv", "tv-api"), "plex": offer("plex", "plex-api"),
"other-tv": offer("other-tv", "tv-api"), "dunst": offer("dunst")}
for _, c := range []struct {
assigned []string
want string
}{
{[]string{"dunst", "tv", "plex"}, "tv"},
{[]string{"dunst", "plex"}, ""},
{[]string{"tv", "other-tv"}, ""},
} {
if got := offeredOn(shelf, c.assigned, "tv-api"); got != c.want {
t.Errorf("%v: %q, want %q", c.assigned, got, c.want)
}
}
}
cmd/mesh-controller/vault_switch_test.go
+413
View File
@@ -0,0 +1,413 @@
package main
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
// Each of the four secret commands honours the switch at its own call site (the review of issue 675): run as the
// terminal runs them, against the real store and a real bus with a stand-in vault serving the vault's seat, a
// module filled the old way takes today's path and the vault is never asked, and one filled the new way reaches
// the vault. Sending every module to the vault at any one call site — or none — fails one of these. Needs the
// store (MESH_TEST_POSTGRES), and is named in testdata/store-tests.
// vaultOnTheBus serves the vault's four intake verbs on the test's bus with a stand-in vault, and says which verbs
// were asked, in order.
type vaultOnTheBus struct {
vault *standInVault
mu sync.Mutex
asked []string
}
func (v *vaultOnTheBus) verbs() string {
v.mu.Lock()
defer v.mu.Unlock()
return strings.Join(v.asked, ",")
}
func (v *vaultOnTheBus) reset() {
v.mu.Lock()
defer v.mu.Unlock()
v.asked = nil
}
// aMeshWithAVault is aMesh with the controller's bus pointed at a test bus where a stand-in vault serves its seat,
// and the switch saying that only the modules named are filled the new way.
func aMeshWithAVault(t *testing.T) (*stores, *vaultOnTheBus, func(moved ...string)) {
t.Helper()
open := aMesh(t)
conn, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(conn.Close)
before := handActConn
handActConn = conn
t.Cleanup(func() { handActConn = before })
served := &vaultOnTheBus{vault: newStandInVault(t)}
for _, verb := range []string{"accept", "secret-ask", "issued", "rotate"} {
sub, err := conn.Subscribe(broker.CalledPattern(link.SeatToolSubject("mesh-vault", verb)), func(m *nats.Msg) {
served.mu.Lock()
served.asked = append(served.asked, verb)
served.mu.Unlock()
var args map[string]any
_ = json.Unmarshal(m.Data, &args)
result, err := served.vault.ask(verb, args)
answer := link.Answer{Result: result}
if err != nil {
answer = link.Answer{Error: err.Error()}
}
raw, _ := json.Marshal(answer)
_ = m.Respond(raw)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
if err := conn.Flush(); err != nil {
t.Fatal(err)
}
switchWas := filledTheNewWay
t.Cleanup(func() { filledTheNewWay = switchWas })
moveOnly := func(moved ...string) {
filledTheNewWay = switchOf(func(module string) bool {
for _, m := range moved {
if m == module {
return true
}
}
return false
})
}
return open, served, moveOnly
}
// aValueFile is a file holding a value, for `--from`.
func aValueFile(t *testing.T, value string) string {
t.Helper()
file := filepath.Join(t.TempDir(), "value")
if err := os.WriteFile(file, []byte(value+"\n"), 0o600); err != nil {
t.Fatal(err)
}
return file
}
// `secret accept` of a module's own secret.
func TestAcceptOfAnOwnSecretTakesThePathOfItsModulesForm(t *testing.T) {
open, vault, moveOnly := aMeshWithAVault(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "dunst", Version: "1",
OwnSecrets: catalogue.OwnSecrets{"token": {Path: "/var/lib/mesh/dunst/token"}}})
if _, err := assign(ctx, open, "anchor", "dunst"); err != nil {
t.Fatal(err)
}
accept := []string{"accept", "anchor", "dunst", "token", "--from", aValueFile(t, "the-bots-token")}
moveOnly("some-other-module")
if err := secretCommand(ctx, accept); err != nil {
t.Fatalf("today's path: %v", err)
}
if asked := vault.verbs(); asked != "" {
t.Fatalf("the vault was asked %s for a module filled the old way", asked)
}
if origin, _, err := open.inventory.OwnSecretOrigin(ctx, "anchor", "dunst", "token"); err != nil || origin == "" {
t.Fatalf("today's store does not hold it: %q %v", origin, err)
}
moveOnly("dunst")
if err := secretCommand(ctx, accept); err != nil {
t.Fatalf("the vault's path: %v", err)
}
if asked := vault.verbs(); asked != "accept,accept" {
t.Fatalf("the vault was asked %q for a module filled the new way", asked)
}
if vault.vault.taken["accept"] != "the-bots-token" {
t.Fatalf("the vault opened %q", vault.vault.taken["accept"])
}
}
// `secret accept --provider`: the provider's module decides, not the consumer's.
func TestAcceptForAProviderTakesThePathOfTheProvidersModule(t *testing.T) {
open, vault, moveOnly := aMeshWithAVault(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "tv", Version: "1",
Provides: []catalogue.Offer{{Name: "tv-api", Scope: catalogue.ScopeMesh}}})
register(t, open, catalogue.Manifest{Module: "requests", Version: "1", Requires: []string{"tv-api"},
Secrets: map[string]string{"tv-api": "/var/lib/mesh/requests/tv-api.secret"}})
for _, a := range [][2]string{{"anchor", "tv"}, {"laptop", "requests"}} {
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
t.Fatal(err)
}
}
accept := []string{"accept", "laptop", "requests", "tv-api", "--provider", "anchor", "--from",
aValueFile(t, "the-pairs-key")}
// The consumer filled the new way, the provider the old: the provider's module decides, today's path.
moveOnly("requests")
if err := secretCommand(ctx, accept); err != nil {
t.Fatalf("today's path: %v", err)
}
if asked := vault.verbs(); asked != "" {
t.Fatalf("the vault was asked %s for a provider filled the old way", asked)
}
moveOnly("tv")
if err := secretCommand(ctx, accept); err != nil {
t.Fatalf("the vault's path: %v", err)
}
if asked := vault.verbs(); asked != "accept,accept" {
t.Fatalf("the vault was asked %q for a provider filled the new way", asked)
}
if said := vault.vault.said(); !strings.Contains(said, `"provider":"anchor"`) || !strings.Contains(said, `"provision":"tv-api"`) {
t.Errorf("the vault was told %s", said)
}
}
// `secret accept` of a provision's credential issued outside the mesh, given once for its provider.
func TestAcceptOfAnOutsideOfferTakesThePathOfTheProvidersModule(t *testing.T) {
open, vault, moveOnly := aMeshWithAVault(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "tv", Version: "1",
Provides: []catalogue.Offer{{Name: "tv-api", Scope: catalogue.ScopeMesh,
Credential: &catalogue.OfferCredential{IssuedBy: catalogue.IssuedOutside}}}})
register(t, open, catalogue.Manifest{Module: "requests", Version: "1", Requires: []string{"tv-api"},
Secrets: map[string]string{"tv-api": "/var/lib/mesh/requests/tv-api.secret"}})
for _, a := range [][2]string{{"anchor", "tv"}, {"laptop", "requests"}} {
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
t.Fatal(err)
}
}
accept := []string{"accept", "anchor", "tv", "tv-api", "--from", aValueFile(t, "the-apps-key")}
moveOnly("requests")
// Today's path seals it to every consumer; whether it then sends is not this test's.
_ = secretCommand(ctx, accept)
if asked := vault.verbs(); asked != "" {
t.Fatalf("the vault was asked %s for a provider filled the old way", asked)
}
moveOnly("tv")
if err := secretCommand(ctx, accept); err != nil {
t.Fatalf("the vault's path: %v", err)
}
if asked := vault.verbs(); asked != "accept,accept" {
t.Fatalf("the vault was asked %q for a provider filled the new way", asked)
}
if vault.vault.taken["accept"] != "the-apps-key" {
t.Fatalf("the vault opened %q", vault.vault.taken["accept"])
}
}
// `secret ask` (and `secret accept --at-desk`): the desk is never answered on this bus, so today's path ends at the
// prompt; the vault's path is one ask of the vault, which asks the desk itself (issue 723), so it is done without
// the controller reaching the desk at all.
func TestAnAskAtTheDeskTakesThePathOfItsModulesForm(t *testing.T) {
open, vault, moveOnly := aMeshWithAVault(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "dunst", Version: "1",
OwnSecrets: catalogue.OwnSecrets{"token": {Path: "/var/lib/mesh/dunst/token"}}})
if _, err := assign(ctx, open, "anchor", "dunst"); err != nil {
t.Fatal(err)
}
for _, line := range [][]string{
{"ask", "anchor", "dunst", "token", "--at", "laptop"},
{"accept", "anchor", "dunst", "token", "--at-desk", "laptop"},
} {
vault.reset()
moveOnly("some-other-module")
err := secretCommand(ctx, line)
if err == nil || !strings.Contains(err.Error(), "could not be asked") {
t.Fatalf("%v: today's path did not reach the desk: %v", line, err)
}
if asked := vault.verbs(); asked != "" {
t.Fatalf("%v: the vault was asked %s for a module filled the old way", line, asked)
}
vault.reset()
moveOnly("dunst")
if err := secretCommand(ctx, line); err != nil {
t.Fatalf("%v: the vault's path reached the desk itself, or failed: %v", line, err)
}
// One ask of the vault, which asks the desk in its own name: the controller carries nothing to it.
if asked := vault.verbs(); asked != "secret-ask" {
t.Fatalf("%v: the vault was asked %q for a module filled the new way", line, asked)
}
}
}
// `secret issued`: the provider's tool is never answered on this bus, so today's path ends at it; the vault's path
// is one ask of the vault, which asks the tool itself (issue 723).
func TestAnIssuedValueTakesThePathOfItsProvidersModule(t *testing.T) {
open, vault, moveOnly := aMeshWithAVault(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "plex", Version: "1", Tools: []string{"plex_token"},
Provides: []catalogue.Offer{{Name: "plex-api", Scope: catalogue.ScopeMesh,
Credential: &catalogue.OfferCredential{IssuedBy: catalogue.IssuedBySoftware, Tool: "plex_token"}}}})
register(t, open, catalogue.Manifest{Module: "kometa", Version: "1", Requires: []string{"plex-api"},
Secrets: map[string]string{"plex-api": "/var/lib/mesh/kometa/plex-api.secret"}})
for _, a := range [][2]string{{"anchor", "plex"}, {"laptop", "kometa"}} {
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
t.Fatal(err)
}
}
// The consumer's send makes the pair, so the provision has a holder to take it for.
plan, settings, err := planFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
if _, err := declarationFor(ctx, open, "laptop", plan, settings); err != nil {
t.Fatal(err)
}
moveOnly("kometa")
err = secretCommand(ctx, []string{"issued", "plex-api"})
if err == nil || !strings.Contains(err.Error(), "could not be asked") {
t.Fatalf("today's path did not reach the provider's tool: %v", err)
}
if asked := vault.verbs(); asked != "" {
t.Fatalf("the vault was asked %s for a provider filled the old way", asked)
}
moveOnly("plex")
if err := secretCommand(ctx, []string{"issued", "plex-api"}); err != nil {
t.Fatalf("the vault's path reached the provider's tool itself, or failed: %v", err)
}
if asked := vault.verbs(); asked != "issued" {
t.Fatalf("the vault was asked %q for a provider filled the new way", asked)
}
}
// `secret rotate` of a module's own secret.
func TestARotationTakesThePathOfItsModulesForm(t *testing.T) {
open, vault, moveOnly := aMeshWithAVault(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "dunst", Version: "1",
OwnSecrets: catalogue.OwnSecrets{"token": {Path: "/var/lib/mesh/dunst/token"}}})
if _, err := assign(ctx, open, "anchor", "dunst"); err != nil {
t.Fatal(err)
}
moveOnly()
if err := secretCommand(ctx, []string{"accept", "anchor", "dunst", "token", "--from", aValueFile(t, "tok")}); err != nil {
t.Fatal(err)
}
rotate := []string{"rotate", "anchor", "dunst", "token", "--why", "the test rotates it"}
moveOnly("some-other-module")
_ = secretCommand(ctx, rotate) // today's path: rotated in the store, or refused there
if asked := vault.verbs(); asked != "" {
t.Fatalf("the vault was asked %s for a module filled the old way", asked)
}
moveOnly("dunst")
if err := secretCommand(ctx, []string{"rotate", "anchor", "dunst", "token"}); err == nil ||
!strings.Contains(err.Error(), "--why") || vault.verbs() != "" {
t.Fatalf("a rotation in the vault without a why: %v, the vault asked %q", err, vault.verbs())
}
if err := secretCommand(ctx, rotate); err != nil {
t.Fatalf("the vault's path: %v", err)
}
if asked := vault.verbs(); asked != "rotate" {
t.Fatalf("the vault was asked %q for a module filled the new way", asked)
}
if said := vault.vault.said(); !strings.Contains(said, `"why":"the test rotates it"`) {
t.Errorf("the vault was not told why: %s", said)
}
}
// A moved module that runs as an account of its own is announced on the operator's channels before the vault is
// asked, and with the announcement failing the vault is never called (issue 836, the N1-give rule).
func TestAcceptForAMovedTrustedModuleAnnouncesBeforeTheVaultIsAsked(t *testing.T) {
open, vault, moveOnly := aMeshWithAVault(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "telegram", Version: "1", RunsAs: "telegram", SecretsOwner: "telegram",
OwnSecrets: catalogue.OwnSecrets{"token": {Path: "/var/lib/mesh/telegram/token"}}})
register(t, open, catalogue.Manifest{Module: "dunst", Version: "1",
OwnSecrets: catalogue.OwnSecrets{"token": {Path: "/var/lib/mesh/dunst/token"}}})
for _, m := range []string{"telegram", "dunst"} {
if _, err := assign(ctx, open, "anchor", m); err != nil {
t.Fatal(err)
}
}
moveOnly("telegram", "dunst")
was := announceTheGiven
t.Cleanup(func() { announceTheGiven = was })
var order []string
announceTheGiven = func(_ context.Context, node, module, name, how, askedBy string) error {
order = append(order, "announce:"+module+":"+vault.verbs())
return nil
}
accept := func(module string) error {
return secretCommand(ctx, []string{"accept", "anchor", module, "token", "--from", aValueFile(t, "the-bots-token")})
}
if err := accept("telegram"); err != nil {
t.Fatal(err)
}
if strings.Join(order, ",") != "announce:telegram:" || vault.verbs() != "accept,accept" {
t.Fatalf("announced %v, the vault was asked %q", order, vault.verbs())
}
vault.reset()
announceTheGiven = func(context.Context, string, string, string, string, string) error {
return errors.New("no channel took it")
}
err := accept("telegram")
if err == nil || !strings.Contains(err.Error(), "nothing was sealed") || vault.verbs() != "" {
t.Fatalf("a failed announcement: %v, the vault was asked %q", err, vault.verbs())
}
// Any other module is kept first and announced after; a failed announcement is said, not undone.
if err := accept("dunst"); err != nil || vault.verbs() != "accept,accept" {
t.Fatalf("an untrusted module: %v, asked %q", err, vault.verbs())
}
}
// issue 851 (hand-act): a moved module's terminal give is recorded in the hand-act log like acceptKept's
// (novox/hq ADR 0380 §6), naming where, and never carrying the value.
func TestAcceptForAMovedModuleRecordsAHandActAndNeverTheValue(t *testing.T) {
open, _, moveOnly := aMeshWithAVault(t)
ctx := t.Context()
api, err := jetstream.New(handActConn)
if err != nil {
t.Fatal(err)
}
if _, err := api.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.HandActsBucket}); err != nil {
t.Fatal(err)
}
register(t, open, catalogue.Manifest{Module: "dunst", Version: "1",
OwnSecrets: catalogue.OwnSecrets{"token": {Path: "/var/lib/mesh/dunst/token"}}})
if _, err := assign(ctx, open, "anchor", "dunst"); err != nil {
t.Fatal(err)
}
moveOnly("dunst")
was := announceTheGiven
t.Cleanup(func() { announceTheGiven = was })
announceTheGiven = func(context.Context, string, string, string, string, string) error { return nil }
if err := secretCommand(ctx, []string{"accept", "anchor", "dunst", "token", "--from", aValueFile(t, "the-bots-token")}); err != nil {
t.Fatal(err)
}
acts, err := link.HandActs(ctx, handActConn, time.Now().Add(-time.Minute))
if err != nil || len(acts) != 1 || acts[0].Verb != "secret accept" || acts[0].Cause != "given-at-the-terminal" ||
!strings.Contains(acts[0].Why, "at the controller's terminal") {
t.Fatalf("recorded %+v (%v)", acts, err)
}
if strings.Contains(acts[0].Why, "the-bots-token") {
t.Error("the act carries the value")
}
}
cmd/mesh-controller/watchdogs.go
+6
View File
@@ -63,6 +63,9 @@ type signalFacts struct {
// failure (S21, novox/hq issue 233).
doing []inventory.Doing
doingErr error
// provFailed is every module whose files a machine's provisioner client last said it failed to place (S21,
// novox/hq ADR 0362 §3): that machine's failed apply.
provFailed []inventory.ProvReport
loop loopFacts
loopErr error
@@ -396,6 +399,9 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
f.handActs, f.handActsErr = w.gatherHandActs(ctx, now)
f.refusedSends, f.refusedSendsErr = inv.RefusedSendsSince(ctx, now.Add(-generationRefusalsSaid))
f.doing, f.doingErr = inv.NotDoingWhatTheyWereTold(ctx)
if f.doingErr == nil && w.js != nil {
f.provFailed, f.doingErr = provisioningFailed(w.js.Context())
}
f.facts.taken, _, f.facts.began, f.facts.err = exportedFacts.last()
return f
}