Keep mesh and node config with their history, and move whole-mesh module layers onto each assigned node (hq issue 630)
An assignment is one module on one node, so a new whole-mesh layer is refused and the ones held are laid under each assigned node's layer, every replaced layer kept, so no machine composes differently.
This commit is contained in:
@@ -117,6 +117,18 @@ func (i *Inventory) RegisterModule(ctx context.Context, m catalogue.Manifest, fr
|
||||
}
|
||||
}
|
||||
}
|
||||
// **A preference falls back only to a key on the catalogue's list, and with its type** (novox/hq issue
|
||||
// 630): mesh and node config hold values for the predefined keys alone, so a `predefined` the list does
|
||||
// not hold is a fallback nobody can ever set.
|
||||
if names := catalogue.PredefinedNamed(m); len(names) > 0 {
|
||||
list, err := i.Predefined(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if problems := catalogue.PredefinedProblems(m, list); len(problems) > 0 {
|
||||
return fmt.Errorf("%s is not registered: %s", m.Module, strings.Join(problems, "; "))
|
||||
}
|
||||
}
|
||||
raw, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -853,6 +865,14 @@ func (i *Inventory) setSettings(ctx context.Context, nodeName, module string, va
|
||||
} else if _, err := i.declared(ctx, module); err != nil {
|
||||
return fmt.Errorf("%w: %s", ErrNoSuchModule, module)
|
||||
}
|
||||
if nodeName == "" && judge {
|
||||
// **There is no module layer for the whole mesh** (novox/hq issue 630): an assignment is one module
|
||||
// on one node. What every node should share is mesh config, for a predefined key. A layer the mesh
|
||||
// already holds is still kept as it is (KeepSettings, the merge gate raising a store from facts).
|
||||
return fmt.Errorf("%s: there is no module layer for the whole mesh — an assignment is one module on "+
|
||||
"one node (novox/hq issue 630). Set it on each node with --node, or, for a predefined key, as mesh "+
|
||||
"config with the config verb. Nothing was changed", module)
|
||||
}
|
||||
if nodeName == "" {
|
||||
// A port is a fact about one machine (novox/hq ADR 0100). Refused here, in composition's
|
||||
// words: stored, it refuses every node running the module at composition, and the mesh
|
||||
@@ -1133,7 +1153,8 @@ func (i *Inventory) ClearSettingsBy(ctx context.Context, nodeName, module, clear
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// SettingsFor is the layers that apply to one module on one node, in the order they are applied.
|
||||
// SettingsFor is the layers that apply to one module on one node, in the order they are applied: mesh
|
||||
// config, node config (both marked Config), then the module's own layer on the node.
|
||||
//
|
||||
// The mesh's first, then the node's, so a node that differs is expressed by differing rather
|
||||
// than by restating everything the rest of the mesh already says.
|
||||
@@ -1151,7 +1172,12 @@ func (i *Inventory) SettingsFor(ctx context.Context, nodeName, module string) ([
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var layers []catalogue.Layer
|
||||
// Mesh config, then node config, for the predefined keys the module's preferences fall back to
|
||||
// (novox/hq issue 630): under the module's own layers, over its defaults.
|
||||
layers, err := i.configLayers(ctx, node.ID, nodeName, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var meshWide bool
|
||||
var raw []byte
|
||||
|
||||
@@ -0,0 +1,339 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// Mesh config and node config (novox/hq issue 630): a value for a predefined key, for the whole mesh or for
|
||||
// one node, that every module whose preference names the key falls back to. The list of predefined keys is
|
||||
// the catalogue's predefined.json, kept here as the last build of the catalogue carried it.
|
||||
|
||||
// ErrNotPredefined is a key mesh or node config was asked to hold that is not on the list.
|
||||
var ErrNotPredefined = errors.New("not a predefined key")
|
||||
|
||||
// SetPredefined replaces the list of predefined keys with the one a build of `repository` at `commit`
|
||||
// carried. Replaced whole: the file is a statement of the whole list.
|
||||
func (i *Inventory) SetPredefined(ctx context.Context, list catalogue.Predefined, repository, commit string) error {
|
||||
tx, err := i.store.Pool().Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
if _, err := tx.Exec(ctx, `delete from predefined_key`); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, key := range list.Keys() {
|
||||
k := list[key]
|
||||
if _, err := tx.Exec(ctx,
|
||||
`insert into predefined_key (key, type, why, repository, commit) values ($1, $2, $3, $4, $5)`,
|
||||
key, k.Type, k.Why, nullable(repository), nullable(commit)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// Predefined is the list of predefined keys the mesh holds; empty before a build of the catalogue carried one.
|
||||
func (i *Inventory) Predefined(ctx context.Context) (catalogue.Predefined, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `select key, type, why from predefined_key`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := catalogue.Predefined{}
|
||||
for rows.Next() {
|
||||
var key string
|
||||
var k catalogue.PredefinedKey
|
||||
if err := rows.Scan(&key, &k.Type, &k.Why); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[key] = k
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ConfigValue is one value of mesh config (Node "") or node config.
|
||||
type ConfigValue struct {
|
||||
Key string
|
||||
Node string
|
||||
Value any
|
||||
SetAt time.Time
|
||||
SetBy string
|
||||
Why string
|
||||
}
|
||||
|
||||
// Config is every value of mesh config and node config, by key, the mesh's first, then by node.
|
||||
func (i *Inventory) Config(ctx context.Context) ([]ConfigValue, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select c.key, coalesce(n.name, ''), c.value, c.set_at, coalesce(c.set_by, ''), c.why
|
||||
from config c left join node n on n.id = c.node
|
||||
order by c.key, c.node is not null, n.name`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []ConfigValue
|
||||
for rows.Next() {
|
||||
var v ConfigValue
|
||||
var raw []byte
|
||||
if err := rows.Scan(&v.Key, &v.Node, &raw, &v.SetAt, &v.SetBy, &v.Why); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(raw, &v.Value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SetConfig sets a predefined key's value for the whole mesh (node "") or for one node. Refused for a key not
|
||||
// on the list and for a value not of the key's type. The value it replaces is kept in config_history.
|
||||
func (i *Inventory) SetConfig(ctx context.Context, key, nodeName string, value any, setBy, why string) error {
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return errors.New("config says why: one sentence, kept with the value")
|
||||
}
|
||||
if err := i.JudgeConfig(ctx, key, nodeName, value); err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
nodeID, err := i.layerNode(ctx, nodeName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tx, err := i.store.Pool().Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
if _, err := tx.Exec(ctx, keepConfigSQL, key, nodeID, "set", why); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx,
|
||||
`delete from config where key = $1 and node is not distinct from $2::uuid`, key, nodeID); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx,
|
||||
`insert into config (key, node, value, set_by, why) values ($1, $2::uuid, $3, $4, $5)`,
|
||||
key, nodeID, raw, nullable(setBy), why); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// JudgeConfig refuses, before anything is done, a value mesh or node config may not hold: a key not on the
|
||||
// list of predefined keys, or a value not of the key's type.
|
||||
func (i *Inventory) JudgeConfig(ctx context.Context, key, nodeName string, value any) error {
|
||||
list, err := i.Predefined(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
k, on := list[key]
|
||||
if !on {
|
||||
return notPredefined(key, nodeName, list)
|
||||
}
|
||||
return catalogue.ConfigValueProblem(fmt.Sprintf("the predefined key %q", key), k.Type, value)
|
||||
}
|
||||
|
||||
// ClearConfig removes a predefined key's value for the whole mesh (node "") or for one node, keeping it in
|
||||
// config_history; false when there was none. A key no longer on the list may still be cleared.
|
||||
func (i *Inventory) ClearConfig(ctx context.Context, key, nodeName, clearedBy, why string) (bool, error) {
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return false, errors.New("config says why: one sentence, kept with the history")
|
||||
}
|
||||
nodeID, err := i.layerNode(ctx, nodeName)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
tx, err := i.store.Pool().Begin(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
if _, err := tx.Exec(ctx, keepConfigSQL, key, nodeID, "clear", why); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if clearedBy != "" {
|
||||
if _, err := tx.Exec(ctx,
|
||||
`update config_history set replaced_by = 'clear by ' || $3
|
||||
where key = $1 and node is not distinct from $2::uuid
|
||||
and replaced_at = (select max(replaced_at) from config_history
|
||||
where key = $1 and node is not distinct from $2::uuid)`,
|
||||
key, nodeID, clearedBy); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `delete from config where key = $1 and node is not distinct from $2::uuid`, key, nodeID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return tag.RowsAffected() > 0, tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// PastConfig is one value of mesh or node config that was replaced or cleared.
|
||||
type PastConfig struct {
|
||||
Value any
|
||||
SetAt time.Time
|
||||
SetBy string
|
||||
Why string
|
||||
ReplacedAt time.Time
|
||||
ReplacedBy string
|
||||
ReplacedWhy string
|
||||
}
|
||||
|
||||
// ConfigHistory is the values a predefined key had for the whole mesh (node "") or one node, latest first.
|
||||
func (i *Inventory) ConfigHistory(ctx context.Context, key, nodeName string) ([]PastConfig, error) {
|
||||
nodeID, err := i.layerNode(ctx, nodeName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select value, coalesce(set_at, 'epoch'), coalesce(set_by, ''), coalesce(why, ''), replaced_at, replaced_by,
|
||||
coalesce(replaced_why, '')
|
||||
from config_history where key = $1 and node is not distinct from $2::uuid order by replaced_at desc`,
|
||||
key, nodeID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []PastConfig
|
||||
for rows.Next() {
|
||||
var p PastConfig
|
||||
var raw []byte
|
||||
if err := rows.Scan(&raw, &p.SetAt, &p.SetBy, &p.Why, &p.ReplacedAt, &p.ReplacedBy, &p.ReplacedWhy); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(raw, &p.Value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// keepConfigSQL copies the value about to be replaced or cleared into the history, with why it goes.
|
||||
const keepConfigSQL = `insert into config_history (key, node, value, set_at, set_by, why, replaced_by, replaced_why)
|
||||
select key, node, value, set_at, set_by, why, $3, $4 from config
|
||||
where key = $1 and node is not distinct from $2::uuid`
|
||||
|
||||
func notPredefined(key, nodeName string, list catalogue.Predefined) error {
|
||||
where := "mesh config"
|
||||
if nodeName != "" {
|
||||
where = "node config"
|
||||
}
|
||||
listed := "the list holds none yet: it arrives with a build of the catalogue carrying " + catalogue.PredefinedFile
|
||||
if len(list) > 0 {
|
||||
listed = "the list holds " + strings.Join(list.Keys(), ", ")
|
||||
}
|
||||
return fmt.Errorf("%w: %s holds values only for the predefined keys, and %q is not one (%s); a module's own "+
|
||||
"value is set with module and node", ErrNotPredefined, where, key, listed)
|
||||
}
|
||||
|
||||
// configLayers are the layers mesh config and node config make for one module on one node: nothing when
|
||||
// neither holds a value, or the module names no predefined key.
|
||||
func (i *Inventory) configLayers(ctx context.Context, nodeID, nodeName, module string) ([]catalogue.Layer, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select key, node is null, value from config where node is null or node = $1`, nodeID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
mesh, node := map[string]any{}, map[string]any{}
|
||||
for rows.Next() {
|
||||
var key string
|
||||
var meshWide bool
|
||||
var raw []byte
|
||||
if err := rows.Scan(&key, &meshWide, &raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var v any
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if meshWide {
|
||||
mesh[key] = v
|
||||
} else {
|
||||
node[key] = v
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(mesh)+len(node) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
// Only the module's preferences, read from its stored definition: a whole catalogue per module per
|
||||
// node would be read for every machine planned.
|
||||
var raw []byte
|
||||
err = i.store.Pool().QueryRow(ctx, `select manifest -> 'preferences' from module where name = $1`, module).Scan(&raw)
|
||||
if errors.Is(err, pgx.ErrNoRows) || len(raw) == 0 || string(raw) == "null" {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var prefs map[string]catalogue.Preference
|
||||
if err := json.Unmarshal(raw, &prefs); err != nil {
|
||||
return nil, fmt.Errorf("%s's preferences cannot be read: %w", module, err)
|
||||
}
|
||||
return catalogue.ConfigLayers(catalogue.Manifest{Module: module, Preferences: prefs}, mesh, node, nodeName), nil
|
||||
}
|
||||
|
||||
// SetModuleValue sets or clears one key of a module's own layer on one node — its module-assignment config —
|
||||
// keeping the rest of the layer, judged as `settings set` judges it, with the layer it replaces kept.
|
||||
// value nil clears the key; the layer goes when no key is left.
|
||||
func (i *Inventory) SetModuleValue(ctx context.Context, nodeName, module, key string, value any, setBy string) (
|
||||
before any, had bool, err error) {
|
||||
if nodeName == "" {
|
||||
return nil, false, errors.New("there is no module layer for the whole mesh: a module's own value is " +
|
||||
"set on one node (novox/hq issue 630); for every node, set mesh config for a predefined key")
|
||||
}
|
||||
layer, has, err := i.Layer(ctx, nodeName, module)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
values := map[string]any{}
|
||||
for k, v := range layer {
|
||||
values[k] = v
|
||||
}
|
||||
before, had = values[key]
|
||||
if value == nil {
|
||||
if !had {
|
||||
return nil, false, nil
|
||||
}
|
||||
delete(values, key)
|
||||
if len(values) == 0 && has {
|
||||
return before, had, i.ClearSettingsBy(ctx, nodeName, module, setBy)
|
||||
}
|
||||
} else {
|
||||
values[key] = value
|
||||
}
|
||||
return before, had, i.SetSettingsBy(ctx, nodeName, module, values, setBy)
|
||||
}
|
||||
|
||||
// ConfigKeysSorted is the keys of a set of config values, sorted, for saying them.
|
||||
func ConfigKeysSorted(values []ConfigValue) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, v := range values {
|
||||
if !seen[v.Key] {
|
||||
seen[v.Key] = true
|
||||
out = append(out, v.Key)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
-- A config value comes from four sources, the last set winning (novox/hq issue 630): the module's
|
||||
-- default, mesh config, node config, and the module's own value on the node. Mesh config and node config
|
||||
-- hold values only for the predefined keys, a fixed list kept in the catalogue (`predefined.json`). An
|
||||
-- assignment is one module on one node: there is no module layer for the whole mesh any more.
|
||||
|
||||
-- The list of predefined keys, as the catalogue's predefined.json last said it. Replaced whole when a
|
||||
-- build of the catalogue carries the file; `repository` and `commit` say which.
|
||||
create table predefined_key (
|
||||
key text primary key,
|
||||
type text not null check (type in ('number', 'string', 'boolean')),
|
||||
why text not null,
|
||||
repository text,
|
||||
commit text,
|
||||
listed_at timestamptz not null default now()
|
||||
);
|
||||
|
||||
-- Mesh config (node null) and node config: one value per predefined key, and per node. Not a foreign key
|
||||
-- to predefined_key: a key the list drops keeps its value, said as reaching nothing, rather than losing it.
|
||||
create table config (
|
||||
key text not null,
|
||||
node uuid references node(id) on delete cascade,
|
||||
value jsonb not null,
|
||||
set_at timestamptz not null default now(),
|
||||
set_by text,
|
||||
why text not null
|
||||
);
|
||||
create unique index config_for_the_mesh on config (key) where node is null;
|
||||
create unique index config_for_a_node on config (key, node) where node is not null;
|
||||
|
||||
-- Every value that was replaced or cleared, with who set it and why, and who replaced it and why.
|
||||
create table config_history (
|
||||
key text not null,
|
||||
node uuid references node(id) on delete cascade,
|
||||
value jsonb not null,
|
||||
set_at timestamptz,
|
||||
set_by text,
|
||||
why text,
|
||||
replaced_at timestamptz not null default now(),
|
||||
-- set · clear
|
||||
replaced_by text not null,
|
||||
replaced_why text
|
||||
);
|
||||
create index config_history_by_key on config_history (key, node, replaced_at desc);
|
||||
|
||||
-- The module layer for the whole mesh goes: each one is laid under the layer of every node the module is
|
||||
-- assigned to — the mesh's values first, the node's over them, maps merged key by key as composition
|
||||
-- merged them — so every machine composes what it did. Every layer replaced, and the mesh-wide layer
|
||||
-- itself, is kept in settings_history (ADR 0217) with the reason `moved-to-nodes`. A mesh-wide layer of
|
||||
-- a module assigned nowhere is kept in the history only: no machine read it.
|
||||
create function pg_temp.config_630_merge(under jsonb, over jsonb) returns jsonb language plpgsql immutable as $$
|
||||
declare
|
||||
merged jsonb;
|
||||
k text;
|
||||
begin
|
||||
if jsonb_typeof(under) is distinct from 'object' or jsonb_typeof(over) is distinct from 'object' then
|
||||
return over;
|
||||
end if;
|
||||
merged := under;
|
||||
for k in select jsonb_object_keys(over) loop
|
||||
if merged ? k then
|
||||
merged := jsonb_set(merged, array[k], pg_temp.config_630_merge(merged -> k, over -> k));
|
||||
else
|
||||
merged := merged || jsonb_build_object(k, over -> k);
|
||||
end if;
|
||||
end loop;
|
||||
return merged;
|
||||
end
|
||||
$$;
|
||||
|
||||
insert into settings_history (node, module, values, set_at, replaced_by, set_by)
|
||||
select t.node, t.module, t.values, t.set_at, 'moved-to-nodes', t.set_by
|
||||
from settings t
|
||||
join settings s on s.module = t.module and s.node is null
|
||||
where t.node is not null
|
||||
and exists (select 1 from assignment a where a.node = t.node and a.module = t.module);
|
||||
|
||||
insert into settings_history (node, module, values, set_at, replaced_by, set_by)
|
||||
select null, module, values, set_at, 'moved-to-nodes', set_by from settings where node is null;
|
||||
|
||||
insert into settings (node, module, values, set_at, set_by)
|
||||
select a.node, s.module, s.values, now(),
|
||||
coalesce(s.set_by || '; ', '') || 'moved from the whole mesh''s layer to this node''s (novox/hq issue 630)'
|
||||
from settings s
|
||||
join assignment a on a.module = s.module
|
||||
where s.node is null
|
||||
on conflict (node, module) where node is not null
|
||||
do update set values = pg_temp.config_630_merge(excluded.values, settings.values),
|
||||
set_at = now(),
|
||||
set_by = coalesce(settings.set_by || '; ', '') ||
|
||||
'the whole mesh''s layer laid under it (novox/hq issue 630)';
|
||||
|
||||
delete from settings where node is null;
|
||||
Reference in New Issue
Block a user