Design 47's last milestones: the keeper, self-check, healers, hand-acts, provisioner, vault, artifact sweep, disk readings, credential forms (issues 774, 1952, 1953, 1954, 1955)
mesh/delivery-group group feat/774-l3-reland rejected: composed together: the checker's verdict at the members' heads (b48c90fea31f): integration: mesh-self-ch…
mesh/delivery closed: a newer head of the same pull request
mesh/check failed: integration: mesh-self-check/cmd/mesh-self-check.TestTheSelfCheckWholeRunsOnTheBus: the run answered {Run:doctor-1793351737-1 At:20…

One commit with exactly the tree of the reviewed head 35b4509c0a, on main 12c2574c9d, so two compiled programs added and removed on the branch leave no trace in history (issue 1577).
This commit is contained in:
2026-10-30 02:21:51 +01:00
parent 12c2574c9d
commit 35ff67bda0
331 changed files with 24352 additions and 1792 deletions
@@ -275,7 +275,7 @@ func (c *Config) change(call Call) (*SetAnswer, error) {
return nil, fmt.Errorf("%s is refused for %s at %s: %s", show(value), call.Key, level,
strings.Join(problems, "; "))
}
if trusted && call.Caller != ControllerCaller && call.approval == nil {
if trusted && call.approval == nil && !c.atTheTerminal(call) {
return nil, fmt.Errorf("%s is trusted (it decides what runs or who can reach what), so it is set only at "+
"the controller's terminal or on the operator's Approve (novox/hq ADR 0348 §8); %s is not the terminal: "+
"propose it instead, with config's action propose", call.Key, call.Caller)
@@ -293,7 +293,7 @@ func (c *Config) change(call Call) (*SetAnswer, error) {
if !had {
return nil, fmt.Errorf("nothing is set for %s at %s, so there is nothing to clear", call.Key, level)
}
if trusted && call.Caller != ControllerCaller && call.approval == nil {
if trusted && call.approval == nil && !c.atTheTerminal(call) {
return nil, fmt.Errorf("%s is trusted, so it is cleared only at the controller's terminal or on the "+
"operator's Approve (novox/hq ADR 0348 §8); %s is not the terminal: propose the clear instead, with "+
"config's action propose and clear", call.Key, call.Caller)
@@ -326,6 +326,13 @@ func (c *Config) change(call Call) (*SetAnswer, error) {
return answer, nil
}
// atTheTerminal says a call is the controller's relay of its terminal, which holds only until the controller's lease is
// handed away: from the lease write a trusted value is set only through propose and the operator's Approve, whoever
// calls (novox/hq ADR 0565 §13).
func (c *Config) atTheTerminal(call Call) bool {
return call.Caller == ControllerCaller && !c.leaseAway()
}
// declared is the rule a value at a level is held to, and whether it is trusted; refused, naming the declared
// keys, for a key the module or the list of predefined keys does not declare (ADR 0348 §10).
func (c *Config) declared(s Sources, level Level, key string) (rule, bool, error) {
+22 -7
View File
@@ -19,6 +19,8 @@ import (
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/handacts"
)
// Seat is the role this module holds, once for the mesh, and Verb its one verb.
@@ -73,9 +75,21 @@ func (runtimeStore) Read(log string) ([]LogEntry, error) {
func (runtimeStore) Emit(event string, body any) error { return stdio.Emit(event, body) }
// HandAct records a change in the controller's hand-act log (to-be 45 §7) through its seat's verb.
// recorder records into the hand-act log of the seat mesh-hand-acts, falling back to the controller's while the seat
// has no holder that answers and the controller still holds its lease (novox/hq ADR 0565 §11).
var recorder = handacts.Recorder{Ask: stdio.Ask, LeaseAway: func() bool {
e, err := stdio.State(LeaseRead).Get(buscompose.ControllerLeaseKey)
var raw []byte
if e != nil {
raw = e.Value
}
away, _ := buscompose.LeaseHandedAway(raw, e != nil, err)
return away
}}
// HandAct records a change in the hand-act log (to-be 45 §7).
func (runtimeStore) HandAct(what, why, cause string) error {
_, err := stdio.Ask("seat:mesh-controller.hand-act", map[string]any{"what": what, "why": why, "cause": cause})
_, err := recorder.Record(handacts.Request{What: what, Why: why, Cause: cause})
return err
}
@@ -90,7 +104,7 @@ func (runtimeAsks) Publish(subject string, body any, id string) error {
func (runtimeAsks) Record(id string) (*stdio.StateEntry, error) { return stdio.SeatRecord("", id) }
func (runtimeAsks) Kinds() ([]stdio.KindHeld, error) { return stdio.SeatKinds() }
func (runtimeAsks) Warranted(ask string) error {
_, err := stdio.Ask("seat:mesh-controller.warranted", map[string]any{"asker": Module, "ask": ask})
_, err := recorder.RecordWarrant(Module, ask)
return err
}
@@ -118,7 +132,8 @@ func resolving(c *Config) {
}
// A build's manifest is written before a want names the build, so a want, a trunk module or a placement moving is
// every change of what a module declares where (issue 1953); the builds themselves are read, not watched.
for _, read := range []string{PlacementRead, WantsRead, ModulesRead, PredefinedRead} {
for _, read := range []string{PlacementRead, WantsRead, ModulesRead, PredefinedRead, IndexerPredefinedRead,
CandidatesRead, MirrorDeclaredRead, TakeoverState} {
go func(read string) {
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
time.Sleep(wait)
@@ -221,9 +236,9 @@ func hearing(c *Config) {
}
}
// Tools is the seat's verb as the runtime serves it.
// Tools is the seat's verb and the module's own tools of the switch, as the runtime serves them.
func Tools(c *Config) []stdio.Tool {
return []stdio.Tool{{
return append(switchTools(c), stdio.Tool{
Name: Seat + "." + Verb,
Description: "Set, clear, list or read the history of config values (novox/hq ADR 0348). A value is set at " +
"one level: mesh config (no module, no node) or node config (a node) for a predefined key, or a " +
@@ -268,7 +283,7 @@ func Tools(c *Config) []stdio.Tool {
call.Caller = stdio.CallerOf(args)
return c.Do(call)
},
}}
})
}
// callOf reads a call's arguments strictly: an argument it does not know is refused, not ignored.
@@ -0,0 +1,73 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
)
// Every state, log and read the manifest declares has a tool that reads it, and every act the module takes by itself
// a tool that starts it (novox/hq ADR 0565 §15): the verb config's actions, and the switch's tools.
func TestEveryStateAndActHasItsTool(t *testing.T) {
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
if err != nil {
t.Fatal(err)
}
var m struct {
State []string `json:"state"`
Logs []string `json:"logs"`
Reads []string `json:"reads"`
Tools []string `json:"tools"`
Claims []struct {
Name string `json:"name"`
Serves []string `json:"serves"`
} `json:"claims"`
}
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
via := map[string]string{
ValuesState: "config list", ResolvedState: "config list", PredefinedState: "config list",
ProposalsState: "config proposals", PullsState: "config list", KeptState: "config list", HistoryLog: "config history",
DeclaredState: DeclaredTool, TakeoverState: DeclaredTool,
PlacementRead: CompareTool, WantsRead: CompareTool, BuildsRead: CompareTool, ModulesRead: DeclaredTool,
ManifestsRead: DeclaredTool, PredefinedRead: CompareTool, MirrorDeclaredRead: CompareTool,
IndexerPredefinedRead: CompareTool, CandidatesRead: CompareTool, LeaseRead: HandBackTool,
"a resolution": ResyncTool, "the indexer's candidates taken": ResyncTool, "a take-over decided": TakeOverTool,
}
served := map[string]bool{}
for _, tool := range Tools(&Config{}) {
served[tool.Name] = true
}
if len(m.Claims) != 1 || m.Claims[0].Name != Seat || strings.Join(m.Claims[0].Serves, ",") != strings.Join(SeatVerbs, ",") {
t.Errorf("claims %+v, want %s serving %v", m.Claims, Seat, SeatVerbs)
}
for _, verb := range SeatVerbs {
if !served[Seat+"."+verb] {
t.Errorf("the seat's verb %s is not served", verb)
}
}
for _, name := range m.Tools {
if !served[name] {
t.Errorf("the manifest lists the tool %s, which is not served", name)
}
}
for _, list := range [][]string{m.State, m.Logs, m.Reads} {
for _, s := range list {
if _, ok := via[s]; !ok {
t.Errorf("%s has no tool that reads it", s)
}
}
}
for what, tool := range via {
name, _, _ := strings.Cut(tool, " ")
if name == "config" {
name = Seat + "." + Verb
}
if !served[name] {
t.Errorf("%s is read or started by %s, which is not served", what, tool)
}
}
}
@@ -758,6 +758,7 @@ func (c *Config) Decide() ([]string, error) {
for _, r := range ready {
said = append(said, c.decide(r.p, r.w))
}
said = append(said, c.decideTakeovers()...)
c.recordWarrants(now)
return said, nil
}
+29 -24
View File
@@ -10,6 +10,8 @@ import (
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// A config value held for the open pull request that introduces its key (novox/hq ADR 0502, issue 1826).
@@ -61,31 +63,22 @@ type PullDecl struct {
At time.Time `json:"at"`
}
// KeptValue is an approved held value, kept apart from the values mesh-config resolves.
type KeptValue struct {
Pull string `json:"pull"`
Head string `json:"head"`
Digest string `json:"declaration"`
Node string `json:"node"`
Module string `json:"module"`
Key string `json:"key"`
Value json.RawMessage `json:"value"`
By string `json:"by"`
Via string `json:"via"`
Why string `json:"why"`
// Approved is who approved it, as the proposal kept it.
Approved string `json:"approved"`
Proposal string `json:"proposal"`
At time.Time `json:"at"`
}
// KeptValue is an approved held value, kept apart from the values mesh-config resolves: mesh-sdk's layout, which the
// checker and the assigner's composition check read (novox/hq ADR 0565 §5).
type KeptValue = delivery.KeptValue
// keptKey is a held value's key in `kept` (delivery.HeldValueKey), the pull request's key form kept for one that is
// not one.
func keptKey(pull, node, module, key string) string {
if k, err := delivery.HeldValueKey(pull, node, module, key); err == nil {
return k
}
return pullKey(pull) + "." + node + "." + module + "." + key
}
func (k KeptValue) stateKey() string { return keptKey(k.Pull, k.Node, k.Module, k.Key) }
func keptStateKey(k KeptValue) string { return keptKey(k.Pull, k.Node, k.Module, k.Key) }
func (k KeptValue) level() Level { return Level{Kind: LevelModule, Node: k.Node, Module: k.Module} }
func keptLevel(k KeptValue) Level { return Level{Kind: LevelModule, Node: k.Node, Module: k.Module} }
// declDigest is the digest of what a declaration holds a value to: a held value stays only while it is the same.
func declDigest(d Declaration) string {
@@ -151,6 +144,12 @@ func (c *Config) PullDeclares(call Call) (*PullDeclaresAnswer, error) {
return nil, fmt.Errorf("pull-declares is the controller's: it reads a pull request's manifests and tells "+
"mesh-config what the head declares; %s is not the controller", call.Caller)
}
if sw, err := c.source(); err != nil {
return nil, err
} else if sw.Source == SourceIndexer {
return nil, errors.New("pull-declares is refused: since the take-over mesh-config reads a pull request's config " +
"declarations from the indexer's config-candidates (novox/hq ADR 0565 §4)")
}
if _, _, _, err := parsePull(call.Pull); err != nil {
return nil, err
}
@@ -185,6 +184,12 @@ func (c *Config) PullDeclares(call Call) (*PullDeclaresAnswer, error) {
pull := strings.TrimSpace(call.Pull)
c.mu.Lock()
defer c.mu.Unlock()
return c.declarePull(pull, head, declares)
}
// declarePull keeps what a pull request's head declares and judges again the values held for it; from the
// controller's pull-declares, or after the take-over from the indexer's config candidates. The caller holds the lock.
func (c *Config) declarePull(pull, head string, declares map[string]map[string]Declaration) (*PullDeclaresAnswer, error) {
if err := c.Store.Put(PullsState, pullKey(pull), PullDecl{Pull: pull, Head: head, Declares: declares,
At: c.now()}); err != nil {
return nil, fmt.Errorf("the declarations were not kept: %v", err)
@@ -235,7 +240,7 @@ func (c *Config) PullDeclares(call Call) (*PullDeclaresAnswer, error) {
// heldChange records a hold, a drop or a merge's set of a held value in the history, the hand-act log and
// config-changed; what could not be done is answered, not dropped.
func (c *Config) heldChange(kv KeptValue, action, why, approved string) []string {
level := kv.level()
level := keptLevel(kv)
ch := Change{Action: action, Level: &level, Key: kv.Key, Value: kv.Value, By: kv.By, Via: kv.Via, Why: why,
Approved: approved, Pull: kv.Pull, Head: kv.Head, At: c.now(), Moved: []Moved{}}
unsaid := c.record(level.ValueKey(kv.Key), ch)
@@ -252,8 +257,8 @@ func (c *Config) heldChange(kv KeptValue, action, why, approved string) []string
// dropKept removes a held value, saying why.
func (c *Config) dropKept(kv KeptValue, why string) []string {
if err := c.Store.Delete(KeptState, kv.stateKey()); err != nil {
return []string{"the held value " + kv.stateKey() + " was not dropped: " + err.Error()}
if err := c.Store.Delete(KeptState, keptStateKey(kv)); err != nil {
return []string{"the held value " + keptStateKey(kv) + " was not dropped: " + err.Error()}
}
c.logf("[mesh-config] dropped %s of %s on %s held for %s: %s", kv.Key, kv.Module, kv.Node, kv.Pull, why)
return c.heldChange(kv, ActionDrop, "dropped, nothing was set: "+why, "")
@@ -408,7 +413,7 @@ func (c *Config) keep(p Proposal) (string, error) {
kv := KeptValue{Pull: p.Pull, Head: pd.Head, Digest: p.DeclDigest, Node: p.Level.Node, Module: p.Level.Module,
Key: p.Key, Value: p.Value, By: "the operator", Via: p.From, Why: p.Why, Approved: p.Approved, Proposal: p.ID,
At: c.now()}
if err := c.Store.Put(KeptState, kv.stateKey(), kv); err != nil {
if err := c.Store.Put(KeptState, keptStateKey(kv), kv); err != nil {
return "", fmt.Errorf("the value was not kept: %v", err)
}
unsaid := c.heldChange(kv, ActionHold, "held for "+p.Pull+" until it merges: "+p.Why, p.Approved)
@@ -471,7 +476,7 @@ func (c *Config) settlePull(pull string, merged bool, head string) []string {
said = append(said, c.dropKept(kv, kv.Pull+" merged, but "+why)...)
continue
}
vk := kv.level().ValueKey(kv.Key)
vk := keptLevel(kv).ValueKey(kv.Key)
v := Value{Value: compact(kv.Value), By: kv.By, Via: kv.Via, Why: kv.Why + " (held for " + kv.Pull +
", merged)", Approved: kv.Approved, At: c.now()}
if err := c.Store.Put(ValuesState, vk, v); err != nil {
@@ -0,0 +1,2 @@
# What these tests read outside their package, as paths from the repository's root (issue 1936).
modules/mesh-config/module.json
+44 -14
View File
@@ -12,10 +12,19 @@ import (
// Reading the sources, writing the resolved values, and publishing the list of predefined keys.
// sources reads everything a resolution reads, from the bus as it is now: which modules are placed where from the
// seat mesh-placement, and each placed module's declarations from its own manifests as built, never from the
// controller (issue 1953, ADR 0565 §3).
// sources reads everything a resolution reads, from the bus as it is now, from the source the switch names
// (takeover.go): which modules are placed where from the seat mesh-placement, and each placed module's declarations
// from its own manifests as built (issue 1953, ADR 0565 §3), or after a hand-back from the controller's mirror.
func (c *Config) sources() (Sources, error) {
sw, err := c.source()
if err != nil {
return Sources{}, err
}
return c.sourcesFrom(sw.Source)
}
// sourcesFrom reads the sources with declarations from one source: the own (interim and indexer) or the mirror.
func (c *Config) sourcesFrom(source string) (Sources, error) {
s := Sources{Declared: map[string]ModuleDeclarations{}, OnNode: map[string]ModuleDeclarations{},
Predefined: map[string]Published{}, Values: map[string]Value{}}
var problems []string
@@ -25,6 +34,9 @@ func (c *Config) sources() (Sources, error) {
if err := c.readDeclarations(&s); err != nil {
return s, fmt.Errorf("the modules' declarations cannot be read: %w", err)
}
if source == SourceController {
c.mirrorDeclarations(&s)
}
for _, u := range s.Unresolved {
problems = append(problems, fmt.Sprintf("%s on %s: %s", u.Module, u.Node, u.Why))
}
@@ -350,6 +362,10 @@ type Resolution struct {
Moved []Moved `json:"moved"`
Published string `json:"published,omitempty"`
Withheld string `json:"withheld,omitempty"`
// Declared are the modules whose `declared` key was written, and Candidates what the indexer's config candidates
// did after the take-over.
Declared []string `json:"declared,omitempty"`
Candidates []string `json:"candidates,omitempty"`
}
// Reconcile publishes the newest list of predefined keys and brings mesh-config_resolved to what the sources
@@ -374,26 +390,39 @@ func (c *Config) Reconcile() (Resolution, error) {
c.record("resolved", Change{Action: "resolve", Why: "an assignment, a declaration or the list of predefined " +
"keys changed", At: c.now(), Moved: moved})
}
if written, derr := c.writeDeclared(s); derr != nil {
c.logf("[mesh-config] declared is not all written: %v", derr)
} else if len(written) > 0 {
r.Declared = written
}
if sw, serr := c.source(); serr == nil && sw.Source == SourceIndexer {
r.Candidates = c.takeCandidates()
}
return r, err
}
// publish makes mesh-config_predefined hold the list the controller kept, when it is newer than the one published
// (ADR 0348 §7: the newest trunk build wins). **A list dropping a key that still holds a value is withheld**: the
// publish makes mesh-config_predefined hold the list its source keeps — the indexer's after the take-over, the
// controller's mirror otherwise — when it is newer than the one published (ADR 0348 §7: the newest trunk build wins;
// the indexer never keeps an older merge's, so its list is newer when its commit differs). **A list dropping a key that still holds a value is withheld**: the
// list published stays, and why is said in every listing and in the log, because a value for a key no longer on
// the list could neither be resolved nor cleared by name (§7: removing a key still holding a value is refused).
// It answers what it published, empty when nothing changed.
func (c *Config) publish() (string, error) {
e, err := c.Store.Get(PredefinedRead, PredefinedMirrorKey)
sw, err := c.source()
if err != nil {
return "", fmt.Errorf("the controller's list cannot be read: %w", err)
return "", err
}
if e == nil {
read, whose := c.mirrorList, "the controller's list"
if sw.Source == SourceIndexer {
read, whose = c.indexerList, "the indexer's list"
}
list, found, err := read()
if err != nil {
return "", fmt.Errorf("%s cannot be read: %w", whose, err)
}
if !found {
return "", nil // no list kept yet
}
var list PredefinedList
if err := json.Unmarshal(e.Value, &list); err != nil || list.Keys == nil {
return "", fmt.Errorf("the controller's list is not one: %s", e.Value)
}
current := map[string]Published{}
if err := c.each(PredefinedState, func(key string, raw []byte) error {
var p Published
@@ -405,11 +434,12 @@ func (c *Config) publish() (string, error) {
}); err != nil {
return "", err
}
generation := 0
generation, sameCommit := 0, len(current) > 0
for _, p := range current {
generation = max(generation, p.Generation)
sameCommit = sameCommit && p.Commit == list.Commit
}
if len(current) > 0 && generation >= list.Generation {
if len(current) > 0 && (sameCommit || (list.Generation > 0 && generation >= list.Generation)) {
c.withheld = ""
return "", nil
}
@@ -0,0 +1,79 @@
package main
// The tools of the switch (novox/hq ADR 0565 §4, §15): compare, take-over and hand-back, served as the seat's verbs
// and as the module's own tools beside them, the read of `declared` and resync.
import (
"strings"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// The switch's tools.
const (
CompareTool = "config_compare"
TakeOverTool = "config_take_over"
HandBackTool = "config_hand_back"
DeclaredTool = "config_declared"
ResyncTool = "config_resync"
)
// SeatVerbs are the seat's verbs, as the manifest claims them.
var SeatVerbs = []string{Verb, CompareVerb, TakeOverVerb, HandBackVerb}
func switchTools(c *Config) []stdio.Tool {
own := ownSwitchTools(c)
byName := map[string]stdio.Tool{}
for _, t := range own {
byName[t.Name] = t
}
for verb, tool := range map[string]string{CompareVerb: CompareTool, TakeOverVerb: TakeOverTool, HandBackVerb: HandBackTool} {
t := byName[tool]
t.Name = Seat + "." + verb
own = append(own, t)
}
return own
}
func ownSwitchTools(c *Config) []stdio.Tool {
why := map[string]any{"type": "object", "properties": map[string]any{
"why": map[string]any{"type": "string", "description": "why, in a sentence"}}, "required": []string{"why"}}
whyOf := func(a map[string]any) string { s, _ := a["why"].(string); return strings.TrimSpace(s) }
return []stdio.Tool{
{Name: CompareTool,
Description: "Every difference between config's own sources and the controller's records (novox/hq ADR 0565 §4): " +
"each placed module's declarations (declared against the mirror declared-config), the list of predefined " +
"keys (the indexer's against the mirror's), each open pull request's config (the indexer's candidate " +
"against the controller's pull-declares), and every placed module's value on every node resolved both " +
"ways; each with its reason. The take-over is refused while any stands.",
Run: func(map[string]any) (any, error) { return c.Compare() }},
{Name: TakeOverTool,
Description: "Ask the operator's Approve, under mesh-config's own name, to switch config to its own sources: " +
"the indexer's list of predefined keys and pull request config, refusing the controller's pull-declares. " +
"Refused while compare shows a difference; on Approve switched if nothing differs then.",
Input: why,
Run: func(a map[string]any) (any, error) {
return c.TakeOver(Call{Why: whyOf(a), Caller: stdio.CallerOf(a)})
}},
{Name: HandBackTool,
Description: "Return config to the controller's records (its mirrors declared-config and predefined, and " +
"pull-declares) until the controller hands its lease away; gone after it. Recorded as a hand act.",
Input: why,
Run: func(a map[string]any) (any, error) {
return c.HandBack(Call{Why: whyOf(a), Caller: stdio.CallerOf(a)})
}},
{Name: DeclaredTool,
Description: "Each placed module's declarations as config keeps them (its state declared): the manifest and " +
"commit they come from and how many keys, or one module whole; and which source holds now.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "one module, whole"}}},
Run: func(a map[string]any) (any, error) {
m, _ := a["module"].(string)
return c.Declared(strings.TrimSpace(m))
}},
{Name: ResyncTool,
Description: "Read every source again and write what differs: the published list of predefined keys, the " +
"resolved values, declared, and after the take-over the indexer's config candidates.",
Run: func(map[string]any) (any, error) { return c.Resync() }},
}
}
@@ -0,0 +1,771 @@
package main
// Config's records out of the controller (novox/hq ADR 0565 §3, §4, §13, §15, and the progressive insight of
// 2026-10-29): mesh-config keeps each placed module's declarations in its own state `declared`; `compare` lists every
// difference between its own sources and the controller's mirrors — each module's declarations, the predefined list,
// each open pull request's config declarations, and every placed module's resolved value on every node both ways;
// `take-over` asks the operator's Approve under mesh-config's own name and, refused while anything differs, writes the
// key `takeover.switch` naming the indexer; `hand-back` returns to the mirrors until the lease write.
//
// **Which source holds** is the key `switch` alone, read by mesh-config alone:
// - none written (the interim of mesh-catalog #558): declarations from each placed module's own manifests, the list
// of predefined keys from the controller's mirror, a pull request's declarations from the controller's
// pull-declares;
// - `indexer` (taken over): declarations as before, the list from the indexer's `predefined`, a pull request's from
// the indexer's `config-candidates`; pull-declares refused;
// - `controller` (handed back): declarations from the mirror `declared-config`, the list from the mirror,
// pull-declares taken.
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The states and reads of the switch.
const (
// DeclaredState is each placed module's declarations at the trunk, one key per module (ADR 0565 §3).
DeclaredState = "declared"
// TakeoverState holds the key SwitchKey and each take-over ask, `ask.<id>`.
TakeoverState = "takeover"
SwitchKey = "switch"
// MirrorDeclaredRead is the controller's mirror of every registered module's declarations, one key per module:
// what `compare` holds the own source to, and the source after a hand-back.
MirrorDeclaredRead = "mesh-controller.declared-config"
// IndexerPredefinedRead and CandidatesRead are the indexer's states (ADR 0565 §2).
IndexerPredefinedRead = delivery.IndexerModule + "." + delivery.IndexerPredefinedState
CandidatesRead = delivery.IndexerModule + "." + delivery.IndexerConfigCandidatesState
// LeaseRead is the controller's lease: after its write hand-back is gone and the terminal sets nothing (§13).
LeaseRead = buscompose.ControllerLease
)
// The sources the switch names.
const (
SourceInterim = "interim"
SourceIndexer = "indexer"
SourceController = "controller"
)
// TakeOverVerb and HandBackVerb are the seat's verbs.
const (
TakeOverVerb = "take-over"
HandBackVerb = "hand-back"
CompareVerb = "compare"
)
// Switch is the key `takeover.switch`: the source that holds, who chose it, why and on which ask.
type Switch struct {
Source string `json:"source"`
At time.Time `json:"at"`
By string `json:"by"`
Why string `json:"why"`
Ask string `json:"ask,omitempty"`
}
// source is the source that holds now.
func (c *Config) source() (Switch, error) {
e, err := c.Store.Get(TakeoverState, SwitchKey)
if err != nil {
return Switch{}, fmt.Errorf("the take-over's switch cannot be read: %w", err)
}
if e == nil {
return Switch{Source: SourceInterim}, nil
}
var s Switch
if err := json.Unmarshal(e.Value, &s); err != nil || (s.Source != SourceIndexer && s.Source != SourceController) {
return Switch{}, fmt.Errorf("the take-over's switch is not one: %s", e.Value)
}
return s, nil
}
// leaseAway says the controller's lease is handed away (ADR 0561 §3's reading).
func (c *Config) leaseAway() bool {
e, err := c.Store.Get(LeaseRead, buscompose.ControllerLeaseKey)
var raw []byte
if e != nil {
raw = e.Value
}
away, _ := buscompose.LeaseHandedAway(raw, e != nil, err)
return away
}
// ---- declared --------------------------------------------------------------------------------------------
// DeclaredKept is one key of `declared`: a module's declarations as the manifest the indexer's `modules.<module>`
// names holds them, with that manifest's digest and commit (ADR 0565 §3).
type DeclaredKept struct {
Module string `json:"module"`
Manifest string `json:"manifest"`
Commit string `json:"commit,omitempty"`
Config map[string]Declaration `json:"settings"`
}
// trunkKept is a module's declarations at the trunk with the manifest they come from.
func (c *Config) trunkKept(module string) (DeclaredKept, error) {
key, err := delivery.ModulesKey(module)
if err != nil {
return DeclaredKept{}, err
}
e, err := c.Store.Get(ModulesRead, key)
if err != nil || e == nil {
return DeclaredKept{}, fmt.Errorf("the indexer's modules.%s cannot be read (%v)", module, err)
}
var m struct {
Manifest string `json:"manifest"`
Commit string `json:"commit"`
}
if err := json.Unmarshal(e.Value, &m); err != nil || m.Manifest == "" {
return DeclaredKept{}, fmt.Errorf("the indexer's modules.%s names no manifest", module)
}
d, err := c.manifestDeclarations(module, m.Manifest)
if err != nil {
return DeclaredKept{}, err
}
return DeclaredKept{Module: module, Manifest: m.Manifest, Commit: m.Commit, Config: d.Config}, nil
}
// writeDeclared makes `declared` hold each placed module's trunk declarations, writing a key only when it differs
// (ADR 0289 §1) and removing one of a module placed nowhere. It answers the modules written.
func (c *Config) writeDeclared(s Sources) ([]string, error) {
held := map[string][]byte{}
if err := c.each(DeclaredState, func(key string, raw []byte) error {
held[key] = raw
return nil
}); err != nil {
return nil, err
}
var written []string
var problems []error
placed := map[string]bool{}
for _, module := range modulesOf(s.Assignments) {
placed[module] = true
k, err := c.trunkKept(module)
if err != nil {
continue // unresolved, said by the resolution; its key stays as it was
}
raw, _ := json.Marshal(k)
if bytes.Equal(raw, held[module]) {
continue
}
if err := c.Store.Put(DeclaredState, module, k); err != nil {
problems = append(problems, fmt.Errorf("%s: %w", module, err))
continue
}
written = append(written, module)
}
for module := range held {
if !placed[module] {
if err := c.Store.Delete(DeclaredState, module); err != nil {
problems = append(problems, fmt.Errorf("%s: %w", module, err))
}
}
}
return written, errors.Join(problems...)
}
// mirrorDeclarations makes the sources' declarations the controller's mirror's, for every placed module: the source
// after a hand-back, and the other side of compare.
func (c *Config) mirrorDeclarations(s *Sources) {
s.Declared, s.OnNode, s.Unresolved = map[string]ModuleDeclarations{}, map[string]ModuleDeclarations{}, nil
for _, module := range modulesOf(s.Assignments) {
e, err := c.Store.Get(MirrorDeclaredRead, module)
var d ModuleDeclarations
if err == nil && e != nil {
d, err = ParseModuleDeclarations(e.Value)
} else if err == nil {
err = errors.New("the mirror holds no key for it")
}
if err != nil {
for _, a := range s.Assignments {
if a.Module == module {
s.Unresolved = append(s.Unresolved, Unresolved{Node: a.Node, Module: module,
Why: "the controller's mirror declared-config: " + err.Error()})
}
}
continue
}
s.Declared[module] = d
}
}
// ---- the predefined list and the pull requests from the indexer ------------------------------------------
// indexerList is the indexer's predefined list as the mirror's list holds one: the keys, and the commit.
func (c *Config) indexerList() (PredefinedList, bool, error) {
e, err := c.Store.Get(IndexerPredefinedRead, delivery.PredefinedListKey)
if err != nil || e == nil {
return PredefinedList{}, false, err
}
var kept delivery.PredefinedKept
if err := json.Unmarshal(e.Value, &kept); err != nil {
return PredefinedList{}, false, fmt.Errorf("the indexer's predefined list is not one: %w", err)
}
list := PredefinedList{Keys: map[string]PredefinedKey{}, Commit: kept.Commit}
if err := json.Unmarshal(kept.List, &list.Keys); err != nil {
return PredefinedList{}, false, fmt.Errorf("the indexer's predefined list is not predefined.json: %w", err)
}
return list, true, nil
}
// mirrorList is the controller's mirror's list.
func (c *Config) mirrorList() (PredefinedList, bool, error) {
e, err := c.Store.Get(PredefinedRead, PredefinedMirrorKey)
if err != nil || e == nil {
return PredefinedList{}, false, err
}
var list PredefinedList
if err := json.Unmarshal(e.Value, &list); err != nil || list.Keys == nil {
return PredefinedList{}, false, fmt.Errorf("the controller's list is not one: %s", e.Value)
}
return list, true, nil
}
// candidates is every open pull request's config declarations as the indexer keeps them, by pull key.
func (c *Config) candidates() (map[string]PullDecl, error) {
out := map[string]PullDecl{}
err := c.each(CandidatesRead, func(key string, raw []byte) error {
var cand delivery.ConfigCandidate
if err := json.Unmarshal(raw, &cand); err != nil || cand.Pull == "" {
return fmt.Errorf("the indexer's config candidate %s is not one", key)
}
p := PullDecl{Pull: cand.Pull, Head: cand.Head, Declares: map[string]map[string]Declaration{}, At: cand.At}
for module, keys := range cand.Declares {
raw, _ := json.Marshal(keys)
var d map[string]Declaration
if err := json.Unmarshal(raw, &d); err != nil {
return fmt.Errorf("the indexer's config candidate %s declares %s unreadably", key, module)
}
for k, decl := range d {
if !decl.HasDefault() {
decl.Default = nil
}
d[k] = decl
}
p.Declares[module] = d
}
out[pullKey(cand.Pull)] = p
return nil
})
return out, err
}
// takeCandidates is the indexer's config candidates taken in place of pull-declares, after the take-over: each one
// whose head or declarations differ from what `pulls` holds is declared again, its held values judged again (ADR 0502
// §6). The caller holds the lock.
func (c *Config) takeCandidates() []string {
cands, err := c.candidates()
if err != nil {
return []string{"the indexer's config candidates cannot be read: " + err.Error()}
}
var s Sources
if err := c.readPulls(&s); err != nil {
return []string{err.Error()}
}
var said []string
for _, k := range sortedKeys(cands) {
p := cands[k]
if have, ok := s.Pulls[k]; ok && have.Head == p.Head && declaresDigest(have.Declares) == declaresDigest(p.Declares) {
continue
}
a, err := c.declarePull(p.Pull, p.Head, p.Declares)
if err != nil {
said = append(said, fmt.Sprintf("%s from the indexer: %v", p.Pull, err))
continue
}
said = append(said, a.Said+" (from the indexer)")
}
return said
}
// declaresDigest is what a pull request's declarations hold a value to, in one encoding.
func declaresDigest(d map[string]map[string]Declaration) string {
flat := map[string]string{}
for m, keys := range d {
for k, decl := range keys {
flat[m+"."+k] = declDigest(decl)
}
}
raw, _ := json.Marshal(flat)
return string(raw)
}
// ---- compare ----------------------------------------------------------------------------------------------
// Difference is one thing the own sources and the controller's mirrors say differently, and why.
type Difference struct {
What string `json:"what"`
Module string `json:"module,omitempty"`
Node string `json:"node,omitempty"`
Key string `json:"key,omitempty"`
Pull string `json:"pull,omitempty"`
Own json.RawMessage `json:"own,omitempty"`
Mirror json.RawMessage `json:"mirror,omitempty"`
Why string `json:"why"`
}
// The kinds of difference.
const (
DiffDeclared = "declared"
DiffPredefined = "predefined"
DiffPull = "pull"
DiffValue = "value"
)
// Comparison is what compare answers.
type Comparison struct {
Source string `json:"source"`
Differences []Difference `json:"differences"`
Count int `json:"count"`
Said string `json:"said"`
}
// Compare lists every difference (ADR 0565 §4 and its insight). The caller holds no lock.
func (c *Config) Compare() (*Comparison, error) {
c.mu.Lock()
defer c.mu.Unlock()
return c.compare()
}
// compare is Compare; the caller holds the lock.
func (c *Config) compare() (*Comparison, error) {
sw, err := c.source()
if err != nil {
return nil, err
}
own, err := c.sourcesFrom(SourceInterim)
if err != nil {
return nil, fmt.Errorf("mesh-config's own sources cannot be read: %w", err)
}
mirror := own
mirror.Declared, mirror.OnNode, mirror.Unresolved = nil, nil, nil
c.mirrorDeclarations(&mirror)
var diffs []Difference
// Each placed module's declarations: `declared` against the mirror's key for it.
for _, module := range modulesOf(own.Assignments) {
var ownRaw []byte
if e, err := c.Store.Get(DeclaredState, module); err == nil && e != nil {
var k DeclaredKept
if json.Unmarshal(e.Value, &k) == nil {
ownRaw, _ = json.Marshal(normal(k.Config))
}
}
var mirrorRaw []byte
if d, ok := mirror.Declared[module]; ok {
mirrorRaw, _ = json.Marshal(normal(d.Config))
}
switch {
case ownRaw == nil && mirrorRaw == nil:
diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Why: "neither side holds its declarations"})
case ownRaw == nil:
diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Mirror: mirrorRaw,
Why: "the mirror holds its declarations and declared does not"})
case mirrorRaw == nil:
diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Own: ownRaw,
Why: "declared holds its declarations and the mirror does not"})
case !bytes.Equal(ownRaw, mirrorRaw):
diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Own: ownRaw, Mirror: mirrorRaw,
Why: "the trunk's manifest declares it otherwise than the mirror"})
}
}
// The predefined list: the indexer's against the mirror's.
il, iok, ierr := c.indexerList()
ml, mok, merr := c.mirrorList()
switch {
case ierr != nil || merr != nil:
diffs = append(diffs, Difference{What: DiffPredefined, Why: "a list cannot be read: " + errors.Join(ierr, merr).Error()})
case !iok || !mok:
if iok != mok {
diffs = append(diffs, Difference{What: DiffPredefined, Why: fmt.Sprintf("the indexer holds a list: %v; the "+
"mirror holds one: %v", iok, mok)})
}
default:
for _, k := range unionKeys(il.Keys, ml.Keys) {
a, aok := il.Keys[k]
b, bok := ml.Keys[k]
ar, _ := json.Marshal(a)
br, _ := json.Marshal(b)
switch {
case !aok:
diffs = append(diffs, Difference{What: DiffPredefined, Key: k, Mirror: br, Why: "only the mirror's list holds it"})
case !bok:
diffs = append(diffs, Difference{What: DiffPredefined, Key: k, Own: ar, Why: "only the indexer's list holds it"})
case !bytes.Equal(ar, br):
diffs = append(diffs, Difference{What: DiffPredefined, Key: k, Own: ar, Mirror: br,
Why: "the two lists declare it otherwise"})
}
}
}
// Each open pull request: the indexer's candidate against what the controller told by pull-declares.
cands, cerr := c.candidates()
if cerr != nil {
diffs = append(diffs, Difference{What: DiffPull, Why: "the indexer's config candidates cannot be read: " + cerr.Error()})
}
for _, k := range unionKeys(cands, own.Pulls) {
a, aok := cands[k]
b, bok := own.Pulls[k]
switch {
case !aok:
diffs = append(diffs, Difference{What: DiffPull, Pull: b.Pull, Why: "the controller told its declarations and the indexer holds no candidate"})
case !bok:
diffs = append(diffs, Difference{What: DiffPull, Pull: a.Pull, Why: "the indexer holds a candidate and the controller told nothing"})
case a.Head != b.Head:
diffs = append(diffs, Difference{What: DiffPull, Pull: a.Pull, Why: fmt.Sprintf("the indexer's candidate is at %s, "+
"the controller told %s", short(a.Head), short(b.Head))})
case declaresDigest(a.Declares) != declaresDigest(b.Declares):
diffs = append(diffs, Difference{What: DiffPull, Pull: a.Pull, Why: "the two declare its config otherwise"})
}
}
// Every placed module's value on every node, resolved both ways.
ownValues, mirrorValues := Resolve(own), Resolve(mirror)
for _, k := range unionKeys(ownValues, mirrorValues) {
a, aok := ownValues[k]
b, bok := mirrorValues[k]
ar, _ := json.Marshal(a)
br, _ := json.Marshal(b)
node, module, key := a.Node, a.Module, a.Key
if !aok {
node, module, key = b.Node, b.Module, b.Key
}
d := Difference{What: DiffValue, Node: node, Module: module, Key: key}
switch {
case !aok:
d.Mirror, d.Why = br, "only the mirror's declarations resolve it on this node"
case !bok:
d.Own, d.Why = ar, "only mesh-config's own declarations (with this node's build) resolve it"
case !bytes.Equal(ar, br):
d.Own, d.Mirror, d.Why = ar, br, "it resolves to another value or source"
default:
continue
}
diffs = append(diffs, d)
}
for _, u := range mirror.Unresolved {
diffs = append(diffs, Difference{What: DiffValue, Node: u.Node, Module: u.Module, Why: "not resolved from the mirror: " + u.Why})
}
for _, u := range own.Unresolved {
diffs = append(diffs, Difference{What: DiffValue, Node: u.Node, Module: u.Module, Why: "not resolved from its own source: " + u.Why})
}
sort.SliceStable(diffs, func(i, j int) bool {
a, b := diffs[i], diffs[j]
return a.What+a.Module+a.Node+a.Key+a.Pull < b.What+b.Module+b.Node+b.Key+b.Pull
})
if diffs == nil {
diffs = []Difference{}
}
cmp := &Comparison{Source: sw.Source, Differences: diffs, Count: len(diffs)}
if len(diffs) == 0 {
cmp.Said = "nothing differs: the take-over may be asked"
} else {
cmp.Said = fmt.Sprintf("%d difference(s): the take-over is refused while any stands", len(diffs))
}
return cmp, nil
}
// normal is declarations in one encoding: a null default as none.
func normal(d map[string]Declaration) map[string]Declaration {
out := map[string]Declaration{}
for k, decl := range d {
if !decl.HasDefault() {
decl.Default = nil
} else {
decl.Default = compact(decl.Default)
}
out[k] = decl
}
return out
}
func unionKeys[A, B any](a map[string]A, b map[string]B) []string {
seen := map[string]bool{}
for k := range a {
seen[k] = true
}
for k := range b {
seen[k] = true
}
return sortedKeys(seen)
}
// ---- take-over and hand-back ---------------------------------------------------------------------------------
// TakeoverAsk is one take-over asked of the operator, `takeover.ask.<id>`: what the ask is composed from again, and
// where it stands.
type TakeoverAsk struct {
ID string `json:"id"`
From string `json:"from"`
Why string `json:"why"`
At time.Time `json:"at"`
State string `json:"state"`
Outcome string `json:"outcome,omitempty"`
Decided time.Time `json:"decided,omitzero"`
Answered string `json:"answered,omitempty"`
Recorded bool `json:"recorded,omitempty"`
}
func askKey(id string) string { return "ask." + id }
// act is what an answer to the take-over binds.
func (t TakeoverAsk) act(option string) asks.Act {
return asks.Act{"verb": Seat + "." + TakeOverVerb, "option": option, "ask": t.ID, "from": t.From,
"at": t.At.UTC().Format(time.RFC3339Nano)}
}
// ask is the take-over's ask, composed from it alone.
func (t TakeoverAsk) ask() asks.Ask {
why, _ := sayable("Why: " + t.Why)
a := asks.Ask{ID: t.ID, Headline: "Switch config to its own sources",
Explanation: strings.Join([]string{
"mesh-config would read the list of predefined keys and each pull request's config from the delivery's " +
"indexer, no longer from the controller. Nothing any node resolves changes: compare showed no difference.",
point(why)}, "\n"),
Who: asks.Operator, Expires: t.At.Add(ProposalLasts),
OnExpiry: "nothing changes; config keeps reading the controller's records", About: "config.takeover",
Details: "Read what was compared with mesh-config's compare verb, through the mesh MCP server. hand-back returns " +
"to the controller's records until the controller's lease is handed away."}
for _, o := range []struct{ id, label, does string }{
{OptionApprove, "Approve", "config switches to its own sources at once, if nothing differs then"},
{OptionDecline, "Decline", "nothing changes"},
} {
binds, _ := asks.ActDigest(t.act(o.id))
a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds})
}
return a
}
// TakeoverAnswer is what take-over and hand-back answer.
type TakeoverAnswer struct {
Said string `json:"said"`
Ask string `json:"ask,omitempty"`
Source string `json:"source"`
}
// TakeOver asks the operator's Approve to switch to the own sources, refused while compare shows a difference.
func (c *Config) TakeOver(call Call) (*TakeoverAnswer, error) {
if strings.TrimSpace(call.Why) == "" {
return nil, errors.New("say why the take-over is asked now: it is shown to the operator")
}
if call.Caller == "" || call.Caller == Unproven {
return nil, errors.New("the call names no proven caller")
}
if c.Asks == nil {
return nil, errors.New("the operator cannot be asked from here, so nothing is switched")
}
c.mu.Lock()
defer c.mu.Unlock()
sw, err := c.source()
if err != nil {
return nil, err
}
if sw.Source == SourceIndexer {
return &TakeoverAnswer{Source: sw.Source, Said: "config reads its own sources already, since " +
sw.At.UTC().Format(time.RFC3339) + " (" + sw.By + ")"}, nil
}
cmp, err := c.compare()
if err != nil {
return nil, err
}
if cmp.Count > 0 {
first := cmp.Differences[0]
return nil, fmt.Errorf("the take-over is refused: compare shows %d difference(s), the first %s %s%s%s%s: %s. "+
"Nothing was asked", cmp.Count, first.What, first.Module, first.Node, first.Key, first.Pull, first.Why)
}
now := c.now()
var open []string
_ = c.each(TakeoverState, func(key string, raw []byte) error {
var t TakeoverAsk
if strings.HasPrefix(key, "ask.") && json.Unmarshal(raw, &t) == nil && t.State == ProposalWaiting &&
now.Before(t.At.Add(ProposalLasts)) {
open = append(open, t.ID)
}
return nil
})
if len(open) > 0 {
return nil, fmt.Errorf("the take-over is asked already, as %s, and waits for the operator's answer", open[0])
}
t := TakeoverAsk{ID: "takeover-" + newProposalID(), From: call.Caller, Why: strings.TrimSpace(call.Why), At: now.UTC(),
State: ProposalWaiting}
ask := t.ask()
if err := ask.Check(now); err != nil {
return nil, err
}
if err := c.Store.Put(TakeoverState, askKey(t.ID), t); err != nil {
return nil, fmt.Errorf("the take-over could not be kept, so the operator was not asked: %w", err)
}
if err := c.Asks.Publish(asks.AskSubject(Module), ask, ask.ID); err != nil {
t.State, t.Outcome = ProposalUnasked, "the ask could not be published: "+err.Error()
_ = c.Store.Put(TakeoverState, askKey(t.ID), t)
return nil, fmt.Errorf("the operator could not be asked: %w", err)
}
return &TakeoverAnswer{Ask: t.ID, Source: sw.Source, Said: fmt.Sprintf("the operator is asked %q under "+
"mesh-config's own name; nothing changes until they approve, and on Approve config switches if nothing differs "+
"then", ask.Headline)}, nil
}
// decideTakeovers acts on each take-over ask the router has closed. The caller holds the lock.
func (c *Config) decideTakeovers() []string {
var said []string
var waiting []TakeoverAsk
_ = c.each(TakeoverState, func(key string, raw []byte) error {
var t TakeoverAsk
if strings.HasPrefix(key, "ask.") && json.Unmarshal(raw, &t) == nil && t.State == ProposalWaiting {
waiting = append(waiting, t)
}
return nil
})
now := c.now()
for _, t := range waiting {
keep := func(state, outcome string) string {
t.State, t.Outcome, t.Decided = state, outcome, now
_ = c.Store.Put(TakeoverState, askKey(t.ID), t)
return "take-over " + t.ID + ": " + outcome
}
e, err := c.Asks.Record(t.ID)
if err != nil || e == nil {
if err == nil && now.After(t.At.Add(ProposalLasts+time.Hour)) {
said = append(said, keep(ProposalEnded, "the router keeps no record of its ask; nothing was switched"))
}
continue
}
var rec routerRecord
if json.Unmarshal(e.Value, &rec) != nil || rec.State == "open" || rec.Warrant == nil {
continue
}
w := *rec.Warrant
if w.Asker != Module || w.Ask != t.ID {
said = append(said, keep(ProposalRefused, "the router's record is for another ask; nothing was switched"))
continue
}
if w.Outcome != asks.OutcomeChosen {
said = append(said, keep(ProposalEnded, "the ask "+string(w.Outcome)+"; nothing was switched"))
continue
}
option, err := w.For(Module, t.ask())
if err == nil {
err = option.Performs(t.act(option.ID))
}
if err != nil {
said = append(said, keep(ProposalRefused, "the answer is refused: "+err.Error()))
continue
}
t.Answered = w.Says()
if option.ID != OptionApprove {
said = append(said, keep(ProposalDeclined, "declined by the operator; nothing was switched"))
continue
}
cmp, err := c.compare()
switch {
case err != nil:
said = append(said, keep(ProposalRefused, "approved, and compare could not be read, so nothing was switched: "+err.Error()))
continue
case cmp.Count > 0:
said = append(said, keep(ProposalRefused, fmt.Sprintf("approved, and refused: compare shows %d difference(s) "+
"now; nothing was switched, ask again once it is empty", cmp.Count)))
continue
}
sw := Switch{Source: SourceIndexer, At: now.UTC(), By: approvedTakeover(w), Why: t.Why, Ask: t.ID}
if err := c.Store.Put(TakeoverState, SwitchKey, sw); err != nil {
said = append(said, fmt.Sprintf("take-over %s: approved, and the switch could not be written (%v); read again "+
"on the next look", t.ID, err))
continue
}
if err := c.Asks.Warranted(t.ID); err == nil {
t.Recorded = true
}
said = append(said, keep(ProposalApproved, "approved by the operator: config reads its own sources from now"))
}
return said
}
func approvedTakeover(w asks.Warrant) string {
if w.By == nil {
return "the operator"
}
return fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity)
}
// HandBack returns to the controller's records until the controller's lease is handed away.
func (c *Config) HandBack(call Call) (*TakeoverAnswer, error) {
if strings.TrimSpace(call.Why) == "" {
return nil, errors.New("say why config hands back: it is kept with the switch")
}
if call.Caller == "" || call.Caller == Unproven {
return nil, errors.New("the call names no proven caller")
}
c.mu.Lock()
defer c.mu.Unlock()
if c.leaseAway() {
return nil, errors.New("hand-back is gone: the controller handed its lease away, and its records end with it " +
"(novox/hq ADR 0565 §4)")
}
sw, err := c.source()
if err != nil {
return nil, err
}
if sw.Source == SourceController {
return &TakeoverAnswer{Source: sw.Source, Said: "config reads the controller's records already"}, nil
}
now := c.now()
back := Switch{Source: SourceController, At: now.UTC(), By: call.Caller, Why: strings.TrimSpace(call.Why)}
if err := c.Store.Put(TakeoverState, SwitchKey, back); err != nil {
return nil, fmt.Errorf("the switch could not be written: %w", err)
}
answer := &TakeoverAnswer{Source: back.Source, Said: "config reads the controller's records again: its mirrors " +
"declared-config and predefined, and pull-declares"}
if err := c.Store.HandAct("handed config back to the controller's records", back.Why, "config-hand-back"); err != nil {
answer.Said += "; the hand-act log does not hold it: " + err.Error()
}
return answer, nil
}
// Declared is the read of `declared`: every module's key, or one whole.
func (c *Config) Declared(module string) (any, error) {
if module != "" {
e, err := c.Store.Get(DeclaredState, module)
if err != nil {
return nil, err
}
if e == nil {
return nil, fmt.Errorf("declared holds no %s: it is placed nowhere, or its manifest cannot be read", module)
}
return json.RawMessage(e.Value), nil
}
rows := map[string]any{}
err := c.each(DeclaredState, func(key string, raw []byte) error {
var k DeclaredKept
if json.Unmarshal(raw, &k) == nil {
rows[key] = map[string]any{"manifest": k.Manifest, "commit": k.Commit, "keys": len(k.Config)}
}
return nil
})
sw, _ := c.source()
return map[string]any{"declared": rows, "count": len(rows), "source": sw}, err
}
// Resync reads every source again and writes what differs: the published list, the resolved values, `declared`, and
// after the take-over the indexer's config candidates (ADR 0565 §15).
func (c *Config) Resync() (any, error) {
r, err := c.Reconcile()
if err != nil {
return r, err
}
c.mu.Lock()
defer c.mu.Unlock()
out := map[string]any{"resolution": r}
if sw, err := c.source(); err == nil && sw.Source == SourceIndexer {
out["candidates"] = c.takeCandidates()
}
return out, nil
}
@@ -0,0 +1,238 @@
package main
import (
"encoding/json"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The trunk's manifests aMesh declares, as the controller's mirror declared-config holds the same declarations.
const (
claudeDeclared = `{"module":"claude-code","settings":{
"managed":{"kind":"","default":{},"why":"","type":"object","description":"managed settings","trusted":true,"predefined":"agent-managed-settings"}}}`
oldDeclared = `{"module":"old","settings":{"workers":{"kind":"preference","default":4,"why":"two per core"}}}`
)
// anAgreeingMesh is aProposingMesh whose controller mirrors say what mesh-config's own sources say: the mirror's
// declarations of every placed module, and the indexer's predefined list equal to the mirror's.
func anAgreeingMesh(t *testing.T) (*Config, *memStore, *standIn) {
t.Helper()
c, m, r := aProposingMesh(t)
m.mirror(t, MirrorDeclaredRead, "dunst", dunstDeclared)
m.mirror(t, MirrorDeclaredRead, "claude-code", claudeDeclared)
m.mirror(t, MirrorDeclaredRead, "old", oldDeclared)
indexerList(t, m, "aaaaaaaaaaaa", `{"agent-managed-settings":{"type":"object","description":"managed settings","trusted":true},
"font-size":{"type":"number","description":"text size, in points","minimum":6,"maximum":72}}`)
if _, err := c.Reconcile(); err != nil {
t.Fatal(err)
}
return c, m, r
}
// indexerList is the indexer's `predefined.list` at a commit.
func indexerList(t *testing.T, m *memStore, commit, list string) {
t.Helper()
raw, _ := json.Marshal(delivery.PredefinedKept{V: 1, Commit: commit, List: json.RawMessage(list)})
m.mirror(t, IndexerPredefinedRead, delivery.PredefinedListKey, string(raw))
}
// candidate is the indexer's config candidate of a pull request.
func candidate(t *testing.T, m *memStore, pull, head, declares string) {
t.Helper()
key, err := delivery.ConfigCandidateKey(pull)
if err != nil {
t.Fatal(err)
}
m.mirror(t, CandidatesRead, key, `{"v":1,"pull":"`+pull+`","head":"`+head+`","declares":`+declares+`}`)
}
func kinds(cmp *Comparison) []string {
var out []string
for _, d := range cmp.Differences {
k := d.What + ":" + d.Module + d.Pull + d.Key
if d.Node != "" {
k += "@" + d.Node
}
if !slices.Contains(out, k) {
out = append(out, k)
}
}
return out
}
// compare lists every difference with its reason — a module's declarations, the predefined list, a pull request's
// config, and each value a node resolves both ways — and the take-over is refused while one stands (ADR 0565 §4 and
// its insight of 2026-10-29; issue 1953 L5-C4).
func TestCompareListsEveryDifferenceAndTheTakeOverWaitsForNone(t *testing.T) {
cases := []struct {
name string
change func(t *testing.T, m *memStore)
want []string
}{
{"all agree", func(*testing.T, *memStore) {}, nil},
{"the mirror declares a default otherwise", func(t *testing.T, m *memStore) {
m.mirror(t, MirrorDeclaredRead, "dunst", strings.Replace(dunstDeclared, `"default":300`, `"default":400`, 1))
}, []string{"declared:dunst", "value:dunstwidth@g14", "value:dunstwidth@novox"}},
{"the mirror lacks a placed module", func(t *testing.T, m *memStore) { _ = m.Delete(MirrorDeclaredRead, "old") },
[]string{"declared:old", "value:old@g14", "value:oldworkers@g14"}},
{"a node's build adds a key the mirror lacks", func(t *testing.T, m *memStore) {
m.want(t, "g14", "dunst", "src2:"+strings.Repeat("ab", 32), strings.Replace(dunstDeclared, `"width"`, `"height":{"type":"number","default":5},"width"`, 1))
}, []string{"value:dunstheight@g14", "value:dunstheight@novox"}}, // a key only a build declares resolves on every node (#558)
{"the lists differ", func(t *testing.T, m *memStore) {
indexerList(t, m, "bbbbbbbbbbbb", `{"font-size":{"type":"number","description":"text size, in points","minimum":6,"maximum":72}}`)
}, []string{"predefined:agent-managed-settings"}},
{"a pull request only the indexer holds", func(t *testing.T, m *memStore) {
candidate(t, m, "novox/mesh-catalog#9", "h1", `{"dunst":{"gap":{"kind":"","type":"number","default":2}}}`)
}, []string{"pull:novox/mesh-catalog#9"}},
}
for _, tc := range cases {
c, m, r := anAgreeingMesh(t)
tc.change(t, m)
cmp, err := c.Compare()
if err != nil {
t.Fatalf("%s: %v", tc.name, err)
}
if got := kinds(cmp); !slices.Equal(got, tc.want) {
t.Errorf("%s: compare lists %v, want %v (%+v)", tc.name, got, tc.want, cmp.Differences)
}
for _, d := range cmp.Differences {
if d.Why == "" {
t.Errorf("%s: a difference without its reason: %+v", tc.name, d)
}
}
a, err := c.TakeOver(Call{Why: "compare is empty", Caller: "person.jochen"})
if len(tc.want) > 0 {
if err == nil || !strings.Contains(err.Error(), "refused") || len(r.subjects) != 0 {
t.Errorf("%s: the take-over was asked while compare differs: %v", tc.name, err)
}
continue
}
if err != nil || a.Ask == "" || len(r.subjects) != 1 {
t.Errorf("%s: the take-over was not asked: %v", tc.name, err)
}
}
}
// The operator's Approve switches config to the indexer: pull-declares is refused, the list and the pull requests'
// config come from the indexer; hand-back returns to the mirrors until the lease write and is gone after it; from the
// lease write a trusted value is set only on an Approve; declared is written once per manifest (ADR 0565 §3, §4, §13).
func TestTheTakeOverAndTheHandBack(t *testing.T) {
c, m, r := anAgreeingMesh(t)
steps := []struct {
name string
do func() (string, error)
want string // what the answer or the error says
}{
{"declared is written when it differs, and once", func() (string, error) {
_ = m.Delete(DeclaredState, "dunst")
first, err := c.Reconcile()
again, _ := c.Reconcile()
return strings.Join(first.Declared, ",") + "|" + strings.Join(again.Declared, ","), err
}, "dunst|"},
{"pull-declares taken before the take-over", func() (string, error) {
a, err := c.PullDeclares(Call{Caller: ControllerCaller, Pull: "novox/mesh-catalog#7", Head: "h0",
Declares: map[string]any{}})
return answered(a), err
}, "keeps what"},
{"the take-over is asked", func() (string, error) {
_ = m.Delete(PullsState, "novox.mesh-catalog.7") // compare holds the told pull request against no candidate
a, err := c.TakeOver(Call{Why: "the release is delivered", Caller: "person.jochen"})
return answered(a), err
}, "is asked"},
{"the operator approves: switched to the indexer", func() (string, error) {
r.choose(t, r.lastAsk(t), OptionApprove, nil)
out, err := c.Decide()
return strings.Join(out, "; "), err
}, "reads its own sources from now"},
{"pull-declares refused after it, naming the indexer", func() (string, error) {
_, err := c.PullDeclares(Call{Caller: ControllerCaller, Pull: "novox/mesh-catalog#8", Head: "h1",
Declares: map[string]any{}})
return "", err
}, "config-candidates"},
{"the list comes from the indexer", func() (string, error) {
indexerList(t, m, "cccccccccccc", `{"font-size":{"type":"number","description":"text size","minimum":6,"maximum":72},
"agent-managed-settings":{"type":"object","description":"managed settings","trusted":true},
"theme":{"type":"string","description":"the desktop's theme"}}`)
res, err := c.Reconcile()
return res.Published, err
}, "theme"},
{"a pull request's config comes from the indexer", func() (string, error) {
candidate(t, m, "novox/mesh-catalog#9", "h9", `{"dunst":{"gap":{"kind":"","type":"number","default":2}}}`)
res, err := c.Reconcile()
return strings.Join(res.Candidates, "; "), err
}, "from the indexer"},
{"hand-back returns to the mirrors", func() (string, error) {
a, err := c.HandBack(Call{Why: "a difference was found after the switch", Caller: "person.jochen"})
return answered(a), err
}, "controller's records again"},
{"after it, declarations resolve from the mirror", func() (string, error) {
m.mirror(t, MirrorDeclaredRead, "dunst", strings.Replace(dunstDeclared, `"default":300`, `"default":400`, 1))
_, err := c.Reconcile()
return string(m.resolved(t, "g14", "dunst", "width").Value), err
}, "400"},
{"hand-back is gone after the lease write", func() (string, error) {
m.mirror(t, LeaseRead, "holder", `{"handed-to":"the operator's hand-over"}`)
_, err := c.HandBack(Call{Why: "again", Caller: "person.jochen"})
return "", err
}, "gone"},
{"the terminal sets no trusted value after the lease write", func() (string, error) {
_, err := c.Change(Call{Action: "set", Key: "managed", Module: "claude-code", Node: "g14", Value: map[string]any{},
Why: "x", Caller: ControllerCaller, By: "person.jochen"})
return "", err
}, "propose"},
}
for _, s := range steps {
got, err := s.do()
text := got
if err != nil {
text = err.Error()
}
if !strings.Contains(text, s.want) {
t.Errorf("%s: said %q, want %q", s.name, text, s.want)
}
}
if !slices.Contains(m.handActs, "handed config back to the controller's records | a difference was found after the switch | config-hand-back") {
t.Errorf("the hand-back is not recorded: %v", m.handActs)
}
if e, _ := m.Get(DeclaredState, "dunst"); e == nil || !strings.Contains(string(e.Value), `"manifest":"`) || !strings.Contains(string(e.Value), `"commit":"c1"`) {
t.Errorf("declared holds no dunst with its manifest: %v", e)
}
// An Approve that finds compare differing at the answer switches nothing.
c2, m2, r2 := anAgreeingMesh(t)
if _, err := c2.TakeOver(Call{Why: "now", Caller: "person.jochen"}); err != nil {
t.Fatal(err)
}
m2.mirror(t, MirrorDeclaredRead, "dunst", strings.Replace(dunstDeclared, `"default":300`, `"default":400`, 1))
r2.choose(t, r2.lastAsk(t), OptionApprove, nil)
out, _ := c2.Decide()
if sw, _ := c2.source(); sw.Source != SourceInterim || !strings.Contains(strings.Join(out, ""), "difference") {
t.Errorf("an Approve while compare differs switched: %s, %v", sw.Source, out)
}
}
// lastAsk is the newest ask the router took.
func (r *standIn) lastAsk(t *testing.T) string {
t.Helper()
r.mu.Lock()
defer r.mu.Unlock()
var newest string
var at time.Time
for id, a := range r.asks {
if newest == "" || a.Expires.After(at) {
newest, at = id, a.Expires
}
}
if newest == "" {
t.Fatal("the router took no ask")
}
return newest
}
func answered(a any) string {
raw, _ := json.Marshal(a)
return string(raw)
}