Files
mesh-catalog/modules/betterbird/cmd/betterbird-tools/derive.go
T
jochen 1835f0efad
mesh/merge-gate head 1835f0efad0f merged into trunk 39e7a06e6293 — pass: builds betterbird, sent nowhere; no bus step; 2 wait(s) for a person; every machin…
mesh/repo-check head 1835f0efad0f merged into trunk 39e7a06e6293 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery superseded: a newer head of the same pull request
betterbird: own Betterbird's whole configuration as settings, shown without a credential (hq issue 586, ADR 0329)
The module owned only the accounts' server fields, so identities, signatures,
calendars, address books, filters and tags could differ between desktop nodes
unseen, and the operator's own preferences were shipped in the catalogue.

prefs becomes one entry per preference (a node's layer overrides one key) and the
defaults file keeps no value of the operator's; identities, calendars,
address-books, filters and tags become settings, adopted like accounts, filters
owned whole per account, and what Betterbird rewrites written only while it is
closed. New tools list each kind, derive the layers from every desktop node and
propose one at a time, and show drift; every answer passes a credential check.
2026-10-13 02:03:04 +02:00

694 lines
23 KiB
Go

package main
// The five settings derived from the desktop nodes' profiles and proposed for the operator's Approve, one layer
// at a time (novox/hq ADR 0329 §10, §12).
//
// betterbird_derive asks every desktop node that holds Betterbird for what its profile holds (its own derive with
// here), and lays the answers out per layer: an entry found on every such node goes into the mesh-wide layer, with
// the value of the node the derive was called on, and the answer lists every field another node holds differently,
// which then shows as drift there; an entry found on some nodes only goes into each of those nodes' layers, so
// nothing is added to another node until the operator moves it. With propose it proposes the next layer through
// the controller's settings verb: first the mesh-wide one, then each node's that needs one, each only once no
// proposal of the module's is open, against the layer as it then stands, which it carries whole beside the derived
// kinds (a layer is replaced whole). The ask shows the values in their shape (ADR 0277 §6); the operator reads them
// whole with `settings proposals <id>` at their desk.
import (
"encoding/json"
"errors"
"fmt"
"regexp"
"sort"
"strings"
)
// derivedEntry is one entry a node's profile holds, in the setting's shape. Match is what makes two nodes' entries
// the same one: an identity's account and address, a calendar's or an address book's address, a tag's key, an
// account's name for its filters.
type derivedEntry struct {
Match string `json:"match"`
Name string `json:"name"`
Value json.RawMessage `json:"value"`
}
// nodeDerivation is what one node's profile holds of the five kinds.
type nodeDerivation struct {
Node string `json:"node"`
Kinds map[string][]derivedEntry `json:"kinds"`
// NotDerived names what a setting does not hold, by name and why: Owl's, an ICS calendar, a local address
// book, an address carrying a password, a filter a setting cannot say.
NotDerived []string `json:"not_derived,omitempty"`
}
var nameChars = regexp.MustCompile(`[^a-z0-9]`)
// entryNameFor is a name of the operator's kind for an entry: lower-case letters and digits from its words,
// unique among those used.
func entryNameFor(words string, fallback string, used map[string]bool) string {
base := nameChars.ReplaceAllString(strings.ToLower(words), "")
if base == "" || base[0] < 'a' || base[0] > 'z' {
base = fallback + base
}
if len(base) > 28 {
base = base[:28]
}
name := base
for i := 2; used[name]; i++ {
name = fmt.Sprintf("%s%d", base, i)
}
used[name] = true
return name
}
func raw(v any) json.RawMessage {
b, _ := json.Marshal(v)
return b
}
// deriveHere reads this node's profile into the five kinds.
func (m *Machine) deriveHere(kinds []string) (nodeDerivation, error) {
d := nodeDerivation{Node: nodeName(), Kinds: map[string][]derivedEntry{}}
p, err := m.profile()
if err != nil {
return d, err
}
prefs := mergedPrefs(p.path)
accounts, _, err := m.readAccounts()
if err != nil {
return d, err
}
nameOf := map[string]string{}
serverOf := map[string]string{}
for _, r := range resolveAccounts(accounts, prefs, m.readRecord(p.Dir)) {
if r.How != refused {
nameOf[r.Keys.Account], serverOf[r.name] = r.name, r.Keys.Server
}
}
want := map[string]bool{}
for _, k := range kinds {
want[k] = true
}
// Identities, by their account's name; the default carries no name or address (the setting accounts' own).
identityName := map[string]string{}
used := map[string]bool{}
byAccount := map[string]bool{}
for _, pi := range m.profileIdentities(prefs) {
account, declared := nameOf[pi.AccountKey]
switch {
case pi.Owl:
if pi.Default {
d.NotDerived = append(d.NotDerived, fmt.Sprintf("identities: %s <%s> is Owl's (novox/hq ADR 0316 §13)",
pi.Value.Name, pi.Value.Address))
}
continue
case !declared:
if !byAccount[pi.AccountKey] {
byAccount[pi.AccountKey] = true
d.NotDerived = append(d.NotDerived, fmt.Sprintf("identities: those of the profile's account %s, which the "+
"setting accounts does not hold", pi.AccountKey))
}
continue
}
v := pi.Value
v.Account = account
e := derivedEntry{}
if pi.Default {
v.Address, v.Name = "", ""
e.Match, e.Name = account+"|", entryNameFor(account, "id", used)
} else {
local, _, _ := strings.Cut(v.Address, "@")
e.Match, e.Name = account+"|"+strings.ToLower(v.Address), entryNameFor(account+local, "id", used)
}
identityName[pi.Key] = e.Name
e.Value = raw(v)
if want[kindIdentities] {
d.Kinds[kindIdentities] = append(d.Kinds[kindIdentities], e)
}
}
// Calendars: CalDAV only; Owl's are listed, an ICS one by its name only, a local one by its name (§13).
used = map[string]bool{}
for _, pc := range profileCalendars(prefs) {
if !want[kindCalendars] {
break
}
switch {
case pc.Kind == "owl" || strings.HasPrefix(pc.Kind, "owl"):
d.NotDerived = append(d.NotDerived, fmt.Sprintf("calendars: %s (%s) is Owl's", pc.Value.Name, pc.Value.URL))
continue
case pc.Kind == "ics":
d.NotDerived = append(d.NotDerived, fmt.Sprintf("calendars: %s is subscribed by an ICS address, which is "+
"often itself the secret: listed by its name only, never declared (novox/hq ADR 0329 §13)", pc.Value.Name))
continue
case pc.Kind != "caldav":
d.NotDerived = append(d.NotDerived, fmt.Sprintf("calendars: %s is %s, kept in the profile: its events are never declared",
pc.Value.Name, pc.Kind))
continue
case addressPassword(pc.Value.URL):
d.NotDerived = append(d.NotDerived, fmt.Sprintf("calendars: %s at %s carries a password in its address: refused "+
"(novox/hq ADR 0329 §2)", pc.Value.Name, withoutPassword(pc.Value.URL)))
continue
}
v := pc.Value
v.Invitations = identityName[pc.IdentityKey]
d.Kinds[kindCalendars] = append(d.Kinds[kindCalendars], derivedEntry{Match: strings.ToLower(strings.TrimRight(v.URL, "/")),
Name: entryNameFor(v.Name, "calendar", used), Value: raw(v)})
}
// Address books: CardDAV only.
used = map[string]bool{}
for _, b := range profileAddressBooks(prefs) {
if !want[kindAddressBooks] {
break
}
switch {
case !b.CardDAV:
d.NotDerived = append(d.NotDerived, fmt.Sprintf("address-books: %s is %s: its contacts are never declared", b.Value.Name, b.Kind))
continue
case addressPassword(b.Value.URL):
d.NotDerived = append(d.NotDerived, fmt.Sprintf("address-books: %s at %s carries a password in its address: "+
"refused (novox/hq ADR 0329 §2)", b.Value.Name, withoutPassword(b.Value.URL)))
continue
}
d.Kinds[kindAddressBooks] = append(d.Kinds[kindAddressBooks], derivedEntry{
Match: strings.ToLower(strings.TrimRight(b.Value.URL, "/")), Name: entryNameFor(b.Value.Name, "book", used), Value: raw(b.Value)})
}
// Tags, by their own key.
if want[kindTags] {
for _, t := range profileTags(prefs) {
d.Kinds[kindTags] = append(d.Kinds[kindTags], derivedEntry{Match: t.Key, Name: t.Key, Value: raw(t.Value)})
}
}
// Filters of each declared account whose file holds one or more: an account with none keeps its file as the
// person's until a setting names it.
if want[kindFilters] {
for _, name := range sortedKeys(serverOf) {
dir := m.accountDirectory(p.path, prefs, serverOf[name])
if dir == "" {
continue
}
text, err := readBounded(dir + "/" + filterFileName)
if err != nil {
continue
}
parsed := parseFilters(string(text), folderBase(prefs, serverOf[name]))
for _, why := range parsed.NotDeclarable {
d.NotDerived = append(d.NotDerived, "filters of "+name+": "+why)
}
if len(parsed.Filters) > 0 {
d.Kinds[kindFilters] = append(d.Kinds[kindFilters], derivedEntry{Match: name, Name: name, Value: raw(parsed.Filters)})
}
}
}
return d, nil
}
// difference is one field another node holds differently from the value the mesh-wide layer gets.
type difference struct {
Kind string `json:"kind"`
Entry string `json:"entry"`
Node string `json:"node"`
Field string `json:"field"`
MeshWide any `json:"mesh_wide"`
There any `json:"there"`
}
// DeriveAnswer is what betterbird_derive answers its caller: the values it derived, which ADR 0329 §10 lets it show.
type DeriveAnswer struct {
// Here is this node's own derivation, when asked with here.
Here *nodeDerivation `json:"here,omitempty"`
// From are the desktop nodes derived from, and NotHeard those holding Betterbird that did not answer.
From []string `json:"derived_from,omitempty"`
NotHeard []string `json:"not_heard,omitempty"`
// Layers are the derived values per layer: "mesh-wide", then each node's own.
Layers map[string]map[string]any `json:"layers,omitempty"`
Differences []difference `json:"differences,omitempty"`
NotDerived map[string][]string `json:"not_derived,omitempty"`
// Next is the layer a propose proposes now, or why none.
Next string `json:"next,omitempty"`
Proposed string `json:"proposed,omitempty"`
Note string `json:"note,omitempty"`
}
const meshWide = "mesh-wide"
// Derive derives the kinds asked for (all five when none) from the desktop nodes, and with propose proposes the
// next layer. prefs are values for the setting prefs to carry in the mesh-wide layer (issue 586: the operator's
// setup moved out of the catalogue); layer names the layer to propose instead of the next in order.
func (m *Machine) Derive(kinds []string, here, propose bool, prefs map[string]any, layer string) (DeriveAnswer, error) {
var a DeriveAnswer
if len(kinds) == 0 {
kinds = allKinds
}
for _, k := range kinds {
if !contains(allKinds, k) {
return a, fmt.Errorf("kinds are %s; %q is not one", strings.Join(allKinds, ", "), k)
}
}
if here {
d, err := m.deriveHere(kinds)
if err != nil {
return a, err
}
a.Here = &d
return a, nil
}
self := nodeName()
if self == "" {
return a, errors.New("this runtime does not say which machine it serves (MESH_NODE), so the node derived on is unknown")
}
holders, err := holdersOfBetterbird()
if err != nil {
return a, err
}
derived := map[string]nodeDerivation{}
for _, n := range holders {
if n == self {
d, err := m.deriveHere(kinds)
if err != nil {
return a, fmt.Errorf("this node's profile: %v", err)
}
derived[n] = d
continue
}
d, err := deriveOn(n, kinds)
if err != nil {
a.NotHeard = append(a.NotHeard, n+": "+err.Error())
continue
}
derived[n] = d
}
if _, ok := derived[self]; !ok {
return a, fmt.Errorf("%s does not hold Betterbird as the controller says: derive on a desktop node", self)
}
a.From = sortedKeys(derived)
a.Layers, a.Differences = layOut(self, derived, kinds)
a.NotDerived = map[string][]string{}
for n, d := range derived {
if len(d.NotDerived) > 0 {
a.NotDerived[n] = d.NotDerived
}
}
if len(a.NotDerived) == 0 {
a.NotDerived = nil
}
if len(prefs) > 0 {
a.Layers[meshWide]["prefs"] = prefs
}
a.Note = "the values each layer would get; an entry on every desktop node is mesh-wide with this node's value, one " +
"on some nodes only is in those nodes' own layers. With propose, the next layer is proposed for the operator's " +
"Approve, carrying the layer as it stands beside these kinds; nothing changes until then (novox/hq ADR 0329 §10)"
if !propose {
return a, nil
}
if len(a.NotHeard) > 0 {
return a, fmt.Errorf("nothing was proposed: %d desktop node(s) holding Betterbird did not answer (%s), and a "+
"layer derived without them would put on every node what only some hold; derive again when they are awake",
len(a.NotHeard), strings.Join(a.NotHeard, "; "))
}
if open, err := openProposals(); err != nil {
return a, fmt.Errorf("nothing was proposed: whether a proposal of the module's is open cannot be read: %v", err)
} else if len(open) > 0 {
return a, fmt.Errorf("nothing was proposed: the module's proposal %s waits for the operator; the next layer is "+
"proposed once it is set or declined (novox/hq ADR 0329 §10, ADR 0277 §10)", strings.Join(open, ", "))
}
order := []string{meshWide}
for _, n := range sortedKeys(a.Layers) {
if n != meshWide {
order = append(order, n)
}
}
if layer != "" {
if _, ok := a.Layers[layer]; !ok {
return a, fmt.Errorf("nothing was proposed: layer is %s", strings.Join(order, " or "))
}
order = []string{layer}
}
for _, l := range order {
node := l
if l == meshWide {
node = ""
}
current, err := readLayer(node)
if err != nil {
return a, fmt.Errorf("nothing was proposed: %v", err)
}
next := withKinds(current, a.Layers[l])
if l == meshWide {
if prefs, ok := next["prefs"]; ok {
body, _ := json.Marshal(map[string]any{"prefs": prefs})
if _, err := parseDeclaredPrefs(body); err != nil {
return a, fmt.Errorf("nothing was proposed: %v", err)
}
}
}
if err := proposable(next, l == meshWide); err != nil {
return a, fmt.Errorf("nothing was proposed for %s: %v", l, err)
}
if mustJSON(next) == mustJSON(current) {
continue
}
a.Next = l
said, err := proposeLayer(node, next, true)
a.Proposed = m.maskValue(said)
if err != nil {
return a, err
}
a.Note = "proposed the " + l + " layer for the operator's Approve; the next layer is proposed by calling again " +
"once this one is set or declined"
return a, nil
}
a.Next = "none: every layer already holds what was derived"
return a, nil
}
// proposable checks a layer's derived kinds as the module would read them, so the operator is never asked to set
// a value the module then refuses. A key to adopt names one node's profile, so the mesh-wide layer holds none (§4).
func proposable(layer map[string]any, mesh bool) error {
conf := map[string]any{}
for _, k := range []string{kindIdentities, kindCalendars, kindAddressBooks, kindFilters} {
if v, ok := layer[k]; ok {
conf[k] = v
}
}
var c Configuration
if err := decodeInto([]byte(mustJSON(conf)), configurationFile, &c); err != nil {
return err
}
if v, ok := layer[kindTags]; ok {
if err := decodeInto([]byte(mustJSON(map[string]any{kindTags: v})), tagsFile, &c); err != nil {
return err
}
}
// A node's layer may name an identity the mesh-wide layer holds for a calendar's invitations: that is judged
// where the layers meet, on the node.
var problems []string
if mesh {
for name, id := range c.Identities {
if id != nil && id.Key != "" {
problems = append(problems, "identities "+name+": a key is given in a node's own layer only")
}
}
for name, cal := range c.Calendars {
if cal != nil && cal.Key != "" {
problems = append(problems, "calendars "+name+": a key is given in a node's own layer only")
}
}
for name, ab := range c.AddressBooks {
if ab != nil && ab.Key != "" {
problems = append(problems, "address-books "+name+": a key is given in a node's own layer only")
}
}
}
for _, p := range configurationProblems(c) {
if !strings.Contains(p, "invitations names") {
problems = append(problems, p)
}
}
if len(problems) > 0 {
return errors.New(strings.Join(problems, "; "))
}
return nil
}
// layOut puts each kind's entries into the layers (§10).
func layOut(self string, derived map[string]nodeDerivation, kinds []string) (map[string]map[string]any, []difference) {
layers := map[string]map[string]any{meshWide: {}}
var diffs []difference
nodes := sortedKeys(derived)
for _, kind := range kinds {
type seen struct {
by map[string]derivedEntry
}
groups := map[string]*seen{}
var order []string
for _, n := range append([]string{self}, nodes...) {
for _, e := range derived[n].Kinds[kind] {
g := groups[e.Match]
if g == nil {
g = &seen{by: map[string]derivedEntry{}}
groups[e.Match] = g
order = append(order, e.Match)
}
if _, already := g.by[n]; !already {
g.by[n] = e
}
}
}
mesh := map[string]any{}
used := map[string]bool{}
for _, match := range order {
g := groups[match]
if len(g.by) == len(nodes) {
mine := g.by[self]
name := uniqueName(mine.Name, used)
var v any
_ = json.Unmarshal(mine.Value, &v)
mesh[name] = v
for _, n := range nodes {
if n == self {
continue
}
diffs = append(diffs, fieldDifferences(kind, name, n, mine.Value, g.by[n].Value)...)
}
}
}
for _, match := range order {
g := groups[match]
if len(g.by) == len(nodes) {
continue
}
for _, n := range sortedKeys(g.by) {
e := g.by[n]
if layers[n] == nil {
layers[n] = map[string]any{}
}
nodeKind, _ := layers[n][kind].(map[string]any)
if nodeKind == nil {
nodeKind = map[string]any{}
layers[n][kind] = nodeKind
}
var v any
_ = json.Unmarshal(e.Value, &v)
nodeKind[uniqueName(e.Name, used)] = v
}
}
if len(mesh) > 0 {
layers[meshWide][kind] = mesh
}
}
return layers, diffs
}
func uniqueName(name string, used map[string]bool) string {
n := name
for i := 2; used[n]; i++ {
n = fmt.Sprintf("%s%d", name, i)
}
used[n] = true
return n
}
// fieldDifferences are the fields a node holds differently from the mesh-wide value of one entry.
func fieldDifferences(kind, entry, node string, mesh, there json.RawMessage) []difference {
var a, b map[string]any
if json.Unmarshal(mesh, &a) != nil || json.Unmarshal(there, &b) != nil {
if mustJSON(json.RawMessage(mesh)) != mustJSON(json.RawMessage(there)) {
var x, y any
_ = json.Unmarshal(mesh, &x)
_ = json.Unmarshal(there, &y)
return []difference{{Kind: kind, Entry: entry, Node: node, Field: "the whole list", MeshWide: x, There: y}}
}
return nil
}
var out []difference
for _, f := range sortedKeys(mergeKeys(a, b)) {
if mustJSON(a[f]) != mustJSON(b[f]) {
out = append(out, difference{Kind: kind, Entry: entry, Node: node, Field: f, MeshWide: a[f], There: b[f]})
}
}
return out
}
func mergeKeys(a, b map[string]any) map[string]bool {
out := map[string]bool{}
for k := range a {
out[k] = true
}
for k := range b {
out[k] = true
}
return out
}
// withKinds is a layer with the derived kinds in place of its own, every other key kept.
func withKinds(layer map[string]any, kinds map[string]any) map[string]any {
out := map[string]any{}
for k, v := range layer {
out[k] = v
}
for k, v := range kinds {
if k == "prefs" {
merged := map[string]any{}
if was, ok := out["prefs"].(map[string]any); ok {
for n, x := range was {
merged[n] = x
}
}
for n, x := range v.(map[string]any) {
merged[n] = x
}
out[k] = merged
continue
}
out[k] = v
}
return out
}
func contains(list []string, s string) bool {
for _, x := range list {
if x == s {
return true
}
}
return false
}
// readLayer is the module's layer as the controller holds it: the mesh-wide one for node "", else that node's.
func readLayer(node string) (map[string]any, error) {
args := map[string]any{"module": "betterbird"}
if node != "" {
args["node"] = node
}
raw, err := asker("seat:mesh-controller.settings", args)
if err != nil {
return nil, fmt.Errorf("the controller's settings verb did not answer: %v", err)
}
said, ok := verbSaid(raw)
if !ok {
return nil, fmt.Errorf("the controller did not answer the layer: %s", said)
}
out := map[string]any{}
text := strings.TrimSpace(said)
if !strings.HasPrefix(text, "{") {
return out, nil
}
dec := json.NewDecoder(strings.NewReader(text))
if err := dec.Decode(&out); err != nil {
return nil, fmt.Errorf("the layer the controller answered cannot be read: %v", err)
}
return out, nil
}
// proposeLayer proposes a layer whole for the operator's Approve (ADR 0277); replace says a key the layer had
// and the proposal does not hold is meant to go, which a re-derive needs when an entry left every profile.
func proposeLayer(node string, values map[string]any, replace bool) (string, error) {
body, err := json.Marshal(values)
if err != nil {
return "", err
}
args := map[string]any{"module": "betterbird", "values": string(body), "propose": "true"}
if replace {
args["replace"] = "true"
}
if node != "" {
args["node"] = node
}
raw, err := asker("seat:mesh-controller.settings", args)
if err != nil {
return "", fmt.Errorf("the controller's settings verb did not answer: %v", err)
}
said, ok := verbSaid(raw)
if !ok {
return said, fmt.Errorf("the controller did not take the proposal: %s", said)
}
return said, nil
}
var (
proposalHead = regexp.MustCompile(`^(\S+)\s+(?:set|clear) betterbird on (.+?)\s+\(`)
waitingLine = regexp.MustCompile(`^\s+waiting for the operator until`)
)
// openProposals are the module's proposals that wait for the operator, by id and layer.
func openProposals() ([]string, error) {
raw, err := asker("seat:mesh-controller.settings", map[string]any{"proposals": "true"})
if err != nil {
return nil, err
}
said, ok := verbSaid(raw)
if !ok {
return nil, errors.New(said)
}
var out []string
current := ""
for _, line := range strings.Split(said, "\n") {
if mm := proposalHead.FindStringSubmatch(line); mm != nil {
current = mm[1] + " (" + mm[2] + ")"
continue
}
if !strings.HasPrefix(line, " ") {
current = ""
}
if current != "" && waitingLine.MatchString(line) {
out = append(out, current)
current = ""
}
}
return out, nil
}
// holdersOfBetterbird are the nodes the controller has assigned the module to: the desktop nodes.
func holdersOfBetterbird() ([]string, error) {
raw, err := asker("seat:mesh-controller.nodes", map[string]any{})
if err != nil {
return nil, fmt.Errorf("the controller's nodes verb did not answer: %v", err)
}
said, _ := verbSaid(raw)
var nodes []struct {
Name string `json:"name"`
}
if err := json.Unmarshal([]byte(said), &nodes); err != nil {
return nil, fmt.Errorf("the controller's nodes cannot be read: %v", err)
}
var out []string
for _, n := range nodes {
raw, err := asker("seat:mesh-controller.node", map[string]any{"node": n.Name})
if err != nil {
return nil, fmt.Errorf("the controller's node verb did not answer for %s: %v", n.Name, err)
}
said, _ := verbSaid(raw)
for _, line := range strings.Split(said, "\n") {
if rest, ok := strings.CutPrefix(strings.TrimSpace(line), "assigned:"); ok {
for _, mod := range strings.Split(rest, ",") {
if strings.TrimSpace(mod) == "betterbird" {
out = append(out, n.Name)
}
}
}
}
}
sort.Strings(out)
if len(out) == 0 {
return nil, errors.New("the controller names no node the module is assigned to")
}
return out, nil
}
// deriveOn asks another node's betterbird for what its profile holds.
func deriveOn(node string, kinds []string) (nodeDerivation, error) {
raw, err := asker("betterbird.betterbird_derive@"+node, map[string]any{"here": true, "kinds": kinds})
if err != nil {
return nodeDerivation{}, err
}
said, ok := verbSaid(raw)
if !ok {
return nodeDerivation{}, errors.New(said)
}
var a DeriveAnswer
if err := json.Unmarshal([]byte(said), &a); err != nil || a.Here == nil {
return nodeDerivation{}, fmt.Errorf("its answer cannot be read")
}
a.Here.Node = node
return *a.Here, nil
}