Files
mesh-catalog/modules/betterbird/cmd/betterbird-tools/derive.go
T
jochen c9a711c259
mesh/merge-gate head c9a711c259ed merged into trunk 827b89dbf109 — pass: builds betterbird, sent nowhere; no bus step; 2 wait(s) for a person; every machin…
mesh/repo-check head c9a711c259ed merged into trunk 827b89dbf109 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
betterbird: write the account lists only while the profile is closed, and never propose over an unreadable layer (hq issue 586, review 121)
writeAccountLists checked only for a running Betterbird and wrote without a sync;
it now waits for the profile's lock too and writes atomically. readLayer read any
answer that was not JSON as an empty layer, and a proposal replaces the layer
whole, so a changed wording would have dropped accounts; it now refuses. The
README says the rollout order that leaves no node on Betterbird's defaults.
2026-10-13 02:12:06 +02:00

701 lines
23 KiB
Go

package main
// The five settings derived from the desktop nodes' profiles and proposed for the operator's Approve, one layer
// at a time (novox/hq ADR 0329 §10, §12).
//
// betterbird_derive asks every desktop node that holds Betterbird for what its profile holds (its own derive with
// here), and lays the answers out per layer: an entry found on every such node goes into the mesh-wide layer, with
// the value of the node the derive was called on, and the answer lists every field another node holds differently,
// which then shows as drift there; an entry found on some nodes only goes into each of those nodes' layers, so
// nothing is added to another node until the operator moves it. With propose it proposes the next layer through
// the controller's settings verb: first the mesh-wide one, then each node's that needs one, each only once no
// proposal of the module's is open, against the layer as it then stands, which it carries whole beside the derived
// kinds (a layer is replaced whole). The ask shows the values in their shape (ADR 0277 §6); the operator reads them
// whole with `settings proposals <id>` at their desk.
import (
"encoding/json"
"errors"
"fmt"
"regexp"
"sort"
"strings"
)
// derivedEntry is one entry a node's profile holds, in the setting's shape. Match is what makes two nodes' entries
// the same one: an identity's account and address, a calendar's or an address book's address, a tag's key, an
// account's name for its filters.
type derivedEntry struct {
Match string `json:"match"`
Name string `json:"name"`
Value json.RawMessage `json:"value"`
}
// nodeDerivation is what one node's profile holds of the five kinds.
type nodeDerivation struct {
Node string `json:"node"`
Kinds map[string][]derivedEntry `json:"kinds"`
// NotDerived names what a setting does not hold, by name and why: Owl's, an ICS calendar, a local address
// book, an address carrying a password, a filter a setting cannot say.
NotDerived []string `json:"not_derived,omitempty"`
}
var nameChars = regexp.MustCompile(`[^a-z0-9]`)
// entryNameFor is a name of the operator's kind for an entry: lower-case letters and digits from its words,
// unique among those used.
func entryNameFor(words string, fallback string, used map[string]bool) string {
base := nameChars.ReplaceAllString(strings.ToLower(words), "")
if base == "" || base[0] < 'a' || base[0] > 'z' {
base = fallback + base
}
if len(base) > 28 {
base = base[:28]
}
name := base
for i := 2; used[name]; i++ {
name = fmt.Sprintf("%s%d", base, i)
}
used[name] = true
return name
}
func raw(v any) json.RawMessage {
b, _ := json.Marshal(v)
return b
}
// deriveHere reads this node's profile into the five kinds.
func (m *Machine) deriveHere(kinds []string) (nodeDerivation, error) {
d := nodeDerivation{Node: nodeName(), Kinds: map[string][]derivedEntry{}}
p, err := m.profile()
if err != nil {
return d, err
}
prefs := mergedPrefs(p.path)
accounts, _, err := m.readAccounts()
if err != nil {
return d, err
}
nameOf := map[string]string{}
serverOf := map[string]string{}
for _, r := range resolveAccounts(accounts, prefs, m.readRecord(p.Dir)) {
if r.How != refused {
nameOf[r.Keys.Account], serverOf[r.name] = r.name, r.Keys.Server
}
}
want := map[string]bool{}
for _, k := range kinds {
want[k] = true
}
// Identities, by their account's name; the default carries no name or address (the setting accounts' own).
identityName := map[string]string{}
used := map[string]bool{}
byAccount := map[string]bool{}
for _, pi := range m.profileIdentities(prefs) {
account, declared := nameOf[pi.AccountKey]
switch {
case pi.Owl:
if pi.Default {
d.NotDerived = append(d.NotDerived, fmt.Sprintf("identities: %s <%s> is Owl's (novox/hq ADR 0316 §13)",
pi.Value.Name, pi.Value.Address))
}
continue
case !declared:
if !byAccount[pi.AccountKey] {
byAccount[pi.AccountKey] = true
d.NotDerived = append(d.NotDerived, fmt.Sprintf("identities: those of the profile's account %s, which the "+
"setting accounts does not hold", pi.AccountKey))
}
continue
}
v := pi.Value
v.Account = account
e := derivedEntry{}
if pi.Default {
v.Address, v.Name = "", ""
e.Match, e.Name = account+"|", entryNameFor(account, "id", used)
} else {
local, _, _ := strings.Cut(v.Address, "@")
e.Match, e.Name = account+"|"+strings.ToLower(v.Address), entryNameFor(account+local, "id", used)
}
identityName[pi.Key] = e.Name
e.Value = raw(v)
if want[kindIdentities] {
d.Kinds[kindIdentities] = append(d.Kinds[kindIdentities], e)
}
}
// Calendars: CalDAV only; Owl's are listed, an ICS one by its name only, a local one by its name (§13).
used = map[string]bool{}
for _, pc := range profileCalendars(prefs) {
if !want[kindCalendars] {
break
}
switch {
case pc.Kind == "owl" || strings.HasPrefix(pc.Kind, "owl"):
d.NotDerived = append(d.NotDerived, fmt.Sprintf("calendars: %s (%s) is Owl's", pc.Value.Name, pc.Value.URL))
continue
case pc.Kind == "ics":
d.NotDerived = append(d.NotDerived, fmt.Sprintf("calendars: %s is subscribed by an ICS address, which is "+
"often itself the secret: listed by its name only, never declared (novox/hq ADR 0329 §13)", pc.Value.Name))
continue
case pc.Kind != "caldav":
d.NotDerived = append(d.NotDerived, fmt.Sprintf("calendars: %s is %s, kept in the profile: its events are never declared",
pc.Value.Name, pc.Kind))
continue
case addressPassword(pc.Value.URL):
d.NotDerived = append(d.NotDerived, fmt.Sprintf("calendars: %s at %s carries a password in its address: refused "+
"(novox/hq ADR 0329 §2)", pc.Value.Name, withoutPassword(pc.Value.URL)))
continue
}
v := pc.Value
v.Invitations = identityName[pc.IdentityKey]
d.Kinds[kindCalendars] = append(d.Kinds[kindCalendars], derivedEntry{Match: strings.ToLower(strings.TrimRight(v.URL, "/")),
Name: entryNameFor(v.Name, "calendar", used), Value: raw(v)})
}
// Address books: CardDAV only.
used = map[string]bool{}
for _, b := range profileAddressBooks(prefs) {
if !want[kindAddressBooks] {
break
}
switch {
case !b.CardDAV:
d.NotDerived = append(d.NotDerived, fmt.Sprintf("address-books: %s is %s: its contacts are never declared", b.Value.Name, b.Kind))
continue
case addressPassword(b.Value.URL):
d.NotDerived = append(d.NotDerived, fmt.Sprintf("address-books: %s at %s carries a password in its address: "+
"refused (novox/hq ADR 0329 §2)", b.Value.Name, withoutPassword(b.Value.URL)))
continue
}
d.Kinds[kindAddressBooks] = append(d.Kinds[kindAddressBooks], derivedEntry{
Match: strings.ToLower(strings.TrimRight(b.Value.URL, "/")), Name: entryNameFor(b.Value.Name, "book", used), Value: raw(b.Value)})
}
// Tags, by their own key.
if want[kindTags] {
for _, t := range profileTags(prefs) {
d.Kinds[kindTags] = append(d.Kinds[kindTags], derivedEntry{Match: t.Key, Name: t.Key, Value: raw(t.Value)})
}
}
// Filters of each declared account whose file holds one or more: an account with none keeps its file as the
// person's until a setting names it.
if want[kindFilters] {
for _, name := range sortedKeys(serverOf) {
dir := m.accountDirectory(p.path, prefs, serverOf[name])
if dir == "" {
continue
}
text, err := readBounded(dir + "/" + filterFileName)
if err != nil {
continue
}
parsed := parseFilters(string(text), folderBase(prefs, serverOf[name]))
for _, why := range parsed.NotDeclarable {
d.NotDerived = append(d.NotDerived, "filters of "+name+": "+why)
}
if len(parsed.Filters) > 0 {
d.Kinds[kindFilters] = append(d.Kinds[kindFilters], derivedEntry{Match: name, Name: name, Value: raw(parsed.Filters)})
}
}
}
return d, nil
}
// difference is one field another node holds differently from the value the mesh-wide layer gets.
type difference struct {
Kind string `json:"kind"`
Entry string `json:"entry"`
Node string `json:"node"`
Field string `json:"field"`
MeshWide any `json:"mesh_wide"`
There any `json:"there"`
}
// DeriveAnswer is what betterbird_derive answers its caller: the values it derived, which ADR 0329 §10 lets it show.
type DeriveAnswer struct {
// Here is this node's own derivation, when asked with here.
Here *nodeDerivation `json:"here,omitempty"`
// From are the desktop nodes derived from, and NotHeard those holding Betterbird that did not answer.
From []string `json:"derived_from,omitempty"`
NotHeard []string `json:"not_heard,omitempty"`
// Layers are the derived values per layer: "mesh-wide", then each node's own.
Layers map[string]map[string]any `json:"layers,omitempty"`
Differences []difference `json:"differences,omitempty"`
NotDerived map[string][]string `json:"not_derived,omitempty"`
// Next is the layer a propose proposes now, or why none.
Next string `json:"next,omitempty"`
Proposed string `json:"proposed,omitempty"`
Note string `json:"note,omitempty"`
}
const meshWide = "mesh-wide"
// Derive derives the kinds asked for (all five when none) from the desktop nodes, and with propose proposes the
// next layer. prefs are values for the setting prefs to carry in the mesh-wide layer (issue 586: the operator's
// setup moved out of the catalogue); layer names the layer to propose instead of the next in order.
func (m *Machine) Derive(kinds []string, here, propose bool, prefs map[string]any, layer string) (DeriveAnswer, error) {
var a DeriveAnswer
if len(kinds) == 0 {
kinds = allKinds
}
for _, k := range kinds {
if !contains(allKinds, k) {
return a, fmt.Errorf("kinds are %s; %q is not one", strings.Join(allKinds, ", "), k)
}
}
if here {
d, err := m.deriveHere(kinds)
if err != nil {
return a, err
}
a.Here = &d
return a, nil
}
self := nodeName()
if self == "" {
return a, errors.New("this runtime does not say which machine it serves (MESH_NODE), so the node derived on is unknown")
}
holders, err := holdersOfBetterbird()
if err != nil {
return a, err
}
derived := map[string]nodeDerivation{}
for _, n := range holders {
if n == self {
d, err := m.deriveHere(kinds)
if err != nil {
return a, fmt.Errorf("this node's profile: %v", err)
}
derived[n] = d
continue
}
d, err := deriveOn(n, kinds)
if err != nil {
a.NotHeard = append(a.NotHeard, n+": "+err.Error())
continue
}
derived[n] = d
}
if _, ok := derived[self]; !ok {
return a, fmt.Errorf("%s does not hold Betterbird as the controller says: derive on a desktop node", self)
}
a.From = sortedKeys(derived)
a.Layers, a.Differences = layOut(self, derived, kinds)
a.NotDerived = map[string][]string{}
for n, d := range derived {
if len(d.NotDerived) > 0 {
a.NotDerived[n] = d.NotDerived
}
}
if len(a.NotDerived) == 0 {
a.NotDerived = nil
}
if len(prefs) > 0 {
a.Layers[meshWide]["prefs"] = prefs
}
a.Note = "the values each layer would get; an entry on every desktop node is mesh-wide with this node's value, one " +
"on some nodes only is in those nodes' own layers. With propose, the next layer is proposed for the operator's " +
"Approve, carrying the layer as it stands beside these kinds; nothing changes until then (novox/hq ADR 0329 §10)"
if !propose {
return a, nil
}
if len(a.NotHeard) > 0 {
return a, fmt.Errorf("nothing was proposed: %d desktop node(s) holding Betterbird did not answer (%s), and a "+
"layer derived without them would put on every node what only some hold; derive again when they are awake",
len(a.NotHeard), strings.Join(a.NotHeard, "; "))
}
if open, err := openProposals(); err != nil {
return a, fmt.Errorf("nothing was proposed: whether a proposal of the module's is open cannot be read: %v", err)
} else if len(open) > 0 {
return a, fmt.Errorf("nothing was proposed: the module's proposal %s waits for the operator; the next layer is "+
"proposed once it is set or declined (novox/hq ADR 0329 §10, ADR 0277 §10)", strings.Join(open, ", "))
}
order := []string{meshWide}
for _, n := range sortedKeys(a.Layers) {
if n != meshWide {
order = append(order, n)
}
}
if layer != "" {
if _, ok := a.Layers[layer]; !ok {
return a, fmt.Errorf("nothing was proposed: layer is %s", strings.Join(order, " or "))
}
order = []string{layer}
}
for _, l := range order {
node := l
if l == meshWide {
node = ""
}
current, err := readLayer(node)
if err != nil {
return a, fmt.Errorf("nothing was proposed: %v", err)
}
next := withKinds(current, a.Layers[l])
if l == meshWide {
if prefs, ok := next["prefs"]; ok {
body, _ := json.Marshal(map[string]any{"prefs": prefs})
if _, err := parseDeclaredPrefs(body); err != nil {
return a, fmt.Errorf("nothing was proposed: %v", err)
}
}
}
if err := proposable(next, l == meshWide); err != nil {
return a, fmt.Errorf("nothing was proposed for %s: %v", l, err)
}
if mustJSON(next) == mustJSON(current) {
continue
}
a.Next = l
said, err := proposeLayer(node, next, true)
a.Proposed = m.maskValue(said)
if err != nil {
return a, err
}
a.Note = "proposed the " + l + " layer for the operator's Approve; the next layer is proposed by calling again " +
"once this one is set or declined"
return a, nil
}
a.Next = "none: every layer already holds what was derived"
return a, nil
}
// proposable checks a layer's derived kinds as the module would read them, so the operator is never asked to set
// a value the module then refuses. A key to adopt names one node's profile, so the mesh-wide layer holds none (§4).
func proposable(layer map[string]any, mesh bool) error {
conf := map[string]any{}
for _, k := range []string{kindIdentities, kindCalendars, kindAddressBooks, kindFilters} {
if v, ok := layer[k]; ok {
conf[k] = v
}
}
var c Configuration
if err := decodeInto([]byte(mustJSON(conf)), configurationFile, &c); err != nil {
return err
}
if v, ok := layer[kindTags]; ok {
if err := decodeInto([]byte(mustJSON(map[string]any{kindTags: v})), tagsFile, &c); err != nil {
return err
}
}
// A node's layer may name an identity the mesh-wide layer holds for a calendar's invitations: that is judged
// where the layers meet, on the node.
var problems []string
if mesh {
for name, id := range c.Identities {
if id != nil && id.Key != "" {
problems = append(problems, "identities "+name+": a key is given in a node's own layer only")
}
}
for name, cal := range c.Calendars {
if cal != nil && cal.Key != "" {
problems = append(problems, "calendars "+name+": a key is given in a node's own layer only")
}
}
for name, ab := range c.AddressBooks {
if ab != nil && ab.Key != "" {
problems = append(problems, "address-books "+name+": a key is given in a node's own layer only")
}
}
}
for _, p := range configurationProblems(c) {
if !strings.Contains(p, "invitations names") {
problems = append(problems, p)
}
}
if len(problems) > 0 {
return errors.New(strings.Join(problems, "; "))
}
return nil
}
// layOut puts each kind's entries into the layers (§10).
func layOut(self string, derived map[string]nodeDerivation, kinds []string) (map[string]map[string]any, []difference) {
layers := map[string]map[string]any{meshWide: {}}
var diffs []difference
nodes := sortedKeys(derived)
for _, kind := range kinds {
type seen struct {
by map[string]derivedEntry
}
groups := map[string]*seen{}
var order []string
for _, n := range append([]string{self}, nodes...) {
for _, e := range derived[n].Kinds[kind] {
g := groups[e.Match]
if g == nil {
g = &seen{by: map[string]derivedEntry{}}
groups[e.Match] = g
order = append(order, e.Match)
}
if _, already := g.by[n]; !already {
g.by[n] = e
}
}
}
mesh := map[string]any{}
used := map[string]bool{}
for _, match := range order {
g := groups[match]
if len(g.by) == len(nodes) {
mine := g.by[self]
name := uniqueName(mine.Name, used)
var v any
_ = json.Unmarshal(mine.Value, &v)
mesh[name] = v
for _, n := range nodes {
if n == self {
continue
}
diffs = append(diffs, fieldDifferences(kind, name, n, mine.Value, g.by[n].Value)...)
}
}
}
for _, match := range order {
g := groups[match]
if len(g.by) == len(nodes) {
continue
}
for _, n := range sortedKeys(g.by) {
e := g.by[n]
if layers[n] == nil {
layers[n] = map[string]any{}
}
nodeKind, _ := layers[n][kind].(map[string]any)
if nodeKind == nil {
nodeKind = map[string]any{}
layers[n][kind] = nodeKind
}
var v any
_ = json.Unmarshal(e.Value, &v)
nodeKind[uniqueName(e.Name, used)] = v
}
}
if len(mesh) > 0 {
layers[meshWide][kind] = mesh
}
}
return layers, diffs
}
func uniqueName(name string, used map[string]bool) string {
n := name
for i := 2; used[n]; i++ {
n = fmt.Sprintf("%s%d", name, i)
}
used[n] = true
return n
}
// fieldDifferences are the fields a node holds differently from the mesh-wide value of one entry.
func fieldDifferences(kind, entry, node string, mesh, there json.RawMessage) []difference {
var a, b map[string]any
if json.Unmarshal(mesh, &a) != nil || json.Unmarshal(there, &b) != nil {
if mustJSON(json.RawMessage(mesh)) != mustJSON(json.RawMessage(there)) {
var x, y any
_ = json.Unmarshal(mesh, &x)
_ = json.Unmarshal(there, &y)
return []difference{{Kind: kind, Entry: entry, Node: node, Field: "the whole list", MeshWide: x, There: y}}
}
return nil
}
var out []difference
for _, f := range sortedKeys(mergeKeys(a, b)) {
if mustJSON(a[f]) != mustJSON(b[f]) {
out = append(out, difference{Kind: kind, Entry: entry, Node: node, Field: f, MeshWide: a[f], There: b[f]})
}
}
return out
}
func mergeKeys(a, b map[string]any) map[string]bool {
out := map[string]bool{}
for k := range a {
out[k] = true
}
for k := range b {
out[k] = true
}
return out
}
// withKinds is a layer with the derived kinds in place of its own, every other key kept.
func withKinds(layer map[string]any, kinds map[string]any) map[string]any {
out := map[string]any{}
for k, v := range layer {
out[k] = v
}
for k, v := range kinds {
if k == "prefs" {
merged := map[string]any{}
if was, ok := out["prefs"].(map[string]any); ok {
for n, x := range was {
merged[n] = x
}
}
for n, x := range v.(map[string]any) {
merged[n] = x
}
out[k] = merged
continue
}
out[k] = v
}
return out
}
func contains(list []string, s string) bool {
for _, x := range list {
if x == s {
return true
}
}
return false
}
// readLayer is the module's layer as the controller holds it: the mesh-wide one for node "", else that node's.
func readLayer(node string) (map[string]any, error) {
args := map[string]any{"module": "betterbird"}
if node != "" {
args["node"] = node
}
raw, err := asker("seat:mesh-controller.settings", args)
if err != nil {
return nil, fmt.Errorf("the controller's settings verb did not answer: %v", err)
}
said, ok := verbSaid(raw)
if !ok {
return nil, fmt.Errorf("the controller did not answer the layer: %s", said)
}
// A missing layer is said in exactly these words; anything else must read as the layer's JSON. A proposal
// replaces the layer whole, so an answer read wrongly as empty would drop what the layer holds: refused, loudly.
text := strings.TrimSpace(said)
where := "the whole mesh"
if node != "" {
where = node
}
if strings.HasPrefix(text, "betterbird on "+where+": no layer") {
return map[string]any{}, nil
}
out := map[string]any{}
if err := json.NewDecoder(strings.NewReader(text)).Decode(&out); err != nil || !strings.HasPrefix(text, "{") {
first, _, _ := strings.Cut(text, "\n")
return nil, fmt.Errorf("the controller's answer for the layer of betterbird on %s cannot be read as the layer "+
"(it begins %q), so nothing is proposed: a layer is replaced whole and would lose what it holds", where, first)
}
return out, nil
}
// proposeLayer proposes a layer whole for the operator's Approve (ADR 0277); replace says a key the layer had
// and the proposal does not hold is meant to go, which a re-derive needs when an entry left every profile.
func proposeLayer(node string, values map[string]any, replace bool) (string, error) {
body, err := json.Marshal(values)
if err != nil {
return "", err
}
args := map[string]any{"module": "betterbird", "values": string(body), "propose": "true"}
if replace {
args["replace"] = "true"
}
if node != "" {
args["node"] = node
}
raw, err := asker("seat:mesh-controller.settings", args)
if err != nil {
return "", fmt.Errorf("the controller's settings verb did not answer: %v", err)
}
said, ok := verbSaid(raw)
if !ok {
return said, fmt.Errorf("the controller did not take the proposal: %s", said)
}
return said, nil
}
var (
proposalHead = regexp.MustCompile(`^(\S+)\s+(?:set|clear) betterbird on (.+?)\s+\(`)
waitingLine = regexp.MustCompile(`^\s+waiting for the operator until`)
)
// openProposals are the module's proposals that wait for the operator, by id and layer.
func openProposals() ([]string, error) {
raw, err := asker("seat:mesh-controller.settings", map[string]any{"proposals": "true"})
if err != nil {
return nil, err
}
said, ok := verbSaid(raw)
if !ok {
return nil, errors.New(said)
}
var out []string
current := ""
for _, line := range strings.Split(said, "\n") {
if mm := proposalHead.FindStringSubmatch(line); mm != nil {
current = mm[1] + " (" + mm[2] + ")"
continue
}
if !strings.HasPrefix(line, " ") {
current = ""
}
if current != "" && waitingLine.MatchString(line) {
out = append(out, current)
current = ""
}
}
return out, nil
}
// holdersOfBetterbird are the nodes the controller has assigned the module to: the desktop nodes.
func holdersOfBetterbird() ([]string, error) {
raw, err := asker("seat:mesh-controller.nodes", map[string]any{})
if err != nil {
return nil, fmt.Errorf("the controller's nodes verb did not answer: %v", err)
}
said, _ := verbSaid(raw)
var nodes []struct {
Name string `json:"name"`
}
if err := json.Unmarshal([]byte(said), &nodes); err != nil {
return nil, fmt.Errorf("the controller's nodes cannot be read: %v", err)
}
var out []string
for _, n := range nodes {
raw, err := asker("seat:mesh-controller.node", map[string]any{"node": n.Name})
if err != nil {
return nil, fmt.Errorf("the controller's node verb did not answer for %s: %v", n.Name, err)
}
said, _ := verbSaid(raw)
for _, line := range strings.Split(said, "\n") {
if rest, ok := strings.CutPrefix(strings.TrimSpace(line), "assigned:"); ok {
for _, mod := range strings.Split(rest, ",") {
if strings.TrimSpace(mod) == "betterbird" {
out = append(out, n.Name)
}
}
}
}
}
sort.Strings(out)
if len(out) == 0 {
return nil, errors.New("the controller names no node the module is assigned to")
}
return out, nil
}
// deriveOn asks another node's betterbird for what its profile holds.
func deriveOn(node string, kinds []string) (nodeDerivation, error) {
raw, err := asker("betterbird.betterbird_derive@"+node, map[string]any{"here": true, "kinds": kinds})
if err != nil {
return nodeDerivation{}, err
}
said, ok := verbSaid(raw)
if !ok {
return nodeDerivation{}, errors.New(said)
}
var a DeriveAnswer
if err := json.Unmarshal([]byte(said), &a); err != nil || a.Here == nil {
return nodeDerivation{}, fmt.Errorf("its answer cannot be read")
}
a.Here.Node = node
return *a.Here, nil
}