Merge branch 'feat/mail-hardening' into mh-711
This commit is contained in:
jochen
@@ -0,0 +1,96 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0377 §3: a move under a module's own `record` rolls out when it recreates none of its containers
|
||||
// and moves none of its data, judged from the two builds' manifests.
|
||||
func TestWhatAMoveUnderRecordHoldsFor(t *testing.T) {
|
||||
base := func() Manifest {
|
||||
raw := `{"module":"mailu","tools":["mailu_status"],
|
||||
"requires":["postgres-database"],
|
||||
"data":{"own":[{"id":"mail","path":"${dir:data-mail}","class":"valuable"}]},
|
||||
"bundles":[{"name":"tools","source":"store/mailu-tools@sha256:aaa","digest":"sha256:aaa"}],
|
||||
"resources":[
|
||||
{"id":"state","type":"directory"},
|
||||
{"id":"mesh-state","type":"directory"},
|
||||
{"id":"data-mail","type":"directory"},
|
||||
{"id":"config-env","type":"file","path":"${dir:state}/mailu.env","content":"A=1"},
|
||||
{"id":"runtime-config","type":"file","path":"${dir:mesh-state}/config.json","content":"{}"},
|
||||
{"id":"inbox-note","type":"file","path":"${dir:data-mail}/README","content":"x"},
|
||||
{"id":"imap","type":"container","image":"registry/imap@sha256:1","volumes":["${dir:data-mail}:/mail"],
|
||||
"env-file":["${dir:state}/mailu.env"]},
|
||||
{"id":"watcher","type":"process","command":"watch"}]}`
|
||||
var m Manifest
|
||||
if err := json.Unmarshal([]byte(raw), &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
resource := func(m *Manifest, id string) map[string]any {
|
||||
for _, r := range m.Resources {
|
||||
if r["id"] == id {
|
||||
return r
|
||||
}
|
||||
}
|
||||
t.Fatalf("no resource %s", id)
|
||||
return nil
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
change func(m *Manifest)
|
||||
holds string // empty: it rolls out
|
||||
}{
|
||||
{"nothing", func(*Manifest) {}, ""},
|
||||
{"a tools bundle", func(m *Manifest) {
|
||||
m.Bundles[0].Digest, m.Bundles[0].Source = "sha256:bbb", "store/mailu-tools@sha256:bbb"
|
||||
}, ""},
|
||||
{"the tool list", func(m *Manifest) { m.Tools = append(m.Tools, "mailu_queue") }, ""},
|
||||
{"a file no container mounts", func(m *Manifest) { resource(m, "runtime-config")["content"] = `{"a":1}` }, ""},
|
||||
{"a container's image", func(m *Manifest) { resource(m, "imap")["image"] = "registry/imap@sha256:2" }, "recreates 1 of mailu's 1"},
|
||||
{"a container's declaration", func(m *Manifest) { resource(m, "imap")["health"] = map[string]any{"kind": "runtime"} }, "recreates 1"},
|
||||
{"a container added", func(m *Manifest) {
|
||||
m.Resources = append(m.Resources, map[string]any{"id": "smtp", "type": "container", "image": "x"})
|
||||
}, "recreates"},
|
||||
{"a mounted file", func(m *Manifest) { resource(m, "config-env")["content"] = "A=2" }, "which its container imap mounts"},
|
||||
{"a file within declared data", func(m *Manifest) { resource(m, "inbox-note")["content"] = "y" }, "which its container imap mounts"},
|
||||
{"a directory holding declared data", func(m *Manifest) { resource(m, "data-mail")["mode"] = "0700" }, "data-mail"},
|
||||
{"the data declaration", func(m *Manifest) { m.Data.Own[0].Class = ClassIrreplaceable }, "data declaration"},
|
||||
{"a provider", func(m *Manifest) { m.Requires = append(m.Requires, "s3") }, "what it requires"},
|
||||
{"what it provides", func(m *Manifest) { m.Provides = append(m.Provides, Offer{Name: "smtp"}) }, "what it provides"},
|
||||
{"a process of a module with data", func(m *Manifest) { resource(m, "watcher")["command"] = "watch2" }, "long-running process watcher"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
from, to := base(), base()
|
||||
c.change(&to)
|
||||
got := Holds(from, to)
|
||||
switch {
|
||||
case c.holds == "" && got != "":
|
||||
t.Fatalf("held: %s", got)
|
||||
case c.holds != "" && !strings.Contains(got, c.holds):
|
||||
t.Fatalf("holds %q; want it to say %q", got, c.holds)
|
||||
}
|
||||
})
|
||||
}
|
||||
// A data directory nothing mounts still holds its data.
|
||||
from, to := base(), base()
|
||||
from.Data.Own = append(from.Data.Own, DataItem{ID: "archive", Path: "${dir:mesh-state}/archive", Class: "valuable"})
|
||||
to.Data = &Data{Own: append([]DataItem(nil), from.Data.Own...)}
|
||||
resource(&to, "runtime-config")["path"] = "${dir:mesh-state}/archive/config.json"
|
||||
resource(&from, "runtime-config")["path"] = "${dir:mesh-state}/archive/config.json"
|
||||
resource(&to, "runtime-config")["content"] = "new"
|
||||
if got := Holds(from, to); !strings.Contains(got, "lies within its data archive") {
|
||||
t.Fatalf("a file within data nothing mounts: %q", got)
|
||||
}
|
||||
// A process of a module with no data of its own may change.
|
||||
from, to = base(), base()
|
||||
from.Data, to.Data = nil, nil
|
||||
resource(&to, "watcher")["command"] = "watch2"
|
||||
if got := Holds(from, to); got != "" {
|
||||
t.Fatalf("a process of a module with no data held: %s", got)
|
||||
}
|
||||
}
|
||||
@@ -109,3 +109,218 @@ func sameDeclaration(a, b any) bool {
|
||||
_ = json.Unmarshal(rb, &nb)
|
||||
return reflect.DeepEqual(na, nb)
|
||||
}
|
||||
|
||||
// What a move under a module's own `record` may do without a person (novox/hq ADR 0377).
|
||||
//
|
||||
// A module records because its manifest says so or because it keeps irreplaceable data: mail, whose containers
|
||||
// recreated together interrupt everyone's mail, and the database server, whose restart drops every consumer. That
|
||||
// hold also held the moves that interrupt nothing — on 2026-10-15 the database server moved with only its tools
|
||||
// bundle changed, its walk said it "recreates none of its containers", and the machine stayed behind until a
|
||||
// person pushed. So such a move rolls out like any other, and only a move that recreates a container or touches
|
||||
// data still waits.
|
||||
|
||||
// Holds is why a move from one build's manifest to another's still waits for a person under a module's own
|
||||
// `record` (ADR 0377 §3), empty when it recreates none of its containers and moves none of its data: no container
|
||||
// changed, added or gone; no file or directory changed, added or gone that a container mounts, or that holds or
|
||||
// lies within a path of the module's declared data; no change to its `data` declaration, nor to what it provides,
|
||||
// requires or contributes; and, for a module that declares data of its own, no change to a long-running process.
|
||||
// Its tools bundles, a file nothing mounts that holds none of its data, and manifest fields that recreate
|
||||
// nothing — its tool list — may differ.
|
||||
func Holds(from, to Manifest) string {
|
||||
module := to.Module
|
||||
if module == "" {
|
||||
module = from.Module
|
||||
}
|
||||
if said := Recreates(from, to).Say(module); said != "" {
|
||||
return said
|
||||
}
|
||||
if !sameDeclaration(from.Data, to.Data) {
|
||||
return "it changes " + module + "'s data declaration"
|
||||
}
|
||||
for _, f := range []struct {
|
||||
what string
|
||||
was, s any
|
||||
}{
|
||||
{"what it provides", from.Provides, to.Provides},
|
||||
{"what it requires", from.Requires, to.Requires},
|
||||
{"which of its requirements are optional", from.OptionalRequires, to.OptionalRequires},
|
||||
{"what it contributes to a provider", from.Contributes, to.Contributes},
|
||||
{"what it contributes to a provider", from.ContributesMany, to.ContributesMany},
|
||||
// A setting is filled into the files the module writes: a changed default changes what a container may
|
||||
// mount without the file's own declaration changing.
|
||||
{"its settings, which the files it writes are filled from", from.Settings, to.Settings},
|
||||
} {
|
||||
if !sameDeclaration(emptyAsNil(f.was), emptyAsNil(f.s)) {
|
||||
return "it changes " + f.what
|
||||
}
|
||||
}
|
||||
mounted := append(mountSources(from), mountSources(to)...)
|
||||
data := append(dataPaths(from), dataPaths(to)...)
|
||||
keepsData := (from.Data != nil && len(from.Data.Own) > 0) || (to.Data != nil && len(to.Data.Own) > 0)
|
||||
before, after := resourcesByID(from), resourcesByID(to)
|
||||
var changed []string
|
||||
for id, r := range after {
|
||||
if was, held := before[id]; !held || !sameDeclaration(was, r) {
|
||||
changed = append(changed, id)
|
||||
}
|
||||
}
|
||||
for id := range before {
|
||||
if _, kept := after[id]; !kept {
|
||||
changed = append(changed, id)
|
||||
}
|
||||
}
|
||||
sort.Strings(changed)
|
||||
dirs := absoluteDirectories(from, to)
|
||||
for _, key := range changed {
|
||||
for _, r := range []map[string]any{before[key], after[key]} {
|
||||
if r == nil {
|
||||
continue
|
||||
}
|
||||
kind, id := fmt.Sprint(r["type"]), fmt.Sprint(r["id"])
|
||||
switch kind {
|
||||
case "file", "directory":
|
||||
for _, p := range pathsOf(r, dirs) {
|
||||
for _, m := range mounted {
|
||||
if pathsMeet(p, expandDir(m.source, dirs)) {
|
||||
return fmt.Sprintf("it changes the %s %s, which its container %s mounts", kind, id, m.container)
|
||||
}
|
||||
}
|
||||
for _, d := range data {
|
||||
if pathsMeet(p, expandDir(d.path, dirs)) {
|
||||
return fmt.Sprintf("it changes the %s %s, which holds or lies within its data %s", kind, id, d.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
case "process", "service":
|
||||
if keepsData {
|
||||
return fmt.Sprintf("it changes its long-running %s %s, and it keeps data of its own", kind, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type mountSource struct{ container, source string }
|
||||
|
||||
// mountSources is every path a container of the module reads from its machine: its volumes' sources and its
|
||||
// environment files.
|
||||
func mountSources(m Manifest) []mountSource {
|
||||
var out []mountSource
|
||||
for _, r := range m.Resources {
|
||||
if fmt.Sprint(r["type"]) != "container" {
|
||||
continue
|
||||
}
|
||||
id := fmt.Sprint(r["id"])
|
||||
vols, _ := r["volumes"].([]any)
|
||||
for _, v := range vols {
|
||||
s, _ := v.(string)
|
||||
if mount := volumeMount.FindStringSubmatch(s); mount != nil {
|
||||
out = append(out, mountSource{id, mount[1]})
|
||||
}
|
||||
}
|
||||
switch files := r["env-file"].(type) {
|
||||
case string:
|
||||
out = append(out, mountSource{id, files})
|
||||
case []any:
|
||||
for _, f := range files {
|
||||
if s, ok := f.(string); ok {
|
||||
out = append(out, mountSource{id, s})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type dataPath struct{ id, path string }
|
||||
|
||||
func dataPaths(m Manifest) []dataPath {
|
||||
if m.Data == nil {
|
||||
return nil
|
||||
}
|
||||
var out []dataPath
|
||||
for _, it := range m.Data.Own {
|
||||
out = append(out, dataPath{it.ID, it.Path})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func resourcesByID(m Manifest) map[string]map[string]any {
|
||||
out := map[string]map[string]any{}
|
||||
for _, r := range m.Resources {
|
||||
out[fmt.Sprint(r["type"])+":"+fmt.Sprint(r["id"])] = r
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// absoluteDirectories is, per directory id, the absolute path a directory of either build states.
|
||||
func absoluteDirectories(ms ...Manifest) map[string]string {
|
||||
out := map[string]string{}
|
||||
for _, m := range ms {
|
||||
for _, r := range m.Resources {
|
||||
if fmt.Sprint(r["type"]) != "directory" {
|
||||
continue
|
||||
}
|
||||
if p, ok := r["path"].(string); ok && strings.HasPrefix(p, "/") {
|
||||
out[fmt.Sprint(r["id"])] = strings.TrimRight(p, "/")
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// pathsOf is every way a file or directory resource is named: its path, and a directory's `${dir:<id>}`.
|
||||
func pathsOf(r map[string]any, dirs map[string]string) []string {
|
||||
var out []string
|
||||
if p, ok := r["path"].(string); ok && p != "" {
|
||||
out = append(out, expandDir(p, dirs))
|
||||
}
|
||||
if fmt.Sprint(r["type"]) == "directory" {
|
||||
out = append(out, expandDir("${dir:"+fmt.Sprint(r["id"])+"}", dirs))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// expandDir writes a leading `${dir:<id>}` as the directory's absolute path when one is stated, so a path named
|
||||
// either way is one path.
|
||||
func expandDir(p string, dirs map[string]string) string {
|
||||
p = strings.TrimRight(p, "/")
|
||||
if ref := dirPlain.FindStringSubmatch(p); ref != nil {
|
||||
if abs, ok := dirs[ref[1]]; ok {
|
||||
return abs
|
||||
}
|
||||
return p
|
||||
}
|
||||
if strings.HasPrefix(p, "${dir:") {
|
||||
end := strings.Index(p, "}")
|
||||
if abs, ok := dirs[p[len("${dir:"):end]]; ok {
|
||||
return abs + p[end+1:]
|
||||
}
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// pathsMeet is whether one path is the other or lies within it.
|
||||
func pathsMeet(a, b string) bool {
|
||||
if a == "" || b == "" {
|
||||
return false
|
||||
}
|
||||
return a == b || strings.HasPrefix(a, b+"/") || strings.HasPrefix(b, a+"/")
|
||||
}
|
||||
|
||||
// emptyAsNil reads an empty list or map as nothing, so a field written empty is no change from one left out.
|
||||
func emptyAsNil(v any) any {
|
||||
rv := reflect.ValueOf(v)
|
||||
switch rv.Kind() {
|
||||
case reflect.Slice, reflect.Map:
|
||||
if rv.Len() == 0 {
|
||||
return nil
|
||||
}
|
||||
case reflect.Ptr:
|
||||
if rv.IsNil() {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
@@ -1476,6 +1476,16 @@ type CurrentBuild struct {
|
||||
Commit string
|
||||
// RollOut is the module's upgrade policy, as Upgrade.RollOut.
|
||||
RollOut bool
|
||||
// From is where the policy came from, as Upgrade.From: a move under a module's own `record` — its manifest's or
|
||||
// its irreplaceable data's — rolls out when it recreates nothing and moves no data (novox/hq ADR 0377), one under
|
||||
// a person's or the bus's never does.
|
||||
From string
|
||||
}
|
||||
|
||||
// OwnRecord is whether a module records because its manifest says so or because it keeps irreplaceable data
|
||||
// (novox/hq ADR 0377 §1): the `record` whose moves that recreate nothing roll out.
|
||||
func (b CurrentBuild) OwnRecord() bool {
|
||||
return !b.RollOut && (b.From == catalogue.FromModule || b.From == catalogue.FromData)
|
||||
}
|
||||
|
||||
// CurrentBuilds is every module's current build and upgrade policy, in one read: what a send records
|
||||
@@ -1497,7 +1507,8 @@ func (i *Inventory) CurrentBuilds(ctx context.Context) (map[string]CurrentBuild,
|
||||
if err := rows.Scan(&name, &b.Commit, &chosen, &together, &why, &by, &manifest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b.RollOut = upgradeFrom(chosen, together, why, by, manifest).RollOut
|
||||
u := upgradeFrom(chosen, together, why, by, manifest)
|
||||
b.RollOut, b.From = u.RollOut, u.From
|
||||
out[name] = b
|
||||
}
|
||||
return out, rows.Err()
|
||||
|
||||
@@ -313,6 +313,11 @@ type PlanModule struct {
|
||||
// Rest is, per machine of the rest, the declaration the send after the first-node gate carried there (novox/hq ADR
|
||||
// 0282 decision 6): the machine's first report of it, applied, is when the build runs there.
|
||||
Rest map[string]SentDeclaration `json:"rest,omitempty"`
|
||||
// Recorded is, for a module whose policy records, per machine running it with a move, which way the walk took it
|
||||
// (novox/hq ADR 0377 §6): "went out: …" when the move recreates none of its containers and moves none of its
|
||||
// data, or "waits: …" with the recreate words, the data change named, the person's or the bus's hold, or why it
|
||||
// could not be judged.
|
||||
Recorded map[string]string `json:"recorded,omitempty"`
|
||||
}
|
||||
|
||||
// PlanGate is one module's rollout record at its gate (to-be 45 §8): the component, the first machine,
|
||||
|
||||
@@ -73,10 +73,10 @@ func TestTheCurrentBuildsAreTheCatalogues(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Rolled out by default, one machine first and gated (novox/hq ADR 0236).
|
||||
if got := current["resolver"]; got != (CurrentBuild{Commit: "c1", RollOut: true}) {
|
||||
if got := current["resolver"]; got != (CurrentBuild{Commit: "c1", RollOut: true, From: catalogue.FromDefault}) {
|
||||
t.Errorf("resolver is at %+v", got)
|
||||
}
|
||||
if got := current["by-hand"]; got != (CurrentBuild{RollOut: true}) {
|
||||
if got := current["by-hand"]; got != (CurrentBuild{RollOut: true, From: catalogue.FromPerson}) {
|
||||
t.Errorf("a module with no source is at %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user