Merge pull request 'A take is a comparison: the host's facts, former targets, and strays (hq ADR 0163)' (#63) from feat/a-take-is-a-comparison-the-hosts-facts into main

This commit was merged in pull request #63.
This commit is contained in:
2026-10-01 19:25:56 +00:00
8 changed files with 436 additions and 7 deletions
+21 -1
View File
@@ -1430,7 +1430,15 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
// node never reads as converged (novox/hq ADR 0100).
for _, h := range updated.Held {
report.Held = append(report.Held, link.Held{ID: h.ID, Module: h.Module, Kind: h.Kind,
Target: h.Target, Since: h.Since, Changed: h.Changed, Kept: h.Kept})
Target: h.Target, Since: h.Since, Changed: h.Changed, Kept: h.Kept, Facts: factsAsReported(h.Facts)})
}
// And what runs here that nobody asked for (novox/hq ADR 0163).
if strays, err := apply.Strays(ctx, apply.ExecRunner, updated); err != nil {
fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not list what else runs here: %v\n", err)
} else {
for _, s := range strays {
report.Strays = append(report.Strays, link.Stray{Kind: s.Kind, Name: s.Name, Detail: s.Detail})
}
}
if declared.Adoption != nil {
if updated.Firewall != nil {
@@ -1638,3 +1646,15 @@ func profileAsReported(detected profile.Profile) map[string]any {
}
return reported
}
// factsAsReported is a held thing's facts as the mesh reads them: the same bytes the host keeps.
func factsAsReported(f *store.Facts) map[string]any {
if f == nil {
return nil
}
out := map[string]any{}
if raw, err := json.Marshal(f); err == nil {
_ = json.Unmarshal(raw, &out)
}
return out
}
+97
View File
@@ -0,0 +1,97 @@
package apply
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-host/internal/store"
)
// A take is a comparison (novox/hq ADR 0163): while a module's container is held, the host reports
// the found image and its age beside the declared one, the networks and who else is on them, the
// mounts and the ports — and says when the declared image is the older.
func TestAHeldContainerCarriesTheFactsATakeCompares(t *testing.T) {
dir, page, m := predecessor(t)
m.containers["hello-web"].image = "web:1.27"
m.containers["hello-web"].imageID = "sha256:found"
m.containers["hello-web"].networks = []string{"predecessor_default"}
m.containers["hello-web"].mounts = []string{"/srv/web:/data"}
m.containers["hello-web"].ports = []string{"80/tcp>0.0.0.0:8080"}
m.images = map[string]string{"sha256:found": "2026-09-17T10:00:00Z", pinned: "2026-08-20T10:00:00Z"}
m.members = map[string][]string{"predecessor_default": {"hello-web", "office", "db"}}
_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
h, ok := state.HeldAt("hello-web.server")
if !ok || h.Facts == nil {
t.Fatalf("a held container carries no facts: %+v", h)
}
f := h.Facts
if f.Image != "web:1.27" || f.ImageCreated != "2026-09-17T10:00:00Z" {
t.Errorf("the found image and its age: %+v", f)
}
if f.DeclaredImage != pinned || f.DeclaredImageCreated != "2026-08-20T10:00:00Z" || !f.Downgrade {
t.Errorf("the declared image, its age, and that it is a downgrade: %+v", f)
}
if got := f.Networks["predecessor_default"]; len(got) != 2 || got[0] != "db" || got[1] != "office" {
t.Errorf("the neighbours on the found network, without the container itself: %v", f.Networks)
}
if len(f.Mounts) != 1 || f.Mounts[0] != "/srv/web:/data" || len(f.Ports) != 1 || f.Ports[0] != "80/tcp>0.0.0.0:8080" {
t.Errorf("mounts and ports as found: %+v", f)
}
// And the held file carries how the declared content differs from what was found.
p, ok := state.HeldAt("hello-web.page")
if !ok || p.Facts == nil || !p.Facts.Differs {
t.Fatalf("a held file that differs from the declared content does not say so: %+v", p)
}
joined := strings.Join(p.Facts.Difference, "\n")
if !strings.Contains(joined, "- the predecessor's page") || !strings.Contains(joined, "+ the mesh's page") {
t.Errorf("the difference does not show what is lost and what is new: %q", joined)
}
_ = os.Remove(filepath.Join(dir, "unused"))
}
// A declared image not yet on the machine leaves its age unknown and the comparison undecided.
func TestAnImageNotYetPulledLeavesTheDowngradeUndecided(t *testing.T) {
dir, page, m := predecessor(t)
m.containers["hello-web"].image = "web:1.27"
m.containers["hello-web"].imageID = "sha256:found"
m.images = map[string]string{"sha256:found": "2026-09-17T10:00:00Z"}
_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
h, _ := state.HeldAt("hello-web.server")
if h.Facts == nil || h.Facts.DeclaredImageCreated != "" || h.Facts.Downgrade {
t.Fatalf("an unknown declared age decided a downgrade: %+v", h.Facts)
}
}
func TestTheDifferenceIsWhatIsLostAndWhatIsNew(t *testing.T) {
differs, lines := differenceOf("a\nprivate scope: local\nb\n", "a\nb\nupstream: public\n")
if !differs || len(lines) != 2 || lines[0] != "- private scope: local" || lines[1] != "+ upstream: public" {
t.Fatalf("got %v %v", differs, lines)
}
if differs, lines := differenceOf("same\n", "same\n"); differs || lines != nil {
t.Fatalf("identical content differs: %v %v", differs, lines)
}
}
// What runs on the machine that the mesh neither wrote nor holds is reported (ADR 0163).
func TestStraysAreWhatRunsHereThatNobodyAsked(t *testing.T) {
m := &machine{containers: map[string]*fakeContainer{
"hello-web": {id: "ours", running: true, image: "web:1"},
"gitea-old": {id: "left-behind", running: true, image: "gitea:1.22"},
"held-thing": {id: "found", running: true, image: "x:1"},
}}
known := store.State{
Resources: []store.Applied{{ID: "hello-web.server", Type: "container", Target: "hello-web"}},
Held: []store.Held{{ID: "other.server", Kind: "container", Target: "held-thing"}},
}
strays, err := Strays(context.Background(), m.run, known)
if err != nil {
t.Fatal(err)
}
if len(strays) != 1 || strays[0].Name != "gitea-old" || !strings.Contains(strays[0].Detail, "gitea:1.22") {
t.Fatalf("strays: %+v", strays)
}
}
+125 -4
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"syscall"
"time"
@@ -433,6 +434,32 @@ type foundContainer struct {
id string
running bool
spec string
// What a take compares (novox/hq ADR 0163): the image and its id, the networks the container
// is on, its mounts and its published ports — empty from a runtime (or a test's fake) that
// answers the short form.
image string
imageID string
networks []string
mounts []string
ports []string
}
// foundFormat is what inspectFound asks the runtime for, tab-separated: the three a hold has
// always needed, then the facts a take compares.
const foundFormat = "{{.Id}}\t{{.State.Running}}\t{{index .Config.Labels \"" + specLabel + "\"}}" +
"\t{{.Config.Image}}\t{{.Image}}" +
"\t{{range $k, $v := .NetworkSettings.Networks}}{{$k}},{{end}}" +
"\t{{range .Mounts}}{{.Source}}:{{.Destination}},{{end}}" +
"\t{{range $p, $b := .NetworkSettings.Ports}}{{$p}}{{range $b}}>{{.HostIp}}:{{.HostPort}}{{end}},{{end}}"
func splitList(s string) []string {
var out []string
for _, part := range strings.Split(s, ",") {
if part = strings.TrimSpace(part); part != "" {
out = append(out, part)
}
}
return out
}
// inspectFound reads a container by name the way a hold needs it: its id, whether it runs, and
@@ -451,8 +478,7 @@ func inspectFound(ctx context.Context, name string, run Runner) (foundContainer,
if err != nil {
return foundContainer{}, false, fmt.Errorf("%w, so nothing can be said about %q", err, name)
}
out, err := run(ctx, cri, "container", "inspect", "--format",
"{{.Id}}\t{{.State.Running}}\t{{index .Config.Labels \""+specLabel+"\"}}", name)
out, err := run(ctx, cri, "container", "inspect", "--format", foundFormat, name)
if err != nil {
if absent(err) {
return foundContainer{}, false, nil
@@ -466,14 +492,100 @@ func inspectFound(ctx context.Context, name string, run Runner) (foundContainer,
name, err)
}
parts := strings.Split(strings.TrimSpace(out), "\t")
for len(parts) < 3 {
for len(parts) < 8 {
parts = append(parts, "")
}
spec := strings.TrimSpace(parts[2])
if spec == "<no value>" {
spec = ""
}
return foundContainer{id: strings.TrimSpace(parts[0]), running: parts[1] == "true", spec: spec}, true, nil
return foundContainer{id: strings.TrimSpace(parts[0]), running: parts[1] == "true", spec: spec,
image: strings.TrimSpace(parts[3]), imageID: strings.TrimSpace(parts[4]),
networks: splitList(parts[5]), mounts: splitList(parts[6]), ports: splitList(parts[7])}, true, nil
}
// factsOf is what a take would compare for a found container (novox/hq ADR 0163): the found
// image and when it was made, the networks and who else is on them, mounts and ports — beside
// what the module declares, and the declared image's date when that image is on the machine.
// Every question the runtime cannot answer leaves its fact empty; a preview says so rather than
// guesses.
func factsOf(ctx context.Context, seen foundContainer, res *declaration.Container, run Runner) *Facts {
cri, err := containerRuntime(ctx, run)
if err != nil {
return nil
}
f := &store.Facts{Image: seen.image, Mounts: seen.mounts, Ports: seen.ports,
DeclaredImage: res.Image, DeclaredPorts: res.Ports, DeclaredVolumes: res.Volumes}
if seen.imageID != "" {
if out, err := run(ctx, cri, "image", "inspect", "--format", "{{.Created}}", seen.imageID); err == nil {
f.ImageCreated = strings.TrimSpace(out)
}
}
if res.Image != "" {
if out, err := run(ctx, cri, "image", "inspect", "--format", "{{.Created}}", res.Image); err == nil {
f.DeclaredImageCreated = strings.TrimSpace(out)
}
}
if found, err := time.Parse(time.RFC3339Nano, f.ImageCreated); err == nil {
if declared, err := time.Parse(time.RFC3339Nano, f.DeclaredImageCreated); err == nil {
f.Downgrade = declared.Before(found)
}
}
for _, network := range seen.networks {
if f.Networks == nil {
f.Networks = map[string][]string{}
}
var members []string
if out, err := run(ctx, cri, "network", "inspect", "--format",
"{{range .Containers}}{{.Name}},{{end}}", network); err == nil {
for _, m := range splitList(out) {
if m != res.Name {
members = append(members, m)
}
}
}
sort.Strings(members)
f.Networks[network] = members
}
return (*Facts)(f)
}
// Facts is store.Facts, named here so hold's callers read as one vocabulary.
type Facts = store.Facts
// differenceOf is how a found file differs from the declared content: the lines only the found
// file has, marked -, then the lines only the declared content has, marked +, in their own order,
// bounded so a report stays a report. Not a diff tool's output: the question a take answers is
// "what would be lost and what would be new", and that is these two lists.
func differenceOf(found, declared string) (bool, []string) {
if found == declared {
return false, nil
}
const bound = 40
count := func(s string) map[string]int {
out := map[string]int{}
for _, line := range strings.Split(s, "\n") {
out[line]++
}
return out
}
inFound, inDeclared := count(found), count(declared)
var out []string
add := func(mark, s string, other map[string]int) {
seen := map[string]int{}
for _, line := range strings.Split(s, "\n") {
seen[line]++
if seen[line] > other[line] && len(out) < bound {
out = append(out, mark+" "+line)
}
}
}
add("-", found, inDeclared)
add("+", declared, inFound)
if len(out) >= bound {
out = append(out, "… and more")
}
return true, out
}
// absent is whether a runtime said the thing is not there, rather than failing to answer. Its own
@@ -540,6 +652,12 @@ func hold(ctx context.Context, sys system.System, r declaration.Resource, module
} else if digestOf(string(content)) != h.Digest {
changed = "rewritten"
}
// What a take would replace it with, and how that differs (novox/hq ADR 0163): a
// file declared whole is compared whole; one written into is not replaced at all.
if res.Into == "" {
differs, lines := differenceOf(string(content), res.Content)
h.Facts = &Facts{Differs: differs, Difference: lines}
}
}
case *declaration.Directory:
info, err := os.Lstat(res.Path)
@@ -628,6 +746,9 @@ func hold(ctx context.Context, sys system.System, r declaration.Resource, module
case h.Running && !seen.running:
changed = "stopped"
}
if exists {
h.Facts = factsOf(ctx, seen, res, run)
}
default:
return out, h, fmt.Errorf("a %s cannot be held", r.Kind())
}
+39 -2
View File
@@ -5,6 +5,7 @@ import (
"errors"
"os"
"path/filepath"
"sort"
"strings"
"testing"
@@ -18,7 +19,11 @@ import (
// label when a host made it. Every command it is asked is written down.
type machine struct {
containers map[string]*fakeContainer
asked []string
// images is what `image inspect --format {{.Created}}` answers per image or id; members is
// what `network inspect` lists per network (ADR 0163).
images map[string]string
members map[string][]string
asked []string
// wgUp is what `wg show interfaces` answers: the tunnels up on the machine.
wgUp string
// handshakes is what `wg show <interface> latest-handshakes` answers, and handshakesFail the
@@ -97,6 +102,9 @@ type fakeContainer struct {
id string
running bool
spec string
// What a take compares (ADR 0163), answered in the long inspect form when set.
image, imageID string
networks, mounts, ports []string
}
func (m *machine) run(_ context.Context, name string, args ...string) (string, error) {
@@ -140,9 +148,38 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e
running = "true"
}
if strings.HasPrefix(args[3], "{{.Id}}") {
return c.id + "\t" + running + "\t" + c.spec + "\n", nil
line := c.id + "\t" + running + "\t" + c.spec
if c.image != "" {
line += "\t" + c.image + "\t" + c.imageID + "\t" + strings.Join(c.networks, ",") + "," +
"\t" + strings.Join(c.mounts, ",") + "," + "\t" + strings.Join(c.ports, ",") + ","
}
return line + "\n", nil
}
return running + "\t" + c.spec + "\n", nil
case "image":
if len(args) > 1 && args[1] == "inspect" {
if created, ok := m.images[args[len(args)-1]]; ok {
return created + "\n", nil
}
return "", errors.New("no such image")
}
return "", nil
case "network":
if len(args) > 1 && args[1] == "inspect" {
return strings.Join(m.members[args[len(args)-1]], ",") + ",\n", nil
}
return "", nil
case "ps":
var lines []string
for name, c := range m.containers {
state := "exited"
if c.running {
state = "running"
}
lines = append(lines, name+"\t"+c.image+"\t"+state)
}
sort.Strings(lines)
return strings.Join(lines, "\n") + "\n", nil
case "rm":
delete(m.containers, args[len(args)-1])
return "", nil
+54
View File
@@ -0,0 +1,54 @@
package apply
import (
"context"
"sort"
"strings"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
)
// Strays is what runs on the machine that the mesh neither wrote nor holds (novox/hq ADR 0163):
// every container the runtime has that no record names and no hold names. The question nothing
// answered on 2026-09-23, when a renamed resource left its old container running for a day; asked
// on every apply now, and reported, so a thing left behind is seen the day it is left.
//
// Containers only, today. A listener nobody declared is harder to attribute to a thing, and the
// machine's own services are not strays; that account is issue 160's.
func Strays(ctx context.Context, run Runner, known store.State) ([]store.Stray, error) {
cri, err := containerRuntime(ctx, run)
if err != nil {
return nil, nil // a machine with no runtime has no containers to stray
}
out, err := run(ctx, cri, "ps", "-a", "--format", "{{.Names}}\t{{.Image}}\t{{.State}}")
if err != nil {
return nil, err
}
ours := map[string]bool{}
for _, r := range known.Resources {
if declaration.Type(r.Type) == declaration.TypeContainer {
ours[r.Target] = true
}
}
for _, h := range known.Held {
if h.Kind == string(declaration.TypeContainer) {
ours[h.Target] = true
}
}
var strays []store.Stray
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
parts := strings.Split(line, "\t")
name := strings.TrimSpace(parts[0])
if name == "" || ours[name] {
continue
}
detail := ""
if len(parts) > 2 {
detail = strings.TrimSpace(parts[1]) + ", " + strings.TrimSpace(parts[2])
}
strays = append(strays, store.Stray{Kind: string(declaration.TypeContainer), Name: name, Detail: detail})
}
sort.Slice(strays, func(i, j int) bool { return strays[i].Name < strays[j].Name })
return strays, nil
}
+14
View File
@@ -110,6 +110,10 @@ type Report struct {
// and the mesh's up in its place, and where the found configuration's original was kept.
Tunnel *CarriedTunnel `json:"tunnel,omitempty"`
// Strays is what runs on the machine that the mesh neither wrote nor holds (novox/hq ADR
// 0163): containers nobody declared and nobody holds, the ones a cutover leaves behind.
Strays []Stray `json:"strays,omitempty"`
// Profile is what this machine can do, detected again by the apply that reports (novox/hq
// ADR 0161) — the same shape enrolment sends — so a capability gained or lost since enrolment,
// a network manager switched, reaches the mesh at the next push rather than never.
@@ -205,6 +209,16 @@ type Held struct {
Changed string `json:"changed,omitempty"`
// Kept is where a file's original was kept.
Kept string `json:"kept,omitempty"`
// Facts is the found thing beside what the module declares — what a take compares (novox/hq
// ADR 0163). The same shape the host keeps; the controller reads it as data.
Facts map[string]any `json:"facts,omitempty"`
}
// A Stray is a container the mesh neither wrote nor holds (ADR 0163).
type Stray struct {
Kind string `json:"kind"`
Name string `json:"name"`
Detail string `json:"detail,omitempty"`
}
// Reach is one thing reachable on the machine: a listening socket, or a published container port.
+29
View File
@@ -0,0 +1,29 @@
package store
import "testing"
// A resource whose target moves leaves what the host wrote under the old target on record as a
// former one, undeclared by construction, so the next apply removes it (novox/hq issue 097, ADR 0163).
func TestARecordWhoseTargetMovedKeepsTheFormerTargetToRemove(t *testing.T) {
s := State{}
s.Record(Applied{ID: "gitea.server", Type: "container", Target: "mesh-gitea", Origin: OriginDeclared})
s.Record(Applied{ID: "gitea.server", Type: "container", Target: "gitea", Origin: OriginDeclared})
if len(s.Resources) != 2 {
t.Fatalf("a moved target produced %d record(s): %+v", len(s.Resources), s.Resources)
}
orphans := s.Orphans(map[string]bool{"gitea.server": true}, OriginDeclared)
if len(orphans) != 1 || orphans[0].Target != "mesh-gitea" || !IsFormer(orphans[0].ID) {
t.Fatalf("the former target is not an orphan to remove: %+v", orphans)
}
s.Forget(orphans[0].ID)
if len(s.Resources) != 1 || s.Resources[0].Target != "gitea" {
t.Fatalf("forgetting the former target touched the current one: %+v", s.Resources)
}
// The same target again is not a move; a carried record is not the host's to remove.
s.Record(Applied{ID: "gitea.server", Type: "container", Target: "gitea", Origin: OriginDeclared})
s.Record(Applied{ID: "bundle", Type: "file", Target: "/a"})
s.Record(Applied{ID: "bundle", Type: "file", Target: "/b"})
if len(s.Resources) != 2 {
t.Fatalf("an unmoved or carried record grew the list: %+v", s.Resources)
}
}
+57
View File
@@ -18,6 +18,7 @@ import (
"os"
"path/filepath"
"sort"
"strings"
"time"
)
@@ -274,6 +275,41 @@ type Held struct {
// reverted: that is how a predecessor still writing is caught.
Changed string `json:"changed,omitempty"`
ChangedAt time.Time `json:"changed_at,omitempty"`
// Facts is what a take would compare: the found thing beside what the module declares
// (novox/hq ADR 0163). Read fresh on every apply while held, so the controller's preview
// speaks of the machine as it is.
Facts *Facts `json:"facts,omitempty"`
}
// Facts is a held thing beside what its module declares — what a take compares (ADR 0163).
type Facts struct {
// A found container: the image it runs and when that image was made; the networks it is on
// and the other containers on each; what it mounts; what it publishes.
Image string `json:"image,omitempty"`
ImageCreated string `json:"image_created,omitempty"`
Networks map[string][]string `json:"networks,omitempty"`
Mounts []string `json:"mounts,omitempty"`
Ports []string `json:"ports,omitempty"`
// What the module declares for it, and the declared image's creation date when the image
// is on the machine already.
DeclaredImage string `json:"declared_image,omitempty"`
DeclaredImageCreated string `json:"declared_image_created,omitempty"`
DeclaredPorts []string `json:"declared_ports,omitempty"`
DeclaredVolumes []string `json:"declared_volumes,omitempty"`
// Downgrade is true when both creation dates are known and the declared image is the older.
Downgrade bool `json:"downgrade,omitempty"`
// A found file: whether the declared content differs from what was found, and how, as lines
// only in the found file (-) and lines only in the declared one (+), bounded.
Differs bool `json:"differs,omitempty"`
Difference []string `json:"difference,omitempty"`
}
// A Stray is something running on the machine that the mesh neither wrote nor holds
// (novox/hq ADR 0163): the answer to "what is here that nobody asked for".
type Stray struct {
Kind string `json:"kind"`
Name string `json:"name"`
Detail string `json:"detail,omitempty"`
}
// Recorded reports whether this host has a record, of any origin, of putting something of this
@@ -462,6 +498,20 @@ func Save(path string, s State) error {
func (s *State) Record(a Applied) {
for i, existing := range s.Resources {
if existing.ID == a.ID {
// A resource whose target moved leaves what the host wrote under the old target
// behind — a container under the old name, a file at the old path. Rewriting the
// record would erase the only trace of it (novox/hq issue 097, ADR 0163), so the old
// target stays on record as a former one, undeclared by construction, until the next
// apply removes it the way it removes anything the host wrote and no longer declares.
// What was found is held, never recorded here, and so never removed by this.
if originOf(existing) == OriginDeclared && existing.Target != "" && a.Target != "" &&
existing.Target != a.Target && existing.Type == a.Type {
former := existing
former.ID = FormerID(existing.ID, existing.Target)
s.Resources[i] = a
s.Resources = append(s.Resources, former)
return
}
s.Resources[i] = a
return
}
@@ -469,6 +519,13 @@ func (s *State) Record(a Applied) {
s.Resources = append(s.Resources, a)
}
// FormerID names the record of a resource's former target: the resource's id and the target it
// had, so the record is distinct from the current one and is never what a declaration names.
func FormerID(id, target string) string { return id + "@former:" + target }
// IsFormer says whether a record names a former target.
func IsFormer(id string) bool { return strings.Contains(id, "@former:") }
// Forget drops a resource from what the node owns.
func (s *State) Forget(id string) {
kept := s.Resources[:0]