diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index c246304..fa43156 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1430,7 +1430,15 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D // node never reads as converged (novox/hq ADR 0100). for _, h := range updated.Held { report.Held = append(report.Held, link.Held{ID: h.ID, Module: h.Module, Kind: h.Kind, - Target: h.Target, Since: h.Since, Changed: h.Changed, Kept: h.Kept}) + Target: h.Target, Since: h.Since, Changed: h.Changed, Kept: h.Kept, Facts: factsAsReported(h.Facts)}) + } + // And what runs here that nobody asked for (novox/hq ADR 0163). + if strays, err := apply.Strays(ctx, apply.ExecRunner, updated); err != nil { + fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not list what else runs here: %v\n", err) + } else { + for _, s := range strays { + report.Strays = append(report.Strays, link.Stray{Kind: s.Kind, Name: s.Name, Detail: s.Detail}) + } } if declared.Adoption != nil { if updated.Firewall != nil { @@ -1638,3 +1646,15 @@ func profileAsReported(detected profile.Profile) map[string]any { } return reported } + +// factsAsReported is a held thing's facts as the mesh reads them: the same bytes the host keeps. +func factsAsReported(f *store.Facts) map[string]any { + if f == nil { + return nil + } + out := map[string]any{} + if raw, err := json.Marshal(f); err == nil { + _ = json.Unmarshal(raw, &out) + } + return out +} diff --git a/internal/apply/facts_test.go b/internal/apply/facts_test.go new file mode 100644 index 0000000..3e444e6 --- /dev/null +++ b/internal/apply/facts_test.go @@ -0,0 +1,97 @@ +package apply + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/store" +) + +// A take is a comparison (novox/hq ADR 0163): while a module's container is held, the host reports +// the found image and its age beside the declared one, the networks and who else is on them, the +// mounts and the ports — and says when the declared image is the older. +func TestAHeldContainerCarriesTheFactsATakeCompares(t *testing.T) { + dir, page, m := predecessor(t) + m.containers["hello-web"].image = "web:1.27" + m.containers["hello-web"].imageID = "sha256:found" + m.containers["hello-web"].networks = []string{"predecessor_default"} + m.containers["hello-web"].mounts = []string{"/srv/web:/data"} + m.containers["hello-web"].ports = []string{"80/tcp>0.0.0.0:8080"} + m.images = map[string]string{"sha256:found": "2026-09-17T10:00:00Z", pinned: "2026-08-20T10:00:00Z"} + m.members = map[string][]string{"predecessor_default": {"hello-web", "office", "db"}} + + _, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) + h, ok := state.HeldAt("hello-web.server") + if !ok || h.Facts == nil { + t.Fatalf("a held container carries no facts: %+v", h) + } + f := h.Facts + if f.Image != "web:1.27" || f.ImageCreated != "2026-09-17T10:00:00Z" { + t.Errorf("the found image and its age: %+v", f) + } + if f.DeclaredImage != pinned || f.DeclaredImageCreated != "2026-08-20T10:00:00Z" || !f.Downgrade { + t.Errorf("the declared image, its age, and that it is a downgrade: %+v", f) + } + if got := f.Networks["predecessor_default"]; len(got) != 2 || got[0] != "db" || got[1] != "office" { + t.Errorf("the neighbours on the found network, without the container itself: %v", f.Networks) + } + if len(f.Mounts) != 1 || f.Mounts[0] != "/srv/web:/data" || len(f.Ports) != 1 || f.Ports[0] != "80/tcp>0.0.0.0:8080" { + t.Errorf("mounts and ports as found: %+v", f) + } + // And the held file carries how the declared content differs from what was found. + p, ok := state.HeldAt("hello-web.page") + if !ok || p.Facts == nil || !p.Facts.Differs { + t.Fatalf("a held file that differs from the declared content does not say so: %+v", p) + } + joined := strings.Join(p.Facts.Difference, "\n") + if !strings.Contains(joined, "- the predecessor's page") || !strings.Contains(joined, "+ the mesh's page") { + t.Errorf("the difference does not show what is lost and what is new: %q", joined) + } + _ = os.Remove(filepath.Join(dir, "unused")) +} + +// A declared image not yet on the machine leaves its age unknown and the comparison undecided. +func TestAnImageNotYetPulledLeavesTheDowngradeUndecided(t *testing.T) { + dir, page, m := predecessor(t) + m.containers["hello-web"].image = "web:1.27" + m.containers["hello-web"].imageID = "sha256:found" + m.images = map[string]string{"sha256:found": "2026-09-17T10:00:00Z"} + _, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) + h, _ := state.HeldAt("hello-web.server") + if h.Facts == nil || h.Facts.DeclaredImageCreated != "" || h.Facts.Downgrade { + t.Fatalf("an unknown declared age decided a downgrade: %+v", h.Facts) + } +} + +func TestTheDifferenceIsWhatIsLostAndWhatIsNew(t *testing.T) { + differs, lines := differenceOf("a\nprivate scope: local\nb\n", "a\nb\nupstream: public\n") + if !differs || len(lines) != 2 || lines[0] != "- private scope: local" || lines[1] != "+ upstream: public" { + t.Fatalf("got %v %v", differs, lines) + } + if differs, lines := differenceOf("same\n", "same\n"); differs || lines != nil { + t.Fatalf("identical content differs: %v %v", differs, lines) + } +} + +// What runs on the machine that the mesh neither wrote nor holds is reported (ADR 0163). +func TestStraysAreWhatRunsHereThatNobodyAsked(t *testing.T) { + m := &machine{containers: map[string]*fakeContainer{ + "hello-web": {id: "ours", running: true, image: "web:1"}, + "gitea-old": {id: "left-behind", running: true, image: "gitea:1.22"}, + "held-thing": {id: "found", running: true, image: "x:1"}, + }} + known := store.State{ + Resources: []store.Applied{{ID: "hello-web.server", Type: "container", Target: "hello-web"}}, + Held: []store.Held{{ID: "other.server", Kind: "container", Target: "held-thing"}}, + } + strays, err := Strays(context.Background(), m.run, known) + if err != nil { + t.Fatal(err) + } + if len(strays) != 1 || strays[0].Name != "gitea-old" || !strings.Contains(strays[0].Detail, "gitea:1.22") { + t.Fatalf("strays: %+v", strays) + } +} diff --git a/internal/apply/hold.go b/internal/apply/hold.go index 4525ced..fbd871b 100644 --- a/internal/apply/hold.go +++ b/internal/apply/hold.go @@ -8,6 +8,7 @@ import ( "fmt" "os" "path/filepath" + "sort" "strings" "syscall" "time" @@ -433,6 +434,32 @@ type foundContainer struct { id string running bool spec string + // What a take compares (novox/hq ADR 0163): the image and its id, the networks the container + // is on, its mounts and its published ports — empty from a runtime (or a test's fake) that + // answers the short form. + image string + imageID string + networks []string + mounts []string + ports []string +} + +// foundFormat is what inspectFound asks the runtime for, tab-separated: the three a hold has +// always needed, then the facts a take compares. +const foundFormat = "{{.Id}}\t{{.State.Running}}\t{{index .Config.Labels \"" + specLabel + "\"}}" + + "\t{{.Config.Image}}\t{{.Image}}" + + "\t{{range $k, $v := .NetworkSettings.Networks}}{{$k}},{{end}}" + + "\t{{range .Mounts}}{{.Source}}:{{.Destination}},{{end}}" + + "\t{{range $p, $b := .NetworkSettings.Ports}}{{$p}}{{range $b}}>{{.HostIp}}:{{.HostPort}}{{end}},{{end}}" + +func splitList(s string) []string { + var out []string + for _, part := range strings.Split(s, ",") { + if part = strings.TrimSpace(part); part != "" { + out = append(out, part) + } + } + return out } // inspectFound reads a container by name the way a hold needs it: its id, whether it runs, and @@ -451,8 +478,7 @@ func inspectFound(ctx context.Context, name string, run Runner) (foundContainer, if err != nil { return foundContainer{}, false, fmt.Errorf("%w, so nothing can be said about %q", err, name) } - out, err := run(ctx, cri, "container", "inspect", "--format", - "{{.Id}}\t{{.State.Running}}\t{{index .Config.Labels \""+specLabel+"\"}}", name) + out, err := run(ctx, cri, "container", "inspect", "--format", foundFormat, name) if err != nil { if absent(err) { return foundContainer{}, false, nil @@ -466,14 +492,100 @@ func inspectFound(ctx context.Context, name string, run Runner) (foundContainer, name, err) } parts := strings.Split(strings.TrimSpace(out), "\t") - for len(parts) < 3 { + for len(parts) < 8 { parts = append(parts, "") } spec := strings.TrimSpace(parts[2]) if spec == "" { spec = "" } - return foundContainer{id: strings.TrimSpace(parts[0]), running: parts[1] == "true", spec: spec}, true, nil + return foundContainer{id: strings.TrimSpace(parts[0]), running: parts[1] == "true", spec: spec, + image: strings.TrimSpace(parts[3]), imageID: strings.TrimSpace(parts[4]), + networks: splitList(parts[5]), mounts: splitList(parts[6]), ports: splitList(parts[7])}, true, nil +} + +// factsOf is what a take would compare for a found container (novox/hq ADR 0163): the found +// image and when it was made, the networks and who else is on them, mounts and ports — beside +// what the module declares, and the declared image's date when that image is on the machine. +// Every question the runtime cannot answer leaves its fact empty; a preview says so rather than +// guesses. +func factsOf(ctx context.Context, seen foundContainer, res *declaration.Container, run Runner) *Facts { + cri, err := containerRuntime(ctx, run) + if err != nil { + return nil + } + f := &store.Facts{Image: seen.image, Mounts: seen.mounts, Ports: seen.ports, + DeclaredImage: res.Image, DeclaredPorts: res.Ports, DeclaredVolumes: res.Volumes} + if seen.imageID != "" { + if out, err := run(ctx, cri, "image", "inspect", "--format", "{{.Created}}", seen.imageID); err == nil { + f.ImageCreated = strings.TrimSpace(out) + } + } + if res.Image != "" { + if out, err := run(ctx, cri, "image", "inspect", "--format", "{{.Created}}", res.Image); err == nil { + f.DeclaredImageCreated = strings.TrimSpace(out) + } + } + if found, err := time.Parse(time.RFC3339Nano, f.ImageCreated); err == nil { + if declared, err := time.Parse(time.RFC3339Nano, f.DeclaredImageCreated); err == nil { + f.Downgrade = declared.Before(found) + } + } + for _, network := range seen.networks { + if f.Networks == nil { + f.Networks = map[string][]string{} + } + var members []string + if out, err := run(ctx, cri, "network", "inspect", "--format", + "{{range .Containers}}{{.Name}},{{end}}", network); err == nil { + for _, m := range splitList(out) { + if m != res.Name { + members = append(members, m) + } + } + } + sort.Strings(members) + f.Networks[network] = members + } + return (*Facts)(f) +} + +// Facts is store.Facts, named here so hold's callers read as one vocabulary. +type Facts = store.Facts + +// differenceOf is how a found file differs from the declared content: the lines only the found +// file has, marked -, then the lines only the declared content has, marked +, in their own order, +// bounded so a report stays a report. Not a diff tool's output: the question a take answers is +// "what would be lost and what would be new", and that is these two lists. +func differenceOf(found, declared string) (bool, []string) { + if found == declared { + return false, nil + } + const bound = 40 + count := func(s string) map[string]int { + out := map[string]int{} + for _, line := range strings.Split(s, "\n") { + out[line]++ + } + return out + } + inFound, inDeclared := count(found), count(declared) + var out []string + add := func(mark, s string, other map[string]int) { + seen := map[string]int{} + for _, line := range strings.Split(s, "\n") { + seen[line]++ + if seen[line] > other[line] && len(out) < bound { + out = append(out, mark+" "+line) + } + } + } + add("-", found, inDeclared) + add("+", declared, inFound) + if len(out) >= bound { + out = append(out, "… and more") + } + return true, out } // absent is whether a runtime said the thing is not there, rather than failing to answer. Its own @@ -540,6 +652,12 @@ func hold(ctx context.Context, sys system.System, r declaration.Resource, module } else if digestOf(string(content)) != h.Digest { changed = "rewritten" } + // What a take would replace it with, and how that differs (novox/hq ADR 0163): a + // file declared whole is compared whole; one written into is not replaced at all. + if res.Into == "" { + differs, lines := differenceOf(string(content), res.Content) + h.Facts = &Facts{Differs: differs, Difference: lines} + } } case *declaration.Directory: info, err := os.Lstat(res.Path) @@ -628,6 +746,9 @@ func hold(ctx context.Context, sys system.System, r declaration.Resource, module case h.Running && !seen.running: changed = "stopped" } + if exists { + h.Facts = factsOf(ctx, seen, res, run) + } default: return out, h, fmt.Errorf("a %s cannot be held", r.Kind()) } diff --git a/internal/apply/hold_test.go b/internal/apply/hold_test.go index 5f67996..3d7d821 100644 --- a/internal/apply/hold_test.go +++ b/internal/apply/hold_test.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "sort" "strings" "testing" @@ -18,7 +19,11 @@ import ( // label when a host made it. Every command it is asked is written down. type machine struct { containers map[string]*fakeContainer - asked []string + // images is what `image inspect --format {{.Created}}` answers per image or id; members is + // what `network inspect` lists per network (ADR 0163). + images map[string]string + members map[string][]string + asked []string // wgUp is what `wg show interfaces` answers: the tunnels up on the machine. wgUp string // handshakes is what `wg show latest-handshakes` answers, and handshakesFail the @@ -97,6 +102,9 @@ type fakeContainer struct { id string running bool spec string + // What a take compares (ADR 0163), answered in the long inspect form when set. + image, imageID string + networks, mounts, ports []string } func (m *machine) run(_ context.Context, name string, args ...string) (string, error) { @@ -140,9 +148,38 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e running = "true" } if strings.HasPrefix(args[3], "{{.Id}}") { - return c.id + "\t" + running + "\t" + c.spec + "\n", nil + line := c.id + "\t" + running + "\t" + c.spec + if c.image != "" { + line += "\t" + c.image + "\t" + c.imageID + "\t" + strings.Join(c.networks, ",") + "," + + "\t" + strings.Join(c.mounts, ",") + "," + "\t" + strings.Join(c.ports, ",") + "," + } + return line + "\n", nil } return running + "\t" + c.spec + "\n", nil + case "image": + if len(args) > 1 && args[1] == "inspect" { + if created, ok := m.images[args[len(args)-1]]; ok { + return created + "\n", nil + } + return "", errors.New("no such image") + } + return "", nil + case "network": + if len(args) > 1 && args[1] == "inspect" { + return strings.Join(m.members[args[len(args)-1]], ",") + ",\n", nil + } + return "", nil + case "ps": + var lines []string + for name, c := range m.containers { + state := "exited" + if c.running { + state = "running" + } + lines = append(lines, name+"\t"+c.image+"\t"+state) + } + sort.Strings(lines) + return strings.Join(lines, "\n") + "\n", nil case "rm": delete(m.containers, args[len(args)-1]) return "", nil diff --git a/internal/apply/strays.go b/internal/apply/strays.go new file mode 100644 index 0000000..f3a0f68 --- /dev/null +++ b/internal/apply/strays.go @@ -0,0 +1,54 @@ +package apply + +import ( + "context" + "sort" + "strings" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// Strays is what runs on the machine that the mesh neither wrote nor holds (novox/hq ADR 0163): +// every container the runtime has that no record names and no hold names. The question nothing +// answered on 2026-09-23, when a renamed resource left its old container running for a day; asked +// on every apply now, and reported, so a thing left behind is seen the day it is left. +// +// Containers only, today. A listener nobody declared is harder to attribute to a thing, and the +// machine's own services are not strays; that account is issue 160's. +func Strays(ctx context.Context, run Runner, known store.State) ([]store.Stray, error) { + cri, err := containerRuntime(ctx, run) + if err != nil { + return nil, nil // a machine with no runtime has no containers to stray + } + out, err := run(ctx, cri, "ps", "-a", "--format", "{{.Names}}\t{{.Image}}\t{{.State}}") + if err != nil { + return nil, err + } + ours := map[string]bool{} + for _, r := range known.Resources { + if declaration.Type(r.Type) == declaration.TypeContainer { + ours[r.Target] = true + } + } + for _, h := range known.Held { + if h.Kind == string(declaration.TypeContainer) { + ours[h.Target] = true + } + } + var strays []store.Stray + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + parts := strings.Split(line, "\t") + name := strings.TrimSpace(parts[0]) + if name == "" || ours[name] { + continue + } + detail := "" + if len(parts) > 2 { + detail = strings.TrimSpace(parts[1]) + ", " + strings.TrimSpace(parts[2]) + } + strays = append(strays, store.Stray{Kind: string(declaration.TypeContainer), Name: name, Detail: detail}) + } + sort.Slice(strays, func(i, j int) bool { return strays[i].Name < strays[j].Name }) + return strays, nil +} diff --git a/internal/link/messages.go b/internal/link/messages.go index 1c1520d..dd63573 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -110,6 +110,10 @@ type Report struct { // and the mesh's up in its place, and where the found configuration's original was kept. Tunnel *CarriedTunnel `json:"tunnel,omitempty"` + // Strays is what runs on the machine that the mesh neither wrote nor holds (novox/hq ADR + // 0163): containers nobody declared and nobody holds, the ones a cutover leaves behind. + Strays []Stray `json:"strays,omitempty"` + // Profile is what this machine can do, detected again by the apply that reports (novox/hq // ADR 0161) — the same shape enrolment sends — so a capability gained or lost since enrolment, // a network manager switched, reaches the mesh at the next push rather than never. @@ -205,6 +209,16 @@ type Held struct { Changed string `json:"changed,omitempty"` // Kept is where a file's original was kept. Kept string `json:"kept,omitempty"` + // Facts is the found thing beside what the module declares — what a take compares (novox/hq + // ADR 0163). The same shape the host keeps; the controller reads it as data. + Facts map[string]any `json:"facts,omitempty"` +} + +// A Stray is a container the mesh neither wrote nor holds (ADR 0163). +type Stray struct { + Kind string `json:"kind"` + Name string `json:"name"` + Detail string `json:"detail,omitempty"` } // Reach is one thing reachable on the machine: a listening socket, or a published container port. diff --git a/internal/store/former_test.go b/internal/store/former_test.go new file mode 100644 index 0000000..57451f4 --- /dev/null +++ b/internal/store/former_test.go @@ -0,0 +1,29 @@ +package store + +import "testing" + +// A resource whose target moves leaves what the host wrote under the old target on record as a +// former one, undeclared by construction, so the next apply removes it (novox/hq issue 097, ADR 0163). +func TestARecordWhoseTargetMovedKeepsTheFormerTargetToRemove(t *testing.T) { + s := State{} + s.Record(Applied{ID: "gitea.server", Type: "container", Target: "mesh-gitea", Origin: OriginDeclared}) + s.Record(Applied{ID: "gitea.server", Type: "container", Target: "gitea", Origin: OriginDeclared}) + if len(s.Resources) != 2 { + t.Fatalf("a moved target produced %d record(s): %+v", len(s.Resources), s.Resources) + } + orphans := s.Orphans(map[string]bool{"gitea.server": true}, OriginDeclared) + if len(orphans) != 1 || orphans[0].Target != "mesh-gitea" || !IsFormer(orphans[0].ID) { + t.Fatalf("the former target is not an orphan to remove: %+v", orphans) + } + s.Forget(orphans[0].ID) + if len(s.Resources) != 1 || s.Resources[0].Target != "gitea" { + t.Fatalf("forgetting the former target touched the current one: %+v", s.Resources) + } + // The same target again is not a move; a carried record is not the host's to remove. + s.Record(Applied{ID: "gitea.server", Type: "container", Target: "gitea", Origin: OriginDeclared}) + s.Record(Applied{ID: "bundle", Type: "file", Target: "/a"}) + s.Record(Applied{ID: "bundle", Type: "file", Target: "/b"}) + if len(s.Resources) != 2 { + t.Fatalf("an unmoved or carried record grew the list: %+v", s.Resources) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 7306725..53296ed 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -18,6 +18,7 @@ import ( "os" "path/filepath" "sort" + "strings" "time" ) @@ -274,6 +275,41 @@ type Held struct { // reverted: that is how a predecessor still writing is caught. Changed string `json:"changed,omitempty"` ChangedAt time.Time `json:"changed_at,omitempty"` + // Facts is what a take would compare: the found thing beside what the module declares + // (novox/hq ADR 0163). Read fresh on every apply while held, so the controller's preview + // speaks of the machine as it is. + Facts *Facts `json:"facts,omitempty"` +} + +// Facts is a held thing beside what its module declares — what a take compares (ADR 0163). +type Facts struct { + // A found container: the image it runs and when that image was made; the networks it is on + // and the other containers on each; what it mounts; what it publishes. + Image string `json:"image,omitempty"` + ImageCreated string `json:"image_created,omitempty"` + Networks map[string][]string `json:"networks,omitempty"` + Mounts []string `json:"mounts,omitempty"` + Ports []string `json:"ports,omitempty"` + // What the module declares for it, and the declared image's creation date when the image + // is on the machine already. + DeclaredImage string `json:"declared_image,omitempty"` + DeclaredImageCreated string `json:"declared_image_created,omitempty"` + DeclaredPorts []string `json:"declared_ports,omitempty"` + DeclaredVolumes []string `json:"declared_volumes,omitempty"` + // Downgrade is true when both creation dates are known and the declared image is the older. + Downgrade bool `json:"downgrade,omitempty"` + // A found file: whether the declared content differs from what was found, and how, as lines + // only in the found file (-) and lines only in the declared one (+), bounded. + Differs bool `json:"differs,omitempty"` + Difference []string `json:"difference,omitempty"` +} + +// A Stray is something running on the machine that the mesh neither wrote nor holds +// (novox/hq ADR 0163): the answer to "what is here that nobody asked for". +type Stray struct { + Kind string `json:"kind"` + Name string `json:"name"` + Detail string `json:"detail,omitempty"` } // Recorded reports whether this host has a record, of any origin, of putting something of this @@ -462,6 +498,20 @@ func Save(path string, s State) error { func (s *State) Record(a Applied) { for i, existing := range s.Resources { if existing.ID == a.ID { + // A resource whose target moved leaves what the host wrote under the old target + // behind — a container under the old name, a file at the old path. Rewriting the + // record would erase the only trace of it (novox/hq issue 097, ADR 0163), so the old + // target stays on record as a former one, undeclared by construction, until the next + // apply removes it the way it removes anything the host wrote and no longer declares. + // What was found is held, never recorded here, and so never removed by this. + if originOf(existing) == OriginDeclared && existing.Target != "" && a.Target != "" && + existing.Target != a.Target && existing.Type == a.Type { + former := existing + former.ID = FormerID(existing.ID, existing.Target) + s.Resources[i] = a + s.Resources = append(s.Resources, former) + return + } s.Resources[i] = a return } @@ -469,6 +519,13 @@ func (s *State) Record(a Applied) { s.Resources = append(s.Resources, a) } +// FormerID names the record of a resource's former target: the resource's id and the target it +// had, so the record is distinct from the current one and is never what a declaration names. +func FormerID(id, target string) string { return id + "@former:" + target } + +// IsFormer says whether a record names a former target. +func IsFormer(id string) bool { return strings.Contains(id, "@former:") } + // Forget drops a resource from what the node owns. func (s *State) Forget(id string) { kept := s.Resources[:0]