From fcc447c21670fc971357d1c6a73df93f7417626d Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 17:11:26 +0200 Subject: [PATCH 01/52] Read a node's adoption from every declaration, so the host knows which modules are untaken (hq ADR 0100) --- internal/declaration/adoption_test.go | 102 ++++++++++++++++++++++++++ internal/declaration/declaration.go | 101 ++++++++++++++++++++++++- 2 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 internal/declaration/adoption_test.go diff --git a/internal/declaration/adoption_test.go b/internal/declaration/adoption_test.go new file mode 100644 index 0000000..8309329 --- /dev/null +++ b/internal/declaration/adoption_test.go @@ -0,0 +1,102 @@ +package declaration + +import ( + "strings" + "testing" +) + +// Defends novox/hq ADR 0100: every declaration says whether the node is adopted and which of its +// modules are taken, and the host refuses one it cannot read that from unambiguously. + +const adoptedResources = `"resources":[ + {"id":"hello-web.page","type":"file","path":"/var/lib/hello-web/index.html","content":"a\n"}, + {"id":"hello-web.server","type":"container","name":"hello-web","image":"sha256:` + sixtyFour + `"}, + {"id":"hello-web.data","type":"directory","path":"/var/lib/hello-web"} + ]` + +const sixtyFour = "0000000000000000000000000000000000000000000000000000000000000000" + +func TestAnAdoptionIsReadWithTheDeclaration(t *testing.T) { + d, err := Parse([]byte(`{"adoption":{"taken":["postgres"],"untaken":{"hello-web":["hello-web.page","hello-web.server"]}}, + "declaration":1,` + adoptedResources + `}`)) + if err != nil { + t.Fatal(err) + } + if d.Adoption == nil { + t.Fatal("the adoption was dropped") + } + if len(d.Adoption.Taken) != 1 || d.Adoption.Taken[0] != "postgres" { + t.Errorf("taken read as %v", d.Adoption.Taken) + } + if module, ok := d.Adoption.UntakenModuleOf("hello-web.server"); !ok || module != "hello-web" { + t.Errorf("the container's untaken module read as %q, %v", module, ok) + } + if _, ok := d.Adoption.UntakenModuleOf("hello-web.data"); ok { + t.Error("a resource the adoption does not name was said to be untaken") + } +} + +func TestADeclarationWithNoAdoptionIsConverged(t *testing.T) { + d, err := Parse([]byte(`{"declaration":1,` + adoptedResources + `}`)) + if err != nil { + t.Fatal(err) + } + if d.Adoption != nil { + t.Errorf("a declaration saying nothing about adoption read as adopted: %+v", d.Adoption) + } + if _, ok := d.Adoption.UntakenModuleOf("hello-web.page"); ok { + t.Error("a converged node has an untaken module") + } +} + +func TestAnAdoptionNamingAnUnknownIDIsRefused(t *testing.T) { + refusal := refusalFor(t, `{"adoption":{"taken":[],"untaken":{"hello-web":["hello-web.missing"]}}, + "declaration":1,`+adoptedResources+`}`) + if !strings.Contains(strings.Join(refusal.Problems, "\n"), "hello-web.missing") { + t.Errorf("the unknown id was not named: %v", refusal.Problems) + } +} + +func TestAnAdoptionMayOnlyHoldFilesAndContainers(t *testing.T) { + refusal := refusalFor(t, `{"adoption":{"taken":[],"untaken":{"hello-web":["hello-web.data"]}}, + "declaration":1,`+adoptedResources+`}`) + if !strings.Contains(strings.Join(refusal.Problems, "\n"), "only a file or a container") { + t.Errorf("a directory was accepted as holdable: %v", refusal.Problems) + } +} + +func TestAnIDUnderTwoModulesIsRefused(t *testing.T) { + refusal := refusalFor(t, `{"adoption":{"taken":[],"untaken":{"a":["hello-web.page"],"b":["hello-web.page"]}}, + "declaration":1,`+adoptedResources+`}`) + if !strings.Contains(strings.Join(refusal.Problems, "\n"), "both") { + t.Errorf("an id under two modules was accepted: %v", refusal.Problems) + } +} + +func TestAModuleBothTakenAndUntakenIsRefused(t *testing.T) { + refusal := refusalFor(t, `{"adoption":{"taken":["hello-web"],"untaken":{"hello-web":["hello-web.page"]}}, + "declaration":1,`+adoptedResources+`}`) + if !strings.Contains(strings.Join(refusal.Problems, "\n"), "both taken and untaken") { + t.Errorf("a module both taken and untaken was accepted: %v", refusal.Problems) + } +} + +func TestTheMeshsOwnResourcesAreNeverUntaken(t *testing.T) { + refusal := refusalFor(t, `{"adoption":{"taken":[],"untaken":{"x":["adoption.guard"]}}, + "declaration":1,"resources":[ + {"id":"adoption.guard","type":"file","path":"/etc/mesh/guard.nft","content":"x"}]}`) + if !strings.Contains(strings.Join(refusal.Problems, "\n"), "belongs to no module") { + t.Errorf("an adoption. id was accepted as untaken: %v", refusal.Problems) + } +} + +func TestAnAdoptionWithAnUnknownFieldIsRefused(t *testing.T) { + refusalFor(t, `{"adoption":{"taken":[],"held":["x"]},"declaration":1,`+adoptedResources+`}`) +} + +func TestACarriedBundleCannotSayTheNodeIsAdopted(t *testing.T) { + _, err := ParseTrusted([]byte(`{"adoption":{"taken":[]},"declaration":1,` + adoptedResources + `}`)) + if err == nil || !strings.Contains(err.Error(), "only the mesh can say") { + t.Fatalf("a bundle claiming adoption was not refused: %v", err) + } +} diff --git a/internal/declaration/declaration.go b/internal/declaration/declaration.go index cd7cbd3..41605f6 100644 --- a/internal/declaration/declaration.go +++ b/internal/declaration/declaration.go @@ -832,6 +832,103 @@ type Declaration struct { // Resources, in the order they are applied. The host does not sort them: ordering is a // decision, and deciding is not what the host does (novox/hq ADR 0005). Resources []Resource + + // Adoption says this node is adopted, and which of its modules have been taken. Nil is a + // converged node — which is every node the mesh raised before adoption existed, and so the + // only form an older controller ever sends (novox/hq ADR 0100). + Adoption *Adoption +} + +// Adoption is a node's mode, as the controller records it: the node is adopted, and these are +// the modules taken on it so far (novox/hq ADR 0100). +// +// **Authoritative, and only ever stated by the controller.** A host does not work out whether it +// is adopted; it is told, in every declaration, so a host restarted from the declaration it kept +// is in the same mode it was in before. +// +// Untaken names, per module assigned here and not yet taken, the ids of its file and container +// resources — the only shapes a predecessor can already have on the machine. The host cannot +// split a resource id into its module, because module names may contain dots, so the controller +// says which ids belong to which module rather than leaving the host to guess. +type Adoption struct { + Taken []string `json:"taken"` + Untaken map[string][]string `json:"untaken,omitempty"` +} + +// AdoptionPrefix is the id prefix of what the mesh itself declares because a node is adopted — +// its openings and its guard. Nothing under it belongs to a module, so none of it is ever held. +const AdoptionPrefix = "adoption." + +// UntakenModuleOf says which untaken module declares a resource, if any. +func (a *Adoption) UntakenModuleOf(id string) (string, bool) { + if a == nil { + return "", false + } + for module, ids := range a.Untaken { + if slices.Contains(ids, id) { + return module, true + } + } + return "", false +} + +// checkAdoption holds what an adoption says against the resources beside it. Every problem is a +// refusal: a host that misread which module is untaken would replace a predecessor's service the +// operator never took. +func checkAdoption(a *Adoption, resources []Resource, allowActions bool) []string { + if a == nil { + return nil + } + if allowActions { + // The bundle is carried with the binary and raises a foundation before any mesh exists. + // Whether a node is adopted is the controller's record, and a bundle that claimed it would + // be the host deciding its own mode (novox/hq ADR 0100). + return []string{"a carried bundle says the node is adopted, and only the mesh can say " + + "that: a node's mode is the controller's record, sent in every declaration"} + } + kinds := map[string]Type{} + for _, r := range resources { + kinds[r.Identity()] = r.Kind() + } + var problems []string + for _, module := range a.Taken { + if _, both := a.Untaken[module]; both { + problems = append(problems, fmt.Sprintf( + "adoption: the module %q is said to be both taken and untaken", module)) + } + } + owner := map[string]string{} + modules := make([]string, 0, len(a.Untaken)) + for module := range a.Untaken { + modules = append(modules, module) + } + sort.Strings(modules) + for _, module := range modules { + for _, id := range a.Untaken[module] { + if strings.HasPrefix(id, AdoptionPrefix) { + problems = append(problems, fmt.Sprintf( + "adoption: %q is the mesh's own and belongs to no module, so it cannot be untaken", id)) + continue + } + if first, twice := owner[id]; twice { + problems = append(problems, fmt.Sprintf( + "adoption: %q is said to belong to both %q and %q", id, first, module)) + continue + } + owner[id] = module + kind, declared := kinds[id] + switch { + case !declared: + problems = append(problems, fmt.Sprintf( + "adoption: %q of the untaken module %q is not in this declaration", id, module)) + case kind != TypeFile && kind != TypeContainer: + problems = append(problems, fmt.Sprintf( + "adoption: %q of the untaken module %q is a %s, and only a file or a "+ + "container can be found on a machine", id, module, kind)) + } + } + } + return problems } // RefusalError refuses a whole declaration, naming every problem at once. @@ -872,6 +969,7 @@ func ParseTrusted(raw []byte) (*Declaration, error) { return parse(raw, true) } type envelope struct { Version int `json:"declaration"` For string `json:"for,omitempty"` + Adoption *Adoption `json:"adoption,omitempty"` Resources []json.RawMessage `json:"resources"` } @@ -889,7 +987,7 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) { env.Version, Version)}} } - d := &Declaration{Version: env.Version, For: env.For} + d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption} var problems []string if len(env.Resources) == 0 { @@ -952,6 +1050,7 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) { problems = append(problems, resource.validate(where, allowActions)...) d.Resources = append(d.Resources, resource) } + problems = append(problems, checkAdoption(env.Adoption, d.Resources, allowActions)...) if len(problems) > 0 { return nil, &RefusalError{Problems: problems} From 3a613113be25eef80f14bb4b600afbfcad2b7357 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 17:14:04 +0200 Subject: [PATCH 02/52] Keep what an adopted node was found holding until its module is taken, and report it held (hq ADR 0100) --- cmd/mesh-host/main.go | 10 +- internal/apply/apply.go | 59 +++++- internal/apply/hold.go | 200 ++++++++++++++++++++ internal/apply/hold_test.go | 365 ++++++++++++++++++++++++++++++++++++ internal/store/store.go | 83 ++++++++ 5 files changed, 714 insertions(+), 3 deletions(-) create mode 100644 internal/apply/hold.go create mode 100644 internal/apply/hold_test.go diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index f4822b0..0bb9631 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -17,6 +17,7 @@ import ( "fmt" "os" "os/signal" + "path/filepath" "sort" "strings" "syscall" @@ -740,8 +741,8 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D // Declared, not carried. A declaration from the mesh removes only what the mesh previously // declared — never what this machine raised for itself from its bundle (04-ISSUES/010). - outcome, updated, applyErr := apply.Apply(ctx, built, declared, known, store.OriginDeclared, - apply.ExecRunner, nil, sealOpener(opts.state)) + outcome, updated, applyErr := apply.ApplyKeeping(ctx, built, declared, known, store.OriginDeclared, + apply.ExecRunner, nil, sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state))) // Saved whichever way it went. Recording only on success would lose the footprint of a // failed apply, and that footprint is on the machine either way. @@ -761,6 +762,11 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw)} for _, change := range outcome.Outcomes { + // What is held is not what this machine owns: it was found, and is kept as it was until + // its module is taken (novox/hq ADR 0100). + if change.Action == "held" { + continue + } report.Applied = append(report.Applied, change.ID) } // Kept whichever way it went, so a node that is disconnected next minute still knows what it diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 893aa17..2e25d56 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -61,7 +61,8 @@ type Report struct { // nothing is the ordinary steady state, and saying so is not the same as saying it failed. func (r Report) Changed() bool { for _, o := range r.Outcomes { - if o.Action != "unchanged" { + // Holding is keeping the machine as it was found, which is not moving it. + if o.Action != "unchanged" && o.Action != "held" { return true } } @@ -121,6 +122,23 @@ func Apply( run Runner, log func(string), unseal Unseal, +) (Report, store.State, error) { + return ApplyKeeping(ctx, sys, d, known, origin, run, log, unseal, nil) +} + +// ApplyKeeping is Apply on a node that may be adopted: keep is where the original of a file found +// there is recorded before anything else happens to it (novox/hq ADR 0100). Nil is a caller that +// can never be handed an adopted declaration — the carried bundle, which may not say it. +func ApplyKeeping( + ctx context.Context, + sys system.System, + d *declaration.Declaration, + known store.State, + origin string, + run Runner, + log func(string), + unseal Unseal, + keep Keep, ) (Report, store.State, error) { if log == nil { log = func(string) {} @@ -182,6 +200,40 @@ func Apply( // a declaration", and they are fixed in different places. var failures []*Error for _, resource := range d.Resources { + // **On an adopted node, what is found is kept until its module is taken** (novox/hq ADR + // 0100). Before anything is applied: a file present with no record of this host writing + // it, or a container present under that name that no host made, is held as it is and + // reported. Once held it stays held — changed or gone — until its module is taken, and + // it is never recorded as applied, so it is never removed as an orphan either. + if d.Adoption != nil && holdable(resource) { + if module, untaken := d.Adoption.UntakenModuleOf(resource.Identity()); untaken { + was, already := known.HeldAt(resource.Identity()) + isFound := false + if !already { + var err error + if isFound, err = found(ctx, resource, run, known); err != nil { + failures = append(failures, &Error{Resource: resource.Identity(), Err: err, Done: report}) + log(fmt.Sprintf(" failed %s (%s): %v", resource.Identity(), resource.Target(), err)) + continue + } + } + if already || isFound { + outcome, held, err := hold(ctx, resource, module, was, already, run, keep, time.Now().UTC()) + if err != nil { + failures = append(failures, &Error{Resource: resource.Identity(), Err: err, Done: report}) + log(fmt.Sprintf(" failed %s (%s): %v", resource.Identity(), outcome.Target, err)) + continue + } + known.RecordHeld(held) + report.Outcomes = append(report.Outcomes, outcome) + if !already || held.Changed != was.Changed { + log(fmt.Sprintf(" held %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail)) + } + continue + } + } + } + was, _ := known.Find(resource.Identity()) outcome, err := applyOne(ctx, sys, resource, run, changed, declares, was, unseal) if err != nil { @@ -229,6 +281,11 @@ func Apply( Wrote: outcome.wrote, Holds: holds(resource), }) + // Its module has been taken, and what was held for it is now the mesh's. + if held, wasHeld := known.HeldAt(resource.Identity()); wasHeld { + known.Release(held.ID) + outcome.Detail = takenDetail(held) + } report.Outcomes = append(report.Outcomes, outcome) if outcome.Action != "unchanged" { changed[resource.Identity()] = true diff --git a/internal/apply/hold.go b/internal/apply/hold.go new file mode 100644 index 0000000..89a4d44 --- /dev/null +++ b/internal/apply/hold.go @@ -0,0 +1,200 @@ +package apply + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// Keep records the original of a file found on an adopted node, before anything else happens to +// it, and says where (novox/hq ADR 0100). It never overwrites an original it already kept: the +// first copy is the one that was there before the mesh. +type Keep func(path string, content []byte, mode os.FileMode) (string, error) + +// KeepIn keeps originals under dir/kept, each named for the path it came from, readable by root +// alone — a predecessor's configuration may carry its credentials. +func KeepIn(dir string) Keep { + return func(path string, content []byte, _ os.FileMode) (string, error) { + sum := sha256.Sum256([]byte(path)) + kept := filepath.Join(dir, "kept", + hex.EncodeToString(sum[:])[:16]+"-"+filepath.Base(path)) + if _, err := os.Lstat(kept); err == nil { + return kept, nil + } + if err := os.MkdirAll(filepath.Dir(kept), 0o700); err != nil { + return "", err + } + if err := writeAtomically(kept, content, 0o600); err != nil { + return "", err + } + back, err := os.ReadFile(kept) + if err != nil || string(back) != string(content) { + return "", fmt.Errorf("kept the original of %s at %s and cannot read it back", path, kept) + } + return kept, nil + } +} + +// holdable is whether a resource is one a predecessor can already have on the machine: a file at +// a path, or a container under a name. +func holdable(r declaration.Resource) bool { + return r.Kind() == declaration.TypeFile || r.Kind() == declaration.TypeContainer +} + +// found is whether a declared file or container is present on the machine with no record of this +// host making it (novox/hq ADR 0100). A container carrying the host's own spec label was made by +// a host, whatever this store says, so it is never found. +func found(ctx context.Context, r declaration.Resource, run Runner, known store.State) (bool, error) { + if known.Recorded(string(r.Kind()), r.Target()) { + return false, nil + } + switch res := r.(type) { + case *declaration.File: + _, err := os.Lstat(res.Path) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + return err == nil, err + case *declaration.Container: + seen, exists, err := inspectFound(ctx, res.Name, run) + if err != nil || !exists { + return false, err + } + return seen.spec == "", nil + } + return false, nil +} + +type foundContainer struct { + id string + running bool + spec string +} + +// inspectFound reads a container by name the way a hold needs it: its id, whether it runs, and +// whether a host made it. +func inspectFound(ctx context.Context, name string, run Runner) (foundContainer, bool, error) { + cri, err := containerRuntime(ctx, run) + if err != nil { + return foundContainer{}, false, fmt.Errorf("%w, so nothing can be said about %q", err, name) + } + out, err := run(ctx, cri, "inspect", "--format", + "{{.Id}}\t{{.State.Running}}\t{{index .Config.Labels \""+specLabel+"\"}}", name) + if err != nil { + return foundContainer{}, false, nil + } + parts := strings.Split(strings.TrimSpace(out), "\t") + for len(parts) < 3 { + parts = append(parts, "") + } + spec := strings.TrimSpace(parts[2]) + if spec == "" { + spec = "" + } + return foundContainer{id: strings.TrimSpace(parts[0]), running: parts[1] == "true", spec: spec}, true, nil +} + +// hold keeps a found file or container as it is, and reports it — the first time by recording +// what was found, every time after by comparing against that. Nothing is reverted, restarted or +// created: a held target that disappears stays held and gone until its module is taken. +func hold(ctx context.Context, r declaration.Resource, module string, was store.Held, already bool, + run Runner, keep Keep, now time.Time) (Outcome, store.Held, error) { + out := begin(r) + h := was + if !already { + h = store.Held{ID: r.Identity(), Module: module, Kind: string(r.Kind()), + Target: r.Target(), Since: now} + } + h.Module = module + + var changed string + switch res := r.(type) { + case *declaration.File: + info, err := os.Lstat(res.Path) + switch { + case errors.Is(err, os.ErrNotExist): + if !already { + return out, h, fmt.Errorf("%s was found and is gone before it could be kept", res.Path) + } + changed = "gone" + case err != nil: + return out, h, err + default: + content, err := os.ReadFile(res.Path) + if err != nil { + return out, h, fmt.Errorf("%s was found and cannot be read to keep it: %w", res.Path, err) + } + if !already { + // The original first, before anything is recorded: a hold with no kept copy + // would be a promise the host cannot keep. + if keep == nil { + return out, h, fmt.Errorf( + "%s was found on this adopted node and this host has nowhere to keep its original", res.Path) + } + kept, err := keep(res.Path, content, info.Mode().Perm()) + if err != nil { + return out, h, fmt.Errorf("keeping the original of %s: %w", res.Path, err) + } + h.Kept = kept + h.Digest = digestOf(string(content)) + h.Mode = fmt.Sprintf("%04o", info.Mode().Perm()) + if st, ok := info.Sys().(*syscall.Stat_t); ok { + h.Owner = fmt.Sprintf("%d:%d", st.Uid, st.Gid) + } + } else if digestOf(string(content)) != h.Digest { + changed = "rewritten" + } + } + case *declaration.Container: + seen, exists, err := inspectFound(ctx, res.Name, run) + if err != nil { + return out, h, err + } + switch { + case !exists && !already: + return out, h, fmt.Errorf("container %s was found and is gone before it could be held", res.Name) + case !already: + h.Container, h.Running = seen.id, seen.running + case !exists: + changed = "gone" + case seen.id != h.Container: + changed = "replaced" + case h.Running && !seen.running: + changed = "stopped" + } + default: + return out, h, fmt.Errorf("a %s cannot be held", r.Kind()) + } + + if changed != h.Changed { + h.Changed = changed + h.ChangedAt = now + if changed == "" { + h.ChangedAt = time.Time{} + } + } + out.Action = "held" + out.Detail = "found on the machine; kept until " + module + " is taken" + if h.Changed != "" { + out.Detail += "; " + h.Changed + " by something other than the mesh since it was found, and not reverted" + } + return out, h, nil +} + +// takenDetail is what an outcome says when a module's cutover replaced what was held for it. +func takenDetail(h store.Held) string { + if h.Kept != "" { + return "taken: replaced what was found; original kept at " + h.Kept + } + return "taken: replaced what was found" +} diff --git a/internal/apply/hold_test.go b/internal/apply/hold_test.go new file mode 100644 index 0000000..6b6e947 --- /dev/null +++ b/internal/apply/hold_test.go @@ -0,0 +1,365 @@ +package apply + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// Defends novox/hq ADR 0100: on an adopted node, what is found is kept until its module is taken. + +// machine is a fake container runtime holding containers by name: id, running, and the host's spec +// label when a host made it. Every command it is asked is written down. +type machine struct { + containers map[string]*fakeContainer + asked []string +} + +type fakeContainer struct { + id string + running bool + spec string +} + +func (m *machine) run(_ context.Context, name string, args ...string) (string, error) { + m.asked = append(m.asked, name+" "+strings.Join(args, " ")) + switch args[0] { + case "info": + return "27.0\n", nil + case "inspect": + c, ok := m.containers[args[len(args)-1]] + if !ok { + return "", errors.New("no such container") + } + running := "false" + if c.running { + running = "true" + } + if strings.HasPrefix(args[2], "{{.Id}}") { + return c.id + "\t" + running + "\t" + c.spec + "\n", nil + } + return running + "\t" + c.spec + "\n", nil + case "rm": + delete(m.containers, args[len(args)-1]) + return "", nil + case "run": + var name, spec string + for i, a := range args { + if a == "--name" { + name = args[i+1] + } + if a == "--label" && strings.HasPrefix(args[i+1], specLabel+"=") { + spec = strings.TrimPrefix(args[i+1], specLabel+"=") + } + } + m.containers[name] = &fakeContainer{id: "made-by-host", running: true, spec: spec} + return "made-by-host\n", nil + } + return "", nil +} + +func (m *machine) removed(name string) bool { + for _, a := range m.asked { + if strings.HasPrefix(a, "docker rm") && strings.HasSuffix(a, " "+name) { + return true + } + } + return false +} + +func adopted(t *testing.T, adoption, resources string) *declaration.Declaration { + t.Helper() + return parse(t, `{"declaration":1,"adoption":`+adoption+`,"resources":[`+resources+`]}`) +} + +const untakenWeb = `{"taken":[],"untaken":{"hello-web":["hello-web.page","hello-web.server"]}}` +const takenWeb = `{"taken":["hello-web"]}` + +func webResources(page string) string { + return `{"id":"hello-web.page","type":"file","path":"` + page + `","content":"the mesh's page\n"}, + {"id":"hello-web.server","type":"container","name":"hello-web","image":"` + pinned + `"}` +} + +func applyAdopted(t *testing.T, d *declaration.Declaration, known store.State, m *machine, keepDir string) (Report, store.State) { + t.Helper() + report, state, err := ApplyKeeping(context.Background(), archHost(t), d, known, + store.OriginDeclared, m.run, nil, nil, KeepIn(keepDir)) + if err != nil { + t.Fatalf("apply failed: %v", err) + } + return report, state +} + +func outcomeOf(r Report, id string) Outcome { + for _, o := range r.Outcomes { + if o.ID == id { + return o + } + } + return Outcome{} +} + +func predecessor(t *testing.T) (dir, page string, m *machine) { + t.Helper() + dir = t.TempDir() + page = filepath.Join(dir, "index.html") + if err := os.WriteFile(page, []byte("the predecessor's page\n"), 0o640); err != nil { + t.Fatal(err) + } + return dir, page, &machine{containers: map[string]*fakeContainer{ + "hello-web": {id: "predecessor-id", running: true}, + }} +} + +func TestAFoundFileOfAnUntakenModuleIsKeptAsItIs(t *testing.T) { + dir, page, m := predecessor(t) + report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) + + got, _ := os.ReadFile(page) + if string(got) != "the predecessor's page\n" { + t.Fatalf("a found file was changed: %q", got) + } + info, _ := os.Stat(page) + if info.Mode().Perm() != 0o640 { + t.Errorf("a found file's mode was changed to %o", info.Mode().Perm()) + } + if o := outcomeOf(report, "hello-web.page"); o.Action != "held" || + !strings.Contains(o.Detail, "kept until hello-web is taken") { + t.Errorf("the found file was not reported held: %+v", o) + } + h, ok := state.HeldAt("hello-web.page") + if !ok || h.Module != "hello-web" || h.Mode != "0640" { + t.Fatalf("the hold was not recorded: %+v", h) + } + kept, err := os.ReadFile(h.Kept) + if err != nil || string(kept) != "the predecessor's page\n" { + t.Fatalf("the original was not kept: %q %v", kept, err) + } + if info, _ := os.Stat(h.Kept); info.Mode().Perm() != 0o600 { + t.Errorf("the kept original is mode %o", info.Mode().Perm()) + } + if _, recorded := state.Find("hello-web.page"); recorded { + t.Error("a held file was recorded as applied, so it would be removed as an orphan") + } + if report.Changed() { + t.Errorf("holding was reported as changing the machine: %+v", report.Outcomes) + } +} + +func TestAFoundContainerOfAnUntakenModuleIsNotReplaced(t *testing.T) { + dir, page, m := predecessor(t) + report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) + + if m.removed("hello-web") { + t.Fatal("a found container was removed") + } + for _, a := range m.asked { + if strings.HasPrefix(a, "docker run") { + t.Fatalf("a container was started over a found one: %s", a) + } + } + if outcomeOf(report, "hello-web.server").Action != "held" { + t.Errorf("the found container was not held: %+v", report.Outcomes) + } + if h, _ := state.HeldAt("hello-web.server"); h.Container != "predecessor-id" || !h.Running { + t.Errorf("the container as found was not recorded: %+v", h) + } +} + +func TestWhatIsNotFoundIsCreatedWhenAssigned(t *testing.T) { + // Assigning prepares: what the module declares that is not there is made. + dir := t.TempDir() + page := filepath.Join(dir, "index.html") + m := &machine{containers: map[string]*fakeContainer{}} + report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) + if o := outcomeOf(report, "hello-web.page"); o.Action != "created" { + t.Errorf("an absent file of an untaken module was not created: %+v", o) + } + if o := outcomeOf(report, "hello-web.server"); o.Action != "created" { + t.Errorf("an absent container of an untaken module was not created: %+v", o) + } + if len(state.Held) != 0 { + t.Errorf("something was held that was not found: %+v", state.Held) + } +} + +func TestAFileThisHostWroteIsNotFound(t *testing.T) { + // Found means present with no record. A record of any origin — carried or declared, this life + // of the node or an earlier one — means this host wrote it. + for _, origin := range []string{store.OriginCarried, store.OriginDeclared} { + dir, page, m := predecessor(t) + known := store.State{Resources: []store.Applied{ + {ID: "earlier-name", Type: "file", Target: page, Origin: origin}}} + report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), known, m, dir) + if o := outcomeOf(report, "hello-web.page"); o.Action == "held" { + t.Errorf("%s: a file this host has a record of was held: %+v", origin, o) + } + if _, held := state.HeldAt("hello-web.page"); held { + t.Errorf("%s: a recorded file was held", origin) + } + } +} + +func TestAContainerAHostMadeIsNotFound(t *testing.T) { + dir, page, m := predecessor(t) + m.containers["hello-web"].spec = "some-spec" + report, _ := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) + if o := outcomeOf(report, "hello-web.server"); o.Action == "held" { + t.Errorf("a container carrying the host's spec label was held: %+v", o) + } +} + +func TestTheGenesisStoreAdoptedInPlaceIsNotFound(t *testing.T) { + // ADR 0078: the foundation's store, raised from the bundle and recorded as carried, is adopted + // as a module by name. It is the mesh's own and must never read as a predecessor's. + dir := t.TempDir() + m := &machine{containers: map[string]*fakeContainer{"mesh-store": {id: "x", running: true}}} + known := store.State{Resources: []store.Applied{{ID: "store", Type: "container", Target: "mesh-store"}}} + d := adopted(t, `{"taken":[],"untaken":{"postgres":["postgres.server"]}}`, + `{"id":"postgres.server","type":"container","name":"mesh-store","image":"`+pinned+`"}`) + report, state := applyAdopted(t, d, known, m, dir) + if o := outcomeOf(report, "postgres.server"); o.Action == "held" { + t.Errorf("the carried store was held: %+v", o) + } + if len(state.Held) != 0 { + t.Errorf("the carried store was held: %+v", state.Held) + } +} + +func TestTakingAModuleReplacesWhatWasHeldAndTheOriginalSurvives(t *testing.T) { + dir, page, m := predecessor(t) + _, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) + h, _ := state.HeldAt("hello-web.page") + + report, state := applyAdopted(t, adopted(t, takenWeb, webResources(page)), state, m, dir) + got, _ := os.ReadFile(page) + if string(got) != "the mesh's page\n" { + t.Fatalf("taking the module did not converge the file: %q", got) + } + if !m.removed("hello-web") || m.containers["hello-web"].id != "made-by-host" { + t.Fatal("taking the module did not replace the found container") + } + if o := outcomeOf(report, "hello-web.page"); !strings.Contains(o.Detail, "original kept at "+h.Kept) { + t.Errorf("the cutover does not say where the original is: %+v", o) + } + if len(state.Held) != 0 { + t.Errorf("what was taken is still held: %+v", state.Held) + } + if _, recorded := state.Find("hello-web.page"); !recorded { + t.Error("a taken file was not recorded as applied") + } + kept, err := os.ReadFile(h.Kept) + if err != nil || string(kept) != "the predecessor's page\n" { + t.Errorf("the kept original did not survive the cutover: %q %v", kept, err) + } +} + +func TestAHeldFileIsNeverRemovedWhenItsModuleIsUnassigned(t *testing.T) { + dir, page, m := predecessor(t) + _, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) + + other := filepath.Join(dir, "other") + _, state = applyAdopted(t, adopted(t, `{"taken":[]}`, + `{"id":"x.other","type":"file","path":"`+other+`","content":"x"}`), state, m, dir) + if got, _ := os.ReadFile(page); string(got) != "the predecessor's page\n" { + t.Fatalf("a held file was touched when its module left: %q", got) + } + if m.removed("hello-web") { + t.Fatal("a held container was removed when its module left") + } + if _, still := state.HeldAt("hello-web.page"); !still { + t.Error("the hold was forgotten, so a return of the module would read the file as the mesh's") + } +} + +func TestAHeldFileRewrittenIsReportedAndNotReverted(t *testing.T) { + dir, page, m := predecessor(t) + d := adopted(t, untakenWeb, webResources(page)) + _, state := applyAdopted(t, d, store.State{}, m, dir) + + if err := os.WriteFile(page, []byte("the predecessor wrote again\n"), 0o640); err != nil { + t.Fatal(err) + } + report, state := applyAdopted(t, d, state, m, dir) + if got, _ := os.ReadFile(page); string(got) != "the predecessor wrote again\n" { + t.Fatalf("a held file was reverted: %q", got) + } + if h, _ := state.HeldAt("hello-web.page"); h.Changed != "rewritten" || h.ChangedAt.IsZero() { + t.Errorf("a rewrite was not recorded: %+v", h) + } + if o := outcomeOf(report, "hello-web.page"); !strings.Contains(o.Detail, "rewritten") { + t.Errorf("a rewrite was not reported: %+v", o) + } + h, _ := state.HeldAt("hello-web.page") + if kept, _ := os.ReadFile(h.Kept); string(kept) != "the predecessor's page\n" { + t.Errorf("the kept original was overwritten by a later write: %q", kept) + } +} + +func TestAHeldContainerStoppedOrReplacedIsReportedAndNotRestarted(t *testing.T) { + for _, c := range []struct { + change func(*machine) + want string + }{ + {func(m *machine) { m.containers["hello-web"].running = false }, "stopped"}, + {func(m *machine) { m.containers["hello-web"].id = "another" }, "replaced"}, + {func(m *machine) { delete(m.containers, "hello-web") }, "gone"}, + } { + dir, page, m := predecessor(t) + d := adopted(t, untakenWeb, webResources(page)) + _, state := applyAdopted(t, d, store.State{}, m, dir) + c.change(m) + m.asked = nil + _, state = applyAdopted(t, d, state, m, dir) + if h, _ := state.HeldAt("hello-web.server"); h.Changed != c.want { + t.Errorf("%s: recorded as %q", c.want, h.Changed) + } + for _, a := range m.asked { + if strings.HasPrefix(a, "docker run") || strings.HasPrefix(a, "docker rm") || + strings.HasPrefix(a, "docker start") { + t.Errorf("%s: the held container was acted on: %s", c.want, a) + } + } + } +} + +func TestAHeldFileThatVanishesIsNotCreated(t *testing.T) { + dir, page, m := predecessor(t) + d := adopted(t, untakenWeb, webResources(page)) + _, state := applyAdopted(t, d, store.State{}, m, dir) + if err := os.Remove(page); err != nil { + t.Fatal(err) + } + _, state = applyAdopted(t, d, state, m, dir) + if _, err := os.Stat(page); !errors.Is(err, os.ErrNotExist) { + t.Fatal("a held file that vanished was created before its module was taken") + } + if h, _ := state.HeldAt("hello-web.page"); h.Changed != "gone" { + t.Errorf("a vanished held file was not reported gone: %+v", h) + } +} + +func TestAConvergedNodeStillReplacesWhatItFinds(t *testing.T) { + // No adoption, no holds: byte for byte what a converged node did before ADR 0100. + dir, page, m := predecessor(t) + d := parse(t, `{"declaration":1,"resources":[`+webResources(page)+`]}`) + report, state := applyAdopted(t, d, store.State{}, m, dir) + if got, _ := os.ReadFile(page); string(got) != "the mesh's page\n" { + t.Errorf("a converged node kept a found file: %q", got) + } + if !m.removed("hello-web") { + t.Error("a converged node kept a found container") + } + if len(state.Held) != 0 || outcomeOf(report, "hello-web.page").Action == "held" { + t.Errorf("a converged node held something: %+v", state.Held) + } + if _, err := os.Stat(filepath.Join(dir, "kept")); !errors.Is(err, os.ErrNotExist) { + t.Error("a converged node kept originals") + } +} diff --git a/internal/store/store.go b/internal/store/store.go index d9c097f..e778420 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -77,6 +77,89 @@ type State struct { // undoing in reverse is the only ordering the host can derive without deciding anything. Resources []Applied `json:"resources"` UpdatedAt time.Time `json:"updated_at"` + + // Held is what this host found on the machine and is keeping as it is, until the module + // declaring it is taken (novox/hq ADR 0100). Never a Resource: nothing here was applied, so + // nothing here is ever removed as an orphan — what is held is not the host's to remove, even + // when its module is unassigned. + Held []Held `json:"held,omitempty"` +} + +// Held is one file or container found on an adopted node — present at a declared path or name, +// with no record of this host having made it — and kept as it was found. +type Held struct { + ID string `json:"id"` + Module string `json:"module"` + Kind string `json:"kind"` + Target string `json:"target"` + // Since is when it was first found. It stays held from then until its module is taken, even + // if it disappears: a vanished file is reported, not recreated. + Since time.Time `json:"since"` + + // A file's content as found, by digest; its mode and owner; and where the original was kept + // before anything else could happen to it. + Digest string `json:"digest,omitempty"` + Mode string `json:"mode,omitempty"` + Owner string `json:"owner,omitempty"` + Kept string `json:"kept,omitempty"` + + // A container's id as found, and whether it was running. + Container string `json:"container,omitempty"` + Running bool `json:"running,omitempty"` + + // Changed is what something other than the mesh has done to it since it was found — + // rewritten, stopped, replaced or gone — and empty while it is as found. Reported, never + // reverted: that is how a predecessor still writing is caught. + Changed string `json:"changed,omitempty"` + ChangedAt time.Time `json:"changed_at,omitempty"` +} + +// Recorded reports whether this host has a record, of any origin, of putting something of this +// kind at this target. What it has a record of is not found: it wrote it, in this life of the node +// or an earlier one — including a foundation raised from the bundle and adopted as modules later +// (novox/hq ADR 0078). +func (s State) Recorded(kind, target string) bool { + for _, r := range s.Resources { + if r.Type == kind && r.Target == target { + return true + } + } + return false +} + +// HeldAt returns what is held under a resource id. +func (s State) HeldAt(id string) (Held, bool) { + for _, h := range s.Held { + if h.ID == id { + return h, true + } + } + return Held{}, false +} + +// RecordHeld adds or replaces what is held under one id, preserving order. +func (s *State) RecordHeld(h Held) { + for i, existing := range s.Held { + if existing.ID == h.ID { + s.Held[i] = h + return + } + } + s.Held = append(s.Held, h) +} + +// Release drops a hold, once its module is taken and the host has converged what was held. +func (s *State) Release(id string) { + kept := s.Held[:0] + for _, h := range s.Held { + if h.ID != id { + kept = append(kept, h) + } + } + s.Held = kept + if len(s.Held) == 0 { + s.Held = nil + } } // Find returns what was applied under an identity. From 3c90d155b3e6037c2d24da2fa983ef1793b6e965 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 17:19:49 +0200 Subject: [PATCH 03/52] Converge openings through the firewall an adopted node was found with, and retire it only when the node converges (hq ADR 0100) --- internal/apply/apply.go | 31 +- internal/apply/opening.go | 109 ++++++ internal/apply/opening_test.go | 204 ++++++++++ internal/declaration/declaration.go | 90 ++++- internal/declaration/declaration_test.go | 12 +- internal/firewall/firewall.go | 431 +++++++++++++++++++++ internal/firewall/firewall_test.go | 335 ++++++++++++++++ internal/firewall/testdata/docker-only.nft | 297 ++++++++++++++ internal/store/store.go | 17 + internal/system/system.go | 3 + 10 files changed, 1522 insertions(+), 7 deletions(-) create mode 100644 internal/apply/opening.go create mode 100644 internal/apply/opening_test.go create mode 100644 internal/firewall/firewall.go create mode 100644 internal/firewall/firewall_test.go create mode 100644 internal/firewall/testdata/docker-only.nft diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 2e25d56..d8a4926 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -150,8 +150,21 @@ func ApplyKeeping( declared[r.Identity()] = true } + // Which firewall is found here, before anything else, since an unsupported one refuses the + // whole declaration (novox/hq ADR 0100). Nothing for a converged node. + fw, err := foundFirewall(ctx, d, &known, run, log) + if err != nil { + return report, known, &Error{Resource: "the firewall found on this machine", Err: err, Done: report} + } + for _, orphan := range known.Orphans(declared, origin) { - action, detail, err := remove(ctx, sys, orphan, run) + var action, detail string + var err error + if declaration.Type(orphan.Type) == declaration.TypeOpening { + action, detail, err = removeOpening(ctx, orphan, run, known.Firewall) + } else { + action, detail, err = remove(ctx, sys, orphan, run) + } if err != nil { return report, known, &Error{Resource: orphan.ID, Err: err, Done: report} } @@ -235,7 +248,13 @@ func ApplyKeeping( } was, _ := known.Find(resource.Identity()) - outcome, err := applyOne(ctx, sys, resource, run, changed, declares, was, unseal) + var outcome Outcome + var err error + if o, isOpening := resource.(*declaration.Opening); isOpening { + outcome, err = applyOpening(ctx, o, run, fw) + } else { + outcome, err = applyOne(ctx, sys, resource, run, changed, declares, was, unseal) + } if err != nil { failed := &Error{Resource: resource.Identity(), Err: err, Done: report} failures = append(failures, failed) @@ -293,6 +312,14 @@ func ApplyKeeping( } } + // A converged node whose found firewall was in force retires it only now, once everything — + // the mesh's derived filter among it — applied cleanly (novox/hq ADR 0100). + if len(failures) == 0 { + if err := retireFirewall(ctx, d, &known, run, log); err != nil { + return report, known, &Error{Resource: "the firewall found on this machine", Err: err, Done: report} + } + } + if len(failures) > 0 { // The first, carrying everything that did happen. One error is what the caller reports // and what a person reads first; the rest are in the report, which is what the mesh diff --git a/internal/apply/opening.go b/internal/apply/opening.go new file mode 100644 index 0000000..3adef33 --- /dev/null +++ b/internal/apply/opening.go @@ -0,0 +1,109 @@ +package apply + +import ( + "context" + "fmt" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/firewall" + "github.com/novox/mesh-host/internal/store" +) + +// foundFirewall settles, before anything else in an apply, which firewall this node has — and on +// an adopted node that the mesh had converged, puts it back in force first (novox/hq ADR 0100). +// +// Only an adopted node asks. It is detected on every apply rather than remembered, so a firewall +// switched on after adoption is spoken to from the next reconcile; what is remembered is what was +// found first, and whether the mesh retired it. An unsupported firewall refuses the whole +// declaration: the mesh could neither open what it needs through it nor say what it would close. +func foundFirewall(ctx context.Context, d *declaration.Declaration, known *store.State, run Runner, + log func(string)) (firewall.Kind, error) { + if d.Adoption == nil { + return "", nil + } + rec := known.Firewall + if rec != nil && rec.DisabledByMesh && rec.Kind == string(firewall.UFW) { + // Returned to adopted: the found firewall is enabled again before the openings are + // converged through it, and the derived filter is gone with this declaration. + if err := firewall.Enable(ctx, run); err != nil { + return "", err + } + rec.DisabledByMesh = false + log(" enabled ufw again: this node is adopted, and the firewall found on it is in force") + } + kind, name, err := firewall.Detect(ctx, run) + if err != nil { + return "", err + } + if kind == firewall.Unsupported { + return "", fmt.Errorf( + "this machine is filtered by %s, and no host speaks that firewall yet. An adopted node "+ + "keeps the firewall it was found with, so the mesh could neither open what it needs "+ + "through it nor say what it would close; this declaration is refused whole", name) + } + if rec == nil { + rec = &store.FoundFirewall{Kind: string(kind), WasActive: kind == firewall.UFW, + FoundAt: time.Now().UTC()} + } else { + rec.Kind = string(kind) + rec.WasActive = rec.WasActive || kind == firewall.UFW + } + known.Firewall = rec + return kind, nil +} + +// retireFirewall disables the found firewall once a converged declaration has applied cleanly, +// which is when the mesh's derived filter has taken its place. Disabled, never flushed: its +// configuration stays on disk for a return to adopted, and the container runtime's rules are not +// its to take. +func retireFirewall(ctx context.Context, d *declaration.Declaration, known *store.State, run Runner, + log func(string)) error { + rec := known.Firewall + if d.Adoption != nil || rec == nil || rec.Kind != string(firewall.UFW) || !rec.WasActive || + rec.DisabledByMesh { + return nil + } + if err := firewall.Disable(ctx, run); err != nil { + return err + } + rec.DisabledByMesh = true + log(" disabled ufw: this node is converged and filtered by the mesh; ufw's configuration is left on disk") + return nil +} + +// applyOpening makes one opening true through the firewall found here. +func applyOpening(ctx context.Context, o *declaration.Opening, run Runner, kind firewall.Kind) (Outcome, error) { + out := begin(o) + switch kind { + case firewall.None: + out.Action = "unchanged" + out.Detail = "no firewall found; nothing filters this port" + return out, nil + case firewall.UFW: + action, err := firewall.Converge(ctx, run, o) + if err != nil { + return out, err + } + out.Action = action + out.Detail = "through ufw, marked " + firewall.Mark(o) + return out, nil + } + return out, fmt.Errorf("no firewall is known for this node, so %s cannot be opened", o.Target()) +} + +// removeOpening deletes the rules the mesh marked for an opening no longer declared, and nothing +// the machine had before. +func removeOpening(ctx context.Context, a store.Applied, run Runner, rec *store.FoundFirewall) (string, string, error) { + if rec == nil || rec.Kind != string(firewall.UFW) { + return "forgotten", "no firewall held a rule for it", nil + } + n, err := firewall.Remove(ctx, run, a.ID) + if err != nil { + return "", "", err + } + if n == 0 { + return "forgotten", "ufw held no rule marked for it", nil + } + return "removed", fmt.Sprintf("%d ufw rule(s) marked as the mesh's deleted", n), nil +} diff --git a/internal/apply/opening_test.go b/internal/apply/opening_test.go new file mode 100644 index 0000000..d51bda5 --- /dev/null +++ b/internal/apply/opening_test.go @@ -0,0 +1,204 @@ +package apply + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// Defends novox/hq ADR 0100: the firewall found on an adopted node stays in force; converging the +// node retires it by disabling it, and returning the node to adopted enables it again. + +type ufwMachine struct { + installed, active bool + rules []string + ruleset string + asked []string +} + +func (u *ufwMachine) run(_ context.Context, name string, args ...string) (string, error) { + u.asked = append(u.asked, name+" "+strings.Join(args, " ")) + switch name { + case "nft": + return u.ruleset, nil + case "ufw": + if !u.installed { + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + } + default: + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + } + switch args[0] { + case "status": + if u.active { + return "Status: active\n", nil + } + return "Status: inactive\n", nil + case "show": + out := "Added user rules (see 'ufw status' for running firewall):\n" + for _, r := range u.rules { + out += "ufw " + r + "\n" + } + return out, nil + case "--force": + u.active = true + return "", nil + case "disable": + u.active = false + return "", nil + case "delete": + want := strings.Join(args[1:], " ") + for i, r := range u.rules { + if strings.ReplaceAll(r, "'", "") == want { + u.rules = append(u.rules[:i], u.rules[i+1:]...) + return "", nil + } + } + return "", errors.New("Could not delete non-existent rule") + default: + // Printed back the way it was given, with the comment quoted as ufw does. + line := strings.Join(args[:len(args)-1], " ") + " '" + args[len(args)-1] + "'" + u.rules = append(u.rules, line) + return "", nil + } +} + +func (u *ufwMachine) index(prefix string) int { + for i, a := range u.asked { + if strings.HasPrefix(a, prefix) { + return i + } + } + return -1 +} + +const busOpening = `{"id":"adoption.opening-tcp-5671-incoming","type":"opening","port":5671,"protocol":"tcp","from":"everywhere","path":"incoming"}` + +func withConf(dir string) string { + return `{"id":"x.conf","type":"file","path":"` + filepath.Join(dir, "x.conf") + `","content":"x\n"}` +} + +func applyWith(t *testing.T, d *declaration.Declaration, known store.State, run Runner) (Report, store.State, error) { + t.Helper() + return ApplyKeeping(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil, + KeepIn(t.TempDir())) +} + +func TestAnOpeningOnAMachineWithNoFirewallChangesNothing(t *testing.T) { + dir := t.TempDir() + u := &ufwMachine{} + report, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) + if err != nil { + t.Fatal(err) + } + o := outcomeOf(report, "adoption.opening-tcp-5671-incoming") + if o.Action != "unchanged" || !strings.Contains(o.Detail, "nothing filters this port") { + t.Errorf("an opening with no firewall: %+v", o) + } + if state.Firewall == nil || state.Firewall.Kind != "none" { + t.Errorf("the firewall found was not recorded: %+v", state.Firewall) + } +} + +func TestAnUnsupportedFirewallRefusesTheWholeDeclaration(t *testing.T) { + dir := t.TempDir() + u := &ufwMachine{ruleset: "table inet filter {\n\tchain input {\n\t\ttype filter hook input priority filter; policy drop;\n\t}\n}\n"} + _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) + if err == nil || !strings.Contains(err.Error(), "no host speaks that firewall") { + t.Fatalf("an unsupported firewall was not refused: %v", err) + } + if _, statErr := os.Stat(filepath.Join(dir, "x.conf")); !errors.Is(statErr, os.ErrNotExist) { + t.Error("part of a refused declaration was applied") + } + if state.Firewall != nil { + t.Errorf("an unsupported firewall was recorded: %+v", state.Firewall) + } +} + +func TestConvergingRetiresTheFoundFirewallAndReturningRestoresIt(t *testing.T) { + dir := t.TempDir() + u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}} + + // Adopted: the opening goes through ufw. + _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) + if err != nil { + t.Fatal(err) + } + if len(u.rules) != 2 || !u.active { + t.Fatalf("adopted: rules %v, active %v", u.rules, u.active) + } + if state.Firewall == nil || state.Firewall.Kind != "ufw" || !state.Firewall.WasActive { + t.Fatalf("adopted: firewall recorded as %+v", state.Firewall) + } + + // Converged: the opening's rule goes, and only then is ufw disabled — never reset. + u.asked = nil + converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`) + _, state, err = applyWith(t, converged, state, u.run) + if err != nil { + t.Fatal(err) + } + if u.active || !state.Firewall.DisabledByMesh { + t.Fatalf("converged: ufw still active (%v) or not recorded as retired (%+v)", u.active, state.Firewall) + } + if len(u.rules) != 1 || u.rules[0] != "allow 22/tcp" { + t.Errorf("converged: the operator's rules were touched, or the mesh's left: %v", u.rules) + } + if del, dis := u.index("ufw delete"), u.index("ufw disable"); del < 0 || dis < del { + t.Errorf("converged: the opening was not removed before ufw was disabled: %v", u.asked) + } + for _, a := range u.asked { + if strings.Contains(a, "reset") { + t.Errorf("converged: ufw was reset: %s", a) + } + } + + // Converged again: nothing more to retire. + u.asked = nil + if _, state, err = applyWith(t, converged, state, u.run); err != nil { + t.Fatal(err) + } + if u.index("ufw") >= 0 { + t.Errorf("a converged node kept talking to a retired ufw: %v", u.asked) + } + + // Returned to adopted: ufw is enabled before the opening is converged through it. + u.asked = nil + _, state, err = applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), state, u.run) + if err != nil { + t.Fatal(err) + } + if !u.active || state.Firewall.DisabledByMesh { + t.Fatalf("returned: ufw active %v, record %+v", u.active, state.Firewall) + } + if en, add := u.index("ufw --force enable"), u.index("ufw allow"); en < 0 || add < en { + t.Errorf("returned: ufw was not enabled before the opening was added: %v", u.asked) + } + if len(u.rules) != 2 { + t.Errorf("returned: the opening was not converged again: %v", u.rules) + } +} + +func TestAConvergedNodeThatWasNeverAdoptedNeverAsksAboutAFirewall(t *testing.T) { + dir := t.TempDir() + u := &ufwMachine{installed: true, active: true} + if _, _, err := applyWith(t, parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`), store.State{}, u.run); err != nil { + t.Fatal(err) + } + if len(u.asked) != 0 { + t.Errorf("a converged apply asked the machine about its firewall: %v", u.asked) + } +} + +func TestAnOpeningOnAConvergedNodeIsRefused(t *testing.T) { + if _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[` + busOpening + `]}`)); err == nil { + t.Error("an opening was accepted on a node the declaration does not say is adopted") + } +} diff --git a/internal/declaration/declaration.go b/internal/declaration/declaration.go index 41605f6..991654f 100644 --- a/internal/declaration/declaration.go +++ b/internal/declaration/declaration.go @@ -78,6 +78,12 @@ const ( // cadence. Tools, hooks and event consumers are not separate modes: they are loaded by a tool // host, which is itself a process that stays up. TypeProcess Type = "process" + + // TypeOpening is a port the mesh needs reachable on an adopted node, converged through the + // firewall found there in that firewall's own terms (novox/hq ADR 0100). A state, not a + // command: the host adds the rule it marks as the mesh's when it is missing, and removes only + // what it marked — which is what lets it travel over the link. + TypeOpening Type = "opening" ) // Resource is one thing that should be true of the machine. @@ -603,6 +609,74 @@ func (s *Service) validate(where string, _ bool) []string { return problems } +// Opening is a port reachable on an adopted node, from where, and on which path. +// +// **From** is everywhere or mesh — the private network, by its interface. **Path** is incoming, +// for something listening on the machine, or forwarded, for a published container port: the found +// firewall sees a published port after the runtime has translated it, so a forwarded opening names +// the container's own port in To as well as the machine's in Port. +type Opening struct { + ID string `json:"id"` + Type Type `json:"type"` + Port int `json:"port"` + Protocol string `json:"protocol"` + From string `json:"from"` + Path string `json:"path"` + To int `json:"to,omitempty"` +} + +// Where an opening admits from, and the path it is on. +const ( + FromEverywhere = "everywhere" + FromMesh = "mesh" + PathIncoming = "incoming" + PathForwarded = "forwarded" +) + +func (o *Opening) Identity() string { return o.ID } +func (o *Opening) Kind() Type { return TypeOpening } + +func (o *Opening) Target() string { + if o.Path == PathForwarded { + return fmt.Sprintf("%s/%d forwarded to %d from %s", o.Protocol, o.Port, o.To, o.From) + } + return fmt.Sprintf("%s/%d %s from %s", o.Protocol, o.Port, o.Path, o.From) +} + +func (o *Opening) validate(where string, _ bool) []string { + var problems []string + if o.Port < 1 || o.Port > 65535 { + problems = append(problems, fmt.Sprintf("%s: an opening's port is 1-65535, not %d", where, o.Port)) + } + if o.Protocol != "tcp" && o.Protocol != "udp" { + problems = append(problems, fmt.Sprintf("%s: an opening is tcp or udp, not %q", where, o.Protocol)) + } + if o.From != FromEverywhere && o.From != FromMesh { + problems = append(problems, fmt.Sprintf( + "%s: an opening is from %q or %q, not %q", where, FromEverywhere, FromMesh, o.From)) + } + switch o.Path { + case PathIncoming: + if o.To != 0 { + problems = append(problems, where+ + ": an incoming opening names no container port; only a forwarded one does") + } + case PathForwarded: + if o.To < 1 || o.To > 65535 { + problems = append(problems, where+ + ": a forwarded opening names the container's port it reaches, as to, 1-65535") + } + default: + problems = append(problems, fmt.Sprintf( + "%s: an opening's path is %q or %q, not %q", where, PathIncoming, PathForwarded, o.Path)) + } + if !strings.HasPrefix(o.ID, AdoptionPrefix) { + problems = append(problems, fmt.Sprintf( + "%s: an opening is the mesh's own, so its id starts %q", where, AdoptionPrefix)) + } + return problems +} + // Package is a package that should be present. // // Present is the whole of what it asserts, never a version: version is the package manager's @@ -810,6 +884,8 @@ func newOf(t Type) Resource { return &Access{} case TypeProcess: return &Process{} + case TypeOpening: + return &Opening{} } return nil } @@ -818,7 +894,7 @@ func newOf(t Type) Resource { func Vocabulary() []Type { return []Type{ TypeAccess, TypeAction, TypeArchive, TypeContainer, TypeDirectory, TypeFile, - TypeNetwork, TypePackage, TypeProcess, TypeService, TypeUser, + TypeNetwork, TypeOpening, TypePackage, TypeProcess, TypeService, TypeUser, } } @@ -1051,6 +1127,18 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) { d.Resources = append(d.Resources, resource) } problems = append(problems, checkAdoption(env.Adoption, d.Resources, allowActions)...) + if env.Adoption == nil { + for _, r := range d.Resources { + if r.Kind() == TypeOpening { + // On a converged node the mesh's own filter admits what is declared, and the + // found firewall is retired; an opening there would be a rule in a firewall the + // mesh has disabled (novox/hq ADR 0100). + problems = append(problems, fmt.Sprintf( + "resource %q: an opening is for an adopted node, and this declaration does not "+ + "say the node is adopted", r.Identity())) + } + } + } if len(problems) > 0 { return nil, &RefusalError{Problems: problems} diff --git a/internal/declaration/declaration_test.go b/internal/declaration/declaration_test.go index b2fd73e..d0823e0 100644 --- a/internal/declaration/declaration_test.go +++ b/internal/declaration/declaration_test.go @@ -266,7 +266,7 @@ func TestAFieldTheNewTypesDoNotUseIsRefused(t *testing.T) { } } -func TestTheVocabularyIsTheElevenShapesTheMeshNeeds(t *testing.T) { +func TestTheVocabularyIsTheTwelveShapesTheMeshNeeds(t *testing.T) { // Six of them the bootstrap uses (novox/hq 07-the-foundation.md), and removing one is a // failing test rather than a discovery during a first-node install. // @@ -282,7 +282,7 @@ func TestTheVocabularyIsTheElevenShapesTheMeshNeeds(t *testing.T) { } for _, want := range []Type{ TypeDirectory, TypeFile, TypeService, TypePackage, TypeContainer, TypeAction, - TypeUser, TypeArchive, TypeNetwork, TypeAccess, TypeProcess, + TypeUser, TypeArchive, TypeNetwork, TypeAccess, TypeProcess, TypeOpening, } { if !speaks[want] { t.Errorf("the host no longer speaks %q", want) @@ -309,8 +309,12 @@ func TestTheVocabularyIsTheElevenShapesTheMeshNeeds(t *testing.T) { // It is a full-host shape rather than a portable one: it needs a process supervisor to install // into. It does NOT need a container runtime, which is the point — only software that // genuinely needs isolation asks for a container. - if len(speaks) != 11 { - t.Errorf("the vocabulary is %d shapes rather than 11; every addition widens what a compromised "+ + // + // `opening` is the twelfth, and novox/hq ADR 0100 is its decision: on an adopted node the + // firewall found there stays in force, and what the mesh needs reachable is converged through + // it as a state the host marks as the mesh's — never a command, which the link may not carry. + if len(speaks) != 12 { + t.Errorf("the vocabulary is %d shapes rather than 12; every addition widens what a compromised "+ "control plane can express, so a change here is a decision: %s", len(speaks), vocabulary()) } diff --git a/internal/firewall/firewall.go b/internal/firewall/firewall.go new file mode 100644 index 0000000..b896c5d --- /dev/null +++ b/internal/firewall/firewall.go @@ -0,0 +1,431 @@ +// Package firewall speaks the firewall found on an adopted node, in that firewall's own terms +// (novox/hq ADR 0100). +// +// **The found firewall stays in force.** On an adopted node the mesh loads nothing that drops by +// default or holds an accept; what it needs reachable it converges as openings through what it +// found, marks each rule as its own, and removes only what it marked. It never resets or flushes: +// the rules the machine already had are the operator's, and they are what keeps it serving. +package firewall + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os/exec" + "regexp" + "strconv" + "strings" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/system" +) + +// Runner executes a command. +type Runner = system.Runner + +// Kind is what firewall a machine has, as far as the mesh is concerned. +type Kind string + +const ( + // UFW is an active ufw — the one kind found on the machines measured, and the one spoken. + UFW Kind = "ufw" + // None is a machine where nothing refuses anything, which needs no openings. + None Kind = "none" + // Unsupported is a firewall no host speaks yet. A machine with one is refused adoption: the + // mesh could neither open what it needs nor know what it would be closing. + Unsupported Kind = "unsupported" +) + +// MeshInterface is the private network's interface, the way an opening from the mesh is known. +// It must be the controller's overlay interface name. +const MeshInterface = "mesh0" + +// Detect says which firewall this machine has. For Unsupported the string names it. +func Detect(ctx context.Context, run Runner) (Kind, string, error) { + if out, err := run(ctx, "firewall-cmd", "--state"); err == nil && strings.TrimSpace(out) == "running" { + return Unsupported, "firewalld", nil + } + ufwActive := false + if out, err := run(ctx, "ufw", "status"); err == nil { + ufwActive = statusActive(out) + } + + out, err := run(ctx, "nft", "list", "ruleset") + switch { + case err == nil: + if refusing := Refusing(out, ufwActive); len(refusing) > 0 { + return Unsupported, "nftables rules that refuse traffic, in " + strings.Join(refusing, ", "), nil + } + case !missing(err): + return "", "", fmt.Errorf("cannot read this machine's packet filter to know what it has: %w", err) + } + + if !ufwActive { + // iptables with the legacy backend is invisible to nft. + for _, legacy := range []string{"iptables-legacy", "ip6tables-legacy"} { + out, err := run(ctx, legacy, "-S") + if err != nil { + continue + } + if refusing := RefusingLegacy(out); len(refusing) > 0 { + return Unsupported, legacy + " rules that refuse traffic, in " + strings.Join(refusing, ", "), nil + } + } + } + + if ufwActive { + return UFW, "ufw", nil + } + return None, "", nil +} + +func missing(err error) bool { + return errors.Is(err, exec.ErrNotFound) +} + +func statusActive(out string) bool { + for _, line := range strings.Split(out, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "Status:") { + return strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line), "Status:")) == "active" + } + } + return false +} + +// Refusing names the tables of an `nft list ruleset` holding something that refuses traffic — a +// drop or reject, or a base chain whose policy drops — and that is neither the mesh's own nor the +// container runtime's. With ufw active, the tables iptables-nft manages are ufw's and the runtime's +// and are not counted. +func Refusing(ruleset string, ufwActive bool) []string { + var refusing []string + managed := map[string]bool{} + var table, chain string + counted := map[string]bool{} + note := func() { + if !counted[table] { + counted[table] = true + refusing = append(refusing, "table "+table) + } + } + for _, raw := range strings.Split(ruleset, "\n") { + line := strings.TrimSpace(raw) + switch { + case strings.HasPrefix(line, "# Warning: table ") && strings.Contains(line, "managed by iptables-nft"): + name := strings.TrimPrefix(line, "# Warning: table ") + name, _, _ = strings.Cut(name, " is managed") + managed[name] = true + continue + case strings.HasPrefix(line, "table "): + table = strings.TrimSuffix(strings.TrimSpace(strings.TrimPrefix(line, "table ")), "{") + table = strings.TrimSpace(table) + chain = "" + continue + case strings.HasPrefix(line, "chain "): + chain = strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(line, "chain "), "{")) + continue + case line == "" || line == "}" || strings.HasPrefix(line, "#") || chain == "": + continue + } + if table == "inet mesh" || table == "inet mesh_guard" { + continue + } + iptables := managed[table] || iptablesTable(table) + if iptables && ufwActive { + continue + } + if strings.HasPrefix(line, "type ") { + if strings.Contains(line, "policy drop") && !(iptables && runtimes(table, chain, line)) { + note() + } + continue + } + if !verdictRefuses(line) { + continue + } + if iptables && runtimes(table, chain, line) { + continue + } + note() + } + return refusing +} + +// iptablesTable is whether a table is one iptables-nft writes. Named rather than read from the +// warning nft prints above it, because nft does not print that for every such table: a captured +// ruleset carried it on ip filter and not on ip raw, where the runtime keeps its drops. +func iptablesTable(table string) bool { + family, name, _ := strings.Cut(table, " ") + if family != "ip" && family != "ip6" { + return false + } + switch name { + case "filter", "nat", "raw", "mangle", "security": + return true + } + return false +} + +// runtimes is whether a refusal in an iptables-nft table is the container runtime's own: in its +// DOCKER chains, its forward policy, or its guard against reaching a container's address directly +// from outside its bridge, in the raw table. +func runtimes(table, chain, line string) bool { + _, name, _ := strings.Cut(table, " ") + switch { + case strings.HasPrefix(chain, "DOCKER"): + return true + case name == "filter" && chain == "FORWARD" && strings.HasPrefix(line, "type "): + return true + case name == "raw" && chain == "PREROUTING": + return strings.Contains(line, "daddr") && strings.Contains(line, "iifname !=") + } + return false +} + +var verdict = regexp.MustCompile(`(^|\s)(drop|reject)(\s|$)`) + +func verdictRefuses(line string) bool { + return verdict.MatchString(line) +} + +// RefusingLegacy names the chains of an `iptables-legacy -S` that refuse traffic outside the +// container runtime's own. +func RefusingLegacy(rules string) []string { + var refusing []string + seen := map[string]bool{} + for _, line := range strings.Split(rules, "\n") { + fields := strings.Fields(line) + if len(fields) < 3 { + continue + } + chain := fields[1] + refuses := false + switch fields[0] { + case "-P": + refuses = fields[2] == "DROP" && chain != "FORWARD" + case "-A": + for i, f := range fields { + if f == "-j" && i+1 < len(fields) && (fields[i+1] == "DROP" || fields[i+1] == "REJECT") { + refuses = !strings.HasPrefix(chain, "DOCKER") + } + } + } + if refuses && !seen[chain] { + seen[chain] = true + refusing = append(refusing, "chain "+chain) + } + } + return refusing +} + +// --- ufw --------------------------------------------------------------------------------------- + +// Mark is the comment every rule the mesh adds carries: whose it is, which opening, and a digest +// of the rule itself, so a rule the opening no longer describes is recognised as stale without the +// host having to know how ufw prints a rule back. +func Mark(o *declaration.Opening) string { + sum := sha256.Sum256([]byte(strings.Join(Rule(o), " "))) + return marker(o.ID) + " " + hex.EncodeToString(sum[:])[:8] +} + +func marker(id string) string { return "mesh-host " + id } + +// markedFor is whether a comment is the mesh's, for this opening. +func markedFor(comment, id string) bool { + return comment == marker(id) || strings.HasPrefix(comment, marker(id)+" ") +} + +// Rule is the ufw rule an opening becomes, without its comment. +// +// incoming from everywhere allow proto tcp to any port P +// incoming from the mesh allow in on mesh0 proto tcp to any port P +// forwarded route allow [in on mesh0] proto tcp to any port +// +// A forwarded opening names the container's port because ufw's route rules are matched after the +// runtime's destination translation. +func Rule(o *declaration.Opening) []string { + var rule []string + port := o.Port + if o.Path == declaration.PathForwarded { + rule = append(rule, "route") + port = o.To + } + rule = append(rule, "allow") + if o.From == declaration.FromMesh { + rule = append(rule, "in", "on", MeshInterface) + } + return append(rule, "proto", o.Protocol, "to", "any", "port", strconv.Itoa(port)) +} + +var commentOf = regexp.MustCompile(`comment '([^']*)'`) + +// added is every rule `ufw show added` lists, each without its leading "ufw". +func added(ctx context.Context, run Runner) ([]string, error) { + out, err := run(ctx, "ufw", "show", "added") + if err != nil { + return nil, fmt.Errorf("reading ufw's rules: %w", err) + } + var rules []string + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "ufw ") { + rules = append(rules, strings.TrimPrefix(line, "ufw ")) + } + } + return rules, nil +} + +func comment(rule string) string { + m := commentOf.FindStringSubmatch(rule) + if m == nil { + return "" + } + return m[1] +} + +// words splits a rule as ufw printed it into arguments, keeping a quoted comment whole. +func words(rule string) []string { + var out []string + var cur strings.Builder + quoted, any := false, false + for _, r := range rule { + switch { + case r == '\'': + quoted = !quoted + any = true + case r == ' ' && !quoted: + if any { + out = append(out, cur.String()) + cur.Reset() + any = false + } + default: + cur.WriteRune(r) + any = true + } + } + if any { + out = append(out, cur.String()) + } + return out +} + +// Converge makes one opening true in ufw: its marked rule present, and any rule marked for it that +// no longer describes it deleted. Nothing unmarked is touched. The outcome is created, updated or +// unchanged, read back from ufw rather than assumed. +func Converge(ctx context.Context, run Runner, o *declaration.Opening) (string, error) { + rules, err := added(ctx, run) + if err != nil { + return "", err + } + mark := Mark(o) + present := false + var stale []string + for _, rule := range rules { + c := comment(rule) + switch { + case c == mark: + present = true + case markedFor(c, o.ID): + stale = append(stale, rule) + } + } + if present && len(stale) == 0 { + return "unchanged", nil + } + for _, rule := range stale { + if _, err := run(ctx, "ufw", append([]string{"delete"}, words(rule)...)...); err != nil { + return "", fmt.Errorf("deleting the mesh's stale ufw rule %q: %w", rule, err) + } + } + if !present { + args := append(Rule(o), "comment", mark) + if _, err := run(ctx, "ufw", args...); err != nil { + return "", fmt.Errorf("adding the ufw rule for %s: %w", o.Target(), err) + } + } + after, err := added(ctx, run) + if err != nil { + return "", err + } + found, leftover := false, 0 + for _, rule := range after { + c := comment(rule) + if c == mark { + found = true + } else if markedFor(c, o.ID) { + leftover++ + } + } + if !found { + return "", fmt.Errorf("ufw was asked for %s and does not list it afterwards", o.Target()) + } + if leftover > 0 { + return "", fmt.Errorf("ufw still lists %d stale rule(s) marked for %s after deleting them", leftover, o.ID) + } + if len(stale) > 0 { + return "updated", nil + } + return "created", nil +} + +// Remove deletes the rules marked for one opening, and nothing else. +func Remove(ctx context.Context, run Runner, id string) (int, error) { + rules, err := added(ctx, run) + if err != nil { + return 0, err + } + removed := 0 + for _, rule := range rules { + if !markedFor(comment(rule), id) { + continue + } + if _, err := run(ctx, "ufw", append([]string{"delete"}, words(rule)...)...); err != nil { + return removed, fmt.Errorf("deleting the mesh's ufw rule %q: %w", rule, err) + } + removed++ + } + after, err := added(ctx, run) + if err != nil { + return removed, err + } + for _, rule := range after { + if markedFor(comment(rule), id) { + return removed, fmt.Errorf("ufw still lists a rule marked for %s after deleting it", id) + } + } + return removed, nil +} + +// Enable turns ufw back on, as found, and reads back that it is. +func Enable(ctx context.Context, run Runner) error { + if _, err := run(ctx, "ufw", "--force", "enable"); err != nil { + return fmt.Errorf("enabling ufw again: %w", err) + } + return expectActive(ctx, run, true) +} + +// Disable retires ufw without flushing it: its configuration stays on disk, and the container +// runtime's rules are not its to remove. +func Disable(ctx context.Context, run Runner) error { + if _, err := run(ctx, "ufw", "disable"); err != nil { + return fmt.Errorf("disabling ufw: %w", err) + } + return expectActive(ctx, run, false) +} + +func expectActive(ctx context.Context, run Runner, want bool) error { + out, err := run(ctx, "ufw", "status") + if err != nil { + return fmt.Errorf("reading ufw's status back: %w", err) + } + if statusActive(out) != want { + state := "inactive" + if want { + state = "active" + } + return fmt.Errorf("ufw was asked to be %s and says: %s", state, strings.TrimSpace(out)) + } + return nil +} diff --git a/internal/firewall/firewall_test.go b/internal/firewall/firewall_test.go new file mode 100644 index 0000000..67d8aa2 --- /dev/null +++ b/internal/firewall/firewall_test.go @@ -0,0 +1,335 @@ +package firewall + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" +) + +// Defends novox/hq ADR 0100: the firewall found on an adopted node stays in force, the mesh opens +// what it needs through it in its own terms, and removes only what it marked. + +func dockerOnly(t *testing.T) string { + t.Helper() + // Captured from a real machine running the container runtime and nothing else that filters: + // its nat, filter and raw tables as iptables-nft writes them. + raw, err := os.ReadFile("testdata/docker-only.nft") + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +const aDroppingTable = ` +table inet filter { + chain input { + type filter hook input priority filter; policy drop; + ct state established,related accept + tcp dport 22 accept + } +} +` + +const ufwChains = ` +# Warning: table ip filter is managed by iptables-nft, do not touch! +table ip filter { + chain INPUT { + type filter hook input priority filter; policy drop; + counter packets 0 bytes 0 jump ufw-before-input + } + chain ufw-user-input { + tcp dport 22 counter packets 0 bytes 0 accept + } + chain ufw-reject-input { + counter packets 0 bytes 0 reject + } +} +` + +const theMeshsOwn = ` +table inet mesh { + chain input { + type filter hook input priority filter; policy drop; + iif lo accept + } +} +table inet mesh_guard { + chain prerouting { + type filter hook prerouting priority raw; policy accept; + iifname != "lo" tcp dport { 5432, 15672 } drop + } +} +` + +func TestTheContainerRuntimesOwnRulesAreNotAFirewall(t *testing.T) { + if got := Refusing(dockerOnly(t), false); len(got) != 0 { + t.Errorf("the runtime's own rules read as a firewall: %v", got) + } +} + +func TestTheMeshsOwnTablesAreNotAFirewall(t *testing.T) { + if got := Refusing(dockerOnly(t)+theMeshsOwn, false); len(got) != 0 { + t.Errorf("the mesh's own tables read as a found firewall: %v", got) + } +} + +func TestATableThatDropsIsAFirewall(t *testing.T) { + got := Refusing(dockerOnly(t)+aDroppingTable, false) + if len(got) != 1 || got[0] != "table inet filter" { + t.Errorf("a dropping table was not named: %v", got) + } +} + +func TestUfwsOwnChainsAreUfwsWhenItIsActive(t *testing.T) { + if got := Refusing(dockerOnly(t)+ufwChains, true); len(got) != 0 { + t.Errorf("ufw's own chains read as a second firewall: %v", got) + } + if got := Refusing(dockerOnly(t)+ufwChains, false); len(got) == 0 { + t.Error("iptables rules that refuse, with ufw not active, were not counted") + } +} + +func TestLegacyIptablesThatRefusesIsAFirewall(t *testing.T) { + docker := "-P INPUT ACCEPT\n-P FORWARD DROP\n-P OUTPUT ACCEPT\n-N DOCKER\n-A DOCKER -i docker0 -j DROP\n" + if got := RefusingLegacy(docker); len(got) != 0 { + t.Errorf("the runtime's legacy rules read as a firewall: %v", got) + } + if got := RefusingLegacy(docker + "-A INPUT -p tcp --dport 25 -j REJECT\n"); len(got) != 1 { + t.Errorf("a legacy reject was not counted: %v", got) + } +} + +// fakeUFW is ufw as far as the host can see it: a status, and user rules it prints back in its +// own canonical form — deliberately not the order the host wrote them in. +type fakeUFW struct { + active bool + installed bool + rules []string + ruleset string + firewalld bool + asked []string +} + +func canonical(args []string) string { + var route, in, port, proto, comment string + for i := 0; i < len(args); i++ { + switch args[i] { + case "route": + route = "route " + case "in": + in = "in on " + args[i+2] + " " + i += 2 + case "port": + port = args[i+1] + i++ + case "proto": + proto = args[i+1] + i++ + case "comment": + comment = args[i+1] + i++ + } + } + line := route + "allow " + in + port + "/" + proto + if comment != "" { + line += " comment '" + comment + "'" + } + return line +} + +func (f *fakeUFW) run(_ context.Context, name string, args ...string) (string, error) { + f.asked = append(f.asked, name+" "+strings.Join(args, " ")) + switch name { + case "firewall-cmd": + if f.firewalld { + return "running\n", nil + } + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + case "nft": + return f.ruleset, nil + case "iptables-legacy", "ip6tables-legacy": + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + case "ufw": + default: + return "", fmt.Errorf("unexpected %s", name) + } + if !f.installed { + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + } + switch { + case args[0] == "status": + if f.active { + return "Status: active\n\nTo Action From\n", nil + } + return "Status: inactive\n", nil + case args[0] == "show": + out := "Added user rules (see 'ufw status' for running firewall):\n" + for _, r := range f.rules { + out += "ufw " + r + "\n" + } + return out, nil + case args[0] == "--force" && args[1] == "enable": + f.active = true + return "Firewall is active and enabled on system startup\n", nil + case args[0] == "disable": + f.active = false + return "Firewall stopped and disabled on system startup\n", nil + case args[0] == "delete": + for i, r := range f.rules { + if strings.Join(words(r), "\x00") == strings.Join(args[1:], "\x00") { + f.rules = append(f.rules[:i], f.rules[i+1:]...) + return "Rule deleted\n", nil + } + } + return "", errors.New("Could not delete non-existent rule") + default: + f.rules = append(f.rules, canonical(args)) + return "Rule added\n", nil + } +} + +func (f *fakeUFW) added() int { + n := 0 + for _, a := range f.asked { + if strings.HasPrefix(a, "ufw allow") || strings.HasPrefix(a, "ufw route") { + n++ + } + } + return n +} + +func opening(id string, port int, from, path string, to int) *declaration.Opening { + return &declaration.Opening{ID: id, Type: declaration.TypeOpening, Port: port, Protocol: "tcp", + From: from, Path: path, To: to} +} + +func TestAnOpeningBecomesTheUfwRuleForItsPathAndOrigin(t *testing.T) { + for _, c := range []struct { + o *declaration.Opening + want string + }{ + {opening("adoption.a", 5671, "everywhere", "incoming", 0), "allow proto tcp to any port 5671"}, + {opening("adoption.b", 5432, "mesh", "incoming", 0), "allow in on mesh0 proto tcp to any port 5432"}, + {opening("adoption.c", 20001, "everywhere", "forwarded", 8080), "route allow proto tcp to any port 8080"}, + {opening("adoption.d", 20001, "mesh", "forwarded", 8080), "route allow in on mesh0 proto tcp to any port 8080"}, + } { + if got := strings.Join(Rule(c.o), " "); got != c.want { + t.Errorf("%s: %q, want %q", c.o.ID, got, c.want) + } + } +} + +func TestAnOpeningIsAddedOnceAndMarkedAsTheMeshs(t *testing.T) { + f := &fakeUFW{installed: true, active: true, rules: []string{"allow 22/tcp"}} + o := opening("adoption.opening-tcp-5671-incoming", 5671, "everywhere", "incoming", 0) + + action, err := Converge(context.Background(), f.run, o) + if err != nil || action != "created" { + t.Fatalf("first converge: %q %v", action, err) + } + if !strings.Contains(f.rules[1], "comment 'mesh-host adoption.opening-tcp-5671-incoming ") { + t.Errorf("the rule is not marked as the mesh's: %v", f.rules) + } + action, err = Converge(context.Background(), f.run, o) + if err != nil || action != "unchanged" { + t.Fatalf("second converge: %q %v", action, err) + } + if f.added() != 1 { + t.Errorf("re-converging added again: %v", f.asked) + } +} + +func TestAnOpeningLostToAReloadIsAddedAgain(t *testing.T) { + f := &fakeUFW{installed: true, active: true} + o := opening("adoption.x", 5671, "everywhere", "incoming", 0) + if _, err := Converge(context.Background(), f.run, o); err != nil { + t.Fatal(err) + } + f.rules = nil // what a reload that lost the rule leaves + action, err := Converge(context.Background(), f.run, o) + if err != nil || action != "created" || len(f.rules) != 1 { + t.Fatalf("a lost opening was not put back: %q %v %v", action, err, f.rules) + } +} + +func TestAChangedOpeningReplacesOnlyItsOwnRule(t *testing.T) { + f := &fakeUFW{installed: true, active: true, rules: []string{"allow 22/tcp", "allow 8080/tcp comment 'someone else'"}} + if _, err := Converge(context.Background(), f.run, opening("adoption.x", 5671, "everywhere", "incoming", 0)); err != nil { + t.Fatal(err) + } + action, err := Converge(context.Background(), f.run, opening("adoption.x", 5671, "mesh", "incoming", 0)) + if err != nil || action != "updated" { + t.Fatalf("%q %v", action, err) + } + if len(f.rules) != 3 || f.rules[0] != "allow 22/tcp" || f.rules[1] != "allow 8080/tcp comment 'someone else'" || + !strings.Contains(f.rules[2], "in on mesh0") { + t.Errorf("rules afterwards: %v", f.rules) + } +} + +func TestRemovingAnOpeningRemovesOnlyWhatWasMarkedForIt(t *testing.T) { + f := &fakeUFW{installed: true, active: true, rules: []string{"allow 22/tcp", "allow 8080/tcp"}} + for _, o := range []*declaration.Opening{ + opening("adoption.a", 5671, "everywhere", "incoming", 0), + opening("adoption.ab", 5000, "everywhere", "incoming", 0), + } { + if _, err := Converge(context.Background(), f.run, o); err != nil { + t.Fatal(err) + } + } + n, err := Remove(context.Background(), f.run, "adoption.a") + if err != nil || n != 1 { + t.Fatalf("removed %d: %v", n, err) + } + if len(f.rules) != 3 || f.rules[0] != "allow 22/tcp" || f.rules[1] != "allow 8080/tcp" || + !strings.Contains(f.rules[2], "adoption.ab") { + t.Errorf("more than the marked rule went: %v", f.rules) + } +} + +func TestEnableAndDisableReadBack(t *testing.T) { + f := &fakeUFW{installed: true, active: true} + if err := Disable(context.Background(), f.run); err != nil || f.active { + t.Fatalf("disable: %v", err) + } + if err := Enable(context.Background(), f.run); err != nil || !f.active { + t.Fatalf("enable: %v", err) + } + for _, a := range f.asked { + if strings.Contains(a, "reset") || strings.Contains(a, "flush") { + t.Errorf("the found firewall was reset: %s", a) + } + } +} + +func TestDetectingTheFoundFirewall(t *testing.T) { + for _, c := range []struct { + name string + f *fakeUFW + want Kind + }{ + {"nothing but the runtime", &fakeUFW{ruleset: dockerOnly(t)}, None}, + {"ufw active", &fakeUFW{installed: true, active: true, ruleset: dockerOnly(t) + ufwChains}, UFW}, + {"ufw installed and inactive", &fakeUFW{installed: true, ruleset: dockerOnly(t)}, None}, + {"firewalld", &fakeUFW{firewalld: true, ruleset: dockerOnly(t)}, Unsupported}, + {"an nftables table of its own", &fakeUFW{ruleset: dockerOnly(t) + aDroppingTable}, Unsupported}, + {"ufw beside an nftables table", &fakeUFW{installed: true, active: true, ruleset: dockerOnly(t) + ufwChains + aDroppingTable}, Unsupported}, + } { + got, name, err := Detect(context.Background(), c.f.run) + if err != nil { + t.Fatalf("%s: %v", c.name, err) + } + if got != c.want { + t.Errorf("%s: detected %s (%s), want %s", c.name, got, name, c.want) + } + if got == Unsupported && name == "" { + t.Errorf("%s: an unsupported firewall was not named", c.name) + } + } +} diff --git a/internal/firewall/testdata/docker-only.nft b/internal/firewall/testdata/docker-only.nft new file mode 100644 index 0000000..7df77ba --- /dev/null +++ b/internal/firewall/testdata/docker-only.nft @@ -0,0 +1,297 @@ +# Warning: table ip nat is managed by iptables-nft, do not touch! +table ip nat { + chain DOCKER { + iifname != "br-c70303d221ee" tcp dport 5680 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-c70303d221ee" tcp dport 15673 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-3636e05760a9" tcp dport 59000 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-3636e05760a9" tcp dport 59001 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-3636e05760a9" tcp dport 55672 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-3636e05760a9" tcp dport 55673 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-3636e05760a9" tcp dport 55432 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "docker0" tcp dport 57732 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "docker0" tcp dport 57733 counter packets 0 bytes 0 xt target "DNAT" + iifname != "docker0" tcp dport 5314 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-613eb68ef5fb" tcp dport 4848 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-b3240c822bce" tcp dport 5679 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-b3240c822bce" tcp dport 15672 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-4b504efd6080" tcp dport 9000 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-4b504efd6080" tcp dport 9001 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-ef6df03f71a0" tcp dport 5432 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-ef6df03f71a0" tcp dport 8081 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-ec480f77ac34" tcp dport 6379 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-af4c9c2aa60a" tcp dport 8001 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-669fda75f1ac" tcp dport 28080 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-af4c9c2aa60a" tcp dport 6789 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-af4c9c2aa60a" tcp dport 8770 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-af4c9c2aa60a" tcp dport 1212 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-af4c9c2aa60a" tcp dport 80 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-af4c9c2aa60a" tcp dport 443 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "docker0" tcp dport 55541 counter packets 0 bytes 0 xt target "DNAT" + } + + chain PREROUTING { + type nat hook prerouting priority dstnat; policy accept; + xt match "addrtype" counter packets 437947 bytes 71410534 jump DOCKER + } + + chain OUTPUT { + type nat hook output priority dstnat; policy accept; + ip daddr != 127.0.0.0/8 xt match "addrtype" counter packets 5761 bytes 423317 jump DOCKER + } + + chain POSTROUTING { + type nat hook postrouting priority srcnat; policy accept; + ip saddr 172.22.0.0/16 oifname != "br-65f6dc782562" counter packets 124 bytes 16328 xt target "MASQUERADE" + ip saddr 172.28.0.0/16 oifname != "br-669fda75f1ac" counter packets 127 bytes 16928 xt target "MASQUERADE" + ip saddr 172.31.0.0/16 oifname != "br-ec480f77ac34" counter packets 127 bytes 16928 xt target "MASQUERADE" + ip saddr 192.168.48.0/20 oifname != "br-ef6df03f71a0" counter packets 127 bytes 16928 xt target "MASQUERADE" + ip saddr 172.25.0.0/16 oifname != "br-4b504efd6080" counter packets 129 bytes 17052 xt target "MASQUERADE" + ip saddr 192.168.32.0/20 oifname != "br-613eb68ef5fb" counter packets 1124 bytes 76748 xt target "MASQUERADE" + ip saddr 172.17.0.0/16 oifname != "docker0" counter packets 1833 bytes 150990 xt target "MASQUERADE" + ip saddr 172.18.0.0/16 oifname != "br-07a5e2f2c42f" counter packets 504 bytes 65112 xt target "MASQUERADE" + ip saddr 172.27.0.0/16 oifname != "br-160f55da427c" counter packets 508 bytes 65784 xt target "MASQUERADE" + ip saddr 192.168.16.0/20 oifname != "br-dba077b9b543" counter packets 503 bytes 64784 xt target "MASQUERADE" + ip saddr 192.168.64.0/20 oifname != "br-d44fef8fd602" counter packets 1282 bytes 111308 xt target "MASQUERADE" + ip saddr 172.30.0.0/16 oifname != "br-af4c9c2aa60a" counter packets 2503 bytes 190324 xt target "MASQUERADE" + ip saddr 172.21.0.0/16 oifname != "br-9d6c95e8d80c" counter packets 1289 bytes 112644 xt target "MASQUERADE" + ip saddr 172.23.0.0/16 oifname != "br-679db9b21e00" counter packets 506 bytes 65384 xt target "MASQUERADE" + ip saddr 172.24.0.0/16 oifname != "br-40094534a5ee" counter packets 508 bytes 65784 xt target "MASQUERADE" + ip saddr 172.26.0.0/16 oifname != "br-3dcb6ef83ea1" counter packets 508 bytes 65784 xt target "MASQUERADE" + ip saddr 172.20.0.0/16 oifname != "br-3a760a74f4f6" counter packets 1153 bytes 104344 xt target "MASQUERADE" + ip saddr 192.168.80.0/20 oifname != "br-3636e05760a9" counter packets 546 bytes 70540 xt target "MASQUERADE" + ip saddr 172.29.0.0/16 oifname != "br-b3240c822bce" counter packets 551 bytes 71492 xt target "MASQUERADE" + ip saddr 172.19.0.0/16 oifname != "br-c70303d221ee" counter packets 1136 bytes 106592 xt target "MASQUERADE" + } +} +# Warning: table ip filter is managed by iptables-nft, do not touch! +table ip filter { + chain DOCKER { + ip daddr 172.17.0.5 iifname != "docker0" oifname "docker0" tcp dport 5432 counter packets 0 bytes 0 accept + ip daddr 172.30.0.2 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 443 counter packets 0 bytes 0 accept + ip daddr 172.30.0.2 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 80 counter packets 0 bytes 0 accept + ip daddr 172.30.0.10 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 3000 counter packets 0 bytes 0 accept + ip daddr 172.30.0.5 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 8000 counter packets 0 bytes 0 accept + ip daddr 172.30.0.3 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 6789 counter packets 0 bytes 0 accept + ip daddr 172.28.0.3 iifname != "br-669fda75f1ac" oifname "br-669fda75f1ac" tcp dport 8080 counter packets 0 bytes 0 accept + ip daddr 172.30.0.4 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 5540 counter packets 0 bytes 0 accept + ip daddr 172.31.0.2 iifname != "br-ec480f77ac34" oifname "br-ec480f77ac34" tcp dport 6379 counter packets 0 bytes 0 accept + ip daddr 192.168.48.3 iifname != "br-ef6df03f71a0" oifname "br-ef6df03f71a0" tcp dport 8081 counter packets 0 bytes 0 accept + ip daddr 192.168.48.2 iifname != "br-ef6df03f71a0" oifname "br-ef6df03f71a0" tcp dport 5432 counter packets 0 bytes 0 accept + ip daddr 172.25.0.2 iifname != "br-4b504efd6080" oifname "br-4b504efd6080" tcp dport 9001 counter packets 0 bytes 0 accept + ip daddr 172.25.0.2 iifname != "br-4b504efd6080" oifname "br-4b504efd6080" tcp dport 9000 counter packets 0 bytes 0 accept + ip daddr 172.29.0.2 iifname != "br-b3240c822bce" oifname "br-b3240c822bce" tcp dport 15672 counter packets 0 bytes 0 accept + ip daddr 172.29.0.2 iifname != "br-b3240c822bce" oifname "br-b3240c822bce" tcp dport 5672 counter packets 0 bytes 0 accept + ip daddr 192.168.32.2 iifname != "br-613eb68ef5fb" oifname "br-613eb68ef5fb" tcp dport 1433 counter packets 0 bytes 0 accept + ip daddr 172.17.0.3 iifname != "docker0" oifname "docker0" tcp dport 5000 counter packets 0 bytes 0 accept + ip daddr 172.17.0.2 iifname != "docker0" oifname "docker0" tcp dport 15672 counter packets 0 bytes 0 accept + ip daddr 172.17.0.2 iifname != "docker0" oifname "docker0" tcp dport 5672 counter packets 0 bytes 0 accept + ip daddr 192.168.80.4 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 5432 counter packets 0 bytes 0 accept + ip daddr 192.168.80.3 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 15672 counter packets 0 bytes 0 accept + ip daddr 192.168.80.3 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 5672 counter packets 0 bytes 0 accept + ip daddr 192.168.80.2 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 9001 counter packets 0 bytes 0 accept + ip daddr 192.168.80.2 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 9000 counter packets 0 bytes 0 accept + ip daddr 172.19.0.2 iifname != "br-c70303d221ee" oifname "br-c70303d221ee" tcp dport 15672 counter packets 0 bytes 0 accept + ip daddr 172.19.0.2 iifname != "br-c70303d221ee" oifname "br-c70303d221ee" tcp dport 5672 counter packets 0 bytes 0 accept + iifname != "br-c70303d221ee" oifname "br-c70303d221ee" counter packets 0 bytes 0 drop + iifname != "br-b3240c822bce" oifname "br-b3240c822bce" counter packets 0 bytes 0 drop + iifname != "br-3636e05760a9" oifname "br-3636e05760a9" counter packets 0 bytes 0 drop + iifname != "br-3a760a74f4f6" oifname "br-3a760a74f4f6" counter packets 0 bytes 0 drop + iifname != "br-3dcb6ef83ea1" oifname "br-3dcb6ef83ea1" counter packets 0 bytes 0 drop + iifname != "br-40094534a5ee" oifname "br-40094534a5ee" counter packets 0 bytes 0 drop + iifname != "br-679db9b21e00" oifname "br-679db9b21e00" counter packets 0 bytes 0 drop + iifname != "br-9d6c95e8d80c" oifname "br-9d6c95e8d80c" counter packets 0 bytes 0 drop + iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + iifname != "br-d44fef8fd602" oifname "br-d44fef8fd602" counter packets 0 bytes 0 drop + iifname != "br-dba077b9b543" oifname "br-dba077b9b543" counter packets 0 bytes 0 drop + iifname != "br-160f55da427c" oifname "br-160f55da427c" counter packets 0 bytes 0 drop + iifname != "br-07a5e2f2c42f" oifname "br-07a5e2f2c42f" counter packets 0 bytes 0 drop + iifname != "docker0" oifname "docker0" counter packets 0 bytes 0 drop + iifname != "br-613eb68ef5fb" oifname "br-613eb68ef5fb" counter packets 0 bytes 0 drop + iifname != "br-4b504efd6080" oifname "br-4b504efd6080" counter packets 0 bytes 0 drop + iifname != "br-ef6df03f71a0" oifname "br-ef6df03f71a0" counter packets 0 bytes 0 drop + iifname != "br-ec480f77ac34" oifname "br-ec480f77ac34" counter packets 0 bytes 0 drop + iifname != "br-669fda75f1ac" oifname "br-669fda75f1ac" counter packets 0 bytes 0 drop + iifname != "br-65f6dc782562" oifname "br-65f6dc782562" counter packets 0 bytes 0 drop + } + + chain DOCKER-FORWARD { + counter packets 6530319 bytes 11196484299 jump DOCKER-CT + counter packets 3312487 bytes 5001091084 jump DOCKER-INTERNAL + counter packets 3312487 bytes 5001091084 jump DOCKER-BRIDGE + iifname "br-c70303d221ee" counter packets 0 bytes 0 accept + iifname "br-b3240c822bce" counter packets 0 bytes 0 accept + iifname "br-3636e05760a9" counter packets 43 bytes 9355 accept + iifname "br-3a760a74f4f6" counter packets 0 bytes 0 accept + iifname "br-3dcb6ef83ea1" counter packets 0 bytes 0 accept + iifname "br-40094534a5ee" counter packets 0 bytes 0 accept + iifname "br-679db9b21e00" counter packets 0 bytes 0 accept + iifname "br-9d6c95e8d80c" counter packets 0 bytes 0 accept + iifname "br-af4c9c2aa60a" counter packets 2761311 bytes 4959915711 accept + iifname "br-d44fef8fd602" counter packets 0 bytes 0 accept + iifname "br-dba077b9b543" counter packets 0 bytes 0 accept + iifname "br-160f55da427c" counter packets 0 bytes 0 accept + iifname "br-07a5e2f2c42f" counter packets 0 bytes 0 accept + iifname "docker0" counter packets 460394 bytes 25754554 accept + iifname "br-613eb68ef5fb" counter packets 10805 bytes 1792862 accept + iifname "br-4b504efd6080" counter packets 33 bytes 2892 accept + iifname "br-ef6df03f71a0" counter packets 0 bytes 0 accept + iifname "br-ec480f77ac34" counter packets 0 bytes 0 accept + iifname "br-669fda75f1ac" counter packets 0 bytes 0 accept + iifname "br-65f6dc782562" counter packets 0 bytes 0 accept + } + + chain DOCKER-BRIDGE { + oifname "br-c70303d221ee" counter packets 0 bytes 0 jump DOCKER + oifname "br-b3240c822bce" counter packets 0 bytes 0 jump DOCKER + oifname "br-3636e05760a9" counter packets 0 bytes 0 jump DOCKER + oifname "br-3a760a74f4f6" counter packets 0 bytes 0 jump DOCKER + oifname "br-3dcb6ef83ea1" counter packets 0 bytes 0 jump DOCKER + oifname "br-40094534a5ee" counter packets 0 bytes 0 jump DOCKER + oifname "br-679db9b21e00" counter packets 0 bytes 0 jump DOCKER + oifname "br-9d6c95e8d80c" counter packets 0 bytes 0 jump DOCKER + oifname "br-af4c9c2aa60a" counter packets 118 bytes 8400 jump DOCKER + oifname "br-d44fef8fd602" counter packets 0 bytes 0 jump DOCKER + oifname "br-dba077b9b543" counter packets 0 bytes 0 jump DOCKER + oifname "br-160f55da427c" counter packets 0 bytes 0 jump DOCKER + oifname "br-07a5e2f2c42f" counter packets 0 bytes 0 jump DOCKER + oifname "docker0" counter packets 0 bytes 0 jump DOCKER + oifname "br-613eb68ef5fb" counter packets 0 bytes 0 jump DOCKER + oifname "br-4b504efd6080" counter packets 0 bytes 0 jump DOCKER + oifname "br-ef6df03f71a0" counter packets 0 bytes 0 jump DOCKER + oifname "br-ec480f77ac34" counter packets 0 bytes 0 jump DOCKER + oifname "br-669fda75f1ac" counter packets 0 bytes 0 jump DOCKER + oifname "br-65f6dc782562" counter packets 0 bytes 0 jump DOCKER + } + + chain DOCKER-CT { + oifname "br-c70303d221ee" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-b3240c822bce" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-3636e05760a9" xt match "conntrack" counter packets 35 bytes 23113 accept + oifname "br-3a760a74f4f6" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-3dcb6ef83ea1" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-40094534a5ee" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-679db9b21e00" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-9d6c95e8d80c" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-af4c9c2aa60a" xt match "conntrack" counter packets 2488066 bytes 1053784742 accept + oifname "br-d44fef8fd602" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-dba077b9b543" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-160f55da427c" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-07a5e2f2c42f" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "docker0" xt match "conntrack" counter packets 666252 bytes 5079577802 accept + oifname "br-613eb68ef5fb" xt match "conntrack" counter packets 7926 bytes 7636543 accept + oifname "br-4b504efd6080" xt match "conntrack" counter packets 29 bytes 10870 accept + oifname "br-ef6df03f71a0" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-ec480f77ac34" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-669fda75f1ac" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-65f6dc782562" xt match "conntrack" counter packets 0 bytes 0 accept + } + + chain DOCKER-INTERNAL { + } + + chain FORWARD { + type filter hook forward priority filter; policy drop; + counter packets 33747166 bytes 176750349038 jump DOCKER-USER + counter packets 6530319 bytes 11196484299 jump DOCKER-FORWARD + } + + chain DOCKER-USER { + oifname "mlab*" counter packets 15657582 bytes 162801189460 accept + iifname "mlab*" counter packets 10958315 bytes 687322055 accept + oifname "incusbr0" counter packets 388768 bytes 2053271282 accept + iifname "incusbr0" counter packets 212182 bytes 12081942 accept + } +} +# Warning: table ip6 nat is managed by iptables-nft, do not touch! +table ip6 nat { + chain DOCKER { + } + + chain PREROUTING { + type nat hook prerouting priority dstnat; policy accept; + xt match "addrtype" counter packets 532 bytes 113834 jump DOCKER + } + + chain OUTPUT { + type nat hook output priority dstnat; policy accept; + ip6 daddr != ::1 xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } +} +table ip6 filter { + chain DOCKER { + } + + chain DOCKER-FORWARD { + counter packets 0 bytes 0 jump DOCKER-CT + counter packets 0 bytes 0 jump DOCKER-INTERNAL + counter packets 0 bytes 0 jump DOCKER-BRIDGE + } + + chain DOCKER-BRIDGE { + } + + chain DOCKER-CT { + } + + chain DOCKER-INTERNAL { + } + + chain FORWARD { + type filter hook forward priority filter; policy accept; + counter packets 0 bytes 0 jump DOCKER-USER + counter packets 0 bytes 0 jump DOCKER-FORWARD + } + + chain DOCKER-USER { + } +} +table ip raw { + chain PREROUTING { + type filter hook prerouting priority raw; policy accept; + ip daddr 172.19.0.2 iifname != "br-c70303d221ee" counter packets 0 bytes 0 drop + ip daddr 192.168.80.2 iifname != "br-3636e05760a9" counter packets 0 bytes 0 drop + ip daddr 192.168.80.3 iifname != "br-3636e05760a9" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 15673 counter packets 0 bytes 0 drop + ip daddr 192.168.80.4 iifname != "br-3636e05760a9" counter packets 0 bytes 0 drop + ip daddr 172.17.0.2 iifname != "docker0" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 57732 counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 57733 counter packets 0 bytes 0 drop + ip daddr 172.17.0.3 iifname != "docker0" counter packets 0 bytes 0 drop + ip daddr 172.17.0.4 iifname != "docker0" counter packets 0 bytes 0 drop + ip daddr 192.168.32.2 iifname != "br-613eb68ef5fb" counter packets 0 bytes 0 drop + ip daddr 172.30.0.7 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 172.29.0.2 iifname != "br-b3240c822bce" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 15672 counter packets 0 bytes 0 drop + ip daddr 172.25.0.2 iifname != "br-4b504efd6080" counter packets 0 bytes 0 drop + ip daddr 172.30.0.9 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 192.168.48.2 iifname != "br-ef6df03f71a0" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 5432 counter packets 0 bytes 0 drop + ip daddr 192.168.48.3 iifname != "br-ef6df03f71a0" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 8081 counter packets 0 bytes 0 drop + ip daddr 172.30.0.8 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 172.31.0.2 iifname != "br-ec480f77ac34" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 6379 counter packets 0 bytes 0 drop + ip daddr 172.30.0.4 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 8001 counter packets 0 bytes 0 drop + ip daddr 172.31.0.3 iifname != "br-ec480f77ac34" counter packets 0 bytes 0 drop + ip daddr 172.28.0.2 iifname != "br-669fda75f1ac" counter packets 0 bytes 0 drop + ip daddr 172.30.0.6 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 172.28.0.3 iifname != "br-669fda75f1ac" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 28080 counter packets 0 bytes 0 drop + ip daddr 172.30.0.3 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 6789 counter packets 0 bytes 0 drop + ip daddr 172.30.0.5 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 172.22.0.2 iifname != "br-65f6dc782562" counter packets 0 bytes 0 drop + ip daddr 172.30.0.10 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 172.22.0.3 iifname != "br-65f6dc782562" counter packets 0 bytes 0 drop + ip daddr 172.30.0.2 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 172.17.0.5 iifname != "docker0" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 55541 counter packets 0 bytes 0 drop + } +} +table ip mangle { + chain FORWARD { + type filter hook forward priority mangle; policy accept; + tcp flags & (syn | rst) == syn counter packets 13760 bytes 825476 xt target "TCPMSS" + } +} diff --git a/internal/store/store.go b/internal/store/store.go index e778420..3e8a82a 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -83,6 +83,23 @@ type State struct { // nothing here is ever removed as an orphan — what is held is not the host's to remove, even // when its module is unassigned. Held []Held `json:"held,omitempty"` + + // Firewall is the firewall found on this machine when it was first adopted, and whether the + // mesh has since retired it (novox/hq ADR 0100). Nil on a node that was never adopted. + Firewall *FoundFirewall `json:"firewall,omitempty"` +} + +// FoundFirewall is what the host found filtering this machine, and what it did about it. +type FoundFirewall struct { + // Kind is ufw or none: an unsupported kind is refused adoption, never recorded. + Kind string `json:"kind"` + // WasActive is whether it was in force when found — which is what converging the node + // retires, and returning it to adopted restores. + WasActive bool `json:"was_active,omitempty"` + // DisabledByMesh is set when converging retired it, so returning to adopted enables it again + // and nothing else ever does. + DisabledByMesh bool `json:"disabled_by_mesh,omitempty"` + FoundAt time.Time `json:"found_at"` } // Held is one file or container found on an adopted node — present at a declared path or name, diff --git a/internal/system/system.go b/internal/system/system.go index 3f928e6..25ae307 100644 --- a/internal/system/system.go +++ b/internal/system/system.go @@ -178,6 +178,9 @@ func everyShape() []declaration.Type { // it: the mesh's own code runs as a process on the machine, and only software that // genuinely needs isolation asks for a container. declaration.TypeProcess, + // An opening is a rule in the firewall found on the machine, which a partial host neither + // has nor can manage (novox/hq ADR 0100). + declaration.TypeOpening, } } From 770f58940123f090428d8a7cff1d7f39b1ee2ad3 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 17:22:31 +0200 Subject: [PATCH 04/52] Report what an adopted node holds, its firewall and what is reachable, and speak unasked when that changes (hq ADR 0100) --- cmd/mesh-host/main.go | 102 +++++++++++- cmd/mesh-host/main_test.go | 34 ++++ internal/identity/identity_test.go | 13 ++ internal/identity/token.go | 5 + internal/link/messages.go | 42 +++++ internal/link/messages_test.go | 8 + internal/link/run.go | 21 ++- internal/reachable/collect.go | 181 ++++++++++++++++++++++ internal/reachable/collect_test.go | 100 ++++++++++++ internal/reachable/testdata/docker-ps.txt | 7 + internal/reachable/testdata/ss.txt | 23 +++ 11 files changed, 528 insertions(+), 8 deletions(-) create mode 100644 internal/reachable/collect.go create mode 100644 internal/reachable/collect_test.go create mode 100644 internal/reachable/testdata/docker-ps.txt create mode 100644 internal/reachable/testdata/ss.txt diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 0bb9631..12f3e42 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -20,6 +20,7 @@ import ( "path/filepath" "sort" "strings" + "sync" "syscall" "text/tabwriter" "time" @@ -27,10 +28,12 @@ import ( "github.com/novox/mesh-host/internal/apply" "github.com/novox/mesh-host/internal/bundle" "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/firewall" "github.com/novox/mesh-host/internal/identity" "github.com/novox/mesh-host/internal/inventory" "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/profile" + "github.com/novox/mesh-host/internal/reachable" "github.com/novox/mesh-host/internal/store" "github.com/novox/mesh-host/internal/system" "github.com/novox/mesh-host/internal/upgrade" @@ -437,6 +440,23 @@ func enrol(ctx context.Context, opts options) error { return err } + // An adopted node keeps the firewall it was found with (novox/hq ADR 0100), so a host that + // cannot speak that firewall must say so now — before the mesh records a node it could never + // open anything on. + if token.Adopted { + kind, name, err := firewall.Detect(ctx, apply.ExecRunner) + if err != nil { + return err + } + if kind == firewall.Unsupported { + return fmt.Errorf( + "this token joins this machine adopted, keeping the firewall found on it, and it is "+ + "filtered by %s, which no host speaks yet. Nothing was enrolled", name) + } + fmt.Printf("joining adopted: what is on this machine is kept, and its firewall (%s) stays in force\n", + string(kind)) + } + fmt.Printf("token for broker %s\n", token.Broker) fmt.Printf(" pinned certificate %s\n", token.Fingerprint) fmt.Printf(" signing key %s\n", @@ -616,7 +636,22 @@ func runLink(ctx context.Context, opts options) error { // new declarations; this holds the machine in the last one whether the link is up or not. A // laptop shut for a week comes back and reconciles — it does not come back and ask what it is // (novox/hq ADR 0004). - go holdTheMachine(ctx, opts, mine, say, sched) + // Reports a reconcile has to make unasked — what an adopted node holds changed, or its + // firewall did — go out over the link when it is up (novox/hq ADR 0100). + outbox := make(chan link.Report, 1) + watch := &adoptionWatch{} + applier = watch.noting(applier) + go holdTheMachine(ctx, opts, mine, say, sched, func(r link.Report) { + if !watch.changed(r) { + return + } + select { + case <-outbox: + // An older one nobody has published yet; this one says everything it did. + default: + } + outbox <- r + }) return link.HoldRoused(ctx, link.Membership{ Node: mine.Node, @@ -624,7 +659,46 @@ func runLink(ctx context.Context, opts options) error { Fingerprint: mine.Membership.Fingerprint, Password: mine.Membership.Password, Signer: mine.Membership.Signer, - }, applier, say, opts.timeout, rousedBySignal(ctx)) + }, applier, say, opts.timeout, rousedBySignal(ctx), outbox) +} + +// adoptionWatch remembers what the node last said about what it holds and its firewall, so a +// reconcile speaks unasked only when that changed. +type adoptionWatch struct { + mu sync.Mutex + last string +} + +// fingerprint is what a report says about adoption: each hold and whether it changed, and the +// firewall. +func adoptionFingerprint(r link.Report) string { + parts := []string{"firewall=" + r.Firewall} + for _, h := range r.Held { + parts = append(parts, h.ID+"="+h.Changed) + } + sort.Strings(parts[1:]) + return strings.Join(parts, "\n") +} + +// changed records a report and says whether it differs from the last one that went out. +func (w *adoptionWatch) changed(r link.Report) bool { + w.mu.Lock() + defer w.mu.Unlock() + now := adoptionFingerprint(r) + if now == w.last { + return false + } + w.last = now + return true +} + +// noting wraps the applier, so a report the link publishes after a delivery counts as said. +func (w *adoptionWatch) noting(apply link.Applier) link.Applier { + return func(ctx context.Context, raw, signature []byte) link.Report { + r := apply(ctx, raw, signature) + w.changed(r) + return r + } } // rousedBySignal is the machine telling this process that its link is probably stale. @@ -672,7 +746,7 @@ func rousedBySignal(ctx context.Context) link.Roused { const ReconcileEvery = 5 * time.Minute func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, say link.Announce, - sched *apply.Scheduler) { + sched *apply.Scheduler, publish func(link.Report)) { ticker := time.NewTicker(ReconcileEvery) defer ticker.Stop() @@ -695,6 +769,12 @@ func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, s } report := applyDeclared(ctx, opts, declared, sched) + // A reconcile is otherwise silent. On an adopted node it speaks when what it holds or + // its firewall changed, because that is how a predecessor still writing is caught + // (novox/hq ADR 0100); publish decides whether anything did. + if publish != nil && report.Refused == "" && (len(report.Held) > 0 || report.Firewall != "") { + publish(report) + } switch { case report.Refused != "": say("what this node was last told no longer applies: " + report.Refused) @@ -761,6 +841,22 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D } report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw)} + // What this node found and holds, its firewall, and what is reachable on it — so an adopted + // node never reads as converged (novox/hq ADR 0100). + for _, h := range updated.Held { + report.Held = append(report.Held, link.Held{ID: h.ID, Module: h.Module, Kind: h.Kind, + Target: h.Target, Since: h.Since, Changed: h.Changed, Kept: h.Kept}) + } + if declared.Adoption != nil { + if updated.Firewall != nil { + report.Firewall = updated.Firewall.Kind + } + reached, err := reachable.Collect(ctx, apply.ExecRunner) + if err != nil { + fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not read what is reachable here: %v\n", err) + } + report.Reachable = reached + } for _, change := range outcome.Outcomes { // What is held is not what this machine owns: it was found, and is kept as it was until // its module is taken (novox/hq ADR 0100). diff --git a/cmd/mesh-host/main_test.go b/cmd/mesh-host/main_test.go index 562c36d..cdacdf7 100644 --- a/cmd/mesh-host/main_test.go +++ b/cmd/mesh-host/main_test.go @@ -1,6 +1,8 @@ package main import ( + "context" + "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/store" "testing" "time" @@ -135,3 +137,35 @@ func TestAFlagAfterAPositionalIsRead(t *testing.T) { } } } + +// Defends novox/hq ADR 0100: a reconcile on an adopted node speaks unasked only when what it holds +// or its firewall changed — which is how a predecessor still writing is caught, without a report +// every five minutes saying nothing new. +func TestAReconcileSpeaksOnlyWhenWhatIsHeldChanged(t *testing.T) { + w := &adoptionWatch{} + held := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page"}, {ID: "hello-web.server"}}} + if !w.changed(held) { + t.Fatal("the first report of a hold was not said") + } + again := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.server"}, {ID: "hello-web.page"}}} + if w.changed(again) { + t.Error("the same holds in another order were said again") + } + rewritten := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page", Changed: "rewritten"}, {ID: "hello-web.server"}}} + if !w.changed(rewritten) { + t.Error("a held file rewritten by something else was not said") + } + if !w.changed(link.Report{Firewall: "none", Held: rewritten.Held}) { + t.Error("a changed firewall was not said") + } +} + +func TestWhatTheLinkPublishedCountsAsSaid(t *testing.T) { + w := &adoptionWatch{} + report := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "a"}}} + applier := w.noting(func(context.Context, []byte, []byte) link.Report { return report }) + applier(context.Background(), nil, nil) + if w.changed(report) { + t.Error("a reconcile repeated what the link had just published") + } +} diff --git a/internal/identity/identity_test.go b/internal/identity/identity_test.go index 2d0c4f9..fcc0b2a 100644 --- a/internal/identity/identity_test.go +++ b/internal/identity/identity_test.go @@ -220,6 +220,19 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) { if len(fields) != 5 { t.Errorf("the token has %d fields, expected 5: %v", len(fields), fields) } + + // novox/hq ADR 0100: an adopted node's token says so, and a converged one's is unchanged. + raw, err = json.Marshal(Token{Version: 1, Secret: "s", Adopted: true}) + if err != nil { + t.Fatal(err) + } + fields = map[string]any{} + if err := json.Unmarshal(raw, &fields); err != nil { + t.Fatal(err) + } + if fields["adopted"] != true { + t.Errorf("an adopted token does not say \"adopted\": %v", fields) + } } func TestACompleteTokenParses(t *testing.T) { diff --git a/internal/identity/token.go b/internal/identity/token.go index 40fab81..4fbaf46 100644 --- a/internal/identity/token.go +++ b/internal/identity/token.go @@ -30,6 +30,11 @@ type Token struct { Fingerprint string `json:"fingerprint,omitempty"` Signer []byte `json:"signer,omitempty"` Secret string `json:"secret"` + + // Adopted says this node joins adopted (novox/hq ADR 0100). The host checks it speaks the + // firewall found here before enrolling, because an adopted node keeps that firewall in force. + // Absent for a converged node. + Adopted bool `json:"adopted,omitempty"` } // ParseToken reads a token a person pasted. diff --git a/internal/link/messages.go b/internal/link/messages.go index 6a48e41..953fe6c 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -1,5 +1,7 @@ package link +import "time" + // The wire formats shared with the control plane, which defines them separately because this // binary requires nothing present and does not import it. A test on each side asserts the field // names, so a rename breaks both at once rather than on a real machine months later. @@ -85,4 +87,44 @@ type Report struct { // than the send, and the machine reads as caught up with words it has not read yet. Clocks // cannot answer "which"; the digest is the answer itself. Declared string `json:"declared,omitempty"` + + // Held is what this adopted node found and is keeping as it was until its module is taken + // (novox/hq ADR 0100). Without it an adopted node reads as converged. + Held []Held `json:"held,omitempty"` + + // Firewall is the firewall found on this machine — "ufw" or "none" — and empty on a node that + // was never asked, which is every converged one. + Firewall string `json:"firewall,omitempty"` + + // Reachable is what can be reached on this machine now: every listening socket and every + // published container port. Only an adopted node reports it; it is what converging the node + // previews, so nothing closes without being named first. + Reachable []Reach `json:"reachable,omitempty"` +} + +// Held is one file or container found on an adopted node and kept as it was. +type Held struct { + ID string `json:"id"` + Module string `json:"module"` + Kind string `json:"kind"` + Target string `json:"target"` + Since time.Time `json:"since"` + // Changed is what something other than the mesh did to it since — rewritten, stopped, + // replaced or gone — and empty while it is as found. + Changed string `json:"changed,omitempty"` + // Kept is where a file's original was kept. + Kept string `json:"kept,omitempty"` +} + +// Reach is one thing reachable on the machine: a listening socket, or a published container port. +type Reach struct { + Protocol string `json:"protocol"` + Address string `json:"address"` + Port int `json:"port"` + // By is what holds it — a process, or a container's name. + By string `json:"by,omitempty"` + // Published is a container port the runtime publishes, reached on the forwarded path; its + // container's own port is ContainerPort. + Published bool `json:"published,omitempty"` + ContainerPort int `json:"container-port,omitempty"` } diff --git a/internal/link/messages_test.go b/internal/link/messages_test.go index 7983221..76a9a7e 100644 --- a/internal/link/messages_test.go +++ b/internal/link/messages_test.go @@ -120,6 +120,14 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) { {Signed{Declaration: []byte("{}"), Signature: []byte("x")}, []string{"declaration", "signature"}}, {Report{Node: "n", Applied: []string{"a"}, Failed: map[string]string{"k": "v"}, Refused: "r"}, []string{"node", "applied", "failed", "refused"}}, + // novox/hq ADR 0100: what an adopted node holds, the firewall it was found with, and what + // is reachable on it. + {Report{Node: "n", Held: []Held{{ID: "i"}}, Firewall: "ufw", Reachable: []Reach{{Port: 1}}}, + []string{"node", "held", "firewall", "reachable"}}, + {Held{ID: "i", Module: "m", Kind: "file", Target: "/t", Changed: "rewritten", Kept: "/k"}, + []string{"id", "module", "kind", "target", "since", "changed", "kept"}}, + {Reach{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "c", Published: true, ContainerPort: 80}, + []string{"protocol", "address", "port", "by", "published", "container-port"}}, } { raw, err := json.Marshal(c.value) if err != nil { diff --git a/internal/link/run.go b/internal/link/run.go index 247d015..5668af4 100644 --- a/internal/link/run.go +++ b/internal/link/run.go @@ -70,15 +70,21 @@ type Announce func(string) type Roused <-chan struct{} func Hold(ctx context.Context, m Membership, apply Applier, say Announce, timeout time.Duration) error { - return HoldRoused(ctx, m, apply, say, timeout, nil) + return HoldRoused(ctx, m, apply, say, timeout, nil, nil) } -// HoldRoused is Hold, told when the machine has reason to think its link is stale. +// Outbox carries reports the node has to say without having been sent anything — what a +// reconcile found changed on an adopted node (novox/hq ADR 0100). Published while the link is up; +// a report made while it is down waits in the channel for the next one. Nil is allowed. +type Outbox <-chan Report + +// HoldRoused is Hold, told when the machine has reason to think its link is stale, and handed +// reports to publish between deliveries. func HoldRoused(ctx context.Context, m Membership, apply Applier, say Announce, - timeout time.Duration, roused Roused) error { + timeout time.Duration, roused Roused, outbox Outbox) error { return holdWith(ctx, func(ctx context.Context) error { - return Run(ctx, m, apply, say, timeout) + return Run(ctx, m, apply, say, timeout, outbox) }, say, roused) } @@ -171,7 +177,8 @@ func holdWith(ctx context.Context, run attempt, say Announce, roused Roused) err // // Outbound only, and nothing listens on this machine. Returns when the link ends, for any reason; // Hold is what decides whether to open it again. -func Run(ctx context.Context, m Membership, apply Applier, say Announce, timeout time.Duration) error { +func Run(ctx context.Context, m Membership, apply Applier, say Announce, timeout time.Duration, + outbox Outbox) error { if say == nil { say = func(string) {} } @@ -254,6 +261,10 @@ func Run(ctx context.Context, m Membership, apply Applier, say Announce, timeout return nil case <-beat.C: publishAlive(ctx, channel, m, say, timeout) + case report := <-outbox: + // Said without having been asked: a reconcile found what an adopted node holds, or + // its firewall, changed since it last said. + publishReport(ctx, channel, m, report, say, timeout) case reason := <-closed: return fmt.Errorf("the link closed: %v", reason) case delivery, ok := <-deliveries: diff --git a/internal/reachable/collect.go b/internal/reachable/collect.go new file mode 100644 index 0000000..537e2e8 --- /dev/null +++ b/internal/reachable/collect.go @@ -0,0 +1,181 @@ +// Package reachable reads what can be reached on this machine now: every listening socket, and +// every container port the runtime publishes (novox/hq ADR 0100). +// +// It is what converging an adopted node previews — each port, whether a module declares it or it +// will close — and what a converged genesis counts before refusing a machine in use. It reads; it +// never decides what is the mesh's. +package reachable + +import ( + "context" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/system" +) + +// Runner executes a command. +type Runner = system.Runner + +// Reach is one thing reachable on this machine, in the words the report carries. +type Reach = link.Reach + +// Collect reads the machine's listening sockets and the runtime's published ports. A published +// port is reported once, as published, rather than again as the runtime's proxy listening for it. +func Collect(ctx context.Context, run Runner) ([]Reach, error) { + out, err := run(ctx, "ss", "-Hltunp") + if err != nil { + return nil, fmt.Errorf("reading this machine's listening sockets: %w", err) + } + sockets := Sockets(out) + + var published []Reach + if ps, err := run(ctx, "docker", "ps", "--format", "{{.Names}}\t{{.Ports}}"); err == nil { + published = Published(ps) + } + return Merge(sockets, published), nil +} + +var process = regexp.MustCompile(`users:\(\("([^"]+)"`) + +// Sockets parses `ss -Hltunp`: each line a netid, a state, two queues, the local address and +// port, the peer, and the process when ss may name it. +func Sockets(out string) []Reach { + var reached []Reach + for _, line := range strings.Split(out, "\n") { + fields := strings.Fields(line) + if len(fields) < 5 { + continue + } + protocol := fields[0] + if protocol != "tcp" && protocol != "udp" { + continue + } + address, port, ok := splitLocal(fields[4]) + if !ok { + continue + } + r := Reach{Protocol: protocol, Address: address, Port: port} + if m := process.FindStringSubmatch(line); m != nil { + r.By = m[1] + } + reached = append(reached, r) + } + return reached +} + +// splitLocal reads "127.0.0.1:53", "[::]:22", "*:22" and "[fe80::1]%veth0:123". +func splitLocal(local string) (string, int, bool) { + i := strings.LastIndex(local, ":") + if i < 0 { + return "", 0, false + } + port, err := strconv.Atoi(local[i+1:]) + if err != nil { + return "", 0, false + } + address := local[:i] + if at := strings.Index(address, "%"); at >= 0 { + address = address[:at] + } + address = strings.TrimSuffix(strings.TrimPrefix(address, "["), "]") + if address == "*" { + address = "0.0.0.0" + } + return address, port, true +} + +// Published parses `docker ps --format '{{.Names}}\t{{.Ports}}'`. Only what is published on the +// machine counts; a port a container exposes and nothing publishes is not reachable from outside it. +func Published(out string) []Reach { + var reached []Reach + for _, line := range strings.Split(out, "\n") { + name, ports, ok := strings.Cut(strings.TrimSpace(line), "\t") + if !ok { + continue + } + for _, mapping := range strings.Split(ports, ",") { + reached = append(reached, mappingOf(name, strings.TrimSpace(mapping))...) + } + } + return reached +} + +// mappingOf reads "0.0.0.0:9000-9001->9000-9001/tcp" into one reach per port. +func mappingOf(name, mapping string) []Reach { + outer, inner, ok := strings.Cut(mapping, "->") + if !ok { + return nil + } + inner, protocol, ok := strings.Cut(inner, "/") + if !ok { + return nil + } + i := strings.LastIndex(outer, ":") + if i < 0 { + return nil + } + address := strings.TrimSuffix(strings.TrimPrefix(outer[:i], "["), "]") + from, to, ok := portRange(outer[i+1:]) + if !ok { + return nil + } + cfrom, _, ok := portRange(inner) + if !ok { + return nil + } + var reached []Reach + for p := from; p <= to; p++ { + reached = append(reached, Reach{Protocol: protocol, Address: address, Port: p, By: name, + Published: true, ContainerPort: cfrom + (p - from)}) + } + return reached +} + +func portRange(s string) (int, int, bool) { + a, b, isRange := strings.Cut(s, "-") + from, err := strconv.Atoi(a) + if err != nil { + return 0, 0, false + } + if !isRange { + return from, from, true + } + to, err := strconv.Atoi(b) + if err != nil || to < from { + return 0, 0, false + } + return from, to, true +} + +// Merge puts the published ports beside the sockets, dropping the runtime proxy's own socket for a +// port that is reported as published already, and sorts the whole by port. +func Merge(sockets, published []Reach) []Reach { + key := func(r Reach) string { return r.Protocol + " " + r.Address + " " + strconv.Itoa(r.Port) } + isPublished := map[string]bool{} + for _, p := range published { + isPublished[key(p)] = true + } + var out []Reach + for _, s := range sockets { + if s.By == "docker-proxy" && isPublished[key(s)] { + continue + } + out = append(out, s) + } + out = append(out, published...) + sort.SliceStable(out, func(i, j int) bool { + if out[i].Port != out[j].Port { + return out[i].Port < out[j].Port + } + if out[i].Protocol != out[j].Protocol { + return out[i].Protocol < out[j].Protocol + } + return out[i].Address < out[j].Address + }) + return out +} diff --git a/internal/reachable/collect_test.go b/internal/reachable/collect_test.go new file mode 100644 index 0000000..1045631 --- /dev/null +++ b/internal/reachable/collect_test.go @@ -0,0 +1,100 @@ +package reachable + +import ( + "context" + "os" + "strings" + "testing" +) + +// Defends novox/hq ADR 0100: converging previews every listening socket and every published +// container port. Fixtures are captured from a real machine. + +func fixture(t *testing.T, name string) string { + t.Helper() + raw, err := os.ReadFile("testdata/" + name) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +func find(rs []Reach, protocol, address string, port int) (Reach, bool) { + for _, r := range rs { + if r.Protocol == protocol && r.Address == address && r.Port == port { + return r, true + } + } + return Reach{}, false +} + +func TestSocketsAreReadWithWhatHoldsThem(t *testing.T) { + got := Sockets(fixture(t, "ss.txt")) + if r, ok := find(got, "tcp", "0.0.0.0", 22); !ok || r.By != "sshd" { + t.Errorf("ssh not read: %+v", r) + } + if r, ok := find(got, "tcp", "::", 445); !ok || r.By != "smbd" { + t.Errorf("an IPv6 wildcard listener not read: %+v", r) + } + if _, ok := find(got, "udp", "fe80::849e:ccff:fea8:24c7", 123); !ok { + t.Error("a link-local address with a scope was not read") + } + if r, ok := find(got, "udp", "127.0.0.1", 53); !ok || r.By != "dnsmasq" { + t.Errorf("a loopback udp socket not read: %+v", r) + } +} + +func TestPublishedPortsNameTheirContainerAndItsPort(t *testing.T) { + got := Published(fixture(t, "docker-ps.txt")) + if r, ok := find(got, "tcp", "0.0.0.0", 8770); !ok || r.By != "whisper" || r.ContainerPort != 8000 || !r.Published { + t.Errorf("a published port: %+v", r) + } + if r, ok := find(got, "tcp", "0.0.0.0", 9001); !ok || r.ContainerPort != 9001 { + t.Errorf("a published range was not expanded: %+v", r) + } + if r, ok := find(got, "tcp", "127.0.0.1", 15673); !ok || r.ContainerPort != 15672 { + t.Errorf("a loopback-published port: %+v", r) + } + for _, r := range got { + if r.By == "umami_db" { + t.Errorf("an exposed and unpublished port was reported reachable: %+v", r) + } + } +} + +func TestAPublishedPortIsReportedOnceAsPublished(t *testing.T) { + merged := Merge(Sockets(fixture(t, "ss.txt")), Published(fixture(t, "docker-ps.txt"))) + n := 0 + for _, r := range merged { + if r.Protocol == "tcp" && r.Address == "0.0.0.0" && r.Port == 8770 { + n++ + if !r.Published { + t.Errorf("the runtime's proxy was reported instead of the published port: %+v", r) + } + } + } + if n != 1 { + t.Errorf("port 8770 reported %d times", n) + } + if _, ok := find(merged, "tcp", "0.0.0.0", 22); !ok { + t.Error("a socket was lost in the merge") + } +} + +func TestCollectAsksSsAndTheRuntime(t *testing.T) { + var asked []string + run := func(_ context.Context, name string, args ...string) (string, error) { + asked = append(asked, name+" "+strings.Join(args, " ")) + if name == "ss" { + return fixture(t, "ss.txt"), nil + } + return fixture(t, "docker-ps.txt"), nil + } + got, err := Collect(context.Background(), run) + if err != nil || len(got) == 0 { + t.Fatalf("%v %v", got, err) + } + if len(asked) != 2 { + t.Errorf("asked %v", asked) + } +} diff --git a/internal/reachable/testdata/docker-ps.txt b/internal/reachable/testdata/docker-ps.txt new file mode 100644 index 0000000..41d56c1 --- /dev/null +++ b/internal/reachable/testdata/docker-ps.txt @@ -0,0 +1,7 @@ +mesh-controller-check-adoption 127.0.0.1:55541->5432/tcp +umami_db 5432/tcp +whisper 0.0.0.0:8770->8000/tcp, [::]:8770->8000/tcp +keycloak 8443/tcp, 127.0.0.1:28080->8080/tcp +minio-lb 0.0.0.0:9000-9001->9000-9001/tcp, [::]:9000-9001->9000-9001/tcp +wonderful_mahavira +anton-lavinmq 0.0.0.0:5680->5672/tcp, [::]:5680->5672/tcp, 127.0.0.1:15673->15672/tcp diff --git a/internal/reachable/testdata/ss.txt b/internal/reachable/testdata/ss.txt new file mode 100644 index 0000000..4d1c61e --- /dev/null +++ b/internal/reachable/testdata/ss.txt @@ -0,0 +1,23 @@ +udp UNCONN 0 0 0.0.0.0:55558 0.0.0.0:* users:(("firefox",pid=2283907,fd=288)) +udp UNCONN 0 0 0.0.0.0:59541 0.0.0.0:* users:(("firefox",pid=2283907,fd=241)) +udp UNCONN 0 0 127.0.0.1:53 0.0.0.0:* users:(("dnsmasq",pid=1189392,fd=6)) +udp UNCONN 0 0 0.0.0.0:33525 0.0.0.0:* users:(("firefox",pid=2283907,fd=304)) +udp UNCONN 0 0 0.0.0.0:41749 0.0.0.0:* users:(("firefox",pid=2283907,fd=351)) +tcp LISTEN 0 4096 127.0.0.1:55541 0.0.0.0:* users:(("docker-proxy",pid=4108732,fd=7)) +tcp LISTEN 0 4096 0.0.0.0:9001 0.0.0.0:* users:(("docker-proxy",pid=1849130,fd=7)) +tcp LISTEN 0 4096 0.0.0.0:8770 0.0.0.0:* users:(("docker-proxy",pid=1920035,fd=7)) +tcp LISTEN 0 50 0.0.0.0:445 0.0.0.0:* users:(("smbd",pid=1248,fd=29)) +tcp LISTEN 0 128 0.0.0.0:22 0.0.0.0:* users:(("sshd",pid=1188536,fd=6)) +tcp LISTEN 0 50 0.0.0.0:139 0.0.0.0:* users:(("smbd",pid=1248,fd=30)) +tcp LISTEN 0 4096 127.0.0.1:5432 0.0.0.0:* users:(("docker-proxy",pid=1854543,fd=7)) +tcp LISTEN 0 32 127.0.0.1:53 0.0.0.0:* users:(("dnsmasq",pid=1189392,fd=7)) +tcp LISTEN 0 4096 127.0.0.1:15673 0.0.0.0:* users:(("docker-proxy",pid=3170,fd=7)) +tcp LISTEN 0 4096 [::]:9001 [::]:* users:(("docker-proxy",pid=1849138,fd=7)) +tcp LISTEN 0 4096 [::]:8770 [::]:* users:(("docker-proxy",pid=1920043,fd=7)) +tcp LISTEN 0 50 [::]:445 [::]:* users:(("smbd",pid=1248,fd=27)) +tcp LISTEN 0 128 [::]:22 [::]:* users:(("sshd",pid=1188536,fd=7)) +tcp LISTEN 0 50 [::]:139 [::]:* users:(("smbd",pid=1248,fd=28)) +udp UNCONN 0 0 [fd42:f8c5:dae:d74c::1]:53 [::]:* +udp UNCONN 0 0 [fe80::849e:ccff:fea8:24c7]%veth6b2b7ba:123 [::]:* +udp UNCONN 0 0 [fe80::e45a:90ff:feca:148f]%vethb5e5a61:123 [::]:* +udp UNCONN 0 0 [fe80::c4ed:ccff:feb1:afd2]%veth005a182:123 [::]:* From 3964d9da0ab1ea18e226244abcd23cf3620695fd Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 17:28:36 +0200 Subject: [PATCH 05/52] Take the foundation's ports as genesis inputs, check them free, and hand them to the controller as the node's settings (hq ADR 0100) --- cmd/mesh-bootstrap/main.go | 61 +++- cmd/mesh-bootstrap/main_test.go | 26 ++ internal/bootstrap/bootstrap.go | 41 ++- internal/bootstrap/module.go | 10 + internal/bootstrap/phase2.go | 10 +- internal/bootstrap/phase2_test.go | 4 +- internal/bootstrap/phase3.go | 3 + internal/bootstrap/phase_packages.go | 21 +- internal/bootstrap/ports.go | 454 +++++++++++++++++++++++++++ internal/bootstrap/ports_test.go | 286 +++++++++++++++++ 10 files changed, 902 insertions(+), 14 deletions(-) create mode 100644 internal/bootstrap/ports.go create mode 100644 internal/bootstrap/ports_test.go diff --git a/cmd/mesh-bootstrap/main.go b/cmd/mesh-bootstrap/main.go index e48aa1a..7a1fba5 100644 --- a/cmd/mesh-bootstrap/main.go +++ b/cmd/mesh-bootstrap/main.go @@ -25,6 +25,7 @@ import ( "net/http" "os" "os/signal" + "strconv" "syscall" "time" @@ -126,6 +127,14 @@ const usage = `mesh-bootstrap — make a bare machine into a mesh --packet-filter which packet filter to run (nftables) --extras catalogue modules beyond the floor, comma-separated + The foundation's ports are this machine's, each checked free before anything is + raised and kept as the node's setting for the module that binds it: + --store-port 5432 --bus-port 5671 --amqp-port 5672 --management-port 15672 + --registry-port 5000 (follows --registry, and must agree with it) + --packages-port 3000 --hub-port 51820/udp + --overlay-range the private network's range (default 10.42.0.0/16); refused + if it overlaps an interface or route the machine already has + The installer carries a builder, not a control plane. What raises a mesh is therefore the same thing that will maintain it, and the control plane a mesh ends up running is one it built itself, from a repository and a commit it can name and build again. @@ -176,7 +185,9 @@ func parseArgs(args []string) (string, bootstrap.Options, bool, error) { HostService: defaultService, // Longer than the host's 10s: these probes reach a container runtime that may be busy // pulling, and a probe that times out on a working machine is a false refusal. - Timeout: 30 * time.Second, + Ports: bootstrap.DefaultPorts(), + OverlayRange: bootstrap.DefaultOverlayRange, + Timeout: 30 * time.Second, // A socket-activated runtime queued behind the network, and a control plane running its // first `initdb`-shaped wait, are both minutes rather than seconds. Wait: 3 * time.Minute, @@ -210,9 +221,38 @@ func parseArgs(args []string) (string, bootstrap.Options, bool, error) { return "", opts, false, fmt.Errorf( "unexpected argument %q — try `mesh-bootstrap help`", positionals[0]) } + if err := registryAgrees(set, &opts); err != nil { + return "", opts, false, err + } return command, opts, jsonOut, nil } +// registryAgrees makes --registry and --registry-port say one port (novox/hq ADR 0100): the +// registry is raised on the port the node gives it, and every node pulls from the address given. +// Either may be said alone and the other follows; said both ways, they must agree. +func registryAgrees(set *flag.FlagSet, opts *bootstrap.Options) error { + said := map[string]bool{} + set.Visit(func(f *flag.Flag) { said[f.Name] = true }) + host, portText, err := net.SplitHostPort(opts.Registry) + if err != nil { + return fmt.Errorf("--registry %q is not host:port: %w", opts.Registry, err) + } + port, err := strconv.Atoi(portText) + if err != nil { + return fmt.Errorf("--registry %q does not end in a port", opts.Registry) + } + switch { + case said["registry-port"] && said["registry"] && port != opts.Ports.Registry: + return fmt.Errorf("--registry %s and --registry-port %d name two ports for one registry", + opts.Registry, opts.Ports.Registry) + case said["registry-port"]: + opts.Registry = net.JoinHostPort(host, strconv.Itoa(opts.Ports.Registry)) + case said["registry"]: + opts.Ports.Registry = port + } + return nil +} + func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet { set := flag.NewFlagSet("mesh-bootstrap", flag.ContinueOnError) set.SetOutput(os.Stderr) @@ -255,6 +295,25 @@ func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet { set.StringVar(&opts.SDKSource.Ref, "sdk-ref", opts.SDKSource.Ref, "what of it to build (default main)") set.StringVar(&opts.Site, "site", "main", "where this machine sits, for the private network") + + // The foundation's ports are this node's (novox/hq ADR 0100): each is checked free before + // anything is raised, and becomes the node's setting for the module that binds it. + for _, p := range []struct { + name, what string + into *int + }{ + {"store-port", "the store", &opts.Ports.Store}, + {"bus-port", "the bus (amqps)", &opts.Ports.Bus}, + {"amqp-port", "the broker's AMQP", &opts.Ports.AMQP}, + {"management-port", "the broker's management, on loopback", &opts.Ports.Management}, + {"registry-port", "the registry", &opts.Ports.Registry}, + {"packages-port", "the package registry", &opts.Ports.Packages}, + {"hub-port", "the private network's hub (udp)", &opts.Ports.Hub}, + } { + set.IntVar(p.into, p.name, *p.into, "the machine's port for "+p.what) + } + set.StringVar(&opts.OverlayRange, "overlay-range", opts.OverlayRange, + "the private network's address range; must not overlap a tunnel the machine already runs") if opts.Answers == nil { opts.Answers = map[string]string{} } diff --git a/cmd/mesh-bootstrap/main_test.go b/cmd/mesh-bootstrap/main_test.go index 2fadbcb..2a4a4bb 100644 --- a/cmd/mesh-bootstrap/main_test.go +++ b/cmd/mesh-bootstrap/main_test.go @@ -164,3 +164,29 @@ func TestTheNodeNameCanBeSaid(t *testing.T) { t.Errorf("--catalog parsed as %q", opts.Catalogue) } } + +// Defends novox/hq ADR 0100: the foundation's ports are inputs to genesis, and the registry's port +// and the address nodes pull from say one port. +func TestTheFoundationsPortsAreGiven(t *testing.T) { + _, opts, _, err := parseArgs([]string{"--store-port", "5433", "--hub-port", "51821", "--overlay-range", "10.77.0.0/16"}) + if err != nil { + t.Fatal(err) + } + if opts.Ports.Store != 5433 || opts.Ports.Hub != 51821 || opts.Ports.Bus != 5671 || opts.OverlayRange != "10.77.0.0/16" { + t.Errorf("ports read as %+v, range %s", opts.Ports, opts.OverlayRange) + } +} + +func TestTheRegistrysPortAndAddressAgree(t *testing.T) { + _, opts, _, err := parseArgs([]string{"--registry-port", "5100"}) + if err != nil || opts.Registry != "127.0.0.1:5100" { + t.Errorf("--registry-port alone: %s %v", opts.Registry, err) + } + _, opts, _, err = parseArgs([]string{"--registry", "192.0.2.10:5100"}) + if err != nil || opts.Ports.Registry != 5100 { + t.Errorf("--registry alone: %d %v", opts.Ports.Registry, err) + } + if _, _, _, err := parseArgs([]string{"--registry", "192.0.2.10:5000", "--registry-port", "5100"}); err == nil { + t.Error("two ports for one registry were accepted") + } +} diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index 9d3d787..95b86b4 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -185,6 +185,20 @@ type Options struct { Prompt func(Choice) (string, error) // Extras are catalogue modules beyond the floor, asked for by name. Extras []string + + // Ports are the ports the foundation binds on this machine (novox/hq ADR 0100). Inputs to + // genesis, each checked free before anything is raised, and then the node's settings for the + // foundation's modules — so adopting the foundation as modules leaves it where it was raised. + // Zero means the catalogue's defaults. + Ports FoundationPorts + // OverlayRange is the private network's address range, checked against every interface and + // route the machine already has. Empty means the mesh's default. + OverlayRange string + + // Adopted raises this machine as an adopted node (novox/hq ADR 0100): what is on it is kept + // until each module is taken, its firewall stays in force, and the mesh guards its own ports + // in a table that only refuses. Without it, a machine in use is refused. + Adopted bool } // pivots reports whether this run goes past the foundation. @@ -287,6 +301,14 @@ type Result struct { // Stopped names why a run went no further. Empty on a run that pivoted. Stopped string `json:"stopped,omitempty"` + + // Adopted, the firewall found, and the ports the foundation was raised on (novox/hq ADR 0100). + Adopted bool `json:"adopted,omitempty"` + Firewall string `json:"firewall,omitempty"` + Ports FoundationPorts `json:"ports"` + // Filter is the packet filter chosen for when the node converges; an adopted genesis loads + // none, and the flip assigns this one. + Filter string `json:"filter-on-converge,omitempty"` } // Run performs the bootstrap, saying what it is doing as it goes. @@ -339,7 +361,12 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro if say == nil { say = func(string) {} } - result := Result{DryRun: o.DryRun} + result := Result{DryRun: o.DryRun, Adopted: o.Adopted} + o.Ports = o.Ports.orDefaults() + result.Ports = o.Ports + if err := o.Ports.Check(); err != nil { + return result, failed(StepPreflight, err) + } // ---- 1. preflight ------------------------------------------------------------------- say("preflight — what has to be true before anything is changed") @@ -399,12 +426,24 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro if err := RefuseExistingServers(ctx, d.Run, creds); err != nil { return result, failed(StepBundle, err) } + // The foundation's ports, its private network's range and its containers' names are checked + // free before anything is raised (novox/hq ADR 0100), each refusal naming what holds it. + if err := CheckTheMachine(ctx, o, d.Run, rewritten.Declaration, say); err != nil { + return result, failed(StepBundle, err) + } // From here on nothing this installer says contains the values it just made. say = Masking(say, creds) root, err := RewriteRoot(&rewritten, creds) if err != nil { return result, failed(StepBundle, err) } + moved, err := RewritePorts(&rewritten, o.Ports, o.OverlayRange) + if err != nil { + return result, failed(StepBundle, err) + } + if moved.Places > 0 { + say(fmt.Sprintf(" ports %d place(s) rewritten to this node's foundation ports", moved.Places)) + } for _, c := range []struct { what, path string made bool diff --git a/internal/bootstrap/module.go b/internal/bootstrap/module.go index 4ba2729..92c4116 100644 --- a/internal/bootstrap/module.go +++ b/internal/bootstrap/module.go @@ -124,6 +124,9 @@ func registerAndAssign(ctx context.Context, o Options, control controlPlane, mod // the only place the reason appears. say(indent(refusal)) } + if err := prepareModule(ctx, o, control, module, say); err != nil { + return out, err + } return out, nil } @@ -209,3 +212,10 @@ func pinPlaceholder(manifest []byte, reference, module string) ([]byte, int, err } return pinned, places, nil } + +// prepareModule is what genesis tells the controller about a module on this node once it is +// assigned and before it is pushed: the ports this node gave it (novox/hq ADR 0100). +func prepareModule(ctx context.Context, o Options, control controlPlane, module string, + say func(string)) error { + return setFoundationSettings(ctx, o, control, module, say) +} diff --git a/internal/bootstrap/phase2.go b/internal/bootstrap/phase2.go index 4eb21a3..431f12c 100644 --- a/internal/bootstrap/phase2.go +++ b/internal/bootstrap/phase2.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "net" + "strconv" "strings" "time" ) @@ -87,6 +88,9 @@ func InstallFromCatalogue(ctx context.Context, o Options, control controlPlane, if _, err := control.tell(ctx, "assign", o.Node, module); err != nil { return err } + if err := prepareModule(ctx, o, control, module, say); err != nil { + return err + } if _, err := pushNode(ctx, o, control, say); err != nil { return err } @@ -120,7 +124,7 @@ func PlaceOnTheNetwork(ctx context.Context, o Options, control controlPlane, Name: "endpoint", Question: "Where do other machines reach this one for the private network? " + "(host:port; the host other machines dial)", - Default: derivedEndpoint(brokerAddress), + Default: derivedEndpoint(brokerAddress, o.Ports.orDefaults().Hub), }, o.Answers["endpoint"], o.Prompt, say) if err != nil { return err @@ -202,12 +206,12 @@ func builds(manifest []byte) bool { // derivedEndpoint is the default place other machines dial for the private network: the same host // they already dial for the broker, on WireGuard's ordinary port. One fact, not two. -func derivedEndpoint(brokerAddress string) string { +func derivedEndpoint(brokerAddress string, hub int) string { host, _, err := net.SplitHostPort(brokerAddress) if err != nil || host == "" { return "" } - return net.JoinHostPort(host, "51820") + return net.JoinHostPort(host, strconv.Itoa(hub)) } func refOr(ref string) string { diff --git a/internal/bootstrap/phase2_test.go b/internal/bootstrap/phase2_test.go index cd212fc..918a553 100644 --- a/internal/bootstrap/phase2_test.go +++ b/internal/bootstrap/phase2_test.go @@ -5,10 +5,10 @@ import "testing" // The endpoint other machines dial defaults to the host they already dial — the broker's — on // WireGuard's port. One fact, not two that drift. func TestTheEndpointDerivesFromTheBrokerAddress(t *testing.T) { - if got := derivedEndpoint("192.0.2.10:5671"); got != "192.0.2.10:51820" { + if got := derivedEndpoint("192.0.2.10:5671", 51820); got != "192.0.2.10:51820" { t.Fatalf("derived %q", got) } - if got := derivedEndpoint(""); got != "" { + if got := derivedEndpoint("", 51820); got != "" { t.Fatalf("an endpoint was invented from nothing: %q", got) } } diff --git a/internal/bootstrap/phase3.go b/internal/bootstrap/phase3.go index e39b9e3..2d60811 100644 --- a/internal/bootstrap/phase3.go +++ b/internal/bootstrap/phase3.go @@ -122,6 +122,9 @@ func installProvider(ctx context.Context, o Options, control controlPlane, modul if _, err := control.tell(ctx, "assign", o.Node, module); err != nil { return err } + if err := prepareModule(ctx, o, control, module, say); err != nil { + return err + } if beforePush != nil { if err := beforePush(); err != nil { return err diff --git a/internal/bootstrap/phase_packages.go b/internal/bootstrap/phase_packages.go index 1a4ceed..cd82e6e 100644 --- a/internal/bootstrap/phase_packages.go +++ b/internal/bootstrap/phase_packages.go @@ -42,8 +42,9 @@ const ( // giteaDBRole/giteaDBName is gitea's own database in the foundation store. giteaDBRole = "mesh_gitea" giteaDBName = "mesh_gitea" - // giteaPort is where the raised server answers on the machine. - giteaPort = 3000 + // defaultGiteaPort is where the raised server answers on the machine unless the node gave the + // package registry another port (novox/hq ADR 0100). + defaultGiteaPort = 3000 ) // RaisePackageRegistry puts a working npm registry in front of the base build. It is idempotent: @@ -60,17 +61,18 @@ func RaisePackageRegistry(ctx context.Context, o Options, d Deps, control contro } say(" seeding gitea's database in the foundation store") + ports := o.Ports.orDefaults() if err := seedGiteaDatabase(ctx, run, o.Timeout, dbPassword, say); err != nil { return err } say(" raising the gitea server on that database") - if err := raiseGiteaServer(ctx, run, o.Timeout, dbPassword, say); err != nil { + if err := raiseGiteaServer(ctx, run, o.Timeout, dbPassword, ports, say); err != nil { return err } say(" waiting for gitea to answer") - base := fmt.Sprintf("http://127.0.0.1:%d", giteaPort) + base := fmt.Sprintf("http://127.0.0.1:%d", ports.Packages) if err := waitForGitea(ctx, d, o, base, say); err != nil { return err } @@ -150,7 +152,7 @@ func seedGiteaDatabase(ctx context.Context, run Runner, timeout time.Duration, p // store's network namespace so `127.0.0.1:5432` reaches postgres, and publishes its own port on the // machine so the builder and this installer can reach it. Started if absent, left alone if present. func raiseGiteaServer(ctx context.Context, run Runner, timeout time.Duration, dbPassword string, - say func(string)) error { + ports FoundationPorts, say func(string)) error { asking, cancel := context.WithTimeout(ctx, timeout) defer cancel() @@ -164,7 +166,7 @@ func raiseGiteaServer(ctx context.Context, run Runner, timeout time.Duration, db env := []string{ "-e", "GITEA__database__DB_TYPE=postgres", // The store is reached on the shared network namespace's loopback. - "-e", "GITEA__database__HOST=127.0.0.1:5432", + "-e", fmt.Sprintf("GITEA__database__HOST=127.0.0.1:%d", ports.Store), "-e", "GITEA__database__NAME=" + giteaDBName, "-e", "GITEA__database__USER=" + giteaDBRole, "-e", "GITEA__database__PASSWD=" + dbPassword, @@ -174,9 +176,14 @@ func raiseGiteaServer(ctx context.Context, run Runner, timeout time.Duration, db // package metadata hands npm a tarball URL built from ROOT_URL, and a client only sends its // stored credential to the host it was stored for. A default ROOT_URL of localhost is a // different host than the binding's 127.0.0.1, so the credential would not be sent. - "-e", fmt.Sprintf("GITEA__server__ROOT_URL=http://127.0.0.1:%d/", giteaPort), + "-e", fmt.Sprintf("GITEA__server__ROOT_URL=http://127.0.0.1:%d/", ports.Packages), "-e", "USER_UID=1000", "-e", "USER_GID=1000", } + if ports.Packages != defaultGiteaPort { + // On the machine's network the server binds its own port, so a port given for it is + // the one it is told to listen on. + env = append(env, "-e", fmt.Sprintf("GITEA__server__HTTP_PORT=%d", ports.Packages)) + } args := append([]string{ "run", "-d", "--name", giteaBootstrap, // Host network, like the control plane: it reaches the foundation store on the machine's diff --git a/internal/bootstrap/ports.go b/internal/bootstrap/ports.go new file mode 100644 index 0000000..4da7560 --- /dev/null +++ b/internal/bootstrap/ports.go @@ -0,0 +1,454 @@ +package bootstrap + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net" + "sort" + "strconv" + "strings" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/reachable" + "github.com/novox/mesh-host/internal/store" +) + +// FoundationPorts are the machine's ports the foundation binds (novox/hq ADR 0100). +// +// **The node's, not the catalogue's.** A machine in use may already hold one — a predecessor's +// registry on 5000, its broker's management port — and a port fixed in the bundle and the manifests +// surfaces as a container that fails to bind, and one changed at genesis would be changed back when +// the foundation is adopted as modules. So each is an input here, checked free, rewritten into the +// bundle, and handed to the controller as that node's setting for the module that binds it. +type FoundationPorts struct { + Store int `json:"store"` + Bus int `json:"bus"` + AMQP int `json:"amqp"` + Management int `json:"management"` + Registry int `json:"registry"` + Packages int `json:"packages"` + Hub int `json:"hub"` +} + +// DefaultPorts are the catalogue's numbers. +func DefaultPorts() FoundationPorts { + return FoundationPorts{Store: 5432, Bus: 5671, AMQP: 5672, Management: 15672, Registry: 5000, + Packages: 3000, Hub: 51820} +} + +// DefaultOverlayRange is the controller's default private-network range. +const DefaultOverlayRange = "10.42.0.0/16" + +// orDefaults fills every port left unsaid. +func (p FoundationPorts) orDefaults() FoundationPorts { + d := DefaultPorts() + for _, f := range []struct{ got, def *int }{ + {&p.Store, &d.Store}, {&p.Bus, &d.Bus}, {&p.AMQP, &d.AMQP}, {&p.Management, &d.Management}, + {&p.Registry, &d.Registry}, {&p.Packages, &d.Packages}, {&p.Hub, &d.Hub}, + } { + if *f.got == 0 { + *f.got = *f.def + } + } + return p +} + +// named is each port with what it is and its protocol, in a fixed order. +func (p FoundationPorts) named() []namedPort { + return []namedPort{ + {"the store", "tcp", p.Store}, {"the bus", "tcp", p.Bus}, {"the broker's AMQP", "tcp", p.AMQP}, + {"the broker's management", "tcp", p.Management}, {"the registry", "tcp", p.Registry}, + {"the package registry", "tcp", p.Packages}, {"the private network's hub", "udp", p.Hub}, + } +} + +type namedPort struct { + what, protocol string + port int +} + +// Check refuses a port out of range, or one port given for two things. +func (p FoundationPorts) Check() error { + seen := map[string]string{} + for _, n := range p.named() { + if n.port < 1 || n.port > 65535 { + return fmt.Errorf("%s's port is %d, and a port is 1-65535", n.what, n.port) + } + key := n.protocol + "/" + strconv.Itoa(n.port) + if other, twice := seen[key]; twice { + return fmt.Errorf("%s and %s were both given %s", other, n.what, key) + } + seen[key] = n.what + } + return nil +} + +// moduleSettings is what each foundation module is told about its ports on this node: the port it +// declares, to the machine's port it is given. Only what differs from the catalogue — a converged +// genesis on the defaults sets nothing, and so changes nothing it did before. +func (p FoundationPorts) moduleSettings() map[string]map[string]int { + d := DefaultPorts() + out := map[string]map[string]int{} + add := func(module string, declared, given int) { + if given == declared { + return + } + if out[module] == nil { + out[module] = map[string]int{} + } + out[module][strconv.Itoa(declared)] = given + } + add("postgres", d.Store, p.Store) + add("lavinmq", d.Bus, p.Bus) + add("lavinmq", d.AMQP, p.AMQP) + add("lavinmq", d.Management, p.Management) + add(RegistryModule, d.Registry, p.Registry) + return out +} + +// PortsSetting is the controller's settings key for a module's given ports. +const PortsSetting = "ports" + +// setFoundationSettings tells the controller the ports this node gave a foundation module — and, +// on an adopted node, that the registry is reached from anywhere, as a node pulls from it before it +// has a private-network address (novox/hq ADR 0100). Done after the module is registered and before +// the push that raises it, so the first declaration already names the node's ports. +func setFoundationSettings(ctx context.Context, o Options, control controlPlane, module string, + say func(string)) error { + values := map[string]any{} + if ports := o.Ports.orDefaults().moduleSettings()[module]; len(ports) > 0 { + values[PortsSetting] = ports + } + if o.Adopted && module == RegistryModule { + values["expose"] = map[string]string{strconv.Itoa(DefaultPorts().Registry): "anywhere"} + } + if len(values) == 0 { + return nil + } + raw, err := json.Marshal(values) + if err != nil { + return err + } + remote := "/" + module + "-settings.json" + if err := control.carrying(ctx, module+"-settings.json", raw, remote); err != nil { + return err + } + if _, err := control.tell(ctx, "settings", "set", module, remote, "--node", o.Node); err != nil { + return err + } + say(" settings " + module + " on " + o.Node + ": " + string(raw)) + return nil +} + +// PortsRewrite says what RewritePorts changed. +type PortsRewrite struct { + Places int +} + +// RewritePorts puts the node's foundation ports into the produced bundle, in place of the +// template's, byte for byte like every other rewrite — so the file keeps its comments and a person +// can read what was applied. A port left at its default is not touched, so a genesis on the +// defaults produces exactly the bundle it did before. +// +// Only the machine's side moves: the outer port of each mapping, the addresses the control plane +// dials on the machine's loopback, and the address nodes are told to dial. What a container listens +// on inside itself, and what an action reaches inside the store's own network, stay as they are. +func RewritePorts(r *Rewritten, p FoundationPorts, overlayRange string) (PortsRewrite, error) { + var out PortsRewrite + p = p.orDefaults() + d := DefaultPorts() + bundle := r.Bundle + var err error + + replace := func(from, to, what string) { + if err != nil || from == to { + return + } + bundle, err = replaceOnce(bundle, from, to, what) + out.Places++ + } + if p.Store != d.Store { + replace(`"ports": ["5432:5432"]`, fmt.Sprintf(`"ports": ["%d:5432"]`, p.Store), "the store's published port") + } + if p.Bus != d.Bus || p.AMQP != d.AMQP || p.Management != d.Management { + replace(`"ports": ["5671:5671", "5672:5672", "127.0.0.1:15672:15672"]`, + fmt.Sprintf(`"ports": ["%d:5671", "%d:5672", "127.0.0.1:%d:15672"]`, p.Bus, p.AMQP, p.Management), + "the broker's published ports") + } + if err != nil { + return out, err + } + + // The control plane runs on the machine's network and dials the store and the broker on its + // loopback, so its connection strings name the machine's ports. The schema step reaches the + // store inside the store's own network and keeps the container's port — so these are found by + // the control plane's environment, not by searching for the text. + control, cerr := controlPlaneIn(r.Declaration) + if cerr != nil { + return out, cerr + } + for _, key := range sortedKeys(control.Env) { + value := control.Env[key] + now := value + now = strings.ReplaceAll(now, "@127.0.0.1:5432/", fmt.Sprintf("@127.0.0.1:%d/", p.Store)) + now = strings.ReplaceAll(now, "@127.0.0.1:5672/", fmt.Sprintf("@127.0.0.1:%d/", p.AMQP)) + if strings.HasSuffix(now, "@127.0.0.1:15672") { + now = strings.TrimSuffix(now, "15672") + strconv.Itoa(p.Management) + } + if key == brokerAddressVar { + if host, port, splitErr := net.SplitHostPort(value); splitErr == nil && port == "5671" { + now = net.JoinHostPort(host, strconv.Itoa(p.Bus)) + } + } + if now != value { + replace(`"`+key+`": "`+value+`"`, `"`+key+`": "`+now+`"`, "the control plane's "+key) + } + } + if err != nil { + return out, err + } + + // The foundation's own filter, where the template carries one: it admits the bus and the + // registry from anywhere, on whatever port they are. + for _, f := range []struct{ def, now int }{{d.Bus, p.Bus}, {d.Registry, p.Registry}} { + if f.def == f.now { + continue + } + for _, form := range []string{"tcp dport %d accept", "ct original proto-dst %d accept"} { + from, to := fmt.Sprintf(form, f.def), fmt.Sprintf(form, f.now) + if n := bytes.Count(bundle, []byte(from)); n > 0 { + bundle = bytes.ReplaceAll(bundle, []byte(from), []byte(to)) + out.Places += n + } + } + } + + // The private network's range, when it is not the default, is the controller's to know. + if overlayRange != "" && overlayRange != DefaultOverlayRange { + replace(`"`+brokerAddressVar+`": `, + `"MESH_OVERLAY_CIDR": "`+overlayRange+`", + "`+brokerAddressVar+`": `, "where the control plane is told the private network's range") + if err != nil { + return out, err + } + } + + if out.Places == 0 { + return out, nil + } + parsed, perr := declaration.ParseFileTrusted(bundle) + if perr != nil { + return out, fmt.Errorf("the bundle stopped being a declaration after its ports were rewritten, which is this installer's fault: %w", perr) + } + r.Bundle, r.Declaration, r.Resources = bundle, parsed, len(parsed.Resources) + if c, cerr := controlPlaneIn(parsed); cerr == nil { + r.BrokerAddress = c.Env[brokerAddressVar] + } + return out, nil +} + +// PortsFree refuses a foundation port something else already holds, naming what holds it. What the +// mesh itself raised on an earlier run of genesis is not counted: ours says which holders are. +func PortsFree(ctx context.Context, run Runner, p FoundationPorts, ours func(reachable.Reach) bool) error { + out, err := run(ctx, "ss", "-Hltunp") + if err != nil { + return fmt.Errorf("cannot read which ports this machine holds, so the foundation's cannot be checked free: %w", err) + } + sockets := reachable.Sockets(out) + var published []reachable.Reach + if ps, err := run(ctx, "docker", "ps", "--format", "{{.Names}}\t{{.Ports}}"); err == nil { + published = reachable.Published(ps) + } + held := reachable.Merge(sockets, published) + + var problems []string + for _, n := range p.orDefaults().named() { + var by []string + for _, r := range held { + if r.Protocol != n.protocol || r.Port != n.port || ours(r) { + continue + } + holder := r.By + if holder == "" { + holder = "something ss does not name" + } + if r.Published { + holder = "the container " + r.By + } + if !contains(by, holder) { + by = append(by, holder) + } + } + if len(by) > 0 { + problems = append(problems, fmt.Sprintf("%s's port %s/%d is held by %s", + n.what, n.protocol, n.port, strings.Join(by, ", "))) + } + } + if len(problems) > 0 { + return fmt.Errorf("the foundation's ports must be free before anything is raised:\n - %s\n"+ + "Give it another with the matching flag (--store-port, --bus-port, --amqp-port, "+ + "--management-port, --registry-port, --packages-port, --hub-port); nothing was changed", + strings.Join(problems, "\n - ")) + } + return nil +} + +// OverlayClear refuses a private-network range that overlaps an address or a route the machine +// already has — a predecessor's tunnel still running — naming the interface. The mesh's own +// interface is not counted. +func OverlayClear(ctx context.Context, run Runner, overlayRange string) error { + if overlayRange == "" { + overlayRange = DefaultOverlayRange + } + _, mine, err := net.ParseCIDR(overlayRange) + if err != nil { + return fmt.Errorf("the private network's range %q is not a range: %w", overlayRange, err) + } + var clashes []string + if out, err := run(ctx, "ip", "-o", "addr", "show"); err == nil { + for _, line := range strings.Split(out, "\n") { + f := strings.Fields(line) + // 3: wg0 inet 10.42.0.1/24 scope global wg0 + if len(f) < 4 || (f[2] != "inet" && f[2] != "inet6") { + continue + } + iface := strings.TrimSuffix(f[1], ":") + if clash(mine, f[3]) && iface != meshInterface { + clashes = append(clashes, fmt.Sprintf("%s holds %s", iface, f[3])) + } + } + } else { + return fmt.Errorf("cannot read this machine's addresses to check the private network's range: %w", err) + } + if out, err := run(ctx, "ip", "-o", "route", "show"); err == nil { + for _, line := range strings.Split(out, "\n") { + f := strings.Fields(line) + // 10.42.0.0/16 dev wg0 proto kernel scope link src 10.42.0.1 + if len(f) < 3 || f[0] == "default" { + continue + } + iface := "" + for i := range f { + if f[i] == "dev" && i+1 < len(f) { + iface = f[i+1] + } + } + if iface != meshInterface && clash(mine, f[0]) { + clashes = append(clashes, fmt.Sprintf("%s routes %s", iface, f[0])) + } + } + } + if len(clashes) > 0 { + return fmt.Errorf("the private network's range %s overlaps what this machine already has: %s.\n"+ + "A tunnel a predecessor still runs would take the mesh's traffic. Give another range with "+ + "--overlay-range; nothing was changed", overlayRange, strings.Join(clashes, "; ")) + } + return nil +} + +// meshInterface is the private network's own interface, which a re-run finds holding its range. +const meshInterface = "mesh0" + +func clash(mine *net.IPNet, other string) bool { + if !strings.Contains(other, "/") { + if ip := net.ParseIP(other); ip != nil { + return mine.Contains(ip) + } + return false + } + ip, theirs, err := net.ParseCIDR(other) + if err != nil { + return false + } + return mine.Contains(theirs.IP) || theirs.Contains(mine.IP) || mine.Contains(ip) +} + +// NamesFree refuses a foundation or bundle container name that a container already has, when no +// host made that container and this node has no record of it — a predecessor's container under the +// mesh's name, which raising the foundation would replace. +func NamesFree(ctx context.Context, run Runner, names []string, known store.State) error { + var taken []string + sorted := append([]string{}, names...) + sort.Strings(sorted) + for _, name := range sorted { + out, err := run(ctx, "docker", "inspect", "--format", + "{{index .Config.Labels \"mesh-host.spec\"}}", name) + if err != nil { + continue // no such container + } + label := strings.TrimSpace(out) + if label != "" && label != "" { + continue + } + if known.Recorded(string(declaration.TypeContainer), name) { + continue + } + taken = append(taken, name) + } + if len(taken) > 0 { + return fmt.Errorf("this machine already runs a container under the name the foundation uses, "+ + "and nothing of the mesh's made it: %s.\nRaising the foundation would replace it. Rename or "+ + "stop it first; nothing was changed", strings.Join(taken, ", ")) + } + return nil +} + +// CheckTheMachine is every check genesis makes before raising anything that this machine does not +// already hold what the foundation needs: its ports, its private network's range, its containers' +// names (novox/hq ADR 0100). A re-run of genesis finds the foundation it raised and does not count +// it. +func CheckTheMachine(ctx context.Context, o Options, run Runner, bundle *declaration.Declaration, + say func(string)) error { + known, err := store.Load(o.State) + if err != nil { + return err + } + rerun := len(known.Resources) > 0 + names := foundationNames(bundle) + mine := map[string]bool{} + for _, n := range names { + mine[n] = true + } + p := o.Ports.orDefaults() + ours := func(r reachable.Reach) bool { + switch { + case mine[r.By]: + return true + case !rerun: + return false + case r.By == "gitea" && r.Port == p.Packages: + // The package registry runs on the machine's network, so ss names its process. + return true + case r.By == "" && r.Protocol == "udp" && r.Port == p.Hub: + // The private network's hub is a kernel interface and has no process. + return true + } + return false + } + if err := PortsFree(ctx, run, p, ours); err != nil { + return err + } + say(fmt.Sprintf(" ports free store %d, bus %d, amqp %d, management %d, registry %d, packages %d, hub %d/udp", + p.Store, p.Bus, p.AMQP, p.Management, p.Registry, p.Packages, p.Hub)) + if err := OverlayClear(ctx, run, o.OverlayRange); err != nil { + return err + } + if err := NamesFree(ctx, run, names, known); err != nil { + return err + } + return nil +} + +// foundationNames are the containers the foundation and genesis raise under fixed names. +func foundationNames(bundle *declaration.Declaration) []string { + names := []string{ControlPlaneModule, giteaBootstrap, "mesh-registry"} + for _, n := range containerNames(bundle) { + if !contains(names, n) { + names = append(names, n) + } + } + sort.Strings(names) + return names +} diff --git a/internal/bootstrap/ports_test.go b/internal/bootstrap/ports_test.go new file mode 100644 index 0000000..49a3e64 --- /dev/null +++ b/internal/bootstrap/ports_test.go @@ -0,0 +1,286 @@ +package bootstrap + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/reachable" + "github.com/novox/mesh-host/internal/store" +) + +// Defends novox/hq ADR 0100: the foundation's ports are the node's — inputs to genesis, checked free, +// rewritten into the bundle, and handed to the controller as the node's settings. + +func producedBundle(t *testing.T) Rewritten { + t.Helper() + template, err := os.ReadFile("../../examples/foundation-first-node.lock") + if err != nil { + t.Skip("no example bundle beside this checkout") + } + r, err := Rewrite(template, "sha256:"+strings.Repeat("ab", 32)) + if err != nil { + t.Fatal(err) + } + if _, err := RewriteRoot(&r, RootCredentials{Store: "s", Broker: "b"}); err != nil { + t.Fatal(err) + } + return r +} + +func containerNamed(d *declaration.Declaration, name string) *declaration.Container { + for _, r := range d.Resources { + if c, ok := r.(*declaration.Container); ok && c.Name == name { + return c + } + } + return nil +} + +func TestTheDefaultPortsLeaveTheBundleAsItWas(t *testing.T) { + r := producedBundle(t) + before := string(r.Bundle) + got, err := RewritePorts(&r, DefaultPorts(), DefaultOverlayRange) + if err != nil { + t.Fatal(err) + } + if got.Places != 0 || string(r.Bundle) != before { + t.Errorf("the default ports rewrote %d place(s)", got.Places) + } +} + +func TestGivenPortsMoveOnlyTheMachinesSide(t *testing.T) { + r := producedBundle(t) + p := FoundationPorts{Store: 5433, Bus: 5771, AMQP: 5772, Management: 15673, Registry: 5100} + got, err := RewritePorts(&r, p, "10.77.0.0/16") + if err != nil { + t.Fatal(err) + } + if got.Places == 0 { + t.Fatal("nothing was rewritten") + } + storeC := containerNamed(r.Declaration, "mesh-store") + if len(storeC.Ports) != 1 || storeC.Ports[0] != "5433:5432" { + t.Errorf("the store publishes %v", storeC.Ports) + } + broker := containerNamed(r.Declaration, "mesh-broker") + if strings.Join(broker.Ports, " ") != "5771:5671 5772:5672 127.0.0.1:15673:15672" { + t.Errorf("the broker publishes %v", broker.Ports) + } + control, err := controlPlaneIn(r.Declaration) + if err != nil { + t.Fatal(err) + } + for key, value := range control.Env { + if strings.HasPrefix(key, "MESH_STORE_") && !strings.Contains(value, "@127.0.0.1:5433/") { + t.Errorf("%s still dials %s", key, value) + } + } + if !strings.Contains(control.Env["MESH_BROKER_AMQP"], "@127.0.0.1:5772/") || + !strings.HasSuffix(control.Env["MESH_BROKER_MANAGEMENT"], "@127.0.0.1:15673") { + t.Errorf("the broker is dialled at %s and %s", control.Env["MESH_BROKER_AMQP"], control.Env["MESH_BROKER_MANAGEMENT"]) + } + if control.Env["MESH_BROKER_ADDRESS"] != "192.0.2.10:5771" || r.BrokerAddress != "192.0.2.10:5771" { + t.Errorf("nodes are told to dial %s (%s)", control.Env["MESH_BROKER_ADDRESS"], r.BrokerAddress) + } + if control.Env["MESH_OVERLAY_CIDR"] != "10.77.0.0/16" { + t.Errorf("the control plane is told the range %q", control.Env["MESH_OVERLAY_CIDR"]) + } + // The schema step reaches the store inside its own network, on the container's port. + text := string(r.Bundle) + if !strings.Contains(text, `MESH_STORE_INVENTORY=postgres://postgres:s@127.0.0.1:5432/inventory`) { + t.Error("the schema step's connection, inside the store's network, was moved off the container's port") + } + for _, want := range []string{"tcp dport 5771 accept", "ct original proto-dst 5771 accept", + "tcp dport 5100 accept", "ct original proto-dst 5100 accept"} { + if !strings.Contains(text, want) { + t.Errorf("the base filter does not say %q", want) + } + } + if strings.Contains(text, "dport 5671 accept") || strings.Contains(text, "dport 5000 accept") { + t.Error("the base filter still admits a default port") + } +} + +func TestATemplateThatDoesNotSayItsPortsAsExpectedIsRefused(t *testing.T) { + r := producedBundle(t) + r.Bundle = []byte(strings.Replace(string(r.Bundle), `"ports": ["5432:5432"]`, `"ports": [ "5432:5432" ]`, 1)) + if _, err := RewritePorts(&r, FoundationPorts{Store: 5433}, ""); err == nil { + t.Error("a store port the installer could not find was silently left") + } +} + +func TestTwoThingsOnOnePortAreRefused(t *testing.T) { + p := DefaultPorts() + p.Registry = p.Store + if err := p.Check(); err == nil { + t.Error("the registry and the store were both given one port") + } + p = DefaultPorts() + p.Hub = 5432 // udp, beside the store's tcp: two different ports + if err := p.Check(); err != nil { + t.Errorf("a udp port beside a tcp one of the same number was refused: %v", err) + } +} + +// machineRunner answers ss, docker ps, docker inspect and ip from fixtures. +type machineRunner struct { + ss, ps, addrs, routes string + unlabelled map[string]bool + labelled map[string]bool +} + +func (m machineRunner) run(_ context.Context, name string, args ...string) (string, error) { + switch { + case name == "ss": + return m.ss, nil + case name == "docker" && args[0] == "ps": + return m.ps, nil + case name == "docker" && args[0] == "inspect": + n := args[len(args)-1] + if m.labelled[n] { + return "abc\n", nil + } + if m.unlabelled[n] { + return "\n", nil + } + return "", errors.New("no such container") + case name == "ip" && args[1] == "addr": + return m.addrs, nil + case name == "ip" && args[1] == "route": + return m.routes, nil + } + return "", nil +} + +func noneOurs(reachable.Reach) bool { return false } + +func TestABusyPortIsRefusedNamingItsHolder(t *testing.T) { + m := machineRunner{ + ss: "tcp LISTEN 0 4096 0.0.0.0:5000 0.0.0.0:* users:((\"docker-proxy\",pid=1,fd=7))\n" + + "tcp LISTEN 0 4096 127.0.0.1:15672 0.0.0.0:* users:((\"beam.smp\",pid=2,fd=7))\n", + ps: "predecessor-registry\t0.0.0.0:5000->5000/tcp\n", + } + err := PortsFree(context.Background(), m.run, DefaultPorts(), noneOurs) + if err == nil { + t.Fatal("held ports were not refused") + } + for _, want := range []string{"predecessor-registry", "beam.smp", "tcp/5000", "tcp/15672", "--registry-port"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal does not say %q: %v", want, err) + } + } + p := DefaultPorts() + p.Registry, p.Management = 5100, 15673 + if err := PortsFree(context.Background(), m.run, p, noneOurs); err != nil { + t.Errorf("other ports given and still refused: %v", err) + } +} + +func TestTheFoundationsOwnContainersAreNotCountedOnARerun(t *testing.T) { + m := machineRunner{ + ss: "tcp LISTEN 0 4096 0.0.0.0:5432 0.0.0.0:* users:((\"docker-proxy\",pid=1,fd=7))\n", + ps: "mesh-store\t0.0.0.0:5432->5432/tcp\n", + } + ours := func(r reachable.Reach) bool { return r.By == "mesh-store" } + if err := PortsFree(context.Background(), m.run, DefaultPorts(), ours); err != nil { + t.Errorf("the foundation's own store was counted as holding its port: %v", err) + } +} + +func TestAnOverlappingTunnelIsRefusedNamingItsInterface(t *testing.T) { + m := machineRunner{ + addrs: "1: lo inet 127.0.0.1/8 scope host lo\n5: wg0 inet 10.42.3.1/24 scope global wg0\n7: mesh0 inet 10.42.0.1/16 scope global mesh0\n", + routes: "default via 192.0.2.1 dev eth0\n10.42.3.0/24 dev wg0 proto kernel scope link src 10.42.3.1\n", + } + err := OverlayClear(context.Background(), m.run, "") + if err == nil || !strings.Contains(err.Error(), "wg0") || strings.Contains(err.Error(), "mesh0") { + t.Fatalf("the overlap was not named by its interface alone: %v", err) + } + if err := OverlayClear(context.Background(), m.run, "10.77.0.0/16"); err != nil { + t.Errorf("a clear range was refused: %v", err) + } +} + +func TestAPredecessorsContainerUnderTheMeshsNameIsRefused(t *testing.T) { + m := machineRunner{unlabelled: map[string]bool{"mesh-registry": true}, labelled: map[string]bool{"mesh-store": true}} + err := NamesFree(context.Background(), m.run, []string{"mesh-store", "mesh-registry", "mesh-broker"}, store.State{}) + if err == nil || !strings.Contains(err.Error(), "mesh-registry") || strings.Contains(err.Error(), "mesh-store") { + t.Fatalf("names: %v", err) + } + known := store.State{Resources: []store.Applied{{ID: "x", Type: "container", Target: "mesh-registry"}}} + if err := NamesFree(context.Background(), m.run, []string{"mesh-registry"}, known); err != nil { + t.Errorf("a container this node has a record of was refused: %v", err) + } +} + +// controlRecorder is a control plane that answers everything and writes down what it was told, +// with the content of every settings file carried to it. +type controlRecorder struct { + told []string + settings map[string]string +} + +func (c *controlRecorder) run(_ context.Context, name string, args ...string) (string, error) { + if name == "docker" && args[0] == "cp" { + raw, _ := os.ReadFile(args[1]) + if strings.HasSuffix(args[2], "-settings.json") { + c.settings[filepath.Base(args[2])] = string(raw) + } + return "", nil + } + if name == "docker" && args[0] == "exec" { + c.told = append(c.told, strings.Join(args[3:], " ")) + } + return "", nil +} + +func (c *controlRecorder) index(prefix string) int { + for i, t := range c.told { + if strings.HasPrefix(t, prefix) { + return i + } + } + return -1 +} + +func TestTheNodesPortsAreSetBeforeTheModuleIsPushed(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + c := &controlRecorder{settings: map[string]string{}} + control := controlPlane{container: "temp-mesh-controller", run: c.run, timeout: time.Second} + o := Options{Node: "anchor", Ports: FoundationPorts{Registry: 5100}, Wait: time.Second} + if _, err := installModule(context.Background(), o, control, RegistryModule, []byte(`{}`), quietly); err != nil { + t.Fatal(err) + } + set, push := c.index("settings set distribution"), c.index("push anchor") + if set < 0 || push < 0 || set > push { + t.Fatalf("settings were not set before the push: %v", c.told) + } + if add := c.index("module add"); add > set { + t.Errorf("settings were set before the module existed: %v", c.told) + } + if !strings.Contains(c.told[set], "--node anchor") { + t.Errorf("the settings are not the node's: %s", c.told[set]) + } + if got := c.settings["distribution-settings.json"]; got != `{"ports":{"5000":5100}}` { + t.Errorf("the registry was told %s", got) + } +} + +func TestAGenesisOnTheDefaultsSetsNoSettings(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + c := &controlRecorder{settings: map[string]string{}} + control := controlPlane{container: "temp-mesh-controller", run: c.run, timeout: time.Second} + o := Options{Node: "anchor", Wait: time.Second} + if _, err := installModule(context.Background(), o, control, RegistryModule, []byte(`{}`), quietly); err != nil { + t.Fatal(err) + } + if c.index("settings") >= 0 { + t.Errorf("a converged genesis on the default ports set settings: %v", c.told) + } +} From 4811f176fdaf59892c32cda5b2b11b29a179865d Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 17:29:32 +0200 Subject: [PATCH 06/52] Refuse a converged genesis on a machine in use, naming every container and listener counted (hq ADR 0100) --- internal/bootstrap/inuse.go | 113 +++++++++++++++++++++++++++++++ internal/bootstrap/inuse_test.go | 99 +++++++++++++++++++++++++++ internal/bootstrap/preflight.go | 5 ++ 3 files changed, 217 insertions(+) create mode 100644 internal/bootstrap/inuse.go create mode 100644 internal/bootstrap/inuse_test.go diff --git a/internal/bootstrap/inuse.go b/internal/bootstrap/inuse.go new file mode 100644 index 0000000..3909de6 --- /dev/null +++ b/internal/bootstrap/inuse.go @@ -0,0 +1,113 @@ +package bootstrap + +import ( + "context" + "fmt" + "net" + "strings" + + "github.com/novox/mesh-host/internal/reachable" + "github.com/novox/mesh-host/internal/store" +) + +// quietUDP are processes whose UDP sockets every fresh machine has — name resolution, address +// configuration, time — and which serve nobody. ss names a process by its first fifteen characters, +// so both spellings are here. +// +// **Still to be measured** (novox/hq ADR 0100): this list is what a fresh machine is expected to +// hold, and it must be checked against a freshly installed lab machine before it is trusted. +var quietUDP = map[string]bool{ + "systemd-resolved": true, "systemd-resolve": true, + "systemd-networkd": true, "systemd-network": true, + "systemd-timesyncd": true, "systemd-timesyn": true, + "dhcpcd": true, +} + +// InUse says what makes this machine a machine in use (novox/hq ADR 0100): every running container +// no host made, and every socket listening on an address other than loopback that is not ssh's — a +// UDP one only when it is held by something other than what every fresh machine runs. ours names +// what the mesh itself runs, which a re-run of genesis finds and does not count. +func InUse(ctx context.Context, run Runner, ours func(name string) bool) ([]string, []reachable.Reach, error) { + var containers []string + out, err := run(ctx, "docker", "ps", "--format", "{{.Names}}\t{{.Label \"mesh-host.spec\"}}") + if err != nil { + return nil, nil, fmt.Errorf("cannot ask the container runtime what is running here: %w", err) + } + for _, line := range strings.Split(out, "\n") { + name, label, _ := strings.Cut(strings.TrimSpace(line), "\t") + label = strings.TrimSpace(label) + if name == "" || (label != "" && label != "") || ours(name) { + continue + } + containers = append(containers, name) + } + + listening, err := run(ctx, "ss", "-Hltunp") + if err != nil { + return nil, nil, fmt.Errorf("cannot read what listens on this machine: %w", err) + } + var listeners []reachable.Reach + for _, r := range reachable.Sockets(listening) { + if counts(r) && !ours(r.By) { + listeners = append(listeners, r) + } + } + return containers, listeners, nil +} + +func counts(r reachable.Reach) bool { + if ip := net.ParseIP(r.Address); ip != nil && ip.IsLoopback() { + return false + } + switch r.Protocol { + case "tcp": + return r.By != "sshd" && !(r.By == "" && r.Port == 22) + case "udp": + return !quietUDP[r.By] + } + return false +} + +// RefuseAMachineInUse is the check a converged genesis makes before changing anything: a machine +// in use is refused, naming every container and listener counted, because raising the foundation's +// filter there would close what it serves — a forgotten --adopted must not close a working machine. +// An adopted genesis is told what it found, and goes on. +func RefuseAMachineInUse(ctx context.Context, o Options, run Runner, say func(string)) error { + known, err := store.Load(o.State) + if err != nil { + return err + } + if len(known.Resources) > 0 { + // What genesis raised on an earlier run is the mesh's, and it is what the machine now + // serves; the question was answered the first time. + say(" in use not asked: this machine carries what an earlier genesis raised") + return nil + } + containers, listeners, err := InUse(ctx, run, func(string) bool { return false }) + if err != nil { + return err + } + if len(containers) == 0 && len(listeners) == 0 { + say(" in use no: no container runs and nothing listens beyond ssh") + return nil + } + var named []string + for _, c := range containers { + named = append(named, "container "+c) + } + for _, l := range listeners { + by := l.By + if by == "" { + by = "an unnamed process" + } + named = append(named, fmt.Sprintf("%s %s:%d by %s", l.Protocol, l.Address, l.Port, by)) + } + if o.Adopted { + say(fmt.Sprintf(" in use yes, and adopted: %d thing(s) found are kept", len(named))) + return nil + } + return fmt.Errorf("this machine is in use, and a converged genesis would close what it serves:\n - %s\n"+ + "If it is meant to join the mesh keeping what it runs, pass --adopted: its firewall stays in "+ + "force and every module is taken on it one at a time. Nothing was changed", + strings.Join(named, "\n - ")) +} diff --git a/internal/bootstrap/inuse_test.go b/internal/bootstrap/inuse_test.go new file mode 100644 index 0000000..b1a1e49 --- /dev/null +++ b/internal/bootstrap/inuse_test.go @@ -0,0 +1,99 @@ +package bootstrap + +import ( + "context" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/store" +) + +// Defends novox/hq ADR 0100: a converged genesis refuses a machine in use, naming every container +// and listener it counted. + +// Lines as `ss -Hltunp` prints them. The ssh, samba, loopback and proxy lines are captured from a +// real machine; the resolver, DHCP and time lines are written in the same shape for the processes a +// fresh machine runs, and still need measuring against one. +const inUseSockets = `tcp LISTEN 0 128 0.0.0.0:22 0.0.0.0:* users:(("sshd",pid=1188536,fd=6)) +tcp LISTEN 0 128 [::]:22 [::]:* users:(("sshd",pid=1188536,fd=7)) +tcp LISTEN 0 32 127.0.0.1:53 0.0.0.0:* users:(("dnsmasq",pid=1189392,fd=7)) +tcp LISTEN 0 4096 127.0.0.1:5432 0.0.0.0:* users:(("docker-proxy",pid=1854543,fd=7)) +udp UNCONN 0 0 0.0.0.0:5355 0.0.0.0:* users:(("systemd-resolve",pid=301,fd=11)) +udp UNCONN 0 0 192.0.2.10%eth0:68 0.0.0.0:* users:(("systemd-network",pid=280,fd=19)) +udp UNCONN 0 0 0.0.0.0:68 0.0.0.0:* users:(("dhcpcd",pid=270,fd=9)) +udp UNCONN 0 0 0.0.0.0:123 0.0.0.0:* users:(("systemd-timesyn",pid=260,fd=9)) +` + +const servingSockets = `tcp LISTEN 0 50 0.0.0.0:445 0.0.0.0:* users:(("smbd",pid=1248,fd=29)) +tcp LISTEN 0 4096 0.0.0.0:8080 0.0.0.0:* users:(("docker-proxy",pid=1920035,fd=7)) +udp UNCONN 0 0 0.0.0.0:123 0.0.0.0:* users:(("ntpd",pid=1070791,fd=17)) +` + +type inUseRunner struct{ ps, ss string } + +func (m inUseRunner) run(_ context.Context, name string, args ...string) (string, error) { + if name == "docker" { + return m.ps, nil + } + return m.ss, nil +} + +func TestAFreshMachineIsNotInUse(t *testing.T) { + containers, listeners, err := InUse(context.Background(), inUseRunner{ss: inUseSockets}.run, + func(string) bool { return false }) + if err != nil { + t.Fatal(err) + } + if len(containers) != 0 || len(listeners) != 0 { + t.Errorf("ssh, loopback, name resolution, DHCP and time were counted: %v %v", containers, listeners) + } +} + +func TestAMachineServingIsInUse(t *testing.T) { + m := inUseRunner{ps: "hello-web\t\nmesh-store\tabc123\n", ss: inUseSockets + servingSockets} + containers, listeners, err := InUse(context.Background(), m.run, func(string) bool { return false }) + if err != nil { + t.Fatal(err) + } + if len(containers) != 1 || containers[0] != "hello-web" { + t.Errorf("containers counted: %v (one a host made is not a predecessor's)", containers) + } + var by []string + for _, l := range listeners { + by = append(by, l.By) + } + if strings.Join(by, " ") != "smbd docker-proxy ntpd" { + t.Errorf("listeners counted: %v", listeners) + } +} + +func TestAConvergedGenesisRefusesAMachineInUseNamingEverything(t *testing.T) { + o := Options{State: filepath.Join(t.TempDir(), "state.json")} + m := inUseRunner{ps: "hello-web\t\n", ss: inUseSockets + servingSockets} + err := RefuseAMachineInUse(context.Background(), o, m.run, quietly) + if err == nil { + t.Fatal("a machine in use was not refused") + } + for _, want := range []string{"container hello-web", "tcp 0.0.0.0:445 by smbd", "tcp 0.0.0.0:8080 by docker-proxy", + "udp 0.0.0.0:123 by ntpd", "--adopted"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal does not name %q: %v", want, err) + } + } + o.Adopted = true + if err := RefuseAMachineInUse(context.Background(), o, m.run, quietly); err != nil { + t.Errorf("an adopted genesis was refused a machine in use: %v", err) + } +} + +func TestARerunOfGenesisIsNotAMachineInUse(t *testing.T) { + o := Options{State: filepath.Join(t.TempDir(), "state.json")} + if err := store.Save(o.State, store.State{Resources: []store.Applied{{ID: "store", Type: "container", Target: "mesh-store"}}}); err != nil { + t.Fatal(err) + } + m := inUseRunner{ps: "mesh-gitea-server\t\n", ss: servingSockets} + if err := RefuseAMachineInUse(context.Background(), o, m.run, quietly); err != nil { + t.Errorf("what an earlier genesis raised was counted as a machine in use: %v", err) + } +} diff --git a/internal/bootstrap/preflight.go b/internal/bootstrap/preflight.go index 5275aeb..7b8d145 100644 --- a/internal/bootstrap/preflight.go +++ b/internal/bootstrap/preflight.go @@ -105,6 +105,11 @@ func Preflight(ctx context.Context, o Options, d Deps, say func(string)) ([]byte if err := waitForRuntime(ctx, d.Run, o.Timeout, o.Wait, say); err != nil { return nil, err } + // A converged genesis refuses a machine in use (novox/hq ADR 0100) — asked once the runtime + // answers, so what it runs can be counted, and before anything changes. + if err := RefuseAMachineInUse(ctx, o, d.Run, say); err != nil { + return nil, err + } // 4. Can this machine reach what the bundle's images come from? // From 5e3dd3f59c9eed816cb0587ce505dd7abbd393e7 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 17:32:44 +0200 Subject: [PATCH 07/52] Raise a machine in use adopted: keep its firewall, load no dropping table, guard the mesh's own ports, and take only the mesh's own modules (hq ADR 0100) --- cmd/mesh-bootstrap/main.go | 7 ++ internal/bootstrap/adopted.go | 192 +++++++++++++++++++++++++++++ internal/bootstrap/adopted_test.go | 192 +++++++++++++++++++++++++++++ internal/bootstrap/bootstrap.go | 37 +++++- internal/bootstrap/enrol.go | 8 +- internal/bootstrap/module.go | 8 +- internal/bootstrap/phase2.go | 12 +- internal/bootstrap/retire.go | 25 ++-- 8 files changed, 467 insertions(+), 14 deletions(-) create mode 100644 internal/bootstrap/adopted.go create mode 100644 internal/bootstrap/adopted_test.go diff --git a/cmd/mesh-bootstrap/main.go b/cmd/mesh-bootstrap/main.go index 7a1fba5..adb27ba 100644 --- a/cmd/mesh-bootstrap/main.go +++ b/cmd/mesh-bootstrap/main.go @@ -135,6 +135,11 @@ const usage = `mesh-bootstrap — make a bare machine into a mesh --overlay-range the private network's range (default 10.42.0.0/16); refused if it overlaps an interface or route the machine already has + --adopted raise a machine in use as an adopted node: what it runs and its + firewall stay as they are, the foundation's filter is not loaded and + the mesh guards its own ports instead, and each module is taken on it + one at a time. Without it, a machine in use is refused + The installer carries a builder, not a control plane. What raises a mesh is therefore the same thing that will maintain it, and the control plane a mesh ends up running is one it built itself, from a repository and a commit it can name and build again. @@ -312,6 +317,8 @@ func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet { } { set.IntVar(p.into, p.name, *p.into, "the machine's port for "+p.what) } + set.BoolVar(&opts.Adopted, "adopted", false, + "raise this machine adopted: keep what it runs and its firewall until each module is taken") set.StringVar(&opts.OverlayRange, "overlay-range", opts.OverlayRange, "the private network's address range; must not overlap a tunnel the machine already runs") if opts.Answers == nil { diff --git a/internal/bootstrap/adopted.go b/internal/bootstrap/adopted.go new file mode 100644 index 0000000..dbbe181 --- /dev/null +++ b/internal/bootstrap/adopted.go @@ -0,0 +1,192 @@ +package bootstrap + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "sort" + "strconv" + "strings" + + "github.com/novox/mesh-host/internal/declaration" +) + +// What an adopted genesis changes about the foundation (novox/hq ADR 0100). +// +// **The firewall found on the machine stays in force.** The foundation's own filter drops by +// default, and every base chain at a hook runs; an accept ends only its own chain and a drop in any +// is final — so loading it would close whatever the machine serves. On an adopted machine it is +// not loaded. Its duty, the store never reachable from outside, passes to the mesh's guard: a table +// of the mesh's own that only refuses, and only the foundation's own ports, which genesis has just +// checked free — so it cannot close anything the machine serves. + +// The guard, as the controller declares it: the same ids, paths and text, so the first push +// finds it already there and takes it over unchanged. +const ( + guardID = declaration.AdoptionPrefix + "guard" + guardUnitID = declaration.AdoptionPrefix + "guard-unit" + guardRunningID = declaration.AdoptionPrefix + "guard-running" + guardPath = "/etc/mesh/guard.nft" + guardUnit = "mesh-guard.service" + guardUnitPath = "/etc/systemd/system/" + guardUnit +) + +// AsGuard renders the mesh's refusal-only table for the given machine ports. It passes everything +// by default; it refuses the ports except from the machine itself — its loopback and the container +// runtime's own networks — and from the private network, known by the interface a packet arrives +// on and never by its source address; at prerouting, ahead of the runtime's destination +// translation, in the inet family so both address families. +// +// Character for character the controller's (mesh-controller internal/catalogue AsGuard); a test +// on each side holds its copy to the same golden text. +func AsGuard(ports []int) string { + sorted := append([]int{}, ports...) + sort.Ints(sorted) + listed := make([]string, len(sorted)) + for i, p := range sorted { + listed[i] = strconv.Itoa(p) + } + var b strings.Builder + b.WriteString("table inet mesh_guard {}\n") + b.WriteString("delete table inet mesh_guard\n") + b.WriteString("table inet mesh_guard {\n") + b.WriteString("\tchain prerouting {\n") + b.WriteString("\t\ttype filter hook prerouting priority raw; policy accept;\n") + fmt.Fprintf(&b, "\t\tiifname != \"lo\" iifname != \"docker0\" iifname != \"br-*\" "+ + "iifname != \"mesh0\" tcp dport { %s } drop\n", strings.Join(listed, ", ")) + b.WriteString("\t}\n") + b.WriteString("}\n") + return b.String() +} + +// guardUnitText is the unit that loads the guard. Stopping it deletes only its own table — never a +// flush, which would take the container runtime's rules and the found firewall with it. +func guardUnitText() string { + return "[Unit]\n" + + "Description=The mesh's guard: refuses its own ports from outside (novox/hq ADR 0100)\n" + + "After=network-pre.target\n" + + "Wants=network-pre.target\n" + + "\n" + + "[Service]\n" + + "Type=oneshot\n" + + "RemainAfterExit=yes\n" + + "ExecStart=nft -f " + guardPath + "\n" + + "ExecReload=nft -f " + guardPath + "\n" + + "ExecStop=nft delete table inet mesh_guard\n" + + "\n" + + "[Install]\n" + + "WantedBy=multi-user.target\n" +} + +// guardResources are the guard as three resources of kinds the host already has. +func guardResources(ports []int) []map[string]any { + return []map[string]any{ + {"id": guardID, "type": "file", "path": guardPath, "content": AsGuard(ports), "mode": "0644"}, + {"id": guardUnitID, "type": "file", "path": guardUnitPath, "content": guardUnitText(), "mode": "0644"}, + {"id": guardRunningID, "type": "service", "unit": guardUnit, "state": "running", + "boot": "enabled", "restart-on": []any{guardID, guardUnitID}}, + } +} + +// guardAfter is where the guard goes: once the container runtime runs, before anything publishes +// a port. +const guardAfter = "container-runtime-running" + +// AdoptedRewrite says what RewriteAdopted did. +type AdoptedRewrite struct { + Removed []string + Guarded []int +} + +// RewriteAdopted makes the produced bundle one for an adopted machine: the foundation's own filter +// taken out, and the mesh's guard put in its place, guarding the store's and the broker's +// management ports on this node. The nftables package stays: the guard is loaded with it, and +// installing a package loads no table. Openings are not the bundle's — the first push declares +// them, once there is a controller to derive them. +func RewriteAdopted(r *Rewritten, p FoundationPorts) (AdoptedRewrite, error) { + var out AdoptedRewrite + p = p.orDefaults() + bundle := r.Bundle + var err error + for _, id := range []string{"base-filter-loaded", "base-filter"} { + if !r.declares(id) { + continue + } + if bundle, err = removeResource(bundle, id); err != nil { + return out, err + } + out.Removed = append(out.Removed, id) + } + + out.Guarded = []int{p.Store, p.Management} + var text bytes.Buffer + text.WriteString(",\n // The mesh's guard (novox/hq ADR 0100): this machine is adopted, so its own firewall\n" + + " // stays in force and the foundation's filter is not loaded. The guard only refuses: the\n" + + " // store's and the broker's management ports, except from the machine and the private network.") + for _, res := range guardResources(out.Guarded) { + var one bytes.Buffer + enc := json.NewEncoder(&one) + enc.SetEscapeHTML(false) + if err := enc.Encode(res); err != nil { + return out, err + } + text.WriteString("\n ") + text.Write(bytes.TrimSpace(one.Bytes())) + text.WriteString(",") + } + insert := bytes.TrimSuffix(text.Bytes(), []byte(",")) + + _, _, to, err := resourceAt(bundle, guardAfter) + if err != nil { + return out, fmt.Errorf("the guard goes after %q, and %w", guardAfter, err) + } + rest := bundle[to:] + joined := make([]byte, 0, len(bundle)+len(insert)) + joined = append(joined, bundle[:to]...) + joined = append(joined, insert...) + // What followed the resource — its own comma, or the end of the list — now follows the guard. + if trimmed := bytes.TrimLeft(rest, " \t\r\n"); len(trimmed) > 0 && trimmed[0] != ',' && trimmed[0] != ']' { + return out, fmt.Errorf("the bundle does not separate %q from what follows it the way a list does", guardAfter) + } + joined = append(joined, rest...) + + parsed, err := declaration.ParseFileTrusted(joined) + if err != nil { + return out, fmt.Errorf("the bundle stopped being a declaration once it was made an adopted one, which is this installer's fault: %w", err) + } + r.Bundle, r.Declaration, r.Resources = joined, parsed, len(parsed.Resources) + return out, nil +} + +// declares is whether the produced bundle names a resource. +func (r Rewritten) declares(id string) bool { + for _, res := range r.Declaration.Resources { + if res.Identity() == id { + return true + } + } + return false +} + +// genesisTakes are the modules an adopted genesis takes as it installs them: the foundation's and +// the mesh's own, whose names genesis checked free, so taking them replaces nothing a predecessor +// ran. The private network is not among them — it rewrites the machine's hosts file and the +// container runtime's configuration whole — and neither is anything the operator installs later. +var genesisTakes = map[string]bool{ + RegistryModule: true, ControlPlaneModule: true, BuilderModule: true, + "postgres": true, "lavinmq": true, "mesh-vault": true, "mesh-catalog": true, +} + +// takeIfAdopted takes one of genesis's own modules on an adopted node, once it is assigned and +// before the push that raises it. +func takeIfAdopted(ctx context.Context, o Options, control controlPlane, module string, say func(string)) error { + if !o.Adopted || !genesisTakes[module] { + return nil + } + if _, err := control.tell(ctx, "take", o.Node, module); err != nil { + return err + } + say(" taken " + module + " on " + o.Node + " — the mesh's own, its name checked free") + return nil +} diff --git a/internal/bootstrap/adopted_test.go b/internal/bootstrap/adopted_test.go new file mode 100644 index 0000000..dd6e8e6 --- /dev/null +++ b/internal/bootstrap/adopted_test.go @@ -0,0 +1,192 @@ +package bootstrap + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/declaration" +) + +// Defends novox/hq ADR 0100: an adopted genesis loads no table that drops by default or holds an +// accept; the mesh guards its own ports in a table that only refuses; and genesis takes the mesh's +// own modules as it installs them, and nothing else. + +// The same golden text the controller's test holds its AsGuard to. +const goldenGuard = `table inet mesh_guard {} +delete table inet mesh_guard +table inet mesh_guard { + chain prerouting { + type filter hook prerouting priority raw; policy accept; + iifname != "lo" iifname != "docker0" iifname != "br-*" iifname != "mesh0" tcp dport { 5432, 15672 } drop + } +} +` + +func TestTheGuardIsExactlyThisTable(t *testing.T) { + if got := AsGuard([]int{15672, 5432}); got != goldenGuard { + t.Fatalf("the guard changed:\n%s", got) + } +} + +func TestAnAdoptedBundleLoadsNoDroppingTableAndExactlyTheGuard(t *testing.T) { + r := producedBundle(t) + p := FoundationPorts{Store: 5433, Management: 15673} + if _, err := RewritePorts(&r, p, ""); err != nil { + t.Fatal(err) + } + got, err := RewriteAdopted(&r, p) + if err != nil { + t.Fatal(err) + } + if strings.Join(got.Removed, ",") != "base-filter-loaded,base-filter" { + t.Errorf("removed %v", got.Removed) + } + at := map[string]int{} + var guards []*declaration.File + for i, res := range r.Declaration.Resources { + at[res.Identity()] = i + if f, ok := res.(*declaration.File); ok { + if strings.Contains(f.Content, "policy drop") || strings.Contains(f.Content, " accept\n") && + !strings.Contains(f.Content, "policy accept") { + t.Errorf("%s loads a table that drops or accepts: %q", f.ID, f.Content) + } + if f.Path == guardPath { + guards = append(guards, f) + } + } + if s, ok := res.(*declaration.Service); ok && s.Unit == "nftables.service" { + t.Errorf("the foundation's filter is still loaded by %s", s.ID) + } + } + if len(guards) != 1 { + t.Fatalf("%d guard table(s)", len(guards)) + } + if !strings.Contains(guards[0].Content, "tcp dport { 5433, 15673 } drop") { + t.Errorf("the guard does not refuse this node's ports: %s", guards[0].Content) + } + if strings.Count(guards[0].Content, "accept") != 1 || !strings.Contains(guards[0].Content, "policy accept") { + t.Errorf("the guard holds an accept of its own: %s", guards[0].Content) + } + for _, id := range []string{guardID, guardUnitID, guardRunningID} { + if _, ok := at[id]; !ok { + t.Errorf("the bundle has no %s", id) + } + } + if !(at["container-runtime-running"] < at[guardID] && at[guardRunningID] < at["store"]) { + t.Errorf("the guard is not between the runtime and the store: %v", at) + } + if _, kept := at["base-filter-package"]; !kept { + t.Error("nft, which loads the guard, is no longer installed") + } + unit := r.Declaration.Resources[at[guardRunningID]].(*declaration.Service) + if unit.Unit != guardUnit || unit.State != "running" || strings.Join(unit.RestartOn, ",") != guardID+","+guardUnitID { + t.Errorf("the guard's service: %+v", unit) + } + stop := r.Declaration.Resources[at[guardUnitID]].(*declaration.File).Content + if !strings.Contains(stop, "ExecStop=nft delete table inet mesh_guard") || strings.Contains(stop, "flush") { + t.Errorf("stopping the guard does not delete only its own table: %s", stop) + } +} + +func TestAConvergedBundleIsNotMadeAnAdoptedOne(t *testing.T) { + // The converged genesis keeps the foundation's filter, byte for byte (novox/hq ADR 0088). + r := producedBundle(t) + for _, res := range r.Declaration.Resources { + if strings.HasPrefix(res.Identity(), declaration.AdoptionPrefix) { + t.Errorf("a converged bundle carries %s", res.Identity()) + } + } + if !r.declares("base-filter-loaded") { + t.Error("a converged bundle lost its filter") + } +} + +func TestAnAdoptedGenesisTakesTheMeshsOwnModulesBeforePushingThem(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + for _, c := range []struct { + module string + takes bool + }{{RegistryModule, true}, {ControlPlaneModule, true}, {BuilderModule, true}, {"gitea", false}} { + rec := &controlRecorder{settings: map[string]string{}} + control := controlPlane{container: "temp-mesh-controller", run: rec.run, timeout: time.Second} + o := Options{Node: "anchor", Adopted: true, Wait: time.Second} + if _, err := installModule(context.Background(), o, control, c.module, []byte(`{}`), quietly); err != nil { + t.Fatal(err) + } + assign, take, push := rec.index("assign anchor "+c.module), rec.index("take anchor "+c.module), rec.index("push anchor") + if !c.takes { + if take >= 0 { + t.Errorf("%s was taken at genesis", c.module) + } + continue + } + if !(assign >= 0 && assign < take && take < push) { + t.Errorf("%s: assign %d, take %d, push %d: %v", c.module, assign, take, push, rec.told) + } + } +} + +func TestAConvergedGenesisTakesNothing(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + rec := &controlRecorder{settings: map[string]string{}} + control := controlPlane{container: "temp-mesh-controller", run: rec.run, timeout: time.Second} + if _, err := installModule(context.Background(), Options{Node: "anchor", Wait: time.Second}, control, + RegistryModule, []byte(`{}`), quietly); err != nil { + t.Fatal(err) + } + if rec.index("take") >= 0 { + t.Errorf("a converged genesis took a module: %v", rec.told) + } +} + +func TestAnAdoptedGenesisOpensTheRegistryFromAnywhere(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + rec := &controlRecorder{settings: map[string]string{}} + control := controlPlane{container: "temp-mesh-controller", run: rec.run, timeout: time.Second} + o := Options{Node: "anchor", Adopted: true, Ports: FoundationPorts{Registry: 5100}, Wait: time.Second} + if _, err := installModule(context.Background(), o, control, RegistryModule, []byte(`{}`), quietly); err != nil { + t.Fatal(err) + } + if got := rec.settings["distribution-settings.json"]; got != `{"expose":{"5000":"anywhere"},"ports":{"5000":5100}}` { + t.Errorf("the registry was told %s", got) + } +} + +func TestAnAdoptedGenesisChoosesTheFilterAndLoadsNone(t *testing.T) { + rec := &controlRecorder{settings: map[string]string{}} + control := controlPlane{container: "mesh-controller", run: rec.run, timeout: time.Second} + o := Options{Node: "anchor", Adopted: true, Answers: map[string]string{"packet-filter": "nftables"}} + filter, err := ChooseAndInstallFilter(context.Background(), o, control, quietly) + if err != nil || filter != "nftables" { + t.Fatalf("%q %v", filter, err) + } + if len(rec.told) != 0 { + t.Errorf("an adopted genesis installed a filter: %v", rec.told) + } +} + +func TestAnAdoptedNodeIsRecordedAdopted(t *testing.T) { + stop := errors.New("stop here") + runtime := &asked{answer: func(name string, args []string) (string, error) { + joined := strings.Join(args, " ") + switch { + case strings.Contains(joined, "node list"): + return "", nil + case strings.Contains(joined, "node add"): + return "", nil + } + return "", fmt.Errorf("%w: %s %v", stop, name, args) + }} + _, _ = Enrol(context.Background(), Options{ + Node: "anchor", Adopted: true, State: t.TempDir() + "/state.json", Timeout: time.Second, + Host: "/usr/local/bin/mesh-host", HostInBackground: true, + }, arch(t), controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second}, + func(string) {}) + if !runtime.ran("node add anchor --adopted") { + t.Errorf("the node was not added adopted: %v", runtime.commands) + } +} diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index 95b86b4..4349bcc 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -34,6 +34,8 @@ import ( "fmt" "strings" "time" + + "github.com/novox/mesh-host/internal/firewall" ) // Step names one stage. A failure says which one, because "the bootstrap failed" is a sentence @@ -377,6 +379,23 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro // Which half of the host applies things here. Asked of the machine and proved, because // `mesh-host` pins this at link time and an installer run by hand has no link time. + // An adopted machine keeps the firewall it was found with, so the mesh must speak it; one no + // host speaks is refused here, before anything changes (novox/hq ADR 0100). + if o.Adopted { + kind, name, err := firewall.Detect(ctx, d.Run) + if err != nil { + return result, failed(StepPreflight, err) + } + if kind == firewall.Unsupported { + return result, failed(StepPreflight, fmt.Errorf( + "this machine is filtered by %s, and no host speaks that firewall yet. An adopted "+ + "machine keeps its firewall in force, so the mesh could neither open what it needs "+ + "through it nor say what it would close. Nothing was changed", name)) + } + result.Firewall = string(kind) + say(" adopted what is on this machine is kept; its firewall (" + string(kind) + ") stays in force") + } + sys, err := WorkOutSystem(ctx, d.Run, o.System) if err != nil { return result, failed(StepPreflight, err) @@ -444,6 +463,14 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro if moved.Places > 0 { say(fmt.Sprintf(" ports %d place(s) rewritten to this node's foundation ports", moved.Places)) } + if o.Adopted { + adopted, err := RewriteAdopted(&rewritten, o.Ports) + if err != nil { + return result, failed(StepBundle, err) + } + say(fmt.Sprintf(" adopted bundle the foundation's filter is not loaded (%s); the mesh's guard refuses %v from outside", + strings.Join(adopted.Removed, ", "), adopted.Guarded)) + } for _, c := range []struct { what, path string made bool @@ -718,7 +745,9 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro // ---- 17. filter ----------------------------------------------------------------------- say("filter — required, so the question is which, not whether") - if err := ChooseAndInstallFilter(ctx, o, permanentControl, say); err != nil { + filter, err := ChooseAndInstallFilter(ctx, o, permanentControl, say) + result.Filter = filter + if err != nil { return result, failed(StepFilter, err) } @@ -736,6 +765,12 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro return result, failed(StepExport, err) } + if o.Adopted { + say("\nthis machine is a mesh of one adopted node: it builds its own software, holds its graph " + + "and sits on its private network, and what it ran before is kept as it was, behind the firewall " + + "it was found with. Take each module on it once its data has moved; converge it when done.") + return result, nil + } say("\nthis machine is a mesh of one node: it builds its own software, holds its graph, " + "sits on its private network, and filters what modules declared.") say("what remains is somebody else's: adding nodes, and assigning what they should run.") diff --git a/internal/bootstrap/enrol.go b/internal/bootstrap/enrol.go index e64b86c..7ee3b71 100644 --- a/internal/bootstrap/enrol.go +++ b/internal/bootstrap/enrol.go @@ -72,7 +72,13 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla if mentions(nodes, o.Node) { say(" already a node " + o.Node) } else { - if _, err := control.tell(ctx, "node", "add", o.Node); err != nil { + add := []string{"node", "add", o.Node} + if o.Adopted { + // The controller records the node's mode; an adopted one keeps what it was found with + // until each module is taken (novox/hq ADR 0100). + add = append(add, "--adopted") + } + if _, err := control.tell(ctx, add...); err != nil { return out, err } out.Added = true diff --git a/internal/bootstrap/module.go b/internal/bootstrap/module.go index 92c4116..6352d4d 100644 --- a/internal/bootstrap/module.go +++ b/internal/bootstrap/module.go @@ -214,8 +214,12 @@ func pinPlaceholder(manifest []byte, reference, module string) ([]byte, int, err } // prepareModule is what genesis tells the controller about a module on this node once it is -// assigned and before it is pushed: the ports this node gave it (novox/hq ADR 0100). +// assigned and before it is pushed: the ports this node gave it, and on an adopted node that the +// module is taken (novox/hq ADR 0100). func prepareModule(ctx context.Context, o Options, control controlPlane, module string, say func(string)) error { - return setFoundationSettings(ctx, o, control, module, say) + if err := setFoundationSettings(ctx, o, control, module, say); err != nil { + return err + } + return takeIfAdopted(ctx, o, control, module, say) } diff --git a/internal/bootstrap/phase2.go b/internal/bootstrap/phase2.go index 431f12c..ee2319e 100644 --- a/internal/bootstrap/phase2.go +++ b/internal/bootstrap/phase2.go @@ -150,16 +150,22 @@ func PlaceOnTheNetwork(ctx context.Context, o Options, control controlPlane, // ChooseAndInstallFilter picks the packet filter — required, so the question is which, not // whether — and installs it. -func ChooseAndInstallFilter(ctx context.Context, o Options, control controlPlane, say func(string)) error { +func ChooseAndInstallFilter(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) { filter, err := decide(Choice{ Name: "packet-filter", Question: "Which packet filter should this machine run?", Options: []string{"nftables"}, }, o.Answers["packet-filter"], o.Prompt, say) if err != nil { - return err + return "", err } - return InstallFromCatalogue(ctx, o, control, filter, say) + if o.Adopted { + // The firewall found here stays in force until the node converges; the filter is + // chosen now and assigned by the flip (novox/hq ADR 0100). + say(" not installed " + filter + " — this machine is adopted; converging it assigns " + filter) + return filter, nil + } + return filter, InstallFromCatalogue(ctx, o, control, filter, say) } // InstallExtras installs what was asked for beyond the floor. diff --git a/internal/bootstrap/retire.go b/internal/bootstrap/retire.go index 2ed13f8..af41291 100644 --- a/internal/bootstrap/retire.go +++ b/internal/bootstrap/retire.go @@ -132,18 +132,29 @@ func RetireTheTemporaryControlPlane(ctx context.Context, o Options, sys system.S // where the comment explaining it lives. A comment that outlives the thing it describes is worse // than no comment: it is the file telling somebody the machine has a control plane it does not. func removeResource(bundle []byte, id string) ([]byte, error) { + previous, from, to, err := resourceAt(bundle, id) + if err != nil { + return nil, err + } + return cut(bundle, previous, from, to), nil +} + +// resourceAt finds one resource's object in a bundle's text by its id: where the one before it +// ended, and where it starts and ends — comments and strings skipped, so an id quoted in a comment +// or a command is never mistaken for the resource. +func resourceAt(bundle []byte, id string) (previous, from, to int, err error) { array := indexOutsideStrings(bundle, `"resources"`) if array < 0 { - return nil, fmt.Errorf("this bundle has no resources array, so there is nothing to take out of it") + return 0, 0, 0, fmt.Errorf("this bundle has no resources array, so there is nothing to take out of it") } open := indexOutsideStrings(bundle[array:], "[") if open < 0 { - return nil, fmt.Errorf("this bundle's resources are not a list") + return 0, 0, 0, fmt.Errorf("this bundle's resources are not a list") } open += array - depth, from := 0, -1 - previous := open + depth := 0 + from, previous = -1, open inString, escaped, inLine, inBlock := false, false, false, false for i := open + 1; i < len(bundle); i++ { c := bundle[i] @@ -181,16 +192,16 @@ func removeResource(bundle []byte, id string) ([]byte, error) { break } if isResource(bundle[from:i+1], id) { - return cut(bundle, previous, from, i+1), nil + return previous, from, i + 1, nil } previous = i + 1 from = -1 case c == ']' && depth == 0: - return nil, fmt.Errorf( + return 0, 0, 0, fmt.Errorf( "this bundle declares no %q, so there is nothing to take out of it", id) } } - return nil, fmt.Errorf("this bundle's resources list does not end") + return 0, 0, 0, fmt.Errorf("this bundle's resources list does not end") } // isResource reports whether one resource's text is the one wanted. From 3e0e6e6b7e540780d97ee8ee5398a8aded663b45 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 17:37:01 +0200 Subject: [PATCH 08/52] =?UTF-8?q?Delete=20a=20forwarded=20opening=20the=20?= =?UTF-8?q?way=20ufw=20accepts=20it,=20and=20read=20a=20fresh=20machine's?= =?UTF-8?q?=20resolver=20as=20not=20in=20use=20=E2=80=94=20both=20measured?= =?UTF-8?q?=20on=20a=20lab=20machine=20(hq=20ADR=200100)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/bootstrap/inuse.go | 20 +- internal/bootstrap/inuse_test.go | 27 +- .../testdata/fresh-machine-listeners.txt | 12 + internal/firewall/firewall.go | 15 +- internal/firewall/firewall_test.go | 95 +++- internal/firewall/testdata/ufw-active.nft | 478 ++++++++++++++++++ internal/firewall/testdata/ufw-delete.txt | 40 ++ internal/firewall/testdata/ufw-show-added.txt | 8 + .../firewall/testdata/ufw-status-active.txt | 21 + 9 files changed, 699 insertions(+), 17 deletions(-) create mode 100644 internal/bootstrap/testdata/fresh-machine-listeners.txt create mode 100644 internal/firewall/testdata/ufw-active.nft create mode 100644 internal/firewall/testdata/ufw-delete.txt create mode 100644 internal/firewall/testdata/ufw-show-added.txt create mode 100644 internal/firewall/testdata/ufw-status-active.txt diff --git a/internal/bootstrap/inuse.go b/internal/bootstrap/inuse.go index 3909de6..58c357b 100644 --- a/internal/bootstrap/inuse.go +++ b/internal/bootstrap/inuse.go @@ -10,13 +10,11 @@ import ( "github.com/novox/mesh-host/internal/store" ) -// quietUDP are processes whose UDP sockets every fresh machine has — name resolution, address -// configuration, time — and which serve nobody. ss names a process by its first fifteen characters, -// so both spellings are here. -// -// **Still to be measured** (novox/hq ADR 0100): this list is what a fresh machine is expected to -// hold, and it must be checked against a freshly installed lab machine before it is trusted. -var quietUDP = map[string]bool{ +// quiet are the processes every fresh machine runs that serve nobody: name resolution (whose +// link-local resolver listens on TCP as well as UDP, on every address), address configuration and +// time. ss names a process by its first fifteen characters, so both spellings are here. Measured +// on a freshly installed lab machine (testdata/fresh-machine-listeners.txt): these and nothing else. +var quiet = map[string]bool{ "systemd-resolved": true, "systemd-resolve": true, "systemd-networkd": true, "systemd-network": true, "systemd-timesyncd": true, "systemd-timesyn": true, @@ -24,8 +22,8 @@ var quietUDP = map[string]bool{ } // InUse says what makes this machine a machine in use (novox/hq ADR 0100): every running container -// no host made, and every socket listening on an address other than loopback that is not ssh's — a -// UDP one only when it is held by something other than what every fresh machine runs. ours names +// no host made, and every socket listening on an address other than loopback that is neither ssh's +// nor held by what every fresh machine runs. ours names // what the mesh itself runs, which a re-run of genesis finds and does not count. func InUse(ctx context.Context, run Runner, ours func(name string) bool) ([]string, []reachable.Reach, error) { var containers []string @@ -61,9 +59,9 @@ func counts(r reachable.Reach) bool { } switch r.Protocol { case "tcp": - return r.By != "sshd" && !(r.By == "" && r.Port == 22) + return r.By != "sshd" && !(r.By == "" && r.Port == 22) && !quiet[r.By] case "udp": - return !quietUDP[r.By] + return !quiet[r.By] } return false } diff --git a/internal/bootstrap/inuse_test.go b/internal/bootstrap/inuse_test.go index b1a1e49..98ea349 100644 --- a/internal/bootstrap/inuse_test.go +++ b/internal/bootstrap/inuse_test.go @@ -2,6 +2,7 @@ package bootstrap import ( "context" + "os" "path/filepath" "strings" "testing" @@ -13,8 +14,8 @@ import ( // and listener it counted. // Lines as `ss -Hltunp` prints them. The ssh, samba, loopback and proxy lines are captured from a -// real machine; the resolver, DHCP and time lines are written in the same shape for the processes a -// fresh machine runs, and still need measuring against one. +// real machine; the resolver, DHCP and time lines are written in the same shape. What a fresh machine +// actually runs is measured in testdata/fresh-machine-listeners.txt. const inUseSockets = `tcp LISTEN 0 128 0.0.0.0:22 0.0.0.0:* users:(("sshd",pid=1188536,fd=6)) tcp LISTEN 0 128 [::]:22 [::]:* users:(("sshd",pid=1188536,fd=7)) tcp LISTEN 0 32 127.0.0.1:53 0.0.0.0:* users:(("dnsmasq",pid=1189392,fd=7)) @@ -97,3 +98,25 @@ func TestARerunOfGenesisIsNotAMachineInUse(t *testing.T) { t.Errorf("what an earlier genesis raised was counted as a machine in use: %v", err) } } + +func TestAFreshlyInstalledMachineAsMeasuredIsNotInUse(t *testing.T) { + // Captured with `ss -Hltunp` on a freshly installed lab machine: its resolver listens on TCP on + // every address, which the record's words alone would count. + raw, err := os.ReadFile("testdata/fresh-machine-listeners.txt") + if err != nil { + t.Fatal(err) + } + run := func(ctx context.Context, name string, args ...string) (string, error) { + if name == "ss" { + return string(raw), nil + } + return "", nil + } + containers, listeners, err := InUse(context.Background(), run, func(string) bool { return false }) + if err != nil { + t.Fatal(err) + } + if len(containers) != 0 || len(listeners) != 0 { + t.Errorf("a fresh machine read as in use: containers %v, listeners %v", containers, listeners) + } +} diff --git a/internal/bootstrap/testdata/fresh-machine-listeners.txt b/internal/bootstrap/testdata/fresh-machine-listeners.txt new file mode 100644 index 0000000..defcf66 --- /dev/null +++ b/internal/bootstrap/testdata/fresh-machine-listeners.txt @@ -0,0 +1,12 @@ +udp UNCONN 0 0 0.0.0.0:5353 0.0.0.0:* users:(("systemd-resolve",pid=262,fd=17)) +udp UNCONN 0 0 0.0.0.0:5355 0.0.0.0:* users:(("systemd-resolve",pid=262,fd=13)) +udp UNCONN 0 0 127.0.0.54:53 0.0.0.0:* users:(("systemd-resolve",pid=262,fd=24)) +udp UNCONN 0 0 127.0.0.53%lo:53 0.0.0.0:* users:(("systemd-resolve",pid=262,fd=22)) +udp UNCONN 0 0 [::]:5353 [::]:* users:(("systemd-resolve",pid=262,fd=18)) +udp UNCONN 0 0 [::]:5355 [::]:* users:(("systemd-resolve",pid=262,fd=15)) +udp UNCONN 0 0 [fe80::1266:6aff:fe24:628d]%enp5s0:546 [::]:* users:(("systemd-network",pid=272,fd=36)) +tcp LISTEN 0 4096 127.0.0.1:39473 0.0.0.0:* users:(("containerd",pid=394,fd=14)) +tcp LISTEN 0 4096 0.0.0.0:5355 0.0.0.0:* users:(("systemd-resolve",pid=262,fd=14)) +tcp LISTEN 0 4096 127.0.0.53%lo:53 0.0.0.0:* users:(("systemd-resolve",pid=262,fd=23)) +tcp LISTEN 0 4096 127.0.0.54:53 0.0.0.0:* users:(("systemd-resolve",pid=262,fd=25)) +tcp LISTEN 0 4096 [::]:5355 [::]:* users:(("systemd-resolve",pid=262,fd=16)) diff --git a/internal/firewall/firewall.go b/internal/firewall/firewall.go index b896c5d..f730afe 100644 --- a/internal/firewall/firewall.go +++ b/internal/firewall/firewall.go @@ -38,6 +38,17 @@ const ( Unsupported Kind = "unsupported" ) +// deletion is the arguments that delete a rule as `ufw show added` printed it. A route rule is +// deleted with `route delete …`: ufw refuses `delete route …` as invalid syntax. And ufw answers +// success when asked to delete a rule it does not hold, so every deletion is read back. +func deletion(rule string) []string { + w := words(rule) + if len(w) > 0 && w[0] == "route" { + return append([]string{"route", "delete"}, w[1:]...) + } + return append([]string{"delete"}, w...) +} + // MeshInterface is the private network's interface, the way an opening from the mesh is known. // It must be the controller's overlay interface name. const MeshInterface = "mesh0" @@ -335,7 +346,7 @@ func Converge(ctx context.Context, run Runner, o *declaration.Opening) (string, return "unchanged", nil } for _, rule := range stale { - if _, err := run(ctx, "ufw", append([]string{"delete"}, words(rule)...)...); err != nil { + if _, err := run(ctx, "ufw", deletion(rule)...); err != nil { return "", fmt.Errorf("deleting the mesh's stale ufw rule %q: %w", rule, err) } } @@ -381,7 +392,7 @@ func Remove(ctx context.Context, run Runner, id string) (int, error) { if !markedFor(comment(rule), id) { continue } - if _, err := run(ctx, "ufw", append([]string{"delete"}, words(rule)...)...); err != nil { + if _, err := run(ctx, "ufw", deletion(rule)...); err != nil { return removed, fmt.Errorf("deleting the mesh's ufw rule %q: %w", rule, err) } removed++ diff --git a/internal/firewall/firewall_test.go b/internal/firewall/firewall_test.go index 67d8aa2..c810a03 100644 --- a/internal/firewall/firewall_test.go +++ b/internal/firewall/firewall_test.go @@ -180,9 +180,17 @@ func (f *fakeUFW) run(_ context.Context, name string, args ...string) (string, e case args[0] == "disable": f.active = false return "Firewall stopped and disabled on system startup\n", nil - case args[0] == "delete": + case args[0] == "delete" && len(args) > 1 && args[1] == "route": + // As the real ufw answers it (captured in testdata/ufw-delete.txt): a route rule is + // deleted with `route delete`, never `delete route`. + return "", errors.New("ERROR: Invalid syntax") + case args[0] == "delete", args[0] == "route" && len(args) > 1 && args[1] == "delete": + rest := args[1:] + if args[0] == "route" { + rest = append([]string{"route"}, args[2:]...) + } for i, r := range f.rules { - if strings.Join(words(r), "\x00") == strings.Join(args[1:], "\x00") { + if strings.Join(words(r), "\x00") == strings.Join(rest, "\x00") { f.rules = append(f.rules[:i], f.rules[i+1:]...) return "Rule deleted\n", nil } @@ -333,3 +341,86 @@ func TestDetectingTheFoundFirewall(t *testing.T) { } } } + +// The fixtures below were captured from a real ufw 0.36.2 on a lab machine, not written by hand: +// ufw prints a rule back in its own shorter form, so the mark in the comment is the only thing the +// host relies on. + +func TestTheMarksAreReadFromWhatUfwReallyPrints(t *testing.T) { + raw, err := os.ReadFile("testdata/ufw-show-added.txt") + if err != nil { + t.Fatal(err) + } + run := func(ctx context.Context, name string, args ...string) (string, error) { return string(raw), nil } + rules, err := added(context.Background(), run) + if err != nil { + t.Fatal(err) + } + if len(rules) != 7 { + t.Fatalf("read %d rules, want 7: %q", len(rules), rules) + } + marked := 0 + for _, r := range rules { + if strings.HasPrefix(comment(r), "mesh-host ") { + marked++ + } + } + if marked != 5 { + t.Errorf("read %d marked rules, want 5", marked) + } + if !markedFor(comment(rules[5]), "adoption.opening-tcp-8443-forwarded") { + t.Errorf("the forwarded rule from the mesh lost its mark: %q", rules[5]) + } +} + +func TestEveryRealRuleIsDeletedInTheFormUfwAccepts(t *testing.T) { + raw, err := os.ReadFile("testdata/ufw-show-added.txt") + if err != nil { + t.Fatal(err) + } + run := func(ctx context.Context, name string, args ...string) (string, error) { return string(raw), nil } + rules, _ := added(context.Background(), run) + // Each of these was run on the lab machine and answered "Rule deleted" (testdata/ufw-delete.txt). + want := map[string]string{ + "allow 5671/tcp comment 'mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d'": "delete allow 5671/tcp comment|mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d", + "allow in on mesh0 to any port 5432 proto tcp comment 'mesh-host adoption.opening-tcp-5432-incoming deadbeef'": "delete allow in on mesh0 to any port 5432 proto tcp comment|mesh-host adoption.opening-tcp-5432-incoming deadbeef", + "route allow 80/tcp comment 'mesh-host adoption.opening-tcp-8081-forwarded 0badf00d'": "route delete allow 80/tcp comment|mesh-host adoption.opening-tcp-8081-forwarded 0badf00d", + "route allow in on mesh0 to any port 443 proto tcp comment 'mesh-host adoption.opening-tcp-8443-forwarded cafe0001'": "route delete allow in on mesh0 to any port 443 proto tcp comment|mesh-host adoption.opening-tcp-8443-forwarded cafe0001", + } + seen := 0 + for _, r := range rules { + w, ok := want[r] + if !ok { + continue + } + seen++ + d := deletion(r) + got := strings.Join(d[:len(d)-1], " ") + "|" + d[len(d)-1] + if got != w { + t.Errorf("deleting %q\n got %s\n want %s", r, got, w) + } + } + if seen != len(want) { + t.Errorf("matched %d of %d captured rules", seen, len(want)) + } +} + +func TestARealUfwRulesetIsUfw(t *testing.T) { + raw, err := os.ReadFile("testdata/ufw-active.nft") + if err != nil { + t.Fatal(err) + } + status, err := os.ReadFile("testdata/ufw-status-active.txt") + if err != nil { + t.Fatal(err) + } + if !statusActive(string(status)) { + t.Fatal("the captured status does not read as active") + } + if refusing := Refusing(string(raw), true); len(refusing) > 0 { + t.Errorf("a machine with ufw active and nothing else read as refusing in %v", refusing) + } + if refusing := Refusing(string(raw), false); len(refusing) == 0 { + t.Error("ufw's drop chains, with ufw not known to be active, read as refusing nothing") + } +} diff --git a/internal/firewall/testdata/ufw-active.nft b/internal/firewall/testdata/ufw-active.nft new file mode 100644 index 0000000..93b24a4 --- /dev/null +++ b/internal/firewall/testdata/ufw-active.nft @@ -0,0 +1,478 @@ +table ip nat { + chain DOCKER { + } + + chain PREROUTING { + type nat hook prerouting priority dstnat; policy accept; + xt match "addrtype" counter packets 2 bytes 1160 jump DOCKER + } + + chain OUTPUT { + type nat hook output priority dstnat; policy accept; + ip daddr != 127.0.0.0/8 xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } + + chain POSTROUTING { + type nat hook postrouting priority srcnat; policy accept; + ip saddr 172.17.0.0/16 oifname != "docker0" counter packets 0 bytes 0 xt target "MASQUERADE" + } +} +table ip filter { + chain DOCKER { + iifname != "docker0" oifname "docker0" counter packets 0 bytes 0 drop + } + + chain DOCKER-FORWARD { + counter packets 0 bytes 0 jump DOCKER-CT + counter packets 0 bytes 0 jump DOCKER-INTERNAL + counter packets 0 bytes 0 jump DOCKER-BRIDGE + iifname "docker0" counter packets 0 bytes 0 accept + } + + chain DOCKER-BRIDGE { + oifname "docker0" counter packets 0 bytes 0 jump DOCKER + } + + chain DOCKER-CT { + oifname "docker0" xt match "conntrack" counter packets 0 bytes 0 accept + } + + chain DOCKER-INTERNAL { + } + + chain FORWARD { + type filter hook forward priority filter; policy drop; + counter packets 0 bytes 0 jump DOCKER-USER + counter packets 0 bytes 0 jump DOCKER-FORWARD + counter packets 0 bytes 0 jump ufw-before-logging-forward + counter packets 0 bytes 0 jump ufw-before-forward + counter packets 0 bytes 0 jump ufw-after-forward + counter packets 0 bytes 0 jump ufw-after-logging-forward + counter packets 0 bytes 0 jump ufw-reject-forward + counter packets 0 bytes 0 jump ufw-track-forward + } + + chain DOCKER-USER { + } + + chain ufw-before-logging-input { + } + + chain ufw-before-logging-output { + } + + chain ufw-before-logging-forward { + } + + chain ufw-before-input { + iifname "lo" counter packets 0 bytes 0 accept + xt match "conntrack" counter packets 0 bytes 0 accept + xt match "conntrack" counter packets 0 bytes 0 jump ufw-logging-deny + xt match "conntrack" counter packets 0 bytes 0 drop + ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept + ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept + ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept + ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept + udp sport 67 udp dport 68 counter packets 0 bytes 0 accept + counter packets 0 bytes 0 jump ufw-not-local + ip daddr 224.0.0.251 udp dport 5353 counter packets 0 bytes 0 accept + ip daddr 239.255.255.250 udp dport 1900 counter packets 0 bytes 0 accept + counter packets 0 bytes 0 jump ufw-user-input + } + + chain ufw-before-output { + oifname "lo" counter packets 0 bytes 0 accept + xt match "conntrack" counter packets 0 bytes 0 accept + counter packets 0 bytes 0 jump ufw-user-output + } + + chain ufw-before-forward { + xt match "conntrack" counter packets 0 bytes 0 accept + ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept + ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept + ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept + ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept + counter packets 0 bytes 0 jump ufw-user-forward + } + + chain ufw-after-input { + udp dport 137 counter packets 0 bytes 0 jump ufw-skip-to-policy-input + udp dport 138 counter packets 0 bytes 0 jump ufw-skip-to-policy-input + tcp dport 139 counter packets 0 bytes 0 jump ufw-skip-to-policy-input + tcp dport 445 counter packets 0 bytes 0 jump ufw-skip-to-policy-input + udp dport 67 counter packets 0 bytes 0 jump ufw-skip-to-policy-input + udp dport 68 counter packets 0 bytes 0 jump ufw-skip-to-policy-input + xt match "addrtype" counter packets 0 bytes 0 jump ufw-skip-to-policy-input + } + + chain ufw-after-output { + } + + chain ufw-after-forward { + } + + chain ufw-after-logging-input { + limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG" + } + + chain ufw-after-logging-output { + } + + chain ufw-after-logging-forward { + limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG" + } + + chain ufw-reject-input { + } + + chain ufw-reject-output { + } + + chain ufw-reject-forward { + } + + chain ufw-track-input { + } + + chain ufw-track-output { + ip protocol tcp xt match "conntrack" counter packets 0 bytes 0 accept + ip protocol udp xt match "conntrack" counter packets 0 bytes 0 accept + } + + chain ufw-track-forward { + } + + chain INPUT { + type filter hook input priority filter; policy drop; + counter packets 1 bytes 76 jump ufw-before-logging-input + counter packets 1 bytes 76 jump ufw-before-input + counter packets 0 bytes 0 jump ufw-after-input + counter packets 0 bytes 0 jump ufw-after-logging-input + counter packets 0 bytes 0 jump ufw-reject-input + counter packets 0 bytes 0 jump ufw-track-input + } + + chain OUTPUT { + type filter hook output priority filter; policy accept; + counter packets 1 bytes 76 jump ufw-before-logging-output + counter packets 1 bytes 76 jump ufw-before-output + counter packets 1 bytes 76 jump ufw-after-output + counter packets 1 bytes 76 jump ufw-after-logging-output + counter packets 1 bytes 76 jump ufw-reject-output + counter packets 1 bytes 76 jump ufw-track-output + } + + chain ufw-logging-deny { + xt match "conntrack" limit rate 3/minute burst 10 packets counter packets 0 bytes 0 return + limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG" + } + + chain ufw-logging-allow { + limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG" + } + + chain ufw-skip-to-policy-input { + counter packets 0 bytes 0 drop + } + + chain ufw-skip-to-policy-output { + counter packets 0 bytes 0 accept + } + + chain ufw-skip-to-policy-forward { + counter packets 0 bytes 0 drop + } + + chain ufw-not-local { + xt match "addrtype" counter packets 0 bytes 0 return + xt match "addrtype" counter packets 0 bytes 0 return + xt match "addrtype" counter packets 0 bytes 0 return + limit rate 3/minute burst 10 packets counter packets 0 bytes 0 jump ufw-logging-deny + counter packets 0 bytes 0 drop + } + + chain ufw-user-input { + tcp dport 22 counter packets 0 bytes 0 accept + tcp dport 8080 counter packets 0 bytes 0 accept + udp dport 51820 counter packets 0 bytes 0 accept + } + + chain ufw-user-output { + } + + chain ufw-user-forward { + tcp dport 80 counter packets 0 bytes 0 accept + iifname "mesh0" tcp dport 443 counter packets 0 bytes 0 accept + } + + chain ufw-user-logging-input { + } + + chain ufw-user-logging-output { + } + + chain ufw-user-logging-forward { + } + + chain ufw-user-limit { + limit rate 3/minute burst 5 packets counter packets 0 bytes 0 xt target "LOG" + counter packets 0 bytes 0 xt target "REJECT" + } + + chain ufw-user-limit-accept { + counter packets 0 bytes 0 accept + } +} +table ip6 nat { + chain DOCKER { + } + + chain PREROUTING { + type nat hook prerouting priority dstnat; policy accept; + xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } + + chain OUTPUT { + type nat hook output priority dstnat; policy accept; + ip6 daddr != ::1 xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } +} +table ip6 filter { + chain DOCKER { + } + + chain DOCKER-FORWARD { + counter packets 0 bytes 0 jump DOCKER-CT + counter packets 0 bytes 0 jump DOCKER-INTERNAL + counter packets 0 bytes 0 jump DOCKER-BRIDGE + } + + chain DOCKER-BRIDGE { + } + + chain DOCKER-CT { + } + + chain DOCKER-INTERNAL { + } + + chain FORWARD { + type filter hook forward priority filter; policy drop; + counter packets 0 bytes 0 jump DOCKER-USER + counter packets 0 bytes 0 jump DOCKER-FORWARD + counter packets 0 bytes 0 jump ufw6-before-logging-forward + counter packets 0 bytes 0 jump ufw6-before-forward + counter packets 0 bytes 0 jump ufw6-after-forward + counter packets 0 bytes 0 jump ufw6-after-logging-forward + counter packets 0 bytes 0 jump ufw6-reject-forward + counter packets 0 bytes 0 jump ufw6-track-forward + } + + chain DOCKER-USER { + } + + chain ufw6-before-logging-input { + } + + chain ufw6-before-logging-output { + } + + chain ufw6-before-logging-forward { + } + + chain ufw6-before-input { + iifname "lo" counter packets 0 bytes 0 accept + xt match "rt" counter packets 0 bytes 0 drop + xt match "conntrack" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + xt match "conntrack" counter packets 0 bytes 0 jump ufw6-logging-deny + xt match "conntrack" counter packets 0 bytes 0 drop + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 ip6 daddr fe80::/10 udp sport 547 udp dport 546 counter packets 0 bytes 0 accept + ip6 daddr ff02::fb udp dport 5353 counter packets 0 bytes 0 accept + ip6 daddr ff02::f udp dport 1900 counter packets 0 bytes 0 accept + counter packets 0 bytes 0 jump ufw6-user-input + } + + chain ufw6-before-output { + oifname "lo" counter packets 0 bytes 0 accept + xt match "rt" counter packets 0 bytes 0 drop + xt match "conntrack" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + counter packets 0 bytes 0 jump ufw6-user-output + } + + chain ufw6-before-forward { + xt match "rt" counter packets 0 bytes 0 drop + xt match "conntrack" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + counter packets 0 bytes 0 jump ufw6-user-forward + } + + chain ufw6-after-input { + udp dport 137 counter packets 0 bytes 0 jump ufw6-skip-to-policy-input + udp dport 138 counter packets 0 bytes 0 jump ufw6-skip-to-policy-input + tcp dport 139 counter packets 0 bytes 0 jump ufw6-skip-to-policy-input + tcp dport 445 counter packets 0 bytes 0 jump ufw6-skip-to-policy-input + udp dport 546 counter packets 0 bytes 0 jump ufw6-skip-to-policy-input + udp dport 547 counter packets 0 bytes 0 jump ufw6-skip-to-policy-input + } + + chain ufw6-after-output { + } + + chain ufw6-after-forward { + } + + chain ufw6-after-logging-input { + limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG" + } + + chain ufw6-after-logging-output { + } + + chain ufw6-after-logging-forward { + limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG" + } + + chain ufw6-reject-input { + } + + chain ufw6-reject-output { + } + + chain ufw6-reject-forward { + } + + chain ufw6-track-input { + } + + chain ufw6-track-output { + meta l4proto tcp xt match "conntrack" counter packets 0 bytes 0 accept + meta l4proto udp xt match "conntrack" counter packets 0 bytes 0 accept + } + + chain ufw6-track-forward { + } + + chain INPUT { + type filter hook input priority filter; policy drop; + counter packets 1 bytes 128 jump ufw6-before-logging-input + counter packets 1 bytes 128 jump ufw6-before-input + counter packets 0 bytes 0 jump ufw6-after-input + counter packets 0 bytes 0 jump ufw6-after-logging-input + counter packets 0 bytes 0 jump ufw6-reject-input + counter packets 0 bytes 0 jump ufw6-track-input + } + + chain OUTPUT { + type filter hook output priority filter; policy accept; + counter packets 4 bytes 304 jump ufw6-before-logging-output + counter packets 4 bytes 304 jump ufw6-before-output + counter packets 0 bytes 0 jump ufw6-after-output + counter packets 0 bytes 0 jump ufw6-after-logging-output + counter packets 0 bytes 0 jump ufw6-reject-output + counter packets 0 bytes 0 jump ufw6-track-output + } + + chain ufw6-logging-deny { + xt match "conntrack" limit rate 3/minute burst 10 packets counter packets 0 bytes 0 return + limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG" + } + + chain ufw6-logging-allow { + limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG" + } + + chain ufw6-skip-to-policy-input { + counter packets 0 bytes 0 drop + } + + chain ufw6-skip-to-policy-output { + counter packets 0 bytes 0 accept + } + + chain ufw6-skip-to-policy-forward { + counter packets 0 bytes 0 drop + } + + chain ufw6-user-input { + tcp dport 22 counter packets 0 bytes 0 accept + tcp dport 8080 counter packets 0 bytes 0 accept + udp dport 51820 counter packets 0 bytes 0 accept + } + + chain ufw6-user-output { + } + + chain ufw6-user-forward { + tcp dport 80 counter packets 0 bytes 0 accept + iifname "mesh0" tcp dport 443 counter packets 0 bytes 0 accept + } + + chain ufw6-user-logging-input { + } + + chain ufw6-user-logging-output { + } + + chain ufw6-user-logging-forward { + } + + chain ufw6-user-limit { + limit rate 3/minute burst 5 packets counter packets 0 bytes 0 xt target "LOG" + counter packets 0 bytes 0 xt target "REJECT" + } + + chain ufw6-user-limit-accept { + counter packets 0 bytes 0 accept + } +} diff --git a/internal/firewall/testdata/ufw-delete.txt b/internal/firewall/testdata/ufw-delete.txt new file mode 100644 index 0000000..b887dad --- /dev/null +++ b/internal/firewall/testdata/ufw-delete.txt @@ -0,0 +1,40 @@ +Rule deleted +Rule deleted (v6) +rc=0 +Rule deleted +Rule deleted (v6) +rc=0 +ERROR: Invalid syntax +rc=1 +===ADDED2 +Added user rules (see 'ufw status' for running firewall): +ufw allow 22/tcp +ufw allow 8080/tcp +ufw route allow 80/tcp comment 'mesh-host adoption.opening-tcp-8081-forwarded 0badf00d' +ufw route allow in on mesh0 to any port 443 proto tcp comment 'mesh-host adoption.opening-tcp-8443-forwarded cafe0001' +ufw allow 51820/udp comment 'mesh-host adoption.opening-udp-51820-incoming 11112222' +Firewall reloaded +reload rc=0 +===AFTERRELOAD +3 +Firewall stopped and disabled on system startup +===DISABLED +Status: inactive +/etc/ufw/user.rules +3 +Firewall is active and enabled on system startup +Status: active +Rule deleted +Rule deleted (v6) +rc=0 +Rule deleted +Rule deleted (v6) +rc=0 +Could not delete non-existent rule +Could not delete non-existent rule (v6) +wrongcomment rc=0 +Added user rules (see 'ufw status' for running firewall): +ufw allow 22/tcp +ufw allow 8080/tcp +ufw allow 51820/udp comment 'mesh-host adoption.opening-udp-51820-incoming 11112222' +Status: active diff --git a/internal/firewall/testdata/ufw-show-added.txt b/internal/firewall/testdata/ufw-show-added.txt new file mode 100644 index 0000000..85d9c6b --- /dev/null +++ b/internal/firewall/testdata/ufw-show-added.txt @@ -0,0 +1,8 @@ +Added user rules (see 'ufw status' for running firewall): +ufw allow 22/tcp +ufw allow 8080/tcp +ufw allow 5671/tcp comment 'mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d' +ufw allow in on mesh0 to any port 5432 proto tcp comment 'mesh-host adoption.opening-tcp-5432-incoming deadbeef' +ufw route allow 80/tcp comment 'mesh-host adoption.opening-tcp-8081-forwarded 0badf00d' +ufw route allow in on mesh0 to any port 443 proto tcp comment 'mesh-host adoption.opening-tcp-8443-forwarded cafe0001' +ufw allow 51820/udp comment 'mesh-host adoption.opening-udp-51820-incoming 11112222' diff --git a/internal/firewall/testdata/ufw-status-active.txt b/internal/firewall/testdata/ufw-status-active.txt new file mode 100644 index 0000000..9f32038 --- /dev/null +++ b/internal/firewall/testdata/ufw-status-active.txt @@ -0,0 +1,21 @@ +Status: active + +To Action From +-- ------ ---- +22/tcp ALLOW Anywhere +8080/tcp ALLOW Anywhere +5671/tcp ALLOW Anywhere # mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d +5432/tcp on mesh0 ALLOW Anywhere # mesh-host adoption.opening-tcp-5432-incoming deadbeef +51820/udp ALLOW Anywhere # mesh-host adoption.opening-udp-51820-incoming 11112222 +22/tcp (v6) ALLOW Anywhere (v6) +8080/tcp (v6) ALLOW Anywhere (v6) +5671/tcp (v6) ALLOW Anywhere (v6) # mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d +5432/tcp (v6) on mesh0 ALLOW Anywhere (v6) # mesh-host adoption.opening-tcp-5432-incoming deadbeef +51820/udp (v6) ALLOW Anywhere (v6) # mesh-host adoption.opening-udp-51820-incoming 11112222 + +80/tcp ALLOW FWD Anywhere # mesh-host adoption.opening-tcp-8081-forwarded 0badf00d +443/tcp ALLOW FWD Anywhere on mesh0 # mesh-host adoption.opening-tcp-8443-forwarded cafe0001 +80/tcp (v6) ALLOW FWD Anywhere (v6) # mesh-host adoption.opening-tcp-8081-forwarded 0badf00d +443/tcp (v6) ALLOW FWD Anywhere (v6) on mesh0 # mesh-host adoption.opening-tcp-8443-forwarded cafe0001 + +===STATUSV From 6eabed63ebb29883c248c338478ebe198fb25634 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 17:38:17 +0200 Subject: [PATCH 09/52] Reload the service manager's units before restarting a service whose files changed, and start the guard before the network as the controller declares it (hq ADR 0100) --- internal/apply/apply.go | 16 ++++++++++++++++ internal/apply/apply_test.go | 12 ++++++++++++ internal/bootstrap/adopted.go | 2 +- internal/system/arch.go | 8 ++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/internal/apply/apply.go b/internal/apply/apply.go index d8a4926..6aa2b14 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -688,11 +688,27 @@ func reflected(r *declaration.Service, changed map[string]bool) []string { return restartedBy(r.RestartOn, changed) } +// unitReloader is a service manager that caches unit files and must be told to read them again. +type unitReloader interface { + ReloadUnits(ctx context.Context, run system.Runner) error +} + func applyService(ctx context.Context, sys system.System, r *declaration.Service, run Runner, changed map[string]bool) (Outcome, error) { out := begin(r) var changes []string + // A file the service reflects changed, and it may be the unit's own file or a drop-in: the + // service manager reads those again only when told to, and a restart without it runs the unit + // it had already loaded. + if reflects(r, changed) { + if u, ok := sys.(unitReloader); ok { + if err := u.ReloadUnits(ctx, run); err != nil { + return out, fmt.Errorf("reloading the service manager's units for %s: %w", r.Unit, err) + } + } + } + // Boot first. A unit asked to be running and enabled should survive this apply failing // half way in the more useful direction: enabled-and-stopped comes back at the next boot, // where running-and-disabled does not. diff --git a/internal/apply/apply_test.go b/internal/apply/apply_test.go index 52a179c..ba5c5aa 100644 --- a/internal/apply/apply_test.go +++ b/internal/apply/apply_test.go @@ -1114,6 +1114,18 @@ func TestAServiceIsRestartedWhenWhatItReflectsChanges(t *testing.T) { if !stopped || !started { t.Errorf("the file changed and the service was not restarted; commands were %v", commands) } + reloaded, stop := -1, -1 + for i, c := range commands { + if strings.Contains(c, "daemon-reload") && reloaded < 0 { + reloaded = i + } + if strings.Contains(c, "stop thing.service") && stop < 0 { + stop = i + } + } + if reloaded < 0 || reloaded > stop { + t.Errorf("the service was restarted without the unit files being read again first; commands were %v", commands) + } } } diff --git a/internal/bootstrap/adopted.go b/internal/bootstrap/adopted.go index dbbe181..0e3dedd 100644 --- a/internal/bootstrap/adopted.go +++ b/internal/bootstrap/adopted.go @@ -65,7 +65,7 @@ func AsGuard(ports []int) string { func guardUnitText() string { return "[Unit]\n" + "Description=The mesh's guard: refuses its own ports from outside (novox/hq ADR 0100)\n" + - "After=network-pre.target\n" + + "Before=network-pre.target\n" + "Wants=network-pre.target\n" + "\n" + "[Service]\n" + diff --git a/internal/system/arch.go b/internal/system/arch.go index 178570d..abee02b 100644 --- a/internal/system/arch.go +++ b/internal/system/arch.go @@ -246,3 +246,11 @@ func (arch) AddUserToGroup(ctx context.Context, run Runner, name, group string) } return nil } + +// ReloadUnits has systemd read its unit files again. A unit file that changed on disk is otherwise +// ignored: a restart runs the unit systemd already loaded, and the new text only takes effect +// after a reload nobody asked for. +func (arch) ReloadUnits(ctx context.Context, run Runner) error { + _, err := run(ctx, "systemctl", "daemon-reload") + return err +} From 0f126d137ca196d5ae8f9b4e235e9bffc8df854a Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 17:48:57 +0200 Subject: [PATCH 10/52] Write into a file the machine shares instead of over it, and reload a service that re-reads its configuration instead of restarting it (hq ADR 0102) --- internal/apply/apply.go | 33 ++++ internal/apply/hold.go | 8 +- internal/apply/into.go | 235 ++++++++++++++++++++++++++++ internal/apply/into_test.go | 213 +++++++++++++++++++++++++ internal/declaration/declaration.go | 34 ++++ internal/declaration/into_test.go | 35 +++++ internal/store/store.go | 13 ++ internal/system/arch.go | 6 + 8 files changed, 575 insertions(+), 2 deletions(-) create mode 100644 internal/apply/into.go create mode 100644 internal/apply/into_test.go create mode 100644 internal/declaration/into_test.go diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 6aa2b14..f7e5d62 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -50,6 +50,8 @@ type Outcome struct { // wrote is a digest of what this apply put there, kept so the next one can tell a machine // that drifted from one the mesh changed its mind about. Not reported: it is bookkeeping. wrote string + // into is what a file written into held before the mesh's keys (novox/hq ADR 0102). + into *store.Into } // Report is what an apply did, in the order it did it. @@ -298,6 +300,7 @@ func ApplyKeeping( ID: resource.Identity(), Type: string(resource.Kind()), Target: outcome.Target, AppliedAt: time.Now().UTC(), Wrote: outcome.wrote, + Into: outcome.into, Holds: holds(resource), }) // Its module has been taken, and what was held for it is now the mesh's. @@ -489,6 +492,9 @@ func applyAccess(r *declaration.Access) (Outcome, error) { } func applyFile(r *declaration.File, previous store.Applied, unseal Unseal) (Outcome, error) { + if r.Into != "" { + return applyInto(r, previous) + } out := begin(r) // What actually goes on disk. For a sealed file the mesh never had this, and neither did @@ -688,6 +694,11 @@ func reflected(r *declaration.Service, changed map[string]bool) []string { return restartedBy(r.RestartOn, changed) } +// serviceReloader is a service manager that can tell a running unit to read its configuration again. +type serviceReloader interface { + ReloadService(ctx context.Context, run system.Runner, unit string) error +} + // unitReloader is a service manager that caches unit files and must be told to read them again. type unitReloader interface { ReloadUnits(ctx context.Context, run system.Runner) error @@ -774,6 +785,25 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service "%s was restarted to pick up a change and is %s", r.Unit, after) } changes = append(changes, "restarted for "+strings.Join(reflected(r, changed), ", ")) + } else if r.State == "running" && len(restartedBy(r.ReloadOn, changed)) > 0 { + // Told to read its configuration again, not stopped: for a service whose restart would + // stop what it runs — every container, for the container runtime (novox/hq ADR 0102). + reloader, ok := sys.(serviceReloader) + if !ok { + return out, fmt.Errorf("%s must be reloaded for %s and this machine's service manager "+ + "cannot reload a unit", r.Unit, strings.Join(restartedBy(r.ReloadOn, changed), ", ")) + } + if err := reloader.ReloadService(ctx, run, r.Unit); err != nil { + return out, fmt.Errorf("reloading %s: %w", r.Unit, err) + } + after, err := sys.ServiceState(ctx, run, r.Unit) + if err != nil { + return out, err + } + if after != "running" { + return out, fmt.Errorf("%s was reloaded to pick up a change and is %s", r.Unit, after) + } + changes = append(changes, "reloaded for "+strings.Join(restartedBy(r.ReloadOn, changed), ", ")) } if len(changes) == 0 { @@ -828,6 +858,9 @@ func remove(ctx context.Context, sys system.System, a store.Applied, run Runner) return "removed", "no longer declared, and empty", nil case declaration.TypeFile: + if a.Into != nil { + return removeInto(a) + } if err := os.RemoveAll(a.Target); err != nil { return "", "", err } diff --git a/internal/apply/hold.go b/internal/apply/hold.go index 89a4d44..d8ddfb4 100644 --- a/internal/apply/hold.go +++ b/internal/apply/hold.go @@ -46,9 +46,13 @@ func KeepIn(dir string) Keep { } // holdable is whether a resource is one a predecessor can already have on the machine: a file at -// a path, or a container under a name. +// a path, or a container under a name. A file written into is not: it replaces nothing that was +// found, only adds the mesh's keys beside it (novox/hq ADR 0102). func holdable(r declaration.Resource) bool { - return r.Kind() == declaration.TypeFile || r.Kind() == declaration.TypeContainer + if f, ok := r.(*declaration.File); ok { + return f.Into == "" + } + return r.Kind() == declaration.TypeContainer } // found is whether a declared file or container is present on the machine with no record of this diff --git a/internal/apply/into.go b/internal/apply/into.go new file mode 100644 index 0000000..d2e4c91 --- /dev/null +++ b/internal/apply/into.go @@ -0,0 +1,235 @@ +package apply + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "sort" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// A file written into, never over (novox/hq ADR 0102). +// +// **The file is the machine's; the mesh owns keys in it.** The container runtime's configuration +// is the case that needed it: the mesh states one fact there — its registry is trusted over the +// private network — and writing the file whole replaced everything the machine had set, down to +// where the runtime keeps its data. So the host reads what is there, sets only the declared keys, +// keeps every other key as it found it, and records what each of its keys held before. Undeclared, +// each key goes back, and a file the mesh created goes only if nothing but its keys is left. + +// applyInto writes a file's declared keys into the object already at its path. +func applyInto(r *declaration.File, previous store.Applied) (Outcome, error) { + out := begin(r) + if r.Into != declaration.IntoJSON { + return out, fmt.Errorf("%s: into %q is not a format this host writes into", r.Path, r.Into) + } + var declared map[string]json.RawMessage + if err := json.Unmarshal([]byte(r.Content), &declared); err != nil { + return out, fmt.Errorf("%s: the keys to write are not a JSON object: %w", r.Path, err) + } + + existing, err := os.ReadFile(r.Path) + existed := err == nil + if err != nil && !errors.Is(err, os.ErrNotExist) { + return out, err + } + object := map[string]json.RawMessage{} + if existed && len(bytes.TrimSpace(existing)) > 0 { + if err := json.Unmarshal(existing, &object); err != nil || object == nil { + // Refused, never replaced: a file the host cannot read as an object is a file it + // cannot write into without losing whatever it is. + return out, fmt.Errorf("%s is not a JSON object, so the mesh cannot write its keys into it "+ + "without replacing what is there; it was left as it is", r.Path) + } + } + + rec := store.Into{Format: declaration.IntoJSON, Before: map[string]json.RawMessage{}} + if previous.Into != nil { + rec.Created = previous.Into.Created + for k, v := range previous.Into.Before { + rec.Before[k] = v + } + rec.Absent = slices.Clone(previous.Into.Absent) + } else { + rec.Created = !existed + } + tracked := func(k string) bool { _, ok := rec.Before[k]; return ok || slices.Contains(rec.Absent, k) } + + // Drift: the machine no longer holds what this host last set in its keys. + drifted := previous.Wrote != "" && existed && digestOf(keysOf(object, keysTracked(rec))) != previous.Wrote + + // Keys the mesh set before and no longer declares go back to what they held. + for _, k := range keysTracked(rec) { + if _, still := declared[k]; still { + continue + } + giveBack(object, &rec, k) + } + // Declared keys: remember what each held the first time, then set it. + for _, k := range keysIn(declared) { + if !tracked(k) { + if v, had := object[k]; had { + rec.Before[k] = v + } else { + rec.Absent = append(rec.Absent, k) + } + } + object[k] = declared[k] + } + + want, err := render(object) + if err != nil { + return out, err + } + same := existed && canonical(existing) == canonical(want) + if !same { + mode := os.FileMode(0o644) + if info, err := os.Stat(r.Path); err == nil { + mode = info.Mode().Perm() // the machine's file keeps the machine's mode + } else if r.Mode != "" { + if m, err := modeOf(r.Mode, mode); err == nil { + mode = m + } + } + if err := os.MkdirAll(filepath.Dir(r.Path), 0o755); err != nil { + return out, err + } + if err := writeAtomically(r.Path, want, mode); err != nil { + return out, err + } + } + // Read back: every declared key holds what was declared. + written, err := os.ReadFile(r.Path) + if err != nil { + return out, fmt.Errorf("wrote into %s and cannot read it back: %w", r.Path, err) + } + var check map[string]json.RawMessage + if err := json.Unmarshal(written, &check); err != nil { + return out, fmt.Errorf("%s is not a JSON object after writing into it: %w", r.Path, err) + } + for k, v := range declared { + if canonical(check[k]) != canonical(v) { + return out, fmt.Errorf("%s does not hold the declared %q after writing into it", r.Path, k) + } + } + + if len(rec.Before) == 0 { + rec.Before = nil + } + out.into = &rec + out.wrote = digestOf(keysOf(check, keysIn(declared))) + switch { + case !existed: + out.Action = "created" + out.Detail = "written into; the file was not there" + case same: + out.Action = "unchanged" + case drifted: + out.Action = "corrected" + out.Detail = "the mesh's keys had been changed on the machine; the rest of the file was kept" + default: + out.Action = "updated" + out.Detail = "the mesh's keys written in; every other key kept as it was" + } + return out, nil +} + +// removeInto gives back what a file written into held before the mesh's keys. +func removeInto(a store.Applied) (string, string, error) { + existing, err := os.ReadFile(a.Target) + if errors.Is(err, os.ErrNotExist) { + return "forgotten", "no longer there", nil + } + if err != nil { + return "", "", err + } + object := map[string]json.RawMessage{} + if len(bytes.TrimSpace(existing)) > 0 { + if err := json.Unmarshal(existing, &object); err != nil || object == nil { + return "kept", "no longer a JSON object, so the mesh's keys were left in it; " + + "remove them by hand", nil + } + } + rec := *a.Into + for _, k := range keysTracked(rec) { + giveBack(object, &rec, k) + } + if a.Into.Created && len(object) == 0 { + if err := os.Remove(a.Target); err != nil { + return "", "", err + } + return "removed", "no longer declared; the mesh had created it and nothing else was in it", nil + } + want, err := render(object) + if err != nil { + return "", "", err + } + info, err := os.Stat(a.Target) + if err != nil { + return "", "", err + } + if err := writeAtomically(a.Target, want, info.Mode().Perm()); err != nil { + return "", "", err + } + return "restored", "no longer declared; the mesh's keys were given back what they held", nil +} + +func giveBack(object map[string]json.RawMessage, rec *store.Into, k string) { + if v, had := rec.Before[k]; had { + object[k] = v + delete(rec.Before, k) + return + } + delete(object, k) + rec.Absent = slices.DeleteFunc(rec.Absent, func(a string) bool { return a == k }) +} + +func keysTracked(rec store.Into) []string { + var keys []string + for k := range rec.Before { + keys = append(keys, k) + } + keys = append(keys, rec.Absent...) + sort.Strings(keys) + return slices.Compact(keys) +} + +func keysOf(object map[string]json.RawMessage, keys []string) string { + var b bytes.Buffer + for _, k := range keys { + b.WriteString(k + "=" + canonical(object[k]) + "\n") + } + return b.String() +} + +func keysIn(m map[string]json.RawMessage) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// canonical is a JSON value compacted, so formatting is not mistaken for a change. +func canonical(raw []byte) string { + var b bytes.Buffer + if err := json.Compact(&b, raw); err != nil { + return string(raw) + } + return b.String() +} + +func render(object map[string]json.RawMessage) ([]byte, error) { + b, err := json.MarshalIndent(object, "", " ") + if err != nil { + return nil, err + } + return append(b, '\n'), nil +} diff --git a/internal/apply/into_test.go b/internal/apply/into_test.go new file mode 100644 index 0000000..421e013 --- /dev/null +++ b/internal/apply/into_test.go @@ -0,0 +1,213 @@ +package apply + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// Defends novox/hq ADR 0102: a file the mesh shares with software it did not install is written +// into, never over, and a service that re-reads its configuration is reloaded, not restarted. + +func intoDecl(t *testing.T, path, keys string) string { + t.Helper() + return fmt.Sprintf(`{"declaration":1,"resources":[ + {"id":"networking.registry-trust","type":"file","path":%q,"into":"json","content":%q} + ]}`, path, keys) +} + +func readObject(t *testing.T, path string) map[string]any { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var o map[string]any + if err := json.Unmarshal(raw, &o); err != nil { + t.Fatalf("%s is not a JSON object: %v\n%s", path, err, raw) + } + return o +} + +// The machine's own runtime settings, the way a predecessor leaves them. +const machinesOwn = `{"data-root":"/srv/docker","log-opts":{"max-size":"10m"},"insecure-registries":["192.0.2.7:5000"]}` + +func TestWritingIntoKeepsEveryKeyTheMachineHad(t *testing.T) { + path := filepath.Join(t.TempDir(), "daemon.json") + if err := os.WriteFile(path, []byte(machinesOwn), 0o600); err != nil { + t.Fatal(err) + } + d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`)) + report, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + o := readObject(t, path) + if o["data-root"] != "/srv/docker" { + t.Errorf("the machine's data directory was not kept: %v", o) + } + if fmt.Sprint(o["log-opts"]) != "map[max-size:10m]" { + t.Errorf("the machine's logging settings were not kept: %v", o) + } + if fmt.Sprint(o["insecure-registries"]) != "[10.42.0.1:5000]" { + t.Errorf("the mesh's key was not written: %v", o) + } + if info, _ := os.Stat(path); info.Mode().Perm() != 0o600 { + t.Errorf("the machine's file mode was changed to %o", info.Mode().Perm()) + } + if got := report.Outcomes[0].Action; got != "updated" { + t.Errorf("writing into was reported as %q", got) + } + + // Again, with nothing changed: nothing to do. + report, state, err = Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if got := report.Outcomes[0].Action; got != "unchanged" { + t.Errorf("a second apply was %q", got) + } + + // Undeclared: the key goes back to what the machine had, and the file stays. + empty := somethingElse(t) + report, _, err = Apply(context.Background(), archHost(t), empty, state, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + o = readObject(t, path) + if fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000]" || o["data-root"] != "/srv/docker" { + t.Errorf("undeclaring did not give the machine back what it had: %v", o) + } + if got := report.Outcomes[0].Action; got != "restored" { + t.Errorf("undeclaring was reported as %q", got) + } +} + +func TestAFileWrittenIntoThatWasNotThereIsRemovedWhenOnlyTheMeshsKeysAreLeft(t *testing.T) { + path := filepath.Join(t.TempDir(), "daemon.json") + d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`)) + report, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if got := report.Outcomes[0].Action; got != "created" { + t.Errorf("writing into a file that was not there was %q", got) + } + // Somebody else adds a key of their own: the file is no longer only the mesh's. + o := readObject(t, path) + o["debug"] = true + raw, _ := json.Marshal(o) + _ = os.WriteFile(path, raw, 0o644) + + empty := somethingElse(t) + if _, _, err := Apply(context.Background(), archHost(t), empty, state, store.OriginDeclared, nil, nil, nil); err != nil { + t.Fatal(err) + } + o = readObject(t, path) + if _, still := o["insecure-registries"]; still || o["debug"] != true { + t.Errorf("undeclaring should remove the mesh's key and keep the other: %v", o) + } + + // Without the other key, the file the mesh created goes. + path2 := filepath.Join(t.TempDir(), "daemon.json") + d2 := parse(t, intoDecl(t, path2, `{"insecure-registries":["10.42.0.1:5000"]}`)) + _, state2, err := Apply(context.Background(), archHost(t), d2, store.State{}, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if _, _, err := Apply(context.Background(), archHost(t), empty, state2, store.OriginDeclared, nil, nil, nil); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(path2); !os.IsNotExist(err) { + t.Errorf("a file the mesh created, holding only its keys, was left behind") + } +} + +func TestAKeyNoLongerDeclaredGoesBackAndANewOneIsRemembered(t *testing.T) { + path := filepath.Join(t.TempDir(), "daemon.json") + _ = os.WriteFile(path, []byte(machinesOwn), 0o644) + first := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`)) + _, state, err := Apply(context.Background(), archHost(t), first, store.State{}, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + second := parse(t, intoDecl(t, path, `{"registry-mirrors":["http://10.42.0.1:5000"]}`)) + if _, _, err := Apply(context.Background(), archHost(t), second, state, store.OriginDeclared, nil, nil, nil); err != nil { + t.Fatal(err) + } + o := readObject(t, path) + if fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000]" { + t.Errorf("a key the mesh stopped declaring was not given back: %v", o) + } + if fmt.Sprint(o["registry-mirrors"]) != "[http://10.42.0.1:5000]" { + t.Errorf("the newly declared key was not written: %v", o) + } +} + +func TestAFileThatIsNotAnObjectIsRefusedAndLeftAlone(t *testing.T) { + path := filepath.Join(t.TempDir(), "daemon.json") + _ = os.WriteFile(path, []byte("# not json at all\n"), 0o644) + d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`)) + if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil); err == nil { + t.Fatal("writing into a file that is not a JSON object was not refused") + } + raw, _ := os.ReadFile(path) + if string(raw) != "# not json at all\n" { + t.Errorf("a file the mesh could not write into was changed: %q", raw) + } +} + +func TestAFileWrittenIntoIsNeverHeldOnAnAdoptedNode(t *testing.T) { + path := filepath.Join(t.TempDir(), "daemon.json") + _ = os.WriteFile(path, []byte(machinesOwn), 0o644) + d := adopted(t, `{"taken":[],"untaken":{"networking":["networking.registry-trust"]}}`, + fmt.Sprintf(`{"id":"networking.registry-trust","type":"file","path":%q,"into":"json","content":%q}`, + path, `{"insecure-registries":["10.42.0.1:5000"]}`)) + m := &machine{} + report, state := applyAdopted(t, d, store.State{}, m, t.TempDir()) + if got := outcomeOf(report, "networking.registry-trust").Action; got == "held" { + t.Fatal("a file written into was held, though it replaces nothing that was found") + } + if len(state.Held) != 0 { + t.Errorf("something was held: %+v", state.Held) + } + o := readObject(t, path) + if o["data-root"] != "/srv/docker" || fmt.Sprint(o["insecure-registries"]) != "[10.42.0.1:5000]" { + t.Errorf("the adopted node's file was not written into: %v", o) + } +} + +func TestAServiceIsReloadedNotRestartedForWhatItReloadsOn(t *testing.T) { + path := filepath.Join(t.TempDir(), "daemon.json") + d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[ + {"id":"trust","type":"file","path":%q,"into":"json","content":%q}, + {"id":"runtime","type":"service","unit":"docker.service","state":"running","reload-on":["trust"]} + ]}`, path, `{"insecure-registries":["10.42.0.1:5000"]}`)) + var commands []string + if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, + recordingServices(&commands), nil, nil); err != nil { + t.Fatal(err) + } + joined := strings.Join(commands, "\n") + if !strings.Contains(joined, "systemctl reload docker.service") { + t.Errorf("the runtime was not reloaded; commands were %v", commands) + } + if strings.Contains(joined, "stop docker.service") || strings.Contains(joined, "restart docker.service") { + t.Errorf("the runtime was stopped, which stops every container on the machine; commands were %v", commands) + } +} + +// somethingElse is a declaration that no longer holds the file: only an unrelated directory. +func somethingElse(t *testing.T) *declaration.Declaration { + t.Helper() + return parse(t, fmt.Sprintf(`{"declaration":1,"resources":[ + {"id":"other","type":"directory","path":%q} + ]}`, filepath.Join(t.TempDir(), "other"))) +} diff --git a/internal/declaration/declaration.go b/internal/declaration/declaration.go index 991654f..a458c8a 100644 --- a/internal/declaration/declaration.go +++ b/internal/declaration/declaration.go @@ -154,6 +154,13 @@ type File struct { // (ADR 0030), and it does not overwrite that either. CreateOnce bool `json:"create-once,omitempty"` + // Into says the file is shared with software the mesh did not install, and the content is + // the mesh's part of it: written into what is there, never over it (novox/hq ADR 0102). Only + // "json" is spoken — the content is a JSON object whose keys the host sets in the file's + // object, keeping every other key as it found it and recording what each of its keys held + // before, so undeclaring the file gives those back. + Into string `json:"into,omitempty"` + // Sealed is content encrypted to this node's sealing key, for a file the mesh must deliver // without being able to read. // @@ -224,6 +231,24 @@ func (f *File) validate(where string, _ bool) []string { if f.Path == "" { problems = append(problems, where+": a file needs a path") } + switch f.Into { + case "": + case IntoJSON: + var object map[string]json.RawMessage + if err := json.Unmarshal([]byte(f.Content), &object); err != nil || object == nil { + problems = append(problems, where+ + ": a file written into JSON carries a JSON object of the keys it sets") + } + if f.Sealed != "" || f.Bytes != "" || len(f.Secrets) > 0 || f.CreateOnce { + problems = append(problems, where+ + ": a file written into says only its keys, in content — not sealed, bytes, "+ + "secrets or create-once") + } + default: + problems = append(problems, fmt.Sprintf( + "%s: into %q; a file is written into \"json\", or omits it to be written whole", + where, f.Into)) + } var said []string for name, value := range map[string]string{ "content": f.Content, "sealed": f.Sealed, "bytes": f.Bytes, @@ -586,6 +611,12 @@ type Service struct { // would be an action, and the link may not carry one (novox/hq ADR 0005) — so this is not a // way around that rule, it is the shape the rule leaves. RestartOn []string `json:"restart-on,omitempty"` + + // ReloadOn names resources whose change means this service must be reloaded — for a service + // that re-reads its configuration when told to, where a restart would stop what it runs: the + // container runtime, whose restart stops every container on the machine (novox/hq ADR 0102). + // A change that is also in RestartOn restarts it, which covers a reload. + ReloadOn []string `json:"reload-on,omitempty"` } func (s *Service) Identity() string { return s.ID } @@ -609,6 +640,9 @@ func (s *Service) validate(where string, _ bool) []string { return problems } +// IntoJSON is the one structured format a file is written into. +const IntoJSON = "json" + // Opening is a port reachable on an adopted node, from where, and on which path. // // **From** is everywhere or mesh — the private network, by its interface. **Path** is incoming, diff --git a/internal/declaration/into_test.go b/internal/declaration/into_test.go new file mode 100644 index 0000000..531e015 --- /dev/null +++ b/internal/declaration/into_test.go @@ -0,0 +1,35 @@ +package declaration + +import ( + "strings" + "testing" +) + +// Defends novox/hq ADR 0102: a file written into carries only a JSON object of its keys, in a +// format the host speaks, and a service may name what it is reloaded on. + +func TestAFileWrittenIntoIsRefusedUnlessItIsAnObjectOfKeys(t *testing.T) { + for name, c := range map[string]struct{ resource, refusal string }{ + "another format": {`{"id":"f","type":"file","path":"/etc/x","into":"toml","content":"a = 1"}`, `into "toml"`}, + "not an object": {`{"id":"f","type":"file","path":"/etc/x","into":"json","content":"[1,2]"}`, "JSON object"}, + "with create-once": {`{"id":"f","type":"file","path":"/etc/x","into":"json","content":"{}","create-once":true}`, "create-once"}, + } { + _, err := Parse([]byte(`{"declaration":1,"resources":[` + c.resource + `]}`)) + if err == nil || !strings.Contains(err.Error(), c.refusal) { + t.Errorf("%s: want a refusal naming %q, got %v", name, c.refusal, err) + } + } + d, err := Parse([]byte(`{"declaration":1,"resources":[ + {"id":"f","type":"file","path":"/etc/x","into":"json","content":"{\"k\":1}"}, + {"id":"s","type":"service","unit":"docker.service","state":"running","reload-on":["f"]} + ]}`)) + if err != nil { + t.Fatal(err) + } + if f := d.Resources[0].(*File); f.Into != IntoJSON { + t.Errorf("into was read as %q", f.Into) + } + if s := d.Resources[1].(*Service); len(s.ReloadOn) != 1 || s.ReloadOn[0] != "f" { + t.Errorf("reload-on was read as %v", s.ReloadOn) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 3e8a82a..78430fa 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -69,6 +69,19 @@ type Applied struct { // person who edits a managed file watches their change vanish every few minutes with nothing // anywhere saying why. Wrote string `json:"wrote,omitempty"` + + // Into is set for a file written into rather than over (novox/hq ADR 0102): the format, what + // each of the mesh's keys held before it set them, which of them were absent, and whether the + // file itself was — so undeclaring it gives the machine back exactly what it had. + Into *Into `json:"into,omitempty"` +} + +// Into is what a file written into held before the mesh's keys. +type Into struct { + Format string `json:"format"` + Before map[string]json.RawMessage `json:"before,omitempty"` + Absent []string `json:"absent,omitempty"` + Created bool `json:"created,omitempty"` } // State is the whole of what a node knows about what it has done. diff --git a/internal/system/arch.go b/internal/system/arch.go index abee02b..20a0cf7 100644 --- a/internal/system/arch.go +++ b/internal/system/arch.go @@ -254,3 +254,9 @@ func (arch) ReloadUnits(ctx context.Context, run Runner) error { _, err := run(ctx, "systemctl", "daemon-reload") return err } + +// ReloadService tells a running unit to read its configuration again, without stopping it. +func (arch) ReloadService(ctx context.Context, run Runner, unit string) error { + _, err := run(ctx, "systemctl", "reload", unit) + return err +} From 1e0c6a351600da824c65fac631b7d3ee2d666a3b Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:00:03 +0200 Subject: [PATCH 11/52] Publish a reconcile report when what is reachable changed, so the controller's converge preview stays fresh (hq ADR 0100) --- cmd/mesh-host/main.go | 12 +++++++++--- cmd/mesh-host/main_test.go | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 12f3e42..cbc7718 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -669,12 +669,18 @@ type adoptionWatch struct { last string } -// fingerprint is what a report says about adoption: each hold and whether it changed, and the -// firewall. +// fingerprint is what a report says about adoption: each hold and whether it changed, the +// firewall, and what is reachable on the machine — which only an adopted node reports, and which +// is what the controller previews a flip from, so a port that opens or closes between deliveries +// must reach it too (novox/hq ADR 0100). func adoptionFingerprint(r link.Report) string { parts := []string{"firewall=" + r.Firewall} for _, h := range r.Held { - parts = append(parts, h.ID+"="+h.Changed) + parts = append(parts, "held "+h.ID+"="+h.Changed) + } + for _, reach := range r.Reachable { + parts = append(parts, fmt.Sprintf("reach %s %s:%d %s %v %d", reach.Protocol, reach.Address, + reach.Port, reach.By, reach.Published, reach.ContainerPort)) } sort.Strings(parts[1:]) return strings.Join(parts, "\n") diff --git a/cmd/mesh-host/main_test.go b/cmd/mesh-host/main_test.go index cdacdf7..72eb1a1 100644 --- a/cmd/mesh-host/main_test.go +++ b/cmd/mesh-host/main_test.go @@ -169,3 +169,24 @@ func TestWhatTheLinkPublishedCountsAsSaid(t *testing.T) { t.Error("a reconcile repeated what the link had just published") } } + +func TestAReconcileSpeaksWhenWhatIsReachableChanged(t *testing.T) { + // The controller previews a flip from what the node last said is reachable; a port that opened + // since must reach it without waiting for the next delivery (novox/hq ADR 0100). + w := &adoptionWatch{} + before := link.Report{Firewall: "ufw", Reachable: []link.Reach{ + {Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}}} + if !w.changed(before) { + t.Fatal("the first report was not said") + } + reordered := link.Report{Firewall: "ufw", Reachable: []link.Reach{ + {Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}}} + if w.changed(reordered) { + t.Error("the same reachable set was said again") + } + opened := link.Report{Firewall: "ufw", Reachable: append(before.Reachable, + link.Reach{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true, ContainerPort: 80})} + if !w.changed(opened) { + t.Error("a newly published port was not said") + } +} From 14b3ffbd409a6175762b870009e7128529925559 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:00:54 +0200 Subject: [PATCH 12/52] Guard only packets addressed to this machine, and load the guard before the network and stop it only at shutdown (hq ADR 0103) --- internal/bootstrap/adopted.go | 9 ++++--- internal/bootstrap/adopted_test.go | 38 +++++++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/internal/bootstrap/adopted.go b/internal/bootstrap/adopted.go index 0e3dedd..c211059 100644 --- a/internal/bootstrap/adopted.go +++ b/internal/bootstrap/adopted.go @@ -36,7 +36,8 @@ const ( // by default; it refuses the ports except from the machine itself — its loopback and the container // runtime's own networks — and from the private network, known by the interface a packet arrives // on and never by its source address; at prerouting, ahead of the runtime's destination -// translation, in the inet family so both address families. +// translation, in the inet family so both address families. It matches only packets addressed to +// this machine: what the machine routes for others is never its business (novox/hq ADR 0103). // // Character for character the controller's (mesh-controller internal/catalogue AsGuard); a test // on each side holds its copy to the same golden text. @@ -53,7 +54,7 @@ func AsGuard(ports []int) string { b.WriteString("table inet mesh_guard {\n") b.WriteString("\tchain prerouting {\n") b.WriteString("\t\ttype filter hook prerouting priority raw; policy accept;\n") - fmt.Fprintf(&b, "\t\tiifname != \"lo\" iifname != \"docker0\" iifname != \"br-*\" "+ + fmt.Fprintf(&b, "\t\tfib daddr type local iifname != \"lo\" iifname != \"docker0\" iifname != \"br-*\" "+ "iifname != \"mesh0\" tcp dport { %s } drop\n", strings.Join(listed, ", ")) b.WriteString("\t}\n") b.WriteString("}\n") @@ -65,8 +66,10 @@ func AsGuard(ports []int) string { func guardUnitText() string { return "[Unit]\n" + "Description=The mesh's guard: refuses its own ports from outside (novox/hq ADR 0100)\n" + - "Before=network-pre.target\n" + + "DefaultDependencies=no\n" + "Wants=network-pre.target\n" + + "Before=network-pre.target shutdown.target\n" + + "Conflicts=shutdown.target\n" + "\n" + "[Service]\n" + "Type=oneshot\n" + diff --git a/internal/bootstrap/adopted_test.go b/internal/bootstrap/adopted_test.go index dd6e8e6..b9e441b 100644 --- a/internal/bootstrap/adopted_test.go +++ b/internal/bootstrap/adopted_test.go @@ -21,7 +21,7 @@ delete table inet mesh_guard table inet mesh_guard { chain prerouting { type filter hook prerouting priority raw; policy accept; - iifname != "lo" iifname != "docker0" iifname != "br-*" iifname != "mesh0" tcp dport { 5432, 15672 } drop + fib daddr type local iifname != "lo" iifname != "docker0" iifname != "br-*" iifname != "mesh0" tcp dport { 5432, 15672 } drop } } ` @@ -32,6 +32,42 @@ func TestTheGuardIsExactlyThisTable(t *testing.T) { } } +// The same golden unit the controller's test holds its guard unit to. It is loaded before the +// network is up, so it carries no default dependencies, and it is stopped only at shutdown. +const goldenGuardUnit = `[Unit] +Description=The mesh's guard: refuses its own ports from outside (novox/hq ADR 0100) +DefaultDependencies=no +Wants=network-pre.target +Before=network-pre.target shutdown.target +Conflicts=shutdown.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=nft -f /etc/mesh/guard.nft +ExecReload=nft -f /etc/mesh/guard.nft +ExecStop=nft delete table inet mesh_guard + +[Install] +WantedBy=multi-user.target +` + +func TestTheGuardUnitIsExactlyThisUnit(t *testing.T) { + if got := guardUnitText(); got != goldenGuardUnit { + t.Fatalf("the guard's unit changed:\n%s", got) + } +} + +func TestTheGuardRefusesOnlyWhatIsAddressedToThisMachine(t *testing.T) { + // A machine that routes for others — a predecessor's private-network hub — must not have a + // packet for another machine's database port refused (novox/hq ADR 0103). + for _, line := range strings.Split(AsGuard([]int{5432}), "\n") { + if strings.Contains(line, " drop") && !strings.HasPrefix(strings.TrimSpace(line), "fib daddr type local ") { + t.Errorf("a refusal matches packets not addressed to this machine: %q", line) + } + } +} + func TestAnAdoptedBundleLoadsNoDroppingTableAndExactlyTheGuard(t *testing.T) { r := producedBundle(t) p := FoundationPorts{Store: 5433, Management: 15673} From c989b57439b6362170dc5e17ed1de80bdaf34bb5 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:01:14 +0200 Subject: [PATCH 13/52] Guard the broker's plaintext port too at an adopted genesis: the filter admits it from the private network only (hq ADR 0103) --- internal/bootstrap/adopted.go | 12 ++++++++---- internal/bootstrap/adopted_test.go | 6 ++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/internal/bootstrap/adopted.go b/internal/bootstrap/adopted.go index c211059..2a1ee1b 100644 --- a/internal/bootstrap/adopted.go +++ b/internal/bootstrap/adopted.go @@ -103,8 +103,11 @@ type AdoptedRewrite struct { } // RewriteAdopted makes the produced bundle one for an adopted machine: the foundation's own filter -// taken out, and the mesh's guard put in its place, guarding the store's and the broker's -// management ports on this node. The nftables package stays: the guard is loaded with it, and +// taken out, and the mesh's guard put in its place, guarding on this node the store's port, the +// broker's management port and the broker's plaintext port. The last is published on every +// interface and the foundation's filter admits it from the private network only, so a found +// firewall that filters only incoming traffic would leave it reachable from anywhere (novox/hq ADR +// 0103). Every one is a port of a module genesis takes. The nftables package stays: the guard is loaded with it, and // installing a package loads no table. Openings are not the bundle's — the first push declares // them, once there is a controller to derive them. func RewriteAdopted(r *Rewritten, p FoundationPorts) (AdoptedRewrite, error) { @@ -122,11 +125,12 @@ func RewriteAdopted(r *Rewritten, p FoundationPorts) (AdoptedRewrite, error) { out.Removed = append(out.Removed, id) } - out.Guarded = []int{p.Store, p.Management} + out.Guarded = []int{p.Store, p.Management, p.AMQP} var text bytes.Buffer text.WriteString(",\n // The mesh's guard (novox/hq ADR 0100): this machine is adopted, so its own firewall\n" + " // stays in force and the foundation's filter is not loaded. The guard only refuses: the\n" + - " // store's and the broker's management ports, except from the machine and the private network.") + " // store's port and the broker's management and plaintext ports, except from the machine and\n" + + " // the private network.") for _, res := range guardResources(out.Guarded) { var one bytes.Buffer enc := json.NewEncoder(&one) diff --git a/internal/bootstrap/adopted_test.go b/internal/bootstrap/adopted_test.go index b9e441b..cbe3a98 100644 --- a/internal/bootstrap/adopted_test.go +++ b/internal/bootstrap/adopted_test.go @@ -70,7 +70,7 @@ func TestTheGuardRefusesOnlyWhatIsAddressedToThisMachine(t *testing.T) { func TestAnAdoptedBundleLoadsNoDroppingTableAndExactlyTheGuard(t *testing.T) { r := producedBundle(t) - p := FoundationPorts{Store: 5433, Management: 15673} + p := FoundationPorts{Store: 5433, Management: 15673, AMQP: 5773} if _, err := RewritePorts(&r, p, ""); err != nil { t.Fatal(err) } @@ -101,7 +101,9 @@ func TestAnAdoptedBundleLoadsNoDroppingTableAndExactlyTheGuard(t *testing.T) { if len(guards) != 1 { t.Fatalf("%d guard table(s)", len(guards)) } - if !strings.Contains(guards[0].Content, "tcp dport { 5433, 15673 } drop") { + // The store's, the broker's plaintext and its management port: each one the filter admits + // from the private network only (novox/hq ADR 0103). + if !strings.Contains(guards[0].Content, "tcp dport { 5433, 5773, 15673 } drop") { t.Errorf("the guard does not refuse this node's ports: %s", guards[0].Content) } if strings.Count(guards[0].Content, "accept") != 1 || !strings.Contains(guards[0].Content, "policy accept") { From 9033e3da988bba331c33acc1574e2e976c7a1ff7 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:01:49 +0200 Subject: [PATCH 14/52] Retire the found firewall only on a converged declaration from the mesh, never on a carried apply (hq ADR 0100) --- internal/apply/apply.go | 2 +- internal/apply/opening.go | 10 +++++++--- internal/apply/opening_test.go | 22 ++++++++++++++++++++++ 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/internal/apply/apply.go b/internal/apply/apply.go index f7e5d62..8bd30a2 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -318,7 +318,7 @@ func ApplyKeeping( // A converged node whose found firewall was in force retires it only now, once everything — // the mesh's derived filter among it — applied cleanly (novox/hq ADR 0100). if len(failures) == 0 { - if err := retireFirewall(ctx, d, &known, run, log); err != nil { + if err := retireFirewall(ctx, d, origin, &known, run, log); err != nil { return report, known, &Error{Resource: "the firewall found on this machine", Err: err, Done: report} } } diff --git a/internal/apply/opening.go b/internal/apply/opening.go index 3adef33..ff6ee1c 100644 --- a/internal/apply/opening.go +++ b/internal/apply/opening.go @@ -57,10 +57,14 @@ func foundFirewall(ctx context.Context, d *declaration.Declaration, known *store // which is when the mesh's derived filter has taken its place. Disabled, never flushed: its // configuration stays on disk for a return to adopted, and the container runtime's rules are not // its to take. -func retireFirewall(ctx context.Context, d *declaration.Declaration, known *store.State, run Runner, - log func(string)) error { +// +// Only a declaration from the mesh converges a node. A carried bundle never says a node is adopted +// — it cannot — so its silence is not the controller's word that the node was converged, and an +// adopted node re-applying its bundle keeps the firewall it was found with. +func retireFirewall(ctx context.Context, d *declaration.Declaration, origin string, known *store.State, + run Runner, log func(string)) error { rec := known.Firewall - if d.Adoption != nil || rec == nil || rec.Kind != string(firewall.UFW) || !rec.WasActive || + if origin != store.OriginDeclared || d.Adoption != nil || rec == nil || rec.Kind != string(firewall.UFW) || !rec.WasActive || rec.DisabledByMesh { return nil } diff --git a/internal/apply/opening_test.go b/internal/apply/opening_test.go index d51bda5..7192925 100644 --- a/internal/apply/opening_test.go +++ b/internal/apply/opening_test.go @@ -202,3 +202,25 @@ func TestAnOpeningOnAConvergedNodeIsRefused(t *testing.T) { t.Error("an opening was accepted on a node the declaration does not say is adopted") } } + +func TestACarriedApplyOnAnAdoptedNodeLeavesItsFirewallInForce(t *testing.T) { + // The bundle, re-applied by the installer or the one-shot CLI, never says a node is adopted. + // That is not the controller converging it, so ufw must stay enabled (novox/hq ADR 0100). + dir := t.TempDir() + u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}} + _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) + if err != nil { + t.Fatal(err) + } + u.asked = nil + carried := parse(t, `{"declaration":1,"resources":[`+withConf(filepath.Join(dir, "bundle"))+`]}`) + _, state, err = ApplyKeeping(context.Background(), archHost(t), carried, state, store.OriginCarried, + u.run, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if !u.active || state.Firewall.DisabledByMesh || u.index("ufw disable") >= 0 { + t.Fatalf("a carried apply retired the found firewall: active %v, record %+v, asked %v", + u.active, state.Firewall, u.asked) + } +} From 588ab71d147df20880d388fbfc2c219bfb41f114 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:02:46 +0200 Subject: [PATCH 15/52] Remove an adopted node's guard and openings last on the flip, and keep them if anything failed (hq ADR 0103) --- internal/apply/apply.go | 31 ++++++++++++++-- internal/apply/opening_test.go | 67 +++++++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 5 deletions(-) diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 8bd30a2..22cdb3b 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -114,7 +114,9 @@ func (e *Error) Unwrap() error { return e.Err } // Removal happens FIRST, and the order is not arbitrary. A resource that leaves a declaration // while another arrives at the same path is an ordinary rename: removing afterwards would // delete the file that had just been written. Removing first risks losing the old state if the -// apply then fails — a recovery concern, where the other is a correctness one. +// apply then fails — a recovery concern, where the other is a correctness one. The one exception +// is what protects an adopted node, the openings and the guard: that goes last, and only when +// everything else applied (novox/hq ADR 0103). func Apply( ctx context.Context, sys system.System, @@ -159,7 +161,7 @@ func ApplyKeeping( return report, known, &Error{Resource: "the firewall found on this machine", Err: err, Done: report} } - for _, orphan := range known.Orphans(declared, origin) { + removeOrphan := func(orphan store.Applied) error { var action, detail string var err error if declaration.Type(orphan.Type) == declaration.TypeOpening { @@ -168,7 +170,7 @@ func ApplyKeeping( action, detail, err = remove(ctx, sys, orphan, run) } if err != nil { - return report, known, &Error{Resource: orphan.ID, Err: err, Done: report} + return &Error{Resource: orphan.ID, Err: err, Done: report} } known.Forget(orphan.ID) report.Outcomes = append(report.Outcomes, Outcome{ @@ -176,6 +178,24 @@ func ApplyKeeping( Action: action, Detail: detail, }) log(fmt.Sprintf(" %s %s (%s)", action, orphan.ID, orphan.Target)) + return nil + } + + // **What protects an adopted node goes last** (novox/hq ADR 0103). The openings and the guard + // are what keep the mesh reachable through the found firewall and the store unreachable from + // outside. When a node is converged they leave the declaration, and removing them first would + // leave the store open from the moment the guard stops until the derived filter loads — and + // for ever, if the filter then fails. So they are removed only once everything else applied + // and the found firewall is retired; if anything failed, they stay, recorded, for the next try. + var protecting []store.Applied + for _, orphan := range known.Orphans(declared, origin) { + if strings.HasPrefix(orphan.ID, declaration.AdoptionPrefix) { + protecting = append(protecting, orphan) + continue + } + if err := removeOrphan(orphan); err != nil { + return report, known, err + } } // What moved in this apply, so a service that must reflect a file can be told the file @@ -321,6 +341,11 @@ func ApplyKeeping( if err := retireFirewall(ctx, d, origin, &known, run, log); err != nil { return report, known, &Error{Resource: "the firewall found on this machine", Err: err, Done: report} } + for _, orphan := range protecting { + if err := removeOrphan(orphan); err != nil { + return report, known, err + } + } } if len(failures) > 0 { diff --git a/internal/apply/opening_test.go b/internal/apply/opening_test.go index 7192925..f4dc76d 100644 --- a/internal/apply/opening_test.go +++ b/internal/apply/opening_test.go @@ -151,8 +151,10 @@ func TestConvergingRetiresTheFoundFirewallAndReturningRestoresIt(t *testing.T) { if len(u.rules) != 1 || u.rules[0] != "allow 22/tcp" { t.Errorf("converged: the operator's rules were touched, or the mesh's left: %v", u.rules) } - if del, dis := u.index("ufw delete"), u.index("ufw disable"); del < 0 || dis < del { - t.Errorf("converged: the opening was not removed before ufw was disabled: %v", u.asked) + // What protected the adopted node goes last: after the derived filter applied and ufw was + // retired (novox/hq ADR 0103). + if del, dis := u.index("ufw delete"), u.index("ufw disable"); dis < 0 || del < dis { + t.Errorf("converged: the opening was removed before ufw was retired: %v", u.asked) } for _, a := range u.asked { if strings.Contains(a, "reset") { @@ -224,3 +226,64 @@ func TestACarriedApplyOnAnAdoptedNodeLeavesItsFirewallInForce(t *testing.T) { u.active, state.Firewall, u.asked) } } + +func TestAFlipThatFailsKeepsTheGuardAndTheOpenings(t *testing.T) { + // Converging a node removes its openings and its guard only once everything else applied and + // the found firewall is retired. A flip that fails part-way keeps them, so the store is never + // left unguarded behind a filter that did not load (novox/hq ADR 0103). + dir := t.TempDir() + guard := filepath.Join(dir, "guard.nft") + guardFile := `{"id":"adoption.guard","type":"file","path":"` + guard + `","content":"table inet mesh_guard {}\n"}` + u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}} + _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+guardFile+","+withConf(dir)), + store.State{}, u.run) + if err != nil { + t.Fatal(err) + } + + // The derived filter cannot be written: its path is under a file. + blocked := filepath.Join(dir, "not-a-directory") + if err := os.WriteFile(blocked, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + filter := `{"id":"nftables.config","type":"file","path":"` + filepath.Join(blocked, "nftables.conf") + `","content":"table inet mesh {}\n"}` + converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`,`+filter+`]}`) + u.asked = nil + _, state, err = applyWith(t, converged, state, u.run) + if err == nil { + t.Fatal("the failing flip reported success") + } + if !u.active || u.index("ufw disable") >= 0 { + t.Errorf("the found firewall was retired by a flip that failed: %v", u.asked) + } + if len(u.rules) != 2 || u.index("ufw delete") >= 0 { + t.Errorf("the opening was removed by a flip that failed: %v", u.rules) + } + if _, statErr := os.Stat(guard); statErr != nil { + t.Errorf("the guard was removed by a flip that failed: %v", statErr) + } + for _, id := range []string{"adoption.guard", "adoption.opening-tcp-5671-incoming"} { + if _, ok := state.Find(id); !ok { + t.Errorf("%s was forgotten, so the next flip would never remove it", id) + } + } + + // Fixed, the next flip completes: filter, retire, and only then the guard and the openings. + if err := os.Remove(blocked); err != nil { + t.Fatal(err) + } + u.asked = nil + _, state, err = applyWith(t, converged, state, u.run) + if err != nil { + t.Fatal(err) + } + if u.active || len(u.rules) != 1 { + t.Errorf("the completed flip left ufw active %v, rules %v", u.active, u.rules) + } + if _, statErr := os.Stat(guard); !errors.Is(statErr, os.ErrNotExist) { + t.Errorf("the guard outlived the completed flip: %v", statErr) + } + if _, ok := state.Find("adoption.guard"); ok { + t.Error("the guard is still recorded after the completed flip") + } +} From 8e2f75454d75874698b099f79c67e746cc394770 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:03:30 +0200 Subject: [PATCH 16/52] Put back the forward policy ufw disable opens when the found firewall is retired, as measured on a lab machine (hq ADR 0100) --- internal/firewall/firewall.go | 52 +++++++- internal/firewall/firewall_test.go | 76 +++++++++++ .../testdata/ufw-disable-iptables-after.txt | 55 ++++++++ .../testdata/ufw-disable-iptables-before.txt | 119 ++++++++++++++++++ 4 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 internal/firewall/testdata/ufw-disable-iptables-after.txt create mode 100644 internal/firewall/testdata/ufw-disable-iptables-before.txt diff --git a/internal/firewall/firewall.go b/internal/firewall/firewall.go index f730afe..43faf6d 100644 --- a/internal/firewall/firewall.go +++ b/internal/firewall/firewall.go @@ -419,11 +419,61 @@ func Enable(ctx context.Context, run Runner) error { // Disable retires ufw without flushing it: its configuration stays on disk, and the container // runtime's rules are not its to remove. +// +// **Nor is the forward policy ufw's to open.** Measured on a lab machine running the container +// runtime with a published port (testdata/ufw-disable-iptables-before.txt and -after.txt): +// `ufw disable` sets every built-in chain's policy to accept, the forward chain's among them. The +// runtime had set that one to drop when it turned forwarding on, and it does not set it again while +// forwarding stays on — not even on a restart. Left so, a retired ufw turns the machine into a +// router for anyone who can reach it. So each family's forward policy is read before, and one that +// was drop is put back and read back. func Disable(ctx context.Context, run Runner) error { + type family struct{ tool, policy string } + var before []family + for _, tool := range []string{"iptables", "ip6tables"} { + if policy, ok := forwardPolicy(ctx, run, tool); ok { + before = append(before, family{tool, policy}) + } + } if _, err := run(ctx, "ufw", "disable"); err != nil { return fmt.Errorf("disabling ufw: %w", err) } - return expectActive(ctx, run, false) + if err := expectActive(ctx, run, false); err != nil { + return err + } + for _, f := range before { + if f.policy != "DROP" { + continue + } + if now, ok := forwardPolicy(ctx, run, f.tool); ok && now == "DROP" { + continue + } + if _, err := run(ctx, f.tool, "-P", "FORWARD", "DROP"); err != nil { + return fmt.Errorf("ufw is disabled, and %s's forward policy, which was drop, could not be put back: %w", + f.tool, err) + } + if now, ok := forwardPolicy(ctx, run, f.tool); !ok || now != "DROP" { + return fmt.Errorf("ufw is disabled, and %s's forward policy was put back to drop and reads %q", + f.tool, now) + } + } + return nil +} + +// forwardPolicy reads the forward chain's policy the way iptables prints it: "-P FORWARD DROP". +// Not ok when the tool is absent or says nothing readable. +func forwardPolicy(ctx context.Context, run Runner, tool string) (string, bool) { + out, err := run(ctx, tool, "-S", "FORWARD") + if err != nil { + return "", false + } + for _, line := range strings.Split(out, "\n") { + f := strings.Fields(line) + if len(f) == 3 && f[0] == "-P" && f[1] == "FORWARD" { + return f[2], true + } + } + return "", false } func expectActive(ctx context.Context, run Runner, want bool) error { diff --git a/internal/firewall/firewall_test.go b/internal/firewall/firewall_test.go index c810a03..4605654 100644 --- a/internal/firewall/firewall_test.go +++ b/internal/firewall/firewall_test.go @@ -114,6 +114,41 @@ type fakeUFW struct { ruleset string firewalld bool asked []string + + // iptablesActive and iptablesInactive are what `iptables -S` prints with ufw active and + // after it is disabled; empty is a machine without iptables. forward is a policy set since. + iptablesActive, iptablesInactive string + forward string +} + +// iptables answers `iptables -S FORWARD` from the captured output for ufw's state, and records +// a forward policy set with -P. +func (f *fakeUFW) iptables(name string, args []string) (string, error) { + if f.iptablesActive == "" { + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + } + if name == "ip6tables" { + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + } + if len(args) == 3 && args[0] == "-P" && args[1] == "FORWARD" { + f.forward = args[2] + return "", nil + } + captured := f.iptablesInactive + if f.active { + captured = f.iptablesActive + } + var out []string + for _, line := range strings.Split(captured, "\n") { + fields := strings.Fields(line) + if len(fields) >= 2 && fields[1] == "FORWARD" { + if fields[0] == "-P" && f.forward != "" && !f.active { + line = "-P FORWARD " + f.forward + } + out = append(out, line) + } + } + return strings.Join(out, "\n") + "\n", nil } func canonical(args []string) string { @@ -155,6 +190,8 @@ func (f *fakeUFW) run(_ context.Context, name string, args ...string) (string, e return f.ruleset, nil case "iptables-legacy", "ip6tables-legacy": return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + case "iptables", "ip6tables": + return f.iptables(name, args) case "ufw": default: return "", fmt.Errorf("unexpected %s", name) @@ -179,6 +216,7 @@ func (f *fakeUFW) run(_ context.Context, name string, args ...string) (string, e return "Firewall is active and enabled on system startup\n", nil case args[0] == "disable": f.active = false + f.forward = "" return "Firewall stopped and disabled on system startup\n", nil case args[0] == "delete" && len(args) > 1 && args[1] == "route": // As the real ufw answers it (captured in testdata/ufw-delete.txt): a route rule is @@ -424,3 +462,41 @@ func TestARealUfwRulesetIsUfw(t *testing.T) { t.Error("ufw's drop chains, with ufw not known to be active, read as refusing nothing") } } + +func TestRetiringUfwKeepsTheMachineFromRoutingForOthers(t *testing.T) { + // Captured on a lab machine running the container runtime with a published port: ufw active, + // then `ufw disable`. Disabling set the forward policy the runtime had set to drop to accept. + before, err := os.ReadFile("testdata/ufw-disable-iptables-before.txt") + if err != nil { + t.Fatal(err) + } + after, err := os.ReadFile("testdata/ufw-disable-iptables-after.txt") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(before), "-P FORWARD DROP") || !strings.Contains(string(after), "-P FORWARD ACCEPT") { + t.Fatal("the captures no longer show ufw disable opening the forward policy") + } + f := &fakeUFW{installed: true, active: true, iptablesActive: string(before), iptablesInactive: string(after)} + if err := Disable(context.Background(), f.run); err != nil { + t.Fatal(err) + } + if f.active { + t.Fatal("ufw is still active") + } + if f.forward != "DROP" { + t.Errorf("the forward policy was left open after ufw was retired: %v", f.asked) + } + for _, a := range f.asked { + if strings.Contains(a, "-F") || strings.Contains(a, "flush") || strings.Contains(a, "reset") { + t.Errorf("retiring ufw flushed something: %s", a) + } + } +} + +func TestRetiringUfwOnAMachineWithoutIptablesStillRetiresIt(t *testing.T) { + f := &fakeUFW{installed: true, active: true} + if err := Disable(context.Background(), f.run); err != nil || f.active { + t.Fatalf("disable: %v, active %v", err, f.active) + } +} diff --git a/internal/firewall/testdata/ufw-disable-iptables-after.txt b/internal/firewall/testdata/ufw-disable-iptables-after.txt new file mode 100644 index 0000000..22dbc77 --- /dev/null +++ b/internal/firewall/testdata/ufw-disable-iptables-after.txt @@ -0,0 +1,55 @@ +-P INPUT ACCEPT +-P FORWARD ACCEPT +-P OUTPUT ACCEPT +-N DOCKER +-N DOCKER-BRIDGE +-N DOCKER-CT +-N DOCKER-FORWARD +-N DOCKER-INTERNAL +-N DOCKER-USER +-N ufw-after-forward +-N ufw-after-input +-N ufw-after-logging-forward +-N ufw-after-logging-input +-N ufw-after-logging-output +-N ufw-after-output +-N ufw-before-forward +-N ufw-before-input +-N ufw-before-logging-forward +-N ufw-before-logging-input +-N ufw-before-logging-output +-N ufw-before-output +-N ufw-reject-forward +-N ufw-reject-input +-N ufw-reject-output +-N ufw-track-forward +-N ufw-track-input +-N ufw-track-output +-A INPUT -j ufw-before-logging-input +-A INPUT -j ufw-before-input +-A INPUT -j ufw-after-input +-A INPUT -j ufw-after-logging-input +-A INPUT -j ufw-reject-input +-A INPUT -j ufw-track-input +-A FORWARD -j DOCKER-USER +-A FORWARD -j DOCKER-FORWARD +-A FORWARD -j ufw-before-logging-forward +-A FORWARD -j ufw-before-forward +-A FORWARD -j ufw-after-forward +-A FORWARD -j ufw-after-logging-forward +-A FORWARD -j ufw-reject-forward +-A FORWARD -j ufw-track-forward +-A OUTPUT -j ufw-before-logging-output +-A OUTPUT -j ufw-before-output +-A OUTPUT -j ufw-after-output +-A OUTPUT -j ufw-after-logging-output +-A OUTPUT -j ufw-reject-output +-A OUTPUT -j ufw-track-output +-A DOCKER -d 172.17.0.2/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 80 -j ACCEPT +-A DOCKER ! -i docker0 -o docker0 -j DROP +-A DOCKER-BRIDGE -o docker0 -j DOCKER +-A DOCKER-CT -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A DOCKER-FORWARD -j DOCKER-CT +-A DOCKER-FORWARD -j DOCKER-INTERNAL +-A DOCKER-FORWARD -j DOCKER-BRIDGE +-A DOCKER-FORWARD -i docker0 -j ACCEPT diff --git a/internal/firewall/testdata/ufw-disable-iptables-before.txt b/internal/firewall/testdata/ufw-disable-iptables-before.txt new file mode 100644 index 0000000..4e18c44 --- /dev/null +++ b/internal/firewall/testdata/ufw-disable-iptables-before.txt @@ -0,0 +1,119 @@ +-P INPUT DROP +-P FORWARD DROP +-P OUTPUT ACCEPT +-N DOCKER +-N DOCKER-BRIDGE +-N DOCKER-CT +-N DOCKER-FORWARD +-N DOCKER-INTERNAL +-N DOCKER-USER +-N ufw-after-forward +-N ufw-after-input +-N ufw-after-logging-forward +-N ufw-after-logging-input +-N ufw-after-logging-output +-N ufw-after-output +-N ufw-before-forward +-N ufw-before-input +-N ufw-before-logging-forward +-N ufw-before-logging-input +-N ufw-before-logging-output +-N ufw-before-output +-N ufw-logging-allow +-N ufw-logging-deny +-N ufw-not-local +-N ufw-reject-forward +-N ufw-reject-input +-N ufw-reject-output +-N ufw-skip-to-policy-forward +-N ufw-skip-to-policy-input +-N ufw-skip-to-policy-output +-N ufw-track-forward +-N ufw-track-input +-N ufw-track-output +-N ufw-user-forward +-N ufw-user-input +-N ufw-user-limit +-N ufw-user-limit-accept +-N ufw-user-logging-forward +-N ufw-user-logging-input +-N ufw-user-logging-output +-N ufw-user-output +-A INPUT -j ufw-before-logging-input +-A INPUT -j ufw-before-input +-A INPUT -j ufw-after-input +-A INPUT -j ufw-after-logging-input +-A INPUT -j ufw-reject-input +-A INPUT -j ufw-track-input +-A FORWARD -j DOCKER-USER +-A FORWARD -j DOCKER-FORWARD +-A FORWARD -j ufw-before-logging-forward +-A FORWARD -j ufw-before-forward +-A FORWARD -j ufw-after-forward +-A FORWARD -j ufw-after-logging-forward +-A FORWARD -j ufw-reject-forward +-A FORWARD -j ufw-track-forward +-A OUTPUT -j ufw-before-logging-output +-A OUTPUT -j ufw-before-output +-A OUTPUT -j ufw-after-output +-A OUTPUT -j ufw-after-logging-output +-A OUTPUT -j ufw-reject-output +-A OUTPUT -j ufw-track-output +-A DOCKER -d 172.17.0.2/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 80 -j ACCEPT +-A DOCKER ! -i docker0 -o docker0 -j DROP +-A DOCKER-BRIDGE -o docker0 -j DOCKER +-A DOCKER-CT -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A DOCKER-FORWARD -j DOCKER-CT +-A DOCKER-FORWARD -j DOCKER-INTERNAL +-A DOCKER-FORWARD -j DOCKER-BRIDGE +-A DOCKER-FORWARD -i docker0 -j ACCEPT +-A ufw-after-input -p udp -m udp --dport 137 -j ufw-skip-to-policy-input +-A ufw-after-input -p udp -m udp --dport 138 -j ufw-skip-to-policy-input +-A ufw-after-input -p tcp -m tcp --dport 139 -j ufw-skip-to-policy-input +-A ufw-after-input -p tcp -m tcp --dport 445 -j ufw-skip-to-policy-input +-A ufw-after-input -p udp -m udp --dport 67 -j ufw-skip-to-policy-input +-A ufw-after-input -p udp -m udp --dport 68 -j ufw-skip-to-policy-input +-A ufw-after-input -m addrtype --dst-type BROADCAST -j ufw-skip-to-policy-input +-A ufw-after-logging-forward -m limit --limit 3/min --limit-burst 10 -j LOG --log-prefix "[UFW BLOCK] " +-A ufw-after-logging-input -m limit --limit 3/min --limit-burst 10 -j LOG --log-prefix "[UFW BLOCK] " +-A ufw-before-forward -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A ufw-before-forward -p icmp -m icmp --icmp-type 3 -j ACCEPT +-A ufw-before-forward -p icmp -m icmp --icmp-type 11 -j ACCEPT +-A ufw-before-forward -p icmp -m icmp --icmp-type 12 -j ACCEPT +-A ufw-before-forward -p icmp -m icmp --icmp-type 8 -j ACCEPT +-A ufw-before-forward -j ufw-user-forward +-A ufw-before-input -i lo -j ACCEPT +-A ufw-before-input -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A ufw-before-input -m conntrack --ctstate INVALID -j ufw-logging-deny +-A ufw-before-input -m conntrack --ctstate INVALID -j DROP +-A ufw-before-input -p icmp -m icmp --icmp-type 3 -j ACCEPT +-A ufw-before-input -p icmp -m icmp --icmp-type 11 -j ACCEPT +-A ufw-before-input -p icmp -m icmp --icmp-type 12 -j ACCEPT +-A ufw-before-input -p icmp -m icmp --icmp-type 8 -j ACCEPT +-A ufw-before-input -p udp -m udp --sport 67 --dport 68 -j ACCEPT +-A ufw-before-input -j ufw-not-local +-A ufw-before-input -d 224.0.0.251/32 -p udp -m udp --dport 5353 -j ACCEPT +-A ufw-before-input -d 239.255.255.250/32 -p udp -m udp --dport 1900 -j ACCEPT +-A ufw-before-input -j ufw-user-input +-A ufw-before-output -o lo -j ACCEPT +-A ufw-before-output -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A ufw-before-output -j ufw-user-output +-A ufw-logging-allow -m limit --limit 3/min --limit-burst 10 -j LOG --log-prefix "[UFW ALLOW] " +-A ufw-logging-deny -m conntrack --ctstate INVALID -m limit --limit 3/min --limit-burst 10 -j RETURN +-A ufw-logging-deny -m limit --limit 3/min --limit-burst 10 -j LOG --log-prefix "[UFW BLOCK] " +-A ufw-not-local -m addrtype --dst-type LOCAL -j RETURN +-A ufw-not-local -m addrtype --dst-type MULTICAST -j RETURN +-A ufw-not-local -m addrtype --dst-type BROADCAST -j RETURN +-A ufw-not-local -m limit --limit 3/min --limit-burst 10 -j ufw-logging-deny +-A ufw-not-local -j DROP +-A ufw-skip-to-policy-forward -j DROP +-A ufw-skip-to-policy-input -j DROP +-A ufw-skip-to-policy-output -j ACCEPT +-A ufw-track-output -p tcp -m conntrack --ctstate NEW -j ACCEPT +-A ufw-track-output -p udp -m conntrack --ctstate NEW -j ACCEPT +-A ufw-user-input -p tcp -m tcp --dport 22 -j ACCEPT +-A ufw-user-input -p tcp -m tcp --dport 5671 -j ACCEPT +-A ufw-user-input -i mesh0 -p tcp -m tcp --dport 5432 -j ACCEPT +-A ufw-user-limit -m limit --limit 3/min -j LOG --log-prefix "[UFW LIMIT BLOCK] " +-A ufw-user-limit -j REJECT --reject-with icmp-port-unreachable +-A ufw-user-limit-accept -j ACCEPT From da65f84c4569bd2ef8e40876f0ebde7df87bfcac Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:04:48 +0200 Subject: [PATCH 17/52] Read fail2ban's bans as no firewall, and an iptables-nft reject as a refusal, from rulesets captured on a lab machine (hq ADR 0100) --- internal/firewall/firewall.go | 189 ++++++++++++++++-- internal/firewall/firewall_test.go | 50 +++++ .../firewall/testdata/fail2ban-iptables-S.txt | 22 ++ .../firewall/testdata/fail2ban-iptables.nft | 103 ++++++++++ .../firewall/testdata/fail2ban-nftables.nft | 105 ++++++++++ 5 files changed, 448 insertions(+), 21 deletions(-) create mode 100644 internal/firewall/testdata/fail2ban-iptables-S.txt create mode 100644 internal/firewall/testdata/fail2ban-iptables.nft create mode 100644 internal/firewall/testdata/fail2ban-nftables.nft diff --git a/internal/firewall/firewall.go b/internal/firewall/firewall.go index 43faf6d..d2630c3 100644 --- a/internal/firewall/firewall.go +++ b/internal/firewall/firewall.go @@ -109,16 +109,31 @@ func statusActive(out string) bool { // drop or reject, or a base chain whose policy drops — and that is neither the mesh's own nor the // container runtime's. With ufw active, the tables iptables-nft manages are ufw's and the runtime's // and are not counted. +// +// **A ban is not a firewall.** fail2ban refuses the sources it banned and passes everything else; +// captured on a lab machine with both of its backends (testdata/fail2ban-nftables.nft, +// testdata/fail2ban-iptables.nft). The mesh opens nothing through it and it closes nothing the +// mesh needs, so a refusal that names the sources it refuses, in a table or a chain that accepts +// nothing and is entered only from chains whose policy accepts, is not counted. func Refusing(ruleset string, ufwActive bool) []string { - var refusing []string + type rule struct{ table, chain, line string } + type chainOf struct { + base, dropping, accepts bool + policyLine string + jumpedFrom []string + } + chains := map[string]*chainOf{} // by "table\x00chain" + tableAccepts := map[string]bool{} + var tables []string + var refusals []rule managed := map[string]bool{} var table, chain string - counted := map[string]bool{} - note := func() { - if !counted[table] { - counted[table] = true - refusing = append(refusing, "table "+table) + get := func(t, c string) *chainOf { + k := t + "\x00" + c + if chains[k] == nil { + chains[k] = &chainOf{} } + return chains[k] } for _, raw := range strings.Split(ruleset, "\n") { line := strings.TrimSpace(raw) @@ -131,34 +146,108 @@ func Refusing(ruleset string, ufwActive bool) []string { case strings.HasPrefix(line, "table "): table = strings.TrimSuffix(strings.TrimSpace(strings.TrimPrefix(line, "table ")), "{") table = strings.TrimSpace(table) + tables = append(tables, table) chain = "" continue case strings.HasPrefix(line, "chain "): chain = strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(line, "chain "), "{")) + get(table, chain) + continue + case strings.HasPrefix(line, "set ") || strings.HasPrefix(line, "map ") || + strings.HasPrefix(line, "flowtable "): + chain = "" continue case line == "" || line == "}" || strings.HasPrefix(line, "#") || chain == "": continue } - if table == "inet mesh" || table == "inet mesh_guard" { - continue - } - iptables := managed[table] || iptablesTable(table) - if iptables && ufwActive { - continue - } + c := get(table, chain) if strings.HasPrefix(line, "type ") { - if strings.Contains(line, "policy drop") && !(iptables && runtimes(table, chain, line)) { - note() + c.base = true + c.policyLine = line + c.dropping = strings.Contains(line, "policy drop") + continue + } + for _, verb := range []string{"jump ", "goto "} { + if i := strings.Index(line, verb); i >= 0 { + target := strings.Fields(line[i+len(verb):]) + if len(target) > 0 { + get(table, target[0]).jumpedFrom = append(get(table, target[0]).jumpedFrom, chain) + } } + } + if accepts(line) { + c.accepts = true + tableAccepts[table] = true + } + if verdictRefuses(line) { + refusals = append(refusals, rule{table, chain, line}) + } + } + + skipped := func(table string) bool { + if table == "inet mesh" || table == "inet mesh_guard" { + return true + } + return (managed[table] || iptablesTable(table)) && ufwActive + } + // onlyBans is whether a refusal only refuses the sources it names: in a table that accepts + // nothing and whose base chains all accept by default, or in a chain that accepts nothing and + // is entered only from base chains that accept by default. + onlyBans := func(r rule) bool { + if !bansSources(r.line) { + return false + } + allAccepting := true + for k, c := range chains { + if strings.HasPrefix(k, r.table+"\x00") && c.base && !strings.Contains(c.policyLine, "policy accept") { + allAccepting = false + } + } + if !tableAccepts[r.table] && allAccepting { + return true + } + c := get(r.table, r.chain) + if c.base || c.accepts || len(c.jumpedFrom) == 0 { + return false + } + for _, from := range c.jumpedFrom { + caller := get(r.table, from) + if !caller.base || !strings.Contains(caller.policyLine, "policy accept") { + return false + } + } + return true + } + + counted := map[string]bool{} + for k, c := range chains { + t, name, _ := strings.Cut(k, "\x00") + if skipped(t) || !c.dropping { continue } - if !verdictRefuses(line) { + if (managed[t] || iptablesTable(t)) && runtimes(t, name, c.policyLine) { continue } - if iptables && runtimes(table, chain, line) { + counted[t] = true + } + for _, r := range refusals { + if skipped(r.table) || counted[r.table] { continue } - note() + if (managed[r.table] || iptablesTable(r.table)) && runtimes(r.table, r.chain, r.line) { + continue + } + if onlyBans(r) { + continue + } + counted[r.table] = true + } + var refusing []string + for _, t := range tables { + if counted[t] { + counted[t] = false + refusing = append(refusing, "table "+t) + } } return refusing } @@ -194,15 +283,73 @@ func runtimes(table, chain, line string) bool { return false } -var verdict = regexp.MustCompile(`(^|\s)(drop|reject)(\s|$)`) +// iptables-nft prints a REJECT target it cannot translate as `xt target "REJECT"`, measured in +// testdata/fail2ban-iptables.nft; a refusal written that way is a refusal too. +var verdict = regexp.MustCompile(`(^|\s)(drop|reject)(\s|$)|xt target "(DROP|REJECT)"`) func verdictRefuses(line string) bool { return verdict.MatchString(line) } +var acceptVerdict = regexp.MustCompile(`(^|\s)accept(\s|;|$)|xt target "ACCEPT"`) + +func accepts(line string) bool { + return acceptVerdict.MatchString(line) +} + +// bansSources is whether a refusal names the sources it refuses — a set or an address — rather +// than refusing everyone but some. +func bansSources(line string) bool { + f := strings.Fields(line) + for i, w := range f { + if (w == "saddr" || w == "-s") && i+1 < len(f) && f[i+1] != "!=" && !strings.HasPrefix(f[i+1], "!") { + return i == 0 || f[i-1] != "!" + } + } + return false +} + // RefusingLegacy names the chains of an `iptables-legacy -S` that refuse traffic outside the -// container runtime's own. +// container runtime's own. A ban — a refusal of the sources it names, in a chain that accepts +// nothing and is entered only from built-in chains whose policy accepts — is not counted, as in +// Refusing (testdata/fail2ban-iptables-S.txt). func RefusingLegacy(rules string) []string { + policy := map[string]string{} + accepting := map[string]bool{} + jumpedFrom := map[string][]string{} + for _, line := range strings.Split(rules, "\n") { + fields := strings.Fields(line) + if len(fields) < 3 { + continue + } + switch fields[0] { + case "-P": + policy[fields[1]] = fields[2] + case "-A": + for i, f := range fields { + if (f == "-j" || f == "-g") && i+1 < len(fields) { + switch fields[i+1] { + case "ACCEPT": + accepting[fields[1]] = true + case "DROP", "REJECT", "RETURN", "LOG": + default: + jumpedFrom[fields[i+1]] = append(jumpedFrom[fields[i+1]], fields[1]) + } + } + } + } + } + ban := func(chain, line string) bool { + if !bansSources(line) || accepting[chain] || len(jumpedFrom[chain]) == 0 { + return false + } + for _, from := range jumpedFrom[chain] { + if policy[from] != "ACCEPT" { + return false + } + } + return true + } var refusing []string seen := map[string]bool{} for _, line := range strings.Split(rules, "\n") { @@ -218,7 +365,7 @@ func RefusingLegacy(rules string) []string { case "-A": for i, f := range fields { if f == "-j" && i+1 < len(fields) && (fields[i+1] == "DROP" || fields[i+1] == "REJECT") { - refuses = !strings.HasPrefix(chain, "DOCKER") + refuses = !strings.HasPrefix(chain, "DOCKER") && !ban(chain, line) } } } diff --git a/internal/firewall/firewall_test.go b/internal/firewall/firewall_test.go index 4605654..878fc71 100644 --- a/internal/firewall/firewall_test.go +++ b/internal/firewall/firewall_test.go @@ -500,3 +500,53 @@ func TestRetiringUfwOnAMachineWithoutIptablesStillRetiresIt(t *testing.T) { t.Fatalf("disable: %v, active %v", err, f.active) } } + +// Captured on a lab machine with fail2ban banning one documentation address in its sshd jail, +// once with its nftables backend and once with its iptables backend (iptables-nft), ufw inactive. + +func captured(t *testing.T, name string) string { + t.Helper() + raw, err := os.ReadFile("testdata/" + name) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +func TestFail2bansBansAreNotAFirewall(t *testing.T) { + for _, name := range []string{"fail2ban-nftables.nft", "fail2ban-iptables.nft"} { + ruleset := captured(t, name) + if !strings.Contains(ruleset, "192.0.2.55") { + t.Fatalf("%s holds no ban", name) + } + if got := Refusing(ruleset, false); len(got) != 0 { + t.Errorf("%s: fail2ban's bans read as a firewall: %v", name, got) + } + kind, what, err := Detect(context.Background(), (&fakeUFW{ruleset: ruleset}).run) + if err != nil || kind != None { + t.Errorf("%s: a machine with only fail2ban detected as %s (%s) %v", name, kind, what, err) + } + } + if got := RefusingLegacy(captured(t, "fail2ban-iptables-S.txt")); len(got) != 0 { + t.Errorf("fail2ban's iptables bans read as a firewall: %v", got) + } +} + +func TestARefusalOfEveryoneButSomeIsStillAFirewall(t *testing.T) { + // A ban names the sources it refuses. A table that refuses every source but some, or every + // port but some, closes what the mesh would open, whatever its policy says. + for name, table := range map[string]string{ + "all but a range": "table inet own {\n\tchain input {\n\t\ttype filter hook input priority filter; policy accept;\n\t\tip saddr != 10.0.0.0/8 drop\n\t}\n}\n", + "all but ssh": "table inet own {\n\tchain input {\n\t\ttype filter hook input priority filter; policy accept;\n\t\ttcp dport != 22 drop\n\t}\n}\n", + "iptables reject": "# Warning: table ip filter is managed by iptables-nft, do not touch!\ntable ip filter {\n\tchain INPUT {\n\t\ttype filter hook input priority filter; policy accept;\n\t\tcounter packets 0 bytes 0 xt target \"REJECT\"\n\t}\n}\n", + "ban beside a dropping policy": "table inet own {\n\tchain input {\n\t\ttype filter hook input priority filter; policy drop;\n\t\tip saddr 192.0.2.9 drop\n\t}\n}\n", + } { + if got := Refusing(dockerOnly(t)+table, false); len(got) == 0 { + t.Errorf("%s: not counted as a firewall", name) + } + } + legacy := "-P INPUT ACCEPT\n-N own\n-A INPUT -j own\n-A own ! -s 10.0.0.0/8 -j DROP\n" + if got := RefusingLegacy(legacy); len(got) == 0 { + t.Error("a legacy refusal of all but a range was not counted") + } +} diff --git a/internal/firewall/testdata/fail2ban-iptables-S.txt b/internal/firewall/testdata/fail2ban-iptables-S.txt new file mode 100644 index 0000000..7b43c32 --- /dev/null +++ b/internal/firewall/testdata/fail2ban-iptables-S.txt @@ -0,0 +1,22 @@ +-P INPUT ACCEPT +-P FORWARD DROP +-P OUTPUT ACCEPT +-N DOCKER +-N DOCKER-BRIDGE +-N DOCKER-CT +-N DOCKER-FORWARD +-N DOCKER-INTERNAL +-N DOCKER-USER +-N f2b-sshd +-A INPUT -p tcp -m multiport --dports 22 -j f2b-sshd +-A FORWARD -j DOCKER-USER +-A FORWARD -j DOCKER-FORWARD +-A DOCKER ! -i docker0 -o docker0 -j DROP +-A DOCKER-BRIDGE -o docker0 -j DOCKER +-A DOCKER-CT -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A DOCKER-FORWARD -j DOCKER-CT +-A DOCKER-FORWARD -j DOCKER-INTERNAL +-A DOCKER-FORWARD -j DOCKER-BRIDGE +-A DOCKER-FORWARD -i docker0 -j ACCEPT +-A f2b-sshd -s 192.0.2.55/32 -j REJECT --reject-with icmp-port-unreachable +-A f2b-sshd -j RETURN diff --git a/internal/firewall/testdata/fail2ban-iptables.nft b/internal/firewall/testdata/fail2ban-iptables.nft new file mode 100644 index 0000000..bbe9f5a --- /dev/null +++ b/internal/firewall/testdata/fail2ban-iptables.nft @@ -0,0 +1,103 @@ +table ip nat { + chain DOCKER { + } + + chain PREROUTING { + type nat hook prerouting priority dstnat; policy accept; + xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } + + chain OUTPUT { + type nat hook output priority dstnat; policy accept; + ip daddr != 127.0.0.0/8 xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } + + chain POSTROUTING { + type nat hook postrouting priority srcnat; policy accept; + ip saddr 172.17.0.0/16 oifname != "docker0" counter packets 0 bytes 0 xt target "MASQUERADE" + } +} +table ip filter { + chain DOCKER { + iifname != "docker0" oifname "docker0" counter packets 0 bytes 0 drop + } + + chain DOCKER-FORWARD { + counter packets 0 bytes 0 jump DOCKER-CT + counter packets 0 bytes 0 jump DOCKER-INTERNAL + counter packets 0 bytes 0 jump DOCKER-BRIDGE + iifname "docker0" counter packets 0 bytes 0 accept + } + + chain DOCKER-BRIDGE { + oifname "docker0" counter packets 0 bytes 0 jump DOCKER + } + + chain DOCKER-CT { + oifname "docker0" xt match "conntrack" counter packets 0 bytes 0 accept + } + + chain DOCKER-INTERNAL { + } + + chain FORWARD { + type filter hook forward priority filter; policy drop; + counter packets 0 bytes 0 jump DOCKER-USER + counter packets 0 bytes 0 jump DOCKER-FORWARD + } + + chain DOCKER-USER { + } + + chain f2b-sshd { + ip saddr 192.0.2.55 counter packets 0 bytes 0 xt target "REJECT" + counter packets 0 bytes 0 return + } + + chain INPUT { + type filter hook input priority filter; policy accept; + ip protocol tcp xt match "multiport" counter packets 0 bytes 0 jump f2b-sshd + } +} +table ip6 nat { + chain DOCKER { + } + + chain PREROUTING { + type nat hook prerouting priority dstnat; policy accept; + xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } + + chain OUTPUT { + type nat hook output priority dstnat; policy accept; + ip6 daddr != ::1 xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } +} +table ip6 filter { + chain DOCKER { + } + + chain DOCKER-FORWARD { + counter packets 0 bytes 0 jump DOCKER-CT + counter packets 0 bytes 0 jump DOCKER-INTERNAL + counter packets 0 bytes 0 jump DOCKER-BRIDGE + } + + chain DOCKER-BRIDGE { + } + + chain DOCKER-CT { + } + + chain DOCKER-INTERNAL { + } + + chain FORWARD { + type filter hook forward priority filter; policy accept; + counter packets 0 bytes 0 jump DOCKER-USER + counter packets 0 bytes 0 jump DOCKER-FORWARD + } + + chain DOCKER-USER { + } +} diff --git a/internal/firewall/testdata/fail2ban-nftables.nft b/internal/firewall/testdata/fail2ban-nftables.nft new file mode 100644 index 0000000..cc38447 --- /dev/null +++ b/internal/firewall/testdata/fail2ban-nftables.nft @@ -0,0 +1,105 @@ +table ip nat { + chain DOCKER { + } + + chain PREROUTING { + type nat hook prerouting priority dstnat; policy accept; + xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } + + chain OUTPUT { + type nat hook output priority dstnat; policy accept; + ip daddr != 127.0.0.0/8 xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } + + chain POSTROUTING { + type nat hook postrouting priority srcnat; policy accept; + ip saddr 172.17.0.0/16 oifname != "docker0" counter packets 0 bytes 0 xt target "MASQUERADE" + } +} +table ip filter { + chain DOCKER { + iifname != "docker0" oifname "docker0" counter packets 0 bytes 0 drop + } + + chain DOCKER-FORWARD { + counter packets 0 bytes 0 jump DOCKER-CT + counter packets 0 bytes 0 jump DOCKER-INTERNAL + counter packets 0 bytes 0 jump DOCKER-BRIDGE + iifname "docker0" counter packets 0 bytes 0 accept + } + + chain DOCKER-BRIDGE { + oifname "docker0" counter packets 0 bytes 0 jump DOCKER + } + + chain DOCKER-CT { + oifname "docker0" xt match "conntrack" counter packets 0 bytes 0 accept + } + + chain DOCKER-INTERNAL { + } + + chain FORWARD { + type filter hook forward priority filter; policy drop; + counter packets 0 bytes 0 jump DOCKER-USER + counter packets 0 bytes 0 jump DOCKER-FORWARD + } + + chain DOCKER-USER { + } +} +table ip6 nat { + chain DOCKER { + } + + chain PREROUTING { + type nat hook prerouting priority dstnat; policy accept; + xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } + + chain OUTPUT { + type nat hook output priority dstnat; policy accept; + ip6 daddr != ::1 xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } +} +table ip6 filter { + chain DOCKER { + } + + chain DOCKER-FORWARD { + counter packets 0 bytes 0 jump DOCKER-CT + counter packets 0 bytes 0 jump DOCKER-INTERNAL + counter packets 0 bytes 0 jump DOCKER-BRIDGE + } + + chain DOCKER-BRIDGE { + } + + chain DOCKER-CT { + } + + chain DOCKER-INTERNAL { + } + + chain FORWARD { + type filter hook forward priority filter; policy accept; + counter packets 0 bytes 0 jump DOCKER-USER + counter packets 0 bytes 0 jump DOCKER-FORWARD + } + + chain DOCKER-USER { + } +} +table inet f2b-table { + set addr-set-sshd { + type ipv4_addr + flags interval + elements = { 192.0.2.55 } + } + + chain f2b-chain { + type filter hook input priority filter - 1; policy accept; + tcp dport 22 ip saddr @addr-set-sshd reject with icmp port-unreachable + } +} From 52e139d96faa69d7a1c88494250c5e0c778145c3 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:06:47 +0200 Subject: [PATCH 18/52] Add no opening a found ufw rule already answers, since ufw takes rules differing only in comment for one, as captured on a lab machine (hq ADR 0103) --- internal/apply/opening.go | 10 +- internal/apply/opening_test.go | 17 ++ internal/firewall/firewall.go | 222 +++++++++++++++++- internal/firewall/firewall_test.go | 159 +++++++++++-- .../firewall/testdata/ufw-comment-only.txt | 37 +++ internal/firewall/testdata/ufw-forms.txt | 16 ++ 6 files changed, 433 insertions(+), 28 deletions(-) create mode 100644 internal/firewall/testdata/ufw-comment-only.txt create mode 100644 internal/firewall/testdata/ufw-forms.txt diff --git a/internal/apply/opening.go b/internal/apply/opening.go index ff6ee1c..95012e9 100644 --- a/internal/apply/opening.go +++ b/internal/apply/opening.go @@ -85,12 +85,18 @@ func applyOpening(ctx context.Context, o *declaration.Opening, run Runner, kind out.Detail = "no firewall found; nothing filters this port" return out, nil case firewall.UFW: - action, err := firewall.Converge(ctx, run, o) + done, err := firewall.Converge(ctx, run, o) if err != nil { return out, err } - out.Action = action + out.Action = done.Action out.Detail = "through ufw, marked " + firewall.Mark(o) + if done.SatisfiedBy != "" { + // ufw would take a rule differing only in its comment for the same one, so the + // mesh's is not added beside it (novox/hq ADR 0103). + out.Detail = "satisfied by a rule found in ufw (" + done.SatisfiedBy + + "); the mesh added nothing and will remove nothing" + } return out, nil } return out, fmt.Errorf("no firewall is known for this node, so %s cannot be opened", o.Target()) diff --git a/internal/apply/opening_test.go b/internal/apply/opening_test.go index f4dc76d..c02aa79 100644 --- a/internal/apply/opening_test.go +++ b/internal/apply/opening_test.go @@ -287,3 +287,20 @@ func TestAFlipThatFailsKeepsTheGuardAndTheOpenings(t *testing.T) { t.Error("the guard is still recorded after the completed flip") } } + +func TestAnOpeningAFoundRuleAnswersIsReportedSatisfied(t *testing.T) { + // novox/hq ADR 0103: the mesh adds nothing beside a rule ufw would take for the same one. + dir := t.TempDir() + u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp", "allow 5671/tcp"}} + report, _, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) + if err != nil { + t.Fatal(err) + } + o := outcomeOf(report, "adoption.opening-tcp-5671-incoming") + if o.Action != "unchanged" || !strings.Contains(o.Detail, "satisfied by a rule found in ufw (allow 5671/tcp)") { + t.Errorf("the opening was not reported satisfied: %+v", o) + } + if len(u.rules) != 2 || u.index("ufw allow") >= 0 { + t.Errorf("a rule was added beside the found one: %v", u.rules) + } +} diff --git a/internal/firewall/firewall.go b/internal/firewall/firewall.go index d2630c3..fdc52db 100644 --- a/internal/firewall/firewall.go +++ b/internal/firewall/firewall.go @@ -119,8 +119,8 @@ func Refusing(ruleset string, ufwActive bool) []string { type rule struct{ table, chain, line string } type chainOf struct { base, dropping, accepts bool - policyLine string - jumpedFrom []string + policyLine string + jumpedFrom []string } chains := map[string]*chainOf{} // by "table\x00chain" tableAccepts := map[string]bool{} @@ -469,17 +469,195 @@ func words(rule string) []string { return out } +// A ufw rule, as `ufw show added` prints it or as it is given, reduced to what ufw compares. +// +// **ufw treats two rules that differ only in their comment as one rule.** Measured on a lab +// machine (testdata/ufw-comment-only.txt): adding `route allow proto tcp to any port 8080 comment +// 'mesh-host …'` beside an operator's `route allow 8080/tcp` answers "Rule updated", and the +// operator's rule now carries the mesh's mark — so removing the opening later would delete the +// operator's rule. The same holds for an incoming rule and for one with a comment of its own. +type ufwRule struct { + route bool + action, in, out string + from, fromPort, to string + port, proto, app string + comment string +} + +// parseRule reads a rule in either of ufw's forms — the short `allow 22/tcp` and the long `allow +// in on mesh0 to any port 5432 proto tcp` — into the fields ufw compares. Not ok for anything it +// does not recognise, which is then never taken to answer an opening. +func parseRule(rule string) (ufwRule, bool) { + w := words(rule) + r := ufwRule{from: "any", to: "any", comment: comment(rule)} + i := 0 + if i < len(w) && w[i] == "route" { + r.route = true + i++ + } + if i >= len(w) { + return r, false + } + switch w[i] { + case "allow", "deny", "reject", "limit": + r.action = w[i] + default: + return r, false + } + i++ + for i < len(w) && (w[i] == "in" || w[i] == "out") { + dir := w[i] + i++ + iface := "" + if i+1 < len(w) && w[i] == "on" { + iface = w[i+1] + i += 2 + } + if dir == "in" { + r.in = iface + } else { + r.out = iface + } + } + if i < len(w) && w[i] != "from" && w[i] != "to" && w[i] != "proto" && w[i] != "comment" && + w[i] != "app" && w[i] != "log" && w[i] != "log-all" { + // The short form: a port with its protocol, a bare port, or an application's name. + port, proto, hasProto := strings.Cut(w[i], "/") + if isPorts(port) { + r.port = port + if hasProto { + r.proto = proto + } + } else { + r.app = w[i] + } + i++ + } + for ; i < len(w); i++ { + next := func() string { + if i+1 < len(w) { + i++ + return w[i] + } + return "" + } + switch w[i] { + case "from": + r.from = next() + if i+1 < len(w) && w[i+1] == "port" { + i++ + r.fromPort = next() + } + case "to": + r.to = next() + if i+1 < len(w) && w[i+1] == "port" { + i++ + r.port = next() + } + case "port": + r.port = next() + case "proto": + r.proto = next() + case "app": + r.app = next() + case "comment": + next() + case "log", "log-all": + default: + return r, false + } + } + if r.proto == "any" { + r.proto = "" + } + return r, true +} + +func isPorts(s string) bool { + if s == "" { + return false + } + for _, c := range s { + if (c < '0' || c > '9') && c != ':' && c != ',' { + return false + } + } + return true +} + +// sameAs is whether ufw would take two rules for one — everything but the comment equal. +func (r ufwRule) sameAs(o ufwRule) bool { + r.comment, o.comment = "", "" + return r == o +} + +// admits is whether a rule already lets through what an opening says: the same path, allowed from +// any source to any address of this machine, on the opening's port and protocol — or on any +// protocol — and on any interface, or the private network's for an opening from it. +func (r ufwRule) admits(o *declaration.Opening) bool { + want, ok := parseRule(strings.Join(Rule(o), " ")) + if !ok || r.route != want.route || r.action != "allow" || r.out != "" || r.app != "" || + r.from != "any" || r.fromPort != "" || r.to != "any" { + return false + } + if r.proto != "" && r.proto != want.proto { + return false + } + if r.in != "" && r.in != want.in { + return false + } + return portsInclude(r.port, want.port) +} + +// portsInclude is whether a ufw port list — 80, 80,443, or 8000:8100 — names a port. +func portsInclude(list, port string) bool { + p, err := strconv.Atoi(port) + if err != nil { + return false + } + for _, part := range strings.Split(list, ",") { + lo, hi, isRange := strings.Cut(part, ":") + a, err := strconv.Atoi(lo) + if err != nil { + continue + } + b := a + if isRange { + if b, err = strconv.Atoi(hi); err != nil { + continue + } + } + if a <= p && p <= b { + return true + } + } + return false +} + +// Converged is what converging an opening did. SatisfiedBy names the rule already there that +// answers the opening, when one does; the mesh then adds nothing, and so will remove nothing. +type Converged struct { + Action string + SatisfiedBy string +} + // Converge makes one opening true in ufw: its marked rule present, and any rule marked for it that // no longer describes it deleted. Nothing unmarked is touched. The outcome is created, updated or // unchanged, read back from ufw rather than assumed. -func Converge(ctx context.Context, run Runner, o *declaration.Opening) (string, error) { +// +// **An opening a rule already answers is not added** (novox/hq ADR 0103). If ufw holds a rule not +// marked for this opening that already admits what it says — the operator's, or one the mesh +// added for another opening — the opening is satisfied by it: adding the mesh's would take that +// rule over if it differs only in its comment, and removing the opening would then delete it. +func Converge(ctx context.Context, run Runner, o *declaration.Opening) (Converged, error) { rules, err := added(ctx, run) if err != nil { - return "", err + return Converged{}, err } mark := Mark(o) present := false var stale []string + satisfiedBy := "" for _, rule := range rules { c := comment(rule) switch { @@ -487,25 +665,45 @@ func Converge(ctx context.Context, run Runner, o *declaration.Opening) (string, present = true case markedFor(c, o.ID): stale = append(stale, rule) + default: + if parsed, ok := parseRule(rule); ok && satisfiedBy == "" && parsed.admits(o) { + satisfiedBy = rule + } } } if present && len(stale) == 0 { - return "unchanged", nil + return Converged{Action: "unchanged"}, nil } for _, rule := range stale { if _, err := run(ctx, "ufw", deletion(rule)...); err != nil { - return "", fmt.Errorf("deleting the mesh's stale ufw rule %q: %w", rule, err) + return Converged{}, fmt.Errorf("deleting the mesh's stale ufw rule %q: %w", rule, err) } } + if !present && satisfiedBy != "" { + after, err := added(ctx, run) + if err != nil { + return Converged{}, err + } + for _, rule := range after { + if markedFor(comment(rule), o.ID) { + return Converged{}, fmt.Errorf("ufw still lists a stale rule marked for %s after deleting it", o.ID) + } + } + action := "unchanged" + if len(stale) > 0 { + action = "updated" + } + return Converged{Action: action, SatisfiedBy: satisfiedBy}, nil + } if !present { args := append(Rule(o), "comment", mark) if _, err := run(ctx, "ufw", args...); err != nil { - return "", fmt.Errorf("adding the ufw rule for %s: %w", o.Target(), err) + return Converged{}, fmt.Errorf("adding the ufw rule for %s: %w", o.Target(), err) } } after, err := added(ctx, run) if err != nil { - return "", err + return Converged{}, err } found, leftover := false, 0 for _, rule := range after { @@ -517,15 +715,15 @@ func Converge(ctx context.Context, run Runner, o *declaration.Opening) (string, } } if !found { - return "", fmt.Errorf("ufw was asked for %s and does not list it afterwards", o.Target()) + return Converged{}, fmt.Errorf("ufw was asked for %s and does not list it afterwards", o.Target()) } if leftover > 0 { - return "", fmt.Errorf("ufw still lists %d stale rule(s) marked for %s after deleting them", leftover, o.ID) + return Converged{}, fmt.Errorf("ufw still lists %d stale rule(s) marked for %s after deleting them", leftover, o.ID) } if len(stale) > 0 { - return "updated", nil + return Converged{Action: "updated"}, nil } - return "created", nil + return Converged{Action: "created"}, nil } // Remove deletes the rules marked for one opening, and nothing else. diff --git a/internal/firewall/firewall_test.go b/internal/firewall/firewall_test.go index 878fc71..33e88ad 100644 --- a/internal/firewall/firewall_test.go +++ b/internal/firewall/firewall_test.go @@ -151,14 +151,17 @@ func (f *fakeUFW) iptables(name string, args []string) (string, error) { return strings.Join(out, "\n") + "\n", nil } -func canonical(args []string) string { - var route, in, port, proto, comment string +// canonical is a rule the way ufw prints it back, as captured (testdata/ufw-comment-only.txt): the +// short form `allow 5671/tcp` for a rule on no interface, the long form `allow in on mesh0 to any +// port 5432 proto tcp` for one on an interface; the comment last. +func canonical(args []string) (rule, commentText string) { + var route, in, port, proto string for i := 0; i < len(args); i++ { switch args[i] { case "route": route = "route " case "in": - in = "in on " + args[i+2] + " " + in = args[i+2] i += 2 case "port": port = args[i+1] @@ -167,15 +170,14 @@ func canonical(args []string) string { proto = args[i+1] i++ case "comment": - comment = args[i+1] + commentText = args[i+1] i++ } } - line := route + "allow " + in + port + "/" + proto - if comment != "" { - line += " comment '" + comment + "'" + if in != "" { + return route + "allow in on " + in + " to any port " + port + " proto " + proto, commentText } - return line + return route + "allow " + port + "/" + proto, commentText } func (f *fakeUFW) run(_ context.Context, name string, args ...string) (string, error) { @@ -235,8 +237,21 @@ func (f *fakeUFW) run(_ context.Context, name string, args ...string) (string, e } return "", errors.New("Could not delete non-existent rule") default: - f.rules = append(f.rules, canonical(args)) - return "Rule added\n", nil + rule, note := canonical(args) + line := rule + if note != "" { + line += " comment '" + note + "'" + } + // As the real ufw does (testdata/ufw-comment-only.txt): a rule differing from one it holds + // only in its comment is the same rule, and its comment is replaced. + for i, r := range f.rules { + if bare, _, _ := strings.Cut(r, " comment '"); bare == rule { + f.rules[i] = line + return "Rule updated\nRule updated (v6)\n", nil + } + } + f.rules = append(f.rules, line) + return "Rule added\nRule added (v6)\n", nil } } @@ -276,14 +291,14 @@ func TestAnOpeningIsAddedOnceAndMarkedAsTheMeshs(t *testing.T) { o := opening("adoption.opening-tcp-5671-incoming", 5671, "everywhere", "incoming", 0) action, err := Converge(context.Background(), f.run, o) - if err != nil || action != "created" { + if err != nil || action.Action != "created" { t.Fatalf("first converge: %q %v", action, err) } if !strings.Contains(f.rules[1], "comment 'mesh-host adoption.opening-tcp-5671-incoming ") { t.Errorf("the rule is not marked as the mesh's: %v", f.rules) } action, err = Converge(context.Background(), f.run, o) - if err != nil || action != "unchanged" { + if err != nil || action.Action != "unchanged" { t.Fatalf("second converge: %q %v", action, err) } if f.added() != 1 { @@ -299,7 +314,7 @@ func TestAnOpeningLostToAReloadIsAddedAgain(t *testing.T) { } f.rules = nil // what a reload that lost the rule leaves action, err := Converge(context.Background(), f.run, o) - if err != nil || action != "created" || len(f.rules) != 1 { + if err != nil || action.Action != "created" || len(f.rules) != 1 { t.Fatalf("a lost opening was not put back: %q %v %v", action, err, f.rules) } } @@ -310,7 +325,7 @@ func TestAChangedOpeningReplacesOnlyItsOwnRule(t *testing.T) { t.Fatal(err) } action, err := Converge(context.Background(), f.run, opening("adoption.x", 5671, "mesh", "incoming", 0)) - if err != nil || action != "updated" { + if err != nil || action.Action != "updated" { t.Fatalf("%q %v", action, err) } if len(f.rules) != 3 || f.rules[0] != "allow 22/tcp" || f.rules[1] != "allow 8080/tcp comment 'someone else'" || @@ -550,3 +565,119 @@ func TestARefusalOfEveryoneButSomeIsStillAFirewall(t *testing.T) { t.Error("a legacy refusal of all but a range was not counted") } } + +// Defends novox/hq ADR 0103: an opening a found rule already answers is not added, because ufw +// takes two rules differing only in their comment for one (testdata/ufw-comment-only.txt). + +func TestUfwTakesTheMeshsRuleAndTheOperatorsForOne(t *testing.T) { + // The capture: each mesh rule answered "Rule updated" beside the operator's equivalent. + raw := captured(t, "ufw-comment-only.txt") + if strings.Count(raw, "Rule updated\n") != 3 { + t.Fatalf("the capture no longer shows ufw updating an equivalent rule:\n%s", raw) + } + for _, c := range []struct { + operators string + o *declaration.Opening + }{ + {"route allow 8080/tcp", opening("adoption.opening-tcp-8080-forwarded", 20001, "everywhere", "forwarded", 8080)}, + {"allow 5671/tcp", opening("adoption.opening-tcp-5671-incoming", 5671, "everywhere", "incoming", 0)}, + {"allow in on mesh0 to any port 5432 proto tcp comment 'operator note'", opening("adoption.opening-tcp-5432-incoming", 5432, "mesh", "incoming", 0)}, + } { + theirs, ok := parseRule(c.operators) + mine, ok2 := parseRule(strings.Join(Rule(c.o), " ") + " comment '" + Mark(c.o) + "'") + if !ok || !ok2 || !theirs.sameAs(mine) { + t.Errorf("%q and the mesh's %v are one rule to ufw, and read as two", c.operators, Rule(c.o)) + } + if !theirs.admits(c.o) { + t.Errorf("%q does not read as answering %s", c.operators, c.o.Target()) + } + } +} + +func TestEveryCapturedRuleFormIsRead(t *testing.T) { + want := map[string]string{ + "allow 22/tcp": "tcp 22 in= from=any", "allow 9200": " 9200 in= from=any", + "allow from 192.0.2.0/24 to any port 9300 proto tcp": "tcp 9300 in= from=192.0.2.0/24", + "allow in on eth0 to any port 9301 proto tcp": "tcp 9301 in=eth0 from=any", + "allow 9500:9510/tcp": "tcp 9500:9510 in= from=any", + "allow 80,443/tcp": "tcp 80,443 in= from=any", + "allow in on mesh0 to any port 5432 proto tcp": "tcp 5432 in=mesh0 from=any", + "route allow 8080/tcp": "tcp 8080 in= from=any", + "allow 9900/tcp": "tcp 9900 in= from=any", + } + rules, err := added(context.Background(), func(context.Context, string, ...string) (string, error) { + return captured(t, "ufw-forms.txt"), nil + }) + if err != nil || len(rules) != 15 { + t.Fatalf("read %d rules: %v", len(rules), err) + } + for _, rule := range rules { + r, ok := parseRule(rule) + if !ok { + t.Errorf("a rule ufw printed was not read: %q", rule) + continue + } + if w, listed := want[rule]; listed { + if got := r.proto + " " + r.port + " in=" + r.in + " from=" + r.from; got != w { + t.Errorf("%q read as %q, want %q", rule, got, w) + } + } + } +} + +func TestAnOpeningAFoundRuleAnswersIsNotAddedAndItsRemovalLeavesTheRule(t *testing.T) { + for _, c := range []struct { + name, operators string + o *declaration.Opening + }{ + {"forwarded, the same rule", "route allow 8080/tcp", opening("adoption.fwd", 20001, "everywhere", "forwarded", 8080)}, + {"incoming, the same rule", "allow 5671/tcp", opening("adoption.bus", 5671, "everywhere", "incoming", 0)}, + {"with a comment of its own", "allow in on mesh0 to any port 5432 proto tcp comment 'operator note'", opening("adoption.store", 5432, "mesh", "incoming", 0)}, + {"broader: from anywhere", "allow 5432/tcp", opening("adoption.store", 5432, "mesh", "incoming", 0)}, + {"broader: any protocol, a range", "allow 5000:5100", opening("adoption.registry", 5000, "everywhere", "incoming", 0)}, + } { + f := &fakeUFW{installed: true, active: true, rules: []string{"allow 22/tcp", c.operators}} + done, err := Converge(context.Background(), f.run, c.o) + if err != nil { + t.Fatalf("%s: %v", c.name, err) + } + if done.SatisfiedBy != c.operators || done.Action != "unchanged" || f.added() != 0 { + t.Errorf("%s: %+v, asked %v", c.name, done, f.asked) + } + if n, err := Remove(context.Background(), f.run, c.o.ID); err != nil || n != 0 { + t.Errorf("%s: removing the opening removed %d: %v", c.name, n, err) + } + if len(f.rules) != 2 || f.rules[1] != c.operators { + t.Errorf("%s: the operator's rule did not survive: %v", c.name, f.rules) + } + } +} + +func TestARuleThatDoesNotAnswerTheOpeningLeavesItToBeAdded(t *testing.T) { + for _, operators := range []string{ + "allow from 192.0.2.0/24 to any port 5671 proto tcp", // narrower: from one range + "allow in on eth0 to any port 5671 proto tcp", // narrower: one interface + "allow 5671/udp", // another protocol + "deny 5671/tcp", // refuses + "route allow 5671/tcp", // another path + "allow to 192.0.2.1 port 5671 proto tcp", // one address + } { + f := &fakeUFW{installed: true, active: true, rules: []string{operators}} + done, err := Converge(context.Background(), f.run, opening("adoption.bus", 5671, "everywhere", "incoming", 0)) + if err != nil || done.Action != "created" || done.SatisfiedBy != "" { + t.Errorf("%q: %+v %v", operators, done, err) + } + } +} + +func TestAnOpeningWhoseFoundRuleIsGoneIsAddedAgain(t *testing.T) { + f := &fakeUFW{installed: true, active: true, rules: []string{"allow 5671/tcp"}} + o := opening("adoption.bus", 5671, "everywhere", "incoming", 0) + if done, err := Converge(context.Background(), f.run, o); err != nil || done.SatisfiedBy == "" { + t.Fatalf("%+v %v", done, err) + } + f.rules = nil // the operator deleted theirs + if done, err := Converge(context.Background(), f.run, o); err != nil || done.Action != "created" { + t.Fatalf("the opening was not added once nothing answered it: %+v %v", done, err) + } +} diff --git a/internal/firewall/testdata/ufw-comment-only.txt b/internal/firewall/testdata/ufw-comment-only.txt new file mode 100644 index 0000000..13f758c --- /dev/null +++ b/internal/firewall/testdata/ufw-comment-only.txt @@ -0,0 +1,37 @@ +$ ufw allow 22/tcp +Rules updated +Rules updated (v6) +$ ufw --force enable +Firewall is active and enabled on system startup +$ ufw route allow 8080/tcp +Rule added +Rule added (v6) +$ ufw route allow proto tcp to any port 8080 comment 'mesh-host adoption.opening-tcp-8080-forwarded 1a2b3c4d' +Rule updated +Rule updated (v6) +$ ufw allow 5671/tcp +Rule added +Rule added (v6) +$ ufw allow proto tcp to any port 5671 comment 'mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d' +Rule updated +Rule updated (v6) +$ ufw allow in on mesh0 to any port 5432 proto tcp comment 'operator note' +Rule added +Rule added (v6) +$ ufw allow in on mesh0 proto tcp to any port 5432 comment 'mesh-host adoption.opening-tcp-5432-incoming 1a2b3c4d' +Rule updated +Rule updated (v6) +$ ufw show added +Added user rules (see 'ufw status' for running firewall): +ufw allow 22/tcp +ufw route allow 8080/tcp comment 'mesh-host adoption.opening-tcp-8080-forwarded 1a2b3c4d' +ufw allow 5671/tcp comment 'mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d' +ufw allow in on mesh0 to any port 5432 proto tcp comment 'mesh-host adoption.opening-tcp-5432-incoming 1a2b3c4d' +$ ufw route delete allow 8080/tcp comment 'mesh-host adoption.opening-tcp-8080-forwarded 1a2b3c4d' +Rule deleted +Rule deleted (v6) +$ ufw show added +Added user rules (see 'ufw status' for running firewall): +ufw allow 22/tcp +ufw allow 5671/tcp comment 'mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d' +ufw allow in on mesh0 to any port 5432 proto tcp comment 'mesh-host adoption.opening-tcp-5432-incoming 1a2b3c4d' diff --git a/internal/firewall/testdata/ufw-forms.txt b/internal/firewall/testdata/ufw-forms.txt new file mode 100644 index 0000000..a39c9c0 --- /dev/null +++ b/internal/firewall/testdata/ufw-forms.txt @@ -0,0 +1,16 @@ +Added user rules (see 'ufw status' for running firewall): +ufw allow 22/tcp +ufw allow 9200 +ufw allow from 192.0.2.0/24 to any port 9300 proto tcp +ufw allow in on eth0 to any port 9301 proto tcp +ufw deny 9400/tcp +ufw limit 2222/tcp +ufw allow 9500:9510/tcp +ufw route allow in on mesh0 out on docker0 to any port 8082 proto tcp +ufw allow to 192.0.2.1 port 9600 proto tcp +ufw allow 9700/udp +ufw route allow in on mesh0 to any port 8083 proto tcp +ufw allow 9900/tcp +ufw allow 80,443/tcp +ufw allow in on mesh0 to any port 5432 proto tcp +ufw route allow 8080/tcp From 078e84c681733bd4902e0d48055c1df101f3e723 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:08:02 +0200 Subject: [PATCH 19/52] Read an adopted or converged node's mode from its state on a re-run of genesis, and refuse a flag that disagrees (hq ADR 0103) --- internal/bootstrap/inuse.go | 31 +++++++++++++++++++++++ internal/bootstrap/inuse_test.go | 43 ++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/internal/bootstrap/inuse.go b/internal/bootstrap/inuse.go index 58c357b..023d322 100644 --- a/internal/bootstrap/inuse.go +++ b/internal/bootstrap/inuse.go @@ -6,6 +6,7 @@ import ( "net" "strings" + "github.com/novox/mesh-host/internal/declaration" "github.com/novox/mesh-host/internal/reachable" "github.com/novox/mesh-host/internal/store" ) @@ -76,6 +77,20 @@ func RefuseAMachineInUse(ctx context.Context, o Options, run Runner, say func(st return err } if len(known.Resources) > 0 { + // **The machine says how it was raised** (novox/hq ADR 0103). A re-run must not change + // the node's mode by a flag forgotten or added: without --adopted the bundle would load + // the foundation's dropping filter over the found firewall, and with it on a converged + // machine the filter the node relies on would be removed as no longer carried. + switch adopted := RecordsAdoption(known); { + case adopted && !o.Adopted: + return fmt.Errorf("this machine was raised adopted, and genesis was run again without --adopted. " + + "Run it again the way it was raised: pass --adopted. Returning it to converged is the " + + "controller's act (converge), never genesis's; nothing was changed") + case !adopted && o.Adopted: + return fmt.Errorf("this machine was raised converged, and genesis was run again with --adopted, " + + "which would remove the foundation's filter it relies on. Run it again without --adopted; " + + "returning a node to adopted is the controller's act (adopt); nothing was changed") + } // What genesis raised on an earlier run is the mesh's, and it is what the machine now // serves; the question was answered the first time. say(" in use not asked: this machine carries what an earlier genesis raised") @@ -109,3 +124,19 @@ func RefuseAMachineInUse(ctx context.Context, o Options, run Runner, say func(st "force and every module is taken on it one at a time. Nothing was changed", strings.Join(named, "\n - ")) } + +// RecordsAdoption is whether this machine's state says it is an adopted node: it holds something +// of the mesh's that only an adopted node has — the guard or an opening, under the adoption prefix +// — or something it found and holds. A node the controller converged has neither any more; the +// record of the firewall it found outlives the flip, so it is not read as the mode. +func RecordsAdoption(known store.State) bool { + if len(known.Held) > 0 { + return true + } + for _, r := range known.Resources { + if strings.HasPrefix(r.ID, declaration.AdoptionPrefix) { + return true + } + } + return false +} diff --git a/internal/bootstrap/inuse_test.go b/internal/bootstrap/inuse_test.go index 98ea349..137ceef 100644 --- a/internal/bootstrap/inuse_test.go +++ b/internal/bootstrap/inuse_test.go @@ -120,3 +120,46 @@ func TestAFreshlyInstalledMachineAsMeasuredIsNotInUse(t *testing.T) { t.Errorf("a fresh machine read as in use: containers %v, listeners %v", containers, listeners) } } + +// Defends novox/hq ADR 0103: a machine raised adopted stays adopted if genesis is run again. The +// installer reads the mode from what the machine records, and refuses a flag that disagrees. +func TestARerunWithoutTheFlagOnAnAdoptedMachineIsRefused(t *testing.T) { + o := Options{State: filepath.Join(t.TempDir(), "state.json")} + adoptedState := store.State{Resources: []store.Applied{ + {ID: "store", Type: "container", Target: "mesh-store", Origin: store.OriginCarried}, + {ID: "adoption.guard", Type: "file", Target: "/etc/mesh/guard.nft", Origin: store.OriginCarried}, + }} + if err := store.Save(o.State, adoptedState); err != nil { + t.Fatal(err) + } + m := inUseRunner{ss: inUseSockets} + err := RefuseAMachineInUse(context.Background(), o, m.run, quietly) + if err == nil || !strings.Contains(err.Error(), "pass --adopted") { + t.Fatalf("a re-run without --adopted on an adopted machine was not refused: %v", err) + } + o.Adopted = true + if err := RefuseAMachineInUse(context.Background(), o, m.run, quietly); err != nil { + t.Errorf("a re-run with --adopted on an adopted machine was refused: %v", err) + } +} + +func TestARerunWithTheFlagOnAConvergedMachineIsRefused(t *testing.T) { + o := Options{State: filepath.Join(t.TempDir(), "state.json"), Adopted: true} + converged := store.State{Resources: []store.Applied{ + {ID: "store", Type: "container", Target: "mesh-store", Origin: store.OriginCarried}, + {ID: "base-filter", Type: "file", Target: "/etc/nftables.conf", Origin: store.OriginCarried}, + }, + // Converged by the controller from adopted: the firewall it found is still recorded. + Firewall: &store.FoundFirewall{Kind: "ufw", WasActive: true, DisabledByMesh: true}} + if err := store.Save(o.State, converged); err != nil { + t.Fatal(err) + } + err := RefuseAMachineInUse(context.Background(), o, inUseRunner{ss: inUseSockets}.run, quietly) + if err == nil || !strings.Contains(err.Error(), "without --adopted") { + t.Fatalf("a re-run with --adopted on a converged machine was not refused: %v", err) + } + o.Adopted = false + if err := RefuseAMachineInUse(context.Background(), o, inUseRunner{ss: inUseSockets}.run, quietly); err != nil { + t.Errorf("a converged re-run of a converged machine was refused: %v", err) + } +} From d59d232656b2d6b03f98c800134b1ed9ca8bb160 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:08:20 +0200 Subject: [PATCH 20/52] Let a re-run of genesis find the package registry it raised itself under its own name (hq ADR 0100) --- internal/bootstrap/ports.go | 6 ++++++ internal/bootstrap/ports_test.go | 13 +++++++++++++ 2 files changed, 19 insertions(+) diff --git a/internal/bootstrap/ports.go b/internal/bootstrap/ports.go index 4da7560..f7a77c3 100644 --- a/internal/bootstrap/ports.go +++ b/internal/bootstrap/ports.go @@ -385,6 +385,12 @@ func NamesFree(ctx context.Context, run Runner, names []string, known store.Stat if known.Recorded(string(declaration.TypeContainer), name) { continue } + if name == giteaBootstrap && len(known.Resources) > 0 { + // Genesis raises the package registry itself, by hand and before the host records + // anything of it, so on a re-run it is found under its own name with no label and no + // record. A machine that carries what an earlier genesis raised made it. + continue + } taken = append(taken, name) } if len(taken) > 0 { diff --git a/internal/bootstrap/ports_test.go b/internal/bootstrap/ports_test.go index 49a3e64..a94371a 100644 --- a/internal/bootstrap/ports_test.go +++ b/internal/bootstrap/ports_test.go @@ -284,3 +284,16 @@ func TestAGenesisOnTheDefaultsSetsNoSettings(t *testing.T) { t.Errorf("a converged genesis on the default ports set settings: %v", c.told) } } + +func TestARerunOfGenesisFindsItsOwnPackageRegistry(t *testing.T) { + // Raised by genesis itself with no label and no record, so a re-run finds it unlabelled. + m := machineRunner{unlabelled: map[string]bool{giteaBootstrap: true}} + rerun := store.State{Resources: []store.Applied{{ID: "store", Type: "container", Target: "mesh-store"}}} + if err := NamesFree(context.Background(), m.run, []string{giteaBootstrap, "mesh-store"}, rerun); err != nil { + t.Errorf("a re-run refused the package registry genesis raised: %v", err) + } + // On a machine genesis never ran on, a container under that name is a predecessor's. + if err := NamesFree(context.Background(), m.run, []string{giteaBootstrap}, store.State{}); err == nil { + t.Error("a container under the package registry's name on a fresh machine was not refused") + } +} From 35ecf68393e9692af9d3a77f6af0f9c7b6acf6fc Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:08:50 +0200 Subject: [PATCH 21/52] Accept --registry as a host alone again, completing it with the registry's port (hq ADR 0100) --- cmd/mesh-bootstrap/main.go | 13 +++++++++++++ cmd/mesh-bootstrap/main_test.go | 14 ++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/cmd/mesh-bootstrap/main.go b/cmd/mesh-bootstrap/main.go index adb27ba..f8ae993 100644 --- a/cmd/mesh-bootstrap/main.go +++ b/cmd/mesh-bootstrap/main.go @@ -18,6 +18,7 @@ package main import ( "context" "encoding/json" + "errors" "flag" "fmt" "io" @@ -239,6 +240,18 @@ func registryAgrees(set *flag.FlagSet, opts *bootstrap.Options) error { said := map[string]bool{} set.Visit(func(f *flag.Flag) { said[f.Name] = true }) host, portText, err := net.SplitHostPort(opts.Registry) + var missing *net.AddrError + if errors.As(err, &missing) && missing.Err == "missing port in address" { + // A host alone, as --registry took before its port became the node's: the registry's + // port — the one given, or the catalogue's — completes it. + port := opts.Ports.Registry + if port == 0 { + port = bootstrap.DefaultPorts().Registry + } + opts.Ports.Registry = port + opts.Registry = net.JoinHostPort(strings.Trim(opts.Registry, "[]"), strconv.Itoa(port)) + return nil + } if err != nil { return fmt.Errorf("--registry %q is not host:port: %w", opts.Registry, err) } diff --git a/cmd/mesh-bootstrap/main_test.go b/cmd/mesh-bootstrap/main_test.go index 2a4a4bb..8d19d6c 100644 --- a/cmd/mesh-bootstrap/main_test.go +++ b/cmd/mesh-bootstrap/main_test.go @@ -190,3 +190,17 @@ func TestTheRegistrysPortAndAddressAgree(t *testing.T) { t.Error("two ports for one registry were accepted") } } + +func TestARegistryGivenAsAHostAloneTakesTheRegistrysPort(t *testing.T) { + _, opts, _, err := parseArgs([]string{"--registry", "192.0.2.10"}) + if err != nil || opts.Registry != "192.0.2.10:5000" || opts.Ports.Registry != 5000 { + t.Errorf("--registry host alone: %s %d %v", opts.Registry, opts.Ports.Registry, err) + } + _, opts, _, err = parseArgs([]string{"--registry", "192.0.2.10", "--registry-port", "5100"}) + if err != nil || opts.Registry != "192.0.2.10:5100" { + t.Errorf("--registry host with --registry-port: %s %v", opts.Registry, err) + } + if _, _, _, err := parseArgs([]string{"--registry", "192.0.2.10:notaport"}); err == nil { + t.Error("a registry with a port that is not a number was accepted") + } +} From 824cb60cbbe7bb5b03f9cc6ca0109e2859da3356 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:14:37 +0200 Subject: [PATCH 22/52] Hold a found directory, a found service's unit, a container that would mount found data, and a step run in a held container on an adopted node (hq ADR 0103) --- internal/apply/apply.go | 50 ++--- internal/apply/hold.go | 281 +++++++++++++++++++++++- internal/apply/hold_test.go | 304 +++++++++++++++++++++----- internal/declaration/adoption_test.go | 15 +- internal/declaration/declaration.go | 18 +- internal/store/store.go | 13 +- 6 files changed, 562 insertions(+), 119 deletions(-) diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 22cdb3b..650c4fe 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -161,6 +161,10 @@ func ApplyKeeping( return report, known, &Error{Resource: "the firewall found on this machine", Err: err, Done: report} } + // What an adopted node's untaken modules find on the machine, looked at before anything in + // this apply — a removal included — could change it or its records (novox/hq ADR 0103). + before := lookBefore(ctx, sys, d, known, run) + removeOrphan := func(orphan store.Applied) error { var action, detail string var err error @@ -236,36 +240,24 @@ func ApplyKeeping( var failures []*Error for _, resource := range d.Resources { // **On an adopted node, what is found is kept until its module is taken** (novox/hq ADR - // 0100). Before anything is applied: a file present with no record of this host writing - // it, or a container present under that name that no host made, is held as it is and - // reported. Once held it stays held — changed or gone — until its module is taken, and - // it is never recorded as applied, so it is never removed as an orphan either. - if d.Adoption != nil && holdable(resource) { - if module, untaken := d.Adoption.UntakenModuleOf(resource.Identity()); untaken { - was, already := known.HeldAt(resource.Identity()) - isFound := false - if !already { - var err error - if isFound, err = found(ctx, resource, run, known); err != nil { - failures = append(failures, &Error{Resource: resource.Identity(), Err: err, Done: report}) - log(fmt.Sprintf(" failed %s (%s): %v", resource.Identity(), resource.Target(), err)) - continue - } - } - if already || isFound { - outcome, held, err := hold(ctx, resource, module, was, already, run, keep, time.Now().UTC()) - if err != nil { - failures = append(failures, &Error{Resource: resource.Identity(), Err: err, Done: report}) - log(fmt.Sprintf(" failed %s (%s): %v", resource.Identity(), outcome.Target, err)) - continue - } - known.RecordHeld(held) - report.Outcomes = append(report.Outcomes, outcome) - if !already || held.Changed != was.Changed { - log(fmt.Sprintf(" held %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail)) - } - continue + // 0100, ADR 0103). Before anything is applied: whatever of a module not yet taken is + // present with no record of this host making it — or would reach what is — is held as it + // is and reported. Once held it stays held until its module is taken, and it is never + // recorded as applied, so it is never removed as an orphan either. + if d.Adoption != nil { + isHeld, news, outcome, err := holdOnAdopted(ctx, sys, resource, d, &known, before, run, keep, + changed, time.Now().UTC()) + if err != nil { + failures = append(failures, &Error{Resource: resource.Identity(), Err: err, Done: report}) + log(fmt.Sprintf(" failed %s (%s): %v", resource.Identity(), resource.Target(), err)) + continue + } + if isHeld { + report.Outcomes = append(report.Outcomes, outcome) + if news { + log(fmt.Sprintf(" held %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail)) } + continue } } diff --git a/internal/apply/hold.go b/internal/apply/hold.go index d8ddfb4..fce255d 100644 --- a/internal/apply/hold.go +++ b/internal/apply/hold.go @@ -14,6 +14,7 @@ import ( "github.com/novox/mesh-host/internal/declaration" "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" ) // Keep records the original of a file found on an adopted node, before anything else happens to @@ -45,14 +46,230 @@ func KeepIn(dir string) Keep { } } -// holdable is whether a resource is one a predecessor can already have on the machine: a file at -// a path, or a container under a name. A file written into is not: it replaces nothing that was -// found, only adds the mesh's keys beside it (novox/hq ADR 0102). -func holdable(r declaration.Resource) bool { - if f, ok := r.(*declaration.File); ok { - return f.Into == "" +// foundBefore is what an adopted apply finds on the machine before it changes anything: each +// directory, service unit and container mount source of an untaken module that is present with no +// record (novox/hq ADR 0103). Looked at first, because the apply itself makes such things — a +// file's parent directory, a unit file a module writes, a package that brings its unit — and what +// the mesh made in this apply was not found. +type foundBefore map[string]bool + +func lookBefore(ctx context.Context, sys system.System, d *declaration.Declaration, known store.State, + run Runner) foundBefore { + seen := foundBefore{} + if d.Adoption == nil { + return seen } - return r.Kind() == declaration.TypeContainer + cri, asked := "", false + for _, r := range d.Resources { + if _, untaken := d.Adoption.UntakenModuleOf(r.Identity()); !untaken { + continue + } + if _, held := known.HeldAt(r.Identity()); held { + continue + } + switch res := r.(type) { + case *declaration.Directory: + if present(res.Path) && !recordedPath(known, res.Path) { + seen["path:"+res.Path] = true + } + case *declaration.Service: + if known.Recorded(string(declaration.TypeService), res.Unit) { + continue + } + // A unit the service manager cannot find is not there; one it can read is, whatever + // state it is in. + if _, err := sys.ServiceState(ctx, run, res.Unit); err == nil { + seen["unit:"+res.Unit] = true + } + case *declaration.Container: + if known.Recorded(string(declaration.TypeContainer), res.Name) { + continue + } + for _, v := range res.Volumes { + src := mountSource(v) + switch { + case src == "": + case strings.HasPrefix(src, "/"): + if present(src) && !recordedPath(known, src) { + seen["path:"+src] = true + } + default: + if !asked { + cri, _ = containerRuntime(ctx, run) + asked = true + } + if cri == "" { + continue + } + if _, err := run(ctx, cri, "volume", "inspect", src); err == nil { + seen["volume:"+src] = true + } + } + } + } + } + return seen +} + +func present(path string) bool { + _, err := os.Lstat(path) + return err == nil +} + +// recordedPath is whether this host has a record of making something at a path. +func recordedPath(known store.State, path string) bool { + for _, kind := range []declaration.Type{declaration.TypeDirectory, declaration.TypeFile, + declaration.TypeArchive, declaration.TypeAccess} { + if known.Recorded(string(kind), path) { + return true + } + } + return false +} + +// mountSource is what a volume mapping mounts: a path on the machine, or a named volume. Empty for +// an anonymous volume, which mounts nothing that could already be there. +func mountSource(mapping string) string { + src, _, ok := strings.Cut(mapping, ":") + if !ok { + return "" + } + return src +} + +// runsIn is the container a resource runs inside, if any: an action's `in`, or a run-once step +// sharing a container's namespace. +func runsIn(r declaration.Resource) string { + switch res := r.(type) { + case *declaration.Action: + return res.In + case *declaration.Container: + if res.RunOnce { + if name, ok := strings.CutPrefix(res.Network, "container:"); ok { + return name + } + } + } + return "" +} + +// heldContainer is what is held under a container's name. +func heldContainer(known store.State, name string) (store.Held, bool) { + for _, h := range known.Held { + if h.Kind == string(declaration.TypeContainer) && h.Target == name { + return h, true + } + } + return store.Held{}, false +} + +// holdOnAdopted decides whether a resource of an adopted node is held rather than applied, and +// holds it (novox/hq ADR 0100, ADR 0103). For a module not yet taken, what is present with no +// record is kept as it is: a file or a container under its name, a directory, a service's unit, +// and a container that would mount a path or a volume found there. An action or a run-once step +// run inside a held container is held with it. Once held, a resource stays held — changed or gone +// — until its module is taken, and it is never recorded as applied, so never removed as an orphan. +// +// Held is false for a resource to apply as usual. News is whether the hold is new or changed, +// which is what is worth a line in the log. +func holdOnAdopted(ctx context.Context, sys system.System, r declaration.Resource, d *declaration.Declaration, + known *store.State, before foundBefore, run Runner, keep Keep, changed map[string]bool, + now time.Time) (held, news bool, out Outcome, err error) { + was, already := known.HeldAt(r.Identity()) + + if in := runsIn(r); in != "" { + if container, isHeld := heldContainer(*known, in); isHeld { + module, untaken := d.Adoption.UntakenModuleOf(r.Identity()) + if !untaken { + module = container.Module + } + h := was + if !already { + h = store.Held{ID: r.Identity(), Kind: string(r.Kind()), Target: r.Target(), Since: now} + } + h.Module, h.Why = module, "runs in "+in + known.RecordHeld(h) + out = begin(r) + out.Action = "held" + out.Detail = fmt.Sprintf("runs in %s, which is held as found; not run until %s is taken", in, module) + return true, !already, out, nil + } + } + + module, untaken := d.Adoption.UntakenModuleOf(r.Identity()) + if !untaken { + return false, false, out, nil + } + why := was.Why + isFound := already + if !already { + switch res := r.(type) { + case *declaration.File: + if res.Into == "" { + if isFound, err = found(ctx, r, run, *known); err != nil { + return false, false, begin(r), err + } + } + case *declaration.Container: + if known.Recorded(string(declaration.TypeContainer), res.Name) { + break + } + _, exists, err := inspectFound(ctx, res.Name, run) + if err != nil { + return false, false, begin(r), err + } + if exists { + if isFound, err = found(ctx, r, run, *known); err != nil { + return false, false, begin(r), err + } + break + } + // Not there under its name, and still it would share what was found: created, it + // would mount the predecessor's data beside the predecessor's own container. + for _, v := range res.Volumes { + src := mountSource(v) + key := "volume:" + src + if strings.HasPrefix(src, "/") { + key = "path:" + src + } + if src != "" && before[key] { + isFound, why = true, "would mount "+src+", found on the machine" + break + } + } + case *declaration.Directory: + isFound = before["path:"+res.Path] + case *declaration.Service: + isFound = before["unit:"+res.Unit] + } + } + if !isFound { + return false, false, out, nil + } + + out, h, err := hold(ctx, sys, r, module, was, already, why, run, keep, now) + if err != nil { + return true, false, out, err + } + // A held service is not started, stopped, enabled or restarted — but a reload stops nothing, + // so one the module names still happens (novox/hq ADR 0102, ADR 0103). + if svc, ok := r.(*declaration.Service); ok && svc.State == "running" { + if which := restartedBy(svc.ReloadOn, changed); len(which) > 0 { + if state, err := sys.ServiceState(ctx, run, svc.Unit); err == nil && state == "running" { + reloader, can := sys.(serviceReloader) + if !can { + return true, false, out, fmt.Errorf("%s must be reloaded for %s and this machine's "+ + "service manager cannot reload a unit", svc.Unit, strings.Join(which, ", ")) + } + if err := reloader.ReloadService(ctx, run, svc.Unit); err != nil { + return true, false, out, fmt.Errorf("reloading the held %s: %w", svc.Unit, err) + } + out.Detail += "; reloaded for " + strings.Join(which, ", ") + ", which stops nothing" + } + } + } + known.RecordHeld(h) + return true, !already || h.Changed != was.Changed, out, nil } // found is whether a declared file or container is present on the machine with no record of this @@ -111,15 +328,16 @@ func inspectFound(ctx context.Context, name string, run Runner) (foundContainer, // hold keeps a found file or container as it is, and reports it — the first time by recording // what was found, every time after by comparing against that. Nothing is reverted, restarted or // created: a held target that disappears stays held and gone until its module is taken. -func hold(ctx context.Context, r declaration.Resource, module string, was store.Held, already bool, - run Runner, keep Keep, now time.Time) (Outcome, store.Held, error) { +func hold(ctx context.Context, sys system.System, r declaration.Resource, module string, was store.Held, + already bool, why string, run Runner, keep Keep, now time.Time) (Outcome, store.Held, error) { out := begin(r) h := was if !already { h = store.Held{ID: r.Identity(), Module: module, Kind: string(r.Kind()), - Target: r.Target(), Since: now} + Target: r.Target(), Since: now, Why: why} } h.Module = module + detail := "found on the machine; kept until " + module + " is taken" var changed string switch res := r.(type) { @@ -159,7 +377,45 @@ func hold(ctx context.Context, r declaration.Resource, module string, was store. changed = "rewritten" } } + case *declaration.Directory: + info, err := os.Lstat(res.Path) + switch { + case errors.Is(err, os.ErrNotExist): + if !already { + return out, h, fmt.Errorf("%s was found and is gone before it could be held", res.Path) + } + changed = "gone" + case err != nil: + return out, h, err + case !already: + // Its mode and owner as found, which the mesh leaves: a database refuses to start + // on a data directory whose mode changed. + h.Mode = fmt.Sprintf("%04o", info.Mode().Perm()) + if st, ok := info.Sys().(*syscall.Stat_t); ok { + h.Owner = fmt.Sprintf("%d:%d", st.Uid, st.Gid) + } + } + detail = "found on the machine; its mode, owner and contents kept until " + module + " is taken" + case *declaration.Service: + state, err := sys.ServiceState(ctx, run, res.Unit) + switch { + case err != nil && !already: + return out, h, fmt.Errorf("the unit %s was found and cannot be read to hold it: %w", res.Unit, err) + case err != nil: + changed = "gone" + case !already: + h.Running = state == "running" + case h.Running && state != "running": + changed = "stopped" + } + detail = "its unit was found on the machine; its state and whether it starts at boot are " + + "kept until " + module + " is taken" case *declaration.Container: + if h.Why != "" && h.Container == "" { + // Held for what it would mount, never created: there is nothing of it to compare. + detail = "not created: it " + h.Why + "; kept until " + module + " is taken" + break + } seen, exists, err := inspectFound(ctx, res.Name, run) if err != nil { return out, h, err @@ -188,7 +444,7 @@ func hold(ctx context.Context, r declaration.Resource, module string, was store. } } out.Action = "held" - out.Detail = "found on the machine; kept until " + module + " is taken" + out.Detail = detail if h.Changed != "" { out.Detail += "; " + h.Changed + " by something other than the mesh since it was found, and not reverted" } @@ -197,6 +453,9 @@ func hold(ctx context.Context, r declaration.Resource, module string, was store. // takenDetail is what an outcome says when a module's cutover replaced what was held for it. func takenDetail(h store.Held) string { + if h.Kind == string(declaration.TypeAction) || (h.Why != "" && h.Container == "") { + return "taken: no longer held (" + h.Why + ")" + } if h.Kept != "" { return "taken: replaced what was found; original kept at " + h.Kept } diff --git a/internal/apply/hold_test.go b/internal/apply/hold_test.go index 6b6e947..9699edd 100644 --- a/internal/apply/hold_test.go +++ b/internal/apply/hold_test.go @@ -19,6 +19,54 @@ import ( type machine struct { containers map[string]*fakeContainer asked []string + + // units are service units by name, as systemd would report them; volumes are the runtime's + // named volumes. + units map[string]*fakeUnit + volumes map[string]bool +} + +type fakeUnit struct { + active, enabled string +} + +// systemctl answers as systemd does for the units the machine has, and "not-found" for any other. +func (m *machine) systemctl(args []string) (string, error) { + unit := args[len(args)-1] + if args[0] == "show" { + unit = args[1] + } + u, ok := m.units[unit] + switch args[0] { + case "show": + if !ok { + return "LoadState=not-found\nActiveState=inactive\nType=simple\n", nil + } + return "LoadState=loaded\nActiveState=" + u.active + "\nType=simple\nRemainAfterExit=no\n", nil + case "is-enabled": + if !ok { + return "", errors.New("not found") + } + return u.enabled + "\n", nil + case "start": + u.active = "active" + case "stop": + u.active = "inactive" + case "enable": + u.enabled = "enabled" + case "disable": + u.enabled = "disabled" + } + return "", nil +} + +func (m *machine) did(prefix string) bool { + for _, a := range m.asked { + if strings.HasPrefix(a, prefix) { + return true + } + } + return false } type fakeContainer struct { @@ -29,7 +77,18 @@ type fakeContainer struct { func (m *machine) run(_ context.Context, name string, args ...string) (string, error) { m.asked = append(m.asked, name+" "+strings.Join(args, " ")) + if name == "systemctl" { + return m.systemctl(args) + } + if name != "docker" { + return "", nil + } switch args[0] { + case "volume": + if m.volumes[args[len(args)-1]] { + return "[]\n", nil + } + return "", errors.New("no such volume") case "info": return "27.0\n", nil case "inspect": @@ -278,88 +337,213 @@ func TestAHeldFileIsNeverRemovedWhenItsModuleIsUnassigned(t *testing.T) { } } -func TestAHeldFileRewrittenIsReportedAndNotReverted(t *testing.T) { - dir, page, m := predecessor(t) - d := adopted(t, untakenWeb, webResources(page)) - _, state := applyAdopted(t, d, store.State{}, m, dir) +// Defends novox/hq ADR 0103: found covers every kind that can reach what the machine already has. - if err := os.WriteFile(page, []byte("the predecessor wrote again\n"), 0o640); err != nil { +// untaken is an adoption with hello-web untaken, listing these of its resources. +func untaken(ids ...string) string { + return `{"taken":[],"untaken":{"hello-web":["` + strings.Join(ids, `","`) + `"]}}` +} + +func TestAFoundDirectoryOfAnUntakenModuleKeepsItsModeOwnerAndContents(t *testing.T) { + dir := t.TempDir() + data := filepath.Join(dir, "data") + if err := os.Mkdir(data, 0o700); err != nil { t.Fatal(err) } - report, state := applyAdopted(t, d, state, m, dir) - if got, _ := os.ReadFile(page); string(got) != "the predecessor wrote again\n" { - t.Fatalf("a held file was reverted: %q", got) + inside := filepath.Join(data, "PG_VERSION") + if err := os.WriteFile(inside, []byte("16\n"), 0o600); err != nil { + t.Fatal(err) } - if h, _ := state.HeldAt("hello-web.page"); h.Changed != "rewritten" || h.ChangedAt.IsZero() { - t.Errorf("a rewrite was not recorded: %+v", h) + m := &machine{containers: map[string]*fakeContainer{}} + report, state := applyAdopted(t, adopted(t, untaken("hello-web.data"), + `{"id":"hello-web.data","type":"directory","path":"`+data+`","mode":"0755"}`), store.State{}, m, dir) + + if info, _ := os.Stat(data); info.Mode().Perm() != 0o700 { + t.Errorf("a found directory was re-moded to %o", info.Mode().Perm()) } - if o := outcomeOf(report, "hello-web.page"); !strings.Contains(o.Detail, "rewritten") { - t.Errorf("a rewrite was not reported: %+v", o) + if got, _ := os.ReadFile(inside); string(got) != "16\n" { + t.Errorf("what is inside a found directory was touched: %q", got) } - h, _ := state.HeldAt("hello-web.page") - if kept, _ := os.ReadFile(h.Kept); string(kept) != "the predecessor's page\n" { - t.Errorf("the kept original was overwritten by a later write: %q", kept) + if o := outcomeOf(report, "hello-web.data"); o.Action != "held" || !strings.Contains(o.Detail, "mode, owner and contents") { + t.Errorf("the found directory was not held: %+v", o) + } + if h, ok := state.HeldAt("hello-web.data"); !ok || h.Mode != "0700" { + t.Errorf("the directory as found was not recorded: %+v", h) + } + if _, recorded := state.Find("hello-web.data"); recorded { + t.Error("a held directory was recorded as applied") } } -func TestAHeldContainerStoppedOrReplacedIsReportedAndNotRestarted(t *testing.T) { +func TestADirectoryMadeInTheSameApplyIsNotFound(t *testing.T) { + // A file's parent is made as the file is written; what the mesh made is not found. + dir := t.TempDir() + data := filepath.Join(dir, "data") + m := &machine{containers: map[string]*fakeContainer{}} + report, state := applyAdopted(t, adopted(t, untaken("hello-web.conf", "hello-web.data"), + `{"id":"hello-web.conf","type":"file","path":"`+filepath.Join(data, "conf")+`","content":"x\n"}, + {"id":"hello-web.data","type":"directory","path":"`+data+`","mode":"0750"}`), store.State{}, m, dir) + if o := outcomeOf(report, "hello-web.data"); o.Action == "held" { + t.Errorf("a directory the apply itself made was held: %+v", o) + } + if info, _ := os.Stat(data); info.Mode().Perm() != 0o750 { + t.Errorf("the mesh's own directory was not converged: %o", info.Mode().Perm()) + } + if len(state.Held) != 0 { + t.Errorf("held: %+v", state.Held) + } +} + +func TestAFoundServiceOfAnUntakenModuleIsNeitherStartedNorEnabledNorRestarted(t *testing.T) { + dir := t.TempDir() + conf := filepath.Join(dir, "hello.conf") + m := &machine{containers: map[string]*fakeContainer{}, + units: map[string]*fakeUnit{"hello.service": {active: "inactive", enabled: "disabled"}}} + report, state := applyAdopted(t, adopted(t, untaken("hello-web.conf", "hello-web.unit"), + `{"id":"hello-web.conf","type":"file","path":"`+conf+`","content":"x\n"}, + {"id":"hello-web.unit","type":"service","unit":"hello.service","state":"running","boot":"enabled", + "restart-on":["hello-web.conf"]}`), store.State{}, m, dir) + + for _, verb := range []string{"systemctl start", "systemctl stop", "systemctl enable", "systemctl disable", "systemctl restart"} { + if m.did(verb) { + t.Errorf("a found service was changed: %s (%v)", verb, m.asked) + } + } + if o := outcomeOf(report, "hello-web.unit"); o.Action != "held" || !strings.Contains(o.Detail, "starts at boot") { + t.Errorf("the found service was not held: %+v", o) + } + if h, ok := state.HeldAt("hello-web.unit"); !ok || h.Running { + t.Errorf("the service as found was not recorded: %+v", h) + } +} + +func TestAHeldServiceIsStillReloadedButNeverRestarted(t *testing.T) { + // A reload stops nothing (novox/hq ADR 0102); a restart would stop the predecessor's service. + dir := t.TempDir() + conf := filepath.Join(dir, "daemon.json") + m := &machine{containers: map[string]*fakeContainer{}, + units: map[string]*fakeUnit{"docker.service": {active: "active", enabled: "enabled"}}} + report, _ := applyAdopted(t, adopted(t, untaken("hello-web.conf", "hello-web.unit"), + `{"id":"hello-web.conf","type":"file","path":"`+conf+`","content":"{}\n"}, + {"id":"hello-web.unit","type":"service","unit":"docker.service","state":"running","boot":"enabled", + "reload-on":["hello-web.conf"]}`), store.State{}, m, dir) + if !m.did("systemctl reload docker.service") { + t.Errorf("a held service was not reloaded for what it re-reads: %v", m.asked) + } + if m.did("systemctl stop") || m.did("systemctl start") { + t.Errorf("a held service was restarted: %v", m.asked) + } + if o := outcomeOf(report, "hello-web.unit"); o.Action != "held" || !strings.Contains(o.Detail, "reloaded for hello-web.conf") { + t.Errorf("the reload was not reported on the hold: %+v", o) + } +} + +func TestAServiceWhoseUnitIsNotThereIsAppliedAsUsual(t *testing.T) { + // No unit before the apply: nothing of a predecessor's to hold. + dir := t.TempDir() + m := &machine{containers: map[string]*fakeContainer{}, units: map[string]*fakeUnit{}} + d := adopted(t, untaken("hello-web.unit"), `{"id":"hello-web.unit","type":"service","unit":"hello.service","state":"running"}`) + _, _, err := ApplyKeeping(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, + m.run, nil, nil, KeepIn(dir)) + if err == nil || !strings.Contains(err.Error(), "does not exist") { + t.Errorf("an absent unit was held rather than applied: %v", err) + } +} + +func TestAContainerThatWouldMountFoundDataIsNotCreated(t *testing.T) { + dir := t.TempDir() + data := filepath.Join(dir, "predecessor-data") + if err := os.Mkdir(data, 0o700); err != nil { + t.Fatal(err) + } for _, c := range []struct { - change func(*machine) - want string + name, volume string + m *machine }{ - {func(m *machine) { m.containers["hello-web"].running = false }, "stopped"}, - {func(m *machine) { m.containers["hello-web"].id = "another" }, "replaced"}, - {func(m *machine) { delete(m.containers, "hello-web") }, "gone"}, + {"a path", data + ":/var/lib/postgresql/data", &machine{containers: map[string]*fakeContainer{}}}, + {"a named volume", "predecessor-pgdata:/var/lib/postgresql/data", + &machine{containers: map[string]*fakeContainer{}, volumes: map[string]bool{"predecessor-pgdata": true}}}, } { - dir, page, m := predecessor(t) - d := adopted(t, untakenWeb, webResources(page)) - _, state := applyAdopted(t, d, store.State{}, m, dir) - c.change(m) - m.asked = nil - _, state = applyAdopted(t, d, state, m, dir) - if h, _ := state.HeldAt("hello-web.server"); h.Changed != c.want { - t.Errorf("%s: recorded as %q", c.want, h.Changed) + report, state := applyAdopted(t, adopted(t, untaken("hello-web.server"), + `{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`", + "volumes":["`+c.volume+`"]}`), store.State{}, c.m, dir) + if c.m.did("docker run") { + t.Errorf("%s: a container mounting found data was created: %v", c.name, c.m.asked) } - for _, a := range m.asked { - if strings.HasPrefix(a, "docker run") || strings.HasPrefix(a, "docker rm") || - strings.HasPrefix(a, "docker start") { - t.Errorf("%s: the held container was acted on: %s", c.want, a) - } + o := outcomeOf(report, "hello-web.server") + if o.Action != "held" || !strings.Contains(o.Detail, "would mount") { + t.Errorf("%s: not held: %+v", c.name, o) + } + if h, ok := state.HeldAt("hello-web.server"); !ok || !strings.Contains(h.Why, "would mount") { + t.Errorf("%s: the hold does not say why: %+v", c.name, h) + } + // Held, it stays held on the next pass, and is still not created. + c.m.asked = nil + _, state = applyAdopted(t, adopted(t, untaken("hello-web.server"), + `{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`", + "volumes":["`+c.volume+`"]}`), state, c.m, dir) + if c.m.did("docker run") || len(state.Held) != 1 { + t.Errorf("%s: a held container was created on the next pass: %v", c.name, c.m.asked) } } } -func TestAHeldFileThatVanishesIsNotCreated(t *testing.T) { +func TestAContainerMountingWhatTheMeshMadeIsCreated(t *testing.T) { + dir := t.TempDir() + data := filepath.Join(dir, "data") + m := &machine{containers: map[string]*fakeContainer{}} + report, _ := applyAdopted(t, adopted(t, untaken("hello-web.data", "hello-web.server"), + `{"id":"hello-web.data","type":"directory","path":"`+data+`"}, + {"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`", + "volumes":["`+data+`:/data"]}`), store.State{}, m, dir) + if o := outcomeOf(report, "hello-web.server"); o.Action != "created" { + t.Errorf("a container mounting only what the mesh made was not created: %+v", o) + } +} + +func TestARunOnceStepInAHeldContainerIsHeld(t *testing.T) { + dir, page, m := predecessor(t) + step := `{"id":"hello-web.migrate","type":"container","name":"hello-web-migrate","image":"` + pinned + `", + "run-once":true,"network":"container:hello-web"}` + report, state := applyAdopted(t, adopted(t, untaken("hello-web.page", "hello-web.server", "hello-web.migrate"), webResources(page)+","+step), store.State{}, m, dir) + if m.did("docker run") { + t.Errorf("a step was run inside a held container: %v", m.asked) + } + o := outcomeOf(report, "hello-web.migrate") + if o.Action != "held" || !strings.Contains(o.Detail, "runs in hello-web") { + t.Errorf("the step was not held: %+v", o) + } + if _, ok := state.HeldAt("hello-web.migrate"); !ok { + t.Error("the held step is not reported held") + } +} + +func TestAnActionInAHeldContainerIsHeldUntilItsModuleIsTaken(t *testing.T) { + // An action cannot arrive over the link today, and a bundle cannot say a node is adopted; the + // host holds one anyway, since what it would run in is the predecessor's. dir, page, m := predecessor(t) d := adopted(t, untakenWeb, webResources(page)) - _, state := applyAdopted(t, d, store.State{}, m, dir) - if err := os.Remove(page); err != nil { - t.Fatal(err) - } - _, state = applyAdopted(t, d, state, m, dir) - if _, err := os.Stat(page); !errors.Is(err, os.ErrNotExist) { - t.Fatal("a held file that vanished was created before its module was taken") - } - if h, _ := state.HeldAt("hello-web.page"); h.Changed != "gone" { - t.Errorf("a vanished held file was not reported gone: %+v", h) - } -} - -func TestAConvergedNodeStillReplacesWhatItFinds(t *testing.T) { - // No adoption, no holds: byte for byte what a converged node did before ADR 0100. - dir, page, m := predecessor(t) - d := parse(t, `{"declaration":1,"resources":[`+webResources(page)+`]}`) + d.Resources = append(d.Resources, &declaration.Action{ID: "hello-web.seed", Type: declaration.TypeAction, + In: "hello-web", Command: []string{"seed"}, Verify: []string{"seeded"}}) report, state := applyAdopted(t, d, store.State{}, m, dir) - if got, _ := os.ReadFile(page); string(got) != "the mesh's page\n" { - t.Errorf("a converged node kept a found file: %q", got) + if m.did("docker exec") { + t.Errorf("an action was run inside a held container: %v", m.asked) } - if !m.removed("hello-web") { - t.Error("a converged node kept a found container") + if o := outcomeOf(report, "hello-web.seed"); o.Action != "held" || !strings.Contains(o.Detail, "not run until hello-web is taken") { + t.Errorf("the action was not held: %+v", o) } - if len(state.Held) != 0 || outcomeOf(report, "hello-web.page").Action == "held" { - t.Errorf("a converged node held something: %+v", state.Held) + if h, ok := state.HeldAt("hello-web.seed"); !ok || h.Module != "hello-web" { + t.Errorf("the held action is not its module's: %+v", h) } - if _, err := os.Stat(filepath.Join(dir, "kept")); !errors.Is(err, os.ErrNotExist) { - t.Error("a converged node kept originals") + + // Taken: the container is the mesh's, and the action runs in it. + taken := adopted(t, takenWeb, webResources(page)) + taken.Resources = append(taken.Resources, d.Resources[len(d.Resources)-1]) + report, state = applyAdopted(t, taken, state, m, dir) + if !m.did("docker exec hello-web ") { + t.Errorf("the action did not run once its module was taken: %v", m.asked) + } + if _, still := state.HeldAt("hello-web.seed"); still || len(state.Held) != 0 { + t.Errorf("holds outlived the take: %+v", state.Held) } } diff --git a/internal/declaration/adoption_test.go b/internal/declaration/adoption_test.go index 8309329..1ca0d3b 100644 --- a/internal/declaration/adoption_test.go +++ b/internal/declaration/adoption_test.go @@ -57,11 +57,16 @@ func TestAnAdoptionNamingAnUnknownIDIsRefused(t *testing.T) { } } -func TestAnAdoptionMayOnlyHoldFilesAndContainers(t *testing.T) { - refusal := refusalFor(t, `{"adoption":{"taken":[],"untaken":{"hello-web":["hello-web.data"]}}, - "declaration":1,`+adoptedResources+`}`) - if !strings.Contains(strings.Join(refusal.Problems, "\n"), "only a file or a container") { - t.Errorf("a directory was accepted as holdable: %v", refusal.Problems) +func TestAnAdoptionMayNameAResourceOfAnyKind(t *testing.T) { + // A directory, a service or an action can reach what was found as surely as a file can, so + // the controller lists every resource of an untaken module (novox/hq ADR 0103). + d, err := Parse([]byte(`{"adoption":{"taken":[],"untaken":{"hello-web":["hello-web.data"]}}, + "declaration":1,` + adoptedResources + `}`)) + if err != nil { + t.Fatalf("a directory of an untaken module was refused: %v", err) + } + if module, ok := d.Adoption.UntakenModuleOf("hello-web.data"); !ok || module != "hello-web" { + t.Errorf("the directory is not its module's: %q %v", module, ok) } } diff --git a/internal/declaration/declaration.go b/internal/declaration/declaration.go index a458c8a..c0bc7af 100644 --- a/internal/declaration/declaration.go +++ b/internal/declaration/declaration.go @@ -956,10 +956,12 @@ type Declaration struct { // is adopted; it is told, in every declaration, so a host restarted from the declaration it kept // is in the same mode it was in before. // -// Untaken names, per module assigned here and not yet taken, the ids of its file and container -// resources — the only shapes a predecessor can already have on the machine. The host cannot -// split a resource id into its module, because module names may contain dots, so the controller -// says which ids belong to which module rather than leaving the host to guess. +// Untaken names, per module assigned here and not yet taken, the ids of its resources. Any kind +// may be listed: a file, a directory, a service's unit or a container can already be on the +// machine, and an action run inside a held container reaches what was found (novox/hq ADR 0103); +// the host decides per kind what can be held. The host cannot split a resource id into its +// module, because module names may contain dots, so the controller says which ids belong to which +// module rather than leaving the host to guess. type Adoption struct { Taken []string `json:"taken"` Untaken map[string][]string `json:"untaken,omitempty"` @@ -1026,15 +1028,9 @@ func checkAdoption(a *Adoption, resources []Resource, allowActions bool) []strin continue } owner[id] = module - kind, declared := kinds[id] - switch { - case !declared: + if _, declared := kinds[id]; !declared { problems = append(problems, fmt.Sprintf( "adoption: %q of the untaken module %q is not in this declaration", id, module)) - case kind != TypeFile && kind != TypeContainer: - problems = append(problems, fmt.Sprintf( - "adoption: %q of the untaken module %q is a %s, and only a file or a "+ - "container can be found on a machine", id, module, kind)) } } } diff --git a/internal/store/store.go b/internal/store/store.go index 78430fa..7eeba41 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -115,8 +115,10 @@ type FoundFirewall struct { FoundAt time.Time `json:"found_at"` } -// Held is one file or container found on an adopted node — present at a declared path or name, -// with no record of this host having made it — and kept as it was found. +// Held is one thing found on an adopted node — a file, directory or container present at a declared +// path or name, or a service's unit, with no record of this host having made it — kept as it was +// found; or what would reach one: a container mounting found data, an action run in a held +// container (novox/hq ADR 0100, ADR 0103). type Held struct { ID string `json:"id"` Module string `json:"module"` @@ -133,10 +135,15 @@ type Held struct { Owner string `json:"owner,omitempty"` Kept string `json:"kept,omitempty"` - // A container's id as found, and whether it was running. + // A container's id as found, and whether it was running — or a service's unit, whether it + // was running. Container string `json:"container,omitempty"` Running bool `json:"running,omitempty"` + // Why says what was found when it is not the resource's own target: the path or volume a + // container would mount, or the held container an action would run in (novox/hq ADR 0103). + Why string `json:"why,omitempty"` + // Changed is what something other than the mesh has done to it since it was found — // rewritten, stopped, replaced or gone — and empty while it is as found. Reported, never // reverted: that is how a predecessor still writing is caught. From 4a095df2f917cf1aafff42f08ce7e6499b9b49ee Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:14:37 +0200 Subject: [PATCH 23/52] Let go of a hold whose resource is no longer declared, touching nothing on disk (hq ADR 0100) --- internal/apply/apply.go | 18 +++++ internal/apply/hold_test.go | 137 +++++++++++++++++++++++++++++++++++- 2 files changed, 153 insertions(+), 2 deletions(-) diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 650c4fe..f9e8d96 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -202,6 +202,24 @@ func ApplyKeeping( } } + // **A hold whose resource is no longer declared is let go, and nothing on disk is touched.** + // What was found stays as it was; only the host's note that it holds it for a module goes, so + // the node stops reporting a hold for a module no longer assigned. Should the module come back, + // what is there is present with no record and is found, and held, again — its first kept + // original is never overwritten (novox/hq ADR 0100). Only a declaration from the mesh says + // what is assigned: a carried bundle's silence is not an unassignment. + if origin == store.OriginDeclared { + for _, h := range append([]store.Held{}, known.Held...) { + if declared[h.ID] { + continue + } + known.Release(h.ID) + report.Outcomes = append(report.Outcomes, Outcome{ID: h.ID, Type: h.Kind, Target: h.Target, + Action: "forgotten", Detail: "no longer declared; left as found"}) + log(fmt.Sprintf(" forgotten %s (%s): no longer declared; left as found", h.ID, h.Target)) + } + } + // What moved in this apply, so a service that must reflect a file can be told the file // moved. Only within one apply: a change from an earlier one has already been reflected, and // restarting for it every time would make a steady machine restart its services for ever. diff --git a/internal/apply/hold_test.go b/internal/apply/hold_test.go index 9699edd..98813b9 100644 --- a/internal/apply/hold_test.go +++ b/internal/apply/hold_test.go @@ -332,8 +332,141 @@ func TestAHeldFileIsNeverRemovedWhenItsModuleIsUnassigned(t *testing.T) { if m.removed("hello-web") { t.Fatal("a held container was removed when its module left") } - if _, still := state.HeldAt("hello-web.page"); !still { - t.Error("the hold was forgotten, so a return of the module would read the file as the mesh's") + if _, still := state.HeldAt("hello-web.page"); still { + t.Error("a hold outlived its resource leaving the declaration, so the node reports it for ever") + } + if _, recorded := state.Find("hello-web.page"); recorded { + t.Error("a file let go was recorded as the mesh's") + } +} + +func TestAHoldNoLongerDeclaredIsLetGoAndFoundAgainIfItsModuleReturns(t *testing.T) { + dir, page, m := predecessor(t) + _, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) + first, _ := state.HeldAt("hello-web.page") + + report, state := applyAdopted(t, adopted(t, `{"taken":[]}`, withConf(dir)), state, m, dir) + if o := outcomeOf(report, "hello-web.page"); o.Action != "forgotten" || o.Detail != "no longer declared; left as found" { + t.Errorf("letting a hold go was not reported: %+v", o) + } + if len(state.Held) != 0 { + t.Errorf("holds outlived their resources: %+v", state.Held) + } + + // The module comes back: what is there is found again, and the original first kept stays. + if err := os.WriteFile(page, []byte("the predecessor wrote again\n"), 0o640); err != nil { + t.Fatal(err) + } + report, state = applyAdopted(t, adopted(t, untakenWeb, webResources(page)), state, m, dir) + if o := outcomeOf(report, "hello-web.page"); o.Action != "held" { + t.Fatalf("a returning module's found file was not held again: %+v", o) + } + again, _ := state.HeldAt("hello-web.page") + if kept, _ := os.ReadFile(again.Kept); again.Kept != first.Kept || string(kept) != "the predecessor's page\n" { + t.Errorf("the first kept original was lost: %s %q", again.Kept, kept) + } + if got, _ := os.ReadFile(page); string(got) != "the predecessor wrote again\n" { + t.Errorf("the found file was touched: %q", got) + } +} + +func TestACarriedApplyLetsNoHoldGo(t *testing.T) { + dir, page, m := predecessor(t) + _, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) + carried := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`) + _, state, err := ApplyKeeping(context.Background(), archHost(t), carried, state, store.OriginCarried, + m.run, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if len(state.Held) != 2 { + t.Errorf("a carried apply let holds go: %+v", state.Held) + } +} + +func TestAHeldFileRewrittenIsReportedAndNotReverted(t *testing.T) { + dir, page, m := predecessor(t) + d := adopted(t, untakenWeb, webResources(page)) + _, state := applyAdopted(t, d, store.State{}, m, dir) + + if err := os.WriteFile(page, []byte("the predecessor wrote again\n"), 0o640); err != nil { + t.Fatal(err) + } + report, state := applyAdopted(t, d, state, m, dir) + if got, _ := os.ReadFile(page); string(got) != "the predecessor wrote again\n" { + t.Fatalf("a held file was reverted: %q", got) + } + if h, _ := state.HeldAt("hello-web.page"); h.Changed != "rewritten" || h.ChangedAt.IsZero() { + t.Errorf("a rewrite was not recorded: %+v", h) + } + if o := outcomeOf(report, "hello-web.page"); !strings.Contains(o.Detail, "rewritten") { + t.Errorf("a rewrite was not reported: %+v", o) + } + h, _ := state.HeldAt("hello-web.page") + if kept, _ := os.ReadFile(h.Kept); string(kept) != "the predecessor's page\n" { + t.Errorf("the kept original was overwritten by a later write: %q", kept) + } +} + +func TestAHeldContainerStoppedOrReplacedIsReportedAndNotRestarted(t *testing.T) { + for _, c := range []struct { + change func(*machine) + want string + }{ + {func(m *machine) { m.containers["hello-web"].running = false }, "stopped"}, + {func(m *machine) { m.containers["hello-web"].id = "another" }, "replaced"}, + {func(m *machine) { delete(m.containers, "hello-web") }, "gone"}, + } { + dir, page, m := predecessor(t) + d := adopted(t, untakenWeb, webResources(page)) + _, state := applyAdopted(t, d, store.State{}, m, dir) + c.change(m) + m.asked = nil + _, state = applyAdopted(t, d, state, m, dir) + if h, _ := state.HeldAt("hello-web.server"); h.Changed != c.want { + t.Errorf("%s: recorded as %q", c.want, h.Changed) + } + for _, a := range m.asked { + if strings.HasPrefix(a, "docker run") || strings.HasPrefix(a, "docker rm") || + strings.HasPrefix(a, "docker start") { + t.Errorf("%s: the held container was acted on: %s", c.want, a) + } + } + } +} + +func TestAHeldFileThatVanishesIsNotCreated(t *testing.T) { + dir, page, m := predecessor(t) + d := adopted(t, untakenWeb, webResources(page)) + _, state := applyAdopted(t, d, store.State{}, m, dir) + if err := os.Remove(page); err != nil { + t.Fatal(err) + } + _, state = applyAdopted(t, d, state, m, dir) + if _, err := os.Stat(page); !errors.Is(err, os.ErrNotExist) { + t.Fatal("a held file that vanished was created before its module was taken") + } + if h, _ := state.HeldAt("hello-web.page"); h.Changed != "gone" { + t.Errorf("a vanished held file was not reported gone: %+v", h) + } +} + +func TestAConvergedNodeStillReplacesWhatItFinds(t *testing.T) { + // No adoption, no holds: byte for byte what a converged node did before ADR 0100. + dir, page, m := predecessor(t) + d := parse(t, `{"declaration":1,"resources":[`+webResources(page)+`]}`) + report, state := applyAdopted(t, d, store.State{}, m, dir) + if got, _ := os.ReadFile(page); string(got) != "the mesh's page\n" { + t.Errorf("a converged node kept a found file: %q", got) + } + if !m.removed("hello-web") { + t.Error("a converged node kept a found container") + } + if len(state.Held) != 0 || outcomeOf(report, "hello-web.page").Action == "held" { + t.Errorf("a converged node held something: %+v", state.Held) + } + if _, err := os.Stat(filepath.Join(dir, "kept")); !errors.Is(err, os.ErrNotExist) { + t.Error("a converged node kept originals") } } From 48a8f4cf9d2dc445f4f92ae2e8db83cc3dbef078 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:16:00 +0200 Subject: [PATCH 24/52] Point the builder's package binding at the port given with --packages-port (hq ADR 0100) --- internal/bootstrap/builder.go | 27 ++++++++++++ internal/bootstrap/builder_test.go | 69 ++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 internal/bootstrap/builder_test.go diff --git a/internal/bootstrap/builder.go b/internal/bootstrap/builder.go index b86c1b1..e8ca324 100644 --- a/internal/bootstrap/builder.go +++ b/internal/bootstrap/builder.go @@ -1,8 +1,10 @@ package bootstrap import ( + "bytes" "context" "fmt" + "strconv" "strings" ) @@ -53,6 +55,9 @@ func InstallBuilder(ctx context.Context, o Options, d Deps, control controlPlane "This is the manifest that makes the builder an ordinary module. Without it the mesh "+ "has the image and no way to run it, so nothing can be built here", err) } + if manifest, err = followPackagesPort(manifest, o.Ports.orDefaults().Packages); err != nil { + return out, err + } pinned, places, err := pinPlaceholder(manifest, published.Reference, BuilderModule) if err != nil { return out, err @@ -84,3 +89,25 @@ func InstallBuilder(ctx context.Context, o Options, d Deps, control controlPlane out.Installed.Pushed, err = pushNode(ctx, o, control, say) return out, err } + +// packagesPortInBinding is the package registry's port as the builder's manifest names it, in the +// binding file it carries — JSON inside a JSON string, so its quotes are escaped. +const packagesPortInBinding = `\"port\": 3000` + +// followPackagesPort points the builder's package binding at the port the node gave the package +// registry (novox/hq ADR 0100). Genesis raises the registry by hand before gitea is a module, so no +// binding the controller resolves can say where it is; the builder carries the address in its own +// manifest, and a port given at genesis must reach it there or the base build dials a port nothing +// answers on. At the default it is left byte for byte as the catalogue has it. +func followPackagesPort(manifest []byte, port int) ([]byte, error) { + if port == defaultGiteaPort { + return manifest, nil + } + if n := bytes.Count(manifest, []byte(packagesPortInBinding)); n != 1 { + return nil, fmt.Errorf("the builder's manifest names the package registry's port %d time(s) where "+ + "this installer looks for it once (%s), so the port given with --packages-port cannot reach "+ + "it; nothing was changed", n, packagesPortInBinding) + } + return bytes.Replace(manifest, []byte(packagesPortInBinding), + []byte(`\"port\": `+strconv.Itoa(port)), 1), nil +} diff --git a/internal/bootstrap/builder_test.go b/internal/bootstrap/builder_test.go new file mode 100644 index 0000000..c2f9848 --- /dev/null +++ b/internal/bootstrap/builder_test.go @@ -0,0 +1,69 @@ +package bootstrap + +import ( + "encoding/json" + "strings" + "testing" +) + +// Defends novox/hq ADR 0100: a port given for the package registry at genesis reaches the one +// thing that dials it by a fixed number, the builder's package binding. + +// The builder's package binding exactly as the catalogue's manifest carries it. +const builderManifest = `{ + "module": "builder", + "resources": [ + { + "id": "package-binding", + "type": "file", + "path": "/var/lib/mesh/builder/package-registry.json", + "mode": "0600", + "content": "{\"provision\": \"package-registry\", \"from\": \"gitea\", \"at\": \"127.0.0.1\", \"as\": \"mesh-builder\", \"serves\": {\"scheme\": \"http\", \"port\": 3000, \"npm-path\": \"/api/packages/novox/npm/\"}}\n" + } + ] +}` + +func bindingPort(t *testing.T, manifest []byte) float64 { + t.Helper() + var m struct { + Resources []struct { + Content string `json:"content"` + } `json:"resources"` + } + if err := json.Unmarshal(manifest, &m); err != nil { + t.Fatal(err) + } + var binding struct { + Serves struct { + Port float64 `json:"port"` + } `json:"serves"` + } + if err := json.Unmarshal([]byte(m.Resources[0].Content), &binding); err != nil { + t.Fatal(err) + } + return binding.Serves.Port +} + +func TestTheBuilderFollowsThePackageRegistrysGivenPort(t *testing.T) { + got, err := followPackagesPort([]byte(builderManifest), 3100) + if err != nil { + t.Fatal(err) + } + if p := bindingPort(t, got); p != 3100 { + t.Errorf("the builder's binding dials %v, not the port given", p) + } +} + +func TestTheBuilderOnTheDefaultPortIsUnchanged(t *testing.T) { + got, err := followPackagesPort([]byte(builderManifest), 3000) + if err != nil || string(got) != builderManifest { + t.Errorf("the default port changed the manifest: %v", err) + } +} + +func TestABuilderManifestThatNoLongerNamesThePortIsRefused(t *testing.T) { + moved := strings.Replace(builderManifest, `\"port\": 3000`, `\"port\": 3001`, 1) + if _, err := followPackagesPort([]byte(moved), 3100); err == nil || !strings.Contains(err.Error(), "--packages-port") { + t.Errorf("a manifest the port cannot reach was accepted: %v", err) + } +} From facf6af46a55d0f082c7f3a3c89efe1f292ec6b1 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:27:51 +0200 Subject: [PATCH 25/52] Add the mesh's members to a list found in a file written into, and take back only those (hq ADR 0102) --- internal/apply/into.go | 153 ++++++++++++++++++++++++++++++++++-- internal/apply/into_test.go | 76 +++++++++++++++++- internal/store/store.go | 4 + 3 files changed, 224 insertions(+), 9 deletions(-) diff --git a/internal/apply/into.go b/internal/apply/into.go index d2e4c91..4847a21 100644 --- a/internal/apply/into.go +++ b/internal/apply/into.go @@ -49,20 +49,28 @@ func applyInto(r *declaration.File, previous store.Applied) (Outcome, error) { } } - rec := store.Into{Format: declaration.IntoJSON, Before: map[string]json.RawMessage{}} + rec := store.Into{Format: declaration.IntoJSON, Before: map[string]json.RawMessage{}, + Added: map[string][]json.RawMessage{}} if previous.Into != nil { rec.Created = previous.Into.Created for k, v := range previous.Into.Before { rec.Before[k] = v } rec.Absent = slices.Clone(previous.Into.Absent) + for k, v := range previous.Into.Added { + rec.Added[k] = slices.Clone(v) + } } else { rec.Created = !existed } - tracked := func(k string) bool { _, ok := rec.Before[k]; return ok || slices.Contains(rec.Absent, k) } + tracked := func(k string) bool { + _, before := rec.Before[k] + _, added := rec.Added[k] + return before || added || slices.Contains(rec.Absent, k) + } // Drift: the machine no longer holds what this host last set in its keys. - drifted := previous.Wrote != "" && existed && digestOf(keysOf(object, keysTracked(rec))) != previous.Wrote + drifted := previous.Wrote != "" && existed && digestOf(viewOf(object, rec, keysTracked(rec))) != previous.Wrote // Keys the mesh set before and no longer declares go back to what they held. for _, k := range keysTracked(rec) { @@ -71,8 +79,28 @@ func applyInto(r *declaration.File, previous store.Applied) (Outcome, error) { } giveBack(object, &rec, k) } - // Declared keys: remember what each held the first time, then set it. + // Declared keys: remember what each held the first time, then set it. A list is the + // machine's too — a predecessor's own trusted registries, say — so the mesh adds its members + // to it rather than replacing it, and remembers exactly which it added. for _, k := range keysIn(declared) { + _, scalar := rec.Before[k] + if isList(declared[k]) && !scalar { + current, had := object[k] + if had && !isList(current) { + return out, fmt.Errorf("%s: the mesh adds to the list %q, and the machine holds something "+ + "other than a list there; it was left as it is", r.Path, k) + } + if !tracked(k) && !had { + rec.Absent = append(rec.Absent, k) + } + merged, added, err := addMembers(current, declared[k], rec.Added[k]) + if err != nil { + return out, fmt.Errorf("%s: %q: %w", r.Path, k, err) + } + rec.Added[k] = added + object[k] = merged + continue + } if !tracked(k) { if v, had := object[k]; had { rec.Before[k] = v @@ -114,6 +142,19 @@ func applyInto(r *declaration.File, previous store.Applied) (Outcome, error) { return out, fmt.Errorf("%s is not a JSON object after writing into it: %w", r.Path, err) } for k, v := range declared { + if _, list := rec.Added[k]; list { + members, _ := membersOf(v) + have, err := membersOf(check[k]) + if err != nil { + return out, fmt.Errorf("%s does not hold a list at %q after writing into it", r.Path, k) + } + for _, m := range members { + if !hasMember(have, m) { + return out, fmt.Errorf("%s does not hold the declared %s in %q after writing into it", r.Path, m, k) + } + } + continue + } if canonical(check[k]) != canonical(v) { return out, fmt.Errorf("%s does not hold the declared %q after writing into it", r.Path, k) } @@ -122,8 +163,11 @@ func applyInto(r *declaration.File, previous store.Applied) (Outcome, error) { if len(rec.Before) == 0 { rec.Before = nil } + if len(rec.Added) == 0 { + rec.Added = nil + } out.into = &rec - out.wrote = digestOf(keysOf(check, keysIn(declared))) + out.wrote = digestOf(viewOf(check, rec, keysIn(declared))) switch { case !existed: out.Action = "created" @@ -181,6 +225,23 @@ func removeInto(a store.Applied) (string, string, error) { } func giveBack(object map[string]json.RawMessage, rec *store.Into, k string) { + if added, list := rec.Added[k]; list { + // Only the members the mesh added go; the list and everything else in it stay, unless + // the mesh made the key and nothing is left in it. + wasAbsent := slices.Contains(rec.Absent, k) + if current, had := object[k]; had && isList(current) { + have, _ := membersOf(current) + have = slices.DeleteFunc(have, func(m json.RawMessage) bool { return hasMember(added, m) }) + if len(have) == 0 && wasAbsent { + delete(object, k) + } else { + object[k] = listOf(have) + } + } + delete(rec.Added, k) + rec.Absent = slices.DeleteFunc(rec.Absent, func(a string) bool { return a == k }) + return + } if v, had := rec.Before[k]; had { object[k] = v delete(rec.Before, k) @@ -195,19 +256,99 @@ func keysTracked(rec store.Into) []string { for k := range rec.Before { keys = append(keys, k) } + for k := range rec.Added { + keys = append(keys, k) + } keys = append(keys, rec.Absent...) sort.Strings(keys) return slices.Compact(keys) } -func keysOf(object map[string]json.RawMessage, keys []string) string { +// viewOf is what the mesh holds itself to in a file written into: each scalar key's value, and for +// a list only whether each member the mesh added is still there — what the machine keeps beside +// them is not the mesh's to judge. +func viewOf(object map[string]json.RawMessage, rec store.Into, keys []string) string { var b bytes.Buffer for _, k := range keys { + if added, list := rec.Added[k]; list { + have, _ := membersOf(object[k]) + b.WriteString(k + " holds") + for _, m := range added { + fmt.Fprintf(&b, " %s=%v", canonical(m), hasMember(have, m)) + } + b.WriteString("\n") + continue + } b.WriteString(k + "=" + canonical(object[k]) + "\n") } return b.String() } +func isList(raw json.RawMessage) bool { + t := bytes.TrimSpace(raw) + return len(t) > 0 && t[0] == '[' +} + +func membersOf(raw json.RawMessage) ([]json.RawMessage, error) { + if len(bytes.TrimSpace(raw)) == 0 { + return nil, nil + } + var members []json.RawMessage + if err := json.Unmarshal(raw, &members); err != nil { + return nil, err + } + return members, nil +} + +func hasMember(list []json.RawMessage, m json.RawMessage) bool { + for _, have := range list { + if canonical(have) == canonical(m) { + return true + } + } + return false +} + +func listOf(members []json.RawMessage) json.RawMessage { + if members == nil { + members = []json.RawMessage{} + } + raw, _ := json.Marshal(members) + return raw +} + +// addMembers adds the declared members to the machine's list, dropping only members the mesh +// added before and no longer declares. It returns the list and exactly which members the mesh +// added — a declared member the machine already had is the machine's, and is never recorded. +func addMembers(current, declared json.RawMessage, addedBefore []json.RawMessage) (json.RawMessage, + []json.RawMessage, error) { + have, err := membersOf(current) + if err != nil { + return nil, nil, err + } + want, err := membersOf(declared) + if err != nil { + return nil, nil, err + } + added := []json.RawMessage{} + for _, a := range addedBefore { + if hasMember(want, a) { + added = append(added, a) + continue + } + have = slices.DeleteFunc(have, func(m json.RawMessage) bool { return canonical(m) == canonical(a) }) + } + for _, m := range want { + if !hasMember(have, m) { + have = append(have, m) + if !hasMember(added, m) { + added = append(added, m) + } + } + } + return listOf(have), added, nil +} + func keysIn(m map[string]json.RawMessage) []string { keys := make([]string, 0, len(m)) for k := range m { diff --git a/internal/apply/into_test.go b/internal/apply/into_test.go index 421e013..09deb25 100644 --- a/internal/apply/into_test.go +++ b/internal/apply/into_test.go @@ -56,8 +56,8 @@ func TestWritingIntoKeepsEveryKeyTheMachineHad(t *testing.T) { if fmt.Sprint(o["log-opts"]) != "map[max-size:10m]" { t.Errorf("the machine's logging settings were not kept: %v", o) } - if fmt.Sprint(o["insecure-registries"]) != "[10.42.0.1:5000]" { - t.Errorf("the mesh's key was not written: %v", o) + if fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000 10.42.0.1:5000]" { + t.Errorf("the mesh's member was not added beside the machine's own: %v", o) } if info, _ := os.Stat(path); info.Mode().Perm() != 0o600 { t.Errorf("the machine's file mode was changed to %o", info.Mode().Perm()) @@ -179,7 +179,7 @@ func TestAFileWrittenIntoIsNeverHeldOnAnAdoptedNode(t *testing.T) { t.Errorf("something was held: %+v", state.Held) } o := readObject(t, path) - if o["data-root"] != "/srv/docker" || fmt.Sprint(o["insecure-registries"]) != "[10.42.0.1:5000]" { + if o["data-root"] != "/srv/docker" || fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000 10.42.0.1:5000]" { t.Errorf("the adopted node's file was not written into: %v", o) } } @@ -211,3 +211,73 @@ func somethingElse(t *testing.T) *declaration.Declaration { {"id":"other","type":"directory","path":%q} ]}`, filepath.Join(t.TempDir(), "other"))) } + +func TestAListIsAddedToNeverReplaced(t *testing.T) { + // The predecessor's own trusted registries are kept; the mesh adds its own and, undeclared, + // takes back only what it added (novox/hq ADR 0102). + path := filepath.Join(t.TempDir(), "daemon.json") + _ = os.WriteFile(path, []byte(`{"insecure-registries":["192.0.2.7:5000","10.42.0.9:5000"]}`), 0o644) + // 10.42.0.9 is declared too, and was already the machine's: it is never the mesh's to remove. + d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000","10.42.0.9:5000"]}`)) + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if got := fmt.Sprint(readObject(t, path)["insecure-registries"]); got != "[192.0.2.7:5000 10.42.0.9:5000 10.42.0.1:5000]" { + t.Fatalf("the list after writing into it: %s", got) + } + rec, _ := state.Find("networking.registry-trust") + if added := rec.Into.Added["insecure-registries"]; len(added) != 1 || canonical(added[0]) != `"10.42.0.1:5000"` { + t.Errorf("recorded as added: %s", added) + } + + // The predecessor adds a member of its own: not the mesh's drift. + o := readObject(t, path) + o["insecure-registries"] = append(o["insecure-registries"].([]any), "198.51.100.3:5000") + raw, _ := json.Marshal(o) + _ = os.WriteFile(path, raw, 0o644) + report, state, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if got := report.Outcomes[0].Action; got != "unchanged" { + t.Errorf("a member the machine added was taken for drift: %q", got) + } + + // Somebody takes the mesh's member out: that is drift, and it is put back. + o = readObject(t, path) + o["insecure-registries"] = []any{"192.0.2.7:5000", "10.42.0.9:5000", "198.51.100.3:5000"} + raw, _ = json.Marshal(o) + _ = os.WriteFile(path, raw, 0o644) + report, state, err = Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if got := report.Outcomes[0].Action; got != "corrected" { + t.Errorf("the mesh's member removed by hand was %q", got) + } + + // Undeclared: only the member the mesh added goes. + if _, _, err := Apply(context.Background(), archHost(t), somethingElse(t), state, store.OriginDeclared, nil, nil, nil); err != nil { + t.Fatal(err) + } + if got := fmt.Sprint(readObject(t, path)["insecure-registries"]); got != "[192.0.2.7:5000 10.42.0.9:5000 198.51.100.3:5000]" { + t.Errorf("undeclaring took more than the mesh added: %s", got) + } +} + +func TestAListTheMeshCreatedGoesWhenEmptied(t *testing.T) { + path := filepath.Join(t.TempDir(), "daemon.json") + _ = os.WriteFile(path, []byte(`{"data-root":"/srv/docker"}`), 0o644) + d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`)) + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if _, _, err := Apply(context.Background(), archHost(t), somethingElse(t), state, store.OriginDeclared, nil, nil, nil); err != nil { + t.Fatal(err) + } + if o := readObject(t, path); fmt.Sprint(o) != "map[data-root:/srv/docker]" { + t.Errorf("the key the mesh created was not removed: %v", o) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 7eeba41..3f135a6 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -82,6 +82,10 @@ type Into struct { Before map[string]json.RawMessage `json:"before,omitempty"` Absent []string `json:"absent,omitempty"` Created bool `json:"created,omitempty"` + // Added is, for each key whose declared value is a list, exactly the members the mesh added + // to the machine's list — never a member that was already there. Undeclared, only these go, + // and drift is judged on these alone (novox/hq ADR 0102). + Added map[string][]json.RawMessage `json:"added,omitempty"` } // State is the whole of what a node knows about what it has done. From b531c4748674feaf715a51c40e7e2624f3fed065 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:29:06 +0200 Subject: [PATCH 26/52] Refuse an opening ufw would merge into a found rule that does other than a plain allow, and read log types in either place (hq ADR 0103) --- internal/firewall/firewall.go | 41 +++++++++++++++++++++++++++--- internal/firewall/firewall_test.go | 37 ++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/internal/firewall/firewall.go b/internal/firewall/firewall.go index fdc52db..789af22 100644 --- a/internal/firewall/firewall.go +++ b/internal/firewall/firewall.go @@ -479,6 +479,7 @@ func words(rule string) []string { type ufwRule struct { route bool action, in, out string + log string from, fromPort, to string port, proto, app string comment string @@ -488,8 +489,23 @@ type ufwRule struct { // in on mesh0 to any port 5432 proto tcp` — into the fields ufw compares. Not ok for anything it // does not recognise, which is then never taken to answer an opening. func parseRule(rule string) (ufwRule, bool) { - w := words(rule) r := ufwRule{from: "any", to: "any", comment: comment(rule)} + // A log type may stand after the action or after the direction; either way it is a property + // of the rule, not of where it matches. The word after `comment` is the comment, whatever it + // says. + var w []string + all := words(rule) + for i := 0; i < len(all); i++ { + switch { + case all[i] == "comment" && i+1 < len(all): + w = append(w, all[i], all[i+1]) + i++ + case all[i] == "log" || all[i] == "log-all": + r.log = all[i] + default: + w = append(w, all[i]) + } + } i := 0 if i < len(w) && w[i] == "route" { r.route = true @@ -585,9 +601,13 @@ func isPorts(s string) bool { return true } -// sameAs is whether ufw would take two rules for one — everything but the comment equal. +// sameAs is whether ufw would take two rules for one: everything equal but the comment, the +// action and the log type. Adding one beside the other updates it in place — its comment, and its +// action or log type — rather than adding a second. func (r ufwRule) sameAs(o ufwRule) bool { r.comment, o.comment = "", "" + r.action, o.action = "", "" + r.log, o.log = "", "" return r == o } @@ -658,6 +678,7 @@ func Converge(ctx context.Context, run Runner, o *declaration.Opening) (Converge present := false var stale []string satisfiedBy := "" + want, _ := parseRule(strings.Join(Rule(o), " ")) for _, rule := range rules { c := comment(rule) switch { @@ -666,7 +687,21 @@ func Converge(ctx context.Context, run Runner, o *declaration.Opening) (Converge case markedFor(c, o.ID): stale = append(stale, rule) default: - if parsed, ok := parseRule(rule); ok && satisfiedBy == "" && parsed.admits(o) { + parsed, ok := parseRule(rule) + if !ok { + continue + } + // ufw would take the mesh's rule for this one and rewrite its action or log type: + // an operator's refusal, or a limit, would silently become an allow — and removing the + // opening would then delete it. A plain allow answers the opening; anything else is a + // conflict the operator decides (novox/hq ADR 0103). + if parsed.sameAs(want) && (parsed.action != "allow" || parsed.log != "") { + return Converged{}, fmt.Errorf("ufw holds %q, which ufw takes for the same rule as the "+ + "mesh's opening for %s, differing in what it does; adding the opening would change "+ + "it, so nothing was added. Change or remove that rule, or have the mesh stop "+ + "declaring the opening", rule, o.Target()) + } + if satisfiedBy == "" && parsed.admits(o) { satisfiedBy = rule } } diff --git a/internal/firewall/firewall_test.go b/internal/firewall/firewall_test.go index 33e88ad..6d0d98c 100644 --- a/internal/firewall/firewall_test.go +++ b/internal/firewall/firewall_test.go @@ -658,7 +658,6 @@ func TestARuleThatDoesNotAnswerTheOpeningLeavesItToBeAdded(t *testing.T) { "allow from 192.0.2.0/24 to any port 5671 proto tcp", // narrower: from one range "allow in on eth0 to any port 5671 proto tcp", // narrower: one interface "allow 5671/udp", // another protocol - "deny 5671/tcp", // refuses "route allow 5671/tcp", // another path "allow to 192.0.2.1 port 5671 proto tcp", // one address } { @@ -681,3 +680,39 @@ func TestAnOpeningWhoseFoundRuleIsGoneIsAddedAgain(t *testing.T) { t.Fatalf("the opening was not added once nothing answered it: %+v %v", done, err) } } + +func TestARuleUfwWouldMergeThatDoesOtherThanAllowRefusesTheOpening(t *testing.T) { + // ufw takes two rules differing only in action or log type for one, and adding the mesh's + // would turn the operator's refusal into an allow (novox/hq ADR 0103). + for _, operators := range []string{ + "deny 5671/tcp", + "reject 5671/tcp", + "limit 5671/tcp", + "allow log 5671/tcp", + "allow log-all proto tcp to any port 5671", + "deny in log to any port 5671 proto tcp comment 'operator note'", + } { + f := &fakeUFW{installed: true, active: true, rules: []string{operators}} + _, err := Converge(context.Background(), f.run, opening("adoption.bus", 5671, "everywhere", "incoming", 0)) + if err == nil || !strings.Contains(err.Error(), operators) { + t.Errorf("%q: the conflict was not refused naming the rule: %v", operators, err) + } + if f.added() != 0 || len(f.rules) != 1 || f.rules[0] != operators { + t.Errorf("%q: something was added or changed: %v %v", operators, f.asked, f.rules) + } + } +} + +func TestALogTypeIsReadInEitherPlace(t *testing.T) { + for rule, want := range map[string]string{ + "allow log 22/tcp": "allow log 22 tcp in=", + "allow in log-all on mesh0 to any port 5432 proto tcp": "allow log-all 5432 tcp in=mesh0", + "route deny log in on mesh0 to any port 80 proto tcp": "deny log 80 tcp in=mesh0", + "allow 22/tcp comment 'log'": "allow 22 tcp in=", + } { + r, ok := parseRule(rule) + if got := r.action + " " + r.log + " " + r.port + " " + r.proto + " in=" + r.in; !ok || got != want { + t.Errorf("%q read as %q (%v), want %q", rule, got, ok, want) + } + } +} From 491e04fb8f39a8e6c3d53237cfd1cff85ed21787 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:30:13 +0200 Subject: [PATCH 27/52] Load the guard before removing the derived filter when a node returns to adopted, and defer the adoption's orphans only on the flip (hq ADR 0103) --- internal/apply/apply.go | 69 ++++++++++++++++++++++++++++------ internal/apply/opening_test.go | 66 ++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+), 11 deletions(-) diff --git a/internal/apply/apply.go b/internal/apply/apply.go index f9e8d96..873e484 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -185,21 +185,53 @@ func ApplyKeeping( return nil } - // **What protects an adopted node goes last** (novox/hq ADR 0103). The openings and the guard - // are what keep the mesh reachable through the found firewall and the store unreachable from - // outside. When a node is converged they leave the declaration, and removing them first would - // leave the store open from the moment the guard stops until the derived filter loads — and - // for ever, if the filter then fails. So they are removed only once everything else applied - // and the found firewall is retired; if anything failed, they stay, recorded, for the next try. - var protecting []store.Applied + // **What protects an adopted node goes last on the flip, and first on the way back** (novox/hq + // ADR 0103). The openings and the guard are what keep the mesh reachable through the found + // firewall and the store unreachable from outside. + // + // When a node is converged they leave the declaration, and removing them first would leave the + // store open from the moment the guard stops until the derived filter loads — and for ever, if + // the filter then fails. So on a converged declaration they are removed only once everything + // else applied and the found firewall is retired; if anything failed, they stay, recorded, for + // the next try. + // + // Returned to adopted, it is the mirror image: removing the derived filter first would leave + // the store open until the guard loads. So the guard's own resources are applied before any + // orphan is removed, and if a removal then fails the guard is already up. A stale opening on an + // adopted node is removed as any orphan is. + var protecting, orphans []store.Applied for _, orphan := range known.Orphans(declared, origin) { - if strings.HasPrefix(orphan.ID, declaration.AdoptionPrefix) { + if d.Adoption == nil && strings.HasPrefix(orphan.ID, declaration.AdoptionPrefix) { protecting = append(protecting, orphan) continue } - if err := removeOrphan(orphan); err != nil { - return report, known, err + orphans = append(orphans, orphan) + } + ordered := d.Resources + guardFirst := 0 + if d.Adoption != nil { + ordered = nil + for _, r := range d.Resources { + if strings.HasPrefix(r.Identity(), guardPrefix) { + ordered = append(ordered, r) + } } + guardFirst = len(ordered) + for _, r := range d.Resources { + if !strings.HasPrefix(r.Identity(), guardPrefix) { + ordered = append(ordered, r) + } + } + } + orphansRemoved := false + removeOrphans := func() error { + orphansRemoved = true + for _, orphan := range orphans { + if err := removeOrphan(orphan); err != nil { + return err + } + } + return nil } // **A hold whose resource is no longer declared is let go, and nothing on disk is touched.** @@ -256,7 +288,12 @@ func ApplyKeeping( // is a different thing — one is "this machine could not do it", the other is "this was never // a declaration", and they are fixed in different places. var failures []*Error - for _, resource := range d.Resources { + for i, resource := range ordered { + if !orphansRemoved && i == guardFirst { + if err := removeOrphans(); err != nil { + return report, known, err + } + } // **On an adopted node, what is found is kept until its module is taken** (novox/hq ADR // 0100, ADR 0103). Before anything is applied: whatever of a module not yet taken is // present with no record of this host making it — or would reach what is — is held as it @@ -345,6 +382,12 @@ func ApplyKeeping( } } + if !orphansRemoved { + if err := removeOrphans(); err != nil { + return report, known, err + } + } + // A converged node whose found firewall was in force retires it only now, once everything — // the mesh's derived filter among it — applied cleanly (novox/hq ADR 0100). if len(failures) == 0 { @@ -370,6 +413,10 @@ func ApplyKeeping( return report, known, nil } +// guardPrefix is the ids of the mesh's guard on an adopted node: its package, table, unit and +// service (novox/hq ADR 0100). +const guardPrefix = declaration.AdoptionPrefix + "guard" + // Unseal opens a value the mesh sealed to this node. Nil when the node has no sealing key, which // makes every sealed file an error rather than a silently skipped one. type Unseal func(sealed string) ([]byte, error) diff --git a/internal/apply/opening_test.go b/internal/apply/opening_test.go index c02aa79..ccdf79d 100644 --- a/internal/apply/opening_test.go +++ b/internal/apply/opening_test.go @@ -304,3 +304,69 @@ func TestAnOpeningAFoundRuleAnswersIsReportedSatisfied(t *testing.T) { t.Errorf("a rule was added beside the found one: %v", u.rules) } } + +// Defends novox/hq ADR 0103: returned to adopted, the guard is up before the derived filter's +// orphans go, and stays up if removing them fails. +func TestReturningToAdoptedLoadsTheGuardBeforeRemovingTheFilter(t *testing.T) { + dir := t.TempDir() + guard := filepath.Join(dir, "guard.nft") + guardFile := `{"id":"adoption.guard","type":"file","path":"` + guard + `","content":"table inet mesh_guard {}\n"}` + for _, stopFails := range []bool{false, true} { + _ = os.Remove(guard) + guardUpAtStop := false + run := func(_ context.Context, name string, args ...string) (string, error) { + if name != "systemctl" { + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + } + switch args[0] { + case "show": + return "LoadState=loaded\nActiveState=active\nType=oneshot\nRemainAfterExit=yes\n", nil + case "stop": + _, err := os.Stat(guard) + guardUpAtStop = err == nil + if stopFails { + return "", errors.New("the filter would not stop") + } + } + return "", nil + } + converged := store.State{Resources: []store.Applied{ + {ID: "nftables.load", Type: "service", Target: "mesh-filter.service", Origin: store.OriginDeclared}}} + _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, withConf(dir)+","+guardFile), converged, run) + if !guardUpAtStop { + t.Errorf("stop fails %v: the derived filter was stopped before the guard was written", stopFails) + } + if stopFails { + if err == nil { + t.Error("a failed removal was not reported") + } + if _, statErr := os.Stat(guard); statErr != nil { + t.Error("the guard is not up after the filter's removal failed") + } + if _, ok := state.Find("adoption.guard"); !ok { + t.Error("the guard applied before the failure was not recorded") + } + } else if err != nil { + t.Fatal(err) + } + } +} + +func TestAStaleOpeningOnAnAdoptedNodeIsRemovedAsAnyOrphan(t *testing.T) { + // Only the flip defers the adoption's own orphans; an adopted node drops a stale opening at + // once, before what replaces it is applied. + dir := t.TempDir() + u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}} + _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) + if err != nil { + t.Fatal(err) + } + u.asked = nil + other := `{"id":"adoption.opening-tcp-5000-incoming","type":"opening","port":5000,"protocol":"tcp","from":"everywhere","path":"incoming"}` + if _, _, err = applyWith(t, adopted(t, `{"taken":[]}`, other+","+withConf(dir)), state, u.run); err != nil { + t.Fatal(err) + } + if del, add := u.index("ufw delete"), u.index("ufw allow"); del < 0 || add < 0 || del > add { + t.Errorf("the stale opening was not removed before the new one was added: %v", u.asked) + } +} From aef4993d101ad89e7c52e70e8352612174273644 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:32:20 +0200 Subject: [PATCH 28/52] Hold an archive, a process's unit and a user found on an adopted node for an untaken module (hq ADR 0103) --- internal/apply/hold.go | 57 ++++++++++++++++++++++++++++ internal/apply/hold_test.go | 74 +++++++++++++++++++++++++++++++++++++ internal/apply/process.go | 5 ++- 3 files changed, 134 insertions(+), 2 deletions(-) diff --git a/internal/apply/hold.go b/internal/apply/hold.go index fce255d..74ca6c7 100644 --- a/internal/apply/hold.go +++ b/internal/apply/hold.go @@ -72,6 +72,25 @@ func lookBefore(ctx context.Context, sys system.System, d *declaration.Declarati if present(res.Path) && !recordedPath(known, res.Path) { seen["path:"+res.Path] = true } + case *declaration.Archive: + // Unpacking over it, and re-owning it recursively, would change the predecessor's + // files. + if present(res.Path) && !recordedPath(known, res.Path) { + seen["path:"+res.Path] = true + } + case *declaration.Process: + // Its unit would be written over and restarted. + if !known.Recorded(string(declaration.TypeProcess), res.Name) && + present(filepath.Join(unitDir, res.Name+".service")) { + seen["unit-file:"+res.Name] = true + } + case *declaration.User: + // Its shell and groups would be changed. + if !known.Recorded(string(declaration.TypeUser), res.Name) { + if _, exists, err := system.LookUpUser(ctx, run, res.Name); err == nil && exists { + seen["user:"+res.Name] = true + } + } case *declaration.Service: if known.Recorded(string(declaration.TypeService), res.Unit) { continue @@ -239,6 +258,12 @@ func holdOnAdopted(ctx context.Context, sys system.System, r declaration.Resourc } case *declaration.Directory: isFound = before["path:"+res.Path] + case *declaration.Archive: + isFound = before["path:"+res.Path] + case *declaration.Process: + isFound = before["unit-file:"+res.Name] + case *declaration.User: + isFound = before["user:"+res.Name] case *declaration.Service: isFound = before["unit:"+res.Unit] } @@ -396,6 +421,38 @@ func hold(ctx context.Context, sys system.System, r declaration.Resource, module } } detail = "found on the machine; its mode, owner and contents kept until " + module + " is taken" + case *declaration.Archive: + if _, err := os.Lstat(res.Path); errors.Is(err, os.ErrNotExist) { + if !already { + return out, h, fmt.Errorf("%s was found and is gone before it could be held", res.Path) + } + changed = "gone" + } else if err != nil { + return out, h, err + } + detail = "something is already at " + res.Path + "; nothing unpacked over it or re-owned until " + + module + " is taken" + case *declaration.Process: + unit := filepath.Join(unitDir, res.Name+".service") + if !present(unit) { + if !already { + return out, h, fmt.Errorf("%s was found and is gone before it could be held", unit) + } + changed = "gone" + } + detail = "its unit " + unit + " was found on the machine; not written over or restarted until " + + module + " is taken" + case *declaration.User: + _, exists, err := system.LookUpUser(ctx, run, res.Name) + switch { + case err != nil: + return out, h, err + case !exists && !already: + return out, h, fmt.Errorf("the user %s was found and is gone before it could be held", res.Name) + case !exists: + changed = "gone" + } + detail = "the user was found on the machine; its shell and groups are kept until " + module + " is taken" case *declaration.Service: state, err := sys.ServiceState(ctx, run, res.Unit) switch { diff --git a/internal/apply/hold_test.go b/internal/apply/hold_test.go index 98813b9..66148e5 100644 --- a/internal/apply/hold_test.go +++ b/internal/apply/hold_test.go @@ -24,6 +24,7 @@ type machine struct { // named volumes. units map[string]*fakeUnit volumes map[string]bool + users map[string]bool } type fakeUnit struct { @@ -80,6 +81,12 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e if name == "systemctl" { return m.systemctl(args) } + if name == "getent" { + if m.users[args[len(args)-1]] { + return args[len(args)-1] + ":x:1500:1500::/home/" + args[len(args)-1] + ":/bin/bash\n", nil + } + return "", errors.New("exit status 2") + } if name != "docker" { return "", nil } @@ -680,3 +687,70 @@ func TestAnActionInAHeldContainerIsHeldUntilItsModuleIsTaken(t *testing.T) { t.Errorf("holds outlived the take: %+v", state.Held) } } + +const sixtyFourZeros = "0000000000000000000000000000000000000000000000000000000000000000" + +func TestAnArchiveOverSomethingFoundIsNotUnpacked(t *testing.T) { + dir := t.TempDir() + at := filepath.Join(dir, "site") + if err := os.Mkdir(at, 0o750); err != nil { + t.Fatal(err) + } + theirs := filepath.Join(at, "index.html") + _ = os.WriteFile(theirs, []byte("the predecessor's site\n"), 0o640) + m := &machine{containers: map[string]*fakeContainer{}} + // The source is unreachable: fetching it would fail the apply, so a pass means it was not tried. + report, state := applyAdopted(t, adopted(t, untaken("hello-web.site"), + `{"id":"hello-web.site","type":"archive","source":"http://192.0.2.1/site.tar.gz", + "digest":"sha256:`+sixtyFourZeros+`","path":"`+at+`","owner":"root"}`), store.State{}, m, dir) + if o := outcomeOf(report, "hello-web.site"); o.Action != "held" || !strings.Contains(o.Detail, "nothing unpacked") { + t.Errorf("an archive over found files was not held: %+v", o) + } + if got, _ := os.ReadFile(theirs); string(got) != "the predecessor's site\n" { + t.Errorf("the found files were touched: %q", got) + } + if _, ok := state.HeldAt("hello-web.site"); !ok { + t.Error("the hold was not recorded") + } +} + +func TestAProcessWhoseUnitIsFoundIsNotWrittenOverOrRestarted(t *testing.T) { + dir := t.TempDir() + was := unitDir + unitDir = dir + t.Cleanup(func() { unitDir = was }) + unit := filepath.Join(dir, "hello-daemon.service") + _ = os.WriteFile(unit, []byte("[Service]\nExecStart=/opt/predecessor/hello\n"), 0o644) + m := &machine{containers: map[string]*fakeContainer{}} + report, state := applyAdopted(t, adopted(t, untaken("hello-web.daemon"), + `{"id":"hello-web.daemon","type":"process","name":"hello-daemon","source":"http://192.0.2.1/d.tar.gz", + "digest":"sha256:`+sixtyFourZeros+`","run":["hello"]}`), store.State{}, m, dir) + if o := outcomeOf(report, "hello-web.daemon"); o.Action != "held" || !strings.Contains(o.Detail, "not written over or restarted") { + t.Errorf("a process whose unit was found was not held: %+v", o) + } + if got, _ := os.ReadFile(unit); string(got) != "[Service]\nExecStart=/opt/predecessor/hello\n" { + t.Errorf("the found unit was written over: %q", got) + } + if m.did("systemctl") { + t.Errorf("the found unit was touched: %v", m.asked) + } + if _, ok := state.HeldAt("hello-web.daemon"); !ok { + t.Error("the hold was not recorded") + } +} + +func TestAUserFoundOnTheMachineKeepsItsShellAndGroups(t *testing.T) { + dir := t.TempDir() + m := &machine{containers: map[string]*fakeContainer{}, users: map[string]bool{"hello": true}} + report, _ := applyAdopted(t, adopted(t, untaken("hello-web.user"), + `{"id":"hello-web.user","type":"user","name":"hello","shell":"/bin/zsh","groups":["docker"]}`), + store.State{}, m, dir) + if o := outcomeOf(report, "hello-web.user"); o.Action != "held" || !strings.Contains(o.Detail, "shell and groups") { + t.Errorf("a found user was not held: %+v", o) + } + for _, a := range m.asked { + if strings.HasPrefix(a, "usermod") || strings.HasPrefix(a, "useradd") { + t.Errorf("a found user was changed: %s", a) + } + } +} diff --git a/internal/apply/process.go b/internal/apply/process.go index 686b7f1..d9956e7 100644 --- a/internal/apply/process.go +++ b/internal/apply/process.go @@ -32,8 +32,9 @@ import ( // owners end up disagreeing about one path. const daemonRoot = "/var/lib/mesh/daemons" -// unitDir is where the mesh writes the units it owns. -const unitDir = "/etc/systemd/system" +// unitDir is where the mesh writes the units it owns. A variable only so a test can point it at a +// directory of its own. +var unitDir = "/etc/systemd/system" func applyProcess(ctx context.Context, r *declaration.Process, run Runner, changed map[string]bool, previous store.Applied) (Outcome, error) { From 444ad8f3cfc8bc9b0a44c2b5a1f886df9b37711f Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:33:09 +0200 Subject: [PATCH 29/52] Record the forward policies before disabling ufw, so a retried retirement restores them (hq ADR 0100) --- internal/apply/opening.go | 8 ++++- internal/apply/opening_test.go | 49 ++++++++++++++++++++++++++++++ internal/firewall/firewall.go | 38 +++++++++++++---------- internal/firewall/firewall_test.go | 6 ++-- internal/store/store.go | 4 +++ 5 files changed, 85 insertions(+), 20 deletions(-) diff --git a/internal/apply/opening.go b/internal/apply/opening.go index 95012e9..8bd415e 100644 --- a/internal/apply/opening.go +++ b/internal/apply/opening.go @@ -30,6 +30,7 @@ func foundFirewall(ctx context.Context, d *declaration.Declaration, known *store return "", err } rec.DisabledByMesh = false + rec.Forward = nil log(" enabled ufw again: this node is adopted, and the firewall found on it is in force") } kind, name, err := firewall.Detect(ctx, run) @@ -68,7 +69,12 @@ func retireFirewall(ctx context.Context, d *declaration.Declaration, origin stri rec.DisabledByMesh { return nil } - if err := firewall.Disable(ctx, run); err != nil { + if rec.Forward == nil { + // Recorded before ufw is touched: disabling it opens the forward policy, and a retry + // must know what it was (novox/hq ADR 0100). + rec.Forward = firewall.ForwardPolicies(ctx, run) + } + if err := firewall.Disable(ctx, run, rec.Forward); err != nil { return err } rec.DisabledByMesh = true diff --git a/internal/apply/opening_test.go b/internal/apply/opening_test.go index ccdf79d..a81a317 100644 --- a/internal/apply/opening_test.go +++ b/internal/apply/opening_test.go @@ -21,6 +21,23 @@ type ufwMachine struct { rules []string ruleset string asked []string + + // forward is iptables' forward policy when set; empty is a machine without iptables. failP + // is how many -P calls fail before one succeeds. + forward string + failP int +} + +func (u *ufwMachine) iptables(args []string) (string, error) { + if len(args) == 3 && args[0] == "-P" { + if u.failP > 0 { + u.failP-- + return "", errors.New("iptables: resource temporarily unavailable") + } + u.forward = args[2] + return "", nil + } + return "-P FORWARD " + u.forward + "\n-A FORWARD -j DOCKER-USER\n", nil } func (u *ufwMachine) run(_ context.Context, name string, args ...string) (string, error) { @@ -28,6 +45,11 @@ func (u *ufwMachine) run(_ context.Context, name string, args ...string) (string switch name { case "nft": return u.ruleset, nil + case "iptables": + if u.forward == "" { + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + } + return u.iptables(args) case "ufw": if !u.installed { return "", &exec.Error{Name: name, Err: exec.ErrNotFound} @@ -52,6 +74,9 @@ func (u *ufwMachine) run(_ context.Context, name string, args ...string) (string return "", nil case "disable": u.active = false + if u.forward != "" { + u.forward = "ACCEPT" // as measured: ufw disable opens the forward policy + } return "", nil case "delete": want := strings.Join(args[1:], " ") @@ -370,3 +395,27 @@ func TestAStaleOpeningOnAnAdoptedNodeIsRemovedAsAnyOrphan(t *testing.T) { t.Errorf("the stale opening was not removed before the new one was added: %v", u.asked) } } + +func TestARetiredFirewallRetriedStillPutsBackTheForwardPolicy(t *testing.T) { + // The forward policy is recorded before ufw is disabled, so a retirement that failed after + // the disable restores what the machine had, not what the disable left (novox/hq ADR 0100). + dir := t.TempDir() + u := &ufwMachine{installed: true, active: true, forward: "DROP", failP: 1} + _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, withConf(dir)), store.State{}, u.run) + if err != nil { + t.Fatal(err) + } + converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`) + if _, state, err = applyWith(t, converged, state, u.run); err == nil { + t.Fatal("the failed restore was not reported") + } + if u.active || u.forward != "ACCEPT" || state.Firewall.Forward["iptables"] != "DROP" { + t.Fatalf("after the failed attempt: active %v, forward %s, recorded %+v", u.active, u.forward, state.Firewall) + } + if _, state, err = applyWith(t, converged, state, u.run); err != nil { + t.Fatal(err) + } + if u.forward != "DROP" || !state.Firewall.DisabledByMesh { + t.Errorf("the retry did not put the forward policy back: %s, %+v", u.forward, state.Firewall) + } +} diff --git a/internal/firewall/firewall.go b/internal/firewall/firewall.go index 789af22..262310d 100644 --- a/internal/firewall/firewall.go +++ b/internal/firewall/firewall.go @@ -806,40 +806,46 @@ func Enable(ctx context.Context, run Runner) error { // runtime had set that one to drop when it turned forwarding on, and it does not set it again while // forwarding stays on — not even on a restart. Left so, a retired ufw turns the machine into a // router for anyone who can reach it. So each family's forward policy is read before, and one that -// was drop is put back and read back. -func Disable(ctx context.Context, run Runner) error { - type family struct{ tool, policy string } - var before []family - for _, tool := range []string{"iptables", "ip6tables"} { - if policy, ok := forwardPolicy(ctx, run, tool); ok { - before = append(before, family{tool, policy}) - } - } +// was drop is put back and read back. before is ForwardPolicies as read before the first attempt. +func Disable(ctx context.Context, run Runner, before map[string]string) error { if _, err := run(ctx, "ufw", "disable"); err != nil { return fmt.Errorf("disabling ufw: %w", err) } if err := expectActive(ctx, run, false); err != nil { return err } - for _, f := range before { - if f.policy != "DROP" { + for _, tool := range []string{"iptables", "ip6tables"} { + if before[tool] != "DROP" { continue } - if now, ok := forwardPolicy(ctx, run, f.tool); ok && now == "DROP" { + if now, ok := forwardPolicy(ctx, run, tool); ok && now == "DROP" { continue } - if _, err := run(ctx, f.tool, "-P", "FORWARD", "DROP"); err != nil { + if _, err := run(ctx, tool, "-P", "FORWARD", "DROP"); err != nil { return fmt.Errorf("ufw is disabled, and %s's forward policy, which was drop, could not be put back: %w", - f.tool, err) + tool, err) } - if now, ok := forwardPolicy(ctx, run, f.tool); !ok || now != "DROP" { + if now, ok := forwardPolicy(ctx, run, tool); !ok || now != "DROP" { return fmt.Errorf("ufw is disabled, and %s's forward policy was put back to drop and reads %q", - f.tool, now) + tool, now) } } return nil } +// ForwardPolicies reads each family's forward policy, by the tool that sets it. Read before ufw is +// disabled and kept by the caller, so a retirement that fails half-way is retried with what the +// machine had — not with what the half-done disable left. +func ForwardPolicies(ctx context.Context, run Runner) map[string]string { + out := map[string]string{} + for _, tool := range []string{"iptables", "ip6tables"} { + if policy, ok := forwardPolicy(ctx, run, tool); ok { + out[tool] = policy + } + } + return out +} + // forwardPolicy reads the forward chain's policy the way iptables prints it: "-P FORWARD DROP". // Not ok when the tool is absent or says nothing readable. func forwardPolicy(ctx context.Context, run Runner, tool string) (string, bool) { diff --git a/internal/firewall/firewall_test.go b/internal/firewall/firewall_test.go index 6d0d98c..02d14bf 100644 --- a/internal/firewall/firewall_test.go +++ b/internal/firewall/firewall_test.go @@ -356,7 +356,7 @@ func TestRemovingAnOpeningRemovesOnlyWhatWasMarkedForIt(t *testing.T) { func TestEnableAndDisableReadBack(t *testing.T) { f := &fakeUFW{installed: true, active: true} - if err := Disable(context.Background(), f.run); err != nil || f.active { + if err := Disable(context.Background(), f.run, nil); err != nil || f.active { t.Fatalf("disable: %v", err) } if err := Enable(context.Background(), f.run); err != nil || !f.active { @@ -493,7 +493,7 @@ func TestRetiringUfwKeepsTheMachineFromRoutingForOthers(t *testing.T) { t.Fatal("the captures no longer show ufw disable opening the forward policy") } f := &fakeUFW{installed: true, active: true, iptablesActive: string(before), iptablesInactive: string(after)} - if err := Disable(context.Background(), f.run); err != nil { + if err := Disable(context.Background(), f.run, ForwardPolicies(context.Background(), f.run)); err != nil { t.Fatal(err) } if f.active { @@ -511,7 +511,7 @@ func TestRetiringUfwKeepsTheMachineFromRoutingForOthers(t *testing.T) { func TestRetiringUfwOnAMachineWithoutIptablesStillRetiresIt(t *testing.T) { f := &fakeUFW{installed: true, active: true} - if err := Disable(context.Background(), f.run); err != nil || f.active { + if err := Disable(context.Background(), f.run, nil); err != nil || f.active { t.Fatalf("disable: %v, active %v", err, f.active) } } diff --git a/internal/store/store.go b/internal/store/store.go index 3f135a6..60aa169 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -116,6 +116,10 @@ type FoundFirewall struct { // DisabledByMesh is set when converging retired it, so returning to adopted enables it again // and nothing else ever does. DisabledByMesh bool `json:"disabled_by_mesh,omitempty"` + // Forward is each family's forward policy as it was before the mesh disabled the firewall, + // by the tool that sets it — recorded before, so a retirement retried puts back what the + // machine had. + Forward map[string]string `json:"forward,omitempty"` FoundAt time.Time `json:"found_at"` } From a4e46320779d7b95eb565aab478ae4015d0f4950 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:33:13 +0200 Subject: [PATCH 30/52] gofmt the store's firewall record --- internal/store/store.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/store/store.go b/internal/store/store.go index 60aa169..d2a2da7 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -115,12 +115,12 @@ type FoundFirewall struct { WasActive bool `json:"was_active,omitempty"` // DisabledByMesh is set when converging retired it, so returning to adopted enables it again // and nothing else ever does. - DisabledByMesh bool `json:"disabled_by_mesh,omitempty"` + DisabledByMesh bool `json:"disabled_by_mesh,omitempty"` // Forward is each family's forward policy as it was before the mesh disabled the firewall, // by the tool that sets it — recorded before, so a retirement retried puts back what the // machine had. Forward map[string]string `json:"forward,omitempty"` - FoundAt time.Time `json:"found_at"` + FoundAt time.Time `json:"found_at"` } // Held is one thing found on an adopted node — a file, directory or container present at a declared From bd3fd17ad851067c4ae321fee73fe90ee3870524 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:33:29 +0200 Subject: [PATCH 31/52] Do not count the machine's own plumbing mounted into a container as found data (hq ADR 0103) --- internal/apply/hold.go | 22 +++++++++++++++++++++- internal/apply/hold_test.go | 17 +++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/internal/apply/hold.go b/internal/apply/hold.go index 74ca6c7..818ecc4 100644 --- a/internal/apply/hold.go +++ b/internal/apply/hold.go @@ -109,7 +109,7 @@ func lookBefore(ctx context.Context, sys system.System, d *declaration.Declarati switch { case src == "": case strings.HasPrefix(src, "/"): - if present(src) && !recordedPath(known, src) { + if !systemPath(src) && present(src) && !recordedPath(known, src) { seen["path:"+src] = true } default: @@ -146,6 +146,26 @@ func recordedPath(known store.State, path string) bool { return false } +// systemPath is whether a bind-mount source is the machine's own plumbing — the runtime's socket, +// the kernel's filesystems, the devices, the clock — which every machine has and no predecessor's +// data lives in. Mounting it shares nothing that was found. +func systemPath(src string) bool { + clean := filepath.Clean(src) + for _, exact := range []string{"/etc/localtime", "/etc/timezone", "/etc/hosts", "/etc/resolv.conf", + "/etc/machine-id", "/etc/passwd", "/etc/group"} { + if clean == exact { + return true + } + } + for _, under := range []string{"/run", "/var/run", "/sys", "/proc", "/dev", "/usr/share/zoneinfo", + "/etc/ssl", "/etc/ca-certificates", "/etc/pki", "/lib/modules", "/usr/lib/modules"} { + if clean == under || strings.HasPrefix(clean, under+"/") { + return true + } + } + return false +} + // mountSource is what a volume mapping mounts: a path on the machine, or a named volume. Empty for // an anonymous volume, which mounts nothing that could already be there. func mountSource(mapping string) string { diff --git a/internal/apply/hold_test.go b/internal/apply/hold_test.go index 66148e5..a6c65b0 100644 --- a/internal/apply/hold_test.go +++ b/internal/apply/hold_test.go @@ -754,3 +754,20 @@ func TestAUserFoundOnTheMachineKeepsItsShellAndGroups(t *testing.T) { } } } + +func TestMountingTheMachinesOwnPlumbingIsNotFoundData(t *testing.T) { + // The runtime's socket, the kernel's filesystems and the clock are on every machine; a + // container mounting them shares nothing a predecessor kept (novox/hq ADR 0103). + dir := t.TempDir() + m := &machine{containers: map[string]*fakeContainer{}} + report, state := applyAdopted(t, adopted(t, untaken("hello-web.server"), + `{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`", + "volumes":["/var/run/docker.sock:/var/run/docker.sock","/etc/localtime:/etc/localtime:ro", + "/proc/cpuinfo:/host/cpuinfo:ro","/dev/null:/data/null"]}`), store.State{}, m, dir) + if o := outcomeOf(report, "hello-web.server"); o.Action != "created" { + t.Errorf("a container mounting only system paths was not created: %+v", o) + } + if len(state.Held) != 0 { + t.Errorf("held: %+v", state.Held) + } +} From b4f3eaf11b4829bfff1aa31538909a387a963cad Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:33:45 +0200 Subject: [PATCH 32/52] Release a whole-file hold once the file is declared written into (hq ADR 0102) --- internal/apply/hold.go | 9 +++++++++ internal/apply/into_test.go | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/internal/apply/hold.go b/internal/apply/hold.go index 818ecc4..f763dac 100644 --- a/internal/apply/hold.go +++ b/internal/apply/hold.go @@ -235,6 +235,15 @@ func holdOnAdopted(ctx context.Context, sys system.System, r declaration.Resourc } } + // A file written into replaces nothing that was found, so it is never held (novox/hq ADR + // 0102) — and a hold from when it was declared whole must not keep the mesh's keys out. + if f, ok := r.(*declaration.File); ok && f.Into != "" { + if already { + known.Release(r.Identity()) + } + return false, false, out, nil + } + module, untaken := d.Adoption.UntakenModuleOf(r.Identity()) if !untaken { return false, false, out, nil diff --git a/internal/apply/into_test.go b/internal/apply/into_test.go index 09deb25..58a7962 100644 --- a/internal/apply/into_test.go +++ b/internal/apply/into_test.go @@ -281,3 +281,24 @@ func TestAListTheMeshCreatedGoesWhenEmptied(t *testing.T) { t.Errorf("the key the mesh created was not removed: %v", o) } } + +func TestAHoldFromAWholeFileDoesNotKeepOutAnIntoWrite(t *testing.T) { + // Declared whole before, the runtime's file was held; declared into now, it is written into. + path := filepath.Join(t.TempDir(), "daemon.json") + _ = os.WriteFile(path, []byte(machinesOwn), 0o644) + known := store.State{Held: []store.Held{{ID: "networking.registry-trust", Module: "networking", + Kind: "file", Target: path}}} + d := adopted(t, `{"taken":[],"untaken":{"networking":["networking.registry-trust"]}}`, + fmt.Sprintf(`{"id":"networking.registry-trust","type":"file","path":%q,"into":"json","content":%q}`, + path, `{"insecure-registries":["10.42.0.1:5000"]}`)) + report, state := applyAdopted(t, d, known, &machine{}, t.TempDir()) + if got := outcomeOf(report, "networking.registry-trust").Action; got != "updated" { + t.Errorf("the file was %q, not written into", got) + } + if len(state.Held) != 0 { + t.Errorf("the old hold outlived the into declaration: %+v", state.Held) + } + if fmt.Sprint(readObject(t, path)["insecure-registries"]) != "[192.0.2.7:5000 10.42.0.1:5000]" { + t.Errorf("the mesh's member was not written in: %v", readObject(t, path)) + } +} From 272e1a65ea4d42c40b45a0adf4f9b53ce45e5bf5 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:33:57 +0200 Subject: [PATCH 33/52] Reload the guard for a changed table and restart it only for a changed unit, as the controller declares it (hq ADR 0103) --- internal/bootstrap/adopted.go | 5 ++++- internal/bootstrap/adopted_test.go | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/bootstrap/adopted.go b/internal/bootstrap/adopted.go index 2a1ee1b..ed7207c 100644 --- a/internal/bootstrap/adopted.go +++ b/internal/bootstrap/adopted.go @@ -87,8 +87,11 @@ func guardResources(ports []int) []map[string]any { return []map[string]any{ {"id": guardID, "type": "file", "path": guardPath, "content": AsGuard(ports), "mode": "0644"}, {"id": guardUnitID, "type": "file", "path": guardUnitPath, "content": guardUnitText(), "mode": "0644"}, + // A changed table is reloaded — the unit's ExecReload loads it in one transaction, so the + // ports are never unguarded — and only a changed unit restarts it. As the controller + // declares it, so the first push finds nothing different. {"id": guardRunningID, "type": "service", "unit": guardUnit, "state": "running", - "boot": "enabled", "restart-on": []any{guardID, guardUnitID}}, + "boot": "enabled", "reload-on": []any{guardID}, "restart-on": []any{guardUnitID}}, } } diff --git a/internal/bootstrap/adopted_test.go b/internal/bootstrap/adopted_test.go index cbe3a98..f52e0e1 100644 --- a/internal/bootstrap/adopted_test.go +++ b/internal/bootstrap/adopted_test.go @@ -121,7 +121,10 @@ func TestAnAdoptedBundleLoadsNoDroppingTableAndExactlyTheGuard(t *testing.T) { t.Error("nft, which loads the guard, is no longer installed") } unit := r.Declaration.Resources[at[guardRunningID]].(*declaration.Service) - if unit.Unit != guardUnit || unit.State != "running" || strings.Join(unit.RestartOn, ",") != guardID+","+guardUnitID { + // A changed table is reloaded, never restarted: a restart deletes the table before loading + // it again, leaving the ports unguarded in between. + if unit.Unit != guardUnit || unit.State != "running" || strings.Join(unit.RestartOn, ",") != guardUnitID || + strings.Join(unit.ReloadOn, ",") != guardID { t.Errorf("the guard's service: %+v", unit) } stop := r.Declaration.Resources[at[guardUnitID]].(*declaration.File).Content From bde5e3461cec23d8e47dedb5c43f2ee0a30e53b6 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:35:03 +0200 Subject: [PATCH 34/52] Keep the original of any file the host writes over without a record of it, on every node, and name where (hq ADR 0100) --- internal/apply/apply.go | 31 +++++++++++++++++++++++++++---- internal/apply/hold_test.go | 36 ++++++++++++++++++++++++++++++++++-- 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 873e484..1f20090 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -322,7 +322,14 @@ func ApplyKeeping( if o, isOpening := resource.(*declaration.Opening); isOpening { outcome, err = applyOpening(ctx, o, run, fw) } else { - outcome, err = applyOne(ctx, sys, resource, run, changed, declares, was, unseal) + // A file this host has no record of, under any id, is the machine's until the mesh + // writes over it — on any node, adopted or not: its original is kept first. + var keepFound Keep + if f, isFile := resource.(*declaration.File); isFile && was.ID == "" && + !known.Recorded(string(declaration.TypeFile), f.Path) { + keepFound = keep + } + outcome, err = applyOne(ctx, sys, resource, run, changed, declares, was, unseal, keepFound) } if err != nil { failed := &Error{Resource: resource.Identity(), Err: err, Done: report} @@ -423,12 +430,12 @@ type Unseal func(sealed string) ([]byte, error) func applyOne(ctx context.Context, sys system.System, r declaration.Resource, run Runner, changed map[string]bool, declares map[string]string, previous store.Applied, - unseal Unseal) (Outcome, error) { + unseal Unseal, keepFound Keep) (Outcome, error) { switch res := r.(type) { case *declaration.Directory: return applyDirectory(res) case *declaration.File: - return applyFile(res, previous, unseal) + return applyFile(res, previous, unseal, keepFound) case *declaration.Service: return applyService(ctx, sys, res, run, changed) case *declaration.Package: @@ -573,7 +580,9 @@ func applyAccess(r *declaration.Access) (Outcome, error) { return out, nil } -func applyFile(r *declaration.File, previous store.Applied, unseal Unseal) (Outcome, error) { +// keepFound, when not nil, is where the original of a file this host has no record of is kept +// before it is written over (novox/hq ADR 0100): once, never overwritten, and named in the outcome. +func applyFile(r *declaration.File, previous store.Applied, unseal Unseal, keepFound Keep) (Outcome, error) { if r.Into != "" { return applyInto(r, previous) } @@ -670,7 +679,15 @@ func applyFile(r *declaration.File, previous store.Applied, unseal Unseal) (Outc drifted := existed && previous.Wrote != "" && digestOf(string(existing)) != previous.Wrote modeSame := existed && beforeMode == mode.Perm() + kept := "" if !contentSame { + if existed && keepFound != nil { + // Before anything is written: a keep that fails stops the write, since the + // original could not be had back otherwise. + if kept, err = keepFound(r.Path, existing, beforeMode); err != nil { + return out, fmt.Errorf("keeping the original of %s before writing over it: %w", r.Path, err) + } + } if err := os.MkdirAll(filepath.Dir(r.Path), 0o755); err != nil { return out, err } @@ -734,6 +751,12 @@ func applyFile(r *declaration.File, previous store.Applied, unseal Unseal) (Outc default: out.Action = "unchanged" } + if kept != "" { + if out.Detail != "" { + out.Detail += "; " + } + out.Detail += "the file found here, which the mesh had no record of, was kept at " + kept + } return out, nil } diff --git a/internal/apply/hold_test.go b/internal/apply/hold_test.go index a6c65b0..5c7b03a 100644 --- a/internal/apply/hold_test.go +++ b/internal/apply/hold_test.go @@ -472,8 +472,40 @@ func TestAConvergedNodeStillReplacesWhatItFinds(t *testing.T) { if len(state.Held) != 0 || outcomeOf(report, "hello-web.page").Action == "held" { t.Errorf("a converged node held something: %+v", state.Held) } - if _, err := os.Stat(filepath.Join(dir, "kept")); !errors.Is(err, os.ErrNotExist) { - t.Error("a converged node kept originals") + // What it writes over that it has no record of, it keeps first — on any node. + o := outcomeOf(report, "hello-web.page") + kept := o.Detail[strings.Index(o.Detail, "kept at ")+len("kept at "):] + if got, err := os.ReadFile(kept); err != nil || string(got) != "the predecessor's page\n" { + t.Errorf("the file written over was not kept, or not named: %q (%s) %v", got, o.Detail, err) + } +} + +func TestAFileWrittenOverIsKeptOnceAndOnlyWhenTheHostHasNoRecordOfIt(t *testing.T) { + dir := t.TempDir() + conf := filepath.Join(dir, "nftables.conf") + _ = os.WriteFile(conf, []byte("# the distribution's own\n"), 0o644) + decl := func(content string) *declaration.Declaration { + return parse(t, `{"declaration":1,"resources":[{"id":"nftables.config","type":"file","path":"`+conf+ + `","content":"`+content+`"}]}`) + } + m := &machine{containers: map[string]*fakeContainer{}} + report, state := applyAdopted(t, decl("table inet mesh {}\\n"), store.State{}, m, dir) + o := outcomeOf(report, "nftables.config") + if !strings.Contains(o.Detail, "had no record of, was kept at ") { + t.Fatalf("writing over an unrecorded file did not keep it: %+v", o) + } + kept := o.Detail[strings.Index(o.Detail, "kept at ")+len("kept at "):] + if got, _ := os.ReadFile(kept); string(got) != "# the distribution's own\n" { + t.Errorf("the kept original is %q", got) + } + + // Now the mesh's: a later change keeps nothing more, and the first original stays. + report, _ = applyAdopted(t, decl("table inet mesh { }\\n"), state, m, dir) + if o := outcomeOf(report, "nftables.config"); o.Action != "updated" || strings.Contains(o.Detail, "kept at") { + t.Errorf("a file the mesh wrote was kept again: %+v", o) + } + if got, _ := os.ReadFile(kept); string(got) != "# the distribution's own\n" { + t.Errorf("the first original was overwritten: %q", got) } } From 60bb3d895c8dd79621afb1acec8e833ea549897e Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:47:48 +0200 Subject: [PATCH 35/52] Count a unit as found only when the machine runs it or starts it at boot, so a packaged unit nothing ran is not held (hq ADR 0103, found by the adoption bed) --- internal/apply/hold.go | 13 ++++++++++--- internal/apply/hold_test.go | 24 +++++++++++++++++++++++- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/internal/apply/hold.go b/internal/apply/hold.go index f763dac..a71bf4d 100644 --- a/internal/apply/hold.go +++ b/internal/apply/hold.go @@ -95,9 +95,16 @@ func lookBefore(ctx context.Context, sys system.System, d *declaration.Declarati if known.Recorded(string(declaration.TypeService), res.Unit) { continue } - // A unit the service manager cannot find is not there; one it can read is, whatever - // state it is in. - if _, err := sys.ServiceState(ctx, run, res.Unit); err == nil { + // Found is what the machine runs: a unit that is running or starts at boot. A unit + // file a package merely ships — a template instance nothing ever started, say the + // private network's own wg-quick@mesh0 — is not a predecessor's service, and holding + // it kept the private network from ever coming up (found by the adoption bed). + state, err := sys.ServiceState(ctx, run, res.Unit) + if err != nil { + continue + } + boot, _ := sys.ServiceBoot(ctx, run, res.Unit) + if state == "running" || boot == "enabled" { seen["unit:"+res.Unit] = true } case *declaration.Container: diff --git a/internal/apply/hold_test.go b/internal/apply/hold_test.go index 5c7b03a..b2efcb6 100644 --- a/internal/apply/hold_test.go +++ b/internal/apply/hold_test.go @@ -570,7 +570,8 @@ func TestAFoundServiceOfAnUntakenModuleIsNeitherStartedNorEnabledNorRestarted(t dir := t.TempDir() conf := filepath.Join(dir, "hello.conf") m := &machine{containers: map[string]*fakeContainer{}, - units: map[string]*fakeUnit{"hello.service": {active: "inactive", enabled: "disabled"}}} + // The predecessor's unit: stopped just now, but it starts at boot, so it is the machine's. + units: map[string]*fakeUnit{"hello.service": {active: "inactive", enabled: "enabled"}}} report, state := applyAdopted(t, adopted(t, untaken("hello-web.conf", "hello-web.unit"), `{"id":"hello-web.conf","type":"file","path":"`+conf+`","content":"x\n"}, {"id":"hello-web.unit","type":"service","unit":"hello.service","state":"running","boot":"enabled", @@ -610,6 +611,27 @@ func TestAHeldServiceIsStillReloadedButNeverRestarted(t *testing.T) { } } +func TestAUnitAPackageOnlyShipsIsNotFound(t *testing.T) { + // The adoption bed found this: the private network's wg-quick@mesh0 is an instance of a unit + // the tunnel package ships. Nothing had ever run it, yet it was held as a predecessor's, and + // the private network never came up. Found is what the machine runs. + dir := t.TempDir() + m := &machine{containers: map[string]*fakeContainer{}, + units: map[string]*fakeUnit{"wg-quick@mesh0.service": {active: "inactive", enabled: "disabled"}}} + report, state := applyAdopted(t, adopted(t, untaken("mesh-wireguard.overlay-up"), + `{"id":"mesh-wireguard.overlay-up","type":"service","unit":"wg-quick@mesh0.service","state":"running","boot":"enabled"}`), + store.State{}, m, dir) + if o := outcomeOf(report, "mesh-wireguard.overlay-up"); o.Action == "held" { + t.Fatalf("a unit nothing runs was held as a predecessor's: %+v", o) + } + if !m.did("systemctl start wg-quick@mesh0.service") || !m.did("systemctl enable wg-quick@mesh0.service") { + t.Errorf("the unit was not started and enabled: %v", m.asked) + } + if len(state.Held) != 0 { + t.Errorf("held: %+v", state.Held) + } +} + func TestAServiceWhoseUnitIsNotThereIsAppliedAsUsual(t *testing.T) { // No unit before the apply: nothing of a predecessor's to hold. dir := t.TempDir() From 0ea646b3849e71a8d83682fe0eaadf715ec5532d Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 19:45:33 +0200 Subject: [PATCH 36/52] Keep the derived filter in force when a guard resource failed on the way back to adopted (hq ADR 0103) --- internal/apply/apply.go | 11 ++++++++++ internal/apply/opening_test.go | 40 ++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 1f20090..d473f3f 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -290,6 +290,17 @@ func ApplyKeeping( var failures []*Error for i, resource := range ordered { if !orphansRemoved && i == guardFirst { + // **Only a guard that is up may let the filter go.** Removing the derived filter's + // resources stops its unit, whose stop deletes the mesh's table; if a guard resource + // failed, doing that would leave the node with neither, and the store open until some + // later reconcile gets the guard up (novox/hq ADR 0103). + if len(failures) > 0 { + first := failures[0] + first.Done = report + first.Others = len(failures) - 1 + log(" kept " + guardPrefix + "*: the guard is not up, so what it replaces was left in force") + return report, known, first + } if err := removeOrphans(); err != nil { return report, known, err } diff --git a/internal/apply/opening_test.go b/internal/apply/opening_test.go index a81a317..d14fbcf 100644 --- a/internal/apply/opening_test.go +++ b/internal/apply/opening_test.go @@ -371,6 +371,9 @@ func TestReturningToAdoptedLoadsTheGuardBeforeRemovingTheFilter(t *testing.T) { if _, ok := state.Find("adoption.guard"); !ok { t.Error("the guard applied before the failure was not recorded") } + if _, still := state.Find("nftables.load"); !still { + t.Error("the filter that would not stop was forgotten, so nothing would stop it later") + } } else if err != nil { t.Fatal(err) } @@ -419,3 +422,40 @@ func TestARetiredFirewallRetriedStillPutsBackTheForwardPolicy(t *testing.T) { t.Errorf("the retry did not put the forward policy back: %s, %+v", u.forward, state.Firewall) } } + +func TestAGuardThatFailsLeavesTheDerivedFilterInForce(t *testing.T) { + // Returning to adopted: if the guard cannot be raised, the filter it replaces must not be + // stopped — its stop deletes the mesh's table, and the node would have neither (novox/hq ADR 0103). + dir := t.TempDir() + blocked := filepath.Join(dir, "not-a-directory") + if err := os.WriteFile(blocked, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + guardFile := `{"id":"adoption.guard","type":"file","path":"` + filepath.Join(blocked, "guard.nft") + + `","content":"table inet mesh_guard {}\n"}` + stopped := false + run := func(_ context.Context, name string, args ...string) (string, error) { + if name != "systemctl" { + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + } + if args[0] == "show" { + return "LoadState=loaded\nActiveState=active\nType=oneshot\nRemainAfterExit=yes\n", nil + } + if args[0] == "stop" { + stopped = true + } + return "", nil + } + converged := store.State{Resources: []store.Applied{ + {ID: "nftables.load", Type: "service", Target: "mesh-filter.service", Origin: store.OriginDeclared}}} + _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, withConf(dir)+","+guardFile), converged, run) + if err == nil { + t.Fatal("a guard that could not be written reported success") + } + if stopped { + t.Error("the derived filter was stopped though the guard is not up") + } + if _, gone := state.Find("nftables.load"); !gone { + t.Error("the filter was forgotten, so nothing would ever stop it") + } +} From da008460ac7ac0cce75cdaa87e6a4b8c77490783 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 19:46:50 +0200 Subject: [PATCH 37/52] Fail a resource when the container runtime cannot answer, instead of reading silence as nothing there (hq ADR 0100) --- internal/apply/hold.go | 74 ++++++++++++++++++++++++++-------- internal/apply/hold_test.go | 79 +++++++++++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 16 deletions(-) diff --git a/internal/apply/hold.go b/internal/apply/hold.go index a71bf4d..8c2497e 100644 --- a/internal/apply/hold.go +++ b/internal/apply/hold.go @@ -51,11 +51,21 @@ func KeepIn(dir string) Keep { // record (novox/hq ADR 0103). Looked at first, because the apply itself makes such things — a // file's parent directory, a unit file a module writes, a package that brings its unit — and what // the mesh made in this apply was not found. -type foundBefore map[string]bool +type foundBefore struct { + is map[string]bool + // trouble is what could not be asked about, by the same key, so the resource that would need + // the answer fails rather than proceeding as if the machine had nothing there. + trouble map[string]string +} + +func (f foundBefore) has(key string) bool { return f.is[key] } + +// why is the reason a key could not be settled, and empty when there was none. +func (f foundBefore) why(key string) string { return f.trouble[key] } func lookBefore(ctx context.Context, sys system.System, d *declaration.Declaration, known store.State, run Runner) foundBefore { - seen := foundBefore{} + seen := foundBefore{is: map[string]bool{}, trouble: map[string]string{}} if d.Adoption == nil { return seen } @@ -70,25 +80,25 @@ func lookBefore(ctx context.Context, sys system.System, d *declaration.Declarati switch res := r.(type) { case *declaration.Directory: if present(res.Path) && !recordedPath(known, res.Path) { - seen["path:"+res.Path] = true + seen.is["path:"+res.Path] = true } case *declaration.Archive: // Unpacking over it, and re-owning it recursively, would change the predecessor's // files. if present(res.Path) && !recordedPath(known, res.Path) { - seen["path:"+res.Path] = true + seen.is["path:"+res.Path] = true } case *declaration.Process: // Its unit would be written over and restarted. if !known.Recorded(string(declaration.TypeProcess), res.Name) && present(filepath.Join(unitDir, res.Name+".service")) { - seen["unit-file:"+res.Name] = true + seen.is["unit-file:"+res.Name] = true } case *declaration.User: // Its shell and groups would be changed. if !known.Recorded(string(declaration.TypeUser), res.Name) { if _, exists, err := system.LookUpUser(ctx, run, res.Name); err == nil && exists { - seen["user:"+res.Name] = true + seen.is["user:"+res.Name] = true } } case *declaration.Service: @@ -105,7 +115,7 @@ func lookBefore(ctx context.Context, sys system.System, d *declaration.Declarati } boot, _ := sys.ServiceBoot(ctx, run, res.Unit) if state == "running" || boot == "enabled" { - seen["unit:"+res.Unit] = true + seen.is["unit:"+res.Unit] = true } case *declaration.Container: if known.Recorded(string(declaration.TypeContainer), res.Name) { @@ -117,7 +127,7 @@ func lookBefore(ctx context.Context, sys system.System, d *declaration.Declarati case src == "": case strings.HasPrefix(src, "/"): if !systemPath(src) && present(src) && !recordedPath(known, src) { - seen["path:"+src] = true + seen.is["path:"+src] = true } default: if !asked { @@ -128,7 +138,10 @@ func lookBefore(ctx context.Context, sys system.System, d *declaration.Declarati continue } if _, err := run(ctx, cri, "volume", "inspect", src); err == nil { - seen["volume:"+src] = true + seen.is["volume:"+src] = true + } else if !absent(err) { + seen.trouble["volume:"+src] = fmt.Sprintf( + "the container runtime could not say whether the volume %s is here: %v", src, err) } } } @@ -287,21 +300,28 @@ func holdOnAdopted(ctx context.Context, sys system.System, r declaration.Resourc if strings.HasPrefix(src, "/") { key = "path:" + src } - if src != "" && before[key] { + if src == "" { + continue + } + if trouble := before.why(key); trouble != "" { + return false, false, begin(r), fmt.Errorf( + "%s, so it is not safe to create a container that would mount it", trouble) + } + if before.has(key) { isFound, why = true, "would mount "+src+", found on the machine" break } } case *declaration.Directory: - isFound = before["path:"+res.Path] + isFound = before.has("path:" + res.Path) case *declaration.Archive: - isFound = before["path:"+res.Path] + isFound = before.has("path:" + res.Path) case *declaration.Process: - isFound = before["unit-file:"+res.Name] + isFound = before.has("unit-file:" + res.Name) case *declaration.User: - isFound = before["user:"+res.Name] + isFound = before.has("user:" + res.Name) case *declaration.Service: - isFound = before["unit:"+res.Unit] + isFound = before.has("unit:" + res.Unit) } } if !isFound { @@ -373,7 +393,16 @@ func inspectFound(ctx context.Context, name string, run Runner) (foundContainer, out, err := run(ctx, cri, "inspect", "--format", "{{.Id}}\t{{.State.Running}}\t{{index .Config.Labels \""+specLabel+"\"}}", name) if err != nil { - return foundContainer{}, false, nil + if absent(err) { + return foundContainer{}, false, nil + } + // **A runtime that could not answer is not a machine with nothing there.** Read as + // absence, a daemon that is down or a permission denied would let the mesh create its own + // container over a predecessor's — the one thing an adopted node must never do + // (novox/hq ADR 0100). + return foundContainer{}, false, fmt.Errorf( + "the container runtime could not say whether %s is here, so it is not safe to make one: %w", + name, err) } parts := strings.Split(strings.TrimSpace(out), "\t") for len(parts) < 3 { @@ -386,6 +415,19 @@ func inspectFound(ctx context.Context, name string, run Runner) (foundContainer, return foundContainer{id: strings.TrimSpace(parts[0]), running: parts[1] == "true", spec: spec}, true, nil } +// absent is whether a runtime said the thing is not there, rather than failing to answer. Its own +// words: docker and podman both say "No such object", "No such container" or "No such volume". +func absent(err error) bool { + said := strings.ToLower(err.Error()) + for _, missing := range []string{"no such object", "no such container", "no such volume", + "no such image"} { + if strings.Contains(said, missing) { + return true + } + } + return false +} + // hold keeps a found file or container as it is, and reports it — the first time by recording // what was found, every time after by comparing against that. Nothing is reverted, restarted or // created: a held target that disappears stays held and gone until its module is taken. diff --git a/internal/apply/hold_test.go b/internal/apply/hold_test.go index b2efcb6..cdf8232 100644 --- a/internal/apply/hold_test.go +++ b/internal/apply/hold_test.go @@ -825,3 +825,82 @@ func TestMountingTheMachinesOwnPlumbingIsNotFoundData(t *testing.T) { t.Errorf("held: %+v", state.Held) } } + +// Defends novox/hq ADR 0100: a runtime that cannot answer is not a machine with nothing there. + +// unreachable is a machine whose container runtime answers everything with a daemon that is down. +type unreachable struct{ asked []string } + +func (u *unreachable) run(_ context.Context, name string, args ...string) (string, error) { + u.asked = append(u.asked, name+" "+strings.Join(args, " ")) + if name == "docker" && args[0] == "info" { + return "27.0\n", nil + } + if name == "docker" { + return "", errors.New("docker exited 1: Cannot connect to the Docker daemon at unix:///var/run/docker.sock") + } + return "", nil +} + +func TestARuntimeThatCannotAnswerNeverLetsTheMeshCreateOverAFoundContainer(t *testing.T) { + dir := t.TempDir() + u := &unreachable{} + d := adopted(t, untaken("hello-web.server"), + `{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`"}`) + _, state, err := ApplyKeeping(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, + u.run, nil, nil, KeepIn(dir)) + if err == nil || !strings.Contains(err.Error(), "not safe to make one") { + t.Fatalf("a runtime that could not answer was read as nothing there: %v", err) + } + for _, a := range u.asked { + if strings.HasPrefix(a, "docker run") || strings.HasPrefix(a, "docker rm") { + t.Errorf("the mesh acted on a container it could not ask about: %s", a) + } + } + if len(state.Held) != 0 { + t.Errorf("something was held on an answer the machine never gave: %+v", state.Held) + } +} + +func TestAHeldContainerStaysHeldWhenTheRuntimeCannotAnswer(t *testing.T) { + dir, page, m := predecessor(t) + d := adopted(t, untaken("hello-web.page", "hello-web.server"), webResources(page)) + _, state := applyAdopted(t, d, store.State{}, m, dir) + if _, held := state.HeldAt("hello-web.server"); !held { + t.Fatal("the found container was not held to begin with") + } + u := &unreachable{} + _, state, err := ApplyKeeping(context.Background(), archHost(t), d, state, store.OriginDeclared, + u.run, nil, nil, KeepIn(dir)) + if err == nil { + t.Fatal("a runtime that could not answer reported success") + } + if h, held := state.HeldAt("hello-web.server"); !held || h.Changed != "" { + t.Errorf("a hold was let go or called changed on an answer the machine never gave: %+v", h) + } +} + +func TestAVolumeTheRuntimeCannotBeAskedAboutStopsTheContainer(t *testing.T) { + dir := t.TempDir() + run := func(_ context.Context, name string, args ...string) (string, error) { + switch { + case name == "docker" && args[0] == "info": + return "27.0\n", nil + case name == "docker" && args[0] == "volume": + return "", errors.New("docker exited 1: Cannot connect to the Docker daemon") + case name == "docker" && args[0] == "inspect": + return "", errors.New("Error: No such object: hello-web") + case name == "docker": + return "", errors.New("docker run must not happen") + } + return "", nil + } + d := adopted(t, untaken("hello-web.server"), + `{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`", + "volumes":["predecessor-data:/data"]}`) + _, _, err := ApplyKeeping(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, + run, nil, nil, KeepIn(dir)) + if err == nil || !strings.Contains(err.Error(), "could not say whether the volume") { + t.Fatalf("a volume the runtime could not be asked about did not stop the container: %v", err) + } +} From dc861fb4a8ca7be65018736eebbf1c3e69a7afe9 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 19:48:05 +0200 Subject: [PATCH 38/52] Do not arm a scheduled step of a module held as found on an adopted node (hq ADR 0103) --- cmd/mesh-host/main.go | 6 +++- internal/apply/schedule.go | 14 ++++++++- internal/apply/schedule_test.go | 55 ++++++++++++++++++++++++++++++--- 3 files changed, 68 insertions(+), 7 deletions(-) diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index cbc7718..168ab4e 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -843,7 +843,11 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D // declared is forgotten — and after a host restart the first apply rebuilds them all. A nil // scheduler is the one-shot CLI path, which exits rather than staying up to fire anything. if sched != nil { - sched.Sync(declared) + held := map[string]bool{} + for _, h := range updated.Held { + held[h.ID] = true + } + sched.Sync(declared, held) } report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw)} diff --git a/internal/apply/schedule.go b/internal/apply/schedule.go index 3057559..ccede49 100644 --- a/internal/apply/schedule.go +++ b/internal/apply/schedule.go @@ -86,7 +86,11 @@ func NewScheduler(clock Clock, run Runner, log func(string)) *Scheduler { // A job whose declaration is unchanged keeps its place in the cadence — its next due time and // whether a run is in flight — so an ordinary reconcile every few minutes does not keep resetting // the clock out from under a schedule and prevent it ever firing. -func (s *Scheduler) Sync(d *declaration.Declaration) { +// held is the ids this node holds as found — what an adopted node keeps until its module is taken +// (novox/hq ADR 0100). A step of a module not yet taken is not armed: run on its cadence it would +// work on the predecessor's data, under the predecessor's service, which is the one thing an +// adopted node must not do. Nil on a converged node, where nothing is held. +func (s *Scheduler) Sync(d *declaration.Declaration, held map[string]bool) { s.mu.Lock() defer s.mu.Unlock() @@ -96,6 +100,14 @@ func (s *Scheduler) Sync(d *declaration.Declaration) { if !ok || c.Schedule == "" { continue } + if module, untaken := d.Adoption.UntakenModuleOf(c.Identity()); untaken || held[c.Identity()] { + if module == "" { + module = "its module" + } + s.log(fmt.Sprintf("scheduled step %s: not armed while %s is held as found on this node", + c.Identity(), module)) + continue + } cron, err := declaration.ParseCron(c.Schedule) if err != nil { // The declaration parser already refused a malformed cron before this runs, so a diff --git a/internal/apply/schedule_test.go b/internal/apply/schedule_test.go index fcca78c..715b0db 100644 --- a/internal/apply/schedule_test.go +++ b/internal/apply/schedule_test.go @@ -244,7 +244,7 @@ func TestAScheduledStepRunsWhenDueAndNotBefore(t *testing.T) { d := &declaration.Declaration{Version: 1, Resources: []declaration.Resource{ scheduledContainer(t, "* * * * *"), // every minute; next due 12:01:00 }} - s.Sync(d) + s.Sync(d, nil) // Not yet due: 12:00:45 is before 12:01:00, so nothing runs. s.Advance(context.Background(), time.Date(2026, 9, 7, 12, 0, 45, 0, time.UTC)) @@ -306,7 +306,7 @@ func TestAFailedRunIsRecordedAndDoesNotFailAnything(t *testing.T) { s := NewScheduler(clock, run, log) s.Sync(&declaration.Declaration{Version: 1, Resources: []declaration.Resource{ scheduledContainer(t, "* * * * *"), - }}) + }}, nil) // Fire a run that exits non-zero. Advance returns nothing — there is no error to fail an apply, // because the run happens outside any apply and outside the store. @@ -371,7 +371,7 @@ func TestASlowRunSkipsTheNextDueRunRatherThanStacking(t *testing.T) { s := NewScheduler(clock, run, log) s.Sync(&declaration.Declaration{Version: 1, Resources: []declaration.Resource{ scheduledContainer(t, "* * * * *"), // every minute - }}) + }}, nil) // First occurrence: 12:01 is due — starts a run that blocks in the runner. s.Advance(context.Background(), time.Date(2026, 9, 7, 12, 1, 5, 0, time.UTC)) @@ -414,7 +414,7 @@ func TestSyncForgetsAScheduleTheDeclarationNoLongerNames(t *testing.T) { s.Sync(&declaration.Declaration{Version: 1, Resources: []declaration.Resource{ scheduledContainer(t, "* * * * *"), - }}) + }}, nil) s.mu.Lock() have := len(s.jobs) s.mu.Unlock() @@ -427,10 +427,55 @@ func TestSyncForgetsAScheduleTheDeclarationNoLongerNames(t *testing.T) { parseTrusted(t, `{"declaration":1,"resources":[ {"id":"web","type":"container","name":"web","image":"`+pinned+`"} ]}`).Resources[0], - }}) + }}, nil) s.Advance(context.Background(), time.Date(2026, 9, 7, 12, 5, 5, 0, time.UTC)) s.Wait() if n := rec.fireCount(); n != 0 { t.Errorf("a schedule the declaration no longer names still fired (%d run(s))", n) } } + +// Defends novox/hq ADR 0103: a scheduled step of a module not yet taken is not armed — run on its +// cadence it would work on the predecessor's data. +func TestAScheduledStepOfAnUntakenModuleIsNotArmed(t *testing.T) { + clock := &fixedClock{now: time.Date(2026, 9, 7, 12, 0, 30, 0, time.UTC)} + rec := &recordingRun{} + var said []string + s := NewScheduler(clock, rec.run, func(line string) { said = append(said, line) }) + + d := &declaration.Declaration{Version: 1, + Adoption: &declaration.Adoption{Untaken: map[string][]string{"backups": {"sync"}}}, + Resources: []declaration.Resource{ + scheduledContainer(t, "* * * * *"), + }} + s.Sync(d, nil) + s.Advance(context.Background(), time.Date(2026, 9, 7, 12, 1, 5, 0, time.UTC)) + s.Wait() + if rec.fireCount() != 0 { + t.Errorf("a held module's scheduled step ran %d time(s)", rec.fireCount()) + } + if len(said) == 0 || !strings.Contains(said[0], "held as found") { + t.Errorf("nothing said why the step was not armed: %v", said) + } + + // Held by id — a step whose own container was found on the machine. + s2 := NewScheduler(clock, rec.run, nil) + s2.Sync(&declaration.Declaration{Version: 1, Resources: []declaration.Resource{ + scheduledContainer(t, "* * * * *"), + }}, map[string]bool{"sync": true}) + s2.Advance(context.Background(), time.Date(2026, 9, 7, 12, 1, 5, 0, time.UTC)) + s2.Wait() + if rec.fireCount() != 0 { + t.Errorf("a held scheduled step ran %d time(s)", rec.fireCount()) + } + + // Taken: the same step is armed and runs. + taken := &declaration.Declaration{Version: 1, Adoption: &declaration.Adoption{Taken: []string{"backups"}}, + Resources: []declaration.Resource{scheduledContainer(t, "* * * * *")}} + s.Sync(taken, nil) + s.Advance(context.Background(), time.Date(2026, 9, 7, 12, 2, 5, 0, time.UTC)) + s.Wait() + if rec.fireCount() == 0 { + t.Error("a taken module's scheduled step never ran") + } +} From d3f25959683902b0c3257665dd10cdc2f9ffc50e Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 19:51:35 +0200 Subject: [PATCH 39/52] Read a ufw rule's direction: an outgoing rule answers no opening, and incoming is the default ufw merges on (hq ADR 0103) --- internal/firewall/firewall.go | 19 ++++++- internal/firewall/firewall_test.go | 56 +++++++++++++++++--- internal/firewall/testdata/ufw-direction.txt | 21 ++++++++ internal/firewall/testdata/ufw-forms.txt | 6 +++ 4 files changed, 92 insertions(+), 10 deletions(-) create mode 100644 internal/firewall/testdata/ufw-direction.txt diff --git a/internal/firewall/firewall.go b/internal/firewall/firewall.go index 262310d..fbf98be 100644 --- a/internal/firewall/firewall.go +++ b/internal/firewall/firewall.go @@ -479,7 +479,10 @@ func words(rule string) []string { type ufwRule struct { route bool action, in, out string - log string + // dir is which way the rule matches: "" (ufw's default, incoming and forwarded), "in" or + // "out". A rule on the outgoing path admits nothing that arrives. + dir string + log string from, fromPort, to string port, proto, app string comment string @@ -529,6 +532,7 @@ func parseRule(rule string) (ufwRule, bool) { iface = w[i+1] i += 2 } + r.dir = dir if dir == "in" { r.in = iface } else { @@ -586,6 +590,12 @@ func parseRule(rule string) (ufwRule, bool) { if r.proto == "any" { r.proto = "" } + // Incoming is ufw's default direction, and it prints `allow in 9005/tcp` back as + // `allow 9005/tcp` and merges the two — captured in testdata/ufw-direction.txt. An outgoing + // rule is its own rule and stays one. + if r.dir == "in" && r.in == "" { + r.dir = "" + } return r, true } @@ -616,10 +626,15 @@ func (r ufwRule) sameAs(o ufwRule) bool { // protocol — and on any interface, or the private network's for an opening from it. func (r ufwRule) admits(o *declaration.Opening) bool { want, ok := parseRule(strings.Join(Rule(o), " ")) - if !ok || r.route != want.route || r.action != "allow" || r.out != "" || r.app != "" || + if !ok || r.route != want.route || r.action != "allow" || r.app != "" || r.from != "any" || r.fromPort != "" || r.to != "any" { return false } + // A rule on the outgoing path lets this machine reach others; it admits nothing that arrives, + // so it never answers an opening. + if r.dir == "out" || r.out != "" { + return false + } if r.proto != "" && r.proto != want.proto { return false } diff --git a/internal/firewall/firewall_test.go b/internal/firewall/firewall_test.go index 02d14bf..426c7d5 100644 --- a/internal/firewall/firewall_test.go +++ b/internal/firewall/firewall_test.go @@ -597,18 +597,24 @@ func TestUfwTakesTheMeshsRuleAndTheOperatorsForOne(t *testing.T) { func TestEveryCapturedRuleFormIsRead(t *testing.T) { want := map[string]string{ "allow 22/tcp": "tcp 22 in= from=any", "allow 9200": " 9200 in= from=any", - "allow from 192.0.2.0/24 to any port 9300 proto tcp": "tcp 9300 in= from=192.0.2.0/24", - "allow in on eth0 to any port 9301 proto tcp": "tcp 9301 in=eth0 from=any", - "allow 9500:9510/tcp": "tcp 9500:9510 in= from=any", - "allow 80,443/tcp": "tcp 80,443 in= from=any", - "allow in on mesh0 to any port 5432 proto tcp": "tcp 5432 in=mesh0 from=any", - "route allow 8080/tcp": "tcp 8080 in= from=any", - "allow 9900/tcp": "tcp 9900 in= from=any", + "allow from 192.0.2.0/24 to any port 9300 proto tcp": "tcp 9300 in= from=192.0.2.0/24", + "allow in on eth0 to any port 9301 proto tcp": "tcp 9301 in=eth0 from=any", + "allow 9500:9510/tcp": "tcp 9500:9510 in= from=any", + "allow 80,443/tcp": "tcp 80,443 in= from=any", + "allow in on mesh0 to any port 5432 proto tcp": "tcp 5432 in=mesh0 from=any", + "route allow 8080/tcp": "tcp 8080 in= from=any", + "allow 9900/tcp": "tcp 9900 in= from=any", + "allow out 5671/tcp": "tcp 5671 in= from=any", + "deny out 5672/tcp": "tcp 5672 in= from=any", + "allow out on eth0 to any port 5673 proto tcp": "tcp 5673 in= from=any", + "allow log 9001/tcp": "tcp 9001 in= from=any", + "route allow log 8084/tcp": "tcp 8084 in= from=any", + "allow in on mesh0 log-all to any port 9002 proto tcp": "tcp 9002 in=mesh0 from=any", } rules, err := added(context.Background(), func(context.Context, string, ...string) (string, error) { return captured(t, "ufw-forms.txt"), nil }) - if err != nil || len(rules) != 15 { + if err != nil || len(rules) != 21 { t.Fatalf("read %d rules: %v", len(rules), err) } for _, rule := range rules { @@ -716,3 +722,37 @@ func TestALogTypeIsReadInEitherPlace(t *testing.T) { } } } + +func TestAnOutgoingRuleNeverAnswersAnOpening(t *testing.T) { + // `ufw allow out 5671/tcp` lets this machine reach others; nothing arrives through it, and + // ufw keeps it as a rule of its own — captured in testdata/ufw-direction.txt. + raw := captured(t, "ufw-direction.txt") + if !strings.Contains(raw, "ufw allow out 9007/tcp\nufw allow 9007/tcp") { + t.Fatalf("the capture no longer shows an outgoing rule standing beside an incoming one:\n%s", raw) + } + for _, operators := range []string{"allow out 5671/tcp", "allow out on eth0 to any port 5671 proto tcp", + "deny out 5671/tcp"} { + f := &fakeUFW{installed: true, active: true, rules: []string{operators}} + done, err := Converge(context.Background(), f.run, opening("adoption.bus", 5671, "everywhere", "incoming", 0)) + if err != nil { + t.Errorf("%q: an outgoing rule was taken for a conflict: %v", operators, err) + continue + } + if done.Action != "created" || done.SatisfiedBy != "" { + t.Errorf("%q: an outgoing rule answered an incoming opening: %+v", operators, done) + } + } +} + +func TestIncomingIsUfwsDefaultDirection(t *testing.T) { + // Captured: `deny in 9006/tcp` and `allow 9006/tcp` are one rule to ufw, so the mesh must read + // them as one too, or it would take an operator's refusal over. + raw := captured(t, "ufw-direction.txt") + if !strings.Contains(raw, "ufw allow 9005/tcp") || strings.Contains(raw, "ufw deny 9006/tcp") { + t.Fatalf("the capture no longer shows `in` as the default direction:\n%s", raw) + } + f := &fakeUFW{installed: true, active: true, rules: []string{"deny in to any port 5671 proto tcp"}} + if _, err := Converge(context.Background(), f.run, opening("adoption.bus", 5671, "everywhere", "incoming", 0)); err == nil { + t.Error("an incoming refusal ufw would merge was not refused") + } +} diff --git a/internal/firewall/testdata/ufw-direction.txt b/internal/firewall/testdata/ufw-direction.txt new file mode 100644 index 0000000..0fa5371 --- /dev/null +++ b/internal/firewall/testdata/ufw-direction.txt @@ -0,0 +1,21 @@ +$ ufw allow in 9005/tcp +Rules updated +Rules updated (v6) +$ ufw deny in 9006/tcp +Rules updated +Rules updated (v6) +$ ufw allow 9006/tcp +Rules updated +Rules updated (v6) +$ ufw allow out 9007/tcp +Rules updated +Rules updated (v6) +$ ufw allow 9007/tcp +Rules updated +Rules updated (v6) +$ ufw show added +Added user rules (see 'ufw status' for running firewall): +ufw allow 9005/tcp +ufw allow 9006/tcp +ufw allow out 9007/tcp +ufw allow 9007/tcp diff --git a/internal/firewall/testdata/ufw-forms.txt b/internal/firewall/testdata/ufw-forms.txt index a39c9c0..7c89bca 100644 --- a/internal/firewall/testdata/ufw-forms.txt +++ b/internal/firewall/testdata/ufw-forms.txt @@ -14,3 +14,9 @@ ufw allow 9900/tcp ufw allow 80,443/tcp ufw allow in on mesh0 to any port 5432 proto tcp ufw route allow 8080/tcp +ufw allow out 5671/tcp +ufw deny out 5672/tcp +ufw allow out on eth0 to any port 5673 proto tcp +ufw allow log 9001/tcp +ufw route allow log 8084/tcp +ufw allow in on mesh0 log-all to any port 9002 proto tcp From 40e8ea9fda0b4b780b994a673826b02415b815c0 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 19:52:37 +0200 Subject: [PATCH 40/52] Hold a unit an administrator installed whatever its state, and a packaged unit only when the machine uses it (hq ADR 0103) --- internal/apply/hold.go | 36 +++++++++++++++++++++--- internal/apply/hold_test.go | 56 ++++++++++++++++++++++++++++++++++++- internal/system/arch.go | 16 +++++++++++ 3 files changed, 103 insertions(+), 5 deletions(-) diff --git a/internal/apply/hold.go b/internal/apply/hold.go index 8c2497e..941fc43 100644 --- a/internal/apply/hold.go +++ b/internal/apply/hold.go @@ -105,14 +105,28 @@ func lookBefore(ctx context.Context, sys system.System, d *declaration.Declarati if known.Recorded(string(declaration.TypeService), res.Unit) { continue } - // Found is what the machine runs: a unit that is running or starts at boot. A unit - // file a package merely ships — a template instance nothing ever started, say the - // private network's own wg-quick@mesh0 — is not a predecessor's service, and holding - // it kept the private network from ever coming up (found by the adoption bed). + // **Found is a unit somebody put on this machine, or one the machine uses.** + // + // Where it comes from first: a unit the service manager loads from outside /usr — + // /etc/systemd/system or /run/systemd/system — was installed by an administrator, so + // it is a predecessor's whatever state it is in, and one deliberately stopped and + // disabled must stay that way (novox/hq ADR 0103). + // + // A unit a package ships, under /usr, is not held by its mere presence: the private + // network's own wg-quick@mesh0 is an instance of a template the tunnel package ships, + // nothing had ever run it, and holding it kept the private network from ever coming up + // (found by the adoption bed). Such a unit is held only if the machine actually uses + // it — running, or started at boot. state, err := sys.ServiceState(ctx, run, res.Unit) if err != nil { continue } + if from, ok := sys.(unitFiles); ok { + if path, err := from.ServiceUnitFile(ctx, run, res.Unit); err == nil && installedByHand(path) { + seen.is["unit:"+res.Unit] = true + continue + } + } boot, _ := sys.ServiceBoot(ctx, run, res.Unit) if state == "running" || boot == "enabled" { seen.is["unit:"+res.Unit] = true @@ -150,6 +164,20 @@ func lookBefore(ctx context.Context, sys system.System, d *declaration.Declarati return seen } +// unitFiles is a service manager that can say where it loads a unit from. +type unitFiles interface { + ServiceUnitFile(ctx context.Context, run Runner, unit string) (string, error) +} + +// installedByHand is whether a unit file is one somebody put on this machine rather than one a +// package ships: anywhere but /usr, where distributions keep what they install. +func installedByHand(path string) bool { + if path == "" { + return false + } + return !strings.HasPrefix(filepath.Clean(path), "/usr/") +} + func present(path string) bool { _, err := os.Lstat(path) return err == nil diff --git a/internal/apply/hold_test.go b/internal/apply/hold_test.go index cdf8232..6b493f6 100644 --- a/internal/apply/hold_test.go +++ b/internal/apply/hold_test.go @@ -29,6 +29,9 @@ type machine struct { type fakeUnit struct { active, enabled string + // fragment is where systemd loads the unit from; empty means /etc/systemd/system, where an + // administrator installs one. + fragment string } // systemctl answers as systemd does for the units the machine has, and "not-found" for any other. @@ -41,8 +44,18 @@ func (m *machine) systemctl(args []string) (string, error) { switch args[0] { case "show": if !ok { + if len(args) > 2 && strings.Contains(args[2], "FragmentPath") { + return "FragmentPath=\n", nil + } return "LoadState=not-found\nActiveState=inactive\nType=simple\n", nil } + if len(args) > 2 && strings.Contains(args[2], "FragmentPath") { + from := u.fragment + if from == "" { + from = "/etc/systemd/system/" + unit + } + return "FragmentPath=" + from + "\n", nil + } return "LoadState=loaded\nActiveState=" + u.active + "\nType=simple\nRemainAfterExit=no\n", nil case "is-enabled": if !ok { @@ -617,7 +630,8 @@ func TestAUnitAPackageOnlyShipsIsNotFound(t *testing.T) { // the private network never came up. Found is what the machine runs. dir := t.TempDir() m := &machine{containers: map[string]*fakeContainer{}, - units: map[string]*fakeUnit{"wg-quick@mesh0.service": {active: "inactive", enabled: "disabled"}}} + units: map[string]*fakeUnit{"wg-quick@mesh0.service": {active: "inactive", enabled: "disabled", + fragment: "/usr/lib/systemd/system/wg-quick@.service"}}} report, state := applyAdopted(t, adopted(t, untaken("mesh-wireguard.overlay-up"), `{"id":"mesh-wireguard.overlay-up","type":"service","unit":"wg-quick@mesh0.service","state":"running","boot":"enabled"}`), store.State{}, m, dir) @@ -904,3 +918,43 @@ func TestAVolumeTheRuntimeCannotBeAskedAboutStopsTheContainer(t *testing.T) { t.Fatalf("a volume the runtime could not be asked about did not stop the container: %v", err) } } + +func TestAUnitSomebodyInstalledIsHeldWhateverStateItIsIn(t *testing.T) { + // A predecessor's unit under /etc, deliberately stopped and disabled: starting it would put + // back a service somebody took down on purpose (novox/hq ADR 0103). + dir := t.TempDir() + m := &machine{containers: map[string]*fakeContainer{}, + units: map[string]*fakeUnit{"hello.service": {active: "inactive", enabled: "disabled", + fragment: "/etc/systemd/system/hello.service"}}} + report, state := applyAdopted(t, adopted(t, untaken("hello-web.unit"), + `{"id":"hello-web.unit","type":"service","unit":"hello.service","state":"running","boot":"enabled"}`), + store.State{}, m, dir) + if o := outcomeOf(report, "hello-web.unit"); o.Action != "held" { + t.Fatalf("a unit an administrator installed was not held: %+v", o) + } + if m.did("systemctl start") || m.did("systemctl enable") { + t.Errorf("a unit somebody had stopped and disabled was started: %v", m.asked) + } + if _, ok := state.HeldAt("hello-web.unit"); !ok { + t.Error("the hold was not recorded") + } +} + +func TestAPackagedUnitTheMachineUsesIsStillHeld(t *testing.T) { + // The predecessor's own service from a package, running: not the mesh's to restart. + dir := t.TempDir() + conf := filepath.Join(dir, "hello.conf") + m := &machine{containers: map[string]*fakeContainer{}, + units: map[string]*fakeUnit{"nginx.service": {active: "active", enabled: "enabled", + fragment: "/usr/lib/systemd/system/nginx.service"}}} + report, _ := applyAdopted(t, adopted(t, untaken("hello-web.conf", "hello-web.unit"), + `{"id":"hello-web.conf","type":"file","path":"`+conf+`","content":"x\n"}, + {"id":"hello-web.unit","type":"service","unit":"nginx.service","state":"running","boot":"enabled", + "restart-on":["hello-web.conf"]}`), store.State{}, m, dir) + if o := outcomeOf(report, "hello-web.unit"); o.Action != "held" { + t.Fatalf("a packaged unit the machine runs was not held: %+v", o) + } + if m.did("systemctl stop") || m.did("systemctl restart") { + t.Errorf("the predecessor's service was restarted: %v", m.asked) + } +} diff --git a/internal/system/arch.go b/internal/system/arch.go index 20a0cf7..9582757 100644 --- a/internal/system/arch.go +++ b/internal/system/arch.go @@ -247,6 +247,22 @@ func (arch) AddUserToGroup(ctx context.Context, run Runner, name, group string) return nil } +// ServiceUnitFile says where the service manager loads a unit from — systemd's FragmentPath. It +// is how the host tells a unit an administrator installed, under /etc or /run, from one a package +// ships under /usr (novox/hq ADR 0103). Empty, with no error, for a unit that loads from nowhere. +func (arch) ServiceUnitFile(ctx context.Context, run Runner, unit string) (string, error) { + out, err := run(ctx, "systemctl", "show", unit, "--property=FragmentPath") + if err != nil { + return "", fmt.Errorf("the service manager did not say where %s comes from: %w", unit, err) + } + for _, line := range strings.Split(out, "\n") { + if path, ok := strings.CutPrefix(strings.TrimSpace(line), "FragmentPath="); ok { + return strings.TrimSpace(path), nil + } + } + return "", nil +} + // ReloadUnits has systemd read its unit files again. A unit file that changed on disk is otherwise // ignored: a restart runs the unit systemd already loaded, and the new text only takes effect // after a reload nobody asked for. From 232ed74940d82e4e8363229bf524c7d91c680e01 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 19:52:40 +0200 Subject: [PATCH 41/52] gofmt the ufw rule's direction field --- internal/firewall/firewall.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/firewall/firewall.go b/internal/firewall/firewall.go index fbf98be..8851659 100644 --- a/internal/firewall/firewall.go +++ b/internal/firewall/firewall.go @@ -477,12 +477,12 @@ func words(rule string) []string { // operator's rule now carries the mesh's mark — so removing the opening later would delete the // operator's rule. The same holds for an incoming rule and for one with a comment of its own. type ufwRule struct { - route bool - action, in, out string + route bool + action, in, out string // dir is which way the rule matches: "" (ufw's default, incoming and forwarded), "in" or // "out". A rule on the outgoing path admits nothing that arrives. - dir string - log string + dir string + log string from, fromPort, to string port, proto, app string comment string From eb2f4fcf53956f217cf8835589accf802ee1d766 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 19:53:15 +0200 Subject: [PATCH 42/52] Keep an original by its path and its content, so a second original at one path is not discarded (hq ADR 0100) --- internal/apply/hold.go | 16 ++++++--- internal/apply/hold_test.go | 66 +++++++++++++++++++++++++++++++++++-- 2 files changed, 75 insertions(+), 7 deletions(-) diff --git a/internal/apply/hold.go b/internal/apply/hold.go index 941fc43..fc82f0f 100644 --- a/internal/apply/hold.go +++ b/internal/apply/hold.go @@ -22,13 +22,19 @@ import ( // first copy is the one that was there before the mesh. type Keep func(path string, content []byte, mode os.FileMode) (string, error) -// KeepIn keeps originals under dir/kept, each named for the path it came from, readable by root -// alone — a predecessor's configuration may carry its credentials. +// KeepIn keeps originals under dir/kept, each named for the path it came from AND for what was in +// it, readable by root alone — a predecessor's configuration may carry its credentials. +// +// **By content as well as path, because a path has more than one original.** A file held, let go +// when its module was unassigned, rewritten by the predecessor and found again is a second +// original; named by path alone the second copy was silently discarded while the report said it +// was kept (novox/hq ADR 0100). The same content at the same path is kept once. func KeepIn(dir string) Keep { return func(path string, content []byte, _ os.FileMode) (string, error) { - sum := sha256.Sum256([]byte(path)) - kept := filepath.Join(dir, "kept", - hex.EncodeToString(sum[:])[:16]+"-"+filepath.Base(path)) + where := sha256.Sum256([]byte(path)) + what := sha256.Sum256(content) + kept := filepath.Join(dir, "kept", hex.EncodeToString(where[:])[:12]+"-"+ + hex.EncodeToString(what[:])[:12]+"-"+filepath.Base(path)) if _, err := os.Lstat(kept); err == nil { return kept, nil } diff --git a/internal/apply/hold_test.go b/internal/apply/hold_test.go index 6b493f6..112f1a7 100644 --- a/internal/apply/hold_test.go +++ b/internal/apply/hold_test.go @@ -382,8 +382,13 @@ func TestAHoldNoLongerDeclaredIsLetGoAndFoundAgainIfItsModuleReturns(t *testing. t.Fatalf("a returning module's found file was not held again: %+v", o) } again, _ := state.HeldAt("hello-web.page") - if kept, _ := os.ReadFile(again.Kept); again.Kept != first.Kept || string(kept) != "the predecessor's page\n" { - t.Errorf("the first kept original was lost: %s %q", again.Kept, kept) + // The predecessor rewrote it while nothing held it, so that is a second original, kept beside + // the first rather than in place of it (novox/hq ADR 0100). + if kept, _ := os.ReadFile(again.Kept); string(kept) != "the predecessor wrote again\n" { + t.Errorf("what is named as kept is %q", kept) + } + if kept, _ := os.ReadFile(first.Kept); string(kept) != "the predecessor's page\n" { + t.Errorf("the first kept original was lost: %q", kept) } if got, _ := os.ReadFile(page); string(got) != "the predecessor wrote again\n" { t.Errorf("the found file was touched: %q", got) @@ -958,3 +963,60 @@ func TestAPackagedUnitTheMachineUsesIsStillHeld(t *testing.T) { t.Errorf("the predecessor's service was restarted: %v", m.asked) } } + +func TestASecondOriginalAtTheSamePathIsKeptToo(t *testing.T) { + // Held, let go when the module was unassigned, rewritten by the predecessor, found again: + // both originals are kept, and the report names the one it kept (novox/hq ADR 0100). + dir := t.TempDir() + page := filepath.Join(dir, "index.html") + keep := KeepIn(dir) + first, err := keep(page, []byte("the predecessor's page\n"), 0o640) + if err != nil { + t.Fatal(err) + } + same, err := keep(page, []byte("the predecessor's page\n"), 0o640) + if err != nil || same != first { + t.Errorf("the same original was kept twice: %s %s %v", first, same, err) + } + second, err := keep(page, []byte("the predecessor wrote again\n"), 0o640) + if err != nil { + t.Fatal(err) + } + if second == first { + t.Fatal("a second original was kept under the first's name") + } + if got, _ := os.ReadFile(first); string(got) != "the predecessor's page\n" { + t.Errorf("the first original is %q", got) + } + if got, _ := os.ReadFile(second); string(got) != "the predecessor wrote again\n" { + t.Errorf("the second original is %q", got) + } +} + +func TestAHoldLetGoAndFoundAgainKeepsBothOriginals(t *testing.T) { + dir, page, m := predecessor(t) + _, state := applyAdopted(t, adopted(t, untaken("hello-web.page", "hello-web.server"), webResources(page)), + store.State{}, m, dir) + first, _ := state.HeldAt("hello-web.page") + + // Unassigned, then the predecessor writes again, then assigned once more. + _, state = applyAdopted(t, adopted(t, `{"taken":[]}`, withConf(dir)), state, m, dir) + if err := os.WriteFile(page, []byte("the predecessor wrote again\n"), 0o640); err != nil { + t.Fatal(err) + } + report, state := applyAdopted(t, adopted(t, untaken("hello-web.page", "hello-web.server"), webResources(page)), + state, m, dir) + again, _ := state.HeldAt("hello-web.page") + if again.Kept == first.Kept { + t.Fatalf("the second original was kept under the first's name: %s", again.Kept) + } + if got, _ := os.ReadFile(again.Kept); string(got) != "the predecessor wrote again\n" { + t.Errorf("what the report names as kept is %q", got) + } + if got, _ := os.ReadFile(first.Kept); string(got) != "the predecessor's page\n" { + t.Errorf("the first original was lost: %q", got) + } + if o := outcomeOf(report, "hello-web.page"); o.Action != "held" { + t.Errorf("the file found again was not held: %+v", o) + } +} From 5f126021b7e5b0670bdf95ed4a031539c71aaea3 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 19:53:50 +0200 Subject: [PATCH 43/52] Keep the originals the carried bundle writes over beside the node's state (hq ADR 0100) --- internal/bootstrap/apply.go | 10 +++++++-- internal/bootstrap/apply_test.go | 38 ++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/internal/bootstrap/apply.go b/internal/bootstrap/apply.go index 372793d..82c6a10 100644 --- a/internal/bootstrap/apply.go +++ b/internal/bootstrap/apply.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "path/filepath" "strings" "github.com/novox/mesh-host/internal/apply" @@ -46,8 +47,13 @@ func ApplyBundle(ctx context.Context, o Options, sys system.System, d *declarati return apply.Report{}, err } - report, updated, applyErr := apply.Apply(ctx, sys, d, known, store.OriginCarried, run, - func(line string) { say(" " + strings.TrimPrefix(line, " ")) }, refuseSealed) + // The bundle writes over whatever the machine has at the paths the foundation needs — a + // distribution's own /etc/nftables.conf among them — so it keeps the original of each file it + // has no record of, beside the node's state, exactly as a declaration from the mesh does + // (novox/hq ADR 0100). + report, updated, applyErr := apply.ApplyKeeping(ctx, sys, d, known, store.OriginCarried, run, + func(line string) { say(" " + strings.TrimPrefix(line, " ")) }, refuseSealed, + apply.KeepIn(filepath.Dir(o.State))) // Saved whichever way it went, for the reason `mesh-host` gives: what was applied before a // failure is on the machine either way, and a host that did not record it would believe it diff --git a/internal/bootstrap/apply_test.go b/internal/bootstrap/apply_test.go index e3f45e6..486b452 100644 --- a/internal/bootstrap/apply_test.go +++ b/internal/bootstrap/apply_test.go @@ -3,8 +3,13 @@ package bootstrap import ( "context" "errors" + "os" + "path/filepath" "strings" "testing" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/system" ) // `mesh-host` is built for one operating system and pins it at link time. An installer run by hand @@ -89,3 +94,36 @@ func TestASealedFileInAFoundationIsRefusedWithAReason(t *testing.T) { t.Errorf("the refusal does not say why there is no key: %v", err) } } + +// Defends novox/hq ADR 0100: the carried bundle keeps the original of a file it writes over that +// the host has no record of — the distribution's own ruleset, say. +func TestTheBundleKeepsTheOriginalOfWhatItWritesOver(t *testing.T) { + dir := t.TempDir() + conf := filepath.Join(dir, "nftables.conf") + if err := os.WriteFile(conf, []byte("# the distribution's own\n"), 0o644); err != nil { + t.Fatal(err) + } + d, err := declaration.ParseFileTrusted([]byte(`{"declaration":1,"resources":[ + {"id":"base-filter","type":"file","path":"` + conf + `","content":"table inet mesh {}\n"}]}`)) + if err != nil { + t.Fatal(err) + } + sys, err := system.For("arch") + if err != nil { + t.Fatal(err) + } + o := Options{State: filepath.Join(dir, "state.json")} + report, err := ApplyBundle(context.Background(), o, sys, d, nil, quietly) + if err != nil { + t.Fatal(err) + } + detail := report.Outcomes[0].Detail + at := strings.Index(detail, "kept at ") + if at < 0 { + t.Fatalf("the bundle wrote over a file it had no record of and kept nothing: %q", detail) + } + if got, err := os.ReadFile(detail[at+len("kept at "):]); err != nil || + string(got) != "# the distribution's own\n" { + t.Errorf("the kept original is %q (%v)", got, err) + } +} From 9839006d48fbb2bf6bbfe069684e5dac2c2bb8c4 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 19:54:43 +0200 Subject: [PATCH 44/52] Retire the found firewall only once the mesh's own filter is loaded on the machine (hq ADR 0100) --- internal/apply/opening.go | 13 +++++++++++++ internal/apply/opening_test.go | 35 ++++++++++++++++++++++++++++++++-- internal/firewall/firewall.go | 24 +++++++++++++++++++++++ 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/internal/apply/opening.go b/internal/apply/opening.go index 8bd415e..216fc7c 100644 --- a/internal/apply/opening.go +++ b/internal/apply/opening.go @@ -69,6 +69,19 @@ func retireFirewall(ctx context.Context, d *declaration.Declaration, origin stri rec.DisabledByMesh { return nil } + // **Nothing is retired until what replaces it is in force** (novox/hq ADR 0100). The flip + // loads the mesh's derived filter in ufw's place; disabling ufw before that table is actually + // loaded — a filter module not assigned, or a unit that did not load — leaves the machine with + // no filter at all. + loaded, err := firewall.MeshTableLoaded(ctx, run) + if err != nil { + return err + } + if !loaded { + return fmt.Errorf("this node is converged and the mesh's own filter (table %s) is not loaded on "+ + "this machine, so ufw was left in force: retiring it would leave the machine filtering "+ + "nothing. Assign a filter module to this node, or return it to adopted", firewall.MeshTable) + } if rec.Forward == nil { // Recorded before ufw is touched: disabling it opens the forward policy, and a retry // must know what it was (novox/hq ADR 0100). diff --git a/internal/apply/opening_test.go b/internal/apply/opening_test.go index d14fbcf..5f976ee 100644 --- a/internal/apply/opening_test.go +++ b/internal/apply/opening_test.go @@ -163,8 +163,10 @@ func TestConvergingRetiresTheFoundFirewallAndReturningRestoresIt(t *testing.T) { t.Fatalf("adopted: firewall recorded as %+v", state.Firewall) } - // Converged: the opening's rule goes, and only then is ufw disabled — never reset. + // Converged: the derived filter is loaded, the opening's rule goes, and only then is ufw + // disabled — never reset. u.asked = nil + u.ruleset = "table inet mesh\n" converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`) _, state, err = applyWith(t, converged, state, u.run) if err != nil { @@ -274,6 +276,7 @@ func TestAFlipThatFailsKeepsTheGuardAndTheOpenings(t *testing.T) { filter := `{"id":"nftables.config","type":"file","path":"` + filepath.Join(blocked, "nftables.conf") + `","content":"table inet mesh {}\n"}` converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`,`+filter+`]}`) u.asked = nil + u.ruleset = "table inet mesh\n" // what the filter's unit loads once the file is written _, state, err = applyWith(t, converged, state, u.run) if err == nil { t.Fatal("the failing flip reported success") @@ -403,7 +406,7 @@ func TestARetiredFirewallRetriedStillPutsBackTheForwardPolicy(t *testing.T) { // The forward policy is recorded before ufw is disabled, so a retirement that failed after // the disable restores what the machine had, not what the disable left (novox/hq ADR 0100). dir := t.TempDir() - u := &ufwMachine{installed: true, active: true, forward: "DROP", failP: 1} + u := &ufwMachine{installed: true, active: true, forward: "DROP", failP: 1, ruleset: "table inet mesh\n"} _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, withConf(dir)), store.State{}, u.run) if err != nil { t.Fatal(err) @@ -459,3 +462,31 @@ func TestAGuardThatFailsLeavesTheDerivedFilterInForce(t *testing.T) { t.Error("the filter was forgotten, so nothing would ever stop it") } } + +func TestUfwIsNotRetiredUntilTheMeshsOwnFilterIsLoaded(t *testing.T) { + // The flip retires the found firewall because the mesh's derived filter takes its place. If + // that table is not loaded, retiring would leave the machine filtering nothing (novox/hq ADR 0100). + dir := t.TempDir() + u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}} + _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) + if err != nil { + t.Fatal(err) + } + converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`) + _, state, err = applyWith(t, converged, state, u.run) + if err == nil || !strings.Contains(err.Error(), "table inet mesh") { + t.Fatalf("ufw was retired with nothing in its place: %v", err) + } + if !u.active || state.Firewall.DisabledByMesh { + t.Errorf("ufw was disabled: active %v, %+v", u.active, state.Firewall) + } + + // Once the table is loaded, the same declaration retires it. + u.ruleset = "table inet mesh\n" + if _, state, err = applyWith(t, converged, state, u.run); err != nil { + t.Fatal(err) + } + if u.active || !state.Firewall.DisabledByMesh { + t.Errorf("ufw was not retired once the mesh's filter was loaded: active %v, %+v", u.active, state.Firewall) + } +} diff --git a/internal/firewall/firewall.go b/internal/firewall/firewall.go index 8851659..91a0f7c 100644 --- a/internal/firewall/firewall.go +++ b/internal/firewall/firewall.go @@ -848,6 +848,30 @@ func Disable(ctx context.Context, run Runner, before map[string]string) error { return nil } +// MeshTable is the derived filter's table, the thing that must be in force before the firewall +// found on a machine is retired. +const MeshTable = "inet mesh" + +// MeshTableLoaded asks the machine whether the mesh's own filter is loaded. Read from the machine +// rather than assumed from the declaration: a table declared and not loaded is exactly the case +// where disabling the found firewall would leave the machine with nothing. +func MeshTableLoaded(ctx context.Context, run Runner) (bool, error) { + out, err := run(ctx, "nft", "list", "tables") + if err != nil { + if missing(err) { + return false, nil + } + return false, fmt.Errorf("cannot read which tables this machine has loaded: %w", err) + } + for _, line := range strings.Split(out, "\n") { + rest, ok := strings.CutPrefix(strings.TrimSpace(line), "table "+MeshTable) + if ok && (rest == "" || strings.HasPrefix(rest, " ") || strings.HasPrefix(rest, "{")) { + return true, nil + } + } + return false, nil +} + // ForwardPolicies reads each family's forward policy, by the tool that sets it. Read before ufw is // disabled and kept by the caller, so a retirement that fails half-way is retried with what the // machine had — not with what the half-done disable left. From 0bd22e50f2df35e814b2de28472fea9239a51488 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 19:56:05 +0200 Subject: [PATCH 45/52] Apply one declaration at a time, so the link and the reconcile do not lose each other's record --- cmd/mesh-host/main.go | 14 ++++++++- cmd/mesh-host/main_test.go | 58 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 168ab4e..62c0477 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -799,10 +799,22 @@ func applyDeclared(ctx context.Context, opts options, raw []byte, sched *apply.S return applyAndKeep(ctx, opts, raw, nil, sched) } +// applying serialises applies on this node. +// +// **Two things apply here: the link and the reconcile loop**, and each reads the node's state, +// acts on the machine, and writes the state back. Run at the same time they interleave, and the +// one that saves last writes a state read before the other acted — losing what the first recorded: +// a hold, the firewall found here, a resource just applied. The machine would then be one thing +// and its record another, which is the fault every read-back in this package exists to prevent. +var applying sync.Mutex + // applyAndKeep applies a declaration and, when it came from the mesh, keeps it so this node can -// go on obeying it while disconnected. +// go on obeying it while disconnected. One at a time, whoever asks. func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.Declared, sched *apply.Scheduler) link.Report { + applying.Lock() + defer applying.Unlock() + declared, err := declaration.Parse(raw) if err != nil { return link.Report{Refused: err.Error()} diff --git a/cmd/mesh-host/main_test.go b/cmd/mesh-host/main_test.go index 72eb1a1..4747647 100644 --- a/cmd/mesh-host/main_test.go +++ b/cmd/mesh-host/main_test.go @@ -2,10 +2,16 @@ package main import ( "context" - "github.com/novox/mesh-host/internal/link" - "github.com/novox/mesh-host/internal/store" + "errors" + "os" + "path/filepath" "testing" "time" + + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" ) // Argument handling gets tests because it already failed silently once: `mesh-host inventory @@ -190,3 +196,51 @@ func TestAReconcileSpeaksWhenWhatIsReachableChanged(t *testing.T) { t.Error("a newly published port was not said") } } + +// Defends the node's own record: the link and the reconcile loop both apply, and each reads the +// state, acts, and writes it back — so they must not run at the same time, or the last save loses +// what the other recorded. +func TestOnlyOneApplyRunsAtATime(t *testing.T) { + // A host is built for one system at link time, and a test binary has no link time: this asks + // the machine it runs on, and stands aside where the answer is no. + built, err := system.For("arch") + if err != nil || built.Confirm(context.Background(), apply.ExecRunner) != nil { + t.Skip("this machine is not one these tests can apply on") + } + was := builtFor + builtFor = "arch" + t.Cleanup(func() { builtFor = was }) + dir := t.TempDir() + opts := options{state: filepath.Join(dir, "state.json")} + raw := []byte(`{"declaration":1,"resources":[{"id":"a","type":"file","path":"` + + filepath.Join(dir, "a.conf") + `","content":"x\n"}]}`) + + // Whatever else is applying — the link, while this is the reconcile — this waits for it. + applying.Lock() + done := make(chan link.Report, 1) + go func() { done <- applyAndKeep(context.Background(), opts, raw, nil, nil) }() + select { + case report := <-done: + applying.Unlock() + t.Fatalf("an apply ran while another held the node: %+v", report) + case <-time.After(50 * time.Millisecond): + } + if _, err := os.Stat(filepath.Join(dir, "a.conf")); !errors.Is(err, os.ErrNotExist) { + applying.Unlock() + t.Fatal("the waiting apply had already touched the machine") + } + applying.Unlock() + + select { + case report := <-done: + if report.Refused != "" { + t.Fatalf("refused: %s", report.Refused) + } + case <-time.After(10 * time.Second): + t.Fatal("the apply never ran once the node was free") + } + known, loadErr := store.Load(opts.state) + if loadErr != nil || len(known.Resources) != 1 { + t.Errorf("the apply recorded %d resource(s): %v", len(known.Resources), loadErr) + } +} From 7beb752be04298068b4f4903cfa00675b9df29ac Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 19:57:15 +0200 Subject: [PATCH 46/52] Take a key or list member the host may have written itself as the mesh's, so undeclaring gives the file back (hq ADR 0102) --- internal/apply/into.go | 26 ++++++++++++++++------- internal/apply/into_test.go | 42 +++++++++++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 9 deletions(-) diff --git a/internal/apply/into.go b/internal/apply/into.go index 4847a21..eefcbeb 100644 --- a/internal/apply/into.go +++ b/internal/apply/into.go @@ -93,7 +93,7 @@ func applyInto(r *declaration.File, previous store.Applied) (Outcome, error) { if !tracked(k) && !had { rec.Absent = append(rec.Absent, k) } - merged, added, err := addMembers(current, declared[k], rec.Added[k]) + merged, added, err := addMembers(current, declared[k], rec.Added[k], previous.Into == nil) if err != nil { return out, fmt.Errorf("%s: %q: %w", r.Path, k, err) } @@ -102,7 +102,14 @@ func applyInto(r *declaration.File, previous store.Applied) (Outcome, error) { continue } if !tracked(k) { - if v, had := object[k]; had { + v, had := object[k] + // **What the host may have written itself is not the machine's.** With no record of + // this file — the first apply, or a host that wrote and died before saving its state — + // a key already holding exactly what the mesh declares cannot be told from one the + // mesh set a moment ago. Remembered as the machine's, it would never be given back: + // undeclaring would leave the mesh's own value behind for ever. So it is the mesh's, + // and undeclaring takes it out (novox/hq ADR 0102). + if had && !(previous.Into == nil && canonical(v) == canonical(declared[k])) { rec.Before[k] = v } else { rec.Absent = append(rec.Absent, k) @@ -320,8 +327,11 @@ func listOf(members []json.RawMessage) json.RawMessage { // addMembers adds the declared members to the machine's list, dropping only members the mesh // added before and no longer declares. It returns the list and exactly which members the mesh // added — a declared member the machine already had is the machine's, and is never recorded. -func addMembers(current, declared json.RawMessage, addedBefore []json.RawMessage) (json.RawMessage, - []json.RawMessage, error) { +// unrecorded says there is no record of this file yet, in which case a declared member already in +// the list may be one the host itself wrote before it could save its state, and is taken as the +// mesh's. +func addMembers(current, declared json.RawMessage, addedBefore []json.RawMessage, + unrecorded bool) (json.RawMessage, []json.RawMessage, error) { have, err := membersOf(current) if err != nil { return nil, nil, err @@ -341,9 +351,11 @@ func addMembers(current, declared json.RawMessage, addedBefore []json.RawMessage for _, m := range want { if !hasMember(have, m) { have = append(have, m) - if !hasMember(added, m) { - added = append(added, m) - } + } else if !unrecorded { + continue // the machine's own, and never the mesh's to take out + } + if !hasMember(added, m) { + added = append(added, m) } } return listOf(have), added, nil diff --git a/internal/apply/into_test.go b/internal/apply/into_test.go index 58a7962..c5f1362 100644 --- a/internal/apply/into_test.go +++ b/internal/apply/into_test.go @@ -217,9 +217,15 @@ func TestAListIsAddedToNeverReplaced(t *testing.T) { // takes back only what it added (novox/hq ADR 0102). path := filepath.Join(t.TempDir(), "daemon.json") _ = os.WriteFile(path, []byte(`{"insecure-registries":["192.0.2.7:5000","10.42.0.9:5000"]}`), 0o644) - // 10.42.0.9 is declared too, and was already the machine's: it is never the mesh's to remove. + // First the mesh's own member alone, so there is a record of this file. + first := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`)) + _, state, err := Apply(context.Background(), archHost(t), first, store.State{}, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + // Now 10.42.0.9 is declared too, and was already the machine's: it is never the mesh's to remove. d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000","10.42.0.9:5000"]}`)) - _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil) + _, state, err = Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, nil, nil, nil) if err != nil { t.Fatal(err) } @@ -302,3 +308,35 @@ func TestAHoldFromAWholeFileDoesNotKeepOutAnIntoWrite(t *testing.T) { t.Errorf("the mesh's member was not written in: %v", readObject(t, path)) } } + +func TestAWriteWithNoRecordOfItIsTheMeshsOwn(t *testing.T) { + // A host that wrote into the file and died before saving its state comes back with no record + // of it. What is there is then exactly what the mesh declares — and remembered as the + // machine's it would never be given back (novox/hq ADR 0102). + path := filepath.Join(t.TempDir(), "daemon.json") + _ = os.WriteFile(path, []byte(`{"data-root":"/srv/docker","insecure-registries":["192.0.2.7:5000"]}`), 0o644) + d := parse(t, intoDecl(t, path, `{"live-restore":true,"insecure-registries":["10.42.0.1:5000"]}`)) + if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil); err != nil { + t.Fatal(err) + } + + // The crash: the state was never saved, so the next apply knows nothing of this file. + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + rec, _ := state.Find("networking.registry-trust") + if _, asTheMachines := rec.Into.Before["live-restore"]; asTheMachines { + t.Error("the mesh's own key was remembered as the machine's") + } + if _, _, err := Apply(context.Background(), archHost(t), somethingElse(t), state, store.OriginDeclared, nil, nil, nil); err != nil { + t.Fatal(err) + } + o := readObject(t, path) + if _, still := o["live-restore"]; still { + t.Errorf("undeclaring left the mesh's key behind: %v", o) + } + if fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000]" || o["data-root"] != "/srv/docker" { + t.Errorf("the machine did not get its file back: %v", o) + } +} From 04665d36c84ea4763eb76280c76adda3c38eb06d Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 19:57:52 +0200 Subject: [PATCH 47/52] Exempt only the daemons a fresh machine was measured to run from counting as in use (hq ADR 0101) --- internal/bootstrap/inuse.go | 14 +++++++++----- internal/bootstrap/inuse_test.go | 24 +++++++++++++++++++----- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/internal/bootstrap/inuse.go b/internal/bootstrap/inuse.go index 023d322..c3cf613 100644 --- a/internal/bootstrap/inuse.go +++ b/internal/bootstrap/inuse.go @@ -12,14 +12,18 @@ import ( ) // quiet are the processes every fresh machine runs that serve nobody: name resolution (whose -// link-local resolver listens on TCP as well as UDP, on every address), address configuration and -// time. ss names a process by its first fifteen characters, so both spellings are here. Measured -// on a freshly installed lab machine (testdata/fresh-machine-listeners.txt): these and nothing else. +// link-local resolver listens on TCP as well as UDP, on every address) and the network manager's +// address configuration. ss names a process by its first fifteen characters, so both spellings are +// here. +// +// **Only what the measurement found** (novox/hq ADR 0101): these two hold every listener on a +// freshly installed lab machine (testdata/fresh-machine-listeners.txt) and nothing else does. A +// daemon joins this list with a measurement of a fresh machine that holds it, never by guess — a +// time client or an address-configuration client listening on a machine that does not run one as +// standard is something somebody installed, and that is a machine in use. var quiet = map[string]bool{ "systemd-resolved": true, "systemd-resolve": true, "systemd-networkd": true, "systemd-network": true, - "systemd-timesyncd": true, "systemd-timesyn": true, - "dhcpcd": true, } // InUse says what makes this machine a machine in use (novox/hq ADR 0100): every running container diff --git a/internal/bootstrap/inuse_test.go b/internal/bootstrap/inuse_test.go index 137ceef..2cc889d 100644 --- a/internal/bootstrap/inuse_test.go +++ b/internal/bootstrap/inuse_test.go @@ -14,16 +14,14 @@ import ( // and listener it counted. // Lines as `ss -Hltunp` prints them. The ssh, samba, loopback and proxy lines are captured from a -// real machine; the resolver, DHCP and time lines are written in the same shape. What a fresh machine -// actually runs is measured in testdata/fresh-machine-listeners.txt. +// real machine; the resolver and network-manager lines are written in the same shape. What a fresh +// machine actually runs is measured in testdata/fresh-machine-listeners.txt. const inUseSockets = `tcp LISTEN 0 128 0.0.0.0:22 0.0.0.0:* users:(("sshd",pid=1188536,fd=6)) tcp LISTEN 0 128 [::]:22 [::]:* users:(("sshd",pid=1188536,fd=7)) tcp LISTEN 0 32 127.0.0.1:53 0.0.0.0:* users:(("dnsmasq",pid=1189392,fd=7)) tcp LISTEN 0 4096 127.0.0.1:5432 0.0.0.0:* users:(("docker-proxy",pid=1854543,fd=7)) udp UNCONN 0 0 0.0.0.0:5355 0.0.0.0:* users:(("systemd-resolve",pid=301,fd=11)) udp UNCONN 0 0 192.0.2.10%eth0:68 0.0.0.0:* users:(("systemd-network",pid=280,fd=19)) -udp UNCONN 0 0 0.0.0.0:68 0.0.0.0:* users:(("dhcpcd",pid=270,fd=9)) -udp UNCONN 0 0 0.0.0.0:123 0.0.0.0:* users:(("systemd-timesyn",pid=260,fd=9)) ` const servingSockets = `tcp LISTEN 0 50 0.0.0.0:445 0.0.0.0:* users:(("smbd",pid=1248,fd=29)) @@ -47,7 +45,7 @@ func TestAFreshMachineIsNotInUse(t *testing.T) { t.Fatal(err) } if len(containers) != 0 || len(listeners) != 0 { - t.Errorf("ssh, loopback, name resolution, DHCP and time were counted: %v %v", containers, listeners) + t.Errorf("ssh, loopback and the daemons a fresh machine runs were counted: %v %v", containers, listeners) } } @@ -163,3 +161,19 @@ func TestARerunWithTheFlagOnAConvergedMachineIsRefused(t *testing.T) { t.Errorf("a converged re-run of a converged machine was refused: %v", err) } } + +func TestOnlyTheDaemonsTheMeasurementFoundAreQuiet(t *testing.T) { + // novox/hq ADR 0101: the exempt daemons are the ones a fresh machine was measured to run — + // the resolver and the network manager. A time client or a DHCP client listening beyond + // loopback is something somebody put there, and that is a machine in use. + sockets := `udp UNCONN 0 0 0.0.0.0:123 0.0.0.0:* users:(("systemd-timesyn",pid=260,fd=9)) +udp UNCONN 0 0 0.0.0.0:68 0.0.0.0:* users:(("dhcpcd",pid=270,fd=9)) +` + _, listeners, err := InUse(context.Background(), inUseRunner{ss: sockets}.run, func(string) bool { return false }) + if err != nil { + t.Fatal(err) + } + if len(listeners) != 2 { + t.Errorf("counted %d listener(s), want the time client and the DHCP client: %+v", len(listeners), listeners) + } +} From c7ff0b9026e5acc7a2d05879c56eec73e190b702 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 19:58:37 +0200 Subject: [PATCH 48/52] Refuse an endpoint whose port is not the one the private network's hub binds (hq ADR 0100) --- internal/bootstrap/phase2.go | 29 +++++++++++++++++++++++++++++ internal/bootstrap/phase2_test.go | 23 ++++++++++++++++++++++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/internal/bootstrap/phase2.go b/internal/bootstrap/phase2.go index ee2319e..ede6305 100644 --- a/internal/bootstrap/phase2.go +++ b/internal/bootstrap/phase2.go @@ -3,6 +3,7 @@ package bootstrap import ( "context" "encoding/json" + "errors" "fmt" "net" "strconv" @@ -133,6 +134,10 @@ func PlaceOnTheNetwork(ctx context.Context, o Options, control controlPlane, return fmt.Errorf("the private network needs an endpoint other machines can dial, and " + "nothing said one: pass --endpoint, or --broker-address so one can be derived") } + endpoint, err = endpointAgrees(endpoint, o.Ports.orDefaults().Hub) + if err != nil { + return err + } if _, err := control.tell(ctx, "assign", o.Node, module); err != nil { return err @@ -210,6 +215,30 @@ func builds(manifest []byte) bool { return m.Build != nil && len(m.Build.Artifacts) > 0 } +// endpointAgrees holds the endpoint other machines dial to the port this node gave the private +// network's hub (novox/hq ADR 0100): the hub binds what --hub-port says, so an endpoint naming +// another port is an address nothing answers on. A host alone takes the hub's port. +func endpointAgrees(endpoint string, hub int) (string, error) { + _, portText, err := net.SplitHostPort(endpoint) + var missing *net.AddrError + if errors.As(err, &missing) && missing.Err == "missing port in address" { + return net.JoinHostPort(strings.Trim(endpoint, "[]"), strconv.Itoa(hub)), nil + } + if err != nil { + return "", fmt.Errorf("--endpoint %q is not host:port: %w", endpoint, err) + } + port, err := strconv.Atoi(portText) + if err != nil { + return "", fmt.Errorf("--endpoint %q does not end in a port", endpoint) + } + if port != hub { + return "", fmt.Errorf("--endpoint %s names port %d and the private network's hub binds %d "+ + "(--hub-port): other machines would dial a port nothing answers on. Give one port for the "+ + "hub; nothing was changed", endpoint, port, hub) + } + return endpoint, nil +} + // derivedEndpoint is the default place other machines dial for the private network: the same host // they already dial for the broker, on WireGuard's ordinary port. One fact, not two. func derivedEndpoint(brokerAddress string, hub int) string { diff --git a/internal/bootstrap/phase2_test.go b/internal/bootstrap/phase2_test.go index 918a553..b1b46f3 100644 --- a/internal/bootstrap/phase2_test.go +++ b/internal/bootstrap/phase2_test.go @@ -1,6 +1,9 @@ package bootstrap -import "testing" +import ( + "strings" + "testing" +) // The endpoint other machines dial defaults to the host they already dial — the broker's — on // WireGuard's port. One fact, not two that drift. @@ -23,3 +26,21 @@ func TestOnlyAManifestWithArtifactsBuilds(t *testing.T) { t.Fatal("a manifest with nothing to build was built anyway") } } + +// Defends novox/hq ADR 0100: the hub's port is the node's, and the endpoint other machines dial +// must name it — an endpoint on another port is an address nothing answers on. +func TestTheEndpointAgreesWithTheHubsPort(t *testing.T) { + if got, err := endpointAgrees("192.0.2.10:51820", 51820); err != nil || got != "192.0.2.10:51820" { + t.Errorf("an endpoint on the hub's port: %q %v", got, err) + } + if got, err := endpointAgrees("192.0.2.10", 51821); err != nil || got != "192.0.2.10:51821" { + t.Errorf("a host alone did not take the hub's port: %q %v", got, err) + } + _, err := endpointAgrees("192.0.2.10:51820", 51821) + if err == nil || !strings.Contains(err.Error(), "--hub-port") { + t.Errorf("two ports for one hub were accepted: %v", err) + } + if _, err := endpointAgrees("192.0.2.10:not-a-port", 51820); err == nil { + t.Error("an endpoint whose port is not a number was accepted") + } +} From b4c21b4f6707eed1373f63bc93b794473abb3943 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 19:59:22 +0200 Subject: [PATCH 49/52] Read a machine's iptables rules when it has no nft, instead of calling it unfiltered (hq ADR 0100) --- internal/firewall/firewall.go | 18 ++++++++++++++--- internal/firewall/firewall_test.go | 31 ++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/internal/firewall/firewall.go b/internal/firewall/firewall.go index 91a0f7c..37b9052 100644 --- a/internal/firewall/firewall.go +++ b/internal/firewall/firewall.go @@ -63,19 +63,31 @@ func Detect(ctx context.Context, run Runner) (Kind, string, error) { ufwActive = statusActive(out) } + noNft := false out, err := run(ctx, "nft", "list", "ruleset") switch { case err == nil: if refusing := Refusing(out, ufwActive); len(refusing) > 0 { return Unsupported, "nftables rules that refuse traffic, in " + strings.Join(refusing, ", "), nil } - case !missing(err): + case missing(err): + // **No nft on this machine does not mean no rules.** iptables-nft writes tables nft would + // have shown, and a machine whose only tool is iptables answers about them through that. + // Read as "nothing filters here", a machine with an iptables firewall would be adopted + // with no openings and nothing would reach the mesh (novox/hq ADR 0100). + noNft = true + default: return "", "", fmt.Errorf("cannot read this machine's packet filter to know what it has: %w", err) } if !ufwActive { - // iptables with the legacy backend is invisible to nft. - for _, legacy := range []string{"iptables-legacy", "ip6tables-legacy"} { + // iptables with the legacy backend is invisible to nft; and where nft is not installed, + // the iptables command is the only way to see anything at all. + tools := []string{"iptables-legacy", "ip6tables-legacy"} + if noNft { + tools = append(tools, "iptables", "ip6tables") + } + for _, legacy := range tools { out, err := run(ctx, legacy, "-S") if err != nil { continue diff --git a/internal/firewall/firewall_test.go b/internal/firewall/firewall_test.go index 426c7d5..acee5d2 100644 --- a/internal/firewall/firewall_test.go +++ b/internal/firewall/firewall_test.go @@ -119,11 +119,20 @@ type fakeUFW struct { // after it is disabled; empty is a machine without iptables. forward is a policy set since. iptablesActive, iptablesInactive string forward string + // noNft is a machine with no nft binary; iptablesRules is what `iptables -S` prints there. + noNft bool + iptablesRules string } // iptables answers `iptables -S FORWARD` from the captured output for ufw's state, and records // a forward policy set with -P. func (f *fakeUFW) iptables(name string, args []string) (string, error) { + if f.iptablesRules != "" && len(args) == 1 && args[0] == "-S" { + if name == "ip6tables" { + return "", nil + } + return f.iptablesRules, nil + } if f.iptablesActive == "" { return "", &exec.Error{Name: name, Err: exec.ErrNotFound} } @@ -189,6 +198,9 @@ func (f *fakeUFW) run(_ context.Context, name string, args ...string) (string, e } return "", &exec.Error{Name: name, Err: exec.ErrNotFound} case "nft": + if f.noNft { + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + } return f.ruleset, nil case "iptables-legacy", "ip6tables-legacy": return "", &exec.Error{Name: name, Err: exec.ErrNotFound} @@ -756,3 +768,22 @@ func TestIncomingIsUfwsDefaultDirection(t *testing.T) { t.Error("an incoming refusal ufw would merge was not refused") } } + +func TestAMachineWithIptablesRulesAndNoNftIsNotReadAsUnfiltered(t *testing.T) { + // nft is not installed, and iptables-nft holds a firewall of somebody's. Read as "nothing + // filters here" the mesh would adopt it, open nothing, and be unreachable (novox/hq ADR 0100). + rules := "-P INPUT DROP\n-P FORWARD DROP\n-P OUTPUT ACCEPT\n-A INPUT -p tcp -m tcp --dport 22 -j ACCEPT\n" + f := &fakeUFW{noNft: true, iptablesRules: rules} + kind, what, err := Detect(context.Background(), f.run) + if err != nil { + t.Fatal(err) + } + if kind != Unsupported { + t.Errorf("a machine filtered by iptables with no nft read as %s (%s)", kind, what) + } + // And a machine with nothing but the runtime's own rules and no nft is still unfiltered. + docker := "-P INPUT ACCEPT\n-P FORWARD DROP\n-P OUTPUT ACCEPT\n-N DOCKER\n-A DOCKER -i docker0 -j DROP\n" + if kind, _, err := Detect(context.Background(), (&fakeUFW{noNft: true, iptablesRules: docker}).run); err != nil || kind != None { + t.Errorf("a machine with only the runtime's rules read as %s: %v", kind, err) + } +} From 01e8affa8927d3c6bc1da918a41eb212a636b736 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 20:00:59 +0200 Subject: [PATCH 50/52] Count an unasked report as said only once the broker has taken it (hq ADR 0100) --- cmd/mesh-host/main.go | 34 ++++++++++++++++++++++++++-------- cmd/mesh-host/main_test.go | 18 ++++++++++++++++++ internal/link/run.go | 24 +++++++++++++++++++----- 3 files changed, 63 insertions(+), 13 deletions(-) diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 62c0477..681eb3c 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -638,11 +638,11 @@ func runLink(ctx context.Context, opts options) error { // (novox/hq ADR 0004). // Reports a reconcile has to make unasked — what an adopted node holds changed, or its // firewall did — go out over the link when it is up (novox/hq ADR 0100). - outbox := make(chan link.Report, 1) + outbox := make(chan link.Unasked, 1) watch := &adoptionWatch{} applier = watch.noting(applier) go holdTheMachine(ctx, opts, mine, say, sched, func(r link.Report) { - if !watch.changed(r) { + if !watch.differs(r) { return } select { @@ -650,7 +650,13 @@ func runLink(ctx context.Context, opts options) error { // An older one nobody has published yet; this one says everything it did. default: } - outbox <- r + // Counted as said only once the broker has taken it: queued and lost — the link down, the + // publish refused — the change would never be said again (novox/hq ADR 0100). + outbox <- link.Unasked{Report: r, Done: func(published bool) { + if published { + watch.said(r) + } + }} }) return link.HoldRoused(ctx, link.Membership{ @@ -686,15 +692,27 @@ func adoptionFingerprint(r link.Report) string { return strings.Join(parts, "\n") } -// changed records a report and says whether it differs from the last one that went out. -func (w *adoptionWatch) changed(r link.Report) bool { +// differs says whether a report says anything the last one that went out did not. It records +// nothing: what was said is what reached the mesh, not what was written down to send. +func (w *adoptionWatch) differs(r link.Report) bool { w.mu.Lock() defer w.mu.Unlock() - now := adoptionFingerprint(r) - if now == w.last { + return adoptionFingerprint(r) != w.last +} + +// said records a report the mesh has actually been told. +func (w *adoptionWatch) said(r link.Report) { + w.mu.Lock() + defer w.mu.Unlock() + w.last = adoptionFingerprint(r) +} + +// changed is differs and said together, for a report published as it is made. +func (w *adoptionWatch) changed(r link.Report) bool { + if !w.differs(r) { return false } - w.last = now + w.said(r) return true } diff --git a/cmd/mesh-host/main_test.go b/cmd/mesh-host/main_test.go index 4747647..4caab47 100644 --- a/cmd/mesh-host/main_test.go +++ b/cmd/mesh-host/main_test.go @@ -244,3 +244,21 @@ func TestOnlyOneApplyRunsAtATime(t *testing.T) { t.Errorf("the apply recorded %d resource(s): %v", len(known.Resources), loadErr) } } + +// Defends novox/hq ADR 0100: a change is counted as said only once the mesh has been told. Queued +// and lost — the link down when the reconcile spoke — it must be said again. +func TestAChangeThatNeverReachedTheMeshIsSaidAgain(t *testing.T) { + w := &adoptionWatch{} + held := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page", Changed: "rewritten"}}} + if !w.differs(held) { + t.Fatal("the first report of a change was not new") + } + // The link was down: nothing published it, so nothing says it was said. + if !w.differs(held) { + t.Error("a change that never reached the mesh was counted as said") + } + w.said(held) + if w.differs(held) { + t.Error("a change the mesh was told was said again") + } +} diff --git a/internal/link/run.go b/internal/link/run.go index 5668af4..1379369 100644 --- a/internal/link/run.go +++ b/internal/link/run.go @@ -76,7 +76,15 @@ func Hold(ctx context.Context, m Membership, apply Applier, say Announce, timeou // Outbox carries reports the node has to say without having been sent anything — what a // reconcile found changed on an adopted node (novox/hq ADR 0100). Published while the link is up; // a report made while it is down waits in the channel for the next one. Nil is allowed. -type Outbox <-chan Report +type Outbox <-chan Unasked + +// Unasked is one such report, with the way to say whether it reached the mesh. Done is called +// with true only when the broker took it — a node that marked a change said because it queued it +// would never say it again, and the mesh would go on believing nothing changed. +type Unasked struct { + Report Report + Done func(published bool) +} // HoldRoused is Hold, told when the machine has reason to think its link is stale, and handed // reports to publish between deliveries. @@ -261,10 +269,13 @@ func Run(ctx context.Context, m Membership, apply Applier, say Announce, timeout return nil case <-beat.C: publishAlive(ctx, channel, m, say, timeout) - case report := <-outbox: + case unasked := <-outbox: // Said without having been asked: a reconcile found what an adopted node holds, or // its firewall, changed since it last said. - publishReport(ctx, channel, m, report, say, timeout) + published := publishReport(ctx, channel, m, unasked.Report, say, timeout) + if unasked.Done != nil { + unasked.Done(published) + } case reason := <-closed: return fmt.Errorf("the link closed: %v", reason) case delivery, ok := <-deliveries: @@ -365,13 +376,14 @@ func handleBody(ctx context.Context, m Membership, body []byte, apply Applier) R return apply(ctx, signed.Declaration, signed.Signature) } +// publishReport tells the mesh what this node did, and says whether the broker took it. func publishReport(ctx context.Context, channel *amqp.Channel, m Membership, report Report, - say Announce, timeout time.Duration) { + say Announce, timeout time.Duration) bool { report.Node = m.Node body, err := json.Marshal(report) if err != nil { say("cannot encode this node's own report: " + err.Error()) - return + return false } publish, cancel := context.WithTimeout(ctx, timeout) defer cancel() @@ -382,7 +394,9 @@ func publishReport(ctx context.Context, channel *amqp.Channel, m Membership, rep if err := channel.PublishWithContext(publish, Exchange, KeyReport, true, false, amqp.Publishing{ContentType: "application/json", Body: body}); err != nil { say(fmt.Sprintf("applied, and could not tell the mesh: %v", err)) + return false } + return true } // publishAlive says this node is here, and nothing else. From c03a31cec5895782ef00fcc82f58a0e7eaf1a94e Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 20:01:14 +0200 Subject: [PATCH 51/52] Count only a record of making something at a path as the mesh's, not an access record (hq ADR 0103) --- internal/apply/hold.go | 7 +++++-- internal/apply/hold_test.go | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/internal/apply/hold.go b/internal/apply/hold.go index fc82f0f..935ec69 100644 --- a/internal/apply/hold.go +++ b/internal/apply/hold.go @@ -189,10 +189,13 @@ func present(path string) bool { return err == nil } -// recordedPath is whether this host has a record of making something at a path. +// recordedPath is whether this host has a record of MAKING something at a path — a directory it +// created, a file it wrote, an archive it unpacked. An access record is not one of those: it says +// the mesh set permissions on a path it does not own, which is exactly what it does to a path +// somebody else's software made, so a path it only has access for is still found (novox/hq ADR 0103). func recordedPath(known store.State, path string) bool { for _, kind := range []declaration.Type{declaration.TypeDirectory, declaration.TypeFile, - declaration.TypeArchive, declaration.TypeAccess} { + declaration.TypeArchive} { if known.Recorded(string(kind), path) { return true } diff --git a/internal/apply/hold_test.go b/internal/apply/hold_test.go index 112f1a7..0651f4f 100644 --- a/internal/apply/hold_test.go +++ b/internal/apply/hold_test.go @@ -1020,3 +1020,24 @@ func TestAHoldLetGoAndFoundAgainKeepsBothOriginals(t *testing.T) { t.Errorf("the file found again was not held: %+v", o) } } + +func TestAPathTheMeshOnlySetAccessOnIsStillFound(t *testing.T) { + // An access record says the mesh set permissions on a path it does not own — which is what it + // does to somebody else's directory. It is not a record of making it (novox/hq ADR 0103). + dir := t.TempDir() + data := filepath.Join(dir, "data") + if err := os.Mkdir(data, 0o700); err != nil { + t.Fatal(err) + } + known := store.State{Resources: []store.Applied{ + {ID: "hello-web.readable", Type: "access", Target: data, Origin: store.OriginDeclared}}} + m := &machine{containers: map[string]*fakeContainer{}} + report, _ := applyAdopted(t, adopted(t, untaken("hello-web.data"), + `{"id":"hello-web.data","type":"directory","path":"`+data+`","mode":"0755"}`), known, m, dir) + if o := outcomeOf(report, "hello-web.data"); o.Action != "held" { + t.Errorf("a directory the mesh only has access for was not held: %+v", o) + } + if info, _ := os.Stat(data); info.Mode().Perm() != 0o700 { + t.Errorf("it was re-moded to %o", info.Mode().Perm()) + } +} From 6dce63b53447ae49843e5ab20dd80e1642ad4932 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 20:01:26 +0200 Subject: [PATCH 52/52] Say plainly where the raised package registry runs, and why an action outside a container still runs --- internal/apply/hold.go | 4 +++- internal/bootstrap/phase_packages.go | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/internal/apply/hold.go b/internal/apply/hold.go index 935ec69..3776c41 100644 --- a/internal/apply/hold.go +++ b/internal/apply/hold.go @@ -234,7 +234,9 @@ func mountSource(mapping string) string { } // runsIn is the container a resource runs inside, if any: an action's `in`, or a run-once step -// sharing a container's namespace. +// sharing a container's namespace. An action with no `in` runs on the machine itself and is not +// held for a container: it reaches nothing a predecessor holds by running there, and holding every +// action of an untaken module would stop a module preparing itself before its cutover. func runsIn(r declaration.Resource) string { switch res := r.(type) { case *declaration.Action: diff --git a/internal/bootstrap/phase_packages.go b/internal/bootstrap/phase_packages.go index cd82e6e..bb8414b 100644 --- a/internal/bootstrap/phase_packages.go +++ b/internal/bootstrap/phase_packages.go @@ -148,9 +148,9 @@ func seedGiteaDatabase(ctx context.Context, run Runner, timeout time.Duration, p return nil } -// raiseGiteaServer starts the gitea server container against the foundation store. It joins the -// store's network namespace so `127.0.0.1:5432` reaches postgres, and publishes its own port on the -// machine so the builder and this installer can reach it. Started if absent, left alone if present. +// raiseGiteaServer starts the gitea server container against the foundation store. It runs on the +// machine's own network, so `127.0.0.1` reaches the store where it publishes its port, and it binds +// its own port there for the builder and this installer. Started if absent, left alone if present. func raiseGiteaServer(ctx context.Context, run Runner, timeout time.Duration, dbPassword string, ports FoundationPorts, say func(string)) error { asking, cancel := context.WithTimeout(ctx, timeout)