diff --git a/cmd/mesh-controller/found_wait_test.go b/cmd/mesh-controller/found_wait_test.go new file mode 100644 index 00000000..10d7b5dd --- /dev/null +++ b/cmd/mesh-controller/found_wait_test.go @@ -0,0 +1,104 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// A directory the node-engine uses as found (novox/hq issue 339) waits for a person to hand it over at the +// machine. Found before this send, it is no fault of the build: the gate passes with the wait carried, so an +// urgent fix of that module still goes through. Found by this send, the send brought it, and the gate holds. +func foundDirectory(module string, since time.Time) inventory.ResourceHealth { + return inventory.ResourceHealth{Module: module, Resource: module + ".data", Kind: link.KindDirectory, + Target: "/srv/" + module, State: link.StateUnhealthy, Since: since, + Reason: link.ReasonUsedAsFound + " owned by 1000:1000, mode 700, as found; root, mode 755 was declared and " + + "not given it — `mesh-host hand-over` at the machine hands it to the mesh"} +} + +func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) { + now := time.Now() + sent := now.Add(-time.Minute) + f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, + Resources: []inventory.ResourceHealth{foundDirectory("notes", sent.Add(-24*time.Hour))}}}} + h, why := moduleHealthWord("notes", "laptop", sent, f) + if h != healthPerson || !strings.Contains(why, "notes.data") || !strings.Contains(why, "hand-over") { + t.Fatalf("a directory found before the send reads %v %q; want a wait for a person", h, why) + } + // Found by this very send: the send brought it, and it is not passed. + f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now, + Resources: []inventory.ResourceHealth{foundDirectory("notes", sent.Add(time.Second))}} + if h, why := moduleHealthWord("notes", "laptop", sent, f); h != healthNotYet { + t.Fatalf("a directory this send found reads %v %q; want not yet", h, why) + } + // A container down beside the old wait is a fault, as before. + f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{ + foundDirectory("notes", sent.Add(-time.Hour)), + {Module: "notes", Resource: "notes.web", Kind: "container", Target: "notes", State: link.StateUnhealthy, Reason: "down"}}} + if h, why := moduleHealthWord("notes", "laptop", sent, f); h != healthNotYet { + t.Fatalf("a container down beside the wait reads %v %q; want not yet", h, why) + } +} + +// The whole walk: a module whose directory was used as found long before still gets its fix to every machine, +// its pass kept and the wait said; a directory this very send found holds it and puts it back. +func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) { + for _, c := range []struct { + name string + found time.Duration // when the directory was found, against now + passes bool + }{ + {"found a day before the send", -24 * time.Hour, true}, + {"found by this send", time.Hour, false}, + } { + t.Run(c.name, func(t *testing.T) { + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + releaseHeard = func(context.Context, *stores) (map[string]bool, error) { + return map[string]bool{"anchor": true, "laptop": true}, nil + } + backlogFacts := gatherGateFacts + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + f, err := backlogFacts(ctx, open, component) + f.health = map[string]inventory.NodeHealth{} + for _, n := range []string{"anchor", "laptop"} { + f.health[n] = inventory.NodeHealth{Node: n, HeardAt: time.Now(), Resources: []inventory.ResourceHealth{ + {Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy}, + foundDirectory("app", time.Now().Add(c.found)), + {Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateHealthy}}} + } + return f, err + } + wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) + gateSettle, gateEvery, gateBound = 0, 0, 300*time.Millisecond + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + advancePlans(ctx, b.open) + if p := b.release(t); p.State != inventory.PlanRolling { + break + } + time.Sleep(20 * time.Millisecond) + } + p := b.release(t) + v, found, err := inv.GateOf(ctx, "build-app-c2") + if c.passes { + if p.State != inventory.PlanDone || err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("the walk is %s (%s); app's verdict %+v: want the fix through", p.State, p.Note, v) + } + if !strings.Contains(v.Why+p.Note, "hand it over") { + t.Errorf("the wait is not carried: verdict %q, walk %q", v.Why, p.Note) + } + return + } + if p.State == inventory.PlanDone { + t.Fatalf("a directory this send found let the walk through: %s", p.Note) + } + }) + } +} diff --git a/cmd/mesh-controller/module_health.go b/cmd/mesh-controller/module_health.go index f34f999d..25717cab 100644 --- a/cmd/mesh-controller/module_health.go +++ b/cmd/mesh-controller/module_health.go @@ -499,6 +499,7 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea if waits && !f.groupsAdded[module] { waits = false } + var found []string for _, r := range h.Resources { if r.Module != module { continue @@ -506,6 +507,14 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea if waits && r.State == link.StateUnhealthy { continue } + // **A directory used as found before this send waits for a person** (novox/hq issue 339): the node-engine + // left its owner and mode, and only someone at the machine can hand it over. It is no fault of this + // build, so it does not hold the module's walk — an urgent fix still goes through — and the verdict + // carries the wait. Found by this very send, the send brought it, and it is judged as unhealthy. + if usedAsFound(r) && r.Since.Before(since) { + found = append(found, r.Resource) + continue + } switch r.State { case link.StateHealthy: case link.StateStarting: @@ -521,12 +530,25 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea reasonAfter(r.Reason)) } } - if waits { - return healthPerson, wait + if waits || len(found) > 0 { + var said []string + if waits { + said = append(said, wait) + } + if len(found) > 0 { + said = append(said, fmt.Sprintf("on %s, %s uses %s as found and waits for a person to hand it over "+ + "(`mesh-host hand-over ` at the machine)", machine, module, strings.Join(found, ", "))) + } + return healthPerson, strings.Join(said, "; ") } return healthGood, "" } +// usedAsFound is a directory the node-engine states it uses as found (novox/hq issue 339). +func usedAsFound(r inventory.ResourceHealth) bool { + return r.Kind == link.KindDirectory && r.State == link.StateUnhealthy && strings.HasPrefix(r.Reason, link.ReasonUsedAsFound) +} + func reasonAfter(s string) string { if s == "" { return "" diff --git a/cmd/mesh-controller/modules.go b/cmd/mesh-controller/modules.go index 7bc6fc3b..352d6227 100644 --- a/cmd/mesh-controller/modules.go +++ b/cmd/mesh-controller/modules.go @@ -1062,12 +1062,12 @@ func declaresTools(m catalogue.Manifest) bool { return false } -// terminalSettings are the keys no verb may change (novox/hq issue 339). `places` says where the node-engine -// creates and, as root, owns a module's directories, with an owner the setting names; `accesses` says which of -// the machine's paths are mounted into a module's container. Set through a verb, either lets any caller of the -// mesh's console — an agent among them — have root hand it a directory, or mount one of the machine's into a -// container it reaches. They are the operator's, typed at the controller's terminal. -var terminalSettings = []string{catalogue.PlacesSetting, catalogue.AccessesSetting} +// The keys no verb may change are catalogue.TerminalKeys (novox/hq issue 339). `places` says where the +// node-engine creates and, as root, owns a module's directories, with an owner the setting names; `accesses` says +// which of the machine's paths are mounted into a module's container; a provider's trust anchors say what every +// consumer trusts. Set through a verb, any of them lets any caller of the mesh's verbs — an agent among them — +// have root hand it a directory, mount one of the machine's into a container it reaches, or have the mesh trust +// an authority of its own. They are the operator's, typed at the controller's terminal. // throughAVerb says whether this process runs a seat verb's command line: the serving controller names the // verb in the environment of every command it runs for one (runVerb), and a person at the terminal runs none. @@ -1085,16 +1085,16 @@ func refuseTerminalSettingsThroughAVerb(before, after map[string]any, module, wh if !through { return nil } - for _, key := range terminalSettings { + for _, key := range catalogue.TerminalKeys(module) { was, _ := json.Marshal(before[key]) now, _ := json.Marshal(after[key]) if string(was) == string(now) { continue } return fmt.Errorf("%s of %s on %s is set at the controller's terminal only, never through a verb (this "+ - "line came through %q): it says where root creates and owns a module's directories, or which of "+ - "the machine's paths are mounted into its container, and whoever may call a verb includes agents "+ - "(novox/hq issue 339). Nothing was changed", key, module, where, verb) + "line came through %q): it says where root creates and owns a module's directories, which of "+ + "the machine's paths are mounted into its container, or what the mesh's consumers trust, and whoever "+ + "may call a verb includes agents (novox/hq issue 339). Nothing was changed", key, module, where, verb) } return nil } diff --git a/cmd/mesh-controller/terminal_settings_test.go b/cmd/mesh-controller/terminal_settings_test.go index 48741817..79e6461f 100644 --- a/cmd/mesh-controller/terminal_settings_test.go +++ b/cmd/mesh-controller/terminal_settings_test.go @@ -150,3 +150,47 @@ func TestPlacesAndAccessesAreRefusedThroughEveryVerb(t *testing.T) { t.Fatalf("a refused call changed the layer: %s, was %s", got, kept) } } + +// The mesh's trust anchors are set at the terminal alone (novox/hq issue 339): through the settings verb, a caller +// could replace the internal authority's root every consumer trusts, or the issuer every login is checked against. +func TestATrustAnchorIsRefusedThroughAVerb(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "step-ca", Version: "1", + Provides: catalogue.FromAnywhere("acme-ca"), + Serves: map[string]map[string]any{"acme-ca": {"root": "", "path": "/acme/acme/directory"}}, + Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/step.conf", "mode": "0644", + "content": "x = ${setting:x}\n"}}}) + register(t, open, catalogue.Manifest{Module: "keycloak", Version: "1", + Provides: catalogue.FromAnywhere("oidc-client"), + Serves: map[string]map[string]any{"oidc-client": {"issuer": "${setting:issuer}"}}, + Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/kc.conf", "mode": "0644", + "content": "issuer = ${setting:issuer}\nx = ${setting:x}\n"}}}) + for _, m := range []string{"step-ca", "keycloak"} { + if _, err := assign(ctx, open, "anchor", m); err != nil { + t.Fatal(err) + } + } + refused := func(what string, err error) { + t.Helper() + if err == nil || !strings.Contains(err.Error(), "controller's terminal") { + t.Fatalf("%s: %v", what, err) + } + } + if err := atTheTerminal(t, "settings", "set", "keycloak", `{"issuer":"https://id.example/realms/mesh","x":0}`, + "--node", "anchor"); err != nil { + t.Fatalf("the issuer at the terminal: %v", err) + } + refused("the issuer through the verb", throughVerb(t, "settings", map[string]any{"module": "keycloak", + "node": "anchor", "values": `{"issuer":"https://evil.example/realms/mesh","x":0}`})) + refused("the authority's root path through the verb", throughVerb(t, "settings", map[string]any{"module": "step-ca", + "node": "anchor", "values": `{"path":"/evil","x":0}`})) + refused("the authority's root path, mesh-wide, through the verb", throughVerb(t, "settings", + map[string]any{"module": "step-ca", "values": `{"path":"/evil"}`})) + refused("clearing the issuer through the verb", throughVerb(t, "settings", map[string]any{"module": "keycloak", + "node": "anchor", "clear": "true"})) + if err := throughVerb(t, "settings", map[string]any{"module": "keycloak", "node": "anchor", + "values": `{"issuer":"https://id.example/realms/mesh","x":1}`}); err != nil { + t.Fatalf("another key through the verb, the issuer kept: %v", err) + } +} diff --git a/internal/catalogue/placement.go b/internal/catalogue/placement.go index 808151ea..24781dd0 100644 --- a/internal/catalogue/placement.go +++ b/internal/catalogue/placement.go @@ -62,7 +62,7 @@ var ownerShape = regexp.MustCompile(`^[0-9]+:[0-9]+$`) var ( systemTrees = []string{"/etc", "/usr", "/boot", "/root", "/run", "/var/run", "/var/lock", "/proc", "/sys", "/dev", "/bin", "/sbin", "/lib", "/lib32", "/lib64", "/var/lib/mesh", "/var/lib/mesh-host", "/var/spool", - "/var/lib/docker", "/var/lib/containers", "/opt"} + "/var/lib/docker", "/var/lib/containers", "/var/lib/containerd", "/opt"} systemRoots = []string{"/", "/var", "/var/lib", "/var/cache", "/var/log", "/var/tmp", "/home", "/mnt", "/media", "/srv", "/tmp", "/storage", "/data", "/services"} ) @@ -438,3 +438,24 @@ func namesOfAccessIDs(accesses map[string]string) []string { sort.Strings(names) return names } + +// trustAnchors are the settings a provider serves its consumers as what they trust, by module (novox/hq issue +// 339): set through a verb, any caller could point every consumer at an authority or an issuer of its own. +// +// - step-ca, the mesh's internal ACME authority: `root`, the root a consumer is handed to trust (the one +// setting that may hold lines, settingsHoldOneLine); `roots` and `path`, where a consumer fetches the roots +// and the ACME directory from, which a setting may override as it may any served fact. +// - keycloak, the identity provider: `issuer`, the issuer every OIDC consumer checks a login's token against. +// +// Named here, not in the manifests, because no manifest field says "this is trusted" yet; the catalogue was read +// for every served fact and every ${setting:…} on 2026-10-09, and these are the ones a consumer trusts. +var trustAnchors = map[string][]string{ + rootModule: {rootSetting, "roots", "path"}, + "keycloak": {"issuer"}, +} + +// TerminalKeys are the settings keys of a module that are set at the controller's terminal alone, never through +// a verb (novox/hq issue 339): places and accesses for every module, and a provider's trust anchors. +func TerminalKeys(module string) []string { + return append([]string{PlacesSetting, AccessesSetting}, trustAnchors[module]...) +} diff --git a/internal/catalogue/terminal_settings_test.go b/internal/catalogue/terminal_settings_test.go index bc3831fe..a8a722f2 100644 --- a/internal/catalogue/terminal_settings_test.go +++ b/internal/catalogue/terminal_settings_test.go @@ -109,7 +109,7 @@ func TestTheRuntimesDataAndAnyHomesSSHAreTheMachinesOwn(t *testing.T) { m := Manifest{Module: "notes", Resources: []map[string]any{{"id": "data", "type": "directory"}}, Accesses: []Access{{ID: "media"}}} for _, path := range []string{"/var/spool", "/var/spool/cron", "/var/lib/docker", "/var/lib/docker/volumes", - "/var/lib/containers/storage", "/opt", "/opt/app", "/home/alice/.ssh", "/home/alice/.ssh/keys", + "/var/lib/containers/storage", "/var/lib/containerd", "/var/lib/containerd/io.containerd.snapshotter.v1", "/opt", "/opt/app", "/home/alice/.ssh", "/home/alice/.ssh/keys", "/srv/backup/.ssh", "/root/.ssh"} { layers := []Layer{{From: "laptop", Values: map[string]any{PlacesSetting: map[string]any{"data": path}}}} if _, err := Places(m, layers); err == nil || !strings.Contains(err.Error(), "issue 339") { @@ -127,3 +127,18 @@ func TestTheRuntimesDataAndAnyHomesSSHAreTheMachinesOwn(t *testing.T) { } } } + +// A trust anchor the mesh hands its consumers is the terminal's too (novox/hq issue 339): the authority's root, +// where its roots and directory are, and the identity provider's issuer. +func TestTrustAnchorsAreTerminalKeys(t *testing.T) { + for module, keys := range map[string][]string{ + "step-ca": {"places", "accesses", "root", "roots", "path"}, + "keycloak": {"places", "accesses", "issuer"}, + "mailu": {"places", "accesses"}, + } { + got := TerminalKeys(module) + if strings.Join(got, ",") != strings.Join(keys, ",") { + t.Errorf("%s: %v; want %v", module, got, keys) + } + } +} diff --git a/internal/link/protocol.go b/internal/link/protocol.go index 3b28d86c..ad959149 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -446,6 +446,14 @@ const KindUnit = "unit" // starting ReasonRelogin when only a new login is missing. const KindAccount = "account" +// KindDirectory is a directory of a module that the node-engine uses as found (novox/hq issue 339): there before +// the mesh, with another owner or mode than declared, and left so until a person hands it over at the machine +// (`mesh-host hand-over`). Stated unhealthy with a reason that starts ReasonUsedAsFound. +const KindDirectory = "directory" + +// ReasonUsedAsFound starts the reason of a directory used as found. +const ReasonUsedAsFound = "used as found:" + // ReasonRelogin starts the reason of an account whose running session began before it was put in a group // (ADR 0252): the build did what it should, and a person has one step left (novox/hq ADR 0254). const ReasonRelogin = "relogin needed"