diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 8293d98..2e570db 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -20,6 +20,7 @@ import ( "os" "os/exec" "path/filepath" + "slices" "sort" "strconv" "strings" @@ -230,6 +231,18 @@ func ApplyKeeping( protecting = append(protecting, orphan) continue } + // **A module the mesh left out is not a module the mesh removed** (novox/hq ADR 0163, rule + // 6): its resources are absent because a setting stored for it cannot compose, and the + // mesh said so by name. What the host wrote for it stays as it is, recorded, until the + // module is declared again or unassigned. + if module, left := d.LeftOutModuleOf(orphan.ID); left { + report.Outcomes = append(report.Outcomes, Outcome{ + ID: orphan.ID, Type: orphan.Type, Target: orphan.Target, + Action: "unchanged", Detail: "kept: " + module + " was left out of this declaration by the mesh, not removed", + }) + log(fmt.Sprintf(" kept %s (%s): %s was left out of this declaration by the mesh, not removed", orphan.ID, orphan.Target, module)) + continue + } orphans = append(orphans, orphan) } ordered := d.Resources @@ -270,6 +283,11 @@ func ApplyKeeping( if declared[h.ID] { continue } + if slices.Contains(d.LeftOut, h.Module) { + // Left out, not unassigned (ADR 0163, rule 6): still held for the module, as the + // mesh asked. + continue + } known.Release(h.ID) report.Outcomes = append(report.Outcomes, Outcome{ID: h.ID, Type: h.Kind, Target: h.Target, Action: "forgotten", Detail: "no longer declared; left as found"}) @@ -1526,6 +1544,11 @@ func containerSpecReading(r *declaration.Container, declares, reads map[string]s if r.IP != "" { b.WriteString("ip " + r.IP + "\n") } + // The networks it also joins are part of what it is (ADR 0163, rule 4): kept or let go, the + // container is recreated, and a neighbour's reach changes with it. + for _, n := range r.Networks { + b.WriteString("also-on " + n + "\n") + } // The cadence is part of what was declared, so a changed schedule is a changed spec — the marker // moves and the install is reported "updated" and re-established. Added only when present, so no // ordinary container's or run-once step's digest moves for a field it does not set. @@ -1766,6 +1789,15 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner, if after.Spec != want { return out, fmt.Errorf("container %s is not the one that was declared after creating it", r.Name) } + // The found networks a per-machine setting keeps for it (novox/hq ADR 0163, rule 4), joined + // once it runs: a runtime starts a container on one network, and the others are connected. + // Refused, not skipped, when one cannot be joined — a neighbour that was promised to keep + // reaching this container by name would silently not. + for _, n := range r.Networks { + if _, err := run(ctx, cri, "network", "connect", n, r.Name); err != nil { + return out, fmt.Errorf("container %s could not join the kept network %s: %w", r.Name, n, err) + } + } out.Action = "created" if existed { diff --git a/internal/apply/left_out_test.go b/internal/apply/left_out_test.go new file mode 100644 index 0000000..f1d1067 --- /dev/null +++ b/internal/apply/left_out_test.go @@ -0,0 +1,136 @@ +package apply + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// A taken container keeps a found network by a per-machine setting (novox/hq ADR 0163, rule 4): +// joined once it runs, part of its spec, and refused when it cannot be joined. +func TestAContainerJoinsTheNetworksItKeeps(t *testing.T) { + var ran []string + connectFails := false + run := func(_ context.Context, name string, args ...string) (string, error) { + if name != "docker" { + return "", errors.New("not installed") + } + ran = append(ran, strings.Join(args, " ")) + switch args[0] { + case "info": + return "29.0.0\n", nil + case "container": + if len(ran) > 2 { + return "true\t" + specOfLast, nil + } + return "false\t\n", errors.New("no such container") + case "run": + return "deadbeef\n", nil + case "network": + if connectFails { + return "", errors.New("network predecessor_default not found") + } + } + return "", nil + } + d := parseTrusted(t, `{"declaration":1,"resources":[ + {"id":"app","type":"container","name":"app","image":"`+pinned+`", + "networks":["predecessor_default"]} + ]}`) + specOfLast = containerSpec(d.Resources[0].(*declaration.Container), inputs{}) + alone := *d.Resources[0].(*declaration.Container) + alone.Networks = nil + if specOfLast == containerSpec(&alone, inputs{}) { + t.Fatal("the kept network is not part of the container's spec: kept or let go, the container would be left alone") + } + report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) + if err != nil { + t.Fatal(err) + } + joined := false + for i, line := range ran { + if line == "network connect predecessor_default app" { + joined = true + if ran[i-1] != "container inspect --format {{.State.Running}}\t{{index .Config.Labels \""+specLabel+"\"}} app" && + !strings.HasPrefix(ran[i-1], "container inspect") { + t.Errorf("joined before the container was read back as running: %v", ran) + } + } + } + if !joined || report.Outcomes[0].Action != "created" { + t.Fatalf("the container did not join the kept network: %v\n%+v", ran, report.Outcomes) + } + + connectFails, ran = true, nil + if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil); err == nil || + !strings.Contains(err.Error(), "could not join the kept network predecessor_default") { + t.Fatalf("a network that cannot be joined was passed over: %v", err) + } +} + +var specOfLast string + +// A module the mesh left out of a declaration is not a module the mesh removed (novox/hq ADR 0163, +// rule 6): what the host wrote for it stays, recorded and said; what it holds for it stays held. +// A module simply absent is removed as it always was. +func TestALeftOutModuleIsNeitherRemovedNorForgotten(t *testing.T) { + var removed []string + gone := map[string]bool{} + run := func(_ context.Context, name string, args ...string) (string, error) { + if name != "docker" { + return "", nil + } + switch args[0] { + case "info": + return "29.0.0\n", nil + case "rm": + removed = append(removed, args[len(args)-1]) + gone[args[len(args)-1]] = true + case "container": + if gone[args[len(args)-1]] { + return "", errors.New("no such container") + } + return "true\tspec", nil + } + return "", nil + } + known := store.State{ + Resources: []store.Applied{ + {ID: "web.server", Type: "container", Target: "web", Origin: store.OriginDeclared}, + {ID: "old.server", Type: "container", Target: "old", Origin: store.OriginDeclared}, + }, + Held: []store.Held{{ID: "web.page", Module: "web", Kind: "file", Target: "/srv/web/index.html"}}, + } + d := parse(t, `{"declaration":1,"left_out":["web"],"resources":[ + {"id":"notes.conf","type":"file","path":"`+t.TempDir()+`/notes.conf","content":"x"} + ]}`) + report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil) + if err != nil { + t.Fatal(err) + } + if len(removed) != 1 || removed[0] != "old" { + t.Fatalf("removed %v; only the module that is absent goes", removed) + } + if _, kept := state.At("container", "web"); !kept { + t.Fatal("the left-out module's record was forgotten") + } + if _, held := state.HeldAt("web.page"); !held { + t.Fatal("the left-out module's hold was released") + } + said := false + for _, o := range report.Outcomes { + if o.ID == "web.server" && o.Action == "unchanged" && strings.Contains(o.Detail, "web was left out of this declaration by the mesh") { + said = true + } + if o.ID == "web.server" && o.Action != "unchanged" { + t.Errorf("the left-out module's container was %s", o.Action) + } + } + if !said { + t.Fatalf("keeping the left-out module's container was not said: %+v", report.Outcomes) + } +} diff --git a/internal/bootstrap/genesis_forge_test.go b/internal/bootstrap/genesis_forge_test.go new file mode 100644 index 0000000..9a75bdb --- /dev/null +++ b/internal/bootstrap/genesis_forge_test.go @@ -0,0 +1,79 @@ +package bootstrap + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// What genesis raises, it raises as the module that succeeds it declares — name, data directory +// and image — so the module adopts it by the found rule that already exists (novox/hq ADR 0163, +// rule 7; issue 090). The network is the one difference left: the bootstrap forge runs on the +// machine's network to reach the store on its loopback, and a take says so. +func TestGenesisRaisesTheForgeAsTheModuleDeclaresIt(t *testing.T) { + var ran [][]string + run := func(_ context.Context, name string, args ...string) (string, error) { + if name == "docker" && args[0] == "container" { + return "", nil // not raised yet + } + ran = append(ran, append([]string{name}, args...)) + return "", nil + } + if err := raiseGiteaServer(context.Background(), run, time.Second, "pw", DefaultPorts(), quietly); err != nil { + t.Fatal(err) + } + var raised []string + for _, r := range ran { + if r[0] == "docker" && r[1] == "run" { + raised = r + } + } + line := strings.Join(raised, " ") + for _, want := range []string{"--name gitea ", "--volume " + giteaDataDir + ":/data", " " + giteaImage} { + if !strings.Contains(line+" ", want) { + t.Errorf("the forge is not raised with %q: %s", want, line) + } + } + if giteaBootstrap != ForgeModule { + t.Errorf("the bootstrap forge is %q and the module names its container %q", giteaBootstrap, ForgeModule) + } + + // Against the module's own manifest, where the catalogue is checked out beside this repository. + var manifest []byte + for _, candidate := range []string{"../../../mesh-catalog/modules/gitea/module.json", "../../../../../mesh-catalog/modules/gitea/module.json"} { + if raw, err := os.ReadFile(filepath.Clean(candidate)); err == nil { + manifest = raw + break + } + } + if manifest == nil { + t.Skip("the catalogue is not beside this checkout; the module's pin is not compared") + } + var m struct { + Resources []struct { + ID, Type, Name, Image string + Volumes []string + } `json:"resources"` + } + if err := json.Unmarshal(manifest, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if r.Type != "container" || r.ID != "server" { + continue + } + if r.Name != giteaBootstrap { + t.Errorf("the module names its container %q; genesis raises %q", r.Name, giteaBootstrap) + } + if r.Image != giteaImage { + t.Errorf("the module pins %s; genesis raises %s — the two must move together", r.Image, giteaImage) + } + if len(r.Volumes) != 1 || !strings.HasSuffix(r.Volumes[0], ":/data") { + t.Errorf("the module mounts %v; genesis mounts %s:/data", r.Volumes, giteaDataDir) + } + } +} diff --git a/internal/bootstrap/inuse_test.go b/internal/bootstrap/inuse_test.go index 2cc889d..594d334 100644 --- a/internal/bootstrap/inuse_test.go +++ b/internal/bootstrap/inuse_test.go @@ -91,7 +91,7 @@ func TestARerunOfGenesisIsNotAMachineInUse(t *testing.T) { if err := store.Save(o.State, store.State{Resources: []store.Applied{{ID: "store", Type: "container", Target: "mesh-store"}}}); err != nil { t.Fatal(err) } - m := inUseRunner{ps: "mesh-gitea-server\t\n", ss: servingSockets} + m := inUseRunner{ps: giteaBootstrap + "\t\n", ss: servingSockets} if err := RefuseAMachineInUse(context.Background(), o, m.run, quietly); err != nil { t.Errorf("what an earlier genesis raised was counted as a machine in use: %v", err) } diff --git a/internal/bootstrap/phase_packages.go b/internal/bootstrap/phase_packages.go index a446b31..a2bf575 100644 --- a/internal/bootstrap/phase_packages.go +++ b/internal/bootstrap/phase_packages.go @@ -25,11 +25,24 @@ const ( // foundationStore is the foundation's postgres container — the mesh's own memory, raised from the // bundle. gitea's bootstrap database lives here too, so a mesh runs one postgres (issue 051). foundationStore = "mesh-store" - // giteaBootstrap is the gitea server raised directly at genesis, before gitea is a module. - giteaBootstrap = "mesh-gitea-server" - // giteaImage is the same upstream image the gitea module runs, pinned identically so the module - // adopts the running server rather than replacing it. - giteaImage = "gitea/gitea@sha256:dfc61e347c8b582df918f4556401bf2cecdfbdb56c5282ae9488dd76fca3e41c" + // giteaBootstrap is the gitea server raised directly at genesis, before gitea is a module — + // under the name the gitea MODULE declares for its container, so the module finds it and holds + // it rather than raising a second forge beside it (novox/hq ADR 0163, rule 7; issue 090). + giteaBootstrap = "gitea" + // giteaImage is the image the gitea module declares for that container, pinned to the same + // digest, so taking the module over is not a downgrade and not an upgrade. **Moves with the + // module's pin**: the two are compared by a take, and a difference is said there — but a + // genesis that raised an older image than the module declares would be taken over as an + // upgrade on first push, which a forge holding the mesh's packages must not have done to it + // unannounced. Checked in TestGenesisRaisesTheForgeAsTheModuleDeclaresIt against the module's + // manifest where the catalogue is beside this checkout. + giteaImage = "gitea/gitea@sha256:87a67ee09d3ae0d1df5fda5dcda3e2a1f9236a45b0a59025d6e00e46adc43bef" + // giteaDataDir is where the module's `data` directory resolves on a machine with the default + // layout (//, novox/hq ADR 0112): mounted at /data as the module mounts + // it, so the repositories, attachments and indexes the bootstrap forge accumulates are the + // module's the day it is taken — before this, the forge had no volume and its data was the + // container's, lost with it. + giteaDataDir = "/var/lib/gitea/data" // packagesOrg is the npm owner: every module consumes `@novox/*` from this gitea org. packagesOrg = "novox" // packagesTeam is the org team whose members may read and write the org's packages. @@ -262,9 +275,13 @@ func raiseGiteaServer(ctx context.Context, run Runner, timeout time.Duration, db "run", "-d", "--name", giteaBootstrap, // Host network, like the control plane: it reaches the foundation store on the machine's // loopback (where the store publishes 5432) and answers on the machine's own 3000, which is - // where mesh-bootstrap and the builder's build containers look for it. + // where mesh-bootstrap and the builder's build containers look for it. The module runs + // bridged and publishes its ports; that is the one difference a take still has to say + // (ADR 0163, rule 7) — the data, the name and the image are the module's already. "--network", "host", "--restart", "unless-stopped", + // The module's data directory, so what the forge accumulates is the module's when taken. + "--volume", giteaDataDir + ":/data", }, env...) args = append(args, giteaImage) diff --git a/internal/declaration/declaration.go b/internal/declaration/declaration.go index d665e7d..4272865 100644 --- a/internal/declaration/declaration.go +++ b/internal/declaration/declaration.go @@ -940,6 +940,13 @@ type Container struct { // its siblings can name before any of them can resolve anything. Dns []string `json:"dns,omitempty"` + // Networks are networks this container also joins once created, by name — a found network a + // per-machine setting keeps for a taken container (novox/hq ADR 0163, rule 4), so a + // neighbour that resolves it there keeps resolving it until the neighbour is taken too. + // Joined after creation, because a runtime starts a container on one network; part of the + // container's spec, so a network kept or let go recreates it. + Networks []string `json:"networks,omitempty"` + // IP is this container's address on its network, passed to the runtime unchanged. // // Only meaningful on a user-defined network, and refused by the runtime elsewhere. Exists for @@ -1032,6 +1039,16 @@ func (c *Container) validate(where string, _ bool) []string { "static address anywhere but a user-defined one") } } + for _, n := range c.Networks { + problems = append(problems, (&Network{Name: n}).validate(where+": networks", false)...) + if n == c.Network { + problems = append(problems, where+": networks names "+n+", which is already the container's network") + } + } + if len(c.Networks) > 0 && (c.RunOnce || c.Schedule != "") { + problems = append(problems, where+": networks is for a container that keeps running; a step "+ + "runs and exits, and joins nothing afterwards") + } return append(problems, checkImage(where, c.Image)...) } @@ -1150,6 +1167,28 @@ type Declaration struct { // backlog longer than the batch, or a slow broker, applied a declaration the mesh had already // superseded. Sequence int64 + + // LeftOut names the modules of this machine's set the mesh left out of this declaration, + // because a setting stored for one cannot compose with its definition (novox/hq ADR 0163, + // rule 6). A machine is told everything or nothing about what it IS told; this is what it is + // not told, said. The host keeps what it holds for a left-out module and touches none of + // what it wrote for it — its resources are absent from the declaration, and absence would + // otherwise read as removal. + LeftOut []string +} + +// LeftOutModuleOf says which left-out module a recorded resource belongs to, if any: its id is the +// module's name, a dot, and the module's own id for it. A module's name may contain a dot, so the +// longest left-out name that prefixes the id wins; a false match keeps a thing an apply would +// otherwise remove, which is the conservative mistake. +func (d *Declaration) LeftOutModuleOf(id string) (string, bool) { + best := "" + for _, m := range d.LeftOut { + if strings.HasPrefix(id, m+".") && len(m) > len(best) { + best = m + } + } + return best, best != "" } // Adoption is a node's mode, as the controller records it: the node is adopted, and these are @@ -1290,6 +1329,8 @@ type envelope struct { // Sequence is optional on the wire, so a controller that does not send one is still // understood: absent reads as zero, which is "no ordering claimed" rather than "first". Sequence int64 `json:"sequence,omitempty"` + // LeftOut is optional on the wire too, and absent when nothing was left out (ADR 0163). + LeftOut []string `json:"left_out,omitempty"` } func parse(raw []byte, allowActions bool) (*Declaration, error) { @@ -1306,8 +1347,20 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) { env.Version, Version)}} } - d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption, Sequence: env.Sequence} + d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption, Sequence: env.Sequence, + LeftOut: env.LeftOut} var problems []string + if len(env.LeftOut) > 0 && allowActions { + // The bundle is carried with the binary and leaves nothing out: which module a setting + // stopped composing for is the mesh's record (ADR 0163). + problems = append(problems, "a carried bundle says modules were left out, and only the "+ + "mesh can say that") + } + for _, m := range env.LeftOut { + if strings.TrimSpace(m) == "" { + problems = append(problems, "left_out names a module with no name") + } + } if len(env.Resources) == 0 && !env.OwnsNothing { problems = append(problems, "no resources. An empty declaration is a mistake, not a "+ diff --git a/internal/declaration/declaration_test.go b/internal/declaration/declaration_test.go index 95c8a16..98e6c92 100644 --- a/internal/declaration/declaration_test.go +++ b/internal/declaration/declaration_test.go @@ -464,3 +464,43 @@ func TestAnExplicitlyEmptyDeclarationIsAccepted(t *testing.T) { t.Fatalf("an unmarked empty declaration must still be refused; got %v", err) } } + +// A container's kept networks are names, not its own network, and not for a step (novox/hq ADR +// 0163, rule 4); and the mesh may say which modules it left out, which a carried bundle may not. +func TestKeptNetworksAndLeftOutModulesAreReadStrictly(t *testing.T) { + pinnedImage := "postgres@sha256:" + strings.Repeat("a", 64) + d, err := Parse([]byte(`{"declaration":1,"left_out":["web"],"resources":[ + {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `","networks":["predecessor_default"]} + ]}`)) + if err != nil { + t.Fatal(err) + } + if got := d.Resources[0].(*Container).Networks; len(got) != 1 || got[0] != "predecessor_default" { + t.Fatalf("the kept network was not read: %v", got) + } + if m, left := d.LeftOutModuleOf("web.server"); !left || m != "web" { + t.Fatalf("web.server is not web's: %q %v", m, left) + } + if _, left := d.LeftOutModuleOf("webapp.server"); left { + t.Fatal("webapp.server was taken for web's") + } + for name, raw := range map[string]string{ + "a bad network name": `{"declaration":1,"resources":[ + {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `","networks":["a/b"]}]}`, + "its own network": `{"declaration":1,"resources":[ + {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `","network":"own","networks":["own"]}]}`, + "a step": `{"declaration":1,"resources":[ + {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `","run-once":true,"networks":["x"]}]}`, + "a nameless module": `{"declaration":1,"left_out":[""],"resources":[ + {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `"}]}`, + } { + if _, err := Parse([]byte(raw)); err == nil { + t.Errorf("%s was accepted", name) + } + } + if _, err := ParseTrusted([]byte(`{"declaration":1,"left_out":["web"],"resources":[ + {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `"}]}`)); err == nil || + !strings.Contains(err.Error(), "only the mesh can say that") { + t.Fatalf("a carried bundle leaving modules out was accepted: %v", err) + } +}