diff --git a/cmd/mesh-controller/held_library.go b/cmd/mesh-controller/held_library.go index 59061b69..be5fab4f 100644 --- a/cmd/mesh-controller/held_library.go +++ b/cmd/mesh-controller/held_library.go @@ -2,25 +2,43 @@ package main import ( "context" + "fmt" "sort" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" ) -// A module that joins keeps the user it ran as until every library it uses is taken (novox/hq ADR 0328 §8). +// A module that joins keeps the user it ran as until it may join (novox/hq ADR 0328 §8, ADR 0334). // -// **Held at the build it ran before, composed exactly as then.** A module whose registered build uses a library not yet -// taken, on a machine that was last sent another build of it, is composed at that build — the manifest the build -// records hold, in place of the registered one, as a recorded module is kept (recorded_kept.go): its resources, its -// ports in the filter, its contributions and bindings are what they were, and nothing of the new contract is sent for -// it. The send records that the machine still carries that build. Once the library is taken, the next composition -// takes the new build. A module the machine never ran has nothing to keep, and is composed as registered; a build the -// records no longer hold cannot be kept, and the module is composed as registered (it then waits as joining modules do). +// **Held at the build it ran before, composed exactly as then.** A module whose registered build joins — declares +// libraries or uses its identity — on a machine that was last sent another build of it, is held there while // -// shelf, when given, has each held module's manifest replaced. It answers what it holds: module → the commit kept. +// - the machine's engine is older than link.OwnGroupContract: it cannot put the tool runner in the module's own group +// nor keep its own directories' group, so the module's tools would lose their read (the re-review of 96e15f86, 2); +// - or a library it uses is not yet taken (§8). +// +// Held, it is composed at that build — the manifest the build records hold, in place of the registered one, as a +// recorded module is kept (recorded_kept.go) — and with the settings it last ran with (SettingsAsOf the machine's last +// send): its resources, its ports in the filter, its contributions and bindings are what they were, and nothing of the +// new contract is sent for it. The send records that the machine still carries that build. A delivery that brings the +// joining build does not send it while it is held: the module waits for a person, the plan says so and judges its +// other modules, and sends it once nothing holds it (release_plan.go, heldIn). A module the machine never ran has +// nothing to keep, and is composed as registered; a build the records no longer hold cannot be kept. + +// heldModule is one module held on a machine: the build it keeps and why, in words with no path. +type heldModule struct { + commit string + // library is the first library it uses that is not taken, empty when none. + library string + why string +} + +// holdJoining is every module held on a machine, by module; shelf, when given, has each one's manifest replaced by +// the build it keeps. func holdJoining(ctx context.Context, inv *inventory.Inventory, node string, - shelf map[string]catalogue.Manifest) (map[string]string, error) { + shelf map[string]catalogue.Manifest) (map[string]heldModule, error) { registered := shelf if registered == nil { var err error @@ -34,7 +52,7 @@ func holdJoining(ctx context.Context, inv *inventory.Inventory, node string, } var joining []string for m := range sent { - if man, has := registered[m]; has && len(man.Libraries) > 0 { + if man, has := registered[m]; has && catalogue.Joins(man) { joining = append(joining, m) } } @@ -42,6 +60,11 @@ func holdJoining(ctx context.Context, inv *inventory.Inventory, node string, return nil, nil } sort.Strings(joining) + stated, had, err := inv.HealthOf(ctx, node) + if err != nil { + return nil, err + } + engineJoins := had && stated.Contract >= link.OwnGroupContract libs, err := inv.Libraries(ctx) if err != nil { return nil, err @@ -54,40 +77,86 @@ func holdJoining(ctx context.Context, inv *inventory.Inventory, node string, if err != nil { return nil, err } - out := map[string]string{} + out := map[string]heldModule{} for _, m := range joining { was := sent[m] if was == "" || sameCommit(was, current[m].Commit) { continue // the machine already runs the registered build: nothing before it to keep } - untaken := "" + h := heldModule{commit: was} for _, u := range registered[m].Libraries { if !taken[u.Library] { - untaken = u.Library + h.library = u.Library break } } - if untaken == "" { + switch { + case !engineJoins: + h.why = fmt.Sprintf("%s keeps the build it runs on %s until that machine's node-engine keeps a module's own "+ + "group (novox/hq ADR 0334)", m, node) + case h.library != "": + h.why = fmt.Sprintf("%s waits for the operator to let the mesh keep the library %s (novox/hq ADR 0328 §8)", + m, h.library) + default: continue } before, found, err := inv.ManifestAt(ctx, m, was) if err != nil { return nil, err } - if !found || usesLibrary(before, untaken) { + if !found || (engineJoins && usesLibrary(before, h.library)) { continue } if shelf != nil { shelf[m] = before } - out[m] = was + out[m] = h } return out, nil } +// heldIn is why a module is held on any of the machines given, or "": what a delivery's first send waits on. +func heldIn(ctx context.Context, inv *inventory.Inventory, module string, machines []string) (string, error) { + for _, node := range machines { + held, err := holdJoining(ctx, inv, node, nil) + if err != nil { + return "", err + } + if h, is := held[module]; is { + return h.why, nil + } + } + return "", nil +} + +// heldSettings puts, for each module held on a machine, the settings it last ran with in place of today's: the layers +// as they stood at the machine's last send. Today's stand where no send is recorded. +func heldSettings(ctx context.Context, inv *inventory.Inventory, node string, held map[string]heldModule, + settings catalogue.SettingsBy) error { + if len(held) == 0 { + return nil + } + last, found, err := inv.LastSend(ctx, node) + if err != nil || !found { + return err + } + for m := range held { + layers, err := inv.SettingsAsOf(ctx, node, m, last.SentAt) + if err != nil { + return err + } + if len(layers) == 0 { + delete(settings, m) + continue + } + settings[m] = layers + } + return nil +} + func usesLibrary(m catalogue.Manifest, library string) bool { for _, u := range m.Libraries { - if u.Library == library { + if library != "" && u.Library == library { return true } } diff --git a/cmd/mesh-controller/held_plan_test.go b/cmd/mesh-controller/held_plan_test.go new file mode 100644 index 00000000..ace5d903 --- /dev/null +++ b/cmd/mesh-controller/held_plan_test.go @@ -0,0 +1,329 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" + "github.com/novox/mesh-controller/internal/link" +) + +// The re-review of 96e15f86, item 1 (novox/hq ADR 0328 §8): a delivery that brings a joining build of a module whose +// library is not taken does not send it — the module waits for a person, the plan stays open saying so, and the other +// modules of the plan are sent and judged as usual. Once the library is taken, the waiting build is sent and judged, +// with no new merge. +func TestADeliveryWaitsForALibraryAndJudgesTheRest(t *testing.T) { + judgeLeavesAsTheCore(t) + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + keeper, _ := withConditionsInMemory(t) + was := doctorFrom + doctorFrom = &doctor{open: open, keeper: keeper, teller: &conditions.Told{}} + t.Cleanup(func() { doctorFrom = was }) + + register(t, open, catalogue.Manifest{Module: catalogue.LibrariesModule, Version: "1"}) + if err := inv.SetSettings(ctx, "", catalogue.LibrariesModule, map[string]any{catalogue.LibrariesSetting: map[string]any{ + "media": map[string]any{"home": "anchor", "root": "/storage/media"}}}); err != nil { + t.Fatal(err) + } + app1 := catalogue.Manifest{Module: "app", Version: "c1"} + app2 := catalogue.Manifest{Module: "app", Version: "c2"} + builds := []struct { + id, module, commit string + m catalogue.Manifest + at time.Time + }{ + {"build-app-1", "app", "c1", app1, time.Now().Add(-2 * time.Hour)}, + {"build-radarr-1", "radarr", "r1", radarrBefore(), time.Now().Add(-2 * time.Hour)}, + {"build-app-2", "app", "c2", app2, time.Now().Add(-time.Minute)}, + {"build-radarr-2", "radarr", "r2", radarrJoined(), time.Now().Add(-time.Minute)}, + } + for _, b := range builds { + raw, _ := json.Marshal(b.m) + if err := inv.RecordBuild(ctx, inventory.Build{ID: b.id, Module: b.module, Commit: b.commit, + Repository: "novox/mesh-media-catalog", Path: "modules/" + b.module, Ref: "main", Manifest: raw, + Asked: b.at, At: b.at}); err != nil { + t.Fatal(err) + } + } + registerAt := func(m catalogue.Manifest, commit string, asked time.Time) { + if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-media-catalog", Seat: "git", + Path: "modules/" + m.Module, Ref: "main", BuiltFrom: commit, Head: commit, Asked: asked}); err != nil { + t.Fatal(err) + } + } + registerAt(app1, "c1", time.Now().Add(-2*time.Hour)) + registerAt(radarrBefore(), "r1", time.Now().Add(-2*time.Hour)) + if _, err := inv.Assign(ctx, "anchor", "app"); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, "anchor", "radarr"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-0", map[string]string{"app": "c1", "radarr": "r1"}); err != nil { + t.Fatal(err) + } + if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.OwnGroupContract, + SaidAt: time.Now(), HeardAt: time.Now()}); err != nil { + t.Fatal(err) + } + registerAt(app2, "c2", time.Now().Add(-time.Minute)) + registerAt(radarrJoined(), "r2", time.Now().Add(-time.Minute)) + + // Every send records what it carries: the registered builds, a held module at the build it runs. + var sent []string + wasSend := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + current, err := open.inventory.CurrentBuilds(ctx) + if err != nil { + return nil, err + } + for _, node := range names { + held, err := holdJoining(ctx, open.inventory, node, nil) + if err != nil { + return nil, err + } + carried := map[string]string{"app": current["app"].Commit, "radarr": current["radarr"].Commit} + for m, h := range held { + carried[m] = h.commit + } + sent = append(sent, node+":app="+carried["app"]+",radarr="+carried["radarr"]) + digest := fmt.Sprintf("d-%s-%d", node, len(sent)) + if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil { + return nil, err + } + if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, + Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { + return nil, err + } + } + return names, nil + } + t.Cleanup(func() { sendRollout = wasSend }) + wasGather := gatherGateFacts + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, + rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} + reports, err := open.inventory.LastReports(ctx) + for _, r := range reports { + f.reports[r.Node] = r + } + return f, err + } + t.Cleanup(func() { gatherGateFacts = wasGather }) + wasSettle, wasEvery := gateSettle, gateEvery + gateSettle, gateEvery = 0, 0 + t.Cleanup(func() { gateSettle, gateEvery = wasSettle, wasEvery }) + + built := time.Now().UTC() + plan := inventory.Plan{ID: "plan-held", Repository: "novox/mesh-media-catalog", Branch: "main", Commit: "c2", + Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app", "radarr"}}, + Modules: map[string]*inventory.PlanModule{ + "app": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-app-2"}, + "radarr": {State: "built", BuiltAt: &built, Commit: "r2", Build: "build-radarr-2"}}} + if err := inv.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + for i := 0; i < 4; i++ { + advancePlans(ctx, open) + } + p, err := inv.PlanByID(ctx, "plan-held") + if err != nil { + t.Fatal(err) + } + if p.State == inventory.PlanSuperseded || p.State == inventory.PlanDone || p.State == inventory.PlanFailed { + t.Fatalf("the plan is %s (%s); want it open, waiting for the library", p.State, p.Note) + } + if g := p.Modules["app"].Gate; g == nil || g.Verdict != inventory.GatePassed { + t.Fatalf("app was not judged and passed beside the held module: %+v (%s)", p.Modules["app"], p.Note) + } + if p.Modules["radarr"].FirstAt != nil || !strings.Contains(p.Note, "library media") { + t.Fatalf("radarr was sent, or the plan does not say it waits for the library: %+v, %q", p.Modules["radarr"], p.Note) + } + if slices.ContainsFunc(sent, func(s string) bool { return strings.Contains(s, "radarr=r2") }) { + t.Fatalf("radarr's joining build was sent while held: %v", sent) + } + + // The library is taken: the waiting build is sent, with no new merge. + if err := inv.RecordTakeStarted(ctx, inventory.TakenLibrary{Library: "media", Home: "anchor", Root: "/storage/media", + By: "the operator", Ask: "c01", Warrant: "c01"}); err != nil { + t.Fatal(err) + } + if err := inv.TakeDone(ctx, "media", "c01"); err != nil { + t.Fatal(err) + } + for i := 0; i < 4; i++ { + advancePlans(ctx, open) + } + if !slices.ContainsFunc(sent, func(s string) bool { return strings.Contains(s, "radarr=r2") }) { + t.Fatalf("radarr's joining build was not sent once its library was taken: %v", sent) + } +} + +// The re-review of 96e15f86, item 2: a module joins only on an engine stating link.OwnGroupContract (7). On an engine +// at 6 a module that ran there before is held at the build it ran, whatever its library — so its tools never lose +// their read of its directories. +func TestAJoiningModuleIsHeldOnAnEngineOlderThanOwnGroups(t *testing.T) { + open, record := heldMesh(t) + ctx := t.Context() + inv := open.inventory + if err := inv.RecordTakeStarted(ctx, inventory.TakenLibrary{Library: "media", Home: "anchor", Root: "/storage/media", + By: "the operator", Ask: "c01", Warrant: "c01"}); err != nil { + t.Fatal(err) + } + if err := inv.TakeDone(ctx, "media", "c01"); err != nil { + t.Fatal(err) + } + _ = record + if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.LibraryContract, + SaidAt: time.Now(), HeardAt: time.Now()}); err != nil { + t.Fatal(err) + } + held, err := holdJoining(ctx, inv, "anchor", nil) + if err != nil { + t.Fatal(err) + } + if held["radarr"].commit != "c1c1c1c1" { + t.Fatalf("on an engine at contract 6 radarr is not held at the build it ran: %+v", held) + } + if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.OwnGroupContract, + SaidAt: time.Now().Add(time.Second), HeardAt: time.Now()}); err != nil { + t.Fatal(err) + } + if held, err = holdJoining(ctx, inv, "anchor", nil); err != nil || len(held) > 0 { + t.Fatalf("on an engine at contract 7 with its library taken radarr is still held: %+v (%v)", held, err) + } +} + +// heldMesh is anchor running radarr's build from before it joined, the joining build registered, media defined. +func heldMesh(t *testing.T) (*stores, inventory.Node) { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + register(t, open, catalogue.Manifest{Module: catalogue.LibrariesModule, Version: "1"}) + if err := inv.SetSettings(ctx, "", catalogue.LibrariesModule, map[string]any{catalogue.LibrariesSetting: map[string]any{ + "media": map[string]any{"home": "anchor", "root": "/storage/media"}}}); err != nil { + t.Fatal(err) + } + for _, b := range []struct { + id, commit string + m catalogue.Manifest + }{{"b1", "c1c1c1c1", radarrBefore()}, {"b2", "c2c2c2c2", radarrJoined()}} { + raw, _ := json.Marshal(b.m) + if err := inv.RecordBuild(ctx, inventory.Build{ID: b.id, Repository: "https://forge.invalid/media.git", Ref: "main", + Module: "radarr", Commit: b.commit, On: "builder", Manifest: raw}); err != nil { + t.Fatal(err) + } + } + if err := inv.RegisterModule(ctx, radarrBefore(), inventory.Source{Repository: "https://forge.invalid/media.git", + Ref: "main", BuiltFrom: "c1c1c1c1"}); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "anchor", "radarr"); err != nil { + t.Fatal(err) + } + record, err := inv.NodeByName(ctx, "anchor") + if err != nil { + t.Fatal(err) + } + if err := inv.RecordSentWith(ctx, record.ID, "d1", map[string]string{"radarr": "c1c1c1c1"}, 1, 1, + inventory.Send{Sequence: 1}); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, radarrJoined(), inventory.Source{Repository: "https://forge.invalid/media.git", + Ref: "main", BuiltFrom: "c2c2c2c2"}); err != nil { + t.Fatal(err) + } + return open, record +} + +// The re-review of 96e15f86, item 3: a held module is composed with the settings it last ran with — a layer changed +// for the joining build (its puid removed) does not reach the build it still runs. +func TestAHeldModuleKeepsTheSettingsItRanWith(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + register(t, open, catalogue.Manifest{Module: catalogue.LibrariesModule, Version: "1"}) + if err := inv.SetSettings(ctx, "", catalogue.LibrariesModule, map[string]any{catalogue.LibrariesSetting: map[string]any{ + "media": map[string]any{"home": "anchor", "root": "/storage/media"}}}); err != nil { + t.Fatal(err) + } + before := radarrBefore() + before.Resources = append(before.Resources, map[string]any{"id": "identity", "type": "file", + "path": "${dir:config}/identity.env", "mode": "0644", "content": "PUID=${setting:puid}\n"}) + joined := radarrJoined() + for _, b := range []struct { + id, commit string + m catalogue.Manifest + }{{"b1", "c1c1c1c1", before}, {"b2", "c2c2c2c2", joined}} { + raw, _ := json.Marshal(b.m) + if err := inv.RecordBuild(ctx, inventory.Build{ID: b.id, Repository: "https://forge.invalid/media.git", Ref: "main", + Module: "radarr", Commit: b.commit, On: "builder", Manifest: raw}); err != nil { + t.Fatal(err) + } + } + if err := inv.RegisterModule(ctx, before, inventory.Source{Repository: "https://forge.invalid/media.git", + Ref: "main", BuiltFrom: "c1c1c1c1"}); err != nil { + t.Fatal(err) + } + if err := inv.SetSettings(ctx, "anchor", "radarr", map[string]any{"puid": "1000"}); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "anchor", "radarr"); err != nil { + t.Fatal(err) + } + record, err := inv.NodeByName(ctx, "anchor") + if err != nil { + t.Fatal(err) + } + if err := inv.RecordSentWith(ctx, record.ID, "d1", map[string]string{"radarr": "c1c1c1c1"}, 1, 1, + inventory.Send{Sequence: 1}); err != nil { + t.Fatal(err) + } + time.Sleep(20 * time.Millisecond) + if err := inv.RegisterModule(ctx, joined, inventory.Source{Repository: "https://forge.invalid/media.git", + Ref: "main", BuiltFrom: "c2c2c2c2"}); err != nil { + t.Fatal(err) + } + if err := inv.ClearSettings(ctx, "anchor", "radarr"); err != nil { // the joining build reads no puid + t.Fatal(err) + } + if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.OwnGroupContract, + SaidAt: time.Now(), HeardAt: time.Now()}); err != nil { + t.Fatal(err) + } + gens, err := generators(ctx, open) + if err != nil { + t.Fatal(err) + } + plan, settings, err := planFor(ctx, open, "anchor") + if err != nil { + t.Fatal(err) + } + held, err := declarationWith(ctx, open, "anchor", plan, settings, gens, Reading) + if err != nil { + t.Fatal(err) + } + if why, left := held.leftOutWhy["radarr"]; left { + t.Fatalf("radarr held was left out: %s", why) + } + for _, r := range held.Resources { + if r["id"] == "radarr.identity" { + if r["content"] != "PUID=1000\n" { + t.Fatalf("radarr held reads %q; want the puid it ran with", r["content"]) + } + return + } + } + t.Fatal("radarr held has no identity file") +} diff --git a/cmd/mesh-controller/kept_libraries_test.go b/cmd/mesh-controller/kept_libraries_test.go index f832d2dd..ed8f588b 100644 --- a/cmd/mesh-controller/kept_libraries_test.go +++ b/cmd/mesh-controller/kept_libraries_test.go @@ -543,3 +543,43 @@ func TestALibraryIsTakenOnlyWhenTheEngineSaysTheActIsDone(t *testing.T) { t.Fatalf("the act done did not make the library taken (%v)", err) } } + +// ADR 0334: the engine states the tool runner's membership as resources of kind runner — `runner:` of that +// module, `runner:library:` of no module. A library's is said for the modules that use it; neither fails the +// first-node gate of a move that leaves the module's libraries as they were, and both hold one that changes them. +func TestARunnersVerdictIsSaidAndJudgedByWhetherTheMoveChangesIt(t *testing.T) { + inv := inventory.ForTest(t) + ctx := t.Context() + if _, err := inv.AddNode(ctx, "ace"); err != nil { + t.Fatal(err) + } + for _, m := range []catalogue.Manifest{radarrManifest, plexManifest} { + if err := inv.RegisterModule(ctx, m, inventory.Source{}); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, "ace", m.Module); err != nil { + t.Fatal(err) + } + } + lib := inventory.ResourceHealth{Resource: "runner:library:media", Kind: link.KindRunner, Target: "jochen", + State: link.StateUnhealthy, Reason: "the tool runner is not in mesh-media yet", Since: time.Now()} + copies := libraryVerdictsFor(ctx, inv, "ace", []inventory.ResourceHealth{lib}) + if len(copies) != 2 || copies[0].Module != "plex" || copies[1].Module != "radarr" { + t.Fatalf("the runner's library verdict is said for %+v; want plex and radarr", copies) + } + own := inventory.ResourceHealth{Module: "radarr", Resource: "runner:radarr", Kind: link.KindRunner, Target: "jochen", + State: link.StateUnhealthy, Reason: "the tool runner is not in mesh-radarr yet", Since: time.Now()} + now := time.Now() + for _, r := range []inventory.ResourceHealth{copies[1], own} { + f := gateFacts{now: now, waits: libraryFactsWith(), groupsAdded: map[string]bool{"radarr": false}, + accessesChanged: map[string]bool{"radarr": false}, + health: map[string]inventory.NodeHealth{"ace": {Node: "ace", HeardAt: now, Resources: []inventory.ResourceHealth{r}}}} + if h, why := moduleHealthWord("radarr", "ace", now.Add(-time.Minute), f); h != healthGood { + t.Errorf("%s, a move that changes no library: %v %q; want not counted", r.Resource, h, why) + } + f.accessesChanged = map[string]bool{"radarr": true} + if h, _ := moduleHealthWord("radarr", "ace", now.Add(-time.Minute), f); h != healthNotYet { + t.Errorf("%s, a move that changes a library: %v; want held to the bound", r.Resource, h) + } + } +} diff --git a/cmd/mesh-controller/module_health.go b/cmd/mesh-controller/module_health.go index de1946fc..63bef360 100644 --- a/cmd/mesh-controller/module_health.go +++ b/cmd/mesh-controller/module_health.go @@ -42,6 +42,9 @@ const ( moduleUnhealthyAfter = 2 ) +// runnerLibrary prefixes the engine's verdict on the tool runner's membership of a library's group (ADR 0334). +const runnerLibrary = "runner:library:" + // libraryVerdictsFor is, for each library verdict in a statement (kind library, no module), one copy for each module // on the machine — assigned, or pulled in by a requirement — whose manifest uses that library: what the module is judged and said by. None when the // assignments or the catalogue cannot be read: the library's own verdict stands alone, and no module is excused by it. @@ -49,7 +52,8 @@ func libraryVerdictsFor(ctx context.Context, inv *inventory.Inventory, node stri rs []inventory.ResourceHealth) []inventory.ResourceHealth { var libraries []inventory.ResourceHealth for _, r := range rs { - if r.Module == "" && r.Kind == link.KindLibrary && strings.HasPrefix(r.Resource, libraryAccess) { + if r.Module == "" && ((r.Kind == link.KindLibrary && strings.HasPrefix(r.Resource, libraryAccess)) || + (r.Kind == link.KindRunner && strings.HasPrefix(r.Resource, runnerLibrary))) { libraries = append(libraries, r) } } @@ -80,7 +84,7 @@ func libraryVerdictsFor(ctx context.Context, inv *inventory.Inventory, node stri } var out []inventory.ResourceHealth for _, lib := range libraries { - name := strings.TrimPrefix(lib.Resource, libraryAccess) + name := strings.TrimPrefix(strings.TrimPrefix(lib.Resource, runnerLibrary), libraryAccess) for _, module := range assigned { for _, u := range shelf[module].Libraries { if u.Library == name { @@ -790,6 +794,11 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea (r.State == link.StateUnknown || (r.Kind == link.KindLibrary && r.State == link.StateStarting)) { continue } + // The tool runner's membership (ADR 0334 §2), of a move that changes no library of the module, is not this + // move's: it is said as the module's condition, and holds a move that changes them to the gate's bound. + if r.Kind == link.KindRunner && !changes { + continue + } switch r.State { case link.StateHealthy: case link.StateStarting: diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index a487aadd..8b2020b6 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -134,8 +134,9 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso if _, err := keepRecorded(ctx, open, nodeName, shelf); err != nil { return catalogue.Resolution{}, nil, err } - // And a module that joins a library not yet taken, at the build it ran before (novox/hq ADR 0328 §8). - if _, err := holdJoining(ctx, inv, nodeName, shelf); err != nil { + // And a module that may not join yet, at the build it ran before (novox/hq ADR 0328 §8, ADR 0334). + held, err := holdJoining(ctx, inv, nodeName, shelf) + if err != nil { return catalogue.Resolution{}, nil, err } @@ -223,6 +224,10 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso } settings[m.Module] = layers } + // A held module with the settings it last ran with (ADR 0328 §8). + if err := heldSettings(ctx, inv, nodeName, held, settings); err != nil { + return catalogue.Resolution{}, nil, err + } return resolved, settings, nil } @@ -490,8 +495,8 @@ func declarationWith(ctx context.Context, open *stores, node string, if kept == nil && len(held) > 0 { kept = map[string]string{} } - for m, was := range held { - kept[m] = was + for m, h := range held { + kept[m] = h.commit } for m, was := range kept { if _, carried := out.Builds[m]; carried { @@ -1035,15 +1040,16 @@ func engineJudgesRoot(ctx context.Context, inv *inventory.Inventory, node string return had && stated.Contract >= link.RootContract, nil } -// engineKeepsLibraries says whether a machine's node-engine makes the mesh's users and library groups and keeps -// libraries (novox/hq ADR 0328), by its own newest statement, for the same reason as engineReadsHealth: an older engine -// parses strictly, and is sent none of it. +// engineKeepsLibraries says whether a machine's node-engine makes the mesh's users and library groups, keeps +// libraries, and keeps a joining module's own groups (novox/hq ADR 0328, ADR 0334: its statement's contract is +// link.OwnGroupContract or later), by its own newest statement. On an older one no module joins: the libraries +// alone are not enough, since its tools would lose their read of a joining module's directories. func engineKeepsLibraries(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) { stated, had, err := inv.HealthOf(ctx, node) if err != nil { return false, err } - return had && stated.Contract >= link.LibraryContract, nil + return had && stated.Contract >= link.OwnGroupContract, nil } // libraryRendering is what a machine's composition needs of ADR 0328. @@ -1083,11 +1089,7 @@ func libraryFacts(ctx context.Context, inv *inventory.Inventory, node string, pl if lf.keeps, err = engineKeepsLibraries(ctx, inv, node); err != nil || !lf.keeps { return lf, err } - if stated, had, err := inv.HealthOf(ctx, node); err != nil { - return lf, err - } else { - lf.ownGroups = had && stated.Contract >= link.OwnGroupContract - } + lf.ownGroups = true // a module joins only on an engine that also keeps its own groups (ADR 0334) lf.identities = map[string]catalogue.ModuleIdentity{} for _, m := range plan.Modules { if !catalogue.Joins(m) { diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index a97cf4a7..749962e2 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -253,9 +253,24 @@ func withoutKept(ctx context.Context, open *stores, node string, moves []invento keeping = keepingRecorded(ctx) } keeps, err := recordedKept(keeping, open, node) - if err != nil || len(keeps) == 0 { + if err != nil { return moves, err } + // And a joining module held at the build it runs (novox/hq ADR 0328 §8): its new build is no move this send makes, + // and judging it would read the gate as superseded by its own send (the re-review of 96e15f86, item 1). + held, err := holdJoining(ctx, open.inventory, node, nil) + if err != nil { + return moves, err + } + if len(held) > 0 && keeps == nil { + keeps = map[string]string{} + } + for m, h := range held { + keeps[m] = h.commit + } + if len(keeps) == 0 { + return moves, nil + } var out []inventory.CarriedMove for _, mv := range moves { if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) { diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 452ae757..9c41f3bd 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -690,7 +690,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, // being behind. Now every module of the tier whose first machine is the same is sent there in one // send, judged there by one gate — kept on the first of them, the others gated by it — and the rest // of the tier's machines are sent once, together, when the gates have passed. - var pending []string + var pending, waitsForAPerson []string firstOn := map[string][]string{} runningOf := map[string][]string{} var rest []string @@ -706,6 +706,20 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, return false, err } runningOf[m] = running + // **A joining build held for a person is not sent** (novox/hq ADR 0328 §8, the re-review of 96e15f86): its + // machines keep the build they run until its library is taken (or their engine keeps own groups), so sending + // it would send nothing new and judge the old build. It waits for a person, said; the plan stays open, the other + // modules are sent and judged, and it is sent on the walk after nothing holds it — no new merge. + if state.FirstAt == nil { + why, err := heldIn(ctx, inv, m, running) + if err != nil { + return false, err + } + if why != "" { + waitsForAPerson = append(waitsForAPerson, why) + continue + } + } // **A build whose source is unchanged is never a move** (novox/hq issue 280): made from the source // of the build every machine running it was sent, it is registered with that build's artifacts — // nothing to send, and nothing for a gate to judge. @@ -868,9 +882,17 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, strings.Join(sent, ", ")) return true, nil } - if len(pending) > 0 { - note := "tier " + fmt.Sprint(p.Tier) + " built; waiting for " + strings.Join(pending, "; ") + - " to report it applied before the rest are sent" + if len(pending) > 0 || len(waitsForAPerson) > 0 { + note := "tier " + fmt.Sprint(p.Tier) + " built; " + if len(pending) > 0 { + note += "waiting for " + strings.Join(pending, "; ") + " to report it applied before the rest are sent" + } + if len(waitsForAPerson) > 0 { + if len(pending) > 0 { + note += "; and " + } + note += "waiting for a person: " + strings.Join(waitsForAPerson, "; ") + } changed := p.State != inventory.PlanRolling || p.Note != note p.State = inventory.PlanRolling p.Note = note diff --git a/go.mod b/go.mod index 46283c1e..496378e5 100644 --- a/go.mod +++ b/go.mod @@ -36,4 +36,4 @@ require ( // committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and // fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without // `go mod vendor` fails loudly, at once, everywhere. -replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261012202816-d8e785bb4b4b +replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261012222214-7a41e120635b diff --git a/go.sum b/go.sum index 7e629522..0184ce90 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-host v0.0.0-20261012202816-d8e785bb4b4b h1:NvXKvFi4lV/oXg29dQ9iUX5oQm+NtPN9ACTa9Ev1i8E= -git.novox.be/novox/mesh-host v0.0.0-20261012202816-d8e785bb4b4b/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c= +git.novox.be/novox/mesh-host v0.0.0-20261012222214-7a41e120635b h1:PGdZJHUU9rCDucdrZIqSs4DGyxlo4lokvwmofpGk9RI= +git.novox.be/novox/mesh-host v0.0.0-20261012222214-7a41e120635b/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= diff --git a/internal/broker/genesis_template_test.go b/internal/broker/genesis_template_test.go index 91b13d31..b598cbae 100644 --- a/internal/broker/genesis_template_test.go +++ b/internal/broker/genesis_template_test.go @@ -82,7 +82,7 @@ func theCarriedAccounts(t *testing.T) string { // the commit the mesh runs, or as captured in testdata/beside (internal/beside, novox/hq issue 432). func theTemplate(t *testing.T) []map[string]any { t.Helper() - path := filepath.Join(beside.Dir(t, "mesh-host"), "examples", "foundation-first-node-nats.lock") + path := filepath.Join(beside.Dir(t, "mesh-host"), "examples", "foundation-first-node.lock") raw, err := os.ReadFile(path) if err != nil { t.Fatal(err) diff --git a/internal/catalogue/libraries_test.go b/internal/catalogue/libraries_test.go index 0b0c9ef4..4ad6faff 100644 --- a/internal/catalogue/libraries_test.go +++ b/internal/catalogue/libraries_test.go @@ -335,6 +335,7 @@ func TestTheEngineTakesWhatIsComposedForLibraries(t *testing.T) { for _, node := range []string{"ace", "novox"} { r := Resolution{Node: node, Account: "jochen", Modules: []Manifest{aRadarr(t), theRuntime(t)}} with := librariesRendering() + with.RunnerJoinsOwnGroups = true // contract 7 (mesh-host #97): the runner on identities, setgid on own directories with.Libraries["books"] = Library{Name: "books", Home: "ace", Root: "/storage/books", Reach: []string{"novox"}} composed := composeLibraries(t, r, with) body, err := json.Marshal(map[string]any{"declaration": validate.Version, "resources": composed.Resources, diff --git a/internal/inventory/libraries.go b/internal/inventory/libraries.go index 522dc9ab..a81f5398 100644 --- a/internal/inventory/libraries.go +++ b/internal/inventory/libraries.go @@ -7,6 +7,7 @@ import ( "fmt" "path/filepath" "strings" + "time" "github.com/jackc/pgx/v5" @@ -290,3 +291,41 @@ func (i *Inventory) TakeFailed(ctx context.Context, library, warrant string) err `delete from kept_library where library = $1 and warrant = $2 and not done`, library, warrant) return err } + +// SettingsAsOf is a module's settings layers on a machine as they stood at a moment: each layer — the whole mesh's, +// then the machine's — as it was set then, read from the layer standing now or from its history. What a module held at +// the build it ran is composed with: the settings it last ran with (novox/hq ADR 0328 §8). +func (i *Inventory) SettingsAsOf(ctx context.Context, nodeName, module string, at time.Time) ([]catalogue.Layer, error) { + node, err := i.NodeByName(ctx, nodeName) + if err != nil { + return nil, err + } + var layers []catalogue.Layer + for _, scope := range []struct { + id *string + from string + }{{nil, catalogue.MeshWideLayer}, {&node.ID, nodeName}} { + var raw []byte + err := i.store.Pool().QueryRow(ctx, + `select values from settings + where module = $1 and node is not distinct from $2::uuid and set_at <= $3 + union all + select values from (select values from settings_history + where module = $1 and node is not distinct from $2::uuid + and (set_at is null or set_at <= $3) and replaced_at > $3 + order by replaced_at asc limit 1) h + limit 1`, module, scope.id, at).Scan(&raw) + if errors.Is(err, pgx.ErrNoRows) { + continue + } + if err != nil { + return nil, err + } + values := map[string]any{} + if err := json.Unmarshal(raw, &values); err != nil { + return nil, err + } + layers = append(layers, catalogue.Layer{From: scope.from, Values: values}) + } + return layers, nil +} diff --git a/internal/link/protocol.go b/internal/link/protocol.go index 923c8280..8ce62a35 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -457,6 +457,10 @@ const LibraryContract = 6 // Identity and put that account in the group mesh-, and accept a directory mode whose first digit is 2. const OwnGroupContract = 7 +// KindRunner is the tool runner's membership as the engine states it (novox/hq ADR 0334 §2; mesh-host internal/link): +// `runner:` of a joining module, in its own group; `runner:library:` of no module, in a library's group. +const KindRunner = "runner" + // KindLibrary is a library's own verdict on its home, as the engine may state it (novox/hq ADR 0328): healthy once // kept. Read beside an access verdict naming the library until the engine's shape is settled. const KindLibrary = "library" diff --git a/testdata/beside/CAPTURED b/testdata/beside/CAPTURED index 996e8683..0750cad9 100644 --- a/testdata/beside/CAPTURED +++ b/testdata/beside/CAPTURED @@ -8,7 +8,7 @@ private addresses. That is acceptable while mesh-controller is private; if it ev a second place to clean besides the catalogue itself. mesh-catalog b9de001833b1b61b297182b8e3bcdae1cbdeace9 modules/*/module.json, modules/nats/Dockerfile -mesh-host d8e785bb4b4bbd4ad99dccec60c588032cd78fa6 examples/foundation-first-node-nats.lock +mesh-host 7a41e120635b3f7b6cb94cca8b3fc4093e4e6110 examples/foundation-first-node.lock mesh-sdk 33917f91ac3cc213b879b32f3c70e2aa70395f98 conformance/events/module-event.json To move them, from this repository's root, with the two repositories checked out beside it: @@ -18,7 +18,7 @@ To move them, from this repository's root, with the two repositories checked out git -C ../mesh-catalog archive modules | tar -x -C testdata/beside/mesh-catalog --wildcards \ 'modules/*/module.json' 'modules/nats/Dockerfile' find testdata/beside -name module.json -exec mv {} {}.captured \; - git -C ../mesh-host archive examples/foundation-first-node-nats.lock | tar -x -C testdata/beside/mesh-host + git -C ../mesh-host archive examples/foundation-first-node.lock | tar -x -C testdata/beside/mesh-host and write the commits here. The SDK is captured at the commit go.mod pins for git.novox.be/novox/mesh-sdk/go (the last part of its pseudo-version; novox/hq issue 449), which internal/link's conformance test holds it to: diff --git a/testdata/beside/mesh-host/examples/foundation-first-node-nats.lock b/testdata/beside/mesh-host/examples/foundation-first-node.lock similarity index 87% rename from testdata/beside/mesh-host/examples/foundation-first-node-nats.lock rename to testdata/beside/mesh-host/examples/foundation-first-node.lock index 25d84627..289e3e2a 100644 --- a/testdata/beside/mesh-host/examples/foundation-first-node-nats.lock +++ b/testdata/beside/mesh-host/examples/foundation-first-node.lock @@ -1,11 +1,17 @@ -// foundation-first-node-nats.lock — what a machine must be before a mesh exists, on the bus being -// built (novox/hq ADR 0106, design 25). +// foundation-first-node.lock — what a machine must be before a mesh exists: the one genesis bundle +// (novox/hq ADR 0331 §2), on the bus the mesh runs on (ADR 0106, ADR 0131, design 25). // -// The same twelve steps as foundation-first-node.lock, with one difference that matters: **the mesh -// writes its own user list, and at genesis there is no mesh yet to write it.** So this carries the -// first one — the controller's own account, at a well-known bootstrap password, exactly as the store -// is reached at `postgres:bootstrap` and the old bus at `guest:guest`. It is rotated with those, and -// from the controller's first composition onward the file is the controller's to write. +// The whole bootstrap (novox/hq 03-DESIGN/01-to-be/07-the-foundation.md): a container runtime, a +// filter, a store, a database per context, those contexts' schemas, the bus with its certificate +// and its first user list, and the control plane running on top of them. +// +// **The mesh writes its own user list, and at genesis there is no mesh yet to write it.** So this +// carries the first one — the controller's own account — and the control plane's address names the +// same account. Both say `bootstrap`, and both are placeholders: the installer makes the account's +// password, puts it in the address and its hash in the list (mesh-host internal/bootstrap +// rootsecrets.go), and refuses a bundle whose bus connection it does not recognise. Until the first +// node has enrolled the installer places the list the controller composes; from the bus's adoption +// as the `nats` module on, the module receives it like any module (ADR 0331 §1, §5). // // The accounts file is its own file beside the server's configuration, because the server's own // settings belong to whoever raises it and the users belong to the mesh (design 25 §4). Both live in @@ -15,7 +21,18 @@ // // No `verify` on the TLS block, deliberately — that setting makes the server demand a *client* // certificate, and nothing in the mesh presents one: a host pins this server's exact certificate and -// authenticates with a password (ADR 0004, design 25 §4). +// authenticates with a password (ADR 0004, design 25 §4). The control plane pins it too, through +// MESH_BROKER_CERTIFICATE, and dials the bus on the machine's own loopback. +// +// PINNED BY DIGEST, and the digest is not decoration: a tag can be made to point at a different +// image, and this file is applied on a machine with no mesh to ask about anything (ADR 0006). The +// registry prefix is the lab's; the digests are upstream's. +// +// The store's data is a NAMED VOLUME, not a directory on the machine. A directory the host +// creates is owned by root, and the database runs as somebody else inside the container — so it +// could not write, and the container crash-looped. A named volume lets the image set up its own +// ownership, and outlives the container, which is what you want for the thing holding the mesh's +// state. { "declaration": 1, diff --git a/testdata/store-tests b/testdata/store-tests index 0757a123..0212aa60 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -47,3 +47,7 @@ ./cmd/mesh-controller TestALibraryIsTakenOnlyWhenTheEngineSaysTheActIsDone ./internal/inventory TestTheSubPathsOfModulesPulledInByARequirementAreKept ./internal/inventory TestALibraryIsTakenWhenTheActIsDone +./cmd/mesh-controller TestADeliveryWaitsForALibraryAndJudgesTheRest +./cmd/mesh-controller TestAJoiningModuleIsHeldOnAnEngineOlderThanOwnGroups +./cmd/mesh-controller TestAHeldModuleKeepsTheSettingsItRanWith +./cmd/mesh-controller TestARunnersVerdictIsSaidAndJudgedByWhetherTheMoveChangesIt diff --git a/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go b/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go index 16ffdc84..4e54fc79 100644 --- a/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go +++ b/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go @@ -133,7 +133,13 @@ func (d *Directory) validate(where string, _ bool) []string { if d.Path == "" { problems = append(problems, where+": a directory needs a path") } - return append(problems, checkMode(where, d.Mode)...) + // A directory's mode may carry the setgid bit (2xxx), so what is made beneath it keeps its group (novox/hq ADR + // 0334 §1); never setuid or sticky, which mean nothing the mesh asks of a directory it owns. + mode := d.Mode + if len(mode) == 4 && mode[0] == '2' { + mode = "0" + mode[1:] + } + return append(problems, checkMode(where, mode)...) } // File is a file with literal content. The host renders nothing. @@ -1564,6 +1570,9 @@ type Identity struct { UID int `json:"uid"` GID int `json:"gid"` Groups []int `json:"groups,omitempty"` + // Runner is the tool runner's account, put in the module's own group (novox/hq ADR 0334 §2) when its tools run + // on this machine. Absent for a module that declares no tools here, and from a controller older than ADR 0334. + Runner string `json:"runner,omitempty"` } // Group is one library's group, as the controller allocated it (ADR 0328 §3). @@ -1707,6 +1716,9 @@ func checkIdentities(ids []Identity, groups []Group) []string { "declaration makes", where, g)) } } + if id.Runner != "" && !accountName.MatchString(id.Runner) { + problems = append(problems, fmt.Sprintf("%s: its runner %q is not an account's name", where, id.Runner)) + } names[id.Name], uids[id.UID] = true, id.Name } return problems diff --git a/vendor/modules.txt b/vendor/modules.txt index 57535ea1..dcb56b10 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -78,7 +78,7 @@ github.com/nats-io/nkeys # github.com/nats-io/nuid v1.0.1 ## explicit github.com/nats-io/nuid -# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261012202816-d8e785bb4b4b +# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261012222214-7a41e120635b ## explicit; go 1.26.0 github.com/novox/mesh-host/internal/declaration github.com/novox/mesh-host/rootsearch @@ -138,4 +138,4 @@ golang.org/x/text/width # golang.org/x/time v0.15.0 ## explicit; go 1.25.0 golang.org/x/time/rate -# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261012202816-d8e785bb4b4b +# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261012222214-7a41e120635b