diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index 7bf565be..be469d09 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -332,6 +332,12 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo, Commit: m.Head, Trunk: m.Trunk}) } + // A branch tried before its pull request reads its outcome from this log, whole (novox/hq issue 686). + if request.Check.Try { + if line, err := json.Marshal(triedOf(v)); err == nil { + say(link.StepTried, string(line)) + } + } } } else if err == nil { // The package-registry credential is a build input, so it is resolved before the clone: a @@ -457,7 +463,8 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec { spec := builder.CheckSpec{ID: request.ID, Repository: request.Repository, Ref: request.Ref, Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{}, Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base, - Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held)} + Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held), + Tests: c.Tests, StoreOnly: c.StoreOnly} for dir, b := range c.Beside { spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref} } @@ -472,6 +479,23 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec { return spec } +// triedOf is a try's outcome as its log line says it (novox/hq issue 686): the repository's own check's verdict, +// or the check's where that did not run — a head that does not merge — and the lines naming what failed. +func triedOf(v builder.CheckVerdict) link.Tried { + t := link.Tried{Verdict: v.Verdict, Summary: v.Summary, Took: v.Took.Round(time.Second).String(), + Head: v.Head, Trunk: v.Trunk} + if v.Repo != nil && (v.Gate == nil || v.Gate.Verdict == "pass") { + t.Verdict, t.Summary = v.Repo.Verdict, v.Repo.Summary + } + if v.Repo != nil && v.Repo.Failed != "" { + t.Failed = strings.Split(v.Repo.Failed, "\n") + } + if t.Verdict == "" { + t.Verdict = "error" + } + return t +} + // checkBuiltOf is a check's builds for its merge as the outcome carries them (novox/hq ADR 0350 §1). func checkBuiltOf(built []builder.CheckBuilt) []link.CheckBuilt { var out []link.CheckBuilt diff --git a/cmd/mesh-builder/tried_test.go b/cmd/mesh-builder/tried_test.go new file mode 100644 index 00000000..06aaf706 --- /dev/null +++ b/cmd/mesh-builder/tried_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "testing" + "time" + + "github.com/novox/mesh-controller/internal/builder" +) + +// A try's outcome, as the build seat says it in the build's log (novox/hq issue 686): the repository's own +// check's verdict, how long it took, and each line naming what failed. +func TestATrysOutcomeNamesWhatFailed(t *testing.T) { + tried := triedOf(builder.CheckVerdict{Verdict: "pass", Summary: "the change touches no module", Took: 303 * time.Second, + Head: "c0ffee", Trunk: "ca3f43c", Gate: &builder.Layer{Verdict: "pass"}, + Repo: &builder.Layer{Verdict: "fail", Summary: "its merge-check.sh failed: TestX", + Failed: "--- FAIL: TestX (0.01s)\n x_test.go:12: got 1, want 2"}}) + if tried.Verdict != "fail" || tried.Took != "5m3s" || len(tried.Failed) != 2 || tried.Trunk != "ca3f43c" || + tried.Summary != "its merge-check.sh failed: TestX" { + t.Fatalf("a failed try: %+v", tried) + } + // A head that does not merge fails both layers: the check's own word. + conflict := triedOf(builder.CheckVerdict{Verdict: "fail", Summary: "does not merge", Gate: &builder.Layer{Verdict: "fail"}, + Repo: &builder.Layer{Verdict: "fail", Summary: "does not merge"}}) + if conflict.Verdict != "fail" || conflict.Summary != "does not merge" { + t.Fatalf("a try that does not merge: %+v", conflict) + } + // No verdict is an error, never a pass. + if none := triedOf(builder.CheckVerdict{}); none.Verdict != "error" { + t.Fatalf("a try with no verdict: %+v", none) + } +} diff --git a/cmd/mesh-controller/adoption.go b/cmd/mesh-controller/adoption.go index 411ae05d..afa90a07 100644 --- a/cmd/mesh-controller/adoption.go +++ b/cmd/mesh-controller/adoption.go @@ -424,8 +424,9 @@ func comparisonOf(module string, c comparison, opts takeOptions) (preview string } // And its settings on this machine, composed against its definition (rule 1, rule 6). for _, layer := range c.layers { - keys := make([]string, 0, len(layer.Values)) - for k := range layer.Values { + said := layer.Whole() + keys := make([]string, 0, len(said)) + for k := range said { keys = append(keys, k) } sort.Strings(keys) diff --git a/cmd/mesh-controller/check.go b/cmd/mesh-controller/check.go index 16e16ba6..64b04429 100644 --- a/cmd/mesh-controller/check.go +++ b/cmd/mesh-controller/check.go @@ -55,6 +55,11 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error { continue } m, err := catalogue.ParseManifest(raw) + if err == nil { + // Each template's text read from the tree beside the manifest, and judged as inline content is + // (novox/hq ADR 0349 §2): a missing template is refused by name. + m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(path)) + } if err != nil { fmt.Fprintf(out, "%s: %v\n", path, err) failed++ diff --git a/cmd/mesh-controller/facts.go b/cmd/mesh-controller/facts.go index 50b891cf..af04f202 100644 --- a/cmd/mesh-controller/facts.go +++ b/cmd/mesh-controller/facts.go @@ -348,11 +348,11 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot if layer.From == catalogue.MeshWideLayer { if !meshWide[module] { meshWide[module] = true - f.Settings = append(f.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)}) + f.Settings = append(f.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Whole())}) } continue } - m.Settings = append(m.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)}) + m.Settings = append(m.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Whole())}) } } m.Declaration = declarationFacts(ctx, open, n.Name, gens, gensErr, scrub) diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index 304186a2..8a1e1a1c 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -1443,6 +1443,12 @@ func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error } // A release held after a failed one, while builds still wait for a gate (ADR 0236). out = append(out, backlogObservation()...) + // And a rest that went silent (novox/hq ADR 0353 §4). + silent, err := silentObservation(ctx, d.open.inventory) + if err != nil { + return nil, err + } + out = append(out, silent...) return sortedFound(dedupeObservations(out)), nil } @@ -1532,6 +1538,12 @@ func gateLine(g *inventory.PlanGate) string { case inventory.GateFailed: what = "FAILED" } + switch { + case g.Verdict == "" && g.Pending: + what = "kept, not yet sent" // novox/hq ADR 0353 §8 + case g.Verdict == "" && g.Silent: + what = "silent on its send, left" // ADR 0353 §4 + } if g.Verdict == "" && g.Waiting() { what = "waits for a busy container on " + g.Busy.Machine } diff --git a/cmd/mesh-controller/held_back.go b/cmd/mesh-controller/held_back.go index 0d90cc19..a0b4c6ba 100644 --- a/cmd/mesh-controller/held_back.go +++ b/cmd/mesh-controller/held_back.go @@ -151,7 +151,7 @@ func heldMachines(ctx context.Context, open *stores, names []string) (map[string for _, mv := range f.moves(node, modules, same, known, false) { if planStillToSend(plans, mv.Module, node) == "" { why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+ - "plan sends it, one machine at a time, judged", mv.Module, buildName(mv.From), buildName(mv.To))) + "plan sends it, one machine first and then the rest at once, judged", mv.Module, buildName(mv.From), buildName(mv.To))) } } if len(why) > 0 { diff --git a/cmd/mesh-controller/issue648_test.go b/cmd/mesh-controller/issue648_test.go new file mode 100644 index 00000000..45261cc0 --- /dev/null +++ b/cmd/mesh-controller/issue648_test.go @@ -0,0 +1,87 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// A tier whose module is no move — made from the source every machine running it runs, so nothing is +// sent — is not waited on for a report after its marking (novox/hq issue 648). Its machines answered the +// send that last carried that build before the walk got there, and nothing will make them report again: +// the walk asks the next tier in the same step, rather than "waiting for app on anchor, laptop" for ever. +func TestANoMoveTierWaitsForNoNewReport(t *testing.T) { + judgeLeavesAsTheCore(t) + g := aGateMesh(t) + ctx := t.Context() + inv := g.open.inventory + + // app is the build machine, so a module of the next tier is built by it: the tier gates the next on app + // running on its machines. + for _, m := range []catalogue.Manifest{ + {Module: "app", Version: "c2", Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}}, + {Module: "web", Version: "w1"}, + } { + if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", + Path: "modules/" + m.Module, BuiltFrom: m.Version, Head: m.Version, Asked: time.Now().Add(-time.Minute)}); err != nil { + t.Fatal(err) + } + } + // What anchor and laptop were last sent (c1) was made from the source the plan's build of c2 is. + for _, b := range []inventory.Build{ + {ID: "build-1s", Module: "app", Commit: "c1", SourceFingerprint: "src1:app", + Asked: time.Now().Add(-30 * time.Second), At: time.Now().Add(-30 * time.Second)}, + {ID: "build-2s", Module: "app", Commit: "c2", SourceFingerprint: "src1:app", Asked: time.Now(), At: time.Now()}, + } { + b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + } + // Each machine answered that send, applied, before the walk's tier was built. + for _, n := range []string{"anchor", "laptop"} { + if _, err := inv.RecordDoing(ctx, nodeID(t, g.open, n), inventory.Doing{Node: n, + Outcome: inventory.OutcomeApplied, Declared: "d-" + n + "-c1", Applied: 1, + At: time.Now().Add(-10 * time.Second)}); err != nil { + t.Fatal(err) + } + } + built := time.Now().UTC() + p := g.plan(t) + p.Tiers = [][]string{{"app"}, {"web"}} + p.Modules["app"].Build, p.Modules["app"].BuiltAt = "build-2s", &built + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + + advancePlans(ctx, g.open) + + if len(g.sent) != 0 { + t.Fatalf("a no-move build was sent: %v", g.sent) + } + p = g.plan(t) + if s := p.Modules["app"]; s == nil || !strings.HasPrefix(s.Why, "no move") { + t.Fatalf("the build was not marked no move: %+v", s) + } + if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") { + t.Fatalf("the walk waits on a no-move tier for a report nothing will cause: tier %d, %s %q", p.Tier, p.State, p.Note) + } + + // A machine that has not answered its last send is still waited on: no move is not "never sent". + p.Tier, p.State, p.Note = 0, inventory.PlanRolling, "" + if err := inv.RecordSent(ctx, nodeID(t, g.open, "laptop"), "d-laptop-later", map[string]string{"app": "c1"}); err != nil { + t.Fatal(err) + } + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + advancePlans(ctx, g.open) + p = g.plan(t) + if p.Tier != 0 || !strings.Contains(p.Note, "waiting for app on laptop") { + t.Fatalf("a machine that has not answered its last send was not waited on: tier %d, %q", p.Tier, p.Note) + } +} diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 1e656622..41d7319d 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -106,6 +106,8 @@ func run() error { return killCommand(ctx, args[1:]) case "forget-checks": return forgetChecksCommand(ctx, args[1:]) + case "try": + return tryCommand(ctx, args[1:]) case "pause", "resume": return pauseCommand(ctx, args[0], args[1:]) case "collection": @@ -381,6 +383,13 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error { // **A merge check builds nothing** (novox/hq to-be 45 §9): its verdict is said, and nothing of it is // recorded or registered. if result.Check != nil || result.Checked != nil { + if isTry(result) { + // A branch tried before its pull request (novox/hq issue 686): its asker reads the outcome from the + // build's log. Never said as `checked`, so the forge's holder sets no status. + fmt.Printf("%s: a try of %s at %.8s, read with `try --id`: said to nobody, nothing taken in\n", result.ID, + result.Repository, result.Ref) + return nil + } checked(ctx, result) return nil } diff --git a/cmd/mesh-controller/merge_gate.go b/cmd/mesh-controller/merge_gate.go index ddcfd609..78b5332a 100644 --- a/cmd/mesh-controller/merge_gate.go +++ b/cmd/mesh-controller/merge_gate.go @@ -576,6 +576,11 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve return err } m, err := catalogue.ParseManifest(raw) + if err == nil { + // Each template's text from the tree beside the manifest, as the build reads it back from the archive it + // packs (novox/hq ADR 0349 §2): so a change to a template alone is a change to its module. + m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(path)) + } if err != nil { if inScope(dir) { failures = append(failures, fmt.Sprintf("%s: the controller judging this (%s) refuses it — %s", diff --git a/cmd/mesh-controller/mesh_for_test.go b/cmd/mesh-controller/mesh_for_test.go index b2e76be4..eea81d13 100644 --- a/cmd/mesh-controller/mesh_for_test.go +++ b/cmd/mesh-controller/mesh_for_test.go @@ -33,7 +33,21 @@ import ( func aMesh(t *testing.T) *stores { t.Helper() inventory.ForTest(t) // raises the store, migrates it, and points the environment at it - licences.ForTest(t) // planning reaches this one too, by name and never by connection + return aMeshOnTheStore(t) +} + +// aMeshBefore is aMesh on a store migrated only below migration `number`, with the function that runs the +// rest: for a test of what a migration that moves data does to what the mesh composes (novox/hq issue 651). +func aMeshBefore(t *testing.T, number int) (*stores, func()) { + t.Helper() + _, migrate := inventory.ForTestBefore(t, number) + return aMeshOnTheStore(t), migrate +} + +// aMeshOnTheStore is the mesh aMesh makes, on the store the environment points at. +func aMeshOnTheStore(t *testing.T) *stores { + t.Helper() + licences.ForTest(t) // planning reaches this one too, by name and never by connection open, err := openStores(t.Context()) if err != nil { diff --git a/cmd/mesh-controller/modules.go b/cmd/mesh-controller/modules.go index 66804655..7517eeca 100644 --- a/cmd/mesh-controller/modules.go +++ b/cmd/mesh-controller/modules.go @@ -8,6 +8,7 @@ import ( "fmt" "net" "os" + "path/filepath" "sort" "strings" "time" @@ -127,6 +128,10 @@ func moduleCommand(ctx context.Context, args []string) error { if err != nil { return err } + // A manifest handed over by hand takes each template's text from the tree beside it (novox/hq ADR 0349 §2). + if m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(positionals[0])); err != nil { + return err + } from, err := whereItComesFrom(*repo, *ref, *commit, *path, *self) if err != nil { return err @@ -537,6 +542,10 @@ func settingsCommand(ctx context.Context, args []string) error { if p.SetBy != "" { by = "; " + p.SetBy } + if p.Why != "" { + // A layer whose placement moved out of it (migration 0097): still the layer, as it stands. + by += "; " + p.Why + } fmt.Printf("%s on %s, until %s (%s%s):\n %s\n", positionals[0], where, p.ReplacedAt.Local().Format("2006-01-02 15:04:05"), p.ReplacedBy, by, shown) } diff --git a/cmd/mesh-controller/placement_moved_test.go b/cmd/mesh-controller/placement_moved_test.go new file mode 100644 index 00000000..5831cc56 --- /dev/null +++ b/cmd/mesh-controller/placement_moved_test.go @@ -0,0 +1,113 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// **Placement leaves the settings layers and no machine is told anything new** (novox/hq ADR 0348 clause +// 11, issue 651). A mesh is seeded as the store holds it before migration 0097 — `places`, `accesses` and a +// kept network among the settings, at the mesh's level and a machine's, one layer of placement only — then +// migrated, and every machine's declaration is composed. What each machine is told of the seeded modules is +// byte for byte what the controller before this change composed from the same seed: the golden files were +// written by the integration branch's controller at bfc79c6 running this same test (MESH_WRITE_GOLDEN=1), +// where there was no migration 0097 to run. +func TestADeclarationIsTheSameAfterPlacementMoves(t *testing.T) { + open, migrate := aMeshBefore(t, 97) + ctx := t.Context() + inv := open.inventory + image := "registry.invalid:5000/library@sha256:" + strings.Repeat("a", 64) + register(t, open, catalogue.Manifest{Module: "library", Version: "1", + Accesses: []catalogue.Access{{ID: "series"}, {ID: "downloads", Mode: catalogue.AccessRead}}, + Resources: []map[string]any{ + {"id": "config", "type": "directory", "mode": "0750"}, + {"id": "server", "type": "container", "image": image, + "volumes": []any{"${dir:config}:/config", "${access:series}:/series", "${access:downloads}:/downloads:ro"}}, + // Today's form: a mergeable file naming no keys takes every key of the layer (ADR 0326 §3). + {"id": "runtime-config", "type": "file", "path": "${dir:config}/runtime.json", "mode": "0640", + "content": `{"log":"info"}`, "merge": "json"}, + }}) + register(t, open, catalogue.Manifest{Module: "found", Version: "1", + Resources: []map[string]any{{"id": "server", "type": "container", "image": image}}}) + if err := inv.SetAdopted(ctx, "anchor", true); err != nil { + t.Fatal(err) + } + for _, l := range []struct{ node, module, values string }{ + {"", "library", `{"accesses": {"downloads": "/storage/downloads"}, "log": "warn"}`}, + {"anchor", "library", `{"places": {"config": {"path": "/var/lib/library/config", "owner": "1001:2000"}}, + "accesses": {"series": "/storage/media/series"}, "workers": 4}`}, + {"laptop", "library", `{"accesses": {"series": "/home/media/series"}}`}, + {"anchor", "found", `{"networks": {"server": ["predecessor_default"]}}`}, + } { + if l.node == "" { + inv.ExecForTest(t, `insert into settings (node, module, values) values (null, $1, $2)`, l.module, l.values) + continue + } + inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`, + l.node, l.module, l.values) + } + + migrate() + // Assigned once the store is migrated: an assignment reads the layers, as this controller holds them. + for _, a := range [][]string{{"anchor", "library", "found"}, {"laptop", "library"}} { + if _, err := assign(ctx, open, a[0], a[1:]...); err != nil { + t.Fatalf("assign %v: %v", a, err) + } + } + + gens, err := generators(ctx, open) + if err != nil { + t.Fatal(err) + } + for _, node := range []string{"anchor", "laptop"} { + plan, settings, err := planFor(ctx, open, node) + if err != nil { + t.Fatal(err) + } + with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Allocating) + if err != nil { + t.Fatal(err) + } + composed, err := plan.Compose(with) + if err != nil { + t.Fatal(err) + } + if len(composed.LeftOut) > 0 { + t.Fatalf("%s leaves out %v", node, composed.LeftOut) + } + var told []map[string]any + for _, r := range composed.Resources { + if owner := composed.Owner[r["id"].(string)]; owner == "library" || owner == "found" { + told = append(told, r) + } + } + sort.Slice(told, func(i, j int) bool { return told[i]["id"].(string) < told[j]["id"].(string) }) + got, err := json.MarshalIndent(told, "", " ") + if err != nil { + t.Fatal(err) + } + golden := filepath.Join("testdata", "placement-moved", node+".json") + if os.Getenv("MESH_WRITE_GOLDEN") != "" { + if err := os.MkdirAll(filepath.Dir(golden), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(golden, append(got, '\n'), 0o644); err != nil { + t.Fatal(err) + } + continue + } + want, err := os.ReadFile(golden) + if err != nil { + t.Fatal(err) + } + if string(want) != string(got)+"\n" { + t.Errorf("%s is told something else after placement moved:\n%s\nwas:\n%s", node, got, want) + } + } +} diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index a5f705b1..7d60b16a 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -456,6 +456,9 @@ var plainWordings = map[string]func(conditions.Observation) words{ "release-held": worded(func(o conditions.Observation) words { return releaseHeldWords(nil, o.Also) }), + "release-silent": worded(func(o conditions.Observation) words { + return releaseSilentWords(o.Also) + }), "facts-stale": worded(func(o conditions.Observation) words { return words{Headline: "Merge checks use outdated facts", Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " + @@ -841,6 +844,19 @@ func releaseHeldWords(modules, machines []string) words { Resolved: "Resolved: the held updates are released"} } +// releaseSilentWords are the plain words of machines that said nothing on an update they were sent (ADR 0353 §4). +func releaseSilentWords(machines []string) words { + where := "Some machines" + if len(machines) > 0 { + where = conditions.Capital(namesWords(machines, 4)) + } + return words{Headline: "Machines did not answer an update", + Needs: "check that they are awake and on the mesh; the next walk sends them what waits.", + Explanation: where + " were sent module updates and said nothing about them within fifteen minutes, so " + + "they were left as they were. What they run is not known to be the update.", + Resolved: "Resolved: the next walk reached them"} +} + // reloginNeeds is what an account waiting for its groups needs (ADR 0252). func reloginNeeds(node string) string { return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node)