diff --git a/cmd/mesh-controller/ask_bus_look_integration_test.go b/cmd/mesh-controller/ask_bus_look_integration_test.go index 9f78de1e..83bbee9f 100644 --- a/cmd/mesh-controller/ask_bus_look_integration_test.go +++ b/cmd/mesh-controller/ask_bus_look_integration_test.go @@ -31,6 +31,7 @@ func TestALookComesBackForATierTheBusRefusedWhole(t *testing.T) { t.Fatal(err) } bus := false + asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it was := askABuild askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) { if !bus { @@ -95,6 +96,7 @@ func TestALookFailsTheTierTheBusRefusedPastTheBound(t *testing.T) { if err := inv.SavePlan(ctx, &p); err != nil { t.Fatal(err) } + asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it was := askABuild askABuild = func(context.Context, buildSource, string, string) (string, error) { return "", fmt.Errorf("%w: nats: no responders available for request", inventory.ErrNoChain) diff --git a/cmd/mesh-controller/ask_bus_moment_test.go b/cmd/mesh-controller/ask_bus_moment_test.go index 42ced920..70163618 100644 --- a/cmd/mesh-controller/ask_bus_moment_test.go +++ b/cmd/mesh-controller/ask_bus_moment_test.go @@ -15,6 +15,15 @@ import ( // seconds) is not a failed build. The module is left unasked and the plan building, and the next look asks it again; only a // module's own fault, or a bus silent for ten minutes, fails the plan. +// asksOfEveryModule makes every module a test names the builder's own, the one module a walk still asks build-agent +// for (ADR 0453 §5): what a test of that ask's waits reads. +func asksOfEveryModule(t *testing.T) { + t.Helper() + was := buildersOwn + buildersOwn = func(string) bool { return true } + t.Cleanup(func() { buildersOwn = was }) +} + func aTierOfThree() (*inventory.Plan, map[string]inventory.Entry) { p := &inventory.Plan{ID: "plan-1", State: inventory.PlanBuilding, Tiers: [][]string{{"a", "b", "c"}}, Modules: map[string]*inventory.PlanModule{}} @@ -26,6 +35,7 @@ func aTierOfThree() (*inventory.Plan, map[string]inventory.Entry) { } func TestABusThatDoesNotAnswerForAMomentDoesNotFailTheTier(t *testing.T) { + asksOfEveryModule(t) p, byName := aTierOfThree() bus := false was := askABuild @@ -68,6 +78,7 @@ func TestABusThatDoesNotAnswerForAMomentDoesNotFailTheTier(t *testing.T) { } func TestABusSilentForTenMinutesFailsTheTierAsBefore(t *testing.T) { + asksOfEveryModule(t) p, byName := aTierOfThree() was := askABuild askABuild = func(context.Context, buildSource, string, string) (string, error) { @@ -119,6 +130,7 @@ func TestAPlanWithAModuleWaitingForTheBusIsOneTheLookComesBackTo(t *testing.T) { } func TestAModulesOwnFaultStillFailsThePlan(t *testing.T) { + asksOfEveryModule(t) p, byName := aTierOfThree() delete(byName, "a") askModule(t.Context(), p, "a", byName) diff --git a/cmd/mesh-controller/away_integration_test.go b/cmd/mesh-controller/away_integration_test.go index 72f18610..1bca1e7c 100644 --- a/cmd/mesh-controller/away_integration_test.go +++ b/cmd/mesh-controller/away_integration_test.go @@ -68,6 +68,7 @@ func TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent(t *testing.T) { var mu sync.Mutex var asks []string var sends [][]string + asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it wasAsk := askABuild askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) { mu.Lock() diff --git a/cmd/mesh-controller/builder_on_the_bus_test.go b/cmd/mesh-controller/builder_on_the_bus_test.go index c20a6d6a..fe277dbd 100644 --- a/cmd/mesh-controller/builder_on_the_bus_test.go +++ b/cmd/mesh-controller/builder_on_the_bus_test.go @@ -117,6 +117,7 @@ func noBuildAsked(t *testing.T) { // the test's own may ask at once, so the record is written under a lock (issue 1543). func asksWithPaths(t *testing.T) *[][3]string { t.Helper() + asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): these tests are of what a walk does around it var asked [][3]string var mu sync.Mutex was := askABuild diff --git a/cmd/mesh-controller/pending_integration_test.go b/cmd/mesh-controller/pending_integration_test.go index c6c5b90d..7000200a 100644 --- a/cmd/mesh-controller/pending_integration_test.go +++ b/cmd/mesh-controller/pending_integration_test.go @@ -305,6 +305,7 @@ func TestAFailedAskIsNotABuildInFlight(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() inv := open.inventory + asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it was := askABuild askABuild = func(context.Context, buildSource, string, string) (string, error) { return "", errors.New("cannot submit a build: nats: timeout") @@ -569,6 +570,7 @@ func TestASecondAssignmentNotMadeKeepsItsCondition(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() asked := 0 + asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it was := askABuild askABuild = func(context.Context, buildSource, string, string) (string, error) { asked++ diff --git a/cmd/mesh-controller/plan_keys.go b/cmd/mesh-controller/plan_keys.go new file mode 100644 index 00000000..e3c48291 --- /dev/null +++ b/cmd/mesh-controller/plan_keys.go @@ -0,0 +1,268 @@ +package main + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/novox/mesh-controller/internal/builderkeys" + "github.com/novox/mesh-controller/internal/inventory" +) + +// A walk takes its modules from the builder's done keys (novox/hq ADR 0503 §6, ADR 0501 §1, ADR 0498 §8; issue 1851, A6 +// and A7). +// +// From R4e a walk asks build-agent for no module but the builder's own (ADR 0453 §5). A module of its tier is marked +// taken from a key (PlanModule.Key) and settled by the look: its fingerprint is read from the indexer's trunk value +// written for the commit the walk carries of its repository; the builder's key of that fingerprint, done with every +// artifact referenced, is taken in (recorded and registered, off the plans' hold) and the module is built; a key not done +// yet, or waiting for a reference, holds the tier and says so; a failed key fails the walk in the builder's own words. +// **No timer**: the builder's and the indexer's writes wake the look (builderkeys.Watch, watchNewModules). + +// isBuildersOwn says a module is the builder's own: the one module build-agent still builds, asked on a merge, until R4g +// (ADR 0453 §5, ADR 0498 §7). +func isBuildersOwn(module string) bool { return buildersOwn(module) } + +// buildersOwn is isBuildersOwn's rule: a variable only so a test of the one ask left (its bus waits, its plan key) may +// name modules of its own as the builder's. +var buildersOwn = func(module string) bool { return module == sdkdelivery.BuilderModule } + +// planKeyReads is what a walk reads of the indexer and the builder (builderkeys.Reader, or a test's). +type planKeyReads interface { + ModuleAt(ctx context.Context, module, commit string) (sdkdelivery.Module, error) + Build(ctx context.Context, module, fp string) (builderkeys.Key, error) +} + +// openPlanKeys opens what a walk reads of the indexer and the builder. A variable so a test needs no bus. +var openPlanKeys = func() (planKeyReads, func(), error) { + r, closeKeys, err := openBuilderKeys() + if err != nil { + return nil, nil, err + } + return r, closeKeys, nil +} + +// takeFromKey marks a module of a walk's tier as taken from the builder's key: asked of nobody (ADR 0501 §6). +func takeFromKey(p *inventory.Plan, state *inventory.PlanModule, e inventory.Entry, now time.Time) { + state.BusWaitFrom = nil + state.State = "asked" + state.AskedAt = &now + state.Key = true + state.Build, state.Fingerprint = "", "" + state.KeyAt = p.CommitOn(e.Source.Repository, followedBranch(e.Source.Ref)) + state.Why = "waits for the builder's key" + state.Commit, state.BuiltAt, state.Made = "", nil, "" + fmt.Printf(" tier %d: %s is taken from the builder's key, asking no build\n", p.Tier, e.Manifest.Module) +} + +// keyTakes are the take-ins of keys a look started and has not seen end, by key build id, and what each said when it +// waits: kept in this process, so a look starts each once and a settle says why it waits. +var keyTakes = struct { + sync.Mutex + running map[string]bool + waits map[string]string + // taken are the keys whose take-in ended registered, or found the module already at a later build (superseded). + taken map[string]bool +}{running: map[string]bool{}, waits: map[string]string{}, taken: map[string]bool{}} + +// settleFromKeys settles every module of the tier taken from a key: built from its key's record, failed with the key's +// words or the take-in's refusal, or still waiting, saying for what. It says whether it changed the plan. It reads only +// the buckets and the records; a done key not recorded yet is taken in off the plans' hold, and its end asks the look. +func settleFromKeys(ctx context.Context, open *stores, keys planKeyReads, p *inventory.Plan, tier []string) (bool, error) { + return settleKeys(ctx, open.inventory, keys, p, tier, func(module string, trunk sdkdelivery.Module, id string) { + go takeKeyOffTheHold(context.WithoutCancel(ctx), open, module, trunk, id) + }) +} + +// keyRecords is what settling from keys reads of the records (*inventory.Inventory, or a test's). +type keyRecords interface { + BuildByID(ctx context.Context, id string) (inventory.Build, bool, error) + RefusedOf(ctx context.Context, ids []string) (map[string]string, error) + SourceOf(ctx context.Context, module string) (inventory.Source, error) +} + +// settleKeys is settleFromKeys on the records handed in, starting a take-in with take. +func settleKeys(ctx context.Context, inv keyRecords, keys planKeyReads, p *inventory.Plan, tier []string, + take func(module string, trunk sdkdelivery.Module, id string)) (bool, error) { + changed := false + set := func(s *inventory.PlanModule, why string) { + if s.Why != why { + s.Why = why + changed = true + } + } + for _, m := range tier { + s := p.Modules[m] + if s == nil || !s.Key || s.State != "asked" { + continue + } + trunk, err := keys.ModuleAt(ctx, m, s.KeyAt) + if errors.Is(err, builderkeys.ErrNoValue) { + set(s, fmt.Sprintf("waits for the indexer's value of %s at %s", m, orNow(s.KeyAt))) + continue + } + if err != nil { + return changed, err + } + if trunk.FP2 == "" { + why := trunk.None2 + if why == "" { + why = "the indexer wrote no fingerprint of the new form" + } + set(s, fmt.Sprintf("waits: %s at %s has no fingerprint to take a build by: %s", m, orNow(s.KeyAt), why)) + continue + } + if s.Fingerprint != trunk.FP2 { + s.Fingerprint, changed = trunk.FP2, true + } + k, err := keys.Build(ctx, m, trunk.FP2) + if err != nil { + return changed, err + } + switch { + case k.Failed(): + // **A failed key fails the walk in the builder's own words** (`digests differ:`, `not stored:`, `not pinned:`, + // `version taken:`), and a rebuild of it is a new revision the next walk takes. + s.State, s.Why = "failed", k.Build.Error + p.State = inventory.PlanFailed + p.Note = fmt.Sprintf("%s failed to build in tier %d: %s", m, p.Tier, firstLine(k.Build.Error)) + changed = true + continue + case !k.Done() || len(k.Unreferenced()) > 0: + set(s, "waits: "+k.Says()) + continue + } + id := keyBuildID(k) + refused, err := inv.RefusedOf(ctx, []string{id}) + if err != nil { + return changed, fmt.Errorf("the refused registrations cannot be read: %w", err) + } + for kept, why := range refusalsKeptFor([]string{id}) { + refused[kept] = why + } + if why, ok := refused[id]; ok { + s.State, s.Why, s.Build = "failed", why, id + p.State = inventory.PlanFailed + p.Note = fmt.Sprintf("%s failed to build in tier %d: %s", m, p.Tier, firstLine(why)) + changed = true + continue + } + b, found, err := inv.BuildByID(ctx, id) + if err != nil { + return changed, err + } + if found && b.Worked() { + keyTakes.Lock() + taken := keyTakes.taken[id] + keyTakes.Unlock() + registered := taken + if !registered { + if registered, err = registeredFrom(ctx, inv, m, b); err != nil { + return changed, err + } + } + if registered { + at := b.At + s.State, s.BuiltAt, s.Commit, s.Build = "built", &at, b.Commit, id + s.Made = fmt.Sprintf("taken from the builder's key %s (revision %d), no build asked", short(trunk.FP2), k.Revision) + s.Why = "" + fmt.Printf("%s: %s taken from the builder's key %s\n", p.ID, m, short(trunk.FP2)) + changed = true + continue + } + } + keyTakes.Lock() + running, waits := keyTakes.running[id], keyTakes.waits[id] + if !running { + keyTakes.running[id] = true + } + keyTakes.Unlock() + if waits != "" { + set(s, firstLine(waits)) + } else { + set(s, "taking in the builder's done key "+short(trunk.FP2)) + } + if !running { + take(m, trunk, id) + } + } + return changed, nil +} + +// registeredFrom says the module's registration stands on the build: the catalogue holds the module at the build's +// commit, or the build was registered as one with an unchanged source (no move). +func registeredFrom(ctx context.Context, inv keyRecords, module string, b inventory.Build) (bool, error) { + src, err := inv.SourceOf(ctx, module) + if errors.Is(err, inventory.ErrNoSuchModule) { + return false, nil + } + if err != nil { + return false, err + } + return sameCommitID(src.BuiltFrom, b.Commit), nil +} + +func sameCommitID(a, b string) bool { + if a == "" || b == "" { + return false + } + n := min(len(a), len(b)) + return n >= 7 && a[:n] == b[:n] +} + +func orNow(commit string) string { + if commit == "" { + return "its value now" + } + return short(commit) +} + +// takeKeyOffTheHold takes a walk's module in from its done key, off the plans' hold: a registration may sweep the +// artifact store, and the plans are held by one look at a time. Its end asks the look, which settles the module from the +// record; a refusal is kept by the key's build id, so the look fails the walk with it; a wait is said by the look. +func takeKeyOffTheHold(ctx context.Context, open *stores, module string, trunk sdkdelivery.Module, id string) { + defer func() { + keyTakes.Lock() + delete(keyTakes.running, id) + keyTakes.Unlock() + askToLook() + }() + keys, closeKeys, err := openBuilderKeys() + if err != nil { + keepKeyWait(id, "waits: the builder's keys cannot be read: "+err.Error()) + return + } + defer closeKeys() + _, _, err = takeInKey(ctx, open.inventory, keys, keyTake{Module: module, Trunk: trunk, + Ask: registrationAsk{Merge: trunk.Commit}}) + switch { + case err == nil, errors.Is(err, inventory.ErrSuperseded): + // Superseded is no failure: the module is already at what a later merge's key built, which a walk that carries + // this merge is answered by, as a build-agent outcome superseded was. + keepKeyWait(id, "") + keyTakes.Lock() + keyTakes.taken[id] = true + keyTakes.Unlock() + case errors.Is(err, errKeyWaits): + keepKeyWait(id, err.Error()) + default: + why := err.Error() + if rerr := open.inventory.RecordRefused(ctx, id, module, why); rerr != nil { + fmt.Printf("plans: %s's refusal could not be kept in the store, only in this controller: %v\n", module, rerr) + } + keepRefusalFor(id, why, time.Now()) + } +} + +func keepKeyWait(id, why string) { + keyTakes.Lock() + defer keyTakes.Unlock() + if why == "" { + delete(keyTakes.waits, id) + return + } + keyTakes.waits[id] = why +} diff --git a/cmd/mesh-controller/plan_keys_test.go b/cmd/mesh-controller/plan_keys_test.go new file mode 100644 index 00000000..e11da201 --- /dev/null +++ b/cmd/mesh-controller/plan_keys_test.go @@ -0,0 +1,190 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/novox/mesh-controller/internal/builderkeys" + "github.com/novox/mesh-controller/internal/inventory" +) + +// planKeysOf is the indexer and the builder as a walk reads them, for a test: values by module and commit, keys by +// module. +type planKeysOf struct { + values map[string]sdkdelivery.Module // by module + "@" + commit + keys map[string]builderkeys.Key +} + +func (k planKeysOf) ModuleAt(_ context.Context, module, commit string) (sdkdelivery.Module, error) { + if v, ok := k.values[module+"@"+commit]; ok { + return v, nil + } + return sdkdelivery.Module{}, builderkeys.ErrNoValue +} + +func (k planKeysOf) Build(_ context.Context, module, fp string) (builderkeys.Key, error) { + if key, ok := k.keys[module]; ok && key.Fingerprint == fp { + return key, nil + } + return builderkeys.Key{Module: module, Fingerprint: fp}, nil +} + +// recordsOf is the build records and refusals a settle reads, for a test. +type recordsOf struct { + builds map[string]inventory.Build + refused map[string]string + sources map[string]inventory.Source +} + +func (r recordsOf) BuildByID(_ context.Context, id string) (inventory.Build, bool, error) { + b, ok := r.builds[id] + return b, ok, nil +} + +func (r recordsOf) RefusedOf(_ context.Context, ids []string) (map[string]string, error) { + out := map[string]string{} + for _, id := range ids { + if why, ok := r.refused[id]; ok { + out[id] = why + } + } + return out, nil +} + +func (r recordsOf) SourceOf(_ context.Context, module string) (inventory.Source, error) { + if s, ok := r.sources[module]; ok { + return s, nil + } + return inventory.Source{}, inventory.ErrNoSuchModule +} + +func aWalkOfOne(module string) (*inventory.Plan, map[string]inventory.Entry) { + p := &inventory.Plan{ID: "plan-1", State: inventory.PlanBuilding, Tiers: [][]string{{module}}, + Modules: map[string]*inventory.PlanModule{}, Repository: "novox/mesh-catalog", Commit: "c0ffee0123456789", + Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{{Repository: "novox/mesh-catalog", Commit: "c0ffee0123456789"}}}} + byName := map[string]inventory.Entry{module: {Source: inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", + Path: "modules/" + module, Ref: "main"}}} + return p, byName +} + +// A6 (ADR 0503 §6, ADR 0498 check §7): a walk whose module has a done key goes asked to built with no build asked: the +// ask stub is never called, the key is taken in once, and `plans` says it was taken from the builder's key. +func TestAWalkTakesADoneKeyAndAsksNoBuild(t *testing.T) { + was := askABuild + askABuild = func(context.Context, buildSource, string, string) (string, error) { + t.Fatal("a walk asked build-agent for a module that is not the builder's own") + return "", nil + } + t.Cleanup(func() { askABuild = was }) + p, byName := aWalkOfOne("sudo") + askModule(t.Context(), p, "sudo", byName) + s := p.Modules["sudo"] + if s.State != "asked" || !s.Key || s.Build != "" || s.KeyAt != "c0ffee0123456789" { + t.Fatalf("the module is not taken from a key: %+v", s) + } + fp := "src2:" + strings.Repeat("a", 64) + k := builderkeys.Key{Module: "sudo", Fingerprint: fp, Found: true, Revision: 2, Build: sdkdelivery.BuilderBuild{ + State: sdkdelivery.BuildDone, Manifest: strings.Repeat("b", 64), Artifacts: []sdkdelivery.BuiltArtifact{{Name: "a", + Digest: "sha256:1", Reference: "artifact-store://sudo/a/blobs/sha256:1"}}}} + keys := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@c0ffee0123456789": {FP2: fp, Commit: "c0ffee0123456789"}}, + keys: map[string]builderkeys.Key{"sudo": k}} + records := recordsOf{builds: map[string]inventory.Build{}, sources: map[string]inventory.Source{}} + var taken []string + take := func(m string, _ sdkdelivery.Module, id string) { taken = append(taken, m+" "+id) } + t.Cleanup(func() { + keyTakes.Lock() + keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{} + keyTakes.Unlock() + }) + + changed, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take) + if err != nil || !changed || len(taken) != 1 || s.State != "asked" || s.Fingerprint != fp { + t.Fatalf("a done key not recorded yet is taken in once: %v %v %v %+v", changed, err, taken, s) + } + // A second look while the take-in runs starts no second one. + if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil || len(taken) != 1 { + t.Fatalf("a second look took the key in again: %v %v", taken, err) + } + // The take-in recorded and registered it: the look settles it built from the record. + id := keyBuildID(k) + records.builds[id] = inventory.Build{ID: id, Module: "sudo", Commit: "c0ffee0123456789", At: time.Now()} + records.sources["sudo"] = inventory.Source{BuiltFrom: "c0ffee0123456789"} + if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil { + t.Fatal(err) + } + if s.State != "built" || s.Build != id || !strings.Contains(s.Made, "taken from the builder's key") { + t.Fatalf("the module is not built from its key: %+v", s) + } + if takesOutcome(*p, &inventory.PlanModule{Key: true, State: "asked", AskedAt: s.AskedAt}, time.Now(), "build-1") { + t.Fatal("a build-agent outcome answers a module taken from a key") + } +} + +// A7: a key building waits and says so, then moves on when done, with no timer and no ask; a failed key fails the walk +// in the builder's words; a value the indexer has not written yet waits, named. +func TestAWalkWaitsForAKeyAndFailsWithItsWords(t *testing.T) { + asksOfNone := askABuild + askABuild = func(context.Context, buildSource, string, string) (string, error) { + t.Fatal("a walk asked a build") + return "", nil + } + t.Cleanup(func() { askABuild = asksOfNone }) + t.Cleanup(func() { + keyTakes.Lock() + keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{} + keyTakes.Unlock() + }) + p, byName := aWalkOfOne("view") + askModule(t.Context(), p, "view", byName) + s := p.Modules["view"] + records := recordsOf{builds: map[string]inventory.Build{}} + take := func(string, sdkdelivery.Module, string) {} + keys := planKeysOf{values: map[string]sdkdelivery.Module{}, keys: map[string]builderkeys.Key{}} + if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil || + !strings.Contains(s.Why, "waits for the indexer's value of view at c0ffee01") { + t.Fatalf("no indexer value yet: %q %v", s.Why, err) + } + fp := "src2:" + strings.Repeat("c", 64) + keys.values["view@c0ffee0123456789"] = sdkdelivery.Module{FP2: fp} + keys.keys["view"] = builderkeys.Key{Module: "view", Fingerprint: fp, Found: true, + Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildBuilding, By: "g14"}} + if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil || + s.State != "asked" || !strings.Contains(s.Why, "building on g14") || p.State != inventory.PlanBuilding { + t.Fatalf("a building key: %+v %v", s, err) + } + keys.keys["view"] = builderkeys.Key{Module: "view", Fingerprint: fp, Found: true, + Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildFailed, Error: "not stored: tools: the store answered 500"}} + if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil { + t.Fatal(err) + } + if s.State != "failed" || p.State != inventory.PlanFailed || !strings.Contains(p.Note, "not stored: tools") { + t.Fatalf("a failed key does not fail the walk in its words: %+v %q", s, p.Note) + } +} + +// A7: a take-in refused (a module from another repository, a manifest refused) fails the walk with the refusal. +func TestARefusedTakeInFailsTheWalk(t *testing.T) { + t.Cleanup(func() { + keyTakes.Lock() + keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{} + keyTakes.Unlock() + }) + p, byName := aWalkOfOne("sudo") + askModule(t.Context(), p, "sudo", byName) + fp := "src2:" + strings.Repeat("d", 64) + k := builderkeys.Key{Module: "sudo", Fingerprint: fp, Found: true, Revision: 1, Build: sdkdelivery.BuilderBuild{ + State: sdkdelivery.BuildDone, Manifest: strings.Repeat("b", 64), Artifacts: []sdkdelivery.BuiltArtifact{{Name: "a", + Digest: "sha256:1", Reference: "artifact-store://sudo/a/blobs/sha256:1"}}}} + keys := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@c0ffee0123456789": {FP2: fp}}, + keys: map[string]builderkeys.Key{"sudo": k}} + records := recordsOf{refused: map[string]string{keyBuildID(k): "sudo is built from novox/mesh-catalog, and this build is of agent/sudo"}} + if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], func(string, sdkdelivery.Module, string) {}); err != nil { + t.Fatal(err) + } + if p.State != inventory.PlanFailed || !strings.Contains(p.Note, "agent/sudo") { + t.Fatalf("a refused take-in did not fail the walk: %s %q", p.State, p.Note) + } +} diff --git a/cmd/mesh-controller/plan_retry.go b/cmd/mesh-controller/plan_retry.go index 9556bef5..86dc803e 100644 --- a/cmd/mesh-controller/plan_retry.go +++ b/cmd/mesh-controller/plan_retry.go @@ -402,7 +402,9 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) { var asked []string for _, m := range failed { askModule(ctx, &p, m, byName) - if s := p.Modules[m]; s.State == "asked" { + if s := p.Modules[m]; s.State == "asked" && s.Key { + asked = append(asked, m+" from the builder's key") + } else if s.State == "asked" { asked = append(asked, m+" as "+s.Build) } } @@ -629,7 +631,9 @@ func retryTierWhole(ctx context.Context, open *stores, p *inventory.Plan, again for _, m := range again { sendAgain(p.Modules[m]) askModule(ctx, p, m, byName) - if s := p.Modules[m]; s.State == "asked" { + if s := p.Modules[m]; s.State == "asked" && s.Key { + asked = append(asked, m+" from the builder's key") + } else if s.State == "asked" { asked = append(asked, m+" as "+s.Build) } } diff --git a/cmd/mesh-controller/plans_building_test.go b/cmd/mesh-controller/plans_building_test.go index 8aab90c0..94d7d3bb 100644 --- a/cmd/mesh-controller/plans_building_test.go +++ b/cmd/mesh-controller/plans_building_test.go @@ -156,6 +156,7 @@ func TestASavedPlanPutsOrDeletesItsKey(t *testing.T) { // The plan's key is there before its first build is asked, so a check cannot take a place between the ask and the plan's save. func TestThePlansKeyIsPutBeforeItsFirstBuildIsAsked(t *testing.T) { + asksOfEveryModule(t) store := newFakeBuildingStore() var s buildingStore = store buildingKeys.Store(&s) diff --git a/cmd/mesh-controller/queue_integration_test.go b/cmd/mesh-controller/queue_integration_test.go index 91d0d484..c0cb98f3 100644 --- a/cmd/mesh-controller/queue_integration_test.go +++ b/cmd/mesh-controller/queue_integration_test.go @@ -431,6 +431,7 @@ func TestARebuildOfACommitAsksWhatTheModuleFollows(t *testing.T) { asked := asksRecorded(t) twoTiers(t, open) var refs []string + asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it was := askABuild askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) { refs = append(refs, ref) diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 4f949fca..d8ec6a75 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -390,6 +390,11 @@ func takesOutcome(p inventory.Plan, state *inventory.PlanModule, asked time.Time if p.Waiting() || state == nil || (state.Build == "" && state.AskedAt == nil) { return false } + // **A module taken from the builder's key is answered by its key alone** (ADR 0501 §1): a build-agent outcome of it + // is somebody else's build. + if state.Key { + return false + } if state.Build != "" { return state.Build == id } @@ -565,6 +570,13 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s p.Note = name + " is no longer in the catalogue" return } + // **Every module but the builder's own is taken from the builder's done key, asking no build** (novox/hq ADR 0503 + // §6, ADR 0501, ADR 0453 §5): the look settles it (settleFromKeys). + if !isBuildersOwn(name) { + sayBuilding(ctx, p.ID, now) + takeFromKey(p, state, e, now) + return + } source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat} fmt.Printf(" tier %d: ", p.Tier) // The branch it follows, never a commit a build once named (novox/hq 04-ISSUES/215): the branch contains every @@ -1071,6 +1083,25 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, // the controller in its first tier: the build that produced the new one is recorded, and the // plan never hears it. The record is the fact; a build recorded after the ask is that tier's // outcome, whoever was listening. + // **Taken from the builder's keys first** (ADR 0503 §6, issue 1851): every module of the tier but the builder's own. + if keyed(p, tier) { + keys, closeKeys, err := openPlanKeys() + if err != nil { + return false, err + } + was := p.State + changed, err := settleFromKeys(ctx, open, keys, p, tier) + closeKeys() + if err != nil { + return false, err + } + if p.State == inventory.PlanFailed && was != inventory.PlanFailed { + sayUnsent(p, rollsOut) + } + if changed { + return true, nil + } + } recorded, byID, err := recordsOfAsked(ctx, inv, p, tier) if err != nil { return false, err @@ -2744,7 +2775,7 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i changed := false for _, m := range tier { s := p.Modules[m] - if s == nil || s.State != "asked" || s.AskedAt == nil { + if s == nil || s.State != "asked" || s.AskedAt == nil || s.Key { continue } var outcome *inventory.Build @@ -2845,3 +2876,13 @@ func stopByHand(p *inventory.Plan, how, by, why string) { } p.Delivery.Stopped, p.Delivery.StoppedWhy = by, why } + +// keyed says a module of the tier still waits on the builder's key. +func keyed(p *inventory.Plan, tier []string) bool { + for _, m := range tier { + if s := p.Modules[m]; s != nil && s.Key && s.State == "asked" { + return true + } + } + return false +} diff --git a/cmd/mesh-controller/replays_integration_test.go b/cmd/mesh-controller/replays_integration_test.go index da467d3d..deb14f0b 100644 --- a/cmd/mesh-controller/replays_integration_test.go +++ b/cmd/mesh-controller/replays_integration_test.go @@ -423,6 +423,7 @@ func TestReplay300AMergeAddingAModuleAsksForItsBuild(t *testing.T) { ctx := t.Context() inv := open.inventory var asked [][3]string + asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it was := askABuild askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) { asked = append(asked, [3]string{source.Repository, path, ref}) diff --git a/cmd/mesh-controller/rollout_events_test.go b/cmd/mesh-controller/rollout_events_test.go index 2110f60f..0c072b33 100644 --- a/cmd/mesh-controller/rollout_events_test.go +++ b/cmd/mesh-controller/rollout_events_test.go @@ -69,6 +69,7 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) { var mu sync.Mutex var asks []string var sends [][]string + asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it wasAsk := askABuild askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) { mu.Lock() diff --git a/cmd/mesh-controller/silent_mid_tier_integration_test.go b/cmd/mesh-controller/silent_mid_tier_integration_test.go index b60784f0..dac3e72b 100644 --- a/cmd/mesh-controller/silent_mid_tier_integration_test.go +++ b/cmd/mesh-controller/silent_mid_tier_integration_test.go @@ -65,6 +65,7 @@ func TestAPlanPassesATierWhoseMachineNeverReportsAfterTheBoundAndSendsTheNextToT var mu sync.Mutex var asks []string var sends [][]string + asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it wasAsk := askABuild askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) { mu.Lock() diff --git a/cmd/mesh-controller/tier_send_integration_test.go b/cmd/mesh-controller/tier_send_integration_test.go index 9d0cdddf..b933516a 100644 --- a/cmd/mesh-controller/tier_send_integration_test.go +++ b/cmd/mesh-controller/tier_send_integration_test.go @@ -302,6 +302,7 @@ func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) { } } + asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it was := askABuild askABuild = func(context.Context, buildSource, string, string) (string, error) { return "build-3", nil } t.Cleanup(func() { askABuild = was }) diff --git a/cmd/mesh-controller/walk_from_keys_integration_test.go b/cmd/mesh-controller/walk_from_keys_integration_test.go new file mode 100644 index 00000000..d4609105 --- /dev/null +++ b/cmd/mesh-controller/walk_from_keys_integration_test.go @@ -0,0 +1,104 @@ +//go:build integration + +package main + +import ( + "reflect" + "strings" + "testing" + "time" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/novox/mesh-controller/internal/inventory" +) + +// A6, A7 whole against the store and a real bus (ADR 0503 §6, ADR 0498 check §7; issue 1851): a walk whose modules the +// builder built goes from asked to built with no build asked, its keys taken in off the plans' hold, then sends its tier +// to the first machine as before; a key still building holds the tier, said, and moves on when the builder's key is done. +func TestAWalkTakesItsTierFromTheBuildersKeysAndSendsIt(t *testing.T) { + withTheGate(t) + judgeLeavesAsTheCore(t) + tm := aTierMesh(t, "app1", "app2") + ctx := t.Context() + inv := tm.open.inventory + noBuildAsked(t) + b := aBuilderOnTheBus(t) + + // The walk carries c3 of the catalogue; its tier is unasked. + p := tm.plan(t) + p.Commit = "c3c3c3c3c3c3c3c3" + for _, m := range []string{"app1", "app2"} { + p.Modules[m] = &inventory.PlanModule{} + } + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + fp1, fp2 := fpOf('1'), fpOf('2') + m1 := b.indexed(t, "app1", "c3", "novox/mesh-catalog", "modules/app1", p.Commit, fp1) + m2 := b.indexed(t, "app2", "c3", "novox/mesh-catalog", "modules/app2", p.Commit, fp2) + b.built(t, "app1", "c3", fp1, m1, sdkdelivery.BuildDone) + b.built(t, "app2", "c3", fp2, m2, sdkdelivery.BuildBuilding) + + advancePlans(ctx, tm.open) + becomes(t, 10*time.Second, "app1 taken from its key", func() bool { + advancePlans(ctx, tm.open) + return tm.plan(t).Modules["app1"].State == "built" + }) + p = tm.plan(t) + if s := p.Modules["app2"]; s.State != "asked" || !s.Key || !strings.Contains(s.Why, "building on ace") { + t.Fatalf("A7: app2, whose key is building, is %+v", s) + } + if len(tm.sent) != 0 { + t.Fatalf("A7: the tier was sent while app2 waits on its key: %v", tm.sent) + } + if s := p.Modules["app1"]; !strings.Contains(s.Made, "taken from the builder's key") || s.Fingerprint != fp1 { + t.Fatalf("A6: plans does not say app1 was taken from the builder's key: %+v", s) + } + + b.built(t, "app2", "c3", fp2, m2, sdkdelivery.BuildDone) + becomes(t, 10*time.Second, "the tier sent to the first machine", func() bool { + advancePlans(ctx, tm.open) + return len(tm.sent) > 0 + }) + p = tm.plan(t) + if !reflect.DeepEqual(tm.sent[0], []string{"anchor"}) { + t.Fatalf("A6: the tier taken from keys was sent %v, not to the first machine", tm.sent) + } + current, err := inv.CurrentBuilds(ctx) + if err != nil { + t.Fatal(err) + } + for _, m := range []string{"app1", "app2"} { + if current[m].Commit != p.Commit || !isKeyBuildID(p.Modules[m].Build) { + t.Fatalf("A6: %s stands on %+v (%s), not the key's build at %s", m, current[m], p.Modules[m].Build, p.Commit) + } + } +} + +// A7: a failed key fails the walk with the builder's words, and nothing is sent. +func TestAFailedKeyFailsTheWalkInTheBuildersWords(t *testing.T) { + tm := aTierMesh(t, "app1") + ctx := t.Context() + inv := tm.open.inventory + noBuildAsked(t) + b := aBuilderOnTheBus(t) + p := tm.plan(t) + p.Commit = "c4c4c4c4c4c4c4c4" + p.Modules["app1"] = &inventory.PlanModule{} + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + fp := fpOf('4') + manifest := b.indexed(t, "app1", "c4", "novox/mesh-catalog", "modules/app1", p.Commit, fp) + b.built(t, "app1", "c4", fp, manifest, sdkdelivery.BuildFailed) + for i := 0; i < 3; i++ { + advancePlans(ctx, tm.open) + } + p = tm.plan(t) + if p.State != inventory.PlanFailed || !strings.Contains(p.Note, "digests differ: ace sha256:1, g14 sha256:2") { + t.Fatalf("A7: the walk is %s: %q", p.State, p.Note) + } + if len(tm.sent) != 0 { + t.Fatalf("A7: a failed walk sent %v", tm.sent) + } +} diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index 109b6104..90af4a0c 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -388,6 +388,15 @@ type PlanModule struct { // data, or "waits: …" with the recreate words, the data change named, the person's or the bus's hold, or why it // could not be judged. Recorded map[string]string `json:"recorded,omitempty"` + // Key says the walk takes this module from the builder's done key and asked no build (novox/hq ADR 0501, ADR 0453 + // §5; issue 1851): from R4e every module but the builder's own. Its outcome is the key's, never a build-agent outcome. + Key bool `json:"key,omitempty"` + // KeyAt is the commit of its repository whose indexer's trunk value names the fingerprint taken (ADR 0501 §2); empty + // for a module of a repository the walk carries no commit of (a dependent through a base), taken at the value now. + KeyAt string `json:"key_at,omitempty"` + // Fingerprint is the fingerprint of the new form (`src2:`) the walk takes the module at, read from that trunk value; + // empty until the indexer has written it. + Fingerprint string `json:"fingerprint,omitempty"` } // PlanGate is one module's rollout record at its gate (to-be 45 §8): the component, the first machine,