From 249d97d1c84b3ffdb12aa2415269270ebdbb97a8 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 8 Oct 2026 16:21:53 +0200 Subject: [PATCH] Make pending assignments safe to race, settle them on a tick, and say only what was checked Review of #150: a withdrawal could land between the look and the act, a failed ask read as a build in flight, a request kept the wrong asker, a build being registered read as not built, build "true" could ask a build nothing waited on, and a status read changed state. Claim a row under the machine's hold before making it, keep a request only once asked, settle on the controller's own tick, raise an assignment not made as a condition until it is answered, and tie each row to its machine. --- cmd/mesh-controller/build.go | 60 ++- cmd/mesh-controller/pending.go | 439 ++++++++++++++---- cmd/mesh-controller/pending_test.go | 368 +++++++++++++-- cmd/mesh-controller/plain_words.go | 13 + cmd/mesh-controller/push.go | 4 + cmd/mesh-controller/queue.go | 8 +- cmd/mesh-controller/release_plan.go | 5 +- cmd/mesh-controller/status.go | 19 +- internal/catalogue/verbs.go | 9 +- ...0082-an-assignment-waits-for-its-build.sql | 49 +- internal/inventory/pending.go | 254 +++++++--- internal/inventory/pending_test.go | 25 + 12 files changed, 1018 insertions(+), 235 deletions(-) create mode 100644 internal/inventory/pending_test.go diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index 56677104..28f5e7ec 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -399,15 +399,43 @@ func buildBehind(ctx context.Context, wait time.Duration) error { // // Separated from the command so `--behind` can walk a list without a second path to the same act. func buildOne(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error { - _, err := buildOneAsked(ctx, source, path, ref, wait, false) + _, err := buildOneAsked(ctx, source, path, ref, wait, false, "build") return err } +// recordAsked keeps a build request once it is asked (novox/hq issue 325), with who asked it, in a store +// opened for the purpose: the asks reach here from processes that hold none. +func recordAsked(ctx context.Context, r inventory.BuildRequest) { + open, err := openStores(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "build %s is asked and not kept as a build request: %v\n", r.ID, err) + return + } + defer open.Close() + recordBuildRequest(ctx, open.inventory, r) +} + +// markNotAsked says a kept build request was not handed over, or not waited for. +func markNotAsked(ctx context.Context, id string, why error) { + open, err := openStores(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "build %s could not be marked as not asked: %v\n", id, err) + return + } + defer open.Close() + if err := open.inventory.MarkNotAsked(ctx, id, why.Error()); err != nil { + fmt.Fprintf(os.Stderr, "build %s could not be marked as not asked: %v\n", id, err) + } +} + // buildOneAsked is buildOne answering the id it asked with — what a plan keeps to match the outcome // by (novox/hq ADR 0219) — and, for an ask not waited for, optionally a dry run: built and looked // at, never taken in (issue 240), which is what `replay` asks unless told to register. +// +// asker is who asked, for the build request kept once the ask is made (novox/hq issue 325); empty for an +// asker that keeps the request itself, with its own name, once it knows the ask was made. func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wait time.Duration, - dryRun bool) (string, error) { + dryRun bool, asker string) (string, error) { if dryRun && wait != 0 { return "", errors.New("a dry run waited for is `build --dry-run`") } @@ -462,17 +490,12 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa } defer ask.Close() fmt.Printf(" of %s\n", seat) - // Kept before it is asked, so `assign` knows a module is coming while its build runs (novox/hq issue 325). - // A dry run registers nothing, and is not kept. - if !dryRun { - if open, err := openStores(ctx); err == nil { - recordBuildRequest(ctx, open.inventory, inventory.BuildRequest{ID: request.ID, - Repository: source.Repository, Seat: source.Seat, Path: path, Ref: ref, For: "build"}) - open.Close() - } else { - fmt.Fprintf(os.Stderr, "build %s is asked and not kept as a build request: %v\n", request.ID, err) - } - } + // Kept once it is asked, so `assign` knows a module is coming while its build runs (novox/hq issue 325); + // never before, so an ask that failed never reads as a build in flight. A dry run registers nothing, and + // is not kept. + keep := !dryRun && asker != "" + asked := inventory.BuildRequest{ID: request.ID, Repository: source.Repository, Seat: source.Seat, Path: path, + Ref: ref, For: asker} if wait == 0 { // Asked and not waited for (novox/hq issue 176): the outcome is the role's event, and the @@ -482,6 +505,9 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa if err := ask.Ask(ctx, request); err != nil { return "", err } + if keep { + recordAsked(ctx, asked) + } if dryRun { fmt.Printf("asked as a dry run, not waited for: `builds --log %s` follows it as it runs; "+ "its outcome is not taken in\n", request.ID) @@ -492,8 +518,16 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa return request.ID, nil } + // Waited for: kept while it is waited for, since that can take minutes, and marked when the hand-over or + // the wait failed — an outcome heard later is still the last word. + if keep { + recordAsked(ctx, asked) + } result, err := ask.Submit(ctx, request, wait) if err != nil { + if keep { + markNotAsked(ctx, request.ID, err) + } return request.ID, err } diff --git a/cmd/mesh-controller/pending.go b/cmd/mesh-controller/pending.go index 4b4af543..af8fc44c 100644 --- a/cmd/mesh-controller/pending.go +++ b/cmd/mesh-controller/pending.go @@ -9,11 +9,12 @@ import ( "strings" "time" + "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) -// A module not registered yet, and the assignment that waits for it (novox/hq issue 325). +// A module not registered yet, and the assignment that waits for it (novox/hq issue 325, ADR 0261). // // On 2026-10-08 the catalogue's pull request adding `sensors` merged; a minute later `assign g14 sensors` // answered "no module of that name: sensors", and a few minutes later the same call worked. The merge had @@ -23,34 +24,55 @@ import ( // So an assignment of a module the catalogue does not hold looks further before it refuses, and says which // of three cases it is in: // -// - **a build is in flight**: a build request for the module's directory has no outcome yet. The -// assignment is kept as a **pending assignment** and made when the build registers the module. -// - **known, not built**: the mesh asked for the directory before, and the build failed, was not +// - **a build is in flight**: a build request for the module's directory has no outcome yet, or its build +// succeeded a moment ago and is being registered. The assignment is kept as a **pending assignment**, +// which the controller makes when the build registers the module (ADR 0261). +// - **known, not built**: the controller asked for the directory before, and the build failed, was not // registered, said nothing within its bound, or could not be asked. Said, with `build` and assign's // own build argument, which asks for the build and keeps the assignment pending on it. -// - **unknown**: no module, build request or merge by that name. Refused as before, with the closest names. +// - **unknown**: no module registered and no build request kept by that name. Refused as before, with the +// closest names. // -// **Pending by default while a build is in flight**, without an argument to ask for it: an assignment is -// what a person meant and is kept even when its machine does not resolve (acts.go), nothing is sent until a -// push, and `unassign` withdraws it. Asking for a second act once the build lands is exactly the two acts by -// hand issue 300 removed. A build is never asked for by default: it runs on another machine, and a directory -// whose last build failed is a fault to look at before it is a thing to retry. +// **Pending by default while a build is in flight** (ADR 0261), without an argument to ask for it: an +// assignment is what a person meant and is kept even when its machine does not resolve (acts.go), and +// `unassign` withdraws it. Asking for a second act once the build lands is the two acts by hand issue 300 +// removed. A build is never asked for by default: it runs on another machine, and a directory whose last +// build failed is a fault to look at before it is a thing to retry. +// +// **Settling is the controller's tick, never a read** (settlingPending): `status`, the board and the summary +// only read the rows. // buildRequestBound is how long a build request may go without an outcome before it is no longer read as in // flight, and a pending assignment waiting for it expires. Generous: a build waits behind others on the // build seat, and a pending assignment that waits a little longer costs nothing. const buildRequestBound = 2 * time.Hour -// pendingShownFor is how long an ended pending assignment is said in `status`. +// registerGrace is how long a build heard as built is read as still in flight while it is not registered: +// the outcome is recorded first and registered after it, in the same take-in. Past it, the build was +// recorded and not registered, and says so. +const registerGrace = 5 * time.Minute + +// pendingShownFor is how long an ended pending assignment is listed in `status`. const pendingShownFor = 24 * time.Hour +// settleEvery is how often the controller settles the pending assignments. +const settleEvery = time.Minute + +// claimStaleAfter is how long a pending assignment may be held as applying before the tick reads the claim +// as left by a controller that stopped, and settles it from what the mesh holds. +const claimStaleAfter = 5 * time.Minute + +// kindPendingEnded is a pending assignment that ended without being made: expired or refused. +const kindPendingEnded = "assignment-not-made" + // assignOptions are what an assignment was asked with beside its machine and modules. type assignOptions struct { // Build asks for the build of a module known and not built, and keeps the assignment pending on it. Build bool } -// recordBuildRequest keeps a build request so `assign` can find it. Said, never fatal: the build was asked. +// recordBuildRequest keeps a build request so `assign` can find it, once it is asked. Said, never fatal: the +// build was asked. func recordBuildRequest(ctx context.Context, inv *inventory.Inventory, r inventory.BuildRequest) { if err := inv.RecordBuildRequest(ctx, r); err != nil { fmt.Fprintf(os.Stderr, "build %s was asked, and could not be kept as a build request, so `assign` "+ @@ -73,25 +95,30 @@ type notHeld struct { said string } -// whereIs looks for a module the catalogue does not hold among the build requests the mesh keeps. +// whereIs looks for a module the catalogue does not hold among the build requests the controller keeps. func whereIs(ctx context.Context, inv *inventory.Inventory, module string, now time.Time) (notHeld, error) { requests, err := inv.RequestsNamed(ctx, module) if err != nil { return notHeld{}, err } if r, ok := inFlight(requests, now); ok { - return notHeld{kind: buildInFlight, request: r, said: fmt.Sprintf("%s is not registered yet: it is being "+ - "built from %s since %s, as build %s; it can be assigned once that build registers it", module, + being := "being built" + if r.Heard { + being = "built and being registered" + } + return notHeld{kind: buildInFlight, request: r, said: fmt.Sprintf("%s is not registered yet: it is %s "+ + "from %s since %s, as build %s; it can be assigned once that build registers it", module, being, requestSource(r), clock(r.At), r.ID)}, nil } if len(requests) > 0 { return notHeld{kind: knownNotBuilt, request: requests[0], said: fmt.Sprintf("%s is known and not "+ "registered: %s", module, whyNotBuilt(requests[0], now))}, nil } - said := fmt.Sprintf("%v: %s — the mesh holds no module, build request or merge by that name", - inventory.ErrNoSuchModule, module) + // Only what was looked at is claimed: the catalogue, and the build requests the controller keeps. + said := fmt.Sprintf("%v: %s — the catalogue holds no module of that name, and no build request the "+ + "controller kept (the last 30 days) is for a directory of that name", inventory.ErrNoSuchModule, module) if near := closestNames(ctx, inv, module); len(near) > 0 { - said += "; the closest it holds: " + strings.Join(near, ", ") + said += "; the closest names it holds: " + strings.Join(near, ", ") } return notHeld{kind: unknownModule, said: said}, nil } @@ -111,6 +138,20 @@ func notInCatalogue(ctx context.Context, open *stores, modules []string) ([]stri return missing, nil } +// openPending is the open pending assignment of a module to a machine, if there is one. +func openPending(ctx context.Context, inv *inventory.Inventory, node, module string) (*inventory.PendingAssignment, error) { + rows, err := inv.PendingFor(ctx, node, module) + if err != nil { + return nil, err + } + for i := range rows { + if rows[i].Open() { + return &rows[i], nil + } + } + return nil, nil +} + // notRegistered answers an assignment of a module the catalogue does not hold: pending on a build in flight, // known and not built, or unknown (novox/hq issue 325). An answer with no error is a pending assignment kept. func notRegistered(ctx context.Context, open *stores, node, module string, opts assignOptions) (string, error) { @@ -123,19 +164,31 @@ func notRegistered(ctx context.Context, open *stores, node, module string, opts if err != nil { return "", err } + waiting, err := openPending(ctx, inv, node, module) + if err != nil { + return "", err + } switch where.kind { case buildInFlight: lead := where.said if opts.Build { lead += "\n no second build was asked: this one is running" } - return keepPending(ctx, open, node, module, where.request, lead) + return keepPending(ctx, open, node, module, where.request, waiting, lead) case knownNotBuilt: last := where.request if !opts.Build { - return "", fmt.Errorf("%w: %s\n assign it again with build \"true\" to ask for its build and keep this "+ + also := "" + if waiting != nil { + also = fmt.Sprintf("\n %s already has a pending assignment of %s, waiting for build %s; build "+ + "\"true\" asks for a new build and makes it wait for that one", node, module, waiting.Build) + } + return "", fmt.Errorf("%w: %s%s\n assign it again with build \"true\" to ask for its build and keep this "+ "assignment until the build registers it; or `build` it (repository %s, path %s) and assign it "+ - "once it is registered", inventory.ErrNoSuchModule, where.said, last.Repository, orRoot(last.Path)) + "once it is registered", inventory.ErrNoSuchModule, where.said, also, last.Repository, orRoot(last.Path)) + } + if waiting != nil && waiting.State == inventory.PendingApplying { + return "", fmt.Errorf("%s is being assigned %s now; nothing was asked", node, module) } source := buildSource{Repository: last.Repository, Seat: last.Seat} id, err := askABuild(ctx, source, last.Path, last.Ref) @@ -147,20 +200,38 @@ func notRegistered(ctx context.Context, open *stores, node, module string, opts recordBuildRequest(ctx, inv, asked.BuildRequest) lead := fmt.Sprintf("%s\n build %s of %s is asked for now; %s can be assigned once it registers it", where.said, id, requestSource(asked), module) - return keepPending(ctx, open, node, module, asked, lead) + return keepPending(ctx, open, node, module, asked, waiting, lead) } answer := strings.TrimPrefix(where.said, inventory.ErrNoSuchModule.Error()) + "\n a module in a repository is built with `build` (its repository and path), then assigned" if opts.Build { - answer += "\n build \"true\" asks for nothing here: the mesh does not know where a module of that name is" + answer += "\n build \"true\" asks for nothing here: the controller has no build request saying where a " + + "module of that name is" } return "", fmt.Errorf("%w%s", inventory.ErrNoSuchModule, answer) } -// keepPending records the assignment as pending on a build request, and says so. -func keepPending(ctx context.Context, open *stores, node, module string, r inventory.RequestOutcome, lead string) (string, error) { +// keepPending records the assignment as pending on a build request — or, where one is already open, makes it +// wait for that build — and says so. +func keepPending(ctx context.Context, open *stores, node, module string, r inventory.RequestOutcome, + waiting *inventory.PendingAssignment, lead string) (string, error) { inv := open.inventory - p, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: node, Module: module, Build: r.ID, + if waiting != nil { + if waiting.Build == r.ID { + return lead + fmt.Sprintf("\n %s already waits for %s on this build: nothing changed", node, module), nil + } + moved, err := inv.RepointPending(ctx, waiting.ID, r.ID, r.Repository, r.Path) + if err != nil { + return "", err + } + if !moved { + return lead + fmt.Sprintf("\n the pending assignment of %s to %s ended while this was asked: `status` "+ + "says how", module, node), nil + } + return lead + fmt.Sprintf("\n the pending assignment of %s to %s, which waited for build %s, now waits "+ + "for build %s", module, node, waiting.Build, r.ID), nil + } + _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: node, Module: module, Build: r.ID, Repository: r.Repository, Path: r.Path}) if errors.Is(err, inventory.ErrAlreadyPending) { return lead + fmt.Sprintf("\n %s already waits for %s: nothing changed", node, module), nil @@ -168,51 +239,63 @@ func keepPending(ctx context.Context, open *stores, node, module string, r inven if err != nil { return "", err } - answer := lead + fmt.Sprintf("\n the assignment is kept as pending: %s is assigned %s when the build "+ - "registers it, and `status` says it until then. If the build fails it expires and says why; "+ - "`unassign %s %s` withdraws it", node, module, node, module) - // The build may have registered the module between the look and the record: made at once, then. - if shelf, err := inv.Catalogue(ctx); err == nil { - if _, now := shelf[module]; now { - if line := applyPending(ctx, open, p, r.ID); line != "" { - answer += "\n " + line - } - } - } - return answer, nil + // The controller makes it, not this act (ADR 0261): when the build registers the module, or at its next + // tick if the module was registered between the look and the record. + return lead + fmt.Sprintf("\n the assignment is kept as pending: the controller assigns %s to %s when the "+ + "build registers it, and `status` lists it until then. If the build fails it expires, says why and "+ + "raises a condition; `unassign %s %s` withdraws it", module, node, node, module), nil } -// inFlight is the newest build request for the module with no outcome yet, asked within the bound. +// inFlight is the newest build request for the module still to register it: no outcome yet within the +// bound, or a successful outcome heard a moment ago and being registered. func inFlight(requests []inventory.RequestOutcome, now time.Time) (inventory.RequestOutcome, bool) { for _, r := range requests { - if r.NotAsked == "" && !r.Heard && now.Sub(r.At) < buildRequestBound { + if stillComing(r, now) { return r, true } } return inventory.RequestOutcome{}, false } -// whyNotBuilt says what came of a module's last build request. +func stillComing(r inventory.RequestOutcome, now time.Time) bool { + if r.Heard { + return r.Failed == "" && now.Sub(r.HeardAt) < registerGrace + } + return r.NotAsked == "" && now.Sub(r.At) < buildRequestBound +} + +// whyNotBuilt says what came of a module's last build request. What was heard comes first: an outcome is the +// last word whatever its asker said. func whyNotBuilt(r inventory.RequestOutcome, now time.Time) string { where := fmt.Sprintf("%s in %s", orRoot(r.Path), r.Repository) switch { - case r.NotAsked != "": - return fmt.Sprintf("%s; the merge that added it (%s) could not ask for its build: %s", where, - short(r.Commit), firstLine(r.NotAsked)) - case !r.Heard: - return fmt.Sprintf("%s; build %s was asked at %s, and no outcome has been heard in %s", where, r.ID, - clock(r.At), now.Sub(r.At).Round(time.Minute)) - case r.Failed != "": + case r.Heard && r.Failed != "": return fmt.Sprintf("%s; its last build, %s at %s, failed: %s", where, r.ID, clock(r.HeardAt), firstLine(r.Failed)) - case r.Module != "" && r.Module != r.Name(): - return fmt.Sprintf("%s; its last build, %s, registered it as %s", where, r.ID, r.Module) - default: + case r.Heard && r.Module != "" && r.Module != r.Name(): + return fmt.Sprintf("%s; its last build, %s, built it as %s", where, r.ID, r.Module) + case r.Heard: return fmt.Sprintf("%s; its last build, %s at %s, was recorded and not registered — `builds` says why", where, r.ID, clock(r.HeardAt)) + case r.NotAsked != "" && r.For == "merge": + return fmt.Sprintf("%s; the merge that added it (%s) could not ask for its build: %s", where, + short(r.Commit), firstLine(r.NotAsked)) + case r.NotAsked != "": + return fmt.Sprintf("%s; build %s, asked by %s, was not handed over or not waited for: %s", where, r.ID, + askerOr(r.For), firstLine(r.NotAsked)) + default: + return fmt.Sprintf("%s; build %s was asked at %s, and no outcome has been heard in %s", where, r.ID, + clock(r.At), now.Sub(r.At).Round(time.Minute)) } } +func askerOr(who string) string { + if who == "" { + return "the controller" + } + return who +} + // requestSource is a build request's source as a person reads it: repository@commit (directory). func requestSource(r inventory.RequestOutcome) string { at := short(r.Commit) @@ -228,8 +311,8 @@ func requestSource(r inventory.RequestOutcome) string { // clock is a moment as a person reads it, in the controller's local time. func clock(t time.Time) string { return t.Local().Format("2006-01-02 15:04:05 MST") } -// closestNames is up to three names the mesh holds that are near the one asked: registered modules and -// directories it asked to build. +// closestNames is up to three names near the one asked: registered modules and directories the controller +// asked to build. func closestNames(ctx context.Context, inv *inventory.Inventory, name string) []string { var candidates []string if shelf, err := inv.Catalogue(ctx); err == nil { @@ -292,26 +375,41 @@ func editDistance(a, b string) int { return prev[len(rb)] } -// applyPending makes a pending assignment now that its module is registered, and settles it: applied, or -// refused with the refusal. Returns what it said, or nothing when another process settled it first. +// applyPending makes a pending assignment now that its module is registered (ADR 0261). Under the machine's +// hold it claims the row (waiting → applying), so a withdrawal cannot land between the look and the act, +// then assigns by the same act a person's assign is, then says what came of it: applied, or refused with the +// refusal. Returns what it said, or nothing when the row no longer waited. func applyPending(ctx context.Context, open *stores, p inventory.PendingAssignment, by string) string { inv := open.inventory + ctx, release, err := holdNodes(ctx, open, []string{p.Node}) + if err != nil { + // Left waiting: the next tick tries again. + return fmt.Sprintf("the pending assignment of %s to %s waits: %s could not be held: %v", p.Module, p.Node, + p.Node, err) + } + defer release() + claimed, err := inv.ClaimPending(ctx, p.ID) + if err != nil { + return fmt.Sprintf("the pending assignment of %s to %s could not be taken for making: %v", p.Module, p.Node, err) + } + if !claimed { + return "" + } answer, err := assign(ctx, open, p.Node, p.Module) state, note := inventory.PendingApplied, "" - if err != nil { + switch { + case err != nil: state = inventory.PendingRefused note = fmt.Sprintf("the pending assignment of %s to %s was refused when build %s registered it: %s", p.Module, p.Node, by, firstLine(err.Error())) - } else { - // Not promised as sent: a walk sends it if the module's upgrade announcement finds it - // assigned, and nothing does if the announcement came first. `status` says which, as for any machine. - note = fmt.Sprintf("%s is assigned %s: build %s registered it (pending since %s); `status` says whether "+ - "%s has been sent it, and `push %s` sends it if not", p.Node, p.Module, by, clock(p.Since), p.Node, p.Node) - if strings.Contains(answer, "already runs") { - note = fmt.Sprintf("%s already runs %s: the pending assignment is met", p.Node, p.Module) - } + case strings.Contains(answer, "already runs"): + note = fmt.Sprintf("%s already runs %s: the pending assignment is met", p.Node, p.Module) + default: + // The same as a person's assign: nothing is sent by this act. + note = fmt.Sprintf("%s is assigned %s: build %s registered it (pending since %s); `push %s` sends it", + p.Node, p.Module, by, clock(p.Since), p.Node) } - settled, serr := inv.SettlePending(ctx, p.ID, state, note) + settled, serr := inv.SettlePending(ctx, p.ID, inventory.PendingApplying, state, note) if serr != nil { return fmt.Sprintf("%s — and it could not be recorded as settled: %v", note, serr) } @@ -321,11 +419,11 @@ func applyPending(ctx context.Context, open *stores, p inventory.PendingAssignme return note } -// expirePending ends a pending assignment whose build will not register its module, with why. +// expirePending ends a waiting pending assignment whose build will not register its module, with why. func expirePending(ctx context.Context, inv *inventory.Inventory, p inventory.PendingAssignment, why string) string { note := fmt.Sprintf("the pending assignment of %s to %s expired: %s; nothing was assigned. Assign it again with "+ "build \"true\" to ask for the build again", p.Module, p.Node, why) - settled, err := inv.SettlePending(ctx, p.ID, inventory.PendingExpired, note) + settled, err := inv.SettlePending(ctx, p.ID, inventory.PendingWaiting, inventory.PendingExpired, note) if err != nil { return fmt.Sprintf("%s — and it could not be recorded: %v", note, err) } @@ -369,21 +467,62 @@ func settlePendingOnBuild(ctx context.Context, open *stores, result link.BuildRe return said } -// settleStalePending settles what a missed outcome left waiting: a module registered meanwhile is assigned, -// a build heard and not registered ends its assignment, and one with no outcome past the bound expires. -// Asked where the mesh is read for status, so a pending assignment never waits silently for ever. -func settleStalePending(ctx context.Context, open *stores, now time.Time) []string { - inv := open.inventory - waiting, err := inv.Pending(ctx, time.Time{}) - if err != nil { - // Said, not swallowed; status reads the pending assignments again and says the same failure there. - return []string{fmt.Sprintf("the pending assignments could not be read to settle them: %v", err)} +// settlingPending is the controller's tick over the pending assignments (ADR 0261), every settleEvery until +// the context ends. What it does is printed to the controller's log, kept in each row's note (which `status` +// lists for a day), and raised as a condition for an assignment that was not made. +func settlingPending(ctx context.Context, open *stores) { + for { + for _, line := range settlePending(ctx, open, time.Now()) { + fmt.Println(line) + } + select { + case <-ctx.Done(): + return + case <-time.After(settleEvery): + } } +} + +// settlePending is one tick: claims left by a controller that stopped are settled from what the mesh holds; +// a module registered meanwhile is assigned; a build heard and not registered, or silent past its bound, +// ends its assignment; ended ones are raised as conditions and cleared once answered; ended rows older than +// KeptFor are deleted. Returns what it did. +func settlePending(ctx context.Context, open *stores, now time.Time) []string { + inv := open.inventory + var said []string + say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) } + shelf, err := inv.Catalogue(ctx) if err != nil { - return []string{fmt.Sprintf("the catalogue could not be read to settle the pending assignments: %v", err)} + say("the pending assignments are not settled this time: the catalogue could not be read: %v", err) + return said + } + stuck, err := inv.Stuck(ctx, now.Add(-claimStaleAfter)) + if err != nil { + say("claims left by a stopped controller could not be read: %v", err) + } + for _, p := range stuck { + assigned, err := inv.Assigned(ctx, p.Node) + if err != nil { + say("the claim on %s for %s could not be settled: %v", p.Node, p.Module, err) + continue + } + if containsString(assigned, p.Module) { + if _, err := inv.SettlePending(ctx, p.ID, inventory.PendingApplying, inventory.PendingApplied, + fmt.Sprintf("%s is assigned %s (settled after the controller that made it stopped)", p.Node, p.Module)); err != nil { + say("the claim on %s for %s could not be settled: %v", p.Node, p.Module, err) + } + continue + } + if err := inv.ReleaseClaim(ctx, p.ID); err != nil { + say("the claim on %s for %s could not be released: %v", p.Node, p.Module, err) + } + } + + waiting, err := inv.Pending(ctx, time.Time{}) + if err != nil { + say("the pending assignments could not be read to settle them: %v", err) } - var said []string for _, p := range waiting { if _, ok := shelf[p.Module]; ok { if line := applyPending(ctx, open, p, p.Build); line != "" { @@ -393,7 +532,7 @@ func settleStalePending(ctx context.Context, open *stores, now time.Time) []stri } requests, err := inv.RequestsNamed(ctx, p.Module) if err != nil { - said = append(said, fmt.Sprintf("the build requests for %s could not be read: %v", p.Module, err)) + say("the build requests for %s could not be read: %v", p.Module, err) continue } var r *inventory.RequestOutcome @@ -406,9 +545,9 @@ func settleStalePending(ctx context.Context, open *stores, now time.Time) []stri switch { case r == nil && now.Sub(p.Since) > buildRequestBound: why = fmt.Sprintf("build %s is no longer on record", p.Build) - case r != nil && r.Heard: + case r != nil && !stillComing(*r, now) && (r.Heard || r.NotAsked != ""): why = whyNotBuilt(*r, now) - case r != nil && now.Sub(r.At) > buildRequestBound: + case r != nil && !stillComing(*r, now): why = fmt.Sprintf("no outcome of build %s was heard within %s of asking", p.Build, buildRequestBound) } if why == "" { @@ -418,25 +557,139 @@ func settleStalePending(ctx context.Context, open *stores, now time.Time) []stri said = append(said, line) } } + + said = append(said, raisePendingEnded(ctx, inv)...) + if n, err := inv.ForgetEndedPending(ctx, now); err != nil { + say("ended pending assignments could not be deleted: %v", err) + } else if n > 0 { + say("deleted %d pending assignment(s) ended more than %s ago", n, inventory.KeptFor) + } return said } -// withdrawPending is `unassign` of a module only pending on a machine: the pending assignment withdrawn. -func withdrawPending(ctx context.Context, inv *inventory.Inventory, node, module string) (string, bool, error) { - waiting, err := inv.Pending(ctx, time.Time{}) +func containsString(xs []string, x string) bool { + for _, y := range xs { + if y == x { + return true + } + } + return false +} + +// pendingObservation is the condition for a pending assignment that was not made. +func pendingObservation(p inventory.PendingAssignment) conditions.Observation { + return conditions.Observation{Scope: conditions.ScopeMachine, ID: p.Node + "." + p.Module, + Token: kindPendingEnded, Kind: kindPendingEnded, Machine: p.Node, Severity: conditions.Warning, + Resolver: conditions.ResolverOperator, Source: "pending assignments", + Summary: p.Note} +} + +// raisePendingEnded raises a condition for each pending assignment that expired or was refused, once, and +// clears it when it is answered: the module was assigned to that machine since, a newer pending assignment of +// it is open, or a person took it back with `unassign`. Where this process keeps no conditions, nothing is +// stamped, and the serving controller's tick raises it. +func raisePendingEnded(ctx context.Context, inv *inventory.Inventory) []string { + keeper := conditionsFrom + if keeper == nil { + return nil + } + var said []string + toRaise, err := inv.ToRaise(ctx) + if err != nil { + return []string{fmt.Sprintf("pending assignments not made could not be read to raise them: %v", err)} + } + for _, p := range toRaise { + if _, err := keeper.Observe(ctx, pendingObservation(p)); err != nil { + said = append(said, fmt.Sprintf("the condition for %s on %s could not be raised: %v", p.Module, p.Node, err)) + continue + } + if err := inv.MarkPending(ctx, p.ID, "raised"); err != nil { + said = append(said, fmt.Sprintf("the condition for %s on %s was raised and not recorded: %v", p.Module, p.Node, err)) + } + } + toClear, err := inv.ToClear(ctx) + if err != nil { + return append(said, fmt.Sprintf("pending assignments raised could not be read to clear them: %v", err)) + } + for _, p := range toClear { + why, answered, err := pendingAnswered(ctx, inv, p) + if err != nil { + said = append(said, fmt.Sprintf("whether %s on %s is answered could not be read: %v", p.Module, p.Node, err)) + continue + } + if !answered { + continue + } + if _, err := keeper.Clear(ctx, pendingObservation(p).Key(), why); err != nil { + said = append(said, fmt.Sprintf("the condition for %s on %s could not be cleared: %v", p.Module, p.Node, err)) + continue + } + if err := inv.MarkPending(ctx, p.ID, "cleared"); err != nil { + said = append(said, fmt.Sprintf("the condition for %s on %s was cleared and not recorded: %v", p.Module, p.Node, err)) + } + } + return said +} + +// pendingAnswered says whether a pending assignment that was not made has been answered since, and how. +func pendingAnswered(ctx context.Context, inv *inventory.Inventory, p inventory.PendingAssignment) (string, bool, error) { + if p.Acknowledged != nil { + return "taken back with unassign", true, nil + } + assigned, err := inv.Assigned(ctx, p.Node) if err != nil { return "", false, err } - for _, p := range waiting { - if p.Node != node || p.Module != module { - continue + if containsString(assigned, p.Module) { + return p.Module + " is assigned to " + p.Node + " since", true, nil + } + rows, err := inv.PendingFor(ctx, p.Node, p.Module) + if err != nil { + return "", false, err + } + for _, r := range rows { + if r.ID != p.ID && r.Since.After(p.Since) { + return "assigned again, pending on another build", true, nil + } + } + return "", false, nil +} + +// withdrawPending is `unassign` of a module only pending on a machine, under the machine's hold: a waiting +// pending assignment is withdrawn; one being made now is refused, never overridden; one that expired or was +// refused is taken back, which answers its condition. False when there is none. +func withdrawPending(ctx context.Context, inv *inventory.Inventory, node, module string) (string, bool, error) { + rows, err := inv.PendingFor(ctx, node, module) + if err != nil { + return "", false, err + } + for _, p := range rows { + switch p.State { + case inventory.PendingApplying: + return "", true, fmt.Errorf("%s is being assigned %s now, as build %s registered it: nothing was "+ + "withdrawn; `unassign %s %s` again once it is assigned takes it off", node, module, p.Build, node, module) + case inventory.PendingWaiting: + note := fmt.Sprintf("the pending assignment of %s to %s is withdrawn; build %s goes on, and registers "+ + "%s assigned nowhere", module, node, p.Build, module) + settled, err := inv.SettlePending(ctx, p.ID, inventory.PendingWaiting, inventory.PendingWithdrawn, note) + if err != nil { + return "", false, err + } + if !settled { + return "", true, fmt.Errorf("the pending assignment of %s to %s changed while it was withdrawn: "+ + "`status` says how; nothing was withdrawn", module, node) + } + return note, true, nil + case inventory.PendingExpired, inventory.PendingRefused: + if p.Cleared != nil || p.Acknowledged != nil { + continue + } + if err := inv.MarkPending(ctx, p.ID, "acknowledged"); err != nil { + return "", false, err + } + return fmt.Sprintf("the pending assignment of %s to %s, which %s, is taken back; its condition clears", + module, node, p.State), true, nil } - note := fmt.Sprintf("the pending assignment of %s to %s is withdrawn; build %s goes on, and registers %s "+ - "assigned nowhere", module, node, p.Build, module) - if _, err := inv.SettlePending(ctx, p.ID, inventory.PendingWithdrawn, note); err != nil { - return "", false, err - } - return note, true, nil } return "", false, nil } diff --git a/cmd/mesh-controller/pending_test.go b/cmd/mesh-controller/pending_test.go index dd3c9d8d..acaf93ae 100644 --- a/cmd/mesh-controller/pending_test.go +++ b/cmd/mesh-controller/pending_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "errors" "slices" @@ -9,6 +10,7 @@ import ( "time" "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -68,21 +70,27 @@ func pendingOf(t *testing.T, open *stores) []inventory.PendingAssignment { return got } -// A build in flight: the merge asked for it, the assignment is kept pending and said so with the build, and -// the build's outcome makes it. Status says it while it waits. +// A build in flight: the merge asked for it, the request is kept as the merge's, the assignment is kept +// pending and said so with the build, status reads it without settling anything, and the build's outcome +// makes it — saying that a push sends it, and nothing more. func TestAnAssignmentWaitsForTheBuildInFlight(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() asksWithPaths(t) mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + requests, err := open.inventory.RequestsNamed(ctx, "sensors") + if err != nil || len(requests) != 1 || requests[0].For != "merge" || requests[0].Commit != "3da80a4b00aa" { + t.Fatalf("the merge's build request: %+v %v", requests, err) + } + said, err := assign(ctx, open, "laptop", "sensors") if err != nil { t.Fatalf("an assignment while its build runs was refused: %v", err) } t.Logf("assign laptop sensors, while its build runs:\n%s", said) for _, want := range []string{"being built from novox/mesh-catalog@3da80a4b (modules/sensors)", - "build b-modules/sensors", "kept as pending", "`unassign laptop sensors`"} { + "build b-modules/sensors", "kept as pending", "the controller assigns sensors to laptop", "`unassign laptop sensors`"} { if !strings.Contains(said, want) { t.Fatalf("the answer does not say %q:\n%s", want, said) } @@ -94,7 +102,6 @@ func TestAnAssignmentWaitsForTheBuildInFlight(t *testing.T) { if len(pending) != 1 || pending[0].State != inventory.PendingWaiting || pending[0].Build != "b-modules/sensors" { t.Fatalf("pending: %+v", pending) } - // Asked twice, it is one pending assignment. if again, err := assign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(again, "already waits") { t.Fatalf("a second assignment: %q %v", again, err) } @@ -127,12 +134,41 @@ func TestAnAssignmentWaitsForTheBuildInFlight(t *testing.T) { } pending = pendingOf(t, open) if len(pending) != 1 || pending[0].State != inventory.PendingApplied || - !strings.Contains(pending[0].Note, "`push laptop` sends it if not") { + !strings.HasSuffix(pending[0].Note, "`push laptop` sends it") { t.Fatalf("pending after the build: %+v", pending) } } -// The build of a pending assignment fails: it expires, saying the build's words, and nothing is assigned. +// **A read settles nothing** (review of #150, point 6): status — and so the board, the summary and the +// probe, which compose from the same reading — leaves a pending assignment whose module is registered as it +// is; the controller's tick makes it. +func TestAStatusReadSettlesNothing(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + asksWithPaths(t) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) + for range 2 { + if _, err := theThreeQuestions(ctx, open); err != nil { + t.Fatal(err) + } + } + if slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingWaiting { + t.Fatalf("a status read settled a pending assignment: %+v", pendingOf(t, open)) + } + said := settlePending(ctx, open, time.Now()) + if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingApplied { + t.Fatalf("the tick did not make it: %v %+v", said, pendingOf(t, open)) + } +} + +// The build of a pending assignment fails: it expires with the build's words and nothing is assigned; the +// tick raises it as a condition once; it then reads as known and not built; and `unassign` takes it back, +// after which the tick clears the condition. Status is not called well while the condition is open, and is +// again once it clears — no fixed day of "not well" (review point 6). func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() @@ -158,15 +194,17 @@ func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) { if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { t.Fatal("a failed build's module was assigned") } - asked, err := theThreeQuestions(ctx, open) - if err != nil { - t.Fatal(err) + + settlePending(ctx, open, time.Now()) + key := pendingObservation(pending[0]).Key() + c, found, err := conditionsFrom.Get(ctx, key) + if err != nil || !found || c.Kind != kindPendingEnded { + t.Fatalf("no condition %s for the assignment not made: %+v %v %v", key, c, found, err) } - if asked.well() { - t.Fatal("status called the mesh well on the day a pending assignment expired") + if asked, err := theThreeQuestions(ctx, open); err != nil || asked.well() || len(asked.conditions) == 0 { + t.Fatalf("status does not hold the open condition: %v", err) } - // Known and not built now: said, and refused without build. _, err = assign(ctx, open, "laptop", "sensors") if !errors.Is(err, inventory.ErrNoSuchModule) { t.Fatalf("a module whose build failed: %v", err) @@ -177,10 +215,54 @@ func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) { t.Fatalf("the refusal does not say %q:\n%v", want, err) } } + + said, err := unassign(ctx, open, "laptop", "sensors") + if err != nil || !strings.Contains(said, "taken back") { + t.Fatalf("unassign of an expired pending assignment: %q %v", said, err) + } + settlePending(ctx, open, time.Now()) + if _, found, _ := conditionsFrom.Get(ctx, key); found { + t.Fatal("the condition stayed open after the assignment was taken back") + } + // Nothing of it keeps status from being well any more: no open pending assignment, no open condition of it. + asked, err := theThreeQuestions(ctx, open) + if err != nil || pendingOpen(asked.pending) { + t.Fatalf("status still counts the pending assignment: %+v %v", asked.pending, err) + } + for _, c := range asked.conditions { + if c.Kind == kindPendingEnded { + t.Fatalf("status still holds the condition: %+v", c) + } + } } -// Known and not built, asked with build: the build is asked from where the last one was, and the assignment -// kept pending on the new build. +// The condition also clears when the module is later assigned to the machine. +func TestTheConditionClearsWhenTheModuleIsAssignedLater(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + asksWithPaths(t) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", + "3da80a4b00aa", "", "boom")); err != nil { + t.Fatal(err) + } + settlePending(ctx, open, time.Now()) + key := pendingObservation(pendingOf(t, open)[0]).Key() + register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + settlePending(ctx, open, time.Now()) + if _, found, _ := conditionsFrom.Get(ctx, key); found { + t.Fatal("the condition stayed open after the module was assigned") + } +} + +// Known and not built, asked with build: the build is asked from where the last one was, the request kept as +// assign's, and the assignment kept pending on the new build. func TestAssignWithBuildAsksForAKnownModule(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() @@ -211,10 +293,146 @@ func TestAssignWithBuildAsksForAKnownModule(t *testing.T) { if len(pending) != 1 || pending[0].Build != "b-modules/sensors" { t.Fatalf("pending: %+v", pending) } + requests, _ := inv.RequestsNamed(ctx, "sensors") + if len(requests) != 2 || requests[0].ID != "b-modules/sensors" || requests[0].For != "assign" { + t.Fatalf("the request is not kept as assign's: %+v", requests) + } } -// Unknown: refused as before, as no module of that name, with the closest names the mesh holds; build asks -// for nothing, since the mesh does not know where it is. +// **build "true" with a pending assignment already waiting** (review point 5): the waiting one is made to +// wait for the new build, and no build is asked that nothing waits on. +func TestAssignWithBuildRepointsTheWaitingAssignment(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-old", Repository: "novox/mesh-catalog", + Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge", At: time.Now().Add(-time.Hour)}); err != nil { + t.Fatal(err) + } + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-old", Repository: "novox/mesh-catalog", Ref: "main", + Path: "modules/sensors", On: "anchor", Failed: "boom"}); err != nil { + t.Fatal(err) + } + // Waiting on the failed build, its expiry not yet settled. + if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "laptop", Module: "sensors", + Build: "build-old", Repository: "novox/mesh-catalog", Path: "modules/sensors"}); err != nil { + t.Fatal(err) + } + _, err := assign(ctx, open, "laptop", "sensors") + if err == nil || !strings.Contains(err.Error(), "already has a pending assignment of sensors, waiting for build build-old") { + t.Fatalf("known and not built with a pending assignment waiting: %v", err) + } + asked := asksWithPaths(t) + said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") + if err != nil || !strings.Contains(said, "which waited for build build-old, now waits for build b-modules/sensors") { + t.Fatalf("assign with build: %q %v", said, err) + } + if len(*asked) != 1 { + t.Fatalf("asked %v", *asked) + } + pending := pendingOf(t, open) + if len(pending) != 1 || pending[0].Build != "b-modules/sensors" || pending[0].State != inventory.PendingWaiting { + t.Fatalf("pending: %+v", pending) + } + // The new build in flight now: build "true" again asks nothing. + if said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors"); err != nil || + !strings.Contains(said, "no second build was asked") || len(*asked) != 1 { + t.Fatalf("a second build true while the build runs: %q %v, asked %v", said, err, *asked) + } +} + +// **A failed ask never reads as in flight** (review point 2): a merge whose ask could not be made keeps the +// request as not asked, and assign says the merge could not ask; a request whose asker could not hand it over +// or stopped waiting reads the same, unless its outcome was heard. +func TestAFailedAskIsNotABuildInFlight(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + was := askABuild + askABuild = func(context.Context, buildSource, string, string) (string, error) { + return "", errors.New("cannot submit a build: nats: timeout") + } + t.Cleanup(func() { askABuild = was }) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + _, err := assign(ctx, open, "laptop", "sensors") + if err == nil || !strings.Contains(err.Error(), "the merge that added it (3da80a4b) could not ask for its build: cannot submit") { + t.Fatalf("a merge that could not ask: %v", err) + } + if len(pendingOf(t, open)) != 0 { + t.Fatal("a pending assignment waits on a build never asked") + } + + if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-waited", Repository: "novox/mesh-catalog", + Seat: "git", Path: "modules/gauges", Ref: "main", For: "build"}); err != nil { + t.Fatal(err) + } + if err := inv.MarkNotAsked(ctx, "build-waited", "cannot submit a build: no responders"); err != nil { + t.Fatal(err) + } + _, err = assign(ctx, open, "laptop", "gauges") + if err == nil || !strings.Contains(err.Error(), "build build-waited, asked by build, was not handed over or not waited for") { + t.Fatalf("a build not handed over: %v", err) + } + // Heard first, the outcome stands: marking it afterwards changes nothing. + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-waited", Repository: "novox/mesh-catalog", Ref: "main", + Path: "modules/gauges", On: "anchor", Failed: "the real failure"}); err != nil { + t.Fatal(err) + } + _, err = assign(ctx, open, "laptop", "gauges") + if err == nil || !strings.Contains(err.Error(), "failed: the real failure") { + t.Fatalf("an outcome heard after a failed wait: %v", err) + } +} + +// A plan's tier keeps its requests as the plan's (review point 3). +func TestAPlansBuildRequestIsThePlans(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + asksWithPaths(t) + if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main", + BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa", + Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true} + if err := (following{open: open}).SourceMoved(ctx, m); err != nil { + t.Fatal(err) + } + requests, err := open.inventory.RequestsNamed(ctx, "app") + if err != nil || len(requests) != 1 || requests[0].For != "plan" { + t.Fatalf("the plan's request: %+v %v", requests, err) + } +} + +// **A build heard as built and not registered yet is in flight** (review point 4): the outcome is recorded +// before the module is registered; for that moment assign keeps the assignment pending. Past the grace it is +// recorded and not registered, and the tick ends the pending assignment with that. +func TestABuildBeingRegisteredIsInFlight(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-done", Repository: "novox/mesh-catalog", + Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge"}); err != nil { + t.Fatal(err) + } + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-done", Repository: "novox/mesh-catalog", Ref: "main", + Path: "modules/sensors", Module: "sensors", On: "anchor"}); err != nil { + t.Fatal(err) + } + said, err := assign(ctx, open, "laptop", "sensors") + if err != nil || !strings.Contains(said, "built and being registered") || !strings.Contains(said, "kept as pending") { + t.Fatalf("a build being registered: %q %v", said, err) + } + settlePending(ctx, open, time.Now().Add(registerGrace+time.Minute)) + p := pendingOf(t, open) + if len(p) != 1 || p[0].State != inventory.PendingExpired || !strings.Contains(p[0].Note, "recorded and not registered") { + t.Fatalf("past the grace: %+v", p) + } +} + +// Unknown: refused as no module of that name, claiming only what was looked at, with the closest names; build +// asks for nothing. func TestAnUnknownModuleIsRefusedWithTheClosestNames(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() @@ -224,12 +442,16 @@ func TestAnUnknownModuleIsRefusedWithTheClosestNames(t *testing.T) { if !errors.Is(err, inventory.ErrNoSuchModule) { t.Fatalf("an unknown module: %v", err) } - for _, want := range []string{"no module of that name: sensors", "no module, build request or merge by that name", - "the closest it holds: sensord", "asks for nothing here"} { + for _, want := range []string{"no module of that name: sensors", "the catalogue holds no module of that name", + "no build request the controller kept (the last 30 days)", "the closest names it holds: sensord", + "asks for nothing here"} { if !strings.Contains(err.Error(), want) { t.Fatalf("the refusal does not say %q:\n%v", want, err) } } + if strings.Contains(err.Error(), "merge") { + t.Fatalf("the refusal claims a merge it never looked at: %v", err) + } if len(*asked) != 0 || len(pendingOf(t, open)) != 0 { t.Fatalf("an unknown module asked %v, pending %+v", *asked, pendingOf(t, open)) } @@ -252,7 +474,8 @@ func TestAnActWithAModuleNotRegisteredIsRefusedWhole(t *testing.T) { } } -// unassign withdraws a pending assignment; the build goes on and registers the module assigned nowhere. +// unassign withdraws a waiting pending assignment; the build goes on and registers the module assigned +// nowhere: a withdrawn row is never claimed. func TestUnassignWithdrawsAPendingAssignment(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() @@ -269,13 +492,80 @@ func TestUnassignWithdrawsAPendingAssignment(t *testing.T) { "3da80a4b00aa", "sensors", "")); err != nil { t.Fatal(err) } + settlePending(ctx, open, time.Now()) if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { t.Fatal("a withdrawn assignment was made") } } -// A build that says nothing within the bound: the pending assignment expires at the next look, saying so, -// rather than waiting for ever; and a build heard by another process meanwhile is made at the next look. +// **A claim is never overridden** (review point 1): a pending assignment being made is claimed +// (waiting → applying); an unassign meanwhile is refused and says so, and leaves the claim; making a row +// already withdrawn makes nothing. A claim left by a controller that stopped is settled by the tick from what +// the mesh holds: back to waiting when the module is not assigned, applied when it is. +func TestAWithdrawalNeverOverridesAClaim(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + asksWithPaths(t) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + p := pendingOf(t, open)[0] + if ok, err := inv.ClaimPending(ctx, p.ID); err != nil || !ok { + t.Fatalf("claim: %v %v", ok, err) + } + _, err := unassign(ctx, open, "laptop", "sensors") + if err == nil || !strings.Contains(err.Error(), "is being assigned sensors now") { + t.Fatalf("unassign of a claimed pending assignment: %v", err) + } + if got := pendingOf(t, open)[0]; got.State != inventory.PendingApplying { + t.Fatalf("the claim was overridden: %+v", got) + } + // Pending(zero) is the waiting ones alone (review point 7). + if waiting, err := inv.Pending(ctx, time.Time{}); err != nil || len(waiting) != 0 { + t.Fatalf("Pending(zero) read a claimed row: %+v %v", waiting, err) + } + + // Left by a controller that stopped: back to waiting, since sensors is not assigned. + settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute)) + if got := pendingOf(t, open)[0]; got.State != inventory.PendingWaiting { + t.Fatalf("a stale claim was not released: %+v", got) + } + + // Withdrawn, then the build registers it: applyPending finds nothing to claim. + if _, err := unassign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) + if line := applyPending(ctx, open, p, "b-modules/sensors"); line != "" { + t.Fatalf("a withdrawn pending assignment was made: %s", line) + } + if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { + t.Fatal("a withdrawn pending assignment was assigned") + } + + // A stale claim whose module was assigned is applied. + q, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "sensors", Build: "b-x"}) + if err != nil { + t.Fatal(err) + } + if ok, _ := inv.ClaimPending(ctx, q.ID); !ok { + t.Fatal("claim") + } + if _, err := inv.Assign(ctx, "anchor", "sensors"); err != nil { + t.Fatal(err) + } + settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute)) + rows, _ := inv.PendingFor(ctx, "anchor", "sensors") + if len(rows) != 1 || rows[0].State != inventory.PendingApplied { + t.Fatalf("a stale claim of an assigned module: %+v", rows) + } +} + +// A build that says nothing within the bound: the pending assignment expires at the next tick, saying so, +// rather than waiting for ever; and a module registered by a process that kept no pending assignment is +// assigned at the next tick. func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() @@ -295,15 +585,11 @@ func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) { t.Fatal(err) } } - // Heard by a process that kept no pending assignment: registered, and nothing made it. register(t, open, catalogue.Manifest{Module: "heard", Version: "1"}) - said := settleStalePending(ctx, open, time.Now().Add(buildRequestBound)) - if len(said) != 2 { - t.Fatalf("settled %v", said) - } + settlePending(ctx, open, time.Now().Add(buildRequestBound)) if !slices.Contains(assignedTo(t, open, "laptop"), "heard") { - t.Fatal("a module registered meanwhile was not assigned at the next look") + t.Fatal("a module registered meanwhile was not assigned at the next tick") } byModule := map[string]inventory.PendingAssignment{} for _, p := range pendingOf(t, open) { @@ -315,6 +601,21 @@ func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) { if p := byModule["heard"]; p.State != inventory.PendingApplied { t.Fatalf("a module registered meanwhile: %+v", p) } + + // Ended rows are deleted after KeptFor (review point 7); open ones never. + if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "lost", Build: "build-lost"}); err != nil { + t.Fatal(err) + } + said := settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) + rows, err := inv.Pending(ctx, time.Unix(0, 0)) + if err != nil { + t.Fatal(err) + } + for _, r := range rows { + if !r.Open() { + t.Fatalf("an ended pending assignment outlived %s: %+v (%v)", inventory.KeptFor, r, said) + } + } } // The seat's assign passes build on; unassign takes none. @@ -327,3 +628,16 @@ func TestTheSeatsAssignPassesBuild(t *testing.T) { t.Fatal("unassign took build") } } + +// The condition's words are plain. +func TestAnAssignmentNotMadeIsSaidPlainly(t *testing.T) { + o := pendingObservation(inventory.PendingAssignment{Node: "g14", Module: "sensors", + Note: "the pending assignment of sensors to g14 expired: build b-1 of modules/sensors failed: boom"}) + w := plainWordings[kindPendingEnded](o) + if why, ok := conditions.PlainWords(w, "g14"); !ok { + t.Fatalf("not plain: %s %+v", why, w) + } + if w.Headline != "sensors was not put on g14" { + t.Fatalf("headline %q", w.Headline) + } +} diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index 54befa7f..4a770d7e 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -153,6 +153,19 @@ var plainWordings = map[string]func(conditions.Observation) words{ "asleep is normal.", m), Resolved: m + " can be reached again"} }), + // A pending assignment that was not made (novox/hq issue 325, ADR 0261). + kindPendingEnded: worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + module := idPart(o, 1) + if module == "" { + module = "a module" + } + return words{Headline: module + " was not put on " + m, + Needs: "decide whether to build it again or take the assignment back; the details say why.", + Explanation: fmt.Sprintf("An assignment of %s to %s waited for its build, and the build did not "+ + "register it, so it was not made.", module, m), + Resolved: "Resolved: " + module + " on " + m + " is settled"} + }), kindBindingKept: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: "A module's data source is held on " + m, diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 5b7a934b..a84dfd56 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -165,6 +165,10 @@ func serve(ctx context.Context) (err error) { // Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for // machines to report moves when they have, and a plan left by a replaced controller resumes. go planTicker(ctx, open) + // And the pending assignments settled on a tick of their own (novox/hq ADR 0261): made once their module + // is registered, ended with why when its build will not register it, raised and cleared as conditions. + // Never by a read. + go settlingPending(ctx, open) // The durations the core's bounds are set from are kept a month (novox/hq to-be 45 Phase 0). go forgettingOldDurations(ctx, inv) // And what the catalogue decided a build meant. The builder's own result is already handled diff --git a/cmd/mesh-controller/queue.go b/cmd/mesh-controller/queue.go index 949ec9f6..fbe64894 100644 --- a/cmd/mesh-controller/queue.go +++ b/cmd/mesh-controller/queue.go @@ -406,6 +406,8 @@ func rebuildCommand(ctx context.Context, args []string) error { if err != nil { return err } + recordAsked(ctx, inventory.BuildRequest{ID: id, Repository: source.Repository, Seat: source.Seat, Path: path, + Ref: ref, For: "rebuild"}) fmt.Printf("rebuild asked as %s\n", id) return nil } @@ -490,7 +492,11 @@ func replayCommand(ctx context.Context, args []string) error { if err := replayRefusal(b, module, history, *register, *older, outstanding); err != nil { return err } - id, err := buildOneAsked(ctx, source, b.Path, b.Commit, 0, !*register) + asker := "" + if *register { + asker = "replay" + } + id, err := buildOneAsked(ctx, source, b.Path, b.Commit, 0, !*register, asker) if err != nil { return err } diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 29fd27d8..c68ea022 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -344,7 +344,8 @@ var askABuild func(ctx context.Context, source buildSource, path, ref string) (s func init() { askABuild = func(ctx context.Context, source buildSource, path, ref string) (string, error) { - return buildOneAsked(ctx, source, path, ref, 0, false) + // Kept by each asker with its own name once the ask is made (novox/hq issue 325). + return buildOneAsked(ctx, source, path, ref, 0, false, "") } } @@ -376,6 +377,8 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s p.Note = fmt.Sprintf("%s could not be asked for: %v", name, err) return } + recordAsked(ctx, inventory.BuildRequest{ID: id, Repository: e.Source.Repository, Seat: e.Source.Seat, + Path: e.Source.Path, Ref: followedBranch(e.Source.Ref), Commit: p.Commit, For: "plan"}) state.State = "asked" state.AskedAt = &now state.Build = id diff --git a/cmd/mesh-controller/status.go b/cmd/mesh-controller/status.go index 4e961db6..a9699cce 100644 --- a/cmd/mesh-controller/status.go +++ b/cmd/mesh-controller/status.go @@ -4,7 +4,6 @@ import ( "context" "flag" "fmt" - "os" "sort" "strings" "time" @@ -469,11 +468,8 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) { if out.conditions, err = openConditions(ctx); err != nil { out.conditionsUnread = err.Error() } - // And the assignments waiting for their module's build (novox/hq issue 325): what a missed outcome left - // waiting is settled first, so one never waits silently past its build. - for _, line := range settleStalePending(ctx, open, time.Now()) { - fmt.Fprintln(os.Stderr, line) - } + // And the assignments waiting for their module's build (novox/hq issue 325, ADR 0261), read only: the + // controller's tick settles them, never a read. if out.pending, err = inv.Pending(ctx, time.Now().Add(-pendingShownFor)); err != nil { out.pendingUnread = err.Error() err = nil @@ -610,13 +606,14 @@ func (a answers) well() bool { return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 && len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 && len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 && - len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingNotWell(a.pending) + len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending) } -// pendingNotWell is whether any pending assignment is waiting, or ended without being made. -func pendingNotWell(pending []inventory.PendingAssignment) bool { +// pendingOpen is whether any pending assignment is still to be made. One that ended without being made is +// not counted here: it is a condition, open until it is answered (ADR 0261). +func pendingOpen(pending []inventory.PendingAssignment) bool { for _, p := range pending { - if p.State != inventory.PendingApplied && p.State != inventory.PendingWithdrawn { + if p.Open() { return true } } @@ -632,7 +629,7 @@ func printPending(pending []inventory.PendingAssignment, unread string) { } var waiting, ended []inventory.PendingAssignment for _, p := range pending { - if p.State == inventory.PendingWaiting { + if p.Open() { waiting = append(waiting, p) } else { ended = append(ended, p) diff --git a/internal/catalogue/verbs.go b/internal/catalogue/verbs.go index 3ab55764..edb2c92b 100644 --- a/internal/catalogue/verbs.go +++ b/internal/catalogue/verbs.go @@ -179,10 +179,11 @@ var ControllerVerbs = []Verb{ }, []string{"node"}, "files", "diff")}, {Name: "assign", Description: "Put a module on a machine. Refused with the mesh's own words when it cannot resolve there, " + "or when a seat its resources are applied through is held by nothing on the machine (novox/hq ADR 0207). " + - "A module not registered yet is looked for (novox/hq issue 325): while its build runs — a merge that adds " + - "a module asks for one — the assignment is kept as pending and made when the build registers it, said in " + - "`status`, and expires with why if the build fails; a module known and not built is said, and build asks " + - "for its build; a name the mesh never heard of is refused with the closest names.", + "A module not registered yet is looked for (novox/hq issue 325, ADR 0261): while its build runs — a merge " + + "that adds a module asks for one — the assignment is kept as pending and the controller makes it when the " + + "build registers it; `status` lists it, and if the build fails it expires with why and raises a condition. " + + "A module known and not built is said, and build asks for its build; a name with no module and no build " + + "request kept is refused with the closest names.", Input: schema(map[string]string{"node": "the machine's name", "module": "the module's name; several comma-separated are judged together", "build": "\"true\": for a module known and not built (its last build failed, or was never asked), ask " + diff --git a/internal/inventory/migrations/0082-an-assignment-waits-for-its-build.sql b/internal/inventory/migrations/0082-an-assignment-waits-for-its-build.sql index cf03c449..09676837 100644 --- a/internal/inventory/migrations/0082-an-assignment-waits-for-its-build.sql +++ b/internal/inventory/migrations/0082-an-assignment-waits-for-its-build.sql @@ -1,15 +1,15 @@ --- An assignment waits for its module's build (novox/hq issue 325). +-- An assignment waits for its module's build (novox/hq issue 325, ADR 0261). -- -- A merge that adds a module asks for its build (issue 300), and the module is registered when the build's -- outcome is taken in, minutes later. An `assign` in between was answered "no module of that name", which -- read as "nobody registered it". The controller now keeps every build it asks for, so `assign` can tell a -- build in flight from a module it never heard of, and an assignment made while the build runs is kept as --- pending and made when the build registers the module, or ended with the reason when it does not. +-- pending and made by the controller when the build registers the module, or ended with the reason. --- Every build the controller asked for, whatever asked it: a merge, a plan's tier, a person's `build`, an --- `assign` with build. Its outcome is the `build` row of the same id, written when the outcome is taken in; --- an ask without one is still running, or was lost. not_asked is why a merge's ask of a new module could --- not be made, and the id is then not a build's. +-- Every build the controller asked for: a merge's new module, a plan's tier, a person's `build`, an `assign` +-- with build. Kept once the ask is made, never before. Its outcome is the `build` row of the same id; +-- an ask without one is still running, or was lost. not_asked is why the ask could not be made or waited +-- for; for a merge that could not ask, the id is the controller's and no build's. create table build_request ( id text primary key, repository text not null, @@ -23,20 +23,27 @@ create table build_request ( ); create index build_request_asked_at on build_request (asked_at); --- An assignment kept until its module is registered. waiting until the build registers the module, then --- applied (the assignment was made), refused (the assignment was refused then, with the refusal), expired --- (the build failed, was not registered, or said nothing within its bound) or withdrawn (`unassign`). +-- An assignment kept until its module is registered (ADR 0261). waiting until the build registers the +-- module; applying while the controller makes it, under the machine's hold; then applied, refused (the +-- assignment was refused then), expired (the build failed, was not registered, or said nothing within its +-- bound) or withdrawn (`unassign`). raised_at is when an expiry or refusal was raised as a condition, +-- acknowledged_at when a person took it back with `unassign`, cleared_at when its condition was cleared. +-- Gone with its machine. Ended rows are deleted after 30 days. create table pending_assignment ( - id bigserial primary key, - node text not null, - module text not null, - build_id text not null, - repository text not null default '', - source_path text not null default '', - since timestamptz not null default now(), - state text not null default 'waiting' - check (state in ('waiting', 'applied', 'refused', 'expired', 'withdrawn')), - note text not null default '', - settled_at timestamptz + id bigserial primary key, + node uuid not null references node (id) on delete cascade, + module text not null, + build_id text not null, + repository text not null default '', + source_path text not null default '', + since timestamptz not null default now(), + state text not null default 'waiting' + check (state in ('waiting', 'applying', 'applied', 'refused', 'expired', 'withdrawn')), + note text not null default '', + settled_at timestamptz, + raised_at timestamptz, + acknowledged_at timestamptz, + cleared_at timestamptz ); -create unique index pending_assignment_waiting on pending_assignment (node, module) where state = 'waiting'; +create unique index pending_assignment_open on pending_assignment (node, module) + where state in ('waiting', 'applying'); diff --git a/internal/inventory/pending.go b/internal/inventory/pending.go index 5e9dcb10..da22226c 100644 --- a/internal/inventory/pending.go +++ b/internal/inventory/pending.go @@ -10,36 +10,36 @@ import ( "github.com/jackc/pgx/v5" ) -// What the controller asked to be built, and the assignments waiting for it (novox/hq issue 325). +// What the controller asked to be built, and the assignments waiting for it (novox/hq issue 325, ADR 0261). // // A build's outcome was kept, and its asking was not: between a merge asking for a new module and the // outcome registering it, the mesh had no record that anything was coming, and `assign` answered "no module -// of that name" for a module minutes from existing. Kept here, an ask lets `assign` say which case it is in, -// and an assignment made while the build runs is kept until the build registers the module. +// of that name" for a module minutes from existing. Kept here, a build request lets `assign` say which case +// it is in, and an assignment made while the build runs is kept until the build registers the module. -// requestKeptFor is how long an ask is kept: long past any build's bound, short enough to stay small. -const requestKeptFor = 30 * 24 * time.Hour +// KeptFor is how long a build request, and an ended pending assignment, is kept. +const KeptFor = 30 * 24 * time.Hour // BuildRequest is one build the controller asked for. type BuildRequest struct { - // ID is the build's id, as its outcome will carry it; for an ask that could not be made, an id of - // the controller's making that no build carries. + // ID is the build's id, as its outcome will carry it; for a merge whose ask could not be made, an id + // of the controller's making that no build carries. ID string // Repository and Seat are the source as the mesh spells it (novox/hq ADR 0111): a path on the seat's // holder, or a URL with no seat. Path is the module's directory in it, Ref the branch or commit asked. Repository, Seat, Path, Ref string // Commit is the commit that made the ask, where one did: a merge's. Commit string - // For says what asked: "merge", "build", "plan", "assign". + // For says who asked: "merge", "plan", "build" (a person), "assign". For string - // NotAsked is why the ask could not be made, empty when it was. + // NotAsked is why the ask could not be made, or not waited for; empty when it was. NotAsked string At time.Time } -// Name is the module this ask is expected to register, read from its directory: the last element of its -// path, or of its repository for a module at the repository's root. A guess until the build says, and the -// one a person reads too: `modules/sensors` is sensors. +// Name is the module this request is expected to register, read from its directory: the last element of +// its path, or of its repository for a module at the repository's root. A guess until the build says, and +// the one a person reads too: `modules/sensors` is sensors. func (a BuildRequest) Name() string { if p := strings.Trim(a.Path, "/"); p != "" && p != "." { return path.Base(p) @@ -47,19 +47,19 @@ func (a BuildRequest) Name() string { return strings.TrimSuffix(path.Base(strings.TrimRight(a.Repository, "/")), ".git") } -// RequestOutcome is an ask beside what came of it, where anything did. +// RequestOutcome is a build request beside what came of it, where anything did. type RequestOutcome struct { BuildRequest // Heard is whether an outcome of this id was taken in; Failed is its builder's words when it failed, - // Module what it registered as, HeardAt when it was taken in. + // Module what the build said it built, HeardAt when it was taken in. Heard bool Failed string Module string HeardAt time.Time } -// RecordBuildRequest keeps one ask. Idempotent on the id: a merge's ask recorded by the merge and by the -// asking itself is one ask. Asks older than requestKeptFor are let go in the same act. +// RecordBuildRequest keeps one build request, once it is asked. Idempotent on the id. Requests older than +// KeptFor are deleted in the same act. func (i *Inventory) RecordBuildRequest(ctx context.Context, a BuildRequest) error { at := a.At if at.IsZero() { @@ -72,17 +72,25 @@ func (i *Inventory) RecordBuildRequest(ctx context.Context, a BuildRequest) erro if _, err := i.store.Pool().Exec(ctx, `insert into build_request (id, repository, seat, source_path, ref, commit_hash, asked_for, not_asked, asked_at) values ($1, $2, $3, $4, $5, $6, $7, $8, $9) - on conflict (id) do update set - commit_hash = case when build_request.commit_hash = '' then excluded.commit_hash else build_request.commit_hash end, - asked_for = case when build_request.asked_for = '' then excluded.asked_for else build_request.asked_for end`, + on conflict (id) do nothing`, a.ID, a.Repository, a.Seat, strings.Trim(a.Path, "/"), a.Ref, a.Commit, a.For, notAsked, at); err != nil { return err } - _, err := i.store.Pool().Exec(ctx, `delete from build_request where asked_at < $1`, time.Now().Add(-requestKeptFor)) + _, err := i.store.Pool().Exec(ctx, `delete from build_request where asked_at < $1`, time.Now().Add(-KeptFor)) return err } -// RequestsNamed is every ask kept whose directory names the module, newest first, each with its outcome. +// MarkNotAsked says a build request was asked and its asker could not hand it over, or stopped waiting for +// it: the words are kept, unless its outcome was heard first. +func (i *Inventory) MarkNotAsked(ctx context.Context, id, why string) error { + _, err := i.store.Pool().Exec(ctx, + `update build_request set not_asked = $2 + where id = $1 and not exists (select 1 from build where build.id = $1)`, id, why) + return err +} + +// RequestsNamed is every build request kept whose directory names the module, or whose build said it built +// it, newest first, each with its outcome. func (i *Inventory) RequestsNamed(ctx context.Context, name string) ([]RequestOutcome, error) { all, err := i.requests(ctx) if err != nil { @@ -97,8 +105,8 @@ func (i *Inventory) RequestsNamed(ctx context.Context, name string) ([]RequestOu return out, nil } -// RequestedNames is the name of every module an ask kept is expected to register: what `assign` offers as a -// closest name beside the modules registered. +// RequestedNames is the name of every module a kept build request is expected to register: what `assign` +// offers as a closest name beside the modules registered. func (i *Inventory) RequestedNames(ctx context.Context) ([]string, error) { all, err := i.requests(ctx) if err != nil { @@ -141,13 +149,14 @@ func (i *Inventory) requests(ctx context.Context) ([]RequestOutcome, error) { // The states of a pending assignment. const ( PendingWaiting = "waiting" + PendingApplying = "applying" PendingApplied = "applied" PendingRefused = "refused" PendingExpired = "expired" PendingWithdrawn = "withdrawn" ) -// PendingAssignment is an assignment kept until its module is registered. +// PendingAssignment is an assignment kept until its module is registered (ADR 0261). type PendingAssignment struct { ID int64 Node string @@ -159,22 +168,48 @@ type PendingAssignment struct { // Note is what ended it, in the words said when it ended. Note string Settled *time.Time + // Raised is when an expiry or refusal was raised as a condition; Acknowledged when a person took it + // back; Cleared when its condition was cleared. + Raised, Acknowledged, Cleared *time.Time +} + +// Open is whether the pending assignment is still to be made. +func (p PendingAssignment) Open() bool { + return p.State == PendingWaiting || p.State == PendingApplying } // ErrAlreadyPending is a second pending assignment of one module to one machine. var ErrAlreadyPending = errors.New("that assignment is already waiting for its build") -// RecordPending keeps an assignment until its module is registered. One waits per machine and module. +const pendingColumns = `p.id, n.name, p.module, p.build_id, p.repository, p.source_path, p.since, p.state, p.note, + p.settled_at, p.raised_at, p.acknowledged_at, p.cleared_at` + +func scanPending(rows pgx.Rows) ([]PendingAssignment, error) { + defer rows.Close() + var out []PendingAssignment + for rows.Next() { + var p PendingAssignment + if err := rows.Scan(&p.ID, &p.Node, &p.Module, &p.Build, &p.Repository, &p.Path, &p.Since, &p.State, + &p.Note, &p.Settled, &p.Raised, &p.Acknowledged, &p.Cleared); err != nil { + return nil, err + } + out = append(out, p) + } + return out, rows.Err() +} + +// RecordPending keeps an assignment until its module is registered. One is open per machine and module. func (i *Inventory) RecordPending(ctx context.Context, p PendingAssignment) (PendingAssignment, error) { - if _, err := i.NodeByName(ctx, p.Node); err != nil { + node, err := i.NodeByName(ctx, p.Node) + if err != nil { return PendingAssignment{}, err } - err := i.store.Pool().QueryRow(ctx, + err = i.store.Pool().QueryRow(ctx, `insert into pending_assignment (node, module, build_id, repository, source_path) values ($1, $2, $3, $4, $5) - on conflict (node, module) where state = 'waiting' do nothing + on conflict (node, module) where state in ('waiting', 'applying') do nothing returning id, since`, - p.Node, p.Module, p.Build, p.Repository, strings.Trim(p.Path, "/")).Scan(&p.ID, &p.Since) + node.ID, p.Module, p.Build, p.Repository, strings.Trim(p.Path, "/")).Scan(&p.ID, &p.Since) if errors.Is(err, pgx.ErrNoRows) { return PendingAssignment{}, ErrAlreadyPending } @@ -185,44 +220,135 @@ func (i *Inventory) RecordPending(ctx context.Context, p PendingAssignment) (Pen return p, nil } -// Pending is every pending assignment still waiting, and every one ended since the time given, oldest -// first. A zero time is the waiting ones alone. -func (i *Inventory) Pending(ctx context.Context, endedSince time.Time) ([]PendingAssignment, error) { - rows, err := i.store.Pool().Query(ctx, - `select id, node, module, build_id, repository, source_path, since, state, note, settled_at - from pending_assignment - where state = 'waiting' or (settled_at is not null and settled_at >= $1) - order by since, id`, endedSince) - if err != nil { - return nil, err - } - defer rows.Close() - var out []PendingAssignment - for rows.Next() { - var p PendingAssignment - if err := rows.Scan(&p.ID, &p.Node, &p.Module, &p.Build, &p.Repository, &p.Path, &p.Since, &p.State, - &p.Note, &p.Settled); err != nil { - return nil, err - } - if endedSince.IsZero() && p.State != PendingWaiting { - continue - } - out = append(out, p) - } - return out, rows.Err() -} - -// SettlePending ends a waiting pending assignment. False when it was no longer waiting: another process -// settled it first, and its word stands. -func (i *Inventory) SettlePending(ctx context.Context, id int64, state, note string) (bool, error) { - if state == PendingWaiting { - return false, errors.New("a pending assignment is settled to applied, refused, expired or withdrawn") - } +// RepointPending makes a waiting pending assignment wait for another build. False when it no longer waits. +func (i *Inventory) RepointPending(ctx context.Context, id int64, build, repository, sourcePath string) (bool, error) { tag, err := i.store.Pool().Exec(ctx, - `update pending_assignment set state = $2, note = $3, settled_at = now() - where id = $1 and state = 'waiting'`, id, state, note) + `update pending_assignment set build_id = $2, repository = $3, source_path = $4 + where id = $1 and state = 'waiting'`, id, build, repository, strings.Trim(sourcePath, "/")) if err != nil { return false, err } return tag.RowsAffected() == 1, nil } + +// Pending is every pending assignment still waiting, oldest first, when endedSince is zero. Otherwise it is +// every open one and every one ended since then. +func (i *Inventory) Pending(ctx context.Context, endedSince time.Time) ([]PendingAssignment, error) { + if endedSince.IsZero() { + rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+` + from pending_assignment p join node n on n.id = p.node + where p.state = 'waiting' order by p.since, p.id`) + if err != nil { + return nil, err + } + return scanPending(rows) + } + rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+` + from pending_assignment p join node n on n.id = p.node + where p.state in ('waiting', 'applying') or p.settled_at >= $1 + order by p.since, p.id`, endedSince) + if err != nil { + return nil, err + } + return scanPending(rows) +} + +// PendingFor is every pending assignment of one module to one machine, newest first. +func (i *Inventory) PendingFor(ctx context.Context, node, module string) ([]PendingAssignment, error) { + rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+` + from pending_assignment p join node n on n.id = p.node + where n.name = $1 and p.module = $2 order by p.since desc, p.id desc`, node, module) + if err != nil { + return nil, err + } + return scanPending(rows) +} + +// Stuck is every pending assignment left applying since before the time given: a controller that claimed +// it and stopped before it said what came of it. +func (i *Inventory) Stuck(ctx context.Context, before time.Time) ([]PendingAssignment, error) { + rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+` + from pending_assignment p join node n on n.id = p.node + where p.state = 'applying' and p.settled_at < $1 order by p.id`, before) + if err != nil { + return nil, err + } + return scanPending(rows) +} + +// ToRaise is every pending assignment that expired or was refused and has not been raised as a condition; +// ToClear every one raised and not cleared. +func (i *Inventory) ToRaise(ctx context.Context) ([]PendingAssignment, error) { + rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+` + from pending_assignment p join node n on n.id = p.node + where p.state in ('expired', 'refused') and p.raised_at is null order by p.id`) + if err != nil { + return nil, err + } + return scanPending(rows) +} + +func (i *Inventory) ToClear(ctx context.Context) ([]PendingAssignment, error) { + rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+` + from pending_assignment p join node n on n.id = p.node + where p.raised_at is not null and p.cleared_at is null order by p.id`) + if err != nil { + return nil, err + } + return scanPending(rows) +} + +// MarkPending stamps a pending assignment raised, acknowledged or cleared, now. +func (i *Inventory) MarkPending(ctx context.Context, id int64, what string) error { + column := map[string]string{"raised": "raised_at", "acknowledged": "acknowledged_at", "cleared": "cleared_at"}[what] + if column == "" { + return errors.New("a pending assignment is marked raised, acknowledged or cleared") + } + _, err := i.store.Pool().Exec(ctx, + `update pending_assignment set `+column+` = coalesce(`+column+`, now()) where id = $1`, id) + return err +} + +// ClaimPending takes a waiting pending assignment for making, so nothing else settles or withdraws it while +// it is made. False when it no longer waits: another process took it, or it was withdrawn. +func (i *Inventory) ClaimPending(ctx context.Context, id int64) (bool, error) { + tag, err := i.store.Pool().Exec(ctx, + `update pending_assignment set state = 'applying', settled_at = now() where id = $1 and state = 'waiting'`, id) + if err != nil { + return false, err + } + return tag.RowsAffected() == 1, nil +} + +// ReleaseClaim puts a claimed pending assignment back to waiting. +func (i *Inventory) ReleaseClaim(ctx context.Context, id int64) error { + _, err := i.store.Pool().Exec(ctx, + `update pending_assignment set state = 'waiting', settled_at = null where id = $1 and state = 'applying'`, id) + return err +} + +// SettlePending ends a pending assignment in the state it is read in (from): waiting for an expiry or a +// withdrawal, applying for the outcome of making it. False when it was no longer in that state. +func (i *Inventory) SettlePending(ctx context.Context, id int64, from, state, note string) (bool, error) { + if state == PendingWaiting || state == PendingApplying { + return false, errors.New("a pending assignment is settled to applied, refused, expired or withdrawn") + } + tag, err := i.store.Pool().Exec(ctx, + `update pending_assignment set state = $3, note = $4, settled_at = now() + where id = $1 and state = $2`, id, from, state, note) + if err != nil { + return false, err + } + return tag.RowsAffected() == 1, nil +} + +// ForgetEndedPending deletes ended pending assignments settled more than KeptFor ago, and says how many. +func (i *Inventory) ForgetEndedPending(ctx context.Context, now time.Time) (int64, error) { + tag, err := i.store.Pool().Exec(ctx, + `delete from pending_assignment where state not in ('waiting', 'applying') and settled_at < $1`, + now.Add(-KeptFor)) + if err != nil { + return 0, err + } + return tag.RowsAffected(), nil +} diff --git a/internal/inventory/pending_test.go b/internal/inventory/pending_test.go new file mode 100644 index 00000000..1ae3b8dd --- /dev/null +++ b/internal/inventory/pending_test.go @@ -0,0 +1,25 @@ +package inventory + +import ( + "testing" + "time" +) + +// A pending assignment goes with its machine (novox/hq issue 325, review of mesh-controller#150 point 7). +func TestAPendingAssignmentGoesWithItsMachine(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + if _, err := inv.AddNode(ctx, "leaving"); err != nil { + t.Fatal(err) + } + if _, err := inv.RecordPending(ctx, PendingAssignment{Node: "leaving", Module: "sensors", Build: "b-1"}); err != nil { + t.Fatal(err) + } + if _, err := inv.store.Pool().Exec(ctx, `delete from node where name = 'leaving'`); err != nil { + t.Fatalf("a machine with a pending assignment could not be removed: %v", err) + } + rows, err := inv.Pending(ctx, time.Unix(0, 0)) + if err != nil || len(rows) != 0 { + t.Fatalf("a pending assignment outlived its machine: %+v %v", rows, err) + } +}