Make pending assignments safe to race, settle them on a tick, and say only what was checked

Review of #150: a withdrawal could land between the look and the act, a
failed ask read as a build in flight, a request kept the wrong asker, a
build being registered read as not built, build "true" could ask a build
nothing waited on, and a status read changed state. Claim a row under the
machine's hold before making it, keep a request only once asked, settle on
the controller's own tick, raise an assignment not made as a condition
until it is answered, and tie each row to its machine.
This commit is contained in:
jochen
2026-10-08 16:38:11 +02:00
parent 7d63d2e68c
commit 249d97d1c8
12 changed files with 1018 additions and 235 deletions
+47 -13
View File
@@ -399,15 +399,43 @@ func buildBehind(ctx context.Context, wait time.Duration) error {
//
// Separated from the command so `--behind` can walk a list without a second path to the same act.
func buildOne(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
_, err := buildOneAsked(ctx, source, path, ref, wait, false)
_, err := buildOneAsked(ctx, source, path, ref, wait, false, "build")
return err
}
// recordAsked keeps a build request once it is asked (novox/hq issue 325), with who asked it, in a store
// opened for the purpose: the asks reach here from processes that hold none.
func recordAsked(ctx context.Context, r inventory.BuildRequest) {
open, err := openStores(ctx)
if err != nil {
fmt.Fprintf(os.Stderr, "build %s is asked and not kept as a build request: %v\n", r.ID, err)
return
}
defer open.Close()
recordBuildRequest(ctx, open.inventory, r)
}
// markNotAsked says a kept build request was not handed over, or not waited for.
func markNotAsked(ctx context.Context, id string, why error) {
open, err := openStores(ctx)
if err != nil {
fmt.Fprintf(os.Stderr, "build %s could not be marked as not asked: %v\n", id, err)
return
}
defer open.Close()
if err := open.inventory.MarkNotAsked(ctx, id, why.Error()); err != nil {
fmt.Fprintf(os.Stderr, "build %s could not be marked as not asked: %v\n", id, err)
}
}
// buildOneAsked is buildOne answering the id it asked with — what a plan keeps to match the outcome
// by (novox/hq ADR 0219) — and, for an ask not waited for, optionally a dry run: built and looked
// at, never taken in (issue 240), which is what `replay` asks unless told to register.
//
// asker is who asked, for the build request kept once the ask is made (novox/hq issue 325); empty for an
// asker that keeps the request itself, with its own name, once it knows the ask was made.
func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wait time.Duration,
dryRun bool) (string, error) {
dryRun bool, asker string) (string, error) {
if dryRun && wait != 0 {
return "", errors.New("a dry run waited for is `build --dry-run`")
}
@@ -462,17 +490,12 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
}
defer ask.Close()
fmt.Printf(" of %s\n", seat)
// Kept before it is asked, so `assign` knows a module is coming while its build runs (novox/hq issue 325).
// A dry run registers nothing, and is not kept.
if !dryRun {
if open, err := openStores(ctx); err == nil {
recordBuildRequest(ctx, open.inventory, inventory.BuildRequest{ID: request.ID,
Repository: source.Repository, Seat: source.Seat, Path: path, Ref: ref, For: "build"})
open.Close()
} else {
fmt.Fprintf(os.Stderr, "build %s is asked and not kept as a build request: %v\n", request.ID, err)
}
}
// Kept once it is asked, so `assign` knows a module is coming while its build runs (novox/hq issue 325);
// never before, so an ask that failed never reads as a build in flight. A dry run registers nothing, and
// is not kept.
keep := !dryRun && asker != ""
asked := inventory.BuildRequest{ID: request.ID, Repository: source.Repository, Seat: source.Seat, Path: path,
Ref: ref, For: asker}
if wait == 0 {
// Asked and not waited for (novox/hq issue 176): the outcome is the role's event, and the
@@ -482,6 +505,9 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
if err := ask.Ask(ctx, request); err != nil {
return "", err
}
if keep {
recordAsked(ctx, asked)
}
if dryRun {
fmt.Printf("asked as a dry run, not waited for: `builds --log %s` follows it as it runs; "+
"its outcome is not taken in\n", request.ID)
@@ -492,8 +518,16 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
return request.ID, nil
}
// Waited for: kept while it is waited for, since that can take minutes, and marked when the hand-over or
// the wait failed — an outcome heard later is still the last word.
if keep {
recordAsked(ctx, asked)
}
result, err := ask.Submit(ctx, request, wait)
if err != nil {
if keep {
markNotAsked(ctx, request.ID, err)
}
return request.ID, err
}
+346 -93
View File
@@ -9,11 +9,12 @@ import (
"strings"
"time"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A module not registered yet, and the assignment that waits for it (novox/hq issue 325).
// A module not registered yet, and the assignment that waits for it (novox/hq issue 325, ADR 0261).
//
// On 2026-10-08 the catalogue's pull request adding `sensors` merged; a minute later `assign g14 sensors`
// answered "no module of that name: sensors", and a few minutes later the same call worked. The merge had
@@ -23,34 +24,55 @@ import (
// So an assignment of a module the catalogue does not hold looks further before it refuses, and says which
// of three cases it is in:
//
// - **a build is in flight**: a build request for the module's directory has no outcome yet. The
// assignment is kept as a **pending assignment** and made when the build registers the module.
// - **known, not built**: the mesh asked for the directory before, and the build failed, was not
// - **a build is in flight**: a build request for the module's directory has no outcome yet, or its build
// succeeded a moment ago and is being registered. The assignment is kept as a **pending assignment**,
// which the controller makes when the build registers the module (ADR 0261).
// - **known, not built**: the controller asked for the directory before, and the build failed, was not
// registered, said nothing within its bound, or could not be asked. Said, with `build` and assign's
// own build argument, which asks for the build and keeps the assignment pending on it.
// - **unknown**: no module, build request or merge by that name. Refused as before, with the closest names.
// - **unknown**: no module registered and no build request kept by that name. Refused as before, with the
// closest names.
//
// **Pending by default while a build is in flight**, without an argument to ask for it: an assignment is
// what a person meant and is kept even when its machine does not resolve (acts.go), nothing is sent until a
// push, and `unassign` withdraws it. Asking for a second act once the build lands is exactly the two acts by
// hand issue 300 removed. A build is never asked for by default: it runs on another machine, and a directory
// whose last build failed is a fault to look at before it is a thing to retry.
// **Pending by default while a build is in flight** (ADR 0261), without an argument to ask for it: an
// assignment is what a person meant and is kept even when its machine does not resolve (acts.go), and
// `unassign` withdraws it. Asking for a second act once the build lands is the two acts by hand issue 300
// removed. A build is never asked for by default: it runs on another machine, and a directory whose last
// build failed is a fault to look at before it is a thing to retry.
//
// **Settling is the controller's tick, never a read** (settlingPending): `status`, the board and the summary
// only read the rows.
// buildRequestBound is how long a build request may go without an outcome before it is no longer read as in
// flight, and a pending assignment waiting for it expires. Generous: a build waits behind others on the
// build seat, and a pending assignment that waits a little longer costs nothing.
const buildRequestBound = 2 * time.Hour
// pendingShownFor is how long an ended pending assignment is said in `status`.
// registerGrace is how long a build heard as built is read as still in flight while it is not registered:
// the outcome is recorded first and registered after it, in the same take-in. Past it, the build was
// recorded and not registered, and says so.
const registerGrace = 5 * time.Minute
// pendingShownFor is how long an ended pending assignment is listed in `status`.
const pendingShownFor = 24 * time.Hour
// settleEvery is how often the controller settles the pending assignments.
const settleEvery = time.Minute
// claimStaleAfter is how long a pending assignment may be held as applying before the tick reads the claim
// as left by a controller that stopped, and settles it from what the mesh holds.
const claimStaleAfter = 5 * time.Minute
// kindPendingEnded is a pending assignment that ended without being made: expired or refused.
const kindPendingEnded = "assignment-not-made"
// assignOptions are what an assignment was asked with beside its machine and modules.
type assignOptions struct {
// Build asks for the build of a module known and not built, and keeps the assignment pending on it.
Build bool
}
// recordBuildRequest keeps a build request so `assign` can find it. Said, never fatal: the build was asked.
// recordBuildRequest keeps a build request so `assign` can find it, once it is asked. Said, never fatal: the
// build was asked.
func recordBuildRequest(ctx context.Context, inv *inventory.Inventory, r inventory.BuildRequest) {
if err := inv.RecordBuildRequest(ctx, r); err != nil {
fmt.Fprintf(os.Stderr, "build %s was asked, and could not be kept as a build request, so `assign` "+
@@ -73,25 +95,30 @@ type notHeld struct {
said string
}
// whereIs looks for a module the catalogue does not hold among the build requests the mesh keeps.
// whereIs looks for a module the catalogue does not hold among the build requests the controller keeps.
func whereIs(ctx context.Context, inv *inventory.Inventory, module string, now time.Time) (notHeld, error) {
requests, err := inv.RequestsNamed(ctx, module)
if err != nil {
return notHeld{}, err
}
if r, ok := inFlight(requests, now); ok {
return notHeld{kind: buildInFlight, request: r, said: fmt.Sprintf("%s is not registered yet: it is being "+
"built from %s since %s, as build %s; it can be assigned once that build registers it", module,
being := "being built"
if r.Heard {
being = "built and being registered"
}
return notHeld{kind: buildInFlight, request: r, said: fmt.Sprintf("%s is not registered yet: it is %s "+
"from %s since %s, as build %s; it can be assigned once that build registers it", module, being,
requestSource(r), clock(r.At), r.ID)}, nil
}
if len(requests) > 0 {
return notHeld{kind: knownNotBuilt, request: requests[0], said: fmt.Sprintf("%s is known and not "+
"registered: %s", module, whyNotBuilt(requests[0], now))}, nil
}
said := fmt.Sprintf("%v: %s — the mesh holds no module, build request or merge by that name",
inventory.ErrNoSuchModule, module)
// Only what was looked at is claimed: the catalogue, and the build requests the controller keeps.
said := fmt.Sprintf("%v: %s — the catalogue holds no module of that name, and no build request the "+
"controller kept (the last 30 days) is for a directory of that name", inventory.ErrNoSuchModule, module)
if near := closestNames(ctx, inv, module); len(near) > 0 {
said += "; the closest it holds: " + strings.Join(near, ", ")
said += "; the closest names it holds: " + strings.Join(near, ", ")
}
return notHeld{kind: unknownModule, said: said}, nil
}
@@ -111,6 +138,20 @@ func notInCatalogue(ctx context.Context, open *stores, modules []string) ([]stri
return missing, nil
}
// openPending is the open pending assignment of a module to a machine, if there is one.
func openPending(ctx context.Context, inv *inventory.Inventory, node, module string) (*inventory.PendingAssignment, error) {
rows, err := inv.PendingFor(ctx, node, module)
if err != nil {
return nil, err
}
for i := range rows {
if rows[i].Open() {
return &rows[i], nil
}
}
return nil, nil
}
// notRegistered answers an assignment of a module the catalogue does not hold: pending on a build in flight,
// known and not built, or unknown (novox/hq issue 325). An answer with no error is a pending assignment kept.
func notRegistered(ctx context.Context, open *stores, node, module string, opts assignOptions) (string, error) {
@@ -123,19 +164,31 @@ func notRegistered(ctx context.Context, open *stores, node, module string, opts
if err != nil {
return "", err
}
waiting, err := openPending(ctx, inv, node, module)
if err != nil {
return "", err
}
switch where.kind {
case buildInFlight:
lead := where.said
if opts.Build {
lead += "\n no second build was asked: this one is running"
}
return keepPending(ctx, open, node, module, where.request, lead)
return keepPending(ctx, open, node, module, where.request, waiting, lead)
case knownNotBuilt:
last := where.request
if !opts.Build {
return "", fmt.Errorf("%w: %s\n assign it again with build \"true\" to ask for its build and keep this "+
also := ""
if waiting != nil {
also = fmt.Sprintf("\n %s already has a pending assignment of %s, waiting for build %s; build "+
"\"true\" asks for a new build and makes it wait for that one", node, module, waiting.Build)
}
return "", fmt.Errorf("%w: %s%s\n assign it again with build \"true\" to ask for its build and keep this "+
"assignment until the build registers it; or `build` it (repository %s, path %s) and assign it "+
"once it is registered", inventory.ErrNoSuchModule, where.said, last.Repository, orRoot(last.Path))
"once it is registered", inventory.ErrNoSuchModule, where.said, also, last.Repository, orRoot(last.Path))
}
if waiting != nil && waiting.State == inventory.PendingApplying {
return "", fmt.Errorf("%s is being assigned %s now; nothing was asked", node, module)
}
source := buildSource{Repository: last.Repository, Seat: last.Seat}
id, err := askABuild(ctx, source, last.Path, last.Ref)
@@ -147,20 +200,38 @@ func notRegistered(ctx context.Context, open *stores, node, module string, opts
recordBuildRequest(ctx, inv, asked.BuildRequest)
lead := fmt.Sprintf("%s\n build %s of %s is asked for now; %s can be assigned once it registers it",
where.said, id, requestSource(asked), module)
return keepPending(ctx, open, node, module, asked, lead)
return keepPending(ctx, open, node, module, asked, waiting, lead)
}
answer := strings.TrimPrefix(where.said, inventory.ErrNoSuchModule.Error()) +
"\n a module in a repository is built with `build` (its repository and path), then assigned"
if opts.Build {
answer += "\n build \"true\" asks for nothing here: the mesh does not know where a module of that name is"
answer += "\n build \"true\" asks for nothing here: the controller has no build request saying where a " +
"module of that name is"
}
return "", fmt.Errorf("%w%s", inventory.ErrNoSuchModule, answer)
}
// keepPending records the assignment as pending on a build request, and says so.
func keepPending(ctx context.Context, open *stores, node, module string, r inventory.RequestOutcome, lead string) (string, error) {
// keepPending records the assignment as pending on a build request — or, where one is already open, makes it
// wait for that build — and says so.
func keepPending(ctx context.Context, open *stores, node, module string, r inventory.RequestOutcome,
waiting *inventory.PendingAssignment, lead string) (string, error) {
inv := open.inventory
p, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: node, Module: module, Build: r.ID,
if waiting != nil {
if waiting.Build == r.ID {
return lead + fmt.Sprintf("\n %s already waits for %s on this build: nothing changed", node, module), nil
}
moved, err := inv.RepointPending(ctx, waiting.ID, r.ID, r.Repository, r.Path)
if err != nil {
return "", err
}
if !moved {
return lead + fmt.Sprintf("\n the pending assignment of %s to %s ended while this was asked: `status` "+
"says how", module, node), nil
}
return lead + fmt.Sprintf("\n the pending assignment of %s to %s, which waited for build %s, now waits "+
"for build %s", module, node, waiting.Build, r.ID), nil
}
_, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: node, Module: module, Build: r.ID,
Repository: r.Repository, Path: r.Path})
if errors.Is(err, inventory.ErrAlreadyPending) {
return lead + fmt.Sprintf("\n %s already waits for %s: nothing changed", node, module), nil
@@ -168,51 +239,63 @@ func keepPending(ctx context.Context, open *stores, node, module string, r inven
if err != nil {
return "", err
}
answer := lead + fmt.Sprintf("\n the assignment is kept as pending: %s is assigned %s when the build "+
"registers it, and `status` says it until then. If the build fails it expires and says why; "+
"`unassign %s %s` withdraws it", node, module, node, module)
// The build may have registered the module between the look and the record: made at once, then.
if shelf, err := inv.Catalogue(ctx); err == nil {
if _, now := shelf[module]; now {
if line := applyPending(ctx, open, p, r.ID); line != "" {
answer += "\n " + line
}
}
}
return answer, nil
// The controller makes it, not this act (ADR 0261): when the build registers the module, or at its next
// tick if the module was registered between the look and the record.
return lead + fmt.Sprintf("\n the assignment is kept as pending: the controller assigns %s to %s when the "+
"build registers it, and `status` lists it until then. If the build fails it expires, says why and "+
"raises a condition; `unassign %s %s` withdraws it", module, node, node, module), nil
}
// inFlight is the newest build request for the module with no outcome yet, asked within the bound.
// inFlight is the newest build request for the module still to register it: no outcome yet within the
// bound, or a successful outcome heard a moment ago and being registered.
func inFlight(requests []inventory.RequestOutcome, now time.Time) (inventory.RequestOutcome, bool) {
for _, r := range requests {
if r.NotAsked == "" && !r.Heard && now.Sub(r.At) < buildRequestBound {
if stillComing(r, now) {
return r, true
}
}
return inventory.RequestOutcome{}, false
}
// whyNotBuilt says what came of a module's last build request.
func stillComing(r inventory.RequestOutcome, now time.Time) bool {
if r.Heard {
return r.Failed == "" && now.Sub(r.HeardAt) < registerGrace
}
return r.NotAsked == "" && now.Sub(r.At) < buildRequestBound
}
// whyNotBuilt says what came of a module's last build request. What was heard comes first: an outcome is the
// last word whatever its asker said.
func whyNotBuilt(r inventory.RequestOutcome, now time.Time) string {
where := fmt.Sprintf("%s in %s", orRoot(r.Path), r.Repository)
switch {
case r.NotAsked != "":
return fmt.Sprintf("%s; the merge that added it (%s) could not ask for its build: %s", where,
short(r.Commit), firstLine(r.NotAsked))
case !r.Heard:
return fmt.Sprintf("%s; build %s was asked at %s, and no outcome has been heard in %s", where, r.ID,
clock(r.At), now.Sub(r.At).Round(time.Minute))
case r.Failed != "":
case r.Heard && r.Failed != "":
return fmt.Sprintf("%s; its last build, %s at %s, failed: %s", where, r.ID, clock(r.HeardAt),
firstLine(r.Failed))
case r.Module != "" && r.Module != r.Name():
return fmt.Sprintf("%s; its last build, %s, registered it as %s", where, r.ID, r.Module)
default:
case r.Heard && r.Module != "" && r.Module != r.Name():
return fmt.Sprintf("%s; its last build, %s, built it as %s", where, r.ID, r.Module)
case r.Heard:
return fmt.Sprintf("%s; its last build, %s at %s, was recorded and not registered — `builds` says why",
where, r.ID, clock(r.HeardAt))
case r.NotAsked != "" && r.For == "merge":
return fmt.Sprintf("%s; the merge that added it (%s) could not ask for its build: %s", where,
short(r.Commit), firstLine(r.NotAsked))
case r.NotAsked != "":
return fmt.Sprintf("%s; build %s, asked by %s, was not handed over or not waited for: %s", where, r.ID,
askerOr(r.For), firstLine(r.NotAsked))
default:
return fmt.Sprintf("%s; build %s was asked at %s, and no outcome has been heard in %s", where, r.ID,
clock(r.At), now.Sub(r.At).Round(time.Minute))
}
}
func askerOr(who string) string {
if who == "" {
return "the controller"
}
return who
}
// requestSource is a build request's source as a person reads it: repository@commit (directory).
func requestSource(r inventory.RequestOutcome) string {
at := short(r.Commit)
@@ -228,8 +311,8 @@ func requestSource(r inventory.RequestOutcome) string {
// clock is a moment as a person reads it, in the controller's local time.
func clock(t time.Time) string { return t.Local().Format("2006-01-02 15:04:05 MST") }
// closestNames is up to three names the mesh holds that are near the one asked: registered modules and
// directories it asked to build.
// closestNames is up to three names near the one asked: registered modules and directories the controller
// asked to build.
func closestNames(ctx context.Context, inv *inventory.Inventory, name string) []string {
var candidates []string
if shelf, err := inv.Catalogue(ctx); err == nil {
@@ -292,26 +375,41 @@ func editDistance(a, b string) int {
return prev[len(rb)]
}
// applyPending makes a pending assignment now that its module is registered, and settles it: applied, or
// refused with the refusal. Returns what it said, or nothing when another process settled it first.
// applyPending makes a pending assignment now that its module is registered (ADR 0261). Under the machine's
// hold it claims the row (waiting → applying), so a withdrawal cannot land between the look and the act,
// then assigns by the same act a person's assign is, then says what came of it: applied, or refused with the
// refusal. Returns what it said, or nothing when the row no longer waited.
func applyPending(ctx context.Context, open *stores, p inventory.PendingAssignment, by string) string {
inv := open.inventory
ctx, release, err := holdNodes(ctx, open, []string{p.Node})
if err != nil {
// Left waiting: the next tick tries again.
return fmt.Sprintf("the pending assignment of %s to %s waits: %s could not be held: %v", p.Module, p.Node,
p.Node, err)
}
defer release()
claimed, err := inv.ClaimPending(ctx, p.ID)
if err != nil {
return fmt.Sprintf("the pending assignment of %s to %s could not be taken for making: %v", p.Module, p.Node, err)
}
if !claimed {
return ""
}
answer, err := assign(ctx, open, p.Node, p.Module)
state, note := inventory.PendingApplied, ""
if err != nil {
switch {
case err != nil:
state = inventory.PendingRefused
note = fmt.Sprintf("the pending assignment of %s to %s was refused when build %s registered it: %s",
p.Module, p.Node, by, firstLine(err.Error()))
} else {
// Not promised as sent: a walk sends it if the module's upgrade announcement finds it
// assigned, and nothing does if the announcement came first. `status` says which, as for any machine.
note = fmt.Sprintf("%s is assigned %s: build %s registered it (pending since %s); `status` says whether "+
"%s has been sent it, and `push %s` sends it if not", p.Node, p.Module, by, clock(p.Since), p.Node, p.Node)
if strings.Contains(answer, "already runs") {
note = fmt.Sprintf("%s already runs %s: the pending assignment is met", p.Node, p.Module)
}
case strings.Contains(answer, "already runs"):
note = fmt.Sprintf("%s already runs %s: the pending assignment is met", p.Node, p.Module)
default:
// The same as a person's assign: nothing is sent by this act.
note = fmt.Sprintf("%s is assigned %s: build %s registered it (pending since %s); `push %s` sends it",
p.Node, p.Module, by, clock(p.Since), p.Node)
}
settled, serr := inv.SettlePending(ctx, p.ID, state, note)
settled, serr := inv.SettlePending(ctx, p.ID, inventory.PendingApplying, state, note)
if serr != nil {
return fmt.Sprintf("%s — and it could not be recorded as settled: %v", note, serr)
}
@@ -321,11 +419,11 @@ func applyPending(ctx context.Context, open *stores, p inventory.PendingAssignme
return note
}
// expirePending ends a pending assignment whose build will not register its module, with why.
// expirePending ends a waiting pending assignment whose build will not register its module, with why.
func expirePending(ctx context.Context, inv *inventory.Inventory, p inventory.PendingAssignment, why string) string {
note := fmt.Sprintf("the pending assignment of %s to %s expired: %s; nothing was assigned. Assign it again with "+
"build \"true\" to ask for the build again", p.Module, p.Node, why)
settled, err := inv.SettlePending(ctx, p.ID, inventory.PendingExpired, note)
settled, err := inv.SettlePending(ctx, p.ID, inventory.PendingWaiting, inventory.PendingExpired, note)
if err != nil {
return fmt.Sprintf("%s — and it could not be recorded: %v", note, err)
}
@@ -369,21 +467,62 @@ func settlePendingOnBuild(ctx context.Context, open *stores, result link.BuildRe
return said
}
// settleStalePending settles what a missed outcome left waiting: a module registered meanwhile is assigned,
// a build heard and not registered ends its assignment, and one with no outcome past the bound expires.
// Asked where the mesh is read for status, so a pending assignment never waits silently for ever.
func settleStalePending(ctx context.Context, open *stores, now time.Time) []string {
inv := open.inventory
waiting, err := inv.Pending(ctx, time.Time{})
if err != nil {
// Said, not swallowed; status reads the pending assignments again and says the same failure there.
return []string{fmt.Sprintf("the pending assignments could not be read to settle them: %v", err)}
// settlingPending is the controller's tick over the pending assignments (ADR 0261), every settleEvery until
// the context ends. What it does is printed to the controller's log, kept in each row's note (which `status`
// lists for a day), and raised as a condition for an assignment that was not made.
func settlingPending(ctx context.Context, open *stores) {
for {
for _, line := range settlePending(ctx, open, time.Now()) {
fmt.Println(line)
}
select {
case <-ctx.Done():
return
case <-time.After(settleEvery):
}
}
}
// settlePending is one tick: claims left by a controller that stopped are settled from what the mesh holds;
// a module registered meanwhile is assigned; a build heard and not registered, or silent past its bound,
// ends its assignment; ended ones are raised as conditions and cleared once answered; ended rows older than
// KeptFor are deleted. Returns what it did.
func settlePending(ctx context.Context, open *stores, now time.Time) []string {
inv := open.inventory
var said []string
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
shelf, err := inv.Catalogue(ctx)
if err != nil {
return []string{fmt.Sprintf("the catalogue could not be read to settle the pending assignments: %v", err)}
say("the pending assignments are not settled this time: the catalogue could not be read: %v", err)
return said
}
stuck, err := inv.Stuck(ctx, now.Add(-claimStaleAfter))
if err != nil {
say("claims left by a stopped controller could not be read: %v", err)
}
for _, p := range stuck {
assigned, err := inv.Assigned(ctx, p.Node)
if err != nil {
say("the claim on %s for %s could not be settled: %v", p.Node, p.Module, err)
continue
}
if containsString(assigned, p.Module) {
if _, err := inv.SettlePending(ctx, p.ID, inventory.PendingApplying, inventory.PendingApplied,
fmt.Sprintf("%s is assigned %s (settled after the controller that made it stopped)", p.Node, p.Module)); err != nil {
say("the claim on %s for %s could not be settled: %v", p.Node, p.Module, err)
}
continue
}
if err := inv.ReleaseClaim(ctx, p.ID); err != nil {
say("the claim on %s for %s could not be released: %v", p.Node, p.Module, err)
}
}
waiting, err := inv.Pending(ctx, time.Time{})
if err != nil {
say("the pending assignments could not be read to settle them: %v", err)
}
var said []string
for _, p := range waiting {
if _, ok := shelf[p.Module]; ok {
if line := applyPending(ctx, open, p, p.Build); line != "" {
@@ -393,7 +532,7 @@ func settleStalePending(ctx context.Context, open *stores, now time.Time) []stri
}
requests, err := inv.RequestsNamed(ctx, p.Module)
if err != nil {
said = append(said, fmt.Sprintf("the build requests for %s could not be read: %v", p.Module, err))
say("the build requests for %s could not be read: %v", p.Module, err)
continue
}
var r *inventory.RequestOutcome
@@ -406,9 +545,9 @@ func settleStalePending(ctx context.Context, open *stores, now time.Time) []stri
switch {
case r == nil && now.Sub(p.Since) > buildRequestBound:
why = fmt.Sprintf("build %s is no longer on record", p.Build)
case r != nil && r.Heard:
case r != nil && !stillComing(*r, now) && (r.Heard || r.NotAsked != ""):
why = whyNotBuilt(*r, now)
case r != nil && now.Sub(r.At) > buildRequestBound:
case r != nil && !stillComing(*r, now):
why = fmt.Sprintf("no outcome of build %s was heard within %s of asking", p.Build, buildRequestBound)
}
if why == "" {
@@ -418,25 +557,139 @@ func settleStalePending(ctx context.Context, open *stores, now time.Time) []stri
said = append(said, line)
}
}
said = append(said, raisePendingEnded(ctx, inv)...)
if n, err := inv.ForgetEndedPending(ctx, now); err != nil {
say("ended pending assignments could not be deleted: %v", err)
} else if n > 0 {
say("deleted %d pending assignment(s) ended more than %s ago", n, inventory.KeptFor)
}
return said
}
// withdrawPending is `unassign` of a module only pending on a machine: the pending assignment withdrawn.
func withdrawPending(ctx context.Context, inv *inventory.Inventory, node, module string) (string, bool, error) {
waiting, err := inv.Pending(ctx, time.Time{})
func containsString(xs []string, x string) bool {
for _, y := range xs {
if y == x {
return true
}
}
return false
}
// pendingObservation is the condition for a pending assignment that was not made.
func pendingObservation(p inventory.PendingAssignment) conditions.Observation {
return conditions.Observation{Scope: conditions.ScopeMachine, ID: p.Node + "." + p.Module,
Token: kindPendingEnded, Kind: kindPendingEnded, Machine: p.Node, Severity: conditions.Warning,
Resolver: conditions.ResolverOperator, Source: "pending assignments",
Summary: p.Note}
}
// raisePendingEnded raises a condition for each pending assignment that expired or was refused, once, and
// clears it when it is answered: the module was assigned to that machine since, a newer pending assignment of
// it is open, or a person took it back with `unassign`. Where this process keeps no conditions, nothing is
// stamped, and the serving controller's tick raises it.
func raisePendingEnded(ctx context.Context, inv *inventory.Inventory) []string {
keeper := conditionsFrom
if keeper == nil {
return nil
}
var said []string
toRaise, err := inv.ToRaise(ctx)
if err != nil {
return []string{fmt.Sprintf("pending assignments not made could not be read to raise them: %v", err)}
}
for _, p := range toRaise {
if _, err := keeper.Observe(ctx, pendingObservation(p)); err != nil {
said = append(said, fmt.Sprintf("the condition for %s on %s could not be raised: %v", p.Module, p.Node, err))
continue
}
if err := inv.MarkPending(ctx, p.ID, "raised"); err != nil {
said = append(said, fmt.Sprintf("the condition for %s on %s was raised and not recorded: %v", p.Module, p.Node, err))
}
}
toClear, err := inv.ToClear(ctx)
if err != nil {
return append(said, fmt.Sprintf("pending assignments raised could not be read to clear them: %v", err))
}
for _, p := range toClear {
why, answered, err := pendingAnswered(ctx, inv, p)
if err != nil {
said = append(said, fmt.Sprintf("whether %s on %s is answered could not be read: %v", p.Module, p.Node, err))
continue
}
if !answered {
continue
}
if _, err := keeper.Clear(ctx, pendingObservation(p).Key(), why); err != nil {
said = append(said, fmt.Sprintf("the condition for %s on %s could not be cleared: %v", p.Module, p.Node, err))
continue
}
if err := inv.MarkPending(ctx, p.ID, "cleared"); err != nil {
said = append(said, fmt.Sprintf("the condition for %s on %s was cleared and not recorded: %v", p.Module, p.Node, err))
}
}
return said
}
// pendingAnswered says whether a pending assignment that was not made has been answered since, and how.
func pendingAnswered(ctx context.Context, inv *inventory.Inventory, p inventory.PendingAssignment) (string, bool, error) {
if p.Acknowledged != nil {
return "taken back with unassign", true, nil
}
assigned, err := inv.Assigned(ctx, p.Node)
if err != nil {
return "", false, err
}
for _, p := range waiting {
if p.Node != node || p.Module != module {
continue
if containsString(assigned, p.Module) {
return p.Module + " is assigned to " + p.Node + " since", true, nil
}
rows, err := inv.PendingFor(ctx, p.Node, p.Module)
if err != nil {
return "", false, err
}
for _, r := range rows {
if r.ID != p.ID && r.Since.After(p.Since) {
return "assigned again, pending on another build", true, nil
}
}
return "", false, nil
}
// withdrawPending is `unassign` of a module only pending on a machine, under the machine's hold: a waiting
// pending assignment is withdrawn; one being made now is refused, never overridden; one that expired or was
// refused is taken back, which answers its condition. False when there is none.
func withdrawPending(ctx context.Context, inv *inventory.Inventory, node, module string) (string, bool, error) {
rows, err := inv.PendingFor(ctx, node, module)
if err != nil {
return "", false, err
}
for _, p := range rows {
switch p.State {
case inventory.PendingApplying:
return "", true, fmt.Errorf("%s is being assigned %s now, as build %s registered it: nothing was "+
"withdrawn; `unassign %s %s` again once it is assigned takes it off", node, module, p.Build, node, module)
case inventory.PendingWaiting:
note := fmt.Sprintf("the pending assignment of %s to %s is withdrawn; build %s goes on, and registers "+
"%s assigned nowhere", module, node, p.Build, module)
settled, err := inv.SettlePending(ctx, p.ID, inventory.PendingWaiting, inventory.PendingWithdrawn, note)
if err != nil {
return "", false, err
}
if !settled {
return "", true, fmt.Errorf("the pending assignment of %s to %s changed while it was withdrawn: "+
"`status` says how; nothing was withdrawn", module, node)
}
return note, true, nil
case inventory.PendingExpired, inventory.PendingRefused:
if p.Cleared != nil || p.Acknowledged != nil {
continue
}
if err := inv.MarkPending(ctx, p.ID, "acknowledged"); err != nil {
return "", false, err
}
return fmt.Sprintf("the pending assignment of %s to %s, which %s, is taken back; its condition clears",
module, node, p.State), true, nil
}
note := fmt.Sprintf("the pending assignment of %s to %s is withdrawn; build %s goes on, and registers %s "+
"assigned nowhere", module, node, p.Build, module)
if _, err := inv.SettlePending(ctx, p.ID, inventory.PendingWithdrawn, note); err != nil {
return "", false, err
}
return note, true, nil
}
return "", false, nil
}
+341 -27
View File
@@ -1,6 +1,7 @@
package main
import (
"context"
"encoding/json"
"errors"
"slices"
@@ -9,6 +10,7 @@ import (
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
@@ -68,21 +70,27 @@ func pendingOf(t *testing.T, open *stores) []inventory.PendingAssignment {
return got
}
// A build in flight: the merge asked for it, the assignment is kept pending and said so with the build, and
// the build's outcome makes it. Status says it while it waits.
// A build in flight: the merge asked for it, the request is kept as the merge's, the assignment is kept
// pending and said so with the build, status reads it without settling anything, and the build's outcome
// makes it — saying that a push sends it, and nothing more.
func TestAnAssignmentWaitsForTheBuildInFlight(t *testing.T) {
open := aCatalogueMesh(t)
ctx := t.Context()
asksWithPaths(t)
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
requests, err := open.inventory.RequestsNamed(ctx, "sensors")
if err != nil || len(requests) != 1 || requests[0].For != "merge" || requests[0].Commit != "3da80a4b00aa" {
t.Fatalf("the merge's build request: %+v %v", requests, err)
}
said, err := assign(ctx, open, "laptop", "sensors")
if err != nil {
t.Fatalf("an assignment while its build runs was refused: %v", err)
}
t.Logf("assign laptop sensors, while its build runs:\n%s", said)
for _, want := range []string{"being built from novox/mesh-catalog@3da80a4b (modules/sensors)",
"build b-modules/sensors", "kept as pending", "`unassign laptop sensors`"} {
"build b-modules/sensors", "kept as pending", "the controller assigns sensors to laptop", "`unassign laptop sensors`"} {
if !strings.Contains(said, want) {
t.Fatalf("the answer does not say %q:\n%s", want, said)
}
@@ -94,7 +102,6 @@ func TestAnAssignmentWaitsForTheBuildInFlight(t *testing.T) {
if len(pending) != 1 || pending[0].State != inventory.PendingWaiting || pending[0].Build != "b-modules/sensors" {
t.Fatalf("pending: %+v", pending)
}
// Asked twice, it is one pending assignment.
if again, err := assign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(again, "already waits") {
t.Fatalf("a second assignment: %q %v", again, err)
}
@@ -127,12 +134,41 @@ func TestAnAssignmentWaitsForTheBuildInFlight(t *testing.T) {
}
pending = pendingOf(t, open)
if len(pending) != 1 || pending[0].State != inventory.PendingApplied ||
!strings.Contains(pending[0].Note, "`push laptop` sends it if not") {
!strings.HasSuffix(pending[0].Note, "`push laptop` sends it") {
t.Fatalf("pending after the build: %+v", pending)
}
}
// The build of a pending assignment fails: it expires, saying the build's words, and nothing is assigned.
// **A read settles nothing** (review of #150, point 6): status — and so the board, the summary and the
// probe, which compose from the same reading — leaves a pending assignment whose module is registered as it
// is; the controller's tick makes it.
func TestAStatusReadSettlesNothing(t *testing.T) {
open := aCatalogueMesh(t)
ctx := t.Context()
asksWithPaths(t)
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
t.Fatal(err)
}
register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"})
for range 2 {
if _, err := theThreeQuestions(ctx, open); err != nil {
t.Fatal(err)
}
}
if slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingWaiting {
t.Fatalf("a status read settled a pending assignment: %+v", pendingOf(t, open))
}
said := settlePending(ctx, open, time.Now())
if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingApplied {
t.Fatalf("the tick did not make it: %v %+v", said, pendingOf(t, open))
}
}
// The build of a pending assignment fails: it expires with the build's words and nothing is assigned; the
// tick raises it as a condition once; it then reads as known and not built; and `unassign` takes it back,
// after which the tick clears the condition. Status is not called well while the condition is open, and is
// again once it clears — no fixed day of "not well" (review point 6).
func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) {
open := aCatalogueMesh(t)
ctx := t.Context()
@@ -158,15 +194,17 @@ func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) {
if slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
t.Fatal("a failed build's module was assigned")
}
asked, err := theThreeQuestions(ctx, open)
if err != nil {
t.Fatal(err)
settlePending(ctx, open, time.Now())
key := pendingObservation(pending[0]).Key()
c, found, err := conditionsFrom.Get(ctx, key)
if err != nil || !found || c.Kind != kindPendingEnded {
t.Fatalf("no condition %s for the assignment not made: %+v %v %v", key, c, found, err)
}
if asked.well() {
t.Fatal("status called the mesh well on the day a pending assignment expired")
if asked, err := theThreeQuestions(ctx, open); err != nil || asked.well() || len(asked.conditions) == 0 {
t.Fatalf("status does not hold the open condition: %v", err)
}
// Known and not built now: said, and refused without build.
_, err = assign(ctx, open, "laptop", "sensors")
if !errors.Is(err, inventory.ErrNoSuchModule) {
t.Fatalf("a module whose build failed: %v", err)
@@ -177,10 +215,54 @@ func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) {
t.Fatalf("the refusal does not say %q:\n%v", want, err)
}
}
said, err := unassign(ctx, open, "laptop", "sensors")
if err != nil || !strings.Contains(said, "taken back") {
t.Fatalf("unassign of an expired pending assignment: %q %v", said, err)
}
settlePending(ctx, open, time.Now())
if _, found, _ := conditionsFrom.Get(ctx, key); found {
t.Fatal("the condition stayed open after the assignment was taken back")
}
// Nothing of it keeps status from being well any more: no open pending assignment, no open condition of it.
asked, err := theThreeQuestions(ctx, open)
if err != nil || pendingOpen(asked.pending) {
t.Fatalf("status still counts the pending assignment: %+v %v", asked.pending, err)
}
for _, c := range asked.conditions {
if c.Kind == kindPendingEnded {
t.Fatalf("status still holds the condition: %+v", c)
}
}
}
// Known and not built, asked with build: the build is asked from where the last one was, and the assignment
// kept pending on the new build.
// The condition also clears when the module is later assigned to the machine.
func TestTheConditionClearsWhenTheModuleIsAssignedLater(t *testing.T) {
open := aCatalogueMesh(t)
ctx := t.Context()
asksWithPaths(t)
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
t.Fatal(err)
}
if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors",
"3da80a4b00aa", "", "boom")); err != nil {
t.Fatal(err)
}
settlePending(ctx, open, time.Now())
key := pendingObservation(pendingOf(t, open)[0]).Key()
register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"})
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
t.Fatal(err)
}
settlePending(ctx, open, time.Now())
if _, found, _ := conditionsFrom.Get(ctx, key); found {
t.Fatal("the condition stayed open after the module was assigned")
}
}
// Known and not built, asked with build: the build is asked from where the last one was, the request kept as
// assign's, and the assignment kept pending on the new build.
func TestAssignWithBuildAsksForAKnownModule(t *testing.T) {
open := aCatalogueMesh(t)
ctx := t.Context()
@@ -211,10 +293,146 @@ func TestAssignWithBuildAsksForAKnownModule(t *testing.T) {
if len(pending) != 1 || pending[0].Build != "b-modules/sensors" {
t.Fatalf("pending: %+v", pending)
}
requests, _ := inv.RequestsNamed(ctx, "sensors")
if len(requests) != 2 || requests[0].ID != "b-modules/sensors" || requests[0].For != "assign" {
t.Fatalf("the request is not kept as assign's: %+v", requests)
}
}
// Unknown: refused as before, as no module of that name, with the closest names the mesh holds; build asks
// for nothing, since the mesh does not know where it is.
// **build "true" with a pending assignment already waiting** (review point 5): the waiting one is made to
// wait for the new build, and no build is asked that nothing waits on.
func TestAssignWithBuildRepointsTheWaitingAssignment(t *testing.T) {
open := aCatalogueMesh(t)
ctx := t.Context()
inv := open.inventory
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-old", Repository: "novox/mesh-catalog",
Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge", At: time.Now().Add(-time.Hour)}); err != nil {
t.Fatal(err)
}
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-old", Repository: "novox/mesh-catalog", Ref: "main",
Path: "modules/sensors", On: "anchor", Failed: "boom"}); err != nil {
t.Fatal(err)
}
// Waiting on the failed build, its expiry not yet settled.
if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "laptop", Module: "sensors",
Build: "build-old", Repository: "novox/mesh-catalog", Path: "modules/sensors"}); err != nil {
t.Fatal(err)
}
_, err := assign(ctx, open, "laptop", "sensors")
if err == nil || !strings.Contains(err.Error(), "already has a pending assignment of sensors, waiting for build build-old") {
t.Fatalf("known and not built with a pending assignment waiting: %v", err)
}
asked := asksWithPaths(t)
said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors")
if err != nil || !strings.Contains(said, "which waited for build build-old, now waits for build b-modules/sensors") {
t.Fatalf("assign with build: %q %v", said, err)
}
if len(*asked) != 1 {
t.Fatalf("asked %v", *asked)
}
pending := pendingOf(t, open)
if len(pending) != 1 || pending[0].Build != "b-modules/sensors" || pending[0].State != inventory.PendingWaiting {
t.Fatalf("pending: %+v", pending)
}
// The new build in flight now: build "true" again asks nothing.
if said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors"); err != nil ||
!strings.Contains(said, "no second build was asked") || len(*asked) != 1 {
t.Fatalf("a second build true while the build runs: %q %v, asked %v", said, err, *asked)
}
}
// **A failed ask never reads as in flight** (review point 2): a merge whose ask could not be made keeps the
// request as not asked, and assign says the merge could not ask; a request whose asker could not hand it over
// or stopped waiting reads the same, unless its outcome was heard.
func TestAFailedAskIsNotABuildInFlight(t *testing.T) {
open := aCatalogueMesh(t)
ctx := t.Context()
inv := open.inventory
was := askABuild
askABuild = func(context.Context, buildSource, string, string) (string, error) {
return "", errors.New("cannot submit a build: nats: timeout")
}
t.Cleanup(func() { askABuild = was })
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
_, err := assign(ctx, open, "laptop", "sensors")
if err == nil || !strings.Contains(err.Error(), "the merge that added it (3da80a4b) could not ask for its build: cannot submit") {
t.Fatalf("a merge that could not ask: %v", err)
}
if len(pendingOf(t, open)) != 0 {
t.Fatal("a pending assignment waits on a build never asked")
}
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-waited", Repository: "novox/mesh-catalog",
Seat: "git", Path: "modules/gauges", Ref: "main", For: "build"}); err != nil {
t.Fatal(err)
}
if err := inv.MarkNotAsked(ctx, "build-waited", "cannot submit a build: no responders"); err != nil {
t.Fatal(err)
}
_, err = assign(ctx, open, "laptop", "gauges")
if err == nil || !strings.Contains(err.Error(), "build build-waited, asked by build, was not handed over or not waited for") {
t.Fatalf("a build not handed over: %v", err)
}
// Heard first, the outcome stands: marking it afterwards changes nothing.
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-waited", Repository: "novox/mesh-catalog", Ref: "main",
Path: "modules/gauges", On: "anchor", Failed: "the real failure"}); err != nil {
t.Fatal(err)
}
_, err = assign(ctx, open, "laptop", "gauges")
if err == nil || !strings.Contains(err.Error(), "failed: the real failure") {
t.Fatalf("an outcome heard after a failed wait: %v", err)
}
}
// A plan's tier keeps its requests as the plan's (review point 3).
func TestAPlansBuildRequestIsThePlans(t *testing.T) {
open := aCatalogueMesh(t)
ctx := t.Context()
asksWithPaths(t)
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"},
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main",
BuiltFrom: "c0", Head: "c0"}); err != nil {
t.Fatal(err)
}
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa",
Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true}
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
t.Fatal(err)
}
requests, err := open.inventory.RequestsNamed(ctx, "app")
if err != nil || len(requests) != 1 || requests[0].For != "plan" {
t.Fatalf("the plan's request: %+v %v", requests, err)
}
}
// **A build heard as built and not registered yet is in flight** (review point 4): the outcome is recorded
// before the module is registered; for that moment assign keeps the assignment pending. Past the grace it is
// recorded and not registered, and the tick ends the pending assignment with that.
func TestABuildBeingRegisteredIsInFlight(t *testing.T) {
open := aCatalogueMesh(t)
ctx := t.Context()
inv := open.inventory
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-done", Repository: "novox/mesh-catalog",
Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge"}); err != nil {
t.Fatal(err)
}
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-done", Repository: "novox/mesh-catalog", Ref: "main",
Path: "modules/sensors", Module: "sensors", On: "anchor"}); err != nil {
t.Fatal(err)
}
said, err := assign(ctx, open, "laptop", "sensors")
if err != nil || !strings.Contains(said, "built and being registered") || !strings.Contains(said, "kept as pending") {
t.Fatalf("a build being registered: %q %v", said, err)
}
settlePending(ctx, open, time.Now().Add(registerGrace+time.Minute))
p := pendingOf(t, open)
if len(p) != 1 || p[0].State != inventory.PendingExpired || !strings.Contains(p[0].Note, "recorded and not registered") {
t.Fatalf("past the grace: %+v", p)
}
}
// Unknown: refused as no module of that name, claiming only what was looked at, with the closest names; build
// asks for nothing.
func TestAnUnknownModuleIsRefusedWithTheClosestNames(t *testing.T) {
open := aCatalogueMesh(t)
ctx := t.Context()
@@ -224,12 +442,16 @@ func TestAnUnknownModuleIsRefusedWithTheClosestNames(t *testing.T) {
if !errors.Is(err, inventory.ErrNoSuchModule) {
t.Fatalf("an unknown module: %v", err)
}
for _, want := range []string{"no module of that name: sensors", "no module, build request or merge by that name",
"the closest it holds: sensord", "asks for nothing here"} {
for _, want := range []string{"no module of that name: sensors", "the catalogue holds no module of that name",
"no build request the controller kept (the last 30 days)", "the closest names it holds: sensord",
"asks for nothing here"} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("the refusal does not say %q:\n%v", want, err)
}
}
if strings.Contains(err.Error(), "merge") {
t.Fatalf("the refusal claims a merge it never looked at: %v", err)
}
if len(*asked) != 0 || len(pendingOf(t, open)) != 0 {
t.Fatalf("an unknown module asked %v, pending %+v", *asked, pendingOf(t, open))
}
@@ -252,7 +474,8 @@ func TestAnActWithAModuleNotRegisteredIsRefusedWhole(t *testing.T) {
}
}
// unassign withdraws a pending assignment; the build goes on and registers the module assigned nowhere.
// unassign withdraws a waiting pending assignment; the build goes on and registers the module assigned
// nowhere: a withdrawn row is never claimed.
func TestUnassignWithdrawsAPendingAssignment(t *testing.T) {
open := aCatalogueMesh(t)
ctx := t.Context()
@@ -269,13 +492,80 @@ func TestUnassignWithdrawsAPendingAssignment(t *testing.T) {
"3da80a4b00aa", "sensors", "")); err != nil {
t.Fatal(err)
}
settlePending(ctx, open, time.Now())
if slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
t.Fatal("a withdrawn assignment was made")
}
}
// A build that says nothing within the bound: the pending assignment expires at the next look, saying so,
// rather than waiting for ever; and a build heard by another process meanwhile is made at the next look.
// **A claim is never overridden** (review point 1): a pending assignment being made is claimed
// (waiting → applying); an unassign meanwhile is refused and says so, and leaves the claim; making a row
// already withdrawn makes nothing. A claim left by a controller that stopped is settled by the tick from what
// the mesh holds: back to waiting when the module is not assigned, applied when it is.
func TestAWithdrawalNeverOverridesAClaim(t *testing.T) {
open := aCatalogueMesh(t)
ctx := t.Context()
inv := open.inventory
asksWithPaths(t)
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
t.Fatal(err)
}
p := pendingOf(t, open)[0]
if ok, err := inv.ClaimPending(ctx, p.ID); err != nil || !ok {
t.Fatalf("claim: %v %v", ok, err)
}
_, err := unassign(ctx, open, "laptop", "sensors")
if err == nil || !strings.Contains(err.Error(), "is being assigned sensors now") {
t.Fatalf("unassign of a claimed pending assignment: %v", err)
}
if got := pendingOf(t, open)[0]; got.State != inventory.PendingApplying {
t.Fatalf("the claim was overridden: %+v", got)
}
// Pending(zero) is the waiting ones alone (review point 7).
if waiting, err := inv.Pending(ctx, time.Time{}); err != nil || len(waiting) != 0 {
t.Fatalf("Pending(zero) read a claimed row: %+v %v", waiting, err)
}
// Left by a controller that stopped: back to waiting, since sensors is not assigned.
settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute))
if got := pendingOf(t, open)[0]; got.State != inventory.PendingWaiting {
t.Fatalf("a stale claim was not released: %+v", got)
}
// Withdrawn, then the build registers it: applyPending finds nothing to claim.
if _, err := unassign(ctx, open, "laptop", "sensors"); err != nil {
t.Fatal(err)
}
register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"})
if line := applyPending(ctx, open, p, "b-modules/sensors"); line != "" {
t.Fatalf("a withdrawn pending assignment was made: %s", line)
}
if slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
t.Fatal("a withdrawn pending assignment was assigned")
}
// A stale claim whose module was assigned is applied.
q, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "sensors", Build: "b-x"})
if err != nil {
t.Fatal(err)
}
if ok, _ := inv.ClaimPending(ctx, q.ID); !ok {
t.Fatal("claim")
}
if _, err := inv.Assign(ctx, "anchor", "sensors"); err != nil {
t.Fatal(err)
}
settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute))
rows, _ := inv.PendingFor(ctx, "anchor", "sensors")
if len(rows) != 1 || rows[0].State != inventory.PendingApplied {
t.Fatalf("a stale claim of an assigned module: %+v", rows)
}
}
// A build that says nothing within the bound: the pending assignment expires at the next tick, saying so,
// rather than waiting for ever; and a module registered by a process that kept no pending assignment is
// assigned at the next tick.
func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) {
open := aCatalogueMesh(t)
ctx := t.Context()
@@ -295,15 +585,11 @@ func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) {
t.Fatal(err)
}
}
// Heard by a process that kept no pending assignment: registered, and nothing made it.
register(t, open, catalogue.Manifest{Module: "heard", Version: "1"})
said := settleStalePending(ctx, open, time.Now().Add(buildRequestBound))
if len(said) != 2 {
t.Fatalf("settled %v", said)
}
settlePending(ctx, open, time.Now().Add(buildRequestBound))
if !slices.Contains(assignedTo(t, open, "laptop"), "heard") {
t.Fatal("a module registered meanwhile was not assigned at the next look")
t.Fatal("a module registered meanwhile was not assigned at the next tick")
}
byModule := map[string]inventory.PendingAssignment{}
for _, p := range pendingOf(t, open) {
@@ -315,6 +601,21 @@ func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) {
if p := byModule["heard"]; p.State != inventory.PendingApplied {
t.Fatalf("a module registered meanwhile: %+v", p)
}
// Ended rows are deleted after KeptFor (review point 7); open ones never.
if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "lost", Build: "build-lost"}); err != nil {
t.Fatal(err)
}
said := settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour))
rows, err := inv.Pending(ctx, time.Unix(0, 0))
if err != nil {
t.Fatal(err)
}
for _, r := range rows {
if !r.Open() {
t.Fatalf("an ended pending assignment outlived %s: %+v (%v)", inventory.KeptFor, r, said)
}
}
}
// The seat's assign passes build on; unassign takes none.
@@ -327,3 +628,16 @@ func TestTheSeatsAssignPassesBuild(t *testing.T) {
t.Fatal("unassign took build")
}
}
// The condition's words are plain.
func TestAnAssignmentNotMadeIsSaidPlainly(t *testing.T) {
o := pendingObservation(inventory.PendingAssignment{Node: "g14", Module: "sensors",
Note: "the pending assignment of sensors to g14 expired: build b-1 of modules/sensors failed: boom"})
w := plainWordings[kindPendingEnded](o)
if why, ok := conditions.PlainWords(w, "g14"); !ok {
t.Fatalf("not plain: %s %+v", why, w)
}
if w.Headline != "sensors was not put on g14" {
t.Fatalf("headline %q", w.Headline)
}
}
+13
View File
@@ -153,6 +153,19 @@ var plainWordings = map[string]func(conditions.Observation) words{
"asleep is normal.", m),
Resolved: m + " can be reached again"}
}),
// A pending assignment that was not made (novox/hq issue 325, ADR 0261).
kindPendingEnded: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
module := idPart(o, 1)
if module == "" {
module = "a module"
}
return words{Headline: module + " was not put on " + m,
Needs: "decide whether to build it again or take the assignment back; the details say why.",
Explanation: fmt.Sprintf("An assignment of %s to %s waited for its build, and the build did not "+
"register it, so it was not made.", module, m),
Resolved: "Resolved: " + module + " on " + m + " is settled"}
}),
kindBindingKept: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: "A module's data source is held on " + m,
+4
View File
@@ -165,6 +165,10 @@ func serve(ctx context.Context) (err error) {
// Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for
// machines to report moves when they have, and a plan left by a replaced controller resumes.
go planTicker(ctx, open)
// And the pending assignments settled on a tick of their own (novox/hq ADR 0261): made once their module
// is registered, ended with why when its build will not register it, raised and cleared as conditions.
// Never by a read.
go settlingPending(ctx, open)
// The durations the core's bounds are set from are kept a month (novox/hq to-be 45 Phase 0).
go forgettingOldDurations(ctx, inv)
// And what the catalogue decided a build meant. The builder's own result is already handled
+7 -1
View File
@@ -406,6 +406,8 @@ func rebuildCommand(ctx context.Context, args []string) error {
if err != nil {
return err
}
recordAsked(ctx, inventory.BuildRequest{ID: id, Repository: source.Repository, Seat: source.Seat, Path: path,
Ref: ref, For: "rebuild"})
fmt.Printf("rebuild asked as %s\n", id)
return nil
}
@@ -490,7 +492,11 @@ func replayCommand(ctx context.Context, args []string) error {
if err := replayRefusal(b, module, history, *register, *older, outstanding); err != nil {
return err
}
id, err := buildOneAsked(ctx, source, b.Path, b.Commit, 0, !*register)
asker := ""
if *register {
asker = "replay"
}
id, err := buildOneAsked(ctx, source, b.Path, b.Commit, 0, !*register, asker)
if err != nil {
return err
}
+4 -1
View File
@@ -344,7 +344,8 @@ var askABuild func(ctx context.Context, source buildSource, path, ref string) (s
func init() {
askABuild = func(ctx context.Context, source buildSource, path, ref string) (string, error) {
return buildOneAsked(ctx, source, path, ref, 0, false)
// Kept by each asker with its own name once the ask is made (novox/hq issue 325).
return buildOneAsked(ctx, source, path, ref, 0, false, "")
}
}
@@ -376,6 +377,8 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
p.Note = fmt.Sprintf("%s could not be asked for: %v", name, err)
return
}
recordAsked(ctx, inventory.BuildRequest{ID: id, Repository: e.Source.Repository, Seat: e.Source.Seat,
Path: e.Source.Path, Ref: followedBranch(e.Source.Ref), Commit: p.Commit, For: "plan"})
state.State = "asked"
state.AskedAt = &now
state.Build = id
+8 -11
View File
@@ -4,7 +4,6 @@ import (
"context"
"flag"
"fmt"
"os"
"sort"
"strings"
"time"
@@ -469,11 +468,8 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
if out.conditions, err = openConditions(ctx); err != nil {
out.conditionsUnread = err.Error()
}
// And the assignments waiting for their module's build (novox/hq issue 325): what a missed outcome left
// waiting is settled first, so one never waits silently past its build.
for _, line := range settleStalePending(ctx, open, time.Now()) {
fmt.Fprintln(os.Stderr, line)
}
// And the assignments waiting for their module's build (novox/hq issue 325, ADR 0261), read only: the
// controller's tick settles them, never a read.
if out.pending, err = inv.Pending(ctx, time.Now().Add(-pendingShownFor)); err != nil {
out.pendingUnread = err.Error()
err = nil
@@ -610,13 +606,14 @@ func (a answers) well() bool {
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 &&
len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingNotWell(a.pending)
len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending)
}
// pendingNotWell is whether any pending assignment is waiting, or ended without being made.
func pendingNotWell(pending []inventory.PendingAssignment) bool {
// pendingOpen is whether any pending assignment is still to be made. One that ended without being made is
// not counted here: it is a condition, open until it is answered (ADR 0261).
func pendingOpen(pending []inventory.PendingAssignment) bool {
for _, p := range pending {
if p.State != inventory.PendingApplied && p.State != inventory.PendingWithdrawn {
if p.Open() {
return true
}
}
@@ -632,7 +629,7 @@ func printPending(pending []inventory.PendingAssignment, unread string) {
}
var waiting, ended []inventory.PendingAssignment
for _, p := range pending {
if p.State == inventory.PendingWaiting {
if p.Open() {
waiting = append(waiting, p)
} else {
ended = append(ended, p)
+5 -4
View File
@@ -179,10 +179,11 @@ var ControllerVerbs = []Verb{
}, []string{"node"}, "files", "diff")},
{Name: "assign", Description: "Put a module on a machine. Refused with the mesh's own words when it cannot resolve there, " +
"or when a seat its resources are applied through is held by nothing on the machine (novox/hq ADR 0207). " +
"A module not registered yet is looked for (novox/hq issue 325): while its build runs — a merge that adds " +
"a module asks for one — the assignment is kept as pending and made when the build registers it, said in " +
"`status`, and expires with why if the build fails; a module known and not built is said, and build asks " +
"for its build; a name the mesh never heard of is refused with the closest names.",
"A module not registered yet is looked for (novox/hq issue 325, ADR 0261): while its build runs — a merge " +
"that adds a module asks for one — the assignment is kept as pending and the controller makes it when the " +
"build registers it; `status` lists it, and if the build fails it expires with why and raises a condition. " +
"A module known and not built is said, and build asks for its build; a name with no module and no build " +
"request kept is refused with the closest names.",
Input: schema(map[string]string{"node": "the machine's name",
"module": "the module's name; several comma-separated are judged together",
"build": "\"true\": for a module known and not built (its last build failed, or was never asked), ask " +
@@ -1,15 +1,15 @@
-- An assignment waits for its module's build (novox/hq issue 325).
-- An assignment waits for its module's build (novox/hq issue 325, ADR 0261).
--
-- A merge that adds a module asks for its build (issue 300), and the module is registered when the build's
-- outcome is taken in, minutes later. An `assign` in between was answered "no module of that name", which
-- read as "nobody registered it". The controller now keeps every build it asks for, so `assign` can tell a
-- build in flight from a module it never heard of, and an assignment made while the build runs is kept as
-- pending and made when the build registers the module, or ended with the reason when it does not.
-- pending and made by the controller when the build registers the module, or ended with the reason.
-- Every build the controller asked for, whatever asked it: a merge, a plan's tier, a person's `build`, an
-- `assign` with build. Its outcome is the `build` row of the same id, written when the outcome is taken in;
-- an ask without one is still running, or was lost. not_asked is why a merge's ask of a new module could
-- not be made, and the id is then not a build's.
-- Every build the controller asked for: a merge's new module, a plan's tier, a person's `build`, an `assign`
-- with build. Kept once the ask is made, never before. Its outcome is the `build` row of the same id;
-- an ask without one is still running, or was lost. not_asked is why the ask could not be made or waited
-- for; for a merge that could not ask, the id is the controller's and no build's.
create table build_request (
id text primary key,
repository text not null,
@@ -23,20 +23,27 @@ create table build_request (
);
create index build_request_asked_at on build_request (asked_at);
-- An assignment kept until its module is registered. waiting until the build registers the module, then
-- applied (the assignment was made), refused (the assignment was refused then, with the refusal), expired
-- (the build failed, was not registered, or said nothing within its bound) or withdrawn (`unassign`).
-- An assignment kept until its module is registered (ADR 0261). waiting until the build registers the
-- module; applying while the controller makes it, under the machine's hold; then applied, refused (the
-- assignment was refused then), expired (the build failed, was not registered, or said nothing within its
-- bound) or withdrawn (`unassign`). raised_at is when an expiry or refusal was raised as a condition,
-- acknowledged_at when a person took it back with `unassign`, cleared_at when its condition was cleared.
-- Gone with its machine. Ended rows are deleted after 30 days.
create table pending_assignment (
id bigserial primary key,
node text not null,
module text not null,
build_id text not null,
repository text not null default '',
source_path text not null default '',
since timestamptz not null default now(),
state text not null default 'waiting'
check (state in ('waiting', 'applied', 'refused', 'expired', 'withdrawn')),
note text not null default '',
settled_at timestamptz
id bigserial primary key,
node uuid not null references node (id) on delete cascade,
module text not null,
build_id text not null,
repository text not null default '',
source_path text not null default '',
since timestamptz not null default now(),
state text not null default 'waiting'
check (state in ('waiting', 'applying', 'applied', 'refused', 'expired', 'withdrawn')),
note text not null default '',
settled_at timestamptz,
raised_at timestamptz,
acknowledged_at timestamptz,
cleared_at timestamptz
);
create unique index pending_assignment_waiting on pending_assignment (node, module) where state = 'waiting';
create unique index pending_assignment_open on pending_assignment (node, module)
where state in ('waiting', 'applying');
+190 -64
View File
@@ -10,36 +10,36 @@ import (
"github.com/jackc/pgx/v5"
)
// What the controller asked to be built, and the assignments waiting for it (novox/hq issue 325).
// What the controller asked to be built, and the assignments waiting for it (novox/hq issue 325, ADR 0261).
//
// A build's outcome was kept, and its asking was not: between a merge asking for a new module and the
// outcome registering it, the mesh had no record that anything was coming, and `assign` answered "no module
// of that name" for a module minutes from existing. Kept here, an ask lets `assign` say which case it is in,
// and an assignment made while the build runs is kept until the build registers the module.
// of that name" for a module minutes from existing. Kept here, a build request lets `assign` say which case
// it is in, and an assignment made while the build runs is kept until the build registers the module.
// requestKeptFor is how long an ask is kept: long past any build's bound, short enough to stay small.
const requestKeptFor = 30 * 24 * time.Hour
// KeptFor is how long a build request, and an ended pending assignment, is kept.
const KeptFor = 30 * 24 * time.Hour
// BuildRequest is one build the controller asked for.
type BuildRequest struct {
// ID is the build's id, as its outcome will carry it; for an ask that could not be made, an id of
// the controller's making that no build carries.
// ID is the build's id, as its outcome will carry it; for a merge whose ask could not be made, an id
// of the controller's making that no build carries.
ID string
// Repository and Seat are the source as the mesh spells it (novox/hq ADR 0111): a path on the seat's
// holder, or a URL with no seat. Path is the module's directory in it, Ref the branch or commit asked.
Repository, Seat, Path, Ref string
// Commit is the commit that made the ask, where one did: a merge's.
Commit string
// For says what asked: "merge", "build", "plan", "assign".
// For says who asked: "merge", "plan", "build" (a person), "assign".
For string
// NotAsked is why the ask could not be made, empty when it was.
// NotAsked is why the ask could not be made, or not waited for; empty when it was.
NotAsked string
At time.Time
}
// Name is the module this ask is expected to register, read from its directory: the last element of its
// path, or of its repository for a module at the repository's root. A guess until the build says, and the
// one a person reads too: `modules/sensors` is sensors.
// Name is the module this request is expected to register, read from its directory: the last element of
// its path, or of its repository for a module at the repository's root. A guess until the build says, and
// the one a person reads too: `modules/sensors` is sensors.
func (a BuildRequest) Name() string {
if p := strings.Trim(a.Path, "/"); p != "" && p != "." {
return path.Base(p)
@@ -47,19 +47,19 @@ func (a BuildRequest) Name() string {
return strings.TrimSuffix(path.Base(strings.TrimRight(a.Repository, "/")), ".git")
}
// RequestOutcome is an ask beside what came of it, where anything did.
// RequestOutcome is a build request beside what came of it, where anything did.
type RequestOutcome struct {
BuildRequest
// Heard is whether an outcome of this id was taken in; Failed is its builder's words when it failed,
// Module what it registered as, HeardAt when it was taken in.
// Module what the build said it built, HeardAt when it was taken in.
Heard bool
Failed string
Module string
HeardAt time.Time
}
// RecordBuildRequest keeps one ask. Idempotent on the id: a merge's ask recorded by the merge and by the
// asking itself is one ask. Asks older than requestKeptFor are let go in the same act.
// RecordBuildRequest keeps one build request, once it is asked. Idempotent on the id. Requests older than
// KeptFor are deleted in the same act.
func (i *Inventory) RecordBuildRequest(ctx context.Context, a BuildRequest) error {
at := a.At
if at.IsZero() {
@@ -72,17 +72,25 @@ func (i *Inventory) RecordBuildRequest(ctx context.Context, a BuildRequest) erro
if _, err := i.store.Pool().Exec(ctx,
`insert into build_request (id, repository, seat, source_path, ref, commit_hash, asked_for, not_asked, asked_at)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9)
on conflict (id) do update set
commit_hash = case when build_request.commit_hash = '' then excluded.commit_hash else build_request.commit_hash end,
asked_for = case when build_request.asked_for = '' then excluded.asked_for else build_request.asked_for end`,
on conflict (id) do nothing`,
a.ID, a.Repository, a.Seat, strings.Trim(a.Path, "/"), a.Ref, a.Commit, a.For, notAsked, at); err != nil {
return err
}
_, err := i.store.Pool().Exec(ctx, `delete from build_request where asked_at < $1`, time.Now().Add(-requestKeptFor))
_, err := i.store.Pool().Exec(ctx, `delete from build_request where asked_at < $1`, time.Now().Add(-KeptFor))
return err
}
// RequestsNamed is every ask kept whose directory names the module, newest first, each with its outcome.
// MarkNotAsked says a build request was asked and its asker could not hand it over, or stopped waiting for
// it: the words are kept, unless its outcome was heard first.
func (i *Inventory) MarkNotAsked(ctx context.Context, id, why string) error {
_, err := i.store.Pool().Exec(ctx,
`update build_request set not_asked = $2
where id = $1 and not exists (select 1 from build where build.id = $1)`, id, why)
return err
}
// RequestsNamed is every build request kept whose directory names the module, or whose build said it built
// it, newest first, each with its outcome.
func (i *Inventory) RequestsNamed(ctx context.Context, name string) ([]RequestOutcome, error) {
all, err := i.requests(ctx)
if err != nil {
@@ -97,8 +105,8 @@ func (i *Inventory) RequestsNamed(ctx context.Context, name string) ([]RequestOu
return out, nil
}
// RequestedNames is the name of every module an ask kept is expected to register: what `assign` offers as a
// closest name beside the modules registered.
// RequestedNames is the name of every module a kept build request is expected to register: what `assign`
// offers as a closest name beside the modules registered.
func (i *Inventory) RequestedNames(ctx context.Context) ([]string, error) {
all, err := i.requests(ctx)
if err != nil {
@@ -141,13 +149,14 @@ func (i *Inventory) requests(ctx context.Context) ([]RequestOutcome, error) {
// The states of a pending assignment.
const (
PendingWaiting = "waiting"
PendingApplying = "applying"
PendingApplied = "applied"
PendingRefused = "refused"
PendingExpired = "expired"
PendingWithdrawn = "withdrawn"
)
// PendingAssignment is an assignment kept until its module is registered.
// PendingAssignment is an assignment kept until its module is registered (ADR 0261).
type PendingAssignment struct {
ID int64
Node string
@@ -159,22 +168,48 @@ type PendingAssignment struct {
// Note is what ended it, in the words said when it ended.
Note string
Settled *time.Time
// Raised is when an expiry or refusal was raised as a condition; Acknowledged when a person took it
// back; Cleared when its condition was cleared.
Raised, Acknowledged, Cleared *time.Time
}
// Open is whether the pending assignment is still to be made.
func (p PendingAssignment) Open() bool {
return p.State == PendingWaiting || p.State == PendingApplying
}
// ErrAlreadyPending is a second pending assignment of one module to one machine.
var ErrAlreadyPending = errors.New("that assignment is already waiting for its build")
// RecordPending keeps an assignment until its module is registered. One waits per machine and module.
const pendingColumns = `p.id, n.name, p.module, p.build_id, p.repository, p.source_path, p.since, p.state, p.note,
p.settled_at, p.raised_at, p.acknowledged_at, p.cleared_at`
func scanPending(rows pgx.Rows) ([]PendingAssignment, error) {
defer rows.Close()
var out []PendingAssignment
for rows.Next() {
var p PendingAssignment
if err := rows.Scan(&p.ID, &p.Node, &p.Module, &p.Build, &p.Repository, &p.Path, &p.Since, &p.State,
&p.Note, &p.Settled, &p.Raised, &p.Acknowledged, &p.Cleared); err != nil {
return nil, err
}
out = append(out, p)
}
return out, rows.Err()
}
// RecordPending keeps an assignment until its module is registered. One is open per machine and module.
func (i *Inventory) RecordPending(ctx context.Context, p PendingAssignment) (PendingAssignment, error) {
if _, err := i.NodeByName(ctx, p.Node); err != nil {
node, err := i.NodeByName(ctx, p.Node)
if err != nil {
return PendingAssignment{}, err
}
err := i.store.Pool().QueryRow(ctx,
err = i.store.Pool().QueryRow(ctx,
`insert into pending_assignment (node, module, build_id, repository, source_path)
values ($1, $2, $3, $4, $5)
on conflict (node, module) where state = 'waiting' do nothing
on conflict (node, module) where state in ('waiting', 'applying') do nothing
returning id, since`,
p.Node, p.Module, p.Build, p.Repository, strings.Trim(p.Path, "/")).Scan(&p.ID, &p.Since)
node.ID, p.Module, p.Build, p.Repository, strings.Trim(p.Path, "/")).Scan(&p.ID, &p.Since)
if errors.Is(err, pgx.ErrNoRows) {
return PendingAssignment{}, ErrAlreadyPending
}
@@ -185,44 +220,135 @@ func (i *Inventory) RecordPending(ctx context.Context, p PendingAssignment) (Pen
return p, nil
}
// Pending is every pending assignment still waiting, and every one ended since the time given, oldest
// first. A zero time is the waiting ones alone.
func (i *Inventory) Pending(ctx context.Context, endedSince time.Time) ([]PendingAssignment, error) {
rows, err := i.store.Pool().Query(ctx,
`select id, node, module, build_id, repository, source_path, since, state, note, settled_at
from pending_assignment
where state = 'waiting' or (settled_at is not null and settled_at >= $1)
order by since, id`, endedSince)
if err != nil {
return nil, err
}
defer rows.Close()
var out []PendingAssignment
for rows.Next() {
var p PendingAssignment
if err := rows.Scan(&p.ID, &p.Node, &p.Module, &p.Build, &p.Repository, &p.Path, &p.Since, &p.State,
&p.Note, &p.Settled); err != nil {
return nil, err
}
if endedSince.IsZero() && p.State != PendingWaiting {
continue
}
out = append(out, p)
}
return out, rows.Err()
}
// SettlePending ends a waiting pending assignment. False when it was no longer waiting: another process
// settled it first, and its word stands.
func (i *Inventory) SettlePending(ctx context.Context, id int64, state, note string) (bool, error) {
if state == PendingWaiting {
return false, errors.New("a pending assignment is settled to applied, refused, expired or withdrawn")
}
// RepointPending makes a waiting pending assignment wait for another build. False when it no longer waits.
func (i *Inventory) RepointPending(ctx context.Context, id int64, build, repository, sourcePath string) (bool, error) {
tag, err := i.store.Pool().Exec(ctx,
`update pending_assignment set state = $2, note = $3, settled_at = now()
where id = $1 and state = 'waiting'`, id, state, note)
`update pending_assignment set build_id = $2, repository = $3, source_path = $4
where id = $1 and state = 'waiting'`, id, build, repository, strings.Trim(sourcePath, "/"))
if err != nil {
return false, err
}
return tag.RowsAffected() == 1, nil
}
// Pending is every pending assignment still waiting, oldest first, when endedSince is zero. Otherwise it is
// every open one and every one ended since then.
func (i *Inventory) Pending(ctx context.Context, endedSince time.Time) ([]PendingAssignment, error) {
if endedSince.IsZero() {
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
from pending_assignment p join node n on n.id = p.node
where p.state = 'waiting' order by p.since, p.id`)
if err != nil {
return nil, err
}
return scanPending(rows)
}
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
from pending_assignment p join node n on n.id = p.node
where p.state in ('waiting', 'applying') or p.settled_at >= $1
order by p.since, p.id`, endedSince)
if err != nil {
return nil, err
}
return scanPending(rows)
}
// PendingFor is every pending assignment of one module to one machine, newest first.
func (i *Inventory) PendingFor(ctx context.Context, node, module string) ([]PendingAssignment, error) {
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
from pending_assignment p join node n on n.id = p.node
where n.name = $1 and p.module = $2 order by p.since desc, p.id desc`, node, module)
if err != nil {
return nil, err
}
return scanPending(rows)
}
// Stuck is every pending assignment left applying since before the time given: a controller that claimed
// it and stopped before it said what came of it.
func (i *Inventory) Stuck(ctx context.Context, before time.Time) ([]PendingAssignment, error) {
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
from pending_assignment p join node n on n.id = p.node
where p.state = 'applying' and p.settled_at < $1 order by p.id`, before)
if err != nil {
return nil, err
}
return scanPending(rows)
}
// ToRaise is every pending assignment that expired or was refused and has not been raised as a condition;
// ToClear every one raised and not cleared.
func (i *Inventory) ToRaise(ctx context.Context) ([]PendingAssignment, error) {
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
from pending_assignment p join node n on n.id = p.node
where p.state in ('expired', 'refused') and p.raised_at is null order by p.id`)
if err != nil {
return nil, err
}
return scanPending(rows)
}
func (i *Inventory) ToClear(ctx context.Context) ([]PendingAssignment, error) {
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
from pending_assignment p join node n on n.id = p.node
where p.raised_at is not null and p.cleared_at is null order by p.id`)
if err != nil {
return nil, err
}
return scanPending(rows)
}
// MarkPending stamps a pending assignment raised, acknowledged or cleared, now.
func (i *Inventory) MarkPending(ctx context.Context, id int64, what string) error {
column := map[string]string{"raised": "raised_at", "acknowledged": "acknowledged_at", "cleared": "cleared_at"}[what]
if column == "" {
return errors.New("a pending assignment is marked raised, acknowledged or cleared")
}
_, err := i.store.Pool().Exec(ctx,
`update pending_assignment set `+column+` = coalesce(`+column+`, now()) where id = $1`, id)
return err
}
// ClaimPending takes a waiting pending assignment for making, so nothing else settles or withdraws it while
// it is made. False when it no longer waits: another process took it, or it was withdrawn.
func (i *Inventory) ClaimPending(ctx context.Context, id int64) (bool, error) {
tag, err := i.store.Pool().Exec(ctx,
`update pending_assignment set state = 'applying', settled_at = now() where id = $1 and state = 'waiting'`, id)
if err != nil {
return false, err
}
return tag.RowsAffected() == 1, nil
}
// ReleaseClaim puts a claimed pending assignment back to waiting.
func (i *Inventory) ReleaseClaim(ctx context.Context, id int64) error {
_, err := i.store.Pool().Exec(ctx,
`update pending_assignment set state = 'waiting', settled_at = null where id = $1 and state = 'applying'`, id)
return err
}
// SettlePending ends a pending assignment in the state it is read in (from): waiting for an expiry or a
// withdrawal, applying for the outcome of making it. False when it was no longer in that state.
func (i *Inventory) SettlePending(ctx context.Context, id int64, from, state, note string) (bool, error) {
if state == PendingWaiting || state == PendingApplying {
return false, errors.New("a pending assignment is settled to applied, refused, expired or withdrawn")
}
tag, err := i.store.Pool().Exec(ctx,
`update pending_assignment set state = $3, note = $4, settled_at = now()
where id = $1 and state = $2`, id, from, state, note)
if err != nil {
return false, err
}
return tag.RowsAffected() == 1, nil
}
// ForgetEndedPending deletes ended pending assignments settled more than KeptFor ago, and says how many.
func (i *Inventory) ForgetEndedPending(ctx context.Context, now time.Time) (int64, error) {
tag, err := i.store.Pool().Exec(ctx,
`delete from pending_assignment where state not in ('waiting', 'applying') and settled_at < $1`,
now.Add(-KeptFor))
if err != nil {
return 0, err
}
return tag.RowsAffected(), nil
}
+25
View File
@@ -0,0 +1,25 @@
package inventory
import (
"testing"
"time"
)
// A pending assignment goes with its machine (novox/hq issue 325, review of mesh-controller#150 point 7).
func TestAPendingAssignmentGoesWithItsMachine(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
if _, err := inv.AddNode(ctx, "leaving"); err != nil {
t.Fatal(err)
}
if _, err := inv.RecordPending(ctx, PendingAssignment{Node: "leaving", Module: "sensors", Build: "b-1"}); err != nil {
t.Fatal(err)
}
if _, err := inv.store.Pool().Exec(ctx, `delete from node where name = 'leaving'`); err != nil {
t.Fatalf("a machine with a pending assignment could not be removed: %v", err)
}
rows, err := inv.Pending(ctx, time.Unix(0, 0))
if err != nil || len(rows) != 0 {
t.Fatalf("a pending assignment outlived its machine: %+v %v", rows, err)
}
}