Make pending assignments safe to race, settle them on a tick, and say only what was checked

Review of #150: a withdrawal could land between the look and the act, a
failed ask read as a build in flight, a request kept the wrong asker, a
build being registered read as not built, build "true" could ask a build
nothing waited on, and a status read changed state. Claim a row under the
machine's hold before making it, keep a request only once asked, settle on
the controller's own tick, raise an assignment not made as a condition
until it is answered, and tie each row to its machine.
This commit is contained in:
jochen
2026-10-08 16:38:11 +02:00
parent 7d63d2e68c
commit 249d97d1c8
12 changed files with 1018 additions and 235 deletions
+47 -13
View File
@@ -399,15 +399,43 @@ func buildBehind(ctx context.Context, wait time.Duration) error {
// //
// Separated from the command so `--behind` can walk a list without a second path to the same act. // Separated from the command so `--behind` can walk a list without a second path to the same act.
func buildOne(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error { func buildOne(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
_, err := buildOneAsked(ctx, source, path, ref, wait, false) _, err := buildOneAsked(ctx, source, path, ref, wait, false, "build")
return err return err
} }
// recordAsked keeps a build request once it is asked (novox/hq issue 325), with who asked it, in a store
// opened for the purpose: the asks reach here from processes that hold none.
func recordAsked(ctx context.Context, r inventory.BuildRequest) {
open, err := openStores(ctx)
if err != nil {
fmt.Fprintf(os.Stderr, "build %s is asked and not kept as a build request: %v\n", r.ID, err)
return
}
defer open.Close()
recordBuildRequest(ctx, open.inventory, r)
}
// markNotAsked says a kept build request was not handed over, or not waited for.
func markNotAsked(ctx context.Context, id string, why error) {
open, err := openStores(ctx)
if err != nil {
fmt.Fprintf(os.Stderr, "build %s could not be marked as not asked: %v\n", id, err)
return
}
defer open.Close()
if err := open.inventory.MarkNotAsked(ctx, id, why.Error()); err != nil {
fmt.Fprintf(os.Stderr, "build %s could not be marked as not asked: %v\n", id, err)
}
}
// buildOneAsked is buildOne answering the id it asked with — what a plan keeps to match the outcome // buildOneAsked is buildOne answering the id it asked with — what a plan keeps to match the outcome
// by (novox/hq ADR 0219) — and, for an ask not waited for, optionally a dry run: built and looked // by (novox/hq ADR 0219) — and, for an ask not waited for, optionally a dry run: built and looked
// at, never taken in (issue 240), which is what `replay` asks unless told to register. // at, never taken in (issue 240), which is what `replay` asks unless told to register.
//
// asker is who asked, for the build request kept once the ask is made (novox/hq issue 325); empty for an
// asker that keeps the request itself, with its own name, once it knows the ask was made.
func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wait time.Duration, func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wait time.Duration,
dryRun bool) (string, error) { dryRun bool, asker string) (string, error) {
if dryRun && wait != 0 { if dryRun && wait != 0 {
return "", errors.New("a dry run waited for is `build --dry-run`") return "", errors.New("a dry run waited for is `build --dry-run`")
} }
@@ -462,17 +490,12 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
} }
defer ask.Close() defer ask.Close()
fmt.Printf(" of %s\n", seat) fmt.Printf(" of %s\n", seat)
// Kept before it is asked, so `assign` knows a module is coming while its build runs (novox/hq issue 325). // Kept once it is asked, so `assign` knows a module is coming while its build runs (novox/hq issue 325);
// A dry run registers nothing, and is not kept. // never before, so an ask that failed never reads as a build in flight. A dry run registers nothing, and
if !dryRun { // is not kept.
if open, err := openStores(ctx); err == nil { keep := !dryRun && asker != ""
recordBuildRequest(ctx, open.inventory, inventory.BuildRequest{ID: request.ID, asked := inventory.BuildRequest{ID: request.ID, Repository: source.Repository, Seat: source.Seat, Path: path,
Repository: source.Repository, Seat: source.Seat, Path: path, Ref: ref, For: "build"}) Ref: ref, For: asker}
open.Close()
} else {
fmt.Fprintf(os.Stderr, "build %s is asked and not kept as a build request: %v\n", request.ID, err)
}
}
if wait == 0 { if wait == 0 {
// Asked and not waited for (novox/hq issue 176): the outcome is the role's event, and the // Asked and not waited for (novox/hq issue 176): the outcome is the role's event, and the
@@ -482,6 +505,9 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
if err := ask.Ask(ctx, request); err != nil { if err := ask.Ask(ctx, request); err != nil {
return "", err return "", err
} }
if keep {
recordAsked(ctx, asked)
}
if dryRun { if dryRun {
fmt.Printf("asked as a dry run, not waited for: `builds --log %s` follows it as it runs; "+ fmt.Printf("asked as a dry run, not waited for: `builds --log %s` follows it as it runs; "+
"its outcome is not taken in\n", request.ID) "its outcome is not taken in\n", request.ID)
@@ -492,8 +518,16 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
return request.ID, nil return request.ID, nil
} }
// Waited for: kept while it is waited for, since that can take minutes, and marked when the hand-over or
// the wait failed — an outcome heard later is still the last word.
if keep {
recordAsked(ctx, asked)
}
result, err := ask.Submit(ctx, request, wait) result, err := ask.Submit(ctx, request, wait)
if err != nil { if err != nil {
if keep {
markNotAsked(ctx, request.ID, err)
}
return request.ID, err return request.ID, err
} }
+346 -93
View File
@@ -9,11 +9,12 @@ import (
"strings" "strings"
"time" "time"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link" "github.com/novox/mesh-controller/internal/link"
) )
// A module not registered yet, and the assignment that waits for it (novox/hq issue 325). // A module not registered yet, and the assignment that waits for it (novox/hq issue 325, ADR 0261).
// //
// On 2026-10-08 the catalogue's pull request adding `sensors` merged; a minute later `assign g14 sensors` // On 2026-10-08 the catalogue's pull request adding `sensors` merged; a minute later `assign g14 sensors`
// answered "no module of that name: sensors", and a few minutes later the same call worked. The merge had // answered "no module of that name: sensors", and a few minutes later the same call worked. The merge had
@@ -23,34 +24,55 @@ import (
// So an assignment of a module the catalogue does not hold looks further before it refuses, and says which // So an assignment of a module the catalogue does not hold looks further before it refuses, and says which
// of three cases it is in: // of three cases it is in:
// //
// - **a build is in flight**: a build request for the module's directory has no outcome yet. The // - **a build is in flight**: a build request for the module's directory has no outcome yet, or its build
// assignment is kept as a **pending assignment** and made when the build registers the module. // succeeded a moment ago and is being registered. The assignment is kept as a **pending assignment**,
// - **known, not built**: the mesh asked for the directory before, and the build failed, was not // which the controller makes when the build registers the module (ADR 0261).
// - **known, not built**: the controller asked for the directory before, and the build failed, was not
// registered, said nothing within its bound, or could not be asked. Said, with `build` and assign's // registered, said nothing within its bound, or could not be asked. Said, with `build` and assign's
// own build argument, which asks for the build and keeps the assignment pending on it. // own build argument, which asks for the build and keeps the assignment pending on it.
// - **unknown**: no module, build request or merge by that name. Refused as before, with the closest names. // - **unknown**: no module registered and no build request kept by that name. Refused as before, with the
// closest names.
// //
// **Pending by default while a build is in flight**, without an argument to ask for it: an assignment is // **Pending by default while a build is in flight** (ADR 0261), without an argument to ask for it: an
// what a person meant and is kept even when its machine does not resolve (acts.go), nothing is sent until a // assignment is what a person meant and is kept even when its machine does not resolve (acts.go), and
// push, and `unassign` withdraws it. Asking for a second act once the build lands is exactly the two acts by // `unassign` withdraws it. Asking for a second act once the build lands is the two acts by hand issue 300
// hand issue 300 removed. A build is never asked for by default: it runs on another machine, and a directory // removed. A build is never asked for by default: it runs on another machine, and a directory whose last
// whose last build failed is a fault to look at before it is a thing to retry. // build failed is a fault to look at before it is a thing to retry.
//
// **Settling is the controller's tick, never a read** (settlingPending): `status`, the board and the summary
// only read the rows.
// buildRequestBound is how long a build request may go without an outcome before it is no longer read as in // buildRequestBound is how long a build request may go without an outcome before it is no longer read as in
// flight, and a pending assignment waiting for it expires. Generous: a build waits behind others on the // flight, and a pending assignment waiting for it expires. Generous: a build waits behind others on the
// build seat, and a pending assignment that waits a little longer costs nothing. // build seat, and a pending assignment that waits a little longer costs nothing.
const buildRequestBound = 2 * time.Hour const buildRequestBound = 2 * time.Hour
// pendingShownFor is how long an ended pending assignment is said in `status`. // registerGrace is how long a build heard as built is read as still in flight while it is not registered:
// the outcome is recorded first and registered after it, in the same take-in. Past it, the build was
// recorded and not registered, and says so.
const registerGrace = 5 * time.Minute
// pendingShownFor is how long an ended pending assignment is listed in `status`.
const pendingShownFor = 24 * time.Hour const pendingShownFor = 24 * time.Hour
// settleEvery is how often the controller settles the pending assignments.
const settleEvery = time.Minute
// claimStaleAfter is how long a pending assignment may be held as applying before the tick reads the claim
// as left by a controller that stopped, and settles it from what the mesh holds.
const claimStaleAfter = 5 * time.Minute
// kindPendingEnded is a pending assignment that ended without being made: expired or refused.
const kindPendingEnded = "assignment-not-made"
// assignOptions are what an assignment was asked with beside its machine and modules. // assignOptions are what an assignment was asked with beside its machine and modules.
type assignOptions struct { type assignOptions struct {
// Build asks for the build of a module known and not built, and keeps the assignment pending on it. // Build asks for the build of a module known and not built, and keeps the assignment pending on it.
Build bool Build bool
} }
// recordBuildRequest keeps a build request so `assign` can find it. Said, never fatal: the build was asked. // recordBuildRequest keeps a build request so `assign` can find it, once it is asked. Said, never fatal: the
// build was asked.
func recordBuildRequest(ctx context.Context, inv *inventory.Inventory, r inventory.BuildRequest) { func recordBuildRequest(ctx context.Context, inv *inventory.Inventory, r inventory.BuildRequest) {
if err := inv.RecordBuildRequest(ctx, r); err != nil { if err := inv.RecordBuildRequest(ctx, r); err != nil {
fmt.Fprintf(os.Stderr, "build %s was asked, and could not be kept as a build request, so `assign` "+ fmt.Fprintf(os.Stderr, "build %s was asked, and could not be kept as a build request, so `assign` "+
@@ -73,25 +95,30 @@ type notHeld struct {
said string said string
} }
// whereIs looks for a module the catalogue does not hold among the build requests the mesh keeps. // whereIs looks for a module the catalogue does not hold among the build requests the controller keeps.
func whereIs(ctx context.Context, inv *inventory.Inventory, module string, now time.Time) (notHeld, error) { func whereIs(ctx context.Context, inv *inventory.Inventory, module string, now time.Time) (notHeld, error) {
requests, err := inv.RequestsNamed(ctx, module) requests, err := inv.RequestsNamed(ctx, module)
if err != nil { if err != nil {
return notHeld{}, err return notHeld{}, err
} }
if r, ok := inFlight(requests, now); ok { if r, ok := inFlight(requests, now); ok {
return notHeld{kind: buildInFlight, request: r, said: fmt.Sprintf("%s is not registered yet: it is being "+ being := "being built"
"built from %s since %s, as build %s; it can be assigned once that build registers it", module, if r.Heard {
being = "built and being registered"
}
return notHeld{kind: buildInFlight, request: r, said: fmt.Sprintf("%s is not registered yet: it is %s "+
"from %s since %s, as build %s; it can be assigned once that build registers it", module, being,
requestSource(r), clock(r.At), r.ID)}, nil requestSource(r), clock(r.At), r.ID)}, nil
} }
if len(requests) > 0 { if len(requests) > 0 {
return notHeld{kind: knownNotBuilt, request: requests[0], said: fmt.Sprintf("%s is known and not "+ return notHeld{kind: knownNotBuilt, request: requests[0], said: fmt.Sprintf("%s is known and not "+
"registered: %s", module, whyNotBuilt(requests[0], now))}, nil "registered: %s", module, whyNotBuilt(requests[0], now))}, nil
} }
said := fmt.Sprintf("%v: %s — the mesh holds no module, build request or merge by that name", // Only what was looked at is claimed: the catalogue, and the build requests the controller keeps.
inventory.ErrNoSuchModule, module) said := fmt.Sprintf("%v: %s — the catalogue holds no module of that name, and no build request the "+
"controller kept (the last 30 days) is for a directory of that name", inventory.ErrNoSuchModule, module)
if near := closestNames(ctx, inv, module); len(near) > 0 { if near := closestNames(ctx, inv, module); len(near) > 0 {
said += "; the closest it holds: " + strings.Join(near, ", ") said += "; the closest names it holds: " + strings.Join(near, ", ")
} }
return notHeld{kind: unknownModule, said: said}, nil return notHeld{kind: unknownModule, said: said}, nil
} }
@@ -111,6 +138,20 @@ func notInCatalogue(ctx context.Context, open *stores, modules []string) ([]stri
return missing, nil return missing, nil
} }
// openPending is the open pending assignment of a module to a machine, if there is one.
func openPending(ctx context.Context, inv *inventory.Inventory, node, module string) (*inventory.PendingAssignment, error) {
rows, err := inv.PendingFor(ctx, node, module)
if err != nil {
return nil, err
}
for i := range rows {
if rows[i].Open() {
return &rows[i], nil
}
}
return nil, nil
}
// notRegistered answers an assignment of a module the catalogue does not hold: pending on a build in flight, // notRegistered answers an assignment of a module the catalogue does not hold: pending on a build in flight,
// known and not built, or unknown (novox/hq issue 325). An answer with no error is a pending assignment kept. // known and not built, or unknown (novox/hq issue 325). An answer with no error is a pending assignment kept.
func notRegistered(ctx context.Context, open *stores, node, module string, opts assignOptions) (string, error) { func notRegistered(ctx context.Context, open *stores, node, module string, opts assignOptions) (string, error) {
@@ -123,19 +164,31 @@ func notRegistered(ctx context.Context, open *stores, node, module string, opts
if err != nil { if err != nil {
return "", err return "", err
} }
waiting, err := openPending(ctx, inv, node, module)
if err != nil {
return "", err
}
switch where.kind { switch where.kind {
case buildInFlight: case buildInFlight:
lead := where.said lead := where.said
if opts.Build { if opts.Build {
lead += "\n no second build was asked: this one is running" lead += "\n no second build was asked: this one is running"
} }
return keepPending(ctx, open, node, module, where.request, lead) return keepPending(ctx, open, node, module, where.request, waiting, lead)
case knownNotBuilt: case knownNotBuilt:
last := where.request last := where.request
if !opts.Build { if !opts.Build {
return "", fmt.Errorf("%w: %s\n assign it again with build \"true\" to ask for its build and keep this "+ also := ""
if waiting != nil {
also = fmt.Sprintf("\n %s already has a pending assignment of %s, waiting for build %s; build "+
"\"true\" asks for a new build and makes it wait for that one", node, module, waiting.Build)
}
return "", fmt.Errorf("%w: %s%s\n assign it again with build \"true\" to ask for its build and keep this "+
"assignment until the build registers it; or `build` it (repository %s, path %s) and assign it "+ "assignment until the build registers it; or `build` it (repository %s, path %s) and assign it "+
"once it is registered", inventory.ErrNoSuchModule, where.said, last.Repository, orRoot(last.Path)) "once it is registered", inventory.ErrNoSuchModule, where.said, also, last.Repository, orRoot(last.Path))
}
if waiting != nil && waiting.State == inventory.PendingApplying {
return "", fmt.Errorf("%s is being assigned %s now; nothing was asked", node, module)
} }
source := buildSource{Repository: last.Repository, Seat: last.Seat} source := buildSource{Repository: last.Repository, Seat: last.Seat}
id, err := askABuild(ctx, source, last.Path, last.Ref) id, err := askABuild(ctx, source, last.Path, last.Ref)
@@ -147,20 +200,38 @@ func notRegistered(ctx context.Context, open *stores, node, module string, opts
recordBuildRequest(ctx, inv, asked.BuildRequest) recordBuildRequest(ctx, inv, asked.BuildRequest)
lead := fmt.Sprintf("%s\n build %s of %s is asked for now; %s can be assigned once it registers it", lead := fmt.Sprintf("%s\n build %s of %s is asked for now; %s can be assigned once it registers it",
where.said, id, requestSource(asked), module) where.said, id, requestSource(asked), module)
return keepPending(ctx, open, node, module, asked, lead) return keepPending(ctx, open, node, module, asked, waiting, lead)
} }
answer := strings.TrimPrefix(where.said, inventory.ErrNoSuchModule.Error()) + answer := strings.TrimPrefix(where.said, inventory.ErrNoSuchModule.Error()) +
"\n a module in a repository is built with `build` (its repository and path), then assigned" "\n a module in a repository is built with `build` (its repository and path), then assigned"
if opts.Build { if opts.Build {
answer += "\n build \"true\" asks for nothing here: the mesh does not know where a module of that name is" answer += "\n build \"true\" asks for nothing here: the controller has no build request saying where a " +
"module of that name is"
} }
return "", fmt.Errorf("%w%s", inventory.ErrNoSuchModule, answer) return "", fmt.Errorf("%w%s", inventory.ErrNoSuchModule, answer)
} }
// keepPending records the assignment as pending on a build request, and says so. // keepPending records the assignment as pending on a build request — or, where one is already open, makes it
func keepPending(ctx context.Context, open *stores, node, module string, r inventory.RequestOutcome, lead string) (string, error) { // wait for that build — and says so.
func keepPending(ctx context.Context, open *stores, node, module string, r inventory.RequestOutcome,
waiting *inventory.PendingAssignment, lead string) (string, error) {
inv := open.inventory inv := open.inventory
p, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: node, Module: module, Build: r.ID, if waiting != nil {
if waiting.Build == r.ID {
return lead + fmt.Sprintf("\n %s already waits for %s on this build: nothing changed", node, module), nil
}
moved, err := inv.RepointPending(ctx, waiting.ID, r.ID, r.Repository, r.Path)
if err != nil {
return "", err
}
if !moved {
return lead + fmt.Sprintf("\n the pending assignment of %s to %s ended while this was asked: `status` "+
"says how", module, node), nil
}
return lead + fmt.Sprintf("\n the pending assignment of %s to %s, which waited for build %s, now waits "+
"for build %s", module, node, waiting.Build, r.ID), nil
}
_, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: node, Module: module, Build: r.ID,
Repository: r.Repository, Path: r.Path}) Repository: r.Repository, Path: r.Path})
if errors.Is(err, inventory.ErrAlreadyPending) { if errors.Is(err, inventory.ErrAlreadyPending) {
return lead + fmt.Sprintf("\n %s already waits for %s: nothing changed", node, module), nil return lead + fmt.Sprintf("\n %s already waits for %s: nothing changed", node, module), nil
@@ -168,51 +239,63 @@ func keepPending(ctx context.Context, open *stores, node, module string, r inven
if err != nil { if err != nil {
return "", err return "", err
} }
answer := lead + fmt.Sprintf("\n the assignment is kept as pending: %s is assigned %s when the build "+ // The controller makes it, not this act (ADR 0261): when the build registers the module, or at its next
"registers it, and `status` says it until then. If the build fails it expires and says why; "+ // tick if the module was registered between the look and the record.
"`unassign %s %s` withdraws it", node, module, node, module) return lead + fmt.Sprintf("\n the assignment is kept as pending: the controller assigns %s to %s when the "+
// The build may have registered the module between the look and the record: made at once, then. "build registers it, and `status` lists it until then. If the build fails it expires, says why and "+
if shelf, err := inv.Catalogue(ctx); err == nil { "raises a condition; `unassign %s %s` withdraws it", module, node, node, module), nil
if _, now := shelf[module]; now {
if line := applyPending(ctx, open, p, r.ID); line != "" {
answer += "\n " + line
}
}
}
return answer, nil
} }
// inFlight is the newest build request for the module with no outcome yet, asked within the bound. // inFlight is the newest build request for the module still to register it: no outcome yet within the
// bound, or a successful outcome heard a moment ago and being registered.
func inFlight(requests []inventory.RequestOutcome, now time.Time) (inventory.RequestOutcome, bool) { func inFlight(requests []inventory.RequestOutcome, now time.Time) (inventory.RequestOutcome, bool) {
for _, r := range requests { for _, r := range requests {
if r.NotAsked == "" && !r.Heard && now.Sub(r.At) < buildRequestBound { if stillComing(r, now) {
return r, true return r, true
} }
} }
return inventory.RequestOutcome{}, false return inventory.RequestOutcome{}, false
} }
// whyNotBuilt says what came of a module's last build request. func stillComing(r inventory.RequestOutcome, now time.Time) bool {
if r.Heard {
return r.Failed == "" && now.Sub(r.HeardAt) < registerGrace
}
return r.NotAsked == "" && now.Sub(r.At) < buildRequestBound
}
// whyNotBuilt says what came of a module's last build request. What was heard comes first: an outcome is the
// last word whatever its asker said.
func whyNotBuilt(r inventory.RequestOutcome, now time.Time) string { func whyNotBuilt(r inventory.RequestOutcome, now time.Time) string {
where := fmt.Sprintf("%s in %s", orRoot(r.Path), r.Repository) where := fmt.Sprintf("%s in %s", orRoot(r.Path), r.Repository)
switch { switch {
case r.NotAsked != "": case r.Heard && r.Failed != "":
return fmt.Sprintf("%s; the merge that added it (%s) could not ask for its build: %s", where,
short(r.Commit), firstLine(r.NotAsked))
case !r.Heard:
return fmt.Sprintf("%s; build %s was asked at %s, and no outcome has been heard in %s", where, r.ID,
clock(r.At), now.Sub(r.At).Round(time.Minute))
case r.Failed != "":
return fmt.Sprintf("%s; its last build, %s at %s, failed: %s", where, r.ID, clock(r.HeardAt), return fmt.Sprintf("%s; its last build, %s at %s, failed: %s", where, r.ID, clock(r.HeardAt),
firstLine(r.Failed)) firstLine(r.Failed))
case r.Module != "" && r.Module != r.Name(): case r.Heard && r.Module != "" && r.Module != r.Name():
return fmt.Sprintf("%s; its last build, %s, registered it as %s", where, r.ID, r.Module) return fmt.Sprintf("%s; its last build, %s, built it as %s", where, r.ID, r.Module)
default: case r.Heard:
return fmt.Sprintf("%s; its last build, %s at %s, was recorded and not registered — `builds` says why", return fmt.Sprintf("%s; its last build, %s at %s, was recorded and not registered — `builds` says why",
where, r.ID, clock(r.HeardAt)) where, r.ID, clock(r.HeardAt))
case r.NotAsked != "" && r.For == "merge":
return fmt.Sprintf("%s; the merge that added it (%s) could not ask for its build: %s", where,
short(r.Commit), firstLine(r.NotAsked))
case r.NotAsked != "":
return fmt.Sprintf("%s; build %s, asked by %s, was not handed over or not waited for: %s", where, r.ID,
askerOr(r.For), firstLine(r.NotAsked))
default:
return fmt.Sprintf("%s; build %s was asked at %s, and no outcome has been heard in %s", where, r.ID,
clock(r.At), now.Sub(r.At).Round(time.Minute))
} }
} }
func askerOr(who string) string {
if who == "" {
return "the controller"
}
return who
}
// requestSource is a build request's source as a person reads it: repository@commit (directory). // requestSource is a build request's source as a person reads it: repository@commit (directory).
func requestSource(r inventory.RequestOutcome) string { func requestSource(r inventory.RequestOutcome) string {
at := short(r.Commit) at := short(r.Commit)
@@ -228,8 +311,8 @@ func requestSource(r inventory.RequestOutcome) string {
// clock is a moment as a person reads it, in the controller's local time. // clock is a moment as a person reads it, in the controller's local time.
func clock(t time.Time) string { return t.Local().Format("2006-01-02 15:04:05 MST") } func clock(t time.Time) string { return t.Local().Format("2006-01-02 15:04:05 MST") }
// closestNames is up to three names the mesh holds that are near the one asked: registered modules and // closestNames is up to three names near the one asked: registered modules and directories the controller
// directories it asked to build. // asked to build.
func closestNames(ctx context.Context, inv *inventory.Inventory, name string) []string { func closestNames(ctx context.Context, inv *inventory.Inventory, name string) []string {
var candidates []string var candidates []string
if shelf, err := inv.Catalogue(ctx); err == nil { if shelf, err := inv.Catalogue(ctx); err == nil {
@@ -292,26 +375,41 @@ func editDistance(a, b string) int {
return prev[len(rb)] return prev[len(rb)]
} }
// applyPending makes a pending assignment now that its module is registered, and settles it: applied, or // applyPending makes a pending assignment now that its module is registered (ADR 0261). Under the machine's
// refused with the refusal. Returns what it said, or nothing when another process settled it first. // hold it claims the row (waiting → applying), so a withdrawal cannot land between the look and the act,
// then assigns by the same act a person's assign is, then says what came of it: applied, or refused with the
// refusal. Returns what it said, or nothing when the row no longer waited.
func applyPending(ctx context.Context, open *stores, p inventory.PendingAssignment, by string) string { func applyPending(ctx context.Context, open *stores, p inventory.PendingAssignment, by string) string {
inv := open.inventory inv := open.inventory
ctx, release, err := holdNodes(ctx, open, []string{p.Node})
if err != nil {
// Left waiting: the next tick tries again.
return fmt.Sprintf("the pending assignment of %s to %s waits: %s could not be held: %v", p.Module, p.Node,
p.Node, err)
}
defer release()
claimed, err := inv.ClaimPending(ctx, p.ID)
if err != nil {
return fmt.Sprintf("the pending assignment of %s to %s could not be taken for making: %v", p.Module, p.Node, err)
}
if !claimed {
return ""
}
answer, err := assign(ctx, open, p.Node, p.Module) answer, err := assign(ctx, open, p.Node, p.Module)
state, note := inventory.PendingApplied, "" state, note := inventory.PendingApplied, ""
if err != nil { switch {
case err != nil:
state = inventory.PendingRefused state = inventory.PendingRefused
note = fmt.Sprintf("the pending assignment of %s to %s was refused when build %s registered it: %s", note = fmt.Sprintf("the pending assignment of %s to %s was refused when build %s registered it: %s",
p.Module, p.Node, by, firstLine(err.Error())) p.Module, p.Node, by, firstLine(err.Error()))
} else { case strings.Contains(answer, "already runs"):
// Not promised as sent: a walk sends it if the module's upgrade announcement finds it note = fmt.Sprintf("%s already runs %s: the pending assignment is met", p.Node, p.Module)
// assigned, and nothing does if the announcement came first. `status` says which, as for any machine. default:
note = fmt.Sprintf("%s is assigned %s: build %s registered it (pending since %s); `status` says whether "+ // The same as a person's assign: nothing is sent by this act.
"%s has been sent it, and `push %s` sends it if not", p.Node, p.Module, by, clock(p.Since), p.Node, p.Node) note = fmt.Sprintf("%s is assigned %s: build %s registered it (pending since %s); `push %s` sends it",
if strings.Contains(answer, "already runs") { p.Node, p.Module, by, clock(p.Since), p.Node)
note = fmt.Sprintf("%s already runs %s: the pending assignment is met", p.Node, p.Module)
}
} }
settled, serr := inv.SettlePending(ctx, p.ID, state, note) settled, serr := inv.SettlePending(ctx, p.ID, inventory.PendingApplying, state, note)
if serr != nil { if serr != nil {
return fmt.Sprintf("%s — and it could not be recorded as settled: %v", note, serr) return fmt.Sprintf("%s — and it could not be recorded as settled: %v", note, serr)
} }
@@ -321,11 +419,11 @@ func applyPending(ctx context.Context, open *stores, p inventory.PendingAssignme
return note return note
} }
// expirePending ends a pending assignment whose build will not register its module, with why. // expirePending ends a waiting pending assignment whose build will not register its module, with why.
func expirePending(ctx context.Context, inv *inventory.Inventory, p inventory.PendingAssignment, why string) string { func expirePending(ctx context.Context, inv *inventory.Inventory, p inventory.PendingAssignment, why string) string {
note := fmt.Sprintf("the pending assignment of %s to %s expired: %s; nothing was assigned. Assign it again with "+ note := fmt.Sprintf("the pending assignment of %s to %s expired: %s; nothing was assigned. Assign it again with "+
"build \"true\" to ask for the build again", p.Module, p.Node, why) "build \"true\" to ask for the build again", p.Module, p.Node, why)
settled, err := inv.SettlePending(ctx, p.ID, inventory.PendingExpired, note) settled, err := inv.SettlePending(ctx, p.ID, inventory.PendingWaiting, inventory.PendingExpired, note)
if err != nil { if err != nil {
return fmt.Sprintf("%s — and it could not be recorded: %v", note, err) return fmt.Sprintf("%s — and it could not be recorded: %v", note, err)
} }
@@ -369,21 +467,62 @@ func settlePendingOnBuild(ctx context.Context, open *stores, result link.BuildRe
return said return said
} }
// settleStalePending settles what a missed outcome left waiting: a module registered meanwhile is assigned, // settlingPending is the controller's tick over the pending assignments (ADR 0261), every settleEvery until
// a build heard and not registered ends its assignment, and one with no outcome past the bound expires. // the context ends. What it does is printed to the controller's log, kept in each row's note (which `status`
// Asked where the mesh is read for status, so a pending assignment never waits silently for ever. // lists for a day), and raised as a condition for an assignment that was not made.
func settleStalePending(ctx context.Context, open *stores, now time.Time) []string { func settlingPending(ctx context.Context, open *stores) {
inv := open.inventory for {
waiting, err := inv.Pending(ctx, time.Time{}) for _, line := range settlePending(ctx, open, time.Now()) {
if err != nil { fmt.Println(line)
// Said, not swallowed; status reads the pending assignments again and says the same failure there. }
return []string{fmt.Sprintf("the pending assignments could not be read to settle them: %v", err)} select {
case <-ctx.Done():
return
case <-time.After(settleEvery):
}
} }
}
// settlePending is one tick: claims left by a controller that stopped are settled from what the mesh holds;
// a module registered meanwhile is assigned; a build heard and not registered, or silent past its bound,
// ends its assignment; ended ones are raised as conditions and cleared once answered; ended rows older than
// KeptFor are deleted. Returns what it did.
func settlePending(ctx context.Context, open *stores, now time.Time) []string {
inv := open.inventory
var said []string
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
shelf, err := inv.Catalogue(ctx) shelf, err := inv.Catalogue(ctx)
if err != nil { if err != nil {
return []string{fmt.Sprintf("the catalogue could not be read to settle the pending assignments: %v", err)} say("the pending assignments are not settled this time: the catalogue could not be read: %v", err)
return said
}
stuck, err := inv.Stuck(ctx, now.Add(-claimStaleAfter))
if err != nil {
say("claims left by a stopped controller could not be read: %v", err)
}
for _, p := range stuck {
assigned, err := inv.Assigned(ctx, p.Node)
if err != nil {
say("the claim on %s for %s could not be settled: %v", p.Node, p.Module, err)
continue
}
if containsString(assigned, p.Module) {
if _, err := inv.SettlePending(ctx, p.ID, inventory.PendingApplying, inventory.PendingApplied,
fmt.Sprintf("%s is assigned %s (settled after the controller that made it stopped)", p.Node, p.Module)); err != nil {
say("the claim on %s for %s could not be settled: %v", p.Node, p.Module, err)
}
continue
}
if err := inv.ReleaseClaim(ctx, p.ID); err != nil {
say("the claim on %s for %s could not be released: %v", p.Node, p.Module, err)
}
}
waiting, err := inv.Pending(ctx, time.Time{})
if err != nil {
say("the pending assignments could not be read to settle them: %v", err)
} }
var said []string
for _, p := range waiting { for _, p := range waiting {
if _, ok := shelf[p.Module]; ok { if _, ok := shelf[p.Module]; ok {
if line := applyPending(ctx, open, p, p.Build); line != "" { if line := applyPending(ctx, open, p, p.Build); line != "" {
@@ -393,7 +532,7 @@ func settleStalePending(ctx context.Context, open *stores, now time.Time) []stri
} }
requests, err := inv.RequestsNamed(ctx, p.Module) requests, err := inv.RequestsNamed(ctx, p.Module)
if err != nil { if err != nil {
said = append(said, fmt.Sprintf("the build requests for %s could not be read: %v", p.Module, err)) say("the build requests for %s could not be read: %v", p.Module, err)
continue continue
} }
var r *inventory.RequestOutcome var r *inventory.RequestOutcome
@@ -406,9 +545,9 @@ func settleStalePending(ctx context.Context, open *stores, now time.Time) []stri
switch { switch {
case r == nil && now.Sub(p.Since) > buildRequestBound: case r == nil && now.Sub(p.Since) > buildRequestBound:
why = fmt.Sprintf("build %s is no longer on record", p.Build) why = fmt.Sprintf("build %s is no longer on record", p.Build)
case r != nil && r.Heard: case r != nil && !stillComing(*r, now) && (r.Heard || r.NotAsked != ""):
why = whyNotBuilt(*r, now) why = whyNotBuilt(*r, now)
case r != nil && now.Sub(r.At) > buildRequestBound: case r != nil && !stillComing(*r, now):
why = fmt.Sprintf("no outcome of build %s was heard within %s of asking", p.Build, buildRequestBound) why = fmt.Sprintf("no outcome of build %s was heard within %s of asking", p.Build, buildRequestBound)
} }
if why == "" { if why == "" {
@@ -418,25 +557,139 @@ func settleStalePending(ctx context.Context, open *stores, now time.Time) []stri
said = append(said, line) said = append(said, line)
} }
} }
said = append(said, raisePendingEnded(ctx, inv)...)
if n, err := inv.ForgetEndedPending(ctx, now); err != nil {
say("ended pending assignments could not be deleted: %v", err)
} else if n > 0 {
say("deleted %d pending assignment(s) ended more than %s ago", n, inventory.KeptFor)
}
return said return said
} }
// withdrawPending is `unassign` of a module only pending on a machine: the pending assignment withdrawn. func containsString(xs []string, x string) bool {
func withdrawPending(ctx context.Context, inv *inventory.Inventory, node, module string) (string, bool, error) { for _, y := range xs {
waiting, err := inv.Pending(ctx, time.Time{}) if y == x {
return true
}
}
return false
}
// pendingObservation is the condition for a pending assignment that was not made.
func pendingObservation(p inventory.PendingAssignment) conditions.Observation {
return conditions.Observation{Scope: conditions.ScopeMachine, ID: p.Node + "." + p.Module,
Token: kindPendingEnded, Kind: kindPendingEnded, Machine: p.Node, Severity: conditions.Warning,
Resolver: conditions.ResolverOperator, Source: "pending assignments",
Summary: p.Note}
}
// raisePendingEnded raises a condition for each pending assignment that expired or was refused, once, and
// clears it when it is answered: the module was assigned to that machine since, a newer pending assignment of
// it is open, or a person took it back with `unassign`. Where this process keeps no conditions, nothing is
// stamped, and the serving controller's tick raises it.
func raisePendingEnded(ctx context.Context, inv *inventory.Inventory) []string {
keeper := conditionsFrom
if keeper == nil {
return nil
}
var said []string
toRaise, err := inv.ToRaise(ctx)
if err != nil {
return []string{fmt.Sprintf("pending assignments not made could not be read to raise them: %v", err)}
}
for _, p := range toRaise {
if _, err := keeper.Observe(ctx, pendingObservation(p)); err != nil {
said = append(said, fmt.Sprintf("the condition for %s on %s could not be raised: %v", p.Module, p.Node, err))
continue
}
if err := inv.MarkPending(ctx, p.ID, "raised"); err != nil {
said = append(said, fmt.Sprintf("the condition for %s on %s was raised and not recorded: %v", p.Module, p.Node, err))
}
}
toClear, err := inv.ToClear(ctx)
if err != nil {
return append(said, fmt.Sprintf("pending assignments raised could not be read to clear them: %v", err))
}
for _, p := range toClear {
why, answered, err := pendingAnswered(ctx, inv, p)
if err != nil {
said = append(said, fmt.Sprintf("whether %s on %s is answered could not be read: %v", p.Module, p.Node, err))
continue
}
if !answered {
continue
}
if _, err := keeper.Clear(ctx, pendingObservation(p).Key(), why); err != nil {
said = append(said, fmt.Sprintf("the condition for %s on %s could not be cleared: %v", p.Module, p.Node, err))
continue
}
if err := inv.MarkPending(ctx, p.ID, "cleared"); err != nil {
said = append(said, fmt.Sprintf("the condition for %s on %s was cleared and not recorded: %v", p.Module, p.Node, err))
}
}
return said
}
// pendingAnswered says whether a pending assignment that was not made has been answered since, and how.
func pendingAnswered(ctx context.Context, inv *inventory.Inventory, p inventory.PendingAssignment) (string, bool, error) {
if p.Acknowledged != nil {
return "taken back with unassign", true, nil
}
assigned, err := inv.Assigned(ctx, p.Node)
if err != nil { if err != nil {
return "", false, err return "", false, err
} }
for _, p := range waiting { if containsString(assigned, p.Module) {
if p.Node != node || p.Module != module { return p.Module + " is assigned to " + p.Node + " since", true, nil
continue }
rows, err := inv.PendingFor(ctx, p.Node, p.Module)
if err != nil {
return "", false, err
}
for _, r := range rows {
if r.ID != p.ID && r.Since.After(p.Since) {
return "assigned again, pending on another build", true, nil
}
}
return "", false, nil
}
// withdrawPending is `unassign` of a module only pending on a machine, under the machine's hold: a waiting
// pending assignment is withdrawn; one being made now is refused, never overridden; one that expired or was
// refused is taken back, which answers its condition. False when there is none.
func withdrawPending(ctx context.Context, inv *inventory.Inventory, node, module string) (string, bool, error) {
rows, err := inv.PendingFor(ctx, node, module)
if err != nil {
return "", false, err
}
for _, p := range rows {
switch p.State {
case inventory.PendingApplying:
return "", true, fmt.Errorf("%s is being assigned %s now, as build %s registered it: nothing was "+
"withdrawn; `unassign %s %s` again once it is assigned takes it off", node, module, p.Build, node, module)
case inventory.PendingWaiting:
note := fmt.Sprintf("the pending assignment of %s to %s is withdrawn; build %s goes on, and registers "+
"%s assigned nowhere", module, node, p.Build, module)
settled, err := inv.SettlePending(ctx, p.ID, inventory.PendingWaiting, inventory.PendingWithdrawn, note)
if err != nil {
return "", false, err
}
if !settled {
return "", true, fmt.Errorf("the pending assignment of %s to %s changed while it was withdrawn: "+
"`status` says how; nothing was withdrawn", module, node)
}
return note, true, nil
case inventory.PendingExpired, inventory.PendingRefused:
if p.Cleared != nil || p.Acknowledged != nil {
continue
}
if err := inv.MarkPending(ctx, p.ID, "acknowledged"); err != nil {
return "", false, err
}
return fmt.Sprintf("the pending assignment of %s to %s, which %s, is taken back; its condition clears",
module, node, p.State), true, nil
} }
note := fmt.Sprintf("the pending assignment of %s to %s is withdrawn; build %s goes on, and registers %s "+
"assigned nowhere", module, node, p.Build, module)
if _, err := inv.SettlePending(ctx, p.ID, inventory.PendingWithdrawn, note); err != nil {
return "", false, err
}
return note, true, nil
} }
return "", false, nil return "", false, nil
} }
+341 -27
View File
@@ -1,6 +1,7 @@
package main package main
import ( import (
"context"
"encoding/json" "encoding/json"
"errors" "errors"
"slices" "slices"
@@ -9,6 +10,7 @@ import (
"time" "time"
"github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link" "github.com/novox/mesh-controller/internal/link"
) )
@@ -68,21 +70,27 @@ func pendingOf(t *testing.T, open *stores) []inventory.PendingAssignment {
return got return got
} }
// A build in flight: the merge asked for it, the assignment is kept pending and said so with the build, and // A build in flight: the merge asked for it, the request is kept as the merge's, the assignment is kept
// the build's outcome makes it. Status says it while it waits. // pending and said so with the build, status reads it without settling anything, and the build's outcome
// makes it — saying that a push sends it, and nothing more.
func TestAnAssignmentWaitsForTheBuildInFlight(t *testing.T) { func TestAnAssignmentWaitsForTheBuildInFlight(t *testing.T) {
open := aCatalogueMesh(t) open := aCatalogueMesh(t)
ctx := t.Context() ctx := t.Context()
asksWithPaths(t) asksWithPaths(t)
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
requests, err := open.inventory.RequestsNamed(ctx, "sensors")
if err != nil || len(requests) != 1 || requests[0].For != "merge" || requests[0].Commit != "3da80a4b00aa" {
t.Fatalf("the merge's build request: %+v %v", requests, err)
}
said, err := assign(ctx, open, "laptop", "sensors") said, err := assign(ctx, open, "laptop", "sensors")
if err != nil { if err != nil {
t.Fatalf("an assignment while its build runs was refused: %v", err) t.Fatalf("an assignment while its build runs was refused: %v", err)
} }
t.Logf("assign laptop sensors, while its build runs:\n%s", said) t.Logf("assign laptop sensors, while its build runs:\n%s", said)
for _, want := range []string{"being built from novox/mesh-catalog@3da80a4b (modules/sensors)", for _, want := range []string{"being built from novox/mesh-catalog@3da80a4b (modules/sensors)",
"build b-modules/sensors", "kept as pending", "`unassign laptop sensors`"} { "build b-modules/sensors", "kept as pending", "the controller assigns sensors to laptop", "`unassign laptop sensors`"} {
if !strings.Contains(said, want) { if !strings.Contains(said, want) {
t.Fatalf("the answer does not say %q:\n%s", want, said) t.Fatalf("the answer does not say %q:\n%s", want, said)
} }
@@ -94,7 +102,6 @@ func TestAnAssignmentWaitsForTheBuildInFlight(t *testing.T) {
if len(pending) != 1 || pending[0].State != inventory.PendingWaiting || pending[0].Build != "b-modules/sensors" { if len(pending) != 1 || pending[0].State != inventory.PendingWaiting || pending[0].Build != "b-modules/sensors" {
t.Fatalf("pending: %+v", pending) t.Fatalf("pending: %+v", pending)
} }
// Asked twice, it is one pending assignment.
if again, err := assign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(again, "already waits") { if again, err := assign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(again, "already waits") {
t.Fatalf("a second assignment: %q %v", again, err) t.Fatalf("a second assignment: %q %v", again, err)
} }
@@ -127,12 +134,41 @@ func TestAnAssignmentWaitsForTheBuildInFlight(t *testing.T) {
} }
pending = pendingOf(t, open) pending = pendingOf(t, open)
if len(pending) != 1 || pending[0].State != inventory.PendingApplied || if len(pending) != 1 || pending[0].State != inventory.PendingApplied ||
!strings.Contains(pending[0].Note, "`push laptop` sends it if not") { !strings.HasSuffix(pending[0].Note, "`push laptop` sends it") {
t.Fatalf("pending after the build: %+v", pending) t.Fatalf("pending after the build: %+v", pending)
} }
} }
// The build of a pending assignment fails: it expires, saying the build's words, and nothing is assigned. // **A read settles nothing** (review of #150, point 6): status — and so the board, the summary and the
// probe, which compose from the same reading — leaves a pending assignment whose module is registered as it
// is; the controller's tick makes it.
func TestAStatusReadSettlesNothing(t *testing.T) {
open := aCatalogueMesh(t)
ctx := t.Context()
asksWithPaths(t)
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
t.Fatal(err)
}
register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"})
for range 2 {
if _, err := theThreeQuestions(ctx, open); err != nil {
t.Fatal(err)
}
}
if slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingWaiting {
t.Fatalf("a status read settled a pending assignment: %+v", pendingOf(t, open))
}
said := settlePending(ctx, open, time.Now())
if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingApplied {
t.Fatalf("the tick did not make it: %v %+v", said, pendingOf(t, open))
}
}
// The build of a pending assignment fails: it expires with the build's words and nothing is assigned; the
// tick raises it as a condition once; it then reads as known and not built; and `unassign` takes it back,
// after which the tick clears the condition. Status is not called well while the condition is open, and is
// again once it clears — no fixed day of "not well" (review point 6).
func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) { func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) {
open := aCatalogueMesh(t) open := aCatalogueMesh(t)
ctx := t.Context() ctx := t.Context()
@@ -158,15 +194,17 @@ func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) {
if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { if slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
t.Fatal("a failed build's module was assigned") t.Fatal("a failed build's module was assigned")
} }
asked, err := theThreeQuestions(ctx, open)
if err != nil { settlePending(ctx, open, time.Now())
t.Fatal(err) key := pendingObservation(pending[0]).Key()
c, found, err := conditionsFrom.Get(ctx, key)
if err != nil || !found || c.Kind != kindPendingEnded {
t.Fatalf("no condition %s for the assignment not made: %+v %v %v", key, c, found, err)
} }
if asked.well() { if asked, err := theThreeQuestions(ctx, open); err != nil || asked.well() || len(asked.conditions) == 0 {
t.Fatal("status called the mesh well on the day a pending assignment expired") t.Fatalf("status does not hold the open condition: %v", err)
} }
// Known and not built now: said, and refused without build.
_, err = assign(ctx, open, "laptop", "sensors") _, err = assign(ctx, open, "laptop", "sensors")
if !errors.Is(err, inventory.ErrNoSuchModule) { if !errors.Is(err, inventory.ErrNoSuchModule) {
t.Fatalf("a module whose build failed: %v", err) t.Fatalf("a module whose build failed: %v", err)
@@ -177,10 +215,54 @@ func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) {
t.Fatalf("the refusal does not say %q:\n%v", want, err) t.Fatalf("the refusal does not say %q:\n%v", want, err)
} }
} }
said, err := unassign(ctx, open, "laptop", "sensors")
if err != nil || !strings.Contains(said, "taken back") {
t.Fatalf("unassign of an expired pending assignment: %q %v", said, err)
}
settlePending(ctx, open, time.Now())
if _, found, _ := conditionsFrom.Get(ctx, key); found {
t.Fatal("the condition stayed open after the assignment was taken back")
}
// Nothing of it keeps status from being well any more: no open pending assignment, no open condition of it.
asked, err := theThreeQuestions(ctx, open)
if err != nil || pendingOpen(asked.pending) {
t.Fatalf("status still counts the pending assignment: %+v %v", asked.pending, err)
}
for _, c := range asked.conditions {
if c.Kind == kindPendingEnded {
t.Fatalf("status still holds the condition: %+v", c)
}
}
} }
// Known and not built, asked with build: the build is asked from where the last one was, and the assignment // The condition also clears when the module is later assigned to the machine.
// kept pending on the new build. func TestTheConditionClearsWhenTheModuleIsAssignedLater(t *testing.T) {
open := aCatalogueMesh(t)
ctx := t.Context()
asksWithPaths(t)
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
t.Fatal(err)
}
if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors",
"3da80a4b00aa", "", "boom")); err != nil {
t.Fatal(err)
}
settlePending(ctx, open, time.Now())
key := pendingObservation(pendingOf(t, open)[0]).Key()
register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"})
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
t.Fatal(err)
}
settlePending(ctx, open, time.Now())
if _, found, _ := conditionsFrom.Get(ctx, key); found {
t.Fatal("the condition stayed open after the module was assigned")
}
}
// Known and not built, asked with build: the build is asked from where the last one was, the request kept as
// assign's, and the assignment kept pending on the new build.
func TestAssignWithBuildAsksForAKnownModule(t *testing.T) { func TestAssignWithBuildAsksForAKnownModule(t *testing.T) {
open := aCatalogueMesh(t) open := aCatalogueMesh(t)
ctx := t.Context() ctx := t.Context()
@@ -211,10 +293,146 @@ func TestAssignWithBuildAsksForAKnownModule(t *testing.T) {
if len(pending) != 1 || pending[0].Build != "b-modules/sensors" { if len(pending) != 1 || pending[0].Build != "b-modules/sensors" {
t.Fatalf("pending: %+v", pending) t.Fatalf("pending: %+v", pending)
} }
requests, _ := inv.RequestsNamed(ctx, "sensors")
if len(requests) != 2 || requests[0].ID != "b-modules/sensors" || requests[0].For != "assign" {
t.Fatalf("the request is not kept as assign's: %+v", requests)
}
} }
// Unknown: refused as before, as no module of that name, with the closest names the mesh holds; build asks // **build "true" with a pending assignment already waiting** (review point 5): the waiting one is made to
// for nothing, since the mesh does not know where it is. // wait for the new build, and no build is asked that nothing waits on.
func TestAssignWithBuildRepointsTheWaitingAssignment(t *testing.T) {
open := aCatalogueMesh(t)
ctx := t.Context()
inv := open.inventory
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-old", Repository: "novox/mesh-catalog",
Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge", At: time.Now().Add(-time.Hour)}); err != nil {
t.Fatal(err)
}
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-old", Repository: "novox/mesh-catalog", Ref: "main",
Path: "modules/sensors", On: "anchor", Failed: "boom"}); err != nil {
t.Fatal(err)
}
// Waiting on the failed build, its expiry not yet settled.
if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "laptop", Module: "sensors",
Build: "build-old", Repository: "novox/mesh-catalog", Path: "modules/sensors"}); err != nil {
t.Fatal(err)
}
_, err := assign(ctx, open, "laptop", "sensors")
if err == nil || !strings.Contains(err.Error(), "already has a pending assignment of sensors, waiting for build build-old") {
t.Fatalf("known and not built with a pending assignment waiting: %v", err)
}
asked := asksWithPaths(t)
said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors")
if err != nil || !strings.Contains(said, "which waited for build build-old, now waits for build b-modules/sensors") {
t.Fatalf("assign with build: %q %v", said, err)
}
if len(*asked) != 1 {
t.Fatalf("asked %v", *asked)
}
pending := pendingOf(t, open)
if len(pending) != 1 || pending[0].Build != "b-modules/sensors" || pending[0].State != inventory.PendingWaiting {
t.Fatalf("pending: %+v", pending)
}
// The new build in flight now: build "true" again asks nothing.
if said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors"); err != nil ||
!strings.Contains(said, "no second build was asked") || len(*asked) != 1 {
t.Fatalf("a second build true while the build runs: %q %v, asked %v", said, err, *asked)
}
}
// **A failed ask never reads as in flight** (review point 2): a merge whose ask could not be made keeps the
// request as not asked, and assign says the merge could not ask; a request whose asker could not hand it over
// or stopped waiting reads the same, unless its outcome was heard.
func TestAFailedAskIsNotABuildInFlight(t *testing.T) {
open := aCatalogueMesh(t)
ctx := t.Context()
inv := open.inventory
was := askABuild
askABuild = func(context.Context, buildSource, string, string) (string, error) {
return "", errors.New("cannot submit a build: nats: timeout")
}
t.Cleanup(func() { askABuild = was })
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
_, err := assign(ctx, open, "laptop", "sensors")
if err == nil || !strings.Contains(err.Error(), "the merge that added it (3da80a4b) could not ask for its build: cannot submit") {
t.Fatalf("a merge that could not ask: %v", err)
}
if len(pendingOf(t, open)) != 0 {
t.Fatal("a pending assignment waits on a build never asked")
}
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-waited", Repository: "novox/mesh-catalog",
Seat: "git", Path: "modules/gauges", Ref: "main", For: "build"}); err != nil {
t.Fatal(err)
}
if err := inv.MarkNotAsked(ctx, "build-waited", "cannot submit a build: no responders"); err != nil {
t.Fatal(err)
}
_, err = assign(ctx, open, "laptop", "gauges")
if err == nil || !strings.Contains(err.Error(), "build build-waited, asked by build, was not handed over or not waited for") {
t.Fatalf("a build not handed over: %v", err)
}
// Heard first, the outcome stands: marking it afterwards changes nothing.
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-waited", Repository: "novox/mesh-catalog", Ref: "main",
Path: "modules/gauges", On: "anchor", Failed: "the real failure"}); err != nil {
t.Fatal(err)
}
_, err = assign(ctx, open, "laptop", "gauges")
if err == nil || !strings.Contains(err.Error(), "failed: the real failure") {
t.Fatalf("an outcome heard after a failed wait: %v", err)
}
}
// A plan's tier keeps its requests as the plan's (review point 3).
func TestAPlansBuildRequestIsThePlans(t *testing.T) {
open := aCatalogueMesh(t)
ctx := t.Context()
asksWithPaths(t)
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"},
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main",
BuiltFrom: "c0", Head: "c0"}); err != nil {
t.Fatal(err)
}
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa",
Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true}
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
t.Fatal(err)
}
requests, err := open.inventory.RequestsNamed(ctx, "app")
if err != nil || len(requests) != 1 || requests[0].For != "plan" {
t.Fatalf("the plan's request: %+v %v", requests, err)
}
}
// **A build heard as built and not registered yet is in flight** (review point 4): the outcome is recorded
// before the module is registered; for that moment assign keeps the assignment pending. Past the grace it is
// recorded and not registered, and the tick ends the pending assignment with that.
func TestABuildBeingRegisteredIsInFlight(t *testing.T) {
open := aCatalogueMesh(t)
ctx := t.Context()
inv := open.inventory
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-done", Repository: "novox/mesh-catalog",
Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge"}); err != nil {
t.Fatal(err)
}
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-done", Repository: "novox/mesh-catalog", Ref: "main",
Path: "modules/sensors", Module: "sensors", On: "anchor"}); err != nil {
t.Fatal(err)
}
said, err := assign(ctx, open, "laptop", "sensors")
if err != nil || !strings.Contains(said, "built and being registered") || !strings.Contains(said, "kept as pending") {
t.Fatalf("a build being registered: %q %v", said, err)
}
settlePending(ctx, open, time.Now().Add(registerGrace+time.Minute))
p := pendingOf(t, open)
if len(p) != 1 || p[0].State != inventory.PendingExpired || !strings.Contains(p[0].Note, "recorded and not registered") {
t.Fatalf("past the grace: %+v", p)
}
}
// Unknown: refused as no module of that name, claiming only what was looked at, with the closest names; build
// asks for nothing.
func TestAnUnknownModuleIsRefusedWithTheClosestNames(t *testing.T) { func TestAnUnknownModuleIsRefusedWithTheClosestNames(t *testing.T) {
open := aCatalogueMesh(t) open := aCatalogueMesh(t)
ctx := t.Context() ctx := t.Context()
@@ -224,12 +442,16 @@ func TestAnUnknownModuleIsRefusedWithTheClosestNames(t *testing.T) {
if !errors.Is(err, inventory.ErrNoSuchModule) { if !errors.Is(err, inventory.ErrNoSuchModule) {
t.Fatalf("an unknown module: %v", err) t.Fatalf("an unknown module: %v", err)
} }
for _, want := range []string{"no module of that name: sensors", "no module, build request or merge by that name", for _, want := range []string{"no module of that name: sensors", "the catalogue holds no module of that name",
"the closest it holds: sensord", "asks for nothing here"} { "no build request the controller kept (the last 30 days)", "the closest names it holds: sensord",
"asks for nothing here"} {
if !strings.Contains(err.Error(), want) { if !strings.Contains(err.Error(), want) {
t.Fatalf("the refusal does not say %q:\n%v", want, err) t.Fatalf("the refusal does not say %q:\n%v", want, err)
} }
} }
if strings.Contains(err.Error(), "merge") {
t.Fatalf("the refusal claims a merge it never looked at: %v", err)
}
if len(*asked) != 0 || len(pendingOf(t, open)) != 0 { if len(*asked) != 0 || len(pendingOf(t, open)) != 0 {
t.Fatalf("an unknown module asked %v, pending %+v", *asked, pendingOf(t, open)) t.Fatalf("an unknown module asked %v, pending %+v", *asked, pendingOf(t, open))
} }
@@ -252,7 +474,8 @@ func TestAnActWithAModuleNotRegisteredIsRefusedWhole(t *testing.T) {
} }
} }
// unassign withdraws a pending assignment; the build goes on and registers the module assigned nowhere. // unassign withdraws a waiting pending assignment; the build goes on and registers the module assigned
// nowhere: a withdrawn row is never claimed.
func TestUnassignWithdrawsAPendingAssignment(t *testing.T) { func TestUnassignWithdrawsAPendingAssignment(t *testing.T) {
open := aCatalogueMesh(t) open := aCatalogueMesh(t)
ctx := t.Context() ctx := t.Context()
@@ -269,13 +492,80 @@ func TestUnassignWithdrawsAPendingAssignment(t *testing.T) {
"3da80a4b00aa", "sensors", "")); err != nil { "3da80a4b00aa", "sensors", "")); err != nil {
t.Fatal(err) t.Fatal(err)
} }
settlePending(ctx, open, time.Now())
if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { if slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
t.Fatal("a withdrawn assignment was made") t.Fatal("a withdrawn assignment was made")
} }
} }
// A build that says nothing within the bound: the pending assignment expires at the next look, saying so, // **A claim is never overridden** (review point 1): a pending assignment being made is claimed
// rather than waiting for ever; and a build heard by another process meanwhile is made at the next look. // (waiting → applying); an unassign meanwhile is refused and says so, and leaves the claim; making a row
// already withdrawn makes nothing. A claim left by a controller that stopped is settled by the tick from what
// the mesh holds: back to waiting when the module is not assigned, applied when it is.
func TestAWithdrawalNeverOverridesAClaim(t *testing.T) {
open := aCatalogueMesh(t)
ctx := t.Context()
inv := open.inventory
asksWithPaths(t)
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
t.Fatal(err)
}
p := pendingOf(t, open)[0]
if ok, err := inv.ClaimPending(ctx, p.ID); err != nil || !ok {
t.Fatalf("claim: %v %v", ok, err)
}
_, err := unassign(ctx, open, "laptop", "sensors")
if err == nil || !strings.Contains(err.Error(), "is being assigned sensors now") {
t.Fatalf("unassign of a claimed pending assignment: %v", err)
}
if got := pendingOf(t, open)[0]; got.State != inventory.PendingApplying {
t.Fatalf("the claim was overridden: %+v", got)
}
// Pending(zero) is the waiting ones alone (review point 7).
if waiting, err := inv.Pending(ctx, time.Time{}); err != nil || len(waiting) != 0 {
t.Fatalf("Pending(zero) read a claimed row: %+v %v", waiting, err)
}
// Left by a controller that stopped: back to waiting, since sensors is not assigned.
settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute))
if got := pendingOf(t, open)[0]; got.State != inventory.PendingWaiting {
t.Fatalf("a stale claim was not released: %+v", got)
}
// Withdrawn, then the build registers it: applyPending finds nothing to claim.
if _, err := unassign(ctx, open, "laptop", "sensors"); err != nil {
t.Fatal(err)
}
register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"})
if line := applyPending(ctx, open, p, "b-modules/sensors"); line != "" {
t.Fatalf("a withdrawn pending assignment was made: %s", line)
}
if slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
t.Fatal("a withdrawn pending assignment was assigned")
}
// A stale claim whose module was assigned is applied.
q, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "sensors", Build: "b-x"})
if err != nil {
t.Fatal(err)
}
if ok, _ := inv.ClaimPending(ctx, q.ID); !ok {
t.Fatal("claim")
}
if _, err := inv.Assign(ctx, "anchor", "sensors"); err != nil {
t.Fatal(err)
}
settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute))
rows, _ := inv.PendingFor(ctx, "anchor", "sensors")
if len(rows) != 1 || rows[0].State != inventory.PendingApplied {
t.Fatalf("a stale claim of an assigned module: %+v", rows)
}
}
// A build that says nothing within the bound: the pending assignment expires at the next tick, saying so,
// rather than waiting for ever; and a module registered by a process that kept no pending assignment is
// assigned at the next tick.
func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) { func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) {
open := aCatalogueMesh(t) open := aCatalogueMesh(t)
ctx := t.Context() ctx := t.Context()
@@ -295,15 +585,11 @@ func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
} }
// Heard by a process that kept no pending assignment: registered, and nothing made it.
register(t, open, catalogue.Manifest{Module: "heard", Version: "1"}) register(t, open, catalogue.Manifest{Module: "heard", Version: "1"})
said := settleStalePending(ctx, open, time.Now().Add(buildRequestBound)) settlePending(ctx, open, time.Now().Add(buildRequestBound))
if len(said) != 2 {
t.Fatalf("settled %v", said)
}
if !slices.Contains(assignedTo(t, open, "laptop"), "heard") { if !slices.Contains(assignedTo(t, open, "laptop"), "heard") {
t.Fatal("a module registered meanwhile was not assigned at the next look") t.Fatal("a module registered meanwhile was not assigned at the next tick")
} }
byModule := map[string]inventory.PendingAssignment{} byModule := map[string]inventory.PendingAssignment{}
for _, p := range pendingOf(t, open) { for _, p := range pendingOf(t, open) {
@@ -315,6 +601,21 @@ func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) {
if p := byModule["heard"]; p.State != inventory.PendingApplied { if p := byModule["heard"]; p.State != inventory.PendingApplied {
t.Fatalf("a module registered meanwhile: %+v", p) t.Fatalf("a module registered meanwhile: %+v", p)
} }
// Ended rows are deleted after KeptFor (review point 7); open ones never.
if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "lost", Build: "build-lost"}); err != nil {
t.Fatal(err)
}
said := settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour))
rows, err := inv.Pending(ctx, time.Unix(0, 0))
if err != nil {
t.Fatal(err)
}
for _, r := range rows {
if !r.Open() {
t.Fatalf("an ended pending assignment outlived %s: %+v (%v)", inventory.KeptFor, r, said)
}
}
} }
// The seat's assign passes build on; unassign takes none. // The seat's assign passes build on; unassign takes none.
@@ -327,3 +628,16 @@ func TestTheSeatsAssignPassesBuild(t *testing.T) {
t.Fatal("unassign took build") t.Fatal("unassign took build")
} }
} }
// The condition's words are plain.
func TestAnAssignmentNotMadeIsSaidPlainly(t *testing.T) {
o := pendingObservation(inventory.PendingAssignment{Node: "g14", Module: "sensors",
Note: "the pending assignment of sensors to g14 expired: build b-1 of modules/sensors failed: boom"})
w := plainWordings[kindPendingEnded](o)
if why, ok := conditions.PlainWords(w, "g14"); !ok {
t.Fatalf("not plain: %s %+v", why, w)
}
if w.Headline != "sensors was not put on g14" {
t.Fatalf("headline %q", w.Headline)
}
}
+13
View File
@@ -153,6 +153,19 @@ var plainWordings = map[string]func(conditions.Observation) words{
"asleep is normal.", m), "asleep is normal.", m),
Resolved: m + " can be reached again"} Resolved: m + " can be reached again"}
}), }),
// A pending assignment that was not made (novox/hq issue 325, ADR 0261).
kindPendingEnded: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
module := idPart(o, 1)
if module == "" {
module = "a module"
}
return words{Headline: module + " was not put on " + m,
Needs: "decide whether to build it again or take the assignment back; the details say why.",
Explanation: fmt.Sprintf("An assignment of %s to %s waited for its build, and the build did not "+
"register it, so it was not made.", module, m),
Resolved: "Resolved: " + module + " on " + m + " is settled"}
}),
kindBindingKept: worded(func(o conditions.Observation) words { kindBindingKept: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine") m := machineOr(o, "a machine")
return words{Headline: "A module's data source is held on " + m, return words{Headline: "A module's data source is held on " + m,
+4
View File
@@ -165,6 +165,10 @@ func serve(ctx context.Context) (err error) {
// Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for // Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for
// machines to report moves when they have, and a plan left by a replaced controller resumes. // machines to report moves when they have, and a plan left by a replaced controller resumes.
go planTicker(ctx, open) go planTicker(ctx, open)
// And the pending assignments settled on a tick of their own (novox/hq ADR 0261): made once their module
// is registered, ended with why when its build will not register it, raised and cleared as conditions.
// Never by a read.
go settlingPending(ctx, open)
// The durations the core's bounds are set from are kept a month (novox/hq to-be 45 Phase 0). // The durations the core's bounds are set from are kept a month (novox/hq to-be 45 Phase 0).
go forgettingOldDurations(ctx, inv) go forgettingOldDurations(ctx, inv)
// And what the catalogue decided a build meant. The builder's own result is already handled // And what the catalogue decided a build meant. The builder's own result is already handled
+7 -1
View File
@@ -406,6 +406,8 @@ func rebuildCommand(ctx context.Context, args []string) error {
if err != nil { if err != nil {
return err return err
} }
recordAsked(ctx, inventory.BuildRequest{ID: id, Repository: source.Repository, Seat: source.Seat, Path: path,
Ref: ref, For: "rebuild"})
fmt.Printf("rebuild asked as %s\n", id) fmt.Printf("rebuild asked as %s\n", id)
return nil return nil
} }
@@ -490,7 +492,11 @@ func replayCommand(ctx context.Context, args []string) error {
if err := replayRefusal(b, module, history, *register, *older, outstanding); err != nil { if err := replayRefusal(b, module, history, *register, *older, outstanding); err != nil {
return err return err
} }
id, err := buildOneAsked(ctx, source, b.Path, b.Commit, 0, !*register) asker := ""
if *register {
asker = "replay"
}
id, err := buildOneAsked(ctx, source, b.Path, b.Commit, 0, !*register, asker)
if err != nil { if err != nil {
return err return err
} }
+4 -1
View File
@@ -344,7 +344,8 @@ var askABuild func(ctx context.Context, source buildSource, path, ref string) (s
func init() { func init() {
askABuild = func(ctx context.Context, source buildSource, path, ref string) (string, error) { askABuild = func(ctx context.Context, source buildSource, path, ref string) (string, error) {
return buildOneAsked(ctx, source, path, ref, 0, false) // Kept by each asker with its own name once the ask is made (novox/hq issue 325).
return buildOneAsked(ctx, source, path, ref, 0, false, "")
} }
} }
@@ -376,6 +377,8 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
p.Note = fmt.Sprintf("%s could not be asked for: %v", name, err) p.Note = fmt.Sprintf("%s could not be asked for: %v", name, err)
return return
} }
recordAsked(ctx, inventory.BuildRequest{ID: id, Repository: e.Source.Repository, Seat: e.Source.Seat,
Path: e.Source.Path, Ref: followedBranch(e.Source.Ref), Commit: p.Commit, For: "plan"})
state.State = "asked" state.State = "asked"
state.AskedAt = &now state.AskedAt = &now
state.Build = id state.Build = id
+8 -11
View File
@@ -4,7 +4,6 @@ import (
"context" "context"
"flag" "flag"
"fmt" "fmt"
"os"
"sort" "sort"
"strings" "strings"
"time" "time"
@@ -469,11 +468,8 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
if out.conditions, err = openConditions(ctx); err != nil { if out.conditions, err = openConditions(ctx); err != nil {
out.conditionsUnread = err.Error() out.conditionsUnread = err.Error()
} }
// And the assignments waiting for their module's build (novox/hq issue 325): what a missed outcome left // And the assignments waiting for their module's build (novox/hq issue 325, ADR 0261), read only: the
// waiting is settled first, so one never waits silently past its build. // controller's tick settles them, never a read.
for _, line := range settleStalePending(ctx, open, time.Now()) {
fmt.Fprintln(os.Stderr, line)
}
if out.pending, err = inv.Pending(ctx, time.Now().Add(-pendingShownFor)); err != nil { if out.pending, err = inv.Pending(ctx, time.Now().Add(-pendingShownFor)); err != nil {
out.pendingUnread = err.Error() out.pendingUnread = err.Error()
err = nil err = nil
@@ -610,13 +606,14 @@ func (a answers) well() bool {
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 && return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 && len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 && len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 &&
len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingNotWell(a.pending) len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending)
} }
// pendingNotWell is whether any pending assignment is waiting, or ended without being made. // pendingOpen is whether any pending assignment is still to be made. One that ended without being made is
func pendingNotWell(pending []inventory.PendingAssignment) bool { // not counted here: it is a condition, open until it is answered (ADR 0261).
func pendingOpen(pending []inventory.PendingAssignment) bool {
for _, p := range pending { for _, p := range pending {
if p.State != inventory.PendingApplied && p.State != inventory.PendingWithdrawn { if p.Open() {
return true return true
} }
} }
@@ -632,7 +629,7 @@ func printPending(pending []inventory.PendingAssignment, unread string) {
} }
var waiting, ended []inventory.PendingAssignment var waiting, ended []inventory.PendingAssignment
for _, p := range pending { for _, p := range pending {
if p.State == inventory.PendingWaiting { if p.Open() {
waiting = append(waiting, p) waiting = append(waiting, p)
} else { } else {
ended = append(ended, p) ended = append(ended, p)
+5 -4
View File
@@ -179,10 +179,11 @@ var ControllerVerbs = []Verb{
}, []string{"node"}, "files", "diff")}, }, []string{"node"}, "files", "diff")},
{Name: "assign", Description: "Put a module on a machine. Refused with the mesh's own words when it cannot resolve there, " + {Name: "assign", Description: "Put a module on a machine. Refused with the mesh's own words when it cannot resolve there, " +
"or when a seat its resources are applied through is held by nothing on the machine (novox/hq ADR 0207). " + "or when a seat its resources are applied through is held by nothing on the machine (novox/hq ADR 0207). " +
"A module not registered yet is looked for (novox/hq issue 325): while its build runs — a merge that adds " + "A module not registered yet is looked for (novox/hq issue 325, ADR 0261): while its build runs — a merge " +
"a module asks for one — the assignment is kept as pending and made when the build registers it, said in " + "that adds a module asks for one — the assignment is kept as pending and the controller makes it when the " +
"`status`, and expires with why if the build fails; a module known and not built is said, and build asks " + "build registers it; `status` lists it, and if the build fails it expires with why and raises a condition. " +
"for its build; a name the mesh never heard of is refused with the closest names.", "A module known and not built is said, and build asks for its build; a name with no module and no build " +
"request kept is refused with the closest names.",
Input: schema(map[string]string{"node": "the machine's name", Input: schema(map[string]string{"node": "the machine's name",
"module": "the module's name; several comma-separated are judged together", "module": "the module's name; several comma-separated are judged together",
"build": "\"true\": for a module known and not built (its last build failed, or was never asked), ask " + "build": "\"true\": for a module known and not built (its last build failed, or was never asked), ask " +
@@ -1,15 +1,15 @@
-- An assignment waits for its module's build (novox/hq issue 325). -- An assignment waits for its module's build (novox/hq issue 325, ADR 0261).
-- --
-- A merge that adds a module asks for its build (issue 300), and the module is registered when the build's -- A merge that adds a module asks for its build (issue 300), and the module is registered when the build's
-- outcome is taken in, minutes later. An `assign` in between was answered "no module of that name", which -- outcome is taken in, minutes later. An `assign` in between was answered "no module of that name", which
-- read as "nobody registered it". The controller now keeps every build it asks for, so `assign` can tell a -- read as "nobody registered it". The controller now keeps every build it asks for, so `assign` can tell a
-- build in flight from a module it never heard of, and an assignment made while the build runs is kept as -- build in flight from a module it never heard of, and an assignment made while the build runs is kept as
-- pending and made when the build registers the module, or ended with the reason when it does not. -- pending and made by the controller when the build registers the module, or ended with the reason.
-- Every build the controller asked for, whatever asked it: a merge, a plan's tier, a person's `build`, an -- Every build the controller asked for: a merge's new module, a plan's tier, a person's `build`, an `assign`
-- `assign` with build. Its outcome is the `build` row of the same id, written when the outcome is taken in; -- with build. Kept once the ask is made, never before. Its outcome is the `build` row of the same id;
-- an ask without one is still running, or was lost. not_asked is why a merge's ask of a new module could -- an ask without one is still running, or was lost. not_asked is why the ask could not be made or waited
-- not be made, and the id is then not a build's. -- for; for a merge that could not ask, the id is the controller's and no build's.
create table build_request ( create table build_request (
id text primary key, id text primary key,
repository text not null, repository text not null,
@@ -23,20 +23,27 @@ create table build_request (
); );
create index build_request_asked_at on build_request (asked_at); create index build_request_asked_at on build_request (asked_at);
-- An assignment kept until its module is registered. waiting until the build registers the module, then -- An assignment kept until its module is registered (ADR 0261). waiting until the build registers the
-- applied (the assignment was made), refused (the assignment was refused then, with the refusal), expired -- module; applying while the controller makes it, under the machine's hold; then applied, refused (the
-- (the build failed, was not registered, or said nothing within its bound) or withdrawn (`unassign`). -- assignment was refused then), expired (the build failed, was not registered, or said nothing within its
-- bound) or withdrawn (`unassign`). raised_at is when an expiry or refusal was raised as a condition,
-- acknowledged_at when a person took it back with `unassign`, cleared_at when its condition was cleared.
-- Gone with its machine. Ended rows are deleted after 30 days.
create table pending_assignment ( create table pending_assignment (
id bigserial primary key, id bigserial primary key,
node text not null, node uuid not null references node (id) on delete cascade,
module text not null, module text not null,
build_id text not null, build_id text not null,
repository text not null default '', repository text not null default '',
source_path text not null default '', source_path text not null default '',
since timestamptz not null default now(), since timestamptz not null default now(),
state text not null default 'waiting' state text not null default 'waiting'
check (state in ('waiting', 'applied', 'refused', 'expired', 'withdrawn')), check (state in ('waiting', 'applying', 'applied', 'refused', 'expired', 'withdrawn')),
note text not null default '', note text not null default '',
settled_at timestamptz settled_at timestamptz,
raised_at timestamptz,
acknowledged_at timestamptz,
cleared_at timestamptz
); );
create unique index pending_assignment_waiting on pending_assignment (node, module) where state = 'waiting'; create unique index pending_assignment_open on pending_assignment (node, module)
where state in ('waiting', 'applying');
+190 -64
View File
@@ -10,36 +10,36 @@ import (
"github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5"
) )
// What the controller asked to be built, and the assignments waiting for it (novox/hq issue 325). // What the controller asked to be built, and the assignments waiting for it (novox/hq issue 325, ADR 0261).
// //
// A build's outcome was kept, and its asking was not: between a merge asking for a new module and the // A build's outcome was kept, and its asking was not: between a merge asking for a new module and the
// outcome registering it, the mesh had no record that anything was coming, and `assign` answered "no module // outcome registering it, the mesh had no record that anything was coming, and `assign` answered "no module
// of that name" for a module minutes from existing. Kept here, an ask lets `assign` say which case it is in, // of that name" for a module minutes from existing. Kept here, a build request lets `assign` say which case
// and an assignment made while the build runs is kept until the build registers the module. // it is in, and an assignment made while the build runs is kept until the build registers the module.
// requestKeptFor is how long an ask is kept: long past any build's bound, short enough to stay small. // KeptFor is how long a build request, and an ended pending assignment, is kept.
const requestKeptFor = 30 * 24 * time.Hour const KeptFor = 30 * 24 * time.Hour
// BuildRequest is one build the controller asked for. // BuildRequest is one build the controller asked for.
type BuildRequest struct { type BuildRequest struct {
// ID is the build's id, as its outcome will carry it; for an ask that could not be made, an id of // ID is the build's id, as its outcome will carry it; for a merge whose ask could not be made, an id
// the controller's making that no build carries. // of the controller's making that no build carries.
ID string ID string
// Repository and Seat are the source as the mesh spells it (novox/hq ADR 0111): a path on the seat's // Repository and Seat are the source as the mesh spells it (novox/hq ADR 0111): a path on the seat's
// holder, or a URL with no seat. Path is the module's directory in it, Ref the branch or commit asked. // holder, or a URL with no seat. Path is the module's directory in it, Ref the branch or commit asked.
Repository, Seat, Path, Ref string Repository, Seat, Path, Ref string
// Commit is the commit that made the ask, where one did: a merge's. // Commit is the commit that made the ask, where one did: a merge's.
Commit string Commit string
// For says what asked: "merge", "build", "plan", "assign". // For says who asked: "merge", "plan", "build" (a person), "assign".
For string For string
// NotAsked is why the ask could not be made, empty when it was. // NotAsked is why the ask could not be made, or not waited for; empty when it was.
NotAsked string NotAsked string
At time.Time At time.Time
} }
// Name is the module this ask is expected to register, read from its directory: the last element of its // Name is the module this request is expected to register, read from its directory: the last element of
// path, or of its repository for a module at the repository's root. A guess until the build says, and the // its path, or of its repository for a module at the repository's root. A guess until the build says, and
// one a person reads too: `modules/sensors` is sensors. // the one a person reads too: `modules/sensors` is sensors.
func (a BuildRequest) Name() string { func (a BuildRequest) Name() string {
if p := strings.Trim(a.Path, "/"); p != "" && p != "." { if p := strings.Trim(a.Path, "/"); p != "" && p != "." {
return path.Base(p) return path.Base(p)
@@ -47,19 +47,19 @@ func (a BuildRequest) Name() string {
return strings.TrimSuffix(path.Base(strings.TrimRight(a.Repository, "/")), ".git") return strings.TrimSuffix(path.Base(strings.TrimRight(a.Repository, "/")), ".git")
} }
// RequestOutcome is an ask beside what came of it, where anything did. // RequestOutcome is a build request beside what came of it, where anything did.
type RequestOutcome struct { type RequestOutcome struct {
BuildRequest BuildRequest
// Heard is whether an outcome of this id was taken in; Failed is its builder's words when it failed, // Heard is whether an outcome of this id was taken in; Failed is its builder's words when it failed,
// Module what it registered as, HeardAt when it was taken in. // Module what the build said it built, HeardAt when it was taken in.
Heard bool Heard bool
Failed string Failed string
Module string Module string
HeardAt time.Time HeardAt time.Time
} }
// RecordBuildRequest keeps one ask. Idempotent on the id: a merge's ask recorded by the merge and by the // RecordBuildRequest keeps one build request, once it is asked. Idempotent on the id. Requests older than
// asking itself is one ask. Asks older than requestKeptFor are let go in the same act. // KeptFor are deleted in the same act.
func (i *Inventory) RecordBuildRequest(ctx context.Context, a BuildRequest) error { func (i *Inventory) RecordBuildRequest(ctx context.Context, a BuildRequest) error {
at := a.At at := a.At
if at.IsZero() { if at.IsZero() {
@@ -72,17 +72,25 @@ func (i *Inventory) RecordBuildRequest(ctx context.Context, a BuildRequest) erro
if _, err := i.store.Pool().Exec(ctx, if _, err := i.store.Pool().Exec(ctx,
`insert into build_request (id, repository, seat, source_path, ref, commit_hash, asked_for, not_asked, asked_at) `insert into build_request (id, repository, seat, source_path, ref, commit_hash, asked_for, not_asked, asked_at)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9) values ($1, $2, $3, $4, $5, $6, $7, $8, $9)
on conflict (id) do update set on conflict (id) do nothing`,
commit_hash = case when build_request.commit_hash = '' then excluded.commit_hash else build_request.commit_hash end,
asked_for = case when build_request.asked_for = '' then excluded.asked_for else build_request.asked_for end`,
a.ID, a.Repository, a.Seat, strings.Trim(a.Path, "/"), a.Ref, a.Commit, a.For, notAsked, at); err != nil { a.ID, a.Repository, a.Seat, strings.Trim(a.Path, "/"), a.Ref, a.Commit, a.For, notAsked, at); err != nil {
return err return err
} }
_, err := i.store.Pool().Exec(ctx, `delete from build_request where asked_at < $1`, time.Now().Add(-requestKeptFor)) _, err := i.store.Pool().Exec(ctx, `delete from build_request where asked_at < $1`, time.Now().Add(-KeptFor))
return err return err
} }
// RequestsNamed is every ask kept whose directory names the module, newest first, each with its outcome. // MarkNotAsked says a build request was asked and its asker could not hand it over, or stopped waiting for
// it: the words are kept, unless its outcome was heard first.
func (i *Inventory) MarkNotAsked(ctx context.Context, id, why string) error {
_, err := i.store.Pool().Exec(ctx,
`update build_request set not_asked = $2
where id = $1 and not exists (select 1 from build where build.id = $1)`, id, why)
return err
}
// RequestsNamed is every build request kept whose directory names the module, or whose build said it built
// it, newest first, each with its outcome.
func (i *Inventory) RequestsNamed(ctx context.Context, name string) ([]RequestOutcome, error) { func (i *Inventory) RequestsNamed(ctx context.Context, name string) ([]RequestOutcome, error) {
all, err := i.requests(ctx) all, err := i.requests(ctx)
if err != nil { if err != nil {
@@ -97,8 +105,8 @@ func (i *Inventory) RequestsNamed(ctx context.Context, name string) ([]RequestOu
return out, nil return out, nil
} }
// RequestedNames is the name of every module an ask kept is expected to register: what `assign` offers as a // RequestedNames is the name of every module a kept build request is expected to register: what `assign`
// closest name beside the modules registered. // offers as a closest name beside the modules registered.
func (i *Inventory) RequestedNames(ctx context.Context) ([]string, error) { func (i *Inventory) RequestedNames(ctx context.Context) ([]string, error) {
all, err := i.requests(ctx) all, err := i.requests(ctx)
if err != nil { if err != nil {
@@ -141,13 +149,14 @@ func (i *Inventory) requests(ctx context.Context) ([]RequestOutcome, error) {
// The states of a pending assignment. // The states of a pending assignment.
const ( const (
PendingWaiting = "waiting" PendingWaiting = "waiting"
PendingApplying = "applying"
PendingApplied = "applied" PendingApplied = "applied"
PendingRefused = "refused" PendingRefused = "refused"
PendingExpired = "expired" PendingExpired = "expired"
PendingWithdrawn = "withdrawn" PendingWithdrawn = "withdrawn"
) )
// PendingAssignment is an assignment kept until its module is registered. // PendingAssignment is an assignment kept until its module is registered (ADR 0261).
type PendingAssignment struct { type PendingAssignment struct {
ID int64 ID int64
Node string Node string
@@ -159,22 +168,48 @@ type PendingAssignment struct {
// Note is what ended it, in the words said when it ended. // Note is what ended it, in the words said when it ended.
Note string Note string
Settled *time.Time Settled *time.Time
// Raised is when an expiry or refusal was raised as a condition; Acknowledged when a person took it
// back; Cleared when its condition was cleared.
Raised, Acknowledged, Cleared *time.Time
}
// Open is whether the pending assignment is still to be made.
func (p PendingAssignment) Open() bool {
return p.State == PendingWaiting || p.State == PendingApplying
} }
// ErrAlreadyPending is a second pending assignment of one module to one machine. // ErrAlreadyPending is a second pending assignment of one module to one machine.
var ErrAlreadyPending = errors.New("that assignment is already waiting for its build") var ErrAlreadyPending = errors.New("that assignment is already waiting for its build")
// RecordPending keeps an assignment until its module is registered. One waits per machine and module. const pendingColumns = `p.id, n.name, p.module, p.build_id, p.repository, p.source_path, p.since, p.state, p.note,
p.settled_at, p.raised_at, p.acknowledged_at, p.cleared_at`
func scanPending(rows pgx.Rows) ([]PendingAssignment, error) {
defer rows.Close()
var out []PendingAssignment
for rows.Next() {
var p PendingAssignment
if err := rows.Scan(&p.ID, &p.Node, &p.Module, &p.Build, &p.Repository, &p.Path, &p.Since, &p.State,
&p.Note, &p.Settled, &p.Raised, &p.Acknowledged, &p.Cleared); err != nil {
return nil, err
}
out = append(out, p)
}
return out, rows.Err()
}
// RecordPending keeps an assignment until its module is registered. One is open per machine and module.
func (i *Inventory) RecordPending(ctx context.Context, p PendingAssignment) (PendingAssignment, error) { func (i *Inventory) RecordPending(ctx context.Context, p PendingAssignment) (PendingAssignment, error) {
if _, err := i.NodeByName(ctx, p.Node); err != nil { node, err := i.NodeByName(ctx, p.Node)
if err != nil {
return PendingAssignment{}, err return PendingAssignment{}, err
} }
err := i.store.Pool().QueryRow(ctx, err = i.store.Pool().QueryRow(ctx,
`insert into pending_assignment (node, module, build_id, repository, source_path) `insert into pending_assignment (node, module, build_id, repository, source_path)
values ($1, $2, $3, $4, $5) values ($1, $2, $3, $4, $5)
on conflict (node, module) where state = 'waiting' do nothing on conflict (node, module) where state in ('waiting', 'applying') do nothing
returning id, since`, returning id, since`,
p.Node, p.Module, p.Build, p.Repository, strings.Trim(p.Path, "/")).Scan(&p.ID, &p.Since) node.ID, p.Module, p.Build, p.Repository, strings.Trim(p.Path, "/")).Scan(&p.ID, &p.Since)
if errors.Is(err, pgx.ErrNoRows) { if errors.Is(err, pgx.ErrNoRows) {
return PendingAssignment{}, ErrAlreadyPending return PendingAssignment{}, ErrAlreadyPending
} }
@@ -185,44 +220,135 @@ func (i *Inventory) RecordPending(ctx context.Context, p PendingAssignment) (Pen
return p, nil return p, nil
} }
// Pending is every pending assignment still waiting, and every one ended since the time given, oldest // RepointPending makes a waiting pending assignment wait for another build. False when it no longer waits.
// first. A zero time is the waiting ones alone. func (i *Inventory) RepointPending(ctx context.Context, id int64, build, repository, sourcePath string) (bool, error) {
func (i *Inventory) Pending(ctx context.Context, endedSince time.Time) ([]PendingAssignment, error) {
rows, err := i.store.Pool().Query(ctx,
`select id, node, module, build_id, repository, source_path, since, state, note, settled_at
from pending_assignment
where state = 'waiting' or (settled_at is not null and settled_at >= $1)
order by since, id`, endedSince)
if err != nil {
return nil, err
}
defer rows.Close()
var out []PendingAssignment
for rows.Next() {
var p PendingAssignment
if err := rows.Scan(&p.ID, &p.Node, &p.Module, &p.Build, &p.Repository, &p.Path, &p.Since, &p.State,
&p.Note, &p.Settled); err != nil {
return nil, err
}
if endedSince.IsZero() && p.State != PendingWaiting {
continue
}
out = append(out, p)
}
return out, rows.Err()
}
// SettlePending ends a waiting pending assignment. False when it was no longer waiting: another process
// settled it first, and its word stands.
func (i *Inventory) SettlePending(ctx context.Context, id int64, state, note string) (bool, error) {
if state == PendingWaiting {
return false, errors.New("a pending assignment is settled to applied, refused, expired or withdrawn")
}
tag, err := i.store.Pool().Exec(ctx, tag, err := i.store.Pool().Exec(ctx,
`update pending_assignment set state = $2, note = $3, settled_at = now() `update pending_assignment set build_id = $2, repository = $3, source_path = $4
where id = $1 and state = 'waiting'`, id, state, note) where id = $1 and state = 'waiting'`, id, build, repository, strings.Trim(sourcePath, "/"))
if err != nil { if err != nil {
return false, err return false, err
} }
return tag.RowsAffected() == 1, nil return tag.RowsAffected() == 1, nil
} }
// Pending is every pending assignment still waiting, oldest first, when endedSince is zero. Otherwise it is
// every open one and every one ended since then.
func (i *Inventory) Pending(ctx context.Context, endedSince time.Time) ([]PendingAssignment, error) {
if endedSince.IsZero() {
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
from pending_assignment p join node n on n.id = p.node
where p.state = 'waiting' order by p.since, p.id`)
if err != nil {
return nil, err
}
return scanPending(rows)
}
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
from pending_assignment p join node n on n.id = p.node
where p.state in ('waiting', 'applying') or p.settled_at >= $1
order by p.since, p.id`, endedSince)
if err != nil {
return nil, err
}
return scanPending(rows)
}
// PendingFor is every pending assignment of one module to one machine, newest first.
func (i *Inventory) PendingFor(ctx context.Context, node, module string) ([]PendingAssignment, error) {
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
from pending_assignment p join node n on n.id = p.node
where n.name = $1 and p.module = $2 order by p.since desc, p.id desc`, node, module)
if err != nil {
return nil, err
}
return scanPending(rows)
}
// Stuck is every pending assignment left applying since before the time given: a controller that claimed
// it and stopped before it said what came of it.
func (i *Inventory) Stuck(ctx context.Context, before time.Time) ([]PendingAssignment, error) {
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
from pending_assignment p join node n on n.id = p.node
where p.state = 'applying' and p.settled_at < $1 order by p.id`, before)
if err != nil {
return nil, err
}
return scanPending(rows)
}
// ToRaise is every pending assignment that expired or was refused and has not been raised as a condition;
// ToClear every one raised and not cleared.
func (i *Inventory) ToRaise(ctx context.Context) ([]PendingAssignment, error) {
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
from pending_assignment p join node n on n.id = p.node
where p.state in ('expired', 'refused') and p.raised_at is null order by p.id`)
if err != nil {
return nil, err
}
return scanPending(rows)
}
func (i *Inventory) ToClear(ctx context.Context) ([]PendingAssignment, error) {
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
from pending_assignment p join node n on n.id = p.node
where p.raised_at is not null and p.cleared_at is null order by p.id`)
if err != nil {
return nil, err
}
return scanPending(rows)
}
// MarkPending stamps a pending assignment raised, acknowledged or cleared, now.
func (i *Inventory) MarkPending(ctx context.Context, id int64, what string) error {
column := map[string]string{"raised": "raised_at", "acknowledged": "acknowledged_at", "cleared": "cleared_at"}[what]
if column == "" {
return errors.New("a pending assignment is marked raised, acknowledged or cleared")
}
_, err := i.store.Pool().Exec(ctx,
`update pending_assignment set `+column+` = coalesce(`+column+`, now()) where id = $1`, id)
return err
}
// ClaimPending takes a waiting pending assignment for making, so nothing else settles or withdraws it while
// it is made. False when it no longer waits: another process took it, or it was withdrawn.
func (i *Inventory) ClaimPending(ctx context.Context, id int64) (bool, error) {
tag, err := i.store.Pool().Exec(ctx,
`update pending_assignment set state = 'applying', settled_at = now() where id = $1 and state = 'waiting'`, id)
if err != nil {
return false, err
}
return tag.RowsAffected() == 1, nil
}
// ReleaseClaim puts a claimed pending assignment back to waiting.
func (i *Inventory) ReleaseClaim(ctx context.Context, id int64) error {
_, err := i.store.Pool().Exec(ctx,
`update pending_assignment set state = 'waiting', settled_at = null where id = $1 and state = 'applying'`, id)
return err
}
// SettlePending ends a pending assignment in the state it is read in (from): waiting for an expiry or a
// withdrawal, applying for the outcome of making it. False when it was no longer in that state.
func (i *Inventory) SettlePending(ctx context.Context, id int64, from, state, note string) (bool, error) {
if state == PendingWaiting || state == PendingApplying {
return false, errors.New("a pending assignment is settled to applied, refused, expired or withdrawn")
}
tag, err := i.store.Pool().Exec(ctx,
`update pending_assignment set state = $3, note = $4, settled_at = now()
where id = $1 and state = $2`, id, from, state, note)
if err != nil {
return false, err
}
return tag.RowsAffected() == 1, nil
}
// ForgetEndedPending deletes ended pending assignments settled more than KeptFor ago, and says how many.
func (i *Inventory) ForgetEndedPending(ctx context.Context, now time.Time) (int64, error) {
tag, err := i.store.Pool().Exec(ctx,
`delete from pending_assignment where state not in ('waiting', 'applying') and settled_at < $1`,
now.Add(-KeptFor))
if err != nil {
return 0, err
}
return tag.RowsAffected(), nil
}
+25
View File
@@ -0,0 +1,25 @@
package inventory
import (
"testing"
"time"
)
// A pending assignment goes with its machine (novox/hq issue 325, review of mesh-controller#150 point 7).
func TestAPendingAssignmentGoesWithItsMachine(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
if _, err := inv.AddNode(ctx, "leaving"); err != nil {
t.Fatal(err)
}
if _, err := inv.RecordPending(ctx, PendingAssignment{Node: "leaving", Module: "sensors", Build: "b-1"}); err != nil {
t.Fatal(err)
}
if _, err := inv.store.Pool().Exec(ctx, `delete from node where name = 'leaving'`); err != nil {
t.Fatalf("a machine with a pending assignment could not be removed: %v", err)
}
rows, err := inv.Pending(ctx, time.Unix(0, 0))
if err != nil || len(rows) != 0 {
t.Fatalf("a pending assignment outlived its machine: %+v %v", rows, err)
}
}