jochen
af63b233db
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery delivered
mesh/delivery-group group feat/setting-defaults failed: a member failed
A key dropped silently ran a module without what its manifest says, and a key inside a block still failed the whole catalogue. Judge a key by what it is about, and narrow the listing to one machine.
753 lines
30 KiB
Go
753 lines
30 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// A module not registered yet, and the assignment that waits for it (novox/hq issue 325, ADR 0261).
|
|
//
|
|
// On 2026-10-08 the catalogue's pull request adding `sensors` merged; a minute later `assign g14 sensors`
|
|
// answered "no module of that name: sensors", and a few minutes later the same call worked. The merge had
|
|
// asked for the build (issue 300) and the build had not finished. The answer read as "you forgot to register
|
|
// it", and nothing in it said a build was on its way.
|
|
//
|
|
// So an assignment of a module the catalogue does not hold looks further before it refuses, and says which
|
|
// of three cases it is in:
|
|
//
|
|
// - **a build is in flight**: a build request for the module's directory has no outcome yet, or its build
|
|
// succeeded a moment ago and is being registered. The assignment is kept as a **pending assignment**,
|
|
// which the controller makes when the build registers the module (ADR 0261).
|
|
// - **known, not built**: the controller asked for the directory before, and the build failed, was not
|
|
// registered, said nothing within its bound, or could not be asked. Said, with `build` and assign's
|
|
// own build argument, which asks for the build and keeps the assignment pending on it.
|
|
// - **unknown**: no module registered and no build request kept by that name. Refused as before, with the
|
|
// closest names.
|
|
//
|
|
// **Pending by default while a build is in flight** (ADR 0261), without an argument to ask for it: an
|
|
// assignment is what a person meant and is kept even when its machine does not resolve (acts.go), and
|
|
// `unassign` withdraws it. Asking for a second act once the build lands is the two acts by hand issue 300
|
|
// removed. A build is never asked for by default: it runs on another machine, and a directory whose last
|
|
// build failed is a fault to look at before it is a thing to retry.
|
|
//
|
|
// **Settling is the controller's tick, never a read** (settlingPending): `status`, the board and the summary
|
|
// only read the rows.
|
|
|
|
// buildRequestBound is how long a build request may go without an outcome before it is no longer read as in
|
|
// flight, and a pending assignment waiting for it expires. Generous: a build waits behind others on the
|
|
// build seat, and a pending assignment that waits a little longer costs nothing.
|
|
const buildRequestBound = 2 * time.Hour
|
|
|
|
// registerGrace is how long a build heard as built is read as still in flight while it is not registered:
|
|
// the outcome is recorded first and registered after it, in the same take-in. Past it, the build was
|
|
// recorded and not registered, and says so.
|
|
const registerGrace = 5 * time.Minute
|
|
|
|
// pendingShownFor is how long an ended pending assignment is listed in `status`.
|
|
const pendingShownFor = 24 * time.Hour
|
|
|
|
// settleEvery is how often the controller settles the pending assignments.
|
|
const settleEvery = time.Minute
|
|
|
|
// claimStaleAfter is how long a pending assignment may be held as applying before the tick reads the claim
|
|
// as left by a controller that stopped, and settles it from what the mesh holds.
|
|
const claimStaleAfter = 5 * time.Minute
|
|
|
|
// kindPendingEnded is a pending assignment that ended without being made: expired or refused.
|
|
const kindPendingEnded = "assignment-not-made"
|
|
|
|
// assignOptions are what an assignment was asked with beside its machine and modules.
|
|
type assignOptions struct {
|
|
// Build asks for the build of a module known and not built, and keeps the assignment pending on it.
|
|
Build bool
|
|
}
|
|
|
|
// recordBuildRequest keeps a build request so `assign` can find it, once it is asked. Said, never fatal: the
|
|
// build was asked.
|
|
func recordBuildRequest(ctx context.Context, inv *inventory.Inventory, r inventory.BuildRequest) {
|
|
if err := inv.RecordBuildRequest(ctx, r); err != nil {
|
|
fmt.Fprintf(os.Stderr, "build %s was asked, and could not be kept as a build request, so `assign` "+
|
|
"will not know it is coming: %v\n", r.ID, err)
|
|
}
|
|
}
|
|
|
|
// The cases of a module the catalogue does not hold (novox/hq issue 325).
|
|
const (
|
|
unknownModule = iota
|
|
buildInFlight
|
|
knownNotBuilt
|
|
)
|
|
|
|
// notHeld is where a module the catalogue does not hold stands: its case, the build request it was read
|
|
// from, and the sentence that says it.
|
|
type notHeld struct {
|
|
kind int
|
|
request inventory.RequestOutcome
|
|
said string
|
|
}
|
|
|
|
// whereIs looks for a module the catalogue does not hold among the build requests the controller keeps.
|
|
func whereIs(ctx context.Context, inv *inventory.Inventory, module string, now time.Time) (notHeld, error) {
|
|
requests, err := inv.RequestsNamed(ctx, module)
|
|
if err != nil {
|
|
return notHeld{}, err
|
|
}
|
|
if r, ok := inFlight(requests, now); ok {
|
|
being := "being built"
|
|
if r.Heard {
|
|
being = "built and being registered"
|
|
}
|
|
return notHeld{kind: buildInFlight, request: r, said: fmt.Sprintf("%s is not registered yet: it is %s "+
|
|
"from %s since %s, as build %s; it can be assigned once that build registers it", module, being,
|
|
requestSource(r), clock(r.At), r.ID)}, nil
|
|
}
|
|
if len(requests) > 0 {
|
|
return notHeld{kind: knownNotBuilt, request: requests[0], said: fmt.Sprintf("%s is known and not "+
|
|
"registered: %s", module, whyNotBuilt(requests[0], now))}, nil
|
|
}
|
|
// Only what was looked at is claimed: the catalogue, and the build requests the controller keeps.
|
|
said := fmt.Sprintf("%v: %s — the catalogue holds no module of that name, and no build request the "+
|
|
"controller kept (the last 30 days) is for a directory of that name", inventory.ErrNoSuchModule, module)
|
|
if near := closestNames(ctx, inv, module); len(near) > 0 {
|
|
said += "; the closest names it holds: " + strings.Join(near, ", ")
|
|
}
|
|
return notHeld{kind: unknownModule, said: said}, nil
|
|
}
|
|
|
|
// notInCatalogue is the modules named that the catalogue does not hold.
|
|
func notInCatalogue(ctx context.Context, open *stores, modules []string) ([]string, error) {
|
|
shelf, err := open.inventory.Catalogue(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var missing []string
|
|
for _, m := range modules {
|
|
if _, ok := shelf[m]; !ok {
|
|
missing = append(missing, m)
|
|
}
|
|
}
|
|
return missing, nil
|
|
}
|
|
|
|
// openPending is the open pending assignment of a module to a machine, if there is one.
|
|
func openPending(ctx context.Context, inv *inventory.Inventory, node, module string) (*inventory.PendingAssignment, error) {
|
|
rows, err := inv.PendingFor(ctx, node, module)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for i := range rows {
|
|
if rows[i].Open() {
|
|
return &rows[i], nil
|
|
}
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
// notRegistered answers an assignment of a module the catalogue does not hold: pending on a build in flight,
|
|
// known and not built, or unknown (novox/hq issue 325). An answer with no error is a pending assignment kept.
|
|
func notRegistered(ctx context.Context, open *stores, node, module string, opts assignOptions) (string, error) {
|
|
inv := open.inventory
|
|
if _, err := inv.NodeByName(ctx, node); err != nil {
|
|
return "", err
|
|
}
|
|
now := time.Now()
|
|
where, err := whereIs(ctx, inv, module, now)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
waiting, err := openPending(ctx, inv, node, module)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
switch where.kind {
|
|
case buildInFlight:
|
|
lead := where.said
|
|
if opts.Build {
|
|
lead += "\n no second build was asked: this one is running"
|
|
}
|
|
return keepPending(ctx, open, node, module, where.request, waiting, lead)
|
|
case knownNotBuilt:
|
|
last := where.request
|
|
if !opts.Build {
|
|
also := ""
|
|
if waiting != nil {
|
|
also = fmt.Sprintf("\n %s already has a pending assignment of %s, waiting for build %s; build "+
|
|
"\"true\" asks for a new build and makes it wait for that one", node, module, waiting.Build)
|
|
}
|
|
return "", fmt.Errorf("%w: %s%s\n assign it again with build \"true\" to ask for its build and keep this "+
|
|
"assignment until the build registers it; or `build` it (repository %s, path %s) and assign it "+
|
|
"once it is registered", inventory.ErrNoSuchModule, where.said, also, last.Repository, orRoot(last.Path))
|
|
}
|
|
if waiting != nil && waiting.State == inventory.PendingApplying {
|
|
return "", fmt.Errorf("%s is being assigned %s now; nothing was asked", node, module)
|
|
}
|
|
source := buildSource{Repository: last.Repository, Seat: last.Seat}
|
|
id, err := askABuild(ctx, source, last.Path, last.Ref)
|
|
if err != nil {
|
|
return "", fmt.Errorf("%s\n its build could not be asked for: %w; nothing was assigned", where.said, err)
|
|
}
|
|
asked := inventory.RequestOutcome{BuildRequest: inventory.BuildRequest{ID: id, Repository: last.Repository,
|
|
Seat: last.Seat, Path: last.Path, Ref: last.Ref, For: "assign", At: now.UTC()}}
|
|
recordBuildRequest(ctx, inv, asked.BuildRequest)
|
|
lead := fmt.Sprintf("%s\n build %s of %s is asked for now; %s can be assigned once it registers it",
|
|
where.said, id, requestSource(asked), module)
|
|
return keepPending(ctx, open, node, module, asked, waiting, lead)
|
|
}
|
|
answer := strings.TrimPrefix(where.said, inventory.ErrNoSuchModule.Error()) +
|
|
"\n a module in a repository is built with `build` (its repository and path), then assigned"
|
|
if opts.Build {
|
|
answer += "\n build \"true\" asks for nothing here: the controller has no build request saying where a " +
|
|
"module of that name is"
|
|
}
|
|
return "", fmt.Errorf("%w%s", inventory.ErrNoSuchModule, answer)
|
|
}
|
|
|
|
// keepPending records the assignment as pending on a build request — or, where one is already open, makes it
|
|
// wait for that build — and says so.
|
|
func keepPending(ctx context.Context, open *stores, node, module string, r inventory.RequestOutcome,
|
|
waiting *inventory.PendingAssignment, lead string) (string, error) {
|
|
inv := open.inventory
|
|
if waiting != nil {
|
|
if waiting.Build == r.ID {
|
|
return lead + fmt.Sprintf("\n %s already waits for %s on this build: nothing changed", node, module), nil
|
|
}
|
|
moved, err := inv.RepointPending(ctx, waiting.ID, r.ID, r.Repository, r.Path)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if !moved {
|
|
return lead + fmt.Sprintf("\n the pending assignment of %s to %s ended while this was asked: `status` "+
|
|
"says how", module, node), nil
|
|
}
|
|
return lead + fmt.Sprintf("\n the pending assignment of %s to %s, which waited for build %s, now waits "+
|
|
"for build %s", module, node, waiting.Build, r.ID), nil
|
|
}
|
|
_, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: node, Module: module, Build: r.ID,
|
|
Repository: r.Repository, Path: r.Path})
|
|
if errors.Is(err, inventory.ErrAlreadyPending) {
|
|
return lead + fmt.Sprintf("\n %s already waits for %s: nothing changed", node, module), nil
|
|
}
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
// The controller makes it, not this act (ADR 0261): when the build registers the module, or at its next
|
|
// tick if the module was registered between the look and the record.
|
|
return lead + fmt.Sprintf("\n the assignment is kept as pending: the controller assigns %s to %s when the "+
|
|
"build registers it, and `status` lists it until then. If the build fails it expires, says why and "+
|
|
"raises a condition; `unassign %s %s` withdraws it", module, node, node, module), nil
|
|
}
|
|
|
|
// inFlight is the newest build request for the module still to register it: no outcome yet within the
|
|
// bound, or a successful outcome heard a moment ago and being registered.
|
|
func inFlight(requests []inventory.RequestOutcome, now time.Time) (inventory.RequestOutcome, bool) {
|
|
for _, r := range requests {
|
|
if stillComing(r, now) {
|
|
return r, true
|
|
}
|
|
}
|
|
return inventory.RequestOutcome{}, false
|
|
}
|
|
|
|
func stillComing(r inventory.RequestOutcome, now time.Time) bool {
|
|
if r.Heard {
|
|
return r.Failed == "" && now.Sub(r.HeardAt) < registerGrace
|
|
}
|
|
return r.NotAsked == "" && now.Sub(r.At) < buildRequestBound
|
|
}
|
|
|
|
// whyNotBuilt says what came of a module's last build request. What was heard comes first: an outcome is the
|
|
// last word whatever its asker said.
|
|
func whyNotBuilt(r inventory.RequestOutcome, now time.Time) string {
|
|
where := fmt.Sprintf("%s in %s", orRoot(r.Path), r.Repository)
|
|
switch {
|
|
case r.Heard && r.Failed != "":
|
|
return fmt.Sprintf("%s; its last build, %s at %s, failed: %s", where, r.ID, clock(r.HeardAt),
|
|
firstLine(r.Failed))
|
|
case r.Heard && r.Module != "" && r.Module != r.Name():
|
|
return fmt.Sprintf("%s; its last build, %s, built it as %s", where, r.ID, r.Module)
|
|
case r.Heard:
|
|
return fmt.Sprintf("%s; its last build, %s at %s, was recorded and not registered — `builds` says why",
|
|
where, r.ID, clock(r.HeardAt))
|
|
case r.NotAsked != "" && r.For == "merge":
|
|
return fmt.Sprintf("%s; the merge that added it (%s) could not ask for its build: %s", where,
|
|
short(r.Commit), firstLine(r.NotAsked))
|
|
case r.NotAsked != "":
|
|
return fmt.Sprintf("%s; build %s, asked by %s, was not handed over: %s", where, r.ID,
|
|
askerOr(r.For), firstLine(r.NotAsked))
|
|
case r.OutcomeUnknown != "":
|
|
return fmt.Sprintf("%s; build %s was asked at %s, its asker stopped waiting (%s), and no outcome has "+
|
|
"been heard in %s", where, r.ID, clock(r.At), firstLine(r.OutcomeUnknown), now.Sub(r.At).Round(time.Minute))
|
|
default:
|
|
return fmt.Sprintf("%s; build %s was asked at %s, and no outcome has been heard in %s", where, r.ID,
|
|
clock(r.At), now.Sub(r.At).Round(time.Minute))
|
|
}
|
|
}
|
|
|
|
func askerOr(who string) string {
|
|
if who == "" {
|
|
return "the controller"
|
|
}
|
|
return who
|
|
}
|
|
|
|
// requestSource is a build request's source as a person reads it: repository@commit (directory).
|
|
func requestSource(r inventory.RequestOutcome) string {
|
|
at := short(r.Commit)
|
|
if at == "" {
|
|
at = r.Ref
|
|
}
|
|
if at == "" {
|
|
at = "its default branch"
|
|
}
|
|
return fmt.Sprintf("%s@%s (%s)", r.Repository, at, orRoot(r.Path))
|
|
}
|
|
|
|
// clock is a moment as a person reads it, in the controller's local time.
|
|
func clock(t time.Time) string { return t.Local().Format("2006-01-02 15:04:05 MST") }
|
|
|
|
// closestNames is up to three names near the one asked: registered modules and directories the controller
|
|
// asked to build.
|
|
func closestNames(ctx context.Context, inv *inventory.Inventory, name string) []string {
|
|
var candidates []string
|
|
if shelf, err := inv.Catalogue(ctx); err == nil {
|
|
for m := range shelf {
|
|
candidates = append(candidates, m)
|
|
}
|
|
}
|
|
if asked, err := inv.RequestedNames(ctx); err == nil {
|
|
candidates = append(candidates, asked...)
|
|
}
|
|
type scored struct {
|
|
name string
|
|
distance int
|
|
}
|
|
seen := map[string]bool{}
|
|
var near []scored
|
|
limit := max(2, len(name)/3)
|
|
for _, c := range candidates {
|
|
if seen[c] || c == name {
|
|
continue
|
|
}
|
|
seen[c] = true
|
|
d := editDistance(name, c)
|
|
if d <= limit || strings.Contains(c, name) || strings.Contains(name, c) {
|
|
near = append(near, scored{c, d})
|
|
}
|
|
}
|
|
sort.Slice(near, func(i, j int) bool {
|
|
if near[i].distance != near[j].distance {
|
|
return near[i].distance < near[j].distance
|
|
}
|
|
return near[i].name < near[j].name
|
|
})
|
|
var out []string
|
|
for i := 0; i < len(near) && i < 3; i++ {
|
|
out = append(out, near[i].name)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// editDistance is the Levenshtein distance between two names.
|
|
func editDistance(a, b string) int {
|
|
ra, rb := []rune(a), []rune(b)
|
|
prev := make([]int, len(rb)+1)
|
|
for j := range prev {
|
|
prev[j] = j
|
|
}
|
|
for i := 1; i <= len(ra); i++ {
|
|
cur := make([]int, len(rb)+1)
|
|
cur[0] = i
|
|
for j := 1; j <= len(rb); j++ {
|
|
cost := 1
|
|
if ra[i-1] == rb[j-1] {
|
|
cost = 0
|
|
}
|
|
cur[j] = min(prev[j]+1, cur[j-1]+1, prev[j-1]+cost)
|
|
}
|
|
prev = cur
|
|
}
|
|
return prev[len(rb)]
|
|
}
|
|
|
|
// applyPending makes a pending assignment now that its module is registered (ADR 0261). Under the machine's
|
|
// hold it claims the row (waiting → applying), so a withdrawal cannot land between the look and the act,
|
|
// then assigns by the same act a person's assign is, then says what came of it: applied, or refused with the
|
|
// refusal. Returns what it said, or nothing when the row no longer waited.
|
|
func applyPending(ctx context.Context, open *stores, p inventory.PendingAssignment, by string) string {
|
|
inv := open.inventory
|
|
ctx, release, err := holdNodes(ctx, open, []string{p.Node})
|
|
if err != nil {
|
|
// Left waiting: the next tick tries again.
|
|
return fmt.Sprintf("the pending assignment of %s to %s waits: %s could not be held: %v", p.Module, p.Node,
|
|
p.Node, err)
|
|
}
|
|
defer release()
|
|
claimed, err := inv.ClaimPending(ctx, p.ID)
|
|
if err != nil {
|
|
return fmt.Sprintf("the pending assignment of %s to %s could not be taken for making: %v", p.Module, p.Node, err)
|
|
}
|
|
if !claimed {
|
|
return ""
|
|
}
|
|
answer, err := assign(ctx, open, p.Node, p.Module)
|
|
state, note := inventory.PendingApplied, ""
|
|
switch {
|
|
case err != nil:
|
|
state = inventory.PendingRefused
|
|
note = fmt.Sprintf("the pending assignment of %s to %s was refused when build %s registered it: %s",
|
|
p.Module, p.Node, by, firstLine(err.Error()))
|
|
case strings.Contains(answer, "already runs"):
|
|
note = fmt.Sprintf("%s already runs %s: the pending assignment is met", p.Node, p.Module)
|
|
default:
|
|
// The same as a person's assign: nothing is sent by this act.
|
|
note = fmt.Sprintf("%s is assigned %s: build %s registered it (pending since %s); `push %s` sends it",
|
|
p.Node, p.Module, by, clock(p.Since), p.Node)
|
|
}
|
|
settled, serr := inv.SettlePending(ctx, p.ID, inventory.PendingApplying, state, note)
|
|
if serr != nil {
|
|
return fmt.Sprintf("%s — and it could not be recorded as settled: %v", note, serr)
|
|
}
|
|
if !settled {
|
|
return ""
|
|
}
|
|
return note
|
|
}
|
|
|
|
// expirePending ends a waiting pending assignment whose build will not register its module, with why.
|
|
func expirePending(ctx context.Context, inv *inventory.Inventory, p inventory.PendingAssignment, why string) string {
|
|
note := fmt.Sprintf("the pending assignment of %s to %s expired: %s; nothing was assigned. Assign it again with "+
|
|
"build \"true\" to ask for the build again", p.Module, p.Node, why)
|
|
settled, err := inv.SettlePending(ctx, p.ID, inventory.PendingWaiting, inventory.PendingExpired, note)
|
|
if err != nil {
|
|
return fmt.Sprintf("%s — and it could not be recorded: %v", note, err)
|
|
}
|
|
if !settled {
|
|
return ""
|
|
}
|
|
return note
|
|
}
|
|
|
|
// settlePendingOnBuild is a build's outcome reaching the assignments waiting for it, wherever the outcome is
|
|
// taken in (novox/hq issue 325): a module registered makes every assignment pending on it; a build of a
|
|
// pending assignment that failed, or was not registered, ends it with why. Returns what was said.
|
|
func settlePendingOnBuild(ctx context.Context, open *stores, result link.BuildResult, module string, takeErr error) []string {
|
|
inv := open.inventory
|
|
waiting, err := inv.Pending(ctx, time.Time{})
|
|
if err != nil {
|
|
return []string{fmt.Sprintf("the pending assignments could not be read after build %s: %v", result.ID, err)}
|
|
}
|
|
registered := module != "" && (takeErr == nil || errors.Is(takeErr, inventory.ErrSuperseded))
|
|
var said []string
|
|
for _, p := range waiting {
|
|
switch {
|
|
case registered && p.Module == module:
|
|
if line := applyPending(ctx, open, p, result.ID); line != "" {
|
|
said = append(said, line)
|
|
}
|
|
case !registered && p.Build == result.ID:
|
|
why := firstLine(result.Failed)
|
|
if why == "" && takeErr != nil {
|
|
why = firstLine(takeErr.Error())
|
|
}
|
|
what := fmt.Sprintf("build %s of %s failed: %s", result.ID, orRoot(p.Path), why)
|
|
if result.Failed == "" {
|
|
what = fmt.Sprintf("build %s of %s was recorded and not registered: %s", result.ID, orRoot(p.Path), why)
|
|
}
|
|
if line := expirePending(ctx, inv, p, what); line != "" {
|
|
said = append(said, line)
|
|
}
|
|
}
|
|
}
|
|
return said
|
|
}
|
|
|
|
// settlingPending is the controller's tick over the pending assignments (ADR 0261), every settleEvery until
|
|
// the context ends. What it does is printed to the controller's log, kept in each row's note (which `status`
|
|
// lists for a day), and raised as a condition for an assignment that was not made.
|
|
func settlingPending(ctx context.Context, open *stores) {
|
|
for {
|
|
for _, line := range settlePending(ctx, open, time.Now()) {
|
|
fmt.Println(line)
|
|
}
|
|
for _, line := range raiseUnknownFields(ctx, open.inventory) {
|
|
fmt.Println(line)
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-time.After(settleEvery):
|
|
}
|
|
}
|
|
}
|
|
|
|
// settlePending is one tick: claims left by a controller that stopped are settled from what the mesh holds;
|
|
// a module registered meanwhile is assigned; a build heard and not registered, or silent past its bound,
|
|
// ends its assignment; ended ones are raised as conditions and cleared once answered; ended rows older than
|
|
// KeptFor are deleted. Returns what it did.
|
|
func settlePending(ctx context.Context, open *stores, now time.Time) []string {
|
|
inv := open.inventory
|
|
var said []string
|
|
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
|
|
|
|
shelf, err := inv.Catalogue(ctx)
|
|
if err != nil {
|
|
say("the pending assignments are not settled this time: the catalogue could not be read: %v", err)
|
|
return said
|
|
}
|
|
stuck, err := inv.Stuck(ctx, now.Add(-claimStaleAfter))
|
|
if err != nil {
|
|
say("claims left by a stopped controller could not be read: %v", err)
|
|
}
|
|
for _, p := range stuck {
|
|
assigned, err := inv.Assigned(ctx, p.Node)
|
|
if err != nil {
|
|
say("the claim on %s for %s could not be settled: %v", p.Node, p.Module, err)
|
|
continue
|
|
}
|
|
if containsString(assigned, p.Module) {
|
|
if _, err := inv.SettlePending(ctx, p.ID, inventory.PendingApplying, inventory.PendingApplied,
|
|
fmt.Sprintf("%s is assigned %s (settled after the controller that made it stopped)", p.Node, p.Module)); err != nil {
|
|
say("the claim on %s for %s could not be settled: %v", p.Node, p.Module, err)
|
|
}
|
|
continue
|
|
}
|
|
if err := inv.ReleaseClaim(ctx, p.ID); err != nil {
|
|
say("the claim on %s for %s could not be released: %v", p.Node, p.Module, err)
|
|
}
|
|
}
|
|
|
|
waiting, err := inv.Pending(ctx, time.Time{})
|
|
if err != nil {
|
|
say("the pending assignments could not be read to settle them: %v", err)
|
|
}
|
|
for _, p := range waiting {
|
|
if _, ok := shelf[p.Module]; ok {
|
|
if line := applyPending(ctx, open, p, p.Build); line != "" {
|
|
said = append(said, line)
|
|
}
|
|
continue
|
|
}
|
|
requests, err := inv.RequestsNamed(ctx, p.Module)
|
|
if err != nil {
|
|
say("the build requests for %s could not be read: %v", p.Module, err)
|
|
continue
|
|
}
|
|
var r *inventory.RequestOutcome
|
|
for i := range requests {
|
|
if requests[i].ID == p.Build {
|
|
r = &requests[i]
|
|
}
|
|
}
|
|
why := ""
|
|
switch {
|
|
case r == nil && now.Sub(p.Since) > buildRequestBound:
|
|
why = fmt.Sprintf("build %s is no longer on record", p.Build)
|
|
case r != nil && !stillComing(*r, now) && (r.Heard || r.NotAsked != "" || r.OutcomeUnknown != ""):
|
|
why = whyNotBuilt(*r, now)
|
|
case r != nil && !stillComing(*r, now):
|
|
why = fmt.Sprintf("no outcome of build %s was heard within %s of asking", p.Build, buildRequestBound)
|
|
}
|
|
if why == "" {
|
|
continue
|
|
}
|
|
if line := expirePending(ctx, inv, p, why); line != "" {
|
|
said = append(said, line)
|
|
}
|
|
}
|
|
|
|
said = append(said, raisePendingEnded(ctx, inv)...)
|
|
if n, err := inv.ForgetEndedPending(ctx, now); err != nil {
|
|
say("ended pending assignments could not be deleted: %v", err)
|
|
} else if n > 0 {
|
|
say("deleted %d pending assignment(s) ended more than %s ago", n, inventory.KeptFor)
|
|
}
|
|
return said
|
|
}
|
|
|
|
func containsString(xs []string, x string) bool {
|
|
for _, y := range xs {
|
|
if y == x {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// pendingObservation is the condition for a pending assignment that was not made: one per pending
|
|
// assignment, its row's id the last part of its id, so one row's condition is never another's.
|
|
func pendingObservation(p inventory.PendingAssignment) conditions.Observation {
|
|
return conditions.Observation{Scope: conditions.ScopeMachine, ID: fmt.Sprintf("%s.%s.%d", p.Node, p.Module, p.ID),
|
|
Token: kindPendingEnded, Kind: kindPendingEnded, Machine: p.Node, Severity: conditions.Warning,
|
|
Resolver: conditions.ResolverOperator, Source: "pending assignments",
|
|
Summary: p.Note}
|
|
}
|
|
|
|
// raisePendingEnded raises a condition for each pending assignment that expired or was refused, once, and
|
|
// clears it when it is answered: the module was assigned to that machine since, a newer pending assignment of
|
|
// it is open, or a person took it back with `unassign`. Where this process keeps no conditions, nothing is
|
|
// stamped, and the serving controller's tick raises it.
|
|
func raisePendingEnded(ctx context.Context, inv *inventory.Inventory) []string {
|
|
keeper := conditionsFrom
|
|
if keeper == nil {
|
|
return nil
|
|
}
|
|
var said []string
|
|
toRaise, err := inv.ToRaise(ctx)
|
|
if err != nil {
|
|
return []string{fmt.Sprintf("pending assignments not made could not be read to raise them: %v", err)}
|
|
}
|
|
for _, p := range toRaise {
|
|
if _, err := keeper.Observe(ctx, pendingObservation(p)); err != nil {
|
|
said = append(said, fmt.Sprintf("the condition for %s on %s could not be raised: %v", p.Module, p.Node, err))
|
|
continue
|
|
}
|
|
if err := inv.MarkPending(ctx, p.ID, "raised"); err != nil {
|
|
said = append(said, fmt.Sprintf("the condition for %s on %s was raised and not recorded: %v", p.Module, p.Node, err))
|
|
}
|
|
}
|
|
toClear, err := inv.ToClear(ctx)
|
|
if err != nil {
|
|
return append(said, fmt.Sprintf("pending assignments raised could not be read to clear them: %v", err))
|
|
}
|
|
for _, p := range toClear {
|
|
why, answered, err := pendingAnswered(ctx, inv, p)
|
|
if err != nil {
|
|
said = append(said, fmt.Sprintf("whether %s on %s is answered could not be read: %v", p.Module, p.Node, err))
|
|
continue
|
|
}
|
|
if !answered {
|
|
continue
|
|
}
|
|
if _, err := keeper.Clear(ctx, pendingObservation(p).Key(), why); err != nil {
|
|
said = append(said, fmt.Sprintf("the condition for %s on %s could not be cleared: %v", p.Module, p.Node, err))
|
|
continue
|
|
}
|
|
if err := inv.MarkPending(ctx, p.ID, "cleared"); err != nil {
|
|
said = append(said, fmt.Sprintf("the condition for %s on %s was cleared and not recorded: %v", p.Module, p.Node, err))
|
|
}
|
|
}
|
|
return append(said, clearOrphaned(ctx, keeper, inv)...)
|
|
}
|
|
|
|
// pendingAnswered says whether a pending assignment that was not made has been answered since, and how.
|
|
func pendingAnswered(ctx context.Context, inv *inventory.Inventory, p inventory.PendingAssignment) (string, bool, error) {
|
|
if p.Acknowledged != nil {
|
|
return "taken back with unassign", true, nil
|
|
}
|
|
assigned, err := inv.Assigned(ctx, p.Node)
|
|
if err != nil {
|
|
return "", false, err
|
|
}
|
|
if containsString(assigned, p.Module) {
|
|
return p.Module + " is assigned to " + p.Node + " since", true, nil
|
|
}
|
|
rows, err := inv.PendingFor(ctx, p.Node, p.Module)
|
|
if err != nil {
|
|
return "", false, err
|
|
}
|
|
// A newer pending assignment answers it only while it is open or once it was made: one that itself
|
|
// ended unmade is its own condition, and answers nothing.
|
|
for _, r := range rows {
|
|
if r.ID != p.ID && r.Since.After(p.Since) && (r.Open() || r.State == inventory.PendingApplied) {
|
|
return "assigned again, pending on another build", true, nil
|
|
}
|
|
}
|
|
return "", false, nil
|
|
}
|
|
|
|
// clearOrphaned clears every open condition of an assignment not made whose pending assignment is no longer
|
|
// on record: its machine was removed, which takes its pending assignments with it.
|
|
func clearOrphaned(ctx context.Context, keeper *conditions.Keeper, inv *inventory.Inventory) []string {
|
|
open, err := keeper.Open(ctx)
|
|
if err != nil {
|
|
return []string{fmt.Sprintf("the open conditions could not be read to clear orphaned ones: %v", err)}
|
|
}
|
|
byID := map[int64]string{}
|
|
var ids []int64
|
|
for _, c := range open {
|
|
if c.Kind != kindPendingEnded {
|
|
continue
|
|
}
|
|
// `<scope>.<node>.<module>.<row>.<kind>`: the row is the part before the kind.
|
|
parts := strings.Split(c.Key, ".")
|
|
if len(parts) < 2 {
|
|
continue
|
|
}
|
|
var id int64
|
|
if _, err := fmt.Sscan(parts[len(parts)-2], &id); err != nil {
|
|
continue
|
|
}
|
|
byID[id] = c.Key
|
|
ids = append(ids, id)
|
|
}
|
|
if len(ids) == 0 {
|
|
return nil
|
|
}
|
|
known, err := inv.PendingKnown(ctx, ids)
|
|
if err != nil {
|
|
return []string{fmt.Sprintf("the pending assignments could not be read to clear orphaned conditions: %v", err)}
|
|
}
|
|
var said []string
|
|
for id, key := range byID {
|
|
if known[id] {
|
|
continue
|
|
}
|
|
if _, err := keeper.Clear(ctx, key, "its pending assignment is no longer on record: its machine was removed"); err != nil {
|
|
said = append(said, fmt.Sprintf("the condition %s could not be cleared: %v", key, err))
|
|
}
|
|
}
|
|
return said
|
|
}
|
|
|
|
// withdrawPending is `unassign` of a module only pending on a machine, under the machine's hold: a waiting
|
|
// pending assignment is withdrawn; one being made now is refused, never overridden; one that expired or was
|
|
// refused is taken back, which answers its condition. False when there is none.
|
|
func withdrawPending(ctx context.Context, inv *inventory.Inventory, node, module string) (string, bool, error) {
|
|
rows, err := inv.PendingFor(ctx, node, module)
|
|
if err != nil {
|
|
return "", false, err
|
|
}
|
|
var takenBack []string
|
|
for _, p := range rows {
|
|
switch p.State {
|
|
case inventory.PendingApplying:
|
|
return "", true, fmt.Errorf("%s is being assigned %s now, as build %s registered it: nothing was "+
|
|
"withdrawn; `unassign %s %s` again once it is assigned takes it off", node, module, p.Build, node, module)
|
|
case inventory.PendingWaiting:
|
|
note := fmt.Sprintf("the pending assignment of %s to %s is withdrawn; build %s goes on, and registers "+
|
|
"%s assigned nowhere", module, node, p.Build, module)
|
|
settled, err := inv.SettlePending(ctx, p.ID, inventory.PendingWaiting, inventory.PendingWithdrawn, note)
|
|
if err != nil {
|
|
return "", false, err
|
|
}
|
|
if !settled {
|
|
return "", true, fmt.Errorf("the pending assignment of %s to %s changed while it was withdrawn: "+
|
|
"`status` says how; nothing was withdrawn", module, node)
|
|
}
|
|
return note, true, nil
|
|
case inventory.PendingExpired, inventory.PendingRefused:
|
|
if p.Cleared != nil || p.Acknowledged != nil {
|
|
continue
|
|
}
|
|
if err := inv.MarkPending(ctx, p.ID, "acknowledged"); err != nil {
|
|
return "", false, err
|
|
}
|
|
takenBack = append(takenBack, fmt.Sprintf("build %s (%s)", p.Build, p.State))
|
|
}
|
|
}
|
|
if len(takenBack) > 0 {
|
|
return fmt.Sprintf("the pending assignment(s) of %s to %s that were not made are taken back: %s; their "+
|
|
"conditions clear", module, node, strings.Join(takenBack, ", ")), true, nil
|
|
}
|
|
return "", false, nil
|
|
}
|