Archived
jochen
596148d8e6
A merge asks for a new module's build, and assign answered "no module of that name" until the build registered it, which read as a module nobody registered (hq issue 325). Keep every build request, tell a build in flight, a module known and not built, and an unknown name apart, and make an assignment made while the build runs when the build registers the module.
443 lines
17 KiB
Go
443 lines
17 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// A module not registered yet, and the assignment that waits for it (novox/hq issue 325).
|
|
//
|
|
// On 2026-10-08 the catalogue's pull request adding `sensors` merged; a minute later `assign g14 sensors`
|
|
// answered "no module of that name: sensors", and a few minutes later the same call worked. The merge had
|
|
// asked for the build (issue 300) and the build had not finished. The answer read as "you forgot to register
|
|
// it", and nothing in it said a build was on its way.
|
|
//
|
|
// So an assignment of a module the catalogue does not hold looks further before it refuses, and says which
|
|
// of three cases it is in:
|
|
//
|
|
// - **a build is in flight**: a build request for the module's directory has no outcome yet. The
|
|
// assignment is kept as a **pending assignment** and made when the build registers the module.
|
|
// - **known, not built**: the mesh asked for the directory before, and the build failed, was not
|
|
// registered, said nothing within its bound, or could not be asked. Said, with `build` and assign's
|
|
// own build argument, which asks for the build and keeps the assignment pending on it.
|
|
// - **unknown**: no module, build request or merge by that name. Refused as before, with the closest names.
|
|
//
|
|
// **Pending by default while a build is in flight**, without an argument to ask for it: an assignment is
|
|
// what a person meant and is kept even when its machine does not resolve (acts.go), nothing is sent until a
|
|
// push, and `unassign` withdraws it. Asking for a second act once the build lands is exactly the two acts by
|
|
// hand issue 300 removed. A build is never asked for by default: it runs on another machine, and a directory
|
|
// whose last build failed is a fault to look at before it is a thing to retry.
|
|
|
|
// buildRequestBound is how long a build request may go without an outcome before it is no longer read as in
|
|
// flight, and a pending assignment waiting for it expires. Generous: a build waits behind others on the
|
|
// build seat, and a pending assignment that waits a little longer costs nothing.
|
|
const buildRequestBound = 2 * time.Hour
|
|
|
|
// pendingShownFor is how long an ended pending assignment is said in `status`.
|
|
const pendingShownFor = 24 * time.Hour
|
|
|
|
// assignOptions are what an assignment was asked with beside its machine and modules.
|
|
type assignOptions struct {
|
|
// Build asks for the build of a module known and not built, and keeps the assignment pending on it.
|
|
Build bool
|
|
}
|
|
|
|
// recordBuildRequest keeps a build request so `assign` can find it. Said, never fatal: the build was asked.
|
|
func recordBuildRequest(ctx context.Context, inv *inventory.Inventory, r inventory.BuildRequest) {
|
|
if err := inv.RecordBuildRequest(ctx, r); err != nil {
|
|
fmt.Fprintf(os.Stderr, "build %s was asked, and could not be kept as a build request, so `assign` "+
|
|
"will not know it is coming: %v\n", r.ID, err)
|
|
}
|
|
}
|
|
|
|
// The cases of a module the catalogue does not hold (novox/hq issue 325).
|
|
const (
|
|
unknownModule = iota
|
|
buildInFlight
|
|
knownNotBuilt
|
|
)
|
|
|
|
// notHeld is where a module the catalogue does not hold stands: its case, the build request it was read
|
|
// from, and the sentence that says it.
|
|
type notHeld struct {
|
|
kind int
|
|
request inventory.RequestOutcome
|
|
said string
|
|
}
|
|
|
|
// whereIs looks for a module the catalogue does not hold among the build requests the mesh keeps.
|
|
func whereIs(ctx context.Context, inv *inventory.Inventory, module string, now time.Time) (notHeld, error) {
|
|
requests, err := inv.RequestsNamed(ctx, module)
|
|
if err != nil {
|
|
return notHeld{}, err
|
|
}
|
|
if r, ok := inFlight(requests, now); ok {
|
|
return notHeld{kind: buildInFlight, request: r, said: fmt.Sprintf("%s is not registered yet: it is being "+
|
|
"built from %s since %s, as build %s; it can be assigned once that build registers it", module,
|
|
requestSource(r), clock(r.At), r.ID)}, nil
|
|
}
|
|
if len(requests) > 0 {
|
|
return notHeld{kind: knownNotBuilt, request: requests[0], said: fmt.Sprintf("%s is known and not "+
|
|
"registered: %s", module, whyNotBuilt(requests[0], now))}, nil
|
|
}
|
|
said := fmt.Sprintf("%v: %s — the mesh holds no module, build request or merge by that name",
|
|
inventory.ErrNoSuchModule, module)
|
|
if near := closestNames(ctx, inv, module); len(near) > 0 {
|
|
said += "; the closest it holds: " + strings.Join(near, ", ")
|
|
}
|
|
return notHeld{kind: unknownModule, said: said}, nil
|
|
}
|
|
|
|
// notInCatalogue is the modules named that the catalogue does not hold.
|
|
func notInCatalogue(ctx context.Context, open *stores, modules []string) ([]string, error) {
|
|
shelf, err := open.inventory.Catalogue(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var missing []string
|
|
for _, m := range modules {
|
|
if _, ok := shelf[m]; !ok {
|
|
missing = append(missing, m)
|
|
}
|
|
}
|
|
return missing, nil
|
|
}
|
|
|
|
// notRegistered answers an assignment of a module the catalogue does not hold: pending on a build in flight,
|
|
// known and not built, or unknown (novox/hq issue 325). An answer with no error is a pending assignment kept.
|
|
func notRegistered(ctx context.Context, open *stores, node, module string, opts assignOptions) (string, error) {
|
|
inv := open.inventory
|
|
if _, err := inv.NodeByName(ctx, node); err != nil {
|
|
return "", err
|
|
}
|
|
now := time.Now()
|
|
where, err := whereIs(ctx, inv, module, now)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
switch where.kind {
|
|
case buildInFlight:
|
|
lead := where.said
|
|
if opts.Build {
|
|
lead += "\n no second build was asked: this one is running"
|
|
}
|
|
return keepPending(ctx, open, node, module, where.request, lead)
|
|
case knownNotBuilt:
|
|
last := where.request
|
|
if !opts.Build {
|
|
return "", fmt.Errorf("%w: %s\n assign it again with build \"true\" to ask for its build and keep this "+
|
|
"assignment until the build registers it; or `build` it (repository %s, path %s) and assign it "+
|
|
"once it is registered", inventory.ErrNoSuchModule, where.said, last.Repository, orRoot(last.Path))
|
|
}
|
|
source := buildSource{Repository: last.Repository, Seat: last.Seat}
|
|
id, err := askABuild(ctx, source, last.Path, last.Ref)
|
|
if err != nil {
|
|
return "", fmt.Errorf("%s\n its build could not be asked for: %w; nothing was assigned", where.said, err)
|
|
}
|
|
asked := inventory.RequestOutcome{BuildRequest: inventory.BuildRequest{ID: id, Repository: last.Repository,
|
|
Seat: last.Seat, Path: last.Path, Ref: last.Ref, For: "assign", At: now.UTC()}}
|
|
recordBuildRequest(ctx, inv, asked.BuildRequest)
|
|
lead := fmt.Sprintf("%s\n build %s of %s is asked for now; %s can be assigned once it registers it",
|
|
where.said, id, requestSource(asked), module)
|
|
return keepPending(ctx, open, node, module, asked, lead)
|
|
}
|
|
answer := strings.TrimPrefix(where.said, inventory.ErrNoSuchModule.Error()) +
|
|
"\n a module in a repository is built with `build` (its repository and path), then assigned"
|
|
if opts.Build {
|
|
answer += "\n build \"true\" asks for nothing here: the mesh does not know where a module of that name is"
|
|
}
|
|
return "", fmt.Errorf("%w%s", inventory.ErrNoSuchModule, answer)
|
|
}
|
|
|
|
// keepPending records the assignment as pending on a build request, and says so.
|
|
func keepPending(ctx context.Context, open *stores, node, module string, r inventory.RequestOutcome, lead string) (string, error) {
|
|
inv := open.inventory
|
|
p, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: node, Module: module, Build: r.ID,
|
|
Repository: r.Repository, Path: r.Path})
|
|
if errors.Is(err, inventory.ErrAlreadyPending) {
|
|
return lead + fmt.Sprintf("\n %s already waits for %s: nothing changed", node, module), nil
|
|
}
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
answer := lead + fmt.Sprintf("\n the assignment is kept as pending: %s is assigned %s when the build "+
|
|
"registers it, and `status` says it until then. If the build fails it expires and says why; "+
|
|
"`unassign %s %s` withdraws it", node, module, node, module)
|
|
// The build may have registered the module between the look and the record: made at once, then.
|
|
if shelf, err := inv.Catalogue(ctx); err == nil {
|
|
if _, now := shelf[module]; now {
|
|
if line := applyPending(ctx, open, p, r.ID); line != "" {
|
|
answer += "\n " + line
|
|
}
|
|
}
|
|
}
|
|
return answer, nil
|
|
}
|
|
|
|
// inFlight is the newest build request for the module with no outcome yet, asked within the bound.
|
|
func inFlight(requests []inventory.RequestOutcome, now time.Time) (inventory.RequestOutcome, bool) {
|
|
for _, r := range requests {
|
|
if r.NotAsked == "" && !r.Heard && now.Sub(r.At) < buildRequestBound {
|
|
return r, true
|
|
}
|
|
}
|
|
return inventory.RequestOutcome{}, false
|
|
}
|
|
|
|
// whyNotBuilt says what came of a module's last build request.
|
|
func whyNotBuilt(r inventory.RequestOutcome, now time.Time) string {
|
|
where := fmt.Sprintf("%s in %s", orRoot(r.Path), r.Repository)
|
|
switch {
|
|
case r.NotAsked != "":
|
|
return fmt.Sprintf("%s; the merge that added it (%s) could not ask for its build: %s", where,
|
|
short(r.Commit), firstLine(r.NotAsked))
|
|
case !r.Heard:
|
|
return fmt.Sprintf("%s; build %s was asked at %s, and no outcome has been heard in %s", where, r.ID,
|
|
clock(r.At), now.Sub(r.At).Round(time.Minute))
|
|
case r.Failed != "":
|
|
return fmt.Sprintf("%s; its last build, %s at %s, failed: %s", where, r.ID, clock(r.HeardAt),
|
|
firstLine(r.Failed))
|
|
case r.Module != "" && r.Module != r.Name():
|
|
return fmt.Sprintf("%s; its last build, %s, registered it as %s", where, r.ID, r.Module)
|
|
default:
|
|
return fmt.Sprintf("%s; its last build, %s at %s, was recorded and not registered — `builds` says why",
|
|
where, r.ID, clock(r.HeardAt))
|
|
}
|
|
}
|
|
|
|
// requestSource is a build request's source as a person reads it: repository@commit (directory).
|
|
func requestSource(r inventory.RequestOutcome) string {
|
|
at := short(r.Commit)
|
|
if at == "" {
|
|
at = r.Ref
|
|
}
|
|
if at == "" {
|
|
at = "its default branch"
|
|
}
|
|
return fmt.Sprintf("%s@%s (%s)", r.Repository, at, orRoot(r.Path))
|
|
}
|
|
|
|
// clock is a moment as a person reads it, in the controller's local time.
|
|
func clock(t time.Time) string { return t.Local().Format("2006-01-02 15:04:05 MST") }
|
|
|
|
// closestNames is up to three names the mesh holds that are near the one asked: registered modules and
|
|
// directories it asked to build.
|
|
func closestNames(ctx context.Context, inv *inventory.Inventory, name string) []string {
|
|
var candidates []string
|
|
if shelf, err := inv.Catalogue(ctx); err == nil {
|
|
for m := range shelf {
|
|
candidates = append(candidates, m)
|
|
}
|
|
}
|
|
if asked, err := inv.RequestedNames(ctx); err == nil {
|
|
candidates = append(candidates, asked...)
|
|
}
|
|
type scored struct {
|
|
name string
|
|
distance int
|
|
}
|
|
seen := map[string]bool{}
|
|
var near []scored
|
|
limit := max(2, len(name)/3)
|
|
for _, c := range candidates {
|
|
if seen[c] || c == name {
|
|
continue
|
|
}
|
|
seen[c] = true
|
|
d := editDistance(name, c)
|
|
if d <= limit || strings.Contains(c, name) || strings.Contains(name, c) {
|
|
near = append(near, scored{c, d})
|
|
}
|
|
}
|
|
sort.Slice(near, func(i, j int) bool {
|
|
if near[i].distance != near[j].distance {
|
|
return near[i].distance < near[j].distance
|
|
}
|
|
return near[i].name < near[j].name
|
|
})
|
|
var out []string
|
|
for i := 0; i < len(near) && i < 3; i++ {
|
|
out = append(out, near[i].name)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// editDistance is the Levenshtein distance between two names.
|
|
func editDistance(a, b string) int {
|
|
ra, rb := []rune(a), []rune(b)
|
|
prev := make([]int, len(rb)+1)
|
|
for j := range prev {
|
|
prev[j] = j
|
|
}
|
|
for i := 1; i <= len(ra); i++ {
|
|
cur := make([]int, len(rb)+1)
|
|
cur[0] = i
|
|
for j := 1; j <= len(rb); j++ {
|
|
cost := 1
|
|
if ra[i-1] == rb[j-1] {
|
|
cost = 0
|
|
}
|
|
cur[j] = min(prev[j]+1, cur[j-1]+1, prev[j-1]+cost)
|
|
}
|
|
prev = cur
|
|
}
|
|
return prev[len(rb)]
|
|
}
|
|
|
|
// applyPending makes a pending assignment now that its module is registered, and settles it: applied, or
|
|
// refused with the refusal. Returns what it said, or nothing when another process settled it first.
|
|
func applyPending(ctx context.Context, open *stores, p inventory.PendingAssignment, by string) string {
|
|
inv := open.inventory
|
|
answer, err := assign(ctx, open, p.Node, p.Module)
|
|
state, note := inventory.PendingApplied, ""
|
|
if err != nil {
|
|
state = inventory.PendingRefused
|
|
note = fmt.Sprintf("the pending assignment of %s to %s was refused when build %s registered it: %s",
|
|
p.Module, p.Node, by, firstLine(err.Error()))
|
|
} else {
|
|
// Not promised as sent: a walk sends it if the module's upgrade announcement finds it
|
|
// assigned, and nothing does if the announcement came first. `status` says which, as for any machine.
|
|
note = fmt.Sprintf("%s is assigned %s: build %s registered it (pending since %s); `status` says whether "+
|
|
"%s has been sent it, and `push %s` sends it if not", p.Node, p.Module, by, clock(p.Since), p.Node, p.Node)
|
|
if strings.Contains(answer, "already runs") {
|
|
note = fmt.Sprintf("%s already runs %s: the pending assignment is met", p.Node, p.Module)
|
|
}
|
|
}
|
|
settled, serr := inv.SettlePending(ctx, p.ID, state, note)
|
|
if serr != nil {
|
|
return fmt.Sprintf("%s — and it could not be recorded as settled: %v", note, serr)
|
|
}
|
|
if !settled {
|
|
return ""
|
|
}
|
|
return note
|
|
}
|
|
|
|
// expirePending ends a pending assignment whose build will not register its module, with why.
|
|
func expirePending(ctx context.Context, inv *inventory.Inventory, p inventory.PendingAssignment, why string) string {
|
|
note := fmt.Sprintf("the pending assignment of %s to %s expired: %s; nothing was assigned. Assign it again with "+
|
|
"build \"true\" to ask for the build again", p.Module, p.Node, why)
|
|
settled, err := inv.SettlePending(ctx, p.ID, inventory.PendingExpired, note)
|
|
if err != nil {
|
|
return fmt.Sprintf("%s — and it could not be recorded: %v", note, err)
|
|
}
|
|
if !settled {
|
|
return ""
|
|
}
|
|
return note
|
|
}
|
|
|
|
// settlePendingOnBuild is a build's outcome reaching the assignments waiting for it, wherever the outcome is
|
|
// taken in (novox/hq issue 325): a module registered makes every assignment pending on it; a build of a
|
|
// pending assignment that failed, or was not registered, ends it with why. Returns what was said.
|
|
func settlePendingOnBuild(ctx context.Context, open *stores, result link.BuildResult, module string, takeErr error) []string {
|
|
inv := open.inventory
|
|
waiting, err := inv.Pending(ctx, time.Time{})
|
|
if err != nil {
|
|
return []string{fmt.Sprintf("the pending assignments could not be read after build %s: %v", result.ID, err)}
|
|
}
|
|
registered := module != "" && (takeErr == nil || errors.Is(takeErr, inventory.ErrSuperseded))
|
|
var said []string
|
|
for _, p := range waiting {
|
|
switch {
|
|
case registered && p.Module == module:
|
|
if line := applyPending(ctx, open, p, result.ID); line != "" {
|
|
said = append(said, line)
|
|
}
|
|
case !registered && p.Build == result.ID:
|
|
why := firstLine(result.Failed)
|
|
if why == "" && takeErr != nil {
|
|
why = firstLine(takeErr.Error())
|
|
}
|
|
what := fmt.Sprintf("build %s of %s failed: %s", result.ID, orRoot(p.Path), why)
|
|
if result.Failed == "" {
|
|
what = fmt.Sprintf("build %s of %s was recorded and not registered: %s", result.ID, orRoot(p.Path), why)
|
|
}
|
|
if line := expirePending(ctx, inv, p, what); line != "" {
|
|
said = append(said, line)
|
|
}
|
|
}
|
|
}
|
|
return said
|
|
}
|
|
|
|
// settleStalePending settles what a missed outcome left waiting: a module registered meanwhile is assigned,
|
|
// a build heard and not registered ends its assignment, and one with no outcome past the bound expires.
|
|
// Asked where the mesh is read for status, so a pending assignment never waits silently for ever.
|
|
func settleStalePending(ctx context.Context, open *stores, now time.Time) []string {
|
|
inv := open.inventory
|
|
waiting, err := inv.Pending(ctx, time.Time{})
|
|
if err != nil {
|
|
// Said, not swallowed; status reads the pending assignments again and says the same failure there.
|
|
return []string{fmt.Sprintf("the pending assignments could not be read to settle them: %v", err)}
|
|
}
|
|
shelf, err := inv.Catalogue(ctx)
|
|
if err != nil {
|
|
return []string{fmt.Sprintf("the catalogue could not be read to settle the pending assignments: %v", err)}
|
|
}
|
|
var said []string
|
|
for _, p := range waiting {
|
|
if _, ok := shelf[p.Module]; ok {
|
|
if line := applyPending(ctx, open, p, p.Build); line != "" {
|
|
said = append(said, line)
|
|
}
|
|
continue
|
|
}
|
|
requests, err := inv.RequestsNamed(ctx, p.Module)
|
|
if err != nil {
|
|
said = append(said, fmt.Sprintf("the build requests for %s could not be read: %v", p.Module, err))
|
|
continue
|
|
}
|
|
var r *inventory.RequestOutcome
|
|
for i := range requests {
|
|
if requests[i].ID == p.Build {
|
|
r = &requests[i]
|
|
}
|
|
}
|
|
why := ""
|
|
switch {
|
|
case r == nil && now.Sub(p.Since) > buildRequestBound:
|
|
why = fmt.Sprintf("build %s is no longer on record", p.Build)
|
|
case r != nil && r.Heard:
|
|
why = whyNotBuilt(*r, now)
|
|
case r != nil && now.Sub(r.At) > buildRequestBound:
|
|
why = fmt.Sprintf("no outcome of build %s was heard within %s of asking", p.Build, buildRequestBound)
|
|
}
|
|
if why == "" {
|
|
continue
|
|
}
|
|
if line := expirePending(ctx, inv, p, why); line != "" {
|
|
said = append(said, line)
|
|
}
|
|
}
|
|
return said
|
|
}
|
|
|
|
// withdrawPending is `unassign` of a module only pending on a machine: the pending assignment withdrawn.
|
|
func withdrawPending(ctx context.Context, inv *inventory.Inventory, node, module string) (string, bool, error) {
|
|
waiting, err := inv.Pending(ctx, time.Time{})
|
|
if err != nil {
|
|
return "", false, err
|
|
}
|
|
for _, p := range waiting {
|
|
if p.Node != node || p.Module != module {
|
|
continue
|
|
}
|
|
note := fmt.Sprintf("the pending assignment of %s to %s is withdrawn; build %s goes on, and registers %s "+
|
|
"assigned nowhere", module, node, p.Build, module)
|
|
if _, err := inv.SettlePending(ctx, p.ID, inventory.PendingWithdrawn, note); err != nil {
|
|
return "", false, err
|
|
}
|
|
return note, true, nil
|
|
}
|
|
return "", false, nil
|
|
}
|