From 7d63d2e68c263323fb06604d1803166a4385b543 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 8 Oct 2026 15:48:36 +0200 Subject: [PATCH] Say why assign finds no module, and keep an assignment pending on its build A merge asks for a new module's build, and assign answered "no module of that name" until the build registered it, which read as a module nobody registered (hq issue 325). Keep every build request, tell a build in flight, a module known and not built, and an unknown name apart, and make an assignment made while the build runs when the build registers the module. --- cmd/mesh-controller/acts.go | 41 ++ cmd/mesh-controller/build.go | 20 + cmd/mesh-controller/main.go | 9 + cmd/mesh-controller/modules.go | 15 +- cmd/mesh-controller/pending.go | 442 ++++++++++++++++++ cmd/mesh-controller/pending_test.go | 329 +++++++++++++ cmd/mesh-controller/readable.go | 23 + cmd/mesh-controller/release_plan.go | 11 +- cmd/mesh-controller/replay325_test.go | 60 +++ cmd/mesh-controller/seatverbs.go | 8 +- cmd/mesh-controller/status.go | 60 ++- cmd/mesh-controller/upgrades.go | 13 +- internal/catalogue/verbs.go | 11 +- ...0082-an-assignment-waits-for-its-build.sql | 42 ++ internal/inventory/pending.go | 228 +++++++++ 15 files changed, 1303 insertions(+), 9 deletions(-) create mode 100644 cmd/mesh-controller/pending.go create mode 100644 cmd/mesh-controller/pending_test.go create mode 100644 cmd/mesh-controller/replay325_test.go create mode 100644 internal/inventory/migrations/0082-an-assignment-waits-for-its-build.sql create mode 100644 internal/inventory/pending.go diff --git a/cmd/mesh-controller/acts.go b/cmd/mesh-controller/acts.go index 5ad5f702..6b1aa6da 100644 --- a/cmd/mesh-controller/acts.go +++ b/cmd/mesh-controller/acts.go @@ -7,8 +7,10 @@ import ( "slices" "sort" "strings" + "time" "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" ) // The things the mesh can be asked to do, separated from how it was asked. @@ -41,6 +43,11 @@ import ( // It costs a resolution per machine. Assignment is a person typing a command, and being told which // machines this just blocked is worth more than the milliseconds. func assign(ctx context.Context, open *stores, node string, modules ...string) (string, error) { + return assignWith(ctx, open, node, assignOptions{}, modules...) +} + +// assignWith is assign asked with its options: build, for a module known and not built (novox/hq issue 325). +func assignWith(ctx context.Context, open *stores, node string, opts assignOptions, modules ...string) (string, error) { if len(modules) == 0 { return "", fmt.Errorf("assign %s names no module", node) } @@ -51,6 +58,29 @@ func assign(ctx context.Context, open *stores, node string, modules ...string) ( return "", err } defer release() + // **A module the catalogue does not hold is looked for further before it is refused** (novox/hq issue + // 325): a build in flight keeps the assignment pending, a module known and not built is said with how to + // build it, and only a name the mesh never heard of is refused as unknown. Several modules in one act + // are judged together (ADR 0207), so an act naming one not registered is not made in part. + if missing, err := notInCatalogue(ctx, open, modules); err != nil { + return "", err + } else if len(missing) > 0 { + if len(modules) == 1 { + return notRegistered(ctx, open, node, modules[0], opts) + } + var lines []string + for _, m := range missing { + where, err := whereIs(ctx, open.inventory, m, time.Now()) + if err != nil { + return "", err + } + lines = append(lines, where.said) + } + return "", fmt.Errorf("%w: nothing was assigned, since modules assigned together are judged together "+ + "(ADR 0207) and %s not registered:\n %s\n assign them together once each is registered, or each "+ + "alone to keep it pending on its build", inventory.ErrNoSuchModule, strings.Join(missing, ", "), + strings.Join(lines, "\n ")) + } // **The one assignment refused for what the node lacks** (novox/hq ADR 0207). Everything else // an assignment leaves unresolved is kept, because assignment is not an ordering; a module whose // resources are applied through a seat nothing on the node holds is refused, because that order @@ -200,6 +230,17 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string) for _, a := range assigned { runs[a] = true } + // A module only pending on the node (novox/hq issue 325) is withdrawn, when it is the act's one module: + // that is the undo of an assignment kept while its build runs. + if len(modules) == 1 && !runs[modules[0]] { + said, withdrawn, err := withdrawPending(ctx, open.inventory, node, modules[0]) + if err != nil { + return "", err + } + if withdrawn { + return said, nil + } + } for _, module := range modules { if !runs[module] { return "", fmt.Errorf("%s is not assigned to %s", module, node) diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index 270eabcc..56677104 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -462,6 +462,17 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa } defer ask.Close() fmt.Printf(" of %s\n", seat) + // Kept before it is asked, so `assign` knows a module is coming while its build runs (novox/hq issue 325). + // A dry run registers nothing, and is not kept. + if !dryRun { + if open, err := openStores(ctx); err == nil { + recordBuildRequest(ctx, open.inventory, inventory.BuildRequest{ID: request.ID, + Repository: source.Repository, Seat: source.Seat, Path: path, Ref: ref, For: "build"}) + open.Close() + } else { + fmt.Fprintf(os.Stderr, "build %s is asked and not kept as a build request: %v\n", request.ID, err) + } + } if wait == 0 { // Asked and not waited for (novox/hq issue 176): the outcome is the role's event, and the @@ -492,6 +503,10 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa } defer open.Close() manifest, kept, err := takeIn(ctx, open.inventory, result) + // The assignments pending on this build, made or ended (novox/hq issue 325). + for _, line := range settlePendingOnBuild(ctx, open, result, manifest.Module, err) { + fmt.Printf(" %s\n", line) + } if err != nil { return request.ID, err } @@ -771,6 +786,11 @@ type answers struct { // handed to the operator; healsUnread why it could not be read. heals *healsCount healsUnread string + // pending is every assignment waiting for its module's build, and every one ended in the last day + // (novox/hq issue 325); pendingUnread why they could not be read. One waiting, or one ended other than + // applied, is not well: an assignment somebody made is not made yet, or will not be. + pending []inventory.PendingAssignment + pendingUnread string } // heldBy is every artifact this mesh has built, for a build that may need one as its base. diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 7e325059..ac5ab1a3 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -371,6 +371,15 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error { return nil } manifest, _, err := takeIn(ctx, b.inv, result) + // The assignments pending on this build, made or ended (novox/hq issue 325), said in the daemon's log + // after what became of the build. + if b.open != nil { + defer func() { + for _, line := range settlePendingOnBuild(ctx, b.open, result, manifest.Module, err) { + fmt.Printf("%s: %s\n", result.ID, line) + } + }() + } // When it was asked, so a plan takes as its outcome only a build asked for it or after it // (novox/hq 04-ISSUES/219). Zero when the id does not say. asked, _ := link.BuildAskedAt(result.ID) diff --git a/cmd/mesh-controller/modules.go b/cmd/mesh-controller/modules.go index 38b8b1bd..5a302da1 100644 --- a/cmd/mesh-controller/modules.go +++ b/cmd/mesh-controller/modules.go @@ -354,9 +354,20 @@ func moduleCommand(ctx context.Context, args []string) error { func assignCommand(ctx context.Context, verb string, args []string) error { // Several modules in one act (novox/hq ADR 0207): holders that depend on each other — the // service manager and the package manager — can only go on, or come off, together. + set := flag.NewFlagSet(verb, flag.ContinueOnError) + // A module known and not built: ask for its build, and keep the assignment pending on it (novox/hq + // issue 325). Only assign reads it. + build := set.Bool("build", false, "for a module known and not built: ask for its build, and assign it when it registers") + args, err := parseAround(set, args) + if err != nil { + return err + } if len(args) < 2 { return fmt.Errorf("%s […]", verb) } + if *build && verb == "unassign" { + return errors.New("unassign takes no --build") + } open, err := openStores(ctx) if err != nil { return err @@ -365,7 +376,9 @@ func assignCommand(ctx context.Context, verb string, args []string) error { // The act itself is in acts.go, so the command API refuses exactly what this refuses // (novox/hq ADR 0035). What differs between the surfaces is how the answer is printed. - act := assign + act := func(ctx context.Context, open *stores, node string, modules ...string) (string, error) { + return assignWith(ctx, open, node, assignOptions{Build: *build}, modules...) + } if verb == "unassign" { act = unassign } diff --git a/cmd/mesh-controller/pending.go b/cmd/mesh-controller/pending.go new file mode 100644 index 00000000..4b4af543 --- /dev/null +++ b/cmd/mesh-controller/pending.go @@ -0,0 +1,442 @@ +package main + +import ( + "context" + "errors" + "fmt" + "os" + "sort" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// A module not registered yet, and the assignment that waits for it (novox/hq issue 325). +// +// On 2026-10-08 the catalogue's pull request adding `sensors` merged; a minute later `assign g14 sensors` +// answered "no module of that name: sensors", and a few minutes later the same call worked. The merge had +// asked for the build (issue 300) and the build had not finished. The answer read as "you forgot to register +// it", and nothing in it said a build was on its way. +// +// So an assignment of a module the catalogue does not hold looks further before it refuses, and says which +// of three cases it is in: +// +// - **a build is in flight**: a build request for the module's directory has no outcome yet. The +// assignment is kept as a **pending assignment** and made when the build registers the module. +// - **known, not built**: the mesh asked for the directory before, and the build failed, was not +// registered, said nothing within its bound, or could not be asked. Said, with `build` and assign's +// own build argument, which asks for the build and keeps the assignment pending on it. +// - **unknown**: no module, build request or merge by that name. Refused as before, with the closest names. +// +// **Pending by default while a build is in flight**, without an argument to ask for it: an assignment is +// what a person meant and is kept even when its machine does not resolve (acts.go), nothing is sent until a +// push, and `unassign` withdraws it. Asking for a second act once the build lands is exactly the two acts by +// hand issue 300 removed. A build is never asked for by default: it runs on another machine, and a directory +// whose last build failed is a fault to look at before it is a thing to retry. + +// buildRequestBound is how long a build request may go without an outcome before it is no longer read as in +// flight, and a pending assignment waiting for it expires. Generous: a build waits behind others on the +// build seat, and a pending assignment that waits a little longer costs nothing. +const buildRequestBound = 2 * time.Hour + +// pendingShownFor is how long an ended pending assignment is said in `status`. +const pendingShownFor = 24 * time.Hour + +// assignOptions are what an assignment was asked with beside its machine and modules. +type assignOptions struct { + // Build asks for the build of a module known and not built, and keeps the assignment pending on it. + Build bool +} + +// recordBuildRequest keeps a build request so `assign` can find it. Said, never fatal: the build was asked. +func recordBuildRequest(ctx context.Context, inv *inventory.Inventory, r inventory.BuildRequest) { + if err := inv.RecordBuildRequest(ctx, r); err != nil { + fmt.Fprintf(os.Stderr, "build %s was asked, and could not be kept as a build request, so `assign` "+ + "will not know it is coming: %v\n", r.ID, err) + } +} + +// The cases of a module the catalogue does not hold (novox/hq issue 325). +const ( + unknownModule = iota + buildInFlight + knownNotBuilt +) + +// notHeld is where a module the catalogue does not hold stands: its case, the build request it was read +// from, and the sentence that says it. +type notHeld struct { + kind int + request inventory.RequestOutcome + said string +} + +// whereIs looks for a module the catalogue does not hold among the build requests the mesh keeps. +func whereIs(ctx context.Context, inv *inventory.Inventory, module string, now time.Time) (notHeld, error) { + requests, err := inv.RequestsNamed(ctx, module) + if err != nil { + return notHeld{}, err + } + if r, ok := inFlight(requests, now); ok { + return notHeld{kind: buildInFlight, request: r, said: fmt.Sprintf("%s is not registered yet: it is being "+ + "built from %s since %s, as build %s; it can be assigned once that build registers it", module, + requestSource(r), clock(r.At), r.ID)}, nil + } + if len(requests) > 0 { + return notHeld{kind: knownNotBuilt, request: requests[0], said: fmt.Sprintf("%s is known and not "+ + "registered: %s", module, whyNotBuilt(requests[0], now))}, nil + } + said := fmt.Sprintf("%v: %s — the mesh holds no module, build request or merge by that name", + inventory.ErrNoSuchModule, module) + if near := closestNames(ctx, inv, module); len(near) > 0 { + said += "; the closest it holds: " + strings.Join(near, ", ") + } + return notHeld{kind: unknownModule, said: said}, nil +} + +// notInCatalogue is the modules named that the catalogue does not hold. +func notInCatalogue(ctx context.Context, open *stores, modules []string) ([]string, error) { + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + return nil, err + } + var missing []string + for _, m := range modules { + if _, ok := shelf[m]; !ok { + missing = append(missing, m) + } + } + return missing, nil +} + +// notRegistered answers an assignment of a module the catalogue does not hold: pending on a build in flight, +// known and not built, or unknown (novox/hq issue 325). An answer with no error is a pending assignment kept. +func notRegistered(ctx context.Context, open *stores, node, module string, opts assignOptions) (string, error) { + inv := open.inventory + if _, err := inv.NodeByName(ctx, node); err != nil { + return "", err + } + now := time.Now() + where, err := whereIs(ctx, inv, module, now) + if err != nil { + return "", err + } + switch where.kind { + case buildInFlight: + lead := where.said + if opts.Build { + lead += "\n no second build was asked: this one is running" + } + return keepPending(ctx, open, node, module, where.request, lead) + case knownNotBuilt: + last := where.request + if !opts.Build { + return "", fmt.Errorf("%w: %s\n assign it again with build \"true\" to ask for its build and keep this "+ + "assignment until the build registers it; or `build` it (repository %s, path %s) and assign it "+ + "once it is registered", inventory.ErrNoSuchModule, where.said, last.Repository, orRoot(last.Path)) + } + source := buildSource{Repository: last.Repository, Seat: last.Seat} + id, err := askABuild(ctx, source, last.Path, last.Ref) + if err != nil { + return "", fmt.Errorf("%s\n its build could not be asked for: %w; nothing was assigned", where.said, err) + } + asked := inventory.RequestOutcome{BuildRequest: inventory.BuildRequest{ID: id, Repository: last.Repository, + Seat: last.Seat, Path: last.Path, Ref: last.Ref, For: "assign", At: now.UTC()}} + recordBuildRequest(ctx, inv, asked.BuildRequest) + lead := fmt.Sprintf("%s\n build %s of %s is asked for now; %s can be assigned once it registers it", + where.said, id, requestSource(asked), module) + return keepPending(ctx, open, node, module, asked, lead) + } + answer := strings.TrimPrefix(where.said, inventory.ErrNoSuchModule.Error()) + + "\n a module in a repository is built with `build` (its repository and path), then assigned" + if opts.Build { + answer += "\n build \"true\" asks for nothing here: the mesh does not know where a module of that name is" + } + return "", fmt.Errorf("%w%s", inventory.ErrNoSuchModule, answer) +} + +// keepPending records the assignment as pending on a build request, and says so. +func keepPending(ctx context.Context, open *stores, node, module string, r inventory.RequestOutcome, lead string) (string, error) { + inv := open.inventory + p, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: node, Module: module, Build: r.ID, + Repository: r.Repository, Path: r.Path}) + if errors.Is(err, inventory.ErrAlreadyPending) { + return lead + fmt.Sprintf("\n %s already waits for %s: nothing changed", node, module), nil + } + if err != nil { + return "", err + } + answer := lead + fmt.Sprintf("\n the assignment is kept as pending: %s is assigned %s when the build "+ + "registers it, and `status` says it until then. If the build fails it expires and says why; "+ + "`unassign %s %s` withdraws it", node, module, node, module) + // The build may have registered the module between the look and the record: made at once, then. + if shelf, err := inv.Catalogue(ctx); err == nil { + if _, now := shelf[module]; now { + if line := applyPending(ctx, open, p, r.ID); line != "" { + answer += "\n " + line + } + } + } + return answer, nil +} + +// inFlight is the newest build request for the module with no outcome yet, asked within the bound. +func inFlight(requests []inventory.RequestOutcome, now time.Time) (inventory.RequestOutcome, bool) { + for _, r := range requests { + if r.NotAsked == "" && !r.Heard && now.Sub(r.At) < buildRequestBound { + return r, true + } + } + return inventory.RequestOutcome{}, false +} + +// whyNotBuilt says what came of a module's last build request. +func whyNotBuilt(r inventory.RequestOutcome, now time.Time) string { + where := fmt.Sprintf("%s in %s", orRoot(r.Path), r.Repository) + switch { + case r.NotAsked != "": + return fmt.Sprintf("%s; the merge that added it (%s) could not ask for its build: %s", where, + short(r.Commit), firstLine(r.NotAsked)) + case !r.Heard: + return fmt.Sprintf("%s; build %s was asked at %s, and no outcome has been heard in %s", where, r.ID, + clock(r.At), now.Sub(r.At).Round(time.Minute)) + case r.Failed != "": + return fmt.Sprintf("%s; its last build, %s at %s, failed: %s", where, r.ID, clock(r.HeardAt), + firstLine(r.Failed)) + case r.Module != "" && r.Module != r.Name(): + return fmt.Sprintf("%s; its last build, %s, registered it as %s", where, r.ID, r.Module) + default: + return fmt.Sprintf("%s; its last build, %s at %s, was recorded and not registered — `builds` says why", + where, r.ID, clock(r.HeardAt)) + } +} + +// requestSource is a build request's source as a person reads it: repository@commit (directory). +func requestSource(r inventory.RequestOutcome) string { + at := short(r.Commit) + if at == "" { + at = r.Ref + } + if at == "" { + at = "its default branch" + } + return fmt.Sprintf("%s@%s (%s)", r.Repository, at, orRoot(r.Path)) +} + +// clock is a moment as a person reads it, in the controller's local time. +func clock(t time.Time) string { return t.Local().Format("2006-01-02 15:04:05 MST") } + +// closestNames is up to three names the mesh holds that are near the one asked: registered modules and +// directories it asked to build. +func closestNames(ctx context.Context, inv *inventory.Inventory, name string) []string { + var candidates []string + if shelf, err := inv.Catalogue(ctx); err == nil { + for m := range shelf { + candidates = append(candidates, m) + } + } + if asked, err := inv.RequestedNames(ctx); err == nil { + candidates = append(candidates, asked...) + } + type scored struct { + name string + distance int + } + seen := map[string]bool{} + var near []scored + limit := max(2, len(name)/3) + for _, c := range candidates { + if seen[c] || c == name { + continue + } + seen[c] = true + d := editDistance(name, c) + if d <= limit || strings.Contains(c, name) || strings.Contains(name, c) { + near = append(near, scored{c, d}) + } + } + sort.Slice(near, func(i, j int) bool { + if near[i].distance != near[j].distance { + return near[i].distance < near[j].distance + } + return near[i].name < near[j].name + }) + var out []string + for i := 0; i < len(near) && i < 3; i++ { + out = append(out, near[i].name) + } + return out +} + +// editDistance is the Levenshtein distance between two names. +func editDistance(a, b string) int { + ra, rb := []rune(a), []rune(b) + prev := make([]int, len(rb)+1) + for j := range prev { + prev[j] = j + } + for i := 1; i <= len(ra); i++ { + cur := make([]int, len(rb)+1) + cur[0] = i + for j := 1; j <= len(rb); j++ { + cost := 1 + if ra[i-1] == rb[j-1] { + cost = 0 + } + cur[j] = min(prev[j]+1, cur[j-1]+1, prev[j-1]+cost) + } + prev = cur + } + return prev[len(rb)] +} + +// applyPending makes a pending assignment now that its module is registered, and settles it: applied, or +// refused with the refusal. Returns what it said, or nothing when another process settled it first. +func applyPending(ctx context.Context, open *stores, p inventory.PendingAssignment, by string) string { + inv := open.inventory + answer, err := assign(ctx, open, p.Node, p.Module) + state, note := inventory.PendingApplied, "" + if err != nil { + state = inventory.PendingRefused + note = fmt.Sprintf("the pending assignment of %s to %s was refused when build %s registered it: %s", + p.Module, p.Node, by, firstLine(err.Error())) + } else { + // Not promised as sent: a walk sends it if the module's upgrade announcement finds it + // assigned, and nothing does if the announcement came first. `status` says which, as for any machine. + note = fmt.Sprintf("%s is assigned %s: build %s registered it (pending since %s); `status` says whether "+ + "%s has been sent it, and `push %s` sends it if not", p.Node, p.Module, by, clock(p.Since), p.Node, p.Node) + if strings.Contains(answer, "already runs") { + note = fmt.Sprintf("%s already runs %s: the pending assignment is met", p.Node, p.Module) + } + } + settled, serr := inv.SettlePending(ctx, p.ID, state, note) + if serr != nil { + return fmt.Sprintf("%s — and it could not be recorded as settled: %v", note, serr) + } + if !settled { + return "" + } + return note +} + +// expirePending ends a pending assignment whose build will not register its module, with why. +func expirePending(ctx context.Context, inv *inventory.Inventory, p inventory.PendingAssignment, why string) string { + note := fmt.Sprintf("the pending assignment of %s to %s expired: %s; nothing was assigned. Assign it again with "+ + "build \"true\" to ask for the build again", p.Module, p.Node, why) + settled, err := inv.SettlePending(ctx, p.ID, inventory.PendingExpired, note) + if err != nil { + return fmt.Sprintf("%s — and it could not be recorded: %v", note, err) + } + if !settled { + return "" + } + return note +} + +// settlePendingOnBuild is a build's outcome reaching the assignments waiting for it, wherever the outcome is +// taken in (novox/hq issue 325): a module registered makes every assignment pending on it; a build of a +// pending assignment that failed, or was not registered, ends it with why. Returns what was said. +func settlePendingOnBuild(ctx context.Context, open *stores, result link.BuildResult, module string, takeErr error) []string { + inv := open.inventory + waiting, err := inv.Pending(ctx, time.Time{}) + if err != nil { + return []string{fmt.Sprintf("the pending assignments could not be read after build %s: %v", result.ID, err)} + } + registered := module != "" && (takeErr == nil || errors.Is(takeErr, inventory.ErrSuperseded)) + var said []string + for _, p := range waiting { + switch { + case registered && p.Module == module: + if line := applyPending(ctx, open, p, result.ID); line != "" { + said = append(said, line) + } + case !registered && p.Build == result.ID: + why := firstLine(result.Failed) + if why == "" && takeErr != nil { + why = firstLine(takeErr.Error()) + } + what := fmt.Sprintf("build %s of %s failed: %s", result.ID, orRoot(p.Path), why) + if result.Failed == "" { + what = fmt.Sprintf("build %s of %s was recorded and not registered: %s", result.ID, orRoot(p.Path), why) + } + if line := expirePending(ctx, inv, p, what); line != "" { + said = append(said, line) + } + } + } + return said +} + +// settleStalePending settles what a missed outcome left waiting: a module registered meanwhile is assigned, +// a build heard and not registered ends its assignment, and one with no outcome past the bound expires. +// Asked where the mesh is read for status, so a pending assignment never waits silently for ever. +func settleStalePending(ctx context.Context, open *stores, now time.Time) []string { + inv := open.inventory + waiting, err := inv.Pending(ctx, time.Time{}) + if err != nil { + // Said, not swallowed; status reads the pending assignments again and says the same failure there. + return []string{fmt.Sprintf("the pending assignments could not be read to settle them: %v", err)} + } + shelf, err := inv.Catalogue(ctx) + if err != nil { + return []string{fmt.Sprintf("the catalogue could not be read to settle the pending assignments: %v", err)} + } + var said []string + for _, p := range waiting { + if _, ok := shelf[p.Module]; ok { + if line := applyPending(ctx, open, p, p.Build); line != "" { + said = append(said, line) + } + continue + } + requests, err := inv.RequestsNamed(ctx, p.Module) + if err != nil { + said = append(said, fmt.Sprintf("the build requests for %s could not be read: %v", p.Module, err)) + continue + } + var r *inventory.RequestOutcome + for i := range requests { + if requests[i].ID == p.Build { + r = &requests[i] + } + } + why := "" + switch { + case r == nil && now.Sub(p.Since) > buildRequestBound: + why = fmt.Sprintf("build %s is no longer on record", p.Build) + case r != nil && r.Heard: + why = whyNotBuilt(*r, now) + case r != nil && now.Sub(r.At) > buildRequestBound: + why = fmt.Sprintf("no outcome of build %s was heard within %s of asking", p.Build, buildRequestBound) + } + if why == "" { + continue + } + if line := expirePending(ctx, inv, p, why); line != "" { + said = append(said, line) + } + } + return said +} + +// withdrawPending is `unassign` of a module only pending on a machine: the pending assignment withdrawn. +func withdrawPending(ctx context.Context, inv *inventory.Inventory, node, module string) (string, bool, error) { + waiting, err := inv.Pending(ctx, time.Time{}) + if err != nil { + return "", false, err + } + for _, p := range waiting { + if p.Node != node || p.Module != module { + continue + } + note := fmt.Sprintf("the pending assignment of %s to %s is withdrawn; build %s goes on, and registers %s "+ + "assigned nowhere", module, node, p.Build, module) + if _, err := inv.SettlePending(ctx, p.ID, inventory.PendingWithdrawn, note); err != nil { + return "", false, err + } + return note, true, nil + } + return "", false, nil +} diff --git a/cmd/mesh-controller/pending_test.go b/cmd/mesh-controller/pending_test.go new file mode 100644 index 00000000..dd3c9d8d --- /dev/null +++ b/cmd/mesh-controller/pending_test.go @@ -0,0 +1,329 @@ +package main + +import ( + "encoding/json" + "errors" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// novox/hq issue 325: an assignment of a module the catalogue does not hold says which case it is in — a +// build in flight (kept pending), known and not built (said, with build), or unknown (refused, with the +// closest names) — and a pending assignment is made when its build registers the module, or ends with why. + +// aCatalogueMesh is aMesh with one module held from the catalogue, so a merge of it is acted on. +func aCatalogueMesh(t *testing.T) *stores { + t.Helper() + open := aMesh(t) + if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: "networkmanager", Version: "1"}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main", + BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + return open +} + +// mergeAdding is the catalogue's merge adding one module's directory, acted on as the bus hands it over. +func mergeAdding(t *testing.T, open *stores, dir, commit string) { + t.Helper() + m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, + Paths: []string{dir + "/module.json"}, ModuleDirs: []string{dir}, ModuleDirsSaid: true} + if err := (following{open: open}).SourceMoved(t.Context(), m); err != nil { + t.Fatal(err) + } +} + +// outcome is a build's outcome as the build seat announces it: registered as module, or failed. +func outcome(id, dir, commit, module, failed string) link.BuildResult { + r := link.BuildResult{ID: id, Repository: "novox/mesh-catalog", Path: dir, Ref: "main", Commit: commit, + On: "anchor", Failed: failed, Source: &link.SourceOnSeat{Repository: "novox/mesh-catalog", Seat: "git"}} + if failed == "" { + r.Module = module + r.Manifest, _ = json.Marshal(catalogue.Manifest{Module: module, Version: "1"}) + } + return r +} + +func assignedTo(t *testing.T, open *stores, node string) []string { + t.Helper() + got, err := open.inventory.Assigned(t.Context(), node) + if err != nil { + t.Fatal(err) + } + return got +} + +func pendingOf(t *testing.T, open *stores) []inventory.PendingAssignment { + t.Helper() + got, err := open.inventory.Pending(t.Context(), time.Now().Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + return got +} + +// A build in flight: the merge asked for it, the assignment is kept pending and said so with the build, and +// the build's outcome makes it. Status says it while it waits. +func TestAnAssignmentWaitsForTheBuildInFlight(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + asksWithPaths(t) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + + said, err := assign(ctx, open, "laptop", "sensors") + if err != nil { + t.Fatalf("an assignment while its build runs was refused: %v", err) + } + t.Logf("assign laptop sensors, while its build runs:\n%s", said) + for _, want := range []string{"being built from novox/mesh-catalog@3da80a4b (modules/sensors)", + "build b-modules/sensors", "kept as pending", "`unassign laptop sensors`"} { + if !strings.Contains(said, want) { + t.Fatalf("the answer does not say %q:\n%s", want, said) + } + } + if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { + t.Fatal("a module not registered was assigned") + } + pending := pendingOf(t, open) + if len(pending) != 1 || pending[0].State != inventory.PendingWaiting || pending[0].Build != "b-modules/sensors" { + t.Fatalf("pending: %+v", pending) + } + // Asked twice, it is one pending assignment. + if again, err := assign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(again, "already waits") { + t.Fatalf("a second assignment: %q %v", again, err) + } + + asked, err := theThreeQuestions(ctx, open) + if err != nil { + t.Fatal(err) + } + if asked.well() { + t.Fatal("status called the mesh well while an assignment waits") + } + body, err := statusAsJSON(asked) + if err != nil { + t.Fatal(err) + } + var doc meshStatus + if err := json.Unmarshal(body, &doc); err != nil { + t.Fatal(err) + } + if len(doc.Pending) != 1 || doc.Pending[0].State != "waiting" || doc.Pending[0].Module != "sensors" { + t.Fatalf("status says pending %+v", doc.Pending) + } + + if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", + "3da80a4b00aa", "sensors", "")); err != nil { + t.Fatal(err) + } + if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") { + t.Fatal("the build registered sensors and the pending assignment was not made") + } + pending = pendingOf(t, open) + if len(pending) != 1 || pending[0].State != inventory.PendingApplied || + !strings.Contains(pending[0].Note, "`push laptop` sends it if not") { + t.Fatalf("pending after the build: %+v", pending) + } +} + +// The build of a pending assignment fails: it expires, saying the build's words, and nothing is assigned. +func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + asksWithPaths(t) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", + "3da80a4b00aa", "", "go build: undefined: sensorsRead")); err != nil { + t.Fatal(err) + } + pending := pendingOf(t, open) + if len(pending) != 1 || pending[0].State != inventory.PendingExpired { + t.Fatalf("pending after a failed build: %+v", pending) + } + for _, want := range []string{"expired", "build b-modules/sensors of modules/sensors failed", + "undefined: sensorsRead", "nothing was assigned", "build \"true\""} { + if !strings.Contains(pending[0].Note, want) { + t.Fatalf("the expiry does not say %q: %s", want, pending[0].Note) + } + } + if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { + t.Fatal("a failed build's module was assigned") + } + asked, err := theThreeQuestions(ctx, open) + if err != nil { + t.Fatal(err) + } + if asked.well() { + t.Fatal("status called the mesh well on the day a pending assignment expired") + } + + // Known and not built now: said, and refused without build. + _, err = assign(ctx, open, "laptop", "sensors") + if !errors.Is(err, inventory.ErrNoSuchModule) { + t.Fatalf("a module whose build failed: %v", err) + } + for _, want := range []string{"sensors is known and not registered", "its last build, b-modules/sensors", + "failed: go build: undefined: sensorsRead", "build \"true\"", "repository novox/mesh-catalog, path modules/sensors"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("the refusal does not say %q:\n%v", want, err) + } + } +} + +// Known and not built, asked with build: the build is asked from where the last one was, and the assignment +// kept pending on the new build. +func TestAssignWithBuildAsksForAKnownModule(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-old", Repository: "novox/mesh-catalog", + Seat: "git", Path: "modules/sensors", Ref: "main", For: "build", At: time.Now().Add(-time.Hour)}); err != nil { + t.Fatal(err) + } + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-old", Repository: "novox/mesh-catalog", Ref: "main", + Path: "modules/sensors", On: "anchor", Failed: "no space left on device"}); err != nil { + t.Fatal(err) + } + asked := asksWithPaths(t) + said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") + if err != nil { + t.Fatal(err) + } + if len(*asked) != 1 || (*asked)[0] != [3]string{"novox/mesh-catalog", "modules/sensors", "main"} { + t.Fatalf("asked %v", *asked) + } + for _, want := range []string{"known and not registered", "no space left on device", + "build b-modules/sensors of novox/mesh-catalog@main (modules/sensors) is asked for now", "kept as pending"} { + if !strings.Contains(said, want) { + t.Fatalf("the answer does not say %q:\n%s", want, said) + } + } + pending := pendingOf(t, open) + if len(pending) != 1 || pending[0].Build != "b-modules/sensors" { + t.Fatalf("pending: %+v", pending) + } +} + +// Unknown: refused as before, as no module of that name, with the closest names the mesh holds; build asks +// for nothing, since the mesh does not know where it is. +func TestAnUnknownModuleIsRefusedWithTheClosestNames(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "sensord", Version: "1"}) + asked := asksWithPaths(t) + _, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") + if !errors.Is(err, inventory.ErrNoSuchModule) { + t.Fatalf("an unknown module: %v", err) + } + for _, want := range []string{"no module of that name: sensors", "no module, build request or merge by that name", + "the closest it holds: sensord", "asks for nothing here"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("the refusal does not say %q:\n%v", want, err) + } + } + if len(*asked) != 0 || len(pendingOf(t, open)) != 0 { + t.Fatalf("an unknown module asked %v, pending %+v", *asked, pendingOf(t, open)) + } +} + +// Several modules in one act, one of them not registered: nothing is assigned and nothing kept pending, and +// each missing one is said. +func TestAnActWithAModuleNotRegisteredIsRefusedWhole(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + asksWithPaths(t) + register(t, open, catalogue.Manifest{Module: "known", Version: "1"}) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + _, err := assign(ctx, open, "laptop", "known", "sensors") + if err == nil || !strings.Contains(err.Error(), "nothing was assigned") || !strings.Contains(err.Error(), "being built") { + t.Fatalf("an act naming a module in flight: %v", err) + } + if slices.Contains(assignedTo(t, open, "laptop"), "known") || len(pendingOf(t, open)) != 0 { + t.Fatal("an act refused whole was made in part") + } +} + +// unassign withdraws a pending assignment; the build goes on and registers the module assigned nowhere. +func TestUnassignWithdrawsAPendingAssignment(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + asksWithPaths(t) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + said, err := unassign(ctx, open, "laptop", "sensors") + if err != nil || !strings.Contains(said, "withdrawn") { + t.Fatalf("unassign of a pending assignment: %q %v", said, err) + } + if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", + "3da80a4b00aa", "sensors", "")); err != nil { + t.Fatal(err) + } + if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { + t.Fatal("a withdrawn assignment was made") + } +} + +// A build that says nothing within the bound: the pending assignment expires at the next look, saying so, +// rather than waiting for ever; and a build heard by another process meanwhile is made at the next look. +func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + for _, r := range []inventory.BuildRequest{ + {ID: "build-lost", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/lost", Ref: "main", + For: "merge", At: time.Now().Add(-buildRequestBound / 2)}, + {ID: "build-heard", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/heard", Ref: "main", + For: "merge", At: time.Now()}, + } { + if err := inv.RecordBuildRequest(ctx, r); err != nil { + t.Fatal(err) + } + } + for _, m := range []string{"lost", "heard"} { + if _, err := assign(ctx, open, "laptop", m); err != nil { + t.Fatal(err) + } + } + // Heard by a process that kept no pending assignment: registered, and nothing made it. + register(t, open, catalogue.Manifest{Module: "heard", Version: "1"}) + + said := settleStalePending(ctx, open, time.Now().Add(buildRequestBound)) + if len(said) != 2 { + t.Fatalf("settled %v", said) + } + if !slices.Contains(assignedTo(t, open, "laptop"), "heard") { + t.Fatal("a module registered meanwhile was not assigned at the next look") + } + byModule := map[string]inventory.PendingAssignment{} + for _, p := range pendingOf(t, open) { + byModule[p.Module] = p + } + if p := byModule["lost"]; p.State != inventory.PendingExpired || !strings.Contains(p.Note, "no outcome of build build-lost") { + t.Fatalf("a build that said nothing: %+v", p) + } + if p := byModule["heard"]; p.State != inventory.PendingApplied { + t.Fatalf("a module registered meanwhile: %+v", p) + } +} + +// The seat's assign passes build on; unassign takes none. +func TestTheSeatsAssignPassesBuild(t *testing.T) { + argv, err := argvFor("assign", map[string]any{"node": "g14", "module": "sensors", "build": "true"}) + if err != nil || !slices.Equal(argv, []string{"assign", "g14", "sensors", "--build"}) { + t.Fatalf("assign with build: %v %v", argv, err) + } + if _, err := argvFor("unassign", map[string]any{"node": "g14", "module": "sensors", "build": "true"}); err == nil { + t.Fatal("unassign took build") + } +} diff --git a/cmd/mesh-controller/readable.go b/cmd/mesh-controller/readable.go index a84a1760..d286b693 100644 --- a/cmd/mesh-controller/readable.go +++ b/cmd/mesh-controller/readable.go @@ -101,6 +101,24 @@ type meshStatus struct { // Overflowing is every module whose identity overflows the bound of a provision it requires, and // so is left out of its provider's grants (novox/hq ADR 0225). Absent when every identity fits. Overflowing []catalogue.Overflow `json:"overflowing,omitempty"` + // Pending is every assignment waiting for its module's build to register it, and every one ended in + // the last day with what ended it (novox/hq issue 325). Absent when there is none; PendingUnread says + // why they could not be read. + Pending []pendingStatus `json:"pending,omitempty"` + PendingUnread string `json:"pendingUnread,omitempty"` +} + +// pendingStatus is one pending assignment as status says it. +type pendingStatus struct { + Node string `json:"node"` + Module string `json:"module"` + State string `json:"state"` + Build string `json:"build"` + Repository string `json:"repository,omitempty"` + Path string `json:"path,omitempty"` + Since time.Time `json:"since"` + Settled *time.Time `json:"settled,omitempty"` + Note string `json:"note,omitempty"` } // machineFiltered is one rule set on a converged machine that the mesh did not write and that @@ -240,6 +258,11 @@ func statusAsJSON(asked answers) ([]byte, error) { out.Conditions, out.ConditionsUnread = inBrief(asked.conditions), asked.conditionsUnread out.Failing = providerStandings(asked.conditions) out.Overflowing = asked.overflowing + out.PendingUnread = asked.pendingUnread + for _, p := range asked.pending { + out.Pending = append(out.Pending, pendingStatus{Node: p.Node, Module: p.Module, State: p.State, + Build: p.Build, Repository: p.Repository, Path: p.Path, Since: p.Since, Settled: p.Settled, Note: p.Note}) + } for name := range asked.refused { out.Unresolved = append(out.Unresolved, machineUnresolved{ Node: name, Problem: asked.refused[name]}) diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index d67db4d8..29fd27d8 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -337,8 +337,15 @@ func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) e // askABuild is how a plan asks for one build, not waited for, and learns the id it asked under. A // variable so a test of what a plan does around an ask needs no build machine. -var askABuild = func(ctx context.Context, source buildSource, path, ref string) (string, error) { - return buildOneAsked(ctx, source, path, ref, 0, false) +// +// Set in init, not where it is declared: an assignment pending on a build asks for one too (novox/hq issue +// 325), and a build's outcome makes pending assignments, so the two refer to each other. +var askABuild func(ctx context.Context, source buildSource, path, ref string) (string, error) + +func init() { + askABuild = func(ctx context.Context, source buildSource, path, ref string) (string, error) { + return buildOneAsked(ctx, source, path, ref, 0, false) + } } // askModule asks the build machine for one module of a plan and marks it asked, with the id it was diff --git a/cmd/mesh-controller/replay325_test.go b/cmd/mesh-controller/replay325_test.go new file mode 100644 index 00000000..d3f0c6f1 --- /dev/null +++ b/cmd/mesh-controller/replay325_test.go @@ -0,0 +1,60 @@ +package main + +import ( + "encoding/json" + "slices" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// novox/hq issue 325, replayed with only what the controller had before its fix, so it can be laid over the +// older commit. On 2026-10-08 the catalogue's pull request adding `sensors` merged, and the merge asked for +// its build (issue 300). About a minute later `assign g14 sensors` answered "no module of that name: +// sensors"; a few minutes later the same call worked, because the build had registered the module. The +// answer read as "nobody registered it". An assignment made while the build runs says the build — where it +// is from, since when, its id — is kept, and is made when the build registers the module. +func TestReplay325(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "networkmanager", Version: "1"}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main", + BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + asksWithPaths(t) + m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "5e450125aa", + Paths: []string{"modules/sensors/module.json", "modules/sensors/cmd/sensors/main.go"}, + ModuleDirs: []string{"modules/sensors"}, ModuleDirsSaid: true} + if err := (following{open: open}).SourceMoved(ctx, m); err != nil { + t.Fatal(err) + } + + said, err := assign(ctx, open, "laptop", "sensors") + if err != nil { + t.Fatalf("assign while the merge's build runs was refused: %v", err) + } + for _, want := range []string{"being built", "novox/mesh-catalog", "modules/sensors", "b-modules/sensors"} { + if !strings.Contains(said, want) { + t.Fatalf("the answer does not say %q:\n%s", want, said) + } + } + + manifest, _ := json.Marshal(catalogue.Manifest{Module: "sensors", Version: "1"}) + if err := (builds{open.inventory, open}).Built(ctx, link.BuildResult{ID: "b-modules/sensors", + Repository: "novox/mesh-catalog", Path: "modules/sensors", Ref: "main", Commit: "5e450125aa", On: "anchor", + Module: "sensors", Manifest: manifest, + Source: &link.SourceOnSeat{Repository: "novox/mesh-catalog", Seat: "git"}}); err != nil { + t.Fatal(err) + } + assigned, err := open.inventory.Assigned(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + if !slices.Contains(assigned, "sensors") { + t.Fatalf("the build registered sensors and laptop runs %v", assigned) + } +} diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index 7bad018f..9b4a9beb 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -418,7 +418,13 @@ func (a *verbArguments) commandLine() ([]string, error) { } // Several modules comma-separated, judged as one act (novox/hq ADR 0207): the holders of // the seats that apply resources depend on each other and go on together. - return append([]string{verb, str("node")}, splitModules(str("module"))...), nil + argv := append([]string{verb, str("node")}, splitModules(str("module"))...) + // A module known and not built: its build asked for, the assignment pending on it (novox/hq + // issue 325). unassign declares no build, so a call giving it is refused before this. + if verb == "assign" && on("build") { + argv = append(argv, "--build") + } + return argv, nil case "pin": if err := need("node", "provision", "from", "module"); err != nil { return nil, err diff --git a/cmd/mesh-controller/status.go b/cmd/mesh-controller/status.go index fc7e18d3..4e961db6 100644 --- a/cmd/mesh-controller/status.go +++ b/cmd/mesh-controller/status.go @@ -4,6 +4,7 @@ import ( "context" "flag" "fmt" + "os" "sort" "strings" "time" @@ -143,6 +144,8 @@ func printStatus(asked answers) error { len(quiet), strings.Join(said, "\n ")) } + printPending(asked.pending, asked.pendingUnread) + if open, late := openPlans(asked.plans, time.Now(), asked.paused, asked.tierBounds); len(open) > 0 { fmt.Printf("%d plan(s) open", len(open)) if late > 0 { @@ -466,6 +469,15 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) { if out.conditions, err = openConditions(ctx); err != nil { out.conditionsUnread = err.Error() } + // And the assignments waiting for their module's build (novox/hq issue 325): what a missed outcome left + // waiting is settled first, so one never waits silently past its build. + for _, line := range settleStalePending(ctx, open, time.Now()) { + fmt.Fprintln(os.Stderr, line) + } + if out.pending, err = inv.Pending(ctx, time.Now().Add(-pendingShownFor)); err != nil { + out.pendingUnread = err.Error() + err = nil + } out.plans, err = inv.RecentPlans(ctx, 5) if err != nil { return answers{}, err @@ -598,7 +610,53 @@ func (a answers) well() bool { return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 && len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 && len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 && - len(a.conditions) == 0 && a.conditionsUnread == "" + len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingNotWell(a.pending) +} + +// pendingNotWell is whether any pending assignment is waiting, or ended without being made. +func pendingNotWell(pending []inventory.PendingAssignment) bool { + for _, p := range pending { + if p.State != inventory.PendingApplied && p.State != inventory.PendingWithdrawn { + return true + } + } + return false +} + +// printPending says the assignments waiting for their module's build, and those ended in the last day +// (novox/hq issue 325). +func printPending(pending []inventory.PendingAssignment, unread string) { + if unread != "" { + fmt.Printf("the assignments waiting for a build could not be read: %s\n\n", unread) + return + } + var waiting, ended []inventory.PendingAssignment + for _, p := range pending { + if p.State == inventory.PendingWaiting { + waiting = append(waiting, p) + } else { + ended = append(ended, p) + } + } + if len(waiting) > 0 { + fmt.Printf("%d assignment(s) wait for their module's build to register it:\n", len(waiting)) + for _, p := range waiting { + fmt.Printf(" %-12s %-20s build %s of %s %s, since %s\n", p.Node, p.Module, p.Build, p.Repository, + orRoot(p.Path), clock(p.Since)) + } + fmt.Printf("\n each is made when its build registers the module; `unassign ` withdraws it\n\n") + } + if len(ended) > 0 { + fmt.Printf("%d pending assignment(s) ended in the last day:\n", len(ended)) + for _, p := range ended { + at := "" + if p.Settled != nil { + at = clock(*p.Settled) + } + fmt.Printf(" %-12s %-20s %-9s %s\n %-12s %s\n", p.Node, p.Module, p.State, at, "", p.Note) + } + fmt.Println() + } } // hostSplit is which machines report which host version, for every version more than one machine diff --git a/cmd/mesh-controller/upgrades.go b/cmd/mesh-controller/upgrades.go index c44bf035..86b08215 100644 --- a/cmd/mesh-controller/upgrades.go +++ b/cmd/mesh-controller/upgrades.go @@ -360,7 +360,7 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error { } // **A new module is built and registered, and sent nowhere** (novox/hq issue 300): the delivery plan // says so, and assigning it is a person's act, which needs it registered first. - built := askNewModules(ctx, m, from, added) + built := askNewModules(ctx, inv, m, from, added) moved := append(append([]inventory.Entry{}, touched...), packaging...) if len(moved) == 0 { if len(built) > 0 { @@ -491,7 +491,8 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error { // new module either. Outside the plan: the plan walks modules the catalogue holds, and a new one has no // machine to send to and nothing standing on it. What could not be asked is said, and left to `build`. // Returns the directories asked for. -func askNewModules(ctx context.Context, m link.SourceMoved, from []inventory.Entry, added []string) []string { +func askNewModules(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, from []inventory.Entry, + added []string) []string { if len(added) == 0 || len(from) == 0 { return nil } @@ -503,11 +504,19 @@ func askNewModules(ctx context.Context, m link.SourceMoved, from []inventory.Ent path = "" } id, err := askABuild(ctx, source, path, m.Base) + // Kept, asked or not, so `assign` can tell a build in flight, or a merge that could not ask for one, + // from a module nobody ever heard of (novox/hq issue 325). + request := inventory.BuildRequest{ID: id, Repository: source.Repository, Seat: source.Seat, Path: path, + Ref: m.Base, Commit: m.Commit, For: "merge"} if err != nil { + request.ID = fmt.Sprintf("not-asked-%s-%s", short(m.Commit), strings.ReplaceAll(dir, "/", "-")) + request.NotAsked = err.Error() + recordBuildRequest(ctx, inv, request) fmt.Printf(" %s is a new module in %s/%s and could not be built: %v — a hand `build` of it "+ "asks again\n", dir, m.Owner, m.Repo, err) continue } + recordBuildRequest(ctx, inv, request) fmt.Printf(" %s is a new module in %s/%s: build %s asked at %s; registered when it lands, assigned "+ "nowhere\n", dir, m.Owner, m.Repo, id, m.Base) asked = append(asked, dir) diff --git a/internal/catalogue/verbs.go b/internal/catalogue/verbs.go index 299fb73c..3ab55764 100644 --- a/internal/catalogue/verbs.go +++ b/internal/catalogue/verbs.go @@ -178,9 +178,16 @@ var ControllerVerbs = []Verb{ "diff": "\"true\": what a push would change on this machine, against what it was last sent; not with files", }, []string{"node"}, "files", "diff")}, {Name: "assign", Description: "Put a module on a machine. Refused with the mesh's own words when it cannot resolve there, " + - "or when a seat its resources are applied through is held by nothing on the machine (novox/hq ADR 0207).", + "or when a seat its resources are applied through is held by nothing on the machine (novox/hq ADR 0207). " + + "A module not registered yet is looked for (novox/hq issue 325): while its build runs — a merge that adds " + + "a module asks for one — the assignment is kept as pending and made when the build registers it, said in " + + "`status`, and expires with why if the build fails; a module known and not built is said, and build asks " + + "for its build; a name the mesh never heard of is refused with the closest names.", Input: schema(map[string]string{"node": "the machine's name", - "module": "the module's name; several comma-separated are judged together"}, []string{"node", "module"})}, + "module": "the module's name; several comma-separated are judged together", + "build": "\"true\": for a module known and not built (its last build failed, or was never asked), ask " + + "for its build and keep this assignment pending until the build registers it"}, + []string{"node", "module"}, "build")}, {Name: "unassign", Description: "Take a module off a machine. Refused when it holds a seat a module left there depends on.", Input: schema(map[string]string{"node": "the machine's name", "module": "the module's name; several comma-separated are judged together"}, []string{"node", "module"})}, diff --git a/internal/inventory/migrations/0082-an-assignment-waits-for-its-build.sql b/internal/inventory/migrations/0082-an-assignment-waits-for-its-build.sql new file mode 100644 index 00000000..cf03c449 --- /dev/null +++ b/internal/inventory/migrations/0082-an-assignment-waits-for-its-build.sql @@ -0,0 +1,42 @@ +-- An assignment waits for its module's build (novox/hq issue 325). +-- +-- A merge that adds a module asks for its build (issue 300), and the module is registered when the build's +-- outcome is taken in, minutes later. An `assign` in between was answered "no module of that name", which +-- read as "nobody registered it". The controller now keeps every build it asks for, so `assign` can tell a +-- build in flight from a module it never heard of, and an assignment made while the build runs is kept as +-- pending and made when the build registers the module, or ended with the reason when it does not. + +-- Every build the controller asked for, whatever asked it: a merge, a plan's tier, a person's `build`, an +-- `assign` with build. Its outcome is the `build` row of the same id, written when the outcome is taken in; +-- an ask without one is still running, or was lost. not_asked is why a merge's ask of a new module could +-- not be made, and the id is then not a build's. +create table build_request ( + id text primary key, + repository text not null, + seat text not null default '', + source_path text not null default '', + ref text not null default '', + commit_hash text not null default '', + asked_for text not null default '', + not_asked text, + asked_at timestamptz not null default now() +); +create index build_request_asked_at on build_request (asked_at); + +-- An assignment kept until its module is registered. waiting until the build registers the module, then +-- applied (the assignment was made), refused (the assignment was refused then, with the refusal), expired +-- (the build failed, was not registered, or said nothing within its bound) or withdrawn (`unassign`). +create table pending_assignment ( + id bigserial primary key, + node text not null, + module text not null, + build_id text not null, + repository text not null default '', + source_path text not null default '', + since timestamptz not null default now(), + state text not null default 'waiting' + check (state in ('waiting', 'applied', 'refused', 'expired', 'withdrawn')), + note text not null default '', + settled_at timestamptz +); +create unique index pending_assignment_waiting on pending_assignment (node, module) where state = 'waiting'; diff --git a/internal/inventory/pending.go b/internal/inventory/pending.go new file mode 100644 index 00000000..5e9dcb10 --- /dev/null +++ b/internal/inventory/pending.go @@ -0,0 +1,228 @@ +package inventory + +import ( + "context" + "errors" + "path" + "strings" + "time" + + "github.com/jackc/pgx/v5" +) + +// What the controller asked to be built, and the assignments waiting for it (novox/hq issue 325). +// +// A build's outcome was kept, and its asking was not: between a merge asking for a new module and the +// outcome registering it, the mesh had no record that anything was coming, and `assign` answered "no module +// of that name" for a module minutes from existing. Kept here, an ask lets `assign` say which case it is in, +// and an assignment made while the build runs is kept until the build registers the module. + +// requestKeptFor is how long an ask is kept: long past any build's bound, short enough to stay small. +const requestKeptFor = 30 * 24 * time.Hour + +// BuildRequest is one build the controller asked for. +type BuildRequest struct { + // ID is the build's id, as its outcome will carry it; for an ask that could not be made, an id of + // the controller's making that no build carries. + ID string + // Repository and Seat are the source as the mesh spells it (novox/hq ADR 0111): a path on the seat's + // holder, or a URL with no seat. Path is the module's directory in it, Ref the branch or commit asked. + Repository, Seat, Path, Ref string + // Commit is the commit that made the ask, where one did: a merge's. + Commit string + // For says what asked: "merge", "build", "plan", "assign". + For string + // NotAsked is why the ask could not be made, empty when it was. + NotAsked string + At time.Time +} + +// Name is the module this ask is expected to register, read from its directory: the last element of its +// path, or of its repository for a module at the repository's root. A guess until the build says, and the +// one a person reads too: `modules/sensors` is sensors. +func (a BuildRequest) Name() string { + if p := strings.Trim(a.Path, "/"); p != "" && p != "." { + return path.Base(p) + } + return strings.TrimSuffix(path.Base(strings.TrimRight(a.Repository, "/")), ".git") +} + +// RequestOutcome is an ask beside what came of it, where anything did. +type RequestOutcome struct { + BuildRequest + // Heard is whether an outcome of this id was taken in; Failed is its builder's words when it failed, + // Module what it registered as, HeardAt when it was taken in. + Heard bool + Failed string + Module string + HeardAt time.Time +} + +// RecordBuildRequest keeps one ask. Idempotent on the id: a merge's ask recorded by the merge and by the +// asking itself is one ask. Asks older than requestKeptFor are let go in the same act. +func (i *Inventory) RecordBuildRequest(ctx context.Context, a BuildRequest) error { + at := a.At + if at.IsZero() { + at = time.Now().UTC() + } + var notAsked *string + if a.NotAsked != "" { + notAsked = &a.NotAsked + } + if _, err := i.store.Pool().Exec(ctx, + `insert into build_request (id, repository, seat, source_path, ref, commit_hash, asked_for, not_asked, asked_at) + values ($1, $2, $3, $4, $5, $6, $7, $8, $9) + on conflict (id) do update set + commit_hash = case when build_request.commit_hash = '' then excluded.commit_hash else build_request.commit_hash end, + asked_for = case when build_request.asked_for = '' then excluded.asked_for else build_request.asked_for end`, + a.ID, a.Repository, a.Seat, strings.Trim(a.Path, "/"), a.Ref, a.Commit, a.For, notAsked, at); err != nil { + return err + } + _, err := i.store.Pool().Exec(ctx, `delete from build_request where asked_at < $1`, time.Now().Add(-requestKeptFor)) + return err +} + +// RequestsNamed is every ask kept whose directory names the module, newest first, each with its outcome. +func (i *Inventory) RequestsNamed(ctx context.Context, name string) ([]RequestOutcome, error) { + all, err := i.requests(ctx) + if err != nil { + return nil, err + } + var out []RequestOutcome + for _, a := range all { + if a.Name() == name || a.Module == name { + out = append(out, a) + } + } + return out, nil +} + +// RequestedNames is the name of every module an ask kept is expected to register: what `assign` offers as a +// closest name beside the modules registered. +func (i *Inventory) RequestedNames(ctx context.Context) ([]string, error) { + all, err := i.requests(ctx) + if err != nil { + return nil, err + } + seen := map[string]bool{} + var out []string + for _, a := range all { + if n := a.Name(); n != "" && !seen[n] { + seen[n] = true + out = append(out, n) + } + } + return out, nil +} + +func (i *Inventory) requests(ctx context.Context) ([]RequestOutcome, error) { + rows, err := i.store.Pool().Query(ctx, + `select a.id, a.repository, a.seat, a.source_path, a.ref, a.commit_hash, a.asked_for, + coalesce(a.not_asked, ''), a.asked_at, + b.id is not null, coalesce(b.failed, ''), coalesce(b.module, ''), coalesce(b.at, a.asked_at) + from build_request a left join build b on b.id = a.id + order by a.asked_at desc, a.id desc`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []RequestOutcome + for rows.Next() { + var a RequestOutcome + if err := rows.Scan(&a.ID, &a.Repository, &a.Seat, &a.Path, &a.Ref, &a.Commit, &a.For, &a.NotAsked, + &a.At, &a.Heard, &a.Failed, &a.Module, &a.HeardAt); err != nil { + return nil, err + } + out = append(out, a) + } + return out, rows.Err() +} + +// The states of a pending assignment. +const ( + PendingWaiting = "waiting" + PendingApplied = "applied" + PendingRefused = "refused" + PendingExpired = "expired" + PendingWithdrawn = "withdrawn" +) + +// PendingAssignment is an assignment kept until its module is registered. +type PendingAssignment struct { + ID int64 + Node string + Module string + // Build is the build it waits for; Repository and Path where that build is from. + Build, Repository, Path string + Since time.Time + State string + // Note is what ended it, in the words said when it ended. + Note string + Settled *time.Time +} + +// ErrAlreadyPending is a second pending assignment of one module to one machine. +var ErrAlreadyPending = errors.New("that assignment is already waiting for its build") + +// RecordPending keeps an assignment until its module is registered. One waits per machine and module. +func (i *Inventory) RecordPending(ctx context.Context, p PendingAssignment) (PendingAssignment, error) { + if _, err := i.NodeByName(ctx, p.Node); err != nil { + return PendingAssignment{}, err + } + err := i.store.Pool().QueryRow(ctx, + `insert into pending_assignment (node, module, build_id, repository, source_path) + values ($1, $2, $3, $4, $5) + on conflict (node, module) where state = 'waiting' do nothing + returning id, since`, + p.Node, p.Module, p.Build, p.Repository, strings.Trim(p.Path, "/")).Scan(&p.ID, &p.Since) + if errors.Is(err, pgx.ErrNoRows) { + return PendingAssignment{}, ErrAlreadyPending + } + if err != nil { + return PendingAssignment{}, err + } + p.State = PendingWaiting + return p, nil +} + +// Pending is every pending assignment still waiting, and every one ended since the time given, oldest +// first. A zero time is the waiting ones alone. +func (i *Inventory) Pending(ctx context.Context, endedSince time.Time) ([]PendingAssignment, error) { + rows, err := i.store.Pool().Query(ctx, + `select id, node, module, build_id, repository, source_path, since, state, note, settled_at + from pending_assignment + where state = 'waiting' or (settled_at is not null and settled_at >= $1) + order by since, id`, endedSince) + if err != nil { + return nil, err + } + defer rows.Close() + var out []PendingAssignment + for rows.Next() { + var p PendingAssignment + if err := rows.Scan(&p.ID, &p.Node, &p.Module, &p.Build, &p.Repository, &p.Path, &p.Since, &p.State, + &p.Note, &p.Settled); err != nil { + return nil, err + } + if endedSince.IsZero() && p.State != PendingWaiting { + continue + } + out = append(out, p) + } + return out, rows.Err() +} + +// SettlePending ends a waiting pending assignment. False when it was no longer waiting: another process +// settled it first, and its word stands. +func (i *Inventory) SettlePending(ctx context.Context, id int64, state, note string) (bool, error) { + if state == PendingWaiting { + return false, errors.New("a pending assignment is settled to applied, refused, expired or withdrawn") + } + tag, err := i.store.Pool().Exec(ctx, + `update pending_assignment set state = $2, note = $3, settled_at = now() + where id = $1 and state = 'waiting'`, id, state, note) + if err != nil { + return false, err + } + return tag.RowsAffected() == 1, nil +}