Archived
Compare commits
26
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fe2ee39365 | ||
|
|
d24132d54f | ||
|
|
04730b54a8 | ||
|
|
45870ab31c | ||
|
|
fd9308d328 | ||
|
|
3479a35462 | ||
|
|
cebc593627 | ||
|
|
2959c81435 | ||
|
|
8933864a49 | ||
|
|
df9e5728a9 | ||
|
|
2d18f438c5 | ||
|
|
3ce5a0239b | ||
|
|
70885ab3c4 | ||
|
|
65508d9df9 | ||
|
|
8123ca5654 | ||
|
|
d9b86376d3 | ||
|
|
ca3f43c400 | ||
|
|
b482122d66 | ||
|
|
2f5cc146fe | ||
|
|
c2ea9b4c51 | ||
|
|
b03d257f81 | ||
|
|
1601cb1743 | ||
|
|
873b19566a | ||
|
|
b0753e2aac | ||
|
|
a69c0e50dc | ||
|
|
b1546ed28a |
@@ -0,0 +1,146 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **The build agent says whether it is busy, and being asked drains it** (novox/hq ADR 0335 §1, §5). Its
|
||||
// container declares `busy` as `mesh-builder busy`: the node-engine runs it inside the container before it
|
||||
// replaces it. It exits 0 while a build is in hand and 1 when none is; and from the first ask the agent takes
|
||||
// no new build request, so its answer ends once the build in hand ends — a replaced builder finishes the check
|
||||
// it runs, instead of the build queue handing it out again from the start (issue 594).
|
||||
//
|
||||
// Two marks in the container's own temporary directory, as the ready mark (ready.go): a new container starts
|
||||
// without them, and a restarted one removes them before it takes work again.
|
||||
//
|
||||
// - in hand: written by the agent when it claims a request, before it reads whether it is drained; removed
|
||||
// when the build ends.
|
||||
// - drained: written by the ask, before it reads whether a build is in hand.
|
||||
//
|
||||
// Each side writes its own mark before it reads the other's, so an ask racing a fetch sees the build in hand,
|
||||
// or the agent sees the drain and hands the request back; never neither.
|
||||
|
||||
func inHandMark() string { return filepath.Join(os.TempDir(), "mesh-builder.in-hand") }
|
||||
func drainMark() string { return filepath.Join(os.TempDir(), "mesh-builder.drained") }
|
||||
|
||||
// The drain lapses when no ask has come for drainLapse: the node-engine asks every 20 seconds for as long as it
|
||||
// really waits, so an agent whose replacement was withdrawn — the declaration put back, the walk stopped —
|
||||
// takes work again rather than staying drained for ever (review of ADR 0335). And an agent drained for longer
|
||||
// than the node-engine's bound and the slack says it is not ready: the replacement it was drained for never
|
||||
// came, which is a fault to see.
|
||||
const (
|
||||
drainLapse = 2 * time.Minute
|
||||
drainTooLong = 15*time.Minute + 5*time.Minute
|
||||
)
|
||||
|
||||
// drainedAt is when the drain began and when it was last asked, from its mark; false when there is no mark.
|
||||
func drainedAt() (began, asked time.Time, ok bool) {
|
||||
info, err := os.Stat(drainMark())
|
||||
if err != nil {
|
||||
return time.Time{}, time.Time{}, false
|
||||
}
|
||||
asked = info.ModTime()
|
||||
began = asked
|
||||
if raw, err := os.ReadFile(drainMark()); err == nil {
|
||||
if t, err := time.Parse(time.RFC3339, strings.TrimSpace(string(raw))); err == nil && !t.After(asked) {
|
||||
began = t
|
||||
}
|
||||
}
|
||||
return began, asked, true
|
||||
}
|
||||
|
||||
// drained says the agent was asked whether it is busy within drainLapse: it takes no new build request.
|
||||
func drained() bool { return drainedSince(time.Now()) }
|
||||
|
||||
func drainedSince(now time.Time) bool {
|
||||
_, asked, ok := drainedAt()
|
||||
return ok && now.Sub(asked) < drainLapse
|
||||
}
|
||||
|
||||
// drainedTooLong is the health check's second half: nil unless the agent has been drained, asked all along, for
|
||||
// longer than its node-engine's bound and the slack.
|
||||
func drainedTooLong(now time.Time) error {
|
||||
began, asked, ok := drainedAt()
|
||||
if !ok || now.Sub(asked) >= drainLapse || now.Sub(began) <= drainTooLong {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("drained since %s and still asked: the replacement its node-engine waits for has not come "+
|
||||
"within %s (novox/hq ADR 0335)", began.UTC().Format(time.RFC3339), drainTooLong)
|
||||
}
|
||||
|
||||
// claim marks a request in hand, unless the agent is drained: then nothing is in hand, and the request goes
|
||||
// back to the queue.
|
||||
func claim() bool {
|
||||
// Made, never written over: one build at a time, and a mark already there is a build in hand.
|
||||
f, err := os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
||||
if err != nil {
|
||||
// Unmarked, an ask would say idle while this build runs: the build is not taken.
|
||||
fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err)
|
||||
return false
|
||||
}
|
||||
_, err = f.WriteString(time.Now().UTC().Format(time.RFC3339) + "\n")
|
||||
if closeErr := f.Close(); err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
if err != nil {
|
||||
_ = os.Remove(inHandMark())
|
||||
fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err)
|
||||
return false
|
||||
}
|
||||
if drained() {
|
||||
_ = os.Remove(inHandMark())
|
||||
fmt.Fprintln(os.Stderr, "asked whether busy, so this agent takes no new build: handed back")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// holding says what is in hand, for the ask to print: written once the request is read.
|
||||
func holding(what string) {
|
||||
_ = os.WriteFile(inHandMark(), []byte(time.Now().UTC().Format(time.RFC3339)+" "+what+"\n"), 0o644)
|
||||
}
|
||||
|
||||
// released ends the build in hand.
|
||||
func released() {
|
||||
if err := os.Remove(inHandMark()); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
fmt.Fprintf(os.Stderr, "cannot remove the mark of the build in hand (%s): %v\n", inHandMark(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// undrained clears both marks at the start of a holder: a container started again is asked afresh.
|
||||
func undrained() error {
|
||||
for _, mark := range []string{drainMark(), inHandMark()} {
|
||||
if err := os.Remove(mark); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("cannot remove %s: %w", mark, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// busy is the ask: it drains the agent, then says whether a build is in hand. Nil — exit 0 — while one is.
|
||||
func busy() (string, error) {
|
||||
// The mark keeps when the drain began while it is asked within drainLapse, and is written again on every
|
||||
// ask: its time is the last ask.
|
||||
began := time.Now().UTC()
|
||||
if was, _, ok := drainedAt(); ok && drained() {
|
||||
began = was.UTC()
|
||||
}
|
||||
if err := os.WriteFile(drainMark(), []byte(began.Format(time.RFC3339)+"\n"), 0o644); err != nil {
|
||||
// Not drained, it could take the next build: said, and answered busy only for what is in hand now.
|
||||
fmt.Fprintf(os.Stderr, "cannot mark this agent drained (%s): %v\n", drainMark(), err)
|
||||
}
|
||||
held, err := os.ReadFile(inHandMark())
|
||||
if err != nil {
|
||||
return "", errors.New("idle: no build in hand, and this agent takes no new one")
|
||||
}
|
||||
since, what, _ := strings.Cut(strings.TrimSpace(string(held)), " ")
|
||||
if what == "" {
|
||||
what = "a build"
|
||||
}
|
||||
return fmt.Sprintf("busy: %s, in hand since %s; taking no new build", what, since), nil
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// `mesh-builder busy` answers 0 while a build is in hand and non-zero when idle, and after its first ask the
|
||||
// agent takes no new build request (novox/hq ADR 0335 §5).
|
||||
func TestBusyAnswersWhileABuildIsInHandAndDrainsTheAgent(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if drained() {
|
||||
t.Fatal("an agent never asked is drained")
|
||||
}
|
||||
// A build is claimed and runs.
|
||||
if !claim() {
|
||||
t.Fatal("an agent never asked refused a build")
|
||||
}
|
||||
holding("novox/mesh-controller (check-280)")
|
||||
said, err := busy()
|
||||
if err != nil || !strings.Contains(said, "novox/mesh-controller (check-280)") {
|
||||
t.Fatalf("an agent with a build in hand did not answer busy with what it holds: %q %v", said, err)
|
||||
}
|
||||
// From that ask on it takes nothing new: the fetch is not made, and one fetched is handed back.
|
||||
if !drained() {
|
||||
t.Fatal("the ask did not drain the agent")
|
||||
}
|
||||
if claim() {
|
||||
t.Fatal("a drained agent took a new build")
|
||||
}
|
||||
// The build in hand is still in hand after the refused claim, and busy goes on saying so.
|
||||
if _, err := busy(); err != nil {
|
||||
t.Fatalf("a refused claim ended the build in hand: %v", err)
|
||||
}
|
||||
// It ends: idle, and still drained.
|
||||
released()
|
||||
if _, err := busy(); err == nil {
|
||||
t.Fatal("an agent with nothing in hand answered busy")
|
||||
}
|
||||
if claim() {
|
||||
t.Fatal("a drained agent took a build after its last one ended")
|
||||
}
|
||||
// Started again, it is asked afresh.
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if drained() || !claim() {
|
||||
t.Fatal("a holder started again is still drained")
|
||||
}
|
||||
released()
|
||||
}
|
||||
|
||||
// An ask racing a fetch sees the build in hand, or the agent sees the drain and hands the request back: never
|
||||
// neither, which would replace the container under a build just taken.
|
||||
func TestAnAskRacingAFetchNeverMissesTheBuild(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
for i := 0; i < 200; i++ {
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var claimed bool
|
||||
var askErr error
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() { defer wg.Done(); claimed = claim() }()
|
||||
go func() { defer wg.Done(); _, askErr = busy() }()
|
||||
wg.Wait()
|
||||
if claimed && askErr != nil {
|
||||
t.Fatalf("round %d: the agent took a build and the ask said idle", i)
|
||||
}
|
||||
released()
|
||||
}
|
||||
}
|
||||
|
||||
// A drain no longer asked lapses: an agent whose replacement was withdrawn after the first ask (a put-back, a
|
||||
// revert, a stopped walk) takes work again after drainLapse, rather than staying drained with its health green.
|
||||
func TestADrainNoLongerAskedLapses(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := busy(); err == nil {
|
||||
t.Fatal("an idle agent answered busy")
|
||||
}
|
||||
if !drained() || claim() {
|
||||
t.Fatal("an agent just asked takes work")
|
||||
}
|
||||
// The last ask was three minutes ago.
|
||||
old := time.Now().Add(-3 * time.Minute)
|
||||
if err := os.Chtimes(drainMark(), old, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if drained() {
|
||||
t.Fatal("a drain no longer asked for three minutes still holds")
|
||||
}
|
||||
if !claim() {
|
||||
t.Fatal("an agent whose drain lapsed takes no work")
|
||||
}
|
||||
released()
|
||||
}
|
||||
|
||||
// An agent drained, and asked all along, for longer than its node-engine's bound and the slack is not ready: the
|
||||
// replacement it was drained for never came. One asked again after a lapse starts a new drain.
|
||||
func TestAnAgentDrainedTooLongIsNotReady(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
taking("mesh-build-machine")
|
||||
if _, err := busy(); err == nil {
|
||||
t.Fatal("an idle agent answered busy")
|
||||
}
|
||||
now := time.Now()
|
||||
if err := drainedTooLong(now); err != nil {
|
||||
t.Fatalf("an agent drained a moment ago is not ready: %v", err)
|
||||
}
|
||||
// The drain began 25 minutes ago, and the node-engine asked a moment ago.
|
||||
began := now.Add(-25 * time.Minute).UTC().Format(time.RFC3339)
|
||||
if err := os.WriteFile(drainMark(), []byte(began+"\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := drainedTooLong(time.Now())
|
||||
if err == nil || !strings.Contains(err.Error(), "has not come") {
|
||||
t.Fatalf("an agent drained for 25 minutes reads ready: %v", err)
|
||||
}
|
||||
// Each ask keeps when the drain began.
|
||||
if _, err := busy(); err == nil {
|
||||
t.Fatal("an idle agent answered busy")
|
||||
}
|
||||
if err := drainedTooLong(time.Now()); err == nil {
|
||||
t.Fatal("an ask started the drain again")
|
||||
}
|
||||
// The asks stopped: the drain lapsed, and so did the fault.
|
||||
old := time.Now().Add(-3 * time.Minute)
|
||||
if err := os.Chtimes(drainMark(), old, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := drainedTooLong(time.Now()); err != nil {
|
||||
t.Fatalf("a lapsed drain still reads not ready: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// An ask this build does not know exits non-zero: an older image asked `busy` reads idle, not busy for the
|
||||
// node-engine's whole bound.
|
||||
func TestAnUnknownCommandFails(t *testing.T) {
|
||||
was := os.Args
|
||||
t.Cleanup(func() { os.Args = was })
|
||||
os.Args = []string{"mesh-builder", "some-later-word"}
|
||||
if err := run(); err == nil {
|
||||
t.Fatal("an unknown command exited 0")
|
||||
}
|
||||
os.Args = []string{"mesh-builder", "help"}
|
||||
if err := run(); err != nil {
|
||||
t.Fatalf("help failed: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -75,6 +75,12 @@ digest of their own configuration. The result is printed as JSON.
|
||||
|
||||
exits 0 once the agent running in this container is bound to its build work and pulls it, and
|
||||
1 before: its container's health check.
|
||||
|
||||
mesh-builder busy
|
||||
|
||||
exits 0 while the agent running in this container has a build in hand, and 1 when it has none;
|
||||
from the first ask the agent takes no new build: its container's busy command, asked before the
|
||||
container is replaced (novox/hq ADR 0335).
|
||||
`
|
||||
|
||||
func run() error {
|
||||
@@ -86,11 +92,29 @@ func run() error {
|
||||
case "build":
|
||||
return buildOnce(context.Background(), os.Args[2:])
|
||||
case "ready":
|
||||
// The container's health check (novox/hq issue 544): exits non-zero until the agent takes work.
|
||||
return ready()
|
||||
default:
|
||||
// The container's health check (novox/hq issue 544): exits non-zero until the agent takes work, and
|
||||
// once it has been drained longer than its node-engine would wait to replace it (ADR 0335).
|
||||
if err := ready(); err != nil {
|
||||
return err
|
||||
}
|
||||
return drainedTooLong(time.Now())
|
||||
case "busy":
|
||||
// The container's busy command (novox/hq ADR 0335): drains the agent, and exits non-zero once
|
||||
// nothing is in hand.
|
||||
said, err := busy()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(said)
|
||||
return nil
|
||||
case "help", "-h", "--help":
|
||||
fmt.Print(usage)
|
||||
return nil
|
||||
default:
|
||||
// **An ask this build does not know fails** (novox/hq ADR 0335): an image older than `busy`, asked
|
||||
// it, must read as idle — exiting 0 would hold its container for the node-engine's whole bound.
|
||||
fmt.Fprint(os.Stderr, usage)
|
||||
return fmt.Errorf("%q is not a command this mesh-builder knows", os.Args[1])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,6 +124,9 @@ func run() error {
|
||||
// **The builder's own version is part of every reuse key** (novox/hq ADR 0350 §1): a builder that builds
|
||||
// differently never takes a build another one made.
|
||||
builder.Version = builder.SelfVersion(version)
|
||||
if err := undrained(); err != nil {
|
||||
return err
|
||||
}
|
||||
credential, err := brokerFrom()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -152,8 +179,13 @@ func run() error {
|
||||
// **What an earlier holder here left goes before anything is taken** (novox/hq issue 306): a holder
|
||||
// recreated mid-check left the check's store and bus running, and the ask went to another machine.
|
||||
h.removeLeftBehind()
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused,
|
||||
Taking: func() { taking(seat) }})
|
||||
// Paused or drained, it takes nothing new (novox/hq ADR 0219, ADR 0335): a drained agent finishes the build
|
||||
// in hand and then says idle, and its container is replaced between builds.
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{
|
||||
Paused: func() bool { return h.Paused() || drained() },
|
||||
Taking: func() { taking(seat) },
|
||||
Claim: claim,
|
||||
})
|
||||
defer machine.Close()
|
||||
|
||||
// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
|
||||
@@ -182,6 +214,8 @@ func run() error {
|
||||
go h.expireChecks(ctx, time.Hour, time.Now)
|
||||
|
||||
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
|
||||
defer released()
|
||||
holding(work.Request().Repository + " (" + work.Request().ID + ")")
|
||||
answer(ctx, publisher, on, workspace, work, h)
|
||||
})
|
||||
}
|
||||
@@ -298,6 +332,12 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo,
|
||||
Commit: m.Head, Trunk: m.Trunk})
|
||||
}
|
||||
// A branch tried before its pull request reads its outcome from this log, whole (novox/hq issue 686).
|
||||
if request.Check.Try {
|
||||
if line, err := json.Marshal(triedOf(v)); err == nil {
|
||||
say(link.StepTried, string(line))
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if err == nil {
|
||||
// The package-registry credential is a build input, so it is resolved before the clone: a
|
||||
@@ -423,7 +463,8 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
|
||||
spec := builder.CheckSpec{ID: request.ID, Repository: request.Repository, Ref: request.Ref,
|
||||
Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{},
|
||||
Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base,
|
||||
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held)}
|
||||
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held),
|
||||
Tests: c.Tests, StoreOnly: c.StoreOnly}
|
||||
for dir, b := range c.Beside {
|
||||
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref}
|
||||
}
|
||||
@@ -438,6 +479,23 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
|
||||
return spec
|
||||
}
|
||||
|
||||
// triedOf is a try's outcome as its log line says it (novox/hq issue 686): the repository's own check's verdict,
|
||||
// or the check's where that did not run — a head that does not merge — and the lines naming what failed.
|
||||
func triedOf(v builder.CheckVerdict) link.Tried {
|
||||
t := link.Tried{Verdict: v.Verdict, Summary: v.Summary, Took: v.Took.Round(time.Second).String(),
|
||||
Head: v.Head, Trunk: v.Trunk}
|
||||
if v.Repo != nil && (v.Gate == nil || v.Gate.Verdict == "pass") {
|
||||
t.Verdict, t.Summary = v.Repo.Verdict, v.Repo.Summary
|
||||
}
|
||||
if v.Repo != nil && v.Repo.Failed != "" {
|
||||
t.Failed = strings.Split(v.Repo.Failed, "\n")
|
||||
}
|
||||
if t.Verdict == "" {
|
||||
t.Verdict = "error"
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// checkBuiltOf is a check's builds for its merge as the outcome carries them (novox/hq ADR 0350 §1).
|
||||
func checkBuiltOf(built []builder.CheckBuilt) []link.CheckBuilt {
|
||||
var out []link.CheckBuilt
|
||||
|
||||
@@ -16,6 +16,7 @@ func TestBuilderDiagnosticsStayOffStdout(t *testing.T) {
|
||||
"version)": true, // --version
|
||||
`"stopping")`: true, // the loop.s shutdown line
|
||||
"usage)": true, // --help text, for a human
|
||||
"said)": true, // busy: the answer its node-engine reads (novox/hq ADR 0335)
|
||||
}
|
||||
for _, file := range []string{"once.go", "main.go"} {
|
||||
src, err := os.ReadFile(file)
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
)
|
||||
|
||||
// A try's outcome, as the build seat says it in the build's log (novox/hq issue 686): the repository's own
|
||||
// check's verdict, how long it took, and each line naming what failed.
|
||||
func TestATrysOutcomeNamesWhatFailed(t *testing.T) {
|
||||
tried := triedOf(builder.CheckVerdict{Verdict: "pass", Summary: "the change touches no module", Took: 303 * time.Second,
|
||||
Head: "c0ffee", Trunk: "ca3f43c", Gate: &builder.Layer{Verdict: "pass"},
|
||||
Repo: &builder.Layer{Verdict: "fail", Summary: "its merge-check.sh failed: TestX",
|
||||
Failed: "--- FAIL: TestX (0.01s)\n x_test.go:12: got 1, want 2"}})
|
||||
if tried.Verdict != "fail" || tried.Took != "5m3s" || len(tried.Failed) != 2 || tried.Trunk != "ca3f43c" ||
|
||||
tried.Summary != "its merge-check.sh failed: TestX" {
|
||||
t.Fatalf("a failed try: %+v", tried)
|
||||
}
|
||||
// A head that does not merge fails both layers: the check's own word.
|
||||
conflict := triedOf(builder.CheckVerdict{Verdict: "fail", Summary: "does not merge", Gate: &builder.Layer{Verdict: "fail"},
|
||||
Repo: &builder.Layer{Verdict: "fail", Summary: "does not merge"}})
|
||||
if conflict.Verdict != "fail" || conflict.Summary != "does not merge" {
|
||||
t.Fatalf("a try that does not merge: %+v", conflict)
|
||||
}
|
||||
// No verdict is an error, never a pass.
|
||||
if none := triedOf(builder.CheckVerdict{}); none.Verdict != "error" {
|
||||
t.Fatalf("a try with no verdict: %+v", none)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **Every user's slow consumer is S9's, not the controller's alone** (novox/hq ADR 0357, issue 270). On
|
||||
// 2026-10-07 the bus closed two clients on the laptop as slow consumers and nothing raised a condition:
|
||||
// the controller heard only its own connection. Said by the bus module and read back, a user the bus
|
||||
// closes is one condition naming the module and its machine, raised at the first look and cleared an hour
|
||||
// after the last closure.
|
||||
func TestAUserTheBusClosedAsASlowConsumerIsOneConditionNamingItsModuleAndMachine(t *testing.T) {
|
||||
closed := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
|
||||
now := closed.Add(20 * time.Second)
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||
t.Cleanup(func() { k.Close(t.Context()) })
|
||||
|
||||
advisories := link.BusClosures([]link.BusClosure{
|
||||
{User: "laptop.claude-code", CID: 10, Reason: "Slow Consumer (Write Deadline)", Opened: closed.Add(-time.Hour),
|
||||
Closed: closed.Add(-10 * time.Minute)},
|
||||
{User: "laptop.claude-code", CID: 44, Reason: "Slow Consumer (Write Deadline)", Opened: closed.Add(-time.Minute),
|
||||
Closed: closed},
|
||||
{User: "anchor.desk", CID: 7, Reason: "Maximum Message Payload Exceeded", Opened: closed.Add(-time.Hour),
|
||||
Closed: closed},
|
||||
}, []string{link.AdvisorySlowConsumer, link.AdvisorySlowConsumer, link.AdvisoryConnectionClosed}, closed.Add(-time.Hour))
|
||||
look := func() {
|
||||
t.Helper()
|
||||
f := &signalFacts{now: now, host: "anchor", advisories: advisories}
|
||||
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
look()
|
||||
|
||||
slow, found, err := conditions.ReadOne(t.Context(), store, "bus.laptop.claude-code.slow-consumer")
|
||||
if err != nil || !found {
|
||||
t.Fatalf("no slow-consumer condition for the user: %v %v", found, err)
|
||||
}
|
||||
if slow.Subject.Machine != "laptop" || slow.Observations != 2 || !slow.LastObserved.Equal(closed) {
|
||||
t.Fatalf("the slow consumer is said on %q, %d closure(s), last %s: %+v", slow.Subject.Machine, slow.Observations,
|
||||
slow.LastObserved, slow)
|
||||
}
|
||||
if !strings.Contains(slow.Summary, "claude-code on laptop") || strings.Contains(slow.Summary, "laptop.claude-code") {
|
||||
t.Fatalf("the summary does not name the module in words, or carries the dotted user: %q", slow.Summary)
|
||||
}
|
||||
if !strings.Contains(slow.Headline, "Claude-code on laptop") {
|
||||
t.Fatalf("the headline is %q", slow.Headline)
|
||||
}
|
||||
cut, found, err := conditions.ReadOne(t.Context(), store, "bus.anchor.desk.connection-closed")
|
||||
if err != nil || !found || cut.Subject.Machine != "anchor" || !strings.Contains(cut.Summary, "maximum message payload exceeded") {
|
||||
t.Fatalf("the user cut off for breaking the bus's rules: %v %v %+v", found, err, cut)
|
||||
}
|
||||
|
||||
// An hour after the newest closure without another, both clear.
|
||||
now = closed.Add(time.Hour + time.Minute)
|
||||
look()
|
||||
for _, key := range []string{"bus.laptop.claude-code.slow-consumer", "bus.anchor.desk.connection-closed"} {
|
||||
if _, found, _ := k.Get(t.Context(), key); found {
|
||||
t.Fatalf("%s is still open an hour after the last closure", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A read of the bus module's events that fails does not blind S9** (review of #326): the controller's own
|
||||
// signals are still said, and the failed read is a condition of its own.
|
||||
func TestABusModuleReadThatFailsKeepsTheControllersOwnSignals(t *testing.T) {
|
||||
at := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
|
||||
f := &signalFacts{now: at.Add(time.Minute), host: "anchor",
|
||||
advisories: []link.Advisory{{Kind: link.AdvisorySlowConsumer, ID: "controller", Said: "the controller fell behind",
|
||||
First: at, Last: at, Count: 1}},
|
||||
busFaultsErr: errors.New("nats: timeout")}
|
||||
keys := map[string]bool{}
|
||||
for _, o := range watchAdvisories(f) {
|
||||
keys[o.Key()] = true
|
||||
}
|
||||
if !keys["bus.controller.slow-consumer"] || !keys["bus.bus-module.bus-faults-unread"] {
|
||||
t.Fatalf("observed %v: the controller's own slow consumer and the unread events, both", keys)
|
||||
}
|
||||
for _, row := range signalsTable {
|
||||
if row.Row == "S9" && row.needs(f) != nil {
|
||||
t.Fatal("the S9 row went blind for a read of the bus module's events")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A dead watch is heard** (review 274): the bus module's watch silent for longer than three looks, or saying
|
||||
// it cannot see, is a condition on the bus's machine — never read as no slow consumers. A stream with no
|
||||
// heartbeat at all is a bus module older than its watch, and says nothing.
|
||||
func TestTheBusWatchSilentOrBlindIsAConditionAndNoHeartbeatAtAllIsNone(t *testing.T) {
|
||||
now := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
|
||||
look := func(b link.BusWatchBeat, heard bool) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, o := range watchAdvisories(&signalFacts{now: now, busWatch: b, busWatchHeard: heard}) {
|
||||
if o.Kind == kindBusWatchDead {
|
||||
out = append(out, o)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
if got := look(link.BusWatchBeat{}, false); len(got) != 0 {
|
||||
t.Fatalf("no heartbeat ever raised %+v", got)
|
||||
}
|
||||
if got := look(link.BusWatchBeat{At: now.Add(-30 * time.Second), Healthy: true, Machine: "anchor"}, true); len(got) != 0 {
|
||||
t.Fatalf("a healthy watch heard 30s ago raised %+v", got)
|
||||
}
|
||||
silent := look(link.BusWatchBeat{At: now.Add(-5 * time.Minute), Healthy: true, Machine: "anchor"}, true)
|
||||
if len(silent) != 1 || silent[0].Machine != "anchor" || !strings.Contains(silent[0].Summary, "said nothing") {
|
||||
t.Fatalf("a watch silent for 5 minutes raised %+v", silent)
|
||||
}
|
||||
blind := look(link.BusWatchBeat{At: now.Add(-10 * time.Second), Healthy: false, Machine: "anchor",
|
||||
Why: "the watch of slow consumers last read the bus's closed connections 95s ago: connection refused"}, true)
|
||||
if len(blind) != 1 || !strings.Contains(blind[0].Said, "connection refused") {
|
||||
t.Fatalf("a watch that says it cannot see raised %+v", blind)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A first-node gate waits for a busy container (novox/hq ADR 0335 §4).
|
||||
//
|
||||
// When a machine reports, on what the gate sent it, that a container of a judged module was left running
|
||||
// because it said busy, the judging has not started: the build is not yet what runs there. The walk says it
|
||||
// waits for a busy container on that machine, counts no pass and no bound, and starts the judging — its
|
||||
// Since — at the report that no longer lists the container waiting. The wait is the node-engine's, bounded
|
||||
// at 15 minutes; the controller's own bound (busyGateBound past the engine's) is for an engine that stopped
|
||||
// reporting while it waited, so no walk waits for ever. An engine older than the field never lists one, and
|
||||
// its gate starts as before.
|
||||
|
||||
// busyGateSlack is how long past the engine's bound the controller still waits for the report that says the
|
||||
// container replaced: a pass of the engine (its asks every 20 seconds) and the report's way here.
|
||||
var busyGateSlack = 5 * time.Minute
|
||||
|
||||
// busyEngineBound is the node-engine's bound on a wait (novox/hq ADR 0335 §3), counted again here on this
|
||||
// controller's clock from when it first read the wait.
|
||||
const busyEngineBound = 15 * time.Minute
|
||||
|
||||
// busyGateEnds is when the gate stops waiting: the engine's own word on its bound, or this controller's count of
|
||||
// it from when it first read the wait, whichever is sooner, and the slack — so neither a clock ahead on the
|
||||
// machine nor a wrong word from it holds a walk longer.
|
||||
func busyGateEnds(b *inventory.GateBusy) time.Time {
|
||||
until := b.Seen.Add(busyEngineBound)
|
||||
if !b.Until.IsZero() && b.Until.Before(until) {
|
||||
until = b.Until
|
||||
}
|
||||
return until.Add(busyGateSlack)
|
||||
}
|
||||
|
||||
// busyWait decides, before a judging, whether the gate waits for a busy container on a machine it judges:
|
||||
// true while it waits, the gate saying so. False when nothing waits, or no longer: a wait that ended moves the
|
||||
// gate's Since to its end. Pure but for the gate it is handed.
|
||||
func busyWait(g *inventory.PlanGate, pairs []judged, f gateFacts, now time.Time) bool {
|
||||
if g.Busy != nil && g.Busy.Ended != nil {
|
||||
return false
|
||||
}
|
||||
for _, j := range pairs {
|
||||
b, waits := busyOn(j, f)
|
||||
if !waits {
|
||||
continue
|
||||
}
|
||||
if g.Busy == nil {
|
||||
g.Busy = &inventory.GateBusy{Machine: j.node, Module: j.module, Container: b.Container, Since: b.Since,
|
||||
Seen: now}
|
||||
}
|
||||
if g.Busy.Seen.IsZero() {
|
||||
g.Busy.Seen = now
|
||||
}
|
||||
g.Busy.Until, g.Busy.Said = b.Until, b.Said
|
||||
if now.After(busyGateEnds(g.Busy)) {
|
||||
// The engine said it would have replaced it by now: the judging starts, and finds the module not
|
||||
// yet healthy while the machine still lists it waiting (busyStill), until the gate's own bound.
|
||||
end := now
|
||||
g.Busy.Ended, g.Busy.PastBound = &end, true
|
||||
g.Since, g.Passes, g.LastPass, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, nil, nil, nil
|
||||
return false
|
||||
}
|
||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, g.Busy.Words(), nil
|
||||
g.Read(now, false, g.Busy.Words())
|
||||
return true
|
||||
}
|
||||
if g.Busy == nil {
|
||||
return false
|
||||
}
|
||||
// Replaced: the judging starts at the report that said so.
|
||||
end := now
|
||||
if r, said := f.reports[g.Busy.Machine]; said && r.At != nil && r.At.After(g.Busy.Since) && r.At.Before(now) {
|
||||
end = r.At.UTC()
|
||||
}
|
||||
g.Busy.Ended = &end
|
||||
g.Since, g.Passes, g.LastPass, g.Last, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, "", nil, nil, nil
|
||||
return false
|
||||
}
|
||||
|
||||
// busyOn is the container of a judged module its machine last reported, on what the gate sent it, as left
|
||||
// running because it said busy.
|
||||
func busyOn(j judged, f gateFacts) (inventory.BusyWait, bool) {
|
||||
r, said := f.reports[j.node]
|
||||
if !said || r.At == nil || !f.reportedOn(j.node, r) {
|
||||
return inventory.BusyWait{}, false
|
||||
}
|
||||
return inventory.WaitingFor(r.Busy, j.module)
|
||||
}
|
||||
|
||||
// busyStill is a judged module whose container its machine still lists waiting once the gate stopped waiting
|
||||
// for it: not yet what was sent, so not healthy, whatever its health says of the container still running.
|
||||
func busyStill(j judged, f gateFacts) (string, bool) {
|
||||
b, waits := busyOn(j, f)
|
||||
if !waits {
|
||||
return "", false
|
||||
}
|
||||
return fmt.Sprintf("%s still runs its busy container %s, past its node-engine's bound of %s",
|
||||
j.node, b.Container, b.Until.UTC().Format(time.RFC3339)), true
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A walk whose module's container waits as busy on its first machine says so, and starts that machine's
|
||||
// first-machine gate only once the machine reports the container replaced (novox/hq ADR 0335 §4).
|
||||
func TestAGateWaitsForABusyContainerAndStartsWhenItIsReplaced(t *testing.T) {
|
||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent,
|
||||
Sent: map[string]inventory.SentDeclaration{"novox": {Digest: "d1", Sequence: 7}}}
|
||||
pairs := []judged{{module: "build-agent", node: "novox"}, {module: "mesh-controller", node: "novox"}}
|
||||
reported := sent.Add(20 * time.Second)
|
||||
waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||
State: inventory.BusyWaiting, Since: reported, Until: reported.Add(15 * time.Minute),
|
||||
Said: "busy: novox/mesh-controller (check of #280), in hand since 00:58:00Z"}}
|
||||
facts := func(at time.Time, busy []inventory.BusyWait) gateFacts {
|
||||
// As judgeMoves hands it: the report held against the gate's own send.
|
||||
return gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"novox": {Node: "novox",
|
||||
Outcome: inventory.OutcomeApplied, At: &at, Declared: "d1", ReportedSequence: 7, Busy: busy}}}
|
||||
}
|
||||
|
||||
now := reported.Add(time.Minute)
|
||||
if !busyWait(g, pairs, facts(reported, waiting), now) {
|
||||
t.Fatal("a gate whose module's container waits as busy judged it")
|
||||
}
|
||||
if !g.Since.Equal(sent) || g.Busy == nil || g.Busy.Machine != "novox" || g.Busy.Ended != nil {
|
||||
t.Fatalf("the gate does not keep the wait: %+v %+v", g.Since, g.Busy)
|
||||
}
|
||||
for _, said := range []string{gateLine(g), g.Last} {
|
||||
if !strings.Contains(said, "waits for a busy container on novox") {
|
||||
t.Errorf("the walk does not say it waits for a busy container there: %s", said)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(g.Last, "check of #280") {
|
||||
t.Errorf("the wait does not say what the container holds: %s", g.Last)
|
||||
}
|
||||
|
||||
// Ten minutes on, still busy, within the engine's bound: still waiting, nothing judged, no bound run.
|
||||
now = reported.Add(10 * time.Minute)
|
||||
if !busyWait(g, pairs, facts(reported.Add(9*time.Minute), waiting), now) || g.Passes != 0 {
|
||||
t.Fatalf("the gate stopped waiting within the bound: %+v", g)
|
||||
}
|
||||
|
||||
// The machine reports it replaced: the judging starts at that report.
|
||||
replacedAt := reported.Add(12 * time.Minute)
|
||||
now = replacedAt.Add(5 * time.Second)
|
||||
replaced := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||
State: "replaced-when-idle", Since: reported, Until: reported.Add(15 * time.Minute)}}
|
||||
if busyWait(g, pairs, facts(replacedAt, replaced), now) {
|
||||
t.Fatal("the gate still waits after the container was replaced")
|
||||
}
|
||||
if g.Since == nil || !g.Since.Equal(replacedAt) || g.Busy.Ended == nil || !g.Busy.Ended.Equal(replacedAt) {
|
||||
t.Fatalf("the gate does not start at the report that said it replaced: since %v, %+v", g.Since, g.Busy)
|
||||
}
|
||||
if strings.Contains(gateLine(g), "waits for a busy container") {
|
||||
t.Errorf("a gate judging still says it waits: %s", gateLine(g))
|
||||
}
|
||||
// And a later report listing it waiting again does not hold a gate that started.
|
||||
if busyWait(g, pairs, facts(now, waiting), now.Add(time.Minute)) {
|
||||
t.Error("a gate that started waits again")
|
||||
}
|
||||
}
|
||||
|
||||
// An engine that stops reporting while it waits holds no walk for ever: past the engine's bound and the slack,
|
||||
// the judging starts, and finds the module not yet healthy while its container is still listed waiting.
|
||||
func TestAGateWaitsNoLongerThanTheEnginesBound(t *testing.T) {
|
||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
|
||||
pairs := []judged{{module: "build-agent", node: "novox"}}
|
||||
waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||
State: inventory.BusyWaiting, Since: sent, Until: sent.Add(15 * time.Minute)}}
|
||||
f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
|
||||
At: &sent, Current: true, Busy: waiting}}}
|
||||
now := sent.Add(15*time.Minute + busyGateSlack + time.Second)
|
||||
if busyWait(g, pairs, f, now) {
|
||||
t.Fatal("a gate waited past the engine's bound and the slack")
|
||||
}
|
||||
if !g.Busy.PastBound || !g.Since.Equal(now) {
|
||||
t.Fatalf("the gate does not start, said past the bound: %+v %v", g.Busy, g.Since)
|
||||
}
|
||||
if why, still := busyStill(pairs[0], f); !still || !strings.Contains(why, "past its node-engine's bound") {
|
||||
t.Fatalf("a module still running its busy container reads as what was sent: %q", why)
|
||||
}
|
||||
}
|
||||
|
||||
// A report from an engine older than the field lists nothing: the gate starts at the send, as before; and a
|
||||
// report not about the gate's send says nothing of it.
|
||||
func TestAGateOfAnEngineThatNeverWaitsStartsAsBefore(t *testing.T) {
|
||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent,
|
||||
Sent: map[string]inventory.SentDeclaration{"ace": {Digest: "d2", Sequence: 9}}}
|
||||
pairs := []judged{{module: "build-agent", node: "ace"}}
|
||||
at := sent.Add(time.Minute)
|
||||
older := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
|
||||
At: &at, Declared: "d2", ReportedSequence: 9}}}
|
||||
if busyWait(g, pairs, older, at) || g.Busy != nil || !g.Since.Equal(sent) {
|
||||
t.Fatalf("a gate on an engine that never waits did not start at the send: %+v %v", g.Busy, g.Since)
|
||||
}
|
||||
elsewhere := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
|
||||
At: &at, Declared: "d1", ReportedSequence: 8, Busy: []inventory.BusyWait{{ID: "build-agent.server",
|
||||
State: inventory.BusyWaiting, Until: at.Add(time.Hour)}}}}}
|
||||
if busyWait(g, pairs, elsewhere, at) {
|
||||
t.Fatal("a wait reported about an earlier declaration held the gate")
|
||||
}
|
||||
}
|
||||
|
||||
// The wait is bounded on this controller's clock: an engine whose clock runs an hour ahead, saying it waits until
|
||||
// then, holds the gate no longer than 15 minutes and the slack from when the controller first read the wait.
|
||||
func TestAGateWaitIsBoundedOnTheControllersClock(t *testing.T) {
|
||||
seen := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &seen}
|
||||
pairs := []judged{{module: "build-agent", node: "novox"}}
|
||||
ahead := seen.Add(time.Hour)
|
||||
f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
|
||||
At: &seen, Current: true, Busy: []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||
State: inventory.BusyWaiting, Since: ahead, Until: ahead.Add(15 * time.Minute)}}}}}
|
||||
if !busyWait(g, pairs, f, seen) || !g.Busy.Seen.Equal(seen) {
|
||||
t.Fatalf("the gate did not start waiting, seen now: %+v", g.Busy)
|
||||
}
|
||||
if !busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack-time.Second)) {
|
||||
t.Fatal("the gate stopped waiting within the bound on its own clock")
|
||||
}
|
||||
if busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack+time.Second)) || !g.Busy.PastBound {
|
||||
t.Fatalf("an engine's later word held the gate past the controller's bound: %+v", g.Busy)
|
||||
}
|
||||
}
|
||||
|
||||
// `plan` says, for a machine whose node-engine is older than the field, each container whose busy is not sent.
|
||||
func TestAPlanSaysABusyAnOlderEngineIsNotSent(t *testing.T) {
|
||||
modules := []catalogue.Manifest{{Module: "build-agent", Resources: []map[string]any{
|
||||
{"id": "server", "type": "container", "busy": map[string]any{"command": "mesh-builder busy"}},
|
||||
{"id": "agent-env", "type": "file"}}}}
|
||||
lines := busyNotSent(modules, false)
|
||||
if len(lines) != 1 || !strings.Contains(lines[0], "build-agent.server declares busy") ||
|
||||
!strings.Contains(lines[0], "replaced at once") {
|
||||
t.Fatalf("the plan does not say the busy not sent: %v", lines)
|
||||
}
|
||||
if lines := busyNotSent(modules, true); len(lines) != 0 {
|
||||
t.Fatalf("a busy sent is said as not sent: %v", lines)
|
||||
}
|
||||
}
|
||||
@@ -624,6 +624,11 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
facts.sentBuilds[j.node] = builds
|
||||
}
|
||||
}
|
||||
// **A busy container on a judged machine holds the judging before it starts** (novox/hq ADR 0335 §4): the
|
||||
// walk says it waits there, and its gate starts at the report that says the container replaced.
|
||||
if busyWait(g, pairs, facts, now) {
|
||||
return "", nil
|
||||
}
|
||||
// **What is wrong with a machine itself is the machine's** (novox/hq issue 281): read once for each
|
||||
// machine judged, apart from what is wrong with a module there, and never pinned on the module the
|
||||
// gate happens to be kept on.
|
||||
@@ -648,6 +653,9 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
for _, j := range pairs {
|
||||
w := words[j.node]
|
||||
h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, w.facts)
|
||||
if why, still := busyStill(j, facts); still && (h == healthGood || h == healthPerson) {
|
||||
h, said = healthNotYet, why
|
||||
}
|
||||
if h == healthGood || h == healthPerson {
|
||||
if on, named := w.on[j.module]; named {
|
||||
h, said = healthNotYet, on
|
||||
@@ -1435,6 +1443,12 @@ func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error
|
||||
}
|
||||
// A release held after a failed one, while builds still wait for a gate (ADR 0236).
|
||||
out = append(out, backlogObservation()...)
|
||||
// And a rest that went silent (novox/hq ADR 0353 §4).
|
||||
silent, err := silentObservation(ctx, d.open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, silent...)
|
||||
return sortedFound(dedupeObservations(out)), nil
|
||||
}
|
||||
|
||||
@@ -1524,6 +1538,15 @@ func gateLine(g *inventory.PlanGate) string {
|
||||
case inventory.GateFailed:
|
||||
what = "FAILED"
|
||||
}
|
||||
switch {
|
||||
case g.Verdict == "" && g.Pending:
|
||||
what = "kept, not yet sent" // novox/hq ADR 0353 §8
|
||||
case g.Verdict == "" && g.Silent:
|
||||
what = "silent on its send, left" // ADR 0353 §4
|
||||
}
|
||||
if g.Verdict == "" && g.Waiting() {
|
||||
what = "waits for a busy container on " + g.Busy.Machine
|
||||
}
|
||||
line := fmt.Sprintf("gate on %s: %s", strings.Join(g.Machines, ", "), what)
|
||||
if g.Component != "" {
|
||||
line += " (core: " + g.Component + ")"
|
||||
|
||||
@@ -151,7 +151,7 @@ func heldMachines(ctx context.Context, open *stores, names []string) (map[string
|
||||
for _, mv := range f.moves(node, modules, same, known, false) {
|
||||
if planStillToSend(plans, mv.Module, node) == "" {
|
||||
why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+
|
||||
"plan sends it, one machine at a time, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
|
||||
"plan sends it, one machine first and then the rest at once, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
|
||||
}
|
||||
}
|
||||
if len(why) > 0 {
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A tier whose module is no move — made from the source every machine running it runs, so nothing is
|
||||
// sent — is not waited on for a report after its marking (novox/hq issue 648). Its machines answered the
|
||||
// send that last carried that build before the walk got there, and nothing will make them report again:
|
||||
// the walk asks the next tier in the same step, rather than "waiting for app on anchor, laptop" for ever.
|
||||
func TestANoMoveTierWaitsForNoNewReport(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
|
||||
// app is the build machine, so a module of the next tier is built by it: the tier gates the next on app
|
||||
// running on its machines.
|
||||
for _, m := range []catalogue.Manifest{
|
||||
{Module: "app", Version: "c2", Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}},
|
||||
{Module: "web", Version: "w1"},
|
||||
} {
|
||||
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
||||
Path: "modules/" + m.Module, BuiltFrom: m.Version, Head: m.Version, Asked: time.Now().Add(-time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// What anchor and laptop were last sent (c1) was made from the source the plan's build of c2 is.
|
||||
for _, b := range []inventory.Build{
|
||||
{ID: "build-1s", Module: "app", Commit: "c1", SourceFingerprint: "src1:app",
|
||||
Asked: time.Now().Add(-30 * time.Second), At: time.Now().Add(-30 * time.Second)},
|
||||
{ID: "build-2s", Module: "app", Commit: "c2", SourceFingerprint: "src1:app", Asked: time.Now(), At: time.Now()},
|
||||
} {
|
||||
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Each machine answered that send, applied, before the walk's tier was built.
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.RecordDoing(ctx, nodeID(t, g.open, n), inventory.Doing{Node: n,
|
||||
Outcome: inventory.OutcomeApplied, Declared: "d-" + n + "-c1", Applied: 1,
|
||||
At: time.Now().Add(-10 * time.Second)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
built := time.Now().UTC()
|
||||
p := g.plan(t)
|
||||
p.Tiers = [][]string{{"app"}, {"web"}}
|
||||
p.Modules["app"].Build, p.Modules["app"].BuiltAt = "build-2s", &built
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
advancePlans(ctx, g.open)
|
||||
|
||||
if len(g.sent) != 0 {
|
||||
t.Fatalf("a no-move build was sent: %v", g.sent)
|
||||
}
|
||||
p = g.plan(t)
|
||||
if s := p.Modules["app"]; s == nil || !strings.HasPrefix(s.Why, "no move") {
|
||||
t.Fatalf("the build was not marked no move: %+v", s)
|
||||
}
|
||||
if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") {
|
||||
t.Fatalf("the walk waits on a no-move tier for a report nothing will cause: tier %d, %s %q", p.Tier, p.State, p.Note)
|
||||
}
|
||||
|
||||
// A machine that has not answered its last send is still waited on: no move is not "never sent".
|
||||
p.Tier, p.State, p.Note = 0, inventory.PlanRolling, ""
|
||||
if err := inv.RecordSent(ctx, nodeID(t, g.open, "laptop"), "d-laptop-later", map[string]string{"app": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, g.open)
|
||||
p = g.plan(t)
|
||||
if p.Tier != 0 || !strings.Contains(p.Note, "waiting for app on laptop") {
|
||||
t.Fatalf("a machine that has not answered its last send was not waited on: tier %d, %q", p.Tier, p.Note)
|
||||
}
|
||||
}
|
||||
@@ -106,6 +106,8 @@ func run() error {
|
||||
return killCommand(ctx, args[1:])
|
||||
case "forget-checks":
|
||||
return forgetChecksCommand(ctx, args[1:])
|
||||
case "try":
|
||||
return tryCommand(ctx, args[1:])
|
||||
case "pause", "resume":
|
||||
return pauseCommand(ctx, args[0], args[1:])
|
||||
case "collection":
|
||||
@@ -381,6 +383,13 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
// **A merge check builds nothing** (novox/hq to-be 45 §9): its verdict is said, and nothing of it is
|
||||
// recorded or registered.
|
||||
if result.Check != nil || result.Checked != nil {
|
||||
if isTry(result) {
|
||||
// A branch tried before its pull request (novox/hq issue 686): its asker reads the outcome from the
|
||||
// build's log. Never said as `checked`, so the forge's holder sets no status.
|
||||
fmt.Printf("%s: a try of %s at %.8s, read with `try --id`: said to nobody, nothing taken in\n", result.ID,
|
||||
result.Repository, result.Ref)
|
||||
return nil
|
||||
}
|
||||
checked(ctx, result)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -456,6 +456,9 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
"release-held": worded(func(o conditions.Observation) words {
|
||||
return releaseHeldWords(nil, o.Also)
|
||||
}),
|
||||
"release-silent": worded(func(o conditions.Observation) words {
|
||||
return releaseSilentWords(o.Also)
|
||||
}),
|
||||
"facts-stale": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Merge checks use outdated facts",
|
||||
Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " +
|
||||
@@ -617,6 +620,24 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
Explanation: "The bus reports a listener too slow to keep up, so messages to it are late.",
|
||||
Resolved: "The listener keeps up again"}
|
||||
}),
|
||||
kindBusWatchDead: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The bus's watch of slow clients is down",
|
||||
Explanation: "The part of the bus that notices clients too slow to keep up has stopped, or cannot see, so " +
|
||||
"such a client would go unnoticed. It is part of the bus's tools on " + machineOr(o, "the bus's machine") + ".",
|
||||
Resolved: "Resolved: the bus's watch of slow clients works again"}
|
||||
}),
|
||||
kindBusFaultsUnread: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Slow clients of the bus cannot be checked",
|
||||
Explanation: "The controller could not read what the bus said about its clients' connections, so a " +
|
||||
"client the bus cut off would go unnoticed. It tries again at every check.",
|
||||
Resolved: "Resolved: the bus's clients are checked again"}
|
||||
}),
|
||||
"connection-closed": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The bus cut a client off",
|
||||
Explanation: "The bus closed a client's connection because it broke the bus's rules, so what it was " +
|
||||
"sending was lost.",
|
||||
Resolved: "Resolved: the client keeps to the bus's rules"}
|
||||
}),
|
||||
kindBusReconnects: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A client keeps losing the bus",
|
||||
Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. " +
|
||||
@@ -841,6 +862,19 @@ func releaseHeldWords(modules, machines []string) words {
|
||||
Resolved: "Resolved: the held updates are released"}
|
||||
}
|
||||
|
||||
// releaseSilentWords are the plain words of machines that said nothing on an update they were sent (ADR 0353 §4).
|
||||
func releaseSilentWords(machines []string) words {
|
||||
where := "Some machines"
|
||||
if len(machines) > 0 {
|
||||
where = conditions.Capital(namesWords(machines, 4))
|
||||
}
|
||||
return words{Headline: "Machines did not answer an update",
|
||||
Needs: "check that they are awake and on the mesh; the next walk sends them what waits.",
|
||||
Explanation: where + " were sent module updates and said nothing about them within fifteen minutes, so " +
|
||||
"they were left as they were. What they run is not known to be the update.",
|
||||
Resolved: "Resolved: the next walk reached them"}
|
||||
}
|
||||
|
||||
// reloginNeeds is what an account waiting for its groups needs (ADR 0252).
|
||||
func reloginNeeds(node string) string {
|
||||
return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node)
|
||||
|
||||
@@ -1008,9 +1008,14 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
readsBusy, err := engineReadsBusy(ctx, inv, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
return catalogue.Rendering{
|
||||
ReadsHealth: readsHealth,
|
||||
JudgesRoot: judgesRoot,
|
||||
ReadsBusy: readsBusy,
|
||||
BusMembership: memberships[node],
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports,
|
||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||
@@ -1044,6 +1049,35 @@ func engineJudgesRoot(ctx context.Context, inv *inventory.Inventory, node string
|
||||
return had && stated.Contract >= link.RootContract, nil
|
||||
}
|
||||
|
||||
// engineReadsBusy says whether a machine's node-engine reads a container's `busy` (novox/hq ADR 0335), by its
|
||||
// own newest statement, for the same reason as engineReadsHealth: an older engine parses strictly.
|
||||
func engineReadsBusy(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) {
|
||||
stated, had, err := inv.HealthOf(ctx, node)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return had && stated.Contract >= link.BusyContract, nil
|
||||
}
|
||||
|
||||
// busyNotSent is, for a node-engine that does not read a container's busy, each container whose module declares
|
||||
// one: not sent it, so replaced at once while it may hold work (novox/hq ADR 0335). Nothing for one that reads it.
|
||||
func busyNotSent(modules []catalogue.Manifest, reads bool) []string {
|
||||
if reads {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, m := range modules {
|
||||
for _, r := range m.Resources {
|
||||
if _, has := r[catalogue.BusyField]; has {
|
||||
out = append(out, fmt.Sprintf("%s.%v declares busy, and this node-engine does not read it yet (its "+
|
||||
"statement's contract is below %d): not sent, so the container is replaced at once, busy or not "+
|
||||
"(novox/hq ADR 0335)", m.Module, r["id"], link.BusyContract))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR
|
||||
// 0199): the zone settled from that node's settings, the node's private address, the port the
|
||||
// answering listen is published on there.
|
||||
@@ -1451,6 +1485,15 @@ func planCommand(ctx context.Context, args []string) error {
|
||||
for _, line := range unboundLines(plan) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
// And a container's busy that this machine's node-engine is too old to be sent (novox/hq ADR 0335): its
|
||||
// container is replaced at once, as before. A statement that cannot be read is said as such.
|
||||
if reads, err := engineReadsBusy(ctx, open.inventory, args[0]); err != nil {
|
||||
fmt.Printf("%s: whether its node-engine reads a container's busy could not be read: %v\n", args[0], err)
|
||||
} else {
|
||||
for _, line := range busyNotSent(plan.Modules, reads) {
|
||||
fmt.Printf("%s: %s\n", args[0], line)
|
||||
}
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, args[0], plan, settings)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
+537
-42
@@ -39,8 +39,8 @@ import (
|
||||
// from the same manifest is no move.
|
||||
//
|
||||
// **The release plan** is what walks the waiting moves through: whenever moves wait for a gate that no
|
||||
// started plan is walking, the mesh opens one — every such machine, one at a time, the control node last,
|
||||
// each sent and judged before the next. A release plan that fails stops the next one opening on its own:
|
||||
// started plan is walking, the mesh opens one — every such machine, the first sent and judged, then the rest
|
||||
// at once, each judged by its own gate (novox/hq ADR 0353, in place of one at a time). A release plan that fails stops the next one opening on its own:
|
||||
// a person releases it again (`upgrade release-backlog --why`), after the condition says why.
|
||||
|
||||
// sendScope is what a send may carry that no gate has seen.
|
||||
@@ -81,6 +81,15 @@ type moveFacts struct {
|
||||
srcs map[string]map[string]string
|
||||
passed map[string]map[string]bool
|
||||
plans []inventory.Plan
|
||||
// self is the release walking these moves, whose own gates are no other walk's (novox/hq ADR 0353).
|
||||
self string
|
||||
}
|
||||
|
||||
type walkingKey struct{}
|
||||
|
||||
// withWalking says a release plan is the one sending: its own gates never hold its own sends back.
|
||||
func withWalking(ctx context.Context, id string) context.Context {
|
||||
return context.WithValue(ctx, walkingKey{}, id)
|
||||
}
|
||||
|
||||
func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, error) {
|
||||
@@ -99,6 +108,7 @@ func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, er
|
||||
return f, err
|
||||
}
|
||||
f.plans, err = inv.OpenPlans(ctx)
|
||||
f.self, _ = ctx.Value(walkingKey{}).(string)
|
||||
return f, err
|
||||
}
|
||||
|
||||
@@ -193,17 +203,24 @@ func (f moveFacts) moves(node string, modules []string, sent map[string]string,
|
||||
// feet. A release keeps no module records: its open gate's carried moves are its walk.
|
||||
func (f moveFacts) walkedBy(module, node, to string) string {
|
||||
for _, p := range f.plans {
|
||||
if !p.Open() {
|
||||
if !p.Open() || (p.Release != nil && p.ID == f.self) {
|
||||
continue
|
||||
}
|
||||
if p.Release != nil {
|
||||
g := p.Release.Gate
|
||||
if g == nil || g.Verdict != "" {
|
||||
continue
|
||||
// The first machine's gate, and every gate of the rest still judging (novox/hq ADR 0353).
|
||||
gates := []*inventory.PlanGate{p.Release.Gate}
|
||||
for _, g := range p.Release.Gates {
|
||||
gates = append(gates, g)
|
||||
}
|
||||
for _, c := range g.Carried {
|
||||
if c.Module == module && !(slices.Contains(g.Machines, node) && sameCommit(c.To, to)) {
|
||||
return p.ID
|
||||
for _, g := range gates {
|
||||
// A send kept and not yet made walks nothing yet (ADR 0353 §8).
|
||||
if g == nil || g.Verdict != "" || g.Silent || g.Pending {
|
||||
continue
|
||||
}
|
||||
for _, c := range g.Carried {
|
||||
if c.Module == module && !(slices.Contains(g.Machines, node) && sameCommit(c.To, to)) {
|
||||
return p.ID
|
||||
}
|
||||
}
|
||||
}
|
||||
continue
|
||||
@@ -303,8 +320,8 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
|
||||
}
|
||||
}
|
||||
if len(refused) > 0 {
|
||||
return nil, fmt.Errorf("%w: %s — a walk sends them, one machine at a time, each judged (novox/hq ADR "+
|
||||
"0236); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))
|
||||
return nil, fmt.Errorf("%w: %s — a walk sends them, one machine first and then the rest at once, each judged (novox/hq ADR "+
|
||||
"0236, ADR 0353); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))
|
||||
}
|
||||
return kept, nil
|
||||
}
|
||||
@@ -320,19 +337,35 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
|
||||
// And the grants step first, when the send changes what the bus's user list must say (novox/hq issue 490):
|
||||
// answered for the gate to keep, nil when there was none.
|
||||
func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, *inventory.GrantsStep, error) {
|
||||
inv := open.inventory
|
||||
f, err := readMoveFacts(ctx, inv)
|
||||
moves, err := gatedMoves(ctx, open, node, owns)
|
||||
if err != nil || len(moves) == 0 {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
sayRecreations(ctx, open, moves)
|
||||
sent, grants, err := sendJudged(ctx, open, node)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
return moves, sent, grants, nil
|
||||
}
|
||||
|
||||
// gatedMoves is what a gated send to one machine would carry, with the build each moves to, made of nothing:
|
||||
// gatedSend's reading, which a walk also keeps before it sends (novox/hq ADR 0353 §8). Refused the same way
|
||||
// while a plan that has started walks one of those builds elsewhere.
|
||||
func gatedMoves(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, error) {
|
||||
inv := open.inventory
|
||||
f, err := readMoveFacts(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
moves, err := machineMoves(ctx, open, f, node, true)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
// A move the send keeps is not made, and is no move for the gate to judge (novox/hq issue 201): a core module
|
||||
// kept over an older record judged as moving back to it would read the gate as superseded by its own send.
|
||||
if moves, err = withoutKept(ctx, open, node, moves); err != nil {
|
||||
return nil, nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
own := func(module string) bool {
|
||||
return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module })
|
||||
@@ -342,7 +375,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
||||
continue
|
||||
}
|
||||
if id := f.walkedBy(mv.Module, node, mv.To); id != "" {
|
||||
return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||
return nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||
mv.Module, short(mv.To), node, id)
|
||||
}
|
||||
}
|
||||
@@ -350,7 +383,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
||||
// plan, or a release, on this machine is not crossed with a newer build; this send waits for its gate.
|
||||
for _, o := range owns {
|
||||
if id := f.walkedBy(o.Module, node, o.To); id != "" {
|
||||
return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||
return nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||
o.Module, short(o.To), node, id)
|
||||
}
|
||||
}
|
||||
@@ -363,30 +396,26 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
||||
moves[i].Build = o.Build
|
||||
}
|
||||
}
|
||||
if len(moves) == 0 && len(owns) == 0 {
|
||||
return nil, nil, nil, nil
|
||||
if len(moves) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
for i := range moves {
|
||||
if moves[i].Build == "" {
|
||||
if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil {
|
||||
return nil, nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module })
|
||||
sayRecreations(ctx, open, moves)
|
||||
sent, grants, err := sendJudged(ctx, open, node)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
return moves, sent, grants, nil
|
||||
return moves, nil
|
||||
}
|
||||
|
||||
// passCarried keeps a pass as the verdict of every build the gate judged beside its own module.
|
||||
func passCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) {
|
||||
seen := map[string]bool{}
|
||||
for _, c := range g.Carried {
|
||||
if c.Module == except || c.Build == "" || seen[c.Build] {
|
||||
// A build put back with another machine's failure is not this gate's to pass (novox/hq ADR 0353 §3).
|
||||
if c.Module == except || c.Build == "" || seen[c.Build] || slices.Contains(g.Returned, c.Module) {
|
||||
continue
|
||||
}
|
||||
seen[c.Build] = true
|
||||
@@ -695,7 +724,7 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl
|
||||
p := inventory.Plan{ID: fmt.Sprintf("release-%d", now.UnixNano()), Repository: releaseRepository,
|
||||
Created: now, State: inventory.PlanRolling, Tiers: [][]string{names}, Modules: map[string]*inventory.PlanModule{},
|
||||
Release: &inventory.PlanRelease{Order: order, By: by},
|
||||
Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine at a time, each judged", len(names),
|
||||
Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine first, then the rest at once, each judged", len(names),
|
||||
strings.Join(order, ", "))}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return nil, err
|
||||
@@ -704,14 +733,23 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl
|
||||
return &p, nil
|
||||
}
|
||||
|
||||
// advanceRelease takes one step of a release plan: the next machine sent everything waiting there under a
|
||||
// gate, or the machine being judged judged once more; the plan done when every machine is.
|
||||
// advanceRelease takes one step of a release plan: the first machine sent everything waiting there under a
|
||||
// gate and judged; once it passed, every other machine sent at once, each under its own gate (novox/hq ADR
|
||||
// 0353 §1), and judged; the plan done when every machine is.
|
||||
//
|
||||
// **A send is kept before it is made** (ADR 0353 §8): the step that chooses a machine keeps its gate pending with
|
||||
// what it will carry, and the next step makes the send. A controller replaced between the send and keeping it
|
||||
// finds the gate pending, reads the machine's last send, and neither sends again nor leaves the machine unjudged.
|
||||
func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) {
|
||||
r := p.Release
|
||||
if r.Settings != nil {
|
||||
return advanceSettingsWalk(ctx, open, p)
|
||||
}
|
||||
ctx = withWalking(ctx, p.ID)
|
||||
now := time.Now().UTC()
|
||||
if r.Rest {
|
||||
return advanceRest(ctx, open, p, now)
|
||||
}
|
||||
if r.Gate == nil {
|
||||
heard, err := releaseHeard(ctx, open)
|
||||
if err != nil {
|
||||
@@ -724,7 +762,7 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
||||
r.Next++
|
||||
continue
|
||||
}
|
||||
moves, sent, grants, err := gatedSend(ctx, open, node, nil)
|
||||
moves, err := gatedMoves(ctx, open, node, nil)
|
||||
if errors.Is(err, errWalkedElsewhere) {
|
||||
note := fmt.Sprintf("waiting before %s: %v", node, err)
|
||||
changed := p.Note != note
|
||||
@@ -732,23 +770,15 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
||||
return changed, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("sending %s what waits there: %w", node, err)
|
||||
return false, fmt.Errorf("reading what waits on %s: %w", node, err)
|
||||
}
|
||||
if len(moves) == 0 {
|
||||
r.Done = append(r.Done, node)
|
||||
r.Next++
|
||||
continue
|
||||
}
|
||||
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves, Sent: sentNow(ctx, open.inventory, sent),
|
||||
Grants: grants}
|
||||
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
|
||||
if said := grantsSaid(grants); said != "" {
|
||||
p.Note += "; " + said
|
||||
}
|
||||
if said := recreationsSaid(moves); said != "" {
|
||||
p.Note += "; " + said
|
||||
}
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
r.Gate = &inventory.PlanGate{Machines: []string{node}, Carried: moves, Pending: true}
|
||||
p.Note = fmt.Sprintf("sending %s %d build(s) that wait for a gate, first", node, len(moves))
|
||||
return true, nil
|
||||
}
|
||||
p.State, p.Tier = inventory.PlanDone, len(p.Tiers)
|
||||
@@ -759,6 +789,19 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
||||
return true, nil
|
||||
}
|
||||
g := r.Gate
|
||||
if g.Pending {
|
||||
node := firstOf(g.Machines)
|
||||
out, err := sendKept(ctx, open, p, node, g, now)
|
||||
if err != nil || out == keptWaiting {
|
||||
return out == keptWaiting, err
|
||||
}
|
||||
if out == keptNothing {
|
||||
r.Gate = nil
|
||||
r.Done = append(r.Done, node)
|
||||
r.Next++
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
verdict, err := judgeMoves(ctx, open, g, nil, now)
|
||||
if err != nil {
|
||||
return false, err
|
||||
@@ -777,6 +820,11 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
||||
r.Done = append(r.Done, firstOf(g.Machines))
|
||||
r.Next++
|
||||
r.Gate = nil
|
||||
// **Then the rest at once** (novox/hq ADR 0353 §1): every other machine heard from, each judged by its own
|
||||
// gate, in place of one machine at a time (ADR 0236 §22).
|
||||
r.Rest = true
|
||||
p.Note = fmt.Sprintf("passed on %s; the rest are sent at once, each judged", firstOf(g.Machines))
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return true, nil
|
||||
case inventory.GateSuperseded:
|
||||
// Another send moved a judged module on the judged machine (novox/hq issue 352): no verdict on what
|
||||
@@ -797,6 +845,453 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// What sendKept did with a kept send.
|
||||
const (
|
||||
keptSent = iota // sent now
|
||||
keptAlready // the machine was already sent it: judged from that send, nothing sent
|
||||
keptNothing // nothing waits there any more: no gate
|
||||
keptWaiting // another walk judges a build it would carry: it waits
|
||||
)
|
||||
|
||||
// sendKept makes a send a gate kept before it was made (novox/hq ADR 0353 §8) — unless the machine's last send
|
||||
// already carries what the gate is for, made before a controller was replaced and before it was kept: then
|
||||
// nothing is sent, and the gate judges from that send (§6).
|
||||
func sendKept(ctx context.Context, open *stores, p *inventory.Plan, node string, g *inventory.PlanGate,
|
||||
now time.Time) (int, error) {
|
||||
inv := open.inventory
|
||||
moves, sent, grants, err := gatedSend(ctx, open, node, nil)
|
||||
if errors.Is(err, errWalkedElsewhere) {
|
||||
p.Note = fmt.Sprintf("waiting before %s: %v", node, err)
|
||||
return keptWaiting, nil
|
||||
}
|
||||
if err != nil {
|
||||
return keptSent, fmt.Errorf("sending %s what waits there: %w", node, err)
|
||||
}
|
||||
if len(moves) > 0 {
|
||||
g.Machines, g.Carried, g.Since, g.Grants, g.Pending = sent, moves, &now, grants, false
|
||||
g.Sent = sentNow(ctx, inv, sent)
|
||||
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
|
||||
if said := grantsSaid(grants); said != "" {
|
||||
p.Note += "; " + said
|
||||
}
|
||||
if said := recreationsSaid(moves); said != "" {
|
||||
p.Note += "; " + said
|
||||
}
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return keptSent, nil
|
||||
}
|
||||
if !alreadySent(ctx, inv, node, g.Carried) {
|
||||
return keptNothing, nil
|
||||
}
|
||||
since := now
|
||||
if s, found, err := inv.SentTo(ctx, node); err == nil && found {
|
||||
if !s.At.IsZero() {
|
||||
since = s.At
|
||||
}
|
||||
g.Sent = map[string]inventory.SentDeclaration{node: s}
|
||||
}
|
||||
g.Since, g.Pending = &since, false
|
||||
p.Note = fmt.Sprintf("%s was already sent the %d build(s) kept for it, at %s: judged from that send, nothing sent again",
|
||||
node, len(g.Carried), since.Local().Format("15:04:05"))
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return keptAlready, nil
|
||||
}
|
||||
|
||||
// alreadySent says a machine's last send carries every build of the moves: a send made and not kept.
|
||||
func alreadySent(ctx context.Context, inv *inventory.Inventory, node string, moves []inventory.CarriedMove) bool {
|
||||
if len(moves) == 0 {
|
||||
return false
|
||||
}
|
||||
sent, known, err := inv.SentBuilds(ctx, node)
|
||||
if err != nil || !known {
|
||||
return false
|
||||
}
|
||||
for _, mv := range moves {
|
||||
if !sameCommit(sent[mv.Module], mv.To) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// advanceRest takes one step of a release plan's rest (novox/hq ADR 0353): the sends kept for a step made, every
|
||||
// machine of the step judged by its own gate, one that fails put back by itself while the others go on to their
|
||||
// verdicts, and the next step — or the end — once every gate of the step has its verdict or is silent.
|
||||
func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.Time) (bool, error) {
|
||||
r := p.Release
|
||||
// The sends kept for this step, in the plan's order: the control node, last in it, last (§1).
|
||||
var pending []string
|
||||
for _, n := range r.Order {
|
||||
if g := r.Gates[n]; g != nil && g.Pending {
|
||||
pending = append(pending, n)
|
||||
}
|
||||
}
|
||||
if len(pending) > 0 {
|
||||
// **A kept send carries only what was kept** (ADR 0353 §2): a build that passed no gate and came to wait on
|
||||
// a machine since its step was chosen could reach a second machine in it; the step is chosen again.
|
||||
if grew, err := keptStepGrew(ctx, open, r, pending); err != nil {
|
||||
return false, err
|
||||
} else if grew != "" {
|
||||
for _, n := range pending {
|
||||
delete(r.Gates, n)
|
||||
}
|
||||
opened, waiting, err := openRestStep(ctx, open, p)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
p.Note = "chosen again: " + grew
|
||||
if !opened && waiting != "" {
|
||||
p.Note += "; " + waiting
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
var sent, already, waiting []string
|
||||
for _, node := range pending {
|
||||
out, err := sendKept(ctx, open, p, node, r.Gates[node], now)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
switch out {
|
||||
case keptSent:
|
||||
sent = append(sent, node)
|
||||
case keptAlready:
|
||||
already = append(already, node)
|
||||
case keptNothing:
|
||||
delete(r.Gates, node)
|
||||
r.Done = append(r.Done, node)
|
||||
case keptWaiting:
|
||||
waiting = append(waiting, node)
|
||||
}
|
||||
}
|
||||
var said []string
|
||||
if len(sent) > 0 {
|
||||
said = append(said, "sent "+strings.Join(sent, ", ")+" at once, each judged there")
|
||||
}
|
||||
if len(already) > 0 {
|
||||
said = append(said, strings.Join(already, ", ")+" already sent it: judged from that send")
|
||||
}
|
||||
if len(waiting) > 0 {
|
||||
said = append(said, "waiting before "+strings.Join(waiting, ", ")+": another walk judges a build it carries")
|
||||
}
|
||||
if len(said) > 0 {
|
||||
p.Note = strings.Join(said, "; ")
|
||||
}
|
||||
return len(sent)+len(already) > 0 || len(waiting) < len(pending), nil
|
||||
}
|
||||
reports, err := open.inventory.LastReports(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
byNode := map[string]inventory.Reported{}
|
||||
for _, rep := range reports {
|
||||
byNode[rep.Node] = rep
|
||||
}
|
||||
changed, noteWas := false, p.Note
|
||||
var notes, judging []string
|
||||
batched, back := batchingRollbacks(ctx)
|
||||
for _, node := range r.Order {
|
||||
g := r.Gates[node]
|
||||
if g == nil || g.Verdict != "" || g.Silent {
|
||||
continue
|
||||
}
|
||||
// What another machine's failure already put back is not this gate's to judge (ADR 0353 §3).
|
||||
if len(g.Carried) > 0 && len(g.Returned) >= len(g.Carried) {
|
||||
decide(g, inventory.GateFailed, "what it was sent was put back with another machine's failure", now)
|
||||
r.Failed = append(r.Failed, node)
|
||||
changed = true
|
||||
continue
|
||||
}
|
||||
// **A machine that says nothing on its send is left silent, not failed** (ADR 0353 §4): it is not judged
|
||||
// until it reports, and past ApplySilentAfter it is listed and left.
|
||||
if !reportedOnSend(byNode[node], node, g) {
|
||||
if now.Sub(*g.Since) > inventory.ApplySilentAfter {
|
||||
g.Silent = true
|
||||
r.Silent = append(r.Silent, node)
|
||||
changed = true
|
||||
notes = append(notes, fmt.Sprintf("%s said nothing on its send within %s: left silent", node,
|
||||
inventory.ApplySilentAfter))
|
||||
continue
|
||||
}
|
||||
judging = append(judging, node+": not yet reported on its send")
|
||||
continue
|
||||
}
|
||||
verdict, err := judgeMoves(ctx, open, g, nil, now)
|
||||
if err != nil {
|
||||
return changed, err
|
||||
}
|
||||
switch verdict {
|
||||
case "":
|
||||
if putBackBroken(ctx, open, p, g, "", nil) {
|
||||
changed = true
|
||||
notes = append(notes, p.Note)
|
||||
}
|
||||
judging = append(judging, node+": "+gateLine(g))
|
||||
case inventory.GatePassed:
|
||||
passCarried(ctx, open, p, g, "")
|
||||
r.Done = append(r.Done, node)
|
||||
changed = true
|
||||
case inventory.GateSuperseded:
|
||||
// Another send moved a judged module there (novox/hq issue 352): nothing judged, nothing put back on
|
||||
// it; what still waits is released again.
|
||||
r.Done = append(r.Done, node)
|
||||
changed = true
|
||||
notes = append(notes, fmt.Sprintf("superseded on %s: %s", node, g.Why))
|
||||
default:
|
||||
// **Put back by itself** (ADR 0353 §3): what was found wanting goes back at once, and every other
|
||||
// machine of the step goes on to its own verdict.
|
||||
stateWas := p.State
|
||||
p.Note = ""
|
||||
failCarried(batched, open, p, g, "")
|
||||
p.State = stateWas
|
||||
r.Failed = append(r.Failed, node)
|
||||
changed = true
|
||||
notes = append(notes, fmt.Sprintf("failed its gate on %s: %s — %s", node, g.Why, p.Note))
|
||||
returnedElsewhere(ctx, open, r, node, g)
|
||||
}
|
||||
}
|
||||
p.Note = ""
|
||||
sendRollbacks(ctx, open, p, back)
|
||||
if p.Note != "" {
|
||||
notes = append(notes, strings.TrimPrefix(p.Note, "; "))
|
||||
}
|
||||
if len(judging) > 0 {
|
||||
note := strings.Join(append(notes, "judging "+strings.Join(judging, "; ")), "; ")
|
||||
p.Note = note
|
||||
return changed || noteWas != note, nil
|
||||
}
|
||||
// Every gate of the step has its verdict or is silent: the next step, unless a machine failed (§3).
|
||||
if len(r.Failed) == 0 {
|
||||
opened, waiting, err := openRestStep(ctx, open, p)
|
||||
if err != nil {
|
||||
return changed, err
|
||||
}
|
||||
if opened {
|
||||
return true, nil
|
||||
}
|
||||
if waiting != "" {
|
||||
note := strings.Join(append(notes, waiting), "; ")
|
||||
p.Note = note
|
||||
return changed || noteWas != note, nil
|
||||
}
|
||||
}
|
||||
p.Tier = len(p.Tiers)
|
||||
end := fmt.Sprintf("released on %s", orNone(strings.Join(r.Done, ", ")))
|
||||
if len(r.Silent) > 0 {
|
||||
end += "; silent on their send, left: " + strings.Join(r.Silent, ", ")
|
||||
}
|
||||
if len(r.Skipped) > 0 {
|
||||
end += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ")
|
||||
}
|
||||
p.State = inventory.PlanDone
|
||||
if len(r.Failed) > 0 {
|
||||
p.State = inventory.PlanFailed
|
||||
end = "failed on " + strings.Join(r.Failed, ", ") + "; " + end
|
||||
} else if len(r.Silent) > 0 && len(r.Gates) == len(r.Silent) {
|
||||
// **A rest that said nothing is no success** (ADR 0353 §4): failure is loud; the condition says it.
|
||||
p.State = inventory.PlanFailed
|
||||
end = "no machine of the rest reported on its send; " + end
|
||||
}
|
||||
p.Note = strings.Join(append(notes, end), "; ")
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// keptStepGrew says why a step's kept sends would now carry a build no gate passed that they were not kept with;
|
||||
// empty when none would.
|
||||
func keptStepGrew(ctx context.Context, open *stores, r *inventory.PlanRelease, pending []string) (string, error) {
|
||||
f, err := readMoveFacts(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, node := range pending {
|
||||
moves, err := gatedMoves(ctx, open, node, nil)
|
||||
if errors.Is(err, errWalkedElsewhere) {
|
||||
continue // sendKept says it waits
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
kept := r.Gates[node].Carried
|
||||
for _, mv := range moves {
|
||||
// The node-engine and the resolver keep one machine at a time, passed or not (ADR 0353 §9).
|
||||
if (f.gated(mv.Module, mv.To) && !oneAtATime(mv.Module, shelf[mv.Module])) || slices.ContainsFunc(kept, func(c inventory.CarriedMove) bool {
|
||||
return c.Module == mv.Module && sameCommit(c.To, mv.To)
|
||||
}) {
|
||||
continue
|
||||
}
|
||||
return fmt.Sprintf("%s's build %s, which this step did not keep for it, came to wait on %s", mv.Module, short(mv.To), node), nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// reportedOnSend says a machine's last report is about the send its gate judges (novox/hq issue 352): the
|
||||
// declaration kept on the gate, or one sequenced after it; a gate that kept none reads the send made last.
|
||||
func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bool {
|
||||
if r.At == nil {
|
||||
return false
|
||||
}
|
||||
if s, kept := g.Sent[node]; kept {
|
||||
return s.ReportsOn(r)
|
||||
}
|
||||
return r.Current
|
||||
}
|
||||
|
||||
// returnedElsewhere marks, on every other gate of the rest still judging, the builds a failed machine's
|
||||
// rollback put back wherever they were sent (ADR 0236 §5): no longer theirs to judge (ADR 0353 §3).
|
||||
func returnedElsewhere(ctx context.Context, open *stores, r *inventory.PlanRelease, failed string, g *inventory.PlanGate) {
|
||||
for _, c := range g.Carried {
|
||||
if c.Build == "" {
|
||||
continue
|
||||
}
|
||||
if marked, err := open.inventory.GateFailed(ctx, c.Build); err != nil || !marked {
|
||||
continue
|
||||
}
|
||||
for node, other := range r.Gates {
|
||||
if node == failed || other.Verdict != "" || slices.Contains(other.Returned, c.Module) {
|
||||
continue
|
||||
}
|
||||
for _, oc := range other.Carried {
|
||||
if oc.Module == c.Module && sameCommit(oc.To, c.To) {
|
||||
other.Returned = append(other.Returned, c.Module)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// openRestStep chooses the next step of a release plan's rest and keeps its sends (novox/hq ADR 0353 §1, §2):
|
||||
// every machine left that is heard from and has something waiting, except that a build no gate has passed goes
|
||||
// to one machine of the step only. Answers whether a step was kept, or why the walk waits.
|
||||
func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, string, error) {
|
||||
r := p.Release
|
||||
heard, err := releaseHeard(ctx, open)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
f, err := readMoveFacts(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
waiting := map[string][]inventory.CarriedMove{}
|
||||
var nodes, walked []string
|
||||
for _, node := range r.Order[min(r.Next, len(r.Order)):] {
|
||||
if r.Gates[node] != nil || slices.Contains(r.Done, node) || slices.Contains(r.Skipped, node) ||
|
||||
slices.Contains(r.Failed, node) || slices.Contains(r.Silent, node) {
|
||||
continue
|
||||
}
|
||||
if !heard[node] {
|
||||
r.Skipped = append(r.Skipped, node)
|
||||
continue
|
||||
}
|
||||
moves, err := gatedMoves(ctx, open, node, nil)
|
||||
if errors.Is(err, errWalkedElsewhere) {
|
||||
walked = append(walked, fmt.Sprintf("%s (%v)", node, err))
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("reading what waits on %s: %w", node, err)
|
||||
}
|
||||
if len(moves) == 0 {
|
||||
r.Done = append(r.Done, node)
|
||||
continue
|
||||
}
|
||||
waiting[node] = moves
|
||||
nodes = append(nodes, node)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
step := restStep(nodes, waiting, f.gated, func(module string) bool { return oneAtATime(module, shelf[module]) })
|
||||
if len(step) == 0 {
|
||||
if len(walked) > 0 {
|
||||
return false, "waiting before " + strings.Join(walked, "; "), nil
|
||||
}
|
||||
return false, "", nil
|
||||
}
|
||||
if r.Gates == nil {
|
||||
r.Gates = map[string]*inventory.PlanGate{}
|
||||
}
|
||||
for _, node := range step {
|
||||
r.Gates[node] = &inventory.PlanGate{Machines: []string{node}, Carried: waiting[node], Pending: true}
|
||||
}
|
||||
p.Note = fmt.Sprintf("sending %s at once, each judged there", strings.Join(step, ", "))
|
||||
if left := len(nodes) - len(step); left > 0 {
|
||||
p.Note += fmt.Sprintf("; %d machine(s) wait for a build that passed no gate yet to pass on one", left)
|
||||
}
|
||||
return true, "", nil
|
||||
}
|
||||
|
||||
// oneAtATime is a module whose builds go to the rest one machine at a time (novox/hq ADR 0353 §9): the
|
||||
// node-engine, which applies every send, and a module holding the mesh's resolver, which names the bus.
|
||||
func oneAtATime(module string, m catalogue.Manifest) bool {
|
||||
if module == hostModule {
|
||||
return true
|
||||
}
|
||||
return slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == resolverSeat })
|
||||
}
|
||||
|
||||
// restStep is the machines of one step of the rest (novox/hq ADR 0353 §2), in the order given: each machine, unless
|
||||
// a build it would carry has passed no gate and an earlier machine of the step already carries it — that build's
|
||||
// first machine. Pure.
|
||||
//
|
||||
// **The node-engine and the mesh's resolver still go one machine at a time** (ADR 0353 §9): a step carries a
|
||||
// build of either to one machine only, judged before the next, since a fault that cuts a machine off the bus
|
||||
// shows only as its silence and no rollback reaches it.
|
||||
func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed func(module, commit string) bool,
|
||||
alone func(module string) bool) []string {
|
||||
first := map[string]string{}
|
||||
var out []string
|
||||
for _, n := range nodes {
|
||||
var unpassed []string
|
||||
for _, mv := range waiting[n] {
|
||||
if !passed(mv.Module, mv.To) || alone(mv.Module) {
|
||||
unpassed = append(unpassed, mv.Module+"@"+mv.To)
|
||||
}
|
||||
}
|
||||
if slices.ContainsFunc(unpassed, func(k string) bool { _, taken := first[k]; return taken }) {
|
||||
continue
|
||||
}
|
||||
for _, k := range unpassed {
|
||||
first[k] = n
|
||||
}
|
||||
out = append(out, n)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// silentObservation is what the gate's probe says of the newest release whose rest went silent (novox/hq ADR 0353
|
||||
// §4): machines that never reported on their send, left as they were — loud, never a success said by nothing.
|
||||
func silentObservation(ctx context.Context, inv *inventory.Inventory) ([]conditions.Observation, error) {
|
||||
recent, err := inv.RecentPlans(ctx, 20)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, p := range recent {
|
||||
if p.Release == nil || p.Release.Settings != nil || p.Open() {
|
||||
continue
|
||||
}
|
||||
if len(p.Release.Silent) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
machines := append([]string(nil), p.Release.Silent...)
|
||||
sort.Strings(machines)
|
||||
o := conditions.Observation{Scope: conditions.ScopeMesh, ID: "release", Token: "silent", Kind: "release-silent",
|
||||
Severity: conditions.Warning, Resolver: conditions.ResolverOperator, Also: machines,
|
||||
Summary: fmt.Sprintf("%s: %s said nothing on the send of the builds that waited for a gate within %s, and were "+
|
||||
"left as they were", p.ID, strings.Join(machines, ", "), inventory.ApplySilentAfter)}
|
||||
w := releaseSilentWords(machines)
|
||||
o.Headline, o.Explanation, o.Resolved, o.Needs = w.Headline, w.Explanation, w.Resolved, w.Needs
|
||||
return []conditions.Observation{o}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// backlogObservation is what the gate's probe says of a release held after a failure.
|
||||
func backlogObservation() []conditions.Observation {
|
||||
if backlogNow.held == "" || len(backlogNow.waiting) == 0 {
|
||||
|
||||
@@ -251,6 +251,20 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool)
|
||||
return out
|
||||
}
|
||||
|
||||
// noMoveWhy is what a plan says of a module of its tier whose build was made from the source every machine
|
||||
// running it runs (novox/hq issue 280): nothing is sent for it.
|
||||
// noMovePrefix opens it, and is what noMove reads: one prefix for the writer and the reader.
|
||||
const (
|
||||
noMovePrefix = "no move"
|
||||
noMoveWhy = noMovePrefix + ": made from the source every machine running it runs"
|
||||
)
|
||||
|
||||
// noMove is whether the plan marked the module no move and sent it nothing. Read from what the plan says,
|
||||
// so a plan kept before the mark had a name of its own is read the same (novox/hq issue 648).
|
||||
func noMove(s *inventory.PlanModule) bool {
|
||||
return s != nil && s.FirstAt == nil && s.Gate == nil && strings.HasPrefix(s.Why, noMovePrefix)
|
||||
}
|
||||
|
||||
// applied says whether every machine running the module has reported since the module was built.
|
||||
// appliedEach is applied with a moment of its own for each machine: the reports that count are the ones
|
||||
// after that machine was sent the build (novox/hq issue 256).
|
||||
@@ -595,7 +609,7 @@ func advancePlans(ctx context.Context, open *stores) {
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
// What waits for a gate, released one machine at a time (ADR 0236).
|
||||
// What waits for a gate, released one machine first, then the rest at once (ADR 0236, ADR 0353).
|
||||
if _, err := releaseBacklog(ctx, open, ""); err != nil {
|
||||
fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err)
|
||||
}
|
||||
@@ -805,7 +819,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
} else if same {
|
||||
now := time.Now().UTC()
|
||||
state.SentAt = &now
|
||||
state.Why = "no move: made from the source every machine running it runs"
|
||||
state.Why = noMoveWhy
|
||||
fmt.Printf("%s: %s built from the source every machine running it runs: no move, nothing sent\n", p.ID, m)
|
||||
continue
|
||||
}
|
||||
@@ -995,6 +1009,25 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
if state.BuiltAt != nil {
|
||||
since = *state.BuiltAt
|
||||
}
|
||||
// **A build that is no move is waited on by the send that last carried it** (novox/hq issue 648).
|
||||
// Nothing was sent for it, so no report follows its marking or its build: a machine that does not
|
||||
// happen to report again held the walk for ever. Each machine runs it once it answered its last
|
||||
// send — the one whose builds made it no move — whenever that was.
|
||||
if noMove(state) {
|
||||
byNode := map[string]inventory.Reported{}
|
||||
for _, r := range reports {
|
||||
byNode[r.Node] = r
|
||||
}
|
||||
if ok, on := appliedEach(m, func(node string) time.Time {
|
||||
if s := byNode[node].Sent; s != nil {
|
||||
return *s
|
||||
}
|
||||
return time.Time{}
|
||||
}, running, reports); !ok {
|
||||
waiting = append(waiting, fmt.Sprintf("%s on %s", m, strings.Join(on, ", ")))
|
||||
}
|
||||
continue
|
||||
}
|
||||
// **Each machine from its own send** (novox/hq issue 256). With one machine first (ADR 0218)
|
||||
// a module is sent twice — the first machine, then the rest — and SentAt is the second.
|
||||
// Asked of every machine, the first machine's report, made between the two sends, read as
|
||||
@@ -1056,10 +1089,36 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
||||
runningOf map[string][]string) error {
|
||||
inv := open.inventory
|
||||
before, beforeKnown, _ := inv.SentBuilds(ctx, node)
|
||||
// **A machine already sent what this send is for is not sent again** (novox/hq ADR 0353 §6): sent by this plan
|
||||
// before a controller was replaced and before it was kept, or by another walk. Judged from that send, and from
|
||||
// the readings of a gate already judging it there.
|
||||
// Only a send the machine reported it applied is one it has; one it never reported on is sent, as before
|
||||
// (issue 280: what it ran before is then that build, and a failure leaves it as it was).
|
||||
if beforeKnown && slices.IndexFunc(modules, func(m string) bool { return !sameCommit(before[m], p.Modules[m].Commit) }) < 0 &&
|
||||
appliedLastSend(ctx, inv, node) {
|
||||
if others, err := gatedMoves(ctx, open, node, nil); err == nil && len(others) == 0 {
|
||||
adoptSend(ctx, open, p, node, modules, before, runningOf)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
// **What the machine ran before is kept before the send is made** (novox/hq ADR 0353 §6): a controller replaced
|
||||
// between the send and keeping it reads it here, and a failure still puts the machine back.
|
||||
kept := false
|
||||
for _, m := range modules {
|
||||
s := p.Modules[m]
|
||||
if beforeKnown && (s.Previous == "" || !sameCommit(before[m], s.Commit)) && s.Previous != before[m] {
|
||||
s.Previous, kept = before[m], true
|
||||
}
|
||||
}
|
||||
if kept {
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
return fmt.Errorf("what %s ran before is not kept, so nothing is sent: %w", node, err)
|
||||
}
|
||||
}
|
||||
owns := make([]inventory.CarriedMove, 0, len(modules))
|
||||
for _, m := range modules {
|
||||
s := p.Modules[m]
|
||||
owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: before[m], To: s.Commit, Build: s.Build})
|
||||
owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: s.Previous, To: s.Commit, Build: s.Build})
|
||||
}
|
||||
// A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges
|
||||
// all of it there.
|
||||
@@ -1074,10 +1133,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
||||
sentWhat := sentNow(ctx, inv, sent)
|
||||
for _, m := range modules {
|
||||
s := p.Modules[m]
|
||||
// What the first machine ran before: what a failed gate puts back (ADR 0236).
|
||||
if beforeKnown {
|
||||
s.Previous = before[m]
|
||||
}
|
||||
// What the first machine ran before (ADR 0236), kept above before the send.
|
||||
s.First, s.FirstAt = sent, &now
|
||||
s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning(sent, runningOf[m]),
|
||||
From: s.Previous, To: s.Commit, Since: &now, Sent: sentWhat}
|
||||
@@ -1106,6 +1162,113 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
||||
return nil
|
||||
}
|
||||
|
||||
// adoptSend records a first send the machine already had (novox/hq ADR 0353 §6, §7): nothing is sent; the gate counts
|
||||
// from the machine's last send, and takes over the readings of an open gate of another walk judging the same build
|
||||
// there — whose carried move also says what the machine ran before, for a rollback.
|
||||
func adoptSend(ctx context.Context, open *stores, p *inventory.Plan, node string, modules []string,
|
||||
before map[string]string, runningOf map[string][]string) {
|
||||
inv := open.inventory
|
||||
since := time.Now().UTC()
|
||||
last, found, err := inv.SentTo(ctx, node)
|
||||
if err == nil && found && !last.At.IsZero() {
|
||||
since = last.At
|
||||
}
|
||||
lead := modules[0]
|
||||
other, id, from := judgingThere(ctx, inv, p.ID, lead, node, p.Modules[lead].Commit, p.Modules[lead].Build)
|
||||
for _, m := range modules {
|
||||
s := p.Modules[m]
|
||||
// What it ran before: kept by this walk before its send was made (§6); else what the other walk's send
|
||||
// moved it from; else the build itself, so a failure leaves it as it was (issue 280).
|
||||
switch {
|
||||
case s.Previous != "":
|
||||
case m == lead && from != "":
|
||||
s.Previous = from
|
||||
default:
|
||||
s.Previous = before[m]
|
||||
}
|
||||
at := since
|
||||
s.First, s.FirstAt = []string{node}, &at
|
||||
s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning([]string{node}, runningOf[m]),
|
||||
From: s.Previous, To: s.Commit, Since: &at}
|
||||
if found {
|
||||
s.Gate.Sent = map[string]inventory.SentDeclaration{node: last}
|
||||
}
|
||||
s.GatedBy = ""
|
||||
if m == lead {
|
||||
s.Gate.TakeReadings(other, modules, id)
|
||||
} else {
|
||||
s.GatedBy = lead
|
||||
}
|
||||
}
|
||||
p.State = inventory.PlanRolling
|
||||
p.Note = fmt.Sprintf("tier %d built; %s already runs %s's build: nothing sent again, judged from that send",
|
||||
p.Tier, node, strings.Join(modules, ", "))
|
||||
if id != "" {
|
||||
p.Note += ", going on from the readings of " + id
|
||||
}
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
}
|
||||
|
||||
// appliedLastSend says a machine reported it applied the declaration it was sent last.
|
||||
func appliedLastSend(ctx context.Context, inv *inventory.Inventory, node string) bool {
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, r := range reports {
|
||||
if r.Node == node {
|
||||
return r.Current && r.At != nil && r.Outcome == inventory.OutcomeApplied
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// judgingThere is an open gate of another walk judging a module's build on a machine (novox/hq ADR 0353 §7), the
|
||||
// walk's id, and what the machine ran before; nil when none does.
|
||||
func judgingThere(ctx context.Context, inv *inventory.Inventory, self, module, node, commit, build string) (*inventory.PlanGate, string, string) {
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, "", ""
|
||||
}
|
||||
// The same build, by its id where both name one, else by its commit.
|
||||
same := func(b, c string) bool {
|
||||
if b != "" && build != "" {
|
||||
return b == build
|
||||
}
|
||||
return sameCommit(c, commit)
|
||||
}
|
||||
usable := func(g *inventory.PlanGate) bool {
|
||||
return g != nil && !g.Pending && g.Since != nil && (g.Verdict == "" || g.Verdict == inventory.GatePassed) &&
|
||||
slices.Contains(g.Machines, node)
|
||||
}
|
||||
for _, q := range plans {
|
||||
if q.ID == self {
|
||||
continue
|
||||
}
|
||||
if r := q.Release; r != nil {
|
||||
gates := []*inventory.PlanGate{r.Gate}
|
||||
for _, g := range r.Gates {
|
||||
gates = append(gates, g)
|
||||
}
|
||||
for _, g := range gates {
|
||||
if !usable(g) {
|
||||
continue
|
||||
}
|
||||
for _, c := range g.Carried {
|
||||
if c.Module == module && c.Node == node && same(c.Build, c.To) {
|
||||
return g, q.ID, c.From
|
||||
}
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if s := q.Modules[module]; s != nil && usable(s.Gate) && same(s.Build, s.Commit) {
|
||||
return s.Gate, q.ID, s.Previous
|
||||
}
|
||||
}
|
||||
return nil, "", ""
|
||||
}
|
||||
|
||||
// passedWith keeps, on a module sent with others, the verdict of the gate that judged the send: the gate
|
||||
// on the first of them passed, and its pass was kept for every build it carried (passCarried).
|
||||
func passedWith(module string, s *inventory.PlanModule, lead string, g *inventory.PlanGate) {
|
||||
@@ -1594,9 +1757,19 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
// A release plan's walk (ADR 0236): machines done, the one judged, those to come.
|
||||
fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "),
|
||||
orNone(strings.Join(r.Done, ", ")), orNone(strings.Join(r.Skipped, ", ")))
|
||||
if r.Gate != nil {
|
||||
fmt.Printf(" %s\n", gateLine(r.Gate))
|
||||
for _, c := range r.Gate.Carried {
|
||||
if len(r.Failed)+len(r.Silent) > 0 {
|
||||
fmt.Printf(" failed: %s; silent: %s\n", orNone(strings.Join(r.Failed, ", ")), orNone(strings.Join(r.Silent, ", ")))
|
||||
}
|
||||
gates := []*inventory.PlanGate{r.Gate}
|
||||
for _, n := range r.Order {
|
||||
gates = append(gates, r.Gates[n])
|
||||
}
|
||||
for _, g := range gates {
|
||||
if g == nil {
|
||||
continue
|
||||
}
|
||||
fmt.Printf(" %s\n", gateLine(g))
|
||||
for _, c := range g.Carried {
|
||||
fmt.Printf(" %-22s %s → %s\n", c.Module, short(c.From), short(c.To))
|
||||
if c.Recreates != "" {
|
||||
fmt.Printf(" %-22s %s\n", "", c.Recreates)
|
||||
|
||||
@@ -0,0 +1,372 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0353: after its first machine passes, a build goes to every other machine at once, each judged by
|
||||
// its own gate and a failing one put back by itself; and a gate's readings outlive the controller that took them —
|
||||
// a machine already sent what a send is for is not sent again, and its judging goes on from what was read.
|
||||
//
|
||||
// Seen 2026-10-14: release-1791930194184991192 sent ace, g14, shanks and the control node one after another in
|
||||
// eleven minutes; the new controller its last send started sent the control node again the controller build it
|
||||
// had just applied, and judged it from its first reading.
|
||||
|
||||
// aBacklogOnThree is the backlog fixture with a third machine, desk, running app at c1 like the others.
|
||||
func aBacklogOnThree(t *testing.T) *backlogMesh {
|
||||
t.Helper()
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
if _, err := inv.AddNode(ctx, "desk"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "desk", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, b.open, "desk"), "d-desk", map[string]string{"app": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wasHeard := releaseHeard
|
||||
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
|
||||
return map[string]bool{"anchor": true, "laptop": true, "desk": true}, nil
|
||||
}
|
||||
t.Cleanup(func() { releaseHeard = wasHeard })
|
||||
return b
|
||||
}
|
||||
|
||||
// events records every send and every judging, in order.
|
||||
func events(t *testing.T) *[]string {
|
||||
t.Helper()
|
||||
var said []string
|
||||
wasSend, wasGather := sendRollout, gatherGateFacts
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
said = append(said, "send "+strings.Join(names, ","))
|
||||
return wasSend(ctx, open, names)
|
||||
}
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
said = append(said, "judge")
|
||||
return wasGather(ctx, open, component)
|
||||
}
|
||||
t.Cleanup(func() { sendRollout, gatherGateFacts = wasSend, wasGather })
|
||||
return &said
|
||||
}
|
||||
|
||||
// Once the first machine passed, the rest are both sent before either is judged (ADR 0353 §1); before, the second
|
||||
// was judged to its verdict before the third was sent.
|
||||
func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
said := events(t)
|
||||
gateEvery = 0
|
||||
for i := 0; i < 12; i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
p := b.release(t)
|
||||
if p.State != inventory.PlanDone || len(p.Release.Done) != 3 {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
desk, laptop := slices.Index(*said, "send desk"), slices.Index(*said, "send laptop")
|
||||
if desk < 0 || laptop < 0 || slices.Index(*said, "send anchor") > min(desk, laptop) {
|
||||
t.Fatalf("sends and judgings: %v", *said)
|
||||
}
|
||||
if slices.Contains((*said)[min(desk, laptop):max(desk, laptop)], "judge") {
|
||||
t.Fatalf("the rest were sent one at a time, each judged before the next: %v", *said)
|
||||
}
|
||||
if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine of the rest whose gate fails is put back at once, and the other machine of the step goes on to its own
|
||||
// verdict (ADR 0353 §3); before, the walk stopped at the failure and the machine after it was never sent.
|
||||
func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
gateEvery = 0
|
||||
b.broken["desk"] = true // desk fails what it is sent in the rest's step
|
||||
advancePlans(ctx, b.open)
|
||||
for i := 0; i < 12 && b.release(t).Open(); i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
p := b.release(t)
|
||||
if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
if !slices.ContainsFunc(b.sent, func(s []string) bool { return slices.Equal(s, []string{"laptop"}) }) {
|
||||
t.Fatalf("laptop was never sent: %v", b.sent)
|
||||
}
|
||||
if !slices.Contains(p.Release.Done, "laptop") {
|
||||
t.Fatalf("laptop has no verdict of its own: %+v", p.Release)
|
||||
}
|
||||
// late waited on laptop alone and was healthy there: its pass is kept, whatever desk did.
|
||||
if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
||||
}
|
||||
// app failed on desk: marked, and put back wherever it was sent (ADR 0236).
|
||||
if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed {
|
||||
t.Fatal("app's build is not marked failed")
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c1" {
|
||||
t.Fatalf("desk was not put back: %v", sent)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk whose first machine already runs its build under another walk's gate sends nothing and goes on from that
|
||||
// gate's readings (ADR 0353 §7); before, it sent the machine again and judged from its first reading.
|
||||
func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
advancePlans(ctx, b.open) // the release sends anchor app c2, and reads it once
|
||||
release := b.release(t)
|
||||
judging := release.Release.Gate
|
||||
if judging == nil || judging.Pending || judging.Passes == 0 {
|
||||
t.Fatalf("the release is not judging anchor: %+v", judging)
|
||||
}
|
||||
built := time.Now().UTC()
|
||||
plan := inventory.Plan{ID: "plan-app", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
||||
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2",
|
||||
Build: "build-app-c2"}}}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, b.open)
|
||||
if len(b.sent) != 1 {
|
||||
t.Fatalf("anchor was sent again what it runs: %v", b.sent)
|
||||
}
|
||||
p, err := inv.PlanByID(ctx, "plan-app")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g := p.Modules["app"].Gate
|
||||
if g == nil || g.Since == nil || !g.Since.Equal(*judging.Since) || g.Passes < judging.Passes || g.TookOver != release.ID {
|
||||
t.Fatalf("the walk's gate starts from zero: %+v (the release's: %+v)", g, judging)
|
||||
}
|
||||
if p.Modules["app"].Previous != "c1" {
|
||||
t.Fatalf("what anchor ran before is %q, not c1", p.Modules["app"].Previous)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk's first send that was made and not kept — the controller replaced between the two — is not made again
|
||||
// by the controller that takes over: the gate counts from that send (ADR 0353 §5, §6).
|
||||
func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
// The old controller sends anchor app c2 and is replaced before it keeps the send.
|
||||
made := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
if _, err := made(ctx, open, names); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, context.Canceled
|
||||
}
|
||||
advancePlans(ctx, g.open)
|
||||
sendRollout = made
|
||||
if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil {
|
||||
t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent)
|
||||
}
|
||||
last, _, err := g.open.inventory.SentTo(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, g.open)
|
||||
if len(g.sent) != 1 {
|
||||
t.Fatalf("the send was made again: %v", g.sent)
|
||||
}
|
||||
s := g.plan(t).Modules["app"]
|
||||
if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) {
|
||||
t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate)
|
||||
}
|
||||
// What anchor ran before was kept before the send, so a failure still puts it back (§5).
|
||||
if s.Previous != "c1" || s.Gate.From != "c1" {
|
||||
t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From)
|
||||
}
|
||||
}
|
||||
|
||||
// A release's send kept before it was made, and made, is judged by the controller that takes over without being
|
||||
// made again (ADR 0353 §8).
|
||||
func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
p, err := releaseBacklog(ctx, b.open, "")
|
||||
if err != nil || p == nil {
|
||||
t.Fatalf("no release opened: %v", err)
|
||||
}
|
||||
if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gate == nil || !p.Release.Gate.Pending {
|
||||
t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The send is made, and the controller is replaced before it keeps it.
|
||||
if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, b.open)
|
||||
if len(b.sent) != 1 {
|
||||
t.Fatalf("anchor was sent again: %v", b.sent)
|
||||
}
|
||||
g := b.release(t).Release.Gate
|
||||
if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 {
|
||||
t.Fatalf("anchor is not judged on the send made: %+v", g)
|
||||
}
|
||||
}
|
||||
|
||||
// A build no gate has passed goes to one machine of a step, its first; a machine that would be its second waits
|
||||
// for the next step, and one whose every build passed goes now (ADR 0353 §2).
|
||||
func TestABuildThatPassedNoGateGoesToOneMachineOfAStep(t *testing.T) {
|
||||
waiting := map[string][]inventory.CarriedMove{
|
||||
"desk": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}},
|
||||
"laptop": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}},
|
||||
"shanks": {{Module: "app", To: "c2"}},
|
||||
"novox": {{Module: "app", To: "c2"}, {Module: "ctl", To: "k1"}},
|
||||
}
|
||||
passed := func(module, commit string) bool { return module == "app" && commit == "c2" }
|
||||
never := func(string) bool { return false }
|
||||
step := restStep([]string{"desk", "laptop", "shanks", "novox"}, waiting, passed, never)
|
||||
if !slices.Equal(step, []string{"desk", "shanks", "novox"}) {
|
||||
t.Fatalf("the step is %v: laptop would be new's second machine before it passed", step)
|
||||
}
|
||||
if step := restStep([]string{"laptop"}, waiting, func(string, string) bool { return true }, never); !slices.Equal(step, []string{"laptop"}) {
|
||||
t.Fatalf("a machine whose every build passed waits: %v", step)
|
||||
}
|
||||
}
|
||||
|
||||
// The node-engine and the mesh's resolver go to the rest one machine at a time, even passed (ADR 0353 §9): a step
|
||||
// carries either to one machine only, while what else passed goes everywhere at once.
|
||||
func TestTheNodeEngineAndTheResolverGoToTheRestOneAtATime(t *testing.T) {
|
||||
waiting := map[string][]inventory.CarriedMove{
|
||||
"desk": {{Module: hostModule, To: "h2"}},
|
||||
"laptop": {{Module: hostModule, To: "h2"}, {Module: "app", To: "c2"}},
|
||||
"shanks": {{Module: "app", To: "c2"}},
|
||||
"novox": {{Module: "dns", To: "d2"}},
|
||||
"ace": {{Module: "dns", To: "d2"}},
|
||||
}
|
||||
resolver := catalogue.Manifest{Module: "dns", Claims: []catalogue.Claim{{Name: resolverSeat}}}
|
||||
alone := func(m string) bool {
|
||||
if m == "dns" {
|
||||
return oneAtATime(m, resolver)
|
||||
}
|
||||
return oneAtATime(m, catalogue.Manifest{Module: m})
|
||||
}
|
||||
step := restStep([]string{"desk", "laptop", "shanks", "novox", "ace"}, waiting,
|
||||
func(string, string) bool { return true }, alone)
|
||||
if !slices.Equal(step, []string{"desk", "shanks", "novox"}) {
|
||||
t.Fatalf("the step is %v: the node-engine and the resolver each reach one machine of it", step)
|
||||
}
|
||||
if oneAtATime("app", catalogue.Manifest{Module: "app"}) {
|
||||
t.Fatal("a leaf module is held to one machine at a time")
|
||||
}
|
||||
}
|
||||
|
||||
// A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent.
|
||||
func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) {
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
gateEvery = 0
|
||||
b.broken["anchor"] = true
|
||||
advancePlans(ctx, b.open)
|
||||
for i := 0; i < 6 && b.release(t).Open(); i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
for _, s := range b.sent {
|
||||
if !slices.Equal(s, []string{"anchor"}) {
|
||||
t.Fatalf("sent %v: the rest after a failed first machine", b.sent)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The control node is sent last in a step of the rest (ADR 0353 §1): the machine holding the controller is
|
||||
// ordered after every other.
|
||||
func TestTheControlNodeIsSentLastInAStep(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
register(t, b.open, catalogue.Manifest{Module: catalogue.ControllerSeatName, Version: "1"})
|
||||
if _, err := b.open.inventory.Assign(ctx, "anchor", catalogue.ControllerSeatName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said := events(t)
|
||||
gateEvery = 0
|
||||
for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
if p := b.release(t); !slices.Equal(p.Release.Order, []string{"desk", "laptop", "anchor"}) {
|
||||
t.Fatalf("the order is %v", p.Release.Order)
|
||||
}
|
||||
laptop, anchor := slices.Index(*said, "send laptop"), slices.Index(*said, "send anchor")
|
||||
if laptop < 0 || anchor < laptop || slices.Contains((*said)[laptop:anchor], "judge") {
|
||||
t.Fatalf("the control node is not sent last in the rest's step: %v", *said)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put
|
||||
// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4).
|
||||
func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
reporting := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing
|
||||
b.sent = append(b.sent, names)
|
||||
current, _ := open.inventory.CurrentBuilds(ctx)
|
||||
return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep",
|
||||
map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit})
|
||||
}
|
||||
return reporting(ctx, open, names)
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = reporting })
|
||||
gateEvery = 0
|
||||
for i := 0; i < 8; i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
p := b.release(t)
|
||||
g := p.Release.Gates["laptop"]
|
||||
if !p.Open() || g == nil || g.Verdict != "" || g.Silent {
|
||||
t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g)
|
||||
}
|
||||
since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute)
|
||||
g.Since = &since
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, b.open)
|
||||
p = b.release(t)
|
||||
if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed {
|
||||
t.Fatal("a silent machine's build was marked failed")
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
|
||||
t.Fatalf("laptop was put back: %v", sent)
|
||||
}
|
||||
obs, err := silentObservation(ctx, inv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) {
|
||||
t.Fatalf("the silence is not said: %+v", obs)
|
||||
}
|
||||
}
|
||||
@@ -457,6 +457,28 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
return nil, err
|
||||
}
|
||||
return []string{"forget-checks", str("owner"), str("repo"), str("number")}, nil
|
||||
case "try":
|
||||
// A branch tried before its pull request (novox/hq issue 686): asked, or read by the id it answered.
|
||||
if id := str("id"); id != "" {
|
||||
return []string{"try", "--id", id}, nil
|
||||
}
|
||||
if a.given["repository"] == "" && a.given["ref"] == "" {
|
||||
// Neither shape named: the command says its usage, which names both, as `build` does.
|
||||
return []string{"try"}, nil
|
||||
}
|
||||
if err := need("repository", "ref"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"try", str("repository"), str("ref")}
|
||||
for _, flag := range []string{"base", "tests"} {
|
||||
if v := str(flag); v != "" {
|
||||
argv = append(argv, "--"+flag, v)
|
||||
}
|
||||
}
|
||||
if on("store-only") {
|
||||
argv = append(argv, "--store-only")
|
||||
}
|
||||
return argv, nil
|
||||
case "cancel", "kill":
|
||||
if err := need("id"); err != nil {
|
||||
return nil, err
|
||||
@@ -1018,6 +1040,8 @@ var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "co
|
||||
"delivery": true,
|
||||
// The build queue, each ask with its position among the waiting (novox/hq issue 455).
|
||||
"queue": true,
|
||||
// A branch tried before its pull request (novox/hq issue 686).
|
||||
"try": true,
|
||||
// A plan followed (novox/hq issue 229): `plans wait` answers JSON; its other forms say text, and so answer no data.
|
||||
"plans": true}
|
||||
|
||||
@@ -1308,6 +1332,10 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
link.AnswersWithin(ctx, bound+planWaitStart)
|
||||
}
|
||||
}
|
||||
if verb == "try" && len(argv) > 1 && argv[1] == "--id" {
|
||||
// Reading a try holds its answer for the verdict, up to its bound (novox/hq issue 686, 229).
|
||||
link.AnswersWithin(ctx, tryWait+planWaitStart)
|
||||
}
|
||||
if answer, read := readHere(ctx, argv); read {
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
@@ -153,11 +153,15 @@ var signalsTable = []signalRow{
|
||||
return newestOf(f.standings, func(c conditions.Condition) time.Time { return c.LastObserved })
|
||||
}},
|
||||
{Row: "S9", Signal: "bus advisories: maximum deliveries, consumer deleted; the controller's own slow " +
|
||||
"consumer and refused subjects", Emitter: "bus server's advisory subjects; the controller's connection",
|
||||
"consumer and refused subjects; every user's connection the server closed as a slow consumer or for " +
|
||||
"breaking the bus's rules (ADR 0357)",
|
||||
Emitter: "bus server's advisory subjects; the controller's connection; the bus module's watch of its " +
|
||||
"server (bus.slow-consumer, bus.connection-closed)",
|
||||
Trigger: "any", Bound: "any occurrence; clears after an hour without another, a deleted consumer " +
|
||||
"once it exists again or the mesh no longer expects it, and a message given up on once DEAD_LETTERS " +
|
||||
"no longer holds it (novox/hq issue 330)",
|
||||
Kind: "slow-consumer, max-deliveries, refused, consumer-lost", Severity: conditions.Warning, Phase: 1,
|
||||
Kind: "slow-consumer, connection-closed, max-deliveries, refused, consumer-lost", Severity: conditions.Warning,
|
||||
Phase: 1,
|
||||
needs: func(f *signalFacts) error { return f.advisoriesErr }, watch: watchAdvisories,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.advisories, func(a link.Advisory) time.Time { return a.Last })
|
||||
@@ -569,6 +573,9 @@ func watchAdvisories(f *signalFacts) []conditions.Observation {
|
||||
// The advisory is remembered and read again on every look for an hour: the condition counts
|
||||
// what the bus said and when, not the looks (novox/hq issue 402).
|
||||
Happened: a.Last, Times: a.Count}
|
||||
if a.User != "" && a.Reason != "" {
|
||||
busUserClosed(&o, a, f.host)
|
||||
}
|
||||
if a.Kind == link.AdvisoryMaxDeliveries && a.Token == link.AdvisoryNotKept {
|
||||
o.Machine = consumerMachine(a.Stream, a.Consumer)
|
||||
o.Headline = clip(conditions.Capital(fmt.Sprintf("%s gave up on a message, not kept",
|
||||
@@ -579,9 +586,92 @@ func watchAdvisories(f *signalFacts) []conditions.Observation {
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
if f.busFaultsErr != nil {
|
||||
out = append(out, busFaultsUnread(f.busFaultsErr))
|
||||
}
|
||||
if o, ok := busWatchDead(f); ok {
|
||||
out = append(out, o)
|
||||
}
|
||||
return append(out, watchDeadLetters(f)...)
|
||||
}
|
||||
|
||||
// kindBusWatchDead is the bus module's watch not heard, or saying it cannot see (novox/hq ADR 0357, review
|
||||
// 274): while it lasts, a slow consumer would read as none.
|
||||
const kindBusWatchDead = "bus-watch-dead"
|
||||
|
||||
// busWatchStale is how long the watch may be silent: three of its looks, and one more for the tick.
|
||||
const busWatchStale = 2 * time.Minute
|
||||
|
||||
// busWatchDead says the watch silent past busWatchStale, or saying it is not healthy. Nothing while the stream
|
||||
// holds no heartbeat at all: a bus module older than its watch says nothing either way.
|
||||
func busWatchDead(f *signalFacts) (conditions.Observation, bool) {
|
||||
if !f.busWatchHeard {
|
||||
return conditions.Observation{}, false
|
||||
}
|
||||
b := f.busWatch
|
||||
o := conditions.Observation{Scope: conditions.ScopeBus, ID: "bus-module", Kind: kindBusWatchDead,
|
||||
Machine: b.Machine, Severity: conditions.Warning}
|
||||
switch {
|
||||
case f.now.Sub(b.At) > busWatchStale:
|
||||
o.Summary = fmt.Sprintf("the bus module's watch of slow consumers has said nothing for %s, so a client "+
|
||||
"the bus cut off would not be said", ago(f.now.Sub(b.At)))
|
||||
o.Said = "last heartbeat " + b.At.UTC().Format(time.RFC3339)
|
||||
return o, true
|
||||
case !b.Healthy:
|
||||
o.Summary = "the bus module's watch of slow consumers says it cannot see, so a client the bus cut off " +
|
||||
"would not be said"
|
||||
o.Said = firstLine(b.Why)
|
||||
return o, true
|
||||
}
|
||||
return conditions.Observation{}, false
|
||||
}
|
||||
|
||||
// kindBusFaultsUnread is the bus module's events not read back (novox/hq ADR 0357): every other user's
|
||||
// slow consumer is unheard while it lasts, and the controller's own signals are still said.
|
||||
const kindBusFaultsUnread = "bus-faults-unread"
|
||||
|
||||
func busFaultsUnread(err error) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeBus, ID: "bus-module", Kind: kindBusFaultsUnread,
|
||||
Severity: conditions.Warning, Confirm: true,
|
||||
Summary: "what the bus module said of its users' connections could not be read back, so a slow consumer " +
|
||||
"or a connection the bus closed is not said until it can",
|
||||
Said: firstLine(err.Error())}
|
||||
}
|
||||
|
||||
// busUserClosed words an advisory about a bus user's connection the server closed (novox/hq ADR 0357): the
|
||||
// user as the operator says it and the machine it is on, the dotted user name kept in the evidence only.
|
||||
func busUserClosed(o *conditions.Observation, a link.Advisory, host string) {
|
||||
who, machine := busUserWords(a.User)
|
||||
if machine == "" && a.User == "controller" {
|
||||
machine = host
|
||||
}
|
||||
o.Machine = machine
|
||||
times, count := "once", "once"
|
||||
if a.Count > 1 {
|
||||
times = fmt.Sprintf("%d times since %s", a.Count, a.First.UTC().Format("15:04 MST"))
|
||||
count = fmt.Sprintf("%d times", a.Count)
|
||||
}
|
||||
switch a.Kind {
|
||||
case link.AdvisorySlowConsumer:
|
||||
o.Summary = fmt.Sprintf("the bus closed the connection of %s as a slow consumer %s in the last hour, "+
|
||||
"newest at %s: it could not take what it was sent fast enough, and what was on its way to it was lost",
|
||||
who, times, a.Last.UTC().Format("15:04 MST"))
|
||||
o.Headline = clip(conditions.Capital(who)+" was too slow for the bus", 60)
|
||||
o.Explanation = conditions.Capital(fmt.Sprintf("%s could not keep up with what the bus sent it, so the bus "+
|
||||
"closed its connection (%s in the last hour) and what was on its way was lost. It connects again "+
|
||||
"by itself; this clears an hour after the last time.", who, count))
|
||||
o.Resolved = "Resolved: " + who + " kept up for an hour"
|
||||
default:
|
||||
o.Summary = fmt.Sprintf("the bus closed the connection of %s for breaking its rules %s in the last hour, "+
|
||||
"newest at %s: %s", who, times, a.Last.UTC().Format("15:04 MST"), strings.ToLower(a.Reason))
|
||||
o.Headline = clip(conditions.Capital(who)+" was cut off by the bus", 60)
|
||||
o.Explanation = conditions.Capital(fmt.Sprintf("The bus closed the connection of %s because it broke "+
|
||||
"the bus's rules (%s, %s in the last hour), so what it was sending was lost. This clears an hour "+
|
||||
"after the last time.", who, strings.ToLower(a.Reason), count))
|
||||
o.Resolved = "Resolved: " + who + " kept to the bus's rules for an hour"
|
||||
}
|
||||
}
|
||||
|
||||
// watchDeadLetters says each consumer that DEAD_LETTERS holds a message for (novox/hq issue 330): open
|
||||
// while it holds any, so it clears when they are delivered again or dropped, never because the server
|
||||
// stopped saying it.
|
||||
|
||||
@@ -0,0 +1,353 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A branch is tried before its pull request (novox/hq issue 686).
|
||||
//
|
||||
// A repository's store tests skip without a store, and no agent may start a database on its machine. So a
|
||||
// builder pushed, a reviewer read, and only the pull request's merge check ran those tests for the first time:
|
||||
// each failure there cost a round of fifteen to thirty minutes (mesh-controller #283, #310, #312).
|
||||
//
|
||||
// `try <owner>/<repository> <ref>` asks the build seat for the repository's own check of a pushed branch or
|
||||
// commit, as the merge check runs it (internal/builder/check.go): the ref merged into the trunk's head, the
|
||||
// trunk's merge-check.sh, each part in its toolchain's container, against a throwaway store and bus. It is a
|
||||
// check and nothing more:
|
||||
//
|
||||
// - **no pull request, so no status**: its outcome is never said as `checked`, which is what the forge's
|
||||
// holder sets statuses from (isTry, in Built);
|
||||
// - **nothing registered, nothing built**: it names no module of the graph, so the merge gate does not run and
|
||||
// no build is made for a merge (novox/hq ADR 0350's scratch builds are a pull request's check's alone);
|
||||
// - **a check in the build queue**: after deliveries (link.KindOf, novox/hq issue 558), and bounded by the
|
||||
// check's own bound (checkTimeout);
|
||||
// - **the code a pull request's check would run anyway**: only a repository of the mesh, whose every pull
|
||||
// request the build seat already checks, in containers that hold no runtime socket.
|
||||
//
|
||||
// A check outlasts a seat verb's one answer, so the ask answers its id at once and `try --id <id>` reads what
|
||||
// came of it from the build's log, where the build seat says it whole (link.StepTried): holding its answer up
|
||||
// to tryWait for a verdict, like `plans wait` (novox/hq issue 229). Nothing is kept beside the log.
|
||||
|
||||
// tryWait is how long reading a try holds its answer for the verdict. A variable so a test need not wait.
|
||||
var tryWait = 20 * time.Second
|
||||
|
||||
// A ref as the forge is asked for it — a branch or a commit, never an option — and a `go test -run` pattern.
|
||||
var (
|
||||
tryRef = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]*$`)
|
||||
tryTests = regexp.MustCompile(`^[A-Za-z0-9_^$|().*+?/\[\]-]+$`)
|
||||
)
|
||||
|
||||
// tryAnswer is what `try` answers, asking or reading.
|
||||
type tryAnswer struct {
|
||||
ID string `json:"id"`
|
||||
// State is asked (just now), waiting (nothing said of it yet: in the build queue), running, or the verdict:
|
||||
// pass, warning (the repository declares no merge-check.sh), fail, or error (it could not run).
|
||||
State string `json:"state"`
|
||||
Said string `json:"said"`
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Base string `json:"base,omitempty"`
|
||||
Trunk string `json:"trunk,omitempty"`
|
||||
On string `json:"on,omitempty"`
|
||||
Took string `json:"took,omitempty"`
|
||||
Failed []string `json:"failed,omitempty"`
|
||||
// Last is the last thing the build seat said of one still running.
|
||||
Last string `json:"last,omitempty"`
|
||||
}
|
||||
|
||||
// isTry says a check's outcome is a try's: marked so, or — echoed by a build seat from before the mark — a
|
||||
// check of no pull request and no group, which is nobody's status either.
|
||||
func isTry(result link.BuildResult) bool {
|
||||
c := result.Checked
|
||||
return c != nil && (c.Try || (c.Number == 0 && c.Group == ""))
|
||||
}
|
||||
|
||||
// asATry makes a check's ask a try's: the repository's own script alone, for no pull request.
|
||||
func asATry(request link.BuildRequest, tests string, storeOnly bool) link.BuildRequest {
|
||||
c := *request.Check
|
||||
c.Number, c.Try, c.Tests, c.StoreOnly = 0, true, tests, storeOnly
|
||||
c.Paths, c.Modules, c.New, c.Dependents, c.Manifests, c.Builds = nil, nil, nil, nil, nil, nil
|
||||
c.Judge, c.Plan, c.Group, c.Members = "", nil, "", nil
|
||||
request.Check = &c
|
||||
return request
|
||||
}
|
||||
|
||||
// maxTries is how many tries may wait or run at once, mesh-wide: each holds a build machine for minutes.
|
||||
const maxTries = 4
|
||||
|
||||
// triesIn reads the build queue for a new try of one commit: the id of a try of that commit already waiting or
|
||||
// running — the answer to a second ask — and whether there is room for another.
|
||||
func triesIn(q link.Queue, owner, repo, commit string) (same string, room bool) {
|
||||
n := 0
|
||||
for _, a := range q.Asks {
|
||||
if !link.IsTry(a.Request) || (a.State != link.AskWaiting && a.State != link.AskInFlight) {
|
||||
continue
|
||||
}
|
||||
n++
|
||||
c := a.Request.Check
|
||||
if same == "" && strings.EqualFold(c.Owner, owner) && strings.EqualFold(c.Repo, repo) && a.Request.Ref == commit {
|
||||
same = a.ID
|
||||
}
|
||||
}
|
||||
return same, n < maxTries
|
||||
}
|
||||
|
||||
// forgeCommit asks the forge, through its module's tools on the bus, which commit a branch or commit of a
|
||||
// repository is, and the repository's default branch. A variable so a test needs no forge.
|
||||
var forgeCommit = func(ctx context.Context, owner, repo, ref string) (commit, trunk string, err error) {
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
defer js.Close()
|
||||
bus := link.OverNATS{Conn: js.Conn()}
|
||||
ask := func(path string, into any) error {
|
||||
raw, _ := json.Marshal(map[string]any{"path": path})
|
||||
answer, err := link.Ask(ctx, bus, "gitea", "gitea_api", raw, 30*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return errors.New(answer.Error)
|
||||
}
|
||||
return json.Unmarshal(answer.Result, into)
|
||||
}
|
||||
var commits struct {
|
||||
Result []struct {
|
||||
SHA string `json:"sha"`
|
||||
} `json:"result"`
|
||||
}
|
||||
if err := ask("/repos/"+owner+"/"+repo+"/commits?limit=1&stat=false&verification=false&files=false&sha="+
|
||||
url.QueryEscape(ref), &commits); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if len(commits.Result) == 0 || commits.Result[0].SHA == "" {
|
||||
return "", "", errors.New("the forge named no commit for it")
|
||||
}
|
||||
var found struct {
|
||||
Result struct {
|
||||
DefaultBranch string `json:"default_branch"`
|
||||
} `json:"result"`
|
||||
}
|
||||
if err := ask("/repos/"+owner+"/"+repo, &found); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return commits.Result[0].SHA, found.Result.DefaultBranch, nil
|
||||
}
|
||||
|
||||
// tryCommand is `try <owner>/<repository> <ref> [--base <branch>] [--tests <pattern>] [--store-only]`, and
|
||||
// `try --id <id>` to read what came of one.
|
||||
func tryCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("try", flag.ContinueOnError)
|
||||
base := set.String("base", "", "the branch it is merged into (the repository's default branch when none)")
|
||||
tests := set.String("tests", "", "only the tests this `go test -run` pattern names")
|
||||
storeOnly := set.Bool("store-only", false, "only the repository's store tests")
|
||||
id := set.String("id", "", "a try's id: what came of it")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
usage := errors.New("try <owner>/<repository> <branch or commit> [--base <branch>] [--tests <pattern>] " +
|
||||
"[--store-only], or try --id <id>")
|
||||
if *id != "" {
|
||||
if len(positionals) != 0 {
|
||||
return usage
|
||||
}
|
||||
// Before it is a token of a bus subject: an id, never a pattern.
|
||||
if !link.SafeBuildID(*id) {
|
||||
return fmt.Errorf("%q is not a try's id", *id)
|
||||
}
|
||||
return tryRead(ctx, *id)
|
||||
}
|
||||
if len(positionals) != 2 {
|
||||
return usage
|
||||
}
|
||||
owner, repo, isPath := strings.Cut(positionals[0], "/")
|
||||
ref := positionals[1]
|
||||
switch {
|
||||
case !isPath || !tryRef.MatchString(owner) || !tryRef.MatchString(repo) || strings.Contains(repo, "/"):
|
||||
return fmt.Errorf("%q is not a repository on the forge, owner/name; nothing was asked", positionals[0])
|
||||
case !tryRef.MatchString(ref):
|
||||
return fmt.Errorf("%q is not a branch or a commit; nothing was asked", ref)
|
||||
case *base != "" && !tryRef.MatchString(*base):
|
||||
return fmt.Errorf("%q is not a branch; nothing was asked", *base)
|
||||
case *tests != "" && !tryTests.MatchString(*tests):
|
||||
return fmt.Errorf("%q is not a `go test -run` pattern of test names; nothing was asked", *tests)
|
||||
}
|
||||
// **An unknown ref is refused here, loudly** — not sent to a machine to fail at its clone minutes later.
|
||||
commit, trunk, err := forgeCommit(ctx, owner, repo, ref)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the forge holds no branch or commit %s in %s/%s — is it pushed? — so nothing was asked: %w",
|
||||
ref, owner, repo, err)
|
||||
}
|
||||
if *base != "" {
|
||||
trunk = *base
|
||||
}
|
||||
if trunk == "" {
|
||||
return fmt.Errorf("the forge named no default branch for %s/%s, and no --base says what %s is merged into; "+
|
||||
"nothing was asked", owner, repo, ref)
|
||||
}
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
entries, read, edges, err := theGraph(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p := link.PullUpdated{Owner: owner, Repo: repo, Base: trunk, Commit: commit}
|
||||
// Whose repository it is, as a pull request's check reckons it: only the mesh's is tried, the code the
|
||||
// build seat runs for every pull request already.
|
||||
mesh := pullScope(p, entries, read, edges)
|
||||
if !mesh.Mesh {
|
||||
return fmt.Errorf("%s/%s is no repository of the mesh — no module is built from it and its owner is not "+
|
||||
"the core's — and only the mesh's are tried on the build seat; nothing was asked", owner, repo)
|
||||
}
|
||||
request, err := checkRequestFor(ctx, open, p, checkScope{Mesh: true, From: mesh.From}, entries)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
request = asATry(request, *tests, *storeOnly)
|
||||
seat := buildSeatHeld(ctx)
|
||||
// **Bounded in number** (novox/hq issue 686): one try of a commit at a time, and maxTries in all.
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
queue, err := link.ReadQueue(ctx, js, seat)
|
||||
js.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
same, room := triesIn(queue, owner, repo, commit)
|
||||
if same != "" {
|
||||
return printTry(tryAnswer{ID: same, State: "asked", Repository: owner + "/" + repo, Ref: ref, Commit: commit,
|
||||
On: seat, Said: fmt.Sprintf("a try of %s/%s at %.12s already waits or runs, as %s: nothing new was asked. "+
|
||||
"Read it with try id=%s", owner, repo, commit, same, same)})
|
||||
}
|
||||
if !room {
|
||||
return fmt.Errorf("%d tries already wait or run on the build seat, the most at once (`queue` lists them); "+
|
||||
"nothing was asked — read one of yours, or ask again when one has ended", maxTries)
|
||||
}
|
||||
ask, err := askOverOn(seat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ask.Close()
|
||||
if err := ask.Ask(ctx, request); err != nil {
|
||||
return err
|
||||
}
|
||||
return printTry(tryAnswer{ID: request.ID, State: "asked", Repository: owner + "/" + repo, Ref: ref, Commit: commit,
|
||||
Base: trunk, On: seat, Said: fmt.Sprintf("asked %s to try %s/%s at %.12s (%s) merged into %s: its own "+
|
||||
"merge-check.sh, as a check in the build queue, bounded at %s; no status is set and nothing is registered. "+
|
||||
"Read it with try id=%s — a full run of this kind takes minutes", seat, owner, repo, commit, ref, trunk,
|
||||
checkTimeout, request.ID)})
|
||||
}
|
||||
|
||||
// tryRead answers what came of a try, holding the answer up to tryWait for its verdict.
|
||||
func tryRead(ctx context.Context, id string) error {
|
||||
until := time.Now().Add(tryWait)
|
||||
for {
|
||||
lines, err := buildLines(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a := tryAnswerOf(id, lines)
|
||||
if (a.State != "waiting" && a.State != "running") || !time.Now().Add(2*time.Second).Before(until) {
|
||||
return printTry(a)
|
||||
}
|
||||
if err := sleepOrDone(ctx, 2*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tryAnswerOf reads a try's answer from what the build seat said of it: its outcome under StepTried; `failed`
|
||||
// for one that could not run — an error, never a pass; else how far it is.
|
||||
func tryAnswerOf(id string, lines []link.BuildLine) tryAnswer {
|
||||
a := tryAnswer{ID: id, State: "waiting", Said: "nothing is said of " + id + " yet: it waits in the build queue " +
|
||||
"(`queue` says where), or no try has that id; ask again"}
|
||||
for _, l := range lines {
|
||||
switch l.Step {
|
||||
case link.StepTried:
|
||||
var t link.Tried
|
||||
if err := json.Unmarshal([]byte(l.Message), &t); err != nil || t.Verdict == "" {
|
||||
return tryAnswer{ID: id, State: "error", Said: "the build seat's word on it cannot be read: " + l.Message}
|
||||
}
|
||||
return tryAnswer{ID: id, State: t.Verdict, Said: t.Summary, Took: t.Took, Commit: t.Head, Trunk: t.Trunk,
|
||||
Failed: t.Failed}
|
||||
case "failed":
|
||||
return tryAnswer{ID: id, State: "error", Said: "it could not run: " + l.Message}
|
||||
case "checked":
|
||||
// A check that is no try, or a build seat from before the verb: its verdict is not a try's.
|
||||
return tryAnswer{ID: id, State: "error", Said: "the build seat checked it and said no try's outcome — " +
|
||||
"it predates the verb (novox/hq issue 686), or " + id + " is no try: " + l.Message + "; `builds` with " +
|
||||
"the id says every line"}
|
||||
}
|
||||
a.State, a.Last = "running", l.Message
|
||||
a.Said = "still running; ask again. `builds` with the id says every line so far"
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
func printTry(a tryAnswer) error {
|
||||
fmt.Fprintln(os.Stderr, a.Said)
|
||||
for _, l := range a.Failed {
|
||||
fmt.Fprintln(os.Stderr, " "+l)
|
||||
}
|
||||
out, err := json.MarshalIndent(a, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(out))
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildLines is everything the build seat said of one build, as the bus keeps it, in order.
|
||||
func buildLines(ctx context.Context, id string) ([]link.BuildLine, error) {
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot reach the bus to read a build's log: %w", err)
|
||||
}
|
||||
defer js.Close()
|
||||
subject := link.BuildLogOf("*", id)
|
||||
sub, err := js.Context().PullSubscribe(subject, "",
|
||||
nats.BindStream(broker.EventsStream), nats.DeliverAll(), nats.AckNone())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot read %s from the bus: %w", subject, err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
var lines []link.BuildLine
|
||||
for {
|
||||
batch, err := sub.Fetch(200, nats.MaxWait(2*time.Second))
|
||||
if err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.DeadlineExceeded) {
|
||||
return nil, fmt.Errorf("reading a build's log: %w", err)
|
||||
}
|
||||
for _, msg := range batch {
|
||||
var line link.BuildLine
|
||||
if json.Unmarshal(msg.Data, &line) == nil {
|
||||
lines = append(lines, line)
|
||||
}
|
||||
}
|
||||
if len(batch) < 200 {
|
||||
return lines, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A branch is tried before its pull request (novox/hq issue 686): the repository's own check asked for a pushed
|
||||
// branch or commit, with no pull request.
|
||||
|
||||
// What a try asks is a check — after deliveries in the build queue, bounded as one — of the repository's own
|
||||
// script alone: no module of the graph, so no gate; nothing built for a merge, so no reusable image.
|
||||
func TestATryAsksACheckOfTheRepositorysOwnScriptAlone(t *testing.T) {
|
||||
request := link.BuildRequest{ID: "build-1", Ref: "c0ffee", Check: &link.CheckRequest{Owner: "novox", Repo: "a",
|
||||
Number: 7, Base: "main", Modules: []string{"a"}, New: []string{"b"}, Manifests: []string{"module.json"},
|
||||
Dependents: []string{"c"}, Builds: []link.CheckBuild{{Module: "a"}}, Judge: link.JudgeSelf,
|
||||
Plan: &link.ChangePlan{}}}
|
||||
tried := asATry(request, "^TestStore", true)
|
||||
c := tried.Check
|
||||
if !c.Try || c.Number != 0 || c.Tests != "^TestStore" || !c.StoreOnly {
|
||||
t.Fatalf("not marked a try, or it kept a pull request's number: %+v", c)
|
||||
}
|
||||
if len(c.Modules)+len(c.New)+len(c.Manifests)+len(c.Dependents)+len(c.Builds) != 0 || c.Judge != "" || c.Plan != nil {
|
||||
t.Fatalf("a try runs the gate or builds for a merge: %+v", c)
|
||||
}
|
||||
if c.Base != "main" || c.Owner != "novox" || tried.Ref != "c0ffee" {
|
||||
t.Fatalf("a try lost what it merges into what: %+v", tried)
|
||||
}
|
||||
if link.KindOf(tried) != link.KindCheck {
|
||||
t.Fatal("a try does not count as a check in the build queue")
|
||||
}
|
||||
}
|
||||
|
||||
// A try's outcome is heard and left there: no `checked` is said, so the forge's holder sets no status, and
|
||||
// nothing is recorded or registered (the handler holds no store here, and would fail reaching for one).
|
||||
func TestATrysOutcomeSetsNoStatusAndRegistersNothing(t *testing.T) {
|
||||
said := &saidOnTheBus{}
|
||||
checkEvents = said
|
||||
defer func() { checkEvents = nil }()
|
||||
for name, result := range map[string]link.BuildResult{
|
||||
"a try": {ID: "build-1", Ref: "c0ffee", Checked: &link.CheckRequest{Owner: "novox", Repo: "a", Try: true},
|
||||
Check: &link.CheckOutcome{Verdict: "pass", RepoCheck: &link.CheckLayer{Verdict: "fail", Summary: "failed"}}},
|
||||
"a try that could not run": {ID: "build-2", Checked: &link.CheckRequest{Owner: "novox", Repo: "a", Try: true},
|
||||
Failed: "cannot clone"},
|
||||
// A build seat from before the mark echoes the ask without it: a check of no pull request and no
|
||||
// group is still nobody's status.
|
||||
"a try echoed by an older build seat": {ID: "build-3", Checked: &link.CheckRequest{Owner: "novox", Repo: "a"},
|
||||
Check: &link.CheckOutcome{Verdict: "pass"}},
|
||||
} {
|
||||
if err := (builds{}).Built(context.Background(), result); err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
}
|
||||
if len(said.events) != 0 {
|
||||
t.Fatalf("a try's outcome was said for the forge: %v", said.events)
|
||||
}
|
||||
// A pull request's check is still said.
|
||||
_ = (builds{}).Built(context.Background(), link.BuildResult{ID: "build-4", Checked: &link.CheckRequest{
|
||||
Owner: "novox", Repo: "a", Number: 7}, Check: &link.CheckOutcome{Verdict: "pass"}})
|
||||
if len(said.events) != 1 {
|
||||
t.Fatalf("a pull request's check was said %d time(s)", len(said.events))
|
||||
}
|
||||
}
|
||||
|
||||
// saidOnTheBus keeps the seat events said, and nothing else.
|
||||
type saidOnTheBus struct {
|
||||
link.Bus
|
||||
events []string
|
||||
}
|
||||
|
||||
func (s *saidOnTheBus) PublishSeatEvent(_ context.Context, seat, event string, _ []byte) error {
|
||||
s.events = append(s.events, seat+" "+event)
|
||||
return nil
|
||||
}
|
||||
|
||||
// A ref the forge does not hold is refused before anything is asked, naming it.
|
||||
func TestATryOfAnUnknownRefIsRefusedLoudly(t *testing.T) {
|
||||
was := forgeCommit
|
||||
defer func() { forgeCommit = was }()
|
||||
forgeCommit = func(_ context.Context, owner, repo, ref string) (string, string, error) {
|
||||
return "", "", errors.New("404 object does not exist [id: " + ref + "]")
|
||||
}
|
||||
err := tryCommand(context.Background(), []string{"novox/mesh-controller", "fix/no-such-branch"})
|
||||
if err == nil {
|
||||
t.Fatal("a try of a ref the forge does not hold was asked")
|
||||
}
|
||||
for _, want := range []string{"fix/no-such-branch", "novox/mesh-controller", "nothing was asked", "404"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
for _, args := range [][]string{{"mesh-controller", "main"}, {"novox/mesh-controller"}, {"novox/a", "--upload-pack=x"},
|
||||
{"novox/a", "main", "--tests", "x; rm -rf /"}, {"novox/a", "main", "--id", "build-1"}} {
|
||||
if err := tryCommand(context.Background(), args); err == nil {
|
||||
t.Errorf("try %v was taken", args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The answer of a try carries the lines that name what failed (novox/hq issue 460's shape) and how long it took.
|
||||
func TestATrysAnswerCarriesTheFailingLines(t *testing.T) {
|
||||
tried, _ := json.Marshal(link.Tried{Verdict: "fail", Summary: "its merge-check.sh failed: TestX", Took: "5m3s",
|
||||
Head: "c0ffee", Trunk: "ca3f43c", Failed: []string{"--- FAIL: TestX (0.01s)", " x_test.go:12: got 1, want 2",
|
||||
"FAIL\tgithub.com/novox/a/cmd\t3.1s"}})
|
||||
lines := []link.BuildLine{{Seq: 1, Step: "check", Message: "novox/a#0 at c0ffee"},
|
||||
{Seq: 2, Step: "output", Message: "ok github.com/novox/a/internal 1.2s"},
|
||||
{Seq: 3, Step: link.StepTried, Message: string(tried)}, {Seq: 4, Step: "checked", Message: "pass: x"}}
|
||||
a := tryAnswerOf("build-1", lines)
|
||||
if a.State != "fail" || a.Took != "5m3s" || len(a.Failed) != 3 || a.Failed[0] != "--- FAIL: TestX (0.01s)" ||
|
||||
a.Trunk != "ca3f43c" || a.ID != "build-1" {
|
||||
t.Fatalf("the answer lost the verdict, the time or the failing lines: %+v", a)
|
||||
}
|
||||
// Not answered yet: said as running, with how far it is — never as a pass.
|
||||
if a := tryAnswerOf("build-1", lines[:2]); a.State != "running" || a.Last == "" {
|
||||
t.Fatalf("a try still running: %+v", a)
|
||||
}
|
||||
if a := tryAnswerOf("build-1", nil); a.State != "waiting" {
|
||||
t.Fatalf("a try nothing was said of: %+v", a)
|
||||
}
|
||||
// One that could not run is an error with the build seat's word, never a pass.
|
||||
if a := tryAnswerOf("build-1", append(lines[:1:1], link.BuildLine{Seq: 2, Step: "failed",
|
||||
Message: "cannot clone"})); a.State != "error" || !strings.Contains(a.Said, "cannot clone") {
|
||||
t.Fatalf("a try that could not run: %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
// One try of a commit at a time, and four in all (novox/hq issue 686): a second ask is answered the first's id,
|
||||
// a fifth is refused; a try that ended, and a pull request's check, count for nothing.
|
||||
func TestTriesAreBoundedInNumber(t *testing.T) {
|
||||
ask := func(id, repo, ref, state string, try bool) link.QueuedAsk {
|
||||
return link.QueuedAsk{ID: id, State: state, Request: link.BuildRequest{ID: id, Ref: ref,
|
||||
Check: &link.CheckRequest{Owner: "novox", Repo: repo, Try: try}}}
|
||||
}
|
||||
q := link.Queue{Asks: []link.QueuedAsk{
|
||||
ask("t1", "a", "c1", link.AskInFlight, true), ask("t2", "a", "c2", link.AskWaiting, true),
|
||||
ask("t3", "b", "c1", link.AskDead, true), ask("p1", "a", "c9", link.AskWaiting, false)}}
|
||||
if same, room := triesIn(q, "novox", "a", "c2"); same != "t2" || !room {
|
||||
t.Fatalf("a second try of a commit: %q %v", same, room)
|
||||
}
|
||||
if same, room := triesIn(q, "novox", "b", "c1"); same != "" || !room {
|
||||
t.Fatalf("a dead try, or another repository's, was taken for this one: %q %v", same, room)
|
||||
}
|
||||
if same, _ := triesIn(q, "novox", "a", "c9"); same != "" {
|
||||
t.Fatalf("a pull request's check was taken for a try: %q", same)
|
||||
}
|
||||
q.Asks = append(q.Asks, ask("t4", "c", "c3", link.AskWaiting, true), ask("t5", "d", "c4", link.AskWaiting, true))
|
||||
if _, room := triesIn(q, "novox", "e", "c5"); room {
|
||||
t.Fatal("a fifth try was let in")
|
||||
}
|
||||
// An id is checked before it is a token of a bus subject.
|
||||
for _, id := range []string{"x.>", "*", "a b"} {
|
||||
if err := tryCommand(context.Background(), []string{"--id", id}); err == nil ||
|
||||
!strings.Contains(err.Error(), "not a try's id") {
|
||||
t.Errorf("try --id %q: %v", id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The verb runs its command, and is declared with the shell habit it replaces (novox/hq ADR 0245).
|
||||
func TestTheTryVerbRunsItsCommand(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
args map[string]any
|
||||
want []string
|
||||
}{
|
||||
{map[string]any{"repository": "novox/a", "ref": "fix/x"}, []string{"try", "novox/a", "fix/x"}},
|
||||
{map[string]any{"repository": "novox/a", "ref": "fix/x", "base": "main", "tests": "^TestX$", "store-only": "true"},
|
||||
[]string{"try", "novox/a", "fix/x", "--base", "main", "--tests", "^TestX$", "--store-only"}},
|
||||
{map[string]any{"id": "build-1"}, []string{"try", "--id", "build-1"}},
|
||||
} {
|
||||
got, err := argvFor("try", c.args)
|
||||
if err != nil || !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("try %v: %v %v, want %v", c.args, got, err, c.want)
|
||||
}
|
||||
}
|
||||
if _, err := argvFor("try", map[string]any{"repository": "novox/a"}); err == nil {
|
||||
t.Error("a try of no ref was taken")
|
||||
}
|
||||
for _, v := range catalogue.ControllerVerbs {
|
||||
if v.Name == "try" {
|
||||
if len(v.Replaces) == 0 {
|
||||
t.Error("try names no shell habit it replaces")
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Error("try is not a verb of the controller seat")
|
||||
}
|
||||
@@ -79,7 +79,7 @@ func (f following) Upgraded(ctx context.Context, u link.Upgraded) error {
|
||||
// `rebuild`, a `replay` registered — was sent here one machine after another without any judging; it
|
||||
// now waits for a gate like any other build, and the release plan walks it through the machines, one
|
||||
// at a time, each judged.
|
||||
fmt.Printf("%s moved to %s outside a plan; a release plan sends it to %s, one machine at a time, each judged "+
|
||||
fmt.Printf("%s moved to %s outside a plan; a release plan sends it to %s, one machine first and then the rest at once, each judged "+
|
||||
"(`upgrade backlog` lists what waits)\n", u.Module, shortCommit(u.Commit), readableList(on))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -177,6 +177,12 @@ func walkReached(p inventory.Plan, running func(string) []string) map[string][]s
|
||||
reached = appendNew(reached, r.Gate.Machines...)
|
||||
carried(r.Gate)
|
||||
}
|
||||
for _, g := range r.Gates {
|
||||
if g != nil && !g.Pending {
|
||||
reached = appendNew(reached, g.Machines...)
|
||||
carried(g)
|
||||
}
|
||||
}
|
||||
for _, tier := range p.Tiers {
|
||||
for _, m := range tier {
|
||||
for _, n := range running(m) {
|
||||
|
||||
@@ -92,6 +92,11 @@ type signalFacts struct {
|
||||
// consumers: the condition counts the messages given up on, not the looks (novox/hq issue 440).
|
||||
deadLettersNewest map[string]time.Time
|
||||
advisoriesErr error
|
||||
// busFaultsErr is why the bus module's events could not be read this tick (novox/hq ADR 0357).
|
||||
busFaultsErr error
|
||||
// busWatch is the bus module's watch's newest heartbeat, when the stream holds one (review 274).
|
||||
busWatch link.BusWatchBeat
|
||||
busWatchHeard bool
|
||||
|
||||
selfCheck selfCheckFacts
|
||||
|
||||
@@ -358,6 +363,18 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
|
||||
}
|
||||
f.advisories = link.Advisories.Since(now.Add(-advisoryQuiet))
|
||||
f.lostConsumers, f.advisoriesErr = w.lostConsumers(ctx, f.advisories)
|
||||
// And what the bus did to every other user's connection, as the bus module said it (novox/hq ADR 0357):
|
||||
// read back for the same hour, and one advisory with the controller's own where both are its.
|
||||
// A read that fails is its own condition, never S9 blind: the controller's own signals are still said.
|
||||
if w.server != nil && w.js != nil {
|
||||
var faults []link.Advisory
|
||||
if faults, f.busFaultsErr = w.server.BusFaults(ctx, now.Add(-advisoryQuiet)); f.busFaultsErr == nil {
|
||||
f.advisories = link.MergeAdvisories(f.advisories, faults)
|
||||
}
|
||||
if f.busFaultsErr == nil {
|
||||
f.busWatch, f.busWatchHeard, f.busFaultsErr = w.server.BusWatchLast(ctx)
|
||||
}
|
||||
}
|
||||
if f.advisoriesErr == nil && w.js != nil {
|
||||
f.deadLetters, f.advisoriesErr = link.HeldDeadLetters(w.js.Context())
|
||||
}
|
||||
|
||||
@@ -36,4 +36,4 @@ require (
|
||||
// committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and
|
||||
// fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without
|
||||
// `go mod vendor` fails loudly, at once, everywhere.
|
||||
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f
|
||||
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261013220009-b9a293f8c8bc
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f h1:8N5OW2mdTNIck2pe4EciTYX5NsrPsHrTLENGNIWYNTU=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261013220009-b9a293f8c8bc h1:h5v2AbdjM/clMY1YxS9p7VuyYtIjkcw85w9Oz1cOYQc=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261013220009-b9a293f8c8bc/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013141520-9588871d6629 h1:7TYFV1Q6Mcz+lsLP/sds87tysET4rKCn3rRIQV8gNmA=
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **The bus module's watch speaks through the runtime that carries it** (novox/hq ADR 0357): the node's
|
||||
// runtime is granted the events the bus module declares, under the module's name, while the bus's own
|
||||
// account keeps the snapshot API and nothing else (ADR 0235).
|
||||
func TestTheRuntimeCarryingTheBusModuleMayPublishItsEventsAndTheBussAccountMayNot(t *testing.T) {
|
||||
events := []string{"bus.slow-consumer", "bus.connection-closed"}
|
||||
runtime := Principal{Kind: KindNodeTools, Node: "anchor", Module: RuntimeModule,
|
||||
Carries: []Declared{{Module: "nats", Emits: events}}}
|
||||
p, err := PermissionsFor(runtime)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, e := range events {
|
||||
if !slices.Contains(p.Publish, "mesh.mod.nats.event."+e) {
|
||||
t.Errorf("the runtime carrying the bus module may not publish %s: %v", e, p.Publish)
|
||||
}
|
||||
}
|
||||
account := Principal{Kind: KindModule, Node: "anchor", Module: "nats", Emits: events, SnapshotsTheBus: true}
|
||||
a, err := PermissionsFor(account)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range a.Publish {
|
||||
if s == "mesh.mod.nats.event.bus.slow-consumer" || s == "mesh.mod.nats.event.bus.connection-closed" {
|
||||
t.Errorf("the bus's own account is granted %s", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1076,6 +1076,16 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
return catalogue.Built{}, fmt.Errorf("%s: building %s failed: %w", module, a.Name, err)
|
||||
}
|
||||
say("image", "built, publishing")
|
||||
// Its bases linked into its repository first, so the push needs none of their blobs from this
|
||||
// machine's store, which may not hold them (novox/hq issue 694). Said and not fatal: a push
|
||||
// from a store that holds them works without, and one that fails says which blob it lacked.
|
||||
if lender, can := publish.(BaseLender); can {
|
||||
if n, err := lender.LendBases(ctx, module+"/"+a.Name, buildArgValues(args)); err != nil {
|
||||
say("image", "its bases could not all be linked into %s/%s before the push: %v", module, a.Name, err)
|
||||
} else if n > 0 {
|
||||
say("image", "%d blob(s) of its bases linked into %s/%s before the push", n, module, a.Name)
|
||||
}
|
||||
}
|
||||
reference, err := publish.PublishImage(ctx, local, module+"/"+a.Name)
|
||||
if err != nil {
|
||||
return catalogue.Built{}, err
|
||||
@@ -1427,6 +1437,19 @@ func standingOn(ctx context.Context, manifest catalogue.Manifest, held map[strin
|
||||
return args, resolved, nil
|
||||
}
|
||||
|
||||
// buildArgValues is what each --build-arg of a build was handed: the bases, as this mesh holds them.
|
||||
func buildArgValues(args []string) []string {
|
||||
var values []string
|
||||
for i := 0; i+1 < len(args); i += 2 {
|
||||
if args[i] == "--build-arg" {
|
||||
if _, value, has := strings.Cut(args[i+1], "="); has {
|
||||
values = append(values, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
// compile runs a module's own code through its toolchain, and says where the result is.
|
||||
//
|
||||
// **In the module's own directory, under the path the toolchain expects.** A module is compiled
|
||||
|
||||
@@ -103,6 +103,23 @@ type CheckSpec struct {
|
||||
// for the walk to take. None for a group's composed check, whose members' own checks build.
|
||||
Builds []CheckBuildSpec
|
||||
BuildWith *CheckBuilding
|
||||
// Tests and StoreOnly narrow what the repository's own script runs, for a branch tried before its pull
|
||||
// request (novox/hq issue 686): handed to the script as EnvTests and EnvStoreOnly, which it may read.
|
||||
Tests string
|
||||
StoreOnly bool
|
||||
}
|
||||
|
||||
// selection is what a check hands the repository's script to narrow what it runs (novox/hq issue 686): nothing
|
||||
// for a pull request's check, which always runs whole.
|
||||
func selection(spec CheckSpec) []string {
|
||||
var env []string
|
||||
if spec.Tests != "" {
|
||||
env = append(env, EnvTests+"="+spec.Tests)
|
||||
}
|
||||
if spec.StoreOnly {
|
||||
env = append(env, EnvStoreOnly+"=1")
|
||||
}
|
||||
return env
|
||||
}
|
||||
|
||||
// GroupHead is one other head of a delivery group's composed check.
|
||||
@@ -193,6 +210,11 @@ const (
|
||||
// EnvGroup is a delivery group's other heads, as JSON, for the gate (novox/hq ADR 0239); empty for a
|
||||
// pull request checked alone.
|
||||
EnvGroup = "MESH_CHECK_GROUP"
|
||||
// EnvTests is a `go test -run` pattern and EnvStoreOnly is 1 when a branch tried before its pull request
|
||||
// asks for only those tests, or only the store tests (novox/hq issue 686). Never set for a pull request's
|
||||
// check. A script that reads neither runs whole.
|
||||
EnvTests = "MESH_CHECK_TESTS"
|
||||
EnvStoreOnly = "MESH_CHECK_STORE_ONLY"
|
||||
)
|
||||
|
||||
// CheckTimeout bounds one check; a check that runs past it is an error, not a pass.
|
||||
@@ -515,6 +537,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
EnvModules + "=" + strings.Join(append(append([]string{}, spec.Modules...), spec.New...), ","),
|
||||
EnvGroup + "=" + groupFile,
|
||||
"GOCACHE=" + filepath.Join(workspace, "go-cache"), "GOMODCACHE=" + filepath.Join(workspace, "go-modules")}
|
||||
env = append(env, selection(spec)...)
|
||||
|
||||
v := CheckVerdict{}
|
||||
modules := append(append([]string{}, spec.Modules...), prefixed("new:", spec.New)...)
|
||||
|
||||
@@ -590,3 +590,15 @@ func TestARepositoryInTwoLanguagesIsCheckedInBoth(t *testing.T) {
|
||||
t.Errorf("a part in a toolchain the mesh does not hold answered %+v — never a pass", layer)
|
||||
}
|
||||
}
|
||||
|
||||
// A branch tried before its pull request may narrow what the repository's script runs (novox/hq issue 686); a
|
||||
// pull request's check is handed nothing, and runs whole.
|
||||
func TestOnlyATrysSelectionReachesTheScript(t *testing.T) {
|
||||
if env := selection(CheckSpec{Number: 7}); len(env) != 0 {
|
||||
t.Fatalf("a pull request's check was handed a selection: %v", env)
|
||||
}
|
||||
env := selection(CheckSpec{Tests: "^TestX$", StoreOnly: true})
|
||||
if len(env) != 2 || env[0] != "MESH_CHECK_TESTS=^TestX$" || env[1] != "MESH_CHECK_STORE_ONLY=1" {
|
||||
t.Fatalf("a try's selection: %v", env)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// **A built image's bases are linked into its repository before it is pushed** (novox/hq issue 694).
|
||||
//
|
||||
// A push reads every blob of the image out of this machine's store, unless the repository it goes to
|
||||
// already holds that blob: the runtime asks the registry first, blob by blob, and reads locally only
|
||||
// what the registry lacks. A runtime keeping its images in containerd's store — the default for a
|
||||
// fresh install from docker 29 on — does not always hold the compressed layers of the base an image
|
||||
// was built FROM: a layer whose unpacked form it already had is never fetched. The push of such an
|
||||
// image then fails on a layer this machine never had ("not all of them are available locally"), while
|
||||
// the very bytes sit in this registry, under the repository the base was mirrored into.
|
||||
//
|
||||
// So the base's blobs are mounted into the image's repository first. A mount moves no bytes, and once
|
||||
// the repository holds a blob the push asks this machine for nothing of it — whichever store the
|
||||
// runtime keeps. A runtime that holds every layer (the classic store) is not changed by this: its push
|
||||
// finds the same blobs already there.
|
||||
|
||||
// BaseLender links what a build's bases hold into the repository its image is about to be pushed to.
|
||||
type BaseLender interface {
|
||||
// LendBases mounts every blob of each base this registry holds — for the platform this machine
|
||||
// builds — into `repository`, and answers how many it mounted.
|
||||
LendBases(ctx context.Context, repository string, bases []string) (int, error)
|
||||
}
|
||||
|
||||
// platformed is an index's entry: a manifest and the platform it is for.
|
||||
type platformed struct {
|
||||
descriptor
|
||||
Platform *struct {
|
||||
OS string `json:"os"`
|
||||
Architecture string `json:"architecture"`
|
||||
} `json:"platform"`
|
||||
}
|
||||
|
||||
// LendBases mounts each base's blobs into the repository. A base this registry does not hold under its
|
||||
// repository — named by tag, or held somewhere else — lends nothing and is no error: the push reads it
|
||||
// from this machine's store as before.
|
||||
func (r Registry) LendBases(ctx context.Context, repository string, bases []string) (int, error) {
|
||||
mounted := 0
|
||||
for _, base := range bases {
|
||||
name, digest, pinned := strings.Cut(base, "@")
|
||||
if !pinned || !strings.HasPrefix(digest, "sha256:") {
|
||||
continue
|
||||
}
|
||||
where, err := parseReference(name + "@" + digest)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
from := where.repository
|
||||
if from == repository {
|
||||
continue
|
||||
}
|
||||
blobs, held, err := r.blobsOf(ctx, from, digest)
|
||||
if err != nil {
|
||||
return mounted, fmt.Errorf("reading %s: %w", base, err)
|
||||
}
|
||||
if !held {
|
||||
continue
|
||||
}
|
||||
for _, blob := range blobs {
|
||||
did, err := r.mount(ctx, repository, from, blob)
|
||||
if err != nil {
|
||||
return mounted, fmt.Errorf("linking %s of %s into %s: %w", blob, base, repository, err)
|
||||
}
|
||||
if did {
|
||||
mounted++
|
||||
}
|
||||
}
|
||||
}
|
||||
return mounted, nil
|
||||
}
|
||||
|
||||
// blobsOf is the config and layers of the manifest under repository@digest — for an index, of each
|
||||
// manifest it names for this machine's platform. False when the registry holds no such manifest there.
|
||||
func (r Registry) blobsOf(ctx context.Context, repository, digest string) ([]string, bool, error) {
|
||||
body, held, err := r.manifest(ctx, repository, digest)
|
||||
if err != nil || !held {
|
||||
return nil, held, err
|
||||
}
|
||||
var document struct {
|
||||
Manifests []platformed `json:"manifests"`
|
||||
Config *descriptor `json:"config"`
|
||||
Layers []descriptor `json:"layers"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &document); err != nil {
|
||||
return nil, false, fmt.Errorf("%s@%s is not a manifest: %w", repository, digest, err)
|
||||
}
|
||||
var blobs []string
|
||||
if document.Config != nil {
|
||||
blobs = append(blobs, document.Config.Digest)
|
||||
}
|
||||
for _, l := range document.Layers {
|
||||
blobs = append(blobs, l.Digest)
|
||||
}
|
||||
for _, m := range document.Manifests {
|
||||
if m.Platform == nil || m.Platform.OS != runtime.GOOS || m.Platform.Architecture != runtime.GOARCH {
|
||||
continue
|
||||
}
|
||||
more, held, err := r.blobsOf(ctx, repository, m.Digest)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if held {
|
||||
blobs = append(blobs, more...)
|
||||
}
|
||||
}
|
||||
return blobs, true, nil
|
||||
}
|
||||
|
||||
// manifest reads one manifest document from this registry; false when it is not there.
|
||||
func (r Registry) manifest(ctx context.Context, repository, digest string) ([]byte, bool, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet,
|
||||
"http://"+r.Address+"/v2/"+repository+"/manifests/"+digest, nil)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
request.Header.Set("Accept", manifestAccept)
|
||||
response, err := r.client().Do(request)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("cannot reach the registry at %s: %w", r.Address, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode == http.StatusNotFound {
|
||||
return nil, false, nil
|
||||
}
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return nil, false, fmt.Errorf("%s/%s@%s: %s", r.Address, repository, digest, response.Status)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(response.Body, 4<<20))
|
||||
return body, err == nil, err
|
||||
}
|
||||
|
||||
// mount links one blob from one repository into another; false when the repository already held it.
|
||||
func (r Registry) mount(ctx context.Context, repository, from, digest string) (bool, error) {
|
||||
base := "http://" + r.Address + "/v2/" + repository
|
||||
if there, err := r.has(ctx, base+"/blobs/"+digest); err != nil {
|
||||
return false, err
|
||||
} else if there {
|
||||
return false, nil
|
||||
}
|
||||
start, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
base+"/blobs/uploads/?mount="+digest+"&from="+from, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
begun, err := r.client().Do(start)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("cannot reach the registry at %s: %w", r.Address, err)
|
||||
}
|
||||
begun.Body.Close()
|
||||
if begun.StatusCode == http.StatusCreated {
|
||||
return true, nil
|
||||
}
|
||||
if begun.StatusCode == http.StatusAccepted {
|
||||
// Not mounted, and an upload begun in its place: let go of it, so nothing is left half-open.
|
||||
if location := begun.Header.Get("Location"); location != "" {
|
||||
if strings.HasPrefix(location, "/") {
|
||||
location = "http://" + r.Address + location
|
||||
}
|
||||
if cancel, err := http.NewRequestWithContext(ctx, http.MethodDelete, location, nil); err == nil {
|
||||
if done, err := r.client().Do(cancel); err == nil {
|
||||
done.Body.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, fmt.Errorf("%s did not mount it from %s", r.Address, from)
|
||||
}
|
||||
return false, fmt.Errorf("%s answered %s when asked to mount it from %s", r.Address, begun.Status, from)
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http/httptest"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A built image's bases are linked into its repository before it is pushed (novox/hq issue 694): a
|
||||
// runtime keeping containerd's image store may not hold a base layer it never had to fetch, and its push
|
||||
// asks this machine only for what the repository lacks.
|
||||
|
||||
// aBaseForThisMachine puts an index into the registry under `repository`: one manifest for this
|
||||
// machine's platform, one for another, and answers the index's digest and each manifest's blobs.
|
||||
func aBaseForThisMachine(t *testing.T, dst *aRegistryOfRepositories, repository string) (string, []string, []string) {
|
||||
t.Helper()
|
||||
put := func(arch string) (string, []string) {
|
||||
config := []byte(`{"architecture":"` + arch + `"}`)
|
||||
layer := []byte(arch + " layer bytes")
|
||||
for _, b := range [][]byte{config, layer} {
|
||||
dst.blobs[digestOf(b)] = b
|
||||
dst.link(repository, digestOf(b))
|
||||
}
|
||||
m, _ := json.Marshal(map[string]any{
|
||||
"schemaVersion": 2, "mediaType": mediaManifestOCI,
|
||||
"config": map[string]any{"mediaType": "application/vnd.oci.image.config.v1+json", "digest": digestOf(config), "size": len(config)},
|
||||
"layers": []map[string]any{{"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": digestOf(layer), "size": len(layer)}},
|
||||
})
|
||||
dst.manifests[repository+"@"+digestOf(m)] = m
|
||||
return digestOf(m), []string{digestOf(config), digestOf(layer)}
|
||||
}
|
||||
other := "s390x"
|
||||
if runtime.GOARCH == other {
|
||||
other = "amd64"
|
||||
}
|
||||
mine, mineBlobs := put(runtime.GOARCH)
|
||||
theirs, theirBlobs := put(other)
|
||||
index, _ := json.Marshal(map[string]any{
|
||||
"schemaVersion": 2, "mediaType": mediaIndexOCI,
|
||||
"manifests": []map[string]any{
|
||||
{"mediaType": mediaManifestOCI, "digest": mine, "size": 1, "platform": map[string]string{"os": runtime.GOOS, "architecture": runtime.GOARCH}},
|
||||
{"mediaType": mediaManifestOCI, "digest": theirs, "size": 1, "platform": map[string]string{"os": "linux", "architecture": other}},
|
||||
},
|
||||
})
|
||||
dst.manifests[repository+"@"+digestOf(index)] = index
|
||||
return digestOf(index), mineBlobs, theirBlobs
|
||||
}
|
||||
|
||||
func TestABuildsBaseIsLinkedIntoItsRepositoryBeforeThePush(t *testing.T) {
|
||||
dst := newRegistryOfRepositories()
|
||||
server := httptest.NewServer(dst.handler())
|
||||
defer server.Close()
|
||||
address := strings.TrimPrefix(server.URL, "http://")
|
||||
r := Registry{Address: address}
|
||||
|
||||
const from = "upstream/docker.io/library/golang"
|
||||
index, mine, theirs := aBaseForThisMachine(t, dst, from)
|
||||
bases := []string{
|
||||
address + "/" + from + "@" + index,
|
||||
// Not bases this registry holds: a package version, a tag, an image held nowhere here.
|
||||
"@novox/mesh-sdk@1.2.3",
|
||||
"golang:1-alpine",
|
||||
"elsewhere.invalid/thing@sha256:" + strings.Repeat("e", 64),
|
||||
}
|
||||
n, err := r.LendBases(context.Background(), "mesh-tools-go/build", bases)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Every blob of this machine's platform is in the image's repository, so a push asks this machine's
|
||||
// store for none of them; another platform's are not, since nothing built here stands on them.
|
||||
for _, blob := range mine {
|
||||
if !dst.links["mesh-tools-go/build"][blob] {
|
||||
t.Fatalf("the base's blob %s is not in the image's repository", blob)
|
||||
}
|
||||
}
|
||||
for _, blob := range theirs {
|
||||
if dst.links["mesh-tools-go/build"][blob] {
|
||||
t.Fatalf("another platform's blob %s was linked", blob)
|
||||
}
|
||||
}
|
||||
if n != len(mine) || dst.mounts != len(mine) || dst.uploads != 0 {
|
||||
t.Fatalf("%d linked, %d mounts, %d uploads; want %d mounts and no bytes moved", n, dst.mounts, dst.uploads, len(mine))
|
||||
}
|
||||
|
||||
// Asked again, nothing is mounted twice.
|
||||
if n, err := r.LendBases(context.Background(), "mesh-tools-go/build", bases); err != nil || n != 0 {
|
||||
t.Fatalf("a second lending mounted %d: %v", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A registry that will not mount is said, not hidden: the push is still tried, from the store.
|
||||
func TestABaseTheRegistryWillNotMountIsAnError(t *testing.T) {
|
||||
dst := newRegistryOfRepositories()
|
||||
dst.noMount = true
|
||||
server := httptest.NewServer(dst.handler())
|
||||
defer server.Close()
|
||||
address := strings.TrimPrefix(server.URL, "http://")
|
||||
index, _, _ := aBaseForThisMachine(t, dst, "upstream/thing")
|
||||
_, err := Registry{Address: address}.LendBases(context.Background(), "hello/server",
|
||||
[]string{address + "/upstream/thing@" + index})
|
||||
if err == nil || !strings.Contains(err.Error(), "did not mount") {
|
||||
t.Fatalf("a refused mount passed silently: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// lending records what a build lent, and when, against the push.
|
||||
type lending struct {
|
||||
*mirroring
|
||||
order []string
|
||||
}
|
||||
|
||||
func (l *lending) LendBases(_ context.Context, repository string, bases []string) (int, error) {
|
||||
l.order = append(l.order, "lend "+repository+" "+strings.Join(bases, " "))
|
||||
return len(bases), nil
|
||||
}
|
||||
|
||||
func (l *lending) PublishImage(ctx context.Context, localTag, repository string) (string, error) {
|
||||
l.order = append(l.order, "push "+repository)
|
||||
return l.mirroring.PublishImage(ctx, localTag, repository)
|
||||
}
|
||||
|
||||
func TestAnImageIsPushedOnlyAfterItsBasesAreLent(t *testing.T) {
|
||||
r, workspace := aRepository(t, anImage, map[string]string{"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}"})
|
||||
r.contents["modules/bus/module.json"] = anImage
|
||||
l := &lending{mirroring: &mirroring{recorded: r, at: "127.0.0.1:5000"}}
|
||||
if _, err := Build(context.Background(), r.run, l, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace,
|
||||
nil, Npmrc{}, GitCredential{}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{
|
||||
"lend bus/server 127.0.0.1:5000/upstream/docker.io/vendor/server@sha256:" + strings.Repeat("1", 64),
|
||||
"push bus/server",
|
||||
}
|
||||
if strings.Join(l.order, "\n") != strings.Join(want, "\n") {
|
||||
t.Fatalf("the build did:\n%s\nwant:\n%s", strings.Join(l.order, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
}
|
||||
@@ -257,6 +257,13 @@ func (s scratchPublisher) PublishArchive(ctx context.Context, repository string,
|
||||
return s.to.PublishArchive(ctx, ScratchPrefix+repository, body, digest)
|
||||
}
|
||||
|
||||
func (s scratchPublisher) LendBases(ctx context.Context, repository string, bases []string) (int, error) {
|
||||
if l, can := s.to.(BaseLender); can {
|
||||
return l.LendBases(ctx, ScratchPrefix+repository, bases)
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (s scratchPublisher) MirrorImage(ctx context.Context, from, repository string) (string, error) {
|
||||
m, can := s.to.(Mirrorer)
|
||||
if !can {
|
||||
|
||||
@@ -21,6 +21,25 @@ func TestTheBussAccountSaysNothingOnTheBus(t *testing.T) {
|
||||
t.Fatalf("a bus module declaring what it emits was not refused, or not for the reason: %v", err)
|
||||
}
|
||||
|
||||
// Its tools bundle says what it emits (novox/hq ADR 0357): the node's runtime carries the bundle and is
|
||||
// granted the module's events, so a bus module that ships one may declare them.
|
||||
bundled := []byte(`{"module":"bus","version":"1","provides":[{"name":"mesh-bus","scope":"mesh"}],
|
||||
"claims":[{"name":"mesh-broker","scope":"mesh"}],"own-secrets":{"broker":"/run/broker"},
|
||||
"emits":["bus.slow-consumer"],"tools":["bus_streams"],
|
||||
"build":{"artifacts":[{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/t","binary":"t"}]}}`)
|
||||
if _, err := ParseManifest(bundled); err != nil {
|
||||
t.Fatalf("a bus module whose tools bundle declares what it emits was refused: %v", err)
|
||||
}
|
||||
// And as built: the resolved manifest registration reads keeps the bundles its build made and drops the
|
||||
// build, and must be registrable just the same.
|
||||
built := []byte(`{"module":"bus","version":"1","provides":[{"name":"mesh-bus","scope":"mesh"}],
|
||||
"claims":[{"name":"mesh-broker","scope":"mesh"}],"own-secrets":{"broker":"/run/broker"},
|
||||
"emits":["bus.slow-consumer"],"tools":["bus_streams"],
|
||||
"bundles":[{"name":"tools","source":"bundles/bus/tools","digest":"sha256:00","language":"go"}]}`)
|
||||
if _, err := ParseManifest(built); err != nil {
|
||||
t.Fatalf("the built bus module, its bundle made, is not registrable: %v", err)
|
||||
}
|
||||
|
||||
quiet := []byte(`{"module":"bus","version":"1","provides":[{"name":"mesh-bus","scope":"mesh"}],
|
||||
"claims":[{"name":"mesh-broker","scope":"mesh"}],"own-secrets":{"broker":"/run/broker"},
|
||||
"tools":["bus_streams"]}`)
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A long-running container may say it is busy (novox/hq ADR 0335 §1).
|
||||
//
|
||||
// **Beside its health, on the container**: a command run inside it, by its shell, as an exec health check
|
||||
// is, that exits 0 while the container holds work a replacement would lose and non-zero when it is idle;
|
||||
// and how long one ask may take. Being asked is also its notice that it is about to be replaced: from the
|
||||
// first ask it takes no new work. The node-engine asks before it replaces the container, leaves it running
|
||||
// while it says busy, and replaces it on the first pass it says idle, or at 15 minutes — the engine's bound,
|
||||
// never the module's.
|
||||
//
|
||||
// Sent only to a node-engine that reads it (link.BusyContract): an older, strict engine refuses a field it
|
||||
// does not know, whole, and to it the container is replaced at once, as before.
|
||||
|
||||
// BusyField is the resource field.
|
||||
const BusyField = "busy"
|
||||
|
||||
// The bounds of one ask (ADR 0335), the node-engine's own, held here near the author.
|
||||
const (
|
||||
BusyTimeoutDefault = 10 * time.Second
|
||||
BusyTimeoutCeiling = time.Minute
|
||||
)
|
||||
|
||||
// Busy is one container's busy, read.
|
||||
type Busy struct {
|
||||
Command string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// ReadBusy reads a resource's `busy` field, its default applied. False when it carries none.
|
||||
func ReadBusy(r map[string]any) (Busy, bool, []string) {
|
||||
raw, present := r[BusyField]
|
||||
if !present {
|
||||
return Busy{}, false, nil
|
||||
}
|
||||
id := fmt.Sprint(r["id"])
|
||||
fields, ok := raw.(map[string]any)
|
||||
if !ok {
|
||||
return Busy{}, true, []string{fmt.Sprintf("%s: busy is %T; it is an object with a command", id, raw)}
|
||||
}
|
||||
var problems []string
|
||||
keys := make([]string, 0, len(fields))
|
||||
for k := range fields {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if k != "command" && k != "timeout" {
|
||||
problems = append(problems, fmt.Sprintf("%s: its busy says %q; busy has a command and a timeout", id, k))
|
||||
}
|
||||
}
|
||||
b := Busy{Timeout: BusyTimeoutDefault}
|
||||
if v, has := fields["command"]; has {
|
||||
s, ok := v.(string)
|
||||
if !ok {
|
||||
problems = append(problems, fmt.Sprintf("%s: its busy's command is %T; it is text", id, v))
|
||||
}
|
||||
b.Command = strings.TrimSpace(s)
|
||||
}
|
||||
if v, has := fields["timeout"]; has {
|
||||
s, _ := v.(string)
|
||||
d, err := time.ParseDuration(strings.TrimSpace(s))
|
||||
switch {
|
||||
case err != nil:
|
||||
problems = append(problems, fmt.Sprintf("%s: its busy's timeout is %v; it is a duration such as \"10s\"", id, v))
|
||||
case d <= 0 || d > BusyTimeoutCeiling:
|
||||
problems = append(problems, fmt.Sprintf("%s: an ask of its busy takes more than nothing and at most %s, not %s",
|
||||
id, BusyTimeoutCeiling, d))
|
||||
default:
|
||||
b.Timeout = d
|
||||
}
|
||||
}
|
||||
return b, true, problems
|
||||
}
|
||||
|
||||
// busyProblems is everything wrong with a manifest's `busy` fields, in the manifest's words: a command on
|
||||
// one line, a timeout within its bounds, and only on a container that stays up.
|
||||
func busyProblems(m Manifest) []string {
|
||||
var problems []string
|
||||
for _, r := range m.Resources {
|
||||
b, has, read := ReadBusy(r)
|
||||
if !has {
|
||||
continue
|
||||
}
|
||||
where := m.Module + ": " + fmt.Sprint(r["id"])
|
||||
for _, p := range read {
|
||||
problems = append(problems, m.Module+": "+p)
|
||||
}
|
||||
if fmt.Sprint(r["type"]) != "container" || !LongRunning(r) {
|
||||
problems = append(problems, fmt.Sprintf("%s declares busy and is not a container that stays up: busy is "+
|
||||
"asked of a running container before it is replaced (ADR 0335)", where))
|
||||
}
|
||||
if b.Command == "" {
|
||||
problems = append(problems, where+"'s busy names no command: one run inside the container, exiting 0 "+
|
||||
"while it is busy")
|
||||
}
|
||||
if strings.ContainsAny(b.Command, "\n\r") {
|
||||
problems = append(problems, where+"'s busy command is one line")
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// busyInto composes a container's `busy` into the node-engine's words, or takes it away: for an engine that
|
||||
// reads it, the command and its timeout written out; for one that does not, the field is not sent, and the
|
||||
// container is replaced at once, as before.
|
||||
func busyInto(resource map[string]any, with Rendering) {
|
||||
if _, has := resource[BusyField]; !has {
|
||||
return
|
||||
}
|
||||
if !with.ReadsBusy {
|
||||
delete(resource, BusyField)
|
||||
return
|
||||
}
|
||||
b, _, _ := ReadBusy(resource)
|
||||
resource[BusyField] = map[string]any{"command": b.Command, "timeout": b.Timeout.String()}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/validate"
|
||||
)
|
||||
|
||||
// A long-running container may say it is busy (novox/hq ADR 0335 §1): `module check` holds the field to its
|
||||
// shape near its author, and it is sent only to a node-engine that reads it, in words the engine's own
|
||||
// validator accepts.
|
||||
|
||||
func busyManifest(t *testing.T, on string, busy any) []byte {
|
||||
t.Helper()
|
||||
extra := map[string]any{"id": "agent", "type": "container", "name": "app-agent",
|
||||
"image": "registry.example/app" + healthDigest}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(healthManifest(t, "", nil, extra), &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range m["resources"].([]any) {
|
||||
if r := r.(map[string]any); r["id"] == on {
|
||||
r[BusyField] = busy
|
||||
}
|
||||
}
|
||||
out, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestABusyIsHeldToItsShape(t *testing.T) {
|
||||
if _, err := ParseManifest(busyManifest(t, "agent", map[string]any{"command": "mesh-builder busy"})); err != nil {
|
||||
t.Fatalf("a well-formed busy was refused: %v", err)
|
||||
}
|
||||
if _, err := ParseManifest(busyManifest(t, "agent", map[string]any{"command": "x", "timeout": "30s"})); err != nil {
|
||||
t.Fatalf("a busy with its timeout was refused: %v", err)
|
||||
}
|
||||
for name, c := range map[string]struct {
|
||||
on string
|
||||
busy any
|
||||
says string
|
||||
}{
|
||||
"text": {"agent", "mesh-builder busy", "it is an object"},
|
||||
"no command": {"agent", map[string]any{"timeout": "5s"}, "names no command"},
|
||||
"two lines": {"agent", map[string]any{"command": "a\nb"}, "one line"},
|
||||
"a long timeout": {"agent", map[string]any{"command": "x", "timeout": "5m"}, "at most 1m0s"},
|
||||
"an unknown key": {"agent", map[string]any{"command": "x", "every": "1m"}, `says "every"`},
|
||||
"on a step": {"seed", map[string]any{"command": "x"}, "not a container that stays up"},
|
||||
"on a service": {"daemon", map[string]any{"command": "x"}, "not a container that stays up"},
|
||||
} {
|
||||
_, err := ParseManifest(busyManifest(t, c.on, c.busy))
|
||||
if err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%s: want a refusal saying %q, got %v", name, c.says, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestABusyIsSentOnlyToAnEngineThatReadsIt(t *testing.T) {
|
||||
m, err := ParseManifest(busyManifest(t, "agent", map[string]any{"command": "mesh-builder busy"}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compose := func(with Rendering) []map[string]any {
|
||||
out, err := Resolution{Node: "laptop", Modules: []Manifest{m}}.Declaration(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
older := compose(Rendering{})
|
||||
if b, sent := older[indexOfID(older, "app.agent")][BusyField]; sent {
|
||||
t.Fatalf("busy was sent to an engine that does not read it: %v", b)
|
||||
}
|
||||
newer := compose(Rendering{ReadsBusy: true})
|
||||
got, _ := newer[indexOfID(newer, "app.agent")][BusyField].(map[string]any)
|
||||
if got["command"] != "mesh-builder busy" || got["timeout"] != "10s" {
|
||||
t.Fatalf("busy composed as %v", got)
|
||||
}
|
||||
// And the engine's own validator takes the container with it.
|
||||
body, err := json.Marshal(map[string]any{"declaration": validate.Version, "resources": newer})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if problems := validate.Declaration(body); len(problems) > 0 {
|
||||
t.Fatalf("the node-engine's validator refuses the composed busy: %v", problems)
|
||||
}
|
||||
if _, has := m.Resources[len(m.Resources)-1][BusyField].(map[string]any)["timeout"]; has {
|
||||
t.Error("composing wrote the timeout into the catalogue's own manifest")
|
||||
}
|
||||
}
|
||||
@@ -252,6 +252,11 @@ type Rendering struct {
|
||||
// statement's contract is link.RootContract or later). To an older, strict engine the field is not
|
||||
// sent, and the account it names is not judged — which the self-check says, as not judged.
|
||||
JudgesRoot bool
|
||||
|
||||
// ReadsBusy says this machine's node-engine reads a container's `busy` (novox/hq ADR 0335: its statement's
|
||||
// contract is link.BusyContract or later). To an older, strict engine the field is not sent, and the
|
||||
// container is replaced at once, as before.
|
||||
ReadsBusy bool
|
||||
}
|
||||
|
||||
// RootField is a user resource's field saying the account must never become root without a person
|
||||
@@ -1062,6 +1067,8 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
// How it is ready, in the node-engine's words: its endpoint as the port this machine
|
||||
// published it on — or not sent at all to an engine older than the field (ADR 0240).
|
||||
healthInto(copied, m, with)
|
||||
// And whether it is busy (novox/hq ADR 0335): sent only to an engine that asks it.
|
||||
busyInto(copied, with)
|
||||
// And a user's `root` (novox/hq ADR 0266): sent only when it asserts something, to an engine
|
||||
// that judges it.
|
||||
rootInto(copied, with)
|
||||
|
||||
@@ -1630,9 +1630,14 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
// **The bus's own account copies the bus and says nothing on it** (novox/hq ADR 0235). The module
|
||||
// holding mesh-broker is granted the snapshot API and nothing else, so anything it declared to say
|
||||
// or hear on the bus would be granted nothing — refused here rather than silently dropped.
|
||||
//
|
||||
// **But its tools bundle may say what it declares** (novox/hq ADR 0357): the node's runtime carries
|
||||
// the bundle and is granted the events the module emits, under the module's name, as for any module it
|
||||
// carries — the bus's account is not who publishes them. So emits are refused only for a bus module
|
||||
// with no bundle to say them.
|
||||
if _, account := m.OwnSecrets["broker"]; account && m.ClaimsSeat(BrokerSeat) {
|
||||
var said []string
|
||||
if len(m.EmitsAll()) > 0 {
|
||||
if len(m.EmitsAll()) > 0 && !m.shipsABundle() {
|
||||
said = append(said, "emits")
|
||||
}
|
||||
for _, f := range []struct {
|
||||
@@ -2336,6 +2341,8 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
// How each long-running resource is ready (novox/hq ADR 0240 rule 2): said near its author, in the
|
||||
// words the node-engine would refuse it in far away.
|
||||
problems = append(problems, healthProblems(m)...)
|
||||
// And whether a container is busy (novox/hq ADR 0335), likewise.
|
||||
problems = append(problems, busyProblems(m)...)
|
||||
if len(problems) > 0 {
|
||||
sort.Strings(problems)
|
||||
return Manifest{}, fmt.Errorf("this manifest cannot be used:\n - %s",
|
||||
@@ -2895,3 +2902,21 @@ func whileStoppedProblems(m Manifest, r map[string]any, scheduled bool) []string
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// shipsABundle says the module builds a tools bundle, which the node's runtime carries and speaks for: as
|
||||
// written, by its build's artifacts; as built and registered, by the bundles its build made — the resolved
|
||||
// manifest keeps those and drops the build (the merge gate found nats not registrable on that, review 274).
|
||||
func (m Manifest) shipsABundle() bool {
|
||||
if len(m.Bundles) > 0 {
|
||||
return true
|
||||
}
|
||||
if m.Build == nil {
|
||||
return false
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Kind == ArtifactBundle {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -362,6 +362,28 @@ var ControllerVerbs = []Verb{
|
||||
"register": "\"true\" to register what it builds",
|
||||
"older": "\"true\", with register: even though a newer build of the module is registered",
|
||||
}, []string{"id"}, "register", "older")},
|
||||
// novox/hq issue 686: a builder ran a branch's store tests for the first time in its pull request's merge check.
|
||||
{Name: "try", Description: "Try a pushed branch or commit before its pull request (novox/hq issue 686): the build " +
|
||||
"seat merges it into the repository's trunk and runs the repository's own merge-check.sh on the merge, in the " +
|
||||
"toolchain containers and against the throwaway store and bus a pull request's merge check gets. Answers at " +
|
||||
"once with an id; asked again with that id it waits up to 20 s and answers pass, fail or error, how long it " +
|
||||
"took and the lines naming what failed — or that it is still waiting or running, to ask again. Sets no " +
|
||||
"status on the forge, registers and builds nothing, runs no merge gate, counts as a check in the build queue " +
|
||||
"(after deliveries and after every pull request's check, however long it waits) and is bounded as one. One " +
|
||||
"try of a commit at a time — a second ask answers the first's id — and at most four waiting or running in " +
|
||||
"all: a fifth is refused. Only for a repository of the mesh; the ref must be on the forge.",
|
||||
Input: schema(map[string]string{
|
||||
"repository": "the repository on the forge, owner/name; with ref",
|
||||
"ref": "the branch or commit to try, as pushed to the forge",
|
||||
"base": "the branch it is merged into (the repository's default branch when none is named). The " +
|
||||
"merge-check.sh that runs is that branch's, never the ref's own: the same trust as a pull request's base",
|
||||
"tests": "only the tests this `go test -run` pattern names, where the repository's merge-check.sh reads " +
|
||||
"MESH_CHECK_TESTS; a script that does not runs whole",
|
||||
"store-only": "\"true\": only the repository's store tests, where its merge-check.sh reads " +
|
||||
"MESH_CHECK_STORE_ONLY",
|
||||
"id": "instead of a repository: the id a try answered, to read what came of it",
|
||||
}, nil, "store-only"),
|
||||
Replaces: []string{"sh merge-check.sh", "docker run postgres"}},
|
||||
// novox/hq ADR 0350 §5: asked by the delivery's owner when the forge says a pull request closed unmerged.
|
||||
{Name: "forget-checks", Description: "Let go of every build a pull request's merge checks made for its merge in " +
|
||||
"the artifact store's scratch namespace (novox/hq ADR 0350 §5): asked when it closed unmerged; a merged one's go " +
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A walk whose first machine waited for a busy container counts the wait as a phase of its own, between the
|
||||
// send to the first machine and the judging, and its delivery time holds it (novox/hq ADR 0335 §4): the
|
||||
// recorded walk again, its judging started only at 17:54:07, a minute after the send, when the machine said
|
||||
// the container replaced.
|
||||
func TestABusyWaitIsAPhaseOfTheDeliveryTime(t *testing.T) {
|
||||
raw := strings.Replace(recordedWalk, `"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z",`,
|
||||
`"gate": {"machines": ["novox"], "since": "2026-10-10T17:54:07.287136827Z",
|
||||
"busy": {"machine": "novox", "module": "mesh-delivery", "container": "mesh-delivery",
|
||||
"since": "2026-10-10T17:53:07Z", "until": "2026-10-10T18:08:07Z", "ended": "2026-10-10T17:54:07.287136827Z"},`, 1)
|
||||
w := walkOf(t, raw).WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil)
|
||||
sumsUp(t, w)
|
||||
if w.TotalMS != 282894 {
|
||||
t.Fatalf("the wait changed the delivery's total: %dms", w.TotalMS)
|
||||
}
|
||||
busy := phase(w, PhaseBusy, 0)
|
||||
if busy.State != PhaseMeasured || busy.TookMS != 60000 {
|
||||
t.Fatalf("the busy wait is not a measured phase of a minute: %+v", busy)
|
||||
}
|
||||
if judged := phase(w, PhaseJudge, 0); judged.TookMS != 148607-60000 {
|
||||
t.Fatalf("the judging still counts from the send, not from the replacement: %+v", judged)
|
||||
}
|
||||
var names []string
|
||||
for _, ph := range w.Phases {
|
||||
names = append(names, ph.Name)
|
||||
}
|
||||
if !strings.Contains(strings.Join(names, " "), PhaseSend+" "+PhaseBusy+" "+PhaseJudge) {
|
||||
t.Fatalf("the busy wait is not between the send and the judging: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
// While the wait goes on, the walk says it waits for a busy container there; a walk where nothing waited has no
|
||||
// such phase.
|
||||
func TestAnOpenBusyWaitIsSaid(t *testing.T) {
|
||||
raw := strings.Replace(recordedWalk, `"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z", "passes": 3,
|
||||
"verdict": "passed", "judged_at": "2026-10-10T17:55:35.894985053Z"}`,
|
||||
`"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z",
|
||||
"busy": {"machine": "novox", "module": "mesh-delivery", "container": "mesh-delivery",
|
||||
"since": "2026-10-10T17:53:07Z", "until": "2026-10-10T18:08:07Z"}}`, 1)
|
||||
raw = strings.Replace(raw, `"state": "done"`, `"state": "rolling"`, 1)
|
||||
p := walkOf(t, raw)
|
||||
if !p.Modules["mesh-delivery"].Gate.Waiting() {
|
||||
t.Fatal("a gate whose busy wait has not ended does not say it waits")
|
||||
}
|
||||
w := p.WalkPhases(time.Date(2026, 10, 10, 17, 58, 0, 0, time.UTC), nil)
|
||||
busy := phase(w, PhaseBusy, 0)
|
||||
if busy.State != PhaseUnknown || !strings.Contains(busy.Said, "waits for a busy container on novox") {
|
||||
t.Fatalf("an open busy wait is not said: %+v", busy)
|
||||
}
|
||||
if plain := walkOf(t, recordedWalk).WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil); phase(plain, PhaseBusy, 0).Name != "" {
|
||||
t.Fatalf("a walk where nothing waited has a busy phase: %+v", plain.Phases)
|
||||
}
|
||||
}
|
||||
|
||||
// A container a report lists waiting is its module's by its id; its output alone is not a new account.
|
||||
func TestABusyContainerIsItsModules(t *testing.T) {
|
||||
list := []BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: BusyWaiting, Said: "a"},
|
||||
{ID: "web.server", State: "replaced-when-idle"}}
|
||||
if b, ok := WaitingFor(list, "build-agent"); !ok || b.Container != "mesh-build-agent" {
|
||||
t.Fatalf("build-agent's container is not found waiting: %+v", b)
|
||||
}
|
||||
if _, ok := WaitingFor(list, "web"); ok {
|
||||
t.Error("a container replaced reads as waiting")
|
||||
}
|
||||
if _, ok := WaitingFor(list, "build"); ok {
|
||||
t.Error("another module's container is found by a prefix of its name")
|
||||
}
|
||||
again := []BusyWait{{ID: "build-agent.server", State: BusyWaiting, Said: "b, a minute on"}, list[1]}
|
||||
if !sameBusy(list, again) {
|
||||
t.Error("a wait whose output changed is a new account")
|
||||
}
|
||||
if sameBusy(list, list[1:]) || !(Doing{Busy: list}).Waiting() || (Doing{Busy: list[1:]}).Waiting() {
|
||||
t.Error("a wait that ended is not a new account")
|
||||
}
|
||||
}
|
||||
|
||||
// A machine's report of a busy container is kept with its last account and read back beside it, and its
|
||||
// starting and its end are news; its output alone is not (novox/hq ADR 0335). Needs the store.
|
||||
func TestAReportOfABusyContainerIsKeptAndReadBack(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t)
|
||||
ctx := t.Context()
|
||||
record, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
since := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
waiting := []BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: BusyWaiting,
|
||||
Since: since, Until: since.Add(15 * time.Minute), Said: "busy: a check"}}
|
||||
keep := func(busy []BusyWait) bool {
|
||||
t.Helper()
|
||||
news, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: "d1", Applied: 1,
|
||||
Busy: busy}, ReportOrder{Epoch: 1, Sequence: 1}, func(ReportOrder) bool { return false })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return news
|
||||
}
|
||||
read := func() []BusyWait {
|
||||
t.Helper()
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range reports {
|
||||
if r.Node == node {
|
||||
return r.Busy
|
||||
}
|
||||
}
|
||||
t.Fatal("the machine's report is not read back")
|
||||
return nil
|
||||
}
|
||||
keep(nil)
|
||||
if got := read(); len(got) != 0 {
|
||||
t.Fatalf("a report with nothing busy reads back %+v", got)
|
||||
}
|
||||
if !keep(waiting) {
|
||||
t.Error("a container starting to wait is not news")
|
||||
}
|
||||
got := read()
|
||||
if b, ok := WaitingFor(got, "build-agent"); !ok || !b.Since.Equal(since) || b.Said != "busy: a check" {
|
||||
t.Fatalf("the wait is not read back: %+v", got)
|
||||
}
|
||||
again := []BusyWait{waiting[0]}
|
||||
again[0].Said = "busy: a check, a minute on"
|
||||
if keep(again) {
|
||||
t.Error("a wait whose output changed is news")
|
||||
}
|
||||
if !keep([]BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: "replaced-when-idle",
|
||||
Since: since, Until: since.Add(15 * time.Minute)}}) {
|
||||
t.Error("a container replaced after it waited is not news")
|
||||
}
|
||||
if _, ok := WaitingFor(read(), "build-agent"); ok {
|
||||
t.Error("a container replaced still reads as waiting")
|
||||
}
|
||||
}
|
||||
|
||||
// A machine of the rest that answered and waits for a busy container is never read as silent, its wait is kept
|
||||
// out of its apply time, and once replaced its report is the end of the wait (novox/hq ADR 0335).
|
||||
func TestARestMachineWaitingForABusyContainerIsNeverSilent(t *testing.T) {
|
||||
c := walk582(t)
|
||||
shanks := c.walk.Modules["build-agent"].Rest["shanks"]
|
||||
ref := shanks.Digest + "@" + shanks.At.Format("2006-01-02T15:04:05.000000")
|
||||
answered := shanks.At.Add(15 * time.Second)
|
||||
// shanks answered, and said a container waits: its apply row is the answer, marked busy.
|
||||
c.rows["shanks"][0].Detail = ApplyBusy
|
||||
at := c.judged.Add(20 * time.Minute) // past ApplySilentAfter from its send
|
||||
w := c.walk.WalkPhases(at, c.lookup)
|
||||
var got RestReport
|
||||
for _, r := range w.Rest {
|
||||
if r.Node == "shanks" {
|
||||
got = r
|
||||
}
|
||||
}
|
||||
if got.State != RestBusy || slicesContains(w.Silent, "shanks") {
|
||||
t.Fatalf("a machine waiting for a busy container reads %q (silent: %v)", got.State, w.Silent)
|
||||
}
|
||||
if w.End != nil || !strings.Contains(phase(w, PhaseApply, -1).Said, "waits for a busy container on shanks") {
|
||||
t.Fatalf("the walk ended, or does not say it waits: %+v", phase(w, PhaseApply, -1))
|
||||
}
|
||||
|
||||
// 17 minutes on, shanks says it replaced the container: its wait is its own duration, and its report is then.
|
||||
c.rows["shanks"] = append(c.rows["shanks"], Duration{Kind: DurationBusy, Subject: "shanks", Node: "shanks", Ref: ref,
|
||||
Started: answered, Took: 17 * time.Minute})
|
||||
w = c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup)
|
||||
sumsUp(t, w)
|
||||
for _, r := range w.Rest {
|
||||
if r.Node == "shanks" {
|
||||
got = r
|
||||
}
|
||||
}
|
||||
if got.State != RestApplied || got.Busy != "17m0s" || !got.Reported.Equal(answered.Add(17*time.Minute)) {
|
||||
t.Fatalf("a machine replaced after its wait reads %+v", got)
|
||||
}
|
||||
if slicesContains(w.Silent, "shanks") || !w.End.Equal(answered.Add(17*time.Minute)) {
|
||||
t.Fatalf("the walk read shanks as silent, or ended elsewhere: %v %v", w.Silent, w.End)
|
||||
}
|
||||
// And the apply time itself is the answer's, never the wait's.
|
||||
if r, _ := FirstReportOf(c.rows["shanks"], shanks, c.walkSent); !r.Answered.Equal(answered) || r.Busy != 17*time.Minute {
|
||||
t.Fatalf("the busy wait is not kept apart from the apply: %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func slicesContains(list []string, s string) bool {
|
||||
for _, x := range list {
|
||||
if x == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -20,10 +20,19 @@ const (
|
||||
// DurationWalkPhase is one phase of an ended walk, per class (novox/hq ADR 0282 decision 6): subject
|
||||
// "<class>/<phase>", and "<class>/total" for its merge to every machine running it.
|
||||
DurationWalkPhase = "walk-phase"
|
||||
// DurationBusy is a machine's wait for a busy container (novox/hq ADR 0335): from its first report of a
|
||||
// send, which said a container waits because it is busy, to the report that says it replaced — the same
|
||||
// send's ref as its `apply` row, whose detail is ApplyBusy. Its own measure, never in the apply's.
|
||||
DurationBusy = "busy-wait"
|
||||
)
|
||||
|
||||
// ApplyBusy is the detail of an `apply` duration whose report said a container waits because it is busy: the
|
||||
// machine answered, and is not yet running what it was sent (novox/hq ADR 0335).
|
||||
const ApplyBusy = "busy"
|
||||
|
||||
// DurationKinds are every kind, in the order `durations` shows them.
|
||||
var DurationKinds = []string{DurationApply, DurationHeartbeatGap, DurationPlanTier, DurationBuild, DurationWalkPhase}
|
||||
var DurationKinds = []string{DurationApply, DurationHeartbeatGap, DurationPlanTier, DurationBuild, DurationWalkPhase,
|
||||
DurationBusy}
|
||||
|
||||
// DurationsKeptFor is how long a duration is kept: long enough to set a bound from, and to correct it
|
||||
// in Phase 1's first live week.
|
||||
@@ -70,6 +79,25 @@ func (i *Inventory) RecordApplyDuration(ctx context.Context, node, declared, out
|
||||
return err
|
||||
}
|
||||
|
||||
// RecordBusyDuration measures a machine's wait for a busy container, at the report that says it replaced
|
||||
// (novox/hq ADR 0335): from the end of its `apply` duration of the send it was last sent, which said it waited, to
|
||||
// now. Nothing when that send's first report did not say a wait, and once per send.
|
||||
func (i *Inventory) RecordBusyDuration(ctx context.Context, node, declared string) error {
|
||||
if declared == "" {
|
||||
return nil
|
||||
}
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`insert into duration (kind, subject, node, ref, started, took_ms, detail)
|
||||
select $1, d.subject, d.node, d.ref, d.started + d.took_ms * interval '1 millisecond',
|
||||
greatest(0, (extract(epoch from (now() - (d.started + d.took_ms * interval '1 millisecond'))) * 1000)::bigint), ''
|
||||
from duration d join node n on n.name = d.node
|
||||
where d.kind = $2 and d.subject = $3 and d.detail = $5 and n.sent = $4 and n.sent_at is not null
|
||||
and d.ref = n.sent || '@' || to_char(n.sent_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US')
|
||||
on conflict do nothing`,
|
||||
DurationBusy, DurationApply, node, declared, ApplyBusy)
|
||||
return err
|
||||
}
|
||||
|
||||
// RecordHeartbeatGap measures the silence before a machine's word: from the last word heard before
|
||||
// it, which the caller read before recording this one.
|
||||
func (i *Inventory) RecordHeartbeatGap(ctx context.Context, node string, before time.Time) error {
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
-- A machine's report says which container it left running because it said busy, and which it replaced
|
||||
-- after it waited (novox/hq ADR 0335, issue 594).
|
||||
--
|
||||
-- A send that replaced the build agent killed the check it ran, and the queue started it again elsewhere.
|
||||
-- A container may now declare `busy`; the node-engine asks it before replacing it, leaves it running while it
|
||||
-- says busy, within 15 minutes, and says so in its report's `busy`: [{"id", "container", "state", "since",
|
||||
-- "until", "said"}], state `waiting`, `replaced-when-idle` or `replaced-at-bound`. Kept with the machine's
|
||||
-- last account, so a walk's first-machine gate waits while its module's container is listed waiting — across
|
||||
-- a restart of the controller, which a merge into mesh-controller makes in the same walk that moves the build
|
||||
-- agent. Empty from an engine that never waits.
|
||||
alter table node_report add column busy jsonb not null default '[]';
|
||||
+86
-11
@@ -10,6 +10,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -846,6 +847,46 @@ type Doing struct {
|
||||
// machine doing what it was told.
|
||||
Since *time.Time
|
||||
Times int
|
||||
|
||||
// Busy is every container the apply left running because it said busy, and every one it replaced after
|
||||
// it waited (novox/hq ADR 0335).
|
||||
Busy []BusyWait
|
||||
}
|
||||
|
||||
// BusyWait is one container a busy command held, as the machine reported it (novox/hq ADR 0335): its
|
||||
// resource id (`<module>.<resource>`), its name, its state (link.BusyWaiting, BusyReplacedIdle or
|
||||
// BusyReplacedAtBound), since when it waits, until when at the latest, and the last output of its command.
|
||||
type BusyWait struct {
|
||||
ID string `json:"id"`
|
||||
Container string `json:"container"`
|
||||
State string `json:"state"`
|
||||
Since time.Time `json:"since"`
|
||||
Until time.Time `json:"until"`
|
||||
Said string `json:"said,omitempty"`
|
||||
}
|
||||
|
||||
// BusyWaiting is the state of a container left running because it said busy (link.BusyWaiting).
|
||||
const BusyWaiting = "waiting"
|
||||
|
||||
// WaitingFor is the container of a module that waits because it said busy, and whether one does: a
|
||||
// resource whose id is `<module>.<resource>`.
|
||||
func WaitingFor(busy []BusyWait, module string) (BusyWait, bool) {
|
||||
for _, b := range busy {
|
||||
if b.State == BusyWaiting && strings.HasPrefix(b.ID, module+".") {
|
||||
return b, true
|
||||
}
|
||||
}
|
||||
return BusyWait{}, false
|
||||
}
|
||||
|
||||
// Waiting says whether any container waits because it said busy.
|
||||
func (d Doing) Waiting() bool {
|
||||
for _, b := range d.Busy {
|
||||
if b.State == BusyWaiting {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FailedResource is one thing a node could not do.
|
||||
@@ -873,6 +914,20 @@ func (d Doing) Stuck() bool { return d.Wrong() && d.Times >= StuckAfter && d.Sin
|
||||
// the same failed resources BY ID. Not by the host's words: an error that carries a duration, a
|
||||
// counter or a temporary path would read as new on every report, and the resource looping on it —
|
||||
// which is what stuck is for — would never be said to be (novox/hq 04-ISSUES/065).
|
||||
// sameBusy says two reports hold the same containers in the same state of a busy wait: its output, which
|
||||
// may change on every ask, is not compared.
|
||||
func sameBusy(a, b []BusyWait) bool {
|
||||
key := func(list []BusyWait) []string {
|
||||
out := make([]string, 0, len(list))
|
||||
for _, w := range list {
|
||||
out = append(out, w.ID+"="+w.State)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
return slices.Equal(key(a), key(b))
|
||||
}
|
||||
|
||||
func sameFailure(a, b Doing) bool {
|
||||
if a.Outcome != b.Outcome || a.Refused != b.Refused || len(a.Failed) != len(b.Failed) {
|
||||
return false
|
||||
@@ -937,13 +992,20 @@ func recordDoing(ctx context.Context, q queries, node string, d Doing) (news boo
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if d.Busy == nil {
|
||||
d.Busy = []BusyWait{}
|
||||
}
|
||||
busy, err := json.Marshal(d.Busy)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
var before Doing
|
||||
var beforeFailed []byte
|
||||
var beforeFailed, beforeBusy []byte
|
||||
found := q.QueryRow(ctx,
|
||||
`select outcome, refused, failed, failing_since, failures, coalesce(declared,'')
|
||||
`select outcome, refused, failed, failing_since, failures, coalesce(declared,''), busy
|
||||
from node_report where node = $1`,
|
||||
node).Scan(&before.Outcome, &before.Refused, &beforeFailed, &before.Since, &before.Times,
|
||||
&before.Declared)
|
||||
&before.Declared, &beforeBusy)
|
||||
switch {
|
||||
case errors.Is(found, pgx.ErrNoRows):
|
||||
news = true
|
||||
@@ -953,7 +1015,13 @@ func recordDoing(ctx context.Context, q queries, node string, d Doing) (news boo
|
||||
if err := json.Unmarshal(beforeFailed, &before.Failed); err != nil {
|
||||
return false, err
|
||||
}
|
||||
news = before.Outcome != d.Outcome || before.Declared != d.Declared || !sameFailure(before, d)
|
||||
if err := json.Unmarshal(beforeBusy, &before.Busy); err != nil {
|
||||
return false, err
|
||||
}
|
||||
// A container that starts or ends a wait because it said busy is news (novox/hq ADR 0335): the walk
|
||||
// waiting on this machine is told by it.
|
||||
news = before.Outcome != d.Outcome || before.Declared != d.Declared || !sameFailure(before, d) ||
|
||||
!sameBusy(before.Busy, d.Busy)
|
||||
}
|
||||
var since *time.Time
|
||||
times := 0
|
||||
@@ -966,13 +1034,13 @@ func recordDoing(ctx context.Context, q queries, node string, d Doing) (news boo
|
||||
}
|
||||
_, err = q.Exec(ctx,
|
||||
`insert into node_report (node, outcome, refused, failed, applied, at, declared,
|
||||
failing_since, failures)
|
||||
values ($1, $2, $3, $4, $5, now(), $6, $7, $8)
|
||||
failing_since, failures, busy)
|
||||
values ($1, $2, $3, $4, $5, now(), $6, $7, $8, $9)
|
||||
on conflict (node) do update set outcome = excluded.outcome, refused = excluded.refused,
|
||||
failed = excluded.failed, applied = excluded.applied, at = excluded.at,
|
||||
declared = excluded.declared,
|
||||
failing_since = excluded.failing_since, failures = excluded.failures`,
|
||||
node, d.Outcome, d.Refused, failed, d.Applied, d.Declared, since, times)
|
||||
failing_since = excluded.failing_since, failures = excluded.failures, busy = excluded.busy`,
|
||||
node, d.Outcome, d.Refused, failed, d.Applied, d.Declared, since, times, busy)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -1059,6 +1127,8 @@ type Reported struct {
|
||||
// gate holds against the send it made, rather than against the send made last (novox/hq issue 352).
|
||||
Declared string
|
||||
ReportedSequence int64
|
||||
// Busy is what the last report said of containers held by their busy command (novox/hq ADR 0335).
|
||||
Busy []BusyWait
|
||||
}
|
||||
|
||||
// SentDeclaration is what one send carried to a machine, as a gate keeps it: the declaration's digest and
|
||||
@@ -1112,8 +1182,8 @@ func (i *Inventory) SentTo(ctx context.Context, name string) (SentDeclaration, b
|
||||
func (i *Inventory) LastReports(ctx context.Context) ([]Reported, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select n.name, coalesce(r.outcome, ''), r.at, n.sent_at,
|
||||
r.declared is not null and r.declared <> '' and r.declared = n.sent,
|
||||
coalesce(r.declared, ''), coalesce(r.reported_sequence, 0)
|
||||
coalesce(r.declared is not null and r.declared <> '' and r.declared = n.sent, false),
|
||||
coalesce(r.declared, ''), coalesce(r.reported_sequence, 0), coalesce(r.busy, '[]')
|
||||
from node n left join node_report r on r.node = n.id
|
||||
order by n.name`)
|
||||
if err != nil {
|
||||
@@ -1123,7 +1193,12 @@ func (i *Inventory) LastReports(ctx context.Context) ([]Reported, error) {
|
||||
var out []Reported
|
||||
for rows.Next() {
|
||||
var r Reported
|
||||
if err := rows.Scan(&r.Node, &r.Outcome, &r.At, &r.Sent, &r.Current, &r.Declared, &r.ReportedSequence); err != nil {
|
||||
var busy []byte
|
||||
if err := rows.Scan(&r.Node, &r.Outcome, &r.At, &r.Sent, &r.Current, &r.Declared, &r.ReportedSequence,
|
||||
&busy); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(busy, &r.Busy); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, r)
|
||||
|
||||
+106
-2
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -387,6 +388,101 @@ type PlanGate struct {
|
||||
// alone to the machine holding the bus, every build there kept, so what the send newly grants is on the
|
||||
// bus before the gate judges it. Nil when the list did not change, or the gate's machine holds the bus.
|
||||
Grants *GrantsStep `json:"grants,omitempty"`
|
||||
// Busy is the wait for a busy container on a judged machine (novox/hq ADR 0335 §4): while the machine
|
||||
// reports its module's container left running because it said busy, the judging waits and has not
|
||||
// started; once it reports the container replaced, Since is moved to then. Nil when nothing waited.
|
||||
Busy *GateBusy `json:"busy,omitempty"`
|
||||
// Pending says the send this gate judges is kept and not yet made (novox/hq ADR 0353 §8): a controller that
|
||||
// takes over reads the machine's last send before making it, and never sends it twice.
|
||||
Pending bool `json:"pending,omitempty"`
|
||||
// Silent says the machine reported nothing on its send within ApplySilentAfter (novox/hq ADR 0353 §4): left,
|
||||
// neither failed nor put back.
|
||||
Silent bool `json:"silent,omitempty"`
|
||||
// TookOver names the gate whose readings this one took over (novox/hq ADR 0353 §7): another walk's judging of
|
||||
// the same build on the same machine, which this gate goes on from instead of starting from zero.
|
||||
TookOver string `json:"took_over,omitempty"`
|
||||
}
|
||||
|
||||
// TakeReadings makes this gate go on from another's readings of the same build on the same machine (novox/hq ADR
|
||||
// 0353 §7): when the judging began, when each machine applied, the passes counted and when, and what each found.
|
||||
// A module's own count is taken only for the modules this gate judges.
|
||||
func (g *PlanGate) TakeReadings(from *PlanGate, modules []string, id string) {
|
||||
if from == nil {
|
||||
return
|
||||
}
|
||||
if from.Since != nil {
|
||||
since := *from.Since
|
||||
g.Since = &since
|
||||
}
|
||||
for n, s := range from.Sent {
|
||||
if slices.Contains(g.Machines, n) {
|
||||
if g.Sent == nil {
|
||||
g.Sent = map[string]SentDeclaration{}
|
||||
}
|
||||
g.Sent[n] = s
|
||||
}
|
||||
}
|
||||
for n, at := range from.Applied {
|
||||
if slices.Contains(g.Machines, n) {
|
||||
if g.Applied == nil {
|
||||
g.Applied = map[string]time.Time{}
|
||||
}
|
||||
g.Applied[n] = at
|
||||
}
|
||||
}
|
||||
g.Passes, g.Faulted, g.HealthySeen, g.Last = from.Passes, from.Faulted, from.HealthySeen, from.Last
|
||||
if from.LastPass != nil {
|
||||
at := *from.LastPass
|
||||
g.LastPass = &at
|
||||
}
|
||||
for _, m := range modules {
|
||||
if n, counted := from.Healthy[m]; counted {
|
||||
if g.Healthy == nil {
|
||||
g.Healthy = map[string]int{}
|
||||
}
|
||||
g.Healthy[m] = n
|
||||
}
|
||||
if at, counted := from.HealthyAt[m]; counted {
|
||||
if g.HealthyAt == nil {
|
||||
g.HealthyAt = map[string]time.Time{}
|
||||
}
|
||||
g.HealthyAt[m] = at
|
||||
}
|
||||
}
|
||||
g.Readings = append([]GateReading(nil), from.Readings...)
|
||||
g.TookOver = id
|
||||
}
|
||||
|
||||
// GateBusy is a first-node gate's wait for a busy container (novox/hq ADR 0335 §4).
|
||||
type GateBusy struct {
|
||||
Machine string `json:"machine"`
|
||||
Module string `json:"module"`
|
||||
Container string `json:"container"`
|
||||
// Since is when the machine's first apply that would have replaced it ran, Until when its node-engine
|
||||
// replaces it at the latest; Ended when the machine reported it replaced and the judging began — or when
|
||||
// the controller stopped waiting past the engine's bound, which PastBound says.
|
||||
Since time.Time `json:"since"`
|
||||
Until time.Time `json:"until"`
|
||||
// Seen is when this controller first read the wait, on its own clock: the gate's bound runs from it, never
|
||||
// from the engine's word alone.
|
||||
Seen time.Time `json:"seen,omitzero"`
|
||||
Ended *time.Time `json:"ended,omitempty"`
|
||||
PastBound bool `json:"past_bound,omitempty"`
|
||||
// Said is the last output of its busy command, as the machine reported it.
|
||||
Said string `json:"said,omitempty"`
|
||||
}
|
||||
|
||||
// Waiting says the gate waits for a busy container and has not started.
|
||||
func (g *PlanGate) Waiting() bool { return g != nil && g.Busy != nil && g.Busy.Ended == nil }
|
||||
|
||||
// Words is the wait in a line, as the walk says it.
|
||||
func (b *GateBusy) Words() string {
|
||||
said := ""
|
||||
if b.Said != "" {
|
||||
said = "; its busy command says: " + b.Said
|
||||
}
|
||||
return fmt.Sprintf("waits for a busy container on %s: %s holds work, and is replaced once it says idle, by %s "+
|
||||
"at the latest (novox/hq ADR 0335)%s", b.Machine, b.Container, b.Until.UTC().Format(time.RFC3339), said)
|
||||
}
|
||||
|
||||
// GrantsStep is the bus's user list sent to the machine holding the bus ahead of a gated send (novox/hq
|
||||
@@ -441,9 +537,17 @@ type CarriedMove struct {
|
||||
type PlanRelease struct {
|
||||
Order []string `json:"order"`
|
||||
Next int `json:"next"`
|
||||
// Gate is the machine being judged; nil between machines.
|
||||
// Gate is the first machine being judged; nil between machines.
|
||||
Gate *PlanGate `json:"gate,omitempty"`
|
||||
Done []string `json:"done,omitempty"`
|
||||
// Rest says the first machine passed (novox/hq ADR 0353 §1): every other machine is sent at once, each judged by
|
||||
// its own gate in Gates, by machine — a build no gate passed going to one machine of a step only (§2).
|
||||
Rest bool `json:"rest,omitempty"`
|
||||
Gates map[string]*PlanGate `json:"gates,omitempty"`
|
||||
// Failed are the machines of the rest whose gate failed, and Silent those that said nothing on their send
|
||||
// within ApplySilentAfter (ADR 0353 §3 and §4).
|
||||
Failed []string `json:"failed,omitempty"`
|
||||
Silent []string `json:"silent,omitempty"`
|
||||
Done []string `json:"done,omitempty"`
|
||||
// Skipped are the machines not heard from when their turn came, left as they were.
|
||||
Skipped []string `json:"skipped,omitempty"`
|
||||
// By is the person who released it, empty when the mesh did.
|
||||
|
||||
@@ -26,6 +26,7 @@ const (
|
||||
PhaseBetween = "between-tiers" // one tier's end to the next tier's ask
|
||||
PhaseBuild = "build" // a tier asked to its last module built
|
||||
PhaseSend = "send-first" // built to sent to the first machines
|
||||
PhaseBusy = "busy-wait" // sent first to a busy container there replaced, when one waited (ADR 0335)
|
||||
PhaseJudge = "judgement" // sent first to the first-node gate's last verdict: its readings
|
||||
PhaseRest = "send-rest" // judged to sent to the rest
|
||||
PhaseApply = "apply" // the last send to every machine of the rest reporting the build applied
|
||||
@@ -90,6 +91,11 @@ type WalkPhase struct {
|
||||
type AppliedReport struct {
|
||||
At time.Time
|
||||
Outcome string
|
||||
// Answered is, for a machine that waited for a busy container (novox/hq ADR 0335), its first report of the
|
||||
// send; Busy how long it then waited; Waiting that it still waits, At being its first report then.
|
||||
Answered time.Time
|
||||
Busy time.Duration
|
||||
Waiting bool
|
||||
}
|
||||
|
||||
// AppliedLookup answers a machine's report of the send the walk made to it: send is what the walk kept of it, and
|
||||
@@ -108,9 +114,13 @@ type RestReport struct {
|
||||
Reported *time.Time `json:"reported,omitempty"`
|
||||
Took string `json:"took,omitempty"`
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
// State is applied, failed, waiting (not yet heard, within ApplySilentAfter) or silent (not heard within it:
|
||||
// listed, never moving the walk's end).
|
||||
// Busy is how long it waited for a busy container before it ran the build (novox/hq ADR 0335): inside Took,
|
||||
// never read as the machine being silent.
|
||||
Busy string `json:"busy,omitempty"`
|
||||
// State is applied, failed, waiting (not yet heard, within ApplySilentAfter), busy (it answered and waits for
|
||||
// a busy container: never silent) or silent (not heard within it: listed, never moving the walk's end).
|
||||
State string `json:"state"`
|
||||
busy time.Duration
|
||||
}
|
||||
|
||||
// The states of a machine of the rest.
|
||||
@@ -118,6 +128,7 @@ const (
|
||||
RestApplied = "applied"
|
||||
RestFailed = "failed"
|
||||
RestWaiting = "waiting"
|
||||
RestBusy = "busy"
|
||||
RestSilent = "silent"
|
||||
)
|
||||
|
||||
@@ -300,8 +311,11 @@ func (p Plan) points(now time.Time, applied AppliedLookup, w *WalkPhases) []poin
|
||||
if t > p.Tier || (t == p.Tier && p.Tier < len(p.Tiers) && !askedAnyOf(p, tier)) {
|
||||
break
|
||||
}
|
||||
var asked, built, first, judged, rest *time.Time
|
||||
var asked, built, first, judged, rest, busyEnded *time.Time
|
||||
allBuilt, anyFirst, allJudged, allRest := true, false, true, true
|
||||
// A busy container on a first machine holds the judging until it is replaced (novox/hq ADR 0335 §4): a
|
||||
// phase of its own, only in a tier where one waited, counted in the delivery time and never exempted.
|
||||
anyBusy, allBusyEnded, busyOn := false, true, ""
|
||||
for _, m := range tier {
|
||||
s := p.Modules[m]
|
||||
if s == nil {
|
||||
@@ -319,6 +333,14 @@ func (p Plan) points(now time.Time, applied AppliedLookup, w *WalkPhases) []poin
|
||||
if s.FirstAt != nil {
|
||||
anyFirst = true
|
||||
first = earliest(first, s.FirstAt)
|
||||
if s.Gate != nil && s.Gate.Busy != nil {
|
||||
anyBusy = true
|
||||
if s.Gate.Busy.Ended == nil {
|
||||
allBusyEnded, busyOn = false, s.Gate.Busy.Machine
|
||||
} else {
|
||||
busyEnded = latest(busyEnded, s.Gate.Busy.Ended)
|
||||
}
|
||||
}
|
||||
if s.Gate != nil {
|
||||
if s.Gate.JudgedAt == nil {
|
||||
allJudged = false
|
||||
@@ -349,7 +371,15 @@ func (p Plan) points(now time.Time, applied AppliedLookup, w *WalkPhases) []poin
|
||||
if !allJudged {
|
||||
judged = nil
|
||||
}
|
||||
out = append(out, point{phase: PhaseSend, tier: t, at: first}, point{phase: PhaseJudge, tier: t, at: judged})
|
||||
out = append(out, point{phase: PhaseSend, tier: t, at: first})
|
||||
if anyBusy {
|
||||
said := ""
|
||||
if !allBusyEnded {
|
||||
busyEnded, said = nil, "waits for a busy container on "+busyOn
|
||||
}
|
||||
out = append(out, point{phase: PhaseBusy, tier: t, at: busyEnded, said: said})
|
||||
}
|
||||
out = append(out, point{phase: PhaseJudge, tier: t, at: judged})
|
||||
} else {
|
||||
out = append(out, point{phase: PhaseSend, tier: t, none: true, said: "no first machine: nothing to judge"},
|
||||
point{phase: PhaseJudge, tier: t, none: true, said: "no first machine: nothing to judge"})
|
||||
@@ -393,6 +423,13 @@ func (p Plan) points(now time.Time, applied AppliedLookup, w *WalkPhases) []poin
|
||||
if r, ok := applied(s.node, s.what, s.walkSent); ok {
|
||||
got := r.At
|
||||
rr.Reported, rr.Outcome, rr.Took = &got, r.Outcome, words(r.At.Sub(at))
|
||||
if r.Busy > 0 {
|
||||
rr.Busy = words(r.Busy)
|
||||
rr.busy = r.Busy
|
||||
}
|
||||
if r.Waiting {
|
||||
rr.Reported, rr.Took, rr.State = nil, "", RestBusy
|
||||
}
|
||||
}
|
||||
reports[key] = rr
|
||||
order = append(order, key)
|
||||
@@ -409,16 +446,20 @@ func (p Plan) points(now time.Time, applied AppliedLookup, w *WalkPhases) []poin
|
||||
if later.Node == rr.Node && later.Sent.After(rr.Sent) && later.Reported != nil &&
|
||||
(rr.Reported == nil || later.Reported.Before(*rr.Reported)) {
|
||||
got := *later.Reported
|
||||
rr.Reported, rr.Outcome, rr.Took = &got, later.Outcome, words(got.Sub(rr.Sent))
|
||||
rr.Reported, rr.Outcome, rr.Took, rr.State = &got, later.Outcome, words(got.Sub(rr.Sent)), ""
|
||||
}
|
||||
}
|
||||
}
|
||||
var end *time.Time
|
||||
var waiting, failed []string
|
||||
var waiting, failed, busy []string
|
||||
for _, k := range order {
|
||||
rr := reports[k]
|
||||
switch {
|
||||
case rr.Reported == nil && now.Sub(rr.Sent) > ApplySilentAfter, rr.Reported != nil && rr.Reported.Sub(rr.Sent) > ApplySilentAfter:
|
||||
case rr.State == RestBusy:
|
||||
// It answered and waits for a busy container: never silent, and the walk's end waits for it.
|
||||
busy = appendOnce(busy, rr.Node)
|
||||
case rr.Reported == nil && now.Sub(rr.Sent) > ApplySilentAfter,
|
||||
rr.Reported != nil && rr.Reported.Sub(rr.Sent)-rr.busy > ApplySilentAfter:
|
||||
rr.State = RestSilent
|
||||
w.Silent = appendOnce(w.Silent, rr.Node)
|
||||
case rr.Reported == nil:
|
||||
@@ -443,9 +484,15 @@ func (p Plan) points(now time.Time, applied AppliedLookup, w *WalkPhases) []poin
|
||||
switch {
|
||||
case len(failed) > 0:
|
||||
return append(out, point{phase: PhaseApply, tier: -1, said: "not applied on " + strings.Join(failed, ", ")})
|
||||
case len(waiting) > 0:
|
||||
return append(out, point{phase: PhaseApply, tier: -1, said: "waiting for " + strings.Join(waiting, ", ") +
|
||||
" to report it applied"})
|
||||
case len(waiting) > 0 || len(busy) > 0:
|
||||
var said []string
|
||||
if len(waiting) > 0 {
|
||||
said = append(said, "waiting for "+strings.Join(waiting, ", ")+" to report it applied")
|
||||
}
|
||||
if len(busy) > 0 {
|
||||
said = append(said, "waits for a busy container on "+strings.Join(busy, ", "))
|
||||
}
|
||||
return append(out, point{phase: PhaseApply, tier: -1, said: strings.Join(said, "; ")})
|
||||
case end == nil:
|
||||
return append(out, point{phase: PhaseApply, tier: -1, said: "no machine of the rest heard from: " +
|
||||
strings.Join(w.Silent, ", ")})
|
||||
@@ -536,19 +583,19 @@ func words(d time.Duration) string {
|
||||
func (i *Inventory) FirstAppliedAfter(ctx context.Context, node string, send SentDeclaration, walkSent time.Time) (AppliedReport, bool, error) {
|
||||
from, to := applyRowsBetween(send, walkSent)
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select ref, started, took_ms, detail from duration
|
||||
where kind = $1 and subject = $2 and started >= $3 and started < $4
|
||||
`select kind, ref, started, took_ms, detail from duration
|
||||
where subject = $2 and ((kind = $1 and started >= $3 and started < $4) or (kind = $5 and started >= $3))
|
||||
order by started`,
|
||||
DurationApply, node, from, to)
|
||||
DurationApply, node, from, to, DurationBusy)
|
||||
if err != nil {
|
||||
return AppliedReport{}, false, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var ds []Duration
|
||||
for rows.Next() {
|
||||
d := Duration{Kind: DurationApply, Subject: node, Node: node}
|
||||
d := Duration{Subject: node, Node: node}
|
||||
var ms int64
|
||||
if err := rows.Scan(&d.Ref, &d.Started, &ms, &d.Detail); err != nil {
|
||||
if err := rows.Scan(&d.Kind, &d.Ref, &d.Started, &ms, &d.Detail); err != nil {
|
||||
return AppliedReport{}, false, err
|
||||
}
|
||||
d.Took = time.Duration(ms) * time.Millisecond
|
||||
@@ -583,16 +630,30 @@ func applyRowsBetween(send SentDeclaration, walkSent time.Time) (time.Time, time
|
||||
// before the walk's record of its send, as it was read before.
|
||||
//
|
||||
// A send past ApplySilentAfter is never read: its machine slept, and is listed silent (ADR 0282 decision 1).
|
||||
//
|
||||
// **A first report that said a container waits because it is busy** (novox/hq ADR 0335) is the machine's answer,
|
||||
// and not yet the build running: its report is the end of its busy wait, the `busy-wait` duration of the same
|
||||
// send, with the wait said apart (Busy). While none is kept the machine is still waiting (Waiting), never silent.
|
||||
func FirstReportOf(ds []Duration, send SentDeclaration, walkSent time.Time) (AppliedReport, bool) {
|
||||
from, to := applyRowsBetween(send, walkSent)
|
||||
for _, d := range ds {
|
||||
if d.Started.Before(from) || !d.Started.Before(to) {
|
||||
if d.Kind == DurationBusy || d.Started.Before(from) || !d.Started.Before(to) {
|
||||
continue
|
||||
}
|
||||
if send.Digest != "" && !strings.HasPrefix(d.Ref, send.Digest+"@") {
|
||||
continue
|
||||
}
|
||||
return AppliedReport{At: d.Started.Add(d.Took).UTC(), Outcome: d.Detail}, true
|
||||
answered := d.Started.Add(d.Took).UTC()
|
||||
if d.Detail != ApplyBusy {
|
||||
return AppliedReport{At: answered, Outcome: d.Detail}, true
|
||||
}
|
||||
for _, b := range ds {
|
||||
if b.Kind == DurationBusy && b.Ref == d.Ref {
|
||||
return AppliedReport{At: b.Started.Add(b.Took).UTC(), Outcome: OutcomeApplied, Answered: answered,
|
||||
Busy: b.Took}, true
|
||||
}
|
||||
}
|
||||
return AppliedReport{At: answered, Outcome: OutcomeApplied, Answered: answered, Waiting: true}, true
|
||||
}
|
||||
return AppliedReport{}, false
|
||||
}
|
||||
|
||||
@@ -24,9 +24,10 @@ import (
|
||||
// which consumer of whose, which subject — for the watchdog to say as a condition, as the refused
|
||||
// reply of issue 265 is said today.
|
||||
//
|
||||
// What the bus says about **other** principals' connections — a module's slow consumer, a module
|
||||
// refused a subject — the server publishes only to a system account, which the mesh's bus does not
|
||||
// have; that half is not heard yet (to-be 45 S9, recorded as deferred).
|
||||
// What the bus does to **other** principals' connections the server does not publish in the mesh's
|
||||
// account. A module's slow consumer it closes and records, and the bus module says that record on the
|
||||
// bus (busfaults.go, novox/hq ADR 0357). A module refused a subject is told and written in the server's
|
||||
// log, and nowhere else: not heard yet (novox/hq issue 702).
|
||||
|
||||
// The advisory kinds, as conditions name them.
|
||||
const (
|
||||
@@ -48,6 +49,11 @@ type Advisory struct {
|
||||
Stream, Consumer string
|
||||
// Token is the condition key's last part, when it is not Kind.
|
||||
Token string
|
||||
// User is the bus user whose connection it is about, when it is about one (novox/hq ADR 0357): the
|
||||
// controller's own, or another user's the bus module said.
|
||||
User string
|
||||
// Reason is the server's reason for closing that connection, as it gave it.
|
||||
Reason string
|
||||
// Said is the newest saying, in the mesh's words.
|
||||
Said string
|
||||
First, Last time.Time
|
||||
|
||||
@@ -164,6 +164,30 @@ type CheckRequest struct {
|
||||
// each, from the merge it judged, into the scratch namespace once it has judged and not failed, for the walk to
|
||||
// take. A build seat from before it builds nothing, and the walk builds as before.
|
||||
Builds []CheckBuild `json:"builds,omitempty"`
|
||||
// Try marks a branch tried before its pull request (novox/hq issue 686): the repository's own check for a
|
||||
// pushed branch or commit, asked by a builder through the controller's `try` verb. Its outcome is read from
|
||||
// the build's log (StepTried) and said to nobody: no status on the forge, nothing recorded or registered,
|
||||
// nothing built for a merge. Tests narrows what the repository's script runs to the tests a pattern names,
|
||||
// and StoreOnly to its store tests; a script that reads neither runs whole.
|
||||
Try bool `json:"try,omitempty"`
|
||||
Tests string `json:"tests,omitempty"`
|
||||
StoreOnly bool `json:"store-only,omitempty"`
|
||||
}
|
||||
|
||||
// StepTried is the step a build seat says a try's outcome under, in the build's log: one line, a Tried as JSON
|
||||
// (novox/hq issue 686).
|
||||
const StepTried = "tried"
|
||||
|
||||
// Tried is what came of a branch tried before its pull request (novox/hq issue 686): the verdict of the
|
||||
// repository's own check for the ref merged into the trunk, how long it took, and the lines that name what
|
||||
// failed, picked from the whole of its output (issue 460).
|
||||
type Tried struct {
|
||||
Verdict string `json:"verdict"`
|
||||
Summary string `json:"summary"`
|
||||
Took string `json:"took"`
|
||||
Head string `json:"head,omitempty"`
|
||||
Trunk string `json:"trunk,omitempty"`
|
||||
Failed []string `json:"failed,omitempty"`
|
||||
}
|
||||
|
||||
// CheckBuild is one module a check builds for its merge: from the merge it made when Merged — a module built from
|
||||
|
||||
@@ -96,8 +96,27 @@ type BuildStart struct {
|
||||
// an earlier taker left unfinished — most often a build agent replaced mid-build — and started
|
||||
// again from the beginning (novox/hq issue 594). 0 from a machine that predates it.
|
||||
Delivery int `json:"delivery,omitempty"`
|
||||
// Try marks the start of a branch tried before its pull request (novox/hq issue 686), which names no
|
||||
// repository and no ref (startOf).
|
||||
Try bool `json:"try,omitempty"`
|
||||
}
|
||||
|
||||
// startOf is what a build machine says of the ask it took. **A try's start names no repository and no commit**
|
||||
// (novox/hq issue 686): the forge's holder writes mesh/merge-gate pending on an open pull request whose head is
|
||||
// the commit a start names, and a try of that head is no check of the pull request — nothing on the forge moves
|
||||
// for a try. Its id, machine and time are all the queue reads of a start.
|
||||
func startOf(r BuildRequest, on string, delivery int, at time.Time) BuildStart {
|
||||
s := BuildStart{ID: r.ID, Repository: r.Repository, Path: r.Path, Ref: r.Ref, On: on,
|
||||
At: at.UTC().Format(time.RFC3339Nano), Delivery: delivery}
|
||||
if IsTry(r) {
|
||||
s.Repository, s.Path, s.Ref, s.Try = "", "", "", true
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// IsTry says an ask is a branch tried before its pull request (novox/hq issue 686).
|
||||
func IsTry(r BuildRequest) bool { return r.Check != nil && r.Check.Try }
|
||||
|
||||
// Restarted is how a build handed out again says so (novox/hq issue 594), in its log and in its check's
|
||||
// report, so a second run never reads as the first. Neutral on why: a holder replaced mid-build, a paused
|
||||
// holder handing an ask back as it arrived, and a bus that held its acknowledgements past the wait all
|
||||
|
||||
@@ -175,6 +175,11 @@ type MachineOptions struct {
|
||||
// can take a build, which is what its container's health check reads (novox/hq issue 544). Nil is
|
||||
// told nothing.
|
||||
Taking func()
|
||||
// Claim is asked once a request is fetched, just before it is handed over: false hands it back to the
|
||||
// queue at once, for another holder. A holder being drained says so here (novox/hq ADR 0335): it marks
|
||||
// the request in hand first and reads its drain after, so an ask of `busy` racing the fetch sees one or
|
||||
// the other, never neither. Nil claims every one.
|
||||
Claim func() bool
|
||||
}
|
||||
|
||||
// MachineOverNATS takes build work from the current build role.
|
||||
@@ -317,6 +322,12 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
endCancelled(ctx, build)
|
||||
continue
|
||||
}
|
||||
// **Drained in the moment it was fetched** (novox/hq ADR 0335): handed back at once, as a pause
|
||||
// landing then is.
|
||||
if m.opts.Claim != nil && !m.opts.Claim() {
|
||||
_ = msg.Nak()
|
||||
continue
|
||||
}
|
||||
// A build outlives the acknowledgement window many times over; said while it runs,
|
||||
// as the controller says it for its own long handlers, so the server neither hands
|
||||
// the ask to a second machine nor counts the wait against its deliveries.
|
||||
@@ -390,10 +401,7 @@ func (b *natsBuild) Done() error { return b.msg.Ack() }
|
||||
// Began publishes that this machine has taken the build, into the stream like the outcome, so a
|
||||
// reader that asks afterwards sees when it started as well as how it ended.
|
||||
func (b *natsBuild) Began(ctx context.Context) error {
|
||||
body, err := json.Marshal(BuildStart{
|
||||
ID: b.request.ID, Repository: b.request.Repository, Path: b.request.Path,
|
||||
Ref: b.request.Ref, On: b.on, At: time.Now().UTC().Format(time.RFC3339Nano), Delivery: b.delivery,
|
||||
})
|
||||
body, err := json.Marshal(startOf(b.request, b.on, b.delivery, time.Now()))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// What the bus did to other users' connections, as the bus module says it (novox/hq ADR 0357, issue 270).
|
||||
//
|
||||
// **The server says another user's slow consumer only in its record of closed connections**, which the
|
||||
// bus module reads on its own machine and says as events: `bus.slow-consumer` for a connection the server
|
||||
// closed as a slow consumer, `bus.connection-closed` for one it closed for breaking the bus's rules. They
|
||||
// are read back from EVENTS on every watchdog tick for the hour S9 says an advisory, rather than taken by
|
||||
// the controller's durable consumer: that one takes a message at a time while a merge builds for
|
||||
// minutes, and read back, a controller that restarts says the last hour again without keeping anything.
|
||||
//
|
||||
// **Said at least once, counted once.** The module says the last hour again whenever it starts; a closure
|
||||
// is its user, its connection id and when it opened, and is counted once however often it is said.
|
||||
|
||||
// BusModule is the module whose events say what the bus did to its users' connections.
|
||||
const BusModule = "nats"
|
||||
|
||||
// The bus module's events, by their local names (its manifest's `emits`).
|
||||
const (
|
||||
BusSlowConsumerEvent = "bus.slow-consumer"
|
||||
BusConnectionClosedEvent = "bus.connection-closed"
|
||||
// BusWatchEvent is the bus module's watch saying it is there after every look, and whether it is healthy
|
||||
// (novox/hq ADR 0357, review 274): a watch that stops saying it is a watch nobody would otherwise hear.
|
||||
BusWatchEvent = "bus.watch"
|
||||
)
|
||||
|
||||
// AdvisoryConnectionClosed is a connection the bus closed for breaking its rules, as conditions name it.
|
||||
const AdvisoryConnectionClosed = "connection-closed"
|
||||
|
||||
// BusClosure is one closed connection as the bus module says it: the user, never a credential.
|
||||
type BusClosure struct {
|
||||
User string `json:"user"`
|
||||
Name string `json:"name,omitempty"`
|
||||
CID uint64 `json:"cid"`
|
||||
Reason string `json:"reason"`
|
||||
Opened time.Time `json:"opened"`
|
||||
Closed time.Time `json:"closed"`
|
||||
Pending int64 `json:"pending_bytes,omitempty"`
|
||||
}
|
||||
|
||||
func (c BusClosure) identity() string {
|
||||
return fmt.Sprintf("%s/%d/%s", c.User, c.CID, c.Opened.UTC().Format(time.RFC3339Nano))
|
||||
}
|
||||
|
||||
// BusClosures are the advisories the bus module's closures make, closed at or after since: one per user
|
||||
// and kind, each closure counted once, the newest said.
|
||||
func BusClosures(closures []BusClosure, kinds []string, since time.Time) []Advisory {
|
||||
type entry struct {
|
||||
a *Advisory
|
||||
seen map[string]bool
|
||||
newest BusClosure
|
||||
}
|
||||
by := map[string]*entry{}
|
||||
var order []string
|
||||
for i, c := range closures {
|
||||
if c.User == "" || c.Closed.Before(since) {
|
||||
continue
|
||||
}
|
||||
kind := kinds[i]
|
||||
key := kind + "/" + c.User
|
||||
e := by[key]
|
||||
if e == nil {
|
||||
e = &entry{a: &Advisory{Kind: kind, ID: c.User, User: c.User}, seen: map[string]bool{}}
|
||||
by[key] = e
|
||||
order = append(order, key)
|
||||
}
|
||||
if e.seen[c.identity()] {
|
||||
continue
|
||||
}
|
||||
e.seen[c.identity()] = true
|
||||
e.a.Count++
|
||||
if e.a.First.IsZero() || c.Closed.Before(e.a.First) {
|
||||
e.a.First = c.Closed
|
||||
}
|
||||
if !c.Closed.Before(e.a.Last) {
|
||||
e.a.Last, e.newest = c.Closed, c
|
||||
}
|
||||
}
|
||||
sort.Strings(order)
|
||||
out := make([]Advisory, 0, len(order))
|
||||
for _, key := range order {
|
||||
e := by[key]
|
||||
c := e.newest
|
||||
name := ""
|
||||
if c.Name != "" && c.Name != c.User {
|
||||
name = fmt.Sprintf(" (connection %q)", c.Name)
|
||||
}
|
||||
pending := ""
|
||||
if c.Pending > 0 {
|
||||
pending = fmt.Sprintf(", %d bytes still to be written to it", c.Pending)
|
||||
}
|
||||
switch e.a.Kind {
|
||||
case AdvisorySlowConsumer:
|
||||
e.a.Said = fmt.Sprintf("the bus closed %s's connection %d%s as a slow consumer: %s%s", c.User, c.CID,
|
||||
name, c.Reason, pending)
|
||||
default:
|
||||
e.a.Said = fmt.Sprintf("the bus closed %s's connection %d%s: %s", c.User, c.CID, name, c.Reason)
|
||||
}
|
||||
e.a.Reason = c.Reason
|
||||
out = append(out, *e.a)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// busEventKind is the advisory kind of one of the bus module's events, by its subject.
|
||||
func busEventKind(subject string) (string, bool) {
|
||||
switch subject {
|
||||
case EventSubject(BusModule, BusSlowConsumerEvent):
|
||||
return AdvisorySlowConsumer, true
|
||||
case EventSubject(BusModule, BusConnectionClosedEvent):
|
||||
return AdvisoryConnectionClosed, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// BusFaults reads back what the bus module said of its users' connections since a moment, from the events
|
||||
// stream, on a consumer of its own that acknowledges nothing (as PowerStates reads), and says it as
|
||||
// advisories. Nothing to read is no advisory, and costs two direct reads: the newest of each event.
|
||||
func (s *Server) BusFaults(ctx context.Context, since time.Time) ([]Advisory, error) {
|
||||
if s.js == nil {
|
||||
return nil, errors.New("this control plane is not on the bus, so it cannot read what the bus module said")
|
||||
}
|
||||
js := s.js.Context()
|
||||
recent := false
|
||||
for _, event := range []string{BusSlowConsumerEvent, BusConnectionClosedEvent} {
|
||||
last, err := js.GetLastMsg(broker.EventsStream, EventSubject(BusModule, event), nats.Context(ctx))
|
||||
if errors.Is(err, nats.ErrMsgNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading what the bus module said of its users' connections: %w", err)
|
||||
}
|
||||
if !last.Time.Before(since) {
|
||||
recent = true
|
||||
}
|
||||
}
|
||||
if !recent {
|
||||
return nil, nil
|
||||
}
|
||||
sub, err := js.SubscribeSync(EventSubject(BusModule, "bus.>"), nats.OrderedConsumer(), nats.StartTime(since))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading what the bus module said of its users' connections: %w", err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
var closures []BusClosure
|
||||
var kinds []string
|
||||
read, cut := 0, false
|
||||
for {
|
||||
if read >= MaxBusEventsRead {
|
||||
cut = true
|
||||
break
|
||||
}
|
||||
read++
|
||||
wait, cancel := context.WithTimeout(ctx, readQuiet)
|
||||
msg, err := sub.NextMsgWithContext(wait)
|
||||
cancel()
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
break
|
||||
}
|
||||
if kind, ok := busEventKind(msg.Subject); ok {
|
||||
var c BusClosure
|
||||
if json.Unmarshal(msg.Data, &c) == nil && c.User != "" && !c.Closed.IsZero() {
|
||||
closures, kinds = append(closures, c), append(kinds, kind)
|
||||
}
|
||||
}
|
||||
if meta, err := msg.Metadata(); err != nil || meta.NumPending == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
out := BusClosures(closures, kinds, since)
|
||||
if cut {
|
||||
// Said, never silent: a busy hour is read up to the bound, and each advisory says it is a floor.
|
||||
for i := range out {
|
||||
out[i].Said += fmt.Sprintf(" (at least: the read stopped at %d of the bus module's events this hour)",
|
||||
MaxBusEventsRead)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// MaxBusEventsRead bounds one tick's read of the bus module's events: a watchdog tick must not spend itself
|
||||
// reading a storm of closures back. A read cut there says so in every advisory it makes.
|
||||
var MaxBusEventsRead = 2000
|
||||
|
||||
// MergeAdvisories is two lists of advisories as one: the same kind about the same thing is one advisory,
|
||||
// counted once (the larger count of the two), said by the newer. The controller's own slow consumer as its client library sees it and
|
||||
// as the server closed it are the same user's (`controller`).
|
||||
func MergeAdvisories(a, b []Advisory) []Advisory {
|
||||
out := append([]Advisory(nil), a...)
|
||||
at := map[string]int{}
|
||||
for i, x := range out {
|
||||
at[x.Kind+"/"+x.ID+"/"+x.Token] = i
|
||||
}
|
||||
for _, y := range b {
|
||||
key := y.Kind + "/" + y.ID + "/" + y.Token
|
||||
i, had := at[key]
|
||||
if !had {
|
||||
at[key] = len(out)
|
||||
out = append(out, y)
|
||||
continue
|
||||
}
|
||||
x := &out[i]
|
||||
// The same fault seen twice, not two faults: the controller's own slow consumer is told to its client
|
||||
// library and closed by the server, and is counted once — by whichever saw more of it.
|
||||
if y.Count > x.Count {
|
||||
x.Count = y.Count
|
||||
}
|
||||
if y.First.Before(x.First) {
|
||||
x.First = y.First
|
||||
}
|
||||
if y.Last.After(x.Last) {
|
||||
x.Last, x.Said = y.Last, y.Said
|
||||
if y.Reason != "" {
|
||||
x.Reason = y.Reason
|
||||
}
|
||||
}
|
||||
if x.User == "" {
|
||||
x.User = y.User
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// BusWatchBeat is the bus module's watch's newest heartbeat.
|
||||
type BusWatchBeat struct {
|
||||
At time.Time `json:"at"`
|
||||
Healthy bool `json:"healthy"`
|
||||
Why string `json:"why,omitempty"`
|
||||
// Machine is the machine the runtime stamped on it (`x-node`).
|
||||
Machine string `json:"-"`
|
||||
}
|
||||
|
||||
// BusWatchLast is the newest heartbeat of the bus module's watch, read directly from the events stream; false
|
||||
// when the stream holds none — a bus module older than its watch, which says nothing either way.
|
||||
func (s *Server) BusWatchLast(ctx context.Context) (BusWatchBeat, bool, error) {
|
||||
if s.js == nil {
|
||||
return BusWatchBeat{}, false, errors.New("this control plane is not on the bus, so it cannot read the bus module's watch")
|
||||
}
|
||||
msg, err := s.js.Context().GetLastMsg(broker.EventsStream, EventSubject(BusModule, BusWatchEvent), nats.Context(ctx))
|
||||
if errors.Is(err, nats.ErrMsgNotFound) {
|
||||
return BusWatchBeat{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return BusWatchBeat{}, false, fmt.Errorf("reading the bus module's watch's heartbeat: %w", err)
|
||||
}
|
||||
var b BusWatchBeat
|
||||
if err := json.Unmarshal(msg.Data, &b); err != nil || b.At.IsZero() {
|
||||
b = BusWatchBeat{At: msg.Time, Why: "its heartbeat could not be read"}
|
||||
}
|
||||
b.Machine = msg.Header.Get("x-node")
|
||||
return b, true, nil
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// What the bus module says of its users' connections, read back as advisories (novox/hq ADR 0357, issue
|
||||
// 270): one per user and kind, each closure counted once however often it is said.
|
||||
//
|
||||
// go test ./internal/link/ -run TestNatsBusFaults (each test on a bus of its own: internal/testbus)
|
||||
|
||||
func aBusWithEvents(t *testing.T) *Server {
|
||||
t.Helper()
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return ConnectNats(js, nil, nil)
|
||||
}
|
||||
|
||||
func sayClosure(t *testing.T, s *Server, event string, c BusClosure) {
|
||||
t.Helper()
|
||||
body, _ := json.Marshal(c)
|
||||
if _, err := s.js.Context().Publish(EventSubject(BusModule, event), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNatsBusFaultsAreOnePerUserAndKindEachClosureCountedOnce(t *testing.T) {
|
||||
s := aBusWithEvents(t)
|
||||
now := time.Now().UTC()
|
||||
slow := BusClosure{User: "laptop.claude-code", Name: "laptop.claude-code", CID: 10, Reason: "Slow Consumer (Write Deadline)",
|
||||
Opened: now.Add(-20 * time.Minute), Closed: now.Add(-2 * time.Minute), Pending: 334000}
|
||||
sayClosure(t, s, BusSlowConsumerEvent, slow)
|
||||
sayClosure(t, s, BusSlowConsumerEvent, slow) // said again: the module started again
|
||||
again := slow
|
||||
again.CID, again.Opened, again.Closed = 31, now.Add(-90*time.Second), now.Add(-time.Minute)
|
||||
sayClosure(t, s, BusSlowConsumerEvent, again)
|
||||
sayClosure(t, s, BusConnectionClosedEvent, BusClosure{User: "anchor.desk", CID: 7,
|
||||
Reason: "Maximum Message Payload Exceeded", Opened: now.Add(-time.Hour / 2), Closed: now.Add(-30 * time.Second)})
|
||||
// Older than the hour: S9 no longer says it.
|
||||
sayClosure(t, s, BusSlowConsumerEvent, BusClosure{User: "anchor.media", CID: 3, Reason: "Slow Consumer (Pending Bytes)",
|
||||
Opened: now.Add(-3 * time.Hour), Closed: now.Add(-2 * time.Hour)})
|
||||
|
||||
got, err := s.BusFaults(t.Context(), now.Add(-time.Hour))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("read %d advisories, want one for each user within the hour: %+v", len(got), got)
|
||||
}
|
||||
for _, a := range got {
|
||||
switch a.User {
|
||||
case "laptop.claude-code":
|
||||
if a.Kind != AdvisorySlowConsumer || a.ID != "laptop.claude-code" || a.Count != 2 ||
|
||||
!a.Last.Equal(again.Closed) || !a.First.Equal(slow.Closed) || !strings.Contains(a.Said, "connection 31") {
|
||||
t.Errorf("the slow consumer read as %+v: two closures, the one said twice counted once", a)
|
||||
}
|
||||
case "anchor.desk":
|
||||
if a.Kind != AdvisoryConnectionClosed || a.Count != 1 || a.Reason != "Maximum Message Payload Exceeded" {
|
||||
t.Errorf("the closed connection read as %+v", a)
|
||||
}
|
||||
default:
|
||||
t.Errorf("an advisory for %s", a.User)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNatsBusFaultsReadNothingWhenTheBusModuleSaidNothing(t *testing.T) {
|
||||
s := aBusWithEvents(t)
|
||||
started := time.Now()
|
||||
got, err := s.BusFaults(t.Context(), time.Now().Add(-time.Hour))
|
||||
if err != nil || len(got) != 0 {
|
||||
t.Fatalf("read %v, %v from a stream the bus module never wrote", got, err)
|
||||
}
|
||||
if took := time.Since(started); took > time.Second {
|
||||
t.Fatalf("reading nothing took %s: a tick must not wait on an empty stream", took)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheControllersOwnSlowConsumerAndItsClosedConnectionAreOneAdvisory(t *testing.T) {
|
||||
at := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
|
||||
own := []Advisory{{Kind: AdvisorySlowConsumer, ID: "controller", Said: "the controller fell behind", First: at, Last: at, Count: 1}}
|
||||
closed := BusClosures([]BusClosure{{User: "controller", CID: 5, Reason: "Slow Consumer (Pending Bytes)",
|
||||
Opened: at.Add(-time.Hour), Closed: at.Add(time.Minute)}}, []string{AdvisorySlowConsumer}, at.Add(-time.Hour))
|
||||
got := MergeAdvisories(own, closed)
|
||||
if len(got) != 1 || got[0].Count != 1 || !got[0].Last.Equal(at.Add(time.Minute)) || got[0].User != "controller" {
|
||||
t.Fatalf("merged as %+v: one fall behind, told to the client and closed by the server, is counted once", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A busy hour is read up to a bound, and says it was cut** (review of #326): a storm of closures must not
|
||||
// spend a watchdog tick, and a count cut short is a floor, said as one.
|
||||
func TestNatsBusFaultsReadAtMostTheBoundAndSaySo(t *testing.T) {
|
||||
s := aBusWithEvents(t)
|
||||
was := MaxBusEventsRead
|
||||
MaxBusEventsRead = 2
|
||||
t.Cleanup(func() { MaxBusEventsRead = was })
|
||||
now := time.Now().UTC()
|
||||
for i := range 3 {
|
||||
sayClosure(t, s, BusSlowConsumerEvent, BusClosure{User: "laptop.claude-code", CID: uint64(10 + i),
|
||||
Reason: "Slow Consumer (Pending Bytes)", Opened: now.Add(-time.Hour / 2), Closed: now.Add(-time.Duration(3-i) * time.Minute)})
|
||||
}
|
||||
got, err := s.BusFaults(t.Context(), now.Add(-time.Hour))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Count != 2 || !strings.Contains(got[0].Said, "at least: the read stopped at 2") {
|
||||
t.Fatalf("a read past its bound read as %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **The bus module's watch is heard by its heartbeat** (review 274): the newest is read directly, with the
|
||||
// machine the runtime stamped on it; none at all is a bus module older than its watch.
|
||||
func TestNatsTheBusWatchsNewestHeartbeatIsRead(t *testing.T) {
|
||||
s := aBusWithEvents(t)
|
||||
if _, heard, err := s.BusWatchLast(t.Context()); err != nil || heard {
|
||||
t.Fatalf("a stream with no heartbeat read as heard %v, %v", heard, err)
|
||||
}
|
||||
at := time.Now().UTC().Truncate(time.Second)
|
||||
for i, healthy := range []bool{true, false} {
|
||||
msg := nats.NewMsg(EventSubject(BusModule, BusWatchEvent))
|
||||
msg.Header.Set("x-node", "anchor")
|
||||
msg.Data, _ = json.Marshal(map[string]any{"at": at.Add(time.Duration(i) * time.Second), "healthy": healthy,
|
||||
"why": "the watch of slow consumers last read the bus's closed connections 95s ago"})
|
||||
if _, err := s.js.Context().PublishMsg(msg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
b, heard, err := s.BusWatchLast(t.Context())
|
||||
if err != nil || !heard || b.Healthy || b.Machine != "anchor" || !b.At.Equal(at.Add(time.Second)) ||
|
||||
!strings.Contains(b.Why, "95s ago") {
|
||||
t.Fatalf("read %+v, %v, %v", b, heard, err)
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
@@ -418,6 +419,12 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
|
||||
for id, why := range report.Failed {
|
||||
doing.Failed = append(doing.Failed, inventory.FailedResource{ID: id, Error: why})
|
||||
}
|
||||
// And a container left running because it said busy, or replaced after it waited (novox/hq ADR 0335):
|
||||
// kept with the account, for the walk whose first-machine gate waits on it.
|
||||
for _, b := range report.Busy {
|
||||
doing.Busy = append(doing.Busy, inventory.BusyWait{ID: b.ID, Container: b.Container, State: b.State,
|
||||
Since: b.Since, Until: b.Until, Said: b.Said})
|
||||
}
|
||||
// Ordered, so two readings of one failure are the same reading.
|
||||
sort.Slice(doing.Failed, func(i, j int) bool { return doing.Failed[i].ID < doing.Failed[j].ID })
|
||||
|
||||
@@ -492,7 +499,22 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
|
||||
}
|
||||
// And how long the machine took, from the send to this first account of it (novox/hq to-be 45
|
||||
// Phase 0): what the sent-not-reported bound will be set from. Said if lost, never a failure.
|
||||
if err := e.Inventory.RecordApplyDuration(ctx, report.Node, report.Declared, doing.Outcome); err != nil {
|
||||
//
|
||||
// **A wait for a busy container is its own measure** (novox/hq ADR 0335): the first report that says one
|
||||
// waits is the machine's answer, kept as its apply with the detail ApplyBusy; the report that says the
|
||||
// container replaced ends the wait, kept as a `busy-wait` of the same send — so the apply's time holds no
|
||||
// busy time, and the walk reads the machine as answered and waiting, never as silent.
|
||||
outcome := doing.Outcome
|
||||
if doing.Waiting() {
|
||||
outcome = inventory.ApplyBusy
|
||||
log.Printf("%s applied declaration %s and leaves a busy container running until it says idle: %s",
|
||||
report.Node, short(report.Declared), busyWords(doing.Busy))
|
||||
} else if replacedAfterWaiting(doing.Busy) {
|
||||
if err := e.Inventory.RecordBusyDuration(ctx, report.Node, report.Declared); err != nil {
|
||||
log.Printf("how long %s waited for a busy container could not be recorded: %v", report.Node, err)
|
||||
}
|
||||
}
|
||||
if err := e.Inventory.RecordApplyDuration(ctx, report.Node, report.Declared, outcome); err != nil {
|
||||
log.Printf("how long %s took to apply could not be recorded: %v", report.Node, err)
|
||||
}
|
||||
|
||||
@@ -504,3 +526,25 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
|
||||
}
|
||||
return news, e.Inventory.RecordOwned(ctx, node.ID, report.Applied)
|
||||
}
|
||||
|
||||
// busyWords is every container waiting because it said busy, in a line (novox/hq ADR 0335).
|
||||
func busyWords(busy []inventory.BusyWait) string {
|
||||
var out []string
|
||||
for _, b := range busy {
|
||||
if b.State == inventory.BusyWaiting {
|
||||
out = append(out, fmt.Sprintf("%s since %s, at the latest until %s", b.Container,
|
||||
b.Since.UTC().Format(time.RFC3339), b.Until.UTC().Format(time.RFC3339)))
|
||||
}
|
||||
}
|
||||
return strings.Join(out, "; ")
|
||||
}
|
||||
|
||||
// replacedAfterWaiting says a report replaced a container that had waited because it was busy.
|
||||
func replacedAfterWaiting(busy []inventory.BusyWait) bool {
|
||||
for _, b := range busy {
|
||||
if b.State != inventory.BusyWaiting {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -272,6 +272,12 @@ type Report struct {
|
||||
// `witness` and `not-reversible` are sent only to a machine whose report carries it.
|
||||
Witness int `json:"witness,omitempty"`
|
||||
|
||||
// Busy is every container the reporting apply left running because it said busy, and every one it
|
||||
// replaced after it had waited (novox/hq ADR 0335; mesh-host internal/link/messages.go): a walk's
|
||||
// first-machine gate waits while its module's container is listed waiting, and starts at the report
|
||||
// that no longer lists it so. Absent from an engine older than BusyContract, which never waits.
|
||||
Busy []Busy `json:"busy,omitempty"`
|
||||
|
||||
// Health is the machine's word on every long-running resource it runs for a module (novox/hq ADR
|
||||
// 0240, to-be 48 §4; mesh-host's internal/liveness): its state, since when, its failing streak and
|
||||
// the restarts its node-engine counted. **Absent from an engine older than the judging**, which is
|
||||
@@ -436,6 +442,31 @@ const ReadinessContract = 2
|
||||
// parses strictly and would refuse the whole declaration for it.
|
||||
const RootContract = 3
|
||||
|
||||
// BusyContract is the statement of an engine that also reads a container's `busy` (novox/hq ADR 0335): it asks
|
||||
// a container that declares it before replacing it and says the wait in its report. Only to such an engine is
|
||||
// the field sent: an older one parses strictly and would refuse the whole declaration for it. The same number as
|
||||
// mesh-host internal/link/messages.go's BusyContract.
|
||||
const BusyContract = 8
|
||||
|
||||
// The states of a busy container in a report (novox/hq ADR 0335 §2, §3).
|
||||
const (
|
||||
BusyWaiting = "waiting"
|
||||
BusyReplacedIdle = "replaced-when-idle"
|
||||
BusyReplacedAtBound = "replaced-at-bound"
|
||||
)
|
||||
|
||||
// Busy is one container a busy command held, as a node-engine reports it (novox/hq ADR 0335). ID is the
|
||||
// resource's id, `<module>.<resource>`; Since when the first apply that would have replaced it ran, Until when
|
||||
// the engine's bound replaces it at the latest, Said the last output of its busy command.
|
||||
type Busy struct {
|
||||
ID string `json:"id"`
|
||||
Container string `json:"container"`
|
||||
State string `json:"state"`
|
||||
Since time.Time `json:"since"`
|
||||
Until time.Time `json:"until"`
|
||||
Said string `json:"said,omitempty"`
|
||||
}
|
||||
|
||||
// ReasonRoot starts the reason of an account verdict that found a way to root (ADR 0266); the node-engine's
|
||||
// own words (mesh-host internal/accounts ReasonRoot).
|
||||
const ReasonRoot = "can become root without a person"
|
||||
|
||||
@@ -427,6 +427,9 @@ func ReadCheckBuilds(ctx context.Context, js *broker.JetStream, seat string, pul
|
||||
// safeKey is an id that can be a key, and a subject token, as it is: a build id, never a pattern.
|
||||
var safeKey = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
|
||||
|
||||
// SafeBuildID says an id can be a subject token as it is: a build's id, never a pattern.
|
||||
func SafeBuildID(id string) bool { return safeKey.MatchString(id) }
|
||||
|
||||
// Cancelled is what the controller writes for an ask it cancelled.
|
||||
type Cancelled struct {
|
||||
At string `json:"at"`
|
||||
|
||||
@@ -26,6 +26,8 @@ import (
|
||||
// - a check that has waited CheckWaitsAtMost goes before every delivery asked after it, so no check
|
||||
// waits for ever behind a stream of deliveries;
|
||||
// - within a kind, first come, first served, by when it was asked;
|
||||
// - a branch tried before its pull request goes after everything else, however long it has waited: it is
|
||||
// never promoted ahead of a delivery, and never ahead of a pull request's check (novox/hq issue 686);
|
||||
// - an ask whose kind cannot be told counts as a check: a delivery taken late breaks a budget, a check
|
||||
// taken late breaks nothing.
|
||||
//
|
||||
@@ -64,8 +66,12 @@ func (a QueuedAsk) askedOrStored() time.Time {
|
||||
// QueueOrder is the order the waiting asks are to be handed out in, now: the deliveries and every check
|
||||
// that has waited CheckWaitsAtMost, by when each was asked, then the other checks, by when each was asked.
|
||||
func QueueOrder(waiting []QueuedAsk, now time.Time) []QueuedAsk {
|
||||
var first, then []QueuedAsk
|
||||
var first, then, tries []QueuedAsk
|
||||
for _, a := range waiting {
|
||||
if IsTry(a.Request) {
|
||||
tries = append(tries, a)
|
||||
continue
|
||||
}
|
||||
if KindOf(a.Request) == KindCheck && now.Sub(a.askedOrStored()) < CheckWaitsAtMost {
|
||||
then = append(then, a)
|
||||
continue
|
||||
@@ -82,7 +88,8 @@ func QueueOrder(waiting []QueuedAsk, now time.Time) []QueuedAsk {
|
||||
}
|
||||
byAsking(first)
|
||||
byAsking(then)
|
||||
return append(first, then...)
|
||||
byAsking(tries)
|
||||
return append(append(first, then...), tries...)
|
||||
}
|
||||
|
||||
// toMove is the asks to move to the tail, in this order, for the queue to stand as order says: everything
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The start of a try names no repository and no commit (novox/hq issue 686): the forge's holder writes
|
||||
// mesh/merge-gate pending on the open pull request whose head a start names, and nothing on the forge moves
|
||||
// for a try. A pull request's check still names both.
|
||||
func TestATrysStartNamesNoCommitTheForgeCouldMatch(t *testing.T) {
|
||||
sha := strings.Repeat("c0ffee", 7)[:40]
|
||||
at := time.Date(2026, 10, 14, 0, 0, 0, 0, time.UTC)
|
||||
try := BuildRequest{ID: "build-1", Repository: "https://forge/novox/a.git", Ref: sha,
|
||||
Check: &CheckRequest{Owner: "novox", Repo: "a", Try: true}}
|
||||
body, _ := json.Marshal(startOf(try, "ace", 1, at))
|
||||
if strings.Contains(string(body), sha) || strings.Contains(string(body), "novox/a") {
|
||||
t.Fatalf("a try's start names what the forge matches a pull request by: %s", body)
|
||||
}
|
||||
if s := startOf(try, "ace", 2, at); s.ID != "build-1" || s.On != "ace" || !s.Try || s.Delivery != 2 || s.At == "" {
|
||||
t.Fatalf("a try's start lost what the queue reads: %+v", s)
|
||||
}
|
||||
pull := try
|
||||
pull.Check = &CheckRequest{Owner: "novox", Repo: "a", Number: 7}
|
||||
if s := startOf(pull, "ace", 1, at); s.Ref != sha || s.Repository == "" || s.Try {
|
||||
t.Fatalf("a pull request's check's start: %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
// A try stays behind every delivery and every pull request's check, however long it has waited: the ten
|
||||
// minutes that put a check ahead of later deliveries never promote a try (novox/hq issue 686).
|
||||
func TestATryIsNeverHandedOutBeforeADeliveryOrAPullRequestsCheck(t *testing.T) {
|
||||
now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
|
||||
try := func(seq uint64, id string, asked time.Time) QueuedAsk {
|
||||
a := anAsk(seq, id, asked, true)
|
||||
a.Request.Check.Try = true
|
||||
return a
|
||||
}
|
||||
waiting := []QueuedAsk{
|
||||
try(1, "try-old", now.Add(-40*time.Minute)),
|
||||
try(2, "try-young", now.Add(-time.Minute)),
|
||||
anAsk(3, "check-young", now.Add(-2*time.Minute), true),
|
||||
anAsk(4, "delivery", now.Add(-time.Minute), false),
|
||||
}
|
||||
want := []string{"delivery", "check-young", "try-old", "try-young"}
|
||||
if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("handed out as %v, want %v", got, want)
|
||||
}
|
||||
if !SafeBuildID("build-20261014T000000-1a2b") || SafeBuildID("x.>") || SafeBuildID("*") || SafeBuildID("") {
|
||||
t.Fatal("an id that is a pattern passes for a build's id")
|
||||
}
|
||||
}
|
||||
+19
-4
@@ -54,11 +54,26 @@ if [ -n "$unformatted" ]; then
|
||||
fi
|
||||
CGO_ENABLED=0 go vet ./...
|
||||
|
||||
if command -v gcc >/dev/null 2>&1; then
|
||||
CGO_ENABLED=1 go test -race -count=1 -timeout 30m ./...
|
||||
suite() {
|
||||
if command -v gcc >/dev/null 2>&1; then
|
||||
CGO_ENABLED=1 go test -race -count=1 -timeout 30m "$@"
|
||||
else
|
||||
echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector"
|
||||
CGO_ENABLED=0 go test -count=1 -timeout 30m "$@"
|
||||
fi
|
||||
}
|
||||
# **A branch tried before its pull request may ask for a part** (novox/hq issue 686): the controller's `try` verb
|
||||
# hands MESH_CHECK_TESTS, a `go test -run` pattern, or MESH_CHECK_STORE_ONLY=1. A pull request's merge check is
|
||||
# handed neither and always runs whole; a part that passes is said as a part, never as the check.
|
||||
if [ -n "${MESH_CHECK_TESTS:-}" ]; then
|
||||
echo "TRIED IN PART: only the tests matching $MESH_CHECK_TESTS, in every package"
|
||||
suite -run "$MESH_CHECK_TESTS" ./...
|
||||
exit 0
|
||||
fi
|
||||
if [ "${MESH_CHECK_STORE_ONLY:-}" = 1 ]; then
|
||||
echo "TRIED IN PART: only the store tests listed in testdata/store-tests"
|
||||
else
|
||||
echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector"
|
||||
CGO_ENABLED=0 go test -count=1 -timeout 30m ./...
|
||||
suite ./...
|
||||
fi
|
||||
|
||||
# **The store tests listed ran** (novox/hq issue 459): a test that needs the store skips without one, and a skip
|
||||
|
||||
@@ -77,6 +77,7 @@
|
||||
"replay",
|
||||
"kill",
|
||||
"forget-checks",
|
||||
"try",
|
||||
"pause",
|
||||
"resume",
|
||||
"hand-act",
|
||||
|
||||
Vendored
+12
@@ -60,3 +60,15 @@
|
||||
./cmd/mesh-controller TestASettingsWalkFailsOnARefusalThatCannotClear
|
||||
./cmd/mesh-controller TestAProviderElsewhereServesItsOwnComposedName
|
||||
./cmd/mesh-controller TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne
|
||||
./internal/inventory TestAReportOfABusyContainerIsKeptAndReadBack
|
||||
./cmd/mesh-controller TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses
|
||||
./cmd/mesh-controller TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn
|
||||
./cmd/mesh-controller TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere
|
||||
./cmd/mesh-controller TestAFirstSendMadeAndNotKeptIsNotSentAgain
|
||||
./cmd/mesh-controller TestAReleaseSendKeptAndMadeIsNotMadeAgain
|
||||
./cmd/mesh-controller TestAFirstMachineThatFailsStillStopsTheWalk
|
||||
./cmd/mesh-controller TestTheControlNodeIsSentLastInAStep
|
||||
./cmd/mesh-controller TestARestMachineSilentPastItsBoundIsLeftAndSaid
|
||||
./cmd/mesh-controller TestANoMoveTierWaitsForNoNewReport
|
||||
./cmd/mesh-controller TestOneProviderDownAndThreeConsumersFailingAreOneCondition
|
||||
./cmd/mesh-controller TestAConsumerFailingBesideAHealthyProviderIsItsOwn
|
||||
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
package declaration
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Busy is how a long-running container says it holds work that would be lost if it were stopped now
|
||||
// (novox/hq ADR 0335 §1): a command run inside it, by its shell, as an exec health check is, that exits 0
|
||||
// while it is busy and non-zero when it is idle. Being asked is also the container's notice that it is
|
||||
// about to be replaced: from the first ask it takes no new work, so its answer ends once the work in hand
|
||||
// ends.
|
||||
//
|
||||
// **Not part of what the container is.** Busy is not in the container's spec: a changed command changes
|
||||
// when the container is replaced, never whether — a container is not recreated for its busy command alone.
|
||||
type Busy struct {
|
||||
Command string `json:"command"`
|
||||
// Timeout is how long one ask may take; empty is BusyTimeoutDefault.
|
||||
Timeout string `json:"timeout,omitempty"`
|
||||
}
|
||||
|
||||
// The bounds of an ask (novox/hq ADR 0335).
|
||||
const (
|
||||
BusyTimeoutDefault = 10 * time.Second
|
||||
// BusyTimeoutCeiling is the longest one ask may take: an ask runs inside an apply, and an apply waits
|
||||
// for nothing longer than a look.
|
||||
BusyTimeoutCeiling = time.Minute
|
||||
// BusyBound is how long a container that says busy is left running after the first apply that would
|
||||
// have replaced it (ADR 0335 §3). The node-engine's, not the module's: no module holds its own
|
||||
// replacement for ever.
|
||||
BusyBound = 15 * time.Minute
|
||||
)
|
||||
|
||||
// UnmarshalJSON reads a busy strictly, as everything a declaration carries is read.
|
||||
func (b *Busy) UnmarshalJSON(raw []byte) error {
|
||||
type plain Busy
|
||||
var p plain
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&p); err != nil {
|
||||
return fmt.Errorf("busy: %w", err)
|
||||
}
|
||||
*b = Busy(p)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Within is how long one ask may take. Validated on arrival, so an unreadable timeout is the default.
|
||||
func (b *Busy) Within() time.Duration {
|
||||
if d, err := time.ParseDuration(b.Timeout); err == nil && d > 0 {
|
||||
return d
|
||||
}
|
||||
return BusyTimeoutDefault
|
||||
}
|
||||
|
||||
// problems holds a container's busy to its shape: a one-line command, a timeout within its bounds, and only
|
||||
// on a container that stays up.
|
||||
func (b *Busy) problems(where string, longRunning bool) []string {
|
||||
if b == nil {
|
||||
return nil
|
||||
}
|
||||
var problems []string
|
||||
say := func(format string, args ...any) {
|
||||
problems = append(problems, where+": "+fmt.Sprintf(format, args...))
|
||||
}
|
||||
if !longRunning {
|
||||
say("busy is asked of a container that stays up; a step or a scheduled run is never replaced mid-run")
|
||||
}
|
||||
if strings.TrimSpace(b.Command) == "" {
|
||||
say("busy needs a command")
|
||||
}
|
||||
if strings.ContainsAny(b.Command, "\n\r") {
|
||||
say("busy's command is one line")
|
||||
}
|
||||
if b.Timeout != "" {
|
||||
d, err := time.ParseDuration(b.Timeout)
|
||||
switch {
|
||||
case err != nil:
|
||||
say("busy's timeout is a duration, not %q", b.Timeout)
|
||||
case d <= 0 || d > BusyTimeoutCeiling:
|
||||
say("an ask of busy takes more than nothing and at most %s, not %s", BusyTimeoutCeiling, d)
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
+354
-5
@@ -12,6 +12,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"slices"
|
||||
@@ -132,7 +133,13 @@ func (d *Directory) validate(where string, _ bool) []string {
|
||||
if d.Path == "" {
|
||||
problems = append(problems, where+": a directory needs a path")
|
||||
}
|
||||
return append(problems, checkMode(where, d.Mode)...)
|
||||
// A directory's mode may carry the setgid bit (2xxx), so what is made beneath it keeps its group (novox/hq ADR
|
||||
// 0334 §1); never setuid or sticky, which mean nothing the mesh asks of a directory it owns.
|
||||
mode := d.Mode
|
||||
if len(mode) == 4 && mode[0] == '2' {
|
||||
mode = "0" + mode[1:]
|
||||
}
|
||||
return append(problems, checkMode(where, mode)...)
|
||||
}
|
||||
|
||||
// File is a file with literal content. The host renders nothing.
|
||||
@@ -456,14 +463,51 @@ const (
|
||||
// **The distinction 04-ISSUES/036 and 026 turn on.** A `directory` resource is the mesh's own —
|
||||
// it creates it, sets its owner and mode, and removes it when empty ([ADR 0030](novox/hq)). A
|
||||
// media library, a download spool is the operator's: it existed before the mesh, several modules
|
||||
// read and write it at once, and the mesh must not create, chown, reconcile or remove it. The
|
||||
// host confirms it is there and mounts it; nothing else.
|
||||
// read and write it at once, and the mesh must not create, chown, chmod or remove it. The host
|
||||
// confirms it is there and mounts it; nothing else. Library data is a library's (novox/hq ADR 0328),
|
||||
// not an access.
|
||||
type Access struct {
|
||||
ID string `json:"id"`
|
||||
Type Type `json:"type"`
|
||||
Path string `json:"path"`
|
||||
// Mode is how this module reaches the path: read or read-write. Absent narrows to read.
|
||||
Mode string `json:"mode,omitempty"`
|
||||
|
||||
// Users and Taken are what a controller speaking ADR 0325 sends (the identities that use the folder, and
|
||||
// that the operator's warrant was carried out): accepted, so its declaration is not refused whole, and not
|
||||
// acted on since ADR 0328 superseded ADR 0325 — no ACL entry is kept.
|
||||
Users []AccessUser `json:"users,omitempty"`
|
||||
Taken bool `json:"taken,omitempty"`
|
||||
|
||||
// Grant is what a controller older than ADR 0325 still sends (ADR 0312's group grant, superseded): accepted
|
||||
// so its declaration is not refused whole, and never acted on — no account is put in a group for data (§5).
|
||||
Grant json.RawMessage `json:"grant,omitempty"`
|
||||
}
|
||||
|
||||
// The uses an access's user has (ADR 0051 §3), and where it runs (ADR 0325 §4).
|
||||
const (
|
||||
FromContainer = "container"
|
||||
FromToolRunner = "tool-runner"
|
||||
)
|
||||
|
||||
// AccessUser is one identity that uses an accessed folder: its name and uid as the controller allocated them, its
|
||||
// use, and where it runs. A tool runner's account may come with no uid (0 or absent): the engine reads it by name.
|
||||
type AccessUser struct {
|
||||
Identity string `json:"identity"`
|
||||
UID int `json:"uid,omitempty"`
|
||||
Use string `json:"use"`
|
||||
From string `json:"from"`
|
||||
}
|
||||
|
||||
// UnmarshalJSON reads a user as strictly as a resource is read.
|
||||
func (u *AccessUser) UnmarshalJSON(raw []byte) error {
|
||||
type plain AccessUser
|
||||
var p plain
|
||||
if err := strictDecode(raw, &p); err != nil {
|
||||
return fmt.Errorf("an access's user: %w", err)
|
||||
}
|
||||
*u = AccessUser(p)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *Access) Identity() string { return a.ID }
|
||||
@@ -482,9 +526,43 @@ func (a *Access) validate(where string, _ bool) []string {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: an access is %q or %q, not %q", where, AccessRead, AccessReadWrite, a.Mode))
|
||||
}
|
||||
for _, u := range a.Users {
|
||||
switch {
|
||||
case u.Identity == "" || !accountName.MatchString(u.Identity):
|
||||
problems = append(problems, fmt.Sprintf("%s: a user of an access is an account's name, and %q is not one",
|
||||
where, u.Identity))
|
||||
case u.UID < 0 || (u.UID == 0 && u.From != FromToolRunner):
|
||||
// Never root (novox/hq ADR 0325 §11): an entry for uid 0 would be a right root has anyway, and a
|
||||
// number below it is no account. A tool runner's account alone may come without its number: the
|
||||
// controller cannot know the operator's account's, and the engine reads it by name
|
||||
// (apply.ResolveRunners), refusing one that is root's.
|
||||
problems = append(problems, fmt.Sprintf("%s: the user %s has uid %d, and a module's identity is never "+
|
||||
"root or below it (novox/hq ADR 0325 §11)", where, u.Identity, u.UID))
|
||||
}
|
||||
if u.Use != AccessRead && u.Use != AccessReadWrite {
|
||||
problems = append(problems, fmt.Sprintf("%s: the user %s uses it %q, and a use is %q or %q", where,
|
||||
u.Identity, u.Use, AccessRead, AccessReadWrite))
|
||||
}
|
||||
if u.Use == AccessReadWrite && a.Mode != AccessReadWrite {
|
||||
problems = append(problems, fmt.Sprintf("%s: the user %s writes an access declared %s", where,
|
||||
u.Identity, orRead(a.Mode)))
|
||||
}
|
||||
if u.From != FromContainer && u.From != FromToolRunner {
|
||||
problems = append(problems, fmt.Sprintf("%s: the user %s runs from %q, and that is %q or %q", where,
|
||||
u.Identity, u.From, FromContainer, FromToolRunner))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// orRead is an access's mode as said: absent narrows to read.
|
||||
func orRead(mode string) string {
|
||||
if mode == "" {
|
||||
return AccessRead
|
||||
}
|
||||
return mode
|
||||
}
|
||||
|
||||
func (u *User) Identity() string { return u.ID }
|
||||
func (u *User) Kind() Type { return TypeUser }
|
||||
func (u *User) Target() string { return u.Name }
|
||||
@@ -1160,6 +1238,83 @@ type Container struct {
|
||||
// Health is how this resource is ready (novox/hq ADR 0240 rule 2, Phase B): one kind and its
|
||||
// timing, judged by the node-engine beside liveness. Absent: judged alive or not, and nothing more.
|
||||
Health *Health `json:"health,omitempty"`
|
||||
|
||||
// Busy is how this container says it holds work a replacement would lose (novox/hq ADR 0335): asked
|
||||
// before it is replaced, and while it says busy it is left running, within the bound. Absent: never
|
||||
// busy, and replaced at once, as before.
|
||||
Busy *Busy `json:"busy,omitempty"`
|
||||
|
||||
// RunAs is who the container runs as, resolved by the controller from the module's identity and the
|
||||
// image's declared interface (novox/hq ADR 0325 §2, §3): told by environment (the image reads PUID and
|
||||
// PGID, say) or by the runtime's own user option. Part of the spec, so a change recreates it. Absent: the
|
||||
// container runs as its image says. Its key is `identity`, the manifest's word; the field is not, because
|
||||
// a resource's Identity is its id.
|
||||
RunAs *ContainerIdentity `json:"identity,omitempty"`
|
||||
}
|
||||
|
||||
// The ways an image is told who to run as (novox/hq ADR 0325 §2).
|
||||
const (
|
||||
IdentityByEnv = "env"
|
||||
IdentityByRun = "run"
|
||||
)
|
||||
|
||||
// ContainerIdentity is who a container runs as, and how its image is told.
|
||||
type ContainerIdentity struct {
|
||||
// Mode is IdentityByEnv or IdentityByRun.
|
||||
Mode string `json:"mode"`
|
||||
// UserEnv and GroupEnv are the variables the image reads the user's and the group's number from, for
|
||||
// IdentityByEnv: "PUID", "PGID".
|
||||
UserEnv string `json:"user_env,omitempty"`
|
||||
GroupEnv string `json:"group_env,omitempty"`
|
||||
UID int `json:"uid"`
|
||||
GID int `json:"gid"`
|
||||
}
|
||||
|
||||
// UnmarshalJSON reads an identity as strictly as a resource is read.
|
||||
func (i *ContainerIdentity) UnmarshalJSON(raw []byte) error {
|
||||
type plain ContainerIdentity
|
||||
var p plain
|
||||
if err := strictDecode(raw, &p); err != nil {
|
||||
return fmt.Errorf("a container's identity: %w", err)
|
||||
}
|
||||
*i = ContainerIdentity(p)
|
||||
return nil
|
||||
}
|
||||
|
||||
// envName is a variable's name as a runtime takes it.
|
||||
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
func (i *ContainerIdentity) problems(where string, env map[string]string) []string {
|
||||
if i == nil {
|
||||
return nil
|
||||
}
|
||||
var problems []string
|
||||
switch i.Mode {
|
||||
case IdentityByEnv:
|
||||
if !envName.MatchString(i.UserEnv) || (i.GroupEnv != "" && !envName.MatchString(i.GroupEnv)) {
|
||||
problems = append(problems, fmt.Sprintf("%s: an identity told by environment names the variables "+
|
||||
"it is told in, and %q and %q are not both names", where, i.UserEnv, i.GroupEnv))
|
||||
}
|
||||
for _, name := range []string{i.UserEnv, i.GroupEnv} {
|
||||
if _, set := env[name]; set && name != "" {
|
||||
problems = append(problems, fmt.Sprintf("%s: env sets %s, which the identity sets; one value "+
|
||||
"for one variable", where, name))
|
||||
}
|
||||
}
|
||||
case IdentityByRun:
|
||||
if i.UserEnv != "" || i.GroupEnv != "" {
|
||||
problems = append(problems, where+": an identity told by the runtime's user option names no variable")
|
||||
}
|
||||
default:
|
||||
problems = append(problems, fmt.Sprintf("%s: an identity is told by %q or %q, not %q", where,
|
||||
IdentityByEnv, IdentityByRun, i.Mode))
|
||||
}
|
||||
// Never root (novox/hq ADR 0325 §11): refused here, so nothing of the declaration is made or started.
|
||||
if i.UID <= 0 || i.GID <= 0 {
|
||||
problems = append(problems, fmt.Sprintf("%s: the identity is uid %d, gid %d, and a module's identity is "+
|
||||
"never root, nor a number below it (novox/hq ADR 0325 §11)", where, i.UID, i.GID))
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
func (c *Container) Identity() string { return c.ID }
|
||||
@@ -1168,6 +1323,7 @@ func (c *Container) Target() string { return c.Name }
|
||||
|
||||
func (c *Container) validate(where string, _ bool) []string {
|
||||
problems := c.Health.problems(where, true, !c.RunOnce && c.Schedule == "")
|
||||
problems = append(problems, c.Busy.problems(where, !c.RunOnce && c.Schedule == "")...)
|
||||
if c.Name == "" {
|
||||
problems = append(problems, where+": a container needs a name")
|
||||
}
|
||||
@@ -1242,6 +1398,7 @@ func (c *Container) validate(where string, _ bool) []string {
|
||||
problems = append(problems, where+": networks is for a container that keeps running; a step "+
|
||||
"runs and exits, and joins nothing afterwards")
|
||||
}
|
||||
problems = append(problems, c.RunAs.problems(where+": container "+c.Name, c.Env)...)
|
||||
return append(problems, checkImage(where, c.Image)...)
|
||||
}
|
||||
|
||||
@@ -1392,6 +1549,185 @@ type Declaration struct {
|
||||
// the generation of what it puts back, which may be older than what the machine applied, and is
|
||||
// not refused for it.
|
||||
PutBack bool
|
||||
|
||||
// Identities is the identity of every module assigned to this machine (novox/hq ADR 0328 §4): an account
|
||||
// and a group `mesh-<module>` with the numbers the controller allocated once for the module, the same on
|
||||
// every machine. The engine makes each before any resource is applied, and removes what it made of one no
|
||||
// longer declared. Absent is a controller older than ADR 0325.
|
||||
Identities []Identity
|
||||
|
||||
// Groups is the group of every library this machine is reached by (novox/hq ADR 0328 §3): `mesh-<library>`
|
||||
// with the number the controller allocated once, mesh-wide. The engine makes each, and removes what it made of
|
||||
// one no longer declared. Absent is a controller older than ADR 0328.
|
||||
Groups []Group
|
||||
|
||||
// Libraries is every library this machine is reached by (ADR 0328 §1, §6, §9): its root here, its group, whether
|
||||
// this is its home, whether the operator's warrant to take it was carried out, the sub-paths its modules ask for,
|
||||
// and the tool runner's account to put in its group (§11). Absent is a controller older than ADR 0328.
|
||||
Libraries []Library
|
||||
}
|
||||
|
||||
// Identity is one module's identity, as the controller allocated it (ADR 0328 §4): its user `mesh-<module>` with
|
||||
// uid, its own group of the same name and number, its primary group gid (its own, or its first library's group),
|
||||
// and the groups of its other libraries as added groups.
|
||||
type Identity struct {
|
||||
Module string `json:"module"`
|
||||
Name string `json:"name"`
|
||||
UID int `json:"uid"`
|
||||
GID int `json:"gid"`
|
||||
Groups []int `json:"groups,omitempty"`
|
||||
// Runner is the tool runner's account, put in the module's own group (novox/hq ADR 0334 §2) when its tools run
|
||||
// on this machine. Absent for a module that declares no tools here, and from a controller older than ADR 0334.
|
||||
Runner string `json:"runner,omitempty"`
|
||||
}
|
||||
|
||||
// Group is one library's group, as the controller allocated it (ADR 0328 §3).
|
||||
type Group struct {
|
||||
Name string `json:"name"`
|
||||
GID int `json:"gid"`
|
||||
}
|
||||
|
||||
// GroupPrefix starts every library group's name, as IdentityPrefix starts every module's.
|
||||
const GroupPrefix = "mesh-"
|
||||
|
||||
// LibraryAccessPrefix starts the access a library's wait and its take act name: `library:<name>`.
|
||||
const LibraryAccessPrefix = "library:"
|
||||
|
||||
// Library is one library as this machine sees it (ADR 0328).
|
||||
type Library struct {
|
||||
Library string `json:"library"`
|
||||
// Root is the library's root on this machine: on its home the data itself, elsewhere the mount point the mounts
|
||||
// module mounts it at.
|
||||
Root string `json:"root"`
|
||||
Group string `json:"group"`
|
||||
GID int `json:"gid"`
|
||||
// Home is that the library's data lives on this machine: only there does the engine keep it.
|
||||
Home bool `json:"home"`
|
||||
// Taken is the controller saying the operator's warrant to take it was carried out (§7).
|
||||
Taken bool `json:"taken,omitempty"`
|
||||
// Paths are the sub-paths its modules ask for, relative to the root.
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
// Runner is the tool runner's account, put in the library's group (§11). Absent when no module's tools use it.
|
||||
Runner string `json:"runner,omitempty"`
|
||||
}
|
||||
|
||||
// checkLibraries judges the groups and libraries on arrival: every library's group declared, named for it, never
|
||||
// root's number; paths that stay below the root; a runner that is an account's name; and no two roots that nest
|
||||
// (§13).
|
||||
func checkLibraries(groups []Group, libs []Library) []string {
|
||||
var problems []string
|
||||
byName := map[string]int{}
|
||||
numbers := map[int]string{}
|
||||
for _, g := range groups {
|
||||
where := fmt.Sprintf("group %q", g.Name)
|
||||
if !strings.HasPrefix(g.Name, GroupPrefix) || !accountName.MatchString(g.Name) {
|
||||
problems = append(problems, fmt.Sprintf("%s: a library's group is named %s<library>", where, GroupPrefix))
|
||||
}
|
||||
if g.GID <= 0 {
|
||||
problems = append(problems, fmt.Sprintf("%s: gid %d, and a library's group is never root, nor a number "+
|
||||
"below it (novox/hq ADR 0328 §15); nothing of this declaration was made", where, g.GID))
|
||||
}
|
||||
if _, twice := byName[g.Name]; twice {
|
||||
problems = append(problems, where+": declared twice")
|
||||
}
|
||||
if other, used := numbers[g.GID]; used {
|
||||
problems = append(problems, fmt.Sprintf("%s: gid %d is %s's too", where, g.GID, other))
|
||||
}
|
||||
byName[g.Name], numbers[g.GID] = g.GID, g.Name
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, l := range libs {
|
||||
where := fmt.Sprintf("library %q", l.Library)
|
||||
switch {
|
||||
case l.Library == "" || strings.ContainsAny(l.Library, "/ \t\n"):
|
||||
problems = append(problems, where+": a library is named by one word")
|
||||
case seen[l.Library]:
|
||||
problems = append(problems, where+": declared twice")
|
||||
}
|
||||
seen[l.Library] = true
|
||||
if !filepath.IsAbs(l.Root) || filepath.Clean(l.Root) != l.Root || l.Root == "/" {
|
||||
problems = append(problems, fmt.Sprintf("%s: its root %q is not an absolute, clean path below /", where,
|
||||
l.Root))
|
||||
}
|
||||
if gid, ok := byName[l.Group]; !ok || gid != l.GID {
|
||||
problems = append(problems, fmt.Sprintf("%s: its group %s (gid %d) is not one this declaration makes",
|
||||
where, l.Group, l.GID))
|
||||
}
|
||||
if l.Group != GroupPrefix+l.Library {
|
||||
problems = append(problems, fmt.Sprintf("%s: its group is %s%s, and %s is not", where, GroupPrefix,
|
||||
l.Library, l.Group))
|
||||
}
|
||||
for _, p := range l.Paths {
|
||||
if p == "" || filepath.IsAbs(p) || filepath.Clean(p) != p || p == ".." || strings.HasPrefix(p, "../") {
|
||||
problems = append(problems, fmt.Sprintf("%s: the path %q is not a clean path below its root", where, p))
|
||||
}
|
||||
}
|
||||
if l.Runner != "" && !accountName.MatchString(l.Runner) {
|
||||
problems = append(problems, fmt.Sprintf("%s: its runner %q is not an account's name", where, l.Runner))
|
||||
}
|
||||
}
|
||||
for i, a := range libs {
|
||||
for _, b := range libs[i+1:] {
|
||||
if a.Root != "" && b.Root != "" && (a.Root == b.Root || strings.HasPrefix(a.Root, b.Root+"/") ||
|
||||
strings.HasPrefix(b.Root, a.Root+"/")) {
|
||||
problems = append(problems, fmt.Sprintf("library %q: its root and library %q's nest, and libraries "+
|
||||
"never overlap (novox/hq ADR 0328 §13)", a.Library, b.Library))
|
||||
}
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// IdentityPrefix starts every module identity's name.
|
||||
const IdentityPrefix = "mesh-"
|
||||
|
||||
// checkIdentities judges the identities on arrival: each named for its module, never root, and no name or number
|
||||
// twice — two modules with one account would write as each other.
|
||||
func checkIdentities(ids []Identity, groups []Group) []string {
|
||||
var problems []string
|
||||
names, uids := map[string]bool{}, map[int]string{}
|
||||
libraryGroups := map[int]bool{}
|
||||
for _, g := range groups {
|
||||
libraryGroups[g.GID] = true
|
||||
}
|
||||
for _, id := range ids {
|
||||
where := fmt.Sprintf("identity %q", id.Name)
|
||||
if id.Module == "" || id.Name != IdentityPrefix+id.Module || !accountName.MatchString(id.Name) {
|
||||
problems = append(problems, fmt.Sprintf("%s: a module's identity is named %s<module>, and this one is "+
|
||||
"for the module %q", where, IdentityPrefix, id.Module))
|
||||
}
|
||||
if id.UID <= 0 || id.GID <= 0 {
|
||||
problems = append(problems, fmt.Sprintf("%s: uid %d, gid %d, and a module's identity is never root, "+
|
||||
"nor a number below it (novox/hq ADR 0328 §15); nothing of this declaration was made", where,
|
||||
id.UID, id.GID))
|
||||
}
|
||||
if names[id.Name] {
|
||||
problems = append(problems, where+": declared twice")
|
||||
}
|
||||
if other, used := uids[id.UID]; used {
|
||||
problems = append(problems, fmt.Sprintf("%s: uid %d is %s's too", where, id.UID, other))
|
||||
}
|
||||
// Its primary group is its own (gid == uid) or a library's this declaration makes (ADR 0328 §4); so is
|
||||
// every added group. A module's own number is never a library group's.
|
||||
if id.GID != id.UID && !libraryGroups[id.GID] {
|
||||
problems = append(problems, fmt.Sprintf("%s: its primary group gid %d is neither its own (%d) nor a "+
|
||||
"library's this declaration makes", where, id.GID, id.UID))
|
||||
}
|
||||
if libraryGroups[id.UID] {
|
||||
problems = append(problems, fmt.Sprintf("%s: uid %d is a library group's number too", where, id.UID))
|
||||
}
|
||||
for _, g := range id.Groups {
|
||||
if !libraryGroups[g] {
|
||||
problems = append(problems, fmt.Sprintf("%s: its added group gid %d is not a library's this "+
|
||||
"declaration makes", where, g))
|
||||
}
|
||||
}
|
||||
if id.Runner != "" && !accountName.MatchString(id.Runner) {
|
||||
problems = append(problems, fmt.Sprintf("%s: its runner %q is not an account's name", where, id.Runner))
|
||||
}
|
||||
names[id.Name], uids[id.UID] = true, id.Name
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// LeftOutModuleOf says which left-out module a recorded resource belongs to, if any: its id is the
|
||||
@@ -1561,6 +1897,13 @@ type envelope struct {
|
||||
// sends them only to a host whose reports say `reads_generation`.
|
||||
Generation int64 `json:"generation,omitempty"`
|
||||
PutBack bool `json:"put_back,omitempty"`
|
||||
// Identities is optional on the wire too (novox/hq ADR 0325 §3, ADR 0328 §4). **An older host refuses this key**; a
|
||||
// controller sends it only to a host whose health says link.IdentityContract.
|
||||
Identities []Identity `json:"identities,omitempty"`
|
||||
// Groups and Libraries are optional too (novox/hq ADR 0328). **An older host refuses these keys**; a controller
|
||||
// sends them only to a host whose health says link.LibraryContract.
|
||||
Groups []Group `json:"groups,omitempty"`
|
||||
Libraries []Library `json:"libraries,omitempty"`
|
||||
}
|
||||
|
||||
func parse(raw []byte, allowActions bool) (*Declaration, error) {
|
||||
@@ -1578,8 +1921,10 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
|
||||
}
|
||||
|
||||
d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption, Sequence: env.Sequence,
|
||||
Epoch: env.Epoch, LeftOut: env.LeftOut, Generation: env.Generation, PutBack: env.PutBack}
|
||||
var problems []string
|
||||
Epoch: env.Epoch, LeftOut: env.LeftOut, Generation: env.Generation, PutBack: env.PutBack,
|
||||
Identities: env.Identities, Groups: env.Groups, Libraries: env.Libraries}
|
||||
problems := checkIdentities(env.Identities, env.Groups)
|
||||
problems = append(problems, checkLibraries(env.Groups, env.Libraries)...)
|
||||
if env.Generation < 0 {
|
||||
// Below zero would read as "none claimed" and pass every refusal of an older generation.
|
||||
problems = append(problems, fmt.Sprintf("an assignment generation below zero (%d) is not one the "+
|
||||
@@ -1732,6 +2077,10 @@ func checkWhileStopped(resources []Resource) []string {
|
||||
return problems
|
||||
}
|
||||
|
||||
// accountName is what an identity's or an access user's name may be: the portable account name the user tools
|
||||
// take, optionally ending in `$`, never starting with a dash — it is handed to them as an argument.
|
||||
var accountName = regexp.MustCompile(`^[A-Za-z0-9_.][A-Za-z0-9_.-]*\$?$`)
|
||||
|
||||
func strictDecode(raw []byte, into any) error {
|
||||
// DisallowUnknownFields is the whole point rather than strictness for its own sake: a
|
||||
// field the host does not know is a thing the control plane believes it asked for.
|
||||
|
||||
Vendored
+2
-2
@@ -78,7 +78,7 @@ github.com/nats-io/nkeys
|
||||
# github.com/nats-io/nuid v1.0.1
|
||||
## explicit
|
||||
github.com/nats-io/nuid
|
||||
# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f
|
||||
# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261013220009-b9a293f8c8bc
|
||||
## explicit; go 1.26.0
|
||||
github.com/novox/mesh-host/internal/declaration
|
||||
github.com/novox/mesh-host/rootsearch
|
||||
@@ -138,4 +138,4 @@ golang.org/x/text/width
|
||||
# golang.org/x/time v0.15.0
|
||||
## explicit; go 1.25.0
|
||||
golang.org/x/time/rate
|
||||
# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f
|
||||
# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261013220009-b9a293f8c8bc
|
||||
|
||||
Reference in New Issue
Block a user