Archived
Compare commits
76
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
df8c1ccf1f | ||
|
|
bbee20adf9 | ||
|
|
afed887217 | ||
|
|
622fb1e9cb | ||
|
|
5d5bdf9e32 | ||
|
|
4af9f1147d | ||
|
|
704b21b3c5 | ||
|
|
5b455b8a24 | ||
|
|
0e97413875 | ||
|
|
da8a5e9bc0 | ||
|
|
847ffd6d12 | ||
|
|
0214b45469 | ||
|
|
2bee290eea | ||
|
|
76304d9db8 | ||
|
|
b7084a19e9 | ||
|
|
c975dbf7b0 | ||
|
|
98e8568749 | ||
|
|
58889235d1 | ||
|
|
48b2906b5c | ||
|
|
ea7d91bff1 | ||
|
|
4a75cc51dd | ||
|
|
535c7641ac | ||
|
|
c707bf18d8 | ||
|
|
060f2a9ca5 | ||
|
|
1666747a22 | ||
|
|
04730b54a8 | ||
|
|
45870ab31c | ||
|
|
fd9308d328 | ||
|
|
3479a35462 | ||
|
|
cebc593627 | ||
|
|
2959c81435 | ||
|
|
8933864a49 | ||
|
|
df9e5728a9 | ||
|
|
2d18f438c5 | ||
|
|
3ce5a0239b | ||
|
|
70885ab3c4 | ||
|
|
65508d9df9 | ||
|
|
8123ca5654 | ||
|
|
d9b86376d3 | ||
|
|
ca3f43c400 | ||
|
|
b482122d66 | ||
|
|
2f5cc146fe | ||
|
|
c2ea9b4c51 | ||
|
|
b03d257f81 | ||
|
|
1601cb1743 | ||
|
|
873b19566a | ||
|
|
b0753e2aac | ||
|
|
a69c0e50dc | ||
|
|
72214ae989 | ||
|
|
5c606fd1d4 | ||
|
|
372faebf28 | ||
|
|
675466a8c3 | ||
|
|
cd49b50c40 | ||
|
|
41eefc2f8c | ||
|
|
c3946e5d72 | ||
|
|
524295926f | ||
|
|
bff5b7f0e4 | ||
|
|
cef22a2526 | ||
|
|
deef9421a2 | ||
|
|
3c19e9ee93 | ||
|
|
3d94f87bb0 | ||
|
|
4644f25edd | ||
|
|
c58e0a571e | ||
|
|
332fbaa9b1 | ||
|
|
7a0c219f96 | ||
|
|
985423ae61 | ||
|
|
cfacda2625 | ||
|
|
bf6beec465 | ||
|
|
9f3aa91e81 | ||
|
|
4bdcef3f73 | ||
|
|
8ddf8b7005 | ||
|
|
8bbb1b19ae | ||
|
|
e32ca00aed | ||
|
|
1219ec8229 | ||
|
|
e324f5d821 | ||
|
|
b1546ed28a |
@@ -0,0 +1,146 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **The build agent says whether it is busy, and being asked drains it** (novox/hq ADR 0335 §1, §5). Its
|
||||
// container declares `busy` as `mesh-builder busy`: the node-engine runs it inside the container before it
|
||||
// replaces it. It exits 0 while a build is in hand and 1 when none is; and from the first ask the agent takes
|
||||
// no new build request, so its answer ends once the build in hand ends — a replaced builder finishes the check
|
||||
// it runs, instead of the build queue handing it out again from the start (issue 594).
|
||||
//
|
||||
// Two marks in the container's own temporary directory, as the ready mark (ready.go): a new container starts
|
||||
// without them, and a restarted one removes them before it takes work again.
|
||||
//
|
||||
// - in hand: written by the agent when it claims a request, before it reads whether it is drained; removed
|
||||
// when the build ends.
|
||||
// - drained: written by the ask, before it reads whether a build is in hand.
|
||||
//
|
||||
// Each side writes its own mark before it reads the other's, so an ask racing a fetch sees the build in hand,
|
||||
// or the agent sees the drain and hands the request back; never neither.
|
||||
|
||||
func inHandMark() string { return filepath.Join(os.TempDir(), "mesh-builder.in-hand") }
|
||||
func drainMark() string { return filepath.Join(os.TempDir(), "mesh-builder.drained") }
|
||||
|
||||
// The drain lapses when no ask has come for drainLapse: the node-engine asks every 20 seconds for as long as it
|
||||
// really waits, so an agent whose replacement was withdrawn — the declaration put back, the walk stopped —
|
||||
// takes work again rather than staying drained for ever (review of ADR 0335). And an agent drained for longer
|
||||
// than the node-engine's bound and the slack says it is not ready: the replacement it was drained for never
|
||||
// came, which is a fault to see.
|
||||
const (
|
||||
drainLapse = 2 * time.Minute
|
||||
drainTooLong = 15*time.Minute + 5*time.Minute
|
||||
)
|
||||
|
||||
// drainedAt is when the drain began and when it was last asked, from its mark; false when there is no mark.
|
||||
func drainedAt() (began, asked time.Time, ok bool) {
|
||||
info, err := os.Stat(drainMark())
|
||||
if err != nil {
|
||||
return time.Time{}, time.Time{}, false
|
||||
}
|
||||
asked = info.ModTime()
|
||||
began = asked
|
||||
if raw, err := os.ReadFile(drainMark()); err == nil {
|
||||
if t, err := time.Parse(time.RFC3339, strings.TrimSpace(string(raw))); err == nil && !t.After(asked) {
|
||||
began = t
|
||||
}
|
||||
}
|
||||
return began, asked, true
|
||||
}
|
||||
|
||||
// drained says the agent was asked whether it is busy within drainLapse: it takes no new build request.
|
||||
func drained() bool { return drainedSince(time.Now()) }
|
||||
|
||||
func drainedSince(now time.Time) bool {
|
||||
_, asked, ok := drainedAt()
|
||||
return ok && now.Sub(asked) < drainLapse
|
||||
}
|
||||
|
||||
// drainedTooLong is the health check's second half: nil unless the agent has been drained, asked all along, for
|
||||
// longer than its node-engine's bound and the slack.
|
||||
func drainedTooLong(now time.Time) error {
|
||||
began, asked, ok := drainedAt()
|
||||
if !ok || now.Sub(asked) >= drainLapse || now.Sub(began) <= drainTooLong {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("drained since %s and still asked: the replacement its node-engine waits for has not come "+
|
||||
"within %s (novox/hq ADR 0335)", began.UTC().Format(time.RFC3339), drainTooLong)
|
||||
}
|
||||
|
||||
// claim marks a request in hand, unless the agent is drained: then nothing is in hand, and the request goes
|
||||
// back to the queue.
|
||||
func claim() bool {
|
||||
// Made, never written over: one build at a time, and a mark already there is a build in hand.
|
||||
f, err := os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
||||
if err != nil {
|
||||
// Unmarked, an ask would say idle while this build runs: the build is not taken.
|
||||
fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err)
|
||||
return false
|
||||
}
|
||||
_, err = f.WriteString(time.Now().UTC().Format(time.RFC3339) + "\n")
|
||||
if closeErr := f.Close(); err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
if err != nil {
|
||||
_ = os.Remove(inHandMark())
|
||||
fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err)
|
||||
return false
|
||||
}
|
||||
if drained() {
|
||||
_ = os.Remove(inHandMark())
|
||||
fmt.Fprintln(os.Stderr, "asked whether busy, so this agent takes no new build: handed back")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// holding says what is in hand, for the ask to print: written once the request is read.
|
||||
func holding(what string) {
|
||||
_ = os.WriteFile(inHandMark(), []byte(time.Now().UTC().Format(time.RFC3339)+" "+what+"\n"), 0o644)
|
||||
}
|
||||
|
||||
// released ends the build in hand.
|
||||
func released() {
|
||||
if err := os.Remove(inHandMark()); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
fmt.Fprintf(os.Stderr, "cannot remove the mark of the build in hand (%s): %v\n", inHandMark(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// undrained clears both marks at the start of a holder: a container started again is asked afresh.
|
||||
func undrained() error {
|
||||
for _, mark := range []string{drainMark(), inHandMark()} {
|
||||
if err := os.Remove(mark); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("cannot remove %s: %w", mark, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// busy is the ask: it drains the agent, then says whether a build is in hand. Nil — exit 0 — while one is.
|
||||
func busy() (string, error) {
|
||||
// The mark keeps when the drain began while it is asked within drainLapse, and is written again on every
|
||||
// ask: its time is the last ask.
|
||||
began := time.Now().UTC()
|
||||
if was, _, ok := drainedAt(); ok && drained() {
|
||||
began = was.UTC()
|
||||
}
|
||||
if err := os.WriteFile(drainMark(), []byte(began.Format(time.RFC3339)+"\n"), 0o644); err != nil {
|
||||
// Not drained, it could take the next build: said, and answered busy only for what is in hand now.
|
||||
fmt.Fprintf(os.Stderr, "cannot mark this agent drained (%s): %v\n", drainMark(), err)
|
||||
}
|
||||
held, err := os.ReadFile(inHandMark())
|
||||
if err != nil {
|
||||
return "", errors.New("idle: no build in hand, and this agent takes no new one")
|
||||
}
|
||||
since, what, _ := strings.Cut(strings.TrimSpace(string(held)), " ")
|
||||
if what == "" {
|
||||
what = "a build"
|
||||
}
|
||||
return fmt.Sprintf("busy: %s, in hand since %s; taking no new build", what, since), nil
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// `mesh-builder busy` answers 0 while a build is in hand and non-zero when idle, and after its first ask the
|
||||
// agent takes no new build request (novox/hq ADR 0335 §5).
|
||||
func TestBusyAnswersWhileABuildIsInHandAndDrainsTheAgent(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if drained() {
|
||||
t.Fatal("an agent never asked is drained")
|
||||
}
|
||||
// A build is claimed and runs.
|
||||
if !claim() {
|
||||
t.Fatal("an agent never asked refused a build")
|
||||
}
|
||||
holding("novox/mesh-controller (check-280)")
|
||||
said, err := busy()
|
||||
if err != nil || !strings.Contains(said, "novox/mesh-controller (check-280)") {
|
||||
t.Fatalf("an agent with a build in hand did not answer busy with what it holds: %q %v", said, err)
|
||||
}
|
||||
// From that ask on it takes nothing new: the fetch is not made, and one fetched is handed back.
|
||||
if !drained() {
|
||||
t.Fatal("the ask did not drain the agent")
|
||||
}
|
||||
if claim() {
|
||||
t.Fatal("a drained agent took a new build")
|
||||
}
|
||||
// The build in hand is still in hand after the refused claim, and busy goes on saying so.
|
||||
if _, err := busy(); err != nil {
|
||||
t.Fatalf("a refused claim ended the build in hand: %v", err)
|
||||
}
|
||||
// It ends: idle, and still drained.
|
||||
released()
|
||||
if _, err := busy(); err == nil {
|
||||
t.Fatal("an agent with nothing in hand answered busy")
|
||||
}
|
||||
if claim() {
|
||||
t.Fatal("a drained agent took a build after its last one ended")
|
||||
}
|
||||
// Started again, it is asked afresh.
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if drained() || !claim() {
|
||||
t.Fatal("a holder started again is still drained")
|
||||
}
|
||||
released()
|
||||
}
|
||||
|
||||
// An ask racing a fetch sees the build in hand, or the agent sees the drain and hands the request back: never
|
||||
// neither, which would replace the container under a build just taken.
|
||||
func TestAnAskRacingAFetchNeverMissesTheBuild(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
for i := 0; i < 200; i++ {
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var claimed bool
|
||||
var askErr error
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() { defer wg.Done(); claimed = claim() }()
|
||||
go func() { defer wg.Done(); _, askErr = busy() }()
|
||||
wg.Wait()
|
||||
if claimed && askErr != nil {
|
||||
t.Fatalf("round %d: the agent took a build and the ask said idle", i)
|
||||
}
|
||||
released()
|
||||
}
|
||||
}
|
||||
|
||||
// A drain no longer asked lapses: an agent whose replacement was withdrawn after the first ask (a put-back, a
|
||||
// revert, a stopped walk) takes work again after drainLapse, rather than staying drained with its health green.
|
||||
func TestADrainNoLongerAskedLapses(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := busy(); err == nil {
|
||||
t.Fatal("an idle agent answered busy")
|
||||
}
|
||||
if !drained() || claim() {
|
||||
t.Fatal("an agent just asked takes work")
|
||||
}
|
||||
// The last ask was three minutes ago.
|
||||
old := time.Now().Add(-3 * time.Minute)
|
||||
if err := os.Chtimes(drainMark(), old, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if drained() {
|
||||
t.Fatal("a drain no longer asked for three minutes still holds")
|
||||
}
|
||||
if !claim() {
|
||||
t.Fatal("an agent whose drain lapsed takes no work")
|
||||
}
|
||||
released()
|
||||
}
|
||||
|
||||
// An agent drained, and asked all along, for longer than its node-engine's bound and the slack is not ready: the
|
||||
// replacement it was drained for never came. One asked again after a lapse starts a new drain.
|
||||
func TestAnAgentDrainedTooLongIsNotReady(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
if err := undrained(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
taking("mesh-build-machine")
|
||||
if _, err := busy(); err == nil {
|
||||
t.Fatal("an idle agent answered busy")
|
||||
}
|
||||
now := time.Now()
|
||||
if err := drainedTooLong(now); err != nil {
|
||||
t.Fatalf("an agent drained a moment ago is not ready: %v", err)
|
||||
}
|
||||
// The drain began 25 minutes ago, and the node-engine asked a moment ago.
|
||||
began := now.Add(-25 * time.Minute).UTC().Format(time.RFC3339)
|
||||
if err := os.WriteFile(drainMark(), []byte(began+"\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := drainedTooLong(time.Now())
|
||||
if err == nil || !strings.Contains(err.Error(), "has not come") {
|
||||
t.Fatalf("an agent drained for 25 minutes reads ready: %v", err)
|
||||
}
|
||||
// Each ask keeps when the drain began.
|
||||
if _, err := busy(); err == nil {
|
||||
t.Fatal("an idle agent answered busy")
|
||||
}
|
||||
if err := drainedTooLong(time.Now()); err == nil {
|
||||
t.Fatal("an ask started the drain again")
|
||||
}
|
||||
// The asks stopped: the drain lapsed, and so did the fault.
|
||||
old := time.Now().Add(-3 * time.Minute)
|
||||
if err := os.Chtimes(drainMark(), old, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := drainedTooLong(time.Now()); err != nil {
|
||||
t.Fatalf("a lapsed drain still reads not ready: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// An ask this build does not know exits non-zero: an older image asked `busy` reads idle, not busy for the
|
||||
// node-engine's whole bound.
|
||||
func TestAnUnknownCommandFails(t *testing.T) {
|
||||
was := os.Args
|
||||
t.Cleanup(func() { os.Args = was })
|
||||
os.Args = []string{"mesh-builder", "some-later-word"}
|
||||
if err := run(); err == nil {
|
||||
t.Fatal("an unknown command exited 0")
|
||||
}
|
||||
os.Args = []string{"mesh-builder", "help"}
|
||||
if err := run(); err != nil {
|
||||
t.Fatalf("help failed: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -41,6 +41,9 @@ type holder struct {
|
||||
say func(link.HolderState) error
|
||||
// remove runs what a kill needs outside the build: the containers left behind.
|
||||
remove builder.Runner
|
||||
// scratch is where check builds are kept (novox/hq ADR 0350 §5): what the week's sweep lets go of. Nil where
|
||||
// this machine publishes to no artifact store.
|
||||
scratch builder.Scratch
|
||||
|
||||
mu sync.Mutex
|
||||
paused bool
|
||||
@@ -131,6 +134,34 @@ func (h *holder) stateWhilePaused(ctx context.Context, every time.Duration) {
|
||||
}
|
||||
}
|
||||
|
||||
// expireChecks lets go of every check build older than a week (novox/hq ADR 0350 §5), at start and every while
|
||||
// after. Every holder sweeps the one store; what one let go of, another finds gone.
|
||||
func (h *holder) expireChecks(ctx context.Context, every time.Duration, now func() time.Time) {
|
||||
sweep := func() {
|
||||
if h.scratch == nil {
|
||||
return
|
||||
}
|
||||
n, err := h.scratch.Expire(ctx, builder.CheckBuildsKept, now())
|
||||
switch {
|
||||
case err != nil:
|
||||
fmt.Fprintf(os.Stderr, "check builds older than %s: %d let go of, and not all: %v\n", builder.CheckBuildsKept, n, err)
|
||||
case n > 0:
|
||||
fmt.Fprintf(os.Stderr, "check builds older than %s: %d check(s)' builds let go of\n", builder.CheckBuildsKept, n)
|
||||
}
|
||||
}
|
||||
sweep()
|
||||
tick := time.NewTicker(every)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
sweep()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// begin records the build this machine took, with the cancel that ends it.
|
||||
func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running {
|
||||
r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})}
|
||||
|
||||
+162
-13
@@ -26,6 +26,7 @@ import (
|
||||
"os/signal"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -75,6 +76,12 @@ digest of their own configuration. The result is printed as JSON.
|
||||
|
||||
exits 0 once the agent running in this container is bound to its build work and pulls it, and
|
||||
1 before: its container's health check.
|
||||
|
||||
mesh-builder busy
|
||||
|
||||
exits 0 while the agent running in this container has a build in hand, and 1 when it has none;
|
||||
from the first ask the agent takes no new build: its container's busy command, asked before the
|
||||
container is replaced (novox/hq ADR 0335).
|
||||
`
|
||||
|
||||
func run() error {
|
||||
@@ -86,17 +93,41 @@ func run() error {
|
||||
case "build":
|
||||
return buildOnce(context.Background(), os.Args[2:])
|
||||
case "ready":
|
||||
// The container's health check (novox/hq issue 544): exits non-zero until the agent takes work.
|
||||
return ready()
|
||||
default:
|
||||
// The container's health check (novox/hq issue 544): exits non-zero until the agent takes work, and
|
||||
// once it has been drained longer than its node-engine would wait to replace it (ADR 0335).
|
||||
if err := ready(); err != nil {
|
||||
return err
|
||||
}
|
||||
return drainedTooLong(time.Now())
|
||||
case "busy":
|
||||
// The container's busy command (novox/hq ADR 0335): drains the agent, and exits non-zero once
|
||||
// nothing is in hand.
|
||||
said, err := busy()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(said)
|
||||
return nil
|
||||
case "help", "-h", "--help":
|
||||
fmt.Print(usage)
|
||||
return nil
|
||||
default:
|
||||
// **An ask this build does not know fails** (novox/hq ADR 0335): an image older than `busy`, asked
|
||||
// it, must read as idle — exiting 0 would hold its container for the node-engine's whole bound.
|
||||
fmt.Fprint(os.Stderr, usage)
|
||||
return fmt.Errorf("%q is not a command this mesh-builder knows", os.Args[1])
|
||||
}
|
||||
}
|
||||
|
||||
if err := notReady(); err != nil {
|
||||
return err
|
||||
}
|
||||
// **The builder's own version is part of every reuse key** (novox/hq ADR 0350 §1): a builder that builds
|
||||
// differently never takes a build another one made.
|
||||
builder.Version = builder.SelfVersion(version)
|
||||
if err := undrained(); err != nil {
|
||||
return err
|
||||
}
|
||||
credential, err := brokerFrom()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -149,8 +180,13 @@ func run() error {
|
||||
// **What an earlier holder here left goes before anything is taken** (novox/hq issue 306): a holder
|
||||
// recreated mid-check left the check's store and bus running, and the ask went to another machine.
|
||||
h.removeLeftBehind()
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused,
|
||||
Taking: func() { taking(seat) }})
|
||||
// Paused or drained, it takes nothing new (novox/hq ADR 0219, ADR 0335): a drained agent finishes the build
|
||||
// in hand and then says idle, and its container is replaced between builds.
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{
|
||||
Paused: func() bool { return h.Paused() || drained() },
|
||||
Taking: func() { taking(seat) },
|
||||
Claim: claim,
|
||||
})
|
||||
defer machine.Close()
|
||||
|
||||
// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
|
||||
@@ -174,8 +210,13 @@ func run() error {
|
||||
|
||||
fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
|
||||
publisher := builder.Registry{Address: registry, Run: builder.Command}
|
||||
// Check builds are let go of after a week (novox/hq ADR 0350 §5).
|
||||
h.scratch = publisher
|
||||
go h.expireChecks(ctx, time.Hour, time.Now)
|
||||
|
||||
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
|
||||
defer released()
|
||||
holding(work.Request().Repository + " (" + work.Request().ID + ")")
|
||||
answer(ctx, publisher, on, workspace, work, h)
|
||||
})
|
||||
}
|
||||
@@ -232,7 +273,12 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
// role's own events under the build's id (novox/hq ADR 0157) — so whoever asked, and anybody
|
||||
// watching, reads the same lines this container's log holds, live, and after the fact from the
|
||||
// stream. Said first, before anything runs, so a build that hangs is one that visibly started.
|
||||
// One line at a time: a check's builds for its merge say their lines while its tests say theirs (novox/hq
|
||||
// issue 716).
|
||||
var saying sync.Mutex
|
||||
say := func(step, message string) {
|
||||
saying.Lock()
|
||||
defer saying.Unlock()
|
||||
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
|
||||
work.Say(step, message)
|
||||
if mine != nil && step != "run" && step != "output" {
|
||||
@@ -262,6 +308,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
if err == nil {
|
||||
say("packages", saysRegistry(npmrc))
|
||||
}
|
||||
registryHere, isRegistry := publisher.(builder.Registry)
|
||||
var built builder.Result
|
||||
if request.Check != nil {
|
||||
// **A pull request's merge check, not a build** (novox/hq to-be 45 §9): nothing is built,
|
||||
@@ -271,17 +318,37 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
if r, ok := publisher.(builder.Registry); ok {
|
||||
registry = r.Address
|
||||
}
|
||||
spec := checkSpecOf(request)
|
||||
// The judge cache the build agent declares as its own data, never in the workspace: only judge builds and
|
||||
// mesh-lab's replays mount it, never a check's containers, so no pull request's script can write what a judge
|
||||
// is built with or kept as, or what the replays run (novox/hq ADR 0358 §6, issue 709).
|
||||
spec.JudgeCache = judgeCache()
|
||||
// What the merge would move, built by the check for the walk to take (novox/hq ADR 0350 §1): into the
|
||||
// scratch namespace. **Never with the package registry's credential**: this is code nobody has approved
|
||||
// yet, and a recipe could read the credential it is handed (issue 462). A module that resolves packages
|
||||
// at build time has no reuse key anyway, so nothing is lost by refusing it here.
|
||||
if isRegistry && len(spec.Builds) > 0 {
|
||||
spec.BuildWith = &builder.CheckBuilding{Publish: publisher, Scratch: registryHere, Held: request.Held,
|
||||
Seats: request.Seats, Named: namedOf(request.Check.Named)}
|
||||
}
|
||||
var v builder.CheckVerdict
|
||||
v, err = builder.Check(building, builder.Command, checkSpecOf(request), workspace, registry, forgeFrom(), say)
|
||||
v, err = builder.Check(building, builder.Command, spec, workspace, registry, forgeFrom(), say)
|
||||
if err == nil {
|
||||
result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report,
|
||||
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo),
|
||||
Head: v.Head, Trunk: v.Trunk, Beside: v.Beside}
|
||||
Head: v.Head, Trunk: v.Trunk, Beside: v.Beside, Built: checkBuiltOf(v.Built),
|
||||
Release: v.Release, JudgedBy: v.JudgedBy, BesideAt: v.BesideAt, Validator: v.Validator}
|
||||
// What each of a group's heads was judged as (novox/hq ADR 0323 §6).
|
||||
for _, m := range v.Members {
|
||||
result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo,
|
||||
Commit: m.Head, Trunk: m.Trunk})
|
||||
}
|
||||
// A branch tried before its pull request reads its outcome from this log, whole (novox/hq issue 686).
|
||||
if request.Check.Try {
|
||||
if line, err := json.Marshal(triedOf(v)); err == nil {
|
||||
say(link.StepTried, string(line))
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if err == nil {
|
||||
// The package-registry credential is a build input, so it is resolved before the clone: a
|
||||
@@ -289,9 +356,22 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
// after a clone that then fails at npm ci.
|
||||
// Every container it starts is labelled with its id, so a kill finds what outlived the
|
||||
// docker client (ADR 0219).
|
||||
built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
|
||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||
forgeFrom(), say, request.Seats)
|
||||
if request.Reuse != nil && isRegistry && !request.DryRun {
|
||||
// **A walk's build takes the check's build of the same key** (novox/hq ADR 0350 §2).
|
||||
reuse := builder.Reuse{Scratch: registryHere}
|
||||
for _, p := range request.Reuse.Pulls {
|
||||
reuse.Pulls = append(reuse.Pulls, builder.PullRef{Owner: p.Owner, Repo: p.Repo, Number: p.Number})
|
||||
}
|
||||
// Only what a verdict names is taken (ADR 0350 §2).
|
||||
reuse.Named = namedOf(request.Reuse.Builds)
|
||||
built, err = builder.BuildReusing(building, builder.Labelled(builder.Command, request.ID), publisher, reuse,
|
||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||
forgeFrom(), say, request.Seats)
|
||||
} else {
|
||||
built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
|
||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||
forgeFrom(), say, request.Seats)
|
||||
}
|
||||
}
|
||||
// What it copied into the store, whatever became of the build (novox/hq ADR 0257).
|
||||
result.Mirrored = built.Mirrored
|
||||
@@ -327,7 +407,19 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
}
|
||||
result.Against = built.Against
|
||||
result.SourceFingerprint = built.Source
|
||||
result.Predefined = built.Predefined
|
||||
// What the walk did instead of building, or why it built (novox/hq ADR 0350 §3, §4).
|
||||
result.ReuseKey = built.ReuseKey
|
||||
if built.Reused != nil {
|
||||
f := built.Reused.From
|
||||
result.Reused = built.Reused.Words()
|
||||
result.ReusedFrom = &link.ReusedFrom{Pull: link.PullRef{Owner: f.Pull.Owner, Repo: f.Pull.Repo,
|
||||
Number: f.Pull.Number}, Head: f.Head, Trunk: f.Trunk, Check: f.Check, Key: f.Key}
|
||||
} else if request.Reuse != nil {
|
||||
result.BuiltWhy = built.Why
|
||||
if result.BuiltWhy == "" {
|
||||
result.BuiltWhy = "it has no artifact to build"
|
||||
}
|
||||
}
|
||||
result.Trunk, result.OnTrunk, result.Branches = built.Trunk, built.OnTrunk, built.Branches
|
||||
for _, r := range built.Read {
|
||||
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
||||
@@ -374,9 +466,14 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
|
||||
spec := builder.CheckSpec{ID: request.ID, Repository: request.Repository, Ref: request.Ref,
|
||||
Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{},
|
||||
Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base,
|
||||
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held)}
|
||||
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held),
|
||||
Tests: c.Tests, StoreOnly: c.StoreOnly, Release: c.Release}
|
||||
for dir, b := range c.Beside {
|
||||
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref}
|
||||
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref, Else: b.Else}
|
||||
}
|
||||
for _, b := range c.Builds {
|
||||
spec.Builds = append(spec.Builds, builder.CheckBuildSpec{Module: b.Module, Repository: b.Repository, Path: b.Path,
|
||||
Ref: b.Ref, Merged: b.Merged, Tier: b.Tier, Skip: b.Skip})
|
||||
}
|
||||
for _, m := range c.Members {
|
||||
spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository,
|
||||
@@ -385,6 +482,58 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
|
||||
return spec
|
||||
}
|
||||
|
||||
// judgeCache is the judge cache the build agent's manifest declares (MESH_JUDGE_CACHE, novox/hq ADR 0358 §6): a
|
||||
// directory mounted at the same path on the host, so a judge build's mounts name what the agent sees. Empty when the
|
||||
// agent declares none, and every judge is then built cold under its check's root.
|
||||
func judgeCache() string { return os.Getenv("MESH_JUDGE_CACHE") }
|
||||
|
||||
// triedOf is a try's outcome as its log line says it (novox/hq issue 686): the repository's own check's verdict,
|
||||
// or the check's where that did not run — a head that does not merge — and the lines naming what failed.
|
||||
func triedOf(v builder.CheckVerdict) link.Tried {
|
||||
t := link.Tried{Verdict: v.Verdict, Summary: v.Summary, Took: v.Took.Round(time.Second).String(),
|
||||
Head: v.Head, Trunk: v.Trunk}
|
||||
if v.Repo != nil && (v.Gate == nil || v.Gate.Verdict == "pass") {
|
||||
t.Verdict, t.Summary = v.Repo.Verdict, v.Repo.Summary
|
||||
}
|
||||
if v.Repo != nil && v.Repo.Failed != "" {
|
||||
t.Failed = strings.Split(v.Repo.Failed, "\n")
|
||||
}
|
||||
if t.Verdict == "" {
|
||||
t.Verdict = "error"
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// namedOf are the builds the seat's verdicts name, as the builder believes a scratch record by them (novox/hq ADR
|
||||
// 0350 §2, issue 703): each module's key with every artifact's digest.
|
||||
func namedOf(builds []link.CheckedBuild) []builder.NamedBuild {
|
||||
var out []builder.NamedBuild
|
||||
for _, b := range builds {
|
||||
named := builder.NamedBuild{Pull: builder.PullRef{Owner: b.Pull.Owner, Repo: b.Pull.Repo,
|
||||
Number: b.Pull.Number}, Check: b.Check, Module: b.Built.Module, Key: b.Built.Key,
|
||||
Digests: map[string]string{}}
|
||||
for _, m := range b.Built.Made {
|
||||
named.Digests[m.Name] = builder.DigestOf(m.Reference)
|
||||
}
|
||||
out = append(out, named)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// checkBuiltOf is a check's builds for its merge as the outcome carries them (novox/hq ADR 0350 §1).
|
||||
func checkBuiltOf(built []builder.CheckBuilt) []link.CheckBuilt {
|
||||
var out []link.CheckBuilt
|
||||
for _, b := range built {
|
||||
c := link.CheckBuilt{Module: b.Module, Key: b.Key, NotBuilt: b.NotBuilt, Took: b.Took.Round(time.Second).String(),
|
||||
Kept: b.Kept}
|
||||
for _, m := range b.Made {
|
||||
c.Made = append(c.Made, link.MadeArtifact{Name: m.Name, Kind: m.Kind, Reference: m.Reference})
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// layerOf is one layer of a check as the outcome carries it.
|
||||
func layerOf(l *builder.Layer) *link.CheckLayer {
|
||||
if l == nil {
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A pull request into a release branch reaches the check as one** (novox/hq ADR 0358): the build seat hands the
|
||||
// check the release branch and, beside it, the trunk each repository is read at when it has no branch of that name;
|
||||
// and it hands the check the judge cache the build agent declares (MESH_JUDGE_CACHE), never a path of its own.
|
||||
func TestTheBuildSeatHandsTheCheckItsReleaseBranch(t *testing.T) {
|
||||
spec := checkSpecOf(link.BuildRequest{ID: "b", Repository: "r", Ref: "abc", Check: &link.CheckRequest{
|
||||
Owner: "novox", Repo: "mesh-catalog", Base: "feat/57", Release: "feat/57",
|
||||
Beside: map[string]link.CheckedOut{"mesh-host": {Repository: "h", Ref: "feat/57", Else: "main"}}}})
|
||||
if spec.Release != "feat/57" || spec.Beside["mesh-host"].Ref != "feat/57" || spec.Beside["mesh-host"].Else != "main" {
|
||||
t.Fatalf("the check was handed release %q and mesh-host %+v", spec.Release, spec.Beside["mesh-host"])
|
||||
}
|
||||
t.Setenv("MESH_JUDGE_CACHE", "/var/lib/mesh/build-agent/judge-cache")
|
||||
if got := judgeCache(); got != "/var/lib/mesh/build-agent/judge-cache" {
|
||||
t.Fatalf("the judge cache the manifest declares is read as %q", got)
|
||||
}
|
||||
t.Setenv("MESH_JUDGE_CACHE", "")
|
||||
if got := judgeCache(); got != "" {
|
||||
t.Fatalf("with no judge cache declared, %q is used", got)
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ func TestBuilderDiagnosticsStayOffStdout(t *testing.T) {
|
||||
"version)": true, // --version
|
||||
`"stopping")`: true, // the loop.s shutdown line
|
||||
"usage)": true, // --help text, for a human
|
||||
"said)": true, // busy: the answer its node-engine reads (novox/hq ADR 0335)
|
||||
}
|
||||
for _, file := range []string{"once.go", "main.go"} {
|
||||
src, err := os.ReadFile(file)
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
)
|
||||
|
||||
// A try's outcome, as the build seat says it in the build's log (novox/hq issue 686): the repository's own
|
||||
// check's verdict, how long it took, and each line naming what failed.
|
||||
func TestATrysOutcomeNamesWhatFailed(t *testing.T) {
|
||||
tried := triedOf(builder.CheckVerdict{Verdict: "pass", Summary: "the change touches no module", Took: 303 * time.Second,
|
||||
Head: "c0ffee", Trunk: "ca3f43c", Gate: &builder.Layer{Verdict: "pass"},
|
||||
Repo: &builder.Layer{Verdict: "fail", Summary: "its merge-check.sh failed: TestX",
|
||||
Failed: "--- FAIL: TestX (0.01s)\n x_test.go:12: got 1, want 2"}})
|
||||
if tried.Verdict != "fail" || tried.Took != "5m3s" || len(tried.Failed) != 2 || tried.Trunk != "ca3f43c" ||
|
||||
tried.Summary != "its merge-check.sh failed: TestX" {
|
||||
t.Fatalf("a failed try: %+v", tried)
|
||||
}
|
||||
// A head that does not merge fails both layers: the check's own word.
|
||||
conflict := triedOf(builder.CheckVerdict{Verdict: "fail", Summary: "does not merge", Gate: &builder.Layer{Verdict: "fail"},
|
||||
Repo: &builder.Layer{Verdict: "fail", Summary: "does not merge"}})
|
||||
if conflict.Verdict != "fail" || conflict.Summary != "does not merge" {
|
||||
t.Fatalf("a try that does not merge: %+v", conflict)
|
||||
}
|
||||
// No verdict is an error, never a pass.
|
||||
if none := triedOf(builder.CheckVerdict{}); none.Verdict != "error" {
|
||||
t.Fatalf("a try with no verdict: %+v", none)
|
||||
}
|
||||
}
|
||||
@@ -26,8 +26,11 @@ import (
|
||||
//
|
||||
// - **The serving controller** takes it before it does anything else — before it asserts the bus's
|
||||
// objects, which are the controller's to write — waiting while another holds it, and renews it.
|
||||
// A renewal refused or failed is the lease lost: the gate closes at once and the process exits, so
|
||||
// its service manager restarts it as a candidate (serve, in push.go).
|
||||
// Another holder seen, or the key's age passed with no renewal made, is the lease lost: the gate
|
||||
// closes at once and the process exits, so its service manager restarts it as a candidate (serve,
|
||||
// in push.go). A renewal the bus does not answer in time is not a loss (novox/hq issue 477): the
|
||||
// lease is kept, no new act starts once half the key's age passed unrenewed, and the next renewal
|
||||
// made opens the gate again.
|
||||
// - **A command run at a shell** — `push` in the installer, the lab, a person repairing a mesh whose
|
||||
// controller is down (issue 201) — acts **under the holder's epoch** when a controller holds the
|
||||
// lease: it is the same mesh's word, composed and sent under the store's hold of each machine like
|
||||
|
||||
@@ -688,7 +688,7 @@ func spelledOf(m inventory.BatchedMerge) string {
|
||||
func namedOf(m inventory.BatchedMerge, repository string) inventory.PlanMerge {
|
||||
k := announcedOf(m)
|
||||
return inventory.PlanMerge{Repository: repository, Commit: m.Commit, Number: k.Number, Title: k.Title, Moves: k.Moves,
|
||||
Merged: m.Merged, Heard: m.Heard}
|
||||
Head: k.HeadSHA, Merged: m.Merged, Heard: m.Heard}
|
||||
}
|
||||
|
||||
// laterMerge says a was merged after b: by the forge's merge time, then by when each was heard.
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
@@ -483,6 +484,7 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
|
||||
Seats: seatBases(ctx),
|
||||
DryRun: dryRun,
|
||||
}
|
||||
request.Reuse = reuseAsked(ctx, dryRun)
|
||||
fmt.Printf("asked for %s", source)
|
||||
if source.Seat != "" {
|
||||
fmt.Printf(" (%s)", repository)
|
||||
@@ -669,6 +671,12 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
|
||||
manifest.Module, short(result.Commit), err)
|
||||
}
|
||||
// **Never a build in the scratch namespace** (novox/hq ADR 0239 §9, ADR 0350 §2): what a check builds is
|
||||
// registered only as the walk's copy in the module's own namespace.
|
||||
if scratch := inScratch(result.Made); scratch != "" {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %s is in the scratch "+
|
||||
"namespace, which only a walk's copy leaves (ADR 0350)", result.On, manifest.Module, short(result.Commit), scratch)
|
||||
}
|
||||
// **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard
|
||||
// twice, or replayed, would otherwise make the build a rollback put back what the module is again,
|
||||
// and the next push would send it.
|
||||
@@ -696,20 +704,6 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
}
|
||||
}
|
||||
}
|
||||
// **The list of predefined keys arrives with a build of the catalogue** (novox/hq issue 630): the
|
||||
// builder reads predefined.json at the catalogue repository's root at the commit built, and it is kept
|
||||
// before the module is registered, so a preference falling back to a key added in the same commit is
|
||||
// judged against the list that adds it. Only from the catalogue: another repository names no list.
|
||||
if len(result.Predefined) > 0 && isCatalogueRepository(result.Repository) {
|
||||
list, err := catalogue.ParsePredefined(result.Predefined)
|
||||
if err != nil {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
|
||||
manifest.Module, short(result.Commit), err)
|
||||
}
|
||||
if err := inv.SetPredefined(ctx, list, result.Repository, result.Commit); err != nil {
|
||||
return manifest, kept, err
|
||||
}
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, manifest, recorded); err != nil {
|
||||
if errors.Is(err, inventory.ErrSuperseded) {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w",
|
||||
@@ -728,6 +722,16 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
return manifest, kept, nil
|
||||
}
|
||||
|
||||
// inScratch is the first artifact a build made into the scratch namespace; empty when none is.
|
||||
func inScratch(made []link.MadeArtifact) string {
|
||||
for _, m := range made {
|
||||
if path, kept := catalogue.InArtifactStore(catalogue.Recorded(m.Reference)); kept && strings.HasPrefix(path, builder.ScratchPrefix) {
|
||||
return m.Reference
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// errOffTheTrunk is a build of a commit off its repository's trunk, which is never published.
|
||||
var errOffTheTrunk = errors.New("the commit is not on its repository's trunk: a commit off the trunk is checked, " +
|
||||
"never published (ADR 0238) — merge it, and the merge builds it")
|
||||
@@ -1039,16 +1043,3 @@ func buildLog(ctx context.Context, id string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// catalogueRepository is the repository the catalogue is: the one whose predefined.json is the list of
|
||||
// predefined keys (novox/hq issue 630).
|
||||
const catalogueRepository = "mesh-catalog"
|
||||
|
||||
// isCatalogueRepository is whether a repository, as a path on the forge or a URL, is the catalogue.
|
||||
func isCatalogueRepository(repository string) bool {
|
||||
trimmed := strings.TrimSuffix(strings.TrimSuffix(strings.TrimSpace(repository), "/"), ".git")
|
||||
if i := strings.LastIndex(trimmed, "/"); i >= 0 {
|
||||
trimmed = trimmed[i+1:]
|
||||
}
|
||||
return trimmed == catalogueRepository
|
||||
}
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **Every user's slow consumer is S9's, not the controller's alone** (novox/hq ADR 0357, issue 270). On
|
||||
// 2026-10-07 the bus closed two clients on the laptop as slow consumers and nothing raised a condition:
|
||||
// the controller heard only its own connection. Said by the bus module and read back, a user the bus
|
||||
// closes is one condition naming the module and its machine, raised at the first look and cleared an hour
|
||||
// after the last closure.
|
||||
func TestAUserTheBusClosedAsASlowConsumerIsOneConditionNamingItsModuleAndMachine(t *testing.T) {
|
||||
closed := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
|
||||
now := closed.Add(20 * time.Second)
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||
t.Cleanup(func() { k.Close(t.Context()) })
|
||||
|
||||
advisories := link.BusClosures([]link.BusClosure{
|
||||
{User: "laptop.claude-code", CID: 10, Reason: "Slow Consumer (Write Deadline)", Opened: closed.Add(-time.Hour),
|
||||
Closed: closed.Add(-10 * time.Minute)},
|
||||
{User: "laptop.claude-code", CID: 44, Reason: "Slow Consumer (Write Deadline)", Opened: closed.Add(-time.Minute),
|
||||
Closed: closed},
|
||||
{User: "anchor.desk", CID: 7, Reason: "Maximum Message Payload Exceeded", Opened: closed.Add(-time.Hour),
|
||||
Closed: closed},
|
||||
}, []string{link.AdvisorySlowConsumer, link.AdvisorySlowConsumer, link.AdvisoryConnectionClosed}, closed.Add(-time.Hour))
|
||||
look := func() {
|
||||
t.Helper()
|
||||
f := &signalFacts{now: now, host: "anchor", advisories: advisories}
|
||||
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
look()
|
||||
|
||||
slow, found, err := conditions.ReadOne(t.Context(), store, "bus.laptop.claude-code.slow-consumer")
|
||||
if err != nil || !found {
|
||||
t.Fatalf("no slow-consumer condition for the user: %v %v", found, err)
|
||||
}
|
||||
if slow.Subject.Machine != "laptop" || slow.Observations != 2 || !slow.LastObserved.Equal(closed) {
|
||||
t.Fatalf("the slow consumer is said on %q, %d closure(s), last %s: %+v", slow.Subject.Machine, slow.Observations,
|
||||
slow.LastObserved, slow)
|
||||
}
|
||||
if !strings.Contains(slow.Summary, "claude-code on laptop") || strings.Contains(slow.Summary, "laptop.claude-code") {
|
||||
t.Fatalf("the summary does not name the module in words, or carries the dotted user: %q", slow.Summary)
|
||||
}
|
||||
if !strings.Contains(slow.Headline, "Claude-code on laptop") {
|
||||
t.Fatalf("the headline is %q", slow.Headline)
|
||||
}
|
||||
cut, found, err := conditions.ReadOne(t.Context(), store, "bus.anchor.desk.connection-closed")
|
||||
if err != nil || !found || cut.Subject.Machine != "anchor" || !strings.Contains(cut.Summary, "maximum message payload exceeded") {
|
||||
t.Fatalf("the user cut off for breaking the bus's rules: %v %v %+v", found, err, cut)
|
||||
}
|
||||
|
||||
// An hour after the newest closure without another, both clear.
|
||||
now = closed.Add(time.Hour + time.Minute)
|
||||
look()
|
||||
for _, key := range []string{"bus.laptop.claude-code.slow-consumer", "bus.anchor.desk.connection-closed"} {
|
||||
if _, found, _ := k.Get(t.Context(), key); found {
|
||||
t.Fatalf("%s is still open an hour after the last closure", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A read of the bus module's events that fails does not blind S9** (review of #326): the controller's own
|
||||
// signals are still said, and the failed read is a condition of its own.
|
||||
func TestABusModuleReadThatFailsKeepsTheControllersOwnSignals(t *testing.T) {
|
||||
at := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
|
||||
f := &signalFacts{now: at.Add(time.Minute), host: "anchor",
|
||||
advisories: []link.Advisory{{Kind: link.AdvisorySlowConsumer, ID: "controller", Said: "the controller fell behind",
|
||||
First: at, Last: at, Count: 1}},
|
||||
busFaultsErr: errors.New("nats: timeout")}
|
||||
keys := map[string]bool{}
|
||||
for _, o := range watchAdvisories(f) {
|
||||
keys[o.Key()] = true
|
||||
}
|
||||
if !keys["bus.controller.slow-consumer"] || !keys["bus.bus-module.bus-faults-unread"] {
|
||||
t.Fatalf("observed %v: the controller's own slow consumer and the unread events, both", keys)
|
||||
}
|
||||
for _, row := range signalsTable {
|
||||
if row.Row == "S9" && row.needs(f) != nil {
|
||||
t.Fatal("the S9 row went blind for a read of the bus module's events")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A first-node gate waits for a busy container (novox/hq ADR 0335 §4).
|
||||
//
|
||||
// When a machine reports, on what the gate sent it, that a container of a judged module was left running
|
||||
// because it said busy, the judging has not started: the build is not yet what runs there. The walk says it
|
||||
// waits for a busy container on that machine, counts no pass and no bound, and starts the judging — its
|
||||
// Since — at the report that no longer lists the container waiting. The wait is the node-engine's, bounded
|
||||
// at 15 minutes; the controller's own bound (busyGateBound past the engine's) is for an engine that stopped
|
||||
// reporting while it waited, so no walk waits for ever. An engine older than the field never lists one, and
|
||||
// its gate starts as before.
|
||||
|
||||
// busyGateSlack is how long past the engine's bound the controller still waits for the report that says the
|
||||
// container replaced: a pass of the engine (its asks every 20 seconds) and the report's way here.
|
||||
var busyGateSlack = 5 * time.Minute
|
||||
|
||||
// busyEngineBound is the node-engine's bound on a wait (novox/hq ADR 0335 §3), counted again here on this
|
||||
// controller's clock from when it first read the wait.
|
||||
const busyEngineBound = 15 * time.Minute
|
||||
|
||||
// busyGateEnds is when the gate stops waiting: the engine's own word on its bound, or this controller's count of
|
||||
// it from when it first read the wait, whichever is sooner, and the slack — so neither a clock ahead on the
|
||||
// machine nor a wrong word from it holds a walk longer.
|
||||
func busyGateEnds(b *inventory.GateBusy) time.Time {
|
||||
until := b.Seen.Add(busyEngineBound)
|
||||
if !b.Until.IsZero() && b.Until.Before(until) {
|
||||
until = b.Until
|
||||
}
|
||||
return until.Add(busyGateSlack)
|
||||
}
|
||||
|
||||
// busyWait decides, before a judging, whether the gate waits for a busy container on a machine it judges:
|
||||
// true while it waits, the gate saying so. False when nothing waits, or no longer: a wait that ended moves the
|
||||
// gate's Since to its end. Pure but for the gate it is handed.
|
||||
func busyWait(g *inventory.PlanGate, pairs []judged, f gateFacts, now time.Time) bool {
|
||||
if g.Busy != nil && g.Busy.Ended != nil {
|
||||
return false
|
||||
}
|
||||
for _, j := range pairs {
|
||||
b, waits := busyOn(j, f)
|
||||
if !waits {
|
||||
continue
|
||||
}
|
||||
if g.Busy == nil {
|
||||
g.Busy = &inventory.GateBusy{Machine: j.node, Module: j.module, Container: b.Container, Since: b.Since,
|
||||
Seen: now}
|
||||
}
|
||||
if g.Busy.Seen.IsZero() {
|
||||
g.Busy.Seen = now
|
||||
}
|
||||
g.Busy.Until, g.Busy.Said = b.Until, b.Said
|
||||
if now.After(busyGateEnds(g.Busy)) {
|
||||
// The engine said it would have replaced it by now: the judging starts, and finds the module not
|
||||
// yet healthy while the machine still lists it waiting (busyStill), until the gate's own bound.
|
||||
end := now
|
||||
g.Busy.Ended, g.Busy.PastBound = &end, true
|
||||
g.Since, g.Passes, g.LastPass, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, nil, nil, nil
|
||||
return false
|
||||
}
|
||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, g.Busy.Words(), nil
|
||||
g.Read(now, false, g.Busy.Words())
|
||||
return true
|
||||
}
|
||||
if g.Busy == nil {
|
||||
return false
|
||||
}
|
||||
// Replaced: the judging starts at the report that said so.
|
||||
end := now
|
||||
if r, said := f.reports[g.Busy.Machine]; said && r.At != nil && r.At.After(g.Busy.Since) && r.At.Before(now) {
|
||||
end = r.At.UTC()
|
||||
}
|
||||
g.Busy.Ended = &end
|
||||
g.Since, g.Passes, g.LastPass, g.Last, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, "", nil, nil, nil
|
||||
return false
|
||||
}
|
||||
|
||||
// busyOn is the container of a judged module its machine last reported, on what the gate sent it, as left
|
||||
// running because it said busy.
|
||||
func busyOn(j judged, f gateFacts) (inventory.BusyWait, bool) {
|
||||
r, said := f.reports[j.node]
|
||||
if !said || r.At == nil || !f.reportedOn(j.node, r) {
|
||||
return inventory.BusyWait{}, false
|
||||
}
|
||||
return inventory.WaitingFor(r.Busy, j.module)
|
||||
}
|
||||
|
||||
// busyStill is a judged module whose container its machine still lists waiting once the gate stopped waiting
|
||||
// for it: not yet what was sent, so not healthy, whatever its health says of the container still running.
|
||||
func busyStill(j judged, f gateFacts) (string, bool) {
|
||||
b, waits := busyOn(j, f)
|
||||
if !waits {
|
||||
return "", false
|
||||
}
|
||||
return fmt.Sprintf("%s still runs its busy container %s, past its node-engine's bound of %s",
|
||||
j.node, b.Container, b.Until.UTC().Format(time.RFC3339)), true
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A walk whose module's container waits as busy on its first machine says so, and starts that machine's
|
||||
// first-machine gate only once the machine reports the container replaced (novox/hq ADR 0335 §4).
|
||||
func TestAGateWaitsForABusyContainerAndStartsWhenItIsReplaced(t *testing.T) {
|
||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent,
|
||||
Sent: map[string]inventory.SentDeclaration{"novox": {Digest: "d1", Sequence: 7}}}
|
||||
pairs := []judged{{module: "build-agent", node: "novox"}, {module: "mesh-controller", node: "novox"}}
|
||||
reported := sent.Add(20 * time.Second)
|
||||
waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||
State: inventory.BusyWaiting, Since: reported, Until: reported.Add(15 * time.Minute),
|
||||
Said: "busy: novox/mesh-controller (check of #280), in hand since 00:58:00Z"}}
|
||||
facts := func(at time.Time, busy []inventory.BusyWait) gateFacts {
|
||||
// As judgeMoves hands it: the report held against the gate's own send.
|
||||
return gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"novox": {Node: "novox",
|
||||
Outcome: inventory.OutcomeApplied, At: &at, Declared: "d1", ReportedSequence: 7, Busy: busy}}}
|
||||
}
|
||||
|
||||
now := reported.Add(time.Minute)
|
||||
if !busyWait(g, pairs, facts(reported, waiting), now) {
|
||||
t.Fatal("a gate whose module's container waits as busy judged it")
|
||||
}
|
||||
if !g.Since.Equal(sent) || g.Busy == nil || g.Busy.Machine != "novox" || g.Busy.Ended != nil {
|
||||
t.Fatalf("the gate does not keep the wait: %+v %+v", g.Since, g.Busy)
|
||||
}
|
||||
for _, said := range []string{gateLine(g), g.Last} {
|
||||
if !strings.Contains(said, "waits for a busy container on novox") {
|
||||
t.Errorf("the walk does not say it waits for a busy container there: %s", said)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(g.Last, "check of #280") {
|
||||
t.Errorf("the wait does not say what the container holds: %s", g.Last)
|
||||
}
|
||||
|
||||
// Ten minutes on, still busy, within the engine's bound: still waiting, nothing judged, no bound run.
|
||||
now = reported.Add(10 * time.Minute)
|
||||
if !busyWait(g, pairs, facts(reported.Add(9*time.Minute), waiting), now) || g.Passes != 0 {
|
||||
t.Fatalf("the gate stopped waiting within the bound: %+v", g)
|
||||
}
|
||||
|
||||
// The machine reports it replaced: the judging starts at that report.
|
||||
replacedAt := reported.Add(12 * time.Minute)
|
||||
now = replacedAt.Add(5 * time.Second)
|
||||
replaced := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||
State: "replaced-when-idle", Since: reported, Until: reported.Add(15 * time.Minute)}}
|
||||
if busyWait(g, pairs, facts(replacedAt, replaced), now) {
|
||||
t.Fatal("the gate still waits after the container was replaced")
|
||||
}
|
||||
if g.Since == nil || !g.Since.Equal(replacedAt) || g.Busy.Ended == nil || !g.Busy.Ended.Equal(replacedAt) {
|
||||
t.Fatalf("the gate does not start at the report that said it replaced: since %v, %+v", g.Since, g.Busy)
|
||||
}
|
||||
if strings.Contains(gateLine(g), "waits for a busy container") {
|
||||
t.Errorf("a gate judging still says it waits: %s", gateLine(g))
|
||||
}
|
||||
// And a later report listing it waiting again does not hold a gate that started.
|
||||
if busyWait(g, pairs, facts(now, waiting), now.Add(time.Minute)) {
|
||||
t.Error("a gate that started waits again")
|
||||
}
|
||||
}
|
||||
|
||||
// An engine that stops reporting while it waits holds no walk for ever: past the engine's bound and the slack,
|
||||
// the judging starts, and finds the module not yet healthy while its container is still listed waiting.
|
||||
func TestAGateWaitsNoLongerThanTheEnginesBound(t *testing.T) {
|
||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
|
||||
pairs := []judged{{module: "build-agent", node: "novox"}}
|
||||
waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||
State: inventory.BusyWaiting, Since: sent, Until: sent.Add(15 * time.Minute)}}
|
||||
f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
|
||||
At: &sent, Current: true, Busy: waiting}}}
|
||||
now := sent.Add(15*time.Minute + busyGateSlack + time.Second)
|
||||
if busyWait(g, pairs, f, now) {
|
||||
t.Fatal("a gate waited past the engine's bound and the slack")
|
||||
}
|
||||
if !g.Busy.PastBound || !g.Since.Equal(now) {
|
||||
t.Fatalf("the gate does not start, said past the bound: %+v %v", g.Busy, g.Since)
|
||||
}
|
||||
if why, still := busyStill(pairs[0], f); !still || !strings.Contains(why, "past its node-engine's bound") {
|
||||
t.Fatalf("a module still running its busy container reads as what was sent: %q", why)
|
||||
}
|
||||
}
|
||||
|
||||
// A report from an engine older than the field lists nothing: the gate starts at the send, as before; and a
|
||||
// report not about the gate's send says nothing of it.
|
||||
func TestAGateOfAnEngineThatNeverWaitsStartsAsBefore(t *testing.T) {
|
||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent,
|
||||
Sent: map[string]inventory.SentDeclaration{"ace": {Digest: "d2", Sequence: 9}}}
|
||||
pairs := []judged{{module: "build-agent", node: "ace"}}
|
||||
at := sent.Add(time.Minute)
|
||||
older := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
|
||||
At: &at, Declared: "d2", ReportedSequence: 9}}}
|
||||
if busyWait(g, pairs, older, at) || g.Busy != nil || !g.Since.Equal(sent) {
|
||||
t.Fatalf("a gate on an engine that never waits did not start at the send: %+v %v", g.Busy, g.Since)
|
||||
}
|
||||
elsewhere := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
|
||||
At: &at, Declared: "d1", ReportedSequence: 8, Busy: []inventory.BusyWait{{ID: "build-agent.server",
|
||||
State: inventory.BusyWaiting, Until: at.Add(time.Hour)}}}}}
|
||||
if busyWait(g, pairs, elsewhere, at) {
|
||||
t.Fatal("a wait reported about an earlier declaration held the gate")
|
||||
}
|
||||
}
|
||||
|
||||
// The wait is bounded on this controller's clock: an engine whose clock runs an hour ahead, saying it waits until
|
||||
// then, holds the gate no longer than 15 minutes and the slack from when the controller first read the wait.
|
||||
func TestAGateWaitIsBoundedOnTheControllersClock(t *testing.T) {
|
||||
seen := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &seen}
|
||||
pairs := []judged{{module: "build-agent", node: "novox"}}
|
||||
ahead := seen.Add(time.Hour)
|
||||
f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
|
||||
At: &seen, Current: true, Busy: []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||
State: inventory.BusyWaiting, Since: ahead, Until: ahead.Add(15 * time.Minute)}}}}}
|
||||
if !busyWait(g, pairs, f, seen) || !g.Busy.Seen.Equal(seen) {
|
||||
t.Fatalf("the gate did not start waiting, seen now: %+v", g.Busy)
|
||||
}
|
||||
if !busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack-time.Second)) {
|
||||
t.Fatal("the gate stopped waiting within the bound on its own clock")
|
||||
}
|
||||
if busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack+time.Second)) || !g.Busy.PastBound {
|
||||
t.Fatalf("an engine's later word held the gate past the controller's bound: %+v", g.Busy)
|
||||
}
|
||||
}
|
||||
|
||||
// `plan` says, for a machine whose node-engine is older than the field, each container whose busy is not sent.
|
||||
func TestAPlanSaysABusyAnOlderEngineIsNotSent(t *testing.T) {
|
||||
modules := []catalogue.Manifest{{Module: "build-agent", Resources: []map[string]any{
|
||||
{"id": "server", "type": "container", "busy": map[string]any{"command": "mesh-builder busy"}},
|
||||
{"id": "agent-env", "type": "file"}}}}
|
||||
lines := busyNotSent(modules, false)
|
||||
if len(lines) != 1 || !strings.Contains(lines[0], "build-agent.server declares busy") ||
|
||||
!strings.Contains(lines[0], "replaced at once") {
|
||||
t.Fatalf("the plan does not say the busy not sent: %v", lines)
|
||||
}
|
||||
if lines := busyNotSent(modules, true); len(lines) != 0 {
|
||||
t.Fatalf("a busy sent is said as not sent: %v", lines)
|
||||
}
|
||||
}
|
||||
@@ -47,13 +47,6 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
shelf := catalogue.Shelf{}
|
||||
faulted := map[string]bool{}
|
||||
failed := 0
|
||||
// The list of predefined keys a preference may fall back to (novox/hq issue 630): the predefined.json
|
||||
// at the root of the repository the manifests are in, found above the first of them.
|
||||
list, listAt, listErr := predefinedAbove(paths[0])
|
||||
if listErr != nil {
|
||||
fmt.Fprintf(out, "%s: %v\n", listAt, listErr)
|
||||
failed++
|
||||
}
|
||||
for _, path := range paths {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
@@ -99,20 +92,6 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
failed += len(named)
|
||||
faulted[m.Module] = true
|
||||
}
|
||||
// A preference falls back only to a key on the list, with its type (novox/hq issue 630).
|
||||
if named := catalogue.PredefinedNamed(m); len(named) > 0 {
|
||||
if listAt == "" {
|
||||
fmt.Fprintf(out, "note: %s: its preferences fall back to %s, and no %s was found above it; "+
|
||||
"registration judges them against the list the mesh holds\n", path, strings.Join(named, ", "),
|
||||
catalogue.PredefinedFile)
|
||||
} else if wrong := catalogue.PredefinedProblems(m, list); len(wrong) > 0 {
|
||||
for _, p := range wrong {
|
||||
fmt.Fprintf(out, "%s: %s\n", path, p)
|
||||
}
|
||||
failed += len(wrong)
|
||||
faulted[m.Module] = true
|
||||
}
|
||||
}
|
||||
shelf[m.Module] = m
|
||||
}
|
||||
|
||||
@@ -306,27 +285,3 @@ func manifestsUnder(dir string) ([]string, error) {
|
||||
sort.Strings(found)
|
||||
return found, err
|
||||
}
|
||||
|
||||
// predefinedAbove reads the predefined.json found in the directory of a manifest or one above it, up to
|
||||
// the repository's root (the directory holding .git). The path it was found at is "" when there is none.
|
||||
func predefinedAbove(manifest string) (catalogue.Predefined, string, error) {
|
||||
dir, err := filepath.Abs(filepath.Dir(manifest))
|
||||
if err != nil {
|
||||
return nil, "", nil
|
||||
}
|
||||
for {
|
||||
at := filepath.Join(dir, catalogue.PredefinedFile)
|
||||
if raw, err := os.ReadFile(at); err == nil {
|
||||
list, err := catalogue.ParsePredefined(raw)
|
||||
return list, at, err
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, ".git")); err == nil {
|
||||
return nil, "", nil
|
||||
}
|
||||
up := filepath.Dir(dir)
|
||||
if up == dir {
|
||||
return nil, "", nil
|
||||
}
|
||||
dir = up
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A pull request's check builds go when it closes (novox/hq ADR 0350 §5): the build seat keeps them in the artifact
|
||||
// store's scratch namespace for the walk of its merge to take. **Let go of where the moment is heard**: the
|
||||
// controller ends the walk answering a merge — after that walk took what it reuses — and the delivery's owner, which
|
||||
// hears the forge say a pull request closed unmerged, asks the `forget-checks` verb; both let go through the store,
|
||||
// as the controller's own collection does. The build seat's holders let go of anything a week old on their own,
|
||||
// so a word lost here leaves a week's builds, never more.
|
||||
|
||||
// checkBuildsIn is the scratch namespace as the controller reaches the artifact store.
|
||||
func checkBuildsIn(ctx context.Context, inv *inventory.Inventory) (builder.Scratch, error) {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
address, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if address == "" {
|
||||
return nil, errors.New("the artifact store is not on the mesh's network")
|
||||
}
|
||||
return builder.Registry{Address: address}, nil
|
||||
}
|
||||
|
||||
// forgetChecks lets go of a pull request's check builds and answers what it did.
|
||||
var forgetChecks = func(ctx context.Context, inv *inventory.Inventory, pull builder.PullRef) (string, error) {
|
||||
scratch, err := checkBuildsIn(ctx, inv)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
n, err := scratch.Forget(ctx, pull)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s: %d check(s)' builds let go of, and not all: %w", pull, n, err)
|
||||
}
|
||||
return fmt.Sprintf("%s: %d check(s)' builds let go of", pull, n), nil
|
||||
}
|
||||
|
||||
// forgetChecksOfWalk lets go, on the save that ends a walk, of the check builds of every pull request whose merge
|
||||
// it answers: the walk took what it reuses. Said, never fatal.
|
||||
func forgetChecksOfWalk(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) {
|
||||
if p.WalkSaid != inventory.WalkEnded || p.Delivery == nil {
|
||||
return
|
||||
}
|
||||
for _, pull := range reuseAskOf(&p).Pulls {
|
||||
said, err := forgetChecks(ctx, inv, builder.PullRef{Owner: pull.Owner, Repo: pull.Repo, Number: pull.Number})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: its check builds stay until the week's sweep: %v\n", p.ID, err)
|
||||
continue
|
||||
}
|
||||
fmt.Printf("%s: %s\n", p.ID, said)
|
||||
}
|
||||
}
|
||||
|
||||
// forgetChecksCommand is `forget-checks <owner> <repo> <number>`: the verb the delivery's owner asks when the forge
|
||||
// says a pull request closed unmerged (novox/hq ADR 0350 §5).
|
||||
func forgetChecksCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 3 {
|
||||
return errors.New("forget-checks <owner> <repo> <number>")
|
||||
}
|
||||
number, err := strconv.Atoi(args[2])
|
||||
if err != nil || number <= 0 {
|
||||
return fmt.Errorf("%q is not a pull request's number", args[2])
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
said, err := forgetChecks(ctx, open.inventory, builder.PullRef{Owner: args[0], Repo: args[1], Number: number})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(said)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// One build per change (novox/hq ADR 0350), as the controller asks it and takes it in.
|
||||
|
||||
// The check of a change is asked to build what a merge of it would move, in the walk's order: a module of the
|
||||
// repository checked from the merge, one standing on it from its own repository as the walk builds it, and one
|
||||
// that reads the repository through a context not at all. A change that moves no module builds nothing.
|
||||
func TestACheckIsAskedToBuildWhatTheMergeWouldMove(t *testing.T) {
|
||||
entries := []inventory.Entry{
|
||||
{Manifest: catalogue.Manifest{Module: "app"}, Source: inventory.Source{Repository: "novox/shop", Seat: "git", Path: "modules/app"}},
|
||||
{Manifest: catalogue.Manifest{Module: "web"}, Source: inventory.Source{Repository: "novox/web", Seat: "git", Ref: "main"}},
|
||||
{Manifest: catalogue.Manifest{Module: "docs"}, Source: inventory.Source{Repository: "novox/docs", Seat: "git"}},
|
||||
}
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "shop", Number: 7, Base: "main", Commit: "abc"}
|
||||
scope := checkScope{Reach: mergeReach{Packaging: entries[2:], Plan: inventory.Plan{Tiers: [][]string{{"app", "docs"}, {"web"}}}}}
|
||||
clone := func(s inventory.Source) (string, error) { return "http://forge.invalid/" + s.Repository + ".git", nil }
|
||||
|
||||
got := checkBuildsOf(p, scope, entries, clone)
|
||||
want := []link.CheckBuild{
|
||||
{Module: "app", Path: "modules/app", Merged: true, Tier: 0},
|
||||
{Module: "docs", Tier: 0, Skip: "its build reads novox/shop through a context, which a check cannot hand the merge"},
|
||||
{Module: "web", Repository: "http://forge.invalid/novox/web.git", Ref: "main", Tier: 1},
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("the check is asked to build\n%+v\nwant\n%+v", got, want)
|
||||
}
|
||||
if got := checkBuildsOf(p, checkScope{}, entries, clone); got != nil {
|
||||
t.Fatalf("a change that moves no module asks for builds: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk's ask carries its leave to take a check's build, naming the pull requests of the merges it answers; a dry
|
||||
// run never does, and neither does an ask that is not a walk's.
|
||||
func TestOnlyAWalksAskTakesAChecksBuildAndADryRunNever(t *testing.T) {
|
||||
p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
|
||||
{Repository: "novox/shop", Number: 7}, {Repository: "http://forge.invalid/novox/web.git", Number: 3},
|
||||
{Repository: "novox/shop", Number: 7}, {Repository: "novox/shop"}}}}
|
||||
asked := reuseAskOf(p)
|
||||
if want := []link.PullRef{{Owner: "novox", Repo: "shop", Number: 7}, {Owner: "novox", Repo: "web", Number: 3}}; !reflect.DeepEqual(asked.Pulls, want) {
|
||||
t.Fatalf("the walk names %+v", asked.Pulls)
|
||||
}
|
||||
walk := withReuse(t.Context(), asked)
|
||||
if reuseAsked(walk, false) != asked {
|
||||
t.Fatal("a walk's ask does not carry its leave")
|
||||
}
|
||||
if reuseAsked(walk, true) != nil {
|
||||
t.Fatal("a dry run carries a leave to take a check's build")
|
||||
}
|
||||
if reuseAsked(t.Context(), false) != nil {
|
||||
t.Fatal("an ask that is not a walk's carries a leave")
|
||||
}
|
||||
}
|
||||
|
||||
// A walk ended lets go of its merges' check builds — only once it ended, after it took what it reuses.
|
||||
func TestAWalkEndedLetsGoOfItsMergesCheckBuilds(t *testing.T) {
|
||||
var forgot []builder.PullRef
|
||||
was := forgetChecks
|
||||
forgetChecks = func(_ context.Context, _ *inventory.Inventory, pull builder.PullRef) (string, error) {
|
||||
forgot = append(forgot, pull)
|
||||
return "", nil
|
||||
}
|
||||
t.Cleanup(func() { forgetChecks = was })
|
||||
p := inventory.Plan{ID: "plan-1", Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
|
||||
{Repository: "novox/shop", Number: 7}}}}
|
||||
|
||||
p.WalkSaid = inventory.WalkStarted
|
||||
forgetChecksOfWalk(t.Context(), nil, p)
|
||||
p.WalkSaid = ""
|
||||
forgetChecksOfWalk(t.Context(), nil, p)
|
||||
if len(forgot) != 0 {
|
||||
t.Fatalf("a walk not ended let go of %v, which it may still take", forgot)
|
||||
}
|
||||
p.WalkSaid = inventory.WalkEnded
|
||||
forgetChecksOfWalk(t.Context(), nil, p)
|
||||
if want := []builder.PullRef{{Owner: "novox", Repo: "shop", Number: 7}}; !reflect.DeepEqual(forgot, want) {
|
||||
t.Fatalf("the walk ended let go of %v", forgot)
|
||||
}
|
||||
}
|
||||
|
||||
// A build in the scratch namespace is never registered: only the walk's copy in the module's own namespace is.
|
||||
func TestABuildInTheScratchNamespaceIsNeverRegistered(t *testing.T) {
|
||||
if got := inScratch([]link.MadeArtifact{{Name: "server", Reference: "store.internal:5000/app/server@sha256:aa"}}); got != "" {
|
||||
t.Fatalf("a module's own build reads as a scratch build: %s", got)
|
||||
}
|
||||
scratch := "store.internal:5000/scratch/app/server@sha256:aa"
|
||||
if got := inScratch([]link.MadeArtifact{{Name: "server", Reference: scratch}}); got != scratch {
|
||||
t.Fatalf("a scratch build was not found: %q", got)
|
||||
}
|
||||
if got := inScratch([]link.MadeArtifact{{Name: "look", Reference: "http://store.internal:5000/v2/scratch/app/look/blobs/sha256:bb"}}); got == "" {
|
||||
t.Fatal("a scratch archive was not found")
|
||||
}
|
||||
}
|
||||
|
||||
// The walk says per module what it did: "reused from …" or "built: <why>" — and a build seat from before says
|
||||
// neither, which is said.
|
||||
func TestAWalksBuildSaysWhetherItReusedOrBuilt(t *testing.T) {
|
||||
if got := (link.BuildResult{Reused: "reused from the check of novox/shop#7"}).MadeWords(); got != "reused from the check of novox/shop#7" {
|
||||
t.Fatalf("said %q", got)
|
||||
}
|
||||
if got := (link.BuildResult{BuiltWhy: "no check built it"}).MadeWords(); got != "built: no check built it" {
|
||||
t.Fatalf("said %q", got)
|
||||
}
|
||||
if got := (link.BuildResult{}).MadeWords(); !strings.Contains(got, "predates ADR 0350") {
|
||||
t.Fatalf("said %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk's module the check's build stood in for is registered by the digest the check built — the copy in the
|
||||
// module's namespace — and the walk keeps "reused from …" for it; its first-node gate judges that registered
|
||||
// build as it judges one the walk built. A scratch reference is refused. Needs the store (MESH_TEST_POSTGRES), and
|
||||
// is named in testdata/store-tests.
|
||||
func TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
gateEvery = 0
|
||||
|
||||
digest := "sha256:" + strings.Repeat("c", 64)
|
||||
reused := func(id, reference string) link.BuildResult {
|
||||
manifest := mustJSON(t, catalogue.Manifest{Module: "app", Version: "c3", Resources: []map[string]any{
|
||||
{"id": "svc", "type": "container", "name": "app", "image": reference}}})
|
||||
return link.BuildResult{ID: id, Repository: "novox/mesh-catalog", Path: "modules/app", Ref: "main", On: "anchor",
|
||||
Commit: "c3", Manifest: manifest, Trunk: "main", OnTrunk: true, Branches: []string{"main"},
|
||||
Made: []link.MadeArtifact{{Name: "server", Kind: "image", Reference: reference}},
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"},
|
||||
Reused: "reused from the check of novox/mesh-catalog#9 at head abcdefabcdef (merged into trunk 123456123456): server " + digest,
|
||||
ReusedFrom: &link.ReusedFrom{Pull: link.PullRef{Owner: "novox", Repo: "mesh-catalog", Number: 9},
|
||||
Head: "abcdefabcdef", Trunk: "123456123456", Key: "r1:" + strings.Repeat("1", 64)}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, inv, reused(link.NewBuildID(time.Now()), "store.internal:5000/scratch/app/server@"+digest)); err == nil ||
|
||||
!strings.Contains(err.Error(), "scratch") {
|
||||
t.Fatalf("a build in the scratch namespace was registered: %v", err)
|
||||
}
|
||||
id := link.NewBuildID(time.Now().Add(time.Second))
|
||||
result := reused(id, "store.internal:5000/app/server@"+digest)
|
||||
if _, _, err := takeIn(ctx, inv, result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil || current["app"].Commit != "c3" {
|
||||
t.Fatalf("the reused build is not what app is registered at: %+v %v", current["app"], err)
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(shelf["app"].Resources)
|
||||
if !strings.Contains(string(raw), "app/server@"+digest) {
|
||||
t.Fatalf("app is registered as %s, not by the check's digest in its own namespace", raw)
|
||||
}
|
||||
|
||||
// The walk asked it: the walk keeps what its build did, and its gate judges the registered build.
|
||||
asked := time.Now().UTC()
|
||||
p := g.plan(t)
|
||||
p.Commit = "c3"
|
||||
p.Modules["app"] = &inventory.PlanModule{State: "asked", AskedAt: &asked, Build: id}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
planBuiltSaying(ctx, g.open, "app", "c3", "", asked, id, result.MadeWords())
|
||||
p = g.plan(t)
|
||||
if m := p.Modules["app"]; m.State != "built" || !strings.HasPrefix(m.Made, "reused from the check of novox/mesh-catalog#9") {
|
||||
t.Fatalf("the walk keeps %+v", m)
|
||||
}
|
||||
for i := 0; i < 6; i++ {
|
||||
advancePlans(ctx, g.open)
|
||||
}
|
||||
p = g.plan(t)
|
||||
gate := p.Modules["app"].Gate
|
||||
if gate == nil || gate.To != "c3" || gate.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("the gate judged %+v, not the reused build", gate)
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c3" {
|
||||
t.Fatalf("the first machine was sent %v, not the reused build", sent)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk's leave names the builds its own pull requests' verdicts made at the head each merge merged — never an
|
||||
// earlier head's — read once per walk, and none when they cannot be read: the builder takes nothing a verdict does
|
||||
// not name (novox/hq ADR 0350 §2, issue 703).
|
||||
func TestAWalksLeaveNamesTheBuildsItsPullRequestsVerdictsMade(t *testing.T) {
|
||||
was := checkedBuildsOf
|
||||
forgetWalkVerdicts()
|
||||
t.Cleanup(func() { checkedBuildsOf = was; forgetWalkVerdicts() })
|
||||
var asked [][]link.PullRef
|
||||
pull := link.PullRef{Owner: "novox", Repo: "shop", Number: 7}
|
||||
atHead := link.CheckedBuild{Pull: pull, Check: "build-2", Head: "bbbbbbbbbbbb", Built: link.CheckBuilt{Module: "app", Key: "r1:2"}}
|
||||
earlier := link.CheckedBuild{Pull: pull, Check: "build-1", Head: "aaaaaaaaaaaa", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
||||
checkedBuildsOf = func(_ context.Context, pulls []link.PullRef) ([]link.CheckedBuild, error) {
|
||||
asked = append(asked, pulls)
|
||||
return []link.CheckedBuild{atHead, earlier}, nil
|
||||
}
|
||||
pulls := []link.PullRef{pull}
|
||||
heads := map[link.PullRef]string{pull: "bbbbbbbbbbbb0123456789"}
|
||||
for range 3 {
|
||||
if got := reuseWithVerdicts(t.Context(), "plan-1", &link.ReuseAsk{Pulls: pulls}, heads); !reflect.DeepEqual(got.Builds,
|
||||
[]link.CheckedBuild{atHead}) {
|
||||
t.Fatalf("the leave names %+v, not only the merged head's build", got.Builds)
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(asked, [][]link.PullRef{pulls}) {
|
||||
t.Fatalf("the verdicts were read %d times for one walk: %v", len(asked), asked)
|
||||
}
|
||||
// A merge whose head was not said takes nothing.
|
||||
if got := reuseWithVerdicts(t.Context(), "plan-1", &link.ReuseAsk{Pulls: pulls}, nil); got.Builds != nil {
|
||||
t.Fatalf("a merge of no said head names %+v", got.Builds)
|
||||
}
|
||||
checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) {
|
||||
return nil, errors.New("no bus")
|
||||
}
|
||||
if got := reuseWithVerdicts(t.Context(), "plan-2", &link.ReuseAsk{Pulls: pulls}, heads); got.Builds != nil {
|
||||
t.Fatalf("verdicts that could not be read named %+v", got.Builds)
|
||||
}
|
||||
}
|
||||
|
||||
// A check is handed only the builds earlier checks of its own pull request at its own head made: another pull
|
||||
// request's build, or an earlier head's, is never kept (novox/hq issue 703). None when they cannot be read.
|
||||
func TestACheckIsHandedOnlyItsOwnHeadsBuilds(t *testing.T) {
|
||||
was := checkedBuildsOf
|
||||
t.Cleanup(func() { checkedBuildsOf = was })
|
||||
pull := link.PullRef{Owner: "novox", Repo: "shop", Number: 7}
|
||||
mine := link.CheckedBuild{Pull: pull, Check: "build-1", Head: "bbbbbbbbbbbb", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
||||
earlier := link.CheckedBuild{Pull: pull, Check: "build-0", Head: "aaaaaaaaaaaa", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
||||
another := link.CheckedBuild{Pull: link.PullRef{Owner: "novox", Repo: "shop", Number: 8}, Check: "build-9",
|
||||
Head: "bbbbbbbbbbbb", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
||||
var asked [][]link.PullRef
|
||||
checkedBuildsOf = func(_ context.Context, pulls []link.PullRef) ([]link.CheckedBuild, error) {
|
||||
asked = append(asked, pulls)
|
||||
return []link.CheckedBuild{mine, earlier, another}, nil
|
||||
}
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "shop", Number: 7, Commit: "bbbbbbbbbbbb0123456789"}
|
||||
builds := []link.CheckBuild{{Module: "app"}}
|
||||
if got := namedForTheCheck(t.Context(), p, builds); !reflect.DeepEqual(got, []link.CheckedBuild{mine}) ||
|
||||
!reflect.DeepEqual(asked, [][]link.PullRef{{pull}}) {
|
||||
t.Fatalf("the check is handed %+v, read for %v", got, asked)
|
||||
}
|
||||
checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) {
|
||||
return nil, errors.New("no bus")
|
||||
}
|
||||
if got := namedForTheCheck(t.Context(), p, builds); got != nil {
|
||||
t.Fatalf("verdicts that could not be read named %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func forgetWalkVerdicts() {
|
||||
walkVerdicts.Lock()
|
||||
defer walkVerdicts.Unlock()
|
||||
walkVerdicts.read = map[string]verdictsRead{}
|
||||
}
|
||||
|
||||
// A walk's merged heads are the forge's word for each merge; a merge announced without its head, or a pull request
|
||||
// merged at two heads in one walk, has none, and its walk takes no check's build (novox/hq issue 703).
|
||||
func TestAWalksMergedHeadsAreTheForgesWord(t *testing.T) {
|
||||
merge := func(number int, head string) inventory.PlanMerge {
|
||||
return inventory.PlanMerge{Repository: "novox/shop", Number: number, Head: head}
|
||||
}
|
||||
p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
|
||||
merge(7, "aaaaaaaaaaaa1"), merge(8, ""), merge(9, "bbbbbbbbbbbb"), merge(9, "cccccccccccc")}}}
|
||||
heads := mergedHeadsOf(p)
|
||||
shop := func(n int) link.PullRef { return link.PullRef{Owner: "novox", Repo: "shop", Number: n} }
|
||||
if heads[shop(7)] != "aaaaaaaaaaaa1" || heads[shop(8)] != "" || heads[shop(9)] != "" {
|
||||
t.Fatalf("merged heads %v", heads)
|
||||
}
|
||||
}
|
||||
+241
-25
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
@@ -248,31 +249,16 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
// Beside it, at what the mesh runs: each core repository by the module the mesh builds from it.
|
||||
beside := map[string]link.CheckedOut{}
|
||||
for _, e := range entries {
|
||||
dir, core := coreModules[e.Manifest.Module]
|
||||
if !core || e.Provided || e.Source.Repository == "" {
|
||||
continue
|
||||
}
|
||||
url, err := clone(e.Source)
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
refs := besideRefs(dir, current[e.Manifest.Module].Commit)
|
||||
beside[dir] = link.CheckedOut{Repository: url, Ref: refs[dir]}
|
||||
if dir == "mesh-controller" {
|
||||
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
|
||||
if e.Source.Seat != "" {
|
||||
for _, sibling := range []string{"mesh-lab", "mesh-sdk"} {
|
||||
if url, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
||||
sibling)}); err == nil {
|
||||
beside[sibling] = link.CheckedOut{Repository: url, Ref: refs[sibling]}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// **A pull request into a release branch is judged by that branch** (novox/hq ADR 0358): asked of the forge once.
|
||||
release, err := releaseBranchOf(ctx, p, scope, entries)
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
beside, err := besideOf(entries, func(module string) string { return current[module].Commit }, clone, release)
|
||||
if err != nil {
|
||||
return link.BuildRequest{}, err
|
||||
}
|
||||
builds := checkBuildsOf(p, scope, entries, clone)
|
||||
return link.BuildRequest{
|
||||
ID: link.NewBuildID(time.Now()),
|
||||
Repository: repository,
|
||||
@@ -282,10 +268,237 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
||||
Source: sourceOnSeat(source),
|
||||
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
|
||||
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
|
||||
Manifests: scope.Manifests, Judge: scope.Judge},
|
||||
Manifests: scope.Manifests, Judge: scope.Judge, Builds: builds, Named: namedForTheCheck(ctx, p, builds),
|
||||
Release: release},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// besideOf is what a check reads beside its own repository, by the directory it is found under: each core repository
|
||||
// by the module the mesh builds from it, at besideRefs — or, for a pull request into a release branch (novox/hq ADR
|
||||
// 0358 §3), at its branch of that name where it has one and at its trunk otherwise, the controller's main kept for
|
||||
// its judge, and mesh-lab at its trunk whatever the base: its replays run with the container runtime's socket. running
|
||||
// is the commit the mesh runs of a module.
|
||||
func besideOf(entries []inventory.Entry, running func(module string) string,
|
||||
clone func(inventory.Source) (string, error), release string) (map[string]link.CheckedOut, error) {
|
||||
beside := map[string]link.CheckedOut{}
|
||||
for _, e := range entries {
|
||||
dir, core := coreModules[e.Manifest.Module]
|
||||
if !core || e.Provided || e.Source.Repository == "" {
|
||||
continue
|
||||
}
|
||||
url, err := clone(e.Source)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs := besideRefs(dir, running(e.Manifest.Module))
|
||||
at := func(dir, trunk string) link.CheckedOut {
|
||||
if release == "" {
|
||||
return link.CheckedOut{Repository: url, Ref: refs[dir]}
|
||||
}
|
||||
if trunk == "" {
|
||||
trunk = "main"
|
||||
}
|
||||
return link.CheckedOut{Repository: url, Ref: release, Else: trunk}
|
||||
}
|
||||
beside[dir] = at(dir, followedBranch(e.Source.Ref))
|
||||
if dir == "mesh-controller" {
|
||||
// The controller's main stays its main: it judges only a check the running controller predates.
|
||||
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
|
||||
if release != "" {
|
||||
// mesh-controller has the release branch: that is what makes it one (ADR 0358 §1).
|
||||
beside[dir] = link.CheckedOut{Repository: url, Ref: release}
|
||||
}
|
||||
if e.Source.Seat != "" {
|
||||
for _, sibling := range []string{"mesh-lab", "mesh-sdk"} {
|
||||
if url, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
||||
sibling)}); err == nil {
|
||||
b := link.CheckedOut{Repository: url, Ref: refs[sibling]}
|
||||
// **mesh-lab is read at its trunk, always** (ADR 0358): its replays run with the container
|
||||
// runtime's socket, so they are only ever reviewed code — never a branch anyone may create.
|
||||
if release != "" && sibling != "mesh-lab" {
|
||||
b = link.CheckedOut{Repository: url, Ref: release, Else: "main"}
|
||||
}
|
||||
beside[sibling] = b
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return beside, nil
|
||||
}
|
||||
|
||||
// releaseBranchOf is the release branch a pull request merges into (novox/hq ADR 0358 §1): its base, when that is not
|
||||
// its repository's trunk and mesh-controller has a branch of that name; empty otherwise, and the pull request is
|
||||
// checked as before it. **A base that is a trunk asks nothing**: the branch a module built from the repository follows
|
||||
// (main when it records none), or mesh-controller's own trunk. Any other base is asked of the forge once, and an
|
||||
// answer it cannot give is an error, never read as "no release branch".
|
||||
func releaseBranchOf(ctx context.Context, p link.PullUpdated, scope checkScope, entries []inventory.Entry) (string, error) {
|
||||
base := p.Base
|
||||
if base == "" {
|
||||
return "", nil
|
||||
}
|
||||
if scope.From != nil && base == trunkOf(scope.From.Source) {
|
||||
return "", nil
|
||||
}
|
||||
var controller *inventory.Entry
|
||||
for i, e := range entries {
|
||||
if e.Manifest.Module == "mesh-controller" && !e.Provided && e.Source.Repository != "" {
|
||||
controller = &entries[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if controller == nil || base == trunkOf(controller.Source) {
|
||||
return "", nil
|
||||
}
|
||||
if !tryRef.MatchString(base) || strings.Contains(base, "..") {
|
||||
return "", nil // no branch the forge could hold: checked as before
|
||||
}
|
||||
owner, repo := sourceOwner(controller.Source.Repository), sourceName(controller.Source.Repository)
|
||||
if owner == "" || repo == "" {
|
||||
return "", nil
|
||||
}
|
||||
has, err := forgeHasBranch(ctx, owner, repo, base)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("whether %s/%s#%d merges into a release branch cannot be told — the forge did not say "+
|
||||
"whether %s/%s has a branch %s (novox/hq ADR 0358): %w", p.Owner, p.Repo, p.Number, owner, repo, base, err)
|
||||
}
|
||||
if !has {
|
||||
return "", nil
|
||||
}
|
||||
return base, nil
|
||||
}
|
||||
|
||||
// trunkOf is the branch a module's source follows: main when it records none, or records a commit.
|
||||
func trunkOf(s inventory.Source) string {
|
||||
if t := strings.TrimSpace(followedBranch(s.Ref)); t != "" {
|
||||
return t
|
||||
}
|
||||
return "main"
|
||||
}
|
||||
|
||||
// sourceName is the name of a recorded repository, a path on the git seat or a URL: novox/mesh-host → mesh-host.
|
||||
func sourceName(repository string) string {
|
||||
parts := strings.Split(strings.Trim(strings.TrimSuffix(repository, ".git"), "/"), "/")
|
||||
if len(parts) < 2 {
|
||||
return ""
|
||||
}
|
||||
return parts[len(parts)-1]
|
||||
}
|
||||
|
||||
// forgeHasBranch asks the forge, through its module's tools on the bus, whether a repository has a branch. A
|
||||
// variable so a test needs no forge.
|
||||
var forgeHasBranch = func(ctx context.Context, owner, repo, branch string) (bool, error) {
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer js.Close()
|
||||
raw, _ := json.Marshal(map[string]any{"path": "/repos/" + owner + "/" + repo + "/branches/" + branch})
|
||||
answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", "gitea_api", raw, 30*time.Second)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return branchAnswered(answer, branch)
|
||||
}
|
||||
|
||||
// branchAnswered reads the forge's answer about a branch: a 404 is "no such branch"; an answer naming the branch is
|
||||
// "it has it"; anything else — another error, an unreadable answer, one naming no branch or another — is an error,
|
||||
// never read as "no release branch".
|
||||
func branchAnswered(answer link.Answer, branch string) (bool, error) {
|
||||
if answer.Error != "" {
|
||||
if strings.Contains(answer.Error, " 404 ") {
|
||||
return false, nil
|
||||
}
|
||||
return false, errors.New(answer.Error)
|
||||
}
|
||||
var found struct {
|
||||
Result struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"result"`
|
||||
}
|
||||
if err := json.Unmarshal(answer.Result, &found); err != nil {
|
||||
return false, fmt.Errorf("the forge's answer about branch %s cannot be read: %w", branch, err)
|
||||
}
|
||||
if found.Result.Name != branch {
|
||||
return false, fmt.Errorf("the forge answered about branch %s naming %q", branch, found.Result.Name)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// namedForTheCheck are the builds earlier checks of the same pull request at the same head named, read once for the
|
||||
// check (novox/hq issue 703): asked again because only the trunk moved, the check keeps what they built of a key
|
||||
// still the same. Another pull request's build, or one of an earlier head, is never kept. None when they cannot be
|
||||
// read, and the check builds, as before.
|
||||
func namedForTheCheck(ctx context.Context, p link.PullUpdated, builds []link.CheckBuild) []link.CheckedBuild {
|
||||
if len(builds) == 0 || p.Commit == "" {
|
||||
return nil
|
||||
}
|
||||
pull := link.PullRef{Owner: p.Owner, Repo: p.Repo, Number: p.Number}
|
||||
read, err := checkedBuildsOf(ctx, []link.PullRef{pull})
|
||||
if err != nil {
|
||||
fmt.Printf(" the earlier checks' builds of %s/%s#%d could not be read, so the check builds: %v\n",
|
||||
p.Owner, p.Repo, p.Number, err)
|
||||
// empty-on-error: said above; a check handed no named build builds every module, as before issue 703
|
||||
return nil
|
||||
}
|
||||
var named []link.CheckedBuild
|
||||
for _, b := range read {
|
||||
if b.Pull == pull && link.SameHead(b.Head, p.Commit) {
|
||||
named = append(named, b)
|
||||
}
|
||||
}
|
||||
return named
|
||||
}
|
||||
|
||||
// checkBuildsOf are the modules a merge of the change would build, in the walk's order, as the check builds them
|
||||
// for the walk to take (novox/hq ADR 0350 §1): a module built from the repository checked, at the branch it merges
|
||||
// into, from the merge the check makes; any other — a module standing on one — from its own repository at the
|
||||
// branch it follows, as the walk builds it. A module whose build reads the repository checked through a context,
|
||||
// or that the merge deletes, cannot be built as the walk would: said, and nothing after it is built. A change that
|
||||
// moves no module builds nothing.
|
||||
func checkBuildsOf(p link.PullUpdated, scope checkScope, entries []inventory.Entry,
|
||||
clone func(inventory.Source) (string, error)) []link.CheckBuild {
|
||||
if len(scope.Reach.Plan.Tiers) == 0 {
|
||||
return nil
|
||||
}
|
||||
m := link.SourceMoved{Owner: p.Owner, Repo: p.Repo, Base: p.Base, CloneURL: p.CloneURL}
|
||||
byName := map[string]inventory.Entry{}
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
skip := map[string]string{}
|
||||
for _, e := range scope.Reach.Packaging {
|
||||
skip[e.Manifest.Module] = "its build reads " + p.Owner + "/" + p.Repo + " through a context, which a check " +
|
||||
"cannot hand the merge"
|
||||
}
|
||||
for _, e := range scope.Reach.Deleted {
|
||||
skip[e.Manifest.Module] = "the merge deletes it"
|
||||
}
|
||||
var out []link.CheckBuild
|
||||
for tier, names := range scope.Reach.Plan.Tiers {
|
||||
for _, name := range names {
|
||||
e, known := byName[name]
|
||||
b := link.CheckBuild{Module: name, Tier: tier, Skip: skip[name]}
|
||||
switch {
|
||||
case !known:
|
||||
b.Skip = "the catalogue does not hold it"
|
||||
case b.Skip != "":
|
||||
case sourceIs(e.Source, m):
|
||||
b.Merged, b.Path = true, e.Source.Path
|
||||
default:
|
||||
url, err := clone(e.Source)
|
||||
if err != nil {
|
||||
b.Skip = "its repository cannot be cloned: " + err.Error()
|
||||
break
|
||||
}
|
||||
b.Repository, b.Path, b.Ref = url, e.Source.Path, followedBranch(e.Source.Ref)
|
||||
}
|
||||
out = append(out, b)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// besideRefs is the ref each repository is cloned at beside a check, by the directory it is found under:
|
||||
// a core repository at the commit the mesh runs of the module built from it (`running`) — but the
|
||||
// catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge
|
||||
@@ -382,6 +595,9 @@ func checkedOf(result link.BuildResult) link.Checked {
|
||||
// What was judged (novox/hq ADR 0323 §3): the head merged into the trunk commit, and the commits of the
|
||||
// repositories read beside it.
|
||||
c.Trunk, c.Beside, c.Took = result.Check.Trunk, result.Check.Beside, result.Check.Took
|
||||
c.Built = result.Check.Built
|
||||
c.Release, c.JudgedBy, c.BesideAt = result.Check.Release, result.Check.JudgedBy, result.Check.BesideAt
|
||||
c.Validator = result.Check.Validator
|
||||
if c.Gate == nil {
|
||||
// A build seat from before the layers: its verdict is the gate's.
|
||||
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
||||
|
||||
@@ -1,383 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// configCommand reads and sets config values (novox/hq issue 630). A value resolves from four sources, the
|
||||
// last set winning: the module's default, mesh config, node config, and the module's own value on a node
|
||||
// (its module-assignment config). Mesh and node config hold values only for the predefined keys, the
|
||||
// catalogue's list; a module's own value is set on one node, never for the whole mesh.
|
||||
//
|
||||
// config [list] [--node <node>] [--module <module>]
|
||||
// config set <key> <value> [--node <node> [--module <module>]] --why <text>
|
||||
// config clear <key> [--node <node> [--module <module>]] --why <text>
|
||||
// config history <key> [--node <node>]
|
||||
//
|
||||
// A set or clear is recorded in the hand-act log with the caller and why, kept in the history, and takes
|
||||
// effect at the next push of the machines it reaches, which it names — as `settings set` does.
|
||||
func configCommand(ctx context.Context, args []string) error {
|
||||
sub := "list"
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
sub, args = args[0], args[1:]
|
||||
}
|
||||
set := flag.NewFlagSet("config", flag.ContinueOnError)
|
||||
node := set.String("node", "", "one node: its node config, or with --module the module's own value there")
|
||||
module := set.String("module", "", "a module: its own value on --node (module-assignment config)")
|
||||
acts := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *module != "" && *node == "" && sub != "list" {
|
||||
return fmt.Errorf("config: %s names no node, and there is no module layer for the whole mesh — an "+
|
||||
"assignment is one module on one node (novox/hq issue 630). Give node, or set mesh config for a "+
|
||||
"predefined key without module. Nothing was changed", *module)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
switch sub {
|
||||
case "list":
|
||||
if len(positionals) > 0 {
|
||||
return errors.New("config list [--node <node>] [--module <module>]")
|
||||
}
|
||||
return configList(ctx, inv, *node, *module)
|
||||
case "history":
|
||||
if len(positionals) != 1 || *module != "" {
|
||||
return errors.New("config history <key> [--node <node>] — a module's own values are in `settings show <module> --node <node> --history`")
|
||||
}
|
||||
past, err := inv.ConfigHistory(ctx, positionals[0], *node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
where := configWhere(*node, "")
|
||||
if len(past) == 0 {
|
||||
fmt.Printf("%s, %s: no value has been replaced\n", positionals[0], where)
|
||||
return nil
|
||||
}
|
||||
for _, p := range past {
|
||||
v, _ := json.Marshal(p.Value)
|
||||
fmt.Printf("%s, %s: %s until %s (%s", positionals[0], where, v,
|
||||
p.ReplacedAt.Local().Format("2006-01-02 15:04:05"), p.ReplacedBy)
|
||||
if p.ReplacedWhy != "" {
|
||||
fmt.Printf(": %s", p.ReplacedWhy)
|
||||
}
|
||||
fmt.Printf("); %s, because %s\n", orUnknown(p.SetBy), orUnknown(p.Why))
|
||||
}
|
||||
return nil
|
||||
case "set", "clear":
|
||||
default:
|
||||
return fmt.Errorf("config has no %q; it has list, set, clear and history", sub)
|
||||
}
|
||||
|
||||
// set and clear.
|
||||
if err := acts.require("config " + sub); err != nil {
|
||||
return err
|
||||
}
|
||||
why := strings.TrimSpace(*acts.why)
|
||||
var key string
|
||||
var value any
|
||||
switch {
|
||||
case sub == "set" && len(positionals) == 2:
|
||||
key = positionals[0]
|
||||
if err := json.Unmarshal([]byte(positionals[1]), &value); err != nil {
|
||||
// A bare word is a string: `config set theme dark` means "dark".
|
||||
value = positionals[1]
|
||||
}
|
||||
if value == nil {
|
||||
return errors.New("config set takes a value; to remove one, config clear")
|
||||
}
|
||||
case sub == "clear" && len(positionals) == 1:
|
||||
key = positionals[0]
|
||||
default:
|
||||
return errors.New("config set <key> <value> [--node <node> [--module <module>]] --why <text>, or " +
|
||||
"config clear <key> [--node <node> [--module <module>]] --why <text>")
|
||||
}
|
||||
if *node != "" {
|
||||
if _, err := inv.NodeByName(ctx, *node); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
where := configWhere(*node, *module)
|
||||
|
||||
// The module's own value on one node: a preference it declares, of its type, judged as `settings set`
|
||||
// judges a layer, its trusted keys refused through a verb exactly as there (novox/hq issue 339, 340).
|
||||
if *module != "" {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m, known := shelf[*module]
|
||||
if !known {
|
||||
return fmt.Errorf("%w: %s", inventory.ErrNoSuchModule, *module)
|
||||
}
|
||||
p, declared := catalogue.PreferencesOf(m)[key]
|
||||
if !declared {
|
||||
return fmt.Errorf("config: %s declares no preference %q; config sets a module's preferences (%s) — "+
|
||||
"any other setting is the settings verb's. Nothing was changed", *module, key,
|
||||
preferencesOrNone(sortedPreferenceKeys(m)))
|
||||
}
|
||||
if value != nil {
|
||||
if err := catalogue.ConfigValueProblem(fmt.Sprintf("%s's preference %q", *module, key), p.Type, value); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
before, _, err := inv.Layer(ctx, *node, *module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
after := map[string]any{}
|
||||
for k, v := range before {
|
||||
after[k] = v
|
||||
}
|
||||
if value == nil {
|
||||
delete(after, key)
|
||||
} else {
|
||||
after[key] = value
|
||||
}
|
||||
if err := refuseTerminalSettingsThroughAVerb(ctx, inv, before, after, *module, *node); err != nil {
|
||||
return err
|
||||
}
|
||||
acts.record(ctx, "config", configActArgs(sub, positionals, *node, *module))
|
||||
was, had, err := inv.SetModuleValue(ctx, *node, *module, key, value, setByWords()+": "+why)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sayConfigChange(key, where, value, was, had)
|
||||
fmt.Printf(" takes effect at the next push: run `plan %s --diff` to see it, `push %s` to send it\n", *node, *node)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Mesh config or node config: a predefined key only, judged before the act is recorded.
|
||||
if value != nil {
|
||||
if err := inv.JudgeConfig(ctx, key, *node, value); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
acts.record(ctx, "config", configActArgs(sub, positionals, *node, ""))
|
||||
before, had := configValueOf(ctx, inv, key, *node)
|
||||
if value == nil {
|
||||
removed, err := inv.ClearConfig(ctx, key, *node, setByWords(), why)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !removed {
|
||||
fmt.Printf("%s, %s: no value was set; nothing changed\n", key, where)
|
||||
return nil
|
||||
}
|
||||
} else if err := inv.SetConfig(ctx, key, *node, value, setByWords(), why); err != nil {
|
||||
return err
|
||||
}
|
||||
sayConfigChange(key, where, value, before, had)
|
||||
reached, err := reachedBy(ctx, inv, key, *node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(reached) == 0 {
|
||||
fmt.Println(" no module assigned there falls back to it yet; nothing to push")
|
||||
return nil
|
||||
}
|
||||
fmt.Printf(" takes effect at the next push of %s: run `push %s` to send it\n", strings.Join(reached, ", "),
|
||||
strings.Join(reached, " "))
|
||||
return nil
|
||||
}
|
||||
|
||||
// configList lists the predefined keys with their mesh and node config, then each module's preferences
|
||||
// with the value on each node it is assigned to and where it comes from.
|
||||
func configList(ctx context.Context, inv *inventory.Inventory, node, module string) error {
|
||||
if node != "" {
|
||||
if _, err := inv.NodeByName(ctx, node); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
list, err := inv.Predefined(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
values, err := inv.Config(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var b strings.Builder
|
||||
if len(list) == 0 {
|
||||
fmt.Fprintf(&b, "no predefined keys: the list arrives with a build of the catalogue carrying %s\n",
|
||||
catalogue.PredefinedFile)
|
||||
} else {
|
||||
fmt.Fprintf(&b, "predefined keys (the catalogue's %s), with mesh config and node config:\n", catalogue.PredefinedFile)
|
||||
}
|
||||
for _, key := range list.Keys() {
|
||||
k := list[key]
|
||||
fmt.Fprintf(&b, " %s (%s): %s\n", key, k.Type, k.Why)
|
||||
held := false
|
||||
for _, v := range values {
|
||||
if v.Key != key || (node != "" && v.Node != "" && v.Node != node) {
|
||||
continue
|
||||
}
|
||||
held = true
|
||||
shown, _ := json.Marshal(v.Value)
|
||||
fmt.Fprintf(&b, " %s: %s — %s, because %s\n", configWhere(v.Node, ""), shown, orUnknown(v.SetBy), v.Why)
|
||||
}
|
||||
if !held {
|
||||
b.WriteString(" no mesh or node config: each module's default\n")
|
||||
}
|
||||
}
|
||||
for _, key := range inventory.ConfigKeysSorted(values) {
|
||||
if _, on := list[key]; on {
|
||||
continue
|
||||
}
|
||||
for _, v := range values {
|
||||
if v.Key == key {
|
||||
shown, _ := json.Marshal(v.Value)
|
||||
fmt.Fprintf(&b, " %s is no longer a predefined key, and %s still holds %s for it: it reaches nothing; "+
|
||||
"config clear %s%s\n", key, configWhere(v.Node, ""), shown, key, nodeFlag(v.Node))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var listed []preferencesOf
|
||||
for _, e := range entries {
|
||||
m := e.Manifest
|
||||
if len(catalogue.PreferencesOf(m)) == 0 || (module != "" && m.Module != module) {
|
||||
continue
|
||||
}
|
||||
if node != "" && !containsString(e.On, node) {
|
||||
continue
|
||||
}
|
||||
p := preferencesOf{Manifest: m, On: map[string][]catalogue.SettingSource{}}
|
||||
for _, n := range e.On {
|
||||
if node != "" && n != node {
|
||||
continue
|
||||
}
|
||||
p.Nodes = append(p.Nodes, n)
|
||||
layers, err := inv.SettingsFor(ctx, n, m.Module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.On[n] = catalogue.Effective(m, layers)
|
||||
}
|
||||
listed = append(listed, p)
|
||||
}
|
||||
b.WriteString("\npreferences, and on each node the value and where it comes from (default, mesh config, " +
|
||||
"node config, or the module on the node):\n")
|
||||
b.WriteString(describePreferences(listed))
|
||||
fmt.Print(b.String())
|
||||
return nil
|
||||
}
|
||||
|
||||
// configWhere is how a config value's place is said.
|
||||
func configWhere(node, module string) string {
|
||||
switch {
|
||||
case module != "":
|
||||
return module + " on " + node
|
||||
case node != "":
|
||||
return catalogue.NodeConfigLayer(node)
|
||||
}
|
||||
return catalogue.MeshConfigLayer
|
||||
}
|
||||
|
||||
// configValueOf is the value mesh config (node "") or a node's config holds for a key.
|
||||
func configValueOf(ctx context.Context, inv *inventory.Inventory, key, node string) (any, bool) {
|
||||
values, err := inv.Config(ctx)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
for _, v := range values {
|
||||
if v.Key == key && v.Node == node {
|
||||
return v.Value, true
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// reachedBy is the nodes a value of mesh config (node "") or a node's config for a key reaches: those
|
||||
// running a module whose preference falls back to it.
|
||||
func reachedBy(ctx context.Context, inv *inventory.Inventory, key, node string) ([]string, error) {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodes := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
if !containsString(catalogue.PredefinedNamed(e.Manifest), key) {
|
||||
continue
|
||||
}
|
||||
for _, n := range e.On {
|
||||
if node == "" || n == node {
|
||||
nodes[n] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(nodes))
|
||||
for n := range nodes {
|
||||
out = append(out, n)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func sayConfigChange(key, where string, value, before any, had bool) {
|
||||
was := "nothing"
|
||||
if had {
|
||||
w, _ := json.Marshal(before)
|
||||
was = string(w)
|
||||
}
|
||||
if value == nil {
|
||||
fmt.Printf("%s, %s: cleared (was %s); the next source down applies\n", key, where, was)
|
||||
return
|
||||
}
|
||||
now, _ := json.Marshal(value)
|
||||
fmt.Printf("%s, %s: %s (was %s); the value it replaced is kept\n", key, where, now, was)
|
||||
}
|
||||
|
||||
func configActArgs(sub string, positionals []string, node, module string) []string {
|
||||
args := append([]string{sub}, positionals...)
|
||||
if node != "" {
|
||||
args = append(args, "--node", node)
|
||||
}
|
||||
if module != "" {
|
||||
args = append(args, "--module", module)
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
func sortedPreferenceKeys(m catalogue.Manifest) []string {
|
||||
declared := catalogue.PreferencesOf(m)
|
||||
keys := make([]string, 0, len(declared))
|
||||
for k := range declared {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
func orUnknown(s string) string {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return "not kept"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func preferencesOrNone(keys []string) string {
|
||||
if len(keys) == 0 {
|
||||
return "it declares none"
|
||||
}
|
||||
return strings.Join(keys, ", ")
|
||||
}
|
||||
@@ -1,187 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// The config verb's line: a listing without key, a set or clear with why, and a module value refused
|
||||
// without a node (novox/hq issue 630).
|
||||
func TestTheConfigVerbsLine(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
args map[string]any
|
||||
want string
|
||||
}{
|
||||
{map[string]any{}, "config list"},
|
||||
{map[string]any{"node": "g14", "module": "dunst"}, "config list --node g14 --module dunst"},
|
||||
{map[string]any{"key": "font-size", "value": "12", "why": "bigger"}, "config set font-size 12 --why bigger"},
|
||||
{map[string]any{"key": "font-size", "value": "13", "node": "g14", "why": "dense"},
|
||||
"config set font-size 13 --node g14 --why dense"},
|
||||
{map[string]any{"key": "font-size", "value": "14", "node": "g14", "module": "dunst", "why": "x"},
|
||||
"config set font-size 14 --node g14 --module dunst --why x"},
|
||||
{map[string]any{"key": "font-size", "clear": "true", "node": "g14", "why": "back"},
|
||||
"config clear font-size --node g14 --why back"},
|
||||
{map[string]any{"key": "font-size", "history": "true"}, "config history font-size"},
|
||||
} {
|
||||
argv, err := argvFor("config", c.args)
|
||||
if err != nil || strings.Join(argv, " ") != c.want {
|
||||
t.Errorf("%v: %v %v, want %s", c.args, argv, err, c.want)
|
||||
}
|
||||
}
|
||||
for want, args := range map[string]map[string]any{
|
||||
"no module layer for the whole mesh": {"key": "font-size", "value": "12", "module": "dunst", "why": "x"},
|
||||
"why": {"key": "font-size", "value": "12"},
|
||||
"names its key": {"value": "12"},
|
||||
"not both": {"key": "font-size", "value": "12", "clear": "true", "why": "x"},
|
||||
} {
|
||||
if _, err := argvFor("config", args); err == nil || !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("%v: %v, want %q", args, err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// End to end through the verb, as the operator's acceptance reads it: cleared → the default; mesh font-size,
|
||||
// then the node's, then dunst's own on the node — each in the node's dunstrc as composed (novox/hq issue 630).
|
||||
func TestConfigReachesTheComposedFileFromEachSource(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.SetPredefined(ctx, catalogue.Predefined{
|
||||
"font-size": {Type: catalogue.TypeNumber, Why: "the size text is drawn at"}}, "", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "dunst", Version: "1",
|
||||
Preferences: map[string]catalogue.Preference{
|
||||
"font-size": {Type: catalogue.TypeNumber, Default: float64(10), Predefined: "font-size", Why: "100 DPI"},
|
||||
},
|
||||
Resources: []map[string]any{{"id": "dunstrc", "type": "file", "path": "/etc/dunst/dunstrc", "mode": "0644", "trusted": false,
|
||||
"content": "font = Inter ${setting:font-size}\n"}}})
|
||||
for _, n := range []string{"laptop", "anchor"} {
|
||||
if _, err := assign(ctx, open, n, "dunst"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
dunstrc := func(node string) string {
|
||||
t.Helper()
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, problems, err := composedAndValidated(ctx, open, node, gens, Allocating)
|
||||
if err != nil {
|
||||
t.Fatalf("%s does not compose: %v %v", node, err, problems)
|
||||
}
|
||||
for _, r := range declared.Resources {
|
||||
if r["path"] == "/etc/dunst/dunstrc" {
|
||||
return fmt.Sprint(r["content"])
|
||||
}
|
||||
}
|
||||
t.Fatalf("%s has no dunstrc: %v", node, problems)
|
||||
return ""
|
||||
}
|
||||
want := func(node, size string) {
|
||||
t.Helper()
|
||||
if got := dunstrc(node); got != "font = Inter "+size+"\n" {
|
||||
t.Fatalf("%s's dunstrc: %q, want size %s", node, got, size)
|
||||
}
|
||||
}
|
||||
verb := func(args map[string]any) error {
|
||||
t.Helper()
|
||||
return throughVerb(t, "config", args)
|
||||
}
|
||||
want("laptop", "10")
|
||||
|
||||
if err := verb(map[string]any{"key": "font-size", "value": "11", "why": "a larger default"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want("laptop", "11")
|
||||
want("anchor", "11")
|
||||
if err := verb(map[string]any{"key": "font-size", "value": "13", "node": "laptop", "why": "a dense screen"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want("laptop", "13")
|
||||
want("anchor", "11")
|
||||
if err := verb(map[string]any{"key": "font-size", "value": "15", "node": "laptop", "module": "dunst",
|
||||
"why": "dunst alone larger"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want("laptop", "15")
|
||||
|
||||
listing := stdoutOf(t, func() error { return verb(map[string]any{}) })
|
||||
for _, said := range []string{"font-size (number): the size text is drawn at", "mesh config: 11",
|
||||
"node config (laptop): 13", "laptop: 15 (dunst on laptop)", "anchor: 11 (mesh config)",
|
||||
"falls back to the predefined key font-size"} {
|
||||
if !strings.Contains(listing, said) {
|
||||
t.Errorf("the listing does not say %q:\n%s", said, listing)
|
||||
}
|
||||
}
|
||||
|
||||
// Cleared, each falls back to the next source down.
|
||||
for _, step := range []struct {
|
||||
args map[string]any
|
||||
size string
|
||||
}{
|
||||
{map[string]any{"key": "font-size", "clear": "true", "node": "laptop", "module": "dunst", "why": "x"}, "13"},
|
||||
{map[string]any{"key": "font-size", "clear": "true", "node": "laptop", "why": "x"}, "11"},
|
||||
{map[string]any{"key": "font-size", "clear": "true", "why": "x"}, "10"},
|
||||
} {
|
||||
if err := verb(step.args); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want("laptop", step.size)
|
||||
}
|
||||
|
||||
// Refused: a key not on the list for mesh config, a module value without a node, a type not the key's.
|
||||
if err := verb(map[string]any{"key": "colour", "value": "blue", "why": "x"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "predefined") {
|
||||
t.Fatalf("mesh config for a key not on the list: %v", err)
|
||||
}
|
||||
if err := runLine(t, []string{"config", "set", "font-size", "12", "--module", "dunst", "--why", "x"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "no module layer for the whole mesh") {
|
||||
t.Fatalf("a module value without a node: %v", err)
|
||||
}
|
||||
if err := verb(map[string]any{"key": "font-size", "value": `"large"`, "node": "laptop", "module": "dunst",
|
||||
"why": "x"}); err == nil || !strings.Contains(err.Error(), "is a number") {
|
||||
t.Fatalf("a string for a number: %v", err)
|
||||
}
|
||||
if err := verb(map[string]any{"key": "nothing", "value": "1", "node": "laptop", "module": "dunst",
|
||||
"why": "x"}); err == nil || !strings.Contains(err.Error(), "declares no preference") {
|
||||
t.Fatalf("a module value for a key it does not declare: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The module check reads the catalogue's predefined.json at the repository's root and refuses a preference
|
||||
// falling back to a key not on it (novox/hq issue 630).
|
||||
func TestTheModuleCheckJudgesPreferencesAgainstTheCataloguesList(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for _, dir := range []string{".git", "modules/dunst"} {
|
||||
if err := os.MkdirAll(filepath.Join(root, dir), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
manifest := filepath.Join(root, "modules/dunst/module.json")
|
||||
write := func(path, body string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
write(manifest, `{"module": "dunst", "version": "1",
|
||||
"preferences": {"font-size": {"type": "number", "default": 10, "predefined": "font-size", "why": "100 DPI"}},
|
||||
"resources": [{"id": "rc", "type": "file", "path": "/etc/rc", "mode": "0644", "trusted": false,
|
||||
"content": "font ${setting:font-size}"}]}`)
|
||||
write(filepath.Join(root, "predefined.json"), `{"theme": {"type": "string", "why": "light or dark"}}`)
|
||||
var out strings.Builder
|
||||
if err := moduleCheck([]string{manifest}, &out); err == nil || !strings.Contains(out.String(), "not on the catalogue's list") {
|
||||
t.Fatalf("a key not on the list passed: %v\n%s", err, out.String())
|
||||
}
|
||||
write(filepath.Join(root, "predefined.json"), `{"font-size": {"type": "number", "why": "the size text is drawn at"}}`)
|
||||
out.Reset()
|
||||
if err := moduleCheck([]string{manifest}, &out); err != nil {
|
||||
t.Fatalf("a key on the list was refused: %v\n%s", err, out.String())
|
||||
}
|
||||
}
|
||||
@@ -263,37 +263,6 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
|
||||
f.Controller.Commit = c.Commit
|
||||
}
|
||||
|
||||
// The list of predefined keys, and mesh and node config (novox/hq issue 630).
|
||||
predefined, err := inv.Predefined(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
for key, k := range predefined {
|
||||
if f.Predefined == nil {
|
||||
f.Predefined = map[string]snapshot.PredefinedKey{}
|
||||
}
|
||||
f.Predefined[key] = snapshot.PredefinedKey{Type: k.Type, Why: k.Why}
|
||||
}
|
||||
config, err := inv.Config(ctx)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
nodeConfig := map[string]map[string]any{}
|
||||
meshConfig := map[string]any{}
|
||||
for _, c := range config {
|
||||
if c.Node == "" {
|
||||
meshConfig[c.Key] = c.Value
|
||||
continue
|
||||
}
|
||||
if nodeConfig[c.Node] == nil {
|
||||
nodeConfig[c.Node] = map[string]any{}
|
||||
}
|
||||
nodeConfig[c.Node][c.Key] = c.Value
|
||||
}
|
||||
if len(meshConfig) > 0 {
|
||||
f.Config = scrub.Values(meshConfig)
|
||||
}
|
||||
|
||||
gens, gensErr := generators(ctx, open)
|
||||
hostShelf := shelf[hostModule]
|
||||
meshWide := map[string]bool{}
|
||||
@@ -311,9 +280,6 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
|
||||
if n.HostVersion != "" {
|
||||
engines[n.HostVersion] = true
|
||||
}
|
||||
if values := nodeConfig[n.Name]; len(values) > 0 {
|
||||
m.Config = scrub.Values(values)
|
||||
}
|
||||
m.System = systemOf(hostShelf, n.HostVersion)
|
||||
m.Libc = libcOf(m.System)
|
||||
reported, err := inv.DescribedOf(ctx, n.Name)
|
||||
@@ -379,10 +345,6 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
for _, layer := range layers {
|
||||
if layer.Config {
|
||||
// Mesh and node config are said once, beside the machine and the mesh (novox/hq issue 630).
|
||||
continue
|
||||
}
|
||||
if layer.From == catalogue.MeshWideLayer {
|
||||
if !meshWide[module] {
|
||||
meshWide[module] = true
|
||||
|
||||
@@ -31,7 +31,7 @@ func aMeshWithSecrets(t *testing.T) (*stores, []string) {
|
||||
}
|
||||
}
|
||||
secrets := []string{"Hunter2-Is-Not-A-Password-9f8e7d", "0123456789abcdefABCDEF0123456789zz"}
|
||||
if err := open.inventory.KeepSettings(ctx, "", "files", map[string]any{
|
||||
if err := open.inventory.SetSettings(ctx, "", "files", map[string]any{
|
||||
"admin_password": secrets[0], "upstream": "10.77.0.9", "hub": "anchor"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ func foundDirectory(module string, since time.Time) inventory.ResourceHealth {
|
||||
}
|
||||
|
||||
func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
now := time.Now()
|
||||
sent := now.Add(-time.Minute)
|
||||
f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
|
||||
@@ -48,6 +49,7 @@ func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) {
|
||||
// The whole walk: a module whose directory was used as found long before still gets its fix to every machine,
|
||||
// its pass kept and the wait said; a directory this very send found holds it and puts it back.
|
||||
func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
found time.Duration // when the directory was found, against now
|
||||
@@ -81,9 +83,13 @@ func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
|
||||
}
|
||||
return f, err
|
||||
}
|
||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
||||
gateSettle, gateEvery, gateBound = 0, 0, 300*time.Millisecond
|
||||
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||
wasWindow := gateWitnessWindow
|
||||
gateWitnessWindow = 0
|
||||
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
|
||||
gateWithin, gateEvery, gateBound = time.Hour, 0, 300*time.Millisecond
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
advancePlans(ctx, b.open)
|
||||
@@ -112,6 +118,7 @@ func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
|
||||
|
||||
// The condition says the wait in its own kind: the operator's, never urgent, and cleared once handed over.
|
||||
func TestADirectoryUsedAsFoundIsItsOwnCondition(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
rs := map[string][]inventory.ResourceHealth{"notes": {foundDirectory("notes", time.Now().Add(-time.Hour))}}
|
||||
|
||||
+211
-36
@@ -28,9 +28,12 @@ import (
|
||||
// that machine reported it applied. A build that applies and then does nothing, crashes, serves no
|
||||
// tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
|
||||
// the component's health — the core's health definitions, or a module's own — within ten minutes of the
|
||||
// apply: a core module passing three times over at least two minutes, a leaf module once, as soon as the
|
||||
// node-engine states it healthy (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it). Only then
|
||||
// are the rest sent.
|
||||
// apply, each module by its rule (gateRuleOf): the node-engine and the mesh's resolver passing on their third
|
||||
// healthy reading, and never later than two minutes after the send while healthy; the controller and the tool runner once, no earlier than the end of the
|
||||
// node-engine's witness window for it; mesh-delivery and a leaf module once, as soon as the node-engine states
|
||||
// it healthy (novox/hq ADR 0351 and ADR 0352, changing ADR 0236 decision 3, ADR 0282 decision 4 and ADR 0304
|
||||
// decision 3).
|
||||
// Only then are the rest sent.
|
||||
//
|
||||
// **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
|
||||
// at its gate — registration refuses it and nothing sends it again on its own — the module's registered
|
||||
@@ -41,22 +44,30 @@ import (
|
||||
|
||||
// The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes.
|
||||
var (
|
||||
// gateSettle is how long the first machine must stay healthy, at the least.
|
||||
gateSettle = 2 * time.Minute
|
||||
// gateWithin is the longest a module under the three-reading rule that stays healthy is held: at it, the
|
||||
// gate passes on the healthy readings it has, the newest healthy and none since the send not (novox/hq ADR
|
||||
// 0352). Readings gateEvery apart, the first at the apply, put three inside it.
|
||||
gateWithin = 2 * time.Minute
|
||||
// gateBound is how long after the apply the build may take to become healthy.
|
||||
gateBound = 10 * time.Minute
|
||||
// gatePasses is how many consecutive judgings must find a core module healthy, gateEvery apart at the
|
||||
// least, after gateSettle. gateLeafPasses is a leaf module's: one, taken once the node-engine states every
|
||||
// resource of it healthy by its own check, with no settle (novox/hq ADR 0282 decision 4 as ADR 0304
|
||||
// decision 3 changed it, issue 524).
|
||||
// gatePasses is how many consecutive judgings must find such a core module healthy, gateEvery apart at
|
||||
// the least, counted from the send: it passes on the third (novox/hq ADR 0352). gateLeafPasses is a leaf module's: one, taken once the
|
||||
// node-engine states every resource of it healthy by its own check, with no settle (novox/hq ADR 0282
|
||||
// decision 4 as ADR 0304 decision 3 changed it, issue 524).
|
||||
gatePasses = 3
|
||||
gateLeafPasses = 1
|
||||
gateLeafSettles = false
|
||||
gateEvery = 40 * time.Second
|
||||
// gateWitnessWindow is the node-engine's witness window for a new controller or tool runner, from its
|
||||
// start (ADR 0236 decision 27, lease.ControllerWithin and lease.NodeToolsWithin): a reading of either
|
||||
// counts only after it has ended, so the build is judged by the witness and the gate both (novox/hq ADR
|
||||
// 0351 decisions 1 and 4).
|
||||
gateWitnessWindow = max(lease.ControllerWithin, lease.NodeToolsWithin)
|
||||
)
|
||||
|
||||
// coreModule is whether a module is judged as the core (novox/hq ADR 0282 decision 1): one on the controller's
|
||||
// own path, or one holding the mesh's resolver. Every other module is a leaf.
|
||||
// own path, or one holding the mesh's resolver. Every other module is a leaf. Its class sets the module's
|
||||
// delivery budget; how many readings its gate takes is gateRuleOf's.
|
||||
func coreModule(module string, m catalogue.Manifest) bool {
|
||||
if _, own := onTheControllersPath[module]; own {
|
||||
return true
|
||||
@@ -69,12 +80,72 @@ func coreModule(module string, m catalogue.Manifest) bool {
|
||||
return coreComponent(module) != ""
|
||||
}
|
||||
|
||||
// gateWants is how many passes a module needs, and whether it must stay healthy gateSettle first.
|
||||
func gateWants(module string, m catalogue.Manifest) (passes int, settles bool) {
|
||||
if coreModule(module, m) {
|
||||
return gatePasses, true
|
||||
// gateRule is how a module's first-node gate judges it: passes healthy readings, gateEvery apart at the
|
||||
// least, counted from the send or, where witnessed, from the end of the node-engine's witness window for it;
|
||||
// and, where within is set, passed at within after the send on fewer, if it was healthy at every reading
|
||||
// (novox/hq ADR 0352). said names the rule in the gate's text.
|
||||
type gateRule struct {
|
||||
passes int
|
||||
within time.Duration
|
||||
witnessed bool
|
||||
said string
|
||||
}
|
||||
|
||||
// gateRuleOf is a module's gate rule (novox/hq ADR 0351): a core module keeps three readings only where its
|
||||
// build, broken on every machine, could not receive its own fix — the node-engine, which applies every send,
|
||||
// and the mesh's resolver, which gives every machine the bus's name (decisions 2, 3 and 6); the bus is never
|
||||
// walked through a gate but kept so (decision 7). The controller, run on one machine, and the tool runner,
|
||||
// whose fix the node-engine applies, are judged by one reading after the witness window (decisions 1 and 4);
|
||||
// mesh-delivery, whose fix goes the controller's own path, by one (decision 5); a leaf module by one (ADR 0304
|
||||
// decision 3).
|
||||
func gateRuleOf(module string, m catalogue.Manifest) gateRule {
|
||||
switch {
|
||||
case coreComponent(module) == lease.ComponentController:
|
||||
return gateRule{passes: 1, witnessed: true, said: "the controller's rule: one reading once it is ready, " +
|
||||
"after the node-engine's witness window (novox/hq ADR 0351)"}
|
||||
case coreComponent(module) == lease.ComponentNodeTools:
|
||||
return gateRule{passes: 1, witnessed: true, said: "the tool runner's rule: one reading once it answers, " +
|
||||
"after the node-engine's witness window (novox/hq ADR 0351)"}
|
||||
case module == catalogue.DeliverySeat:
|
||||
return gateRule{passes: 1, said: "mesh-delivery's rule: one reading (novox/hq ADR 0351)"}
|
||||
case coreModule(module, m):
|
||||
return gateRule{passes: gatePasses, within: gateWithin, said: fmt.Sprintf("the core's rule: %d readings, "+
|
||||
"or healthy throughout %s after the send (novox/hq ADR 0352)", gatePasses, gateWithin)}
|
||||
}
|
||||
return gateLeafPasses, gateLeafSettles
|
||||
r := gateRule{passes: gateLeafPasses, said: "a leaf module's rule: one reading (novox/hq ADR 0304)"}
|
||||
if gateLeafSettles {
|
||||
r.within = gateWithin
|
||||
r.said = fmt.Sprintf("a leaf module's rule, judged as the core: %d readings, or healthy throughout %s after the send",
|
||||
gateLeafPasses, gateWithin)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// gateFloor is the earliest a module's reading counts by its rule: the send and, for a witnessed module, the
|
||||
// end of the node-engine's witness window on every machine judged (novox/hq ADR 0351 decisions 1 and 4). A
|
||||
// module under the three-reading rule counts from the send (novox/hq ADR 0352). The window opens when the node-engine starts the build, which is before the machine reports it
|
||||
// applied (applied, as the gate first read it) and, for the controller, before the new controller took the
|
||||
// lease (holder): the later of those, the send's time at the least, plus the window, is never before its end.
|
||||
// A controller that starts again takes the lease again, and so moves its floor on. Pure.
|
||||
func gateFloor(module string, rule gateRule, since time.Time, machines []string, applied map[string]time.Time,
|
||||
holder *lease.Holder) time.Time {
|
||||
floor := since
|
||||
if !rule.witnessed {
|
||||
return floor
|
||||
}
|
||||
for _, n := range machines {
|
||||
start := since
|
||||
if at, read := applied[n]; read && at.After(start) {
|
||||
start = at
|
||||
}
|
||||
if coreComponent(module) == lease.ComponentController && holder != nil && holder.Taken.After(start) {
|
||||
start = holder.Taken
|
||||
}
|
||||
if end := start.Add(gateWitnessWindow); end.After(floor) {
|
||||
floor = end
|
||||
}
|
||||
}
|
||||
return floor
|
||||
}
|
||||
|
||||
// gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions
|
||||
@@ -501,6 +572,13 @@ func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module stri
|
||||
return judgeMoves(ctx, open, g, pairs, now)
|
||||
}
|
||||
|
||||
// dueAtTheMost says a gate healthy since its send has reached gateWithin and not been judged since: the one
|
||||
// judging that may come sooner than gateEvery after the last pass (novox/hq ADR 0352).
|
||||
func dueAtTheMost(g *inventory.PlanGate, now time.Time) bool {
|
||||
most := g.Since.Add(gateWithin)
|
||||
return !g.Faulted && g.LastPass != nil && !now.Before(most) && g.LastPass.Before(most)
|
||||
}
|
||||
|
||||
// judged is one module on one machine, as a gate judges it.
|
||||
type judged struct{ module, node string }
|
||||
|
||||
@@ -511,7 +589,9 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
if g.Verdict != "" {
|
||||
return g.Verdict, nil
|
||||
}
|
||||
if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery {
|
||||
// A judging comes gateEvery after the last pass, and once at gateWithin after the send whatever the spacing:
|
||||
// a module healthy throughout is never held past it (novox/hq ADR 0352).
|
||||
if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery && !dueAtTheMost(g, now) {
|
||||
return "", nil
|
||||
}
|
||||
for _, c := range g.Carried {
|
||||
@@ -544,6 +624,11 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
facts.sentBuilds[j.node] = builds
|
||||
}
|
||||
}
|
||||
// **A busy container on a judged machine holds the judging before it starts** (novox/hq ADR 0335 §4): the
|
||||
// walk says it waits there, and its gate starts at the report that says the container replaced.
|
||||
if busyWait(g, pairs, facts, now) {
|
||||
return "", nil
|
||||
}
|
||||
// **What is wrong with a machine itself is the machine's** (novox/hq issue 281): read once for each
|
||||
// machine judged, apart from what is wrong with a module there, and never pinned on the module the
|
||||
// gate happens to be kept on.
|
||||
@@ -568,6 +653,9 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
for _, j := range pairs {
|
||||
w := words[j.node]
|
||||
h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, w.facts)
|
||||
if why, still := busyStill(j, facts); still && (h == healthGood || h == healthPerson) {
|
||||
h, said = healthNotYet, why
|
||||
}
|
||||
if h == healthGood || h == healthPerson {
|
||||
if on, named := w.on[j.module]; named {
|
||||
h, said = healthNotYet, on
|
||||
@@ -608,6 +696,51 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
why = said
|
||||
}
|
||||
}
|
||||
// **Each module counts its readings by its own rule** (novox/hq ADR 0351): from when the gate first read
|
||||
// each machine report its send applied, and who holds the controller lease, the earliest its reading counts.
|
||||
if g.Applied == nil {
|
||||
g.Applied = map[string]time.Time{}
|
||||
}
|
||||
for node := range byMachine {
|
||||
r, said := facts.reports[node]
|
||||
if _, read := g.Applied[node]; !read && said && r.At != nil && r.Outcome == inventory.OutcomeApplied &&
|
||||
facts.reportedOn(node, r) {
|
||||
g.Applied[node] = *r.At
|
||||
}
|
||||
}
|
||||
rules, floors := map[string]gateRule{}, map[string]time.Time{}
|
||||
for _, m := range modules {
|
||||
var on []string
|
||||
for _, j := range pairs {
|
||||
if j.module == m {
|
||||
on = append(on, j.node)
|
||||
}
|
||||
}
|
||||
rules[m] = gateRuleOf(m, shelf[m])
|
||||
floors[m] = gateFloor(m, rules[m], *g.Since, on, g.Applied, facts.holder)
|
||||
}
|
||||
tallyGate(g, gateJudging{modules: modules, reading: reading, waits: waits, worst: worst, why: why,
|
||||
rules: rules, floors: floors}, now)
|
||||
return g.Verdict, nil
|
||||
}
|
||||
|
||||
// gateJudging is one judging's readings, as tallyGate counts them: the modules judged, each one's reading
|
||||
// (the worst of its machines'), its waits for a person, the send's worst reading and why, and each module's
|
||||
// rule and the earliest its reading counts.
|
||||
type gateJudging struct {
|
||||
modules []string
|
||||
reading map[string]health
|
||||
waits map[string]string
|
||||
worst health
|
||||
why string
|
||||
rules map[string]gateRule
|
||||
floors map[string]time.Time
|
||||
}
|
||||
|
||||
// tallyGate counts one judging into the gate: a pass counted, a pass missed (and why), or the verdict. Pure but
|
||||
// for the gate it writes.
|
||||
func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) {
|
||||
modules, reading, waits, worst, why := j.modules, j.reading, j.waits, j.worst, j.why
|
||||
if g.Healthy == nil {
|
||||
g.Healthy = map[string]int{}
|
||||
}
|
||||
@@ -623,38 +756,53 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
delete(g.HealthyAt, m)
|
||||
continue
|
||||
}
|
||||
// **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more
|
||||
// often while another module beside it is not yet healthy counts no faster.
|
||||
if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery {
|
||||
g.Healthy[m]++
|
||||
g.HealthyAt[m] = now
|
||||
}
|
||||
if w := waits[m]; w != "" {
|
||||
if g.Waits == nil {
|
||||
g.Waits = map[string]string{}
|
||||
}
|
||||
g.Waits[m] = w
|
||||
}
|
||||
// **A reading counts only from the module's floor** (novox/hq ADR 0351 decisions 1 and 4): the
|
||||
// controller's and the tool runner's after the witness window, any other module's from the send.
|
||||
if now.Before(j.floors[m]) {
|
||||
continue
|
||||
}
|
||||
// **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more
|
||||
// often while another module beside it is not yet healthy counts no faster.
|
||||
if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery {
|
||||
g.Healthy[m]++
|
||||
g.HealthyAt[m] = now
|
||||
}
|
||||
}
|
||||
settled := now.Sub(*g.Since) >= gateSettle
|
||||
// What the send needs is what its most demanding module needs: a send carrying a core module is judged
|
||||
// as the core.
|
||||
wants, settles := 0, false
|
||||
// What the send needs is what its most demanding module needs — the most readings, then the latest
|
||||
// floor: a send carrying the node-engine is judged by its three readings, or by being healthy throughout
|
||||
// its two minutes (novox/hq ADR 0352).
|
||||
wants, floor, within, said := 0, time.Time{}, time.Duration(0), ""
|
||||
for _, m := range modules {
|
||||
p, s := gateWants(m, shelf[m])
|
||||
wants, settles = max(wants, p), settles || s
|
||||
r, f := j.rules[m], j.floors[m]
|
||||
if r.passes > wants || (r.passes == wants && f.After(floor)) {
|
||||
wants, within, said = r.passes, r.within, r.said
|
||||
}
|
||||
if f.After(floor) {
|
||||
floor = f
|
||||
}
|
||||
}
|
||||
if len(modules) == 0 {
|
||||
// A send judged on no machine of its modules (its first machine not running them) has no class to
|
||||
// be judged by: it is judged as the core, never passed at once without a reading.
|
||||
wants, settles = gatePasses, true
|
||||
wants, floor, within = gatePasses, *g.Since, gateWithin
|
||||
said = fmt.Sprintf("the core's rule: %d readings, or healthy throughout %s after the send", gatePasses, gateWithin)
|
||||
}
|
||||
// heldLongEnough is a rule's most: within after the send, healthy at every reading since it.
|
||||
heldLongEnough := func(within time.Duration) bool {
|
||||
return within > 0 && !g.Faulted && !now.Before(g.Since.Add(within))
|
||||
}
|
||||
g.Wants = wants
|
||||
// A module's own pass is its own class's: a leaf beside a core module that fails keeps its build after
|
||||
// A module's own pass is its own rule's: a leaf beside a core module that fails keeps its build after
|
||||
// its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524).
|
||||
passedAlone := func(m string) bool {
|
||||
p, s := gateWants(m, shelf[m])
|
||||
return (settled || !s) && reading[m] <= healthPerson && g.Healthy[m] >= p
|
||||
r := j.rules[m]
|
||||
return reading[m] <= healthPerson && (g.Healthy[m] >= r.passes || (g.Healthy[m] >= 1 && heldLongEnough(r.within)))
|
||||
}
|
||||
// fail decides the send failed: what passed on its own keeps its pass, everything else is put back.
|
||||
fail := func(why string) {
|
||||
@@ -671,6 +819,7 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
pastBound := now.Sub(*g.Since) > gateBound
|
||||
switch {
|
||||
case worst == healthBroken:
|
||||
g.Faulted = true
|
||||
g.Read(now, false, g.BrokenWhy)
|
||||
var judging []string
|
||||
for _, m := range modules {
|
||||
@@ -689,25 +838,36 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
case worst == healthWaiting:
|
||||
// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
|
||||
// the provider's own condition says what is wrong (ADR 0240 rule 5).
|
||||
g.Faulted = g.Faulted || g.HealthySeen // not healthy after a healthy reading; before one, still starting
|
||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||
g.Read(now, false, why)
|
||||
case worst == healthNotYet:
|
||||
g.Faulted = g.Faulted || g.HealthySeen // not healthy after a healthy reading; before one, still starting
|
||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||
g.Read(now, false, why)
|
||||
if pastBound {
|
||||
fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why))
|
||||
}
|
||||
case now.Before(floor):
|
||||
// Healthy before its rule lets a reading count: not a pass yet, and not a fault (novox/hq ADR 0351).
|
||||
g.HealthySeen = true
|
||||
// The bound still holds: a module that cannot be read healthy after its floor fails at it.
|
||||
g.Last, g.Failing = fmt.Sprintf("healthy, and a reading counts from %s by %s",
|
||||
floor.UTC().Format(time.RFC3339), said), nil
|
||||
if pastBound {
|
||||
fail(fmt.Sprintf("not judged within %s of its apply: %s", gateBound, g.Last))
|
||||
}
|
||||
default:
|
||||
// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
|
||||
g.HealthySeen = true
|
||||
g.Passes++
|
||||
g.LastPass, g.Last, g.Failing = &now, "", nil
|
||||
g.Read(now, true, "")
|
||||
if g.Passes >= wants && (settled || !settles) {
|
||||
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
|
||||
now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
|
||||
if g.Passes >= wants || heldLongEnough(within) {
|
||||
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s, by %s", g.Passes,
|
||||
now.Sub(*g.Since).Round(time.Second), said)+waitsSaid(g.Waits), now)
|
||||
}
|
||||
}
|
||||
return g.Verdict, nil
|
||||
}
|
||||
|
||||
// judgedCommits is the commit of each module a gate judges: the gate's own To for its module, and each
|
||||
@@ -1283,6 +1443,12 @@ func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error
|
||||
}
|
||||
// A release held after a failed one, while builds still wait for a gate (ADR 0236).
|
||||
out = append(out, backlogObservation()...)
|
||||
// And a rest that went silent (novox/hq ADR 0353 §4).
|
||||
silent, err := silentObservation(ctx, d.open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, silent...)
|
||||
return sortedFound(dedupeObservations(out)), nil
|
||||
}
|
||||
|
||||
@@ -1372,6 +1538,15 @@ func gateLine(g *inventory.PlanGate) string {
|
||||
case inventory.GateFailed:
|
||||
what = "FAILED"
|
||||
}
|
||||
switch {
|
||||
case g.Verdict == "" && g.Pending:
|
||||
what = "kept, not yet sent" // novox/hq ADR 0353 §8
|
||||
case g.Verdict == "" && g.Silent:
|
||||
what = "silent on its send, left" // ADR 0353 §4
|
||||
}
|
||||
if g.Verdict == "" && g.Waiting() {
|
||||
what = "waits for a busy container on " + g.Busy.Machine
|
||||
}
|
||||
line := fmt.Sprintf("gate on %s: %s", strings.Join(g.Machines, ", "), what)
|
||||
if g.Component != "" {
|
||||
line += " (core: " + g.Component + ")"
|
||||
|
||||
@@ -21,11 +21,15 @@ import (
|
||||
// wait, and a provider waiting for a login says who waits on it.
|
||||
|
||||
// withGateBounds sets the gate's bounds for one test.
|
||||
func withGateBounds(t *testing.T, settle, every, bound time.Duration) {
|
||||
func withGateBounds(t *testing.T, within, every, bound time.Duration) {
|
||||
t.Helper()
|
||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||
gateSettle, gateEvery, gateBound = settle, every, bound
|
||||
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
||||
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||
wasWindow := gateWitnessWindow
|
||||
gateWitnessWindow = 0
|
||||
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||
gateWithin, gateEvery, gateBound = within, every, bound
|
||||
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
|
||||
}
|
||||
|
||||
// factsOn is a judging's facts for one machine that applied its send: the node tools answer, and what it says
|
||||
@@ -53,10 +57,11 @@ func healthyContainer(module string) inventory.ResourceHealth {
|
||||
// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass,
|
||||
// counted over the gate's spacing — before it says its own. The broken one alone is put back.
|
||||
func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second)
|
||||
withGateBounds(t, time.Hour, 20*time.Millisecond, 10*time.Second)
|
||||
since := time.Now().Add(-time.Second)
|
||||
base := factsOn(t, "laptop", since, healthyContainer("app"))
|
||||
wasGather := gatherGateFacts
|
||||
@@ -95,9 +100,10 @@ const broker_runtime = "node-tools"
|
||||
// **A pass is counted only the gate's spacing after the one before** (review (c)): a send judged every tick
|
||||
// while a module beside it is not yet healthy counts the healthy one once.
|
||||
func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
withGateBounds(t, 0, time.Hour, time.Hour)
|
||||
withGateBounds(t, time.Hour, time.Hour, time.Hour)
|
||||
since := time.Now().Add(-time.Minute)
|
||||
base := factsOn(t, "laptop", since, healthyContainer("app"), inventory.ResourceHealth{Module: "late",
|
||||
Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, Reason: "down"})
|
||||
@@ -119,6 +125,7 @@ func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) {
|
||||
|
||||
// **A carried build's pass carries its own wait, never another's** (review (e)).
|
||||
func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
@@ -143,6 +150,7 @@ func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) {
|
||||
// carried its own module, healthy, and a build waiting on that machine that never became healthy. At the
|
||||
// bound the waiting one is put back and marked; the plan's own module keeps its pass, so no walk waits on it.
|
||||
func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
@@ -223,6 +231,7 @@ func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) {
|
||||
// **A provider waiting for a login says who waits on it** (review (g)): its consumer, failing a check that
|
||||
// needs it, is held under it, and the provider's relogin-needed condition lists it and is urgent.
|
||||
func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
@@ -285,6 +294,7 @@ func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
|
||||
// tools are marked and put back in the very judging that found them broken — their registered build is the
|
||||
// one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass.
|
||||
func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
@@ -376,7 +386,7 @@ func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testi
|
||||
f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{app}}}
|
||||
return f, nil
|
||||
}
|
||||
withGateBounds(t, 100*time.Millisecond, 0, 10*time.Second)
|
||||
withGateBounds(t, time.Hour, 0, 10*time.Second)
|
||||
|
||||
release := func() inventory.Plan {
|
||||
t.Helper()
|
||||
@@ -427,6 +437,7 @@ func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testi
|
||||
// **A move from a build not known excuses no wait** (issue 318 review): a plan's own module whose previous
|
||||
// build was not recorded, or whose previous manifest is not kept, cannot be shown to have added the group.
|
||||
func TestAMoveFromAnUnknownBuildExcusesNoWait(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
@@ -0,0 +1,338 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// The first-node gate's readings by module (novox/hq ADR 0351, ADR 0352): the controller and the tool runner pass
|
||||
// on one reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's
|
||||
// resolver on their third healthy reading and never later than two minutes while healthy; the bus is kept so.
|
||||
|
||||
// withTheGatesTimes sets the gate's production bounds for a test, whatever another test left.
|
||||
func withTheGatesTimes(t *testing.T) {
|
||||
t.Helper()
|
||||
wasWithin, wasEvery, wasBound, wasWindow := gateWithin, gateEvery, gateBound, gateWitnessWindow
|
||||
wasPasses, wasLeafPasses, wasLeafSettles := gatePasses, gateLeafPasses, gateLeafSettles
|
||||
gateWithin, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second
|
||||
gatePasses, gateLeafPasses, gateLeafSettles = 3, 1, false
|
||||
t.Cleanup(func() {
|
||||
gateWithin, gateEvery, gateBound, gateWitnessWindow = wasWithin, wasEvery, wasBound, wasWindow
|
||||
gatePasses, gateLeafPasses, gateLeafSettles = wasPasses, wasLeafPasses, wasLeafSettles
|
||||
})
|
||||
}
|
||||
|
||||
func TestEachModuleHasItsGateRule(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
for _, c := range []struct {
|
||||
module string
|
||||
claims []catalogue.Claim
|
||||
passes int
|
||||
within time.Duration
|
||||
witnessed bool
|
||||
said string
|
||||
}{
|
||||
{"mesh-controller", nil, 1, 0, true, "the controller's rule"},
|
||||
{"node-tools", nil, 1, 0, true, "the tool runner's rule"},
|
||||
{catalogue.DeliverySeat, nil, 1, 0, false, "mesh-delivery's rule"},
|
||||
{"mesh-host", nil, 3, 2 * time.Minute, false, "the core's rule"},
|
||||
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, 2 * time.Minute, false, "the core's rule"},
|
||||
{"nats", nil, 3, 2 * time.Minute, false, "the core's rule"},
|
||||
{"app", nil, 1, 0, false, "a leaf module's rule"},
|
||||
} {
|
||||
r := gateRuleOf(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
|
||||
if r.passes != c.passes || r.within != c.within || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) {
|
||||
t.Errorf("%s: %+v; want %d readings, within %s, witnessed %v, said %q", c.module, r, c.passes, c.within,
|
||||
c.witnessed, c.said)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Every module that was core stays core for its delivery budget and its walk: only its readings change.
|
||||
func TestTheCoreStaysTheCore(t *testing.T) {
|
||||
for _, m := range []string{"mesh-controller", "mesh-host", "node-tools", "nats", catalogue.DeliverySeat} {
|
||||
if !coreModule(m, catalogue.Manifest{Module: m}) {
|
||||
t.Errorf("%s is no longer judged as the core for its class", m)
|
||||
}
|
||||
}
|
||||
if !coreModule("dnsmasq", catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}) {
|
||||
t.Error("the resolver's holder is no longer judged as the core for its class")
|
||||
}
|
||||
}
|
||||
|
||||
// gateOn judges one module on one machine, healthy at every reading.
|
||||
type gateOn struct {
|
||||
module, node string
|
||||
g *inventory.PlanGate
|
||||
holder *lease.Holder
|
||||
}
|
||||
|
||||
func aGateOn(module, node string, since time.Time) *gateOn {
|
||||
return &gateOn{module: module, node: node, g: &inventory.PlanGate{Machines: []string{node}, Since: &since,
|
||||
Applied: map[string]time.Time{}}}
|
||||
}
|
||||
|
||||
func (o *gateOn) read(at time.Time, h health, why string) {
|
||||
r := gateRuleOf(o.module, catalogue.Manifest{Module: o.module})
|
||||
floor := gateFloor(o.module, r, *o.g.Since, []string{o.node}, o.g.Applied, o.holder)
|
||||
tallyGate(o.g, gateJudging{modules: []string{o.module}, reading: map[string]health{o.module: h},
|
||||
worst: h, why: why, rules: map[string]gateRule{o.module: r}, floors: map[string]time.Time{o.module: floor}}, at)
|
||||
}
|
||||
|
||||
func TestTheControllerPassesOnOneReadingAfterTheWitnessWindow(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
sent := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC)
|
||||
o := aGateOn("mesh-controller", "novox", sent)
|
||||
o.g.Applied["novox"] = sent.Add(15 * time.Second)
|
||||
o.holder = &lease.Holder{Instance: "new", Taken: sent.Add(20 * time.Second)}
|
||||
|
||||
// Ready, but inside the witness window, which opened before it took the lease: not counted.
|
||||
for _, after := range []time.Duration{30 * time.Second, 60 * time.Second, 79 * time.Second} {
|
||||
o.read(sent.Add(after), healthGood, "")
|
||||
if o.g.Verdict != "" || o.g.Passes != 0 || !strings.Contains(o.g.Last, "counts from") {
|
||||
t.Fatalf("at %s: %+v; want no reading counted inside the witness window", after, o.g)
|
||||
}
|
||||
}
|
||||
o.read(sent.Add(80*time.Second), healthGood, "")
|
||||
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || o.g.Wants != 1 {
|
||||
t.Fatalf("after the witness window: %+v; want passed on one reading", o.g)
|
||||
}
|
||||
if !strings.Contains(o.g.Why, "healthy 1 times") || !strings.Contains(o.g.Why, "the controller's rule") ||
|
||||
!strings.Contains(o.g.Why, "ADR 0351") {
|
||||
t.Errorf("the verdict says %q; want the rule it applied", o.g.Why)
|
||||
}
|
||||
}
|
||||
|
||||
// A controller that starts again takes the lease again: its witness window starts again, and so does the
|
||||
// gate's wait.
|
||||
func TestAControllerThatStartsAgainWaitsOutItsNewWindow(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
sent := time.Now()
|
||||
o := aGateOn("mesh-controller", "novox", sent)
|
||||
o.g.Applied["novox"] = sent.Add(10 * time.Second)
|
||||
o.holder = &lease.Holder{Instance: "again", Taken: sent.Add(70 * time.Second)}
|
||||
o.read(sent.Add(90*time.Second), healthGood, "")
|
||||
if o.g.Verdict != "" {
|
||||
t.Fatalf("passed %s after a start the witness still judges: %+v", 20*time.Second, o.g)
|
||||
}
|
||||
o.read(sent.Add(130*time.Second), healthGood, "")
|
||||
if o.g.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("not passed after the new window: %+v", o.g)
|
||||
}
|
||||
}
|
||||
|
||||
// Without the lease's word (a controller carried by another gate), the window counts from the applied report,
|
||||
// and from the send at the least.
|
||||
func TestTheWitnessWindowCountsFromTheSendAtTheLeast(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
sent := time.Now()
|
||||
r := gateRuleOf("mesh-controller", catalogue.Manifest{})
|
||||
if f := gateFloor("mesh-controller", r, sent, []string{"novox"}, nil, nil); !f.Equal(sent.Add(time.Minute)) {
|
||||
t.Errorf("floor %s after the send; want 1m", f.Sub(sent))
|
||||
}
|
||||
if f := gateFloor("node-tools", gateRuleOf("node-tools", catalogue.Manifest{}), sent, []string{"ace"},
|
||||
map[string]time.Time{"ace": sent.Add(12 * time.Second)}, nil); !f.Equal(sent.Add(72 * time.Second)) {
|
||||
t.Errorf("the tool runner's floor %s after the send; want 1m12s", f.Sub(sent))
|
||||
}
|
||||
}
|
||||
|
||||
// A controller that a witness put back, or that never becomes ready, fails its gate as before.
|
||||
func TestAControllerThatFailsIsStillFailed(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
sent := time.Now()
|
||||
o := aGateOn("mesh-controller", "novox", sent)
|
||||
o.read(sent.Add(30*time.Second), healthBroken, "the witness on novox judged the controller and rolled-back")
|
||||
if o.g.Verdict != inventory.GateFailed {
|
||||
t.Fatalf("a controller put back by its witness: %+v; want failed", o.g)
|
||||
}
|
||||
o = aGateOn("mesh-controller", "novox", sent)
|
||||
o.read(sent.Add(5*time.Minute), healthNotYet, "the controller holding the lease does not say it is ready")
|
||||
if o.g.Verdict != "" {
|
||||
t.Fatalf("failed before the bound: %+v", o.g)
|
||||
}
|
||||
o.read(sent.Add(10*time.Minute+time.Second), healthNotYet, "the controller holding the lease does not say it is ready")
|
||||
if o.g.Verdict != inventory.GateFailed || !strings.Contains(o.g.Why, "not healthy within") {
|
||||
t.Fatalf("never ready: %+v; want failed at the bound", o.g)
|
||||
}
|
||||
}
|
||||
|
||||
// threeReadings judges a module under the three-reading rule on one machine, the send at sent.
|
||||
func threeReadings(t *testing.T, module string, sent time.Time) (*inventory.PlanGate, func(time.Duration, health)) {
|
||||
t.Helper()
|
||||
g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent}
|
||||
rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}})
|
||||
floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil)
|
||||
return g, func(after time.Duration, h health) {
|
||||
why := ""
|
||||
if h != healthGood {
|
||||
why = "the node-engine reports another build"
|
||||
}
|
||||
tallyGate(g, gateJudging{modules: []string{module}, reading: map[string]health{module: h}, worst: h, why: why,
|
||||
rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}}, sent.Add(after))
|
||||
}
|
||||
}
|
||||
|
||||
// A healthy node-engine or resolver passes at its third healthy reading, 40 s apart from the apply: before two
|
||||
// minutes (novox/hq ADR 0352).
|
||||
func TestTheNodeEngineAndTheResolverPassAtTheirThirdReading(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
for _, module := range []string{"mesh-host", "dnsmasq"} {
|
||||
g, judge := threeReadings(t, module, time.Now())
|
||||
judge(15*time.Second, healthGood)
|
||||
judge(55*time.Second, healthGood)
|
||||
if g.Verdict != "" || g.Passes != 2 {
|
||||
t.Fatalf("%s: after two readings %+v; want still judging", module, g)
|
||||
}
|
||||
judge(95*time.Second, healthGood)
|
||||
if g.Verdict != inventory.GatePassed || g.Passes != 3 || g.Wants != 3 ||
|
||||
!strings.Contains(g.Why, "over 1m35s") || !strings.Contains(g.Why, "the core's rule: 3 readings") {
|
||||
t.Fatalf("%s: after three readings %+v; want passed at 1m35s by the core's rule", module, g)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// One healthy throughout whose readings came late passes at two minutes on the readings it has; the judging at
|
||||
// two minutes comes sooner than the spacing after the last pass.
|
||||
func TestAHealthyNodeEngineIsNeverHeldPastTwoMinutes(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
sent := time.Now()
|
||||
last := sent.Add(100 * time.Second)
|
||||
held := &inventory.PlanGate{Since: &sent, LastPass: &last}
|
||||
if dueAtTheMost(held, sent.Add(119*time.Second)) || !dueAtTheMost(held, sent.Add(2*time.Minute)) {
|
||||
t.Fatal("the judging at two minutes is not due, or is due before")
|
||||
}
|
||||
held.Faulted = true
|
||||
if dueAtTheMost(held, sent.Add(2*time.Minute)) {
|
||||
t.Fatal("a send that was not healthy at a reading is judged at two minutes")
|
||||
}
|
||||
|
||||
g, judge := threeReadings(t, "mesh-host", time.Now())
|
||||
judge(100*time.Second, healthGood)
|
||||
judge(2*time.Minute, healthGood)
|
||||
if g.Verdict != inventory.GatePassed || g.Passes != 2 {
|
||||
t.Fatalf("late readings, healthy throughout, at two minutes: %+v; want passed on two", g)
|
||||
}
|
||||
g, judge = threeReadings(t, "mesh-host", time.Now())
|
||||
judge(2*time.Minute, healthGood)
|
||||
if g.Verdict != inventory.GatePassed || g.Passes != 1 {
|
||||
t.Fatalf("one healthy reading, at two minutes: %+v; want passed on it", g)
|
||||
}
|
||||
}
|
||||
|
||||
// A reading not yet healthy before the first healthy one is the build still starting: the two minutes still hold.
|
||||
func TestANodeEngineStillStartingKeepsItsTwoMinutes(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
g, judge := threeReadings(t, "mesh-host", time.Now())
|
||||
judge(10*time.Second, healthNotYet)
|
||||
judge(100*time.Second, healthGood)
|
||||
judge(2*time.Minute, healthGood)
|
||||
if g.Verdict != inventory.GatePassed || g.Faulted {
|
||||
t.Fatalf("%+v; want passed at two minutes", g)
|
||||
}
|
||||
}
|
||||
|
||||
// A reading that is not healthy after a healthy one takes the two minutes away: the module needs its three readings, and one
|
||||
// never healthy fails at the bound and is put back; one found broken fails.
|
||||
func TestANodeEngineThatFailsAReadingIsNotPassedAtTwoMinutes(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
g, judge := threeReadings(t, "mesh-host", time.Now())
|
||||
judge(20*time.Second, healthNotYet)
|
||||
judge(60*time.Second, healthGood)
|
||||
judge(100*time.Second, healthGood)
|
||||
judge(2*time.Minute+10*time.Second, healthGood)
|
||||
if g.Verdict != inventory.GatePassed || g.Passes != 3 {
|
||||
t.Fatalf("%+v; want passed only on its third healthy reading", g)
|
||||
}
|
||||
g, judge = threeReadings(t, "mesh-host", time.Now())
|
||||
judge(60*time.Second, healthGood)
|
||||
judge(100*time.Second, healthNotYet)
|
||||
judge(2*time.Minute, healthGood)
|
||||
if g.Verdict != "" || !g.Faulted {
|
||||
t.Fatalf("passed at two minutes after a reading that was not healthy: %+v", g)
|
||||
}
|
||||
judge(10*time.Minute+time.Second, healthNotYet)
|
||||
if g.Verdict != inventory.GateFailed || !strings.Contains(g.Why, "not healthy within") {
|
||||
t.Fatalf("never healthy again: %+v; want failed at the bound", g)
|
||||
}
|
||||
g, judge = threeReadings(t, "mesh-host", time.Now())
|
||||
judge(60*time.Second, healthGood)
|
||||
judge(100*time.Second, healthBroken)
|
||||
if g.Verdict != inventory.GateFailed {
|
||||
t.Fatalf("a broken reading: %+v; want failed, to be put back", g)
|
||||
}
|
||||
}
|
||||
|
||||
// The tool runner passes on one reading after its witness window; mesh-delivery on one, at once.
|
||||
func TestTheToolRunnerAndMeshDeliveryPassOnOneReading(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
sent := time.Now()
|
||||
o := aGateOn("node-tools", "ace", sent)
|
||||
o.g.Applied["ace"] = sent.Add(10 * time.Second)
|
||||
o.read(sent.Add(30*time.Second), healthGood, "")
|
||||
if o.g.Verdict != "" {
|
||||
t.Fatalf("the tool runner passed inside its witness window: %+v", o.g)
|
||||
}
|
||||
o.read(sent.Add(70*time.Second), healthGood, "")
|
||||
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "the tool runner's rule") {
|
||||
t.Fatalf("the tool runner: %+v; want passed on one reading", o.g)
|
||||
}
|
||||
|
||||
o = aGateOn(catalogue.DeliverySeat, "novox", sent)
|
||||
o.read(sent.Add(5*time.Second), healthGood, "")
|
||||
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "mesh-delivery's rule") {
|
||||
t.Fatalf("mesh-delivery: %+v; want passed on one reading", o.g)
|
||||
}
|
||||
}
|
||||
|
||||
// A dip after a healthy reading taken before the controller's witness floor is a fault: the send is not passed at
|
||||
// two minutes (novox/hq ADR 0352 decision 3).
|
||||
func TestADipAfterAnUncountedHealthyReadingIsAFault(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
sent := time.Now()
|
||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
|
||||
modules := []string{"mesh-controller", "mesh-host"}
|
||||
rules, floors := map[string]gateRule{}, map[string]time.Time{}
|
||||
for _, m := range modules {
|
||||
rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m})
|
||||
}
|
||||
floors["mesh-host"] = sent
|
||||
floors["mesh-controller"] = sent.Add(90 * time.Second)
|
||||
judge := func(after time.Duration, h health) {
|
||||
tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": h, "mesh-host": h},
|
||||
worst: h, why: "not yet", rules: rules, floors: floors}, sent.Add(after))
|
||||
}
|
||||
judge(30*time.Second, healthGood)
|
||||
if g.Passes != 0 {
|
||||
t.Fatalf("a reading before the controller's floor was counted: %+v", g)
|
||||
}
|
||||
judge(70*time.Second, healthNotYet)
|
||||
judge(2*time.Minute, healthGood)
|
||||
if !g.Faulted || g.Verdict != "" {
|
||||
t.Fatalf("%+v; want the dip a fault, and no pass at two minutes", g)
|
||||
}
|
||||
}
|
||||
|
||||
// A send carrying the controller and the node-engine is judged by the node-engine's three readings.
|
||||
func TestASendIsJudgedByItsMostDemandingRule(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
sent := time.Now()
|
||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
|
||||
modules := []string{"mesh-controller", "mesh-host"}
|
||||
rules, floors := map[string]gateRule{}, map[string]time.Time{}
|
||||
for _, m := range modules {
|
||||
rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m})
|
||||
floors[m] = gateFloor(m, rules[m], sent, []string{"novox"}, nil, nil)
|
||||
}
|
||||
for _, after := range []time.Duration{70 * time.Second, 110 * time.Second} {
|
||||
tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": healthGood,
|
||||
"mesh-host": healthGood}, worst: healthGood, rules: rules, floors: floors}, sent.Add(after))
|
||||
}
|
||||
if g.Verdict != "" || g.Wants != 3 || g.Passes != 2 || g.Healthy["mesh-controller"] != 2 {
|
||||
t.Fatalf("%+v; want the send judged by the node-engine's three readings, the controller's own counted", g)
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,10 @@ func aGateMesh(t *testing.T) *gateMesh {
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
g := &gateMesh{open: open, health: map[string]health{}}
|
||||
// The gate's own tests judge a first machine, as walks a controller before novox/hq ADR 0360 started still are.
|
||||
wasGated := gateFirstMachine
|
||||
gateFirstMachine = true
|
||||
t.Cleanup(func() { gateFirstMachine = wasGated })
|
||||
g.keeper, _ = withConditionsInMemory(t)
|
||||
g.told = &conditions.Told{}
|
||||
was := doctorFrom
|
||||
@@ -124,10 +128,14 @@ func aGateMesh(t *testing.T) *gateMesh {
|
||||
}
|
||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||
|
||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||
wasWindow := gateWitnessWindow
|
||||
gateWitnessWindow = 0
|
||||
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||
// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
|
||||
gateSettle, gateEvery = 0, time.Hour
|
||||
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
||||
gateWithin, gateEvery = time.Hour, time.Hour
|
||||
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
|
||||
|
||||
built := time.Now().UTC()
|
||||
plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
||||
@@ -267,12 +275,40 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A build rolls everywhere with no hand and no gate (novox/hq ADR 0360 §1): both machines in one send, no health
|
||||
// judged, and the walk done once both reported on the send. Before, anchor was judged first, then laptop sent.
|
||||
func TestABuildRollsToEveryMachineAtOnceWithNoGate(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
gateFirstMachine = false
|
||||
ctx := t.Context()
|
||||
judged := false
|
||||
wasGather := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
judged = true
|
||||
return wasGather(ctx, open, component)
|
||||
}
|
||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||
for i := 0; i < 6; i++ {
|
||||
advancePlans(ctx, g.open)
|
||||
}
|
||||
p := g.plan(t)
|
||||
if !reflect.DeepEqual(g.sent, [][]string{{"anchor", "laptop"}}) {
|
||||
t.Fatalf("sent %v, not both machines in one send", g.sent)
|
||||
}
|
||||
if s := p.Modules["app"]; p.State != inventory.PlanDone || s.Gate != nil || s.FirstAt != nil || judged {
|
||||
t.Fatalf("the plan is %s (%s), app %+v, judged %v", p.State, p.Note, s, judged)
|
||||
}
|
||||
if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 {
|
||||
t.Fatalf("a rolled build left a condition: %+v %v", obs, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the
|
||||
// core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524).
|
||||
func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
gateSettle = time.Hour // a core module could not pass for an hour
|
||||
gateWithin = time.Hour // a core module could not pass on fewer readings for an hour
|
||||
advancePlans(ctx, g.open) // the first machine is sent the new build
|
||||
advancePlans(ctx, g.open) // one healthy judging
|
||||
p := g.plan(t)
|
||||
@@ -285,31 +321,6 @@ func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Which modules are judged as the core: those on the controller's own path and the resolver's holder, three
|
||||
// readings after the settle time; every other module one (novox/hq ADR 0282 decision 1, ADR 0304 decision 3).
|
||||
func TestAModuleIsJudgedByItsClass(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
module string
|
||||
claims []catalogue.Claim
|
||||
passes int
|
||||
settles bool
|
||||
}{
|
||||
{"app", nil, 1, false},
|
||||
{"nzbget", nil, 1, false},
|
||||
{"mesh-controller", nil, 3, true},
|
||||
{"mesh-host", nil, 3, true},
|
||||
{"node-tools", nil, 3, true},
|
||||
{"nats", nil, 3, true},
|
||||
{catalogue.DeliverySeat, nil, 3, true},
|
||||
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, true},
|
||||
} {
|
||||
p, s := gateWants(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
|
||||
if p != c.passes || s != c.settles {
|
||||
t.Errorf("%s: %d passes, settles %v; want %d, %v", c.module, p, s, c.passes, c.settles)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A first machine that never becomes healthy fails its gate at the bound, and is put back.
|
||||
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
|
||||
@@ -89,8 +89,6 @@ var handActVerbs = []handActVerb{
|
||||
{Verb: handActIssued, Decision: "a credential the provider's software issued, kept for its consumers (ADR 0305)"},
|
||||
{Verb: "hand-act drill", Decision: "a drill is a person's deliberate test of the mesh, never a repair"},
|
||||
// A person's decisions by design.
|
||||
// A config value is what a person chooses the mesh's modules use (novox/hq issue 630), never a repair.
|
||||
{Verb: "config", Decision: "a config value is a person's choice of what the modules use, not a repair (issue 630)"},
|
||||
{Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"},
|
||||
{Verb: "retire reject", Decision: "keeping a consumer active is a person's word (ADR 0230)"},
|
||||
{Verb: "cleanup delete", Decision: "nothing retired is deleted without a person (ADR 0230)"},
|
||||
|
||||
@@ -151,7 +151,7 @@ func heldMachines(ctx context.Context, open *stores, names []string) (map[string
|
||||
for _, mv := range f.moves(node, modules, same, known, false) {
|
||||
if planStillToSend(plans, mv.Module, node) == "" {
|
||||
why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+
|
||||
"plan sends it, one machine at a time, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
|
||||
"plan sends it, one machine first and then the rest at once, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
|
||||
}
|
||||
}
|
||||
if len(why) > 0 {
|
||||
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
// another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module
|
||||
// to another build supersedes the judging: no verdict, nothing put back.
|
||||
func TestReplay352(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn)
|
||||
t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging)
|
||||
@@ -67,6 +68,7 @@ func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) {
|
||||
|
||||
// A plan's own first send waits while a release judges the same module on that machine with another build.
|
||||
func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
@@ -82,6 +84,7 @@ func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
|
||||
|
||||
// A merge plan's judging is superseded the same way: another send moved its module on the first machine.
|
||||
func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
@@ -140,6 +143,7 @@ func testASendThatMovesTheModuleSupersedesTheJudging(t *testing.T) {
|
||||
// A report is on the gate's own send: the declaration itself, or one sequenced after it; a gate kept
|
||||
// without its send reads the report against the send made last, as before. Pure.
|
||||
func TestAReportIsHeldAgainstTheGatesOwnSend(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
sent := inventory.SentDeclaration{Digest: "d-490", Sequence: 490}
|
||||
for _, c := range []struct {
|
||||
r inventory.Reported
|
||||
@@ -185,6 +189,7 @@ func TestAReportIsHeldAgainstTheGatesOwnSend(t *testing.T) {
|
||||
// A gate judges only the build the machine was last sent: last sent another build of the module, the
|
||||
// judging is superseded, whatever the machine reports. Pure.
|
||||
func TestAGateJudgesOnlyTheBuildTheMachineWasLastSent(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
at := time.Now()
|
||||
f := gateFacts{now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor", Outcome: inventory.OutcomeApplied,
|
||||
At: &at, Current: true}}, engines: map[string]string{}, served: map[string]served{}, rolledBack: map[string][]lease.Rollback{},
|
||||
@@ -212,6 +217,7 @@ func TestAGateJudgesOnlyTheBuildTheMachineWasLastSent(t *testing.T) {
|
||||
// A move of another build of a module to a machine where a release or a plan is judging that module
|
||||
// waits for that judging; the same build to that machine is already there. Pure.
|
||||
func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
at := time.Now()
|
||||
release := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{
|
||||
Gate: &inventory.PlanGate{Machines: []string{"novox"}, Carried: []inventory.CarriedMove{
|
||||
@@ -249,6 +255,7 @@ func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) {
|
||||
// The controller is never put back to a build that reaches less of the store's schema than the store
|
||||
// has, or to one that never said what it reaches: the current build is kept, and the condition says so.
|
||||
func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
@@ -345,6 +352,7 @@ func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T)
|
||||
|
||||
// The lease's holder names the build the declaration told it it is, and the version stamp only without one.
|
||||
func TestTheHolderNamesTheBuildTheDeclarationToldIt(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
t.Setenv(RunningBuildVar, " ad62528c47c7 ")
|
||||
if h := holderOf("x"); h.Build != "ad62528c47c7" {
|
||||
t.Fatalf("the holder's build is %q", h.Build)
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A tier whose module is no move — made from the source every machine running it runs, so nothing is
|
||||
// sent — is not waited on for a report after its marking (novox/hq issue 648). Its machines answered the
|
||||
// send that last carried that build before the walk got there, and nothing will make them report again:
|
||||
// the walk asks the next tier in the same step, rather than "waiting for app on anchor, laptop" for ever.
|
||||
func TestANoMoveTierWaitsForNoNewReport(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
|
||||
// app holds the build seat, so a module of the next tier is built by it; since novox/hq ADR 0360 §3 that gates
|
||||
// nothing.
|
||||
for _, m := range []catalogue.Manifest{
|
||||
{Module: "app", Version: "c2", Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}},
|
||||
{Module: "web", Version: "w1"},
|
||||
} {
|
||||
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
||||
Path: "modules/" + m.Module, BuiltFrom: m.Version, Head: m.Version, Asked: time.Now().Add(-time.Minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// What anchor and laptop were last sent (c1) was made from the source the plan's build of c2 is.
|
||||
for _, b := range []inventory.Build{
|
||||
{ID: "build-1s", Module: "app", Commit: "c1", SourceFingerprint: "src1:app",
|
||||
Asked: time.Now().Add(-30 * time.Second), At: time.Now().Add(-30 * time.Second)},
|
||||
{ID: "build-2s", Module: "app", Commit: "c2", SourceFingerprint: "src1:app", Asked: time.Now(), At: time.Now()},
|
||||
} {
|
||||
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Each machine answered that send, applied, before the walk's tier was built.
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.RecordDoing(ctx, nodeID(t, g.open, n), inventory.Doing{Node: n,
|
||||
Outcome: inventory.OutcomeApplied, Declared: "d-" + n + "-c1", Applied: 1,
|
||||
At: time.Now().Add(-10 * time.Second)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
built := time.Now().UTC()
|
||||
p := g.plan(t)
|
||||
p.Tiers = [][]string{{"app"}, {"web"}}
|
||||
p.Modules["app"].Build, p.Modules["app"].BuiltAt = "build-2s", &built
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
advancePlans(ctx, g.open)
|
||||
|
||||
if len(g.sent) != 0 {
|
||||
t.Fatalf("a no-move build was sent: %v", g.sent)
|
||||
}
|
||||
p = g.plan(t)
|
||||
if s := p.Modules["app"]; s == nil || !strings.HasPrefix(s.Why, "no move") {
|
||||
t.Fatalf("the build was not marked no move: %+v", s)
|
||||
}
|
||||
if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") {
|
||||
t.Fatalf("the walk waits on a no-move tier for a report nothing will cause: tier %d, %s %q", p.Tier, p.State, p.Note)
|
||||
}
|
||||
|
||||
// A machine that has not answered its last send is not waited on either: a built-by edge gates no tier
|
||||
// (novox/hq ADR 0360 §3). Before, the walk waited for app on laptop.
|
||||
p.Tier, p.State, p.Note = 0, inventory.PlanRolling, ""
|
||||
if err := inv.RecordSent(ctx, nodeID(t, g.open, "laptop"), "d-laptop-later", map[string]string{"app": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, g.open)
|
||||
p = g.plan(t)
|
||||
if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") {
|
||||
t.Fatalf("the walk waits on a built-by edge: tier %d, %q", p.Tier, p.Note)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0360 §4, §5: a walk ends on its machines' reports — every machine sent has reported on its send,
|
||||
// or is silent past ApplySilentAfter — and names a failed or refused apply; it never waits for a health reading.
|
||||
func TestAWalkEndsOnItsMachinesReports(t *testing.T) {
|
||||
now := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
|
||||
sent := now.Add(-time.Minute)
|
||||
long := now.Add(-inventory.ApplySilentAfter - time.Minute)
|
||||
at := now.Add(-10 * time.Second)
|
||||
p := inventory.Plan{Modules: map[string]*inventory.PlanModule{
|
||||
"app": {Rest: map[string]inventory.SentDeclaration{
|
||||
"ace": {Digest: "d-ace", At: sent},
|
||||
"novox": {Digest: "d-novox", At: sent},
|
||||
"shanks": {Digest: "d-shanks", At: sent},
|
||||
"laptop": {Digest: "d-laptop", At: long},
|
||||
}},
|
||||
"proxy": {Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d-ace", At: sent}}},
|
||||
}}
|
||||
reports := []inventory.Reported{
|
||||
{Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"},
|
||||
{Node: "novox", Outcome: inventory.OutcomeFailed, At: &at, Declared: "d-novox"},
|
||||
{Node: "shanks", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-before"},
|
||||
}
|
||||
waiting, failed, silent := walkReports(p, reports, now)
|
||||
if !slices.Equal(waiting, []string{"shanks"}) || !slices.Equal(failed, []string{"novox"}) ||
|
||||
!slices.Equal(silent, []string{"laptop"}) {
|
||||
t.Fatalf("waiting %v, failed %v, silent %v", waiting, failed, silent)
|
||||
}
|
||||
reports[2].Declared = "d-shanks"
|
||||
if waiting, _, _ := walkReports(p, reports, now); len(waiting) != 0 {
|
||||
t.Fatalf("the walk still waits once every machine reported: %v", waiting)
|
||||
}
|
||||
}
|
||||
|
||||
// withTheGate turns the first-machine gate on for a test of its own path: kept for the walks a controller before
|
||||
// novox/hq ADR 0360 started.
|
||||
func withTheGate(t *testing.T) {
|
||||
t.Helper()
|
||||
was := gateFirstMachine
|
||||
gateFirstMachine = true
|
||||
t.Cleanup(func() { gateFirstMachine = was })
|
||||
}
|
||||
|
||||
// Every machine heard from is sent at once and passed on its applied report, with no judging (novox/hq ADR 0360 §7);
|
||||
// before, the first machine was judged before the rest were sent (ADR 0353 §1).
|
||||
func TestAReleaseSendsEveryMachineAtOnce(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
said := events(t)
|
||||
for i := 0; i < 12; i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
p := b.release(t)
|
||||
if p.State != inventory.PlanDone || len(p.Release.Done) != 3 {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
for _, n := range []string{"desk", "laptop", "anchor"} {
|
||||
if !slices.Contains(*said, "send "+n) {
|
||||
t.Fatalf("%s was not sent: %v", n, *said)
|
||||
}
|
||||
}
|
||||
if slices.Contains(*said, "judge") {
|
||||
t.Fatalf("a machine was judged by its health: %v", *said)
|
||||
}
|
||||
if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine that reports its apply failed is named and fails the release, with nothing put back, and every other
|
||||
// machine goes on to its own report (novox/hq ADR 0360 §5, §7); before, it was put back (ADR 0353 §3).
|
||||
func TestAMachineThatFailsItsApplyIsNamedAndNothingIsPutBack(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
b.broken["desk"] = true // desk fails what it is sent
|
||||
advancePlans(ctx, b.open)
|
||||
for i := 0; i < 12 && b.release(t).Open(); i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
p := b.release(t)
|
||||
if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
if !slices.Contains(p.Release.Done, "laptop") || !slices.Contains(p.Release.Done, "anchor") {
|
||||
t.Fatalf("the others have no verdict of their own: %+v", p.Release)
|
||||
}
|
||||
if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c2" {
|
||||
t.Fatalf("desk was put back: %v", sent)
|
||||
}
|
||||
}
|
||||
|
||||
// A release's send kept before it was made, and made, is judged by the controller that takes over without being
|
||||
// made again (ADR 0353 §8), with every machine sent at once (ADR 0360 §7).
|
||||
func TestAReleaseSendKeptAndMadeIsNotMadeAgainAtOnce(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
p, err := releaseBacklog(ctx, b.open, "")
|
||||
if err != nil || p == nil {
|
||||
t.Fatalf("no release opened: %v", err)
|
||||
}
|
||||
if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gates["anchor"] == nil || !p.Release.Gates["anchor"].Pending {
|
||||
t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The send is made, and the controller is replaced before it keeps it.
|
||||
if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, b.open)
|
||||
if n := len(slices.DeleteFunc(slices.Clone(b.sent), func(s []string) bool { return !slices.Equal(s, []string{"anchor"}) })); n != 1 {
|
||||
t.Fatalf("anchor was sent again: %v", b.sent)
|
||||
}
|
||||
g := b.release(t).Release.Gates["anchor"]
|
||||
if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 {
|
||||
t.Fatalf("anchor is not judged on the send made: %+v", g)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put
|
||||
// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4); its step also sent anchor, so the release is done, not failed (ADR 0360 §7).
|
||||
func TestAMachineSilentPastItsBoundIsLeftAndSaidAtOnce(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
reporting := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing
|
||||
b.sent = append(b.sent, names)
|
||||
current, _ := open.inventory.CurrentBuilds(ctx)
|
||||
return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep",
|
||||
map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit})
|
||||
}
|
||||
return reporting(ctx, open, names)
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = reporting })
|
||||
for i := 0; i < 8; i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
p := b.release(t)
|
||||
g := p.Release.Gates["laptop"]
|
||||
if !p.Open() || g == nil || g.Verdict != "" || g.Silent {
|
||||
t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g)
|
||||
}
|
||||
since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute)
|
||||
g.Since = &since
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, b.open)
|
||||
p = b.release(t)
|
||||
// anchor applied in the same step, so the release is done, laptop named silent (novox/hq ADR 0360 §7).
|
||||
if p.State != inventory.PlanDone || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed {
|
||||
t.Fatal("a silent machine's build was marked failed")
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
|
||||
t.Fatalf("laptop was put back: %v", sent)
|
||||
}
|
||||
obs, err := silentObservation(ctx, inv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) {
|
||||
t.Fatalf("the silence is not said: %+v", obs)
|
||||
}
|
||||
}
|
||||
@@ -104,6 +104,10 @@ func run() error {
|
||||
return replayCommand(ctx, args[1:])
|
||||
case "kill":
|
||||
return killCommand(ctx, args[1:])
|
||||
case "forget-checks":
|
||||
return forgetChecksCommand(ctx, args[1:])
|
||||
case "try":
|
||||
return tryCommand(ctx, args[1:])
|
||||
case "pause", "resume":
|
||||
return pauseCommand(ctx, args[0], args[1:])
|
||||
case "collection":
|
||||
@@ -171,8 +175,6 @@ func run() error {
|
||||
return adoptCommand(ctx, args[1:])
|
||||
case "settings":
|
||||
return settingsCommand(ctx, args[1:])
|
||||
case "config":
|
||||
return configCommand(ctx, args[1:])
|
||||
case "secret":
|
||||
return secretCommand(ctx, args[1:])
|
||||
case "operator":
|
||||
@@ -381,6 +383,13 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
// **A merge check builds nothing** (novox/hq to-be 45 §9): its verdict is said, and nothing of it is
|
||||
// recorded or registered.
|
||||
if result.Check != nil || result.Checked != nil {
|
||||
if isTry(result) {
|
||||
// A branch tried before its pull request (novox/hq issue 686): its asker reads the outcome from the
|
||||
// build's log. Never said as `checked`, so the forge's holder sets no status.
|
||||
fmt.Printf("%s: a try of %s at %.8s, read with `try --id`: said to nobody, nothing taken in\n", result.ID,
|
||||
result.Repository, result.Ref)
|
||||
return nil
|
||||
}
|
||||
checked(ctx, result)
|
||||
return nil
|
||||
}
|
||||
@@ -431,8 +440,14 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
}
|
||||
fmt.Printf("%s: %s %s registered, built on %s from %s\n",
|
||||
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
|
||||
// What the walk's build did, kept on the walk whose ask it answers (novox/hq ADR 0350 §4); a build seat from
|
||||
// before it says nothing, and that is what is kept.
|
||||
made := result.MadeWords()
|
||||
if result.Reused != "" || result.BuiltWhy != "" {
|
||||
fmt.Printf("%s: %s %s\n", result.ID, manifest.Module, made)
|
||||
}
|
||||
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
|
||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
||||
planBuiltSaying(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID, made)
|
||||
// A module registered may be one a machine is now behind: `status` is composed again.
|
||||
statusFrom.nudge()
|
||||
return nil
|
||||
|
||||
@@ -296,24 +296,6 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error)
|
||||
}
|
||||
v.Failures = append(v.Failures, failures...)
|
||||
}
|
||||
// The list of predefined keys a head's catalogue tree carries, which the change composes with
|
||||
// (novox/hq issue 630); the mesh's own list otherwise.
|
||||
var changedList catalogue.Predefined
|
||||
for _, h := range heads {
|
||||
if h.Tree == "" {
|
||||
continue
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(h.Tree, catalogue.PredefinedFile))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
list, err := catalogue.ParsePredefined(raw)
|
||||
if err != nil {
|
||||
v.Failures = append(v.Failures, err.Error())
|
||||
continue
|
||||
}
|
||||
changedList = list
|
||||
}
|
||||
if len(in.group) > 0 {
|
||||
say("a delivery group: %d head(s) composed together", len(heads))
|
||||
}
|
||||
@@ -337,7 +319,7 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error)
|
||||
|
||||
// Both compositions, each in a store of its own.
|
||||
say("composing every machine as the mesh is")
|
||||
before, _, notes, err := composeEveryMachine(ctx, in, base, sources, nil, nil)
|
||||
before, _, notes, err := composeEveryMachine(ctx, in, base, sources, nil)
|
||||
if err != nil {
|
||||
return v, fmt.Errorf("the mesh as the snapshot says it is could not be raised: %w", err)
|
||||
}
|
||||
@@ -350,7 +332,7 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error)
|
||||
}
|
||||
}
|
||||
sort.Strings(added)
|
||||
after, tried, notes, err := composeEveryMachine(ctx, in, change, sources, added, changedList)
|
||||
after, tried, notes, err := composeEveryMachine(ctx, in, change, sources, added)
|
||||
if err != nil {
|
||||
return v, fmt.Errorf("the mesh with the change could not be raised: %w", err)
|
||||
}
|
||||
@@ -766,13 +748,13 @@ func standInBuild(m catalogue.Manifest) (catalogue.Manifest, error) {
|
||||
// grant to compose — and answers each machine's composition. Notes are what of the snapshot could not
|
||||
// be raised: a refusal to register a module, keep a setting or a pin.
|
||||
func composeEveryMachine(ctx context.Context, in mergeCheckInput, shelf map[string]catalogue.Manifest,
|
||||
sources map[string]inventory.Source, trials []string, list catalogue.Predefined) (map[string]mergeComposed, map[string]mergeTrial, []string, error) {
|
||||
sources map[string]inventory.Source, trials []string) (map[string]mergeComposed, map[string]mergeTrial, []string, error) {
|
||||
open, drop, err := throwawayStores(ctx, in.admin)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
defer drop()
|
||||
notes, err := raiseFromFacts(ctx, open, in.facts, shelf, sources, list)
|
||||
notes, err := raiseFromFacts(ctx, open, in.facts, shelf, sources)
|
||||
if err != nil {
|
||||
return nil, nil, notes, err
|
||||
}
|
||||
@@ -968,22 +950,9 @@ func throwawayStores(ctx context.Context, admin string) (*stores, func(), error)
|
||||
// for each standing in for the one it holds, its assignments, the seats it holds, its pins and its
|
||||
// settings, and a stand-in for every secret a person gave.
|
||||
func raiseFromFacts(ctx context.Context, open *stores, f snapshot.Facts, shelf map[string]catalogue.Manifest,
|
||||
sources map[string]inventory.Source, list catalogue.Predefined) ([]string, error) {
|
||||
sources map[string]inventory.Source) ([]string, error) {
|
||||
inv := open.inventory
|
||||
var notes []string
|
||||
// The list of predefined keys before any module, which registration judges its preferences against
|
||||
// (novox/hq issue 630): the change's own when its catalogue tree carries one, the mesh's otherwise.
|
||||
if list == nil && len(f.Predefined) > 0 {
|
||||
list = catalogue.Predefined{}
|
||||
for key, k := range f.Predefined {
|
||||
list[key] = catalogue.PredefinedKey{Type: k.Type, Why: k.Why}
|
||||
}
|
||||
}
|
||||
if len(list) > 0 {
|
||||
if err := inv.SetPredefined(ctx, list, "", ""); err != nil {
|
||||
notes = append(notes, fmt.Sprintf("the list of predefined keys is not kept: %s", oneLine(err.Error())))
|
||||
}
|
||||
}
|
||||
names := make([]string, 0, len(shelf))
|
||||
for name := range shelf {
|
||||
names = append(names, name)
|
||||
@@ -1076,20 +1045,6 @@ func raiseFromFacts(ctx context.Context, open *stores, f snapshot.Facts, shelf m
|
||||
}
|
||||
}
|
||||
}
|
||||
// Mesh and node config, as the mesh holds them (novox/hq issue 630).
|
||||
for _, key := range sortedAnyKeys(f.Config) {
|
||||
if err := inv.SetConfig(ctx, key, "", f.Config[key], "the facts snapshot", "raised from the facts snapshot"); err != nil {
|
||||
notes = append(notes, fmt.Sprintf("the mesh config of %s is not kept: %s", key, oneLine(err.Error())))
|
||||
}
|
||||
}
|
||||
for _, m := range f.Machines {
|
||||
for _, key := range sortedAnyKeys(m.Config) {
|
||||
if err := inv.SetConfig(ctx, key, m.Name, m.Config[key], "the facts snapshot", "raised from the facts snapshot"); err != nil {
|
||||
notes = append(notes, fmt.Sprintf("%s's node config of %s is not kept: %s", m.Described(), key,
|
||||
oneLine(err.Error())))
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, s := range f.Settings {
|
||||
if err := inv.KeepSettings(ctx, "", s.Module, standInPaths(s.Values)); err != nil {
|
||||
notes = append(notes, fmt.Sprintf("the mesh's settings of %s are not kept: %s", s.Module, oneLine(err.Error())))
|
||||
@@ -1357,13 +1312,3 @@ func firstOr(items, otherwise []string) string {
|
||||
}
|
||||
return "it says nothing more"
|
||||
}
|
||||
|
||||
// sortedAnyKeys is a map's keys, sorted.
|
||||
func sortedAnyKeys(in map[string]any) []string {
|
||||
keys := make([]string, 0, len(in))
|
||||
for k := range in {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
@@ -240,6 +240,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
continue
|
||||
}
|
||||
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
||||
declaredSeverity(&o, hold, m)
|
||||
if hold != nil {
|
||||
if by, held := hold.heldWith(node, m, unhealthy[m]); held {
|
||||
// Held under the provider's condition: nothing of its own, and the provider's says it waits. The
|
||||
@@ -347,6 +348,7 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata
|
||||
} else {
|
||||
_, rest := besideAWait(p.Module, rs)
|
||||
o = moduleUnhealthyObservation(p.Module, p.Node, rest)
|
||||
declaredSeverity(&o, hold, p.Module)
|
||||
if said, waits := personWait(p.Module, p.Node, rest); waits {
|
||||
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rest)
|
||||
}
|
||||
@@ -481,6 +483,19 @@ func sayWaitingOn(o *conditions.Observation, waiters []string) {
|
||||
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
|
||||
}
|
||||
|
||||
// declaredSeverity raises a module's unhealthy condition urgent when its manifest declares `unhealthy` urgent
|
||||
// (novox/hq ADR 0355 §1). Only that condition changes (ADR 0355 §4): a wait for a new login, a module that needs
|
||||
// the operator and a directory used as found keep their severities, and without the field, or without a manifest
|
||||
// to read it from, it stays the warning it was.
|
||||
func declaredSeverity(o *conditions.Observation, hold *holding, module string) {
|
||||
if o.Kind != kindModuleUnhealthy || hold == nil {
|
||||
return
|
||||
}
|
||||
if m, ok := hold.manifestOf(module); ok && m.UnhealthyIsUrgent() {
|
||||
o.Severity = conditions.Urgent
|
||||
}
|
||||
}
|
||||
|
||||
// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
|
||||
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
|
||||
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
|
||||
|
||||
@@ -571,14 +571,18 @@ func settingsCommand(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
m, ok := known[positionals[0]]
|
||||
if !ok || len(catalogue.PreferencesOf(m)) == 0 {
|
||||
if !ok || len(m.Settings) == 0 {
|
||||
return nil
|
||||
}
|
||||
var layers []catalogue.Layer
|
||||
if *node != "" {
|
||||
// Every source on the node: mesh config, node config, then the module's own (novox/hq issue 630).
|
||||
if layers, err = inv.SettingsFor(ctx, *node, positionals[0]); err != nil {
|
||||
if mesh, has, err := inv.Layer(ctx, "", positionals[0]); err != nil {
|
||||
return err
|
||||
} else if has {
|
||||
layers = append(layers, catalogue.Layer{From: catalogue.MeshWideLayer, Values: mesh})
|
||||
}
|
||||
if has {
|
||||
layers = append(layers, catalogue.Layer{From: *node, Values: values})
|
||||
}
|
||||
} else if has {
|
||||
layers = append(layers, catalogue.Layer{From: catalogue.MeshWideLayer, Values: values})
|
||||
@@ -608,7 +612,7 @@ func settingsCommand(ctx context.Context, args []string) error {
|
||||
var listed []preferencesOf
|
||||
for _, e := range entries {
|
||||
m := e.Manifest
|
||||
if len(catalogue.PreferencesOf(m)) == 0 || (only != "" && m.Module != only) {
|
||||
if len(m.Settings) == 0 || (only != "" && m.Module != only) {
|
||||
continue
|
||||
}
|
||||
if *node != "" && !containsString(e.On, *node) {
|
||||
@@ -678,11 +682,7 @@ func describeEffective(module, where string, values []catalogue.SettingSource) s
|
||||
fmt.Fprintf(&b, "%s on %s, every value and where it comes from:\n", module, where)
|
||||
for _, v := range values {
|
||||
value, _ := json.Marshal(v.Value)
|
||||
from := v.From
|
||||
if v.From == where && where != "the whole mesh" {
|
||||
from = valueSource(module, where, v)
|
||||
}
|
||||
fmt.Fprintf(&b, " %s = %s (%s", v.Key, value, from)
|
||||
fmt.Fprintf(&b, " %s = %s (%s", v.Key, value, v.From)
|
||||
if v.HasDefault && !v.FromDefault {
|
||||
d, _ := json.Marshal(v.Default)
|
||||
fmt.Fprintf(&b, "; the default is %s", d)
|
||||
@@ -723,27 +723,28 @@ func describePreferences(modules []preferencesOf) string {
|
||||
on = "on " + strings.Join(p.Nodes, ", ")
|
||||
}
|
||||
fmt.Fprintf(&b, "%s (%s)\n", p.Manifest.Module, on)
|
||||
declared := catalogue.PreferencesOf(p.Manifest)
|
||||
keys := make([]string, 0, len(declared))
|
||||
for k := range declared {
|
||||
keys := make([]string, 0, len(p.Manifest.Settings))
|
||||
for k := range p.Manifest.Settings {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
d := declared[k]
|
||||
d := p.Manifest.Settings[k]
|
||||
def, _ := json.Marshal(d.Default)
|
||||
falls := ""
|
||||
if d.Predefined != "" {
|
||||
falls = ", falls back to the predefined key " + d.Predefined
|
||||
}
|
||||
fmt.Fprintf(&b, " %s, default %s%s: %s\n", k, def, falls, d.Why)
|
||||
fmt.Fprintf(&b, " %s, default %s: %s\n", k, def, d.Why)
|
||||
for _, n := range p.Nodes {
|
||||
for _, s := range p.On[n] {
|
||||
if s.Key != k {
|
||||
continue
|
||||
}
|
||||
v, _ := json.Marshal(s.Value)
|
||||
fmt.Fprintf(&b, " %s: %s (%s)\n", n, v, valueSource(p.Manifest.Module, n, s))
|
||||
from := s.From
|
||||
if s.FromDefault {
|
||||
from = catalogue.DefaultLayer
|
||||
} else if s.From != catalogue.MeshWideLayer {
|
||||
from = "the node"
|
||||
}
|
||||
fmt.Fprintf(&b, " %s: %s (%s)\n", n, v, from)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -751,18 +752,6 @@ func describePreferences(modules []preferencesOf) string {
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// valueSource is where a value on a node comes from, as the four sources are said (novox/hq issue 630):
|
||||
// "default", "mesh config", "node config (<node>)", or "<module> on <node>" for the module's own value.
|
||||
func valueSource(module, node string, s catalogue.SettingSource) string {
|
||||
switch {
|
||||
case s.FromDefault:
|
||||
return catalogue.DefaultLayer
|
||||
case s.From == catalogue.MeshConfigLayer, s.From == catalogue.NodeConfigLayer(node), s.From == catalogue.MeshWideLayer:
|
||||
return s.From
|
||||
}
|
||||
return module + " on " + node
|
||||
}
|
||||
|
||||
// nodeFlag is ` --node <node>` for a machine's layer, nothing for the whole mesh's.
|
||||
func nodeFlag(node string) string {
|
||||
if node == "" {
|
||||
|
||||
@@ -684,7 +684,7 @@ func artifactStoreOnNetwork(ctx context.Context, inv *inventory.Inventory,
|
||||
if !offers {
|
||||
continue
|
||||
}
|
||||
serves, err := servedOnNode(ctx, inv, machine, m, catalogue.ArtifactStoreProvision)
|
||||
serves, _, err := servedOnNode(ctx, inv, machine, m, catalogue.ArtifactStoreProvision)
|
||||
if err != nil {
|
||||
return "", "", false, fmt.Errorf("reading where %s puts the artifact store: %w", machine, err)
|
||||
}
|
||||
|
||||
@@ -88,7 +88,7 @@ func checkWait(module, machine string, w inventory.Wait, f operatorWaitFacts) er
|
||||
case w.Provision != "":
|
||||
return checkProvisionWait(module, machine, w, m, f)
|
||||
case w.Setting != "":
|
||||
if !catalogue.DeclaresPreference(m, w.Setting) {
|
||||
if _, declared := m.Settings[w.Setting]; !declared {
|
||||
return fmt.Errorf("says it waits for the setting %s, which %s does not declare", w.Setting, module)
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -30,6 +30,7 @@ func userUnit(module, account string) inventory.ResourceHealth {
|
||||
// Only what depends on the new login alone is a wait; anything else unhealthy of the module is judged as
|
||||
// before, so that a fault is never excused.
|
||||
func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
notInGroup := relogin("lights", "operator")
|
||||
notInGroup.Reason = "not in the group lights: the apply has not put operator there; its outcome says why"
|
||||
systemUnit := userUnit("lights", "operator")
|
||||
@@ -74,6 +75,7 @@ func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) {
|
||||
// The gate reads a wait for a person as its own reading, never a fault, and passes with it; the same module
|
||||
// with a container down beside it is not yet healthy, as before.
|
||||
func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
now := time.Now()
|
||||
since := now.Add(-time.Minute)
|
||||
account := relogin("lights", "operator")
|
||||
@@ -102,6 +104,7 @@ func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) {
|
||||
// The module's condition says the wait in one sentence for a person, and clears on the first statement that
|
||||
// no longer says it — said as the module's own fault when its unit still fails after the new login.
|
||||
func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
k, _ := withConditionsInMemory(t)
|
||||
@@ -158,6 +161,7 @@ func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) {
|
||||
// A module that is healthy on its own keeps its pass when another module of the same send fails at the
|
||||
// bound: it is neither put back nor left without a verdict for every other walk to wait on.
|
||||
func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
@@ -207,6 +211,7 @@ func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) {
|
||||
// What the operator reads of the wait (ADR 0253, ADR 0254): it needs them, so it is never quiet, and it offers
|
||||
// no button, since nothing but their own new login can do it.
|
||||
func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{
|
||||
relogin("openrazer", "operator"), userUnit("openrazer", "operator")})
|
||||
plainExample(t, o, "openrazer waits for a new login on g14",
|
||||
@@ -238,6 +243,7 @@ func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) {
|
||||
// When a module not working turns into one waiting for a new login, and back, the clearing line says what it
|
||||
// became, never that it works again (issue 318 review).
|
||||
func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
store := conditions.NewInMemory()
|
||||
@@ -286,6 +292,7 @@ func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) {
|
||||
|
||||
// Which moves add an account group, read from the builds' manifests (issue 318 review).
|
||||
func TestOnlyAMoveThatAddsAnAccountGroupCanBringAWait(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
user := func(groups ...any) catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "lights", Resources: []map[string]any{{"id": "operator", "type": "user",
|
||||
"name": "operator", "groups": groups}}}
|
||||
|
||||
@@ -456,6 +456,9 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
"release-held": worded(func(o conditions.Observation) words {
|
||||
return releaseHeldWords(nil, o.Also)
|
||||
}),
|
||||
"release-silent": worded(func(o conditions.Observation) words {
|
||||
return releaseSilentWords(o.Also)
|
||||
}),
|
||||
"facts-stale": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Merge checks use outdated facts",
|
||||
Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " +
|
||||
@@ -617,6 +620,18 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
Explanation: "The bus reports a listener too slow to keep up, so messages to it are late.",
|
||||
Resolved: "The listener keeps up again"}
|
||||
}),
|
||||
kindBusFaultsUnread: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Slow clients of the bus cannot be checked",
|
||||
Explanation: "The controller could not read what the bus said about its clients' connections, so a " +
|
||||
"client the bus cut off would go unnoticed. It tries again at every check.",
|
||||
Resolved: "Resolved: the bus's clients are checked again"}
|
||||
}),
|
||||
"connection-closed": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "The bus cut a client off",
|
||||
Explanation: "The bus closed a client's connection because it broke the bus's rules, so what it was " +
|
||||
"sending was lost.",
|
||||
Resolved: "Resolved: the client keeps to the bus's rules"}
|
||||
}),
|
||||
kindBusReconnects: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A client keeps losing the bus",
|
||||
Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. " +
|
||||
@@ -841,6 +856,19 @@ func releaseHeldWords(modules, machines []string) words {
|
||||
Resolved: "Resolved: the held updates are released"}
|
||||
}
|
||||
|
||||
// releaseSilentWords are the plain words of machines that said nothing on an update they were sent (ADR 0353 §4).
|
||||
func releaseSilentWords(machines []string) words {
|
||||
where := "Some machines"
|
||||
if len(machines) > 0 {
|
||||
where = conditions.Capital(namesWords(machines, 4))
|
||||
}
|
||||
return words{Headline: "Machines did not answer an update",
|
||||
Needs: "check that they are awake and on the mesh; the next walk sends them what waits.",
|
||||
Explanation: where + " were sent module updates and said nothing about them within fifteen minutes, so " +
|
||||
"they were left as they were. What they run is not known to be the update.",
|
||||
Resolved: "Resolved: the next walk reached them"}
|
||||
}
|
||||
|
||||
// reloginNeeds is what an account waiting for its groups needs (ADR 0252).
|
||||
func reloginNeeds(node string) string {
|
||||
return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node)
|
||||
|
||||
@@ -289,9 +289,14 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
||||
continue
|
||||
}
|
||||
caps, _ := inv.ProfileOf(ctx, n.Name)
|
||||
// Its public domain too (novox/hq ADR 0346): a provider elsewhere serves its own composed names,
|
||||
// and composes them as its own plan does, under its own domain.
|
||||
// Read as its profile is: a machine whose domain cannot be read (removed since the list was read)
|
||||
// composes no public name, which leaves the key out and says so, and fails no other machine's plan.
|
||||
domain, _ := inv.PublicDomainOf(ctx, n.Name)
|
||||
others = append(others, candidate{
|
||||
catalogue.Node{Name: n.Name, Site: siteOf[n.Name], Capabilities: caps,
|
||||
At: onNetwork[n.Name]}, theirs})
|
||||
At: onNetwork[n.Name], PublicDomain: domain}, theirs})
|
||||
}
|
||||
|
||||
offered := map[string][]catalogue.Provider{}
|
||||
@@ -310,12 +315,18 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
||||
// What that module says a consumer needs to know, with that node's settings on
|
||||
// it: a port somebody moved on the provider is a port its consumers must be told
|
||||
// about, and the two coming from different places is how they come to disagree.
|
||||
serves, err := servedOnNode(ctx, inv, o.node.Name, m, name)
|
||||
serves, layers, err := servedOnNode(ctx, inv, o.node.Name, m, name)
|
||||
if err != nil {
|
||||
return catalogue.World{}, err
|
||||
}
|
||||
// And its own composed names, as that node composes them (novox/hq ADR 0346): a name it
|
||||
// does not compose leaves the key out, said under each consumer's need.
|
||||
serves, leftOut, err := got.ServedOwnNames(m, name, serves, catalogue.WithDefaults(m, layers))
|
||||
if err != nil {
|
||||
return catalogue.World{}, err
|
||||
}
|
||||
offered[name] = append(offered[name], catalogue.Provider{
|
||||
Node: o.node.Name, At: o.node.At, Serves: serves, Module: m.Module})
|
||||
Node: o.node.Name, At: o.node.At, Serves: serves, Module: m.Module, LeftOut: leftOut})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -997,9 +1008,14 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
readsBusy, err := engineReadsBusy(ctx, inv, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
return catalogue.Rendering{
|
||||
ReadsHealth: readsHealth,
|
||||
JudgesRoot: judgesRoot,
|
||||
ReadsBusy: readsBusy,
|
||||
BusMembership: memberships[node],
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports,
|
||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||
@@ -1033,6 +1049,35 @@ func engineJudgesRoot(ctx context.Context, inv *inventory.Inventory, node string
|
||||
return had && stated.Contract >= link.RootContract, nil
|
||||
}
|
||||
|
||||
// engineReadsBusy says whether a machine's node-engine reads a container's `busy` (novox/hq ADR 0335), by its
|
||||
// own newest statement, for the same reason as engineReadsHealth: an older engine parses strictly.
|
||||
func engineReadsBusy(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) {
|
||||
stated, had, err := inv.HealthOf(ctx, node)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return had && stated.Contract >= link.BusyContract, nil
|
||||
}
|
||||
|
||||
// busyNotSent is, for a node-engine that does not read a container's busy, each container whose module declares
|
||||
// one: not sent it, so replaced at once while it may hold work (novox/hq ADR 0335). Nothing for one that reads it.
|
||||
func busyNotSent(modules []catalogue.Manifest, reads bool) []string {
|
||||
if reads {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, m := range modules {
|
||||
for _, r := range m.Resources {
|
||||
if _, has := r[catalogue.BusyField]; has {
|
||||
out = append(out, fmt.Sprintf("%s.%v declares busy, and this node-engine does not read it yet (its "+
|
||||
"statement's contract is below %d): not sent, so the container is replaced at once, busy or not "+
|
||||
"(novox/hq ADR 0335)", m.Module, r["id"], link.BusyContract))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR
|
||||
// 0199): the zone settled from that node's settings, the node's private address, the port the
|
||||
// answering listen is published on there.
|
||||
@@ -1426,10 +1471,29 @@ func planCommand(ctx context.Context, args []string) error {
|
||||
// in this output would have told anybody that.
|
||||
for _, n := range plan.Needs {
|
||||
fmt.Printf(" needs %s from %s, for %s\n", n.Name, n.From, n.For)
|
||||
// And a served key the provider left out, because its machine composes no name it asks for
|
||||
// (novox/hq ADR 0346 §3).
|
||||
for _, line := range n.LeftOut {
|
||||
fmt.Printf(" %s\n", line)
|
||||
}
|
||||
}
|
||||
// What the providers on this machine leave out of what they serve, and why: said where the remedy
|
||||
// is (novox/hq ADR 0346 §3).
|
||||
for _, line := range plan.ServedLeftOut(settings) {
|
||||
fmt.Printf("%s: %s\n", args[0], line)
|
||||
}
|
||||
for _, line := range unboundLines(plan) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
// And a container's busy that this machine's node-engine is too old to be sent (novox/hq ADR 0335): its
|
||||
// container is replaced at once, as before. A statement that cannot be read is said as such.
|
||||
if reads, err := engineReadsBusy(ctx, open.inventory, args[0]); err != nil {
|
||||
fmt.Printf("%s: whether its node-engine reads a container's busy could not be read: %v\n", args[0], err)
|
||||
} else {
|
||||
for _, line := range busyNotSent(plan.Modules, reads) {
|
||||
fmt.Printf("%s: %s\n", args[0], line)
|
||||
}
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, args[0], plan, settings)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1566,19 +1630,20 @@ func managerPublicKeyFor(
|
||||
// artifact store's trust and the references composed through it (04-ISSUES/102). Unreadable
|
||||
// given ports are that node's refusal to report, not this reader's.
|
||||
func servedOnNode(ctx context.Context, inv *inventory.Inventory, node string,
|
||||
m catalogue.Manifest, provision string) (map[string]any, error) {
|
||||
m catalogue.Manifest, provision string) (map[string]any, []catalogue.Layer, error) {
|
||||
ports, layers, err := portsGivenOn(ctx, inv, node, m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
serves := catalogue.ServedOn(m, provision, ports)
|
||||
if len(serves) > 0 {
|
||||
serves, err = catalogue.Settle(serves, catalogue.WithDefaults(m, layers))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
return serves, nil
|
||||
// The layers too, read once, for what is filled after (novox/hq ADR 0346).
|
||||
return serves, layers, nil
|
||||
}
|
||||
|
||||
// portsGivenOn is where a node puts a module's ports — assigned, then given over them — and the
|
||||
|
||||
@@ -133,7 +133,7 @@ func TestASharedRepositoryIsPlannedFromTheRecordedBuildSources(t *testing.T) {
|
||||
{"the route proxy's program: the route proxy alone", "mesh-controller",
|
||||
[]string{"examples/route-proxy/main.go"}, "route-proxy", true},
|
||||
{"a package all three build from: all three", "mesh-controller",
|
||||
[]string{"internal/broker/broker.go"}, "mesh-controller | build-agent | route-proxy", false},
|
||||
[]string{"internal/broker/broker.go"}, "mesh-controller,route-proxy | build-agent", false},
|
||||
{"the route proxy's directory in the catalogue: it alone", "mesh-catalog",
|
||||
[]string{"modules/route-proxy/module.json"}, "route-proxy", false},
|
||||
{"the build agent's directory: it alone, nothing it builds", "mesh-catalog",
|
||||
|
||||
@@ -145,8 +145,9 @@ func TestAPlanIsWhatTheChangeTouchedAndWhatIsBuiltOnIt(t *testing.T) {
|
||||
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a,x"},
|
||||
{what: "built-by never widens", edges: []inventory.Edge{dep("x", builtBy, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a"},
|
||||
{what: "built-by orders", edges: []inventory.Edge{dep("x", builtBy, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a | x"},
|
||||
// Whichever builder runs builds it (novox/hq ADR 0360 §3): built-by orders nothing, as before it ordered.
|
||||
{what: "built-by does not order", edges: []inventory.Edge{dep("x", builtBy, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a,x"},
|
||||
{what: "worker-of never widens", edges: []inventory.Edge{dep("x", workerOf, "a")},
|
||||
repo: "one", paths: []string{"modules/a/x"}, want: "a"},
|
||||
{what: "worker-of orders", edges: []inventory.Edge{dep("x", workerOf, "a")},
|
||||
@@ -296,18 +297,18 @@ func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) {
|
||||
[]string{"internal/inventory/migrations/0088-a-build-says-its-build-source.sql"}, read,
|
||||
"mesh-controller | build-agent", ""},
|
||||
// A package all three build from: all three; the build seat's holder after the controller whose worker
|
||||
// it binds (worker-of, issue 206), the proxy after the holder that builds it (built-by).
|
||||
// it binds (worker-of, issue 206), the proxy beside the controller: built-by orders nothing (ADR 0360 §3).
|
||||
{"a package all three build from: all three", "mesh-controller",
|
||||
[]string{"internal/broker/broker.go"}, read, "mesh-controller | build-agent | route-proxy", ""},
|
||||
[]string{"internal/broker/broker.go"}, read, "mesh-controller,route-proxy | build-agent", ""},
|
||||
{"a vendored package all three build from: all three", "mesh-controller",
|
||||
[]string{"vendor/github.com/nats-io/nats.go/nats.go"}, read, "mesh-controller | build-agent | route-proxy", ""},
|
||||
[]string{"vendor/github.com/nats-io/nats.go/nats.go"}, read, "mesh-controller,route-proxy | build-agent", ""},
|
||||
{"go.sum: all three", "mesh-controller",
|
||||
[]string{"go.sum"}, read, "mesh-controller | build-agent | route-proxy", ""},
|
||||
[]string{"go.sum"}, read, "mesh-controller,route-proxy | build-agent", ""},
|
||||
{"a file added to the proxy's package: the proxy", "mesh-controller",
|
||||
[]string{"examples/route-proxy/new.go"}, read, "route-proxy", ""},
|
||||
// Before any build source is said: as before.
|
||||
{"no build source said: a README moves all three, as before", "mesh-controller",
|
||||
[]string{"README.md"}, unsaid, "mesh-controller | build-agent | route-proxy", ""},
|
||||
[]string{"README.md"}, unsaid, "mesh-controller,route-proxy | build-agent", ""},
|
||||
// In the catalogue, where they live, each by its own build source.
|
||||
{"the route proxy's recipe: it alone", "mesh-catalog",
|
||||
[]string{"modules/route-proxy/Dockerfile"}, read, "route-proxy", ""},
|
||||
@@ -331,7 +332,7 @@ func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) {
|
||||
// Files not all said: everything the repository builds, as before.
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Commit: "head",
|
||||
Paths: []string{"README.md"}, PathsTruncated: true}
|
||||
if got := tiered(reachOfMerge(m, entries, read, edges).Plan.Tiers); got != "mesh-controller | build-agent | route-proxy" {
|
||||
if got := tiered(reachOfMerge(m, entries, read, edges).Plan.Tiers); got != "mesh-controller,route-proxy | build-agent" {
|
||||
t.Errorf("a merge whose files were not all said: planned %q, wanted all three", got)
|
||||
}
|
||||
}
|
||||
@@ -452,8 +453,9 @@ func TestAPlanIsTheTouchedModulesAndWhatIsReachableAlongTheWideningEdges(t *test
|
||||
kinds := []string{inventory.EdgeStandsOn, inventory.EdgeDeclared, inventory.EdgePackages,
|
||||
inventory.EdgeBuiltBy, inventory.EdgeWorkerOf}
|
||||
widens := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true}
|
||||
// Built-by orders nothing since novox/hq ADR 0360 §3.
|
||||
orders := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true,
|
||||
inventory.EdgeBuiltBy: true, inventory.EdgeWorkerOf: true}
|
||||
inventory.EdgeWorkerOf: true}
|
||||
// Directory names drawn from one pool, so two repositories hold directories of the same name, and one
|
||||
// is a prefix of another.
|
||||
dirs := []string{"a", "ab", "b", "c", "lib/x", "lib/y", "modules/a", "modules/a/sub"}
|
||||
|
||||
@@ -211,7 +211,7 @@ func (r *rootReader) executeServed(ctx context.Context, machine string) (bool, s
|
||||
}
|
||||
holders++
|
||||
var setting *any
|
||||
if catalogue.DeclaresPreference(e.Manifest, loginShellVerb) {
|
||||
if _, declared := e.Manifest.Settings[loginShellVerb]; declared {
|
||||
layers, err := r.settings(ctx, machine, e.Manifest.Module)
|
||||
if err != nil {
|
||||
served = true
|
||||
|
||||
@@ -18,8 +18,9 @@ package main
|
||||
// it beside the layer (`settings` says it back), and records the warrant in the hand-act log on the bus, where a
|
||||
// person's decisions are read; the router edits the ask on every channel to its outcome. No seat event of its
|
||||
// own: nothing consumes one, and the installer's first user list in the node-engine's repository names every
|
||||
// controller event, so one would cost a node-engine change for a fact without a reader. The push afterwards is a
|
||||
// separate act, as it is for a layer set at the terminal.
|
||||
// controller event, so one would cost a node-engine change for a fact without a reader. No push is left for
|
||||
// afterwards: the warrant that sets the layer opens the walk that sends it (novox/hq ADR 0344, settings_walk.go),
|
||||
// so the operator's one Approve is the act that makes the change take effect.
|
||||
//
|
||||
// **What the operator reads** is the module, the machine, each key with its exact new value and, for a changed
|
||||
// key, the value it replaces. A value that may not leave the mesh (an address, a path, a secret's shape: the
|
||||
@@ -27,6 +28,13 @@ package main
|
||||
// ask says so, and the whole is read with `settings proposals <id>` — whose fingerprint must be the one on the
|
||||
// phone. Fail closed: a proposal nothing can carry to the operator is refused at once, in words, and never left
|
||||
// waiting for an answer that cannot come.
|
||||
//
|
||||
// **On a channel that proves who answers, the values are shown WHOLE** (novox/hq issue 383, the operator's
|
||||
// decision of 2026-10-10): a mount point, a share name or a private address is the thing being approved and must
|
||||
// be readable, so the ask carries them whole beside the explanation (asks.Ask.Whole), the router shows that only
|
||||
// on a kind that verifies its sender, and only a value shaped like a secret is withheld there. The message is the
|
||||
// headline, one line per key, who proposed it and when; the fingerprint and how to read the proposal whole are
|
||||
// the Details answer's (asks.Ask.Details). The masking stays for conditions and for channels that prove nothing.
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -47,6 +55,7 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/outward"
|
||||
)
|
||||
@@ -154,47 +163,48 @@ func (p settingsProposal) ask(id string, machines []string) (asks.Ask, map[strin
|
||||
if len([]rune(headline)) > asks.HeadlineLength {
|
||||
headline = verb + " settings?"
|
||||
}
|
||||
shown, whole := p.change(machines)
|
||||
var b strings.Builder
|
||||
if p.Clear {
|
||||
fmt.Fprintf(&b, "Clear the settings of %s on %s, back to what the module says?\n\n", p.Module, p.where())
|
||||
} else {
|
||||
fmt.Fprintf(&b, "Set the settings of %s on %s to these values?\n\n", p.Module, p.where())
|
||||
// The message reads at a glance (issues 383 and 639): a summary of what changes where, then at most three
|
||||
// points, a key each; every key with its whole value, who proposed it, the fingerprint and how to read it at the
|
||||
// terminal are its Details, which the router notes after the answers.
|
||||
shown, shownAll, shownWhole := p.change(machines, false)
|
||||
whole, wholeAll, _ := p.change(machines, true)
|
||||
summary := p.summary()
|
||||
compose := func(points []string) string {
|
||||
return strings.Join(append([]string{summary}, points...), "\n")
|
||||
}
|
||||
fmt.Fprintf(&b, "Proposed by %s, at %s. ", sayable(p.From, machines), p.At.Local().Format("15:04 on 2 Jan"))
|
||||
switch {
|
||||
case p.Clear && p.HadLayer:
|
||||
b.WriteString("The layer it removes:\n\n")
|
||||
case p.Clear:
|
||||
b.WriteString("There is no layer to remove; approving changes nothing.")
|
||||
case !p.HadLayer:
|
||||
b.WriteString("There is no layer yet; this is the whole of it:\n\n")
|
||||
default:
|
||||
b.WriteString("The layer is replaced whole; what changes against it:\n\n")
|
||||
// The Details: every setting it sets, one line each, uncut — whole where the sender is proven (WholeDetails),
|
||||
// masked elsewhere — then who proposed it, the fingerprint and how to read it at the terminal.
|
||||
details := func(lines []string) string {
|
||||
var d strings.Builder
|
||||
d.WriteString(strings.Join(lines, "\n") + "\n")
|
||||
d.WriteString(p.onApprove() + "\n")
|
||||
fmt.Fprintf(&d, "Proposed by %s at %s. Fingerprint %s.\n", sayable(p.From, machines), p.At.Local().Format("15:04 on 2 Jan"),
|
||||
fingerprint(p.Digest))
|
||||
if !shownWhole {
|
||||
d.WriteString("Where who answers is not proven, its values are shown as ‹address›, ‹path› or ‹withheld›.\n")
|
||||
}
|
||||
fmt.Fprintf(&d, "Read it whole, with this fingerprint: settings proposals %s at the controller's terminal, or the "+
|
||||
"controller's settings verb with proposal %s through the mesh MCP server.", id, id)
|
||||
return d.String()
|
||||
}
|
||||
b.WriteString(shown)
|
||||
fmt.Fprintf(&b, "\n\nFingerprint %s.", fingerprint(p.Digest))
|
||||
if !whole {
|
||||
b.WriteString(" Parts shown as ‹address›, ‹path› or ‹withheld› may not leave the mesh: read it whole, with " +
|
||||
"this fingerprint, through the mesh MCP server (mesh-controller.settings, proposal " + id + ") or with " +
|
||||
"mesh-cli settings proposals " + id + ".")
|
||||
}
|
||||
if p.Clear {
|
||||
b.WriteString(" Approved, the layer is removed at once and the machine takes it at its next push.")
|
||||
} else {
|
||||
b.WriteString(" Approved, the layer is set at once and the machine takes it at its next push.")
|
||||
}
|
||||
q := asks.Ask{ID: id, Headline: headline, Explanation: b.String(), Who: asks.Operator,
|
||||
q := asks.Ask{ID: id, Headline: headline, Explanation: compose(shown), Who: asks.Operator,
|
||||
Expires: p.At.Add(askApproveFor), OnExpiry: "the proposal is discarded; nothing changes",
|
||||
About: p.about()}
|
||||
About: p.about(), Details: details(shownAll)}
|
||||
// Only where a value was masked: an ask whose values all pass the content rule shows the same everywhere.
|
||||
if w := compose(whole); w != q.Explanation {
|
||||
q.Whole = w
|
||||
}
|
||||
if w := details(wholeAll); w != q.Details {
|
||||
q.WholeDetails = w
|
||||
}
|
||||
options := map[string]int{}
|
||||
for i, act := range p.actions(id) {
|
||||
binds, _ := asks.ActDigest(boundAct(act))
|
||||
oid := optionID(act.Label)
|
||||
options[oid] = i
|
||||
does := "the settings are set; nothing is pushed yet"
|
||||
does := "the settings are set and sent to " + p.reaches() + " at once, each machine judged"
|
||||
if p.Clear {
|
||||
does = "the layer is removed; nothing is pushed yet"
|
||||
does = "the layer is removed and that is sent to " + p.reaches() + " at once, each machine judged"
|
||||
}
|
||||
if act.Arguments["answer"] == answerDecline {
|
||||
does = "nothing changes; the proposal is discarded"
|
||||
@@ -205,55 +215,141 @@ func (p settingsProposal) ask(id string, machines []string) (asks.Ask, map[strin
|
||||
return q, options
|
||||
}
|
||||
|
||||
// shownMost is the most of a change the phone is shown, in bytes; the rest is read whole with `settings proposals`.
|
||||
const shownMost = 1400
|
||||
// summary is the proposal's first line, what changes where: how many settings, how many stay, and that Approve sends
|
||||
// the layer at once by a walk, one machine at a time (novox/hq ADR 0344); the rest of what Approve does
|
||||
// (onApprove) is the Details'.
|
||||
func (p settingsProposal) summary() string {
|
||||
if p.Clear {
|
||||
if !p.HadLayer {
|
||||
return "There is no layer to remove; approving changes nothing."
|
||||
}
|
||||
return fmt.Sprintf("Removes the %d setting(s) of %s set on %s; approved, the layer is removed and that is sent at once "+
|
||||
"by a walk to %s, one machine at a time, each judged.", len(leaves(p.Before, "")), p.Module, p.where(), p.reaches())
|
||||
}
|
||||
added, changed, removed := settingsChange(p.Before, p.Values)
|
||||
n := len(added) + len(changed) + len(removed)
|
||||
unchanged := len(leaves(p.Values, "")) - len(added) - len(changed)
|
||||
if n == 0 {
|
||||
return fmt.Sprintf("Nothing changes: the %d setting(s) of %s on %s are as they stand.", unchanged, p.Module, p.where())
|
||||
}
|
||||
stay := ""
|
||||
if unchanged > 0 {
|
||||
stay = fmt.Sprintf(", and %d stay as they are", unchanged)
|
||||
}
|
||||
return fmt.Sprintf("Changes %d setting(s) of %s on %s%s; approved, the layer is set and sent at once by a walk to %s, "+
|
||||
"one machine at a time, each judged.", n, p.Module, p.where(), stay, p.reaches())
|
||||
}
|
||||
|
||||
// change is what changes, line by line, each value shown under the content rule, and whether every value was
|
||||
// shown whole: "+ key: value" added, "~ key: value (was: old)" changed, "- key (was: old)" removed, and the
|
||||
// count of keys unchanged.
|
||||
func (p settingsProposal) change(machines []string) (string, bool) {
|
||||
// change is what changes: as at most asks.MostPoints points for the glance, as every line whole for the note, and
|
||||
// whether every value was shown exactly: "+ key: value"
|
||||
// added, "~ key: value (was: old)" changed, "- key (was: old)" removed. Each value is shown under the content rule
|
||||
// (sayableValue), or — exact, for a channel that proves who answers — as it is, withheld only when shaped like a
|
||||
// secret (wholeValue). More changes than points: the first two, and the keys of the rest; a point longer than a
|
||||
// phone shows at a glance is cut and says so. The note carries every line uncut (the review of #304: Approve sets
|
||||
// exactly what the operator can read).
|
||||
func (p settingsProposal) change(machines []string, exact bool) ([]string, []string, bool) {
|
||||
whole := true
|
||||
say := func(v any) string {
|
||||
s, w := sayableValue(v, machines)
|
||||
if exact {
|
||||
s, w = wholeValue(v, machines)
|
||||
}
|
||||
whole = whole && w
|
||||
return s
|
||||
}
|
||||
var lines []string
|
||||
type line struct{ key, text string }
|
||||
var lines []line
|
||||
if p.Clear {
|
||||
was := leaves(p.Before, "")
|
||||
for _, k := range layerKeys(was) {
|
||||
lines = append(lines, fmt.Sprintf("- %s: %s", k, say(was[k])))
|
||||
lines = append(lines, line{k, fmt.Sprintf("- %s: %s", k, say(was[k]))})
|
||||
}
|
||||
} else {
|
||||
added, changed, removed := settingsChange(p.Before, p.Values)
|
||||
was, now := leaves(p.Before, ""), leaves(p.Values, "")
|
||||
for _, k := range added {
|
||||
lines = append(lines, fmt.Sprintf("+ %s: %s", k, say(now[k])))
|
||||
lines = append(lines, line{k, fmt.Sprintf("+ %s: %s", k, say(now[k]))})
|
||||
}
|
||||
for _, k := range changed {
|
||||
lines = append(lines, fmt.Sprintf("~ %s: %s (was: %s)", k, say(now[k]), say(was[k])))
|
||||
lines = append(lines, line{k, fmt.Sprintf("~ %s: %s (was: %s)", k, say(now[k]), say(was[k]))})
|
||||
}
|
||||
for _, k := range removed {
|
||||
lines = append(lines, fmt.Sprintf("- %s (was: %s)", k, say(was[k])))
|
||||
}
|
||||
unchanged := len(now) - len(added) - len(changed)
|
||||
switch {
|
||||
case len(lines) == 0 && unchanged > 0:
|
||||
lines = append(lines, fmt.Sprintf("= nothing changes: the %d key(s) are as they stand", unchanged))
|
||||
case unchanged > 0:
|
||||
lines = append(lines, fmt.Sprintf("= %d key(s) unchanged", unchanged))
|
||||
lines = append(lines, line{k, fmt.Sprintf("- %s (was: %s)", k, say(was[k]))})
|
||||
}
|
||||
}
|
||||
out := strings.Join(lines, "\n")
|
||||
if len(out) > shownMost {
|
||||
cut := shownMost
|
||||
for cut > 0 && out[cut] != '\n' {
|
||||
cut--
|
||||
// **Each line is judged as the router will judge it** (the review of #304, ADR 0347 §1 and §3): a line the
|
||||
// content rule (masked) or the secret rule (whole) would refuse — a value given to a secret's name, say
|
||||
// "+ password: hunter2" — is said with its key still named and its value withheld, never left for the
|
||||
// router to withhold whole and unnamed. A line refused even so is the propose's refusal (judgeLines).
|
||||
for i, l := range lines {
|
||||
if lineLeaves(l.text, exact, machines) {
|
||||
continue
|
||||
}
|
||||
out = out[:cut] + fmt.Sprintf("\n… NOT SHOWN IN FULL here: %d more bytes", len(strings.Join(lines, "\n"))-cut)
|
||||
lines[i].text = l.text[:2] + l.key + " (value withheld)"
|
||||
whole = false
|
||||
}
|
||||
return out, whole
|
||||
all := make([]string, len(lines))
|
||||
for i, l := range lines {
|
||||
all[i] = l.text
|
||||
}
|
||||
// What the note holds of a value: whole where the glance is the whole one (shown only where the sender is
|
||||
// proven, with the whole details beside it), under the content rule elsewhere — so the marker says no more than
|
||||
// the note it points to (the review of ADR 0347).
|
||||
inNote := "in the note"
|
||||
if exact {
|
||||
inNote = "whole in the note"
|
||||
}
|
||||
var out []string
|
||||
for i, l := range lines {
|
||||
if len(lines) > asks.MostPoints && i == asks.MostPoints-1 {
|
||||
var rest []string
|
||||
for _, r := range lines[i:] {
|
||||
rest = append(rest, r.key)
|
||||
}
|
||||
out = append(out, fmt.Sprintf("and %d more, each %s: %s", len(rest), inNote, strings.Join(rest, ", ")))
|
||||
break
|
||||
}
|
||||
out = append(out, l.text)
|
||||
}
|
||||
// A point too long for a glance is cut, and says so: its value is the note's.
|
||||
cut := "… (cut: " + inNote + ")"
|
||||
for i, l := range out {
|
||||
if r := []rune(l); len(r) > asks.PointLength {
|
||||
out[i] = string(r[:asks.PointLength-len([]rune(cut))]) + cut
|
||||
}
|
||||
}
|
||||
return out, all, whole
|
||||
}
|
||||
|
||||
// lineLeaves says whether a line may leave the mesh as the router judges it: under the content rule for the masked
|
||||
// form, the secret rule alone for the whole one.
|
||||
func lineLeaves(line string, exact bool, machines []string) bool {
|
||||
if exact {
|
||||
_, ok := outward.Secret(line, machines...)
|
||||
return ok
|
||||
}
|
||||
_, ok := outward.Check(line, machines...)
|
||||
return ok
|
||||
}
|
||||
|
||||
// judgeLines is the first line of an ask's details the router would refuse, or withhold, and why: masked under
|
||||
// the content rule line by line, whole under the secret rule. Empty when every line may leave.
|
||||
func judgeLines(details string, exact bool, machines []string) (string, string) {
|
||||
for _, line := range strings.Split(details, "\n") {
|
||||
if strings.TrimSpace(line) == "" {
|
||||
continue
|
||||
}
|
||||
if exact {
|
||||
if refusal, ok := outward.Secret(line, machines...); !ok {
|
||||
return line, refusal.String()
|
||||
}
|
||||
continue
|
||||
}
|
||||
if refusal, ok := outward.Check(line, machines...); !ok {
|
||||
return line, refusal.String()
|
||||
}
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
|
||||
func layerKeys(m map[string]any) []string {
|
||||
@@ -300,6 +396,27 @@ func sayableValue(v any, machines []string) (string, bool) {
|
||||
return out, out == text
|
||||
}
|
||||
|
||||
// wholeValue is a value as a channel that proves who answers is shown it (asks.Ask.Whole), and whether it was
|
||||
// shown whole: as it is, unless it is shaped like a secret (outward.Secret), which is withheld whole — never in
|
||||
// part, so no half of a key reaches the phone.
|
||||
func wholeValue(v any, machines []string) (string, bool) {
|
||||
text, ok := v.(string)
|
||||
if !ok {
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return markWithheld, false
|
||||
}
|
||||
text = string(raw)
|
||||
}
|
||||
if text == "" {
|
||||
return `""`, true
|
||||
}
|
||||
if _, ok := outward.Secret(text, machines...); !ok {
|
||||
return markWithheld, false
|
||||
}
|
||||
return text, true
|
||||
}
|
||||
|
||||
// sayable is a text with what may not leave the mesh replaced in place by a marker; what the markers cannot make
|
||||
// pass is withheld whole.
|
||||
func sayable(text string, machines []string) string {
|
||||
@@ -432,6 +549,23 @@ func (pr proposer) propose(ctx context.Context, in proposeInput) (string, error)
|
||||
return refuse("the ask's words would carry %s, which may not leave the mesh, and the change could not be "+
|
||||
"shown without it; %s", refusal, atTheTerminalInstead(in))
|
||||
}
|
||||
// The whole words are shown only where the sender is proven, and never a secret's shape: wholeValue withholds
|
||||
// one, and the router would refuse the ask if one were left, so it is refused here first, in words.
|
||||
if refusal, ok := outward.Secret(q.Whole, machines...); !ok {
|
||||
return refuse("the change shown whole would carry %s, which may not leave the mesh on any channel; %s",
|
||||
refusal, atTheTerminalInstead(in))
|
||||
}
|
||||
// So are the details (ADR 0347 §1, §3): every line the router would withhold from the note, or refuse the ask
|
||||
// for, is refused here, in the controller's own words, naming the line's key — Approve never sets a value the
|
||||
// operator was shown nothing of.
|
||||
if line, why := judgeLines(q.Details, false, machines); line != "" {
|
||||
return refuse("the change's line %q would carry %s, which the operator's note may not show; %s",
|
||||
keyOfLine(line), why, atTheTerminalInstead(in))
|
||||
}
|
||||
if line, why := judgeLines(q.WholeDetails, true, machines); line != "" {
|
||||
return refuse("the change's line %q shown whole would carry %s, which may not leave the mesh on any channel; %s",
|
||||
keyOfLine(line), why, atTheTerminalInstead(in))
|
||||
}
|
||||
// Fail closed, before anything is kept: no router, no grant, no channel means no ask.
|
||||
if pr.routerHere != nil {
|
||||
here, err := pr.routerHere(ctx)
|
||||
@@ -519,15 +653,29 @@ func (pr proposer) propose(ctx context.Context, in proposeInput) (string, error)
|
||||
fingerprint(p.Digest))
|
||||
fmt.Fprintf(&b, " until %s nothing changes: the layer is set only on Approve, and discarded on Decline or at expiry\n",
|
||||
q.Expires.Local().Format("2006-01-02 15:04"))
|
||||
fmt.Fprintf(&b, " `settings proposals %s` shows it whole; once approved, `push %s` sends it", id, pushWord(p.Node))
|
||||
fmt.Fprintf(&b, " `settings proposals %s` shows it whole; once approved, a walk sends it to %s at once, one machine at "+
|
||||
"a time, each judged, and its end is told as every walk's is (novox/hq ADR 0344)", id, p.reaches())
|
||||
return b.String(), nil
|
||||
}
|
||||
|
||||
func pushWord(node string) string {
|
||||
if node == "" {
|
||||
return "--behind"
|
||||
// reaches is the machines an approved layer is sent to, in words: the one it names, or every machine running the
|
||||
// module.
|
||||
func (p settingsProposal) reaches() string {
|
||||
if p.Node == "" {
|
||||
return "every machine running " + p.Module
|
||||
}
|
||||
return node
|
||||
return p.Node
|
||||
}
|
||||
|
||||
// onApprove is what the ask says Approve does (novox/hq ADR 0344 §4): the one Approve sets the layer and sends it.
|
||||
func (p settingsProposal) onApprove() string {
|
||||
what := "the layer is set"
|
||||
if p.Clear {
|
||||
what = "the layer is removed"
|
||||
}
|
||||
return fmt.Sprintf("Approved, %s and sent at once to %s, one machine at a time, each judged, every build there "+
|
||||
"kept; its end is told as every walk's is. A module that reads its settings only when it starts takes them at "+
|
||||
"its next start.", what, p.reaches())
|
||||
}
|
||||
|
||||
func newProposalID() string {
|
||||
@@ -542,11 +690,6 @@ const (
|
||||
|
||||
// proposeCommand is `settings propose`, on this controller's stores and bus.
|
||||
func proposeCommand(ctx context.Context, module, node, valuesArg string, clear, replace bool) error {
|
||||
if node == "" && !clear {
|
||||
// There is no module layer for the whole mesh (novox/hq issue 630); clearing one left over still works.
|
||||
return fmt.Errorf("%s: there is no module layer for the whole mesh — an assignment is one module on one "+
|
||||
"node (novox/hq issue 630). Propose it for one node with --node. Nothing was proposed", module)
|
||||
}
|
||||
var values map[string]any
|
||||
if !clear {
|
||||
if valuesArg == "" {
|
||||
@@ -793,6 +936,40 @@ func setLayerIn(open *stores) setOnWarrant {
|
||||
changed = "nothing changed"
|
||||
}
|
||||
}
|
||||
return changed + " (ask " + askID + ")", nil
|
||||
// **And sends it** (novox/hq ADR 0344): the layer the operator approved is walked to the machines it reaches
|
||||
// at once. A walk that cannot be opened is said as the act's failure — the layer is set and nothing sends it.
|
||||
walked, err := openSettingsWalk(ctx, open, inventory.PlanSettings{Module: p.Module, Node: p.Node, Ask: askID,
|
||||
Change: changed})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s (ask %s) is set, and the walk that sends it could not be opened: %v — `push %s` sends it",
|
||||
changed, askID, err, pushTo(p.Node))
|
||||
}
|
||||
return changed + " (ask " + askID + "); " + walked, nil
|
||||
}
|
||||
}
|
||||
|
||||
// pushTo is the push that sends a layer by hand when its walk could not: the machine, or every one behind.
|
||||
func pushTo(node string) string {
|
||||
if node == "" {
|
||||
return "--behind"
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
// keyOfLine is what a refusal names of a line: the key of a change's line ("+ key: …"), or the line's first words.
|
||||
func keyOfLine(line string) string {
|
||||
if len(line) > 2 && strings.ContainsRune("+~-=", rune(line[0])) && line[1] == ' ' {
|
||||
rest := line[2:]
|
||||
if i := strings.IndexAny(rest, ": "); i > 0 {
|
||||
rest = rest[:i]
|
||||
}
|
||||
if _, ok := outward.Secret(rest); !ok {
|
||||
return "whose key is shaped like a secret"
|
||||
}
|
||||
return rest
|
||||
}
|
||||
if r := []rune(line); len(r) > 40 {
|
||||
return string(r[:40]) + "…"
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
@@ -116,24 +117,43 @@ func TestAProposalAsksAtTheLevelApproveWithTheExactChange(t *testing.T) {
|
||||
if q.Options[0].Binds == q.Options[1].Binds {
|
||||
t.Error("Approve and Decline bind the same act")
|
||||
}
|
||||
for _, line := range []string{
|
||||
"Set the settings of mounts on shanks to these values?",
|
||||
"Proposed by g14/claude-code, through the mesh-controller seat, at " + r.now.Local().Format("15:04 on 2 Jan") + ".",
|
||||
"+ sources: recalbox=‹address›@‹path›:ro",
|
||||
"~ shares: library=‹path› (was: none)",
|
||||
"- old (was: x)",
|
||||
"Fingerprint " + fingerprint(layerDigest(mountsSources)) + ".",
|
||||
"read it whole, with this fingerprint",
|
||||
} {
|
||||
if !strings.Contains(q.Explanation, line) {
|
||||
t.Errorf("the explanation lacks %q:\n%s", line, q.Explanation)
|
||||
}
|
||||
// The message reads at a glance (issues 383 and 639): a summary, then one point per key — and nothing else:
|
||||
// who proposed it, the fingerprint and the how-to are the Details, which the router notes after the answers.
|
||||
summary := "Changes 3 setting(s) of mounts on shanks; approved, the layer is set and sent at once by a walk to shanks, one machine at a time, each judged."
|
||||
if q.Explanation != summary+"\n+ sources: recalbox=‹address›@‹path›:ro\n~ shares: library=‹path› (was: none)\n- old (was: x)" {
|
||||
t.Errorf("the explanation:\n%s", q.Explanation)
|
||||
}
|
||||
if err := asks.Glance(q.Explanation); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
for _, leak := range []string{"nas.lan", "/mnt/recalbox", "/mnt/library", "smb://"} {
|
||||
if strings.Contains(q.Explanation, leak) {
|
||||
t.Errorf("the explanation carries %q, which may not leave the mesh", leak)
|
||||
}
|
||||
}
|
||||
// Where the sender is proven, the values whole: the mount point and the share are what is approved.
|
||||
if q.Whole != summary+"\n+ sources: recalbox=smb://nas.lan/recalbox@/mnt/recalbox:ro\n~ shares: library=/mnt/library (was: none)\n- old (was: x)" ||
|
||||
asks.Glance(q.Whole) != nil {
|
||||
t.Errorf("the whole words:\n%s", q.Whole)
|
||||
}
|
||||
for _, line := range []string{
|
||||
"Proposed by g14/claude-code, through the mesh-controller seat at " + r.now.Local().Format("15:04 on 2 Jan") +
|
||||
". Fingerprint " + fingerprint(layerDigest(mountsSources)) + ".",
|
||||
"Where who answers is not proven, its values are shown as ‹address›, ‹path› or ‹withheld›.",
|
||||
"Read it whole, with this fingerprint: settings proposals " + kept(r).ID + " at the controller's terminal",
|
||||
} {
|
||||
if !strings.Contains(q.Details, line) {
|
||||
t.Errorf("Details lack %q:\n%s", line, q.Details)
|
||||
}
|
||||
}
|
||||
if refusal, ok := outward.Check(q.Details, "anchor", "laptop", "shanks"); !ok {
|
||||
t.Errorf("the router would withhold a line of the details: %s", refusal)
|
||||
}
|
||||
for _, howTo := range []string{"ingerprint", "settings proposals", "settings verb", "not proven", "Proposed by"} {
|
||||
if strings.Contains(q.Explanation, howTo) || strings.Contains(q.Whole, howTo) {
|
||||
t.Errorf("the message carries the how-to %q", howTo)
|
||||
}
|
||||
}
|
||||
all := []string{q.Headline, q.Explanation, q.OnExpiry}
|
||||
for _, o := range q.Options {
|
||||
all = append(all, o.Label, o.Does)
|
||||
@@ -141,6 +161,9 @@ func TestAProposalAsksAtTheLevelApproveWithTheExactChange(t *testing.T) {
|
||||
if refusal, ok := outward.Check(strings.Join(all, "\n"), "anchor", "laptop", "shanks"); !ok {
|
||||
t.Errorf("the router would refuse the ask: %s", refusal)
|
||||
}
|
||||
if refusal, ok := outward.Secret(q.Whole, "anchor", "laptop", "shanks"); !ok {
|
||||
t.Errorf("the router would refuse the whole words: %s", refusal)
|
||||
}
|
||||
// Kept as the controller's own ask, about no condition, with the proposal whole and its digests.
|
||||
kept := r.theProposal(t)
|
||||
p := kept.Proposal
|
||||
@@ -173,11 +196,98 @@ func TestAMeshWideLayerIsProposed(t *testing.T) {
|
||||
}
|
||||
q := r.askSent(t)
|
||||
if q.Headline != "Set notes on the whole mesh?" || q.About != "settings.notes.mesh" ||
|
||||
!strings.Contains(q.Explanation, "Set the settings of notes on the whole mesh to these values?") {
|
||||
q.Explanation != "Changes 1 setting(s) of notes on the whole mesh; approved, the layer is set and sent at once by a walk to every machine running notes, one machine at a time, each judged.\n+ x: 1" ||
|
||||
q.Whole != "" {
|
||||
t.Errorf("%+v", q)
|
||||
}
|
||||
}
|
||||
|
||||
// kept is the one proposal the rig keeps.
|
||||
func kept(r *proposerRig) asked {
|
||||
for _, a := range r.store {
|
||||
if a.Proposal != nil {
|
||||
return a
|
||||
}
|
||||
}
|
||||
return asked{}
|
||||
}
|
||||
|
||||
// The switch between the masked and the whole change (issue 383): the same change, under the content rule and
|
||||
// exact, differs in the masked values alone; a value shaped like a secret is withheld in both, whole, with its
|
||||
// key still named; and a change no value of which is masked carries no whole words of its own.
|
||||
func TestAProposalShowsItsValuesWholeOnlyWhereTheSenderIsProvenAndNeverASecret(t *testing.T) {
|
||||
machines := []string{"anchor", "laptop", "shanks"}
|
||||
r := newProposerRig(t)
|
||||
values := map[string]any{"shares": "media=/storage/media", "sources": "recalbox=smb://nas.lan/recalbox@/mnt/recalbox", "font": "Inter 13"}
|
||||
if _, err := r.pr.propose(context.Background(), proposeInput{module: "mounts", node: "shanks", values: values}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q := r.askSent(t)
|
||||
masked, _, _ := kept(r).Proposal.change(machines, false)
|
||||
whole, _, _ := kept(r).Proposal.change(machines, true)
|
||||
if !strings.HasSuffix(q.Explanation, "\n"+strings.Join(masked, "\n")) || !strings.HasSuffix(q.Whole, "\n"+strings.Join(whole, "\n")) {
|
||||
t.Fatalf("the ask does not carry the change masked and whole:\n%s\n--\n%s", q.Explanation, q.Whole)
|
||||
}
|
||||
if want := []string{"+ font: Inter 13", "+ shares: media=‹path›", "+ sources: recalbox=‹address›@‹path›"}; strings.Join(masked, "\n") != strings.Join(want, "\n") {
|
||||
t.Errorf("masked: %q", masked)
|
||||
}
|
||||
if want := []string{"+ font: Inter 13", "+ shares: media=/storage/media", "+ sources: recalbox=smb://nas.lan/recalbox@/mnt/recalbox"}; strings.Join(whole, "\n") != strings.Join(want, "\n") {
|
||||
t.Errorf("whole: %q", whole)
|
||||
}
|
||||
if _, ok := outward.Secret(q.Whole, "shanks"); !ok {
|
||||
t.Error("the router would refuse the whole words")
|
||||
}
|
||||
// Mutation: the masked and the whole change differ in exactly the lines whose value was masked.
|
||||
differ := 0
|
||||
for i := range masked {
|
||||
if masked[i] != whole[i] {
|
||||
differ++
|
||||
if !strings.Contains(masked[i], "‹") || strings.Contains(whole[i], "‹") {
|
||||
t.Errorf("the lines differ otherwise than by the mask:\n%s\n%s", masked[i], whole[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
if differ != 2 {
|
||||
t.Errorf("%d lines differ, and the mask covered 2", differ)
|
||||
}
|
||||
// A value shaped like a secret is withheld in both, whole, with its key still named.
|
||||
r1 := newProposerRig(t)
|
||||
secrets := map[string]any{"github": "ghp_abcdefghijklmnopqrstuvwxyz0123456789", "cert": "-----BEGIN CERTIFICATE-----"}
|
||||
if _, err := r1.pr.propose(context.Background(), proposeInput{module: "mounts", node: "shanks", values: secrets}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q1 := r1.askSent(t)
|
||||
for _, line := range []string{"+ cert: ‹withheld›", "+ github: ‹withheld›"} {
|
||||
if !strings.Contains(q1.Explanation, line) {
|
||||
t.Errorf("lacks %q:\n%s", line, q1.Explanation)
|
||||
}
|
||||
}
|
||||
for _, secret := range []string{"ghp_", "BEGIN CERTIFICATE"} {
|
||||
if strings.Contains(q1.Explanation, secret) || strings.Contains(q1.Whole, secret) || strings.Contains(q1.Details, secret) {
|
||||
t.Errorf("the secret %q reaches the phone", secret)
|
||||
}
|
||||
}
|
||||
// More keys than points (issue 639): the first two, then the keys of the rest, read whole at the terminal.
|
||||
r3 := newProposerRig(t)
|
||||
five := map[string]any{"a": "1", "b": "2", "c": "3", "d": "4", "e": "5"}
|
||||
if _, err := r3.pr.propose(context.Background(), proposeInput{module: "notes", node: "laptop", values: five}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if q3 := r3.askSent(t); !strings.HasSuffix(q3.Explanation, "\n+ a: 1\n+ b: 2\nand 3 more, each in the note: c, d, e") || asks.Glance(q3.Explanation) != nil ||
|
||||
!strings.Contains(q3.Details, "settings proposals") {
|
||||
t.Errorf("five keys:\n%s", q3.Explanation)
|
||||
}
|
||||
// No value masked: the message is the same everywhere, and the ask says no whole words.
|
||||
r2 := newProposerRig(t)
|
||||
if _, err := r2.pr.propose(context.Background(), proposeInput{module: "dunst", node: "laptop", values: map[string]any{"font-size": 13, "width": 500}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if q2 := r2.askSent(t); q2.Whole != "" || !strings.HasSuffix(q2.Explanation, "\n+ font-size: 13\n+ width: 500") ||
|
||||
strings.Contains(q2.Details, "not proven") {
|
||||
t.Errorf("%+v", q2)
|
||||
}
|
||||
}
|
||||
|
||||
// A proposal expired unanswered is kept so by the reconciling, and a warrant for it afterwards sets nothing.
|
||||
func TestAnExpiredProposalIsKeptExpired(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
@@ -205,7 +315,8 @@ func TestAClearIsProposedAndShowsWhatItRemoves(t *testing.T) {
|
||||
}
|
||||
q := r.askSent(t)
|
||||
if q.Headline != "Clear notes on laptop?" || !strings.Contains(q.Explanation, "- places.data.owner: 1001:1001") ||
|
||||
!strings.Contains(q.Explanation, "- places.data.path: ‹path›") {
|
||||
!strings.Contains(q.Explanation, "- places.data.path: ‹path›") || !strings.Contains(q.Whole, "- places.data.path: /srv/notes") ||
|
||||
!strings.HasPrefix(q.Explanation, "Removes the 2 setting(s) of notes set on laptop; approved, the layer is removed and that is sent at once by a walk to laptop, one machine at a time") {
|
||||
t.Errorf("%+v", q)
|
||||
}
|
||||
if p := r.theProposal(t).Proposal; !p.Clear || p.Digest != layerDigest(r.before) || len(r.judged) != 0 {
|
||||
@@ -744,3 +855,125 @@ func captureStdout(t *testing.T, f func()) string {
|
||||
_ = w.Close()
|
||||
return <-done
|
||||
}
|
||||
|
||||
// The review of #304: Approve sets exactly what the operator can read. A proposal of five keys, one value 300
|
||||
// characters long, shows two keys at a glance, says what it cut, and carries every key with its whole value in its
|
||||
// Details for a channel that proves who answers (WholeDetails), and every key masked for one that does not.
|
||||
func TestEveryKeyAndItsWholeValueReachTheOperator(t *testing.T) {
|
||||
var long strings.Builder
|
||||
long.WriteString("library=/storage")
|
||||
for i := 0; long.Len() < 300; i++ {
|
||||
long.WriteString(fmt.Sprintf("/album-%02d", i))
|
||||
}
|
||||
value := long.String()[:300]
|
||||
values := map[string]any{"a": "1", "b": "2", "c": "3", "d": "4", "shares": value}
|
||||
r := newProposerRig(t)
|
||||
if _, err := r.pr.propose(context.Background(), proposeInput{module: "mounts", node: "shanks", values: values}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q := r.askSent(t)
|
||||
if err := q.Check(r.now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// What the operator reads on a proving channel: the whole words, then the note's whole details.
|
||||
visible := q.Whole + "\n" + q.WholeDetails
|
||||
if q.Whole == "" {
|
||||
visible = q.Explanation + "\n" + q.WholeDetails
|
||||
}
|
||||
for k, v := range values {
|
||||
if !strings.Contains(visible, "+ "+k+": "+v.(string)) {
|
||||
t.Errorf("the operator never reads %s whole:\n%s", k, visible)
|
||||
}
|
||||
}
|
||||
// The masked glance points to a note that holds the values masked, and says no more; the whole glance, shown only
|
||||
// beside the whole details, says they are whole there.
|
||||
if asks.Glance(q.Explanation) != nil || !strings.HasSuffix(q.Explanation, "\nand 3 more, each in the note: c, d, shares") ||
|
||||
strings.Contains(q.Explanation, "whole in the note") || !strings.HasSuffix(q.Whole, "\nand 3 more, each whole in the note: c, d, shares") {
|
||||
t.Errorf("the glance:\n%s", q.Explanation)
|
||||
}
|
||||
for k := range values {
|
||||
if !strings.Contains(q.Details, "+ "+k+": ") {
|
||||
t.Errorf("the masked details lack %s:\n%s", k, q.Details)
|
||||
}
|
||||
}
|
||||
if strings.Contains(q.Details, "/storage") {
|
||||
t.Errorf("the masked details show a path:\n%s", q.Details)
|
||||
}
|
||||
if refusal, ok := outward.Check(q.Details, "shanks"); !ok {
|
||||
t.Errorf("the router would withhold the masked details: %s", refusal)
|
||||
}
|
||||
if _, ok := outward.Secret(q.WholeDetails, "shanks"); !ok {
|
||||
t.Error("the router would refuse the whole details")
|
||||
}
|
||||
// A value too long for a glance is cut there, and says so.
|
||||
r2 := newProposerRig(t)
|
||||
if _, err := r2.pr.propose(context.Background(), proposeInput{module: "mounts", node: "shanks", values: map[string]any{"shares": value}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q2 := r2.askSent(t)
|
||||
// The cut point is 160 characters, its marker inside the bound.
|
||||
if last := q2.Whole[strings.LastIndex(q2.Whole, "\n")+1:]; len([]rune(last)) != asks.PointLength {
|
||||
t.Errorf("the cut point is %d characters, not %d: %q", len([]rune(last)), asks.PointLength, last)
|
||||
}
|
||||
if !strings.HasSuffix(q2.Whole, "… (cut: whole in the note)") || !strings.HasSuffix(q2.Explanation, "‹path›") && !strings.HasSuffix(q2.Explanation, "… (cut: in the note)") || !strings.Contains(q2.WholeDetails, "+ shares: "+value) || asks.Glance(q2.Whole) != nil {
|
||||
t.Errorf("the cut is not said, or the note lacks the whole:\n%s\n--\n%s", q2.Whole, q2.WholeDetails)
|
||||
}
|
||||
}
|
||||
|
||||
// The review of #304: a value given to a secret's name, past the second key, is said with its key named and its
|
||||
// value withheld, in the note on every channel; the router has nothing left to withhold unnamed or refuse. A line
|
||||
// the router would refuse even so is refused at propose, in the controller's words.
|
||||
func TestASecretNamedKeyIsNamedAndItsValueWithheld(t *testing.T) {
|
||||
r := newProposerRig(t)
|
||||
values := map[string]any{"a": "1", "b": "2", "c": "3", "password": "hunter2"}
|
||||
if _, err := r.pr.propose(context.Background(), proposeInput{module: "notes", node: "laptop", values: values}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q := r.askSent(t)
|
||||
for _, form := range []string{q.Details, q.WholeDetails, q.Explanation, q.Whole} {
|
||||
if strings.Contains(form, "hunter2") {
|
||||
t.Errorf("the value reaches the phone:\n%s", form)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(q.Details, "+ password (value withheld)") {
|
||||
t.Errorf("the key is not named with its value withheld:\n%s", q.Details)
|
||||
}
|
||||
if q.WholeDetails != "" && !strings.Contains(q.WholeDetails, "+ password (value withheld)") {
|
||||
t.Errorf("the whole details do not name the key:\n%s", q.WholeDetails)
|
||||
}
|
||||
if line, why := judgeLines(q.Details, false, nil); line != "" {
|
||||
t.Errorf("the router would withhold %q: %s", line, why)
|
||||
}
|
||||
if line, why := judgeLines(q.WholeDetails, true, nil); line != "" {
|
||||
t.Errorf("the router would refuse %q: %s", line, why)
|
||||
}
|
||||
if !strings.HasSuffix(q.Explanation, "\nand 2 more, each in the note: c, password") {
|
||||
t.Errorf("the glance:\n%s", q.Explanation)
|
||||
}
|
||||
// A key shaped like a secret cannot be named on any channel: refused at propose, in the controller's words, and
|
||||
// the key itself not echoed.
|
||||
r2 := newProposerRig(t)
|
||||
secretKey := "ghp_abcdefghijklmnopqrstuvwxyz0123456789"
|
||||
_, err := r2.pr.propose(context.Background(), proposeInput{module: "notes", node: "laptop",
|
||||
values: map[string]any{"a": "1", "b": "2", secretKey: "3"}})
|
||||
if err == nil || !strings.Contains(err.Error(), "Nothing was proposed") || strings.Contains(err.Error(), secretKey) {
|
||||
t.Errorf("a secret-shaped key: %v", err)
|
||||
}
|
||||
if len(r2.sent) != 0 {
|
||||
t.Errorf("the refused proposal was asked")
|
||||
}
|
||||
}
|
||||
|
||||
// keyOfLine names a change's key, and never one shaped like a secret.
|
||||
func TestARefusalNamesTheKeyAndNeverASecret(t *testing.T) {
|
||||
for line, want := range map[string]string{
|
||||
"+ password (value withheld)": "password",
|
||||
"~ shares: ‹path› (was: none)": "shares",
|
||||
"+ ghp_abcdefghijklmnopqrstuvwxyz0123456789: 1": "whose key is shaped like a secret",
|
||||
"Proposed by g14 at 15:04.": "Proposed by g14 at 15:04.",
|
||||
} {
|
||||
if got := keyOfLine(line); got != want {
|
||||
t.Errorf("%q: %q, want %q", line, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -249,7 +249,13 @@ func serve(ctx context.Context) (err error) {
|
||||
// (novox/hq to-be 45 §9).
|
||||
checkEvents = bus
|
||||
// And every walk this controller keeps in a new state, for the delivery it walks (novox/hq ADR 0239).
|
||||
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { sayPlanMoved(ctx, bus, open.inventory, p) }
|
||||
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) {
|
||||
sayPlanMoved(ctx, bus, open.inventory, p)
|
||||
// A walk ended has taken what it reuses: its merges' check builds go (novox/hq ADR 0350 §5).
|
||||
if p.WalkSaid == inventory.WalkEnded {
|
||||
go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p)
|
||||
}
|
||||
}
|
||||
if err := server.Checks(following{open}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -616,6 +616,7 @@ func TestKillAndPauseAskTheHolderOnTheMachine(t *testing.T) {
|
||||
// A plan that stopped at its first machine is retried: the module sent to that machine again, the
|
||||
// send recorded as the first anew, and the plan goes on — unless a newer plan holds the module.
|
||||
func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := asksRecorded(t)
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
@@ -83,6 +84,27 @@ func everyBuildKept(ctx context.Context, node string) bool {
|
||||
return on != "" && on == node
|
||||
}
|
||||
|
||||
type keepForSettingsKey struct{}
|
||||
|
||||
// keepingBuildsForSettings is a context whose sends compose every module of one machine — recorded or rolling out —
|
||||
// at the build that machine was last sent, as keepingEveryBuild does, for a settings walk's send (novox/hq ADR 0344):
|
||||
// the layer the operator approved, and nothing of any module's new code. A build held for a person, or waiting
|
||||
// for a gate, stays where it is; the walk that is its own carries it.
|
||||
func keepingBuildsForSettings(ctx context.Context, node string) context.Context {
|
||||
return context.WithValue(ctx, keepForSettingsKey{}, node)
|
||||
}
|
||||
|
||||
// errSettingsCannotKeep is a settings walk's send refused because a build of its machine cannot be kept: what the
|
||||
// machine was last sent is not known, or a build it runs is gone from the records. It does not clear by itself, so
|
||||
// the walk fails on it rather than trying again (ADR 0344).
|
||||
var errSettingsCannotKeep = errors.New("the settings cannot be sent with every build kept")
|
||||
|
||||
// settingsKeptOn is whether this context is a settings walk's send to this machine.
|
||||
func settingsKeptOn(ctx context.Context, node string) bool {
|
||||
on, _ := ctx.Value(keepForSettingsKey{}).(string)
|
||||
return on != "" && on == node
|
||||
}
|
||||
|
||||
// recordedKept is, for a send under keepingRecorded, every recorded module the machine was last sent a
|
||||
// build of that the mesh's build is not identical to: module → the commit it keeps. Nil when the context
|
||||
// keeps nothing, or when what the machine was last sent is not known (it is then held whole elsewhere —
|
||||
@@ -92,7 +114,7 @@ func everyBuildKept(ctx context.Context, node string) bool {
|
||||
// 490), and a machine whose last send is not known refuses the send: there is nothing to keep it at, and
|
||||
// composing the mesh's builds would be the very move the grants step must not make.
|
||||
func recordedKept(ctx context.Context, open *stores, node string) (map[string]string, error) {
|
||||
every := everyBuildKept(ctx, node)
|
||||
every := everyBuildKept(ctx, node) || settingsKeptOn(ctx, node)
|
||||
skip, on := keptExcept(ctx)
|
||||
if !on && !every {
|
||||
return nil, nil
|
||||
@@ -106,6 +128,9 @@ func recordedKept(ctx context.Context, open *stores, node string) (map[string]st
|
||||
return nil, err
|
||||
}
|
||||
if !known {
|
||||
if settingsKeptOn(ctx, node) {
|
||||
return nil, fmt.Errorf("%w: what %s was last sent is not known (novox/hq ADR 0344)", errSettingsCannotKeep, node)
|
||||
}
|
||||
if every {
|
||||
return nil, fmt.Errorf("what %s was last sent is not known, so the bus's user list cannot be sent "+
|
||||
"there alone with every build kept (novox/hq issue 490)", node)
|
||||
@@ -166,6 +191,10 @@ func keepRecorded(ctx context.Context, open *stores, node string, shelf map[stri
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found && settingsKeptOn(ctx, node) {
|
||||
return nil, fmt.Errorf("%w: %s runs %s's build %s, which the build records no longer hold (novox/hq ADR 0344)",
|
||||
errSettingsCannotKeep, node, m, short(kept[m]))
|
||||
}
|
||||
if !found && everyBuildKept(ctx, node) {
|
||||
return nil, fmt.Errorf("%s runs %s's build %s, which the build records no longer hold: the bus's user "+
|
||||
"list cannot be sent there alone with it kept (novox/hq issue 490)", node, m, short(kept[m]))
|
||||
|
||||
+578
-43
@@ -39,8 +39,8 @@ import (
|
||||
// from the same manifest is no move.
|
||||
//
|
||||
// **The release plan** is what walks the waiting moves through: whenever moves wait for a gate that no
|
||||
// started plan is walking, the mesh opens one — every such machine, one at a time, the control node last,
|
||||
// each sent and judged before the next. A release plan that fails stops the next one opening on its own:
|
||||
// started plan is walking, the mesh opens one — every such machine, the first sent and judged, then the rest
|
||||
// at once, each judged by its own gate (novox/hq ADR 0353, in place of one at a time). A release plan that fails stops the next one opening on its own:
|
||||
// a person releases it again (`upgrade release-backlog --why`), after the condition says why.
|
||||
|
||||
// sendScope is what a send may carry that no gate has seen.
|
||||
@@ -81,6 +81,15 @@ type moveFacts struct {
|
||||
srcs map[string]map[string]string
|
||||
passed map[string]map[string]bool
|
||||
plans []inventory.Plan
|
||||
// self is the release walking these moves, whose own gates are no other walk's (novox/hq ADR 0353).
|
||||
self string
|
||||
}
|
||||
|
||||
type walkingKey struct{}
|
||||
|
||||
// withWalking says a release plan is the one sending: its own gates never hold its own sends back.
|
||||
func withWalking(ctx context.Context, id string) context.Context {
|
||||
return context.WithValue(ctx, walkingKey{}, id)
|
||||
}
|
||||
|
||||
func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, error) {
|
||||
@@ -99,6 +108,7 @@ func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, er
|
||||
return f, err
|
||||
}
|
||||
f.plans, err = inv.OpenPlans(ctx)
|
||||
f.self, _ = ctx.Value(walkingKey{}).(string)
|
||||
return f, err
|
||||
}
|
||||
|
||||
@@ -193,17 +203,24 @@ func (f moveFacts) moves(node string, modules []string, sent map[string]string,
|
||||
// feet. A release keeps no module records: its open gate's carried moves are its walk.
|
||||
func (f moveFacts) walkedBy(module, node, to string) string {
|
||||
for _, p := range f.plans {
|
||||
if !p.Open() {
|
||||
if !p.Open() || (p.Release != nil && p.ID == f.self) {
|
||||
continue
|
||||
}
|
||||
if p.Release != nil {
|
||||
g := p.Release.Gate
|
||||
if g == nil || g.Verdict != "" {
|
||||
continue
|
||||
// The first machine's gate, and every gate of the rest still judging (novox/hq ADR 0353).
|
||||
gates := []*inventory.PlanGate{p.Release.Gate}
|
||||
for _, g := range p.Release.Gates {
|
||||
gates = append(gates, g)
|
||||
}
|
||||
for _, c := range g.Carried {
|
||||
if c.Module == module && !(slices.Contains(g.Machines, node) && sameCommit(c.To, to)) {
|
||||
return p.ID
|
||||
for _, g := range gates {
|
||||
// A send kept and not yet made walks nothing yet (ADR 0353 §8).
|
||||
if g == nil || g.Verdict != "" || g.Silent || g.Pending {
|
||||
continue
|
||||
}
|
||||
for _, c := range g.Carried {
|
||||
if c.Module == module && !(slices.Contains(g.Machines, node) && sameCommit(c.To, to)) {
|
||||
return p.ID
|
||||
}
|
||||
}
|
||||
}
|
||||
continue
|
||||
@@ -303,8 +320,8 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
|
||||
}
|
||||
}
|
||||
if len(refused) > 0 {
|
||||
return nil, fmt.Errorf("%w: %s — a walk sends them, one machine at a time, each judged (novox/hq ADR "+
|
||||
"0236); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))
|
||||
return nil, fmt.Errorf("%w: %s — a walk sends them, one machine first and then the rest at once, each judged (novox/hq ADR "+
|
||||
"0236, ADR 0353); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))
|
||||
}
|
||||
return kept, nil
|
||||
}
|
||||
@@ -320,19 +337,35 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
|
||||
// And the grants step first, when the send changes what the bus's user list must say (novox/hq issue 490):
|
||||
// answered for the gate to keep, nil when there was none.
|
||||
func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, *inventory.GrantsStep, error) {
|
||||
inv := open.inventory
|
||||
f, err := readMoveFacts(ctx, inv)
|
||||
moves, err := gatedMoves(ctx, open, node, owns)
|
||||
if err != nil || len(moves) == 0 {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
sayRecreations(ctx, open, moves)
|
||||
sent, grants, err := sendJudged(ctx, open, node)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
return moves, sent, grants, nil
|
||||
}
|
||||
|
||||
// gatedMoves is what a gated send to one machine would carry, with the build each moves to, made of nothing:
|
||||
// gatedSend's reading, which a walk also keeps before it sends (novox/hq ADR 0353 §8). Refused the same way
|
||||
// while a plan that has started walks one of those builds elsewhere.
|
||||
func gatedMoves(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, error) {
|
||||
inv := open.inventory
|
||||
f, err := readMoveFacts(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
moves, err := machineMoves(ctx, open, f, node, true)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
// A move the send keeps is not made, and is no move for the gate to judge (novox/hq issue 201): a core module
|
||||
// kept over an older record judged as moving back to it would read the gate as superseded by its own send.
|
||||
if moves, err = withoutKept(ctx, open, node, moves); err != nil {
|
||||
return nil, nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
own := func(module string) bool {
|
||||
return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module })
|
||||
@@ -342,7 +375,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
||||
continue
|
||||
}
|
||||
if id := f.walkedBy(mv.Module, node, mv.To); id != "" {
|
||||
return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||
return nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||
mv.Module, short(mv.To), node, id)
|
||||
}
|
||||
}
|
||||
@@ -350,7 +383,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
||||
// plan, or a release, on this machine is not crossed with a newer build; this send waits for its gate.
|
||||
for _, o := range owns {
|
||||
if id := f.walkedBy(o.Module, node, o.To); id != "" {
|
||||
return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||
return nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||
o.Module, short(o.To), node, id)
|
||||
}
|
||||
}
|
||||
@@ -363,30 +396,26 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
||||
moves[i].Build = o.Build
|
||||
}
|
||||
}
|
||||
if len(moves) == 0 && len(owns) == 0 {
|
||||
return nil, nil, nil, nil
|
||||
if len(moves) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
for i := range moves {
|
||||
if moves[i].Build == "" {
|
||||
if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil {
|
||||
return nil, nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module })
|
||||
sayRecreations(ctx, open, moves)
|
||||
sent, grants, err := sendJudged(ctx, open, node)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
return moves, sent, grants, nil
|
||||
return moves, nil
|
||||
}
|
||||
|
||||
// passCarried keeps a pass as the verdict of every build the gate judged beside its own module.
|
||||
func passCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) {
|
||||
seen := map[string]bool{}
|
||||
for _, c := range g.Carried {
|
||||
if c.Module == except || c.Build == "" || seen[c.Build] {
|
||||
// A build put back with another machine's failure is not this gate's to pass (novox/hq ADR 0353 §3).
|
||||
if c.Module == except || c.Build == "" || seen[c.Build] || slices.Contains(g.Returned, c.Module) {
|
||||
continue
|
||||
}
|
||||
seen[c.Build] = true
|
||||
@@ -616,6 +645,10 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl
|
||||
}
|
||||
for _, p := range plans {
|
||||
if p.Release != nil {
|
||||
if by != "" && p.Release.Settings != nil {
|
||||
return nil, fmt.Errorf("%s is sending %s; what waits is released once it ends — `plans %s` says where it is",
|
||||
p.ID, p.Repository, p.ID)
|
||||
}
|
||||
if by != "" {
|
||||
return nil, fmt.Errorf("%s is already releasing what waits; `plans %s` says where it is", p.ID, p.ID)
|
||||
}
|
||||
@@ -641,7 +674,9 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl
|
||||
return nil, err
|
||||
}
|
||||
for _, p := range recent {
|
||||
if p.Release == nil {
|
||||
// A settings walk that failed holds nothing back (novox/hq ADR 0344 §6): its failure is its module's
|
||||
// settings on one machine, said on its own, not a build no gate passed.
|
||||
if p.Release == nil || p.Release.Settings != nil {
|
||||
continue
|
||||
}
|
||||
if p.State == inventory.PlanFailed {
|
||||
@@ -689,7 +724,7 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl
|
||||
p := inventory.Plan{ID: fmt.Sprintf("release-%d", now.UnixNano()), Repository: releaseRepository,
|
||||
Created: now, State: inventory.PlanRolling, Tiers: [][]string{names}, Modules: map[string]*inventory.PlanModule{},
|
||||
Release: &inventory.PlanRelease{Order: order, By: by},
|
||||
Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine at a time, each judged", len(names),
|
||||
Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine first, then the rest at once, each judged", len(names),
|
||||
strings.Join(order, ", "))}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return nil, err
|
||||
@@ -698,11 +733,28 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl
|
||||
return &p, nil
|
||||
}
|
||||
|
||||
// advanceRelease takes one step of a release plan: the next machine sent everything waiting there under a
|
||||
// gate, or the machine being judged judged once more; the plan done when every machine is.
|
||||
// advanceRelease takes one step of a release plan: the first machine sent everything waiting there under a
|
||||
// gate and judged; once it passed, every other machine sent at once, each under its own gate (novox/hq ADR
|
||||
// 0353 §1), and judged; the plan done when every machine is.
|
||||
//
|
||||
// **A send is kept before it is made** (ADR 0353 §8): the step that chooses a machine keeps its gate pending with
|
||||
// what it will carry, and the next step makes the send. A controller replaced between the send and keeping it
|
||||
// finds the gate pending, reads the machine's last send, and neither sends again nor leaves the machine unjudged.
|
||||
func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) {
|
||||
r := p.Release
|
||||
if r.Settings != nil {
|
||||
return advanceSettingsWalk(ctx, open, p)
|
||||
}
|
||||
ctx = withWalking(ctx, p.ID)
|
||||
now := time.Now().UTC()
|
||||
if r.Rest {
|
||||
return advanceRest(ctx, open, p, now)
|
||||
}
|
||||
if r.Gate == nil && !gateFirstMachine {
|
||||
// **No first machine** (novox/hq ADR 0360 §7): every machine heard from is sent at once, in one step.
|
||||
r.Rest = true
|
||||
return advanceRest(ctx, open, p, now)
|
||||
}
|
||||
if r.Gate == nil {
|
||||
heard, err := releaseHeard(ctx, open)
|
||||
if err != nil {
|
||||
@@ -715,7 +767,7 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
||||
r.Next++
|
||||
continue
|
||||
}
|
||||
moves, sent, grants, err := gatedSend(ctx, open, node, nil)
|
||||
moves, err := gatedMoves(ctx, open, node, nil)
|
||||
if errors.Is(err, errWalkedElsewhere) {
|
||||
note := fmt.Sprintf("waiting before %s: %v", node, err)
|
||||
changed := p.Note != note
|
||||
@@ -723,23 +775,15 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
||||
return changed, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("sending %s what waits there: %w", node, err)
|
||||
return false, fmt.Errorf("reading what waits on %s: %w", node, err)
|
||||
}
|
||||
if len(moves) == 0 {
|
||||
r.Done = append(r.Done, node)
|
||||
r.Next++
|
||||
continue
|
||||
}
|
||||
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves, Sent: sentNow(ctx, open.inventory, sent),
|
||||
Grants: grants}
|
||||
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
|
||||
if said := grantsSaid(grants); said != "" {
|
||||
p.Note += "; " + said
|
||||
}
|
||||
if said := recreationsSaid(moves); said != "" {
|
||||
p.Note += "; " + said
|
||||
}
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
r.Gate = &inventory.PlanGate{Machines: []string{node}, Carried: moves, Pending: true}
|
||||
p.Note = fmt.Sprintf("sending %s %d build(s) that wait for a gate, first", node, len(moves))
|
||||
return true, nil
|
||||
}
|
||||
p.State, p.Tier = inventory.PlanDone, len(p.Tiers)
|
||||
@@ -750,6 +794,19 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
||||
return true, nil
|
||||
}
|
||||
g := r.Gate
|
||||
if g.Pending {
|
||||
node := firstOf(g.Machines)
|
||||
out, err := sendKept(ctx, open, p, node, g, now)
|
||||
if err != nil || out == keptWaiting {
|
||||
return out == keptWaiting, err
|
||||
}
|
||||
if out == keptNothing {
|
||||
r.Gate = nil
|
||||
r.Done = append(r.Done, node)
|
||||
r.Next++
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
verdict, err := judgeMoves(ctx, open, g, nil, now)
|
||||
if err != nil {
|
||||
return false, err
|
||||
@@ -768,6 +825,11 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
||||
r.Done = append(r.Done, firstOf(g.Machines))
|
||||
r.Next++
|
||||
r.Gate = nil
|
||||
// **Then the rest at once** (novox/hq ADR 0353 §1): every other machine heard from, each judged by its own
|
||||
// gate, in place of one machine at a time (ADR 0236 §22).
|
||||
r.Rest = true
|
||||
p.Note = fmt.Sprintf("passed on %s; the rest are sent at once, each judged", firstOf(g.Machines))
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return true, nil
|
||||
case inventory.GateSuperseded:
|
||||
// Another send moved a judged module on the judged machine (novox/hq issue 352): no verdict on what
|
||||
@@ -788,6 +850,479 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// What sendKept did with a kept send.
|
||||
const (
|
||||
keptSent = iota // sent now
|
||||
keptAlready // the machine was already sent it: judged from that send, nothing sent
|
||||
keptNothing // nothing waits there any more: no gate
|
||||
keptWaiting // another walk judges a build it would carry: it waits
|
||||
)
|
||||
|
||||
// sendKept makes a send a gate kept before it was made (novox/hq ADR 0353 §8) — unless the machine's last send
|
||||
// already carries what the gate is for, made before a controller was replaced and before it was kept: then
|
||||
// nothing is sent, and the gate judges from that send (§6).
|
||||
func sendKept(ctx context.Context, open *stores, p *inventory.Plan, node string, g *inventory.PlanGate,
|
||||
now time.Time) (int, error) {
|
||||
inv := open.inventory
|
||||
moves, sent, grants, err := gatedSend(ctx, open, node, nil)
|
||||
if errors.Is(err, errWalkedElsewhere) {
|
||||
p.Note = fmt.Sprintf("waiting before %s: %v", node, err)
|
||||
return keptWaiting, nil
|
||||
}
|
||||
if err != nil {
|
||||
return keptSent, fmt.Errorf("sending %s what waits there: %w", node, err)
|
||||
}
|
||||
if len(moves) > 0 {
|
||||
g.Machines, g.Carried, g.Since, g.Grants, g.Pending = sent, moves, &now, grants, false
|
||||
g.Sent = sentNow(ctx, inv, sent)
|
||||
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
|
||||
if said := grantsSaid(grants); said != "" {
|
||||
p.Note += "; " + said
|
||||
}
|
||||
if said := recreationsSaid(moves); said != "" {
|
||||
p.Note += "; " + said
|
||||
}
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return keptSent, nil
|
||||
}
|
||||
if !alreadySent(ctx, inv, node, g.Carried) {
|
||||
return keptNothing, nil
|
||||
}
|
||||
since := now
|
||||
if s, found, err := inv.SentTo(ctx, node); err == nil && found {
|
||||
if !s.At.IsZero() {
|
||||
since = s.At
|
||||
}
|
||||
g.Sent = map[string]inventory.SentDeclaration{node: s}
|
||||
}
|
||||
g.Since, g.Pending = &since, false
|
||||
p.Note = fmt.Sprintf("%s was already sent the %d build(s) kept for it, at %s: judged from that send, nothing sent again",
|
||||
node, len(g.Carried), since.Local().Format("15:04:05"))
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return keptAlready, nil
|
||||
}
|
||||
|
||||
// alreadySent says a machine's last send carries every build of the moves: a send made and not kept.
|
||||
func alreadySent(ctx context.Context, inv *inventory.Inventory, node string, moves []inventory.CarriedMove) bool {
|
||||
if len(moves) == 0 {
|
||||
return false
|
||||
}
|
||||
sent, known, err := inv.SentBuilds(ctx, node)
|
||||
if err != nil || !known {
|
||||
return false
|
||||
}
|
||||
for _, mv := range moves {
|
||||
if !sameCommit(sent[mv.Module], mv.To) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// advanceRest takes one step of a release plan's rest (novox/hq ADR 0353): the sends kept for a step made, every
|
||||
// machine of the step judged by its own gate, one that fails put back by itself while the others go on to their
|
||||
// verdicts, and the next step — or the end — once every gate of the step has its verdict or is silent.
|
||||
func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.Time) (bool, error) {
|
||||
r := p.Release
|
||||
// The sends kept for this step, in the plan's order: the control node, last in it, last (§1).
|
||||
var pending []string
|
||||
for _, n := range r.Order {
|
||||
if g := r.Gates[n]; g != nil && g.Pending {
|
||||
pending = append(pending, n)
|
||||
}
|
||||
}
|
||||
if len(pending) > 0 {
|
||||
// **A kept send carries only what was kept** (ADR 0353 §2): a build that passed no gate and came to wait on
|
||||
// a machine since its step was chosen could reach a second machine in it; the step is chosen again.
|
||||
if grew, err := keptStepGrew(ctx, open, r, pending); err != nil {
|
||||
return false, err
|
||||
} else if grew != "" {
|
||||
for _, n := range pending {
|
||||
delete(r.Gates, n)
|
||||
}
|
||||
opened, waiting, err := openRestStep(ctx, open, p)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
p.Note = "chosen again: " + grew
|
||||
if !opened && waiting != "" {
|
||||
p.Note += "; " + waiting
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
var sent, already, waiting []string
|
||||
for _, node := range pending {
|
||||
out, err := sendKept(ctx, open, p, node, r.Gates[node], now)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
switch out {
|
||||
case keptSent:
|
||||
sent = append(sent, node)
|
||||
case keptAlready:
|
||||
already = append(already, node)
|
||||
case keptNothing:
|
||||
delete(r.Gates, node)
|
||||
r.Done = append(r.Done, node)
|
||||
case keptWaiting:
|
||||
waiting = append(waiting, node)
|
||||
}
|
||||
}
|
||||
var said []string
|
||||
if len(sent) > 0 {
|
||||
said = append(said, "sent "+strings.Join(sent, ", ")+" at once, each judged there")
|
||||
}
|
||||
if len(already) > 0 {
|
||||
said = append(said, strings.Join(already, ", ")+" already sent it: judged from that send")
|
||||
}
|
||||
if len(waiting) > 0 {
|
||||
said = append(said, "waiting before "+strings.Join(waiting, ", ")+": another walk judges a build it carries")
|
||||
}
|
||||
if len(said) > 0 {
|
||||
p.Note = strings.Join(said, "; ")
|
||||
}
|
||||
return len(sent)+len(already) > 0 || len(waiting) < len(pending), nil
|
||||
}
|
||||
reports, err := open.inventory.LastReports(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
byNode := map[string]inventory.Reported{}
|
||||
for _, rep := range reports {
|
||||
byNode[rep.Node] = rep
|
||||
}
|
||||
changed, noteWas := false, p.Note
|
||||
var notes, judging []string
|
||||
batched, back := batchingRollbacks(ctx)
|
||||
for _, node := range r.Order {
|
||||
g := r.Gates[node]
|
||||
if g == nil || g.Verdict != "" || g.Silent {
|
||||
continue
|
||||
}
|
||||
// What another machine's failure already put back is not this gate's to judge (ADR 0353 §3).
|
||||
if len(g.Carried) > 0 && len(g.Returned) >= len(g.Carried) {
|
||||
decide(g, inventory.GateFailed, "what it was sent was put back with another machine's failure", now)
|
||||
r.Failed = append(r.Failed, node)
|
||||
changed = true
|
||||
continue
|
||||
}
|
||||
// **A machine that says nothing on its send is left silent, not failed** (ADR 0353 §4): it is not judged
|
||||
// until it reports, and past ApplySilentAfter it is listed and left.
|
||||
if !reportedOnSend(byNode[node], node, g) {
|
||||
if now.Sub(*g.Since) > inventory.ApplySilentAfter {
|
||||
g.Silent = true
|
||||
r.Silent = append(r.Silent, node)
|
||||
changed = true
|
||||
notes = append(notes, fmt.Sprintf("%s said nothing on its send within %s: left silent", node,
|
||||
inventory.ApplySilentAfter))
|
||||
continue
|
||||
}
|
||||
judging = append(judging, node+": not yet reported on its send")
|
||||
continue
|
||||
}
|
||||
if !gateFirstMachine {
|
||||
// **Judged by its report alone** (novox/hq ADR 0360 §7): applied passes; failed or refused is named and
|
||||
// fails the walk, with nothing put back — apply-failing says it.
|
||||
switch rep := byNode[node]; rep.Outcome {
|
||||
case inventory.OutcomeApplied:
|
||||
decide(g, inventory.GatePassed, node+" reported it applied its send (novox/hq ADR 0360)", now)
|
||||
passCarried(ctx, open, p, g, "")
|
||||
r.Done = append(r.Done, node)
|
||||
changed = true
|
||||
case inventory.OutcomeFailed, inventory.OutcomeRefused:
|
||||
decide(g, inventory.GateFailed, fmt.Sprintf("%s %s its send", node, rep.Outcome), now)
|
||||
r.Failed = append(r.Failed, node)
|
||||
changed = true
|
||||
notes = append(notes, fmt.Sprintf("%s %s its send: nothing put back, apply-failing says it", node, rep.Outcome))
|
||||
default:
|
||||
judging = append(judging, node+": not yet reported on its send")
|
||||
}
|
||||
continue
|
||||
}
|
||||
verdict, err := judgeMoves(ctx, open, g, nil, now)
|
||||
if err != nil {
|
||||
return changed, err
|
||||
}
|
||||
switch verdict {
|
||||
case "":
|
||||
if putBackBroken(ctx, open, p, g, "", nil) {
|
||||
changed = true
|
||||
notes = append(notes, p.Note)
|
||||
}
|
||||
judging = append(judging, node+": "+gateLine(g))
|
||||
case inventory.GatePassed:
|
||||
passCarried(ctx, open, p, g, "")
|
||||
r.Done = append(r.Done, node)
|
||||
changed = true
|
||||
case inventory.GateSuperseded:
|
||||
// Another send moved a judged module there (novox/hq issue 352): nothing judged, nothing put back on
|
||||
// it; what still waits is released again.
|
||||
r.Done = append(r.Done, node)
|
||||
changed = true
|
||||
notes = append(notes, fmt.Sprintf("superseded on %s: %s", node, g.Why))
|
||||
default:
|
||||
// **Put back by itself** (ADR 0353 §3): what was found wanting goes back at once, and every other
|
||||
// machine of the step goes on to its own verdict.
|
||||
stateWas := p.State
|
||||
p.Note = ""
|
||||
failCarried(batched, open, p, g, "")
|
||||
p.State = stateWas
|
||||
r.Failed = append(r.Failed, node)
|
||||
changed = true
|
||||
notes = append(notes, fmt.Sprintf("failed its gate on %s: %s — %s", node, g.Why, p.Note))
|
||||
returnedElsewhere(ctx, open, r, node, g)
|
||||
}
|
||||
}
|
||||
p.Note = ""
|
||||
sendRollbacks(ctx, open, p, back)
|
||||
if p.Note != "" {
|
||||
notes = append(notes, strings.TrimPrefix(p.Note, "; "))
|
||||
}
|
||||
if len(judging) > 0 {
|
||||
note := strings.Join(append(notes, "judging "+strings.Join(judging, "; ")), "; ")
|
||||
p.Note = note
|
||||
return changed || noteWas != note, nil
|
||||
}
|
||||
// Every gate of the step has its verdict or is silent: the next step, unless a machine failed (§3).
|
||||
if len(r.Failed) == 0 {
|
||||
opened, waiting, err := openRestStep(ctx, open, p)
|
||||
if err != nil {
|
||||
return changed, err
|
||||
}
|
||||
if opened {
|
||||
return true, nil
|
||||
}
|
||||
if waiting != "" {
|
||||
note := strings.Join(append(notes, waiting), "; ")
|
||||
p.Note = note
|
||||
return changed || noteWas != note, nil
|
||||
}
|
||||
}
|
||||
p.Tier = len(p.Tiers)
|
||||
end := fmt.Sprintf("released on %s", orNone(strings.Join(r.Done, ", ")))
|
||||
if len(r.Silent) > 0 {
|
||||
end += "; silent on their send, left: " + strings.Join(r.Silent, ", ")
|
||||
}
|
||||
if len(r.Skipped) > 0 {
|
||||
end += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ")
|
||||
}
|
||||
p.State = inventory.PlanDone
|
||||
if len(r.Failed) > 0 {
|
||||
p.State = inventory.PlanFailed
|
||||
end = "failed on " + strings.Join(r.Failed, ", ") + "; " + end
|
||||
} else if len(r.Silent) > 0 && len(r.Gates) == len(r.Silent) {
|
||||
// **A rest that said nothing is no success** (ADR 0353 §4): failure is loud; the condition says it.
|
||||
p.State = inventory.PlanFailed
|
||||
end = "no machine of the rest reported on its send; " + end
|
||||
}
|
||||
p.Note = strings.Join(append(notes, end), "; ")
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// keptStepGrew says why a step's kept sends would now carry a build no gate passed that they were not kept with;
|
||||
// empty when none would.
|
||||
func keptStepGrew(ctx context.Context, open *stores, r *inventory.PlanRelease, pending []string) (string, error) {
|
||||
if !gateFirstMachine {
|
||||
return "", nil // every machine goes at once (novox/hq ADR 0360 §7): a send carries what waits when it is made
|
||||
}
|
||||
f, err := readMoveFacts(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, node := range pending {
|
||||
moves, err := gatedMoves(ctx, open, node, nil)
|
||||
if errors.Is(err, errWalkedElsewhere) {
|
||||
continue // sendKept says it waits
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
kept := r.Gates[node].Carried
|
||||
for _, mv := range moves {
|
||||
// The node-engine and the resolver keep one machine at a time, passed or not (ADR 0353 §9).
|
||||
if (f.gated(mv.Module, mv.To) && !oneAtATime(mv.Module, shelf[mv.Module])) || slices.ContainsFunc(kept, func(c inventory.CarriedMove) bool {
|
||||
return c.Module == mv.Module && sameCommit(c.To, mv.To)
|
||||
}) {
|
||||
continue
|
||||
}
|
||||
return fmt.Sprintf("%s's build %s, which this step did not keep for it, came to wait on %s", mv.Module, short(mv.To), node), nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// reportedOnSend says a machine's last report is about the send its gate judges (novox/hq issue 352): the
|
||||
// declaration kept on the gate, or one sequenced after it; a gate that kept none reads the send made last.
|
||||
func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bool {
|
||||
if r.At == nil {
|
||||
return false
|
||||
}
|
||||
if s, kept := g.Sent[node]; kept {
|
||||
return s.ReportsOn(r)
|
||||
}
|
||||
return r.Current
|
||||
}
|
||||
|
||||
// returnedElsewhere marks, on every other gate of the rest still judging, the builds a failed machine's
|
||||
// rollback put back wherever they were sent (ADR 0236 §5): no longer theirs to judge (ADR 0353 §3).
|
||||
func returnedElsewhere(ctx context.Context, open *stores, r *inventory.PlanRelease, failed string, g *inventory.PlanGate) {
|
||||
for _, c := range g.Carried {
|
||||
if c.Build == "" {
|
||||
continue
|
||||
}
|
||||
if marked, err := open.inventory.GateFailed(ctx, c.Build); err != nil || !marked {
|
||||
continue
|
||||
}
|
||||
for node, other := range r.Gates {
|
||||
if node == failed || other.Verdict != "" || slices.Contains(other.Returned, c.Module) {
|
||||
continue
|
||||
}
|
||||
for _, oc := range other.Carried {
|
||||
if oc.Module == c.Module && sameCommit(oc.To, c.To) {
|
||||
other.Returned = append(other.Returned, c.Module)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// openRestStep chooses the next step of a release plan's rest and keeps its sends (novox/hq ADR 0353 §1, §2):
|
||||
// every machine left that is heard from and has something waiting, except that a build no gate has passed goes
|
||||
// to one machine of the step only. Answers whether a step was kept, or why the walk waits.
|
||||
func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, string, error) {
|
||||
r := p.Release
|
||||
heard, err := releaseHeard(ctx, open)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
f, err := readMoveFacts(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
waiting := map[string][]inventory.CarriedMove{}
|
||||
var nodes, walked []string
|
||||
for _, node := range r.Order[min(r.Next, len(r.Order)):] {
|
||||
if r.Gates[node] != nil || slices.Contains(r.Done, node) || slices.Contains(r.Skipped, node) ||
|
||||
slices.Contains(r.Failed, node) || slices.Contains(r.Silent, node) {
|
||||
continue
|
||||
}
|
||||
if !heard[node] {
|
||||
r.Skipped = append(r.Skipped, node)
|
||||
continue
|
||||
}
|
||||
moves, err := gatedMoves(ctx, open, node, nil)
|
||||
if errors.Is(err, errWalkedElsewhere) {
|
||||
walked = append(walked, fmt.Sprintf("%s (%v)", node, err))
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("reading what waits on %s: %w", node, err)
|
||||
}
|
||||
if len(moves) == 0 {
|
||||
r.Done = append(r.Done, node)
|
||||
continue
|
||||
}
|
||||
waiting[node] = moves
|
||||
nodes = append(nodes, node)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
// **Every machine at once** (novox/hq ADR 0360 §2, §7): no build waits for one machine to pass first.
|
||||
step := nodes
|
||||
if gateFirstMachine {
|
||||
step = restStep(nodes, waiting, f.gated, func(module string) bool { return oneAtATime(module, shelf[module]) })
|
||||
}
|
||||
if len(step) == 0 {
|
||||
if len(walked) > 0 {
|
||||
return false, "waiting before " + strings.Join(walked, "; "), nil
|
||||
}
|
||||
return false, "", nil
|
||||
}
|
||||
if r.Gates == nil {
|
||||
r.Gates = map[string]*inventory.PlanGate{}
|
||||
}
|
||||
for _, node := range step {
|
||||
r.Gates[node] = &inventory.PlanGate{Machines: []string{node}, Carried: waiting[node], Pending: true}
|
||||
}
|
||||
p.Note = fmt.Sprintf("sending %s at once, each judged there", strings.Join(step, ", "))
|
||||
if left := len(nodes) - len(step); left > 0 {
|
||||
p.Note += fmt.Sprintf("; %d machine(s) wait for a build that passed no gate yet to pass on one", left)
|
||||
}
|
||||
return true, "", nil
|
||||
}
|
||||
|
||||
// oneAtATime is a module whose builds go to the rest one machine at a time (novox/hq ADR 0353 §9): the
|
||||
// node-engine, which applies every send, and a module holding the mesh's resolver, which names the bus.
|
||||
func oneAtATime(module string, m catalogue.Manifest) bool {
|
||||
if module == hostModule {
|
||||
return true
|
||||
}
|
||||
return slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == resolverSeat })
|
||||
}
|
||||
|
||||
// restStep is the machines of one step of the rest (novox/hq ADR 0353 §2), in the order given: each machine, unless
|
||||
// a build it would carry has passed no gate and an earlier machine of the step already carries it — that build's
|
||||
// first machine. Pure.
|
||||
//
|
||||
// **The node-engine and the mesh's resolver still go one machine at a time** (ADR 0353 §9): a step carries a
|
||||
// build of either to one machine only, judged before the next, since a fault that cuts a machine off the bus
|
||||
// shows only as its silence and no rollback reaches it.
|
||||
func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed func(module, commit string) bool,
|
||||
alone func(module string) bool) []string {
|
||||
first := map[string]string{}
|
||||
var out []string
|
||||
for _, n := range nodes {
|
||||
var unpassed []string
|
||||
for _, mv := range waiting[n] {
|
||||
if !passed(mv.Module, mv.To) || alone(mv.Module) {
|
||||
unpassed = append(unpassed, mv.Module+"@"+mv.To)
|
||||
}
|
||||
}
|
||||
if slices.ContainsFunc(unpassed, func(k string) bool { _, taken := first[k]; return taken }) {
|
||||
continue
|
||||
}
|
||||
for _, k := range unpassed {
|
||||
first[k] = n
|
||||
}
|
||||
out = append(out, n)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// silentObservation is what the gate's probe says of the newest release whose rest went silent (novox/hq ADR 0353
|
||||
// §4): machines that never reported on their send, left as they were — loud, never a success said by nothing.
|
||||
func silentObservation(ctx context.Context, inv *inventory.Inventory) ([]conditions.Observation, error) {
|
||||
recent, err := inv.RecentPlans(ctx, 20)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, p := range recent {
|
||||
if p.Release == nil || p.Release.Settings != nil || p.Open() {
|
||||
continue
|
||||
}
|
||||
if len(p.Release.Silent) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
machines := append([]string(nil), p.Release.Silent...)
|
||||
sort.Strings(machines)
|
||||
o := conditions.Observation{Scope: conditions.ScopeMesh, ID: "release", Token: "silent", Kind: "release-silent",
|
||||
Severity: conditions.Warning, Resolver: conditions.ResolverOperator, Also: machines,
|
||||
Summary: fmt.Sprintf("%s: %s said nothing on the send of the builds that waited for a gate within %s, and were "+
|
||||
"left as they were", p.ID, strings.Join(machines, ", "), inventory.ApplySilentAfter)}
|
||||
w := releaseSilentWords(machines)
|
||||
o.Headline, o.Explanation, o.Resolved, o.Needs = w.Headline, w.Explanation, w.Resolved, w.Needs
|
||||
return []conditions.Observation{o}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// backlogObservation is what the gate's probe says of a release held after a failure.
|
||||
func backlogObservation() []conditions.Observation {
|
||||
if backlogNow.held == "" || len(backlogNow.waiting) == 0 {
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A pull request into a release branch is judged by that branch** (novox/hq ADR 0358, issue 704): a base that is
|
||||
// not its repository's trunk, where mesh-controller has a branch of that name, is a release branch; the check then
|
||||
// reads every repository beside it at its branch of that name where it has one, else at its trunk. Any other base is
|
||||
// checked exactly as before.
|
||||
|
||||
const aRelease = "feat/57-config-and-provisioning"
|
||||
|
||||
// theCore is the mesh's core as a check reads it beside: the controller, the node-engine, the tool runner and the
|
||||
// catalogue's bus, on the git seat.
|
||||
func theCore() []inventory.Entry {
|
||||
onSeat := func(module, repository string) inventory.Entry {
|
||||
e := fromRepo(module, repository, "")
|
||||
e.Source.Seat = "git"
|
||||
return e
|
||||
}
|
||||
nats := onSeat("nats", "novox/mesh-catalog")
|
||||
nats.Source.Path = "modules/nats"
|
||||
return []inventory.Entry{onSeat("mesh-controller", "novox/mesh-controller"), onSeat("mesh-host", "novox/mesh-host"),
|
||||
onSeat("node-tools", "novox/mesh-tools"), nats, onSeat("gitea", "novox/mesh-catalog")}
|
||||
}
|
||||
|
||||
// aForgeWithBranches stands in for the forge: which branches each repository has, and how often it was asked.
|
||||
func aForgeWithBranches(t *testing.T, branches map[string][]string, failing error) *int {
|
||||
t.Helper()
|
||||
asked := 0
|
||||
was := forgeHasBranch
|
||||
t.Cleanup(func() { forgeHasBranch = was })
|
||||
forgeHasBranch = func(_ context.Context, owner, repo, branch string) (bool, error) {
|
||||
asked++
|
||||
if failing != nil {
|
||||
return false, failing
|
||||
}
|
||||
for _, b := range branches[owner+"/"+repo] {
|
||||
if b == branch {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
return &asked
|
||||
}
|
||||
|
||||
func TestABaseIsAReleaseBranchWhenTheControllerHasABranchOfThatName(t *testing.T) {
|
||||
entries := theCore()
|
||||
scopeOf := func(p link.PullUpdated) checkScope { return pullScope(p, entries, nil, nil) }
|
||||
pull := func(repo, base string) link.PullUpdated {
|
||||
return link.PullUpdated{Owner: "novox", Repo: repo, Number: 286, Base: base, Commit: "abc",
|
||||
Paths: []string{"modules/gitea/index.ts"}}
|
||||
}
|
||||
|
||||
asked := aForgeWithBranches(t, map[string][]string{"novox/mesh-controller": {"main", aRelease}}, nil)
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
p link.PullUpdated
|
||||
release string
|
||||
asks int
|
||||
}{
|
||||
{"a pull request into the trunk asks nothing and is no release", pull("mesh-catalog", "main"), "", 0},
|
||||
{"a catalogue pull request into the release branch", pull("mesh-catalog", aRelease), aRelease, 1},
|
||||
{"a controller pull request into the release branch", pull("mesh-controller", aRelease), aRelease, 1},
|
||||
{"a base mesh-controller has no branch of is checked as today", pull("mesh-catalog", "feat/other"), "", 1},
|
||||
{"no base: as today", pull("mesh-catalog", ""), "", 0},
|
||||
} {
|
||||
*asked = 0
|
||||
got, err := releaseBranchOf(t.Context(), c.p, scopeOf(c.p), entries)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", c.what, err)
|
||||
}
|
||||
if got != c.release || *asked != c.asks {
|
||||
t.Errorf("%s: release %q after %d question(s) to the forge, not %q after %d", c.what, got, *asked,
|
||||
c.release, c.asks)
|
||||
}
|
||||
}
|
||||
|
||||
// The branch a module built from the repository follows is its trunk: never asked about.
|
||||
photos := fromRepo("photos", "novox/photos", "")
|
||||
photos.Source.Ref = "nox-mesh"
|
||||
withPhotos := append(theCore(), photos)
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "photos", Base: "nox-mesh", Commit: "abc", Paths: []string{"x.js"}}
|
||||
*asked = 0
|
||||
if got, err := releaseBranchOf(t.Context(), p, pullScope(p, withPhotos, nil, nil), withPhotos); err != nil ||
|
||||
got != "" || *asked != 0 {
|
||||
t.Fatalf("a module's own trunk read as release %q (%v) after %d question(s)", got, err, *asked)
|
||||
}
|
||||
|
||||
// A forge that cannot answer is an error, never "no release branch".
|
||||
aForgeWithBranches(t, nil, errors.New("nothing serves gitea.gitea_api"))
|
||||
q := pull("mesh-catalog", aRelease)
|
||||
if _, err := releaseBranchOf(t.Context(), q, scopeOf(q), entries); err == nil ||
|
||||
!strings.Contains(err.Error(), "cannot be told") {
|
||||
t.Fatalf("an unanswered forge was read as an answer: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBesideAReleaseBranchCheckEachRepositoryIsReadAtThatBranchElseItsTrunk(t *testing.T) {
|
||||
entries := theCore()
|
||||
clone := func(s inventory.Source) (string, error) { return "http://git/" + s.Repository + ".git", nil }
|
||||
running := func(module string) string { return "running-" + module }
|
||||
|
||||
beside, err := besideOf(entries, running, clone, aRelease)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
at := func(repo string, ref, orElse string) link.CheckedOut {
|
||||
return link.CheckedOut{Repository: "http://git/novox/" + repo + ".git", Ref: ref, Else: orElse}
|
||||
}
|
||||
want := map[string]link.CheckedOut{
|
||||
// The judge: mesh-controller's release branch, which it has.
|
||||
"mesh-controller": at("mesh-controller", aRelease, ""),
|
||||
"mesh-controller-main": at("mesh-controller", "main", ""),
|
||||
// A controller pull request into the release branch reads mesh-host at that branch where it has one.
|
||||
"mesh-host": at("mesh-host", aRelease, "main"),
|
||||
"mesh-tools": at("mesh-tools", aRelease, "main"),
|
||||
"mesh-catalog": at("mesh-catalog", aRelease, "main"),
|
||||
// mesh-lab at its trunk, always: its replays run with the container runtime's socket.
|
||||
"mesh-lab": at("mesh-lab", "main", ""),
|
||||
"mesh-sdk": at("mesh-sdk", aRelease, "main"),
|
||||
}
|
||||
if !reflect.DeepEqual(beside, want) {
|
||||
t.Fatalf("beside a release-branch check:\n got %v\nwant %v", beside, want)
|
||||
}
|
||||
|
||||
// Any other base: exactly as before — what the mesh runs, the catalogue's and the lab's main.
|
||||
beside, err = besideOf(entries, running, clone, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want = map[string]link.CheckedOut{
|
||||
"mesh-controller": at("mesh-controller", "running-mesh-controller", ""),
|
||||
"mesh-controller-main": at("mesh-controller", "main", ""),
|
||||
"mesh-host": at("mesh-host", "running-mesh-host", ""),
|
||||
"mesh-tools": at("mesh-tools", "running-node-tools", ""),
|
||||
"mesh-catalog": at("mesh-catalog", "main", ""),
|
||||
"mesh-lab": at("mesh-lab", "main", ""),
|
||||
"mesh-sdk": at("mesh-sdk", sdkPinned(), ""),
|
||||
}
|
||||
if !reflect.DeepEqual(beside, want) {
|
||||
t.Fatalf("beside a trunk check:\n got %v\nwant %v", beside, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReleaseBranchVerdictNamesTheBranchTheJudgeAndWhatItRead(t *testing.T) {
|
||||
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 286, Base: aRelease, Release: aRelease}
|
||||
c := checkedOf(link.BuildResult{ID: "b", Ref: "aaaaaaaaaaaa0000", Checked: asked, Check: &link.CheckOutcome{
|
||||
Verdict: "pass", Head: "aaaaaaaaaaaa", Trunk: "111111111111", Gate: &link.CheckLayer{Verdict: "pass"},
|
||||
RepoCheck: &link.CheckLayer{Verdict: "pass"}, Release: aRelease, JudgedBy: "fcdecb436117",
|
||||
Beside: map[string]string{"mesh-host": "222222222222", "mesh-lab": "333333333333"},
|
||||
BesideAt: map[string]string{"mesh-host": aRelease, "mesh-lab": "main"},
|
||||
Validator: "mesh-host's validator of " + aRelease + " at 222222222222"}})
|
||||
if c.Release != aRelease || c.JudgedBy != "fcdecb436117" {
|
||||
t.Fatalf("the verdict names release %q judged by %q", c.Release, c.JudgedBy)
|
||||
}
|
||||
if c.BesideAt["mesh-host"] != aRelease || c.BesideAt["mesh-lab"] != "main" || c.Beside["mesh-host"] != "222222222222" ||
|
||||
c.Validator != "mesh-host's validator of "+aRelease+" at 222222222222" {
|
||||
t.Fatalf("the verdict names what it read beside it as %v at %v", c.Beside, c.BesideAt)
|
||||
}
|
||||
}
|
||||
|
||||
// **Clause 5** (ADR 0358 §5): a release branch's own pull request into its trunk is no release-branch check: the forge
|
||||
// is asked nothing, the request names no release branch, and the repositories beside it are read as for any trunk
|
||||
// pull request. Its delivery group's composed check is asked as for any other (askGroupCheck, unchanged).
|
||||
func TestAReleaseBranchsPullRequestIntoItsTrunkIsCheckedAsAnyTrunkPullRequest(t *testing.T) {
|
||||
entries := theCore()
|
||||
asked := aForgeWithBranches(t, map[string][]string{"novox/mesh-controller": {"main", aRelease}}, nil)
|
||||
for _, repo := range []string{"mesh-catalog", "mesh-controller", "mesh-host"} {
|
||||
p := link.PullUpdated{Owner: "novox", Repo: repo, Number: 400, Head: aRelease, Base: "main", Commit: "abc",
|
||||
Paths: []string{"modules/gitea/index.ts"}}
|
||||
got, err := releaseBranchOf(t.Context(), p, pullScope(p, entries, nil, nil), entries)
|
||||
if err != nil || got != "" || *asked != 0 {
|
||||
t.Fatalf("%s's release branch into main read as release %q (%v) after %d question(s)", repo, got, err, *asked)
|
||||
}
|
||||
}
|
||||
clone := func(s inventory.Source) (string, error) { return "http://git/" + s.Repository + ".git", nil }
|
||||
beside, err := besideOf(entries, func(module string) string { return "running-" + module }, clone, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for dir, b := range beside {
|
||||
if b.Ref == aRelease || b.Else != "" {
|
||||
t.Fatalf("beside a trunk pull request %s is read at %+v", dir, b)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A forge answer that names no branch is an error**, never "no release branch"; a 404 is "no such branch".
|
||||
func TestTheForgesAnswerAboutABranchIsReadStrictly(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
answer link.Answer
|
||||
has bool
|
||||
fails bool
|
||||
}{
|
||||
{"the branch named", link.Answer{Result: []byte(`{"result":{"name":"` + aRelease + `"}}`)}, true, false},
|
||||
{"not found", link.Answer{Error: "Gitea API /repos/novox/mesh-controller/branches/x: 404 {\"message\":\"not found\"}"},
|
||||
false, false},
|
||||
{"an empty branch name", link.Answer{Result: []byte(`{"result":{"name":""}}`)}, false, true},
|
||||
{"no result", link.Answer{Result: []byte(`{}`)}, false, true},
|
||||
{"another branch named", link.Answer{Result: []byte(`{"result":{"name":"main"}}`)}, false, true},
|
||||
{"unreadable", link.Answer{Result: []byte(`not json`)}, false, true},
|
||||
{"another error", link.Answer{Error: "Gitea API: 500 internal"}, false, true},
|
||||
} {
|
||||
has, err := branchAnswered(c.answer, aRelease)
|
||||
if has != c.has || (err != nil) != c.fails {
|
||||
t.Errorf("%s: has %v, error %v", c.what, has, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **The trunk is the repository's own, not a name**: a repository whose trunk is not main, one that follows main, and
|
||||
// a controller whose trunk is not main.
|
||||
func TestATrunkIsTheBranchTheRepositoryFollows(t *testing.T) {
|
||||
asked := aForgeWithBranches(t, map[string][]string{"novox/mesh-controller": {"main", aRelease, "master"}}, nil)
|
||||
legacy := fromRepo("legacy", "novox/legacy", "")
|
||||
legacy.Source.Ref = "master"
|
||||
onMain := fromRepo("photos", "novox/photos", "")
|
||||
onMain.Source.Ref = "main"
|
||||
entries := append(theCore(), legacy, onMain)
|
||||
for _, c := range []struct {
|
||||
what, repo, base, release string
|
||||
asks int
|
||||
}{
|
||||
{"a repository whose trunk is master, into master", "legacy", "master", "", 0},
|
||||
{"a repository whose trunk is master, into the release branch", "legacy", aRelease, aRelease, 1},
|
||||
{"a repository whose module follows main, into main", "photos", "main", "", 0},
|
||||
{"a repository whose module follows main, into the release branch", "photos", aRelease, aRelease, 1},
|
||||
} {
|
||||
*asked = 0
|
||||
p := link.PullUpdated{Owner: "novox", Repo: c.repo, Base: c.base, Commit: "abc", Paths: []string{"x"}}
|
||||
got, err := releaseBranchOf(t.Context(), p, pullScope(p, entries, nil, nil), entries)
|
||||
if err != nil || got != c.release || *asked != c.asks {
|
||||
t.Errorf("%s: release %q (%v) after %d question(s), not %q after %d", c.what, got, err, *asked, c.release, c.asks)
|
||||
}
|
||||
}
|
||||
|
||||
// A controller whose trunk is not main: main may be a release branch there, and its own trunk never is.
|
||||
core := theCore()
|
||||
core[0].Source.Ref = "trunk"
|
||||
for _, c := range []struct {
|
||||
base, release string
|
||||
asks int
|
||||
}{{"trunk", "", 0}, {"main", "main", 1}} {
|
||||
*asked = 0
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "hq", Base: c.base, Commit: "abc", Paths: []string{"x"}}
|
||||
got, err := releaseBranchOf(t.Context(), p, pullScope(p, core, nil, nil), core)
|
||||
if err != nil || got != c.release || *asked != c.asks {
|
||||
t.Errorf("hq into %s, the controller following trunk: release %q (%v) after %d question(s)", c.base, got, err,
|
||||
*asked)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// trunkOf: the branch a source follows, and main when it records none or records a commit (issue 215's rule).
|
||||
func TestATrunkIsMainWhenTheSourceRecordsNone(t *testing.T) {
|
||||
for ref, want := range map[string]string{"": "main", "main": "main", "master": "master", "nox-mesh": "nox-mesh",
|
||||
"0123456789abcdef0123456789abcdef01234567": "main", " ": "main"} {
|
||||
if got := trunkOf(inventory.Source{Ref: ref}); got != want {
|
||||
t.Errorf("a source recording %q follows %q, not %q", ref, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9,10 +9,13 @@ import (
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -25,6 +28,10 @@ import (
|
||||
// alone cannot (a tier waiting for machines to report); a controller replaced mid-plan finds the
|
||||
// plan where it left it.
|
||||
|
||||
// gateFirstMachine sends a module that rolls out to one machine first and judges it there (ADR 0236). Off since
|
||||
// novox/hq ADR 0360: the gate's code is kept for the walks a controller before it started, and its tests turn it on.
|
||||
var gateFirstMachine = false
|
||||
|
||||
// planWaitBound is how long a plan may wait on one thing before `status` names it red.
|
||||
const planWaitBound = 30 * time.Minute
|
||||
|
||||
@@ -41,29 +48,15 @@ func tiersOf(set []string, edges []inventory.Edge) [][]string {
|
||||
for _, m := range set {
|
||||
deps[m] = map[string]bool{}
|
||||
}
|
||||
// The build seat's holders follow the controller that defines their worker (EdgeWorkerOf,
|
||||
// novox/hq issue 206), so the built-by edge from that controller to such a holder yields: the
|
||||
// controller is built by whichever build machine is running, as the runtime image always was.
|
||||
worker := map[string]map[string]bool{}
|
||||
for _, e := range edges {
|
||||
if e.Kind == inventory.EdgeWorkerOf && in[e.From] && in[e.To] {
|
||||
if worker[e.To] == nil {
|
||||
worker[e.To] = map[string]bool{}
|
||||
}
|
||||
worker[e.To][e.From] = true
|
||||
}
|
||||
}
|
||||
for _, e := range edges {
|
||||
// A code dependency — B packages A's source — rebuilds B with A, in the same tier: B's
|
||||
// build needs nothing of A's first. The other kinds order: stands-on and declared after
|
||||
// the base is built, built-by after the build machine is built and running — except for
|
||||
// what the build machine itself stands on, and for the controller whose worker the build
|
||||
// machine binds. The runtime image is built by the builder and the builder is built on the
|
||||
// runtime image; the image comes first, built by the builder that is running.
|
||||
// build needs nothing of A's first. Stands-on and declared order after the base is built, and the
|
||||
// build seat's holder follows the controller that defines its worker (EdgeWorkerOf, issue 206).
|
||||
if !in[e.From] || !in[e.To] || e.From == e.To || e.Kind == inventory.EdgePackages {
|
||||
continue
|
||||
}
|
||||
if e.Kind == inventory.EdgeBuiltBy && (isBaseOf(e.From, e.To, edges, in) || worker[e.From][e.To]) {
|
||||
// **Being built by the builder orders nothing** (novox/hq ADR 0360 §3): whichever builder runs builds it.
|
||||
if e.Kind == inventory.EdgeBuiltBy {
|
||||
continue
|
||||
}
|
||||
deps[e.From][e.To] = true
|
||||
@@ -104,29 +97,6 @@ func tiersOf(set []string, edges []inventory.Edge) [][]string {
|
||||
return tiers
|
||||
}
|
||||
|
||||
// isBaseOf says whether `to` stands on `base`, directly or through other bases in the set, along
|
||||
// the build edges alone.
|
||||
func isBaseOf(base, to string, edges []inventory.Edge, in map[string]bool) bool {
|
||||
seen := map[string]bool{}
|
||||
var walk func(string) bool
|
||||
walk = func(m string) bool {
|
||||
if m == base {
|
||||
return true
|
||||
}
|
||||
if seen[m] {
|
||||
return false
|
||||
}
|
||||
seen[m] = true
|
||||
for _, e := range edges {
|
||||
if e.From == m && in[e.To] && (e.Kind == inventory.EdgeStandsOn || e.Kind == inventory.EdgeDeclared) && walk(e.To) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
return walk(to)
|
||||
}
|
||||
|
||||
// reachableFrom is the moved modules plus everything that depends on them, through every layer:
|
||||
// what a merge rebuilds. Along the code and build edges only: a module *built by* the build machine
|
||||
// is not changed by a new build machine, so a built-by edge orders and gates a plan and never
|
||||
@@ -162,7 +132,7 @@ func reachableFrom(moved []string, edges []inventory.Edge) []string {
|
||||
}
|
||||
|
||||
// hasCycle says whether the tiers' last tier holds modules that still depend on each other. A packages
|
||||
// edge orders nothing (tiersOf), so a module and what packages its source share a tier by rule: that is
|
||||
// or built-by edge orders nothing (tiersOf), so a module and what packages its source share a tier by rule: that is
|
||||
// no cycle, and saying one was is a false report in every such plan's log.
|
||||
func hasCycle(tiers [][]string, edges []inventory.Edge) bool {
|
||||
if len(tiers) == 0 {
|
||||
@@ -173,7 +143,7 @@ func hasCycle(tiers [][]string, edges []inventory.Edge) bool {
|
||||
last[m] = true
|
||||
}
|
||||
for _, e := range edges {
|
||||
if e.Kind == inventory.EdgePackages {
|
||||
if e.Kind == inventory.EdgePackages || e.Kind == inventory.EdgeBuiltBy {
|
||||
continue
|
||||
}
|
||||
if last[e.From] && last[e.To] {
|
||||
@@ -219,12 +189,6 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool)
|
||||
return nil
|
||||
}
|
||||
inTier := map[string]bool{}
|
||||
all := map[string]bool{}
|
||||
for _, tier := range p.Tiers {
|
||||
for _, m := range tier {
|
||||
all[m] = true
|
||||
}
|
||||
}
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
inTier[m] = true
|
||||
}
|
||||
@@ -239,8 +203,8 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool)
|
||||
for _, e := range edges {
|
||||
// A delivery group's order inside a walk (novox/hq ADR 0276 decision 5) gates as built-by does: the
|
||||
// member before is running on its machines before the member after is asked.
|
||||
ordered := e.Kind == edgeGroupOrder ||
|
||||
e.Kind == inventory.EdgeBuiltBy && !isBaseOf(e.From, e.To, edges, all)
|
||||
// A built-by edge waits for nothing since novox/hq ADR 0360 §3.
|
||||
ordered := e.Kind == edgeGroupOrder
|
||||
if later[e.From] && inTier[e.To] && ordered && !seen[e.To] && rollsOut(e.To) {
|
||||
seen[e.To] = true
|
||||
out = append(out, e.To)
|
||||
@@ -250,6 +214,20 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool)
|
||||
return out
|
||||
}
|
||||
|
||||
// noMoveWhy is what a plan says of a module of its tier whose build was made from the source every machine
|
||||
// running it runs (novox/hq issue 280): nothing is sent for it.
|
||||
// noMovePrefix opens it, and is what noMove reads: one prefix for the writer and the reader.
|
||||
const (
|
||||
noMovePrefix = "no move"
|
||||
noMoveWhy = noMovePrefix + ": made from the source every machine running it runs"
|
||||
)
|
||||
|
||||
// noMove is whether the plan marked the module no move and sent it nothing. Read from what the plan says,
|
||||
// so a plan kept before the mark had a name of its own is read the same (novox/hq issue 648).
|
||||
func noMove(s *inventory.PlanModule) bool {
|
||||
return s != nil && s.FirstAt == nil && s.Gate == nil && strings.HasPrefix(s.Why, noMovePrefix)
|
||||
}
|
||||
|
||||
// applied says whether every machine running the module has reported since the module was built.
|
||||
// appliedEach is applied with a moment of its own for each machine: the reports that count are the ones
|
||||
// after that machine was sent the build (novox/hq issue 256).
|
||||
@@ -313,6 +291,144 @@ func init() {
|
||||
}
|
||||
}
|
||||
|
||||
// reuseKey carries a walk's leave to take a check's build from askModule to the ask it makes (novox/hq ADR 0350 §2).
|
||||
type reuseKey struct{}
|
||||
|
||||
// withReuse is ctx carrying a walk's leave to take a check's build.
|
||||
func withReuse(ctx context.Context, r *link.ReuseAsk) context.Context {
|
||||
return context.WithValue(ctx, reuseKey{}, r)
|
||||
}
|
||||
|
||||
// reuseIn is the leave ctx carries; nil for every ask but a walk's.
|
||||
func reuseIn(ctx context.Context) *link.ReuseAsk {
|
||||
r, _ := ctx.Value(reuseKey{}).(*link.ReuseAsk)
|
||||
return r
|
||||
}
|
||||
|
||||
// reuseAsked is the leave an ask carries: a walk's, and never a dry run's — a dry run registers nothing, so it
|
||||
// takes nothing either (novox/hq ADR 0350 §2, §6).
|
||||
func reuseAsked(ctx context.Context, dryRun bool) *link.ReuseAsk {
|
||||
if dryRun {
|
||||
return nil
|
||||
}
|
||||
return reuseIn(ctx)
|
||||
}
|
||||
|
||||
// reuseAskOf is a walk's leave to take a check's build: the pull requests of the merges it answers.
|
||||
func reuseAskOf(p *inventory.Plan) *link.ReuseAsk {
|
||||
r := &link.ReuseAsk{}
|
||||
if p.Delivery == nil {
|
||||
return r
|
||||
}
|
||||
seen := map[link.PullRef]bool{}
|
||||
for _, m := range p.Delivery.Merges {
|
||||
pull, ok := pullOf(m)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if !seen[pull] {
|
||||
seen[pull] = true
|
||||
r.Pulls = append(r.Pulls, pull)
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// checkedBuildsOf are the builds the build seat's checks of those pull requests said they made for their merges,
|
||||
// within the week check builds are kept (novox/hq ADR 0350 §2): read from the seat's own outcomes on the bus. A
|
||||
// variable so a test needs no bus.
|
||||
var checkedBuildsOf = func(ctx context.Context, pulls []link.PullRef) ([]link.CheckedBuild, error) {
|
||||
if len(pulls) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer js.Close()
|
||||
return link.ReadCheckBuilds(ctx, js, buildSeatHeld(ctx), pulls, time.Now().Add(-builder.CheckBuildsKept))
|
||||
}
|
||||
|
||||
// mergedHeadsOf are the head commits the walk's merges merged, by pull request (novox/hq issue 703): a pull request
|
||||
// merged twice in one walk, or whose head the announcement did not say, has none, and takes no check's build.
|
||||
func mergedHeadsOf(p *inventory.Plan) map[link.PullRef]string {
|
||||
heads := map[link.PullRef]string{}
|
||||
if p.Delivery == nil {
|
||||
return heads
|
||||
}
|
||||
seen := map[link.PullRef]bool{}
|
||||
for _, m := range p.Delivery.Merges {
|
||||
pull, ok := pullOf(m)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if seen[pull] && heads[pull] != m.Head {
|
||||
heads[pull] = ""
|
||||
continue
|
||||
}
|
||||
seen[pull] = true
|
||||
heads[pull] = m.Head
|
||||
}
|
||||
return heads
|
||||
}
|
||||
|
||||
// walkVerdicts are the checks' builds a walk read, by the walk's plan: read once per walk (novox/hq issue 649), not
|
||||
// once per module it asks for. Kept a day; a walk lasts minutes.
|
||||
var walkVerdicts = struct {
|
||||
sync.Mutex
|
||||
read map[string]verdictsRead
|
||||
}{read: map[string]verdictsRead{}}
|
||||
|
||||
type verdictsRead struct {
|
||||
at time.Time
|
||||
builds []link.CheckedBuild
|
||||
}
|
||||
|
||||
// reuseWithVerdicts is a walk's leave with the builds its pull requests' verdicts name: the only builds the builder
|
||||
// may take (ADR 0350 §2), and **only those of a check of the head each merge merged** (novox/hq issue 703): a build
|
||||
// a check made at an earlier head of the same pull request is never registered. Read once per walk. When they
|
||||
// cannot be read the leave names none, and the walk builds, saying why.
|
||||
func reuseWithVerdicts(ctx context.Context, walk string, r *link.ReuseAsk, heads map[link.PullRef]string) *link.ReuseAsk {
|
||||
walkVerdicts.Lock()
|
||||
read, known := walkVerdicts.read[walk]
|
||||
walkVerdicts.Unlock()
|
||||
if !known || walk == "" {
|
||||
builds, err := checkedBuildsOf(ctx, r.Pulls)
|
||||
if err != nil {
|
||||
fmt.Printf(" the checks' builds of %d pull request(s) could not be read, so the walk builds: %v\n", len(r.Pulls), err)
|
||||
return r
|
||||
}
|
||||
read = verdictsRead{at: time.Now(), builds: builds}
|
||||
if walk != "" {
|
||||
walkVerdicts.Lock()
|
||||
for id, w := range walkVerdicts.read {
|
||||
if time.Since(w.at) > 24*time.Hour {
|
||||
delete(walkVerdicts.read, id)
|
||||
}
|
||||
}
|
||||
walkVerdicts.read[walk] = read
|
||||
walkVerdicts.Unlock()
|
||||
}
|
||||
}
|
||||
r.Builds = nil
|
||||
for _, b := range read.builds {
|
||||
if link.SameHead(b.Head, heads[b.Pull]) {
|
||||
r.Builds = append(r.Builds, b)
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// pullOf is a merge's pull request: owner and repository, the last two parts of however the repository is spelled
|
||||
// (owner/repo, a path on the git seat, a URL), and its number. False when the merge names none.
|
||||
func pullOf(m inventory.PlanMerge) (link.PullRef, bool) {
|
||||
parts := strings.Split(strings.Trim(strings.TrimSuffix(m.Repository, ".git"), "/"), "/")
|
||||
if len(parts) < 2 || m.Number <= 0 || parts[len(parts)-2] == "" || parts[len(parts)-1] == "" {
|
||||
return link.PullRef{}, false
|
||||
}
|
||||
return link.PullRef{Owner: parts[len(parts)-2], Repo: parts[len(parts)-1], Number: m.Number}, true
|
||||
}
|
||||
|
||||
// askModule asks the build machine for one module of a plan and marks it asked, with the id it was
|
||||
// asked under (novox/hq ADR 0219) — or failed, with the plan, when it could not be asked.
|
||||
func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[string]inventory.Entry) {
|
||||
@@ -340,7 +456,10 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
|
||||
if p.Delivery != nil && p.Delivery.Alone && carried != "" {
|
||||
ref = carried
|
||||
}
|
||||
id, err := askABuild(ctx, source, e.Source.Path, ref)
|
||||
// **The walk takes the check's build of the same key** (novox/hq ADR 0350 §2): its ask says so, with the pull
|
||||
// requests it answers, whose checks name the part of a differing key.
|
||||
id, err := askABuild(withReuse(ctx, reuseWithVerdicts(ctx, p.ID, reuseAskOf(p), mergedHeadsOf(p))), source,
|
||||
e.Source.Path, ref)
|
||||
if err != nil {
|
||||
state.State = "failed"
|
||||
state.Why = err.Error()
|
||||
@@ -359,7 +478,7 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
|
||||
state.State = "asked"
|
||||
state.AskedAt = &now
|
||||
state.Build = id
|
||||
state.Why, state.Commit, state.BuiltAt = "", "", nil
|
||||
state.Why, state.Commit, state.BuiltAt, state.Made = "", "", nil, ""
|
||||
}
|
||||
|
||||
// planBuilt marks a module built (or failed) in every open plan whose current tier holds it, and
|
||||
@@ -371,6 +490,12 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
|
||||
// would send machines, and the next tier, what the later merge replaced. asked is zero when the
|
||||
// build's request time is not known, and such an outcome is taken as before.
|
||||
func planBuilt(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id string) {
|
||||
planBuiltSaying(ctx, open, module, commit, failed, asked, id, "")
|
||||
}
|
||||
|
||||
// planBuiltSaying is planBuilt keeping what the build did — "reused from <check build>" or "built: <why>" (novox/hq
|
||||
// ADR 0350 §4) — on the module of the walk it answers.
|
||||
func planBuiltSaying(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id, made string) {
|
||||
inv := open.inventory
|
||||
// One controller works the plans at a time (novox/hq issue 213); an outcome waits its turn rather
|
||||
// than write over what the holder is about to save. Not taken, it is still in the build records,
|
||||
@@ -436,6 +561,7 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
|
||||
state.State = "built"
|
||||
state.BuiltAt = &now
|
||||
state.Commit = commit
|
||||
state.Made = made
|
||||
}
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
||||
@@ -505,7 +631,7 @@ func advancePlans(ctx context.Context, open *stores) {
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
// What waits for a gate, released one machine at a time (ADR 0236).
|
||||
// What waits for a gate, released one machine first, then the rest at once (ADR 0236, ADR 0353).
|
||||
if _, err := releaseBacklog(ctx, open, ""); err != nil {
|
||||
fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err)
|
||||
}
|
||||
@@ -715,7 +841,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
} else if same {
|
||||
now := time.Now().UTC()
|
||||
state.SentAt = &now
|
||||
state.Why = "no move: made from the source every machine running it runs"
|
||||
state.Why = noMoveWhy
|
||||
fmt.Printf("%s: %s built from the source every machine running it runs: no move, nothing sent\n", p.ID, m)
|
||||
continue
|
||||
}
|
||||
@@ -724,6 +850,12 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
// **Every machine at once** (novox/hq ADR 0360 §1, §2): a module that rolls out is sent to every machine
|
||||
// running it in one step, with no first machine and no gate, whatever module it is. A module a controller
|
||||
// before ADR 0360 already sent to its first machine finishes under its gate.
|
||||
if !gateFirstMachine && state.FirstAt == nil {
|
||||
policy.Together = true
|
||||
}
|
||||
var reports []inventory.Reported
|
||||
if !policy.Together {
|
||||
// Read for the choice of the first machine as well as for its report.
|
||||
@@ -905,6 +1037,25 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
if state.BuiltAt != nil {
|
||||
since = *state.BuiltAt
|
||||
}
|
||||
// **A build that is no move is waited on by the send that last carried it** (novox/hq issue 648).
|
||||
// Nothing was sent for it, so no report follows its marking or its build: a machine that does not
|
||||
// happen to report again held the walk for ever. Each machine runs it once it answered its last
|
||||
// send — the one whose builds made it no move — whenever that was.
|
||||
if noMove(state) {
|
||||
byNode := map[string]inventory.Reported{}
|
||||
for _, r := range reports {
|
||||
byNode[r.Node] = r
|
||||
}
|
||||
if ok, on := appliedEach(m, func(node string) time.Time {
|
||||
if s := byNode[node].Sent; s != nil {
|
||||
return *s
|
||||
}
|
||||
return time.Time{}
|
||||
}, running, reports); !ok {
|
||||
waiting = append(waiting, fmt.Sprintf("%s on %s", m, strings.Join(on, ", ")))
|
||||
}
|
||||
continue
|
||||
}
|
||||
// **Each machine from its own send** (novox/hq issue 256). With one machine first (ADR 0218)
|
||||
// a module is sent twice — the first machine, then the rest — and SentAt is the second.
|
||||
// Asked of every machine, the first machine's report, made between the two sends, read as
|
||||
@@ -934,15 +1085,92 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
return changed, nil
|
||||
}
|
||||
}
|
||||
// **A walk ends on its machines' reports** (novox/hq ADR 0360 §4, §5): after its last tier, every machine it sent
|
||||
// has reported on that send, or is silent past ApplySilentAfter; never a health reading.
|
||||
end := ""
|
||||
if p.Tier == len(p.Tiers)-1 {
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
waiting, failed, silent := walkReports(*p, reports, time.Now().UTC())
|
||||
if len(waiting) > 0 {
|
||||
note := "sent; waiting for " + strings.Join(waiting, ", ") + " to report on their send"
|
||||
changed := p.State != inventory.PlanRolling || p.Note != note
|
||||
p.State = inventory.PlanRolling
|
||||
p.Note = note
|
||||
return changed, nil
|
||||
}
|
||||
var said []string
|
||||
if len(failed) > 0 {
|
||||
said = append(said, "failed or refused its send (apply-failing says it): "+strings.Join(failed, ", "))
|
||||
}
|
||||
if len(silent) > 0 {
|
||||
said = append(said, fmt.Sprintf("said nothing on its send within %s: %s", inventory.ApplySilentAfter,
|
||||
strings.Join(silent, ", ")))
|
||||
}
|
||||
end = strings.Join(said, "; ")
|
||||
// **The new controller's user list follows its walk** (ADR 0236 §34, read without the gate since ADR 0360):
|
||||
// once a walk that moved the controller ends, the bus's machine is sent the list it composes, if behind.
|
||||
if s := p.Modules[catalogue.ControllerSeatName]; s != nil && s.SentAt != nil && s.Gate == nil && !noMove(s) {
|
||||
carryUserList(ctx, open, p)
|
||||
}
|
||||
}
|
||||
p.Tier++
|
||||
p.State = inventory.PlanBuilding
|
||||
p.Note = ""
|
||||
p.Note = end
|
||||
if p.Tier < len(p.Tiers) {
|
||||
fmt.Printf("%s: tier %d done; asking tier %d: %s\n", p.ID, p.Tier-1, p.Tier, strings.Join(p.Tiers[p.Tier], ", "))
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// walkReports is, for every machine a walk sent its rest, whether it reported on that send (novox/hq ADR 0360 §4):
|
||||
// those still waiting, those that reported a failed or refused apply, and those silent past ApplySilentAfter,
|
||||
// each sorted. Pure.
|
||||
func walkReports(p inventory.Plan, reports []inventory.Reported, now time.Time) (waiting, failed, silent []string) {
|
||||
byNode := map[string]inventory.Reported{}
|
||||
for _, r := range reports {
|
||||
byNode[r.Node] = r
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, s := range p.Modules {
|
||||
if s == nil {
|
||||
continue
|
||||
}
|
||||
for node, sent := range s.Rest {
|
||||
if seen[node] {
|
||||
continue
|
||||
}
|
||||
r := byNode[node]
|
||||
at := sent.At
|
||||
if at.IsZero() && s.SentAt != nil {
|
||||
at = *s.SentAt
|
||||
}
|
||||
switch {
|
||||
case r.At != nil && sent.ReportsOn(r):
|
||||
seen[node] = true
|
||||
if r.Outcome == inventory.OutcomeFailed || r.Outcome == inventory.OutcomeRefused {
|
||||
failed = append(failed, node)
|
||||
}
|
||||
case !at.IsZero() && now.Sub(at) > inventory.ApplySilentAfter:
|
||||
seen[node] = true
|
||||
silent = append(silent, node)
|
||||
default:
|
||||
if !slices.Contains(waiting, node) {
|
||||
waiting = append(waiting, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// A machine one module's send was reported on and another's not yet is the same send: reported.
|
||||
waiting = slices.DeleteFunc(waiting, func(n string) bool { return seen[n] })
|
||||
sort.Strings(waiting)
|
||||
sort.Strings(failed)
|
||||
sort.Strings(silent)
|
||||
return waiting, failed, silent
|
||||
}
|
||||
|
||||
// restOf is, for one module, every machine of its rest that the send reached and what it carried there.
|
||||
func restOf(to, sent []string, what map[string]inventory.SentDeclaration) map[string]inventory.SentDeclaration {
|
||||
out := map[string]inventory.SentDeclaration{}
|
||||
@@ -966,10 +1194,36 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
||||
runningOf map[string][]string) error {
|
||||
inv := open.inventory
|
||||
before, beforeKnown, _ := inv.SentBuilds(ctx, node)
|
||||
// **A machine already sent what this send is for is not sent again** (novox/hq ADR 0353 §6): sent by this plan
|
||||
// before a controller was replaced and before it was kept, or by another walk. Judged from that send, and from
|
||||
// the readings of a gate already judging it there.
|
||||
// Only a send the machine reported it applied is one it has; one it never reported on is sent, as before
|
||||
// (issue 280: what it ran before is then that build, and a failure leaves it as it was).
|
||||
if beforeKnown && slices.IndexFunc(modules, func(m string) bool { return !sameCommit(before[m], p.Modules[m].Commit) }) < 0 &&
|
||||
appliedLastSend(ctx, inv, node) {
|
||||
if others, err := gatedMoves(ctx, open, node, nil); err == nil && len(others) == 0 {
|
||||
adoptSend(ctx, open, p, node, modules, before, runningOf)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
// **What the machine ran before is kept before the send is made** (novox/hq ADR 0353 §6): a controller replaced
|
||||
// between the send and keeping it reads it here, and a failure still puts the machine back.
|
||||
kept := false
|
||||
for _, m := range modules {
|
||||
s := p.Modules[m]
|
||||
if beforeKnown && (s.Previous == "" || !sameCommit(before[m], s.Commit)) && s.Previous != before[m] {
|
||||
s.Previous, kept = before[m], true
|
||||
}
|
||||
}
|
||||
if kept {
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
return fmt.Errorf("what %s ran before is not kept, so nothing is sent: %w", node, err)
|
||||
}
|
||||
}
|
||||
owns := make([]inventory.CarriedMove, 0, len(modules))
|
||||
for _, m := range modules {
|
||||
s := p.Modules[m]
|
||||
owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: before[m], To: s.Commit, Build: s.Build})
|
||||
owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: s.Previous, To: s.Commit, Build: s.Build})
|
||||
}
|
||||
// A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges
|
||||
// all of it there.
|
||||
@@ -984,10 +1238,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
||||
sentWhat := sentNow(ctx, inv, sent)
|
||||
for _, m := range modules {
|
||||
s := p.Modules[m]
|
||||
// What the first machine ran before: what a failed gate puts back (ADR 0236).
|
||||
if beforeKnown {
|
||||
s.Previous = before[m]
|
||||
}
|
||||
// What the first machine ran before (ADR 0236), kept above before the send.
|
||||
s.First, s.FirstAt = sent, &now
|
||||
s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning(sent, runningOf[m]),
|
||||
From: s.Previous, To: s.Commit, Since: &now, Sent: sentWhat}
|
||||
@@ -1016,6 +1267,113 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
||||
return nil
|
||||
}
|
||||
|
||||
// adoptSend records a first send the machine already had (novox/hq ADR 0353 §6, §7): nothing is sent; the gate counts
|
||||
// from the machine's last send, and takes over the readings of an open gate of another walk judging the same build
|
||||
// there — whose carried move also says what the machine ran before, for a rollback.
|
||||
func adoptSend(ctx context.Context, open *stores, p *inventory.Plan, node string, modules []string,
|
||||
before map[string]string, runningOf map[string][]string) {
|
||||
inv := open.inventory
|
||||
since := time.Now().UTC()
|
||||
last, found, err := inv.SentTo(ctx, node)
|
||||
if err == nil && found && !last.At.IsZero() {
|
||||
since = last.At
|
||||
}
|
||||
lead := modules[0]
|
||||
other, id, from := judgingThere(ctx, inv, p.ID, lead, node, p.Modules[lead].Commit, p.Modules[lead].Build)
|
||||
for _, m := range modules {
|
||||
s := p.Modules[m]
|
||||
// What it ran before: kept by this walk before its send was made (§6); else what the other walk's send
|
||||
// moved it from; else the build itself, so a failure leaves it as it was (issue 280).
|
||||
switch {
|
||||
case s.Previous != "":
|
||||
case m == lead && from != "":
|
||||
s.Previous = from
|
||||
default:
|
||||
s.Previous = before[m]
|
||||
}
|
||||
at := since
|
||||
s.First, s.FirstAt = []string{node}, &at
|
||||
s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning([]string{node}, runningOf[m]),
|
||||
From: s.Previous, To: s.Commit, Since: &at}
|
||||
if found {
|
||||
s.Gate.Sent = map[string]inventory.SentDeclaration{node: last}
|
||||
}
|
||||
s.GatedBy = ""
|
||||
if m == lead {
|
||||
s.Gate.TakeReadings(other, modules, id)
|
||||
} else {
|
||||
s.GatedBy = lead
|
||||
}
|
||||
}
|
||||
p.State = inventory.PlanRolling
|
||||
p.Note = fmt.Sprintf("tier %d built; %s already runs %s's build: nothing sent again, judged from that send",
|
||||
p.Tier, node, strings.Join(modules, ", "))
|
||||
if id != "" {
|
||||
p.Note += ", going on from the readings of " + id
|
||||
}
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
}
|
||||
|
||||
// appliedLastSend says a machine reported it applied the declaration it was sent last.
|
||||
func appliedLastSend(ctx context.Context, inv *inventory.Inventory, node string) bool {
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, r := range reports {
|
||||
if r.Node == node {
|
||||
return r.Current && r.At != nil && r.Outcome == inventory.OutcomeApplied
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// judgingThere is an open gate of another walk judging a module's build on a machine (novox/hq ADR 0353 §7), the
|
||||
// walk's id, and what the machine ran before; nil when none does.
|
||||
func judgingThere(ctx context.Context, inv *inventory.Inventory, self, module, node, commit, build string) (*inventory.PlanGate, string, string) {
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, "", ""
|
||||
}
|
||||
// The same build, by its id where both name one, else by its commit.
|
||||
same := func(b, c string) bool {
|
||||
if b != "" && build != "" {
|
||||
return b == build
|
||||
}
|
||||
return sameCommit(c, commit)
|
||||
}
|
||||
usable := func(g *inventory.PlanGate) bool {
|
||||
return g != nil && !g.Pending && g.Since != nil && (g.Verdict == "" || g.Verdict == inventory.GatePassed) &&
|
||||
slices.Contains(g.Machines, node)
|
||||
}
|
||||
for _, q := range plans {
|
||||
if q.ID == self {
|
||||
continue
|
||||
}
|
||||
if r := q.Release; r != nil {
|
||||
gates := []*inventory.PlanGate{r.Gate}
|
||||
for _, g := range r.Gates {
|
||||
gates = append(gates, g)
|
||||
}
|
||||
for _, g := range gates {
|
||||
if !usable(g) {
|
||||
continue
|
||||
}
|
||||
for _, c := range g.Carried {
|
||||
if c.Module == module && c.Node == node && same(c.Build, c.To) {
|
||||
return g, q.ID, c.From
|
||||
}
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if s := q.Modules[module]; s != nil && usable(s.Gate) && same(s.Build, s.Commit) {
|
||||
return s.Gate, q.ID, s.Previous
|
||||
}
|
||||
}
|
||||
return nil, "", ""
|
||||
}
|
||||
|
||||
// passedWith keeps, on a module sent with others, the verdict of the gate that judged the send: the gate
|
||||
// on the first of them passed, and its pass was kept for every build it carried (passCarried).
|
||||
func passedWith(module string, s *inventory.PlanModule, lead string, g *inventory.PlanGate) {
|
||||
@@ -1504,9 +1862,19 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
// A release plan's walk (ADR 0236): machines done, the one judged, those to come.
|
||||
fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "),
|
||||
orNone(strings.Join(r.Done, ", ")), orNone(strings.Join(r.Skipped, ", ")))
|
||||
if r.Gate != nil {
|
||||
fmt.Printf(" %s\n", gateLine(r.Gate))
|
||||
for _, c := range r.Gate.Carried {
|
||||
if len(r.Failed)+len(r.Silent) > 0 {
|
||||
fmt.Printf(" failed: %s; silent: %s\n", orNone(strings.Join(r.Failed, ", ")), orNone(strings.Join(r.Silent, ", ")))
|
||||
}
|
||||
gates := []*inventory.PlanGate{r.Gate}
|
||||
for _, n := range r.Order {
|
||||
gates = append(gates, r.Gates[n])
|
||||
}
|
||||
for _, g := range gates {
|
||||
if g == nil {
|
||||
continue
|
||||
}
|
||||
fmt.Printf(" %s\n", gateLine(g))
|
||||
for _, c := range g.Carried {
|
||||
fmt.Printf(" %-22s %s → %s\n", c.Module, short(c.From), short(c.To))
|
||||
if c.Recreates != "" {
|
||||
fmt.Printf(" %-22s %s\n", "", c.Recreates)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -49,8 +50,10 @@ func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) {
|
||||
if pos["mesh-tools"] != 0 || pos["builder"] != 1 {
|
||||
t.Fatalf("the runtime then the builder: %v", tiers)
|
||||
}
|
||||
if !(pos["shop"] > pos["builder"] && pos["postgres"] > pos["builder"]) {
|
||||
t.Fatalf("what the builder builds comes after the builder: %v", tiers)
|
||||
// Built-by orders nothing (novox/hq ADR 0360 §3): what the builder builds is beside it, after the runtime it
|
||||
// stands on.
|
||||
if pos["shop"] != 1 || pos["postgres"] != 1 {
|
||||
t.Fatalf("what the builder builds waits for the builder: %v", tiers)
|
||||
}
|
||||
if pos["shop-plugin"] <= pos["shop"] {
|
||||
t.Fatalf("a plugin after what it is declared on: %v", tiers)
|
||||
@@ -67,24 +70,23 @@ func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) {
|
||||
// A change to a package all three build from moves all three. The builder holds
|
||||
// the build seat, whose worker the controller defines, so it follows the controller (worker-of,
|
||||
// novox/hq issue 206), and the controller's built-by edge to it yields: the controller is built by the
|
||||
// build machine that is running. The proxy is built by the new builder: the controller, the builder,
|
||||
// the proxy — the live plan of every controller merge. (This read "the builder, then the controller
|
||||
// and the proxy together" before issue 206, and the fixture had no worker-of edge.)
|
||||
// build machine that is running. The proxy is built by whichever builder runs, so it goes beside the
|
||||
// controller (novox/hq ADR 0360 §3; it read "the controller, the builder, the proxy" before).
|
||||
small := reachableFrom([]string{"mesh-controller", "builder", "route-proxy"}, edges)
|
||||
smallTiers := tiersOf(small, edges)
|
||||
if got := tiered(smallTiers); got != "mesh-controller | builder | route-proxy" {
|
||||
t.Fatalf("the controller, then the builder, then the proxy: %v", smallTiers)
|
||||
if got := tiered(smallTiers); got != "mesh-controller,route-proxy | builder" {
|
||||
t.Fatalf("the controller and the proxy, then the builder: %v", smallTiers)
|
||||
}
|
||||
// The builder alone moved: the builder, and nothing it builds.
|
||||
if only := reachableFrom([]string{"builder"}, edges); len(only) != 1 {
|
||||
t.Fatalf("a build machine change rebuilds the build machine alone: %v", only)
|
||||
}
|
||||
|
||||
// Only a runtime dependency gates on deployment, and only when the module rolls out.
|
||||
// Nothing waits on deployment for a built-by edge (novox/hq ADR 0360 §3): the builder gates no tier after it.
|
||||
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"}, []string{"mesh-tools"}, edges)
|
||||
p.Tier = pos["builder"]
|
||||
rollsOut := func(m string) bool { return m == "builder" }
|
||||
if g := gates(p, edges, rollsOut); len(g) != 1 || g[0] != "builder" {
|
||||
if g := gates(p, edges, rollsOut); len(g) != 0 {
|
||||
t.Fatalf("the builder gates the tier after it: %v", g)
|
||||
}
|
||||
p.Tier = 0
|
||||
@@ -253,10 +255,10 @@ func TestAChangeToTheBuildAgentRebuildsTheBuildAgentAlone(t *testing.T) {
|
||||
t.Fatalf("%v planned %v: the build agent alone", paths, p.Tiers)
|
||||
}
|
||||
}
|
||||
// With what it builds moved beside it, the agent comes first and they follow: an order, not a widening.
|
||||
// With what it builds moved beside it, both go in one tier (novox/hq ADR 0360 §3): built-by orders nothing.
|
||||
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Commit: "abc"},
|
||||
[]string{"build-agent", "redis"}, edges)
|
||||
if len(p.Tiers) != 2 || p.Tiers[0][0] != "build-agent" || p.Tiers[1][0] != "redis" || len(p.Modules) != 2 {
|
||||
t.Fatalf("the agent, then what moved beside it: %v", p.Tiers)
|
||||
if len(p.Tiers) != 1 || !slices.Equal(p.Tiers[0], []string{"build-agent", "redis"}) || len(p.Modules) != 2 {
|
||||
t.Fatalf("the agent and what moved beside it, in one tier: %v", p.Tiers)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,8 +111,13 @@ func aBacklog(t *testing.T) *backlogMesh {
|
||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// A broken machine reports its apply failed (novox/hq ADR 0360 §7: the report is the verdict).
|
||||
outcome := inventory.OutcomeApplied
|
||||
if b.broken[node] {
|
||||
outcome = inventory.OutcomeFailed
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
||||
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
||||
Outcome: outcome, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
@@ -133,11 +138,15 @@ func aBacklog(t *testing.T) *backlogMesh {
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||
gateSettle, gateEvery = 0, time.Hour
|
||||
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||
wasWindow := gateWitnessWindow
|
||||
gateWitnessWindow = 0
|
||||
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||
gateWithin, gateEvery = time.Hour, time.Hour
|
||||
t.Cleanup(func() {
|
||||
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
|
||||
gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound
|
||||
gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound
|
||||
})
|
||||
return b
|
||||
}
|
||||
@@ -160,6 +169,7 @@ func (b *backlogMesh) release(t *testing.T) inventory.Plan {
|
||||
// The backlog goes out one machine at a time: the first judged before the second is sent, each build's
|
||||
// pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused.
|
||||
func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
@@ -215,6 +225,7 @@ func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) {
|
||||
// A release that fails on its first machine puts back what it carried there, goes no further, and the
|
||||
// next one waits for a person, said as a condition; a person releases it with why.
|
||||
func TestAFailedReleaseIsPutBackAndTheNextWaitsForAPerson(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
|
||||
@@ -31,6 +31,7 @@ import (
|
||||
// nothing is put back, the walk goes on to the workstation, and openrazer's condition says, in one sentence
|
||||
// for the operator, that a new login is needed on the laptop.
|
||||
func TestReplay318(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
@@ -112,10 +113,14 @@ func TestReplay318(t *testing.T) {
|
||||
}
|
||||
|
||||
wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts
|
||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||
wasWindow := gateWitnessWindow
|
||||
gateWitnessWindow = 0
|
||||
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||
t.Cleanup(func() {
|
||||
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
|
||||
gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound
|
||||
gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound
|
||||
})
|
||||
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
|
||||
modules, err := open.inventory.Assigned(ctx, node)
|
||||
@@ -184,7 +189,7 @@ func TestReplay318(t *testing.T) {
|
||||
return f, nil
|
||||
}
|
||||
// The bound is reached at the first judging that is not a pass: what happened at 11:21 happens at once.
|
||||
gateSettle, gateEvery, gateBound = 0, 0, 0
|
||||
gateWithin, gateEvery, gateBound = time.Hour, 0, 0
|
||||
|
||||
for i := 0; i < 12; i++ {
|
||||
advancePlans(ctx, open)
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
// at its gate with the pass's own words, and put it back. Written with only what the controller had before
|
||||
// its fix, so it is laid over the commit before.
|
||||
func TestReplay335(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
t.Run("the first machine's later reports", testAPassedGateIsNotJudgedAgainFromTheFirstMachinesLaterReports)
|
||||
t.Run("a walk stopped after the pass", testAWalkStoppedAfterItsGatePassedKeepsThePass)
|
||||
}
|
||||
@@ -77,6 +78,7 @@ func testAWalkStoppedAfterItsGatePassedKeepsThePass(t *testing.T) {
|
||||
// the first send is older than the wait for a first machine's report. Read before the pass is taken over, the
|
||||
// carried module's step said the first machine never applied it, and the passed build was put back.
|
||||
func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
tm := aTierMesh(t, "m01", "m02")
|
||||
ctx := t.Context()
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
// had before its fix — the stores, register, assign, a plan saved and advanced, the watchdogs' gathering and
|
||||
// seeing — so it is laid over the commit before.
|
||||
func TestReplay336(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
@@ -139,6 +139,7 @@ func TestReplay273AConsumerBesideItsStoreStaysBoundToIt(t *testing.T) {
|
||||
// one, kept below. Heard as bytes, so an older controller reads them as it reads any report — this file is
|
||||
// written only with what the controller had before the judging, for the prover to lay over that commit.
|
||||
func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
@@ -248,9 +249,13 @@ func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) {
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = wasSend })
|
||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||
gateSettle, gateEvery = 0, 0
|
||||
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
||||
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||
wasWindow := gateWitnessWindow
|
||||
gateWitnessWindow = 0
|
||||
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||
gateWithin, gateEvery = time.Hour, 0
|
||||
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
|
||||
|
||||
built := time.Now().UTC()
|
||||
plan := inventory.Plan{ID: "plan-crashloop", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
||||
|
||||
@@ -0,0 +1,382 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0353, kept behind gateFirstMachine for walks a controller before ADR 0360 started: after its first machine passes, a build goes to every other machine at once, each judged by
|
||||
// its own gate and a failing one put back by itself; and a gate's readings outlive the controller that took them —
|
||||
// a machine already sent what a send is for is not sent again, and its judging goes on from what was read.
|
||||
//
|
||||
// Seen 2026-10-14: release-1791930194184991192 sent ace, g14, shanks and the control node one after another in
|
||||
// eleven minutes; the new controller its last send started sent the control node again the controller build it
|
||||
// had just applied, and judged it from its first reading.
|
||||
|
||||
// aBacklogOnThree is the backlog fixture with a third machine, desk, running app at c1 like the others.
|
||||
func aBacklogOnThree(t *testing.T) *backlogMesh {
|
||||
t.Helper()
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
if _, err := inv.AddNode(ctx, "desk"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "desk", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, b.open, "desk"), "d-desk", map[string]string{"app": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wasHeard := releaseHeard
|
||||
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
|
||||
return map[string]bool{"anchor": true, "laptop": true, "desk": true}, nil
|
||||
}
|
||||
t.Cleanup(func() { releaseHeard = wasHeard })
|
||||
return b
|
||||
}
|
||||
|
||||
// events records every send and every judging, in order.
|
||||
func events(t *testing.T) *[]string {
|
||||
t.Helper()
|
||||
var said []string
|
||||
wasSend, wasGather := sendRollout, gatherGateFacts
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
said = append(said, "send "+strings.Join(names, ","))
|
||||
return wasSend(ctx, open, names)
|
||||
}
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
said = append(said, "judge")
|
||||
return wasGather(ctx, open, component)
|
||||
}
|
||||
t.Cleanup(func() { sendRollout, gatherGateFacts = wasSend, wasGather })
|
||||
return &said
|
||||
}
|
||||
|
||||
// Once the first machine passed, the rest are both sent before either is judged (ADR 0353 §1); before, the second
|
||||
// was judged to its verdict before the third was sent.
|
||||
func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) {
|
||||
withTheGate(t)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
said := events(t)
|
||||
gateEvery = 0
|
||||
for i := 0; i < 12; i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
p := b.release(t)
|
||||
if p.State != inventory.PlanDone || len(p.Release.Done) != 3 {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
desk, laptop := slices.Index(*said, "send desk"), slices.Index(*said, "send laptop")
|
||||
if desk < 0 || laptop < 0 || slices.Index(*said, "send anchor") > min(desk, laptop) {
|
||||
t.Fatalf("sends and judgings: %v", *said)
|
||||
}
|
||||
if slices.Contains((*said)[min(desk, laptop):max(desk, laptop)], "judge") {
|
||||
t.Fatalf("the rest were sent one at a time, each judged before the next: %v", *said)
|
||||
}
|
||||
if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine of the rest whose gate fails is put back at once, and the other machine of the step goes on to its own
|
||||
// verdict (ADR 0353 §3); before, the walk stopped at the failure and the machine after it was never sent.
|
||||
func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) {
|
||||
withTheGate(t)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
gateEvery = 0
|
||||
b.broken["desk"] = true // desk fails what it is sent in the rest's step
|
||||
advancePlans(ctx, b.open)
|
||||
for i := 0; i < 12 && b.release(t).Open(); i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
p := b.release(t)
|
||||
if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
if !slices.ContainsFunc(b.sent, func(s []string) bool { return slices.Equal(s, []string{"laptop"}) }) {
|
||||
t.Fatalf("laptop was never sent: %v", b.sent)
|
||||
}
|
||||
if !slices.Contains(p.Release.Done, "laptop") {
|
||||
t.Fatalf("laptop has no verdict of its own: %+v", p.Release)
|
||||
}
|
||||
// late waited on laptop alone and was healthy there: its pass is kept, whatever desk did.
|
||||
if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
||||
}
|
||||
// app failed on desk: marked, and put back wherever it was sent (ADR 0236).
|
||||
if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed {
|
||||
t.Fatal("app's build is not marked failed")
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c1" {
|
||||
t.Fatalf("desk was not put back: %v", sent)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk whose first machine already runs its build under another walk's gate sends nothing and goes on from that
|
||||
// gate's readings (ADR 0353 §7); before, it sent the machine again and judged from its first reading.
|
||||
func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) {
|
||||
withTheGate(t)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
advancePlans(ctx, b.open) // the release sends anchor app c2, and reads it once
|
||||
release := b.release(t)
|
||||
judging := release.Release.Gate
|
||||
if judging == nil || judging.Pending || judging.Passes == 0 {
|
||||
t.Fatalf("the release is not judging anchor: %+v", judging)
|
||||
}
|
||||
built := time.Now().UTC()
|
||||
plan := inventory.Plan{ID: "plan-app", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
||||
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2",
|
||||
Build: "build-app-c2"}}}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, b.open)
|
||||
if len(b.sent) != 1 {
|
||||
t.Fatalf("anchor was sent again what it runs: %v", b.sent)
|
||||
}
|
||||
p, err := inv.PlanByID(ctx, "plan-app")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g := p.Modules["app"].Gate
|
||||
if g == nil || g.Since == nil || !g.Since.Equal(*judging.Since) || g.Passes < judging.Passes || g.TookOver != release.ID {
|
||||
t.Fatalf("the walk's gate starts from zero: %+v (the release's: %+v)", g, judging)
|
||||
}
|
||||
if p.Modules["app"].Previous != "c1" {
|
||||
t.Fatalf("what anchor ran before is %q, not c1", p.Modules["app"].Previous)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk's first send that was made and not kept — the controller replaced between the two — is not made again
|
||||
// by the controller that takes over: the gate counts from that send (ADR 0353 §5, §6).
|
||||
func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) {
|
||||
withTheGate(t)
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
// The old controller sends anchor app c2 and is replaced before it keeps the send.
|
||||
made := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
if _, err := made(ctx, open, names); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, context.Canceled
|
||||
}
|
||||
advancePlans(ctx, g.open)
|
||||
sendRollout = made
|
||||
if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil {
|
||||
t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent)
|
||||
}
|
||||
last, _, err := g.open.inventory.SentTo(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, g.open)
|
||||
if len(g.sent) != 1 {
|
||||
t.Fatalf("the send was made again: %v", g.sent)
|
||||
}
|
||||
s := g.plan(t).Modules["app"]
|
||||
if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) {
|
||||
t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate)
|
||||
}
|
||||
// What anchor ran before was kept before the send, so a failure still puts it back (§5).
|
||||
if s.Previous != "c1" || s.Gate.From != "c1" {
|
||||
t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From)
|
||||
}
|
||||
}
|
||||
|
||||
// A release's send kept before it was made, and made, is judged by the controller that takes over without being
|
||||
// made again (ADR 0353 §8).
|
||||
func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) {
|
||||
withTheGate(t)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
p, err := releaseBacklog(ctx, b.open, "")
|
||||
if err != nil || p == nil {
|
||||
t.Fatalf("no release opened: %v", err)
|
||||
}
|
||||
if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gate == nil || !p.Release.Gate.Pending {
|
||||
t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The send is made, and the controller is replaced before it keeps it.
|
||||
if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, b.open)
|
||||
if len(b.sent) != 1 {
|
||||
t.Fatalf("anchor was sent again: %v", b.sent)
|
||||
}
|
||||
g := b.release(t).Release.Gate
|
||||
if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 {
|
||||
t.Fatalf("anchor is not judged on the send made: %+v", g)
|
||||
}
|
||||
}
|
||||
|
||||
// A build no gate has passed goes to one machine of a step, its first; a machine that would be its second waits
|
||||
// for the next step, and one whose every build passed goes now (ADR 0353 §2).
|
||||
func TestABuildThatPassedNoGateGoesToOneMachineOfAStep(t *testing.T) {
|
||||
withTheGate(t)
|
||||
waiting := map[string][]inventory.CarriedMove{
|
||||
"desk": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}},
|
||||
"laptop": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}},
|
||||
"shanks": {{Module: "app", To: "c2"}},
|
||||
"novox": {{Module: "app", To: "c2"}, {Module: "ctl", To: "k1"}},
|
||||
}
|
||||
passed := func(module, commit string) bool { return module == "app" && commit == "c2" }
|
||||
never := func(string) bool { return false }
|
||||
step := restStep([]string{"desk", "laptop", "shanks", "novox"}, waiting, passed, never)
|
||||
if !slices.Equal(step, []string{"desk", "shanks", "novox"}) {
|
||||
t.Fatalf("the step is %v: laptop would be new's second machine before it passed", step)
|
||||
}
|
||||
if step := restStep([]string{"laptop"}, waiting, func(string, string) bool { return true }, never); !slices.Equal(step, []string{"laptop"}) {
|
||||
t.Fatalf("a machine whose every build passed waits: %v", step)
|
||||
}
|
||||
}
|
||||
|
||||
// The node-engine and the mesh's resolver go to the rest one machine at a time, even passed (ADR 0353 §9): a step
|
||||
// carries either to one machine only, while what else passed goes everywhere at once.
|
||||
func TestTheNodeEngineAndTheResolverGoToTheRestOneAtATime(t *testing.T) {
|
||||
withTheGate(t)
|
||||
waiting := map[string][]inventory.CarriedMove{
|
||||
"desk": {{Module: hostModule, To: "h2"}},
|
||||
"laptop": {{Module: hostModule, To: "h2"}, {Module: "app", To: "c2"}},
|
||||
"shanks": {{Module: "app", To: "c2"}},
|
||||
"novox": {{Module: "dns", To: "d2"}},
|
||||
"ace": {{Module: "dns", To: "d2"}},
|
||||
}
|
||||
resolver := catalogue.Manifest{Module: "dns", Claims: []catalogue.Claim{{Name: resolverSeat}}}
|
||||
alone := func(m string) bool {
|
||||
if m == "dns" {
|
||||
return oneAtATime(m, resolver)
|
||||
}
|
||||
return oneAtATime(m, catalogue.Manifest{Module: m})
|
||||
}
|
||||
step := restStep([]string{"desk", "laptop", "shanks", "novox", "ace"}, waiting,
|
||||
func(string, string) bool { return true }, alone)
|
||||
if !slices.Equal(step, []string{"desk", "shanks", "novox"}) {
|
||||
t.Fatalf("the step is %v: the node-engine and the resolver each reach one machine of it", step)
|
||||
}
|
||||
if oneAtATime("app", catalogue.Manifest{Module: "app"}) {
|
||||
t.Fatal("a leaf module is held to one machine at a time")
|
||||
}
|
||||
}
|
||||
|
||||
// A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent.
|
||||
func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) {
|
||||
withTheGate(t)
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
gateEvery = 0
|
||||
b.broken["anchor"] = true
|
||||
advancePlans(ctx, b.open)
|
||||
for i := 0; i < 6 && b.release(t).Open(); i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
for _, s := range b.sent {
|
||||
if !slices.Equal(s, []string{"anchor"}) {
|
||||
t.Fatalf("sent %v: the rest after a failed first machine", b.sent)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The control node is sent last in a step of the rest (ADR 0353 §1): the machine holding the controller is
|
||||
// ordered after every other.
|
||||
func TestTheControlNodeIsSentLastInAStep(t *testing.T) {
|
||||
withTheGate(t)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
register(t, b.open, catalogue.Manifest{Module: catalogue.ControllerSeatName, Version: "1"})
|
||||
if _, err := b.open.inventory.Assign(ctx, "anchor", catalogue.ControllerSeatName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said := events(t)
|
||||
gateEvery = 0
|
||||
for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
if p := b.release(t); !slices.Equal(p.Release.Order, []string{"desk", "laptop", "anchor"}) {
|
||||
t.Fatalf("the order is %v", p.Release.Order)
|
||||
}
|
||||
laptop, anchor := slices.Index(*said, "send laptop"), slices.Index(*said, "send anchor")
|
||||
if laptop < 0 || anchor < laptop || slices.Contains((*said)[laptop:anchor], "judge") {
|
||||
t.Fatalf("the control node is not sent last in the rest's step: %v", *said)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put
|
||||
// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4).
|
||||
func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) {
|
||||
withTheGate(t)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
reporting := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing
|
||||
b.sent = append(b.sent, names)
|
||||
current, _ := open.inventory.CurrentBuilds(ctx)
|
||||
return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep",
|
||||
map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit})
|
||||
}
|
||||
return reporting(ctx, open, names)
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = reporting })
|
||||
gateEvery = 0
|
||||
for i := 0; i < 8; i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
p := b.release(t)
|
||||
g := p.Release.Gates["laptop"]
|
||||
if !p.Open() || g == nil || g.Verdict != "" || g.Silent {
|
||||
t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g)
|
||||
}
|
||||
since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute)
|
||||
g.Since = &since
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, b.open)
|
||||
p = b.release(t)
|
||||
if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed {
|
||||
t.Fatal("a silent machine's build was marked failed")
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
|
||||
t.Fatalf("laptop was put back: %v", sent)
|
||||
}
|
||||
obs, err := silentObservation(ctx, inv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) {
|
||||
t.Fatalf("the silence is not said: %+v", obs)
|
||||
}
|
||||
}
|
||||
@@ -452,6 +452,33 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
case "queue":
|
||||
// As data, the table its output (novox/hq issue 455).
|
||||
return []string{"queue", "--json"}, nil
|
||||
case "forget-checks":
|
||||
if err := need("owner", "repo", "number"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{"forget-checks", str("owner"), str("repo"), str("number")}, nil
|
||||
case "try":
|
||||
// A branch tried before its pull request (novox/hq issue 686): asked, or read by the id it answered.
|
||||
if id := str("id"); id != "" {
|
||||
return []string{"try", "--id", id}, nil
|
||||
}
|
||||
if a.given["repository"] == "" && a.given["ref"] == "" {
|
||||
// Neither shape named: the command says its usage, which names both, as `build` does.
|
||||
return []string{"try"}, nil
|
||||
}
|
||||
if err := need("repository", "ref"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"try", str("repository"), str("ref")}
|
||||
for _, flag := range []string{"base", "tests"} {
|
||||
if v := str(flag); v != "" {
|
||||
argv = append(argv, "--"+flag, v)
|
||||
}
|
||||
}
|
||||
if on("store-only") {
|
||||
argv = append(argv, "--store-only")
|
||||
}
|
||||
return argv, nil
|
||||
case "cancel", "kill":
|
||||
if err := need("id"); err != nil {
|
||||
return nil, err
|
||||
@@ -881,48 +908,6 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
argv = append(argv, "--adopted")
|
||||
}
|
||||
return argv, nil
|
||||
case "config":
|
||||
// Config values from four sources (novox/hq issue 630): mesh config and node config for a predefined
|
||||
// key, and a module's own value on one node. A listing without key; a set or clear says why.
|
||||
key := str("key")
|
||||
if str("module") != "" && str("node") == "" && (key != "" || on("clear")) {
|
||||
return nil, errors.New("config: there is no module layer for the whole mesh — an assignment is one module " +
|
||||
"on one node (novox/hq issue 630); give node, or leave module out to set mesh config for a predefined key")
|
||||
}
|
||||
place := func(argv []string) []string {
|
||||
if n := str("node"); n != "" {
|
||||
argv = append(argv, "--node", n)
|
||||
}
|
||||
if m := str("module"); m != "" {
|
||||
argv = append(argv, "--module", m)
|
||||
}
|
||||
return argv
|
||||
}
|
||||
switch {
|
||||
case key == "":
|
||||
if str("value") != "" || on("clear") || on("history") || str("why") != "" {
|
||||
return nil, errors.New("config: a set, a clear or a history names its key")
|
||||
}
|
||||
return place([]string{"config", "list"}), nil
|
||||
case on("history"):
|
||||
if str("value") != "" || on("clear") || str("module") != "" || str("why") != "" {
|
||||
return nil, errors.New("config: history reads a key's mesh or node config, and takes only node")
|
||||
}
|
||||
return place([]string{"config", "history", key}), nil
|
||||
case on("clear") && str("value") != "":
|
||||
return nil, errors.New("config: a value or clear, not both")
|
||||
case on("clear"):
|
||||
if err := need("why"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(place([]string{"config", "clear", key}), "--why", str("why")), nil
|
||||
case str("value") != "":
|
||||
if err := need("why"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(place([]string{"config", "set", key, str("value")}), "--why", str("why")), nil
|
||||
}
|
||||
return nil, errors.New("config: with key, give value, clear or history")
|
||||
case "settings":
|
||||
// Every module's preferences, their defaults and each machine's value (novox/hq ADR 0262):
|
||||
// the one interface for them, so no module builds a settings tool of its own. Asked for by
|
||||
@@ -1055,6 +1040,8 @@ var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "co
|
||||
"delivery": true,
|
||||
// The build queue, each ask with its position among the waiting (novox/hq issue 455).
|
||||
"queue": true,
|
||||
// A branch tried before its pull request (novox/hq issue 686).
|
||||
"try": true,
|
||||
// A plan followed (novox/hq issue 229): `plans wait` answers JSON; its other forms say text, and so answer no data.
|
||||
"plans": true}
|
||||
|
||||
@@ -1345,6 +1332,10 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
link.AnswersWithin(ctx, bound+planWaitStart)
|
||||
}
|
||||
}
|
||||
if verb == "try" && len(argv) > 1 && argv[1] == "--id" {
|
||||
// Reading a try holds its answer for the verdict, up to its bound (novox/hq issue 686, 229).
|
||||
link.AnswersWithin(ctx, tryWait+planWaitStart)
|
||||
}
|
||||
if answer, read := readHere(ctx, argv); read {
|
||||
return answer, nil
|
||||
}
|
||||
@@ -1656,7 +1647,6 @@ var commandReadForms = map[string]func(rest []string) bool{
|
||||
"node": func(r []string) bool { return subIn(r, "list", "show") },
|
||||
"module": func(r []string) bool { return subIn(r, "list") },
|
||||
"settings": func(r []string) bool { return subIn(r, "show", "preferences", "proposals") },
|
||||
"config": func(r []string) bool { return flagsOnly(r) || subIn(r, "list", "history") },
|
||||
"retire": func(r []string) bool { return subIn(r, "list") },
|
||||
"cleanup": func(r []string) bool { return subIn(r, "list") },
|
||||
"delivery": func(r []string) bool { return subIn(r, "plan", "walks") },
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0346, issue 624: a provider on another machine serves its consumers its own composed
|
||||
// names, as that machine composes them; one it does not compose is left out, said under the need.
|
||||
func TestAProviderElsewhereServesItsOwnComposedName(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
forge, err := catalogue.ParseManifest([]byte(`{"module":"forge","version":"1","requires":["route"],
|
||||
"provides":[{"name":"npm-package-registry","scope":"mesh"}],
|
||||
"contributes":{"route":{"web":{"label":"git","port":3000}}},
|
||||
"serves":{"npm-package-registry":{"scheme":"http","port":3000,"npm-path":"/api/packages/novox/npm/",
|
||||
"public-root":"https://${bound:route:name-web}/api/packages/novox/npm/"}}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "proxy", Version: "1", Provides: catalogue.FromAnywhere("route"),
|
||||
Receives: map[string]string{"route": "/var/lib/proxy/routes.json"}})
|
||||
register(t, open, forge)
|
||||
register(t, open, catalogue.Manifest{Module: "lab", Version: "1", Requires: []string{"npm-package-registry"},
|
||||
Binds: map[string]string{"npm-package-registry": "/var/lib/lab/package-registry.json"}})
|
||||
for _, a := range [][2]string{{"anchor", "proxy"}, {"anchor", "forge"}, {"laptop", "lab"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := open.inventory.SetPublicDomain(ctx, "anchor", "example.test"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n := need(t, plan, "lab", "npm-package-registry")
|
||||
if n.Serves["public-root"] != "https://git.example.test/api/packages/novox/npm/" || len(n.LeftOut) != 0 {
|
||||
t.Fatalf("the consumer elsewhere is served the forge's public root: %v, left out %v", n.Serves, n.LeftOut)
|
||||
}
|
||||
|
||||
if err := open.inventory.SetPublicDomain(ctx, "anchor", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, _, err = planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n = need(t, plan, "lab", "npm-package-registry")
|
||||
if _, there := n.Serves["public-root"]; there {
|
||||
t.Fatalf("anchor composes no public name, so the key is left out: %v", n.Serves)
|
||||
}
|
||||
if len(n.LeftOut) != 1 || !strings.Contains(n.LeftOut[0], `without "public-root"`) {
|
||||
t.Fatalf("and the need says so: %v", n.LeftOut)
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,7 @@ func TestSettingsSayWhereEachValueComesFrom(t *testing.T) {
|
||||
{Key: "width", Value: float64(250), From: catalogue.DefaultLayer, FromDefault: true, Default: float64(250), HasDefault: true},
|
||||
})
|
||||
want := "dunst on laptop, every value and where it comes from:\n" +
|
||||
" font-size = 13 (dunst on laptop; the default is 10)\n" +
|
||||
" font-size = 13 (laptop; the default is 10)\n" +
|
||||
" width = 250 (default)\n"
|
||||
if got != want {
|
||||
t.Fatalf("said\n%s\nwant\n%s", got, want)
|
||||
@@ -66,7 +66,7 @@ func TestPreferencesSayEachMachinesValueAndItsSource(t *testing.T) {
|
||||
want := "dunst (on desk, laptop)\n" +
|
||||
" font-size, default 10: readable at 100 DPI\n" +
|
||||
" desk: 10 (default)\n" +
|
||||
" laptop: 16 (dunst on laptop)\n" +
|
||||
" laptop: 16 (the node)\n" +
|
||||
" width, default 250: forty characters\n" +
|
||||
" desk: 300 (the mesh)\n" +
|
||||
" laptop: 250 (default)\n"
|
||||
@@ -96,7 +96,7 @@ func TestPreferencesListedFromTheStores(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"preferences"}) })
|
||||
if !strings.Contains(all, "notes (on laptop)") || !strings.Contains(all, "laptop: 16 (notes on laptop)") ||
|
||||
if !strings.Contains(all, "notes (on laptop)") || !strings.Contains(all, "laptop: 16 (the node)") ||
|
||||
!strings.Contains(all, "font-size, default 10: readable at 100 DPI") {
|
||||
t.Fatalf("the listing:\n%s", all)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// An approved settings change is sent by a walk at once, and the operator's one Approve is its push (novox/hq ADR
|
||||
// 0344, issue 641).
|
||||
//
|
||||
// On 2026-10-13 the operator approved a betterbird proposal on Telegram and nothing happened: the warrant set the
|
||||
// layer, and the push stayed a separate act (ADR 0277 §18) that nobody took. Now the warrant that sets the layer
|
||||
// opens a **settings walk**: the walk ADR 0236 opens for builds waiting for a gate, with a layer in place of the
|
||||
// builds. Its order is every machine running the module that the layer reaches, the control node last; each is
|
||||
// sent and its module judged there, from the machine's report on that send, before the next is sent. The send is
|
||||
// the layer alone: every module there stays at the build its machine was last sent (keepingBuildsForSettings), so a
|
||||
// build held for a person (ADR 0242) or waiting for a gate stays where it is — a policy of `record` holds a build,
|
||||
// never a setting — and a send that would move a running module's data stops the walk (ADR 0217 §3). The walk says its start and end on `plan-moved` (ADR 0308), and the holder of the operator's
|
||||
// channel tells its end once. A failure stops the walk and leaves the layer as approved; it holds back nothing
|
||||
// else.
|
||||
|
||||
// settingsWalkPrefix names a settings walk's id.
|
||||
const settingsWalkPrefix = "settings-"
|
||||
|
||||
// settingsWalkFor is what a settings walk says it is for, where a merge's walk says its repository.
|
||||
func settingsWalkFor(s inventory.PlanSettings) string {
|
||||
where := s.Node
|
||||
if where == "" {
|
||||
where = "the whole mesh"
|
||||
}
|
||||
return fmt.Sprintf("the settings of %s on %s", s.Module, where)
|
||||
}
|
||||
|
||||
// settingsWalkOrder is the machines a layer reaches, in the order a settings walk sends them: every machine
|
||||
// running the module, or the one the layer names when it runs it there, sorted, the control node last. Pure.
|
||||
func settingsWalkOrder(node string, running, controllers []string) []string {
|
||||
var order, last []string
|
||||
for _, n := range running {
|
||||
if node != "" && n != node {
|
||||
continue
|
||||
}
|
||||
if slices.Contains(controllers, n) {
|
||||
last = append(last, n)
|
||||
} else {
|
||||
order = append(order, n)
|
||||
}
|
||||
}
|
||||
sort.Strings(order)
|
||||
sort.Strings(last)
|
||||
return append(order, last...)
|
||||
}
|
||||
|
||||
// openSettingsWalk opens the walk that sends an approved layer, and answers the words of what it does: the
|
||||
// machines it sends, or that no machine runs the module and nothing is sent. Called on the operator's warrant,
|
||||
// after the layer is set.
|
||||
var openSettingsWalk = func(ctx context.Context, open *stores, s inventory.PlanSettings) (string, error) {
|
||||
inv := open.inventory
|
||||
running, err := inv.Running(ctx, s.Module)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
controllers, err := inv.Running(ctx, catalogue.ControllerSeatName)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
order := settingsWalkOrder(s.Node, running, controllers)
|
||||
if len(order) == 0 {
|
||||
return fmt.Sprintf("no machine runs %s there, so there is nothing to send", s.Module), nil
|
||||
}
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
now := time.Now().UTC()
|
||||
p := inventory.Plan{ID: fmt.Sprintf("%s%d", settingsWalkPrefix, now.UnixNano()), Repository: settingsWalkFor(s),
|
||||
Created: now, State: inventory.PlanRolling, Tiers: [][]string{{s.Module}}, Modules: map[string]*inventory.PlanModule{},
|
||||
Release: &inventory.PlanRelease{Order: order, Settings: &s},
|
||||
Note: fmt.Sprintf("approved (ask %s): sent to %s, one machine at a time, each judged", s.Ask,
|
||||
strings.Join(order, ", "))}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return "", err
|
||||
}
|
||||
fmt.Printf("%s: %s — %s\n", p.ID, p.Repository, p.Note)
|
||||
return fmt.Sprintf("%s sends it to %s, one machine at a time, each judged", p.ID, strings.Join(order, ", ")), nil
|
||||
}
|
||||
|
||||
// settingsDataMoves is what a settings walk's send to a machine would move of a running module's data: composed as
|
||||
// that send composes it, every build kept, and compared with what the machine was last sent (ADR 0217 §3).
|
||||
var settingsDataMoves = func(ctx context.Context, open *stores, node string) ([]dataMove, error) {
|
||||
keeping := keepingBuildsForSettings(keepingRecorded(ctx), node)
|
||||
gens, err := generators(keeping, open)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plan, settings, err := planFor(keeping, open, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
declared, err := declarationWith(keeping, open, node, plan, settings, gens, Reading)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return heldMoves(keeping, open.inventory, node, body, nil)
|
||||
}
|
||||
|
||||
// judgingOn is the open walk, other than the one named, whose gate is judging on a machine; empty when none is. A
|
||||
// settings walk sends a machine only when no other walk judges there: its send would set aside the declaration
|
||||
// that walk's gate reads the machine's report against.
|
||||
var judgingOn = func(ctx context.Context, open *stores, self, node string) (string, error) {
|
||||
plans, err := open.inventory.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return judgingIn(plans, self, node), nil
|
||||
}
|
||||
|
||||
// judgingIn is judgingOn over plans read. Pure.
|
||||
func judgingIn(plans []inventory.Plan, self, node string) string {
|
||||
judging := func(g *inventory.PlanGate) bool {
|
||||
return g != nil && g.Verdict == "" && slices.Contains(g.Machines, node)
|
||||
}
|
||||
for _, p := range plans {
|
||||
if p.ID == self || !p.Open() {
|
||||
continue
|
||||
}
|
||||
if p.Release != nil {
|
||||
if judging(p.Release.Gate) {
|
||||
return p.ID
|
||||
}
|
||||
continue
|
||||
}
|
||||
for _, s := range p.Modules {
|
||||
if s != nil && judging(s.Gate) {
|
||||
return p.ID
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// advanceSettingsWalk takes one step of a settings walk: the next machine sent and its module judged there, or the
|
||||
// machine being judged judged once more; done when every machine is.
|
||||
func advanceSettingsWalk(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) {
|
||||
r, s := p.Release, p.Release.Settings
|
||||
now := time.Now().UTC()
|
||||
wait := func(note string) (bool, error) {
|
||||
changed := p.Note != note
|
||||
p.Note = note
|
||||
return changed, nil
|
||||
}
|
||||
if r.Gate == nil {
|
||||
heard, err := releaseHeard(ctx, open)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for r.Next < len(r.Order) {
|
||||
node := r.Order[r.Next]
|
||||
if !heard[node] {
|
||||
r.Skipped = append(r.Skipped, node)
|
||||
r.Next++
|
||||
continue
|
||||
}
|
||||
if id, err := judgingOn(ctx, open, p.ID, node); err != nil {
|
||||
return false, err
|
||||
} else if id != "" {
|
||||
return wait(fmt.Sprintf("waiting before %s: %s is judging there", node, id))
|
||||
}
|
||||
// **A running module's data that would move is a person's word** (ADR 0217 §3): the walk stops before
|
||||
// that machine, failed and said, and `push --move` sends it.
|
||||
held, err := settingsDataMoves(ctx, open, node)
|
||||
if errors.Is(err, errSettingsCannotKeep) {
|
||||
return failBefore(p, node, err.Error())
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("whether %s's data would move could not be read: %w", node, err)
|
||||
}
|
||||
if len(held) > 0 {
|
||||
var said strings.Builder
|
||||
sayDataHeld(&said, node, held)
|
||||
return failBefore(p, node, "a running module's data would move there — "+
|
||||
strings.Join(strings.Fields(said.String()), " "))
|
||||
}
|
||||
// A build that cannot be kept is refused by the composition above, which is the send's own, and fails the
|
||||
// walk there: the send flattens its refusals into words, so it is not looked for again here.
|
||||
sent, grants, err := sendJudged(keepingBuildsForSettings(ctx, node), open, node)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("sending %s %s: %w", node, settingsWalkFor(*s), err)
|
||||
}
|
||||
// **The module's build is the one the gate judges** (issue 641 review): kept by this send, it is what a
|
||||
// crossing send — a merge's walk sending the machine a new build of the module — supersedes, so this gate
|
||||
// never reads that send's report as its own.
|
||||
kept := ""
|
||||
if builds, known, err := open.inventory.SentBuilds(ctx, node); err == nil && known {
|
||||
kept = builds[s.Module]
|
||||
}
|
||||
r.Gate = &inventory.PlanGate{Component: coreComponent(s.Module), Machines: sent, Since: &now,
|
||||
From: kept, To: kept, Sent: sentNow(ctx, open.inventory, sent), Grants: grants}
|
||||
p.Note = fmt.Sprintf("sent %s %s (ask %s), every build there kept; judging %s there", node,
|
||||
settingsWalkFor(*s), s.Ask, s.Module)
|
||||
if said := grantsSaid(grants); said != "" {
|
||||
p.Note += "; " + said
|
||||
}
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return true, nil
|
||||
}
|
||||
p.State, p.Tier = inventory.PlanDone, len(p.Tiers)
|
||||
p.Note = fmt.Sprintf("%s (ask %s) delivered on %s", settingsWalkFor(*s), s.Ask, strings.Join(r.Done, ", "))
|
||||
if len(r.Done) == 0 {
|
||||
// Sent nowhere is not delivered: said as the walk's failure, so the operator is not told it landed.
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("%s (ask %s) was sent to no machine", settingsWalkFor(*s), s.Ask)
|
||||
}
|
||||
if len(r.Skipped) > 0 {
|
||||
p.Note += "; not heard from, left as they were, their next send carries it: " + strings.Join(r.Skipped, ", ")
|
||||
}
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return true, nil
|
||||
}
|
||||
g := r.Gate
|
||||
node := r.Order[r.Next]
|
||||
verdict, err := judgeMoves(ctx, open, g, []judged{{module: s.Module, node: node}}, now)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
switch verdict {
|
||||
case "":
|
||||
return wait(fmt.Sprintf("judging %s on %s: %s", s.Module, node, gateLine(g)))
|
||||
case inventory.GatePassed:
|
||||
r.Done = append(r.Done, node)
|
||||
r.Next++
|
||||
r.Gate = nil
|
||||
return true, nil
|
||||
case inventory.GateSuperseded:
|
||||
p.State = inventory.PlanSuperseded
|
||||
p.Note = fmt.Sprintf("superseded on %s: %s — nothing judged, nothing put back; the layer is set, and the "+
|
||||
"machines not yet sent take it at their next send", node, g.Why)
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return true, nil
|
||||
}
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("%s failed its gate on %s: %s — the layer stays as the operator approved it, and the one it "+
|
||||
"replaced is in `settings %s` history", settingsWalkFor(*s), node, g.Why, s.Module) + unreached(r)
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// failBefore ends a settings walk failed before it sent a machine, saying why, and that the layer stays.
|
||||
func failBefore(p *inventory.Plan, node, why string) (bool, error) {
|
||||
r, s := p.Release, p.Release.Settings
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("%s not sent to %s: %s; the layer stays as the operator approved it", settingsWalkFor(*s),
|
||||
node, why) + unreached(r)
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// unreached is what a stopped settings walk left without the layer, in words: the machines after the one it stopped
|
||||
// at, and those passed over as away (ADR 0344 §6). Empty when there are none.
|
||||
func unreached(r *inventory.PlanRelease) string {
|
||||
var out string
|
||||
if r.Next+1 < len(r.Order) {
|
||||
out += "; not sent: " + strings.Join(r.Order[r.Next+1:], ", ")
|
||||
}
|
||||
if len(r.Skipped) > 0 {
|
||||
out += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ")
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,572 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0344, issue 641: the operator's one Approve on a settings proposal is the act that makes it take
|
||||
// effect. The warrant sets the layer and opens a settings walk, which sends it to the machines it reaches one at a
|
||||
// time, judging the module on each; the ask says so before the Approve, and nothing names a push.
|
||||
|
||||
// The order: every machine running the module, or the one the layer names, the control node last.
|
||||
func TestASettingsWalkGoesToTheMachinesTheLayerReachesTheControlNodeLast(t *testing.T) {
|
||||
running := []string{"shanks", "novox", "g14", "ace"}
|
||||
for _, c := range []struct {
|
||||
node string
|
||||
want []string
|
||||
}{
|
||||
{"", []string{"ace", "g14", "shanks", "novox"}},
|
||||
{"g14", []string{"g14"}},
|
||||
{"novox", []string{"novox"}},
|
||||
{"elsewhere", nil},
|
||||
} {
|
||||
if got := settingsWalkOrder(c.node, running, []string{"novox"}); !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("a layer on %q goes to %v, want %v", c.node, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A settings walk sends a machine only when no other walk judges there.
|
||||
func TestASettingsWalkWaitsWhileAnotherWalkJudgesOnTheMachine(t *testing.T) {
|
||||
judging := &inventory.PlanGate{Machines: []string{"g14"}}
|
||||
passed := &inventory.PlanGate{Machines: []string{"g14"}, Verdict: inventory.GatePassed}
|
||||
plans := []inventory.Plan{
|
||||
{ID: "settings-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: judging}},
|
||||
{ID: "release-2", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: passed}},
|
||||
{ID: "plan-3", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{"app": {Gate: judging}}},
|
||||
{ID: "plan-4", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{"app": {Gate: judging}}},
|
||||
}
|
||||
if id := judgingIn(plans, "settings-1", "g14"); id != "plan-3" {
|
||||
t.Errorf("on g14 %q judges, want plan-3 (itself, a passed gate and an ended walk are not judging)", id)
|
||||
}
|
||||
if id := judgingIn(plans[1:2], "x", "g14"); id != "" {
|
||||
t.Errorf("a passed gate judges: %q", id)
|
||||
}
|
||||
if id := judgingIn(plans, "x", "shanks"); id != "" {
|
||||
t.Errorf("on shanks %q judges", id)
|
||||
}
|
||||
}
|
||||
|
||||
// The ask says, before the Approve, that the layer is sent at once and judged, and that its end is told as every
|
||||
// walk's is; neither the ask nor the proposer's answer names a push any more.
|
||||
func TestTheAskSaysAnApprovedLayerIsSentAtOnce(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
in proposeInput
|
||||
where string
|
||||
had bool
|
||||
}{
|
||||
{proposeInput{module: "mounts", node: "shanks", values: mountsSources}, "shanks", false},
|
||||
{proposeInput{module: "notes", values: map[string]any{"x": "1"}}, "every machine running notes", true},
|
||||
{proposeInput{module: "notes", node: "laptop", clear: true}, "laptop", true},
|
||||
} {
|
||||
r := newProposerRig(t)
|
||||
if c.had {
|
||||
r.before, r.had = map[string]any{"x": "0"}, true
|
||||
}
|
||||
words, err := r.pr.propose(context.Background(), c.in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q := r.askSent(t)
|
||||
if err := q.Check(r.now); err != nil {
|
||||
t.Fatalf("the ask is refused: %v", err)
|
||||
}
|
||||
// At a glance (ADR 0345): the summary says it is sent at once by a walk, one machine at a time; the rest of
|
||||
// what Approve does is the Details', which the router notes after the answers.
|
||||
if !strings.Contains(q.Explanation, "sent at once by a walk to "+c.where+", one machine at a time, each judged") ||
|
||||
!strings.Contains(q.Details, "and sent at once to "+c.where+", one machine at a time, each judged, "+
|
||||
"every build there kept; its end is told as every walk's is") {
|
||||
t.Errorf("the ask does not say the layer is sent at once to %s:\n%s", c.where, q.Explanation)
|
||||
}
|
||||
approve := q.Options[0]
|
||||
if approve.Label != "Approve" || !strings.Contains(approve.Does, "sent to "+c.where+" at once") {
|
||||
t.Errorf("Approve does %q", approve.Does)
|
||||
}
|
||||
if !strings.Contains(words, "a walk sends it to "+c.where+" at once") {
|
||||
t.Errorf("the proposer's answer:\n%s", words)
|
||||
}
|
||||
for _, said := range []string{q.Explanation, q.Whole, approve.Does, words} {
|
||||
if strings.Contains(said, "push") {
|
||||
t.Errorf("a push is still named, as if a second act were needed:\n%s", said)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- on the real stores -----------------------------------------------------------------------------
|
||||
|
||||
// The warrant sets the layer and opens the walk that sends it, naming the machines it reaches; a layer for a
|
||||
// machine that does not run the module opens none and says so.
|
||||
func TestAnApprovedLayerOpensTheWalkThatSendsIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "notes", Version: "1",
|
||||
Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644",
|
||||
"content": "x = ${setting:x}\n"}}})
|
||||
if _, err := assign(ctx, open, "laptop", "notes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A layer set by hand opens no walk: it still waits for a push (ADR 0344 §7).
|
||||
if err := atTheTerminal(t, "settings", "set", "notes", `{"x":"0"}`, "--node", "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plans, _ := open.inventory.OpenPlans(ctx); len(plans) != 0 {
|
||||
t.Fatalf("a layer set at the terminal opened %d walk(s)", len(plans))
|
||||
}
|
||||
set := setLayerIn(open)
|
||||
now := time.Now()
|
||||
values := map[string]any{"x": "1"}
|
||||
p := settingsProposal{Module: "notes", Values: values, From: "g14/claude-code", At: now,
|
||||
Digest: layerDigest(values), BeforeDigest: layerDigest(nil)}
|
||||
words, err := set(ctx, p, "s9", "approved by the operator (ask s9)")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(words, "+ x (ask s9); settings-") || !strings.Contains(words, "sends it to laptop, one machine at a time") {
|
||||
t.Errorf("the warrant's words: %q", words)
|
||||
}
|
||||
plans, err := open.inventory.OpenPlans(ctx)
|
||||
if err != nil || len(plans) != 1 {
|
||||
t.Fatalf("open walks: %+v %v", plans, err)
|
||||
}
|
||||
w := plans[0]
|
||||
if w.Release == nil || w.Release.Settings == nil || w.Release.Settings.Module != "notes" || w.Release.Settings.Ask != "s9" ||
|
||||
!reflect.DeepEqual(w.Release.Order, []string{"laptop"}) || w.Repository != "the settings of notes on the whole mesh" ||
|
||||
!reflect.DeepEqual(w.Tiers, [][]string{{"notes"}}) {
|
||||
t.Fatalf("the settings walk: %+v %+v", w, w.Release)
|
||||
}
|
||||
|
||||
// A machine's layer where the module does not run: set, and nothing to send.
|
||||
q := settingsProposal{Module: "notes", Node: "anchor", Values: values, From: "g14/claude-code", At: now,
|
||||
Digest: layerDigest(values), BeforeDigest: layerDigest(nil)}
|
||||
words, err = set(ctx, q, "s10", "approved by the operator (ask s10)")
|
||||
if err != nil || !strings.Contains(words, "no machine runs notes there, so there is nothing to send") {
|
||||
t.Fatalf("a layer nobody runs: %q %v", words, err)
|
||||
}
|
||||
if plans, _ := open.inventory.OpenPlans(ctx); len(plans) != 1 {
|
||||
t.Fatalf("a walk was opened for a machine that does not run the module: %d open", len(plans))
|
||||
}
|
||||
|
||||
// A walk that cannot be opened is the act's failure, said with the push that sends it by hand.
|
||||
was := openSettingsWalk
|
||||
openSettingsWalk = func(context.Context, *stores, inventory.PlanSettings) (string, error) {
|
||||
return "", fmt.Errorf("the plans are not to be had")
|
||||
}
|
||||
t.Cleanup(func() { openSettingsWalk = was })
|
||||
again := map[string]any{"x": "2"}
|
||||
byHand := map[string]any{"x": "0"}
|
||||
r := settingsProposal{Module: "notes", Node: "laptop", Values: again, Before: byHand, HadLayer: true,
|
||||
From: "g14/claude-code", At: now, Digest: layerDigest(again), BeforeDigest: layerDigest(byHand)}
|
||||
if _, err := set(ctx, r, "s11", "approved"); err == nil || !strings.Contains(err.Error(), "is set, and the walk that "+
|
||||
"sends it could not be opened: the plans are not to be had — `push laptop` sends it") {
|
||||
t.Errorf("an unopened walk: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// walkMesh is a mesh where `app` runs c1 on anchor and laptop and nothing waits for a gate: every send applied at
|
||||
// once, recorded, and judged from the reports, with a machine named broken failing what it was sent.
|
||||
type walkMesh struct {
|
||||
open *stores
|
||||
sent [][]string
|
||||
broken map[string]bool
|
||||
// kept says, per send, whether it kept every build of its machine; moves is the data a machine's send would move.
|
||||
kept []bool
|
||||
moves map[string][]dataMove
|
||||
heard map[string]bool
|
||||
}
|
||||
|
||||
func aSettledMesh(t *testing.T) *walkMesh {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
w := &walkMesh{open: open, broken: map[string]bool{}, moves: map[string][]dataMove{},
|
||||
heard: map[string]bool{"anchor": true, "laptop": true}}
|
||||
withConditionsInMemory(t)
|
||||
was := doctorFrom
|
||||
doctorFrom = nil
|
||||
t.Cleanup(func() { doctorFrom = was })
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c1", Module: "app", Commit: "c1",
|
||||
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: old, At: old,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-1"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c1", Head: "c1", Asked: old}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, n, "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, map[string]string{"app": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
wasMoves, wasHeard, wasSend, wasGather, wasData := machineMoves, releaseHeard, sendRollout, gatherGateFacts, settingsDataMoves
|
||||
settingsDataMoves = func(_ context.Context, _ *stores, node string) ([]dataMove, error) { return w.moves[node], nil }
|
||||
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
|
||||
modules, err := open.inventory.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sent, known, err := open.inventory.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.moves(node, modules, sent, known, all), nil
|
||||
}
|
||||
releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return w.heard, nil }
|
||||
n := 0
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
w.sent = append(w.sent, append([]string(nil), names...))
|
||||
w.kept = append(w.kept, len(names) == 1 && settingsKeptOn(ctx, names[0]))
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range names {
|
||||
modules, _ := open.inventory.Assigned(ctx, node)
|
||||
carried := map[string]string{}
|
||||
for _, m := range modules {
|
||||
carried[m] = current[m].Commit
|
||||
}
|
||||
n++
|
||||
digest := fmt.Sprintf("d-%s-%d", node, n)
|
||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
||||
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
||||
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
|
||||
reports, err := open.inventory.LastReports(ctx)
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
for _, r := range reports {
|
||||
if w.broken[r.Node] {
|
||||
r.Outcome = inventory.OutcomeFailed
|
||||
}
|
||||
f.reports[r.Node] = r
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||
wasWindow := gateWitnessWindow
|
||||
gateWitnessWindow = 0
|
||||
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||
gateWithin, gateEvery = time.Hour, time.Hour
|
||||
t.Cleanup(func() {
|
||||
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
|
||||
settingsDataMoves = wasData
|
||||
gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound
|
||||
})
|
||||
return w
|
||||
}
|
||||
|
||||
func (w *walkMesh) walk(t *testing.T) inventory.Plan {
|
||||
t.Helper()
|
||||
plans, err := w.open.inventory.RecentPlans(t.Context(), 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, p := range plans {
|
||||
if p.Release != nil && p.Release.Settings != nil {
|
||||
return p
|
||||
}
|
||||
}
|
||||
t.Fatal("no settings walk")
|
||||
return inventory.Plan{}
|
||||
}
|
||||
|
||||
// The walk sends the first machine alone and judges the module there before it sends the next, and ends
|
||||
// delivered on both; nothing waited for a gate, so each send carried the layer alone.
|
||||
func TestASettingsWalkSendsOneMachineAtATimeEachJudged(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
w := aSettledMesh(t)
|
||||
ctx := t.Context()
|
||||
words, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s1", Change: "~ x"})
|
||||
if err != nil || !strings.Contains(words, "sends it to anchor, laptop") {
|
||||
t.Fatalf("%q %v", words, err)
|
||||
}
|
||||
advancePlans(ctx, w.open)
|
||||
if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) {
|
||||
t.Fatalf("sent %v: the first machine alone, before it is judged", w.sent)
|
||||
}
|
||||
p := w.walk(t)
|
||||
if p.Release.Gate == nil || p.Release.Next != 0 || !strings.Contains(p.Note, "judging app on anchor") {
|
||||
t.Fatalf("the walk after its first send: %s %+v", p.Note, p.Release)
|
||||
}
|
||||
gateEvery = 0
|
||||
for i := 0; i < 6; i++ {
|
||||
advancePlans(ctx, w.open)
|
||||
}
|
||||
if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}, {"laptop"}}) {
|
||||
t.Fatalf("sent %v", w.sent)
|
||||
}
|
||||
p = w.walk(t)
|
||||
if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, []string{"anchor", "laptop"}) ||
|
||||
p.Outcome() != inventory.WalkDelivered || !strings.Contains(p.Note, "delivered on anchor, laptop") {
|
||||
t.Fatalf("the walk is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
// It said its start and its end, so the operator is told once (ADR 0308).
|
||||
if p.Times == nil || p.Times.Started == nil || p.Times.Ended == nil {
|
||||
t.Fatalf("the walk's start and end are not marked: %+v", p.Times)
|
||||
}
|
||||
// Nothing waits, and no walk of waiting builds opened beside it.
|
||||
if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r != nil {
|
||||
t.Fatalf("%+v %v", r, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine that fails what it was sent stops the walk there: the next is not sent, the layer stays, and the
|
||||
// failure holds back no walk of waiting builds.
|
||||
func TestAFailedSettingsWalkStopsAndHoldsNothingBack(t *testing.T) {
|
||||
w := aSettledMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := w.open.inventory
|
||||
w.broken["anchor"] = true
|
||||
if err := inv.KeepSettings(ctx, "", "app", map[string]any{"x": "1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s2"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 4; i++ {
|
||||
advancePlans(ctx, w.open)
|
||||
}
|
||||
p := w.walk(t)
|
||||
if p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed ||
|
||||
!strings.Contains(p.Note, "the settings of app on the whole mesh failed its gate on anchor") ||
|
||||
!strings.Contains(p.Note, "the layer stays as the operator approved it") || !strings.Contains(p.Note, "not sent: laptop") {
|
||||
t.Fatalf("the walk is %s: %s", p.State, p.Note)
|
||||
}
|
||||
if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) {
|
||||
t.Fatalf("sent %v: anchor alone, nothing to laptop", w.sent)
|
||||
}
|
||||
if layer, has, err := inv.Layer(ctx, "", "app"); err != nil || !has || layer["x"] != "1" {
|
||||
t.Fatalf("the layer after the failure: %v %v %v", layer, has, err)
|
||||
}
|
||||
// A build that waits for a gate is still walked: the failed settings walk holds nothing back.
|
||||
w.broken["anchor"] = false
|
||||
now := time.Now().Add(-time.Minute)
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2",
|
||||
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r == nil {
|
||||
t.Fatalf("the waiting build is held back by a failed settings walk: %+v %v", r, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Each send of a settings walk keeps every build of its machine: the layer goes, and a build held for a person or
|
||||
// waiting for a gate stays where it is (ADR 0344 §3).
|
||||
func TestASettingsWalkSendsTheLayerAloneEveryBuildKept(t *testing.T) {
|
||||
w := aSettledMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := w.open.inventory
|
||||
gateEvery = 0
|
||||
if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s3"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 4; i++ {
|
||||
advancePlans(ctx, w.open)
|
||||
}
|
||||
if !reflect.DeepEqual(w.kept, []bool{true, true}) || w.walk(t).State != inventory.PlanDone {
|
||||
t.Fatalf("the sends kept every build: %v; the walk: %s", w.kept, w.walk(t).Note)
|
||||
}
|
||||
// What such a send keeps, on the store: a rolled-out build waiting there is kept at the one last sent.
|
||||
now := time.Now().Add(-time.Minute)
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2",
|
||||
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept, err := recordedKept(keepingBuildsForSettings(ctx, "anchor"), w.open, "anchor")
|
||||
if err != nil || kept["app"] != "c1" {
|
||||
t.Fatalf("a settings send to anchor keeps %v (%v), want app at c1", kept, err)
|
||||
}
|
||||
if kept, _ := recordedKept(ctx, w.open, "anchor"); kept["app"] != "" {
|
||||
t.Fatalf("an ordinary send keeps a rolled-out build: %v", kept)
|
||||
}
|
||||
// And so the ungated build waiting on anchor is no move of the settings send's: nothing for a gate to see.
|
||||
f, err := readMoveFacts(ctx, inv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waiting, _ := machineMoves(ctx, w.open, f, "anchor", false)
|
||||
if len(waiting) != 1 || waiting[0].Module != "app" {
|
||||
t.Fatalf("anchor waits for %+v, want app's c2", waiting)
|
||||
}
|
||||
if moved, err := withoutKept(keepingBuildsForSettings(ctx, "anchor"), w.open, "anchor", waiting); err != nil || len(moved) != 0 {
|
||||
t.Fatalf("a settings send to anchor would move %+v (%v)", moved, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine whose send would move a running module's data stops the walk there, failed and said with the push that
|
||||
// sends it on a person's word (ADR 0217 §3); one not heard from is passed over and named; and a walk that reached
|
||||
// no machine is not said delivered.
|
||||
func TestASettingsWalkStopsBeforeADataMoveAndPassesOverAMachineAway(t *testing.T) {
|
||||
w := aSettledMesh(t)
|
||||
ctx := t.Context()
|
||||
gateEvery = 0
|
||||
w.moves["laptop"] = []dataMove{{Container: "app-server", Inside: "/data", From: "/srv/a", To: "/srv/b"}}
|
||||
if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s4"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 4; i++ {
|
||||
advancePlans(ctx, w.open)
|
||||
}
|
||||
p := w.walk(t)
|
||||
if p.State != inventory.PlanFailed || !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) ||
|
||||
!strings.Contains(p.Note, "not sent to laptop: a running module's data would move there") ||
|
||||
!strings.Contains(p.Note, "push laptop --move app") {
|
||||
t.Fatalf("sent %v; the walk is %s: %s", w.sent, p.State, p.Note)
|
||||
}
|
||||
|
||||
v := aSettledMesh(t)
|
||||
v.heard = map[string]bool{"laptop": true}
|
||||
gateEvery = 0
|
||||
if _, err := openSettingsWalk(ctx, v.open, inventory.PlanSettings{Module: "app", Ask: "s5"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 4; i++ {
|
||||
advancePlans(ctx, v.open)
|
||||
}
|
||||
p = v.walk(t)
|
||||
if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Skipped, []string{"anchor"}) ||
|
||||
!strings.Contains(p.Note, "delivered on laptop; not heard from, left as they were, their next send carries it: anchor") {
|
||||
t.Fatalf("the walk with anchor away is %s: %s", p.State, p.Note)
|
||||
}
|
||||
|
||||
u := aSettledMesh(t)
|
||||
u.heard = map[string]bool{}
|
||||
if _, err := openSettingsWalk(ctx, u.open, inventory.PlanSettings{Module: "app", Ask: "s6"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, u.open)
|
||||
if p := u.walk(t); p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed ||
|
||||
!strings.Contains(p.Note, "was sent to no machine") || len(u.sent) != 0 {
|
||||
t.Fatalf("a walk that reached nobody is %s: %s", p.State, p.Note)
|
||||
}
|
||||
}
|
||||
|
||||
// The gate keeps the module's build its send kept, so a crossing send — a merge's walk sending the machine a new
|
||||
// build of the module while the settings gate judges there — supersedes it, and its report is never read as the
|
||||
// settings send's (issue 641 review, finding 1).
|
||||
func TestACrossingSendSupersedesASettingsGate(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
w := aSettledMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := w.open.inventory
|
||||
if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s7"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, w.open)
|
||||
p := w.walk(t)
|
||||
if g := p.Release.Gate; g == nil || g.To != "c1" || g.From != "c1" {
|
||||
t.Fatalf("the settings gate judges no build: %+v", p.Release.Gate)
|
||||
}
|
||||
// Another walk sends anchor app's c2 while the settings gate judges there.
|
||||
now := time.Now().Add(-time.Minute)
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2",
|
||||
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, w.open, "anchor"), "d-merge", map[string]string{"app": "c2"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.RecordDoing(ctx, nodeID(t, w.open, "anchor"), inventory.Doing{Node: "anchor",
|
||||
Outcome: inventory.OutcomeApplied, Declared: "d-merge", Applied: 1, At: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gateEvery = 0
|
||||
advancePlans(ctx, w.open)
|
||||
p = w.walk(t)
|
||||
if p.State != inventory.PlanSuperseded || p.Release.Gate == nil || p.Release.Gate.Verdict != inventory.GateSuperseded ||
|
||||
!strings.Contains(p.Note, "superseded on anchor") || len(p.Release.Done) != 0 {
|
||||
t.Fatalf("the settings walk after a crossing send is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) {
|
||||
t.Fatalf("sent %v: laptop was sent after a superseded judging", w.sent)
|
||||
}
|
||||
}
|
||||
|
||||
// A send refusal that cannot clear by itself — what the machine was last sent is not known, or a build it runs is
|
||||
// gone — fails the walk with why, rather than keeping it open and holding the walk of waiting builds; and a
|
||||
// failure names the machines passed over as away (issue 641 review, findings 2 and 3).
|
||||
func TestASettingsWalkFailsOnARefusalThatCannotClear(t *testing.T) {
|
||||
w := aSettledMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := w.open.inventory
|
||||
// What a send to a machine never sent can keep: nothing, and said as the refusal the walk fails on.
|
||||
if _, err := inv.AddNode(ctx, "fresh"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := recordedKept(keepingBuildsForSettings(ctx, "fresh"), w.open, "fresh"); !errors.Is(err, errSettingsCannotKeep) {
|
||||
t.Fatalf("a settings send to a machine never sent: %v", err)
|
||||
}
|
||||
w.heard = map[string]bool{"laptop": true}
|
||||
settingsDataMoves = func(_ context.Context, _ *stores, node string) ([]dataMove, error) {
|
||||
return nil, fmt.Errorf("%w: what %s was last sent is not known", errSettingsCannotKeep, node)
|
||||
}
|
||||
if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s8"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, w.open)
|
||||
p := w.walk(t)
|
||||
if p.State != inventory.PlanFailed || len(w.sent) != 0 ||
|
||||
!strings.Contains(p.Note, "not sent to laptop: the settings cannot be sent with every build kept: what laptop was last sent is not known") ||
|
||||
!strings.Contains(p.Note, "not heard from, left as they were: anchor") {
|
||||
t.Fatalf("the walk is %s: %s", p.State, p.Note)
|
||||
}
|
||||
// It is not open, so a build waiting for a gate is walked.
|
||||
now := time.Now().Add(-time.Minute)
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2",
|
||||
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r == nil {
|
||||
t.Fatalf("the waiting build is held by the failed settings walk: %+v %v", r, err)
|
||||
}
|
||||
}
|
||||
@@ -153,11 +153,15 @@ var signalsTable = []signalRow{
|
||||
return newestOf(f.standings, func(c conditions.Condition) time.Time { return c.LastObserved })
|
||||
}},
|
||||
{Row: "S9", Signal: "bus advisories: maximum deliveries, consumer deleted; the controller's own slow " +
|
||||
"consumer and refused subjects", Emitter: "bus server's advisory subjects; the controller's connection",
|
||||
"consumer and refused subjects; every user's connection the server closed as a slow consumer or for " +
|
||||
"breaking the bus's rules (ADR 0357)",
|
||||
Emitter: "bus server's advisory subjects; the controller's connection; the bus module's watch of its " +
|
||||
"server (bus.slow-consumer, bus.connection-closed)",
|
||||
Trigger: "any", Bound: "any occurrence; clears after an hour without another, a deleted consumer " +
|
||||
"once it exists again or the mesh no longer expects it, and a message given up on once DEAD_LETTERS " +
|
||||
"no longer holds it (novox/hq issue 330)",
|
||||
Kind: "slow-consumer, max-deliveries, refused, consumer-lost", Severity: conditions.Warning, Phase: 1,
|
||||
Kind: "slow-consumer, connection-closed, max-deliveries, refused, consumer-lost", Severity: conditions.Warning,
|
||||
Phase: 1,
|
||||
needs: func(f *signalFacts) error { return f.advisoriesErr }, watch: watchAdvisories,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.advisories, func(a link.Advisory) time.Time { return a.Last })
|
||||
@@ -569,6 +573,9 @@ func watchAdvisories(f *signalFacts) []conditions.Observation {
|
||||
// The advisory is remembered and read again on every look for an hour: the condition counts
|
||||
// what the bus said and when, not the looks (novox/hq issue 402).
|
||||
Happened: a.Last, Times: a.Count}
|
||||
if a.User != "" && a.Reason != "" {
|
||||
busUserClosed(&o, a, f.host)
|
||||
}
|
||||
if a.Kind == link.AdvisoryMaxDeliveries && a.Token == link.AdvisoryNotKept {
|
||||
o.Machine = consumerMachine(a.Stream, a.Consumer)
|
||||
o.Headline = clip(conditions.Capital(fmt.Sprintf("%s gave up on a message, not kept",
|
||||
@@ -579,9 +586,58 @@ func watchAdvisories(f *signalFacts) []conditions.Observation {
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
if f.busFaultsErr != nil {
|
||||
out = append(out, busFaultsUnread(f.busFaultsErr))
|
||||
}
|
||||
return append(out, watchDeadLetters(f)...)
|
||||
}
|
||||
|
||||
// kindBusFaultsUnread is the bus module's events not read back (novox/hq ADR 0357): every other user's
|
||||
// slow consumer is unheard while it lasts, and the controller's own signals are still said.
|
||||
const kindBusFaultsUnread = "bus-faults-unread"
|
||||
|
||||
func busFaultsUnread(err error) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeBus, ID: "bus-module", Kind: kindBusFaultsUnread,
|
||||
Severity: conditions.Warning, Confirm: true,
|
||||
Summary: "what the bus module said of its users' connections could not be read back, so a slow consumer " +
|
||||
"or a connection the bus closed is not said until it can",
|
||||
Said: firstLine(err.Error())}
|
||||
}
|
||||
|
||||
// busUserClosed words an advisory about a bus user's connection the server closed (novox/hq ADR 0357): the
|
||||
// user as the operator says it and the machine it is on, the dotted user name kept in the evidence only.
|
||||
func busUserClosed(o *conditions.Observation, a link.Advisory, host string) {
|
||||
who, machine := busUserWords(a.User)
|
||||
if machine == "" && a.User == "controller" {
|
||||
machine = host
|
||||
}
|
||||
o.Machine = machine
|
||||
times, count := "once", "once"
|
||||
if a.Count > 1 {
|
||||
times = fmt.Sprintf("%d times since %s", a.Count, a.First.UTC().Format("15:04 MST"))
|
||||
count = fmt.Sprintf("%d times", a.Count)
|
||||
}
|
||||
switch a.Kind {
|
||||
case link.AdvisorySlowConsumer:
|
||||
o.Summary = fmt.Sprintf("the bus closed the connection of %s as a slow consumer %s in the last hour, "+
|
||||
"newest at %s: it could not take what it was sent fast enough, and what was on its way to it was lost",
|
||||
who, times, a.Last.UTC().Format("15:04 MST"))
|
||||
o.Headline = clip(conditions.Capital(who)+" was too slow for the bus", 60)
|
||||
o.Explanation = conditions.Capital(fmt.Sprintf("%s could not keep up with what the bus sent it, so the bus "+
|
||||
"closed its connection (%s in the last hour) and what was on its way was lost. It connects again "+
|
||||
"by itself; this clears an hour after the last time.", who, count))
|
||||
o.Resolved = "Resolved: " + who + " kept up for an hour"
|
||||
default:
|
||||
o.Summary = fmt.Sprintf("the bus closed the connection of %s for breaking its rules %s in the last hour, "+
|
||||
"newest at %s: %s", who, times, a.Last.UTC().Format("15:04 MST"), strings.ToLower(a.Reason))
|
||||
o.Headline = clip(conditions.Capital(who)+" was cut off by the bus", 60)
|
||||
o.Explanation = conditions.Capital(fmt.Sprintf("The bus closed the connection of %s because it broke "+
|
||||
"the bus's rules (%s, %s in the last hour), so what it was sending was lost. This clears an hour "+
|
||||
"after the last time.", who, strings.ToLower(a.Reason), count))
|
||||
o.Resolved = "Resolved: " + who + " kept to the bus's rules for an hour"
|
||||
}
|
||||
}
|
||||
|
||||
// watchDeadLetters says each consumer that DEAD_LETTERS holds a message for (novox/hq issue 330): open
|
||||
// while it holds any, so it clears when they are delivered again or dropped, never because the server
|
||||
// stopped saying it.
|
||||
@@ -806,7 +862,7 @@ func watchLease(f *signalFacts) []conditions.Observation {
|
||||
}
|
||||
}
|
||||
if len(lost) > 0 {
|
||||
how := "lost it: its renewal was refused or could not be made"
|
||||
how := "lost it: another holder wrote its key, or no renewal was made within the key's age"
|
||||
if newest.How == inventory.EpochExpired {
|
||||
how = "stopped renewing it without giving it back, and was taken over"
|
||||
}
|
||||
|
||||
@@ -263,18 +263,8 @@ func TestTheAgentsManagedSettingsAreRefusedThroughAVerb(t *testing.T) {
|
||||
if err := atTheTerminal(t, "settings", "set", "claude-code", allow, "--node", "laptop"); err != nil {
|
||||
t.Fatalf("the managed settings at the terminal: %v", err)
|
||||
}
|
||||
// There is no module layer for the whole mesh (novox/hq issue 630), at the terminal either; one the mesh
|
||||
// still holds from before is kept as it is, and is the terminal's as before.
|
||||
if err := atTheTerminal(t, "settings", "set", "claude-code", server); err == nil ||
|
||||
!strings.Contains(err.Error(), "no module layer for the whole mesh") {
|
||||
t.Fatalf("a tool server for the whole mesh at the terminal: %v", err)
|
||||
}
|
||||
var held map[string]any
|
||||
if err := json.Unmarshal([]byte(server), &held); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.KeepSettings(ctx, "", "claude-code", held); err != nil {
|
||||
t.Fatalf("a whole-mesh layer held from before: %v", err)
|
||||
if err := atTheTerminal(t, "settings", "set", "claude-code", server); err != nil {
|
||||
t.Fatalf("a tool server for the mesh at the terminal: %v", err)
|
||||
}
|
||||
kept := layer("laptop")
|
||||
// Through a verb they are neither changed, dropped nor cleared.
|
||||
@@ -369,15 +359,15 @@ func TestOnlyATrustedFilesNamedKeyIsRefusedThroughAVerb(t *testing.T) {
|
||||
if _, err := assign(ctx, open, "anchor", "mailer"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := throughVerb(t, "settings", map[string]any{"module": "mailer", "node": "anchor", "values": `{"tags":{"todo":{"name":"To do"}}}`}); err != nil {
|
||||
if err := throughVerb(t, "settings", map[string]any{"module": "mailer", "values": `{"tags":{"todo":{"name":"To do"}}}`}); err != nil {
|
||||
t.Fatalf("a key of a file nothing trusts is the verb's: %v", err)
|
||||
}
|
||||
err := throughVerb(t, "settings", map[string]any{"module": "mailer", "node": "anchor", "replace": "true",
|
||||
err := throughVerb(t, "settings", map[string]any{"module": "mailer", "replace": "true",
|
||||
"values": `{"tags":{"todo":{"name":"To do"}},"accounts":{"mail":{"address":"me@mail.example"}}}`})
|
||||
if err == nil || !strings.Contains(err.Error(), "controller's terminal") || !strings.Contains(err.Error(), "accounts") {
|
||||
t.Fatalf("a key of a trusted file is the terminal's: %v", err)
|
||||
}
|
||||
values, _, err := open.inventory.Layer(ctx, "anchor", "mailer")
|
||||
values, _, err := open.inventory.Layer(ctx, "", "mailer")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -130,10 +130,14 @@ func aTierMesh(t *testing.T, modules ...string) *tierMesh {
|
||||
}
|
||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||
|
||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||
wasWindow := gateWitnessWindow
|
||||
gateWitnessWindow = 0
|
||||
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||
// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
|
||||
gateSettle, gateEvery = 0, time.Hour
|
||||
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
||||
gateWithin, gateEvery = time.Hour, time.Hour
|
||||
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
|
||||
|
||||
built := time.Now().UTC()
|
||||
states := map[string]*inventory.PlanModule{}
|
||||
@@ -167,6 +171,7 @@ func (tm *tierMesh) raise(machine, kind, summary string) {
|
||||
// The incident's tier, small: a dozen modules whose first machine is the same are sent there in one
|
||||
// send, judged by one gate, and the rest of their machines are sent once — two sends, not two dozen.
|
||||
func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
var modules []string
|
||||
for i := 1; i <= 12; i++ {
|
||||
@@ -219,9 +224,13 @@ func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) {
|
||||
// settling, is the node tools' — inside their settle window, the gate's bound — and no failure of the
|
||||
// modules sent with them: at the bound only the node tools are put back.
|
||||
func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
tm := aTierMesh(t, "app1", "app2", broker.RuntimeModule)
|
||||
ctx := t.Context()
|
||||
inv := tm.open.inventory
|
||||
// The node tools are held inside their witness window, which the node-engine keeps while they settle: their
|
||||
// one reading does not count yet, so the core behind raised meanwhile is theirs (novox/hq ADR 0351 decision 4).
|
||||
gateWitnessWindow = time.Hour
|
||||
|
||||
advancePlans(ctx, tm.open)
|
||||
if len(tm.sent) != 1 {
|
||||
@@ -269,6 +278,7 @@ func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) {
|
||||
// A machine-level condition about anything else holds the machine back as a whole: everything the send
|
||||
// moved there fails together, once, with that reason — not one module for it.
|
||||
func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
tm := aTierMesh(t, "app1", "app2", "app3")
|
||||
ctx := t.Context()
|
||||
@@ -300,6 +310,7 @@ func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) {
|
||||
// The machine-level conditions, sorted: one naming a module is that module's; the core behind is the
|
||||
// core's when the send moved it and nobody's when it did not; anything else is the machine's as a whole.
|
||||
func TestTheConditionsAboutAMachineItself(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
since := time.Now().Add(-time.Minute)
|
||||
about := func(kind, id string) conditions.Condition {
|
||||
return conditions.Condition{Key: "machine." + id + "." + kind, Kind: kind, Raised: time.Now(),
|
||||
@@ -332,6 +343,7 @@ func TestTheConditionsAboutAMachineItself(t *testing.T) {
|
||||
// rollback that failed; and `plans retry` asks it again rather than refusing a build that was never
|
||||
// judged (the incident's "NOT put back: no build kept").
|
||||
func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
@@ -387,6 +399,7 @@ func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) {
|
||||
// them, they are registered back and marked in the judging that finds them broken, the plan goes on judging
|
||||
// the other module to its own pass, and only then fails.
|
||||
func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
for _, order := range [][]string{{broker.RuntimeModule, "app1"}, {"app1", broker.RuntimeModule}} {
|
||||
t.Run("gate kept on "+order[0], func(t *testing.T) {
|
||||
@@ -405,7 +418,10 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
|
||||
}
|
||||
return f, err
|
||||
}
|
||||
gateEvery, gateSettle, gateBound = 0, 100*time.Millisecond, 10*time.Second
|
||||
// app1, judged as the core, counts a reading at most every 100 ms: it cannot have its three in the
|
||||
// judgings that find the node tools broken, and is judged to its own pass after them (the hold the
|
||||
// settle time gave before novox/hq ADR 0352 counted readings from the send).
|
||||
gateEvery, gateWithin, gateBound = 100*time.Millisecond, time.Hour, 10*time.Second
|
||||
for i := 0; i < 20 && len(seen) < 2; i++ {
|
||||
advancePlans(ctx, tm.open)
|
||||
}
|
||||
@@ -440,15 +456,16 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// **A send carrying a core module waits for the core's readings** (issue 524 review): the leaf beside the node
|
||||
// tools has its one reading at once, but the send is judged as the core: not before the settle time, and three
|
||||
// readings.
|
||||
func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) {
|
||||
// **A send carrying the tool runner waits for its witness window** (issue 524 review, novox/hq ADR 0351
|
||||
// decision 4): the leaf beside the node tools has its one reading at once, but the send is judged by the tool
|
||||
// runner's rule: one reading, not before the node-engine's witness window for it has ended.
|
||||
func TestASendCarryingTheToolRunnerWaitsForItsWitnessWindow(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
tm := aTierMesh(t, broker.RuntimeModule, "app1")
|
||||
ctx := t.Context()
|
||||
// One reading counted per advance, as every test with a gate left open does: a gate judged with no time
|
||||
// between readings would count a pass at every look and never let an advance rest.
|
||||
gateEvery, gateSettle = time.Hour, time.Hour
|
||||
gateEvery, gateWitnessWindow = time.Hour, time.Hour
|
||||
for i := 0; i < 6; i++ {
|
||||
advancePlans(ctx, tm.open)
|
||||
}
|
||||
@@ -457,10 +474,10 @@ func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) {
|
||||
if lead == nil {
|
||||
lead = p.Modules["app1"].Gate
|
||||
}
|
||||
if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != gatePasses {
|
||||
t.Fatalf("passed before the core's settle time: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead)
|
||||
if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != 1 {
|
||||
t.Fatalf("passed inside the witness window: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead)
|
||||
}
|
||||
gateEvery, gateSettle = 0, 0
|
||||
gateEvery, gateWitnessWindow = 0, 0
|
||||
for i := 0; i < 6; i++ {
|
||||
advancePlans(ctx, tm.open)
|
||||
}
|
||||
@@ -469,8 +486,9 @@ func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) {
|
||||
t.Fatalf("the plan is %s (%s), sent %v", p.State, p.Note, tm.sent)
|
||||
}
|
||||
for _, m := range []string{broker.RuntimeModule, "app1"} {
|
||||
if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < gatePasses) {
|
||||
t.Fatalf("%s's gate: %+v; want passed on %d readings", m, g, gatePasses)
|
||||
if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < 1 ||
|
||||
!strings.Contains(g.Why, "the tool runner's rule")) {
|
||||
t.Fatalf("%s's gate: %+v; want passed on one reading by the tool runner's rule", m, g)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,353 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A branch is tried before its pull request (novox/hq issue 686).
|
||||
//
|
||||
// A repository's store tests skip without a store, and no agent may start a database on its machine. So a
|
||||
// builder pushed, a reviewer read, and only the pull request's merge check ran those tests for the first time:
|
||||
// each failure there cost a round of fifteen to thirty minutes (mesh-controller #283, #310, #312).
|
||||
//
|
||||
// `try <owner>/<repository> <ref>` asks the build seat for the repository's own check of a pushed branch or
|
||||
// commit, as the merge check runs it (internal/builder/check.go): the ref merged into the trunk's head, the
|
||||
// trunk's merge-check.sh, each part in its toolchain's container, against a throwaway store and bus. It is a
|
||||
// check and nothing more:
|
||||
//
|
||||
// - **no pull request, so no status**: its outcome is never said as `checked`, which is what the forge's
|
||||
// holder sets statuses from (isTry, in Built);
|
||||
// - **nothing registered, nothing built**: it names no module of the graph, so the merge gate does not run and
|
||||
// no build is made for a merge (novox/hq ADR 0350's scratch builds are a pull request's check's alone);
|
||||
// - **a check in the build queue**: after deliveries (link.KindOf, novox/hq issue 558), and bounded by the
|
||||
// check's own bound (checkTimeout);
|
||||
// - **the code a pull request's check would run anyway**: only a repository of the mesh, whose every pull
|
||||
// request the build seat already checks, in containers that hold no runtime socket.
|
||||
//
|
||||
// A check outlasts a seat verb's one answer, so the ask answers its id at once and `try --id <id>` reads what
|
||||
// came of it from the build's log, where the build seat says it whole (link.StepTried): holding its answer up
|
||||
// to tryWait for a verdict, like `plans wait` (novox/hq issue 229). Nothing is kept beside the log.
|
||||
|
||||
// tryWait is how long reading a try holds its answer for the verdict. A variable so a test need not wait.
|
||||
var tryWait = 20 * time.Second
|
||||
|
||||
// A ref as the forge is asked for it — a branch or a commit, never an option — and a `go test -run` pattern.
|
||||
var (
|
||||
tryRef = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]*$`)
|
||||
tryTests = regexp.MustCompile(`^[A-Za-z0-9_^$|().*+?/\[\]-]+$`)
|
||||
)
|
||||
|
||||
// tryAnswer is what `try` answers, asking or reading.
|
||||
type tryAnswer struct {
|
||||
ID string `json:"id"`
|
||||
// State is asked (just now), waiting (nothing said of it yet: in the build queue), running, or the verdict:
|
||||
// pass, warning (the repository declares no merge-check.sh), fail, or error (it could not run).
|
||||
State string `json:"state"`
|
||||
Said string `json:"said"`
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Base string `json:"base,omitempty"`
|
||||
Trunk string `json:"trunk,omitempty"`
|
||||
On string `json:"on,omitempty"`
|
||||
Took string `json:"took,omitempty"`
|
||||
Failed []string `json:"failed,omitempty"`
|
||||
// Last is the last thing the build seat said of one still running.
|
||||
Last string `json:"last,omitempty"`
|
||||
}
|
||||
|
||||
// isTry says a check's outcome is a try's: marked so, or — echoed by a build seat from before the mark — a
|
||||
// check of no pull request and no group, which is nobody's status either.
|
||||
func isTry(result link.BuildResult) bool {
|
||||
c := result.Checked
|
||||
return c != nil && (c.Try || (c.Number == 0 && c.Group == ""))
|
||||
}
|
||||
|
||||
// asATry makes a check's ask a try's: the repository's own script alone, for no pull request.
|
||||
func asATry(request link.BuildRequest, tests string, storeOnly bool) link.BuildRequest {
|
||||
c := *request.Check
|
||||
c.Number, c.Try, c.Tests, c.StoreOnly = 0, true, tests, storeOnly
|
||||
c.Paths, c.Modules, c.New, c.Dependents, c.Manifests, c.Builds = nil, nil, nil, nil, nil, nil
|
||||
c.Judge, c.Plan, c.Group, c.Members = "", nil, "", nil
|
||||
request.Check = &c
|
||||
return request
|
||||
}
|
||||
|
||||
// maxTries is how many tries may wait or run at once, mesh-wide: each holds a build machine for minutes.
|
||||
const maxTries = 4
|
||||
|
||||
// triesIn reads the build queue for a new try of one commit: the id of a try of that commit already waiting or
|
||||
// running — the answer to a second ask — and whether there is room for another.
|
||||
func triesIn(q link.Queue, owner, repo, commit string) (same string, room bool) {
|
||||
n := 0
|
||||
for _, a := range q.Asks {
|
||||
if !link.IsTry(a.Request) || (a.State != link.AskWaiting && a.State != link.AskInFlight) {
|
||||
continue
|
||||
}
|
||||
n++
|
||||
c := a.Request.Check
|
||||
if same == "" && strings.EqualFold(c.Owner, owner) && strings.EqualFold(c.Repo, repo) && a.Request.Ref == commit {
|
||||
same = a.ID
|
||||
}
|
||||
}
|
||||
return same, n < maxTries
|
||||
}
|
||||
|
||||
// forgeCommit asks the forge, through its module's tools on the bus, which commit a branch or commit of a
|
||||
// repository is, and the repository's default branch. A variable so a test needs no forge.
|
||||
var forgeCommit = func(ctx context.Context, owner, repo, ref string) (commit, trunk string, err error) {
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
defer js.Close()
|
||||
bus := link.OverNATS{Conn: js.Conn()}
|
||||
ask := func(path string, into any) error {
|
||||
raw, _ := json.Marshal(map[string]any{"path": path})
|
||||
answer, err := link.Ask(ctx, bus, "gitea", "gitea_api", raw, 30*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return errors.New(answer.Error)
|
||||
}
|
||||
return json.Unmarshal(answer.Result, into)
|
||||
}
|
||||
var commits struct {
|
||||
Result []struct {
|
||||
SHA string `json:"sha"`
|
||||
} `json:"result"`
|
||||
}
|
||||
if err := ask("/repos/"+owner+"/"+repo+"/commits?limit=1&stat=false&verification=false&files=false&sha="+
|
||||
url.QueryEscape(ref), &commits); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if len(commits.Result) == 0 || commits.Result[0].SHA == "" {
|
||||
return "", "", errors.New("the forge named no commit for it")
|
||||
}
|
||||
var found struct {
|
||||
Result struct {
|
||||
DefaultBranch string `json:"default_branch"`
|
||||
} `json:"result"`
|
||||
}
|
||||
if err := ask("/repos/"+owner+"/"+repo, &found); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return commits.Result[0].SHA, found.Result.DefaultBranch, nil
|
||||
}
|
||||
|
||||
// tryCommand is `try <owner>/<repository> <ref> [--base <branch>] [--tests <pattern>] [--store-only]`, and
|
||||
// `try --id <id>` to read what came of one.
|
||||
func tryCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("try", flag.ContinueOnError)
|
||||
base := set.String("base", "", "the branch it is merged into (the repository's default branch when none)")
|
||||
tests := set.String("tests", "", "only the tests this `go test -run` pattern names")
|
||||
storeOnly := set.Bool("store-only", false, "only the repository's store tests")
|
||||
id := set.String("id", "", "a try's id: what came of it")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
usage := errors.New("try <owner>/<repository> <branch or commit> [--base <branch>] [--tests <pattern>] " +
|
||||
"[--store-only], or try --id <id>")
|
||||
if *id != "" {
|
||||
if len(positionals) != 0 {
|
||||
return usage
|
||||
}
|
||||
// Before it is a token of a bus subject: an id, never a pattern.
|
||||
if !link.SafeBuildID(*id) {
|
||||
return fmt.Errorf("%q is not a try's id", *id)
|
||||
}
|
||||
return tryRead(ctx, *id)
|
||||
}
|
||||
if len(positionals) != 2 {
|
||||
return usage
|
||||
}
|
||||
owner, repo, isPath := strings.Cut(positionals[0], "/")
|
||||
ref := positionals[1]
|
||||
switch {
|
||||
case !isPath || !tryRef.MatchString(owner) || !tryRef.MatchString(repo) || strings.Contains(repo, "/"):
|
||||
return fmt.Errorf("%q is not a repository on the forge, owner/name; nothing was asked", positionals[0])
|
||||
case !tryRef.MatchString(ref):
|
||||
return fmt.Errorf("%q is not a branch or a commit; nothing was asked", ref)
|
||||
case *base != "" && !tryRef.MatchString(*base):
|
||||
return fmt.Errorf("%q is not a branch; nothing was asked", *base)
|
||||
case *tests != "" && !tryTests.MatchString(*tests):
|
||||
return fmt.Errorf("%q is not a `go test -run` pattern of test names; nothing was asked", *tests)
|
||||
}
|
||||
// **An unknown ref is refused here, loudly** — not sent to a machine to fail at its clone minutes later.
|
||||
commit, trunk, err := forgeCommit(ctx, owner, repo, ref)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the forge holds no branch or commit %s in %s/%s — is it pushed? — so nothing was asked: %w",
|
||||
ref, owner, repo, err)
|
||||
}
|
||||
if *base != "" {
|
||||
trunk = *base
|
||||
}
|
||||
if trunk == "" {
|
||||
return fmt.Errorf("the forge named no default branch for %s/%s, and no --base says what %s is merged into; "+
|
||||
"nothing was asked", owner, repo, ref)
|
||||
}
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
entries, read, edges, err := theGraph(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p := link.PullUpdated{Owner: owner, Repo: repo, Base: trunk, Commit: commit}
|
||||
// Whose repository it is, as a pull request's check reckons it: only the mesh's is tried, the code the
|
||||
// build seat runs for every pull request already.
|
||||
mesh := pullScope(p, entries, read, edges)
|
||||
if !mesh.Mesh {
|
||||
return fmt.Errorf("%s/%s is no repository of the mesh — no module is built from it and its owner is not "+
|
||||
"the core's — and only the mesh's are tried on the build seat; nothing was asked", owner, repo)
|
||||
}
|
||||
request, err := checkRequestFor(ctx, open, p, checkScope{Mesh: true, From: mesh.From}, entries)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
request = asATry(request, *tests, *storeOnly)
|
||||
seat := buildSeatHeld(ctx)
|
||||
// **Bounded in number** (novox/hq issue 686): one try of a commit at a time, and maxTries in all.
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
queue, err := link.ReadQueue(ctx, js, seat)
|
||||
js.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
same, room := triesIn(queue, owner, repo, commit)
|
||||
if same != "" {
|
||||
return printTry(tryAnswer{ID: same, State: "asked", Repository: owner + "/" + repo, Ref: ref, Commit: commit,
|
||||
On: seat, Said: fmt.Sprintf("a try of %s/%s at %.12s already waits or runs, as %s: nothing new was asked. "+
|
||||
"Read it with try id=%s", owner, repo, commit, same, same)})
|
||||
}
|
||||
if !room {
|
||||
return fmt.Errorf("%d tries already wait or run on the build seat, the most at once (`queue` lists them); "+
|
||||
"nothing was asked — read one of yours, or ask again when one has ended", maxTries)
|
||||
}
|
||||
ask, err := askOverOn(seat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ask.Close()
|
||||
if err := ask.Ask(ctx, request); err != nil {
|
||||
return err
|
||||
}
|
||||
return printTry(tryAnswer{ID: request.ID, State: "asked", Repository: owner + "/" + repo, Ref: ref, Commit: commit,
|
||||
Base: trunk, On: seat, Said: fmt.Sprintf("asked %s to try %s/%s at %.12s (%s) merged into %s: its own "+
|
||||
"merge-check.sh, as a check in the build queue, bounded at %s; no status is set and nothing is registered. "+
|
||||
"Read it with try id=%s — a full run of this kind takes minutes", seat, owner, repo, commit, ref, trunk,
|
||||
checkTimeout, request.ID)})
|
||||
}
|
||||
|
||||
// tryRead answers what came of a try, holding the answer up to tryWait for its verdict.
|
||||
func tryRead(ctx context.Context, id string) error {
|
||||
until := time.Now().Add(tryWait)
|
||||
for {
|
||||
lines, err := buildLines(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a := tryAnswerOf(id, lines)
|
||||
if (a.State != "waiting" && a.State != "running") || !time.Now().Add(2*time.Second).Before(until) {
|
||||
return printTry(a)
|
||||
}
|
||||
if err := sleepOrDone(ctx, 2*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tryAnswerOf reads a try's answer from what the build seat said of it: its outcome under StepTried; `failed`
|
||||
// for one that could not run — an error, never a pass; else how far it is.
|
||||
func tryAnswerOf(id string, lines []link.BuildLine) tryAnswer {
|
||||
a := tryAnswer{ID: id, State: "waiting", Said: "nothing is said of " + id + " yet: it waits in the build queue " +
|
||||
"(`queue` says where), or no try has that id; ask again"}
|
||||
for _, l := range lines {
|
||||
switch l.Step {
|
||||
case link.StepTried:
|
||||
var t link.Tried
|
||||
if err := json.Unmarshal([]byte(l.Message), &t); err != nil || t.Verdict == "" {
|
||||
return tryAnswer{ID: id, State: "error", Said: "the build seat's word on it cannot be read: " + l.Message}
|
||||
}
|
||||
return tryAnswer{ID: id, State: t.Verdict, Said: t.Summary, Took: t.Took, Commit: t.Head, Trunk: t.Trunk,
|
||||
Failed: t.Failed}
|
||||
case "failed":
|
||||
return tryAnswer{ID: id, State: "error", Said: "it could not run: " + l.Message}
|
||||
case "checked":
|
||||
// A check that is no try, or a build seat from before the verb: its verdict is not a try's.
|
||||
return tryAnswer{ID: id, State: "error", Said: "the build seat checked it and said no try's outcome — " +
|
||||
"it predates the verb (novox/hq issue 686), or " + id + " is no try: " + l.Message + "; `builds` with " +
|
||||
"the id says every line"}
|
||||
}
|
||||
a.State, a.Last = "running", l.Message
|
||||
a.Said = "still running; ask again. `builds` with the id says every line so far"
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
func printTry(a tryAnswer) error {
|
||||
fmt.Fprintln(os.Stderr, a.Said)
|
||||
for _, l := range a.Failed {
|
||||
fmt.Fprintln(os.Stderr, " "+l)
|
||||
}
|
||||
out, err := json.MarshalIndent(a, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(out))
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildLines is everything the build seat said of one build, as the bus keeps it, in order.
|
||||
func buildLines(ctx context.Context, id string) ([]link.BuildLine, error) {
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot reach the bus to read a build's log: %w", err)
|
||||
}
|
||||
defer js.Close()
|
||||
subject := link.BuildLogOf("*", id)
|
||||
sub, err := js.Context().PullSubscribe(subject, "",
|
||||
nats.BindStream(broker.EventsStream), nats.DeliverAll(), nats.AckNone())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot read %s from the bus: %w", subject, err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
var lines []link.BuildLine
|
||||
for {
|
||||
batch, err := sub.Fetch(200, nats.MaxWait(2*time.Second))
|
||||
if err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.DeadlineExceeded) {
|
||||
return nil, fmt.Errorf("reading a build's log: %w", err)
|
||||
}
|
||||
for _, msg := range batch {
|
||||
var line link.BuildLine
|
||||
if json.Unmarshal(msg.Data, &line) == nil {
|
||||
lines = append(lines, line)
|
||||
}
|
||||
}
|
||||
if len(batch) < 200 {
|
||||
return lines, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A branch is tried before its pull request (novox/hq issue 686): the repository's own check asked for a pushed
|
||||
// branch or commit, with no pull request.
|
||||
|
||||
// What a try asks is a check — after deliveries in the build queue, bounded as one — of the repository's own
|
||||
// script alone: no module of the graph, so no gate; nothing built for a merge, so no reusable image.
|
||||
func TestATryAsksACheckOfTheRepositorysOwnScriptAlone(t *testing.T) {
|
||||
request := link.BuildRequest{ID: "build-1", Ref: "c0ffee", Check: &link.CheckRequest{Owner: "novox", Repo: "a",
|
||||
Number: 7, Base: "main", Modules: []string{"a"}, New: []string{"b"}, Manifests: []string{"module.json"},
|
||||
Dependents: []string{"c"}, Builds: []link.CheckBuild{{Module: "a"}}, Judge: link.JudgeSelf,
|
||||
Plan: &link.ChangePlan{}}}
|
||||
tried := asATry(request, "^TestStore", true)
|
||||
c := tried.Check
|
||||
if !c.Try || c.Number != 0 || c.Tests != "^TestStore" || !c.StoreOnly {
|
||||
t.Fatalf("not marked a try, or it kept a pull request's number: %+v", c)
|
||||
}
|
||||
if len(c.Modules)+len(c.New)+len(c.Manifests)+len(c.Dependents)+len(c.Builds) != 0 || c.Judge != "" || c.Plan != nil {
|
||||
t.Fatalf("a try runs the gate or builds for a merge: %+v", c)
|
||||
}
|
||||
if c.Base != "main" || c.Owner != "novox" || tried.Ref != "c0ffee" {
|
||||
t.Fatalf("a try lost what it merges into what: %+v", tried)
|
||||
}
|
||||
if link.KindOf(tried) != link.KindCheck {
|
||||
t.Fatal("a try does not count as a check in the build queue")
|
||||
}
|
||||
}
|
||||
|
||||
// A try's outcome is heard and left there: no `checked` is said, so the forge's holder sets no status, and
|
||||
// nothing is recorded or registered (the handler holds no store here, and would fail reaching for one).
|
||||
func TestATrysOutcomeSetsNoStatusAndRegistersNothing(t *testing.T) {
|
||||
said := &saidOnTheBus{}
|
||||
checkEvents = said
|
||||
defer func() { checkEvents = nil }()
|
||||
for name, result := range map[string]link.BuildResult{
|
||||
"a try": {ID: "build-1", Ref: "c0ffee", Checked: &link.CheckRequest{Owner: "novox", Repo: "a", Try: true},
|
||||
Check: &link.CheckOutcome{Verdict: "pass", RepoCheck: &link.CheckLayer{Verdict: "fail", Summary: "failed"}}},
|
||||
"a try that could not run": {ID: "build-2", Checked: &link.CheckRequest{Owner: "novox", Repo: "a", Try: true},
|
||||
Failed: "cannot clone"},
|
||||
// A build seat from before the mark echoes the ask without it: a check of no pull request and no
|
||||
// group is still nobody's status.
|
||||
"a try echoed by an older build seat": {ID: "build-3", Checked: &link.CheckRequest{Owner: "novox", Repo: "a"},
|
||||
Check: &link.CheckOutcome{Verdict: "pass"}},
|
||||
} {
|
||||
if err := (builds{}).Built(context.Background(), result); err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
}
|
||||
if len(said.events) != 0 {
|
||||
t.Fatalf("a try's outcome was said for the forge: %v", said.events)
|
||||
}
|
||||
// A pull request's check is still said.
|
||||
_ = (builds{}).Built(context.Background(), link.BuildResult{ID: "build-4", Checked: &link.CheckRequest{
|
||||
Owner: "novox", Repo: "a", Number: 7}, Check: &link.CheckOutcome{Verdict: "pass"}})
|
||||
if len(said.events) != 1 {
|
||||
t.Fatalf("a pull request's check was said %d time(s)", len(said.events))
|
||||
}
|
||||
}
|
||||
|
||||
// saidOnTheBus keeps the seat events said, and nothing else.
|
||||
type saidOnTheBus struct {
|
||||
link.Bus
|
||||
events []string
|
||||
}
|
||||
|
||||
func (s *saidOnTheBus) PublishSeatEvent(_ context.Context, seat, event string, _ []byte) error {
|
||||
s.events = append(s.events, seat+" "+event)
|
||||
return nil
|
||||
}
|
||||
|
||||
// A ref the forge does not hold is refused before anything is asked, naming it.
|
||||
func TestATryOfAnUnknownRefIsRefusedLoudly(t *testing.T) {
|
||||
was := forgeCommit
|
||||
defer func() { forgeCommit = was }()
|
||||
forgeCommit = func(_ context.Context, owner, repo, ref string) (string, string, error) {
|
||||
return "", "", errors.New("404 object does not exist [id: " + ref + "]")
|
||||
}
|
||||
err := tryCommand(context.Background(), []string{"novox/mesh-controller", "fix/no-such-branch"})
|
||||
if err == nil {
|
||||
t.Fatal("a try of a ref the forge does not hold was asked")
|
||||
}
|
||||
for _, want := range []string{"fix/no-such-branch", "novox/mesh-controller", "nothing was asked", "404"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
for _, args := range [][]string{{"mesh-controller", "main"}, {"novox/mesh-controller"}, {"novox/a", "--upload-pack=x"},
|
||||
{"novox/a", "main", "--tests", "x; rm -rf /"}, {"novox/a", "main", "--id", "build-1"}} {
|
||||
if err := tryCommand(context.Background(), args); err == nil {
|
||||
t.Errorf("try %v was taken", args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The answer of a try carries the lines that name what failed (novox/hq issue 460's shape) and how long it took.
|
||||
func TestATrysAnswerCarriesTheFailingLines(t *testing.T) {
|
||||
tried, _ := json.Marshal(link.Tried{Verdict: "fail", Summary: "its merge-check.sh failed: TestX", Took: "5m3s",
|
||||
Head: "c0ffee", Trunk: "ca3f43c", Failed: []string{"--- FAIL: TestX (0.01s)", " x_test.go:12: got 1, want 2",
|
||||
"FAIL\tgithub.com/novox/a/cmd\t3.1s"}})
|
||||
lines := []link.BuildLine{{Seq: 1, Step: "check", Message: "novox/a#0 at c0ffee"},
|
||||
{Seq: 2, Step: "output", Message: "ok github.com/novox/a/internal 1.2s"},
|
||||
{Seq: 3, Step: link.StepTried, Message: string(tried)}, {Seq: 4, Step: "checked", Message: "pass: x"}}
|
||||
a := tryAnswerOf("build-1", lines)
|
||||
if a.State != "fail" || a.Took != "5m3s" || len(a.Failed) != 3 || a.Failed[0] != "--- FAIL: TestX (0.01s)" ||
|
||||
a.Trunk != "ca3f43c" || a.ID != "build-1" {
|
||||
t.Fatalf("the answer lost the verdict, the time or the failing lines: %+v", a)
|
||||
}
|
||||
// Not answered yet: said as running, with how far it is — never as a pass.
|
||||
if a := tryAnswerOf("build-1", lines[:2]); a.State != "running" || a.Last == "" {
|
||||
t.Fatalf("a try still running: %+v", a)
|
||||
}
|
||||
if a := tryAnswerOf("build-1", nil); a.State != "waiting" {
|
||||
t.Fatalf("a try nothing was said of: %+v", a)
|
||||
}
|
||||
// One that could not run is an error with the build seat's word, never a pass.
|
||||
if a := tryAnswerOf("build-1", append(lines[:1:1], link.BuildLine{Seq: 2, Step: "failed",
|
||||
Message: "cannot clone"})); a.State != "error" || !strings.Contains(a.Said, "cannot clone") {
|
||||
t.Fatalf("a try that could not run: %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
// One try of a commit at a time, and four in all (novox/hq issue 686): a second ask is answered the first's id,
|
||||
// a fifth is refused; a try that ended, and a pull request's check, count for nothing.
|
||||
func TestTriesAreBoundedInNumber(t *testing.T) {
|
||||
ask := func(id, repo, ref, state string, try bool) link.QueuedAsk {
|
||||
return link.QueuedAsk{ID: id, State: state, Request: link.BuildRequest{ID: id, Ref: ref,
|
||||
Check: &link.CheckRequest{Owner: "novox", Repo: repo, Try: try}}}
|
||||
}
|
||||
q := link.Queue{Asks: []link.QueuedAsk{
|
||||
ask("t1", "a", "c1", link.AskInFlight, true), ask("t2", "a", "c2", link.AskWaiting, true),
|
||||
ask("t3", "b", "c1", link.AskDead, true), ask("p1", "a", "c9", link.AskWaiting, false)}}
|
||||
if same, room := triesIn(q, "novox", "a", "c2"); same != "t2" || !room {
|
||||
t.Fatalf("a second try of a commit: %q %v", same, room)
|
||||
}
|
||||
if same, room := triesIn(q, "novox", "b", "c1"); same != "" || !room {
|
||||
t.Fatalf("a dead try, or another repository's, was taken for this one: %q %v", same, room)
|
||||
}
|
||||
if same, _ := triesIn(q, "novox", "a", "c9"); same != "" {
|
||||
t.Fatalf("a pull request's check was taken for a try: %q", same)
|
||||
}
|
||||
q.Asks = append(q.Asks, ask("t4", "c", "c3", link.AskWaiting, true), ask("t5", "d", "c4", link.AskWaiting, true))
|
||||
if _, room := triesIn(q, "novox", "e", "c5"); room {
|
||||
t.Fatal("a fifth try was let in")
|
||||
}
|
||||
// An id is checked before it is a token of a bus subject.
|
||||
for _, id := range []string{"x.>", "*", "a b"} {
|
||||
if err := tryCommand(context.Background(), []string{"--id", id}); err == nil ||
|
||||
!strings.Contains(err.Error(), "not a try's id") {
|
||||
t.Errorf("try --id %q: %v", id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The verb runs its command, and is declared with the shell habit it replaces (novox/hq ADR 0245).
|
||||
func TestTheTryVerbRunsItsCommand(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
args map[string]any
|
||||
want []string
|
||||
}{
|
||||
{map[string]any{"repository": "novox/a", "ref": "fix/x"}, []string{"try", "novox/a", "fix/x"}},
|
||||
{map[string]any{"repository": "novox/a", "ref": "fix/x", "base": "main", "tests": "^TestX$", "store-only": "true"},
|
||||
[]string{"try", "novox/a", "fix/x", "--base", "main", "--tests", "^TestX$", "--store-only"}},
|
||||
{map[string]any{"id": "build-1"}, []string{"try", "--id", "build-1"}},
|
||||
} {
|
||||
got, err := argvFor("try", c.args)
|
||||
if err != nil || !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("try %v: %v %v, want %v", c.args, got, err, c.want)
|
||||
}
|
||||
}
|
||||
if _, err := argvFor("try", map[string]any{"repository": "novox/a"}); err == nil {
|
||||
t.Error("a try of no ref was taken")
|
||||
}
|
||||
for _, v := range catalogue.ControllerVerbs {
|
||||
if v.Name == "try" {
|
||||
if len(v.Replaces) == 0 {
|
||||
t.Error("try names no shell habit it replaces")
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Error("try is not a verb of the controller seat")
|
||||
}
|
||||
@@ -79,7 +79,7 @@ func (f following) Upgraded(ctx context.Context, u link.Upgraded) error {
|
||||
// `rebuild`, a `replay` registered — was sent here one machine after another without any judging; it
|
||||
// now waits for a gate like any other build, and the release plan walks it through the machines, one
|
||||
// at a time, each judged.
|
||||
fmt.Printf("%s moved to %s outside a plan; a release plan sends it to %s, one machine at a time, each judged "+
|
||||
fmt.Printf("%s moved to %s outside a plan; a release plan sends it to %s, one machine first and then the rest at once, each judged "+
|
||||
"(`upgrade backlog` lists what waits)\n", u.Module, shortCommit(u.Commit), readableList(on))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A module may declare its unhealthy urgent (novox/hq ADR 0355): its module-unhealthy condition is then raised
|
||||
// urgent; without the field it stays a warning; a wait for a new login stays a warning whatever the module says.
|
||||
|
||||
// judgedTwice judges one machine's statement twice, over a catalogue in which mail declares its unhealthy
|
||||
// urgent and notes says nothing, and returns what is open.
|
||||
func judgedTwice(t *testing.T, unhealthy map[string][]inventory.ResourceHealth) []conditions.Condition {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
k, _ := withConditionsInMemory(t)
|
||||
was := readHoldingFor
|
||||
t.Cleanup(func() { readHoldingFor = was })
|
||||
readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
||||
return &holding{ctx: context.Background(), open: open, providers: map[string]providerLookup{},
|
||||
shelf: map[string]catalogue.Manifest{
|
||||
"mail": {Module: "mail", Version: "1", Unhealthy: &catalogue.UnhealthyPolicy{
|
||||
Severity: catalogue.UnhealthyUrgent, Why: "mail is refused or stuck for everyone with a mailbox"}},
|
||||
"notes": {Module: "notes", Version: "1"},
|
||||
}}, nil
|
||||
}
|
||||
streaks := map[string]int{}
|
||||
for look := 1; look <= 2; look++ {
|
||||
for m := range unhealthy {
|
||||
streaks[m] = look
|
||||
}
|
||||
if err := judgeModuleHealth(ctx, nil, k, "anchor", unhealthy, streaks, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
func containerDown(module string) inventory.ResourceHealth {
|
||||
return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
|
||||
State: link.StateUnhealthy, Reason: "down"}
|
||||
}
|
||||
|
||||
func TestAModuleThatDeclaresItsUnhealthyUrgentIsRaisedUrgent(t *testing.T) {
|
||||
open := judgedTwice(t, map[string][]inventory.ResourceHealth{"mail": {containerDown("mail")}})
|
||||
c := conditionOf(open, "module.mail.anchor.unhealthy")
|
||||
if c == nil {
|
||||
t.Fatalf("mail's unhealthy condition was not raised: %+v", open)
|
||||
}
|
||||
if c.Severity != conditions.Urgent {
|
||||
t.Errorf("mail declares its unhealthy urgent, and it was raised %s", c.Severity)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModuleThatSaysNothingOfItsUnhealthyStaysAWarning(t *testing.T) {
|
||||
open := judgedTwice(t, map[string][]inventory.ResourceHealth{"notes": {containerDown("notes")}})
|
||||
c := conditionOf(open, "module.notes.anchor.unhealthy")
|
||||
if c == nil {
|
||||
t.Fatalf("notes' unhealthy condition was not raised: %+v", open)
|
||||
}
|
||||
if c.Severity != conditions.Warning {
|
||||
t.Errorf("notes says nothing of its unhealthy, and it was raised %s", c.Severity)
|
||||
}
|
||||
}
|
||||
|
||||
// Only the module-unhealthy condition changes (ADR 0355 §4): a wait for a new login (ADR 0254) of a module that
|
||||
// declares its unhealthy urgent stays a warning.
|
||||
func TestAWaitForANewLoginStaysAWarningWhenTheModuleDeclaresItsUnhealthyUrgent(t *testing.T) {
|
||||
open := judgedTwice(t, map[string][]inventory.ResourceHealth{"mail": {relogin("mail", "operator"),
|
||||
userUnit("mail", "operator")}})
|
||||
c := conditionOf(open, "module.mail.anchor.relogin-needed")
|
||||
if c == nil {
|
||||
t.Fatalf("mail's relogin-needed condition was not raised: %+v", open)
|
||||
}
|
||||
if c.Severity != conditions.Warning {
|
||||
t.Errorf("a wait for a new login was raised %s; want a warning", c.Severity)
|
||||
}
|
||||
if conditionOf(open, "module.mail.anchor.unhealthy") != nil {
|
||||
t.Errorf("a wait for a new login raised mail's unhealthy condition beside it: %+v", open)
|
||||
}
|
||||
}
|
||||
@@ -177,6 +177,12 @@ func walkReached(p inventory.Plan, running func(string) []string) map[string][]s
|
||||
reached = appendNew(reached, r.Gate.Machines...)
|
||||
carried(r.Gate)
|
||||
}
|
||||
for _, g := range r.Gates {
|
||||
if g != nil && !g.Pending {
|
||||
reached = appendNew(reached, g.Machines...)
|
||||
carried(g)
|
||||
}
|
||||
}
|
||||
for _, tier := range p.Tiers {
|
||||
for _, m := range tier {
|
||||
for _, n := range running(m) {
|
||||
|
||||
@@ -92,6 +92,8 @@ type signalFacts struct {
|
||||
// consumers: the condition counts the messages given up on, not the looks (novox/hq issue 440).
|
||||
deadLettersNewest map[string]time.Time
|
||||
advisoriesErr error
|
||||
// busFaultsErr is why the bus module's events could not be read this tick (novox/hq ADR 0357).
|
||||
busFaultsErr error
|
||||
|
||||
selfCheck selfCheckFacts
|
||||
|
||||
@@ -358,6 +360,15 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
|
||||
}
|
||||
f.advisories = link.Advisories.Since(now.Add(-advisoryQuiet))
|
||||
f.lostConsumers, f.advisoriesErr = w.lostConsumers(ctx, f.advisories)
|
||||
// And what the bus did to every other user's connection, as the bus module said it (novox/hq ADR 0357):
|
||||
// read back for the same hour, and one advisory with the controller's own where both are its.
|
||||
// A read that fails is its own condition, never S9 blind: the controller's own signals are still said.
|
||||
if w.server != nil && w.js != nil {
|
||||
var faults []link.Advisory
|
||||
if faults, f.busFaultsErr = w.server.BusFaults(ctx, now.Add(-advisoryQuiet)); f.busFaultsErr == nil {
|
||||
f.advisories = link.MergeAdvisories(f.advisories, faults)
|
||||
}
|
||||
}
|
||||
if f.advisoriesErr == nil && w.js != nil {
|
||||
f.deadLetters, f.advisoriesErr = link.HeldDeadLetters(w.js.Context())
|
||||
}
|
||||
|
||||
@@ -38,8 +38,9 @@ func TestTheBuildSeatsHolderFollowsTheControllerThatDefinesItsWorker(t *testing.
|
||||
if pos["build-agent"] <= pos["mesh-controller"] {
|
||||
t.Fatalf("the build machine after the controller that defines its worker: %v", tiers)
|
||||
}
|
||||
if pos["route-proxy"] <= pos["build-agent"] {
|
||||
t.Fatalf("what the build machine builds comes after it: %v", tiers)
|
||||
// Built-by orders nothing (novox/hq ADR 0360 §3): whichever builder runs builds the proxy, beside the controller.
|
||||
if pos["route-proxy"] != 0 {
|
||||
t.Fatalf("what the builder builds waits for a new one: %v", tiers)
|
||||
}
|
||||
if hasCycle(tiers, edges) {
|
||||
t.Fatalf("no cycle here: %v", tiers)
|
||||
|
||||
@@ -3,7 +3,7 @@ module github.com/novox/mesh-controller
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
@@ -36,4 +36,4 @@ require (
|
||||
// committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and
|
||||
// fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without
|
||||
// `go mod vendor` fails loudly, at once, everywhere.
|
||||
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f
|
||||
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261013220009-b9a293f8c8bc
|
||||
|
||||
@@ -1,7 +1,13 @@
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f h1:8N5OW2mdTNIck2pe4EciTYX5NsrPsHrTLENGNIWYNTU=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261013220009-b9a293f8c8bc h1:h5v2AbdjM/clMY1YxS9p7VuyYtIjkcw85w9Oz1cOYQc=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261013220009-b9a293f8c8bc/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013141520-9588871d6629 h1:7TYFV1Q6Mcz+lsLP/sds87tysET4rKCn3rRIQV8gNmA=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013141520-9588871d6629/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013145622-f4745447ba6d h1:HEJyqorKktrkJPeeJdwEo0kdrJjH3fm/YVS9MGftCqY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013145622-f4745447ba6d/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44 h1:r5xIo5qWOXQdCSMxvqJy8kV4U+P/QWXFK6971Ha6wU8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Listing and letting go of tagged documents: the check builds the scratch namespace keeps (novox/hq ADR 0350
|
||||
// §5). A check build is found by its tag, and the build seat's holder lets each go when its pull request closes
|
||||
// or it is seven days old, so the store keeps a week of check builds and no more.
|
||||
|
||||
// Tags is every tag repository holds, in the store's order. None for a repository the store does not hold.
|
||||
func (s Store) Tags(ctx context.Context, repository string) ([]string, error) {
|
||||
if s.Address == "" {
|
||||
return nil, fmt.Errorf("this mesh has no artifact store on its network to list %s in", repository)
|
||||
}
|
||||
var tags []string
|
||||
next := "http://" + s.Address + "/v2/" + repository + "/tags/list?n=1000"
|
||||
for next != "" {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, next, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
|
||||
}
|
||||
switch response.StatusCode {
|
||||
case http.StatusOK:
|
||||
case http.StatusNotFound:
|
||||
response.Body.Close()
|
||||
return tags, nil
|
||||
default:
|
||||
response.Body.Close()
|
||||
return nil, fmt.Errorf("the artifact store answered %s for the tags of %s", response.Status, repository)
|
||||
}
|
||||
var page struct {
|
||||
Tags []string `json:"tags"`
|
||||
}
|
||||
err = json.NewDecoder(io.LimitReader(response.Body, 8<<20)).Decode(&page)
|
||||
link := response.Header.Get("Link")
|
||||
response.Body.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the tags of %s could not be read: %w", repository, err)
|
||||
}
|
||||
tags = append(tags, page.Tags...)
|
||||
next = nextPage(link, s.Address)
|
||||
}
|
||||
return tags, nil
|
||||
}
|
||||
|
||||
// nextPage is the registry's `Link: </v2/…>; rel="next"`, made whole; empty when there is no next page.
|
||||
func nextPage(link, address string) string {
|
||||
if !strings.Contains(link, `rel="next"`) {
|
||||
return ""
|
||||
}
|
||||
start, end := strings.Index(link, "<"), strings.Index(link, ">")
|
||||
if start < 0 || end <= start+1 {
|
||||
return ""
|
||||
}
|
||||
target := link[start+1 : end]
|
||||
if strings.HasPrefix(target, "/") {
|
||||
return "http://" + address + target
|
||||
}
|
||||
return target
|
||||
}
|
||||
|
||||
// DeleteTagged lets go of the document tag names in repository: its manifest, by digest, so the next
|
||||
// collection takes its layer. Gone when the tag names nothing.
|
||||
func (s Store) DeleteTagged(ctx context.Context, repository, tag string) error {
|
||||
if s.Address == "" {
|
||||
return fmt.Errorf("this mesh has no artifact store on its network to let go of %s:%s in", repository, tag)
|
||||
}
|
||||
digest, err := s.manifestDigest(ctx, repository, tag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if digest == "" {
|
||||
return Gone
|
||||
}
|
||||
err = s.remove(ctx, s.url(repository, "manifests", digest), repository+":"+tag)
|
||||
if errors.Is(err, Gone) {
|
||||
return Gone
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **The bus module's watch speaks through the runtime that carries it** (novox/hq ADR 0357): the node's
|
||||
// runtime is granted the events the bus module declares, under the module's name, while the bus's own
|
||||
// account keeps the snapshot API and nothing else (ADR 0235).
|
||||
func TestTheRuntimeCarryingTheBusModuleMayPublishItsEventsAndTheBussAccountMayNot(t *testing.T) {
|
||||
events := []string{"bus.slow-consumer", "bus.connection-closed"}
|
||||
runtime := Principal{Kind: KindNodeTools, Node: "anchor", Module: RuntimeModule,
|
||||
Carries: []Declared{{Module: "nats", Emits: events}}}
|
||||
p, err := PermissionsFor(runtime)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, e := range events {
|
||||
if !slices.Contains(p.Publish, "mesh.mod.nats.event."+e) {
|
||||
t.Errorf("the runtime carrying the bus module may not publish %s: %v", e, p.Publish)
|
||||
}
|
||||
}
|
||||
account := Principal{Kind: KindModule, Node: "anchor", Module: "nats", Emits: events, SnapshotsTheBus: true}
|
||||
a, err := PermissionsFor(account)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range a.Publish {
|
||||
if s == "mesh.mod.nats.event.bus.slow-consumer" || s == "mesh.mod.nats.event.bus.connection-closed" {
|
||||
t.Errorf("the bus's own account is granted %s", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
+265
-27
@@ -91,6 +91,16 @@ type Result struct {
|
||||
// (novox/hq issue 280).
|
||||
Source string
|
||||
|
||||
// ReuseKey is the build's reuse key (novox/hq ADR 0350 §1): its source fingerprint's parts with the builder's
|
||||
// version, and ReuseParts those parts; empty, with NoKey saying why, where the source does not pin the build.
|
||||
ReuseKey string
|
||||
ReuseParts map[string]string
|
||||
NoKey string
|
||||
// Reused is the check build a walk took instead of building (ADR 0350 §2), and Why, when it built, why it did
|
||||
// not take one (§3). Both empty for a build not asked to reuse.
|
||||
Reused *Reused
|
||||
Why string
|
||||
|
||||
// Sources are what this build was made from, as files (novox/hq ADR 0267 rule 1): per repository, the
|
||||
// entries a changed file is tested against (SourceHolds). The module's own repository has an empty
|
||||
// Repository. Said only for a build of a commit on the trunk, and only for a repository whose every
|
||||
@@ -98,11 +108,6 @@ type Result struct {
|
||||
// recipe and the package it compiles; for any other, nothing is said and the whole of what the build
|
||||
// sees stays its source, as before.
|
||||
Sources []BuildSource
|
||||
|
||||
// Predefined is the repository's list of predefined keys (catalogue.PredefinedFile at its root) as
|
||||
// read at the commit built, when it has one (novox/hq issue 630): what the controller keeps as the
|
||||
// list mesh and node config hold values for. Empty for a repository without one.
|
||||
Predefined json.RawMessage
|
||||
}
|
||||
|
||||
// BuildSource is the build source a build read in one repository (novox/hq ADR 0267).
|
||||
@@ -141,14 +146,48 @@ func Build(ctx context.Context, run Runner, publish Publisher,
|
||||
// one record that it is there.
|
||||
var mirrored []string
|
||||
result, err := build(ctx, run, publish, repository, path, ref, workspace, held, npmrc, forge, log,
|
||||
&mirrored, seats...)
|
||||
&mirrored, nil, seats...)
|
||||
result.Mirrored = mirrored
|
||||
return result, err
|
||||
}
|
||||
|
||||
// Reuse is what a walk's build may take instead of building (novox/hq ADR 0350 §2): the scratch namespace's check
|
||||
// builds, and the pull requests the walk answers, whose checks name a differing key.
|
||||
type Reuse struct {
|
||||
Scratch Scratch
|
||||
Pulls []PullRef
|
||||
// Named are the builds the verdicts of the walk's own pull requests name (novox/hq ADR 0350 §2): a scratch build
|
||||
// is taken only when one of them names its key and every digest. The record in the store alone is never enough.
|
||||
Named []NamedBuild
|
||||
// keyFirst is a check's build (§1): its key is taken before anything is built, and a module with none is not
|
||||
// built at all — nothing could ever take it.
|
||||
keyFirst bool
|
||||
// pull and head are the check's own pull request and the head it judged (novox/hq issue 703): it keeps only a
|
||||
// build an earlier check of the same pull request at the same head made.
|
||||
pull PullRef
|
||||
head string
|
||||
}
|
||||
|
||||
// errNoKey is a check's build of a module whose source does not pin it: not built (ADR 0350 §1).
|
||||
var errNoKey = errors.New("it has no reuse key")
|
||||
|
||||
// BuildReusing is Build for the walk of a trunk commit (novox/hq ADR 0350 §2): before building, it takes the reuse
|
||||
// key of the commit's tree, and where a check build holds the same key it copies that build into the module's
|
||||
// namespace by digest and answers with it, building nothing. Otherwise it builds, and Result.Why says why (§3).
|
||||
// Only a commit on its trunk takes a check build: anything else builds, as Build does.
|
||||
func BuildReusing(ctx context.Context, run Runner, publish Publisher, reuse Reuse,
|
||||
repository, path, ref, workspace string, held map[string]string, npmrc Npmrc,
|
||||
forge GitCredential, log Log, seats ...map[string]string) (Result, error) {
|
||||
var mirrored []string
|
||||
result, err := build(ctx, run, publish, repository, path, ref, workspace, held, npmrc, forge, log,
|
||||
&mirrored, &reuse, seats...)
|
||||
result.Mirrored = mirrored
|
||||
return result, err
|
||||
}
|
||||
|
||||
func build(ctx context.Context, run Runner, publish Publisher,
|
||||
repository, path, ref, workspace string, held map[string]string, npmrc Npmrc,
|
||||
forge GitCredential, log Log, mirrored *[]string, seats ...map[string]string) (Result, error) {
|
||||
forge GitCredential, log Log, mirrored *[]string, reuse *Reuse, seats ...map[string]string) (Result, error) {
|
||||
// The clone base of each seat a context may name (novox/hq ADR 0155); variadic so the callers
|
||||
// that hand none — tests of everything but contexts — read as they did.
|
||||
var seatBases map[string]string
|
||||
@@ -238,25 +277,6 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
||||
say("manifest", "INVALID: %v", err)
|
||||
return Result{}, err
|
||||
}
|
||||
// The repository's list of predefined keys, at its root (novox/hq issue 630): read and judged here,
|
||||
// where a changed list is first read, and handed on for the controller to keep. A preference that
|
||||
// falls back to a key the list does not hold, or with another type, is refused here as at registration.
|
||||
var predefined json.RawMessage
|
||||
if raw, err := os.ReadFile(filepath.Join(tree, catalogue.PredefinedFile)); err == nil {
|
||||
list, err := catalogue.ParsePredefined(raw)
|
||||
if err != nil {
|
||||
say("manifest", "INVALID: %v", err)
|
||||
return Result{}, err
|
||||
}
|
||||
if problems := catalogue.PredefinedProblems(manifest, list); len(problems) > 0 {
|
||||
err := fmt.Errorf("%s", strings.Join(problems, "; "))
|
||||
say("manifest", "INVALID: %v", err)
|
||||
return Result{}, err
|
||||
}
|
||||
predefined, _ = json.Marshal(list)
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return Result{}, fmt.Errorf("%s's %s cannot be read: %w", repository, catalogue.PredefinedFile, err)
|
||||
}
|
||||
say("manifest", "%s v%s — %d artifact(s)", manifest.Module, manifest.Version, artifactCount(manifest))
|
||||
|
||||
// What it is made from, for its source fingerprint: the module's own tree first.
|
||||
@@ -308,6 +328,9 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
||||
var built []catalogue.Built
|
||||
// stoodOn is every base the build was handed, as resolved — the edges the catalogue derives.
|
||||
var stoodOn []string
|
||||
// reused is the check build a walk took, and builtWhy why it built instead (novox/hq ADR 0350).
|
||||
var reused *Reused
|
||||
builtWhy := ""
|
||||
if manifest.Build != nil {
|
||||
// What this module said it stands on, answered with what this mesh actually holds. Done
|
||||
// before anything is built, so a missing base is refused in front of the person who can
|
||||
@@ -351,7 +374,40 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
||||
// Ordered, so two builds of one commit do the same work in the same sequence and their
|
||||
// logs can be compared.
|
||||
sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name })
|
||||
// **A walk takes the check's build of the same key** (novox/hq ADR 0350 §2): only a commit on its trunk, and
|
||||
// only when asked — a check, a dry run, `build --ref` and `rebuild` never are. Its key is taken by the same
|
||||
// steps a build takes, stopped before anything is built; what it takes is copied by digest.
|
||||
if reuse != nil && reuse.keyFirst {
|
||||
keyed, key, err := keyFor(ctx, run, publish, manifest, within, workspace, commit, credentials, artifacts, args,
|
||||
held, npmrcPath, npmrc, seatBases, src, say)
|
||||
if err == nil && key.Key == "" {
|
||||
err = fmt.Errorf("%w: %s", errNoKey, key.Why)
|
||||
}
|
||||
if err != nil {
|
||||
say("reuse", "not built for the merge: %v", err)
|
||||
return Result{Commit: commit, NoKey: err.Error()}, err
|
||||
}
|
||||
// **A check keeps a build of the same key a verdict names** (novox/hq issue 703): asked again when only
|
||||
// the trunk moved, it builds nothing the earlier check built of an unchanged source.
|
||||
if kept := keptCheckBuild(ctx, *reuse, manifest.Module, key.Key, len(artifacts), say); kept != nil {
|
||||
built, src, reused = kept.Made, keyed, kept
|
||||
}
|
||||
} else if reuse != nil && reuse.Scratch != nil {
|
||||
taken, keyed, why := takeACheckBuild(ctx, run, publish, *reuse, manifest, within, workspace, commit,
|
||||
credentials, artifacts, args, held, npmrcPath, npmrc, seatBases, src, onTrunk, say)
|
||||
if taken != nil {
|
||||
built = taken.Made
|
||||
src = keyed
|
||||
reused = taken
|
||||
} else {
|
||||
builtWhy = why
|
||||
say("reuse", "built: %s", why)
|
||||
}
|
||||
}
|
||||
for _, a := range artifacts {
|
||||
if reused != nil {
|
||||
break
|
||||
}
|
||||
say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a))
|
||||
made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, npmrc, seatBases, src, say)
|
||||
if err != nil {
|
||||
@@ -387,9 +443,148 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
||||
say("source", "%d path(s) of %s", len(s.Paths), where)
|
||||
}
|
||||
}
|
||||
// The reuse key (novox/hq ADR 0350 §1): what a check keeps its build under, and what a walk takes one by.
|
||||
key := keyOf(src, manifest, Version)
|
||||
return Result{Manifest: resolved, Commit: commit, Built: built,
|
||||
Against: against(within, manifest, stoodOn), Read: readBy(manifest), Source: fingerprint,
|
||||
Trunk: trunk, OnTrunk: onTrunk, Branches: branches, Sources: sources, Predefined: predefined}, nil
|
||||
Trunk: trunk, OnTrunk: onTrunk, Branches: branches, Sources: sources,
|
||||
ReuseKey: key.Key, ReuseParts: key.Parts, NoKey: key.Why, Reused: reused, Why: builtWhy}, nil
|
||||
}
|
||||
|
||||
// takeACheckBuild is the walk's look for a check build of its key (novox/hq ADR 0350 §2, §3): the key taken by
|
||||
// the steps a build takes, each artifact stopped before it is built; the build kept under it copied by digest into
|
||||
// the module's namespace. Answers what it took and the source inputs its key was taken from, or why it builds. A
|
||||
// copy that fails is said, and the walk builds (§3): it never fails the walk on its own.
|
||||
func takeACheckBuild(ctx context.Context, run Runner, publish Publisher, reuse Reuse, manifest catalogue.Manifest,
|
||||
within, workspace, commit, credentials string, artifacts []catalogue.Artifact, args []string,
|
||||
held map[string]string, npmrcPath string, npmrc Npmrc, seats map[string]string, src *sourceInputs, onTrunk bool,
|
||||
say func(step, format string, args ...any)) (*Reused, *sourceInputs, string) {
|
||||
if !onTrunk {
|
||||
return nil, nil, "the commit is not on its trunk, and only a trunk commit's walk takes a check's build"
|
||||
}
|
||||
keyed, key, err := keyFor(ctx, run, publish, manifest, within, workspace, commit, credentials, artifacts, args,
|
||||
held, npmrcPath, npmrc, seats, src, say)
|
||||
if err != nil {
|
||||
return nil, nil, err.Error()
|
||||
}
|
||||
if key.Key == "" {
|
||||
return nil, nil, "it has no reuse key: " + key.Why
|
||||
}
|
||||
found, ok, err := reuse.Scratch.Find(ctx, manifest.Module, key.Key)
|
||||
if err != nil {
|
||||
return nil, nil, "the scratch namespace could not be read: " + err.Error()
|
||||
}
|
||||
if !ok {
|
||||
return nil, nil, notKeptWhy(ctx, reuse, manifest.Module, key)
|
||||
}
|
||||
// **Only a build a verdict names** (ADR 0350 §2): the store takes writes from anything on the network, so a
|
||||
// record is believed only when a check verdict of the walk's own pull requests names its key and its digests.
|
||||
if why := unnamed(reuse.Named, manifest.Module, key.Key, found); why != "" {
|
||||
return nil, nil, why
|
||||
}
|
||||
taken := &Reused{From: found}
|
||||
for _, made := range found.Made {
|
||||
copied, err := reuse.Scratch.Take(ctx, manifest.Module, made)
|
||||
if err != nil {
|
||||
why := fmt.Sprintf("copying %s's %s from %s failed (%v)", manifest.Module, made.Name, found.Said(), err)
|
||||
say("reuse", "%s; built instead", why)
|
||||
return nil, nil, why
|
||||
}
|
||||
taken.Made = append(taken.Made, copied)
|
||||
}
|
||||
// Every artifact the manifest declares, or none: a check build missing one is not this build.
|
||||
if len(taken.Made) != len(artifacts) {
|
||||
return nil, nil, fmt.Sprintf("%s holds %d artifact(s) of %s, which declares %d", found.Said(), len(taken.Made),
|
||||
manifest.Module, len(artifacts))
|
||||
}
|
||||
say("reuse", "%s", taken.Words())
|
||||
return taken, keyed, ""
|
||||
}
|
||||
|
||||
// keptCheckBuild is the build a check keeps instead of building (novox/hq issue 703): the one kept under the
|
||||
// module's key by an earlier check of the same pull request at the same head — only the trunk moved — when a verdict
|
||||
// of the build seat names its key and every digest — the record alone is never trusted (ADR 0350 §2) — and it holds
|
||||
// every artifact the module declares. Another pull request's build, or an earlier head's, is never kept. Nil, and
|
||||
// the check builds, otherwise.
|
||||
func keptCheckBuild(ctx context.Context, reuse Reuse, module, key string, artifacts int,
|
||||
say func(step, format string, args ...any)) *Reused {
|
||||
if reuse.Scratch == nil || reuse.head == "" {
|
||||
return nil
|
||||
}
|
||||
var same []NamedBuild
|
||||
for _, n := range reuse.Named {
|
||||
if n.Pull == reuse.pull {
|
||||
same = append(same, n)
|
||||
}
|
||||
}
|
||||
if len(same) == 0 {
|
||||
return nil
|
||||
}
|
||||
found, ok, err := reuse.Scratch.Find(ctx, module, key)
|
||||
switch {
|
||||
case err != nil:
|
||||
say("reuse", "the scratch namespace could not be read, so the check builds: %v", err)
|
||||
return nil
|
||||
case !ok:
|
||||
return nil
|
||||
case found.Pull != reuse.pull || found.Head == "" || twelve(found.Head) != twelve(reuse.head):
|
||||
say("reuse", "the build kept under %s is %s, not this pull request's at this head: built", twelve(key[3:]),
|
||||
found.Said())
|
||||
return nil
|
||||
case unnamed(same, module, key, found) != "":
|
||||
say("reuse", "a build is kept under %s and no verdict names it with its digests: built", twelve(key[3:]))
|
||||
return nil
|
||||
case len(found.Made) != artifacts:
|
||||
return nil
|
||||
}
|
||||
kept := &Reused{From: found, Made: found.Made}
|
||||
say("reuse", "kept, not built: %s", strings.Replace(kept.Words(), "reused from", "the build of", 1))
|
||||
return kept
|
||||
}
|
||||
|
||||
// keyFor takes a build's reuse key by the steps a build takes, each artifact stopped before anything is built or
|
||||
// published (ADR 0350 §2): its own copy of the source inputs, so the build's are untouched.
|
||||
func keyFor(ctx context.Context, run Runner, publish Publisher, manifest catalogue.Manifest,
|
||||
within, workspace, commit, credentials string, artifacts []catalogue.Artifact, args []string,
|
||||
held map[string]string, npmrcPath string, npmrc Npmrc, seats map[string]string, src *sourceInputs,
|
||||
say func(step, format string, args ...any)) (*sourceInputs, reuseKey, error) {
|
||||
keyed := src.copy()
|
||||
keyed.onlyKey = true
|
||||
for _, a := range artifacts {
|
||||
if _, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held,
|
||||
npmrcPath, npmrc, seats, keyed, say); err != nil {
|
||||
return nil, reuseKey{}, fmt.Errorf("its reuse key could not be taken: %w", err)
|
||||
}
|
||||
}
|
||||
keyed.onlyKey = false
|
||||
return keyed, keyOf(keyed, manifest, Version), nil
|
||||
}
|
||||
|
||||
// notKeptWhy is why no check build has the key (§3): no check of the walk's pull requests built the module, the
|
||||
// key differs — naming the part — or the build it made is gone.
|
||||
func notKeptWhy(ctx context.Context, reuse Reuse, module string, key reuseKey) string {
|
||||
if len(reuse.Pulls) == 0 {
|
||||
return "no check built it with this key"
|
||||
}
|
||||
builds, err := reuse.Scratch.OfPulls(ctx, reuse.Pulls)
|
||||
if err != nil {
|
||||
return "no check build has this key, and which checks built it could not be read: " + err.Error()
|
||||
}
|
||||
for _, b := range builds {
|
||||
if b.Module != module {
|
||||
continue
|
||||
}
|
||||
if b.Key == key.Key {
|
||||
return b.Said() + " built it with this key, and it was removed"
|
||||
}
|
||||
return "the key differs from " + b.Said() + "'s in " + strings.Join(differing(b.Parts, key.Parts), ", ")
|
||||
}
|
||||
var pulls []string
|
||||
for _, p := range reuse.Pulls {
|
||||
pulls = append(pulls, p.String())
|
||||
}
|
||||
return "no check of " + strings.Join(pulls, ", ") + " built it (a module its check did not move, or a check " +
|
||||
"that did not finish)"
|
||||
}
|
||||
|
||||
// branchesHolding is every branch of a fresh clone's origin the commit is on, without `origin/`.
|
||||
@@ -778,6 +973,10 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
|
||||
switch a.Kind {
|
||||
case catalogue.ArtifactUpstream:
|
||||
// Its key is the image it copies, read from the manifest (keyOf): nothing to take here.
|
||||
if src.keyOnly() {
|
||||
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
|
||||
}
|
||||
// Mirrored, not built: copied under a name of the mesh's own, so what a machine fetches is
|
||||
// pinned by a digest this registry assigned rather than by a tag somebody else can move.
|
||||
//
|
||||
@@ -906,6 +1105,10 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
} else {
|
||||
src.ownWhole()
|
||||
}
|
||||
// Taken for a reuse key (novox/hq ADR 0350 §2): everything its fingerprint reads is read; nothing is built.
|
||||
if src.keyOnly() {
|
||||
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
|
||||
}
|
||||
invocation := append([]string{"build", "-f", recipePath, "-t", local}, args...)
|
||||
if a.Target != "" {
|
||||
invocation = append(invocation, "--target", a.Target)
|
||||
@@ -923,6 +1126,16 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
return catalogue.Built{}, fmt.Errorf("%s: building %s failed: %w", module, a.Name, err)
|
||||
}
|
||||
say("image", "built, publishing")
|
||||
// Its bases linked into its repository first, so the push needs none of their blobs from this
|
||||
// machine's store, which may not hold them (novox/hq issue 694). Said and not fatal: a push
|
||||
// from a store that holds them works without, and one that fails says which blob it lacked.
|
||||
if lender, can := publish.(BaseLender); can {
|
||||
if n, err := lender.LendBases(ctx, module+"/"+a.Name, buildArgValues(args)); err != nil {
|
||||
say("image", "its bases could not all be linked into %s/%s before the push: %v", module, a.Name, err)
|
||||
} else if n > 0 {
|
||||
say("image", "%d blob(s) of its bases linked into %s/%s before the push", n, module, a.Name)
|
||||
}
|
||||
}
|
||||
reference, err := publish.PublishImage(ctx, local, module+"/"+a.Name)
|
||||
if err != nil {
|
||||
return catalogue.Built{}, err
|
||||
@@ -973,6 +1186,9 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
src.notPinned(a.Name + " resolves packages of its own at build time")
|
||||
}
|
||||
}
|
||||
if src.keyOnly() {
|
||||
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
|
||||
}
|
||||
// The module's own packages first, where the compiler and the bundler resolve them from
|
||||
// (dependencies.go); nothing at all for a module whose package.json names only the SDK.
|
||||
if err := installOwn(ctx, run, tree, chain, base, registry, say); err != nil {
|
||||
@@ -1016,6 +1232,9 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
say("package", "building and publishing %s (%s)", a.Name, a.Language)
|
||||
src.ownWhole()
|
||||
src.notPinned(a.Name + " is a package, built from what the registry holds when it is built")
|
||||
if src.keyOnly() {
|
||||
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
|
||||
}
|
||||
reference, err := publishPackage(ctx, run, module, tree, a, npmrc, say)
|
||||
if err != nil {
|
||||
return catalogue.Built{}, fmt.Errorf("%s: publishing %s failed: %w", module, a.Name, err)
|
||||
@@ -1024,6 +1243,12 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil
|
||||
|
||||
case catalogue.ArtifactArchive:
|
||||
if src.keyOnly() {
|
||||
if a.URL == "" {
|
||||
src.ownHas(cleanEntry(a.From) + "/**")
|
||||
}
|
||||
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
|
||||
}
|
||||
var body []byte
|
||||
var err error
|
||||
if a.URL != "" {
|
||||
@@ -1262,6 +1487,19 @@ func standingOn(ctx context.Context, manifest catalogue.Manifest, held map[strin
|
||||
return args, resolved, nil
|
||||
}
|
||||
|
||||
// buildArgValues is what each --build-arg of a build was handed: the bases, as this mesh holds them.
|
||||
func buildArgValues(args []string) []string {
|
||||
var values []string
|
||||
for i := 0; i+1 < len(args); i += 2 {
|
||||
if args[i] == "--build-arg" {
|
||||
if _, value, has := strings.Cut(args[i+1], "="); has {
|
||||
values = append(values, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
// compile runs a module's own code through its toolchain, and says where the result is.
|
||||
//
|
||||
// **In the module's own directory, under the path the toolchain expects.** A module is compiled
|
||||
|
||||
+661
-26
@@ -13,7 +13,9 @@ import (
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
@@ -87,6 +89,19 @@ type CheckSpec struct {
|
||||
// Judge is who judges the gate: "" the controller the mesh runs, "self" the change's own, "validator"
|
||||
// the running one with the change's validator.
|
||||
Judge string
|
||||
// Release is the release branch the pull request merges into (novox/hq ADR 0358), empty for any other base. The
|
||||
// gate is then judged by the controller beside it as mesh-controller — mesh-controller's release branch, never
|
||||
// the controller the mesh runs nor its main — with mesh-host's validator from beside it when mesh-host was read at
|
||||
// the release branch too; a change to the controller still judges itself and a change to the node-engine brings
|
||||
// its own validator. The verdict names the branch, the judge's commit and the branch each beside was read at.
|
||||
Release string
|
||||
// JudgeCache is the judge cache (novox/hq ADR 0358 §6): a directory of the build agent's own, outside the
|
||||
// workspace, that only judge builds mount — never a check's containers, so no pull request's script can write it.
|
||||
// It holds the Go build cache every judge is built with (judges stay warm), the release-branch judges kept by the
|
||||
// commits they are built from (judges/, one per release-branch head), and each judge build's fresh home (homes/,
|
||||
// removed after the build). Empty: each judge is built cold under the check's own root, which every check empties
|
||||
// before it clones, and nothing is kept.
|
||||
JudgeCache string
|
||||
// Toolchain is the image a check's Go runs in: the mesh's own Go toolchain, as it holds it.
|
||||
// Toolchains is every toolchain the mesh holds, by language, for a script that declares another.
|
||||
Toolchain string
|
||||
@@ -98,6 +113,28 @@ type CheckSpec struct {
|
||||
// and composed with it by the gate as one future state. The repository's own check is not run for a
|
||||
// group: each member's pull request runs its own.
|
||||
Group []GroupHead
|
||||
// Builds are the modules the delivery plan moves, in the walk's order, and BuildWith what they are built with
|
||||
// (novox/hq ADR 0350 §1): built from the merge the check judged into the scratch namespace once it has judged,
|
||||
// for the walk to take. None for a group's composed check, whose members' own checks build.
|
||||
Builds []CheckBuildSpec
|
||||
BuildWith *CheckBuilding
|
||||
// Tests and StoreOnly narrow what the repository's own script runs, for a branch tried before its pull
|
||||
// request (novox/hq issue 686): handed to the script as EnvTests and EnvStoreOnly, which it may read.
|
||||
Tests string
|
||||
StoreOnly bool
|
||||
}
|
||||
|
||||
// selection is what a check hands the repository's script to narrow what it runs (novox/hq issue 686): nothing
|
||||
// for a pull request's check, which always runs whole.
|
||||
func selection(spec CheckSpec) []string {
|
||||
var env []string
|
||||
if spec.Tests != "" {
|
||||
env = append(env, EnvTests+"="+spec.Tests)
|
||||
}
|
||||
if spec.StoreOnly {
|
||||
env = append(env, EnvStoreOnly+"=1")
|
||||
}
|
||||
return env
|
||||
}
|
||||
|
||||
// GroupHead is one other head of a delivery group's composed check.
|
||||
@@ -131,10 +168,12 @@ func ToolchainsOf(held map[string]string) map[string]string {
|
||||
return out
|
||||
}
|
||||
|
||||
// Beside is one repository cloned next to the one checked.
|
||||
// Beside is one repository cloned next to the one checked: at Ref, or at Else when Else is said and the repository
|
||||
// has no branch Ref (novox/hq ADR 0358 §3).
|
||||
type Beside struct {
|
||||
Repository string
|
||||
Ref string
|
||||
Else string
|
||||
}
|
||||
|
||||
// CheckVerdict is what came of one check. Verdict and Summary are the gate's; Gate and Repo each layer.
|
||||
@@ -155,6 +194,18 @@ type CheckVerdict struct {
|
||||
// Members are a delivery group's other heads as the composed check judged them, each with the trunk
|
||||
// commit it was merged into (ADR 0323 §6).
|
||||
Members []JudgedHead
|
||||
// Built are the builds it made for its merge (novox/hq ADR 0350 §1), each with its key, or why not.
|
||||
Built []CheckBuilt
|
||||
// Release is the release branch it was judged for (novox/hq ADR 0358 §4), JudgedBy the commit of the controller
|
||||
// that judged its gate (twelve characters), and BesideAt the branch each repository beside it was read at, by
|
||||
// directory. Empty for a check of any other base.
|
||||
Release string
|
||||
JudgedBy string
|
||||
BesideAt map[string]string
|
||||
// Validator says which node-engine validator the judge of a release-branch check was built with (ADR 0358 §2):
|
||||
// mesh-host's of the release branch with its commit, the one the release branch's controller vendors, the
|
||||
// change's own, or the one the change's own controller vendors. Empty for a check of any other base.
|
||||
Validator string
|
||||
}
|
||||
|
||||
// Layer is one layer of a check, judged.
|
||||
@@ -186,6 +237,11 @@ const (
|
||||
// EnvGroup is a delivery group's other heads, as JSON, for the gate (novox/hq ADR 0239); empty for a
|
||||
// pull request checked alone.
|
||||
EnvGroup = "MESH_CHECK_GROUP"
|
||||
// EnvTests is a `go test -run` pattern and EnvStoreOnly is 1 when a branch tried before its pull request
|
||||
// asks for only those tests, or only the store tests (novox/hq issue 686). Never set for a pull request's
|
||||
// check. A script that reads neither runs whole.
|
||||
EnvTests = "MESH_CHECK_TESTS"
|
||||
EnvStoreOnly = "MESH_CHECK_STORE_ONLY"
|
||||
)
|
||||
|
||||
// CheckTimeout bounds one check; a check that runs past it is an error, not a pass.
|
||||
@@ -245,7 +301,17 @@ func ScriptToolchain(script []byte) string {
|
||||
// Check runs one merge check. An error is that it could not run; the verdict is then "error".
|
||||
func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry string, forge GitCredential,
|
||||
log Log) (CheckVerdict, error) {
|
||||
// Everything a check says goes to the build's log, which the bus keeps: said redacted (novox/hq issue 462).
|
||||
// Everything a check says goes to the build's log, which the bus keeps: said redacted (novox/hq issue 462). A line
|
||||
// at a time: its builds for the merge say theirs while its tests run (novox/hq issue 716).
|
||||
if log != nil {
|
||||
var saying sync.Mutex
|
||||
inner := log
|
||||
log = func(step, message string) {
|
||||
saying.Lock()
|
||||
defer saying.Unlock()
|
||||
inner(step, message)
|
||||
}
|
||||
}
|
||||
redact := redactorFor(forge)
|
||||
say := logging(func(step, message string) {
|
||||
if log != nil {
|
||||
@@ -274,13 +340,32 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
if err := forgetLeftCredentials(workspace); err != nil {
|
||||
return CheckVerdict{}, err
|
||||
}
|
||||
clone := func(repository, ref, dir string) error {
|
||||
// **mesh-lab's replays run where no pull request reaches** (novox/hq issue 709, ADR 0358 §6): they hold the
|
||||
// container runtime's socket, so their sources, home, Go cache and module cache are none of the workspace's.
|
||||
replay, err := replayDirs(spec.JudgeCache, workspace, root, spec.ID)
|
||||
if err != nil {
|
||||
return CheckVerdict{}, err
|
||||
}
|
||||
if replay.base != "" {
|
||||
defer os.RemoveAll(replay.base)
|
||||
}
|
||||
// cloneAt clones a repository and checks out ref — or, when the repository has no branch ref and orElse is said,
|
||||
// orElse (novox/hq ADR 0358 §3); readAt keeps which, by directory.
|
||||
readAt := map[string]string{}
|
||||
cloneAt := func(repository, ref, orElse, dir string) error {
|
||||
if _, err := run(ctx, root, "git", cloneWith(credentials, "clone", "--quiet", repository, dir)...); err != nil {
|
||||
return fmt.Errorf("cannot clone %s: %s", redact.redact(repository), redact.redact(err.Error()))
|
||||
}
|
||||
if err := recordedWithoutUserinfo(ctx, run, filepath.Join(root, dir), repository); err != nil {
|
||||
return err
|
||||
}
|
||||
if orElse != "" && ref != "" {
|
||||
if _, err := run(ctx, filepath.Join(root, dir), "git", "rev-parse", "--verify", "--quiet",
|
||||
"refs/remotes/origin/"+ref); err != nil {
|
||||
ref = orElse
|
||||
}
|
||||
}
|
||||
readAt[dir] = ref
|
||||
if ref != "" {
|
||||
if _, err := run(ctx, filepath.Join(root, dir), "git", "checkout", "--quiet", ref); err != nil {
|
||||
return fmt.Errorf("%s has no %s: %w", redact.redact(repository), ref, err)
|
||||
@@ -288,6 +373,9 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
}
|
||||
return nil
|
||||
}
|
||||
clone := func(repository, ref, dir string) error {
|
||||
return cloneAt(repository, ref, "", dir)
|
||||
}
|
||||
name := spec.Repo
|
||||
if name == "" || !safeName.MatchString(name) {
|
||||
name = "checked"
|
||||
@@ -323,8 +411,28 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
}
|
||||
named := func(v CheckVerdict) CheckVerdict {
|
||||
v.Head, v.Trunk, v.Merge, v.Beside, v.Members = judged.Head, judged.Trunk, judged.Merge, judged.Beside, judged.Members
|
||||
v.Release, v.JudgedBy, v.BesideAt, v.Validator = spec.Release, judged.JudgedBy, judged.BesideAt, judged.Validator
|
||||
return v
|
||||
}
|
||||
// **What the merge would move is built from a checkout nothing of the check touches** (novox/hq issue 703): the
|
||||
// gate and the repository's own check run in containers that mount the workspace read-write, so they can change
|
||||
// its tree, its objects and its caches. The builds are made from a copy of the merge's objects, cloned before
|
||||
// anything of the change runs, outside the workspace — so what is kept under a key is what the key describes. Each
|
||||
// build's working copy is cloned from it into the workspace only after the check has ended (forTheMerge): the
|
||||
// host's container engine sees no other directory of the holder's (novox/hq issue 713).
|
||||
buildingTree := ""
|
||||
if len(spec.Builds) > 0 && spec.BuildWith != nil && len(spec.Group) == 0 {
|
||||
at := judged.Merge
|
||||
if at == "" {
|
||||
at = spec.Ref
|
||||
}
|
||||
var building string
|
||||
building, buildingTree, err = checkoutForTheBuilds(ctx, run, tree, at, workspace)
|
||||
if err != nil {
|
||||
return CheckVerdict{}, err
|
||||
}
|
||||
defer os.RemoveAll(building)
|
||||
}
|
||||
// **The base branch's check judges the change** (novox/hq issue 310), never the one the change brings.
|
||||
judging, err := TheCheckThatJudges(ctx, tree, spec.Base)
|
||||
if err != nil {
|
||||
@@ -347,9 +455,35 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
if dir == name || !safeName.MatchString(dir) {
|
||||
continue
|
||||
}
|
||||
if err := clone(b.Repository, b.Ref, dir); err != nil {
|
||||
if dir == "mesh-lab" {
|
||||
// **mesh-lab is never read at a release branch** (novox/hq ADR 0358): its replays run with the container
|
||||
// runtime's socket, so a branch of a release's name — which anyone may create — is never what runs there,
|
||||
// whether it is asked with a fallback or without.
|
||||
if b.Else != "" || (spec.Release != "" && b.Ref == spec.Release) {
|
||||
b.Ref = b.Else
|
||||
if b.Ref == "" {
|
||||
b.Ref = "main"
|
||||
}
|
||||
}
|
||||
b.Else = ""
|
||||
}
|
||||
if err := cloneAt(b.Repository, b.Ref, b.Else, dir); err != nil {
|
||||
return CheckVerdict{}, fmt.Errorf("beside it, %w", err)
|
||||
}
|
||||
if dir == "mesh-lab" {
|
||||
// The copy the replays run from, made before anything of the change runs, outside every check container
|
||||
// (novox/hq issue 709) — and whether it holds replays, read now, so nothing written later decides it.
|
||||
if err := replay.copyLab(ctx, run, filepath.Join(root, dir)); err != nil {
|
||||
return CheckVerdict{}, fmt.Errorf("mesh-lab could not be copied for its replays: %w", err)
|
||||
}
|
||||
}
|
||||
if spec.Release != "" {
|
||||
// The branch each was read at is named with its commit (ADR 0358 §4).
|
||||
if judged.BesideAt == nil {
|
||||
judged.BesideAt = map[string]string{}
|
||||
}
|
||||
judged.BesideAt[dir] = readAt[dir]
|
||||
}
|
||||
// The commit each one read is named in the verdict (ADR 0323 §3): its move is issue 568's question.
|
||||
if at := headAt(ctx, run, filepath.Join(root, dir)); at != "" {
|
||||
if judged.Beside == nil {
|
||||
@@ -357,7 +491,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
}
|
||||
judged.Beside[dir] = at
|
||||
}
|
||||
say("check", "beside it %s at %s", dir, short(b.Ref))
|
||||
say("check", "beside it %s at %s", dir, refWords(readAt[dir]))
|
||||
}
|
||||
|
||||
// A delivery group's other heads (novox/hq ADR 0239), each at its head, for the gate to compose with this.
|
||||
@@ -409,6 +543,25 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
// Every clone is made: the credential is gone before anything of the check runs (novox/hq issue 462).
|
||||
forget()
|
||||
|
||||
// **What the merge moves is built beside the tests** (novox/hq issue 716) when the build agent has a judge cache to
|
||||
// build it in, where no container of the check reaches; without one, after the check has judged.
|
||||
logged := func(step, message string) {
|
||||
if log != nil {
|
||||
log(step, redact.redact(message))
|
||||
}
|
||||
}
|
||||
var beside *besideTheTests
|
||||
if buildingTree != "" {
|
||||
place, err := buildsBesideTheTests(spec.JudgeCache, workspace, spec.ID)
|
||||
if err != nil {
|
||||
return CheckVerdict{}, err
|
||||
}
|
||||
if place != "" {
|
||||
beside = startBesideTheTests(ctx, run, spec, place, buildingTree, judged, forge, logged, say)
|
||||
defer beside.abandon()
|
||||
}
|
||||
}
|
||||
|
||||
// The facts, and the versions they say the mesh runs.
|
||||
if registry == "" {
|
||||
return CheckVerdict{}, errors.New("no artifact store to read the facts snapshot from")
|
||||
@@ -471,21 +624,56 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
user = fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid())
|
||||
}
|
||||
in := func(image, dir string, env []string, command ...string) []string {
|
||||
home := workspace
|
||||
for _, e := range env {
|
||||
if strings.HasPrefix(e, "HOME=") {
|
||||
home = strings.TrimPrefix(e, "HOME=")
|
||||
}
|
||||
}
|
||||
// As the builder itself: what a check writes into the workspace is the builder's to remove.
|
||||
args := []string{"run", "--rm", "--network", "host", "--volume", workspace + ":" + workspace, "--workdir", dir,
|
||||
"--user", user, "--env", "HOME=" + workspace,
|
||||
"--user", user, "--env", "HOME=" + home,
|
||||
// The check's own checkouts, whoever cloned them: git in a container of another user than the
|
||||
// one that cloned refuses a repository it does not own ("dubious ownership"), and Go's build
|
||||
// stamps the version from git — a judge that would not build for want of it.
|
||||
"--env", "GIT_CONFIG_COUNT=1", "--env", "GIT_CONFIG_KEY_0=safe.directory", "--env", "GIT_CONFIG_VALUE_0=*"}
|
||||
for _, e := range env {
|
||||
args = append(args, "--env", e)
|
||||
if !strings.HasPrefix(e, "HOME=") {
|
||||
args = append(args, "--env", e)
|
||||
}
|
||||
}
|
||||
return append(append(args, image), command...)
|
||||
}
|
||||
inToolchain := func(dir string, env []string, command ...string) []string {
|
||||
return in(spec.Toolchain, dir, env, command...)
|
||||
}
|
||||
// **Every judge is built where no pull request can reach** (novox/hq issue 482, ADR 0358 review): the workspace's
|
||||
// Go cache, HOME and git configuration are written by pull requests' own scripts, and git runs during a Go build
|
||||
// to stamp its version — a planted hook would rewrite the judge's sources before they compile. So a judge is built
|
||||
// with a HOME and configuration made fresh for this check and a Go cache only judge builds mount (judgeBuildDirs),
|
||||
// no global or system git configuration, no Go environment file, and no version stamping from git.
|
||||
judgeHome, judgeGoCache, err := judgeBuildDirs(spec.JudgeCache, workspace, root, spec.ID)
|
||||
if err != nil {
|
||||
return CheckVerdict{}, err
|
||||
}
|
||||
defer os.RemoveAll(judgeHome)
|
||||
inJudgeBuild := func(dir string, _ []string, command ...string) []string {
|
||||
args := in(spec.Toolchain, dir, JudgeBuildEnv(judgeHome, judgeGoCache), command...)
|
||||
if spec.JudgeCache == "" {
|
||||
return args // under the check's root, in the workspace mounted already
|
||||
}
|
||||
// The judge cache's parts this build needs, mounted into judge builds alone (ADR 0358 §6).
|
||||
return append([]string{args[0], "--volume", judgeHome + ":" + judgeHome, "--volume",
|
||||
judgeGoCache + ":" + judgeGoCache}, args[1:]...)
|
||||
}
|
||||
inReplays := func(env []string, command ...string) []string {
|
||||
args := in(spec.Toolchain, filepath.Join(replay.lab, "replays"), append(ReplayEnv(replay.home, replay.goCache,
|
||||
replay.modCache), env...), command...)
|
||||
// The socket goes to the replays alone, never to the change's own code.
|
||||
mounts := []string{"--volume", "/var/run/docker.sock:/var/run/docker.sock", "--volume", replay.base + ":" +
|
||||
replay.base, "--volume", replay.goCache + ":" + replay.goCache, "--volume", replay.modCache + ":" + replay.modCache}
|
||||
return append(append([]string{args[0]}, mounts...), args[1:]...)
|
||||
}
|
||||
var out tail
|
||||
// running runs one container of the check, its output into the report, in a process group of its own.
|
||||
running := func(args []string) error {
|
||||
@@ -497,10 +685,23 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
|
||||
// The judge. Its failing to build is the change's fault when the change is the judge or its validator,
|
||||
// and the check's when it is the controller the mesh runs.
|
||||
gate, judgeFault, err := judgeFor(ctx, labelled, run, spec, root, tree, inToolchain, say)
|
||||
gate, judgeFault, err := judgeFor(ctx, labelled, run, spec, root, tree, judged.BesideAt, inJudgeBuild, say)
|
||||
if err != nil {
|
||||
return CheckVerdict{}, err
|
||||
}
|
||||
if spec.Release != "" {
|
||||
// **A release-branch judgement says so** (novox/hq ADR 0358 §4): the branch, the controller's commit that
|
||||
// judged, and each repository read beside it at its branch and commit.
|
||||
judgedBy := filepath.Join(root, "mesh-controller")
|
||||
if spec.Judge == "self" {
|
||||
judgedBy = tree
|
||||
}
|
||||
judged.JudgedBy = headAt(ctx, run, judgedBy)
|
||||
judged.Validator = releaseValidator(spec, judged.BesideAt, judged.Beside)
|
||||
said := releaseWords(spec.Release, judged.JudgedBy, judged.Validator, judged.BesideAt, judged.Beside)
|
||||
fmt.Fprintf(&out, "--- %s\n", said)
|
||||
say("check", "%s", said)
|
||||
}
|
||||
verdictFile := filepath.Join(root, "verdict.json")
|
||||
env := []string{EnvFacts + "=" + factsFile, EnvGate + "=" + gate, EnvGateStore + "=" + storeURL,
|
||||
EnvTestStore + "=" + storeURL, EnvTestBus + "=nats://" + bus, EnvRepository + "=" + spec.Owner + "/" + spec.Repo,
|
||||
@@ -508,6 +709,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
EnvModules + "=" + strings.Join(append(append([]string{}, spec.Modules...), spec.New...), ","),
|
||||
EnvGroup + "=" + groupFile,
|
||||
"GOCACHE=" + filepath.Join(workspace, "go-cache"), "GOMODCACHE=" + filepath.Join(workspace, "go-modules")}
|
||||
env = append(env, selection(spec)...)
|
||||
|
||||
v := CheckVerdict{}
|
||||
modules := append(append([]string{}, spec.Modules...), prefixed("new:", spec.New)...)
|
||||
@@ -525,7 +727,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
say("check", "the gate: %d module(s) of the graph touched — %s", len(modules), strings.Join(modules, ", "))
|
||||
fmt.Fprintf(&out, "--- the gate: %s\n", strings.Join(modules, ", "))
|
||||
v.Gate.Verdict, v.Gate.Summary = gateLayer(ctx, spec, tree, root, gate, verdictFile, env, inToolchain,
|
||||
running, &out, bus, workspace, name, say)
|
||||
running, &out, bus, name, replay.hasReplays, replay.refused(), inReplays, say)
|
||||
}
|
||||
if timedOut() {
|
||||
v.Gate.Verdict, v.Gate.Summary = "error", fmt.Sprintf("the check ran past %s and was ended", CheckTimeout)
|
||||
@@ -565,6 +767,11 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
}
|
||||
}
|
||||
|
||||
if spec.Release != "" {
|
||||
// The statuses name the release branch and the judge first, so the forge's short description keeps them.
|
||||
mark := releaseMark(spec.Release, judged.JudgedBy)
|
||||
v.Gate.Summary, v.Repo.Summary = mark+v.Gate.Summary, mark+v.Repo.Summary
|
||||
}
|
||||
// What the check printed travels in the verdict to the forge and the controller: redacted as the log is.
|
||||
v.Gate.Summary, v.Repo.Summary, v.Repo.Failed = redact.redact(v.Gate.Summary), redact.redact(v.Repo.Summary),
|
||||
redact.redact(v.Repo.Failed)
|
||||
@@ -572,9 +779,82 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
v.Report, v.Took = redact.redact(withWhatFailed(out.String(), v.Repo)), time.Since(began)
|
||||
say("check", "gate %s — %s; repository %s — %s (%s)", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary,
|
||||
strings.ToUpper(v.Repo.Verdict), v.Repo.Summary, v.Took.Round(time.Second))
|
||||
|
||||
// **What the merge would move, built once, here** (novox/hq ADR 0350 §1): after the judging, from the merge it
|
||||
// judged, into the scratch namespace — so the walk of the merge commit takes it rather than building again.
|
||||
switch {
|
||||
case beside != nil:
|
||||
v.Built = beside.settle(ctx, v)
|
||||
case len(spec.Builds) > 0 && spec.BuildWith != nil && len(spec.Group) == 0:
|
||||
v.Built = forTheMerge(ctx, run, spec, v, workspace, buildingTree, judged, forge, logged, say)
|
||||
}
|
||||
return named(v), nil
|
||||
}
|
||||
|
||||
// forTheMerge builds what the merge moves once the check has judged it, from merged — the copy of the merge
|
||||
// checkoutForTheBuilds made outside the workspace — **in the builder's own workspace** (novox/hq issue 713). A build's
|
||||
// containers are run by the host's container engine, which mounts a path as the host names it: the build seat's
|
||||
// holder runs in a container that shares only its workspace with the host, at the same path, so a working copy in
|
||||
// its own temporary directory reaches the toolchain as an empty directory ("go.mod file not found"), every check
|
||||
// build failed, and every walk built again. The copy stays outside the workspace, so nothing the tests ran reaches it;
|
||||
// the working copy cloned from it is made only now, after every container of the check has ended.
|
||||
func forTheMerge(ctx context.Context, run Runner, spec CheckSpec, verdict CheckVerdict, workspace, merged string,
|
||||
judged CheckVerdict, forge GitCredential, log Log, say func(step, format string, args ...any)) []CheckBuilt {
|
||||
if why := skipsTheBuilds(verdict); why != "" {
|
||||
say("check", "%s", why)
|
||||
return nil
|
||||
}
|
||||
began := time.Now()
|
||||
built := buildForTheMerge(ctx, run, spec, *spec.BuildWith, workspace, merged, judged.Merge, judged, forge, log, say)
|
||||
say("check", "built for the merge: %d of %d module(s) (%s)", builtCount(built), len(built),
|
||||
time.Since(began).Round(time.Second))
|
||||
return built
|
||||
}
|
||||
|
||||
// checkoutForTheBuilds is a clone of the commit a check builds for its merge, made before anything of the check runs,
|
||||
// outside the workspace its containers mount (novox/hq issue 703): the directory to remove, and the clone in it.
|
||||
// Cloned over git's own transport, never by linking the objects: what the check's containers later write into the
|
||||
// workspace's objects cannot reach it.
|
||||
func checkoutForTheBuilds(ctx context.Context, run Runner, tree, at, workspace string) (string, string, error) {
|
||||
dir, err := os.MkdirTemp("", "mesh-check-builds-")
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if withinDir(workspace, dir) {
|
||||
os.RemoveAll(dir)
|
||||
return "", "", fmt.Errorf("the temporary directory %s is inside the workspace %s, which a check's containers "+
|
||||
"mount: the merge is not built from where its tests could change it", dir, workspace)
|
||||
}
|
||||
commit, err := run(ctx, tree, "git", "rev-parse", "--verify", at+"^{commit}")
|
||||
if err != nil {
|
||||
os.RemoveAll(dir)
|
||||
return "", "", fmt.Errorf("the commit to build for the merge, %s: %w", at, err)
|
||||
}
|
||||
commit = strings.TrimSpace(commit)
|
||||
if _, err := run(ctx, tree, "git", "update-ref", "refs/heads/mesh-check-builds", commit); err != nil {
|
||||
os.RemoveAll(dir)
|
||||
return "", "", err
|
||||
}
|
||||
cloned := filepath.Join(dir, "merged")
|
||||
if _, err := run(ctx, dir, "git", "clone", "--quiet", "--no-local", "--no-checkout", "--branch",
|
||||
"mesh-check-builds", tree, cloned); err != nil {
|
||||
os.RemoveAll(dir)
|
||||
return "", "", fmt.Errorf("cannot copy the merge to build it: %w", err)
|
||||
}
|
||||
return dir, cloned, nil
|
||||
}
|
||||
|
||||
// builtCount is how many of a check's builds for its merge were built.
|
||||
func builtCount(built []CheckBuilt) int {
|
||||
n := 0
|
||||
for _, b := range built {
|
||||
if b.NotBuilt == "" {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// ownCheck runs the repository's own check, part by part, each in its toolchain's own container: the
|
||||
// layer's verdict. Nil when the check was ended from outside (not past its bound), which the caller
|
||||
// answers as the context's error.
|
||||
@@ -735,8 +1015,9 @@ const SomeManifestsEnv = "MESH_MODULE_CHECK_SOME=1"
|
||||
// gateLayer runs the gate: the touched manifests through `module check`, every machine composed with the
|
||||
// change, and the replays of what the mesh runs. It answers the gate's verdict and summary.
|
||||
func gateLayer(ctx context.Context, spec CheckSpec, tree, root, gate, verdictFile string, env []string,
|
||||
inToolchain func(string, []string, ...string) []string, running func([]string) error, out *tail, bus, workspace,
|
||||
name string, say func(step, format string, args ...any)) (string, string) {
|
||||
inToolchain func(string, []string, ...string) []string, running func([]string) error, out *tail, bus,
|
||||
name string, hasReplays bool, replaysRefused string, inReplays func([]string, ...string) []string,
|
||||
say func(step, format string, args ...any)) (string, string) {
|
||||
// 1. The manifests the change touches, as the judge reads them: a manifest it cannot read, or one with a
|
||||
// problem the change brings, fails here. **What was already so on the base branch is said and fails
|
||||
// nothing** — the gate's own rule: a module whose manifest the running controller already finds fault
|
||||
@@ -828,24 +1109,37 @@ func gateLayer(ctx context.Context, spec CheckSpec, tree, root, gate, verdictFil
|
||||
// 3. **The replays of what the mesh runs** (to-be 45 §9, M9): mesh-lab's, from its main — reviewed code,
|
||||
// so given the container runtime the resolver replay raises containers with — against the bus of the
|
||||
// release the mesh runs and the change's own catalogue when the change is to the catalogue.
|
||||
if lab := filepath.Join(root, "mesh-lab", "replays"); ctx.Err() == nil {
|
||||
if _, err := os.Stat(lab); err == nil {
|
||||
if ctx.Err() == nil && hasReplays && replaysRefused != "" {
|
||||
// **Never from the workspace** (novox/hq issue 709): without the judge cache there is no place outside it.
|
||||
say("check", "%s", replaysRefused)
|
||||
fmt.Fprintf(out, "--- the replays: %s\n", replaysRefused)
|
||||
return "error", replaysRefused
|
||||
}
|
||||
if ctx.Err() == nil {
|
||||
if hasReplays {
|
||||
catalogue := filepath.Join(root, "mesh-catalog")
|
||||
if name == "mesh-catalog" {
|
||||
catalogue = tree
|
||||
}
|
||||
say("check", "the replays of what the mesh runs, from mesh-lab")
|
||||
fmt.Fprintln(out, "--- the replays (mesh-lab replays/)")
|
||||
args := inToolchain(lab, []string{EnvTestBus + "=nats://" + bus, "MESH_REPLAY_CATALOGUE=" + catalogue,
|
||||
"GOCACHE=" + filepath.Join(workspace, "go-cache"), "GOMODCACHE=" + filepath.Join(workspace, "go-modules")},
|
||||
// In the replays' own environment (inReplays): reviewed code, the socket, and nothing the workspace holds
|
||||
// but the catalogue they read.
|
||||
args := inReplays([]string{EnvTestBus + "=nats://" + bus, "MESH_REPLAY_CATALOGUE=" + catalogue},
|
||||
"go", "test", "-count=1", "./...")
|
||||
// The socket goes to the replays alone, never to the change's own code above.
|
||||
args = append([]string{args[0], "--volume", "/var/run/docker.sock:/var/run/docker.sock"}, args[1:]...)
|
||||
if err := running(args); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return "error", "the check was ended during the replays"
|
||||
}
|
||||
return "fail", "a replay of a core incident fails with this change: " + lastLine(out.String())
|
||||
if replaysNotStarted(err) {
|
||||
// The container runtime could not start them (a mount, an image): the check's fault, never the change's.
|
||||
said := "the replays could not be started, so the change is not judged by them: " + lastLine(out.String())
|
||||
say("check", "%s", said)
|
||||
return "error", said
|
||||
}
|
||||
said := "a replay of a core incident fails with this change: " + lastLine(out.String())
|
||||
say("check", "%s", said)
|
||||
return "fail", said
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -860,21 +1154,32 @@ func gateLayer(ctx context.Context, spec CheckSpec, tree, root, gate, verdictFil
|
||||
// the mesh runs. It answers the judge's path; or, when the change makes its own judge unbuildable, why —
|
||||
// the change's fault, a failing gate; or an error when the check cannot build a judge at all. A check
|
||||
// whose gate does not run builds a judge only to hand its scripts one, and goes on without when it cannot.
|
||||
func judgeFor(ctx context.Context, labelled, run Runner, spec CheckSpec, root, tree string,
|
||||
//
|
||||
// **A pull request into a release branch is judged by that branch's controller** (novox/hq ADR 0358 §2): the one
|
||||
// beside it as mesh-controller, read at the release branch — with mesh-host's validator from beside it when mesh-host
|
||||
// has that branch too, or with the change's own validator for a change to the node-engine — never the controller the
|
||||
// mesh runs. besideAt is the branch each repository beside it was read at.
|
||||
func judgeFor(ctx context.Context, labelled, run Runner, spec CheckSpec, root, tree string, besideAt map[string]string,
|
||||
inToolchain func(string, []string, ...string) []string, say func(step, format string, args ...any)) (string, string, error) {
|
||||
gated := spec.Gated()
|
||||
switch spec.Judge {
|
||||
case "self":
|
||||
bin := filepath.Join(root, "bin", "judge-of-itself")
|
||||
if _, err := labelled(ctx, root, "docker", inToolchain(tree,
|
||||
[]string{"CGO_ENABLED=0", "GOFLAGS=-mod=vendor", "GOPROXY=off", "GOCACHE=" + filepath.Join(filepath.Dir(root), "go-cache")},
|
||||
if _, err := labelled(ctx, root, "docker", inToolchain(tree, nil,
|
||||
"go", "build", "-o", bin, "./cmd/mesh-controller")...); err != nil {
|
||||
return "", "the change's controller, which judges itself, does not build: " + firstLine(err.Error()), nil
|
||||
}
|
||||
say("check", "judged by the change's own controller")
|
||||
return bin, "", nil
|
||||
}
|
||||
bin, judgeTree, err := judge(ctx, labelled, run, root, inToolchain, say)
|
||||
var bin, judgeTree string
|
||||
var err error
|
||||
if spec.Release != "" {
|
||||
withHost := spec.Judge != "validator" && besideAt["mesh-host"] == spec.Release
|
||||
bin, judgeTree, err = releaseJudge(ctx, labelled, run, spec, root, withHost, inToolchain, say)
|
||||
} else {
|
||||
bin, judgeTree, err = judge(ctx, labelled, run, root, inToolchain, say)
|
||||
}
|
||||
if err != nil {
|
||||
if !gated {
|
||||
say("check", "no judge for its script: %v", err)
|
||||
@@ -893,15 +1198,346 @@ func judgeFor(ctx context.Context, labelled, run Runner, spec CheckSpec, root, t
|
||||
}
|
||||
}
|
||||
withValidator := filepath.Join(root, "bin", "judge-with-this-validator")
|
||||
if _, err := labelled(ctx, root, "docker", inToolchain(filepath.Join(root, judgeTree),
|
||||
[]string{"CGO_ENABLED=0", "GOFLAGS=-mod=vendor", "GOPROXY=off", "GOCACHE=" + filepath.Join(filepath.Dir(root), "go-cache")},
|
||||
if _, err := labelled(ctx, root, "docker", inToolchain(filepath.Join(root, judgeTree), nil,
|
||||
"go", "build", "-o", withValidator, "./cmd/mesh-controller")...); err != nil {
|
||||
if spec.Release != "" {
|
||||
return "", spec.Release + "'s controller does not build with this change's validator: " +
|
||||
firstLine(err.Error()), nil
|
||||
}
|
||||
return "", "the controller the mesh runs does not build with this change's validator: " + firstLine(err.Error()), nil
|
||||
}
|
||||
if spec.Release != "" {
|
||||
say("check", "judged by %s's controller, with this change's validator", spec.Release)
|
||||
return withValidator, "", nil
|
||||
}
|
||||
say("check", "judged by the controller the mesh runs, with this change's validator")
|
||||
return withValidator, "", nil
|
||||
}
|
||||
|
||||
// releaseJudge builds the controller that judges a pull request into a release branch (novox/hq ADR 0358 §2): the
|
||||
// one beside the check as mesh-controller, read at that branch — with mesh-host's validator from beside it in place
|
||||
// of the one it vendors when withHost. **Built once per release-branch head**: kept in the spec's judge cache by the
|
||||
// commits it is built from and taken from there while they stand. It answers the binary and the directory it was
|
||||
// built from.
|
||||
func releaseJudge(ctx context.Context, run, plain Runner, spec CheckSpec, root string, withHost bool,
|
||||
inToolchain func(string, []string, ...string) []string, say func(step, format string, args ...any)) (string, string, error) {
|
||||
const dir = "mesh-controller"
|
||||
if _, err := os.Stat(filepath.Join(root, dir)); err != nil {
|
||||
return "", "", fmt.Errorf("no controller of %s beside the check to judge it with", spec.Release)
|
||||
}
|
||||
key := fullHeadAt(ctx, plain, filepath.Join(root, dir))
|
||||
if key == "" {
|
||||
return "", "", fmt.Errorf("the controller of %s beside the check names no commit", spec.Release)
|
||||
}
|
||||
with := ""
|
||||
if withHost {
|
||||
host := fullHeadAt(ctx, plain, filepath.Join(root, "mesh-host"))
|
||||
if host == "" {
|
||||
return "", "", fmt.Errorf("mesh-host of %s beside the check names no commit", spec.Release)
|
||||
}
|
||||
key += "-with-mesh-host-" + host
|
||||
with = ", with mesh-host's validator of " + spec.Release + " at " + twelve(host)
|
||||
}
|
||||
bin := filepath.Join(root, "bin", "mesh-controller")
|
||||
if took := judgeKept(keptJudges(spec.JudgeCache), key, bin); took {
|
||||
say("check", "judged by %s's controller at %s%s, built for an earlier check of that head", spec.Release,
|
||||
twelve(key), with)
|
||||
return bin, dir, nil
|
||||
}
|
||||
if withHost {
|
||||
vendored := filepath.Join(root, dir, "vendor", "github.com", "novox", "mesh-host")
|
||||
for _, pkg := range []string{"validate", filepath.Join("internal", "declaration")} {
|
||||
if err := replaceGoFiles(filepath.Join(root, "mesh-host", pkg), filepath.Join(vendored, pkg)); err != nil {
|
||||
return "", "", fmt.Errorf("mesh-host's validator of %s could not be put in its controller: %w", spec.Release, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Built where no pull request reaches (JudgeBuildEnv): a judge kept for this head is the head's, nothing else.
|
||||
if _, err := run(ctx, root, "docker", inToolchain(filepath.Join(root, dir), nil,
|
||||
"go", "build", "-o", bin, "./cmd/mesh-controller")...); err != nil {
|
||||
return "", "", fmt.Errorf("the controller of %s does not build%s: %w", spec.Release, with, err)
|
||||
}
|
||||
// Kept before anything of the change runs: nothing a pull request's script writes can reach the cache.
|
||||
keepJudge(keptJudges(spec.JudgeCache), key, bin, say)
|
||||
say("check", "judged by %s's controller at %s%s", spec.Release, twelve(key), with)
|
||||
return bin, dir, nil
|
||||
}
|
||||
|
||||
// JudgeBuildEnv is the environment every judge is built in (novox/hq ADR 0358 §6, issue 482): home, made fresh for the
|
||||
// build, as HOME and configuration; goCache, the judge cache's Go cache that no check's container mounts; no global
|
||||
// or system git configuration; no Go environment file; and no version stamped from git, so no git runs at all.
|
||||
func JudgeBuildEnv(home, goCache string) []string {
|
||||
return []string{"HOME=" + home, "XDG_CONFIG_HOME=" + filepath.Join(home, ".config"),
|
||||
"GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_NOSYSTEM=1", "GOENV=off",
|
||||
"GOCACHE=" + goCache, "GOPATH=" + filepath.Join(home, "go"),
|
||||
"CGO_ENABLED=0", "GOFLAGS=-mod=vendor -buildvcs=false", "GOPROXY=off"}
|
||||
}
|
||||
|
||||
// judgeBuildDirs are the home and the Go cache a check's judge builds use (ADR 0358 §6). With a judge cache: a home
|
||||
// made fresh under its homes/ for this check, and its go-cache/, kept warm between checks — both on the host path the
|
||||
// build agent declares, so the container runtime mounts what the agent sees, and nothing is left in a temporary
|
||||
// directory of the host. Without one: both under the check's root, which every check empties before it clones. A
|
||||
// judge cache inside the workspace is refused: a check's containers mount the workspace.
|
||||
func judgeBuildDirs(cache, workspace, root, id string) (string, string, error) {
|
||||
if cache == "" {
|
||||
home, goCache := filepath.Join(root, "judge-home"), filepath.Join(root, "judge-go-cache")
|
||||
for _, d := range []string{home, goCache} {
|
||||
if err := os.MkdirAll(d, 0o700); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
}
|
||||
return home, goCache, nil
|
||||
}
|
||||
if withinDir(workspace, cache) {
|
||||
return "", "", fmt.Errorf("the judge cache %s is inside the workspace %s, which a check's containers mount: "+
|
||||
"no judge is built where a pull request could reach it", cache, workspace)
|
||||
}
|
||||
if !safeName.MatchString(id) {
|
||||
return "", "", fmt.Errorf("the check's id %q cannot name its place in the judge cache", id)
|
||||
}
|
||||
home, goCache := filepath.Join(cache, "homes", id), filepath.Join(cache, "go-cache")
|
||||
if err := os.RemoveAll(home); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
for _, d := range []string{home, goCache} {
|
||||
if err := os.MkdirAll(d, 0o700); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
}
|
||||
return home, goCache, nil
|
||||
}
|
||||
|
||||
// replayPlace is where mesh-lab's replays run from and with (novox/hq issue 709), never anything a pull request can
|
||||
// write: base, the directory made for this check under the judge cache's homes/ (the same path on the host and in the
|
||||
// build agent), removed after it; lab, the copy of mesh-lab they run; home, their HOME and configuration; and the judge
|
||||
// cache's Go build and module caches. Empty without a judge cache: the replays are then refused. hasReplays says whether
|
||||
// mesh-lab held replays when it was cloned, before anything of the change ran.
|
||||
type replayPlace struct {
|
||||
base, lab, home, goCache, modCache string
|
||||
hasReplays bool
|
||||
}
|
||||
|
||||
// replaysNeedTheJudgeCache is the gate's error when mesh-lab holds replays and the build agent has no judge cache.
|
||||
const replaysNeedTheJudgeCache = "the replays need the build agent's judge cache (mesh-catalog#297); refused rather " +
|
||||
"than run from the workspace"
|
||||
|
||||
// refused is why the replays cannot run here, empty when they can.
|
||||
func (p replayPlace) refused() string {
|
||||
if p.base == "" {
|
||||
return replaysNeedTheJudgeCache
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// replayDirs makes the replays' place (issue 709, ADR 0358 §6). The replays' build is as trusted as a judge's: it
|
||||
// compiles mesh-lab's trunk, reviewed code, and nothing of the pull request, which hands them only its catalogue, as
|
||||
// data. Never the workspace: the gate runs before the replays, a pull request's own code among its containers. Without
|
||||
// a judge cache there is no place: the build agent declares no other host path, and the replays are refused.
|
||||
func replayDirs(cache, workspace, root, id string) (replayPlace, error) {
|
||||
if cache == "" {
|
||||
return replayPlace{}, nil
|
||||
}
|
||||
if withinDir(workspace, cache) {
|
||||
return replayPlace{}, fmt.Errorf("the judge cache %s is inside the workspace %s, which a check's containers "+
|
||||
"mount: no replay is run where a pull request could reach it", cache, workspace)
|
||||
}
|
||||
if !safeName.MatchString(id) {
|
||||
return replayPlace{}, fmt.Errorf("the check's id %q cannot name its place in the judge cache", id)
|
||||
}
|
||||
base := filepath.Join(cache, "homes", id+"-replays")
|
||||
p := replayPlace{base: base, lab: filepath.Join(base, "mesh-lab"), home: filepath.Join(base, "home"),
|
||||
goCache: filepath.Join(cache, "go-cache"), modCache: filepath.Join(cache, "go-modules")}
|
||||
if err := os.RemoveAll(base); err != nil {
|
||||
return replayPlace{}, err
|
||||
}
|
||||
for _, d := range []string{p.home, p.goCache, p.modCache} {
|
||||
if err := os.MkdirAll(d, 0o700); err != nil {
|
||||
return replayPlace{}, err
|
||||
}
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// copyLab reads whether mesh-lab's checkout, freshly cloned and before anything of the change runs, holds replays, and
|
||||
// copies it over git's own transport to where the replays run it — when there is such a place.
|
||||
func (p *replayPlace) copyLab(ctx context.Context, run Runner, from string) error {
|
||||
_, err := os.Stat(filepath.Join(from, "replays"))
|
||||
p.hasReplays = err == nil
|
||||
if p.lab == "" {
|
||||
return nil
|
||||
}
|
||||
return copyCheckout(ctx, run, from, p.lab)
|
||||
}
|
||||
|
||||
// replaysNotStarted says the container runtime could not start the replays' container — its own exit status 125, or
|
||||
// 126 and 127 for a command it could not run — rather than that a replay failed.
|
||||
func replaysNotStarted(err error) bool {
|
||||
var exit *exec.ExitError
|
||||
if !errors.As(err, &exit) {
|
||||
return true
|
||||
}
|
||||
code := exit.ExitCode()
|
||||
return code == 125 || code == 126 || code == 127
|
||||
}
|
||||
|
||||
// ReplayEnv is the environment mesh-lab's replays run in (novox/hq issue 709): home as HOME and configuration, the Go
|
||||
// build and module caches given, no global or system git configuration, no Go environment file, and no version
|
||||
// stamped from git — as a judge is built (JudgeBuildEnv), with the modules the replays fetch.
|
||||
func ReplayEnv(home, goCache, modCache string) []string {
|
||||
return []string{"HOME=" + home, "XDG_CONFIG_HOME=" + filepath.Join(home, ".config"),
|
||||
"GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_NOSYSTEM=1", "GOENV=off",
|
||||
"GOCACHE=" + goCache, "GOMODCACHE=" + modCache, "GOPATH=" + filepath.Join(home, "go"),
|
||||
"GOFLAGS=-buildvcs=false", "GOWORK=off"}
|
||||
}
|
||||
|
||||
// copyCheckout copies a checkout at the commit it stands at into dir, over git's own transport — never by linking
|
||||
// objects — so what is written into the source afterwards does not reach the copy.
|
||||
func copyCheckout(ctx context.Context, run Runner, from, dir string) error {
|
||||
commit := fullHeadAt(ctx, run, from)
|
||||
if commit == "" {
|
||||
return fmt.Errorf("%s names no commit", from)
|
||||
}
|
||||
if _, err := run(ctx, filepath.Dir(dir), "git", "clone", "--quiet", "--no-local", "--no-checkout", from, dir); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := run(ctx, dir, "git", "checkout", "--quiet", "--detach", commit)
|
||||
return err
|
||||
}
|
||||
|
||||
// keptJudges is where a judge cache keeps the release-branch judges, empty when there is no judge cache.
|
||||
func keptJudges(cache string) string {
|
||||
if cache == "" {
|
||||
return ""
|
||||
}
|
||||
return filepath.Join(cache, "judges")
|
||||
}
|
||||
|
||||
// judgesKept is how many release-branch judges a judge cache keeps: a head moved past is not built from again.
|
||||
const judgesKept = 4
|
||||
|
||||
// judgeKept puts the judge kept under key in the cache at bin, and says whether there was one.
|
||||
func judgeKept(cache, key, bin string) bool {
|
||||
if cache == "" {
|
||||
return false
|
||||
}
|
||||
body, err := os.ReadFile(filepath.Join(cache, key, "mesh-controller"))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(bin), 0o755); err != nil {
|
||||
return false
|
||||
}
|
||||
if err := os.WriteFile(bin, body, 0o755); err != nil {
|
||||
return false
|
||||
}
|
||||
now := time.Now()
|
||||
_ = os.Chtimes(filepath.Join(cache, key), now, now)
|
||||
return true
|
||||
}
|
||||
|
||||
// keepJudge keeps a judge just built under key, and lets go of all but the judgesKept most recently used. A judge
|
||||
// that cannot be kept is said and built again next time: the cache only saves time.
|
||||
func keepJudge(cache, key, bin string, say func(step, format string, args ...any)) {
|
||||
if cache == "" {
|
||||
return
|
||||
}
|
||||
body, err := os.ReadFile(bin)
|
||||
if err == nil {
|
||||
err = os.MkdirAll(filepath.Join(cache, key), 0o755)
|
||||
}
|
||||
if err == nil {
|
||||
kept := filepath.Join(cache, key, "mesh-controller")
|
||||
if err = os.WriteFile(kept+".part", body, 0o755); err == nil {
|
||||
err = os.Rename(kept+".part", kept)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
say("check", "the judge could not be kept for the next check, which builds it again: %v", err)
|
||||
return
|
||||
}
|
||||
entries, err := os.ReadDir(cache)
|
||||
if err != nil || len(entries) <= judgesKept {
|
||||
return
|
||||
}
|
||||
type used struct {
|
||||
name string
|
||||
at time.Time
|
||||
}
|
||||
var all []used
|
||||
for _, e := range entries {
|
||||
if info, err := e.Info(); err == nil && e.IsDir() {
|
||||
all = append(all, used{e.Name(), info.ModTime()})
|
||||
}
|
||||
}
|
||||
sort.Slice(all, func(i, j int) bool { return all[i].at.After(all[j].at) })
|
||||
for _, u := range all[min(judgesKept, len(all)):] {
|
||||
if u.name != key {
|
||||
_ = os.RemoveAll(filepath.Join(cache, u.name))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// fullHeadAt is the whole commit a checkout stands at, empty when it names none.
|
||||
func fullHeadAt(ctx context.Context, run Runner, tree string) string {
|
||||
out, err := run(ctx, tree, "git", "rev-parse", "--verify", "--quiet", "HEAD^{commit}")
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(out)
|
||||
}
|
||||
|
||||
// releaseValidator says which node-engine validator a release-branch check's judge is built with (novox/hq ADR 0358
|
||||
// §2), by the rule judgeFor builds it by.
|
||||
func releaseValidator(spec CheckSpec, at, commits map[string]string) string {
|
||||
switch {
|
||||
case spec.Judge == "self":
|
||||
return "the validator the change's own controller vendors"
|
||||
case spec.Judge == "validator":
|
||||
return "the change's own validator"
|
||||
case at["mesh-host"] == spec.Release:
|
||||
return fmt.Sprintf("mesh-host's validator of %s at %s", spec.Release, commits["mesh-host"])
|
||||
}
|
||||
return fmt.Sprintf("the validator %s's controller vendors (mesh-host has no branch %s)", spec.Release, spec.Release)
|
||||
}
|
||||
|
||||
// releaseWords is how a release-branch check says what judged it and what it read (novox/hq ADR 0358 §4).
|
||||
func releaseWords(release, judgedBy, validator string, at, commits map[string]string) string {
|
||||
var read []string
|
||||
for _, dir := range sortedKeys(at) {
|
||||
read = append(read, fmt.Sprintf("%s %s at %s", dir, refWords(at[dir]), commits[dir]))
|
||||
}
|
||||
said := fmt.Sprintf("judged for the release branch %s by its controller at %s, with %s", release, judgedBy, validator)
|
||||
if len(read) > 0 {
|
||||
said += "; read beside it: " + strings.Join(read, ", ")
|
||||
}
|
||||
return said
|
||||
}
|
||||
|
||||
// releaseMark opens each layer's summary of a release-branch check, so a status says it first.
|
||||
func releaseMark(release, judgedBy string) string {
|
||||
return fmt.Sprintf("[release %s, judged by its controller %s] ", release, judgedBy)
|
||||
}
|
||||
|
||||
// refWords is a ref as a line says it: a commit by its first eight characters, a branch whole.
|
||||
func refWords(ref string) string {
|
||||
if commitLike.MatchString(ref) {
|
||||
return short(ref)
|
||||
}
|
||||
return ref
|
||||
}
|
||||
|
||||
// commitLike is a ref that names a commit.
|
||||
var commitLike = regexp.MustCompile(`^[0-9a-f]{7,40}$`)
|
||||
|
||||
func sortedKeys(m map[string]string) []string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
// replaceGoFiles puts a package's Go files — never its tests — in place of another copy's.
|
||||
func replaceGoFiles(from, into string) error {
|
||||
old, err := filepath.Glob(filepath.Join(into, "*.go"))
|
||||
@@ -1136,8 +1772,7 @@ func judge(ctx context.Context, run, plain Runner, root string, inToolchain func
|
||||
say func(step, format string, args ...any)) (string, string, error) {
|
||||
bin := filepath.Join(root, "bin", "mesh-controller")
|
||||
build := func(dir string) error {
|
||||
_, err := run(ctx, root, "docker", inToolchain(filepath.Join(root, dir),
|
||||
[]string{"CGO_ENABLED=0", "GOFLAGS=-mod=vendor", "GOPROXY=off", "GOCACHE=" + filepath.Join(filepath.Dir(root), "go-cache")},
|
||||
_, err := run(ctx, root, "docker", inToolchain(filepath.Join(root, dir), nil,
|
||||
"go", "build", "-o", bin, "./cmd/mesh-controller")...)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,334 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// The builds a merge check makes for its merge (novox/hq ADR 0350 §1): every artifact of every module the delivery
|
||||
// plan moves, built from the merged tree the check judged into the scratch namespace, kept under its reuse key, so
|
||||
// the walk of the merge commit takes it instead of building again. Built after the check has judged, and only when
|
||||
// it did not fail: a change that fails is not merged, and nothing it built would be taken.
|
||||
|
||||
// CheckBuildSpec is one module a check builds for its merge, in the walk's order (Tier).
|
||||
type CheckBuildSpec struct {
|
||||
Module string
|
||||
// Repository, Path and Ref are where it is built from; Merged says from the merge the check made instead — the
|
||||
// module is built from the repository checked, at the branch merged into.
|
||||
Repository string
|
||||
Path string
|
||||
Ref string
|
||||
Merged bool
|
||||
Tier int
|
||||
// Skip says why it cannot be built as the walk would build it: not built, and nothing after it.
|
||||
Skip string
|
||||
}
|
||||
|
||||
// CheckBuilding is what a check builds with: where it publishes and keeps its builds, and what a build is handed.
|
||||
type CheckBuilding struct {
|
||||
Publish Publisher
|
||||
Scratch Scratch
|
||||
Held map[string]string
|
||||
Npmrc Npmrc
|
||||
Seats map[string]string
|
||||
// Named are the builds the build seat's verdicts of earlier checks of this pull request at this head name
|
||||
// (novox/hq issue 703): a build kept under a module's key is kept again, instead of built, only when one of them
|
||||
// names its key and every digest, and the build is this pull request's at this head. The record alone is never
|
||||
// enough.
|
||||
Named []NamedBuild
|
||||
}
|
||||
|
||||
// CheckBuilt is one module a check built for its merge, as its verdict lists it (§1): its key and its artifacts, or
|
||||
// why it was not built.
|
||||
type CheckBuilt struct {
|
||||
Module string
|
||||
Key string
|
||||
Made []catalogue.Built
|
||||
// NotBuilt says why it was not: no key, a build that failed, or a module after one of those.
|
||||
NotBuilt string
|
||||
Took time.Duration
|
||||
// Kept is the check whose build this check kept instead of building, its key unchanged (novox/hq issue 703):
|
||||
// the verdict names that build, by key and digest, as that check's. Empty when this check built it.
|
||||
Kept string
|
||||
}
|
||||
|
||||
// skipsTheBuilds is why a check builds nothing for its merge; empty when it builds.
|
||||
func skipsTheBuilds(v CheckVerdict) string {
|
||||
for _, l := range []*Layer{v.Gate, v.Repo} {
|
||||
if l != nil && (l.Verdict == "fail" || l.Verdict == "error") {
|
||||
return "the check did not pass, so nothing is built for its merge"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// buildForTheMerge builds each module the plan moves into the scratch namespace (makeForTheMerge), and keeps what it
|
||||
// built under its keys (keepForTheMerge): the order of a check that builds after it has judged.
|
||||
func buildForTheMerge(ctx context.Context, run Runner, spec CheckSpec, with CheckBuilding, workspace, tree, merge string,
|
||||
judged CheckVerdict, forge GitCredential, log Log, say func(step, format string, args ...any)) []CheckBuilt {
|
||||
out, kept := makeForTheMerge(ctx, run, spec, with, workspace, tree, merge, judged, forge, log, say)
|
||||
return keepForTheMerge(ctx, spec, with, out, kept, say)
|
||||
}
|
||||
|
||||
// makeForTheMerge builds each module the plan moves, tier by tier, into the scratch namespace — **the modules of a
|
||||
// tier at the same time** (novox/hq issue 716): none of them stands on another, and a tier waits only for the ones
|
||||
// before it, whose builds it is handed. A module standing after one not built is not built either: what it stands on
|
||||
// in the walk is not what it would stand on here. tree is the check's copy of the merge and merge the merge commit it
|
||||
// made there; workspace is where each build's working copy is made, one directory per module, which the host's
|
||||
// container engine sees at the same path (novox/hq issue 713). Answers what each module came to, in the plan's order,
|
||||
// and the builds to keep — nothing is kept here.
|
||||
func makeForTheMerge(ctx context.Context, run Runner, spec CheckSpec, with CheckBuilding, workspace, tree, merge string,
|
||||
judged CheckVerdict, forge GitCredential, log Log, say func(step, format string, args ...any)) ([]CheckBuilt, []CheckBuild) {
|
||||
if len(spec.Builds) == 0 || with.Publish == nil || with.Scratch == nil {
|
||||
return nil, nil
|
||||
}
|
||||
// What an earlier check left, ended before it could remove it, is not built on.
|
||||
place := filepath.Join(workspace, "for-the-merge")
|
||||
_ = os.RemoveAll(place)
|
||||
defer os.RemoveAll(place)
|
||||
// One log, said to by builds running at once: a line at a time.
|
||||
var saying sync.Mutex
|
||||
if log != nil {
|
||||
inner := log
|
||||
log = func(step, message string) {
|
||||
saying.Lock()
|
||||
defer saying.Unlock()
|
||||
inner(step, message)
|
||||
}
|
||||
}
|
||||
inner := say
|
||||
say = func(step, format string, args ...any) {
|
||||
saying.Lock()
|
||||
defer saying.Unlock()
|
||||
inner(step, format, args...)
|
||||
}
|
||||
ref := merge
|
||||
if ref == "" {
|
||||
ref = spec.Ref
|
||||
}
|
||||
held := map[string]string{}
|
||||
for k, v := range with.Held {
|
||||
held[k] = v
|
||||
}
|
||||
out := make([]CheckBuilt, len(spec.Builds))
|
||||
results := make([]Result, len(spec.Builds))
|
||||
var kept []CheckBuild
|
||||
stopAt, stopWhy := -1, ""
|
||||
for start := 0; start < len(spec.Builds); {
|
||||
tier := spec.Builds[start].Tier
|
||||
end := start
|
||||
for end < len(spec.Builds) && spec.Builds[end].Tier == tier {
|
||||
end++
|
||||
}
|
||||
// What this tier's builds are handed: the builds of the tiers before it, read alone while they run.
|
||||
handed := map[string]string{}
|
||||
for k, v := range held {
|
||||
handed[k] = v
|
||||
}
|
||||
var building sync.WaitGroup
|
||||
for i := start; i < end; i++ {
|
||||
b := spec.Builds[i]
|
||||
if stopAt >= 0 && b.Tier > stopAt {
|
||||
out[i] = CheckBuilt{Module: b.Module, NotBuilt: stopWhy}
|
||||
continue
|
||||
}
|
||||
if b.Skip != "" {
|
||||
say("build", "%s not built for the merge: %s", b.Module, b.Skip)
|
||||
out[i] = CheckBuilt{Module: b.Module, NotBuilt: b.Skip}
|
||||
continue
|
||||
}
|
||||
repository, at := b.Repository, b.Ref
|
||||
if b.Merged {
|
||||
repository, at = tree, ref
|
||||
}
|
||||
name := b.Module
|
||||
if !safeName.MatchString(name) {
|
||||
name = fmt.Sprintf("module-%d", i)
|
||||
}
|
||||
building.Add(1)
|
||||
go func(i int, b CheckBuildSpec, repository, at, dir string) {
|
||||
defer building.Done()
|
||||
began := time.Now()
|
||||
say("build", "for the merge: %s (tier %d)", b.Module, b.Tier)
|
||||
var mirrored []string
|
||||
result, err := build(ctx, run, scratchPublisher{to: with.Publish}, repository, b.Path, at, dir, handed,
|
||||
with.Npmrc, forge, log, &mirrored, &Reuse{keyFirst: true, Scratch: with.Scratch, Named: with.Named,
|
||||
pull: PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number}, head: judged.Head},
|
||||
with.Seats)
|
||||
built := CheckBuilt{Module: b.Module, Key: result.ReuseKey, Took: time.Since(began)}
|
||||
switch {
|
||||
case errors.Is(err, errNoKey):
|
||||
built.NotBuilt = err.Error()
|
||||
case err != nil:
|
||||
built.NotBuilt = "its build failed: " + err.Error()
|
||||
case result.ReuseKey == "":
|
||||
built.NotBuilt = "it has no reuse key: " + result.NoKey
|
||||
}
|
||||
out[i], results[i] = built, result
|
||||
}(i, b, repository, at, filepath.Join(place, name))
|
||||
}
|
||||
building.Wait()
|
||||
for i := start; i < end; i++ {
|
||||
b, built, result := spec.Builds[i], &out[i], results[i]
|
||||
if built.NotBuilt != "" {
|
||||
if built.Key != "" || built.Took > 0 {
|
||||
say("build", "%s not built for the merge: %s", b.Module, built.NotBuilt)
|
||||
}
|
||||
if stopAt < 0 || b.Tier < stopAt {
|
||||
stopAt, stopWhy = b.Tier, "it stands after "+b.Module+", which was not built for the merge"
|
||||
}
|
||||
continue
|
||||
}
|
||||
built.Made = result.Built
|
||||
for _, m := range result.Built {
|
||||
held[b.Module+"/"+m.Name] = m.Reference
|
||||
}
|
||||
if result.Reused != nil {
|
||||
// **Kept, not built** (novox/hq issue 703): the earlier check's build, named as that check's, so the
|
||||
// walk takes it as the verdict names it; this check's document names it too, so it stays while this
|
||||
// check does (ADR 0350 §5).
|
||||
built.Kept = result.Reused.From.Check
|
||||
say("build", "%s kept for the merge under %s: %s", b.Module, twelve(result.ReuseKey[3:]),
|
||||
result.Reused.From.Said())
|
||||
kept = append(kept, result.Reused.From)
|
||||
continue
|
||||
}
|
||||
say("build", "%s built for the merge under %s (%s)", b.Module, twelve(result.ReuseKey[3:]),
|
||||
built.Took.Round(time.Second))
|
||||
if len(result.Built) > 0 {
|
||||
kept = append(kept, CheckBuild{Key: result.ReuseKey, Module: b.Module, Parts: result.ReuseParts,
|
||||
Made: result.Built, Pull: PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number},
|
||||
Head: judged.Head, Trunk: judged.Trunk, Check: spec.ID, At: time.Now().UTC()})
|
||||
}
|
||||
}
|
||||
start = end
|
||||
}
|
||||
return out, kept
|
||||
}
|
||||
|
||||
// keepForTheMerge keeps a check's builds under their keys, its own document naming them, so the walk of its merge
|
||||
// finds them; what cannot be kept is let go of, said, and each module says it — never the check's verdict.
|
||||
func keepForTheMerge(ctx context.Context, spec CheckSpec, with CheckBuilding, out []CheckBuilt, kept []CheckBuild,
|
||||
say func(step, format string, args ...any)) []CheckBuilt {
|
||||
if len(kept) == 0 {
|
||||
return out
|
||||
}
|
||||
if err := with.Scratch.Keep(ctx, PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number}, spec.ID,
|
||||
time.Now().UTC(), kept); err != nil {
|
||||
// Built and not findable: the walk builds, and says no check built it. Said, never the check's verdict.
|
||||
say("build", "the builds for the merge could not be kept, and the walk will build: %v", err)
|
||||
if err := with.Scratch.Discard(ctx, kept); err != nil {
|
||||
say("build", "and what they made could not all be let go of: %v", err)
|
||||
}
|
||||
for i := range out {
|
||||
if out[i].NotBuilt == "" {
|
||||
out[i].NotBuilt = fmt.Sprintf("built, and could not be kept: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// discardForTheMerge lets go of builds a check made beside its tests when the check did not pass (novox/hq issue 716):
|
||||
// a change that fails is not merged, and nothing it built is kept. A build another check names stays (Discard).
|
||||
func discardForTheMerge(ctx context.Context, with CheckBuilding, kept []CheckBuild, why string,
|
||||
say func(step, format string, args ...any)) {
|
||||
say("check", "%s: the %d build(s) made beside it are let go of", why, len(kept))
|
||||
if len(kept) == 0 {
|
||||
return
|
||||
}
|
||||
if err := with.Scratch.Discard(ctx, kept); err != nil {
|
||||
say("build", "what they made could not all be let go of, and goes with the week's sweep: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// besideTheTests are a check's builds for its merge running while its gate and its own check run (novox/hq issue
|
||||
// 716): started once every clone is made, waited for when the check has judged, kept when it passed and let go of
|
||||
// when it did not.
|
||||
type besideTheTests struct {
|
||||
spec CheckSpec
|
||||
with CheckBuilding
|
||||
cancel context.CancelFunc
|
||||
done chan struct{}
|
||||
place string
|
||||
out []CheckBuilt
|
||||
kept []CheckBuild
|
||||
began time.Time
|
||||
settled bool
|
||||
say func(step, format string, args ...any)
|
||||
}
|
||||
|
||||
// buildsBesideTheTests is where a check's builds run beside its tests: a directory made for this check under the judge
|
||||
// cache's homes/ (ADR 0358 §6), on the host path the build agent declares — so the host's container engine mounts what
|
||||
// the agent sees (issue 713) — and never mounted by a check's container, so nothing a pull request's script runs can
|
||||
// change a build while it is made (issue 703). Empty, and the builds wait for the check, without a judge cache.
|
||||
func buildsBesideTheTests(cache, workspace, id string) (string, error) {
|
||||
if cache == "" {
|
||||
return "", nil
|
||||
}
|
||||
if withinDir(workspace, cache) {
|
||||
return "", fmt.Errorf("the judge cache %s is inside the workspace %s, which a check's containers mount: "+
|
||||
"nothing is built for the merge where a pull request could reach it", cache, workspace)
|
||||
}
|
||||
if !safeName.MatchString(id) {
|
||||
return "", fmt.Errorf("the check's id %q cannot name its place in the judge cache", id)
|
||||
}
|
||||
place := filepath.Join(cache, "homes", id+"-builds")
|
||||
if err := os.RemoveAll(place); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.MkdirAll(place, 0o700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return place, nil
|
||||
}
|
||||
|
||||
// startBesideTheTests starts the builds for the merge in place, from merged, the check's copy of the merge.
|
||||
func startBesideTheTests(ctx context.Context, run Runner, spec CheckSpec, place, merged string, judged CheckVerdict,
|
||||
forge GitCredential, log Log, say func(step, format string, args ...any)) *besideTheTests {
|
||||
building, cancel := context.WithCancel(ctx)
|
||||
b := &besideTheTests{spec: spec, with: *spec.BuildWith, cancel: cancel, done: make(chan struct{}), place: place,
|
||||
began: time.Now(), say: say}
|
||||
say("check", "building what the merge moves (%d module(s)) beside its tests", len(spec.Builds))
|
||||
go func() {
|
||||
defer close(b.done)
|
||||
b.out, b.kept = makeForTheMerge(building, run, spec, b.with, place, merged, judged.Merge, judged, forge, log, say)
|
||||
}()
|
||||
return b
|
||||
}
|
||||
|
||||
// settle waits for the builds and keeps them when the check passed; a check that did not pass lets go of them, and
|
||||
// its verdict names none.
|
||||
func (b *besideTheTests) settle(ctx context.Context, verdict CheckVerdict) []CheckBuilt {
|
||||
<-b.done
|
||||
b.settled = true
|
||||
defer os.RemoveAll(b.place)
|
||||
if why := skipsTheBuilds(verdict); why != "" {
|
||||
discardForTheMerge(ctx, b.with, b.kept, why, b.say)
|
||||
return nil
|
||||
}
|
||||
built := keepForTheMerge(ctx, b.spec, b.with, b.out, b.kept, b.say)
|
||||
b.say("check", "built for the merge: %d of %d module(s), beside the tests (%s)", builtCount(built), len(built),
|
||||
time.Since(b.began).Round(time.Second))
|
||||
return built
|
||||
}
|
||||
|
||||
// abandon ends builds a check that ended without a verdict leaves, and lets go of what they made.
|
||||
func (b *besideTheTests) abandon() {
|
||||
if b.settled {
|
||||
return
|
||||
}
|
||||
b.cancel()
|
||||
<-b.done
|
||||
defer os.RemoveAll(b.place)
|
||||
letGo, done := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer done()
|
||||
discardForTheMerge(letGo, b.with, b.kept, "the check ended without a verdict", b.say)
|
||||
}
|
||||
@@ -210,3 +210,53 @@ func TestACheckMergeRunsNoHook(t *testing.T) {
|
||||
t.Fatalf("a hook ran when the check made its merge: %s", said)
|
||||
}
|
||||
}
|
||||
|
||||
// What a check's tests change in the workspace never reaches what it builds for the merge (novox/hq issue 703): the
|
||||
// builds are made from a copy of the merge, outside the workspace, cloned before anything of the change runs.
|
||||
func TestWhatTheTestsChangeInTheWorkspaceNeverReachesTheBuild(t *testing.T) {
|
||||
repo, head, _ := aTrunkAndABranch(t, map[string]string{"main.go": "package main // the merge\n"},
|
||||
map[string]string{"trunk-only": "t"}, map[string]string{"head-only": "h"})
|
||||
workspace := t.TempDir()
|
||||
tree := filepath.Join(workspace, "check", "checked")
|
||||
if out, err := exec.Command("git", "clone", "--quiet", repo, tree).CombinedOutput(); err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
dir, cloned, err := checkoutForTheBuilds(t.Context(), Command, tree, head, workspace)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
if withinDir(workspace, cloned) {
|
||||
t.Fatalf("the builds' checkout %s is inside the workspace %s the tests mount", cloned, workspace)
|
||||
}
|
||||
|
||||
// The tests rewrite the tree, its branch and every object it holds.
|
||||
if err := os.WriteFile(filepath.Join(tree, "main.go"), []byte("package main // planted\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := filepath.WalkDir(filepath.Join(tree, ".git", "objects"), func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() {
|
||||
return err
|
||||
}
|
||||
_ = os.Chmod(path, 0o644)
|
||||
return os.WriteFile(path, []byte("planted"), 0o644)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The build clones the merge from its own checkout, as buildForTheMerge does, and reads what the merge holds.
|
||||
build := filepath.Join(t.TempDir(), "source")
|
||||
if out, err := exec.Command("git", "clone", "--quiet", cloned, build).CombinedOutput(); err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
if out, err := exec.Command("git", "-C", build, "checkout", "--quiet", head).CombinedOutput(); err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
body, err := os.ReadFile(filepath.Join(build, "main.go"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(body) != "package main // the merge\n" {
|
||||
t.Fatalf("the build read %q: what the tests wrote reached it", body)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -590,3 +590,15 @@ func TestARepositoryInTwoLanguagesIsCheckedInBoth(t *testing.T) {
|
||||
t.Errorf("a part in a toolchain the mesh does not hold answered %+v — never a pass", layer)
|
||||
}
|
||||
}
|
||||
|
||||
// A branch tried before its pull request may narrow what the repository's script runs (novox/hq issue 686); a
|
||||
// pull request's check is handed nothing, and runs whole.
|
||||
func TestOnlyATrysSelectionReachesTheScript(t *testing.T) {
|
||||
if env := selection(CheckSpec{Number: 7}); len(env) != 0 {
|
||||
t.Fatalf("a pull request's check was handed a selection: %v", env)
|
||||
}
|
||||
env := selection(CheckSpec{Tests: "^TestX$", StoreOnly: true})
|
||||
if len(env) != 2 || env[0] != "MESH_CHECK_TESTS=^TestX$" || env[1] != "MESH_CHECK_STORE_ONLY=1" {
|
||||
t.Fatalf("a try's selection: %v", env)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// **A built image's bases are linked into its repository before it is pushed** (novox/hq issue 694).
|
||||
//
|
||||
// A push reads every blob of the image out of this machine's store, unless the repository it goes to
|
||||
// already holds that blob: the runtime asks the registry first, blob by blob, and reads locally only
|
||||
// what the registry lacks. A runtime keeping its images in containerd's store — the default for a
|
||||
// fresh install from docker 29 on — does not always hold the compressed layers of the base an image
|
||||
// was built FROM: a layer whose unpacked form it already had is never fetched. The push of such an
|
||||
// image then fails on a layer this machine never had ("not all of them are available locally"), while
|
||||
// the very bytes sit in this registry, under the repository the base was mirrored into.
|
||||
//
|
||||
// So the base's blobs are mounted into the image's repository first. A mount moves no bytes, and once
|
||||
// the repository holds a blob the push asks this machine for nothing of it — whichever store the
|
||||
// runtime keeps. A runtime that holds every layer (the classic store) is not changed by this: its push
|
||||
// finds the same blobs already there.
|
||||
|
||||
// BaseLender links what a build's bases hold into the repository its image is about to be pushed to.
|
||||
type BaseLender interface {
|
||||
// LendBases mounts every blob of each base this registry holds — for the platform this machine
|
||||
// builds — into `repository`, and answers how many it mounted.
|
||||
LendBases(ctx context.Context, repository string, bases []string) (int, error)
|
||||
}
|
||||
|
||||
// platformed is an index's entry: a manifest and the platform it is for.
|
||||
type platformed struct {
|
||||
descriptor
|
||||
Platform *struct {
|
||||
OS string `json:"os"`
|
||||
Architecture string `json:"architecture"`
|
||||
} `json:"platform"`
|
||||
}
|
||||
|
||||
// LendBases mounts each base's blobs into the repository. A base this registry does not hold under its
|
||||
// repository — named by tag, or held somewhere else — lends nothing and is no error: the push reads it
|
||||
// from this machine's store as before.
|
||||
func (r Registry) LendBases(ctx context.Context, repository string, bases []string) (int, error) {
|
||||
mounted := 0
|
||||
for _, base := range bases {
|
||||
name, digest, pinned := strings.Cut(base, "@")
|
||||
if !pinned || !strings.HasPrefix(digest, "sha256:") {
|
||||
continue
|
||||
}
|
||||
where, err := parseReference(name + "@" + digest)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
from := where.repository
|
||||
if from == repository {
|
||||
continue
|
||||
}
|
||||
blobs, held, err := r.blobsOf(ctx, from, digest)
|
||||
if err != nil {
|
||||
return mounted, fmt.Errorf("reading %s: %w", base, err)
|
||||
}
|
||||
if !held {
|
||||
continue
|
||||
}
|
||||
for _, blob := range blobs {
|
||||
did, err := r.mount(ctx, repository, from, blob)
|
||||
if err != nil {
|
||||
return mounted, fmt.Errorf("linking %s of %s into %s: %w", blob, base, repository, err)
|
||||
}
|
||||
if did {
|
||||
mounted++
|
||||
}
|
||||
}
|
||||
}
|
||||
return mounted, nil
|
||||
}
|
||||
|
||||
// blobsOf is the config and layers of the manifest under repository@digest — for an index, of each
|
||||
// manifest it names for this machine's platform. False when the registry holds no such manifest there.
|
||||
func (r Registry) blobsOf(ctx context.Context, repository, digest string) ([]string, bool, error) {
|
||||
body, held, err := r.manifest(ctx, repository, digest)
|
||||
if err != nil || !held {
|
||||
return nil, held, err
|
||||
}
|
||||
var document struct {
|
||||
Manifests []platformed `json:"manifests"`
|
||||
Config *descriptor `json:"config"`
|
||||
Layers []descriptor `json:"layers"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &document); err != nil {
|
||||
return nil, false, fmt.Errorf("%s@%s is not a manifest: %w", repository, digest, err)
|
||||
}
|
||||
var blobs []string
|
||||
if document.Config != nil {
|
||||
blobs = append(blobs, document.Config.Digest)
|
||||
}
|
||||
for _, l := range document.Layers {
|
||||
blobs = append(blobs, l.Digest)
|
||||
}
|
||||
for _, m := range document.Manifests {
|
||||
if m.Platform == nil || m.Platform.OS != runtime.GOOS || m.Platform.Architecture != runtime.GOARCH {
|
||||
continue
|
||||
}
|
||||
more, held, err := r.blobsOf(ctx, repository, m.Digest)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if held {
|
||||
blobs = append(blobs, more...)
|
||||
}
|
||||
}
|
||||
return blobs, true, nil
|
||||
}
|
||||
|
||||
// manifest reads one manifest document from this registry; false when it is not there.
|
||||
func (r Registry) manifest(ctx context.Context, repository, digest string) ([]byte, bool, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet,
|
||||
"http://"+r.Address+"/v2/"+repository+"/manifests/"+digest, nil)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
request.Header.Set("Accept", manifestAccept)
|
||||
response, err := r.client().Do(request)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("cannot reach the registry at %s: %w", r.Address, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode == http.StatusNotFound {
|
||||
return nil, false, nil
|
||||
}
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return nil, false, fmt.Errorf("%s/%s@%s: %s", r.Address, repository, digest, response.Status)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(response.Body, 4<<20))
|
||||
return body, err == nil, err
|
||||
}
|
||||
|
||||
// mount links one blob from one repository into another; false when the repository already held it.
|
||||
func (r Registry) mount(ctx context.Context, repository, from, digest string) (bool, error) {
|
||||
base := "http://" + r.Address + "/v2/" + repository
|
||||
if there, err := r.has(ctx, base+"/blobs/"+digest); err != nil {
|
||||
return false, err
|
||||
} else if there {
|
||||
return false, nil
|
||||
}
|
||||
start, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
base+"/blobs/uploads/?mount="+digest+"&from="+from, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
begun, err := r.client().Do(start)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("cannot reach the registry at %s: %w", r.Address, err)
|
||||
}
|
||||
begun.Body.Close()
|
||||
if begun.StatusCode == http.StatusCreated {
|
||||
return true, nil
|
||||
}
|
||||
if begun.StatusCode == http.StatusAccepted {
|
||||
// Not mounted, and an upload begun in its place: let go of it, so nothing is left half-open.
|
||||
if location := begun.Header.Get("Location"); location != "" {
|
||||
if strings.HasPrefix(location, "/") {
|
||||
location = "http://" + r.Address + location
|
||||
}
|
||||
if cancel, err := http.NewRequestWithContext(ctx, http.MethodDelete, location, nil); err == nil {
|
||||
if done, err := r.client().Do(cancel); err == nil {
|
||||
done.Body.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, fmt.Errorf("%s did not mount it from %s", r.Address, from)
|
||||
}
|
||||
return false, fmt.Errorf("%s answered %s when asked to mount it from %s", r.Address, begun.Status, from)
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http/httptest"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A built image's bases are linked into its repository before it is pushed (novox/hq issue 694): a
|
||||
// runtime keeping containerd's image store may not hold a base layer it never had to fetch, and its push
|
||||
// asks this machine only for what the repository lacks.
|
||||
|
||||
// aBaseForThisMachine puts an index into the registry under `repository`: one manifest for this
|
||||
// machine's platform, one for another, and answers the index's digest and each manifest's blobs.
|
||||
func aBaseForThisMachine(t *testing.T, dst *aRegistryOfRepositories, repository string) (string, []string, []string) {
|
||||
t.Helper()
|
||||
put := func(arch string) (string, []string) {
|
||||
config := []byte(`{"architecture":"` + arch + `"}`)
|
||||
layer := []byte(arch + " layer bytes")
|
||||
for _, b := range [][]byte{config, layer} {
|
||||
dst.blobs[digestOf(b)] = b
|
||||
dst.link(repository, digestOf(b))
|
||||
}
|
||||
m, _ := json.Marshal(map[string]any{
|
||||
"schemaVersion": 2, "mediaType": mediaManifestOCI,
|
||||
"config": map[string]any{"mediaType": "application/vnd.oci.image.config.v1+json", "digest": digestOf(config), "size": len(config)},
|
||||
"layers": []map[string]any{{"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": digestOf(layer), "size": len(layer)}},
|
||||
})
|
||||
dst.manifests[repository+"@"+digestOf(m)] = m
|
||||
return digestOf(m), []string{digestOf(config), digestOf(layer)}
|
||||
}
|
||||
other := "s390x"
|
||||
if runtime.GOARCH == other {
|
||||
other = "amd64"
|
||||
}
|
||||
mine, mineBlobs := put(runtime.GOARCH)
|
||||
theirs, theirBlobs := put(other)
|
||||
index, _ := json.Marshal(map[string]any{
|
||||
"schemaVersion": 2, "mediaType": mediaIndexOCI,
|
||||
"manifests": []map[string]any{
|
||||
{"mediaType": mediaManifestOCI, "digest": mine, "size": 1, "platform": map[string]string{"os": runtime.GOOS, "architecture": runtime.GOARCH}},
|
||||
{"mediaType": mediaManifestOCI, "digest": theirs, "size": 1, "platform": map[string]string{"os": "linux", "architecture": other}},
|
||||
},
|
||||
})
|
||||
dst.manifests[repository+"@"+digestOf(index)] = index
|
||||
return digestOf(index), mineBlobs, theirBlobs
|
||||
}
|
||||
|
||||
func TestABuildsBaseIsLinkedIntoItsRepositoryBeforeThePush(t *testing.T) {
|
||||
dst := newRegistryOfRepositories()
|
||||
server := httptest.NewServer(dst.handler())
|
||||
defer server.Close()
|
||||
address := strings.TrimPrefix(server.URL, "http://")
|
||||
r := Registry{Address: address}
|
||||
|
||||
const from = "upstream/docker.io/library/golang"
|
||||
index, mine, theirs := aBaseForThisMachine(t, dst, from)
|
||||
bases := []string{
|
||||
address + "/" + from + "@" + index,
|
||||
// Not bases this registry holds: a package version, a tag, an image held nowhere here.
|
||||
"@novox/mesh-sdk@1.2.3",
|
||||
"golang:1-alpine",
|
||||
"elsewhere.invalid/thing@sha256:" + strings.Repeat("e", 64),
|
||||
}
|
||||
n, err := r.LendBases(context.Background(), "mesh-tools-go/build", bases)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Every blob of this machine's platform is in the image's repository, so a push asks this machine's
|
||||
// store for none of them; another platform's are not, since nothing built here stands on them.
|
||||
for _, blob := range mine {
|
||||
if !dst.links["mesh-tools-go/build"][blob] {
|
||||
t.Fatalf("the base's blob %s is not in the image's repository", blob)
|
||||
}
|
||||
}
|
||||
for _, blob := range theirs {
|
||||
if dst.links["mesh-tools-go/build"][blob] {
|
||||
t.Fatalf("another platform's blob %s was linked", blob)
|
||||
}
|
||||
}
|
||||
if n != len(mine) || dst.mounts != len(mine) || dst.uploads != 0 {
|
||||
t.Fatalf("%d linked, %d mounts, %d uploads; want %d mounts and no bytes moved", n, dst.mounts, dst.uploads, len(mine))
|
||||
}
|
||||
|
||||
// Asked again, nothing is mounted twice.
|
||||
if n, err := r.LendBases(context.Background(), "mesh-tools-go/build", bases); err != nil || n != 0 {
|
||||
t.Fatalf("a second lending mounted %d: %v", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A registry that will not mount is said, not hidden: the push is still tried, from the store.
|
||||
func TestABaseTheRegistryWillNotMountIsAnError(t *testing.T) {
|
||||
dst := newRegistryOfRepositories()
|
||||
dst.noMount = true
|
||||
server := httptest.NewServer(dst.handler())
|
||||
defer server.Close()
|
||||
address := strings.TrimPrefix(server.URL, "http://")
|
||||
index, _, _ := aBaseForThisMachine(t, dst, "upstream/thing")
|
||||
_, err := Registry{Address: address}.LendBases(context.Background(), "hello/server",
|
||||
[]string{address + "/upstream/thing@" + index})
|
||||
if err == nil || !strings.Contains(err.Error(), "did not mount") {
|
||||
t.Fatalf("a refused mount passed silently: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// lending records what a build lent, and when, against the push.
|
||||
type lending struct {
|
||||
*mirroring
|
||||
order []string
|
||||
}
|
||||
|
||||
func (l *lending) LendBases(_ context.Context, repository string, bases []string) (int, error) {
|
||||
l.order = append(l.order, "lend "+repository+" "+strings.Join(bases, " "))
|
||||
return len(bases), nil
|
||||
}
|
||||
|
||||
func (l *lending) PublishImage(ctx context.Context, localTag, repository string) (string, error) {
|
||||
l.order = append(l.order, "push "+repository)
|
||||
return l.mirroring.PublishImage(ctx, localTag, repository)
|
||||
}
|
||||
|
||||
func TestAnImageIsPushedOnlyAfterItsBasesAreLent(t *testing.T) {
|
||||
r, workspace := aRepository(t, anImage, map[string]string{"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}"})
|
||||
r.contents["modules/bus/module.json"] = anImage
|
||||
l := &lending{mirroring: &mirroring{recorded: r, at: "127.0.0.1:5000"}}
|
||||
if _, err := Build(context.Background(), r.run, l, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace,
|
||||
nil, Npmrc{}, GitCredential{}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{
|
||||
"lend bus/server 127.0.0.1:5000/upstream/docker.io/vendor/server@sha256:" + strings.Repeat("1", 64),
|
||||
"push bus/server",
|
||||
}
|
||||
if strings.Join(l.order, "\n") != strings.Join(want, "\n") {
|
||||
t.Fatalf("the build did:\n%s\nwant:\n%s", strings.Join(l.order, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,863 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **A pull request into a release branch is judged by the controller of that branch** (novox/hq ADR 0358, issue
|
||||
// 704): the build seat's check builds the judge from mesh-controller read at the release branch — with mesh-host's
|
||||
// validator of that branch when mesh-host has one — reads every repository beside it at that branch where it has one
|
||||
// and at its trunk otherwise, and names the branch, the judge's commit and each repository read with its commit.
|
||||
|
||||
const releaseBranch = "feat/57-config-and-provisioning"
|
||||
|
||||
// gitInRepo runs git in a repository made for a test.
|
||||
func gitInRepo(t *testing.T, dir string, args ...string) string {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd.Dir = dir
|
||||
cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org",
|
||||
"GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org")
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("git %v: %v\n%s", args, err, out)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
// commitFiles writes files into a repository and commits them on the branch checked out.
|
||||
func commitFiles(t *testing.T, dir string, files map[string]string) string {
|
||||
t.Helper()
|
||||
for name, body := range files {
|
||||
if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
gitInRepo(t, dir, "add", "-A")
|
||||
gitInRepo(t, dir, "commit", "--quiet", "-m", "x")
|
||||
return gitInRepo(t, dir, "rev-parse", "HEAD")
|
||||
}
|
||||
|
||||
// aRepositoryWithARelease is a repository whose main holds mainFiles and, when releaseFiles is not nil, a release
|
||||
// branch over them holding releaseFiles. It answers the repository, main's head and the release branch's head.
|
||||
func aRepositoryWithARelease(t *testing.T, mainFiles, releaseFiles map[string]string) (string, string, string) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
gitInRepo(t, dir, "init", "--quiet", "-b", "main")
|
||||
main := commitFiles(t, dir, mainFiles)
|
||||
if releaseFiles == nil {
|
||||
return dir, main, ""
|
||||
}
|
||||
gitInRepo(t, dir, "checkout", "--quiet", "-b", releaseBranch)
|
||||
head := commitFiles(t, dir, releaseFiles)
|
||||
gitInRepo(t, dir, "checkout", "--quiet", "main")
|
||||
return dir, main, head
|
||||
}
|
||||
|
||||
// aJudgeOf is a controller that judges as told: its gate passes, saying who it is and the word of the node-engine
|
||||
// validator it vendors — the one it was built with.
|
||||
func aJudgeOf(who string) map[string]string {
|
||||
return map[string]string{
|
||||
"go.mod": "module example.org/judge\n\ngo 1.22\n\nrequire github.com/novox/mesh-host v0.0.0\n",
|
||||
"vendor/modules.txt": "# github.com/novox/mesh-host v0.0.0\n## explicit; go 1.22\ngithub.com/novox/mesh-host/validate\n",
|
||||
"vendor/github.com/novox/mesh-host/validate/v.go": "package validate\n\nconst Word = \"the vendored validator\"\n",
|
||||
"vendor/github.com/novox/mesh-host/internal/declaration/d.go": "package declaration\n",
|
||||
"cmd/mesh-controller/main.go": `package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/novox/mesh-host/validate"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) == 2 && os.Args[1] == "merge-gate" {
|
||||
fmt.Println("usage: merge-gate --facts <file> --store <postgres>")
|
||||
os.Exit(2)
|
||||
}
|
||||
fmt.Printf("{\"verdict\":\"pass\",\"summary\":\"%s with %s\"}\n", "` + who + `", validate.Word)
|
||||
}
|
||||
`,
|
||||
}
|
||||
}
|
||||
|
||||
// aHostSaying is the node-engine's validator, saying a word.
|
||||
func aHostSaying(word string) map[string]string {
|
||||
return map[string]string{
|
||||
"validate/v.go": "package validate\n\nconst Word = \"" + word + "\"\n",
|
||||
"internal/declaration/d.go": "package declaration\n",
|
||||
"validate/v_test.go": "package validate\n",
|
||||
"internal/declaration/x_test.go": "package declaration\n",
|
||||
}
|
||||
}
|
||||
|
||||
// aRuntimeThatBuildsTheJudge puts a container runtime on PATH that runs every container's command on this machine,
|
||||
// in the directory and with the environment the check gives it — the judge's build too — and a store and a bus that
|
||||
// answer. It answers the file each `go` the check runs is written to.
|
||||
func aRuntimeThatBuildsTheJudge(t *testing.T) string {
|
||||
t.Helper()
|
||||
if _, err := exec.LookPath("go"); err != nil {
|
||||
t.Skip("no Go toolchain on this machine to build a judge with")
|
||||
}
|
||||
bus, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { bus.Close() })
|
||||
go func() {
|
||||
for {
|
||||
c, err := bus.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
c.Close()
|
||||
}
|
||||
}()
|
||||
dir := t.TempDir()
|
||||
built := filepath.Join(dir, "built")
|
||||
runtime := `#!/bin/sh
|
||||
cmd="$1"; shift
|
||||
case "$cmd" in
|
||||
ps|rm|exec) exit 0 ;;
|
||||
volume)
|
||||
if [ -n "$VOLUMES" ]; then echo "$*" >> "$VOLUMES"; fi
|
||||
exit 0 ;;
|
||||
port) echo "` + bus.Addr().String() + `"; exit 0 ;;
|
||||
run)
|
||||
if [ -n "$RUNS" ]; then echo "$*" >> "$RUNS"; fi
|
||||
workdir=""; detached=""
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
-d) detached=1; shift ;;
|
||||
--rm) shift ;;
|
||||
--workdir) workdir="$2"; shift 2 ;;
|
||||
--env) export "$2"; shift 2 ;;
|
||||
--network|--volume|--user|--label|--name|-p|-e) shift 2 ;;
|
||||
*) break ;;
|
||||
esac
|
||||
done
|
||||
if [ -n "$detached" ]; then echo a-container; exit 0; fi
|
||||
shift
|
||||
if [ "$1" = go ]; then echo "GOCACHE=$GOCACHE GOENV=$GOENV $*" >> "` + built + `"; fi
|
||||
if [ "$1" = go ] && [ "$2" = build ] && [ -n "$JUDGE_HOMES" ]; then echo "$HOME" >> "$JUDGE_HOMES"; fi
|
||||
if [ "$1" = go ] && [ "$2" = test ] && [ -n "$REPLAYS_CANNOT_START" ]; then
|
||||
echo "docker: Error response from daemon: invalid mount config for type bind" >&2; exit 125
|
||||
fi
|
||||
if [ "$1" = go ] && [ "$2" = test ] && [ -n "$REPLAYS" ]; then
|
||||
echo "HOME=$HOME GOCACHE=$GOCACHE GOMODCACHE=$GOMODCACHE GOENV=$GOENV GOFLAGS=$GOFLAGS DIR=$workdir" >> "$REPLAYS"
|
||||
fi
|
||||
cd "$workdir" && exec "$@" ;;
|
||||
esac
|
||||
exit 1
|
||||
`
|
||||
if err := os.WriteFile(filepath.Join(dir, "docker"), []byte(runtime), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
t.Setenv("GOTOOLCHAIN", "local")
|
||||
return built
|
||||
}
|
||||
|
||||
// builds is how many times the check ran go to build a judge.
|
||||
func builds(t *testing.T, file string) int {
|
||||
t.Helper()
|
||||
body, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return strings.Count(string(body), "go build")
|
||||
}
|
||||
|
||||
// theReleasedMesh is the core beside a release-branch check: a controller whose main and release branch judge
|
||||
// differently, a node-engine with a validator of each, and a lab with no release branch.
|
||||
func theReleasedMesh(t *testing.T) (map[string]Beside, string, string, string) {
|
||||
t.Helper()
|
||||
controller, _, controllerRelease := aRepositoryWithARelease(t, aJudgeOf("main's controller"),
|
||||
aJudgeOf("the release branch's controller"))
|
||||
host, _, hostRelease := aRepositoryWithARelease(t, aHostSaying("main's validator"),
|
||||
aHostSaying("the release branch's validator"))
|
||||
lab, labMain, _ := aRepositoryWithARelease(t, map[string]string{"README": "the lab"}, nil)
|
||||
return map[string]Beside{
|
||||
"mesh-controller": {Repository: controller, Ref: releaseBranch},
|
||||
"mesh-controller-main": {Repository: controller, Ref: "main"},
|
||||
"mesh-host": {Repository: host, Ref: releaseBranch, Else: "main"},
|
||||
"mesh-lab": {Repository: lab, Ref: releaseBranch, Else: "main"},
|
||||
}, controllerRelease, hostRelease, labMain
|
||||
}
|
||||
|
||||
// aPullRequestInto is a repository whose pull request's head is cut from base, with a change.
|
||||
func aPullRequestInto(t *testing.T, files map[string]string, base string, change map[string]string) (string, string) {
|
||||
t.Helper()
|
||||
repo, _, _ := aRepositoryWithARelease(t, files, map[string]string{"released": "r"})
|
||||
gitInRepo(t, repo, "checkout", "--quiet", "-b", "change", base)
|
||||
head := commitFiles(t, repo, change)
|
||||
gitInRepo(t, repo, "checkout", "--quiet", "main")
|
||||
return repo, head
|
||||
}
|
||||
|
||||
func TestACataloguePullRequestIntoAReleaseBranchIsJudgedByThatBranchsController(t *testing.T) {
|
||||
built := aRuntimeThatBuildsTheJudge(t)
|
||||
registry := aFactsRegistry(t)
|
||||
beside, controllerRelease, hostRelease, labMain := theReleasedMesh(t)
|
||||
catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`,
|
||||
CheckScript: "true\n"}, releaseBranch, map[string]string{"modules/gitea/index.ts": "x"})
|
||||
cache := t.TempDir()
|
||||
check := func(id string) CheckVerdict {
|
||||
t.Helper()
|
||||
v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: catalogue, Ref: head, Owner: "novox",
|
||||
Repo: "mesh-catalog", Number: 286, Base: releaseBranch, Release: releaseBranch, JudgeCache: cache,
|
||||
Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"}, Beside: beside, Toolchain: "golang"},
|
||||
t.TempDir(), registry, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
v := check("check-704-catalogue")
|
||||
// The release branch's controller judged, with mesh-host's validator of the release branch.
|
||||
if v.Gate == nil || v.Gate.Verdict != "pass" ||
|
||||
!strings.HasSuffix(v.Gate.Summary, "the release branch's controller with the release branch's validator") {
|
||||
t.Fatalf("the gate was not judged by the release branch's controller with its node-engine's validator: %+v\n%s",
|
||||
v.Gate, v.Report)
|
||||
}
|
||||
// It names the release branch, the judge's commit, and each repository read beside it at its branch and commit.
|
||||
mark := "[release " + releaseBranch + ", judged by its controller " + controllerRelease[:12] + "] "
|
||||
if !strings.HasPrefix(v.Gate.Summary, mark) || !strings.HasPrefix(v.Repo.Summary, mark) {
|
||||
t.Fatalf("the statuses do not name the release branch and its judge first:\n%s\n%s", v.Gate.Summary, v.Repo.Summary)
|
||||
}
|
||||
if v.Release != releaseBranch || v.JudgedBy != controllerRelease[:12] {
|
||||
t.Fatalf("the verdict names release %q judged by %q, not %s by %s", v.Release, v.JudgedBy, releaseBranch,
|
||||
controllerRelease[:12])
|
||||
}
|
||||
if v.BesideAt["mesh-host"] != releaseBranch || v.Beside["mesh-host"] != hostRelease[:12] {
|
||||
t.Fatalf("mesh-host was read at %q (%s), not its release branch at %s", v.BesideAt["mesh-host"],
|
||||
v.Beside["mesh-host"], hostRelease[:12])
|
||||
}
|
||||
// A repository with no release branch is read at its trunk, and says so.
|
||||
if v.BesideAt["mesh-lab"] != "main" || v.Beside["mesh-lab"] != labMain[:12] {
|
||||
t.Fatalf("the lab, with no release branch, was read at %q (%s), not main at %s", v.BesideAt["mesh-lab"],
|
||||
v.Beside["mesh-lab"], labMain[:12])
|
||||
}
|
||||
if !strings.Contains(v.Report, "read beside it: ") || !strings.Contains(v.Report, "mesh-host "+releaseBranch+" at "+hostRelease[:12]) {
|
||||
t.Fatalf("the report does not say what was read beside it:\n%s", v.Report)
|
||||
}
|
||||
// It says which validator judged: mesh-host's of the release branch, with its commit.
|
||||
if want := "mesh-host's validator of " + releaseBranch + " at " + hostRelease[:12]; v.Validator != want ||
|
||||
!strings.Contains(v.Report, want) {
|
||||
t.Fatalf("the verdict says the validator was %q, not %q:\n%s", v.Validator, want, v.Report)
|
||||
}
|
||||
// Each repository read beside it is named with the branch and the commit read.
|
||||
for dir := range beside {
|
||||
if v.BesideAt[dir] == "" || v.Beside[dir] == "" ||
|
||||
!strings.Contains(v.Report, dir+" "+v.BesideAt[dir]+" at "+v.Beside[dir]) {
|
||||
t.Fatalf("%s is not named with its branch and commit: at %q, commit %q\n%s", dir, v.BesideAt[dir],
|
||||
v.Beside[dir], v.Report)
|
||||
}
|
||||
}
|
||||
if n := builds(t, built); n != 1 {
|
||||
t.Fatalf("the judge was built %d times for one check", n)
|
||||
}
|
||||
// **The judge is built with the judge cache's Go cache** (ADR 0358 §6), which no check's container mounts, and no
|
||||
// Go environment file.
|
||||
said, _ := os.ReadFile(built)
|
||||
if !strings.Contains(string(said), "GOCACHE="+filepath.Join(cache, "go-cache")+" GOENV=off go build") {
|
||||
t.Fatalf("the release judge was not built with the judge cache's Go cache:\n%s", said)
|
||||
}
|
||||
// **Clause 5**: the head was merged onto the release branch's head, which the verdict names as its trunk commit,
|
||||
// so the release branch's move puts it out of date as a trunk's move does (ADR 0323 §4, the forge's holder).
|
||||
if releaseHead := gitInRepo(t, catalogue, "rev-parse", releaseBranch); v.Trunk != releaseHead[:12] {
|
||||
t.Fatalf("the verdict names trunk %q, not the release branch's head %s", v.Trunk, releaseHead[:12])
|
||||
}
|
||||
|
||||
// **Built once per release-branch head**: a second check while that head stands takes the judge kept.
|
||||
v = check("check-704-catalogue-again")
|
||||
if !strings.HasSuffix(v.Gate.Summary, "the release branch's controller with the release branch's validator") {
|
||||
t.Fatalf("the kept judge did not judge: %+v\n%s", v.Gate, v.Report)
|
||||
}
|
||||
if n := builds(t, built); n != 1 {
|
||||
t.Fatalf("the judge of the same release-branch head was built again: %d builds", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAControllerPullRequestIntoAReleaseBranchReadsTheNodeEngineAtThatBranch(t *testing.T) {
|
||||
aRuntimeThatBuildsTheJudge(t)
|
||||
registry := aFactsRegistry(t)
|
||||
beside, _, hostRelease, _ := theReleasedMesh(t)
|
||||
delete(beside, "mesh-controller")
|
||||
controller, head := aPullRequestInto(t, aJudgeOf("the change's controller"), releaseBranch,
|
||||
map[string]string{"cmd/x.txt": "x"})
|
||||
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-controller", Repository: controller, Ref: head,
|
||||
Owner: "novox", Repo: "mesh-controller", Number: 315, Base: releaseBranch, Release: releaseBranch,
|
||||
JudgeCache: t.TempDir(), Paths: []string{"cmd/x.txt"}, Modules: []string{"mesh-controller"}, Judge: "self",
|
||||
Beside: beside, Toolchain: "golang"}, t.TempDir(), registry, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v.Gate == nil || !strings.HasSuffix(v.Gate.Summary, "the change's controller with the vendored validator") {
|
||||
t.Fatalf("a controller change into a release branch did not judge itself: %+v\n%s", v.Gate, v.Report)
|
||||
}
|
||||
if v.BesideAt["mesh-host"] != releaseBranch || v.Beside["mesh-host"] != hostRelease[:12] {
|
||||
t.Fatalf("mesh-host was read beside it at %q (%s), not at the release branch's %s", v.BesideAt["mesh-host"],
|
||||
v.Beside["mesh-host"], hostRelease[:12])
|
||||
}
|
||||
if v.Release != releaseBranch || v.JudgedBy == "" || v.JudgedBy != v.Merge[:12] {
|
||||
t.Fatalf("the verdict names release %q judged by %q, not its own merge %s", v.Release, v.JudgedBy, v.Merge)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPullRequestIntoTheTrunkIsJudgedAsBefore(t *testing.T) {
|
||||
aRuntimeThatBuildsTheJudge(t)
|
||||
registry := aFactsRegistry(t)
|
||||
controller, controllerMain, _ := aRepositoryWithARelease(t, aJudgeOf("main's controller"),
|
||||
aJudgeOf("the release branch's controller"))
|
||||
host, hostMain, _ := aRepositoryWithARelease(t, aHostSaying("main's validator"),
|
||||
aHostSaying("the release branch's validator"))
|
||||
catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`,
|
||||
CheckScript: "true\n"}, "main", map[string]string{"modules/gitea/index.ts": "x"})
|
||||
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-trunk", Repository: catalogue, Ref: head,
|
||||
Owner: "novox", Repo: "mesh-catalog", Number: 1, Base: "main", JudgeCache: t.TempDir(),
|
||||
Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"}, Toolchain: "golang",
|
||||
Beside: map[string]Beside{"mesh-controller": {Repository: controller, Ref: controllerMain},
|
||||
"mesh-host": {Repository: host, Ref: hostMain}}}, t.TempDir(), registry, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v.Gate == nil || v.Gate.Summary != "main's controller with the vendored validator" {
|
||||
t.Fatalf("a trunk pull request was not judged by the controller beside it as before: %+v\n%s", v.Gate, v.Report)
|
||||
}
|
||||
if v.Release != "" || v.JudgedBy != "" || v.BesideAt != nil || strings.Contains(v.Report, "release branch") {
|
||||
t.Fatalf("a trunk pull request's verdict speaks of a release branch: %q %q %v", v.Release, v.JudgedBy, v.BesideAt)
|
||||
}
|
||||
if v.Beside["mesh-host"] != hostMain[:12] {
|
||||
t.Fatalf("mesh-host was read at %s, not the commit asked %s", v.Beside["mesh-host"], hostMain[:12])
|
||||
}
|
||||
}
|
||||
|
||||
// **Without mesh-host's release branch, the release branch's controller keeps the validator it vendors** (ADR 0358
|
||||
// §2), and the verdict says so.
|
||||
func TestWithoutTheNodeEnginesReleaseBranchTheJudgeKeepsItsVendoredValidator(t *testing.T) {
|
||||
aRuntimeThatBuildsTheJudge(t)
|
||||
registry := aFactsRegistry(t)
|
||||
beside, controllerRelease, _, _ := theReleasedMesh(t)
|
||||
host, hostMain, _ := aRepositoryWithARelease(t, aHostSaying("main's validator"), nil)
|
||||
beside["mesh-host"] = Beside{Repository: host, Ref: releaseBranch, Else: "main"}
|
||||
catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`,
|
||||
CheckScript: "true\n"}, releaseBranch, map[string]string{"modules/gitea/index.ts": "x"})
|
||||
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-vendored", Repository: catalogue, Ref: head,
|
||||
Owner: "novox", Repo: "mesh-catalog", Number: 286, Base: releaseBranch, Release: releaseBranch,
|
||||
JudgeCache: t.TempDir(), Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"}, Beside: beside,
|
||||
Toolchain: "golang"}, t.TempDir(), registry, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v.Gate == nil || !strings.HasSuffix(v.Gate.Summary, "the release branch's controller with the vendored validator") {
|
||||
t.Fatalf("without mesh-host's release branch the judge did not keep its vendored validator: %+v\n%s", v.Gate, v.Report)
|
||||
}
|
||||
if v.JudgedBy != controllerRelease[:12] || v.BesideAt["mesh-host"] != "main" || v.Beside["mesh-host"] != hostMain[:12] {
|
||||
t.Fatalf("judged by %q, mesh-host read at %q (%s)", v.JudgedBy, v.BesideAt["mesh-host"], v.Beside["mesh-host"])
|
||||
}
|
||||
if !strings.Contains(v.Validator, "controller vendors") || !strings.Contains(v.Report, v.Validator) {
|
||||
t.Fatalf("the verdict does not say the vendored validator judged: %q\n%s", v.Validator, v.Report)
|
||||
}
|
||||
}
|
||||
|
||||
// **Clause 7** (ADR 0358 §7): a release branch's own pull request into its trunk is checked as any pull request into a
|
||||
// trunk — by the controller beside it as the mesh runs it, the repositories at what a trunk check reads — and its
|
||||
// verdict names no release branch.
|
||||
func TestAReleaseBranchsPullRequestIntoItsTrunkIsJudgedByTheControllerTheMeshRuns(t *testing.T) {
|
||||
aRuntimeThatBuildsTheJudge(t)
|
||||
registry := aFactsRegistry(t)
|
||||
controller, controllerMain, _ := aRepositoryWithARelease(t, aJudgeOf("the controller the mesh runs"),
|
||||
aJudgeOf("the release branch's controller"))
|
||||
host, hostMain, _ := aRepositoryWithARelease(t, aHostSaying("main's validator"),
|
||||
aHostSaying("the release branch's validator"))
|
||||
catalogue, _, releaseHead := aRepositoryWithARelease(t, map[string]string{
|
||||
"modules/gitea/module.json": `{"module":"gitea"}`, CheckScript: "true\n"},
|
||||
map[string]string{"modules/gitea/index.ts": "released"})
|
||||
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-release-into-trunk", Repository: catalogue,
|
||||
Ref: releaseHead, Owner: "novox", Repo: "mesh-catalog", Number: 300, Base: "main", JudgeCache: t.TempDir(),
|
||||
Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"}, Toolchain: "golang",
|
||||
Beside: map[string]Beside{"mesh-controller": {Repository: controller, Ref: controllerMain},
|
||||
"mesh-host": {Repository: host, Ref: hostMain}}}, t.TempDir(), registry, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v.Gate == nil || v.Gate.Summary != "the controller the mesh runs with the vendored validator" {
|
||||
t.Fatalf("the release branch's pull request into its trunk was not judged by the controller the mesh runs: %+v\n%s",
|
||||
v.Gate, v.Report)
|
||||
}
|
||||
if v.Release != "" || v.JudgedBy != "" || v.BesideAt != nil || v.Validator != "" ||
|
||||
strings.Contains(v.Report, "release branch") || strings.Contains(v.Gate.Summary, "[release") {
|
||||
t.Fatalf("its verdict names a release branch: %q %q %v %q", v.Release, v.JudgedBy, v.BesideAt, v.Validator)
|
||||
}
|
||||
if v.Beside["mesh-host"] != hostMain[:12] || v.Beside["mesh-controller"] != controllerMain[:12] {
|
||||
t.Fatalf("beside it were read %v, not the trunk's %s and %s", v.Beside, controllerMain[:12], hostMain[:12])
|
||||
}
|
||||
}
|
||||
|
||||
// **mesh-lab is read at its trunk, always** (ADR 0358): its replays run with the container runtime's socket, so a
|
||||
// branch of the release's name in mesh-lab — which anyone may create — is never what a check reads.
|
||||
func TestAReleaseBranchCheckReadsMeshLabAtItsTrunkEvenWhenItHasThatBranch(t *testing.T) {
|
||||
aRuntimeThatBuildsTheJudge(t)
|
||||
registry := aFactsRegistry(t)
|
||||
beside, _, _, _ := theReleasedMesh(t)
|
||||
lab, labMain, labRelease := aRepositoryWithARelease(t, map[string]string{"README": "the lab"},
|
||||
map[string]string{"README": "a branch anyone could push"})
|
||||
beside["mesh-lab"] = Beside{Repository: lab, Ref: releaseBranch, Else: "main"}
|
||||
catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`,
|
||||
CheckScript: "true\n"}, releaseBranch, map[string]string{"modules/gitea/index.ts": "x"})
|
||||
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-lab", Repository: catalogue, Ref: head,
|
||||
Owner: "novox", Repo: "mesh-catalog", Number: 286, Base: releaseBranch, Release: releaseBranch,
|
||||
JudgeCache: t.TempDir(), Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"}, Beside: beside,
|
||||
Toolchain: "golang"}, t.TempDir(), registry, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v.BesideAt["mesh-lab"] != "main" || v.Beside["mesh-lab"] != labMain[:12] || v.Beside["mesh-lab"] == labRelease[:12] {
|
||||
t.Fatalf("mesh-lab was read at %q (%s), not its trunk main at %s", v.BesideAt["mesh-lab"], v.Beside["mesh-lab"],
|
||||
labMain[:12])
|
||||
}
|
||||
}
|
||||
|
||||
// aPlantedWorkspace is a workspace a pull request's own script has written into, as it can: a git configuration in its
|
||||
// HOME whose fsmonitor hook says each directory git runs in, a git configuration under XDG_CONFIG_HOME doing the same,
|
||||
// and a Go environment file that would hand the judge's build a tool of its own. It answers the workspace and the
|
||||
// file the hook writes to.
|
||||
func aPlantedWorkspace(t *testing.T) (string, string) {
|
||||
t.Helper()
|
||||
workspace, logs := t.TempDir(), t.TempDir()
|
||||
ran := filepath.Join(logs, "hook-ran")
|
||||
hook := filepath.Join(workspace, "hook.sh")
|
||||
files := map[string]string{
|
||||
hook: "#!/bin/sh\necho \"$PWD\" >> " + ran + "\n",
|
||||
filepath.Join(workspace, ".gitconfig"): "[core]\n\tfsmonitor = " + hook + "\n",
|
||||
filepath.Join(workspace, ".config", "git", "config"): "[core]\n\tfsmonitor = " + hook + "\n",
|
||||
filepath.Join(workspace, ".config", "go", "env"): "GOFLAGS=-toolexec=" + hook + "\n",
|
||||
}
|
||||
for name, body := range files {
|
||||
if err := os.MkdirAll(filepath.Dir(name), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(name, []byte(body), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return workspace, ran
|
||||
}
|
||||
|
||||
// judgeRanTheHook says whether the planted hook ran in a judge's tree: the controller beside the check, or the
|
||||
// change's own when it judges itself.
|
||||
func judgeRanTheHook(t *testing.T, ran string) string {
|
||||
t.Helper()
|
||||
body, _ := os.ReadFile(ran)
|
||||
for _, line := range strings.Split(string(body), "\n") {
|
||||
if strings.HasSuffix(line, "/check/mesh-controller") {
|
||||
return string(body)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// **Every judge is built where no pull request reaches** (issue 482, ADR 0358 review): a git configuration with a
|
||||
// hook, and a Go environment file, planted in the workspace by an earlier pull request's script, do not run during
|
||||
// the release judge's build — while the pull request's own script, in the workspace, still meets them.
|
||||
func TestAHookPlantedInTheWorkspaceDoesNotRunDuringTheReleaseJudgesBuild(t *testing.T) {
|
||||
aRuntimeThatBuildsTheJudge(t)
|
||||
homes := filepath.Join(t.TempDir(), "homes")
|
||||
t.Setenv("JUDGE_HOMES", homes)
|
||||
registry := aFactsRegistry(t)
|
||||
workspace, ran := aPlantedWorkspace(t)
|
||||
beside, _, _, _ := theReleasedMesh(t)
|
||||
catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`,
|
||||
CheckScript: "git status >/dev/null 2>&1; true\n"}, releaseBranch, map[string]string{"modules/gitea/index.ts": "x"})
|
||||
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-planted", Repository: catalogue, Ref: head,
|
||||
Owner: "novox", Repo: "mesh-catalog", Number: 286, Base: releaseBranch, Release: releaseBranch,
|
||||
JudgeCache: t.TempDir(), Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"}, Beside: beside,
|
||||
Toolchain: "golang"}, workspace, registry, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v.Gate == nil || v.Gate.Verdict != "pass" {
|
||||
t.Fatalf("the check did not judge: %+v\n%s", v.Gate, v.Report)
|
||||
}
|
||||
if body, _ := os.ReadFile(ran); !strings.Contains(string(body), "/check/mesh-catalog") {
|
||||
t.Fatalf("the planted hook never ran, so this test proves nothing: %q", body)
|
||||
}
|
||||
if said := judgeRanTheHook(t, ran); said != "" {
|
||||
t.Fatalf("the hook planted in the workspace ran during the judge's build:\n%s", said)
|
||||
}
|
||||
if body, _ := os.ReadFile(homes); len(body) == 0 || strings.Contains(string(body), workspace) {
|
||||
t.Fatalf("the judge was built with a HOME in the workspace: %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
// The same for the judge built with a node-engine change's own validator, into a release branch (ADR 0358 §2).
|
||||
func TestAHookPlantedInTheWorkspaceDoesNotRunDuringTheValidatorJudgesBuild(t *testing.T) {
|
||||
aRuntimeThatBuildsTheJudge(t)
|
||||
homes := filepath.Join(t.TempDir(), "homes")
|
||||
t.Setenv("JUDGE_HOMES", homes)
|
||||
registry := aFactsRegistry(t)
|
||||
workspace, ran := aPlantedWorkspace(t)
|
||||
beside, _, _, _ := theReleasedMesh(t)
|
||||
delete(beside, "mesh-host")
|
||||
files := aHostSaying("the release branch's validator")
|
||||
files["module.json"] = `{"module":"mesh-host"}`
|
||||
files[CheckScript] = "git status >/dev/null 2>&1; true\n"
|
||||
host, head := aPullRequestInto(t, files, releaseBranch, aHostSaying("this change's validator"))
|
||||
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-planted-validator", Repository: host, Ref: head,
|
||||
Owner: "novox", Repo: "mesh-host", Number: 111, Base: releaseBranch, Release: releaseBranch, Judge: "validator",
|
||||
JudgeCache: t.TempDir(), Paths: []string{"validate/v.go"}, Modules: []string{"mesh-host"}, Beside: beside,
|
||||
Toolchain: "golang"}, workspace, registry, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v.Gate == nil || !strings.HasSuffix(v.Gate.Summary, "the release branch's controller with this change's validator") {
|
||||
t.Fatalf("a node-engine change into a release branch was not judged by that branch's controller with its own "+
|
||||
"validator: %+v\n%s", v.Gate, v.Report)
|
||||
}
|
||||
if body, _ := os.ReadFile(ran); !strings.Contains(string(body), "/check/mesh-host") {
|
||||
t.Fatalf("the planted hook never ran, so this test proves nothing: %q", body)
|
||||
}
|
||||
if said := judgeRanTheHook(t, ran); said != "" {
|
||||
t.Fatalf("the hook planted in the workspace ran during a judge's build:\n%s", said)
|
||||
}
|
||||
if body, _ := os.ReadFile(homes); strings.Count(string(body), "\n") != 2 || strings.Contains(string(body), workspace) {
|
||||
t.Fatalf("the judges were not both built with a HOME outside the workspace: %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
// **mesh-lab is never read at a release branch**, asked with a fallback or without (ADR 0358).
|
||||
func TestTheBuildSeatNeverReadsMeshLabAtAReleaseBranch(t *testing.T) {
|
||||
aRuntimeThatBuildsTheJudge(t)
|
||||
registry := aFactsRegistry(t)
|
||||
lab, labMain, _ := aRepositoryWithARelease(t, map[string]string{"README": "the lab"},
|
||||
map[string]string{"README": "a branch anyone could push"})
|
||||
for _, asked := range []Beside{{Repository: lab, Ref: releaseBranch}, {Repository: lab, Ref: releaseBranch, Else: "main"}} {
|
||||
beside, _, _, _ := theReleasedMesh(t)
|
||||
beside["mesh-lab"] = asked
|
||||
catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`,
|
||||
CheckScript: "true\n"}, releaseBranch, map[string]string{"modules/gitea/index.ts": "x"})
|
||||
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-lab-asked", Repository: catalogue, Ref: head,
|
||||
Owner: "novox", Repo: "mesh-catalog", Number: 286, Base: releaseBranch, Release: releaseBranch,
|
||||
JudgeCache: t.TempDir(), Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"}, Beside: beside,
|
||||
Toolchain: "golang"}, t.TempDir(), registry, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v.BesideAt["mesh-lab"] != "main" || v.Beside["mesh-lab"] != labMain[:12] {
|
||||
t.Fatalf("mesh-lab asked as %+v was read at %q (%s), not main at %s", asked, v.BesideAt["mesh-lab"],
|
||||
v.Beside["mesh-lab"], labMain[:12])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Judges are built warm and where no pull request reaches** (ADR 0358 §6): every judge build uses the judge cache's
|
||||
// one Go cache, kept between checks; its home is made fresh under the judge cache and removed after; nothing is left in
|
||||
// a temporary directory of the host; and no container a check runs for the pull request mounts the judge cache.
|
||||
func TestJudgesAreBuiltWarmFromAJudgeCacheNoCheckContainerMounts(t *testing.T) {
|
||||
built := aRuntimeThatBuildsTheJudge(t)
|
||||
runs := filepath.Join(t.TempDir(), "runs")
|
||||
t.Setenv("RUNS", runs)
|
||||
tmp := t.TempDir()
|
||||
t.Setenv("TMPDIR", tmp)
|
||||
registry := aFactsRegistry(t)
|
||||
cache := t.TempDir()
|
||||
controller, controllerMain, _ := aRepositoryWithARelease(t, aJudgeOf("the controller the mesh runs"), nil)
|
||||
beside := map[string]Beside{"mesh-controller": {Repository: controller, Ref: controllerMain}}
|
||||
for i := range 2 {
|
||||
catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`,
|
||||
CheckScript: "true\n"}, "main", map[string]string{"modules/gitea/index.ts": fmt.Sprint(i)})
|
||||
v, err := Check(t.Context(), Command, CheckSpec{ID: fmt.Sprintf("check-704-warm-%d", i), Repository: catalogue,
|
||||
Ref: head, Owner: "novox", Repo: "mesh-catalog", Number: 1, Base: "main", JudgeCache: cache,
|
||||
Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"}, Toolchain: "golang", Beside: beside},
|
||||
t.TempDir(), registry, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v.Gate == nil || v.Gate.Summary != "the controller the mesh runs with the vendored validator" {
|
||||
t.Fatalf("check %d was not judged: %+v\n%s", i, v.Gate, v.Report)
|
||||
}
|
||||
if i == 0 {
|
||||
if entries, _ := os.ReadDir(filepath.Join(cache, "go-cache")); len(entries) == 0 {
|
||||
t.Fatal("the judge's build left nothing in the judge cache's Go cache to build the next judge warm")
|
||||
}
|
||||
}
|
||||
}
|
||||
// Both judges built with the one Go cache of the judge cache.
|
||||
said, _ := os.ReadFile(built)
|
||||
if n := strings.Count(string(said), "GOCACHE="+filepath.Join(cache, "go-cache")+" GOENV=off go build"); n != 2 {
|
||||
t.Fatalf("%d of 2 judge builds used the judge cache's Go cache:\n%s", n, said)
|
||||
}
|
||||
// Nothing left behind: no home under the judge cache, nothing in the host's temporary directory.
|
||||
if left, _ := os.ReadDir(filepath.Join(cache, "homes")); len(left) != 0 {
|
||||
t.Fatalf("judge builds left %d home(s) under the judge cache", len(left))
|
||||
}
|
||||
if left, _ := os.ReadDir(tmp); len(left) != 0 {
|
||||
t.Fatalf("a check left %d entr(ies) in the temporary directory, e.g. %s", len(left), left[0].Name())
|
||||
}
|
||||
// Only a judge build mounts the judge cache: no container the check runs for the pull request does.
|
||||
body, _ := os.ReadFile(runs)
|
||||
judgeBuilds := 0
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(body)), "\n") {
|
||||
judging := strings.Contains(line, " go build ")
|
||||
if judging {
|
||||
judgeBuilds++
|
||||
if !strings.Contains(line, "--volume "+filepath.Join(cache, "go-cache")+":") {
|
||||
t.Fatalf("a judge build did not mount the judge cache's Go cache: %s", line)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if strings.Contains(line, cache) {
|
||||
t.Fatalf("a container the check runs for the pull request names the judge cache: %s", line)
|
||||
}
|
||||
}
|
||||
if judgeBuilds != 2 {
|
||||
t.Fatalf("%d judge builds were run, not 2", judgeBuilds)
|
||||
}
|
||||
}
|
||||
|
||||
// Without a judge cache, a judge is built cold under the check's own root — never in a temporary directory of the
|
||||
// host — and a judge cache inside the workspace is refused.
|
||||
func TestWithoutAJudgeCacheAJudgeIsBuiltColdUnderTheChecksRoot(t *testing.T) {
|
||||
built := aRuntimeThatBuildsTheJudge(t)
|
||||
tmp := t.TempDir()
|
||||
t.Setenv("TMPDIR", tmp)
|
||||
registry := aFactsRegistry(t)
|
||||
controller, controllerMain, _ := aRepositoryWithARelease(t, aJudgeOf("the controller the mesh runs"), nil)
|
||||
beside := map[string]Beside{"mesh-controller": {Repository: controller, Ref: controllerMain}}
|
||||
catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`,
|
||||
CheckScript: "true\n"}, "main", map[string]string{"modules/gitea/index.ts": "x"})
|
||||
workspace := t.TempDir()
|
||||
spec := CheckSpec{ID: "check-704-cold", Repository: catalogue, Ref: head, Owner: "novox", Repo: "mesh-catalog",
|
||||
Number: 1, Base: "main", Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"},
|
||||
Toolchain: "golang", Beside: beside}
|
||||
v, err := Check(t.Context(), Command, spec, workspace, registry, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v.Gate == nil || v.Gate.Verdict != "pass" {
|
||||
t.Fatalf("the check was not judged: %+v\n%s", v.Gate, v.Report)
|
||||
}
|
||||
said, _ := os.ReadFile(built)
|
||||
if !strings.Contains(string(said), "GOCACHE="+filepath.Join(workspace, "check", "judge-go-cache")+" GOENV=off go build") {
|
||||
t.Fatalf("without a judge cache the judge was not built under the check's root:\n%s", said)
|
||||
}
|
||||
if left, _ := os.ReadDir(tmp); len(left) != 0 {
|
||||
t.Fatalf("a check left %d entr(ies) in the temporary directory", len(left))
|
||||
}
|
||||
spec.JudgeCache = filepath.Join(workspace, "judge-cache")
|
||||
if _, err := Check(t.Context(), Command, spec, workspace, registry, GitCredential{}, nil); err == nil ||
|
||||
!strings.Contains(err.Error(), "inside the workspace") {
|
||||
t.Fatalf("a judge cache inside the workspace was taken: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// aLabWithReplays is mesh-lab with replays: a Go module of its own whose one test passes, needing nothing fetched.
|
||||
func aLabWithReplays(t *testing.T) (string, string) {
|
||||
t.Helper()
|
||||
lab, labMain, _ := aRepositoryWithARelease(t, map[string]string{
|
||||
"replays/go.mod": "module example.org/replays\n\ngo 1.22\n",
|
||||
"replays/replay_test.go": "package replays\n\nimport \"testing\"\n\nfunc TestAReplay(t *testing.T) {}\n",
|
||||
}, nil)
|
||||
return lab, labMain
|
||||
}
|
||||
|
||||
// **mesh-lab's replays run where no pull request reaches** (novox/hq issue 709, ADR 0358 §6): they hold the container
|
||||
// runtime's socket, so a Go environment file with a -toolexec hook and a Go cache and module cache planted in the
|
||||
// workspace by a pull request's script are neither run nor read — the replays run from a copy of mesh-lab outside
|
||||
// the workspace with a fresh home and the judge cache's caches, all removed or kept outside every check container.
|
||||
func TestTheReplaysRunWhereNoPullRequestReaches(t *testing.T) {
|
||||
aRuntimeThatBuildsTheJudge(t)
|
||||
replays := filepath.Join(t.TempDir(), "replays")
|
||||
t.Setenv("REPLAYS", replays)
|
||||
registry := aFactsRegistry(t)
|
||||
workspace, ran := aPlantedWorkspace(t)
|
||||
for _, poisoned := range []string{filepath.Join(workspace, "go-cache", "README"),
|
||||
filepath.Join(workspace, "go-modules", "cache", "download", "planted")} {
|
||||
if err := os.MkdirAll(filepath.Dir(poisoned), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(poisoned, []byte("planted by a pull request"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
cache := t.TempDir()
|
||||
controller, controllerMain, _ := aRepositoryWithARelease(t, aJudgeOf("the controller the mesh runs"), nil)
|
||||
lab, labMain := aLabWithReplays(t)
|
||||
catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`,
|
||||
CheckScript: "true\n"}, "main", map[string]string{"modules/gitea/index.ts": "x"})
|
||||
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-709", Repository: catalogue, Ref: head, Owner: "novox",
|
||||
Repo: "mesh-catalog", Number: 1, Base: "main", JudgeCache: cache, Paths: []string{"modules/gitea/index.ts"},
|
||||
Modules: []string{"gitea"}, Toolchain: "golang", Beside: map[string]Beside{
|
||||
"mesh-controller": {Repository: controller, Ref: controllerMain}, "mesh-lab": {Repository: lab, Ref: labMain}}},
|
||||
workspace, registry, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v.Gate == nil || v.Gate.Verdict != "pass" || !strings.Contains(v.Report, "--- the replays") {
|
||||
t.Fatalf("the replays did not run and pass: %+v\n%s", v.Gate, v.Report)
|
||||
}
|
||||
body, _ := os.ReadFile(replays)
|
||||
said := strings.TrimSpace(string(body))
|
||||
if said == "" || strings.Count(said, "\n") != 0 {
|
||||
t.Fatalf("the replays ran %q, not once", said)
|
||||
}
|
||||
for _, want := range []string{"GOCACHE=" + filepath.Join(cache, "go-cache") + " ",
|
||||
"GOMODCACHE=" + filepath.Join(cache, "go-modules") + " ", "GOENV=off ", "GOFLAGS=-buildvcs=false ",
|
||||
"HOME=" + filepath.Join(cache, "homes", "check-709-replays", "home") + " ",
|
||||
"DIR=" + filepath.Join(cache, "homes", "check-709-replays", "mesh-lab", "replays")} {
|
||||
if !strings.Contains(said, want) {
|
||||
t.Fatalf("the replays did not run with %q: %s", want, said)
|
||||
}
|
||||
}
|
||||
if strings.Contains(said, workspace) {
|
||||
t.Fatalf("the replays ran with something of the workspace: %s", said)
|
||||
}
|
||||
if body, _ := os.ReadFile(ran); strings.Contains(string(body), "replays") || strings.Contains(string(body), "mesh-lab") {
|
||||
t.Fatalf("a hook planted in the workspace ran during the replays:\n%s", body)
|
||||
}
|
||||
if left, _ := os.ReadDir(filepath.Join(cache, "homes")); len(left) != 0 {
|
||||
t.Fatalf("the replays left %d entr(ies) under the judge cache's homes", len(left))
|
||||
}
|
||||
}
|
||||
|
||||
// aSelfJudgingControllerThatWritesIntoTheReplays is a controller pull request whose own judge, while the gate runs it,
|
||||
// writes a test into the replays beside the check — the reviewer's case for issue 709: code of the pull request's,
|
||||
// run before the replays, reaching what they run.
|
||||
func aSelfJudgingControllerThatWritesIntoTheReplays(t *testing.T, marker string) (string, string) {
|
||||
t.Helper()
|
||||
files := aJudgeOf("the change's controller")
|
||||
evil := "package replays\n\nimport (\n\t\"os\"\n\t\"testing\"\n)\n\nfunc TestEvil(t *testing.T) {\n\t_ = os.WriteFile(" +
|
||||
fmt.Sprintf("%q", marker) + ", []byte(\"ran\"), 0o644)\n}\n"
|
||||
files["cmd/mesh-controller/main.go"] = strings.Replace(files["cmd/mesh-controller/main.go"], "func main() {",
|
||||
"func main() {\n\tif len(os.Args) > 2 && os.Args[1] == \"merge-gate\" {\n\t\t_ = os.WriteFile(os.Getenv(\"MESH_CHECK_BESIDE\")+"+
|
||||
"\"/mesh-lab/replays/evil_test.go\", []byte("+fmt.Sprintf("%q", evil)+"), 0o644)\n\t\t_ = os.WriteFile("+
|
||||
fmt.Sprintf("%q", marker+".wrote")+", nil, 0o644)\n\t}", 1)
|
||||
return aPullRequestInto(t, files, "main", map[string]string{"cmd/x.txt": "x"})
|
||||
}
|
||||
|
||||
// **The replays never read what a pull request wrote** (issue 709, reviews 12303 and 12308): with a judge cache they
|
||||
// run from a copy under it, and a self-judging controller pull request writing a test into the replays beside the
|
||||
// check, a planted GOFLAGS hook and a poisoned cache entry in the workspace neither run nor are read. Without one the
|
||||
// replays are refused — the gate an error that says why, never a pass, never the workspace — they never start, and
|
||||
// nothing is made on the host.
|
||||
func TestTheReplaysNeverReadWhatAPullRequestWroteAndNeedAJudgeCache(t *testing.T) {
|
||||
for _, withCache := range []bool{true, false} {
|
||||
t.Run(fmt.Sprintf("judge cache %v", withCache), func(t *testing.T) {
|
||||
aRuntimeThatBuildsTheJudge(t)
|
||||
replays, volumes := filepath.Join(t.TempDir(), "replays"), filepath.Join(t.TempDir(), "volumes")
|
||||
t.Setenv("REPLAYS", replays)
|
||||
t.Setenv("VOLUMES", volumes)
|
||||
tmp := t.TempDir()
|
||||
t.Setenv("TMPDIR", tmp)
|
||||
registry := aFactsRegistry(t)
|
||||
workspace, ran := aPlantedWorkspace(t)
|
||||
if err := os.MkdirAll(filepath.Join(workspace, "go-cache"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(workspace, "go-cache", "README"), []byte("planted"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
evilRan := filepath.Join(t.TempDir(), "evil-ran")
|
||||
controller, head := aSelfJudgingControllerThatWritesIntoTheReplays(t, evilRan)
|
||||
lab, labMain := aLabWithReplays(t)
|
||||
cache := ""
|
||||
if withCache {
|
||||
cache = t.TempDir()
|
||||
}
|
||||
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-709-self", Repository: controller, Ref: head,
|
||||
Owner: "novox", Repo: "mesh-controller", Number: 330, Base: "main", Judge: "self", JudgeCache: cache,
|
||||
Paths: []string{"cmd/x.txt"}, Modules: []string{"mesh-controller"}, Toolchain: "golang",
|
||||
Beside: map[string]Beside{"mesh-lab": {Repository: lab, Ref: labMain}}}, workspace, registry,
|
||||
GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(evilRan + ".wrote"); err != nil {
|
||||
t.Fatalf("the self-judging pull request did not write into the replays beside it, so this proves nothing: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(evilRan); err == nil {
|
||||
t.Fatal("the test the pull request wrote into the replays ran with the socket")
|
||||
}
|
||||
if body, _ := os.ReadFile(ran); strings.Contains(string(body), "replays") || strings.Contains(string(body), "mesh-lab") {
|
||||
t.Fatalf("a hook planted in the workspace ran during the replays:\n%s", body)
|
||||
}
|
||||
body, _ := os.ReadFile(replays)
|
||||
said := strings.TrimSpace(string(body))
|
||||
if vols, _ := os.ReadFile(volumes); len(vols) != 0 {
|
||||
t.Fatalf("the check made a container volume: %q", vols)
|
||||
}
|
||||
if left, _ := os.ReadDir(tmp); len(left) != 0 {
|
||||
t.Fatalf("a check left %d entr(ies) in the temporary directory", len(left))
|
||||
}
|
||||
if !withCache {
|
||||
if v.Gate == nil || v.Gate.Verdict != "error" || !strings.Contains(v.Gate.Summary, replaysNeedTheJudgeCache) ||
|
||||
v.Verdict != "error" {
|
||||
t.Fatalf("without a judge cache the gate said %+v, not an error saying the replays need it", v.Gate)
|
||||
}
|
||||
if said != "" {
|
||||
t.Fatalf("without a judge cache the replays started: %s", said)
|
||||
}
|
||||
return
|
||||
}
|
||||
if v.Gate == nil || v.Gate.Verdict != "pass" || !strings.Contains(v.Report, "--- the replays") {
|
||||
t.Fatalf("the replays did not run and pass: %+v\n%s", v.Gate, v.Report)
|
||||
}
|
||||
base := filepath.Join(cache, "homes", "check-709-self-replays")
|
||||
if said == "" || strings.Contains(said, workspace) || !strings.Contains(said, "GOENV=off ") ||
|
||||
!strings.Contains(said, "DIR="+filepath.Join(base, "mesh-lab", "replays")) {
|
||||
t.Fatalf("the replays did not run from the copy under %s, apart from the workspace: %q", base, said)
|
||||
}
|
||||
if _, err := os.Stat(base); err == nil {
|
||||
t.Fatalf("the replays' place %s was left after the check", base)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// **Replays the container runtime could not start are the check's error, never the change's failure**, said in the
|
||||
// build's log (review 12308); and a check whose id cannot name a place in the judge cache is refused, never given a
|
||||
// shared one.
|
||||
func TestReplaysThatCannotStartAreAnErrorAndAnUnsafeIDIsRefused(t *testing.T) {
|
||||
aRuntimeThatBuildsTheJudge(t)
|
||||
t.Setenv("REPLAYS_CANNOT_START", "1")
|
||||
registry := aFactsRegistry(t)
|
||||
controller, controllerMain, _ := aRepositoryWithARelease(t, aJudgeOf("the controller the mesh runs"), nil)
|
||||
lab, labMain := aLabWithReplays(t)
|
||||
catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`,
|
||||
CheckScript: "true\n"}, "main", map[string]string{"modules/gitea/index.ts": "x"})
|
||||
spec := CheckSpec{ID: "check-709-start", Repository: catalogue, Ref: head, Owner: "novox", Repo: "mesh-catalog",
|
||||
Number: 1, Base: "main", JudgeCache: t.TempDir(), Paths: []string{"modules/gitea/index.ts"},
|
||||
Modules: []string{"gitea"}, Toolchain: "golang", Beside: map[string]Beside{
|
||||
"mesh-controller": {Repository: controller, Ref: controllerMain}, "mesh-lab": {Repository: lab, Ref: labMain}}}
|
||||
var logged []string
|
||||
v, err := Check(t.Context(), Command, spec, t.TempDir(), registry, GitCredential{}, func(step, message string) {
|
||||
logged = append(logged, message)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v.Gate == nil || v.Gate.Verdict != "error" || !strings.Contains(v.Gate.Summary, "could not be started") ||
|
||||
!strings.Contains(v.Gate.Summary, "invalid mount config") {
|
||||
t.Fatalf("replays that could not start were said %+v, not an error naming why", v.Gate)
|
||||
}
|
||||
if !strings.Contains(strings.Join(logged, "\n"), "the replays could not be started") {
|
||||
t.Fatalf("the build's log does not say what failed:\n%s", strings.Join(logged, "\n"))
|
||||
}
|
||||
|
||||
spec.ID = "Check 1/../../x"
|
||||
if _, err := Check(t.Context(), Command, spec, t.TempDir(), registry, GitCredential{}, nil); err == nil ||
|
||||
!strings.Contains(err.Error(), "cannot name its place in the judge cache") {
|
||||
t.Fatalf("a check with an unsafe id was taken: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq issue 713: after ADR 0350 the walk of novox/mesh-controller#331 built mesh-controller, build-agent and
|
||||
// route-proxy again, each "built: no check of novox/mesh-controller#331 built it". The check had built none: its
|
||||
// build log said "built for the merge: 0 of 3 module(s)", mesh-controller's go bundle failing with "go.mod file not
|
||||
// found". The build seat's holder runs in a container sharing only its workspace with the host, and a build's
|
||||
// toolchain container is run by the host's container engine, which mounts a path as the host names it. The check
|
||||
// built in a temporary directory of the holder's own (issue 703's copy), which the engine mounted as an empty one.
|
||||
|
||||
// aHostsEngine runs as the host's container engine does for a holder in a container: a mount of a path outside the
|
||||
// workspace the holder shares with the host is an empty directory, and a Go toolchain finds no go.mod in it.
|
||||
type aHostsEngine struct {
|
||||
onTrunk
|
||||
shared string
|
||||
}
|
||||
|
||||
func (h aHostsEngine) run(ctx context.Context, dir, name string, args ...string) (string, error) {
|
||||
if name == "docker" && len(args) > 0 && args[0] == "run" {
|
||||
for i, a := range args {
|
||||
if a != "--volume" || i+1 >= len(args) {
|
||||
continue
|
||||
}
|
||||
from, _, _ := strings.Cut(args[i+1], ":")
|
||||
if !withinDir(h.shared, from) {
|
||||
return "", errors.New("go: go.mod file not found in current directory or any parent directory")
|
||||
}
|
||||
}
|
||||
}
|
||||
return h.onTrunk.run(ctx, dir, name, args...)
|
||||
}
|
||||
|
||||
// aGoModule is a module of one Go bundle built from a repository's root.
|
||||
func aGoModule(module string) map[string]string {
|
||||
files := aSharedRepository("one", "1")
|
||||
files[ManifestName] = `{"module":"` + module + `","version":"1","build":{"artifacts":[
|
||||
{"name":"server","kind":"bundle","language":"go","system":"arch","from":"cmd/ctl","binary":"ctl"}]},
|
||||
"resources":[{"id":"server","type":"process","name":"` + module + `","artifact":"server","run":["./ctl"]}]}`
|
||||
return files
|
||||
}
|
||||
|
||||
// A controller change whose walk moves mesh-controller, build-agent and route-proxy (#331's) has its check build all
|
||||
// three, in the workspace the host's engine sees, and the walk of its merge takes each from the check by its key.
|
||||
func TestReplay713TheWalkOfAControllerChangeTakesAllThreeFromItsCheck(t *testing.T) {
|
||||
withVersion(t, "builder-1")
|
||||
pull := PullRef{Owner: "novox", Repo: "mesh-controller", Number: 331}
|
||||
held := map[string]string{"mesh-tools-go/build": "registry.invalid/mesh-tools-go/build@sha256:" + strings.Repeat("b", 64)}
|
||||
repositories := map[string]string{"build-agent": "https://forge.invalid/mesh-catalog-build-agent.git",
|
||||
"route-proxy": "https://forge.invalid/mesh-catalog-route-proxy.git"}
|
||||
checked := func(shared string) (*recorded, aHostsEngine) {
|
||||
r := &recorded{tree: "tree-of-the-merge", contents: aGoModule("mesh-controller"),
|
||||
secondary: map[string]map[string]string{}}
|
||||
for module, repository := range repositories {
|
||||
r.secondary[repository] = aGoModule(module)
|
||||
}
|
||||
return r, aHostsEngine{onTrunk: onTrunk{recorded: r}, shared: shared}
|
||||
}
|
||||
spec := CheckSpec{ID: "build-331", Owner: pull.Owner, Repo: pull.Repo, Number: pull.Number,
|
||||
Builds: []CheckBuildSpec{{Module: "mesh-controller", Merged: true, Tier: 0},
|
||||
{Module: "build-agent", Repository: repositories["build-agent"], Ref: "main", Tier: 1},
|
||||
{Module: "route-proxy", Repository: repositories["route-proxy"], Ref: "main", Tier: 2}}}
|
||||
judged := CheckVerdict{Head: "98e856874913", Trunk: "c975dbf7b053", Merge: "2d22279b1d07"}
|
||||
passed := CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "pass"}}
|
||||
|
||||
// As it ran: the builds made in the holder's own temporary directory, which the host's engine does not see.
|
||||
shared, outside := t.TempDir(), t.TempDir()
|
||||
merged := filepath.Join(outside, "merged")
|
||||
r, engine := checked(shared)
|
||||
asItRan := buildForTheMerge(t.Context(), engine.run, spec, CheckBuilding{Publish: r, Scratch: newScratch(), Held: held},
|
||||
outside, merged, judged.Merge, judged, GitCredential{}, nil, logging(nil))
|
||||
if builtCount(asItRan) != 0 {
|
||||
t.Fatalf("the engine of this test sees the holder's temporary directory: %+v", asItRan)
|
||||
}
|
||||
|
||||
// The check as it builds now: every module its merge moves, kept under its key.
|
||||
scratch := newScratch()
|
||||
spec.BuildWith = &CheckBuilding{Publish: r, Scratch: scratch, Held: held}
|
||||
built := forTheMerge(t.Context(), engine.run, spec, passed, shared, merged, judged, GitCredential{}, nil,
|
||||
logging(nil))
|
||||
if len(built) != 3 || builtCount(built) != 3 {
|
||||
t.Fatalf("the check built %d of %d module(s) its merge moves: %+v", builtCount(built), len(built), built)
|
||||
}
|
||||
var names []NamedBuild
|
||||
for _, b := range built {
|
||||
n := NamedBuild{Pull: pull, Check: spec.ID, Module: b.Module, Key: b.Key, Digests: map[string]string{}}
|
||||
for _, m := range b.Made {
|
||||
d := m.Digest
|
||||
if d == "" {
|
||||
d = DigestOf(m.Reference)
|
||||
}
|
||||
n.Digests[m.Name] = d
|
||||
}
|
||||
names = append(names, n)
|
||||
}
|
||||
|
||||
// The walk of the merge commit, tier by tier: each module taken from the check, none built.
|
||||
walked := map[string]string{"mesh-controller": "https://forge.invalid/mesh-controller.git"}
|
||||
for module, repository := range repositories {
|
||||
walked[module] = repository
|
||||
}
|
||||
for _, module := range []string{"mesh-controller", "build-agent", "route-proxy"} {
|
||||
w, walk := checked(shared)
|
||||
w.secondary[walked["mesh-controller"]] = aGoModule("mesh-controller")
|
||||
got, err := BuildReusing(t.Context(), walk.run, w, Reuse{Scratch: scratch, Pulls: []PullRef{pull}, Named: names},
|
||||
walked[module], "", "main", t.TempDir(), held, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", module, err)
|
||||
}
|
||||
if got.Reused == nil || got.Why != "" {
|
||||
t.Fatalf("the walk built %s again: %s", module, got.Why)
|
||||
}
|
||||
for _, line := range w.ran {
|
||||
if strings.HasPrefix(line, "docker run") && strings.Contains(line, "go build") {
|
||||
t.Fatalf("the walk compiled %s: %s", module, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// novox/hq issue 716: a check built the images its merge moves one after another, and only after its merge-check.sh
|
||||
// had passed — 3 to 6 minutes on a check of about 5. The modules of a tier stand on none of each other and are built
|
||||
// at the same time; with a judge cache, beside the tests; a check that does not pass lets go of them.
|
||||
|
||||
// atOnce is a fake engine whose compiles wait until every one of a tier's has started: builds one after another never
|
||||
// get there, and the first one's compile says so.
|
||||
type atOnce struct {
|
||||
mu sync.Mutex
|
||||
r *recorded
|
||||
want int
|
||||
started int
|
||||
all chan struct{}
|
||||
shared string
|
||||
}
|
||||
|
||||
func (a *atOnce) run(ctx context.Context, dir, name string, args ...string) (string, error) {
|
||||
if name == "docker" && len(args) > 0 && args[0] == "run" && strings.Contains(strings.Join(args, " "), "go build") {
|
||||
a.mu.Lock()
|
||||
a.started++
|
||||
if a.started == a.want {
|
||||
close(a.all)
|
||||
}
|
||||
a.mu.Unlock()
|
||||
select {
|
||||
case <-a.all:
|
||||
case <-time.After(5 * time.Second):
|
||||
return "", errors.New("built alone: no other module of its tier was being built")
|
||||
}
|
||||
}
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return onTrunk{recorded: a.r}.run(ctx, dir, name, args...)
|
||||
}
|
||||
|
||||
func (a *atOnce) PublishImage(ctx context.Context, localTag, repository string) (string, error) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return a.r.PublishImage(ctx, localTag, repository)
|
||||
}
|
||||
|
||||
func (a *atOnce) PublishArchive(ctx context.Context, repository string, body []byte, digest string) (string, error) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return a.r.PublishArchive(ctx, repository, body, digest)
|
||||
}
|
||||
|
||||
// aLockedScratch is the scratch namespace in memory, said to by builds at once.
|
||||
type aLockedScratch struct {
|
||||
mu sync.Mutex
|
||||
*aScratch
|
||||
discarded []CheckBuild
|
||||
}
|
||||
|
||||
func (s *aLockedScratch) Keep(ctx context.Context, p PullRef, c string, at time.Time, b []CheckBuild) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.aScratch.Keep(ctx, p, c, at, b)
|
||||
}
|
||||
|
||||
func (s *aLockedScratch) Find(ctx context.Context, module, key string) (CheckBuild, bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.aScratch.Find(ctx, module, key)
|
||||
}
|
||||
|
||||
func (s *aLockedScratch) Discard(_ context.Context, b []CheckBuild) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.discarded = append(s.discarded, b...)
|
||||
return nil
|
||||
}
|
||||
|
||||
// threeAtOnce are three Go modules in one tier, each from its own repository.
|
||||
func threeAtOnce(t *testing.T) (*atOnce, CheckSpec, map[string]string) {
|
||||
t.Helper()
|
||||
r := &recorded{tree: "tree-of-the-merge", contents: aGoModule("mesh-controller"),
|
||||
secondary: map[string]map[string]string{}}
|
||||
spec := CheckSpec{ID: "build-716", Owner: "novox", Repo: "mesh-controller", Number: 716,
|
||||
Builds: []CheckBuildSpec{{Module: "mesh-controller", Merged: true, Tier: 0}}}
|
||||
for _, module := range []string{"build-agent", "route-proxy"} {
|
||||
repository := "https://forge.invalid/" + module + ".git"
|
||||
r.secondary[repository] = aGoModule(module)
|
||||
spec.Builds = append(spec.Builds, CheckBuildSpec{Module: module, Repository: repository, Ref: "main", Tier: 0})
|
||||
}
|
||||
held := map[string]string{"mesh-tools-go/build": "registry.invalid/mesh-tools-go/build@sha256:" + strings.Repeat("b", 64)}
|
||||
return &atOnce{r: r, want: 3, all: make(chan struct{})}, spec, held
|
||||
}
|
||||
|
||||
// The modules of one tier are built at the same time, each in a working copy of its own.
|
||||
func TestReplay716TheModulesOfATierAreBuiltAtTheSameTime(t *testing.T) {
|
||||
withVersion(t, "builder-1")
|
||||
engine, spec, held := threeAtOnce(t)
|
||||
scratch := &aLockedScratch{aScratch: newScratch()}
|
||||
built := buildForTheMerge(t.Context(), engine.run, spec, CheckBuilding{Publish: engine, Scratch: scratch, Held: held},
|
||||
t.TempDir(), filepath.Join(t.TempDir(), "merged"), "2d22279b1d07",
|
||||
CheckVerdict{Head: "98e856874913", Trunk: "c975dbf7b053"}, GitCredential{}, nil, logging(nil))
|
||||
if builtCount(built) != 3 {
|
||||
t.Fatalf("the check built %d of 3 module(s) of one tier: %+v", builtCount(built), built)
|
||||
}
|
||||
if len(scratch.builds) != 3 {
|
||||
t.Fatalf("the check kept %d build(s), not 3", len(scratch.builds))
|
||||
}
|
||||
}
|
||||
|
||||
// With a judge cache, the builds run while the tests run, where no container of the check reaches; a check that passed
|
||||
// keeps them, and one that failed lets go of every one and names none.
|
||||
func TestReplay716TheBuildsRunBesideTheTestsAndAFailedCheckLetsGoOfThem(t *testing.T) {
|
||||
withVersion(t, "builder-1")
|
||||
workspace := t.TempDir()
|
||||
if _, err := buildsBesideTheTests(filepath.Join(workspace, "judge-cache"), workspace, "build-716"); err == nil {
|
||||
t.Fatal("a judge cache inside the workspace was built in, where a pull request's script reaches")
|
||||
}
|
||||
for _, passed := range []bool{true, false} {
|
||||
cache := t.TempDir()
|
||||
place, err := buildsBesideTheTests(cache, workspace, "build-716")
|
||||
if err != nil || !withinDir(cache, place) || withinDir(workspace, place) {
|
||||
t.Fatalf("the builds' place %q (%v) is not the judge cache's, outside the workspace", place, err)
|
||||
}
|
||||
engine, spec, held := threeAtOnce(t)
|
||||
scratch := &aLockedScratch{aScratch: newScratch()}
|
||||
spec.BuildWith = &CheckBuilding{Publish: engine, Scratch: scratch, Held: held}
|
||||
judged := CheckVerdict{Head: "98e856874913", Trunk: "c975dbf7b053", Merge: "2d22279b1d07"}
|
||||
beside := startBesideTheTests(t.Context(), engine.run, spec, place, filepath.Join(t.TempDir(), "merged"),
|
||||
judged, GitCredential{}, nil, logging(nil))
|
||||
// The tests: the builds are under way before the check has a verdict.
|
||||
select {
|
||||
case <-engine.all:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("nothing was built while the tests ran")
|
||||
}
|
||||
verdict := CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "pass"}}
|
||||
if !passed {
|
||||
verdict.Repo.Verdict = "fail"
|
||||
}
|
||||
built := beside.settle(t.Context(), verdict)
|
||||
beside.abandon()
|
||||
switch {
|
||||
case passed && (builtCount(built) != 3 || len(scratch.builds) != 3 || len(scratch.discarded) != 0):
|
||||
t.Fatalf("a check that passed: built %+v, kept %d, let go of %d", built, len(scratch.builds),
|
||||
len(scratch.discarded))
|
||||
case !passed && (built != nil || len(scratch.builds) != 0 || len(scratch.discarded) != 3):
|
||||
t.Fatalf("a check that failed: named %+v, kept %d, let go of %d", built, len(scratch.builds),
|
||||
len(scratch.discarded))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,623 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// One build per change (novox/hq ADR 0350).
|
||||
//
|
||||
// **The merge check builds what the merge would move** (§1): every artifact of every module the delivery plan
|
||||
// moves, from the merged tree it checks, into the artifact store's scratch namespace — never a module's own —
|
||||
// kept under a **reuse key**: the source fingerprint as the build takes it (the module's tree, its contexts' trees,
|
||||
// its bases and toolchains by digest; issue 280) with the builder's own version and, for an upstream artifact, the
|
||||
// image it copies. **The walk of a trunk commit takes that build** (§2): before building, it takes the reuse key
|
||||
// of the trunk commit's tree the same way, and where a check build holds the same key it copies it by digest into
|
||||
// the module's namespace and answers with that, building nothing. Otherwise it builds and says why (§3): no check
|
||||
// built it, the key differs (naming the part), or the check build is gone; a copy that fails is said and built.
|
||||
//
|
||||
// **What the key names is what a fresh build could differ by.** The tree is every file the recipe and the
|
||||
// compiler see; the contexts are the other repositories it reads, by tree; the bases and toolchains are what it
|
||||
// stands on, by digest; the builder's version is how it is built. A build whose source does not pin it — one that
|
||||
// resolves packages from the mesh's registry when it is built, or copies an upstream image named by tag — has no
|
||||
// key, and is always built.
|
||||
|
||||
// Version is this builder's own version, as the reuse key takes it: the digest of the running executable, so any
|
||||
// change to how it builds is a change of key (§1). Set once by the build seat's holder (SelfVersion); empty — no
|
||||
// key, every build its own — when it cannot be said.
|
||||
var Version string
|
||||
|
||||
// SelfVersion is the version the reuse key takes: the running executable's digest, or the version stamped at build
|
||||
// when the executable cannot be read. Empty when neither says anything ("development" says nothing: every unstamped
|
||||
// builder would read as one).
|
||||
func SelfVersion(stamped string) string {
|
||||
if exe, err := os.Executable(); err == nil {
|
||||
if f, err := os.Open(exe); err == nil {
|
||||
defer f.Close()
|
||||
sum := sha256.New()
|
||||
if _, err := io.Copy(sum, f); err == nil {
|
||||
return "exe:sha256:" + hex.EncodeToString(sum.Sum(nil))
|
||||
}
|
||||
}
|
||||
}
|
||||
if stamped == "" || stamped == "development" {
|
||||
return ""
|
||||
}
|
||||
return stamped
|
||||
}
|
||||
|
||||
// ScratchPrefix is the namespace a check builds into (novox/hq ADR 0239 §9, ADR 0350 §1): `scratch/<module>/<artifact>`.
|
||||
// Registration refuses a reference into it; only the walk's copy, in the module's own namespace, is registered.
|
||||
const ScratchPrefix = "scratch/"
|
||||
|
||||
// checksRepository holds one document per check that built: which builds it made, of which pull request, when.
|
||||
// What the holder lets go of by (§5).
|
||||
const checksRepository = ScratchPrefix + "checks"
|
||||
|
||||
// reuseTagPrefix names a check build's record under its module's scratch repository: `r1-<hex>`.
|
||||
const reuseTagPrefix = "r1-"
|
||||
|
||||
// CheckBuildsKept is how long a check build is kept in the scratch namespace (§5).
|
||||
const CheckBuildsKept = 7 * 24 * time.Hour
|
||||
|
||||
// The parts of a reuse key, as the walk names the one that differs (§3).
|
||||
const (
|
||||
partModule = "the module"
|
||||
partTree = "the build source"
|
||||
partBases = "a base image"
|
||||
partBuilder = "the builder's version"
|
||||
partContext = "a context"
|
||||
partTool = "a toolchain"
|
||||
partUpstream = "an upstream image"
|
||||
)
|
||||
|
||||
// reuseKey is a build's reuse key and the parts it was made from; Key empty, with why, when it has none.
|
||||
type reuseKey struct {
|
||||
Key string
|
||||
Parts map[string]string
|
||||
Why string
|
||||
}
|
||||
|
||||
// keyOf is the reuse key of what src says the build was made from, built by version (ADR 0350 §1).
|
||||
func keyOf(src *sourceInputs, manifest catalogue.Manifest, version string) reuseKey {
|
||||
if src == nil || src.unpinned != "" || src.tree == "" {
|
||||
why := "its source does not pin the build"
|
||||
if src != nil && src.unpinned != "" {
|
||||
why = src.unpinned
|
||||
} else if src != nil && src.tree == "" {
|
||||
why = "its tree could not be named"
|
||||
}
|
||||
return reuseKey{Why: why}
|
||||
}
|
||||
if version == "" {
|
||||
return reuseKey{Why: "the builder's version cannot be said"}
|
||||
}
|
||||
parts := map[string]string{partModule: src.module, partTree: src.tree, partBuilder: version}
|
||||
var bases []string
|
||||
seen := map[string]bool{}
|
||||
for _, b := range src.bases {
|
||||
if d := referenceDigest(b); !seen[d] {
|
||||
seen[d] = true
|
||||
bases = append(bases, d)
|
||||
}
|
||||
}
|
||||
sort.Strings(bases)
|
||||
parts[partBases] = strings.Join(bases, " ")
|
||||
for a, t := range src.contexts {
|
||||
parts[partContext+" "+a] = t
|
||||
}
|
||||
for a, t := range src.toolchains {
|
||||
parts[partTool+" "+a] = t
|
||||
}
|
||||
// An upstream artifact is copied, not built: what it copies is in module.json, and so in the tree — but a tag
|
||||
// names whatever its registry says today, so only an image pinned by digest is the same image tomorrow.
|
||||
if manifest.Build != nil {
|
||||
for _, a := range manifest.Build.Artifacts {
|
||||
if a.Kind != catalogue.ArtifactUpstream {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(a.From, "@sha256:") {
|
||||
return reuseKey{Why: fmt.Sprintf("%s copies %s, named by tag, which can move", a.Name, a.From)}
|
||||
}
|
||||
parts[partUpstream+" "+a.Name] = a.From
|
||||
}
|
||||
}
|
||||
lines := make([]string, 0, len(parts))
|
||||
for k, v := range parts {
|
||||
lines = append(lines, k+"="+v)
|
||||
}
|
||||
sort.Strings(lines)
|
||||
sum := sha256.Sum256([]byte(strings.Join(lines, "\n")))
|
||||
return reuseKey{Key: "r1:" + hex.EncodeToString(sum[:]), Parts: parts}
|
||||
}
|
||||
|
||||
// tagOf is the tag a key's record is kept under.
|
||||
func tagOf(key string) string { return reuseTagPrefix + strings.TrimPrefix(key, "r1:") }
|
||||
|
||||
// differing names the parts of two keys that differ, in a fixed order: what the walk says when it builds (§3).
|
||||
func differing(was, now map[string]string) []string {
|
||||
names := map[string]bool{}
|
||||
for k := range was {
|
||||
names[k] = true
|
||||
}
|
||||
for k := range now {
|
||||
names[k] = true
|
||||
}
|
||||
var out []string
|
||||
for k := range names {
|
||||
if was[k] != now[k] {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// PullRef names a pull request.
|
||||
type PullRef struct {
|
||||
Owner string `json:"owner"`
|
||||
Repo string `json:"repo"`
|
||||
Number int `json:"number"`
|
||||
}
|
||||
|
||||
func (p PullRef) String() string { return fmt.Sprintf("%s/%s#%d", p.Owner, p.Repo, p.Number) }
|
||||
|
||||
// slug is the pull request as a tag's first part: owner.repo.number, every other character a dash.
|
||||
func (p PullRef) slug() string {
|
||||
clean := func(s string) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_' || r == '-' {
|
||||
return r
|
||||
}
|
||||
return '-'
|
||||
}, s)
|
||||
}
|
||||
return clean(p.Owner) + "." + clean(p.Repo) + "." + strconv.Itoa(p.Number)
|
||||
}
|
||||
|
||||
// CheckBuild is one module a check built into the scratch namespace, kept under its reuse key (§1).
|
||||
type CheckBuild struct {
|
||||
Key string `json:"key"`
|
||||
Module string `json:"module"`
|
||||
Parts map[string]string `json:"parts"`
|
||||
// Made are its artifacts, in the scratch namespace.
|
||||
Made []catalogue.Built `json:"made"`
|
||||
// Pull is the pull request checked, Head its head and Trunk the trunk commit it was merged into; Check the
|
||||
// check's ask; At when it was built.
|
||||
Pull PullRef `json:"pull"`
|
||||
Head string `json:"head,omitempty"`
|
||||
Trunk string `json:"trunk,omitempty"`
|
||||
Check string `json:"check"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// Said is the check build as the walk names it: "the check of owner/repo#n at head H (trunk T)".
|
||||
func (c CheckBuild) Said() string {
|
||||
s := "the check of " + c.Pull.String()
|
||||
if c.Head != "" {
|
||||
s += " at head " + twelve(c.Head)
|
||||
}
|
||||
if c.Trunk != "" {
|
||||
s += " (merged into trunk " + twelve(c.Trunk) + ")"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// checkIndex is one check's document under checksRepository: every build it made.
|
||||
type checkIndex struct {
|
||||
Pull PullRef `json:"pull"`
|
||||
Check string `json:"check"`
|
||||
At time.Time `json:"at"`
|
||||
Builds []CheckBuild `json:"builds"`
|
||||
}
|
||||
|
||||
// Scratch is where check builds are kept and taken from: the artifact store's scratch namespace (ADR 0350). The
|
||||
// Registry is one; a test's is a map.
|
||||
type Scratch interface {
|
||||
// Keep records a check's builds: each under its module and key, and the check's own document.
|
||||
Keep(ctx context.Context, pull PullRef, check string, at time.Time, builds []CheckBuild) error
|
||||
// Find is the check build kept under a module's key; false when none is.
|
||||
Find(ctx context.Context, module, key string) (CheckBuild, bool, error)
|
||||
// OfPulls are the check builds kept for those pull requests, newest first: what the walk names a differing
|
||||
// key from.
|
||||
OfPulls(ctx context.Context, pulls []PullRef) ([]CheckBuild, error)
|
||||
// Take copies one artifact of a check build by digest into the module's own namespace, and answers it there.
|
||||
Take(ctx context.Context, module string, made catalogue.Built) (catalogue.Built, error)
|
||||
// Forget lets go of every check build of a pull request, and Expire of every check older than the bound;
|
||||
// each answers how many checks it let go of.
|
||||
Forget(ctx context.Context, pull PullRef) (int, error)
|
||||
Expire(ctx context.Context, older time.Duration, now time.Time) (int, error)
|
||||
// Discard lets go of what builds a check could not keep made: nothing would ever find them.
|
||||
Discard(ctx context.Context, builds []CheckBuild) error
|
||||
}
|
||||
|
||||
// scratchPublisher is a Publisher into the scratch namespace: what a check builds is never under a module's
|
||||
// own name (ADR 0239 §9). A base the build stands on is copied where every build copies it.
|
||||
type scratchPublisher struct{ to Publisher }
|
||||
|
||||
func (s scratchPublisher) PublishImage(ctx context.Context, localTag, repository string) (string, error) {
|
||||
return s.to.PublishImage(ctx, localTag, ScratchPrefix+repository)
|
||||
}
|
||||
|
||||
func (s scratchPublisher) PublishArchive(ctx context.Context, repository string, body []byte, digest string) (string, error) {
|
||||
return s.to.PublishArchive(ctx, ScratchPrefix+repository, body, digest)
|
||||
}
|
||||
|
||||
func (s scratchPublisher) LendBases(ctx context.Context, repository string, bases []string) (int, error) {
|
||||
if l, can := s.to.(BaseLender); can {
|
||||
return l.LendBases(ctx, ScratchPrefix+repository, bases)
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (s scratchPublisher) MirrorImage(ctx context.Context, from, repository string) (string, error) {
|
||||
m, can := s.to.(Mirrorer)
|
||||
if !can {
|
||||
return "", errors.New("the artifact store here cannot copy an upstream image")
|
||||
}
|
||||
return m.MirrorImage(ctx, from, ScratchPrefix+repository)
|
||||
}
|
||||
|
||||
func (s scratchPublisher) MirrorBase(ctx context.Context, from, repository, former string) (string, error) {
|
||||
if m, can := s.to.(BaseMirrorer); can {
|
||||
return m.MirrorBase(ctx, from, repository, former)
|
||||
}
|
||||
if m, can := s.to.(Mirrorer); can {
|
||||
return m.MirrorImage(ctx, from, repository)
|
||||
}
|
||||
return "", errors.New("the artifact store here cannot copy a base")
|
||||
}
|
||||
|
||||
// The registry as the scratch namespace.
|
||||
|
||||
func (r Registry) store() artifacts.Store {
|
||||
return artifacts.Store{Address: r.Address, HTTP: r.client()}
|
||||
}
|
||||
|
||||
const recordMedia = "application/vnd.novox.mesh.check-build.v1+json"
|
||||
|
||||
// Keep puts each build's record under its module's scratch repository, by key, and the check's document.
|
||||
func (r Registry) Keep(ctx context.Context, pull PullRef, check string, at time.Time, builds []CheckBuild) error {
|
||||
store := r.store()
|
||||
for _, b := range builds {
|
||||
body, err := json.Marshal(b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := store.PutTagged(ctx, ScratchPrefix+b.Module, tagOf(b.Key), recordMedia, body); err != nil {
|
||||
return fmt.Errorf("keeping %s's check build: %w", b.Module, err)
|
||||
}
|
||||
}
|
||||
body, err := json.Marshal(checkIndex{Pull: pull, Check: check, At: at, Builds: builds})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := store.PutTagged(ctx, checksRepository, pull.slug()+"."+check, recordMedia, body); err != nil {
|
||||
return fmt.Errorf("keeping which builds the check of %s made: %w", pull, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Find reads the record kept under a module's key.
|
||||
func (r Registry) Find(ctx context.Context, module, key string) (CheckBuild, bool, error) {
|
||||
body, _, err := r.store().GetTagged(ctx, ScratchPrefix+module, tagOf(key))
|
||||
if errors.Is(err, artifacts.ErrNoTag) {
|
||||
return CheckBuild{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return CheckBuild{}, false, err
|
||||
}
|
||||
var b CheckBuild
|
||||
if err := json.Unmarshal(body, &b); err != nil {
|
||||
return CheckBuild{}, false, fmt.Errorf("%s's check build under %s is not one: %w", module, tagOf(key), err)
|
||||
}
|
||||
if b.Key != key || b.Module != module {
|
||||
return CheckBuild{}, false, nil
|
||||
}
|
||||
return b, true, nil
|
||||
}
|
||||
|
||||
// checks are the check documents kept, by tag; only those of the pull requests named when any are.
|
||||
func (r Registry) checks(ctx context.Context, pulls []PullRef) (map[string]checkIndex, error) {
|
||||
store := r.store()
|
||||
tags, err := store.Tags(ctx, checksRepository)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
wanted := map[string]PullRef{}
|
||||
for _, p := range pulls {
|
||||
wanted[p.slug()+"."] = p
|
||||
}
|
||||
out := map[string]checkIndex{}
|
||||
for _, tag := range tags {
|
||||
if len(pulls) > 0 {
|
||||
match := false
|
||||
for prefix := range wanted {
|
||||
match = match || strings.HasPrefix(tag, prefix)
|
||||
}
|
||||
if !match {
|
||||
continue
|
||||
}
|
||||
}
|
||||
body, _, err := store.GetTagged(ctx, checksRepository, tag)
|
||||
if errors.Is(err, artifacts.ErrNoTag) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var c checkIndex
|
||||
if err := json.Unmarshal(body, &c); err != nil {
|
||||
continue // not a document this wrote: left to nobody, said by nothing — it names no build
|
||||
}
|
||||
if len(pulls) > 0 {
|
||||
if p := wanted[c.Pull.slug()+"."]; p != c.Pull {
|
||||
continue // a slug two pull requests share: only the one named
|
||||
}
|
||||
}
|
||||
out[tag] = c
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// OfPulls are the check builds of those pull requests, newest first.
|
||||
func (r Registry) OfPulls(ctx context.Context, pulls []PullRef) ([]CheckBuild, error) {
|
||||
if len(pulls) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
checks, err := r.checks(ctx, pulls)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []CheckBuild
|
||||
for _, c := range checks {
|
||||
out = append(out, c.Builds...)
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].At.After(out[j].At) })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Take copies one artifact of a check build into the module's namespace by digest: an image's manifests as they
|
||||
// are, so the digest registered is the digest the check built; an archive's blob mounted across, and held.
|
||||
func (r Registry) Take(ctx context.Context, module string, made catalogue.Built) (catalogue.Built, error) {
|
||||
repository := module + "/" + made.Name
|
||||
from := ScratchPrefix + repository
|
||||
if made.Digest != "" || strings.Contains(made.Reference, "/blobs/") {
|
||||
digest := made.Digest
|
||||
if digest == "" {
|
||||
_, digest, _ = strings.Cut(made.Reference, "/blobs/")
|
||||
}
|
||||
where := upstream{base: "http://" + r.Address, repository: from, reference: digest}
|
||||
if err := r.copyBlob(ctx, &source{client: r.client(), mountFrom: from}, where, digest, repository); err != nil {
|
||||
return catalogue.Built{}, err
|
||||
}
|
||||
size, err := r.blobSize(ctx, repository, digest)
|
||||
if err != nil {
|
||||
return catalogue.Built{}, err
|
||||
}
|
||||
if err := r.hold(ctx, repository, digest, int(size)); err != nil {
|
||||
return catalogue.Built{}, err
|
||||
}
|
||||
return catalogue.Built{Name: made.Name, Kind: made.Kind, Digest: digest, Launchers: made.Launchers,
|
||||
Reference: "http://" + r.Address + "/v2/" + repository + "/blobs/" + digest}, nil
|
||||
}
|
||||
_, digest, pinned := strings.Cut(made.Reference, "@")
|
||||
if !pinned || !strings.HasPrefix(digest, "sha256:") {
|
||||
return catalogue.Built{}, fmt.Errorf("%s's check build %s is not pinned by digest", module, made.Reference)
|
||||
}
|
||||
reference, err := r.MirrorBase(ctx, r.Address+"/"+from+"@"+digest, repository, from)
|
||||
if err != nil {
|
||||
return catalogue.Built{}, err
|
||||
}
|
||||
if !strings.HasSuffix(reference, "@"+digest) {
|
||||
return catalogue.Built{}, fmt.Errorf("%s's check build was copied as %s, not as %s", module, reference, digest)
|
||||
}
|
||||
return catalogue.Built{Name: made.Name, Kind: made.Kind, Reference: reference, Launchers: made.Launchers}, nil
|
||||
}
|
||||
|
||||
// blobSize is the size the store says a blob has.
|
||||
func (r Registry) blobSize(ctx context.Context, repository, digest string) (int64, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodHead, "http://"+r.Address+"/v2/"+repository+"/blobs/"+digest, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
response, err := r.client().Do(request)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK || response.ContentLength < 0 {
|
||||
return 0, fmt.Errorf("%s/blobs/%s: %s", repository, digest, response.Status)
|
||||
}
|
||||
return response.ContentLength, nil
|
||||
}
|
||||
|
||||
// Forget lets go of every check build of a pull request (§5): what each check built in the scratch namespace, the
|
||||
// record under its key when it is still that check's, and the check's document.
|
||||
func (r Registry) Forget(ctx context.Context, pull PullRef) (int, error) {
|
||||
all, err := r.checks(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
mine := map[string]checkIndex{}
|
||||
for tag, c := range all {
|
||||
if c.Pull == pull {
|
||||
mine[tag] = c
|
||||
}
|
||||
}
|
||||
return r.letGo(ctx, mine, all)
|
||||
}
|
||||
|
||||
// Expire lets go of every check older than the bound (§5).
|
||||
func (r Registry) Expire(ctx context.Context, older time.Duration, now time.Time) (int, error) {
|
||||
checks, err := r.checks(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
old := map[string]checkIndex{}
|
||||
for tag, c := range checks {
|
||||
if now.Sub(c.At) > older {
|
||||
old[tag] = c
|
||||
}
|
||||
}
|
||||
return r.letGo(ctx, old, checks)
|
||||
}
|
||||
|
||||
// Discard lets go of builds no check document names.
|
||||
func (r Registry) Discard(ctx context.Context, builds []CheckBuild) error {
|
||||
all, err := r.checks(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = r.letGo(ctx, map[string]checkIndex{"": {Builds: builds}}, all)
|
||||
return err
|
||||
}
|
||||
|
||||
// letGo lets go of each check's builds, then its document; a check whose builds could not all be let go of keeps
|
||||
// its document, so the next sweep asks again. **A build another check still names stays**: deletion is by digest,
|
||||
// and two checks that built one digest in one repository — a check asked again, a dependent two pull requests move —
|
||||
// share it; it goes with the last check that names it. all are every check kept, the ones going among them.
|
||||
func (r Registry) letGo(ctx context.Context, checks, all map[string]checkIndex) (int, error) {
|
||||
store := r.store()
|
||||
stays := map[string]bool{}
|
||||
// recorded are the records under a key another check still names (novox/hq issue 703): a later check that kept
|
||||
// the build names it as the check's that built it, and the walk finds it by that record.
|
||||
recorded := map[string]bool{}
|
||||
for tag, c := range all {
|
||||
if _, going := checks[tag]; going {
|
||||
continue
|
||||
}
|
||||
for _, b := range c.Builds {
|
||||
recorded[b.Module+" "+b.Key+" "+b.Check] = true
|
||||
for _, m := range b.Made {
|
||||
stays[catalogue.Recorded(m.Reference)] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
gone := 0
|
||||
var failed []string
|
||||
for tag, c := range checks {
|
||||
ok := true
|
||||
for _, b := range c.Builds {
|
||||
for _, m := range b.Made {
|
||||
reference := catalogue.Recorded(m.Reference)
|
||||
if path, kept := catalogue.InArtifactStore(reference); !kept || !strings.HasPrefix(path, ScratchPrefix) {
|
||||
continue // never anything outside the scratch namespace
|
||||
}
|
||||
if stays[reference] {
|
||||
continue // another check still names it
|
||||
}
|
||||
if err := store.LetGo(ctx, reference); err != nil && !errors.Is(err, artifacts.Gone) {
|
||||
ok = false
|
||||
failed = append(failed, fmt.Sprintf("%s: %v", m.Reference, err))
|
||||
}
|
||||
}
|
||||
// The record under the key, only while it is still this check's: a later check of the same key
|
||||
// replaced it, and that one is its own check's to let go of. A build a check kept is that build's
|
||||
// check's record, and goes with the last check naming it (novox/hq issue 703).
|
||||
if kept, found, err := r.Find(ctx, b.Module, b.Key); err == nil && found &&
|
||||
(kept.Check == c.Check || kept.Check == b.Check) && !recorded[b.Module+" "+b.Key+" "+kept.Check] {
|
||||
if err := store.DeleteTagged(ctx, ScratchPrefix+b.Module, tagOf(b.Key)); err != nil && !errors.Is(err, artifacts.Gone) {
|
||||
ok = false
|
||||
failed = append(failed, fmt.Sprintf("%s's record: %v", b.Module, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if tag == "" {
|
||||
gone++
|
||||
continue
|
||||
}
|
||||
if err := store.DeleteTagged(ctx, checksRepository, tag); err != nil && !errors.Is(err, artifacts.Gone) {
|
||||
failed = append(failed, fmt.Sprintf("%s: %v", tag, err))
|
||||
continue
|
||||
}
|
||||
gone++
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
sort.Strings(failed)
|
||||
return gone, fmt.Errorf("not let go of: %s", strings.Join(failed, "; "))
|
||||
}
|
||||
return gone, nil
|
||||
}
|
||||
|
||||
// Reused is what the walk took from a check instead of building (§2, §4).
|
||||
type Reused struct {
|
||||
From CheckBuild
|
||||
// Made are the copies in the module's namespace: what is registered.
|
||||
Made []catalogue.Built
|
||||
}
|
||||
|
||||
// Words is the walk's word for a module it took: "reused from <check build>", with each digest (§4).
|
||||
func (r Reused) Words() string {
|
||||
var digests []string
|
||||
for _, m := range r.Made {
|
||||
d := m.Digest
|
||||
if d == "" {
|
||||
_, d, _ = strings.Cut(m.Reference, "@")
|
||||
}
|
||||
digests = append(digests, m.Name+" "+d)
|
||||
}
|
||||
return "reused from " + r.From.Said() + ": " + strings.Join(digests, ", ")
|
||||
}
|
||||
|
||||
// NamedBuild is a build a check's verdict names (ADR 0350 §2): its module, key and each artifact's digest.
|
||||
type NamedBuild struct {
|
||||
Pull PullRef
|
||||
Check string
|
||||
Module string
|
||||
Key string
|
||||
Digests map[string]string
|
||||
}
|
||||
|
||||
// DigestOf is the digest a reference names: an image's after `@`, an archive's after `/blobs/`.
|
||||
func DigestOf(reference string) string {
|
||||
if _, d, ok := strings.Cut(reference, "/blobs/"); ok {
|
||||
return d
|
||||
}
|
||||
if _, d, ok := strings.Cut(reference, "@"); ok {
|
||||
return d
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// unnamed is why a record found under a key is not taken: no verdict of the walk's pull requests names that module,
|
||||
// key and every one of its digests. Empty when one does.
|
||||
func unnamed(named []NamedBuild, module, key string, found CheckBuild) string {
|
||||
for _, n := range named {
|
||||
if n.Module != module || n.Key != key || n.Check != found.Check || len(n.Digests) != len(found.Made) {
|
||||
continue
|
||||
}
|
||||
same := true
|
||||
for _, m := range found.Made {
|
||||
d := m.Digest
|
||||
if d == "" {
|
||||
d = DigestOf(m.Reference)
|
||||
}
|
||||
same = same && d != "" && n.Digests[m.Name] == d
|
||||
}
|
||||
if same {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
// A build of another pull request, or of a head the forge did not merge, is in no verdict the walk is handed
|
||||
// (novox/hq issue 703): said with the check that made it, whose head names which head it was.
|
||||
return "a build is kept under this key, by " + found.Said() + ", and no check verdict of the walk's pull requests " +
|
||||
"at the head the forge merged names it with its digests: not taken"
|
||||
}
|
||||
@@ -0,0 +1,779 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// One build per change (novox/hq ADR 0350): the merge check builds what the merge would move into the scratch
|
||||
// namespace under its reuse key, and the walk of the trunk commit takes that build by digest instead of building.
|
||||
|
||||
// aScratch is the scratch namespace in memory: check builds by module and key, and what the walk took.
|
||||
type aScratch struct {
|
||||
builds map[string]CheckBuild
|
||||
took []string
|
||||
failTake bool
|
||||
}
|
||||
|
||||
func newScratch() *aScratch { return &aScratch{builds: map[string]CheckBuild{}} }
|
||||
|
||||
func (s *aScratch) Keep(_ context.Context, _ PullRef, _ string, _ time.Time, builds []CheckBuild) error {
|
||||
for _, b := range builds {
|
||||
s.builds[b.Module+" "+b.Key] = b
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *aScratch) Find(_ context.Context, module, key string) (CheckBuild, bool, error) {
|
||||
b, ok := s.builds[module+" "+key]
|
||||
return b, ok, nil
|
||||
}
|
||||
|
||||
func (s *aScratch) OfPulls(_ context.Context, pulls []PullRef) ([]CheckBuild, error) {
|
||||
var out []CheckBuild
|
||||
for _, b := range s.builds {
|
||||
if slices.Contains(pulls, b.Pull) {
|
||||
out = append(out, b)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *aScratch) Take(_ context.Context, module string, made catalogue.Built) (catalogue.Built, error) {
|
||||
if s.failTake {
|
||||
return catalogue.Built{}, errors.New("the store refused the mount")
|
||||
}
|
||||
s.took = append(s.took, made.Reference)
|
||||
made.Reference = strings.Replace(made.Reference, ScratchPrefix, "", 1)
|
||||
return made, nil
|
||||
}
|
||||
|
||||
func (s *aScratch) Forget(context.Context, PullRef) (int, error) { return 0, nil }
|
||||
func (s *aScratch) Expire(context.Context, time.Duration, time.Time) (int, error) { return 0, nil }
|
||||
func (s *aScratch) Discard(context.Context, []CheckBuild) error { return nil }
|
||||
|
||||
const reusedModule = `{"module":"meshboard","version":"1",
|
||||
"build":{"artifacts":[
|
||||
{"name":"server","kind":"image","from":"Dockerfile"},
|
||||
{"name":"look","kind":"archive","from":"files"}]},
|
||||
"resources":[
|
||||
{"id":"svc","type":"container","name":"meshboard","artifact":"server"},
|
||||
{"id":"theme","type":"archive","path":"/opt/meshboard","artifact":"look"}]}`
|
||||
|
||||
// aModuleAt is a repository holding the module under mod/, whose directory git names as tree, on its trunk.
|
||||
func aModuleAt(t *testing.T, tree string) (*recorded, Runner, string) {
|
||||
t.Helper()
|
||||
r := &recorded{tree: tree, contents: map[string]string{"mod/" + ManifestName: reusedModule,
|
||||
"mod/Dockerfile": "FROM scratch", "mod/files/theme.conf": "dark"}}
|
||||
run := func(ctx context.Context, dir, name string, args ...string) (string, error) {
|
||||
if name == "git" && len(args) > 0 && args[0] == "symbolic-ref" {
|
||||
r.ran = append(r.ran, name+" "+strings.Join(args, " "))
|
||||
return "origin/main\n", nil
|
||||
}
|
||||
return r.run(ctx, dir, name, args...)
|
||||
}
|
||||
return r, run, t.TempDir()
|
||||
}
|
||||
|
||||
func dockerBuilt(r *recorded) bool {
|
||||
for _, line := range r.ran {
|
||||
if strings.HasPrefix(line, "docker build") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var aPull = PullRef{Owner: "novox", Repo: "meshboard", Number: 7}
|
||||
|
||||
// aCheckBuilt is the check of aPull building the module from a tree, into the scratch namespace.
|
||||
func aCheckBuilt(t *testing.T, scratch *aScratch, tree string) CheckBuilt {
|
||||
t.Helper()
|
||||
r, run, workspace := aModuleAt(t, tree)
|
||||
spec := CheckSpec{ID: "build-1", Owner: aPull.Owner, Repo: aPull.Repo, Number: aPull.Number,
|
||||
Builds: []CheckBuildSpec{{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod",
|
||||
Ref: "main"}}}
|
||||
built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: scratch}, workspace, "", "",
|
||||
CheckVerdict{Head: "abcdefabcdef", Trunk: "123456123456"}, GitCredential{}, nil, logging(nil))
|
||||
if len(built) != 1 || built[0].NotBuilt != "" || built[0].Key == "" {
|
||||
t.Fatalf("the check did not build the module for its merge: %+v", built)
|
||||
}
|
||||
if !dockerBuilt(r) {
|
||||
t.Fatal("the check built no image")
|
||||
}
|
||||
for repository := range r.images {
|
||||
if !strings.HasPrefix(repository, ScratchPrefix) {
|
||||
t.Fatalf("the check published %s outside the scratch namespace", repository)
|
||||
}
|
||||
}
|
||||
for repository := range r.archives {
|
||||
if !strings.HasPrefix(repository, ScratchPrefix) {
|
||||
t.Fatalf("the check published %s outside the scratch namespace", repository)
|
||||
}
|
||||
}
|
||||
return built[0]
|
||||
}
|
||||
|
||||
// named is the walk's leave naming what a check's verdict says it built, as the controller reads it.
|
||||
func named(c CheckBuilt) []NamedBuild {
|
||||
n := NamedBuild{Pull: aPull, Check: "build-1", Module: c.Module, Key: c.Key, Digests: map[string]string{}}
|
||||
for _, m := range c.Made {
|
||||
d := m.Digest
|
||||
if d == "" {
|
||||
d = DigestOf(m.Reference)
|
||||
}
|
||||
n.Digests[m.Name] = d
|
||||
}
|
||||
return []NamedBuild{n}
|
||||
}
|
||||
|
||||
func withVersion(t *testing.T, v string) {
|
||||
t.Helper()
|
||||
was := Version
|
||||
Version = v
|
||||
t.Cleanup(func() { Version = was })
|
||||
}
|
||||
|
||||
// A merge whose reuse key equals its check's builds nothing again: the walk copies the check's build by digest
|
||||
// and registers that digest.
|
||||
func TestAWalkWhoseKeyIsItsChecksBuildsNothingAndRegistersTheChecksDigest(t *testing.T) {
|
||||
withVersion(t, "builder-1")
|
||||
scratch := newScratch()
|
||||
checked := aCheckBuilt(t, scratch, "tree-one")
|
||||
|
||||
r, run, workspace := aModuleAt(t, "tree-one")
|
||||
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)},
|
||||
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if dockerBuilt(r) || len(r.images) > 0 || len(r.archives) > 0 {
|
||||
t.Fatalf("the walk built again: ran %v, published %v %v", r.ran, r.images, r.archives)
|
||||
}
|
||||
if got.Reused == nil || got.Why != "" {
|
||||
t.Fatalf("the walk did not say it reused the check's build: reused %+v, why %q", got.Reused, got.Why)
|
||||
}
|
||||
if got.ReuseKey != checked.Key {
|
||||
t.Fatalf("the walk's key %s is not the check's %s", got.ReuseKey, checked.Key)
|
||||
}
|
||||
if len(scratch.took) != 2 {
|
||||
t.Fatalf("the walk copied %v, not the check's two artifacts", scratch.took)
|
||||
}
|
||||
for _, made := range checked.Made {
|
||||
registered := strings.Replace(made.Reference, ScratchPrefix, "", 1)
|
||||
found := false
|
||||
for _, b := range got.Built {
|
||||
found = found || (b.Reference == registered && b.Digest == made.Digest)
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("the walk registers %+v, not the check's %s copied", got.Built, made.Reference)
|
||||
}
|
||||
}
|
||||
if image, _ := got.Manifest.Resources[0]["image"].(string); image != "registry.invalid/meshboard/server@sha256:"+strings.Repeat("a", 64) {
|
||||
t.Fatalf("the manifest pins %q, not the check's digest in the module's namespace", image)
|
||||
}
|
||||
words := got.Reused.Words()
|
||||
if !strings.HasPrefix(words, "reused from the check of novox/meshboard#7 at head abcdefabcdef (merged into trunk 123456123456)") ||
|
||||
!strings.Contains(words, "sha256:") {
|
||||
t.Fatalf("the walk says %q", words)
|
||||
}
|
||||
}
|
||||
|
||||
// A record no verdict names is refused and built instead, and the walk says so: the store takes writes from anything
|
||||
// on its network, so a record planted under another merge's key is never registered.
|
||||
func TestARecordNoVerdictNamesIsRefusedAndBuilt(t *testing.T) {
|
||||
withVersion(t, "builder-1")
|
||||
scratch := newScratch()
|
||||
checked := aCheckBuilt(t, scratch, "tree-one")
|
||||
for name, leave := range map[string][]NamedBuild{
|
||||
"no verdict": nil,
|
||||
"another digest": func() []NamedBuild {
|
||||
n := named(checked)
|
||||
n[0].Digests["server"] = "sha256:" + strings.Repeat("e", 64)
|
||||
return n
|
||||
}(),
|
||||
"another check": func() []NamedBuild { n := named(checked); n[0].Check = "build-9"; return n }(),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
scratch.took = nil
|
||||
r, run, workspace := aModuleAt(t, "tree-one")
|
||||
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: leave},
|
||||
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Reused != nil || len(scratch.took) > 0 || !dockerBuilt(r) {
|
||||
t.Fatalf("a record no verdict names was taken: %+v, copied %v", got.Reused, scratch.took)
|
||||
}
|
||||
if !strings.Contains(got.Why, "no check verdict of the walk's pull requests at the head the forge merged names it") ||
|
||||
!strings.Contains(got.Why, "at head abcdefabcdef") {
|
||||
t.Fatalf("the walk says %q", got.Why)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A differing key builds, and says which part differs.
|
||||
func TestAWalkWhoseKeyDiffersBuildsAndSaysWhy(t *testing.T) {
|
||||
withVersion(t, "builder-1")
|
||||
scratch := newScratch()
|
||||
checked := aCheckBuilt(t, scratch, "tree-one")
|
||||
|
||||
r, run, workspace := aModuleAt(t, "tree-two")
|
||||
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)},
|
||||
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !dockerBuilt(r) || got.Reused != nil {
|
||||
t.Fatalf("a walk whose source differs did not build: reused %+v", got.Reused)
|
||||
}
|
||||
if !strings.Contains(got.Why, "the key differs") || !strings.Contains(got.Why, partTree) {
|
||||
t.Fatalf("the walk says %q, not which part of the key differs", got.Why)
|
||||
}
|
||||
|
||||
// And a builder of another version builds too, saying so.
|
||||
withVersion(t, "builder-2")
|
||||
r, run, workspace = aModuleAt(t, "tree-one")
|
||||
got, err = BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)},
|
||||
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !dockerBuilt(r) || !strings.Contains(got.Why, partBuilder) {
|
||||
t.Fatalf("a newer builder took the older one's build, or did not say why not: %q", got.Why)
|
||||
}
|
||||
|
||||
// No check of the walk's pull requests built it: said.
|
||||
r, run, workspace = aModuleAt(t, "tree-one")
|
||||
got, _ = BuildReusing(t.Context(), run, r, Reuse{Scratch: newScratch(), Pulls: []PullRef{aPull}},
|
||||
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if !dockerBuilt(r) || !strings.Contains(got.Why, "no check of novox/meshboard#7 built it") {
|
||||
t.Fatalf("the walk says %q", got.Why)
|
||||
}
|
||||
}
|
||||
|
||||
// A copy that fails is said and falls back to a build; it never fails the walk.
|
||||
func TestAFailedCopyOfAChecksBuildFallsBackToBuilding(t *testing.T) {
|
||||
withVersion(t, "builder-1")
|
||||
scratch := newScratch()
|
||||
checked := aCheckBuilt(t, scratch, "tree-one")
|
||||
scratch.failTake = true
|
||||
|
||||
r, run, workspace := aModuleAt(t, "tree-one")
|
||||
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)},
|
||||
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("a failed copy failed the walk: %v", err)
|
||||
}
|
||||
if !dockerBuilt(r) || got.Reused != nil || len(r.images) == 0 {
|
||||
t.Fatal("a failed copy did not fall back to a build")
|
||||
}
|
||||
if !strings.Contains(got.Why, "copying meshboard's") || !strings.Contains(got.Why, "refused the mount") {
|
||||
t.Fatalf("the walk says %q", got.Why)
|
||||
}
|
||||
for repository := range r.images {
|
||||
if strings.HasPrefix(repository, ScratchPrefix) {
|
||||
t.Fatalf("the walk's own build went into the scratch namespace: %s", repository)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Only a walk takes a check's build: a plain build — a check's, a dry run's, `build --ref`'s — builds, and a commit
|
||||
// off the trunk never takes one.
|
||||
func TestOnlyAWalkOfATrunkCommitTakesAChecksBuild(t *testing.T) {
|
||||
withVersion(t, "builder-1")
|
||||
scratch := newScratch()
|
||||
checked := aCheckBuilt(t, scratch, "tree-one")
|
||||
|
||||
r, run, workspace := aModuleAt(t, "tree-one")
|
||||
if _, err := Build(t.Context(), run, r, "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil,
|
||||
Npmrc{}, GitCredential{}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !dockerBuilt(r) || len(scratch.took) > 0 {
|
||||
t.Fatal("a build not asked to reuse took a check's build")
|
||||
}
|
||||
|
||||
r, _, workspace = aModuleAt(t, "tree-one") // no trunk said: off it
|
||||
got, err := BuildReusing(t.Context(), r.run, r, Reuse{Scratch: scratch, Named: named(checked)}, "https://forge.invalid/meshboard.git",
|
||||
"mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !dockerBuilt(r) || got.Reused != nil || !strings.Contains(got.Why, "not on its trunk") {
|
||||
t.Fatalf("a commit off the trunk took a check's build: %q", got.Why)
|
||||
}
|
||||
}
|
||||
|
||||
// A check that did not pass builds nothing for its merge, and one asked to build nothing builds nothing.
|
||||
func TestACheckThatFailedBuildsNothingForItsMerge(t *testing.T) {
|
||||
if skipsTheBuilds(CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "fail"}}) == "" {
|
||||
t.Fatal("a check whose repository layer failed builds for its merge")
|
||||
}
|
||||
if skipsTheBuilds(CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "warning"}}) != "" {
|
||||
t.Fatal("a check that passed with a warning builds nothing for its merge")
|
||||
}
|
||||
r, run, workspace := aModuleAt(t, "tree-one")
|
||||
if built := buildForTheMerge(t.Context(), run, CheckSpec{ID: "build-1"}, CheckBuilding{Publish: r, Scratch: newScratch()},
|
||||
workspace, "", "", CheckVerdict{}, GitCredential{}, nil, logging(nil)); built != nil || len(r.ran) > 0 {
|
||||
t.Fatalf("a check moving no module built %v, ran %v", built, r.ran)
|
||||
}
|
||||
}
|
||||
|
||||
// A module whose source does not pin its build has no key and is not built by the check; what stands after it
|
||||
// is not built either.
|
||||
func TestACheckBuildsNoModuleWithoutAKeyNorWhatStandsAfterIt(t *testing.T) {
|
||||
withVersion(t, "builder-1")
|
||||
r, run, workspace := aModuleAt(t, "tree-one")
|
||||
r.contents["mod/"+ManifestName] = `{"module":"meshboard","version":"1","build":{"artifacts":[
|
||||
{"name":"mirror","kind":"upstream","from":"docker.io/library/alpine:3"}]}}`
|
||||
spec := CheckSpec{ID: "build-1", Owner: "novox", Repo: "meshboard", Number: 7, Builds: []CheckBuildSpec{
|
||||
{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod", Tier: 0},
|
||||
{Module: "after", Repository: "https://forge.invalid/after.git", Tier: 1}}}
|
||||
built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: newScratch()}, workspace, "", "",
|
||||
CheckVerdict{}, GitCredential{}, nil, logging(nil))
|
||||
if len(built) != 2 || !strings.Contains(built[0].NotBuilt, "named by tag") ||
|
||||
!strings.Contains(built[1].NotBuilt, "stands after meshboard") {
|
||||
t.Fatalf("built %+v", built)
|
||||
}
|
||||
if len(r.images) > 0 {
|
||||
t.Fatalf("a module without a key was published: %v", r.images)
|
||||
}
|
||||
}
|
||||
|
||||
// What the key names: every input a fresh build could differ by.
|
||||
func TestTheReuseKeyNamesEveryInputOfTheBuild(t *testing.T) {
|
||||
src := newSourceInputs("m")
|
||||
src.tree = "t1"
|
||||
src.bases = []string{"store:5000/b/x@sha256:" + strings.Repeat("b", 64)}
|
||||
src.contexts["server"] = "ctx1"
|
||||
src.toolchains["tools"] = "go sha256:c 1234"
|
||||
manifest := catalogue.Manifest{Module: "m"}
|
||||
base := keyOf(src, manifest, "v1")
|
||||
if base.Key == "" {
|
||||
t.Fatalf("no key: %s", base.Why)
|
||||
}
|
||||
for name, change := range map[string]func(*sourceInputs) string{
|
||||
partTree: func(s *sourceInputs) string { s.tree = "t2"; return "v1" },
|
||||
partBases: func(s *sourceInputs) string {
|
||||
s.bases = []string{"other@sha256:" + strings.Repeat("c", 64)}
|
||||
return "v1"
|
||||
},
|
||||
partContext: func(s *sourceInputs) string { s.contexts["server"] = "ctx2"; return "v1" },
|
||||
partTool: func(s *sourceInputs) string { s.toolchains["tools"] = "go sha256:d 1234"; return "v1" },
|
||||
partBuilder: func(*sourceInputs) string { return "v2" },
|
||||
} {
|
||||
s := src.copy()
|
||||
version := change(s)
|
||||
if k := keyOf(s, manifest, version); k.Key == base.Key {
|
||||
t.Errorf("a change of %s left the key the same", name)
|
||||
}
|
||||
}
|
||||
// A base copied to another address is the same base: the key names it by digest.
|
||||
moved := src.copy()
|
||||
moved.bases = []string{"elsewhere:5000/scratch/b/x@sha256:" + strings.Repeat("b", 64)}
|
||||
if keyOf(moved, manifest, "v1").Key != base.Key {
|
||||
t.Error("the same base at another address changed the key")
|
||||
}
|
||||
if k := keyOf(src, manifest, ""); k.Key != "" {
|
||||
t.Error("a builder whose version cannot be said keyed its build")
|
||||
}
|
||||
unpinned := src.copy()
|
||||
unpinned.notPinned("it resolves packages from the mesh's registry at build time")
|
||||
if k := keyOf(unpinned, manifest, "v1"); k.Key != "" || !strings.Contains(k.Why, "registry") {
|
||||
t.Errorf("a build its source does not pin was keyed: %+v", k)
|
||||
}
|
||||
}
|
||||
|
||||
// aTaggingRegistry is the in-memory registry with tags and their listing, as the store's scratch namespace needs.
|
||||
type aTaggingRegistry struct {
|
||||
*aRegistryOfRepositories
|
||||
tags map[string]map[string]string // repository → tag → manifest digest
|
||||
}
|
||||
|
||||
func (m *aTaggingRegistry) serve() http.Handler {
|
||||
under := m.handler()
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if repository, ok := strings.CutSuffix(strings.TrimPrefix(r.URL.Path, "/v2/"), "/tags/list"); ok {
|
||||
m.mu.Lock()
|
||||
var tags []string
|
||||
for tag := range m.tags[repository] {
|
||||
tags = append(tags, tag)
|
||||
}
|
||||
m.mu.Unlock()
|
||||
if tags == nil {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"name": repository, "tags": tags})
|
||||
return
|
||||
}
|
||||
repository, kind, rest := splitPath(r.URL.Path)
|
||||
if kind == "blobs" && r.Method == http.MethodHead {
|
||||
m.mu.Lock()
|
||||
body, linked := m.blobs[rest], m.links[repository][rest]
|
||||
m.mu.Unlock()
|
||||
if !linked {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
if kind != "manifests" {
|
||||
under.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
if !strings.HasPrefix(rest, "sha256:") {
|
||||
m.mu.Lock()
|
||||
switch r.Method {
|
||||
case http.MethodPut:
|
||||
body, _ := readAll(r)
|
||||
if m.tags[repository] == nil {
|
||||
m.tags[repository] = map[string]string{}
|
||||
}
|
||||
m.tags[repository][rest] = digestOf(body)
|
||||
m.manifests[repository+"@"+digestOf(body)] = body
|
||||
m.mu.Unlock()
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
return
|
||||
default:
|
||||
digest, ok := m.tags[repository][rest]
|
||||
body, held := m.manifests[repository+"@"+digest]
|
||||
m.mu.Unlock()
|
||||
if !ok || !held {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Docker-Content-Digest", digest)
|
||||
w.Header().Set("Content-Type", mediaTypeOf(body))
|
||||
if r.Method == http.MethodGet {
|
||||
_, _ = w.Write(body)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
}
|
||||
if r.Method == http.MethodDelete {
|
||||
m.mu.Lock()
|
||||
for _, tags := range m.tags {
|
||||
for tag, digest := range tags {
|
||||
if digest == rest {
|
||||
delete(tags, tag)
|
||||
}
|
||||
}
|
||||
}
|
||||
m.mu.Unlock()
|
||||
}
|
||||
under.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// A check's builds go after seven days, and when its pull request closes or is merged; the walk's copy in the
|
||||
// module's own namespace stays.
|
||||
func TestCheckBuildsExpireAfterSevenDaysAndGoWhenTheirPullRequestCloses(t *testing.T) {
|
||||
m := &aTaggingRegistry{aRegistryOfRepositories: newRegistryOfRepositories(), tags: map[string]map[string]string{}}
|
||||
server := httptest.NewServer(m.serve())
|
||||
defer server.Close()
|
||||
address := strings.TrimPrefix(server.URL, "http://")
|
||||
registry := Registry{Address: address}
|
||||
ctx := t.Context()
|
||||
now := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC)
|
||||
|
||||
image := func(repository, body string) string {
|
||||
digest := digestOf([]byte(body))
|
||||
m.manifests[repository+"@"+digest] = []byte(body)
|
||||
return address + "/" + repository + "@" + digest
|
||||
}
|
||||
check := func(pull PullRef, id string, at time.Time, key string) string {
|
||||
reference := image("scratch/meshboard/server", `{"mediaType":"`+mediaManifestOCI+`","layers":[],"n":"`+id+`"}`)
|
||||
build := CheckBuild{Key: key, Module: "meshboard", Made: []catalogue.Built{{Name: "server", Kind: "image",
|
||||
Reference: reference}}, Pull: pull, Check: id, At: at}
|
||||
if err := registry.Keep(ctx, pull, id, at, []CheckBuild{build}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return reference
|
||||
}
|
||||
held := func(reference string) bool {
|
||||
_, digest, _ := strings.Cut(reference, "@")
|
||||
_, ok := m.manifests["scratch/meshboard/server@"+digest]
|
||||
return ok
|
||||
}
|
||||
other := PullRef{Owner: "novox", Repo: "meshboard", Number: 8}
|
||||
old := check(aPull, "build-1", now.Add(-8*24*time.Hour), "r1:"+strings.Repeat("1", 64))
|
||||
fresh := check(aPull, "build-2", now.Add(-time.Hour), "r1:"+strings.Repeat("2", 64))
|
||||
others := check(other, "build-3", now.Add(-time.Hour), "r1:"+strings.Repeat("3", 64))
|
||||
walked := image("meshboard/server", `{"mediaType":"`+mediaManifestOCI+`","layers":[],"n":"walk"}`)
|
||||
|
||||
if _, found, err := registry.Find(ctx, "meshboard", "r1:"+strings.Repeat("2", 64)); err != nil || !found {
|
||||
t.Fatalf("a check build kept is not found by its key: %v %v", found, err)
|
||||
}
|
||||
n, err := registry.Expire(ctx, CheckBuildsKept, now)
|
||||
if err != nil || n != 1 {
|
||||
t.Fatalf("expired %d checks (%v), want the one eight days old", n, err)
|
||||
}
|
||||
if held(old) || !held(fresh) || !held(others) {
|
||||
t.Fatalf("after the week's sweep: old %v, fresh %v, another's %v", held(old), held(fresh), held(others))
|
||||
}
|
||||
if _, found, _ := registry.Find(ctx, "meshboard", "r1:"+strings.Repeat("1", 64)); found {
|
||||
t.Fatal("the expired check build's record is still found")
|
||||
}
|
||||
|
||||
n, err = registry.Forget(ctx, aPull)
|
||||
if err != nil || n != 1 {
|
||||
t.Fatalf("forgot %d checks of %s (%v)", n, aPull, err)
|
||||
}
|
||||
if held(fresh) || !held(others) {
|
||||
t.Fatalf("after %s closed: its build held %v, another pull request's %v", aPull, held(fresh), held(others))
|
||||
}
|
||||
if _, ok := m.manifests["meshboard/server@"+strings.SplitN(walked, "@", 2)[1]]; !ok {
|
||||
t.Fatal("letting go of check builds took the walk's copy in the module's namespace")
|
||||
}
|
||||
if left, _ := registry.OfPulls(ctx, []PullRef{aPull}); len(left) != 0 {
|
||||
t.Fatalf("%s still has check builds: %+v", aPull, left)
|
||||
}
|
||||
}
|
||||
|
||||
// The walk copies a check build by digest: the digest registered is the digest the check built, and the copy is
|
||||
// in the module's own namespace.
|
||||
func TestAChecksBuildIsCopiedByDigestIntoTheModulesNamespace(t *testing.T) {
|
||||
m := &aTaggingRegistry{aRegistryOfRepositories: newRegistryOfRepositories(), tags: map[string]map[string]string{}}
|
||||
server := httptest.NewServer(m.serve())
|
||||
defer server.Close()
|
||||
address := strings.TrimPrefix(server.URL, "http://")
|
||||
registry := Registry{Address: address}
|
||||
|
||||
layer := []byte("layer")
|
||||
m.blobs[digestOf(layer)] = layer
|
||||
m.link("scratch/meshboard/server", digestOf(layer))
|
||||
manifest := []byte(`{"mediaType":"` + mediaManifestOCI + `","layers":[{"digest":"` + digestOf(layer) + `","size":5}]}`)
|
||||
m.manifests["scratch/meshboard/server@"+digestOf(manifest)] = manifest
|
||||
archive := []byte("archive")
|
||||
m.blobs[digestOf(archive)] = archive
|
||||
m.link("scratch/meshboard/look", digestOf(archive))
|
||||
|
||||
got, err := registry.Take(t.Context(), "meshboard", catalogue.Built{Name: "server", Kind: "image",
|
||||
Reference: address + "/scratch/meshboard/server@" + digestOf(manifest)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Reference != address+"/meshboard/server@"+digestOf(manifest) {
|
||||
t.Fatalf("copied as %s", got.Reference)
|
||||
}
|
||||
if m.uploads != 0 || m.mounts != 1 {
|
||||
t.Fatalf("the image's layer was moved (%d uploads, %d mounts), not mounted", m.uploads, m.mounts)
|
||||
}
|
||||
got, err = registry.Take(t.Context(), "meshboard", catalogue.Built{Name: "look", Kind: "archive",
|
||||
Digest: digestOf(archive), Reference: "http://" + address + "/v2/scratch/meshboard/look/blobs/" + digestOf(archive)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Digest != digestOf(archive) || got.Reference != "http://"+address+"/v2/meshboard/look/blobs/"+digestOf(archive) ||
|
||||
!m.links["meshboard/look"][digestOf(archive)] {
|
||||
t.Fatalf("the archive was copied as %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Two checks that built one digest in one repository share it: letting go of one check's builds keeps it while the
|
||||
// other still names it, and the last to go takes it.
|
||||
func TestABuildTwoChecksShareGoesWithTheLastOfThem(t *testing.T) {
|
||||
m := &aTaggingRegistry{aRegistryOfRepositories: newRegistryOfRepositories(), tags: map[string]map[string]string{}}
|
||||
server := httptest.NewServer(m.serve())
|
||||
defer server.Close()
|
||||
address := strings.TrimPrefix(server.URL, "http://")
|
||||
registry := Registry{Address: address}
|
||||
ctx := t.Context()
|
||||
now := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC)
|
||||
|
||||
body := `{"mediaType":"` + mediaManifestOCI + `","layers":[]}`
|
||||
digest := digestOf([]byte(body))
|
||||
m.manifests["scratch/meshboard/server@"+digest] = []byte(body)
|
||||
reference := address + "/scratch/meshboard/server@" + digest
|
||||
other := PullRef{Owner: "novox", Repo: "meshboard", Number: 8}
|
||||
for i, pull := range []PullRef{aPull, other} {
|
||||
id := "build-" + strconv.Itoa(i+1)
|
||||
b := CheckBuild{Key: "r1:" + strings.Repeat(strconv.Itoa(i+1), 64), Module: "meshboard", Pull: pull, Check: id,
|
||||
At: now, Made: []catalogue.Built{{Name: "server", Kind: "image", Reference: reference}}}
|
||||
if err := registry.Keep(ctx, pull, id, now, []CheckBuild{b}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
held := func() bool { _, ok := m.manifests["scratch/meshboard/server@"+digest]; return ok }
|
||||
|
||||
if n, err := registry.Forget(ctx, aPull); err != nil || n != 1 {
|
||||
t.Fatalf("forgot %d (%v)", n, err)
|
||||
}
|
||||
if !held() {
|
||||
t.Fatal("letting go of one check's builds took a build another check still names")
|
||||
}
|
||||
if n, err := registry.Forget(ctx, other); err != nil || n != 1 {
|
||||
t.Fatalf("forgot %d (%v)", n, err)
|
||||
}
|
||||
if held() {
|
||||
t.Fatal("the build stayed after the last check naming it went")
|
||||
}
|
||||
}
|
||||
|
||||
// aCheckAgain is a later check (build-2) of aPull building the module from a tree, with the builds the build seat's
|
||||
// verdicts name for it (novox/hq issue 703).
|
||||
func aCheckAgain(t *testing.T, scratch *aScratch, tree string, leave []NamedBuild) (CheckBuilt, *recorded) {
|
||||
t.Helper()
|
||||
return aCheckOf(t, scratch, aPull, "abcdefabcdef", tree, leave)
|
||||
}
|
||||
|
||||
// aCheckOf is a later check (build-2) of a pull request at a head, the trunk moved since build-1.
|
||||
func aCheckOf(t *testing.T, scratch *aScratch, pull PullRef, head, tree string, leave []NamedBuild) (CheckBuilt, *recorded) {
|
||||
t.Helper()
|
||||
r, run, workspace := aModuleAt(t, tree)
|
||||
spec := CheckSpec{ID: "build-2", Owner: pull.Owner, Repo: pull.Repo, Number: pull.Number,
|
||||
Builds: []CheckBuildSpec{{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod",
|
||||
Ref: "main"}}}
|
||||
built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: scratch, Named: leave},
|
||||
workspace, "", "", CheckVerdict{Head: head, Trunk: "654321654321"}, GitCredential{}, nil, logging(nil))
|
||||
if len(built) != 1 || built[0].NotBuilt != "" || built[0].Key == "" {
|
||||
t.Fatalf("the second check did not answer the module for its merge: %+v", built)
|
||||
}
|
||||
return built[0], r
|
||||
}
|
||||
|
||||
// A check asked again when only the trunk moved, whose key is unchanged, builds nothing: it keeps the build an
|
||||
// earlier verdict names and names it by key and digest, so the walk can take it (novox/hq issue 703).
|
||||
func TestASecondCheckWhoseKeyIsUnchangedBuildsNothingAndNamesTheEarlierBuild(t *testing.T) {
|
||||
withVersion(t, "builder-1")
|
||||
scratch := newScratch()
|
||||
first := aCheckBuilt(t, scratch, "tree-one")
|
||||
|
||||
again, r := aCheckAgain(t, scratch, "tree-one", named(first))
|
||||
if dockerBuilt(r) || len(r.images) > 0 || len(r.archives) > 0 {
|
||||
t.Fatalf("the second check built again: ran %v, published %v %v", r.ran, r.images, r.archives)
|
||||
}
|
||||
if again.Key != first.Key || again.Kept != "build-1" || !reflect.DeepEqual(again.Made, first.Made) {
|
||||
t.Fatalf("the second check names %+v, not the first check's build %+v", again, first)
|
||||
}
|
||||
// The walk takes it by what the second verdict names.
|
||||
leave := named(again)
|
||||
leave[0].Check = again.Kept
|
||||
w, run, workspace := aModuleAt(t, "tree-one")
|
||||
got, err := BuildReusing(t.Context(), run, w, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: leave},
|
||||
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil || got.Reused == nil || dockerBuilt(w) {
|
||||
t.Fatalf("the walk did not take the kept build: %v, why %q", err, got.Why)
|
||||
}
|
||||
}
|
||||
|
||||
// A check whose key changed builds, as before.
|
||||
func TestASecondCheckWhoseKeyChangedBuilds(t *testing.T) {
|
||||
withVersion(t, "builder-1")
|
||||
scratch := newScratch()
|
||||
first := aCheckBuilt(t, scratch, "tree-one")
|
||||
|
||||
again, r := aCheckAgain(t, scratch, "tree-two", named(first))
|
||||
if !dockerBuilt(r) || again.Kept != "" || again.Key == first.Key {
|
||||
t.Fatalf("a check whose key changed did not build: %+v", again)
|
||||
}
|
||||
}
|
||||
|
||||
// A scratch record no verdict names is never kept: the store takes writes from anything on its network, so a
|
||||
// check builds rather than name a planted record (novox/hq issue 703, ADR 0350 §2).
|
||||
func TestASecondCheckKeepsNoRecordAVerdictDoesNotName(t *testing.T) {
|
||||
withVersion(t, "builder-1")
|
||||
scratch := newScratch()
|
||||
first := aCheckBuilt(t, scratch, "tree-one")
|
||||
for name, leave := range map[string][]NamedBuild{
|
||||
"no verdict": nil,
|
||||
"another digest": func() []NamedBuild {
|
||||
n := named(first)
|
||||
n[0].Digests["server"] = "sha256:" + strings.Repeat("e", 64)
|
||||
return n
|
||||
}(),
|
||||
"another key": func() []NamedBuild { n := named(first); n[0].Key = "r1:" + strings.Repeat("f", 64); return n }(),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
again, r := aCheckAgain(t, scratch, "tree-one", leave)
|
||||
if !dockerBuilt(r) || again.Kept != "" {
|
||||
t.Fatalf("a record no verdict names was kept: %+v", again)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A build a later check kept stays when the check that built it goes: its artifacts and its record, which the later
|
||||
// check's verdict names (novox/hq issue 703, ADR 0350 §5).
|
||||
func TestABuildALaterCheckKeptStaysWhenTheCheckThatBuiltItGoes(t *testing.T) {
|
||||
m := &aTaggingRegistry{aRegistryOfRepositories: newRegistryOfRepositories(), tags: map[string]map[string]string{}}
|
||||
server := httptest.NewServer(m.serve())
|
||||
defer server.Close()
|
||||
address := strings.TrimPrefix(server.URL, "http://")
|
||||
registry := Registry{Address: address}
|
||||
ctx := t.Context()
|
||||
now := time.Date(2026, 10, 14, 9, 0, 0, 0, time.UTC)
|
||||
|
||||
body := `{"mediaType":"` + mediaManifestOCI + `","layers":[]}`
|
||||
digest := digestOf([]byte(body))
|
||||
m.manifests["scratch/meshboard/server@"+digest] = []byte(body)
|
||||
key := "r1:" + strings.Repeat("1", 64)
|
||||
b := CheckBuild{Key: key, Module: "meshboard", Pull: aPull, Check: "build-1", At: now,
|
||||
Made: []catalogue.Built{{Name: "server", Kind: "image", Reference: address + "/scratch/meshboard/server@" + digest}}}
|
||||
if err := registry.Keep(ctx, aPull, "build-1", now, []CheckBuild{b}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Another pull request's check keeps it: its own document names the build, which stays the first check's.
|
||||
other := PullRef{Owner: "novox", Repo: "meshboard", Number: 8}
|
||||
if err := registry.Keep(ctx, other, "build-2", now, []CheckBuild{b}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held := func() bool { _, ok := m.manifests["scratch/meshboard/server@"+digest]; return ok }
|
||||
found := func() bool { _, ok, _ := registry.Find(ctx, "meshboard", key); return ok }
|
||||
|
||||
if n, err := registry.Forget(ctx, aPull); err != nil || n != 1 {
|
||||
t.Fatalf("forgot %d (%v)", n, err)
|
||||
}
|
||||
if !held() || !found() {
|
||||
t.Fatalf("the build a later check names went with the check that built it: held %v, record %v", held(), found())
|
||||
}
|
||||
if n, err := registry.Forget(ctx, other); err != nil || n != 1 {
|
||||
t.Fatalf("forgot %d (%v)", n, err)
|
||||
}
|
||||
if held() || found() {
|
||||
t.Fatalf("the build stayed after the last check naming it went: held %v, record %v", held(), found())
|
||||
}
|
||||
}
|
||||
|
||||
// Only an earlier check of the same pull request at the same head is kept from: another head of it, or another pull
|
||||
// request whose verdict names the same key and digests, builds (novox/hq issue 703).
|
||||
func TestACheckKeepsNoBuildOfAnotherHeadOrPullRequest(t *testing.T) {
|
||||
withVersion(t, "builder-1")
|
||||
other := PullRef{Owner: "novox", Repo: "meshboard", Number: 8}
|
||||
for name, c := range map[string]struct {
|
||||
pull PullRef
|
||||
head string
|
||||
}{
|
||||
"another head": {aPull, "999999999999"},
|
||||
"another pull request": {other, "abcdefabcdef"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
scratch := newScratch()
|
||||
first := aCheckBuilt(t, scratch, "tree-one")
|
||||
leave := named(first)
|
||||
leave[0].Pull = c.pull // as if its own verdict named it
|
||||
again, r := aCheckOf(t, scratch, c.pull, c.head, "tree-one", leave)
|
||||
if !dockerBuilt(r) || again.Kept != "" {
|
||||
t.Fatalf("a build of %s was kept: %+v", name, again)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,43 @@ type sourceInputs struct {
|
||||
// contextsAtHead is false once a context was cloned at something other than its trunk's head: its
|
||||
// closure is not the one the next merge there meets, and the build says no build source.
|
||||
contextsAtHead bool
|
||||
// onlyKey says the inputs are being taken for a reuse key (novox/hq ADR 0350 §2): each artifact goes as far
|
||||
// as what its source fingerprint reads, and nothing is built or published.
|
||||
onlyKey bool
|
||||
}
|
||||
|
||||
// keyOnly is whether these inputs are only being taken for a reuse key.
|
||||
func (s *sourceInputs) keyOnly() bool { return s != nil && s.onlyKey }
|
||||
|
||||
// copy is the inputs taken so far, apart: a walk's key is taken from them without touching the build's own.
|
||||
func (s *sourceInputs) copy() *sourceInputs {
|
||||
c := newSourceInputs(s.module)
|
||||
c.tree, c.unpinned, c.prefix, c.ownIsAll, c.contextsAtHead = s.tree, s.unpinned, s.prefix, s.ownIsAll, s.contextsAtHead
|
||||
c.bases = append([]string{}, s.bases...)
|
||||
for k, v := range s.contexts {
|
||||
c.contexts[k] = v
|
||||
}
|
||||
for k, v := range s.toolchains {
|
||||
c.toolchains[k] = v
|
||||
}
|
||||
for k, v := range s.own {
|
||||
c.own[k] = v
|
||||
}
|
||||
for k, v := range s.read {
|
||||
if v == nil {
|
||||
c.read[k] = nil
|
||||
continue
|
||||
}
|
||||
set := map[string]bool{}
|
||||
for e := range v {
|
||||
set[e] = true
|
||||
}
|
||||
c.read[k] = set
|
||||
}
|
||||
for k, v := range s.readAs {
|
||||
c.readAs[k] = v
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// contextOffHead says a context was not its trunk's head.
|
||||
|
||||
@@ -127,6 +127,7 @@ func boundNeed(n Needed, bound Chosen, catalogue map[string]Manifest, node Node,
|
||||
identity = pm.IdentityBoundOf(n.Name)
|
||||
}
|
||||
held.From, held.At, held.Module, held.Serves, held.Identity = p.Node, p.At, p.Module, p.Serves, identity
|
||||
held.LeftOut = p.LeftOut
|
||||
return held, true, ""
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +21,16 @@ func TestTheBussAccountSaysNothingOnTheBus(t *testing.T) {
|
||||
t.Fatalf("a bus module declaring what it emits was not refused, or not for the reason: %v", err)
|
||||
}
|
||||
|
||||
// Its tools bundle says what it emits (novox/hq ADR 0357): the node's runtime carries the bundle and is
|
||||
// granted the module's events, so a bus module that ships one may declare them.
|
||||
bundled := []byte(`{"module":"bus","version":"1","provides":[{"name":"mesh-bus","scope":"mesh"}],
|
||||
"claims":[{"name":"mesh-broker","scope":"mesh"}],"own-secrets":{"broker":"/run/broker"},
|
||||
"emits":["bus.slow-consumer"],"tools":["bus_streams"],
|
||||
"build":{"artifacts":[{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/t","binary":"t"}]}}`)
|
||||
if _, err := ParseManifest(bundled); err != nil {
|
||||
t.Fatalf("a bus module whose tools bundle declares what it emits was refused: %v", err)
|
||||
}
|
||||
|
||||
quiet := []byte(`{"module":"bus","version":"1","provides":[{"name":"mesh-bus","scope":"mesh"}],
|
||||
"claims":[{"name":"mesh-broker","scope":"mesh"}],"own-secrets":{"broker":"/run/broker"},
|
||||
"tools":["bus_streams"]}`)
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A long-running container may say it is busy (novox/hq ADR 0335 §1).
|
||||
//
|
||||
// **Beside its health, on the container**: a command run inside it, by its shell, as an exec health check
|
||||
// is, that exits 0 while the container holds work a replacement would lose and non-zero when it is idle;
|
||||
// and how long one ask may take. Being asked is also its notice that it is about to be replaced: from the
|
||||
// first ask it takes no new work. The node-engine asks before it replaces the container, leaves it running
|
||||
// while it says busy, and replaces it on the first pass it says idle, or at 15 minutes — the engine's bound,
|
||||
// never the module's.
|
||||
//
|
||||
// Sent only to a node-engine that reads it (link.BusyContract): an older, strict engine refuses a field it
|
||||
// does not know, whole, and to it the container is replaced at once, as before.
|
||||
|
||||
// BusyField is the resource field.
|
||||
const BusyField = "busy"
|
||||
|
||||
// The bounds of one ask (ADR 0335), the node-engine's own, held here near the author.
|
||||
const (
|
||||
BusyTimeoutDefault = 10 * time.Second
|
||||
BusyTimeoutCeiling = time.Minute
|
||||
)
|
||||
|
||||
// Busy is one container's busy, read.
|
||||
type Busy struct {
|
||||
Command string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// ReadBusy reads a resource's `busy` field, its default applied. False when it carries none.
|
||||
func ReadBusy(r map[string]any) (Busy, bool, []string) {
|
||||
raw, present := r[BusyField]
|
||||
if !present {
|
||||
return Busy{}, false, nil
|
||||
}
|
||||
id := fmt.Sprint(r["id"])
|
||||
fields, ok := raw.(map[string]any)
|
||||
if !ok {
|
||||
return Busy{}, true, []string{fmt.Sprintf("%s: busy is %T; it is an object with a command", id, raw)}
|
||||
}
|
||||
var problems []string
|
||||
keys := make([]string, 0, len(fields))
|
||||
for k := range fields {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if k != "command" && k != "timeout" {
|
||||
problems = append(problems, fmt.Sprintf("%s: its busy says %q; busy has a command and a timeout", id, k))
|
||||
}
|
||||
}
|
||||
b := Busy{Timeout: BusyTimeoutDefault}
|
||||
if v, has := fields["command"]; has {
|
||||
s, ok := v.(string)
|
||||
if !ok {
|
||||
problems = append(problems, fmt.Sprintf("%s: its busy's command is %T; it is text", id, v))
|
||||
}
|
||||
b.Command = strings.TrimSpace(s)
|
||||
}
|
||||
if v, has := fields["timeout"]; has {
|
||||
s, _ := v.(string)
|
||||
d, err := time.ParseDuration(strings.TrimSpace(s))
|
||||
switch {
|
||||
case err != nil:
|
||||
problems = append(problems, fmt.Sprintf("%s: its busy's timeout is %v; it is a duration such as \"10s\"", id, v))
|
||||
case d <= 0 || d > BusyTimeoutCeiling:
|
||||
problems = append(problems, fmt.Sprintf("%s: an ask of its busy takes more than nothing and at most %s, not %s",
|
||||
id, BusyTimeoutCeiling, d))
|
||||
default:
|
||||
b.Timeout = d
|
||||
}
|
||||
}
|
||||
return b, true, problems
|
||||
}
|
||||
|
||||
// busyProblems is everything wrong with a manifest's `busy` fields, in the manifest's words: a command on
|
||||
// one line, a timeout within its bounds, and only on a container that stays up.
|
||||
func busyProblems(m Manifest) []string {
|
||||
var problems []string
|
||||
for _, r := range m.Resources {
|
||||
b, has, read := ReadBusy(r)
|
||||
if !has {
|
||||
continue
|
||||
}
|
||||
where := m.Module + ": " + fmt.Sprint(r["id"])
|
||||
for _, p := range read {
|
||||
problems = append(problems, m.Module+": "+p)
|
||||
}
|
||||
if fmt.Sprint(r["type"]) != "container" || !LongRunning(r) {
|
||||
problems = append(problems, fmt.Sprintf("%s declares busy and is not a container that stays up: busy is "+
|
||||
"asked of a running container before it is replaced (ADR 0335)", where))
|
||||
}
|
||||
if b.Command == "" {
|
||||
problems = append(problems, where+"'s busy names no command: one run inside the container, exiting 0 "+
|
||||
"while it is busy")
|
||||
}
|
||||
if strings.ContainsAny(b.Command, "\n\r") {
|
||||
problems = append(problems, where+"'s busy command is one line")
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// busyInto composes a container's `busy` into the node-engine's words, or takes it away: for an engine that
|
||||
// reads it, the command and its timeout written out; for one that does not, the field is not sent, and the
|
||||
// container is replaced at once, as before.
|
||||
func busyInto(resource map[string]any, with Rendering) {
|
||||
if _, has := resource[BusyField]; !has {
|
||||
return
|
||||
}
|
||||
if !with.ReadsBusy {
|
||||
delete(resource, BusyField)
|
||||
return
|
||||
}
|
||||
b, _, _ := ReadBusy(resource)
|
||||
resource[BusyField] = map[string]any{"command": b.Command, "timeout": b.Timeout.String()}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/validate"
|
||||
)
|
||||
|
||||
// A long-running container may say it is busy (novox/hq ADR 0335 §1): `module check` holds the field to its
|
||||
// shape near its author, and it is sent only to a node-engine that reads it, in words the engine's own
|
||||
// validator accepts.
|
||||
|
||||
func busyManifest(t *testing.T, on string, busy any) []byte {
|
||||
t.Helper()
|
||||
extra := map[string]any{"id": "agent", "type": "container", "name": "app-agent",
|
||||
"image": "registry.example/app" + healthDigest}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(healthManifest(t, "", nil, extra), &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range m["resources"].([]any) {
|
||||
if r := r.(map[string]any); r["id"] == on {
|
||||
r[BusyField] = busy
|
||||
}
|
||||
}
|
||||
out, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestABusyIsHeldToItsShape(t *testing.T) {
|
||||
if _, err := ParseManifest(busyManifest(t, "agent", map[string]any{"command": "mesh-builder busy"})); err != nil {
|
||||
t.Fatalf("a well-formed busy was refused: %v", err)
|
||||
}
|
||||
if _, err := ParseManifest(busyManifest(t, "agent", map[string]any{"command": "x", "timeout": "30s"})); err != nil {
|
||||
t.Fatalf("a busy with its timeout was refused: %v", err)
|
||||
}
|
||||
for name, c := range map[string]struct {
|
||||
on string
|
||||
busy any
|
||||
says string
|
||||
}{
|
||||
"text": {"agent", "mesh-builder busy", "it is an object"},
|
||||
"no command": {"agent", map[string]any{"timeout": "5s"}, "names no command"},
|
||||
"two lines": {"agent", map[string]any{"command": "a\nb"}, "one line"},
|
||||
"a long timeout": {"agent", map[string]any{"command": "x", "timeout": "5m"}, "at most 1m0s"},
|
||||
"an unknown key": {"agent", map[string]any{"command": "x", "every": "1m"}, `says "every"`},
|
||||
"on a step": {"seed", map[string]any{"command": "x"}, "not a container that stays up"},
|
||||
"on a service": {"daemon", map[string]any{"command": "x"}, "not a container that stays up"},
|
||||
} {
|
||||
_, err := ParseManifest(busyManifest(t, c.on, c.busy))
|
||||
if err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%s: want a refusal saying %q, got %v", name, c.says, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestABusyIsSentOnlyToAnEngineThatReadsIt(t *testing.T) {
|
||||
m, err := ParseManifest(busyManifest(t, "agent", map[string]any{"command": "mesh-builder busy"}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compose := func(with Rendering) []map[string]any {
|
||||
out, err := Resolution{Node: "laptop", Modules: []Manifest{m}}.Declaration(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
older := compose(Rendering{})
|
||||
if b, sent := older[indexOfID(older, "app.agent")][BusyField]; sent {
|
||||
t.Fatalf("busy was sent to an engine that does not read it: %v", b)
|
||||
}
|
||||
newer := compose(Rendering{ReadsBusy: true})
|
||||
got, _ := newer[indexOfID(newer, "app.agent")][BusyField].(map[string]any)
|
||||
if got["command"] != "mesh-builder busy" || got["timeout"] != "10s" {
|
||||
t.Fatalf("busy composed as %v", got)
|
||||
}
|
||||
// And the engine's own validator takes the container with it.
|
||||
body, err := json.Marshal(map[string]any{"declaration": validate.Version, "resources": newer})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if problems := validate.Declaration(body); len(problems) > 0 {
|
||||
t.Fatalf("the node-engine's validator refuses the composed busy: %v", problems)
|
||||
}
|
||||
if _, has := m.Resources[len(m.Resources)-1][BusyField].(map[string]any)["timeout"]; has {
|
||||
t.Error("composing wrote the timeout into the catalogue's own manifest")
|
||||
}
|
||||
}
|
||||
@@ -135,6 +135,9 @@ func CheckServes(m Manifest) []string {
|
||||
for _, key := range sortedAnyKeys(m.Serves[provision]) {
|
||||
text, ok := m.Serves[provision][key].(string)
|
||||
if !ok {
|
||||
// A provider's own name is filled only in a value of its own (novox/hq ADR 0346): one
|
||||
// nested in a list or an object would be served as the placeholder, unsaid.
|
||||
problems = append(problems, nestedOwnNames(m, provision, key, m.Serves[provision][key])...)
|
||||
continue
|
||||
}
|
||||
// A probe identity, because what is checked is the shape of the statement and not
|
||||
@@ -143,6 +146,8 @@ func CheckServes(m Manifest) []string {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s serves %s, and the value it serves as %q %s", m.Module, provision, key, err))
|
||||
}
|
||||
// And the provider's own names it may serve (novox/hq ADR 0346).
|
||||
problems = append(problems, checkServedOwnNames(m, provision, key, text)...)
|
||||
// A label longer than DNS keeps is not truncated here (asDNSLabel), so the offer's own
|
||||
// bound has to keep the identity inside one (ADR 0225).
|
||||
if strings.Contains(text, "${consumer:as:dns}") {
|
||||
|
||||
@@ -252,6 +252,11 @@ type Rendering struct {
|
||||
// statement's contract is link.RootContract or later). To an older, strict engine the field is not
|
||||
// sent, and the account it names is not judged — which the self-check says, as not judged.
|
||||
JudgesRoot bool
|
||||
|
||||
// ReadsBusy says this machine's node-engine reads a container's `busy` (novox/hq ADR 0335: its statement's
|
||||
// contract is link.BusyContract or later). To an older, strict engine the field is not sent, and the
|
||||
// container is replaced at once, as before.
|
||||
ReadsBusy bool
|
||||
}
|
||||
|
||||
// RootField is a user resource's field saying the account must never become root without a person
|
||||
@@ -1062,6 +1067,8 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
// How it is ready, in the node-engine's words: its endpoint as the port this machine
|
||||
// published it on — or not sent at all to an engine older than the field (ADR 0240).
|
||||
healthInto(copied, m, with)
|
||||
// And whether it is busy (novox/hq ADR 0335): sent only to an engine that asks it.
|
||||
busyInto(copied, with)
|
||||
// And a user's `root` (novox/hq ADR 0266): sent only when it asserts something, to an engine
|
||||
// that judges it.
|
||||
rootInto(copied, with)
|
||||
@@ -2039,7 +2046,13 @@ func (r Resolution) servedOnThisMachine(provision string, with Rendering) (map[s
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("%s serving %s: %w", m.Module, provision, err)
|
||||
}
|
||||
return settled, true, nil
|
||||
// And the provider's own names, as this machine composes them (novox/hq ADR 0346); one it does
|
||||
// not compose leaves its key out, said by the plan on this machine (ServedLeftOut).
|
||||
named, _, err := r.ServedOwnNames(m, provision, settled, WithDefaults(m, with.Settings[m.Module]))
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("%s serving %s: %w", m.Module, provision, err)
|
||||
}
|
||||
return named, true, nil
|
||||
}
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
@@ -530,12 +530,6 @@ type Manifest struct {
|
||||
// stays refused by name until a layer sets it. The mesh's layer, then the node's, override it.
|
||||
Settings map[string]SettingDeclaration `json:"settings,omitempty"`
|
||||
|
||||
// Preferences are the values this module reads that a machine may tune (novox/hq issue 630): each
|
||||
// key's type, default and why, and the predefined key it falls back to, if any. The value resolves
|
||||
// from the default, then mesh config and node config for the predefined key, then this module's own
|
||||
// value on the node. `settings` entries of kind preference are read as preferences while they move.
|
||||
Preferences map[string]Preference `json:"preferences,omitempty"`
|
||||
|
||||
// unknown is the first key this manifest has that this controller does not know, at any depth, when
|
||||
// it was read from the store (novox/hq ADR 0262): a manifest a newer controller registered.
|
||||
// ParseManifest refuses it; the store's catalogue still loads, and the module is left out of every
|
||||
@@ -558,6 +552,10 @@ type Manifest struct {
|
||||
// why. A person's choice through the `upgrade` verb stands over it; the bus records whatever it says.
|
||||
Upgrade *UpgradePolicy `json:"upgrade,omitempty"`
|
||||
|
||||
// Unhealthy is how loud this module's unhealthy condition is (novox/hq ADR 0355): a warning when unsaid;
|
||||
// `urgent`, with why, for a module whose failure costs people something they rely on.
|
||||
Unhealthy *UnhealthyPolicy `json:"unhealthy,omitempty"`
|
||||
|
||||
// Reads are other modules' state this module reads and watches, each `<module>.<name>`
|
||||
// (novox/hq ADR 0201). Read-only: only the owner's instances write.
|
||||
Reads []string `json:"reads,omitempty"`
|
||||
@@ -1636,9 +1634,14 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
// **The bus's own account copies the bus and says nothing on it** (novox/hq ADR 0235). The module
|
||||
// holding mesh-broker is granted the snapshot API and nothing else, so anything it declared to say
|
||||
// or hear on the bus would be granted nothing — refused here rather than silently dropped.
|
||||
//
|
||||
// **But its tools bundle may say what it declares** (novox/hq ADR 0357): the node's runtime carries
|
||||
// the bundle and is granted the events the module emits, under the module's name, as for any module it
|
||||
// carries — the bus's account is not who publishes them. So emits are refused only for a bus module
|
||||
// with no bundle to say them.
|
||||
if _, account := m.OwnSecrets["broker"]; account && m.ClaimsSeat(BrokerSeat) {
|
||||
var said []string
|
||||
if len(m.EmitsAll()) > 0 {
|
||||
if len(m.EmitsAll()) > 0 && !m.shipsABundle() {
|
||||
said = append(said, "emits")
|
||||
}
|
||||
for _, f := range []struct {
|
||||
@@ -2327,6 +2330,7 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
problems = append(problems, m.placementProblems()...)
|
||||
problems = append(problems, m.dataProblems()...)
|
||||
problems = append(problems, m.upgradeProblems()...)
|
||||
problems = append(problems, m.unhealthyProblems()...)
|
||||
|
||||
for i, r := range m.Resources {
|
||||
id, _ := r["id"].(string)
|
||||
@@ -2342,6 +2346,8 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
// How each long-running resource is ready (novox/hq ADR 0240 rule 2): said near its author, in the
|
||||
// words the node-engine would refuse it in far away.
|
||||
problems = append(problems, healthProblems(m)...)
|
||||
// And whether a container is busy (novox/hq ADR 0335), likewise.
|
||||
problems = append(problems, busyProblems(m)...)
|
||||
if len(problems) > 0 {
|
||||
sort.Strings(problems)
|
||||
return Manifest{}, fmt.Errorf("this manifest cannot be used:\n - %s",
|
||||
@@ -2901,3 +2907,16 @@ func whileStoppedProblems(m Manifest, r map[string]any, scheduled bool) []string
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// shipsABundle says the module builds a tools bundle, which the node's runtime carries and speaks for.
|
||||
func (m Manifest) shipsABundle() bool {
|
||||
if m.Build == nil {
|
||||
return false
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Kind == ArtifactBundle {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -1,270 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A config value resolves from four sources, the last set winning (novox/hq issue 630):
|
||||
//
|
||||
// 1. the default in the module's manifest;
|
||||
// 2. mesh config: a mesh-wide value for a predefined key, such as `font-size`;
|
||||
// 3. node config: a value for a predefined key for one node;
|
||||
// 4. module-assignment config: that module's own value on that node, the most specific.
|
||||
//
|
||||
// An assignment is one module on one node: there is no module layer for the whole mesh. Mesh and node
|
||||
// config hold values only for the keys on a fixed, reviewable list — the catalogue's `predefined.json` —
|
||||
// that any module may fall back to, by naming one as its preference's `predefined`. A preference without
|
||||
// `predefined` reads only its default and its module-assignment config. A preference is never trusted.
|
||||
|
||||
// Preference is one value a module reads that a machine may tune, as the manifest declares it under
|
||||
// `preferences`.
|
||||
type Preference struct {
|
||||
// Type is "number", "string" or "boolean".
|
||||
Type string `json:"type"`
|
||||
// Default is the value when no source sets it, of Type.
|
||||
Default any `json:"default"`
|
||||
// Predefined names the key on the catalogue's list this preference falls back to: mesh config,
|
||||
// then node config, for that key. Empty reads only the default and the module-assignment config.
|
||||
Predefined string `json:"predefined,omitempty"`
|
||||
// Why this default: one sentence for whoever wonders whether to change it.
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// The types a preference and a predefined key may have.
|
||||
const (
|
||||
TypeNumber = "number"
|
||||
TypeString = "string"
|
||||
TypeBoolean = "boolean"
|
||||
)
|
||||
|
||||
// PredefinedFile is the catalogue repository's root file holding the predefined keys.
|
||||
const PredefinedFile = "predefined.json"
|
||||
|
||||
// PredefinedKey is one key mesh and node config may hold a value for.
|
||||
type PredefinedKey struct {
|
||||
Type string `json:"type"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// Predefined is the list of predefined keys, by key.
|
||||
type Predefined map[string]PredefinedKey
|
||||
|
||||
// Keys is the list's keys, sorted.
|
||||
func (p Predefined) Keys() []string {
|
||||
keys := make([]string, 0, len(p))
|
||||
for k := range p {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
// predefinedName is what a predefined key looks like: the shape of a setting key.
|
||||
var predefinedName = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]*$`)
|
||||
|
||||
// ParsePredefined reads `predefined.json` strictly, and refuses a key whose name, type or why is wrong.
|
||||
func ParsePredefined(raw []byte) (Predefined, error) {
|
||||
decoder := json.NewDecoder(bytes.NewReader(raw))
|
||||
decoder.DisallowUnknownFields()
|
||||
var list Predefined
|
||||
if err := decoder.Decode(&list); err != nil {
|
||||
return nil, fmt.Errorf("%s is not a list of predefined keys ({\"<key>\": {\"type\": …, \"why\": …}}): %w",
|
||||
PredefinedFile, err)
|
||||
}
|
||||
var problems []string
|
||||
for _, key := range list.Keys() {
|
||||
k := list[key]
|
||||
if !predefinedName.MatchString(key) {
|
||||
problems = append(problems, fmt.Sprintf("%q is not a usable key: lower-case letters, digits, dots, dashes and underscores", key))
|
||||
}
|
||||
if meshWords[key] {
|
||||
problems = append(problems, fmt.Sprintf("%q is a key the mesh reads itself", key))
|
||||
}
|
||||
if !knownType(k.Type) {
|
||||
problems = append(problems, fmt.Sprintf("%q has the type %q; a type is %q, %q or %q", key, k.Type,
|
||||
TypeNumber, TypeString, TypeBoolean))
|
||||
}
|
||||
if strings.TrimSpace(k.Why) == "" {
|
||||
problems = append(problems, fmt.Sprintf("%q has no why: say in one sentence what the key is for", key))
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return nil, fmt.Errorf("%s: %s", PredefinedFile, strings.Join(problems, "; "))
|
||||
}
|
||||
if list == nil {
|
||||
list = Predefined{}
|
||||
}
|
||||
return list, nil
|
||||
}
|
||||
|
||||
func knownType(t string) bool {
|
||||
return t == TypeNumber || t == TypeString || t == TypeBoolean
|
||||
}
|
||||
|
||||
// TypeOf is the type a JSON value has, as a preference names it, or "" for one that is none of them.
|
||||
func TypeOf(v any) string {
|
||||
switch v.(type) {
|
||||
case float64, int, int64, json.Number:
|
||||
return TypeNumber
|
||||
case string:
|
||||
return TypeString
|
||||
case bool:
|
||||
return TypeBoolean
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// PreferencesOf is every preference a module declares: those under `preferences`, and — while the
|
||||
// catalogue moves to them — every `settings` entry of kind preference, its type read from its default.
|
||||
func PreferencesOf(m Manifest) map[string]Preference {
|
||||
if len(m.Preferences) == 0 && len(m.Settings) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]Preference, len(m.Preferences)+len(m.Settings))
|
||||
for key, d := range m.Settings {
|
||||
out[key] = Preference{Type: TypeOf(d.Default), Default: d.Default, Why: d.Why}
|
||||
}
|
||||
for key, p := range m.Preferences {
|
||||
out[key] = p
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// DeclaresPreference is whether a module declares a preference by this key.
|
||||
func DeclaresPreference(m Manifest, key string) bool {
|
||||
_, has := m.Preferences[key]
|
||||
if !has {
|
||||
_, has = m.Settings[key]
|
||||
}
|
||||
return has
|
||||
}
|
||||
|
||||
// PredefinedProblems is every way a module's preferences disagree with the list of predefined keys: a
|
||||
// `predefined` not on it, or a type other than the list's.
|
||||
func PredefinedProblems(m Manifest, list Predefined) []string {
|
||||
var problems []string
|
||||
keys := make([]string, 0, len(m.Preferences))
|
||||
for k := range m.Preferences {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
p := m.Preferences[key]
|
||||
if p.Predefined == "" {
|
||||
continue
|
||||
}
|
||||
k, on := list[p.Predefined]
|
||||
if !on {
|
||||
listed := "the list holds none"
|
||||
if len(list) > 0 {
|
||||
listed = "the list holds " + strings.Join(list.Keys(), ", ")
|
||||
}
|
||||
problems = append(problems, fmt.Sprintf("%s's preference %q falls back to the predefined key %q, which "+
|
||||
"is not on the catalogue's list of predefined keys (%s; %s)", m.Module, key, p.Predefined, PredefinedFile,
|
||||
listed))
|
||||
continue
|
||||
}
|
||||
if k.Type != p.Type {
|
||||
problems = append(problems, fmt.Sprintf("%s's preference %q is a %s, and the predefined key %q it falls "+
|
||||
"back to is a %s", m.Module, key, p.Type, p.Predefined, k.Type))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// Where a value from mesh or node config is said to come from.
|
||||
const MeshConfigLayer = "mesh config"
|
||||
|
||||
// NodeConfigLayer is what the node config of one node is called where a value's source is said.
|
||||
func NodeConfigLayer(node string) string { return "node config (" + node + ")" }
|
||||
|
||||
// ConfigLayers are the layers mesh config and node config make for one module: each of its preferences
|
||||
// that names a predefined key, given that key's value. Laid between the module's defaults and its
|
||||
// module-assignment config; like the defaults they fill `${setting:<key>}` only.
|
||||
func ConfigLayers(m Manifest, mesh, node map[string]any, nodeName string) []Layer {
|
||||
var out []Layer
|
||||
for _, source := range []struct {
|
||||
from string
|
||||
values map[string]any
|
||||
}{{MeshConfigLayer, mesh}, {NodeConfigLayer(nodeName), node}} {
|
||||
values := map[string]any{}
|
||||
for key, p := range m.Preferences {
|
||||
if p.Predefined == "" {
|
||||
continue
|
||||
}
|
||||
if v, set := source.values[p.Predefined]; set {
|
||||
values[key] = v
|
||||
}
|
||||
}
|
||||
if len(values) > 0 {
|
||||
out = append(out, Layer{From: source.from, Values: values, Config: true})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ConfigValueProblem refuses a value of the wrong type for a predefined key or a preference.
|
||||
func ConfigValueProblem(what string, want string, v any) error {
|
||||
if got := TypeOf(v); got != want {
|
||||
shown, _ := json.Marshal(v)
|
||||
return fmt.Errorf("%s is a %s, and %s is %s", what, want, shown, aOrNothing(got))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func aOrNothing(t string) string {
|
||||
if t == "" {
|
||||
return "none of a number, a string or a boolean"
|
||||
}
|
||||
return "a " + t
|
||||
}
|
||||
|
||||
// preferenceProblems is every way a module's `preferences` are wrong in themselves.
|
||||
func preferenceProblems(m Manifest) []string {
|
||||
var problems []string
|
||||
keys := make([]string, 0, len(m.Preferences))
|
||||
for k := range m.Preferences {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
p := m.Preferences[key]
|
||||
say := func(format string, args ...any) {
|
||||
problems = append(problems, fmt.Sprintf("%s's preference %q: ", m.Module, key)+fmt.Sprintf(format, args...))
|
||||
}
|
||||
if _, twice := m.Settings[key]; twice {
|
||||
say("declared under settings too; declare it once, under preferences")
|
||||
}
|
||||
if !knownType(p.Type) {
|
||||
say("the type is %q; a type is %q, %q or %q", p.Type, TypeNumber, TypeString, TypeBoolean)
|
||||
} else if p.Default == nil {
|
||||
say("no default: a preference has one")
|
||||
} else if got := TypeOf(p.Default); got != p.Type {
|
||||
shown, _ := json.Marshal(p.Default)
|
||||
say("it is a %s, and its default %s is %s", p.Type, shown, aOrNothing(got))
|
||||
}
|
||||
if p.Predefined != "" && !predefinedName.MatchString(p.Predefined) {
|
||||
say("%q is not a predefined key's name", p.Predefined)
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// PredefinedNamed is every predefined key a module's preferences fall back to, sorted.
|
||||
func PredefinedNamed(m Manifest) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, p := range m.Preferences {
|
||||
if p.Predefined != "" && !seen[p.Predefined] {
|
||||
seen[p.Predefined] = true
|
||||
out = append(out, p.Predefined)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The list of predefined keys is read strictly: a type and a why for each key (novox/hq issue 630).
|
||||
func TestThePredefinedListIsReadStrictly(t *testing.T) {
|
||||
list, err := ParsePredefined([]byte(`{"font-size": {"type": "number", "why": "the size text is drawn at"},
|
||||
"theme": {"type": "string", "why": "light or dark"}}`))
|
||||
if err != nil || len(list) != 2 || list["font-size"].Type != TypeNumber {
|
||||
t.Fatalf("a good list: %v %v", list, err)
|
||||
}
|
||||
for raw, want := range map[string]string{
|
||||
`{"font-size": {"type": "integer", "why": "x"}}`: `the type "integer"`,
|
||||
`{"font-size": {"type": "number"}}`: "no why",
|
||||
`{"Font Size": {"type": "number", "why": "x"}}`: "not a usable key",
|
||||
`{"font-size": {"type": "number", "why": "x", "a": 1}}`: "unknown field",
|
||||
`{"ports": {"type": "number", "why": "x"}}`: "the mesh reads itself",
|
||||
} {
|
||||
if _, err := ParsePredefined([]byte(raw)); err == nil || !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("%s: %v, want %q", raw, err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A manifest declares preferences with a type, a default of that type, a why and maybe a predefined key;
|
||||
// a `settings` entry of kind preference is read as one, its type from its default.
|
||||
func TestAManifestDeclaresPreferences(t *testing.T) {
|
||||
file := `"resources": [{"id": "rc", "type": "file", "path": "/etc/rc", "mode": "0644", "trusted": false,
|
||||
"content": "size = ${setting:font-size} width = ${setting:width}"}]`
|
||||
m, err := ParseManifest([]byte(`{"module": "dunst", "version": "1",
|
||||
"preferences": {"font-size": {"type": "number", "default": 10, "predefined": "font-size", "why": "100 DPI"}},
|
||||
"settings": {"width": {"kind": "preference", "default": 250, "why": "forty characters"}}, ` + file + `}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prefs := PreferencesOf(m)
|
||||
if prefs["font-size"].Predefined != "font-size" || prefs["width"].Type != TypeNumber || prefs["width"].Default != float64(250) {
|
||||
t.Fatalf("the preferences: %+v", prefs)
|
||||
}
|
||||
if got := PredefinedNamed(m); len(got) != 1 || got[0] != "font-size" {
|
||||
t.Fatalf("the predefined keys named: %v", got)
|
||||
}
|
||||
for body, want := range map[string]string{
|
||||
`"preferences": {"font-size": {"type": "number", "default": "10", "why": "x"}, "width": {"type": "number", "default": 1, "why": "x"}}`: `its default "10" is a string`,
|
||||
`"preferences": {"font-size": {"type": "float", "default": 10, "why": "x"}, "width": {"type": "number", "default": 1, "why": "x"}}`: `the type is "float"`,
|
||||
`"preferences": {"font-size": {"type": "number", "why": "x"}, "width": {"type": "number", "default": 1, "why": "x"}}`: "no default",
|
||||
`"preferences": {"font-size": {"type": "number", "default": 10}, "width": {"type": "number", "default": 1, "why": "x"}}`: "no why",
|
||||
} {
|
||||
if _, err := ParseManifest([]byte(`{"module": "dunst", "version": "1", ` + body + `, ` + file + `}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), want) {
|
||||
t.Errorf("%s: %v, want %q", body, err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A preference falls back only to a key on the list, with the list's type.
|
||||
func TestAPreferencesPredefinedKeyIsOnTheListWithItsType(t *testing.T) {
|
||||
m := Manifest{Module: "dunst", Preferences: map[string]Preference{
|
||||
"font-size": {Type: TypeNumber, Default: float64(10), Predefined: "font-size", Why: "x"}}}
|
||||
if p := PredefinedProblems(m, Predefined{"font-size": {Type: TypeNumber, Why: "x"}}); len(p) != 0 {
|
||||
t.Fatalf("on the list: %v", p)
|
||||
}
|
||||
if p := PredefinedProblems(m, Predefined{"theme": {Type: TypeString, Why: "x"}}); len(p) != 1 ||
|
||||
!strings.Contains(p[0], "not on the catalogue's list") || !strings.Contains(p[0], "theme") {
|
||||
t.Fatalf("not on the list: %v", p)
|
||||
}
|
||||
if p := PredefinedProblems(m, Predefined{"font-size": {Type: TypeString, Why: "x"}}); len(p) != 1 ||
|
||||
!strings.Contains(p[0], "is a string") {
|
||||
t.Fatalf("another type: %v", p)
|
||||
}
|
||||
}
|
||||
|
||||
// Mesh and node config fill `${setting:<key>}` between the default and the module's own layer, and — like
|
||||
// the default — never reach a mergeable file.
|
||||
func TestConfigLayersFillAPlaceholderAndNoMergeableFile(t *testing.T) {
|
||||
m := Manifest{Module: "dunst", Preferences: map[string]Preference{
|
||||
"font-size": {Type: TypeNumber, Default: float64(10), Predefined: "font-size", Why: "x"},
|
||||
"width": {Type: TypeNumber, Default: float64(250), Why: "x"}}}
|
||||
fill := func(layers []Layer) string {
|
||||
t.Helper()
|
||||
r := map[string]any{"id": "rc", "type": "file", "content": "font ${setting:font-size} width ${setting:width}"}
|
||||
if err := settingInto(r, WithDefaults(m, layers), m.Module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r["content"].(string)
|
||||
}
|
||||
mesh := map[string]any{"font-size": float64(11), "width": float64(999)}
|
||||
node := map[string]any{"font-size": float64(13)}
|
||||
if got := fill(nil); got != "font 10 width 250" {
|
||||
t.Fatalf("the defaults: %s", got)
|
||||
}
|
||||
if got := fill(ConfigLayers(m, mesh, nil, "g14")); got != "font 11 width 250" {
|
||||
t.Fatalf("mesh config, which a preference without predefined does not read: %s", got)
|
||||
}
|
||||
both := ConfigLayers(m, mesh, node, "g14")
|
||||
if got := fill(both); got != "font 13 width 250" {
|
||||
t.Fatalf("node config over mesh config: %s", got)
|
||||
}
|
||||
own := append(both, Layer{From: "g14", Values: map[string]any{"font-size": float64(15)}})
|
||||
if got := fill(own); got != "font 15 width 250" {
|
||||
t.Fatalf("the module's own value over both: %s", got)
|
||||
}
|
||||
sources := map[string]string{}
|
||||
for _, s := range Effective(m, both) {
|
||||
sources[s.Key] = s.From
|
||||
}
|
||||
if sources["font-size"] != NodeConfigLayer("g14") || sources["width"] != DefaultLayer {
|
||||
t.Fatalf("where each comes from: %v", sources)
|
||||
}
|
||||
merged, err := ApplySettings(map[string]any{"id": "conf", "type": "file", "merge": MergeJSON, "content": "{}"}, both)
|
||||
if err != nil || merged["content"] != "{}\n" {
|
||||
t.Fatalf("config reached a mergeable file: %v %v", merged["content"], err)
|
||||
}
|
||||
}
|
||||
@@ -111,6 +111,9 @@ type Provider struct {
|
||||
// (novox/hq to-be 23), and the pair is what tells the holder of a seat apart from another module
|
||||
// providing the same thing (ADR 0110).
|
||||
Module string
|
||||
// LeftOut says each served key left out of Serves because that node composes no name it asks
|
||||
// for (novox/hq ADR 0346 §3): said under each consumer's need.
|
||||
LeftOut []string
|
||||
}
|
||||
|
||||
// Held is a claim somebody already has, used for the scopes wider than one node.
|
||||
@@ -230,6 +233,9 @@ type Needed struct {
|
||||
// state, not a consumer missing its key. Set by the plan, which is the only layer that knows a
|
||||
// licence's manager; empty for every consumer.
|
||||
Manager bool
|
||||
// LeftOut says each served key the provider left out, because its machine composes no name the
|
||||
// value asks for (novox/hq ADR 0346 §3).
|
||||
LeftOut []string
|
||||
// Module is the providing module on From, empty for a need answered by a record or by nothing
|
||||
// that serves it. A provider is a (node, module) pair (novox/hq to-be 23), and a binding to data
|
||||
// is a binding to the pair (ADR 0232).
|
||||
@@ -582,7 +588,7 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
||||
bound = pm.IdentityBoundOf(want)
|
||||
}
|
||||
needs = append(needs, Needed{Name: want, From: p.Node, At: p.At, Module: p.Module,
|
||||
Serves: p.Serves, For: because[want], SharedOwn: shared, Identity: bound})
|
||||
Serves: p.Serves, For: because[want], SharedOwn: shared, Identity: bound, LeftOut: p.LeftOut})
|
||||
}
|
||||
switch {
|
||||
case world.Unchecked:
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user