Compare commits
107
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
60cc9ec363 | ||
|
|
5d22d7df86 | ||
|
|
86057f4551 | ||
|
|
e4acab32d6 | ||
|
|
743e6478a7 | ||
|
|
ede2772b3c | ||
|
|
b34055f580 | ||
|
|
ee13e75bf3 | ||
|
|
480657d95b | ||
|
|
38c87a7e47 | ||
|
|
6acce60f8b | ||
|
|
4390589c61 | ||
|
|
e6a28a2d28 | ||
|
|
32bd8b5d5f | ||
|
|
c2ec0a601a | ||
|
|
34050dc6a3 | ||
|
|
a5500f732f | ||
|
|
3ec4473b29 | ||
|
|
df8c1ccf1f | ||
|
|
bbee20adf9 | ||
|
|
afed887217 | ||
|
|
622fb1e9cb | ||
|
|
5d5bdf9e32 | ||
|
|
4af9f1147d | ||
|
|
704b21b3c5 | ||
|
|
5b455b8a24 | ||
|
|
0e97413875 | ||
|
|
da8a5e9bc0 | ||
|
|
847ffd6d12 | ||
|
|
0214b45469 | ||
|
|
2bee290eea | ||
|
|
76304d9db8 | ||
|
|
b7084a19e9 | ||
|
|
c975dbf7b0 | ||
|
|
98e8568749 | ||
|
|
58889235d1 | ||
|
|
48b2906b5c | ||
|
|
ea7d91bff1 | ||
|
|
4a75cc51dd | ||
|
|
535c7641ac | ||
|
|
c707bf18d8 | ||
|
|
060f2a9ca5 | ||
|
|
1666747a22 | ||
|
|
04730b54a8 | ||
|
|
45870ab31c | ||
|
|
fd9308d328 | ||
|
|
3479a35462 | ||
|
|
cebc593627 | ||
|
|
2959c81435 | ||
|
|
8933864a49 | ||
|
|
df9e5728a9 | ||
|
|
2d18f438c5 | ||
|
|
3ce5a0239b | ||
|
|
70885ab3c4 | ||
|
|
65508d9df9 | ||
|
|
8123ca5654 | ||
|
|
d9b86376d3 | ||
|
|
ca3f43c400 | ||
|
|
b482122d66 | ||
|
|
2f5cc146fe | ||
|
|
c2ea9b4c51 | ||
|
|
b03d257f81 | ||
|
|
1601cb1743 | ||
|
|
873b19566a | ||
|
|
b0753e2aac | ||
|
|
a69c0e50dc | ||
|
|
72214ae989 | ||
|
|
5c606fd1d4 | ||
|
|
372faebf28 | ||
|
|
675466a8c3 | ||
|
|
cd49b50c40 | ||
|
|
41eefc2f8c | ||
|
|
c3946e5d72 | ||
|
|
524295926f | ||
|
|
bff5b7f0e4 | ||
|
|
cef22a2526 | ||
|
|
deef9421a2 | ||
|
|
3c19e9ee93 | ||
|
|
3d94f87bb0 | ||
|
|
4644f25edd | ||
|
|
c58e0a571e | ||
|
|
332fbaa9b1 | ||
|
|
7a0c219f96 | ||
|
|
985423ae61 | ||
|
|
cfacda2625 | ||
|
|
bf6beec465 | ||
|
|
9f3aa91e81 | ||
|
|
4bdcef3f73 | ||
|
|
8ddf8b7005 | ||
|
|
8bbb1b19ae | ||
|
|
e32ca00aed | ||
|
|
1219ec8229 | ||
|
|
e324f5d821 | ||
|
|
6f1a05366c | ||
|
|
0a1dc5f93f | ||
|
|
ae0b384497 | ||
|
|
f3875a6b60 | ||
|
|
f09c5ebdec | ||
|
|
6d2e4c299a | ||
|
|
b654b320a1 | ||
|
|
4cc8588afb | ||
|
|
584aa2d7d7 | ||
|
|
c9dd07f919 | ||
|
|
404b4409e7 | ||
|
|
6e6cd5e26d | ||
|
|
e658a14296 | ||
|
|
b1546ed28a |
@@ -0,0 +1,146 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **The build agent says whether it is busy, and being asked drains it** (novox/hq ADR 0335 §1, §5). Its
|
||||||
|
// container declares `busy` as `mesh-builder busy`: the node-engine runs it inside the container before it
|
||||||
|
// replaces it. It exits 0 while a build is in hand and 1 when none is; and from the first ask the agent takes
|
||||||
|
// no new build request, so its answer ends once the build in hand ends — a replaced builder finishes the check
|
||||||
|
// it runs, instead of the build queue handing it out again from the start (issue 594).
|
||||||
|
//
|
||||||
|
// Two marks in the container's own temporary directory, as the ready mark (ready.go): a new container starts
|
||||||
|
// without them, and a restarted one removes them before it takes work again.
|
||||||
|
//
|
||||||
|
// - in hand: written by the agent when it claims a request, before it reads whether it is drained; removed
|
||||||
|
// when the build ends.
|
||||||
|
// - drained: written by the ask, before it reads whether a build is in hand.
|
||||||
|
//
|
||||||
|
// Each side writes its own mark before it reads the other's, so an ask racing a fetch sees the build in hand,
|
||||||
|
// or the agent sees the drain and hands the request back; never neither.
|
||||||
|
|
||||||
|
func inHandMark() string { return filepath.Join(os.TempDir(), "mesh-builder.in-hand") }
|
||||||
|
func drainMark() string { return filepath.Join(os.TempDir(), "mesh-builder.drained") }
|
||||||
|
|
||||||
|
// The drain lapses when no ask has come for drainLapse: the node-engine asks every 20 seconds for as long as it
|
||||||
|
// really waits, so an agent whose replacement was withdrawn — the declaration put back, the walk stopped —
|
||||||
|
// takes work again rather than staying drained for ever (review of ADR 0335). And an agent drained for longer
|
||||||
|
// than the node-engine's bound and the slack says it is not ready: the replacement it was drained for never
|
||||||
|
// came, which is a fault to see.
|
||||||
|
const (
|
||||||
|
drainLapse = 2 * time.Minute
|
||||||
|
drainTooLong = 15*time.Minute + 5*time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
// drainedAt is when the drain began and when it was last asked, from its mark; false when there is no mark.
|
||||||
|
func drainedAt() (began, asked time.Time, ok bool) {
|
||||||
|
info, err := os.Stat(drainMark())
|
||||||
|
if err != nil {
|
||||||
|
return time.Time{}, time.Time{}, false
|
||||||
|
}
|
||||||
|
asked = info.ModTime()
|
||||||
|
began = asked
|
||||||
|
if raw, err := os.ReadFile(drainMark()); err == nil {
|
||||||
|
if t, err := time.Parse(time.RFC3339, strings.TrimSpace(string(raw))); err == nil && !t.After(asked) {
|
||||||
|
began = t
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return began, asked, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// drained says the agent was asked whether it is busy within drainLapse: it takes no new build request.
|
||||||
|
func drained() bool { return drainedSince(time.Now()) }
|
||||||
|
|
||||||
|
func drainedSince(now time.Time) bool {
|
||||||
|
_, asked, ok := drainedAt()
|
||||||
|
return ok && now.Sub(asked) < drainLapse
|
||||||
|
}
|
||||||
|
|
||||||
|
// drainedTooLong is the health check's second half: nil unless the agent has been drained, asked all along, for
|
||||||
|
// longer than its node-engine's bound and the slack.
|
||||||
|
func drainedTooLong(now time.Time) error {
|
||||||
|
began, asked, ok := drainedAt()
|
||||||
|
if !ok || now.Sub(asked) >= drainLapse || now.Sub(began) <= drainTooLong {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("drained since %s and still asked: the replacement its node-engine waits for has not come "+
|
||||||
|
"within %s (novox/hq ADR 0335)", began.UTC().Format(time.RFC3339), drainTooLong)
|
||||||
|
}
|
||||||
|
|
||||||
|
// claim marks a request in hand, unless the agent is drained: then nothing is in hand, and the request goes
|
||||||
|
// back to the queue.
|
||||||
|
func claim() bool {
|
||||||
|
// Made, never written over: one build at a time, and a mark already there is a build in hand.
|
||||||
|
f, err := os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
||||||
|
if err != nil {
|
||||||
|
// Unmarked, an ask would say idle while this build runs: the build is not taken.
|
||||||
|
fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
_, err = f.WriteString(time.Now().UTC().Format(time.RFC3339) + "\n")
|
||||||
|
if closeErr := f.Close(); err == nil {
|
||||||
|
err = closeErr
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
_ = os.Remove(inHandMark())
|
||||||
|
fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if drained() {
|
||||||
|
_ = os.Remove(inHandMark())
|
||||||
|
fmt.Fprintln(os.Stderr, "asked whether busy, so this agent takes no new build: handed back")
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// holding says what is in hand, for the ask to print: written once the request is read.
|
||||||
|
func holding(what string) {
|
||||||
|
_ = os.WriteFile(inHandMark(), []byte(time.Now().UTC().Format(time.RFC3339)+" "+what+"\n"), 0o644)
|
||||||
|
}
|
||||||
|
|
||||||
|
// released ends the build in hand.
|
||||||
|
func released() {
|
||||||
|
if err := os.Remove(inHandMark()); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
fmt.Fprintf(os.Stderr, "cannot remove the mark of the build in hand (%s): %v\n", inHandMark(), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// undrained clears both marks at the start of a holder: a container started again is asked afresh.
|
||||||
|
func undrained() error {
|
||||||
|
for _, mark := range []string{drainMark(), inHandMark()} {
|
||||||
|
if err := os.Remove(mark); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return fmt.Errorf("cannot remove %s: %w", mark, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// busy is the ask: it drains the agent, then says whether a build is in hand. Nil — exit 0 — while one is.
|
||||||
|
func busy() (string, error) {
|
||||||
|
// The mark keeps when the drain began while it is asked within drainLapse, and is written again on every
|
||||||
|
// ask: its time is the last ask.
|
||||||
|
began := time.Now().UTC()
|
||||||
|
if was, _, ok := drainedAt(); ok && drained() {
|
||||||
|
began = was.UTC()
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(drainMark(), []byte(began.Format(time.RFC3339)+"\n"), 0o644); err != nil {
|
||||||
|
// Not drained, it could take the next build: said, and answered busy only for what is in hand now.
|
||||||
|
fmt.Fprintf(os.Stderr, "cannot mark this agent drained (%s): %v\n", drainMark(), err)
|
||||||
|
}
|
||||||
|
held, err := os.ReadFile(inHandMark())
|
||||||
|
if err != nil {
|
||||||
|
return "", errors.New("idle: no build in hand, and this agent takes no new one")
|
||||||
|
}
|
||||||
|
since, what, _ := strings.Cut(strings.TrimSpace(string(held)), " ")
|
||||||
|
if what == "" {
|
||||||
|
what = "a build"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("busy: %s, in hand since %s; taking no new build", what, since), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// `mesh-builder busy` answers 0 while a build is in hand and non-zero when idle, and after its first ask the
|
||||||
|
// agent takes no new build request (novox/hq ADR 0335 §5).
|
||||||
|
func TestBusyAnswersWhileABuildIsInHandAndDrainsTheAgent(t *testing.T) {
|
||||||
|
t.Setenv("TMPDIR", t.TempDir())
|
||||||
|
if err := undrained(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if drained() {
|
||||||
|
t.Fatal("an agent never asked is drained")
|
||||||
|
}
|
||||||
|
// A build is claimed and runs.
|
||||||
|
if !claim() {
|
||||||
|
t.Fatal("an agent never asked refused a build")
|
||||||
|
}
|
||||||
|
holding("novox/mesh-controller (check-280)")
|
||||||
|
said, err := busy()
|
||||||
|
if err != nil || !strings.Contains(said, "novox/mesh-controller (check-280)") {
|
||||||
|
t.Fatalf("an agent with a build in hand did not answer busy with what it holds: %q %v", said, err)
|
||||||
|
}
|
||||||
|
// From that ask on it takes nothing new: the fetch is not made, and one fetched is handed back.
|
||||||
|
if !drained() {
|
||||||
|
t.Fatal("the ask did not drain the agent")
|
||||||
|
}
|
||||||
|
if claim() {
|
||||||
|
t.Fatal("a drained agent took a new build")
|
||||||
|
}
|
||||||
|
// The build in hand is still in hand after the refused claim, and busy goes on saying so.
|
||||||
|
if _, err := busy(); err != nil {
|
||||||
|
t.Fatalf("a refused claim ended the build in hand: %v", err)
|
||||||
|
}
|
||||||
|
// It ends: idle, and still drained.
|
||||||
|
released()
|
||||||
|
if _, err := busy(); err == nil {
|
||||||
|
t.Fatal("an agent with nothing in hand answered busy")
|
||||||
|
}
|
||||||
|
if claim() {
|
||||||
|
t.Fatal("a drained agent took a build after its last one ended")
|
||||||
|
}
|
||||||
|
// Started again, it is asked afresh.
|
||||||
|
if err := undrained(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if drained() || !claim() {
|
||||||
|
t.Fatal("a holder started again is still drained")
|
||||||
|
}
|
||||||
|
released()
|
||||||
|
}
|
||||||
|
|
||||||
|
// An ask racing a fetch sees the build in hand, or the agent sees the drain and hands the request back: never
|
||||||
|
// neither, which would replace the container under a build just taken.
|
||||||
|
func TestAnAskRacingAFetchNeverMissesTheBuild(t *testing.T) {
|
||||||
|
t.Setenv("TMPDIR", t.TempDir())
|
||||||
|
for i := 0; i < 200; i++ {
|
||||||
|
if err := undrained(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var claimed bool
|
||||||
|
var askErr error
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
wg.Add(2)
|
||||||
|
go func() { defer wg.Done(); claimed = claim() }()
|
||||||
|
go func() { defer wg.Done(); _, askErr = busy() }()
|
||||||
|
wg.Wait()
|
||||||
|
if claimed && askErr != nil {
|
||||||
|
t.Fatalf("round %d: the agent took a build and the ask said idle", i)
|
||||||
|
}
|
||||||
|
released()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A drain no longer asked lapses: an agent whose replacement was withdrawn after the first ask (a put-back, a
|
||||||
|
// revert, a stopped walk) takes work again after drainLapse, rather than staying drained with its health green.
|
||||||
|
func TestADrainNoLongerAskedLapses(t *testing.T) {
|
||||||
|
t.Setenv("TMPDIR", t.TempDir())
|
||||||
|
if err := undrained(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := busy(); err == nil {
|
||||||
|
t.Fatal("an idle agent answered busy")
|
||||||
|
}
|
||||||
|
if !drained() || claim() {
|
||||||
|
t.Fatal("an agent just asked takes work")
|
||||||
|
}
|
||||||
|
// The last ask was three minutes ago.
|
||||||
|
old := time.Now().Add(-3 * time.Minute)
|
||||||
|
if err := os.Chtimes(drainMark(), old, old); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if drained() {
|
||||||
|
t.Fatal("a drain no longer asked for three minutes still holds")
|
||||||
|
}
|
||||||
|
if !claim() {
|
||||||
|
t.Fatal("an agent whose drain lapsed takes no work")
|
||||||
|
}
|
||||||
|
released()
|
||||||
|
}
|
||||||
|
|
||||||
|
// An agent drained, and asked all along, for longer than its node-engine's bound and the slack is not ready: the
|
||||||
|
// replacement it was drained for never came. One asked again after a lapse starts a new drain.
|
||||||
|
func TestAnAgentDrainedTooLongIsNotReady(t *testing.T) {
|
||||||
|
t.Setenv("TMPDIR", t.TempDir())
|
||||||
|
if err := undrained(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
taking("mesh-build-machine")
|
||||||
|
if _, err := busy(); err == nil {
|
||||||
|
t.Fatal("an idle agent answered busy")
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
if err := drainedTooLong(now); err != nil {
|
||||||
|
t.Fatalf("an agent drained a moment ago is not ready: %v", err)
|
||||||
|
}
|
||||||
|
// The drain began 25 minutes ago, and the node-engine asked a moment ago.
|
||||||
|
began := now.Add(-25 * time.Minute).UTC().Format(time.RFC3339)
|
||||||
|
if err := os.WriteFile(drainMark(), []byte(began+"\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
err := drainedTooLong(time.Now())
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "has not come") {
|
||||||
|
t.Fatalf("an agent drained for 25 minutes reads ready: %v", err)
|
||||||
|
}
|
||||||
|
// Each ask keeps when the drain began.
|
||||||
|
if _, err := busy(); err == nil {
|
||||||
|
t.Fatal("an idle agent answered busy")
|
||||||
|
}
|
||||||
|
if err := drainedTooLong(time.Now()); err == nil {
|
||||||
|
t.Fatal("an ask started the drain again")
|
||||||
|
}
|
||||||
|
// The asks stopped: the drain lapsed, and so did the fault.
|
||||||
|
old := time.Now().Add(-3 * time.Minute)
|
||||||
|
if err := os.Chtimes(drainMark(), old, old); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := drainedTooLong(time.Now()); err != nil {
|
||||||
|
t.Fatalf("a lapsed drain still reads not ready: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An ask this build does not know exits non-zero: an older image asked `busy` reads idle, not busy for the
|
||||||
|
// node-engine's whole bound.
|
||||||
|
func TestAnUnknownCommandFails(t *testing.T) {
|
||||||
|
was := os.Args
|
||||||
|
t.Cleanup(func() { os.Args = was })
|
||||||
|
os.Args = []string{"mesh-builder", "some-later-word"}
|
||||||
|
if err := run(); err == nil {
|
||||||
|
t.Fatal("an unknown command exited 0")
|
||||||
|
}
|
||||||
|
os.Args = []string{"mesh-builder", "help"}
|
||||||
|
if err := run(); err != nil {
|
||||||
|
t.Fatalf("help failed: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -41,6 +41,9 @@ type holder struct {
|
|||||||
say func(link.HolderState) error
|
say func(link.HolderState) error
|
||||||
// remove runs what a kill needs outside the build: the containers left behind.
|
// remove runs what a kill needs outside the build: the containers left behind.
|
||||||
remove builder.Runner
|
remove builder.Runner
|
||||||
|
// scratch is where check builds are kept (novox/hq ADR 0350 §5): what the week's sweep lets go of. Nil where
|
||||||
|
// this machine publishes to no artifact store.
|
||||||
|
scratch builder.Scratch
|
||||||
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
paused bool
|
paused bool
|
||||||
@@ -131,6 +134,34 @@ func (h *holder) stateWhilePaused(ctx context.Context, every time.Duration) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// expireChecks lets go of every check build older than a week (novox/hq ADR 0350 §5), at start and every while
|
||||||
|
// after. Every holder sweeps the one store; what one let go of, another finds gone.
|
||||||
|
func (h *holder) expireChecks(ctx context.Context, every time.Duration, now func() time.Time) {
|
||||||
|
sweep := func() {
|
||||||
|
if h.scratch == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
n, err := h.scratch.Expire(ctx, builder.CheckBuildsKept, now())
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
fmt.Fprintf(os.Stderr, "check builds older than %s: %d let go of, and not all: %v\n", builder.CheckBuildsKept, n, err)
|
||||||
|
case n > 0:
|
||||||
|
fmt.Fprintf(os.Stderr, "check builds older than %s: %d check(s)' builds let go of\n", builder.CheckBuildsKept, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sweep()
|
||||||
|
tick := time.NewTicker(every)
|
||||||
|
defer tick.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-tick.C:
|
||||||
|
sweep()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// begin records the build this machine took, with the cancel that ends it.
|
// begin records the build this machine took, with the cancel that ends it.
|
||||||
func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running {
|
func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running {
|
||||||
r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})}
|
r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})}
|
||||||
|
|||||||
+162
-12
@@ -26,6 +26,7 @@ import (
|
|||||||
"os/signal"
|
"os/signal"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -75,6 +76,12 @@ digest of their own configuration. The result is printed as JSON.
|
|||||||
|
|
||||||
exits 0 once the agent running in this container is bound to its build work and pulls it, and
|
exits 0 once the agent running in this container is bound to its build work and pulls it, and
|
||||||
1 before: its container's health check.
|
1 before: its container's health check.
|
||||||
|
|
||||||
|
mesh-builder busy
|
||||||
|
|
||||||
|
exits 0 while the agent running in this container has a build in hand, and 1 when it has none;
|
||||||
|
from the first ask the agent takes no new build: its container's busy command, asked before the
|
||||||
|
container is replaced (novox/hq ADR 0335).
|
||||||
`
|
`
|
||||||
|
|
||||||
func run() error {
|
func run() error {
|
||||||
@@ -86,17 +93,41 @@ func run() error {
|
|||||||
case "build":
|
case "build":
|
||||||
return buildOnce(context.Background(), os.Args[2:])
|
return buildOnce(context.Background(), os.Args[2:])
|
||||||
case "ready":
|
case "ready":
|
||||||
// The container's health check (novox/hq issue 544): exits non-zero until the agent takes work.
|
// The container's health check (novox/hq issue 544): exits non-zero until the agent takes work, and
|
||||||
return ready()
|
// once it has been drained longer than its node-engine would wait to replace it (ADR 0335).
|
||||||
default:
|
if err := ready(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return drainedTooLong(time.Now())
|
||||||
|
case "busy":
|
||||||
|
// The container's busy command (novox/hq ADR 0335): drains the agent, and exits non-zero once
|
||||||
|
// nothing is in hand.
|
||||||
|
said, err := busy()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(said)
|
||||||
|
return nil
|
||||||
|
case "help", "-h", "--help":
|
||||||
fmt.Print(usage)
|
fmt.Print(usage)
|
||||||
return nil
|
return nil
|
||||||
|
default:
|
||||||
|
// **An ask this build does not know fails** (novox/hq ADR 0335): an image older than `busy`, asked
|
||||||
|
// it, must read as idle — exiting 0 would hold its container for the node-engine's whole bound.
|
||||||
|
fmt.Fprint(os.Stderr, usage)
|
||||||
|
return fmt.Errorf("%q is not a command this mesh-builder knows", os.Args[1])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := notReady(); err != nil {
|
if err := notReady(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// **The builder's own version is part of every reuse key** (novox/hq ADR 0350 §1): a builder that builds
|
||||||
|
// differently never takes a build another one made.
|
||||||
|
builder.Version = builder.SelfVersion(version)
|
||||||
|
if err := undrained(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
credential, err := brokerFrom()
|
credential, err := brokerFrom()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -149,8 +180,13 @@ func run() error {
|
|||||||
// **What an earlier holder here left goes before anything is taken** (novox/hq issue 306): a holder
|
// **What an earlier holder here left goes before anything is taken** (novox/hq issue 306): a holder
|
||||||
// recreated mid-check left the check's store and bus running, and the ask went to another machine.
|
// recreated mid-check left the check's store and bus running, and the ask went to another machine.
|
||||||
h.removeLeftBehind()
|
h.removeLeftBehind()
|
||||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused,
|
// Paused or drained, it takes nothing new (novox/hq ADR 0219, ADR 0335): a drained agent finishes the build
|
||||||
Taking: func() { taking(seat) }})
|
// in hand and then says idle, and its container is replaced between builds.
|
||||||
|
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{
|
||||||
|
Paused: func() bool { return h.Paused() || drained() },
|
||||||
|
Taking: func() { taking(seat) },
|
||||||
|
Claim: claim,
|
||||||
|
})
|
||||||
defer machine.Close()
|
defer machine.Close()
|
||||||
|
|
||||||
// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
|
// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
|
||||||
@@ -174,8 +210,13 @@ func run() error {
|
|||||||
|
|
||||||
fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
|
fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
|
||||||
publisher := builder.Registry{Address: registry, Run: builder.Command}
|
publisher := builder.Registry{Address: registry, Run: builder.Command}
|
||||||
|
// Check builds are let go of after a week (novox/hq ADR 0350 §5).
|
||||||
|
h.scratch = publisher
|
||||||
|
go h.expireChecks(ctx, time.Hour, time.Now)
|
||||||
|
|
||||||
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
|
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
|
||||||
|
defer released()
|
||||||
|
holding(work.Request().Repository + " (" + work.Request().ID + ")")
|
||||||
answer(ctx, publisher, on, workspace, work, h)
|
answer(ctx, publisher, on, workspace, work, h)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -232,7 +273,12 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
|||||||
// role's own events under the build's id (novox/hq ADR 0157) — so whoever asked, and anybody
|
// role's own events under the build's id (novox/hq ADR 0157) — so whoever asked, and anybody
|
||||||
// watching, reads the same lines this container's log holds, live, and after the fact from the
|
// watching, reads the same lines this container's log holds, live, and after the fact from the
|
||||||
// stream. Said first, before anything runs, so a build that hangs is one that visibly started.
|
// stream. Said first, before anything runs, so a build that hangs is one that visibly started.
|
||||||
|
// One line at a time: a check's builds for its merge say their lines while its tests say theirs (novox/hq
|
||||||
|
// issue 716).
|
||||||
|
var saying sync.Mutex
|
||||||
say := func(step, message string) {
|
say := func(step, message string) {
|
||||||
|
saying.Lock()
|
||||||
|
defer saying.Unlock()
|
||||||
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
|
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
|
||||||
work.Say(step, message)
|
work.Say(step, message)
|
||||||
if mine != nil && step != "run" && step != "output" {
|
if mine != nil && step != "run" && step != "output" {
|
||||||
@@ -262,6 +308,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
say("packages", saysRegistry(npmrc))
|
say("packages", saysRegistry(npmrc))
|
||||||
}
|
}
|
||||||
|
registryHere, isRegistry := publisher.(builder.Registry)
|
||||||
var built builder.Result
|
var built builder.Result
|
||||||
if request.Check != nil {
|
if request.Check != nil {
|
||||||
// **A pull request's merge check, not a build** (novox/hq to-be 45 §9): nothing is built,
|
// **A pull request's merge check, not a build** (novox/hq to-be 45 §9): nothing is built,
|
||||||
@@ -271,17 +318,37 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
|||||||
if r, ok := publisher.(builder.Registry); ok {
|
if r, ok := publisher.(builder.Registry); ok {
|
||||||
registry = r.Address
|
registry = r.Address
|
||||||
}
|
}
|
||||||
|
spec := checkSpecOf(request)
|
||||||
|
// The judge cache the build agent declares as its own data, never in the workspace: only judge builds and
|
||||||
|
// mesh-lab's replays mount it, never a check's containers, so no pull request's script can write what a judge
|
||||||
|
// is built with or kept as, or what the replays run (novox/hq ADR 0358 §6, issue 709).
|
||||||
|
spec.JudgeCache = judgeCache()
|
||||||
|
// What the merge would move, built by the check for the walk to take (novox/hq ADR 0350 §1): into the
|
||||||
|
// scratch namespace. **Never with the package registry's credential**: this is code nobody has approved
|
||||||
|
// yet, and a recipe could read the credential it is handed (issue 462). A module that resolves packages
|
||||||
|
// at build time has no reuse key anyway, so nothing is lost by refusing it here.
|
||||||
|
if isRegistry && len(spec.Builds) > 0 {
|
||||||
|
spec.BuildWith = &builder.CheckBuilding{Publish: publisher, Scratch: registryHere, Held: request.Held,
|
||||||
|
Running: request.Check.Running, Seats: request.Seats, Named: namedOf(request.Check.Named)}
|
||||||
|
}
|
||||||
var v builder.CheckVerdict
|
var v builder.CheckVerdict
|
||||||
v, err = builder.Check(building, builder.Command, checkSpecOf(request), workspace, registry, forgeFrom(), say)
|
v, err = builder.Check(building, builder.Command, spec, workspace, registry, forgeFrom(), say)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report,
|
result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report,
|
||||||
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo),
|
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo),
|
||||||
Head: v.Head, Trunk: v.Trunk, Beside: v.Beside}
|
Head: v.Head, Trunk: v.Trunk, Beside: v.Beside, Built: checkBuiltOf(v.Built),
|
||||||
|
Release: v.Release, JudgedBy: v.JudgedBy, BesideAt: v.BesideAt, Validator: v.Validator}
|
||||||
// What each of a group's heads was judged as (novox/hq ADR 0323 §6).
|
// What each of a group's heads was judged as (novox/hq ADR 0323 §6).
|
||||||
for _, m := range v.Members {
|
for _, m := range v.Members {
|
||||||
result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo,
|
result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo,
|
||||||
Commit: m.Head, Trunk: m.Trunk})
|
Commit: m.Head, Trunk: m.Trunk})
|
||||||
}
|
}
|
||||||
|
// A branch tried before its pull request reads its outcome from this log, whole (novox/hq issue 686).
|
||||||
|
if request.Check.Try {
|
||||||
|
if line, err := json.Marshal(triedOf(v)); err == nil {
|
||||||
|
say(link.StepTried, string(line))
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else if err == nil {
|
} else if err == nil {
|
||||||
// The package-registry credential is a build input, so it is resolved before the clone: a
|
// The package-registry credential is a build input, so it is resolved before the clone: a
|
||||||
@@ -289,9 +356,22 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
|||||||
// after a clone that then fails at npm ci.
|
// after a clone that then fails at npm ci.
|
||||||
// Every container it starts is labelled with its id, so a kill finds what outlived the
|
// Every container it starts is labelled with its id, so a kill finds what outlived the
|
||||||
// docker client (ADR 0219).
|
// docker client (ADR 0219).
|
||||||
built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
|
if request.Reuse != nil && isRegistry && !request.DryRun {
|
||||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
// **A walk's build takes the check's build of the same key** (novox/hq ADR 0350 §2).
|
||||||
forgeFrom(), say, request.Seats)
|
reuse := builder.Reuse{Scratch: registryHere}
|
||||||
|
for _, p := range request.Reuse.Pulls {
|
||||||
|
reuse.Pulls = append(reuse.Pulls, builder.PullRef{Owner: p.Owner, Repo: p.Repo, Number: p.Number})
|
||||||
|
}
|
||||||
|
// Only what a verdict names is taken (ADR 0350 §2).
|
||||||
|
reuse.Named = namedOf(request.Reuse.Builds)
|
||||||
|
built, err = builder.BuildReusing(building, builder.Labelled(builder.Command, request.ID), publisher, reuse,
|
||||||
|
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||||
|
forgeFrom(), say, request.Seats)
|
||||||
|
} else {
|
||||||
|
built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
|
||||||
|
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||||
|
forgeFrom(), say, request.Seats)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// What it copied into the store, whatever became of the build (novox/hq ADR 0257).
|
// What it copied into the store, whatever became of the build (novox/hq ADR 0257).
|
||||||
result.Mirrored = built.Mirrored
|
result.Mirrored = built.Mirrored
|
||||||
@@ -327,6 +407,19 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
|||||||
}
|
}
|
||||||
result.Against = built.Against
|
result.Against = built.Against
|
||||||
result.SourceFingerprint = built.Source
|
result.SourceFingerprint = built.Source
|
||||||
|
// What the walk did instead of building, or why it built (novox/hq ADR 0350 §3, §4).
|
||||||
|
result.ReuseKey = built.ReuseKey
|
||||||
|
if built.Reused != nil {
|
||||||
|
f := built.Reused.From
|
||||||
|
result.Reused = built.Reused.Words()
|
||||||
|
result.ReusedFrom = &link.ReusedFrom{Pull: link.PullRef{Owner: f.Pull.Owner, Repo: f.Pull.Repo,
|
||||||
|
Number: f.Pull.Number}, Head: f.Head, Trunk: f.Trunk, Check: f.Check, Key: f.Key}
|
||||||
|
} else if request.Reuse != nil {
|
||||||
|
result.BuiltWhy = built.Why
|
||||||
|
if result.BuiltWhy == "" {
|
||||||
|
result.BuiltWhy = "it has no artifact to build"
|
||||||
|
}
|
||||||
|
}
|
||||||
result.Trunk, result.OnTrunk, result.Branches = built.Trunk, built.OnTrunk, built.Branches
|
result.Trunk, result.OnTrunk, result.Branches = built.Trunk, built.OnTrunk, built.Branches
|
||||||
for _, r := range built.Read {
|
for _, r := range built.Read {
|
||||||
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
||||||
@@ -373,9 +466,14 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
|
|||||||
spec := builder.CheckSpec{ID: request.ID, Repository: request.Repository, Ref: request.Ref,
|
spec := builder.CheckSpec{ID: request.ID, Repository: request.Repository, Ref: request.Ref,
|
||||||
Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{},
|
Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{},
|
||||||
Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base,
|
Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base,
|
||||||
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held)}
|
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held),
|
||||||
|
Tests: c.Tests, StoreOnly: c.StoreOnly, Release: c.Release}
|
||||||
for dir, b := range c.Beside {
|
for dir, b := range c.Beside {
|
||||||
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref}
|
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref, Else: b.Else}
|
||||||
|
}
|
||||||
|
for _, b := range c.Builds {
|
||||||
|
spec.Builds = append(spec.Builds, builder.CheckBuildSpec{Module: b.Module, Repository: b.Repository, Path: b.Path,
|
||||||
|
Ref: b.Ref, Merged: b.Merged, Tier: b.Tier, Skip: b.Skip})
|
||||||
}
|
}
|
||||||
for _, m := range c.Members {
|
for _, m := range c.Members {
|
||||||
spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository,
|
spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository,
|
||||||
@@ -384,6 +482,58 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
|
|||||||
return spec
|
return spec
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// judgeCache is the judge cache the build agent's manifest declares (MESH_JUDGE_CACHE, novox/hq ADR 0358 §6): a
|
||||||
|
// directory mounted at the same path on the host, so a judge build's mounts name what the agent sees. Empty when the
|
||||||
|
// agent declares none, and every judge is then built cold under its check's root.
|
||||||
|
func judgeCache() string { return os.Getenv("MESH_JUDGE_CACHE") }
|
||||||
|
|
||||||
|
// triedOf is a try's outcome as its log line says it (novox/hq issue 686): the repository's own check's verdict,
|
||||||
|
// or the check's where that did not run — a head that does not merge — and the lines naming what failed.
|
||||||
|
func triedOf(v builder.CheckVerdict) link.Tried {
|
||||||
|
t := link.Tried{Verdict: v.Verdict, Summary: v.Summary, Took: v.Took.Round(time.Second).String(),
|
||||||
|
Head: v.Head, Trunk: v.Trunk}
|
||||||
|
if v.Repo != nil && (v.Gate == nil || v.Gate.Verdict == "pass") {
|
||||||
|
t.Verdict, t.Summary = v.Repo.Verdict, v.Repo.Summary
|
||||||
|
}
|
||||||
|
if v.Repo != nil && v.Repo.Failed != "" {
|
||||||
|
t.Failed = strings.Split(v.Repo.Failed, "\n")
|
||||||
|
}
|
||||||
|
if t.Verdict == "" {
|
||||||
|
t.Verdict = "error"
|
||||||
|
}
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
|
||||||
|
// namedOf are the builds the seat's verdicts name, as the builder believes a scratch record by them (novox/hq ADR
|
||||||
|
// 0350 §2, issue 703): each module's key with every artifact's digest.
|
||||||
|
func namedOf(builds []link.CheckedBuild) []builder.NamedBuild {
|
||||||
|
var out []builder.NamedBuild
|
||||||
|
for _, b := range builds {
|
||||||
|
named := builder.NamedBuild{Pull: builder.PullRef{Owner: b.Pull.Owner, Repo: b.Pull.Repo,
|
||||||
|
Number: b.Pull.Number}, Check: b.Check, Module: b.Built.Module, Key: b.Built.Key,
|
||||||
|
Digests: map[string]string{}}
|
||||||
|
for _, m := range b.Built.Made {
|
||||||
|
named.Digests[m.Name] = builder.DigestOf(m.Reference)
|
||||||
|
}
|
||||||
|
out = append(out, named)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkBuiltOf is a check's builds for its merge as the outcome carries them (novox/hq ADR 0350 §1).
|
||||||
|
func checkBuiltOf(built []builder.CheckBuilt) []link.CheckBuilt {
|
||||||
|
var out []link.CheckBuilt
|
||||||
|
for _, b := range built {
|
||||||
|
c := link.CheckBuilt{Module: b.Module, Key: b.Key, NotBuilt: b.NotBuilt, Took: b.Took.Round(time.Second).String(),
|
||||||
|
Kept: b.Kept, Same: b.Same, SameAs: b.SameAs}
|
||||||
|
for _, m := range b.Made {
|
||||||
|
c.Made = append(c.Made, link.MadeArtifact{Name: m.Name, Kind: m.Kind, Reference: m.Reference})
|
||||||
|
}
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// layerOf is one layer of a check as the outcome carries it.
|
// layerOf is one layer of a check as the outcome carries it.
|
||||||
func layerOf(l *builder.Layer) *link.CheckLayer {
|
func layerOf(l *builder.Layer) *link.CheckLayer {
|
||||||
if l == nil {
|
if l == nil {
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **A pull request into a release branch reaches the check as one** (novox/hq ADR 0358): the build seat hands the
|
||||||
|
// check the release branch and, beside it, the trunk each repository is read at when it has no branch of that name;
|
||||||
|
// and it hands the check the judge cache the build agent declares (MESH_JUDGE_CACHE), never a path of its own.
|
||||||
|
func TestTheBuildSeatHandsTheCheckItsReleaseBranch(t *testing.T) {
|
||||||
|
spec := checkSpecOf(link.BuildRequest{ID: "b", Repository: "r", Ref: "abc", Check: &link.CheckRequest{
|
||||||
|
Owner: "novox", Repo: "mesh-catalog", Base: "feat/57", Release: "feat/57",
|
||||||
|
Beside: map[string]link.CheckedOut{"mesh-host": {Repository: "h", Ref: "feat/57", Else: "main"}}}})
|
||||||
|
if spec.Release != "feat/57" || spec.Beside["mesh-host"].Ref != "feat/57" || spec.Beside["mesh-host"].Else != "main" {
|
||||||
|
t.Fatalf("the check was handed release %q and mesh-host %+v", spec.Release, spec.Beside["mesh-host"])
|
||||||
|
}
|
||||||
|
t.Setenv("MESH_JUDGE_CACHE", "/var/lib/mesh/build-agent/judge-cache")
|
||||||
|
if got := judgeCache(); got != "/var/lib/mesh/build-agent/judge-cache" {
|
||||||
|
t.Fatalf("the judge cache the manifest declares is read as %q", got)
|
||||||
|
}
|
||||||
|
t.Setenv("MESH_JUDGE_CACHE", "")
|
||||||
|
if got := judgeCache(); got != "" {
|
||||||
|
t.Fatalf("with no judge cache declared, %q is used", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -16,6 +16,7 @@ func TestBuilderDiagnosticsStayOffStdout(t *testing.T) {
|
|||||||
"version)": true, // --version
|
"version)": true, // --version
|
||||||
`"stopping")`: true, // the loop.s shutdown line
|
`"stopping")`: true, // the loop.s shutdown line
|
||||||
"usage)": true, // --help text, for a human
|
"usage)": true, // --help text, for a human
|
||||||
|
"said)": true, // busy: the answer its node-engine reads (novox/hq ADR 0335)
|
||||||
}
|
}
|
||||||
for _, file := range []string{"once.go", "main.go"} {
|
for _, file := range []string{"once.go", "main.go"} {
|
||||||
src, err := os.ReadFile(file)
|
src, err := os.ReadFile(file)
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/builder"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A try's outcome, as the build seat says it in the build's log (novox/hq issue 686): the repository's own
|
||||||
|
// check's verdict, how long it took, and each line naming what failed.
|
||||||
|
func TestATrysOutcomeNamesWhatFailed(t *testing.T) {
|
||||||
|
tried := triedOf(builder.CheckVerdict{Verdict: "pass", Summary: "the change touches no module", Took: 303 * time.Second,
|
||||||
|
Head: "c0ffee", Trunk: "ca3f43c", Gate: &builder.Layer{Verdict: "pass"},
|
||||||
|
Repo: &builder.Layer{Verdict: "fail", Summary: "its merge-check.sh failed: TestX",
|
||||||
|
Failed: "--- FAIL: TestX (0.01s)\n x_test.go:12: got 1, want 2"}})
|
||||||
|
if tried.Verdict != "fail" || tried.Took != "5m3s" || len(tried.Failed) != 2 || tried.Trunk != "ca3f43c" ||
|
||||||
|
tried.Summary != "its merge-check.sh failed: TestX" {
|
||||||
|
t.Fatalf("a failed try: %+v", tried)
|
||||||
|
}
|
||||||
|
// A head that does not merge fails both layers: the check's own word.
|
||||||
|
conflict := triedOf(builder.CheckVerdict{Verdict: "fail", Summary: "does not merge", Gate: &builder.Layer{Verdict: "fail"},
|
||||||
|
Repo: &builder.Layer{Verdict: "fail", Summary: "does not merge"}})
|
||||||
|
if conflict.Verdict != "fail" || conflict.Summary != "does not merge" {
|
||||||
|
t.Fatalf("a try that does not merge: %+v", conflict)
|
||||||
|
}
|
||||||
|
// No verdict is an error, never a pass.
|
||||||
|
if none := triedOf(builder.CheckVerdict{}); none.Verdict != "error" {
|
||||||
|
t.Fatalf("a try with no verdict: %+v", none)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A provision whose credential only a person outside the mesh can give (novox/hq ADR 0337, issue 164, issue 165):
|
||||||
|
// given once for its provider, at the controller's terminal, and sealed in one act to every consumer that binds the
|
||||||
|
// provision from that provider now. The value is never read back, printed or kept.
|
||||||
|
|
||||||
|
// outsideOffer is the provider module's own name when `secret accept <node> <module> <name>` names a provision that
|
||||||
|
// module offers with a credential issued outside the mesh, and "" when it names anything else (an own secret, or a
|
||||||
|
// pair with --provider).
|
||||||
|
func outsideOffer(ctx context.Context, module, name, provider string) (string, error) {
|
||||||
|
if provider != "" {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
shelf, err := open.inventory.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if m, known := shelf[module]; known && m.IssuedOutsideOf(name) {
|
||||||
|
return module, nil
|
||||||
|
}
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// offeredOutsideBy is a module that offers the provision with a credential issued outside the mesh, or "".
|
||||||
|
func offeredOutsideBy(ctx context.Context, provision string) (string, error) {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
shelf, err := open.inventory.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
module, _ := catalogue.OffersIssuedOutside(shelf, provision)
|
||||||
|
return module, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// consumersFrom is every consumer that binds a provision from one provider now, found by resolving every machine as
|
||||||
|
// its next send would, and the machines that could not be resolved, with why.
|
||||||
|
func consumersFrom(ctx context.Context, open *stores, provider, providerModule, provision string) (
|
||||||
|
[]inventory.PairEnd, []string, error) {
|
||||||
|
nodes, err := open.inventory.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
var out []inventory.PairEnd
|
||||||
|
var unresolved []string
|
||||||
|
for _, n := range nodes {
|
||||||
|
plan, _, err := planFor(ctx, open, n.Name)
|
||||||
|
if err != nil {
|
||||||
|
unresolved = append(unresolved, fmt.Sprintf("%s: %s", n.Name, oneLine(err.Error())))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, bindsFrom(plan, n.Name, provider, providerModule, provision)...)
|
||||||
|
}
|
||||||
|
return out, unresolved, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// bindsFrom is every consumer on one machine whose resolution answers the provision from that provider. Pure.
|
||||||
|
func bindsFrom(plan catalogue.Resolution, machine, provider, providerModule, provision string) []inventory.PairEnd {
|
||||||
|
var out []inventory.PairEnd
|
||||||
|
for _, need := range plan.Needs {
|
||||||
|
if need.ByRecord || need.Name != provision || need.From != provider {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if need.Module != "" && need.Module != providerModule {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, inventory.PairEnd{Consumer: machine, ConsumerModule: need.For, Local: need.Local})
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool {
|
||||||
|
return out[i].ConsumerModule+"\x00"+out[i].Local < out[j].ConsumerModule+"\x00"+out[j].Local
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// acceptForProvision gives a provision's credential issued outside the mesh, once for its provider (ADR 0337 §3).
|
||||||
|
func acceptForProvision(ctx context.Context, provider, providerModule, provision, value string) error {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
consumers, unresolved, err := consumersFrom(ctx, open, provider, providerModule, provision)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
accepted, err := open.inventory.AcceptSecretForProvision(ctx, provision, provider, providerModule, value, consumers)
|
||||||
|
value = ""
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Print(acceptedSaid(provider, providerModule, provision, accepted, unresolved))
|
||||||
|
if len(accepted.Sealed) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := sendTo(ctx, open, accepted.Machines); err != nil {
|
||||||
|
return fmt.Errorf("%s is sealed and could not be sent: %w; `push %s` sends it", provision, err,
|
||||||
|
strings.Join(accepted.Machines, "`, `push "))
|
||||||
|
}
|
||||||
|
fmt.Printf(" sent %s; each consumer starts again on it\n", strings.Join(accepted.Machines, ", "))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// acceptedSaid is what one accept for a provider did, in words that never carry the value. Pure.
|
||||||
|
func acceptedSaid(provider, providerModule, provision string, a inventory.ProvisionAccepted, unresolved []string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
if len(a.Sealed) == 0 {
|
||||||
|
fmt.Fprintf(&b, "%s from %s on %s: no consumer binds it now, so nothing was sealed; a consumer that binds "+
|
||||||
|
"later waits for it, and this line given again seals it to every consumer then\n", provision,
|
||||||
|
providerModule, provider)
|
||||||
|
} else {
|
||||||
|
var names []string
|
||||||
|
for _, c := range a.Sealed {
|
||||||
|
names = append(names, fmt.Sprintf("%s on %s%s", c.ConsumerModule, c.Consumer, asLocal(c.Local)))
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, "%s from %s on %s is given, sealed to %s and to %s\n", provision, providerModule, provider,
|
||||||
|
strings.Join(names, ", "), provider)
|
||||||
|
fmt.Fprintf(&b, " the mesh cannot read it back, never makes one of its own for it, and will not rotate it\n")
|
||||||
|
}
|
||||||
|
for _, c := range a.Unsealable {
|
||||||
|
fmt.Fprintf(&b, " not sealed to %s on %s: no sealing key on that machine; give it again once it has one\n",
|
||||||
|
c.ConsumerModule, c.Consumer)
|
||||||
|
}
|
||||||
|
for _, u := range unresolved {
|
||||||
|
fmt.Fprintf(&b, " not known whether a consumer on %s binds it, which did not resolve; give it again once it does\n", u)
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
@@ -26,8 +26,11 @@ import (
|
|||||||
//
|
//
|
||||||
// - **The serving controller** takes it before it does anything else — before it asserts the bus's
|
// - **The serving controller** takes it before it does anything else — before it asserts the bus's
|
||||||
// objects, which are the controller's to write — waiting while another holds it, and renews it.
|
// objects, which are the controller's to write — waiting while another holds it, and renews it.
|
||||||
// A renewal refused or failed is the lease lost: the gate closes at once and the process exits, so
|
// Another holder seen, or the key's age passed with no renewal made, is the lease lost: the gate
|
||||||
// its service manager restarts it as a candidate (serve, in push.go).
|
// closes at once and the process exits, so its service manager restarts it as a candidate (serve,
|
||||||
|
// in push.go). A renewal the bus does not answer in time is not a loss (novox/hq issue 477): the
|
||||||
|
// lease is kept, no new act starts once half the key's age passed unrenewed, and the next renewal
|
||||||
|
// made opens the gate again.
|
||||||
// - **A command run at a shell** — `push` in the installer, the lab, a person repairing a mesh whose
|
// - **A command run at a shell** — `push` in the installer, the lab, a person repairing a mesh whose
|
||||||
// controller is down (issue 201) — acts **under the holder's epoch** when a controller holds the
|
// controller is down (issue 201) — acts **under the holder's epoch** when a controller holds the
|
||||||
// lease: it is the same mesh's word, composed and sent under the store's hold of each machine like
|
// lease: it is the same mesh's word, composed and sent under the store's hold of each machine like
|
||||||
|
|||||||
@@ -203,8 +203,9 @@ func seatDependenciesOnAssign(ctx context.Context, open *stores, node string, mo
|
|||||||
if _, err := catalogue.AssignRefusal(shelf, node, assigned, adding); err != nil {
|
if _, err := catalogue.AssignRefusal(shelf, node, assigned, adding); err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
// Two modules declaring one package, path or unit is refused before anything is recorded
|
// Two modules declaring one path or unit, or one package present and absent, is refused before anything
|
||||||
// (novox/hq ADR 0210, 04-ISSUES/235): kept, the node would not resolve until one came off again.
|
// is recorded (novox/hq ADR 0210, ADR 0303, 04-ISSUES/235): kept, the node would not resolve until one came
|
||||||
|
// off again. Two modules declaring one package present is not a collision (ADR 0303, issue 628).
|
||||||
if err := catalogue.CollisionRefusal(shelf, node, assigned, adding); err != nil {
|
if err := catalogue.CollisionRefusal(shelf, node, assigned, adding); err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -140,7 +140,7 @@ func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testin
|
|||||||
}
|
}
|
||||||
|
|
||||||
// And a replay to the catalogue says where the store is now.
|
// And a replay to the catalogue says where the store is now.
|
||||||
announced, err := following{open}.Announceable(ctx)
|
announced, err := following{open: open}.Announceable(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -722,7 +722,7 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
|
|||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
derived := derivedFilter{rules: rules, foundation: with.Foundation, mesh: with.Mesh,
|
derived := derivedFilter{rules: rules, foundation: with.Foundation, mesh: with.Mesh,
|
||||||
outward: plan.PublicDomain != "", outwardLinks: with.OutwardLinks}
|
outward: plan.PublicDomain != "", outwardLinks: with.OutwardLinks, local: with.LocalNetworks}
|
||||||
filtering, err := inv.FilteringOf(ctx, node)
|
filtering, err := inv.FilteringOf(ctx, node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
@@ -930,6 +930,9 @@ type derivedFilter struct {
|
|||||||
// The filter constrains what arrives on them; everything arriving elsewhere is this machine's
|
// The filter constrains what arrives on them; everything arriving elsewhere is this machine's
|
||||||
// own guest and is not filtered.
|
// own guest and is not filtered.
|
||||||
outwardLinks []string
|
outwardLinks []string
|
||||||
|
// local is the machine's local networks, which a port from `local-network` stays open to (novox/hq
|
||||||
|
// ADR 0340).
|
||||||
|
local []string
|
||||||
}
|
}
|
||||||
|
|
||||||
// closesOutside is what a narrowing from everywhere to the private network is called: it closes.
|
// closesOutside is what a narrowing from everywhere to the private network is called: it closes.
|
||||||
@@ -977,6 +980,17 @@ func (d derivedFilter) fate(r inventory.Reach) string {
|
|||||||
if !onMesh {
|
if !onMesh {
|
||||||
return fmt.Sprintf("%s — declared by %s from the mesh only", closesOutside, by)
|
return fmt.Sprintf("%s — declared by %s from the mesh only", closesOutside, by)
|
||||||
}
|
}
|
||||||
|
case catalogue.FromLocalNetwork:
|
||||||
|
// The private network and this machine's local networks (novox/hq ADR 0340): it closes to
|
||||||
|
// everything else outside, and says which local networks stay.
|
||||||
|
if !onMesh {
|
||||||
|
if len(d.local) == 0 {
|
||||||
|
return fmt.Sprintf("%s — declared by %s from the local network, and this machine "+
|
||||||
|
"reported none", closesOutside, by)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("WILL CLOSE to everything outside the private network and %s — declared "+
|
||||||
|
"by %s from the local network", strings.Join(d.local, ", "), by)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return fmt.Sprintf("declared by %s (from %s)", by, rule.From)
|
return fmt.Sprintf("declared by %s (from %s)", by, rule.From)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -688,7 +688,7 @@ func spelledOf(m inventory.BatchedMerge) string {
|
|||||||
func namedOf(m inventory.BatchedMerge, repository string) inventory.PlanMerge {
|
func namedOf(m inventory.BatchedMerge, repository string) inventory.PlanMerge {
|
||||||
k := announcedOf(m)
|
k := announcedOf(m)
|
||||||
return inventory.PlanMerge{Repository: repository, Commit: m.Commit, Number: k.Number, Title: k.Title, Moves: k.Moves,
|
return inventory.PlanMerge{Repository: repository, Commit: m.Commit, Number: k.Number, Title: k.Title, Moves: k.Moves,
|
||||||
Merged: m.Merged, Heard: m.Heard}
|
Head: k.HeadSHA, Merged: m.Merged, Heard: m.Heard}
|
||||||
}
|
}
|
||||||
|
|
||||||
// laterMerge says a was merged after b: by the forge's merge time, then by when each was heard.
|
// laterMerge says a was merged after b: by the forge's merge time, then by when each was heard.
|
||||||
@@ -1000,10 +1000,15 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
|||||||
plan.Delivery.Merges = named
|
plan.Delivery.Merges = named
|
||||||
plan.Delivery.Alone = batch.Delivery != nil && batch.Delivery.Alone
|
plan.Delivery.Alone = batch.Delivery != nil && batch.Delivery.Alone
|
||||||
plan.Delivery.Withheld = withheld
|
plan.Delivery.Withheld = withheld
|
||||||
|
// **Left where it is, a module whose program did not change** (novox/hq issue 721): its check found it the same as
|
||||||
|
// the build the mesh holds.
|
||||||
|
for n, why := range leaveSameAsRunning(ctx, inv, &plan, moved, append(append([]inventory.Edge{}, edges...), ordered...)) {
|
||||||
|
unchanged[n] = why
|
||||||
|
}
|
||||||
plan.Delivery.Unchanged = unchangedLeft(unchanged, plan.Modules)
|
plan.Delivery.Unchanged = unchangedLeft(unchanged, plan.Modules)
|
||||||
// **Its moments and its class** (novox/hq ADR 0282 decision 6): measured, never acted on.
|
// **Its moments and its class** (novox/hq ADR 0282 decision 6): measured, never acted on.
|
||||||
plan.Times = walkTimesAtCut(*batch, plan, entries, now)
|
plan.Times = walkTimesAtCut(*batch, plan, entries, now)
|
||||||
if len(moved) == 0 {
|
if len(moved) == 0 || len(plan.Modules) == 0 {
|
||||||
plan.State = inventory.PlanDone
|
plan.State = inventory.PlanDone
|
||||||
plan.Tiers = [][]string{}
|
plan.Tiers = [][]string{}
|
||||||
plan.Note = "moved nothing the mesh holds: nothing to walk"
|
plan.Note = "moved nothing the mesh holds: nothing to walk"
|
||||||
|
|||||||
@@ -758,7 +758,7 @@ func TestTheCatchUpKeepsWhatACutMadeHistory(t *testing.T) {
|
|||||||
return entries, read, err
|
return entries, read, err
|
||||||
}
|
}
|
||||||
if err := catchUpOnMerges(ctx, time.Now(), unheardMerges{announcedList(lost), open.inventory}, catalogued,
|
if err := catchUpOnMerges(ctx, time.Now(), unheardMerges{announcedList(lost), open.inventory}, catalogued,
|
||||||
(following{open}).SourceMoved, t.Logf); err != nil {
|
(following{open: open}).SourceMoved, t.Logf); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/builder"
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
@@ -483,6 +484,7 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
|
|||||||
Seats: seatBases(ctx),
|
Seats: seatBases(ctx),
|
||||||
DryRun: dryRun,
|
DryRun: dryRun,
|
||||||
}
|
}
|
||||||
|
request.Reuse = reuseAsked(ctx, dryRun)
|
||||||
fmt.Printf("asked for %s", source)
|
fmt.Printf("asked for %s", source)
|
||||||
if source.Seat != "" {
|
if source.Seat != "" {
|
||||||
fmt.Printf(" (%s)", repository)
|
fmt.Printf(" (%s)", repository)
|
||||||
@@ -669,6 +671,12 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
|||||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
|
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
|
||||||
manifest.Module, short(result.Commit), err)
|
manifest.Module, short(result.Commit), err)
|
||||||
}
|
}
|
||||||
|
// **Never a build in the scratch namespace** (novox/hq ADR 0239 §9, ADR 0350 §2): what a check builds is
|
||||||
|
// registered only as the walk's copy in the module's own namespace.
|
||||||
|
if scratch := inScratch(result.Made); scratch != "" {
|
||||||
|
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %s is in the scratch "+
|
||||||
|
"namespace, which only a walk's copy leaves (ADR 0350)", result.On, manifest.Module, short(result.Commit), scratch)
|
||||||
|
}
|
||||||
// **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard
|
// **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard
|
||||||
// twice, or replayed, would otherwise make the build a rollback put back what the module is again,
|
// twice, or replayed, would otherwise make the build a rollback put back what the module is again,
|
||||||
// and the next push would send it.
|
// and the next push would send it.
|
||||||
@@ -714,6 +722,16 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
|||||||
return manifest, kept, nil
|
return manifest, kept, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// inScratch is the first artifact a build made into the scratch namespace; empty when none is.
|
||||||
|
func inScratch(made []link.MadeArtifact) string {
|
||||||
|
for _, m := range made {
|
||||||
|
if path, kept := catalogue.InArtifactStore(catalogue.Recorded(m.Reference)); kept && strings.HasPrefix(path, builder.ScratchPrefix) {
|
||||||
|
return m.Reference
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
// errOffTheTrunk is a build of a commit off its repository's trunk, which is never published.
|
// errOffTheTrunk is a build of a commit off its repository's trunk, which is never published.
|
||||||
var errOffTheTrunk = errors.New("the commit is not on its repository's trunk: a commit off the trunk is checked, " +
|
var errOffTheTrunk = errors.New("the commit is not on its repository's trunk: a commit off the trunk is checked, " +
|
||||||
"never published (ADR 0238) — merge it, and the merge builds it")
|
"never published (ADR 0238) — merge it, and the merge builds it")
|
||||||
@@ -881,6 +899,34 @@ type answers struct {
|
|||||||
//
|
//
|
||||||
// Routed through the artifact store as the network reaches it now (novox/hq 04-ISSUES/102): a
|
// Routed through the artifact store as the network reaches it now (novox/hq 04-ISSUES/102): a
|
||||||
// base is recorded by digest and path, and a build machine needs something it can pull.
|
// base is recorded by digest and path, and a build machine needs something it can pull.
|
||||||
|
// runningBy is, keyed "<module>/<artifact>", each artifact of the build the mesh has registered for its module, as a
|
||||||
|
// builder reaches the artifact store (novox/hq issue 721): what a check compares its builds with. Empty when it cannot
|
||||||
|
// be read, said, and then nothing is said unchanged.
|
||||||
|
func runningBy(ctx context.Context) map[string]string {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not read what this mesh runs, so the check compares nothing: %v\n", err)
|
||||||
|
// empty-on-error: said above; a check given nothing to compare with says no module unchanged, and all move
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
running, err := open.inventory.RegisteredHeld(ctx)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not read what this mesh runs, so the check compares nothing: %v\n", err)
|
||||||
|
// empty-on-error: said above; a check given nothing to compare with says no module unchanged, and all move
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
address, err := whereABuilderReachesTheStore(ctx, open.inventory)
|
||||||
|
if err != nil || address == "" {
|
||||||
|
return running
|
||||||
|
}
|
||||||
|
routed := make(map[string]string, len(running))
|
||||||
|
for key, reference := range running {
|
||||||
|
routed[key] = catalogue.Rerouted(reference, address)
|
||||||
|
}
|
||||||
|
return routed
|
||||||
|
}
|
||||||
|
|
||||||
func heldBy(ctx context.Context) map[string]string {
|
func heldBy(ctx context.Context) map[string]string {
|
||||||
open, err := openStores(ctx)
|
open, err := openStores(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -0,0 +1,92 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **Every user's slow consumer is S9's, not the controller's alone** (novox/hq ADR 0357, issue 270). On
|
||||||
|
// 2026-10-07 the bus closed two clients on the laptop as slow consumers and nothing raised a condition:
|
||||||
|
// the controller heard only its own connection. Said by the bus module and read back, a user the bus
|
||||||
|
// closes is one condition naming the module and its machine, raised at the first look and cleared an hour
|
||||||
|
// after the last closure.
|
||||||
|
func TestAUserTheBusClosedAsASlowConsumerIsOneConditionNamingItsModuleAndMachine(t *testing.T) {
|
||||||
|
closed := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
|
||||||
|
now := closed.Add(20 * time.Second)
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||||
|
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||||
|
t.Cleanup(func() { k.Close(t.Context()) })
|
||||||
|
|
||||||
|
advisories := link.BusClosures([]link.BusClosure{
|
||||||
|
{User: "laptop.claude-code", CID: 10, Reason: "Slow Consumer (Write Deadline)", Opened: closed.Add(-time.Hour),
|
||||||
|
Closed: closed.Add(-10 * time.Minute)},
|
||||||
|
{User: "laptop.claude-code", CID: 44, Reason: "Slow Consumer (Write Deadline)", Opened: closed.Add(-time.Minute),
|
||||||
|
Closed: closed},
|
||||||
|
{User: "anchor.desk", CID: 7, Reason: "Maximum Message Payload Exceeded", Opened: closed.Add(-time.Hour),
|
||||||
|
Closed: closed},
|
||||||
|
}, []string{link.AdvisorySlowConsumer, link.AdvisorySlowConsumer, link.AdvisoryConnectionClosed}, closed.Add(-time.Hour))
|
||||||
|
look := func() {
|
||||||
|
t.Helper()
|
||||||
|
f := &signalFacts{now: now, host: "anchor", advisories: advisories}
|
||||||
|
if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
look()
|
||||||
|
|
||||||
|
slow, found, err := conditions.ReadOne(t.Context(), store, "bus.laptop.claude-code.slow-consumer")
|
||||||
|
if err != nil || !found {
|
||||||
|
t.Fatalf("no slow-consumer condition for the user: %v %v", found, err)
|
||||||
|
}
|
||||||
|
if slow.Subject.Machine != "laptop" || slow.Observations != 2 || !slow.LastObserved.Equal(closed) {
|
||||||
|
t.Fatalf("the slow consumer is said on %q, %d closure(s), last %s: %+v", slow.Subject.Machine, slow.Observations,
|
||||||
|
slow.LastObserved, slow)
|
||||||
|
}
|
||||||
|
if !strings.Contains(slow.Summary, "claude-code on laptop") || strings.Contains(slow.Summary, "laptop.claude-code") {
|
||||||
|
t.Fatalf("the summary does not name the module in words, or carries the dotted user: %q", slow.Summary)
|
||||||
|
}
|
||||||
|
if !strings.Contains(slow.Headline, "Claude-code on laptop") {
|
||||||
|
t.Fatalf("the headline is %q", slow.Headline)
|
||||||
|
}
|
||||||
|
cut, found, err := conditions.ReadOne(t.Context(), store, "bus.anchor.desk.connection-closed")
|
||||||
|
if err != nil || !found || cut.Subject.Machine != "anchor" || !strings.Contains(cut.Summary, "maximum message payload exceeded") {
|
||||||
|
t.Fatalf("the user cut off for breaking the bus's rules: %v %v %+v", found, err, cut)
|
||||||
|
}
|
||||||
|
|
||||||
|
// An hour after the newest closure without another, both clear.
|
||||||
|
now = closed.Add(time.Hour + time.Minute)
|
||||||
|
look()
|
||||||
|
for _, key := range []string{"bus.laptop.claude-code.slow-consumer", "bus.anchor.desk.connection-closed"} {
|
||||||
|
if _, found, _ := k.Get(t.Context(), key); found {
|
||||||
|
t.Fatalf("%s is still open an hour after the last closure", key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A read of the bus module's events that fails does not blind S9** (review of #326): the controller's own
|
||||||
|
// signals are still said, and the failed read is a condition of its own.
|
||||||
|
func TestABusModuleReadThatFailsKeepsTheControllersOwnSignals(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
|
||||||
|
f := &signalFacts{now: at.Add(time.Minute), host: "anchor",
|
||||||
|
advisories: []link.Advisory{{Kind: link.AdvisorySlowConsumer, ID: "controller", Said: "the controller fell behind",
|
||||||
|
First: at, Last: at, Count: 1}},
|
||||||
|
busFaultsErr: errors.New("nats: timeout")}
|
||||||
|
keys := map[string]bool{}
|
||||||
|
for _, o := range watchAdvisories(f) {
|
||||||
|
keys[o.Key()] = true
|
||||||
|
}
|
||||||
|
if !keys["bus.controller.slow-consumer"] || !keys["bus.bus-module.bus-faults-unread"] {
|
||||||
|
t.Fatalf("observed %v: the controller's own slow consumer and the unread events, both", keys)
|
||||||
|
}
|
||||||
|
for _, row := range signalsTable {
|
||||||
|
if row.Row == "S9" && row.needs(f) != nil {
|
||||||
|
t.Fatal("the S9 row went blind for a read of the bus module's events")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -60,9 +60,19 @@ func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Ra
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
// And every module's own stream (novox/hq ADR 0363), from the catalogue, before the consumers on it.
|
||||||
|
modStreams, err := inv.DeclaredStreams(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
// Every one tried, and every failure named: one module's consumer the bus refuses is no reason
|
// Every one tried, and every failure named: one module's consumer the bus refuses is no reason
|
||||||
// the modules after it in the list hear nothing (novox/hq issue 208, where this runs on each send).
|
// the modules after it in the list hear nothing (novox/hq issue 208, where this runs on each send).
|
||||||
var failed []error
|
var failed []error
|
||||||
|
for _, s := range broker.ModuleStreams(modStreams) {
|
||||||
|
if err := r.EnsureStream(s); err != nil {
|
||||||
|
failed = append(failed, fmt.Errorf("the module stream %s: %w", s.Name, err))
|
||||||
|
}
|
||||||
|
}
|
||||||
for _, s := range trafficStreams {
|
for _, s := range trafficStreams {
|
||||||
if err := r.EnsureStream(s); err != nil {
|
if err := r.EnsureStream(s); err != nil {
|
||||||
failed = append(failed, fmt.Errorf("the work queue %s: %w", s.Name, err))
|
failed = append(failed, fmt.Errorf("the work queue %s: %w", s.Name, err))
|
||||||
@@ -73,11 +83,30 @@ func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Ra
|
|||||||
failed = append(failed, fmt.Errorf("the worker %s on %s: %w", c.Name, c.Stream, err))
|
failed = append(failed, fmt.Errorf("the worker %s on %s: %w", c.Name, c.Stream, err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
restricted := map[string]bool{}
|
||||||
|
for _, s := range modStreams {
|
||||||
|
restricted[s.Name] = s.Restricted
|
||||||
|
}
|
||||||
for _, c := range consumers {
|
for _, c := range consumers {
|
||||||
if err := r.EnsureConsumer(c.Consumer); err != nil {
|
if err := broker.EnsureHeard(r, c.Consumer, restricted[c.Consumer.Stream]); err != nil {
|
||||||
failed = append(failed, fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err))
|
failed = append(failed, fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// **And a consumer no longer granted loses what it reads through** (novox/hq ADR 0363 §6): every consumer
|
||||||
|
// on a module's stream the composition no longer asks for is removed, on this very send.
|
||||||
|
if p, prunes := r.(broker.Pruner); prunes && len(modStreams) > 0 {
|
||||||
|
var wanted []broker.Consumer
|
||||||
|
for _, c := range consumers {
|
||||||
|
wanted = append(wanted, c.Consumer)
|
||||||
|
}
|
||||||
|
removed, err := broker.PruneHeard(p, modStreams, wanted)
|
||||||
|
if err != nil {
|
||||||
|
failed = append(failed, err)
|
||||||
|
}
|
||||||
|
for _, gone := range removed {
|
||||||
|
fmt.Printf(" removed the consumer %s: its module is no longer granted that stream (novox/hq ADR 0363)\n", gone)
|
||||||
|
}
|
||||||
|
}
|
||||||
if len(failed) > 0 {
|
if len(failed) > 0 {
|
||||||
return nil, errors.Join(failed...)
|
return nil, errors.Join(failed...)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,103 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A first-node gate waits for a busy container (novox/hq ADR 0335 §4).
|
||||||
|
//
|
||||||
|
// When a machine reports, on what the gate sent it, that a container of a judged module was left running
|
||||||
|
// because it said busy, the judging has not started: the build is not yet what runs there. The walk says it
|
||||||
|
// waits for a busy container on that machine, counts no pass and no bound, and starts the judging — its
|
||||||
|
// Since — at the report that no longer lists the container waiting. The wait is the node-engine's, bounded
|
||||||
|
// at 15 minutes; the controller's own bound (busyGateBound past the engine's) is for an engine that stopped
|
||||||
|
// reporting while it waited, so no walk waits for ever. An engine older than the field never lists one, and
|
||||||
|
// its gate starts as before.
|
||||||
|
|
||||||
|
// busyGateSlack is how long past the engine's bound the controller still waits for the report that says the
|
||||||
|
// container replaced: a pass of the engine (its asks every 20 seconds) and the report's way here.
|
||||||
|
var busyGateSlack = 5 * time.Minute
|
||||||
|
|
||||||
|
// busyEngineBound is the node-engine's bound on a wait (novox/hq ADR 0335 §3), counted again here on this
|
||||||
|
// controller's clock from when it first read the wait.
|
||||||
|
const busyEngineBound = 15 * time.Minute
|
||||||
|
|
||||||
|
// busyGateEnds is when the gate stops waiting: the engine's own word on its bound, or this controller's count of
|
||||||
|
// it from when it first read the wait, whichever is sooner, and the slack — so neither a clock ahead on the
|
||||||
|
// machine nor a wrong word from it holds a walk longer.
|
||||||
|
func busyGateEnds(b *inventory.GateBusy) time.Time {
|
||||||
|
until := b.Seen.Add(busyEngineBound)
|
||||||
|
if !b.Until.IsZero() && b.Until.Before(until) {
|
||||||
|
until = b.Until
|
||||||
|
}
|
||||||
|
return until.Add(busyGateSlack)
|
||||||
|
}
|
||||||
|
|
||||||
|
// busyWait decides, before a judging, whether the gate waits for a busy container on a machine it judges:
|
||||||
|
// true while it waits, the gate saying so. False when nothing waits, or no longer: a wait that ended moves the
|
||||||
|
// gate's Since to its end. Pure but for the gate it is handed.
|
||||||
|
func busyWait(g *inventory.PlanGate, pairs []judged, f gateFacts, now time.Time) bool {
|
||||||
|
if g.Busy != nil && g.Busy.Ended != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, j := range pairs {
|
||||||
|
b, waits := busyOn(j, f)
|
||||||
|
if !waits {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if g.Busy == nil {
|
||||||
|
g.Busy = &inventory.GateBusy{Machine: j.node, Module: j.module, Container: b.Container, Since: b.Since,
|
||||||
|
Seen: now}
|
||||||
|
}
|
||||||
|
if g.Busy.Seen.IsZero() {
|
||||||
|
g.Busy.Seen = now
|
||||||
|
}
|
||||||
|
g.Busy.Until, g.Busy.Said = b.Until, b.Said
|
||||||
|
if now.After(busyGateEnds(g.Busy)) {
|
||||||
|
// The engine said it would have replaced it by now: the judging starts, and finds the module not
|
||||||
|
// yet healthy while the machine still lists it waiting (busyStill), until the gate's own bound.
|
||||||
|
end := now
|
||||||
|
g.Busy.Ended, g.Busy.PastBound = &end, true
|
||||||
|
g.Since, g.Passes, g.LastPass, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, nil, nil, nil
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, g.Busy.Words(), nil
|
||||||
|
g.Read(now, false, g.Busy.Words())
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if g.Busy == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
// Replaced: the judging starts at the report that said so.
|
||||||
|
end := now
|
||||||
|
if r, said := f.reports[g.Busy.Machine]; said && r.At != nil && r.At.After(g.Busy.Since) && r.At.Before(now) {
|
||||||
|
end = r.At.UTC()
|
||||||
|
}
|
||||||
|
g.Busy.Ended = &end
|
||||||
|
g.Since, g.Passes, g.LastPass, g.Last, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, "", nil, nil, nil
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// busyOn is the container of a judged module its machine last reported, on what the gate sent it, as left
|
||||||
|
// running because it said busy.
|
||||||
|
func busyOn(j judged, f gateFacts) (inventory.BusyWait, bool) {
|
||||||
|
r, said := f.reports[j.node]
|
||||||
|
if !said || r.At == nil || !f.reportedOn(j.node, r) {
|
||||||
|
return inventory.BusyWait{}, false
|
||||||
|
}
|
||||||
|
return inventory.WaitingFor(r.Busy, j.module)
|
||||||
|
}
|
||||||
|
|
||||||
|
// busyStill is a judged module whose container its machine still lists waiting once the gate stopped waiting
|
||||||
|
// for it: not yet what was sent, so not healthy, whatever its health says of the container still running.
|
||||||
|
func busyStill(j judged, f gateFacts) (string, bool) {
|
||||||
|
b, waits := busyOn(j, f)
|
||||||
|
if !waits {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s still runs its busy container %s, past its node-engine's bound of %s",
|
||||||
|
j.node, b.Container, b.Until.UTC().Format(time.RFC3339)), true
|
||||||
|
}
|
||||||
@@ -0,0 +1,148 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A walk whose module's container waits as busy on its first machine says so, and starts that machine's
|
||||||
|
// first-machine gate only once the machine reports the container replaced (novox/hq ADR 0335 §4).
|
||||||
|
func TestAGateWaitsForABusyContainerAndStartsWhenItIsReplaced(t *testing.T) {
|
||||||
|
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||||
|
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent,
|
||||||
|
Sent: map[string]inventory.SentDeclaration{"novox": {Digest: "d1", Sequence: 7}}}
|
||||||
|
pairs := []judged{{module: "build-agent", node: "novox"}, {module: "mesh-controller", node: "novox"}}
|
||||||
|
reported := sent.Add(20 * time.Second)
|
||||||
|
waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||||
|
State: inventory.BusyWaiting, Since: reported, Until: reported.Add(15 * time.Minute),
|
||||||
|
Said: "busy: novox/mesh-controller (check of #280), in hand since 00:58:00Z"}}
|
||||||
|
facts := func(at time.Time, busy []inventory.BusyWait) gateFacts {
|
||||||
|
// As judgeMoves hands it: the report held against the gate's own send.
|
||||||
|
return gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"novox": {Node: "novox",
|
||||||
|
Outcome: inventory.OutcomeApplied, At: &at, Declared: "d1", ReportedSequence: 7, Busy: busy}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
now := reported.Add(time.Minute)
|
||||||
|
if !busyWait(g, pairs, facts(reported, waiting), now) {
|
||||||
|
t.Fatal("a gate whose module's container waits as busy judged it")
|
||||||
|
}
|
||||||
|
if !g.Since.Equal(sent) || g.Busy == nil || g.Busy.Machine != "novox" || g.Busy.Ended != nil {
|
||||||
|
t.Fatalf("the gate does not keep the wait: %+v %+v", g.Since, g.Busy)
|
||||||
|
}
|
||||||
|
for _, said := range []string{gateLine(g), g.Last} {
|
||||||
|
if !strings.Contains(said, "waits for a busy container on novox") {
|
||||||
|
t.Errorf("the walk does not say it waits for a busy container there: %s", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(g.Last, "check of #280") {
|
||||||
|
t.Errorf("the wait does not say what the container holds: %s", g.Last)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ten minutes on, still busy, within the engine's bound: still waiting, nothing judged, no bound run.
|
||||||
|
now = reported.Add(10 * time.Minute)
|
||||||
|
if !busyWait(g, pairs, facts(reported.Add(9*time.Minute), waiting), now) || g.Passes != 0 {
|
||||||
|
t.Fatalf("the gate stopped waiting within the bound: %+v", g)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The machine reports it replaced: the judging starts at that report.
|
||||||
|
replacedAt := reported.Add(12 * time.Minute)
|
||||||
|
now = replacedAt.Add(5 * time.Second)
|
||||||
|
replaced := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||||
|
State: "replaced-when-idle", Since: reported, Until: reported.Add(15 * time.Minute)}}
|
||||||
|
if busyWait(g, pairs, facts(replacedAt, replaced), now) {
|
||||||
|
t.Fatal("the gate still waits after the container was replaced")
|
||||||
|
}
|
||||||
|
if g.Since == nil || !g.Since.Equal(replacedAt) || g.Busy.Ended == nil || !g.Busy.Ended.Equal(replacedAt) {
|
||||||
|
t.Fatalf("the gate does not start at the report that said it replaced: since %v, %+v", g.Since, g.Busy)
|
||||||
|
}
|
||||||
|
if strings.Contains(gateLine(g), "waits for a busy container") {
|
||||||
|
t.Errorf("a gate judging still says it waits: %s", gateLine(g))
|
||||||
|
}
|
||||||
|
// And a later report listing it waiting again does not hold a gate that started.
|
||||||
|
if busyWait(g, pairs, facts(now, waiting), now.Add(time.Minute)) {
|
||||||
|
t.Error("a gate that started waits again")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An engine that stops reporting while it waits holds no walk for ever: past the engine's bound and the slack,
|
||||||
|
// the judging starts, and finds the module not yet healthy while its container is still listed waiting.
|
||||||
|
func TestAGateWaitsNoLongerThanTheEnginesBound(t *testing.T) {
|
||||||
|
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||||
|
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
|
||||||
|
pairs := []judged{{module: "build-agent", node: "novox"}}
|
||||||
|
waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||||
|
State: inventory.BusyWaiting, Since: sent, Until: sent.Add(15 * time.Minute)}}
|
||||||
|
f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
|
||||||
|
At: &sent, Current: true, Busy: waiting}}}
|
||||||
|
now := sent.Add(15*time.Minute + busyGateSlack + time.Second)
|
||||||
|
if busyWait(g, pairs, f, now) {
|
||||||
|
t.Fatal("a gate waited past the engine's bound and the slack")
|
||||||
|
}
|
||||||
|
if !g.Busy.PastBound || !g.Since.Equal(now) {
|
||||||
|
t.Fatalf("the gate does not start, said past the bound: %+v %v", g.Busy, g.Since)
|
||||||
|
}
|
||||||
|
if why, still := busyStill(pairs[0], f); !still || !strings.Contains(why, "past its node-engine's bound") {
|
||||||
|
t.Fatalf("a module still running its busy container reads as what was sent: %q", why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A report from an engine older than the field lists nothing: the gate starts at the send, as before; and a
|
||||||
|
// report not about the gate's send says nothing of it.
|
||||||
|
func TestAGateOfAnEngineThatNeverWaitsStartsAsBefore(t *testing.T) {
|
||||||
|
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||||
|
g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent,
|
||||||
|
Sent: map[string]inventory.SentDeclaration{"ace": {Digest: "d2", Sequence: 9}}}
|
||||||
|
pairs := []judged{{module: "build-agent", node: "ace"}}
|
||||||
|
at := sent.Add(time.Minute)
|
||||||
|
older := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
|
||||||
|
At: &at, Declared: "d2", ReportedSequence: 9}}}
|
||||||
|
if busyWait(g, pairs, older, at) || g.Busy != nil || !g.Since.Equal(sent) {
|
||||||
|
t.Fatalf("a gate on an engine that never waits did not start at the send: %+v %v", g.Busy, g.Since)
|
||||||
|
}
|
||||||
|
elsewhere := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
|
||||||
|
At: &at, Declared: "d1", ReportedSequence: 8, Busy: []inventory.BusyWait{{ID: "build-agent.server",
|
||||||
|
State: inventory.BusyWaiting, Until: at.Add(time.Hour)}}}}}
|
||||||
|
if busyWait(g, pairs, elsewhere, at) {
|
||||||
|
t.Fatal("a wait reported about an earlier declaration held the gate")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The wait is bounded on this controller's clock: an engine whose clock runs an hour ahead, saying it waits until
|
||||||
|
// then, holds the gate no longer than 15 minutes and the slack from when the controller first read the wait.
|
||||||
|
func TestAGateWaitIsBoundedOnTheControllersClock(t *testing.T) {
|
||||||
|
seen := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||||
|
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &seen}
|
||||||
|
pairs := []judged{{module: "build-agent", node: "novox"}}
|
||||||
|
ahead := seen.Add(time.Hour)
|
||||||
|
f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
|
||||||
|
At: &seen, Current: true, Busy: []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||||
|
State: inventory.BusyWaiting, Since: ahead, Until: ahead.Add(15 * time.Minute)}}}}}
|
||||||
|
if !busyWait(g, pairs, f, seen) || !g.Busy.Seen.Equal(seen) {
|
||||||
|
t.Fatalf("the gate did not start waiting, seen now: %+v", g.Busy)
|
||||||
|
}
|
||||||
|
if !busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack-time.Second)) {
|
||||||
|
t.Fatal("the gate stopped waiting within the bound on its own clock")
|
||||||
|
}
|
||||||
|
if busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack+time.Second)) || !g.Busy.PastBound {
|
||||||
|
t.Fatalf("an engine's later word held the gate past the controller's bound: %+v", g.Busy)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// `plan` says, for a machine whose node-engine is older than the field, each container whose busy is not sent.
|
||||||
|
func TestAPlanSaysABusyAnOlderEngineIsNotSent(t *testing.T) {
|
||||||
|
modules := []catalogue.Manifest{{Module: "build-agent", Resources: []map[string]any{
|
||||||
|
{"id": "server", "type": "container", "busy": map[string]any{"command": "mesh-builder busy"}},
|
||||||
|
{"id": "agent-env", "type": "file"}}}}
|
||||||
|
lines := busyNotSent(modules, false)
|
||||||
|
if len(lines) != 1 || !strings.Contains(lines[0], "build-agent.server declares busy") ||
|
||||||
|
!strings.Contains(lines[0], "replaced at once") {
|
||||||
|
t.Fatalf("the plan does not say the busy not sent: %v", lines)
|
||||||
|
}
|
||||||
|
if lines := busyNotSent(modules, true); len(lines) != 0 {
|
||||||
|
t.Fatalf("a busy sent is said as not sent: %v", lines)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/builder"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A pull request's check builds go when it closes (novox/hq ADR 0350 §5): the build seat keeps them in the artifact
|
||||||
|
// store's scratch namespace for the walk of its merge to take. **Let go of where the moment is heard**: the
|
||||||
|
// controller ends the walk answering a merge — after that walk took what it reuses — and the delivery's owner, which
|
||||||
|
// hears the forge say a pull request closed unmerged, asks the `forget-checks` verb; both let go through the store,
|
||||||
|
// as the controller's own collection does. The build seat's holders let go of anything a week old on their own,
|
||||||
|
// so a word lost here leaves a week's builds, never more.
|
||||||
|
|
||||||
|
// checkBuildsIn is the scratch namespace as the controller reaches the artifact store.
|
||||||
|
func checkBuildsIn(ctx context.Context, inv *inventory.Inventory) (builder.Scratch, error) {
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
address, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if address == "" {
|
||||||
|
return nil, errors.New("the artifact store is not on the mesh's network")
|
||||||
|
}
|
||||||
|
return builder.Registry{Address: address}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// forgetChecks lets go of a pull request's check builds and answers what it did.
|
||||||
|
var forgetChecks = func(ctx context.Context, inv *inventory.Inventory, pull builder.PullRef) (string, error) {
|
||||||
|
scratch, err := checkBuildsIn(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
n, err := scratch.Forget(ctx, pull)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("%s: %d check(s)' builds let go of, and not all: %w", pull, n, err)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s: %d check(s)' builds let go of", pull, n), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// forgetChecksOfWalk lets go, on the save that ends a walk, of the check builds of every pull request whose merge
|
||||||
|
// it answers: the walk took what it reuses. Said, never fatal.
|
||||||
|
func forgetChecksOfWalk(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) {
|
||||||
|
if p.WalkSaid != inventory.WalkEnded || p.Delivery == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, pull := range reuseAskOf(&p).Pulls {
|
||||||
|
said, err := forgetChecks(ctx, inv, builder.PullRef{Owner: pull.Owner, Repo: pull.Repo, Number: pull.Number})
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "%s: its check builds stay until the week's sweep: %v\n", p.ID, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fmt.Printf("%s: %s\n", p.ID, said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// forgetChecksCommand is `forget-checks <owner> <repo> <number>`: the verb the delivery's owner asks when the forge
|
||||||
|
// says a pull request closed unmerged (novox/hq ADR 0350 §5).
|
||||||
|
func forgetChecksCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) != 3 {
|
||||||
|
return errors.New("forget-checks <owner> <repo> <number>")
|
||||||
|
}
|
||||||
|
number, err := strconv.Atoi(args[2])
|
||||||
|
if err != nil || number <= 0 {
|
||||||
|
return fmt.Errorf("%q is not a pull request's number", args[2])
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
said, err := forgetChecks(ctx, open.inventory, builder.PullRef{Owner: args[0], Repo: args[1], Number: number})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(said)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,280 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/builder"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// One build per change (novox/hq ADR 0350), as the controller asks it and takes it in.
|
||||||
|
|
||||||
|
// The check of a change is asked to build what a merge of it would move, in the walk's order: a module of the
|
||||||
|
// repository checked from the merge, one standing on it from its own repository as the walk builds it, and one
|
||||||
|
// that reads the repository through a context not at all. A change that moves no module builds nothing.
|
||||||
|
func TestACheckIsAskedToBuildWhatTheMergeWouldMove(t *testing.T) {
|
||||||
|
entries := []inventory.Entry{
|
||||||
|
{Manifest: catalogue.Manifest{Module: "app"}, Source: inventory.Source{Repository: "novox/shop", Seat: "git", Path: "modules/app"}},
|
||||||
|
{Manifest: catalogue.Manifest{Module: "web"}, Source: inventory.Source{Repository: "novox/web", Seat: "git", Ref: "main"}},
|
||||||
|
{Manifest: catalogue.Manifest{Module: "docs"}, Source: inventory.Source{Repository: "novox/docs", Seat: "git"}},
|
||||||
|
}
|
||||||
|
p := link.PullUpdated{Owner: "novox", Repo: "shop", Number: 7, Base: "main", Commit: "abc"}
|
||||||
|
scope := checkScope{Reach: mergeReach{Packaging: entries[2:], Plan: inventory.Plan{Tiers: [][]string{{"app", "docs"}, {"web"}}}}}
|
||||||
|
clone := func(s inventory.Source) (string, error) { return "http://forge.invalid/" + s.Repository + ".git", nil }
|
||||||
|
|
||||||
|
got := checkBuildsOf(p, scope, entries, clone)
|
||||||
|
want := []link.CheckBuild{
|
||||||
|
{Module: "app", Path: "modules/app", Merged: true, Tier: 0},
|
||||||
|
{Module: "docs", Tier: 0, Skip: "its build reads novox/shop through a context, which a check cannot hand the merge"},
|
||||||
|
{Module: "web", Repository: "http://forge.invalid/novox/web.git", Ref: "main", Tier: 1},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("the check is asked to build\n%+v\nwant\n%+v", got, want)
|
||||||
|
}
|
||||||
|
if got := checkBuildsOf(p, checkScope{}, entries, clone); got != nil {
|
||||||
|
t.Fatalf("a change that moves no module asks for builds: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A walk's ask carries its leave to take a check's build, naming the pull requests of the merges it answers; a dry
|
||||||
|
// run never does, and neither does an ask that is not a walk's.
|
||||||
|
func TestOnlyAWalksAskTakesAChecksBuildAndADryRunNever(t *testing.T) {
|
||||||
|
p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
|
||||||
|
{Repository: "novox/shop", Number: 7}, {Repository: "http://forge.invalid/novox/web.git", Number: 3},
|
||||||
|
{Repository: "novox/shop", Number: 7}, {Repository: "novox/shop"}}}}
|
||||||
|
asked := reuseAskOf(p)
|
||||||
|
if want := []link.PullRef{{Owner: "novox", Repo: "shop", Number: 7}, {Owner: "novox", Repo: "web", Number: 3}}; !reflect.DeepEqual(asked.Pulls, want) {
|
||||||
|
t.Fatalf("the walk names %+v", asked.Pulls)
|
||||||
|
}
|
||||||
|
walk := withReuse(t.Context(), asked)
|
||||||
|
if reuseAsked(walk, false) != asked {
|
||||||
|
t.Fatal("a walk's ask does not carry its leave")
|
||||||
|
}
|
||||||
|
if reuseAsked(walk, true) != nil {
|
||||||
|
t.Fatal("a dry run carries a leave to take a check's build")
|
||||||
|
}
|
||||||
|
if reuseAsked(t.Context(), false) != nil {
|
||||||
|
t.Fatal("an ask that is not a walk's carries a leave")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A walk ended lets go of its merges' check builds — only once it ended, after it took what it reuses.
|
||||||
|
func TestAWalkEndedLetsGoOfItsMergesCheckBuilds(t *testing.T) {
|
||||||
|
var forgot []builder.PullRef
|
||||||
|
was := forgetChecks
|
||||||
|
forgetChecks = func(_ context.Context, _ *inventory.Inventory, pull builder.PullRef) (string, error) {
|
||||||
|
forgot = append(forgot, pull)
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { forgetChecks = was })
|
||||||
|
p := inventory.Plan{ID: "plan-1", Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
|
||||||
|
{Repository: "novox/shop", Number: 7}}}}
|
||||||
|
|
||||||
|
p.WalkSaid = inventory.WalkStarted
|
||||||
|
forgetChecksOfWalk(t.Context(), nil, p)
|
||||||
|
p.WalkSaid = ""
|
||||||
|
forgetChecksOfWalk(t.Context(), nil, p)
|
||||||
|
if len(forgot) != 0 {
|
||||||
|
t.Fatalf("a walk not ended let go of %v, which it may still take", forgot)
|
||||||
|
}
|
||||||
|
p.WalkSaid = inventory.WalkEnded
|
||||||
|
forgetChecksOfWalk(t.Context(), nil, p)
|
||||||
|
if want := []builder.PullRef{{Owner: "novox", Repo: "shop", Number: 7}}; !reflect.DeepEqual(forgot, want) {
|
||||||
|
t.Fatalf("the walk ended let go of %v", forgot)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A build in the scratch namespace is never registered: only the walk's copy in the module's own namespace is.
|
||||||
|
func TestABuildInTheScratchNamespaceIsNeverRegistered(t *testing.T) {
|
||||||
|
if got := inScratch([]link.MadeArtifact{{Name: "server", Reference: "store.internal:5000/app/server@sha256:aa"}}); got != "" {
|
||||||
|
t.Fatalf("a module's own build reads as a scratch build: %s", got)
|
||||||
|
}
|
||||||
|
scratch := "store.internal:5000/scratch/app/server@sha256:aa"
|
||||||
|
if got := inScratch([]link.MadeArtifact{{Name: "server", Reference: scratch}}); got != scratch {
|
||||||
|
t.Fatalf("a scratch build was not found: %q", got)
|
||||||
|
}
|
||||||
|
if got := inScratch([]link.MadeArtifact{{Name: "look", Reference: "http://store.internal:5000/v2/scratch/app/look/blobs/sha256:bb"}}); got == "" {
|
||||||
|
t.Fatal("a scratch archive was not found")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The walk says per module what it did: "reused from …" or "built: <why>" — and a build seat from before says
|
||||||
|
// neither, which is said.
|
||||||
|
func TestAWalksBuildSaysWhetherItReusedOrBuilt(t *testing.T) {
|
||||||
|
if got := (link.BuildResult{Reused: "reused from the check of novox/shop#7"}).MadeWords(); got != "reused from the check of novox/shop#7" {
|
||||||
|
t.Fatalf("said %q", got)
|
||||||
|
}
|
||||||
|
if got := (link.BuildResult{BuiltWhy: "no check built it"}).MadeWords(); got != "built: no check built it" {
|
||||||
|
t.Fatalf("said %q", got)
|
||||||
|
}
|
||||||
|
if got := (link.BuildResult{}).MadeWords(); !strings.Contains(got, "predates ADR 0350") {
|
||||||
|
t.Fatalf("said %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A walk's module the check's build stood in for is registered by the digest the check built — the copy in the
|
||||||
|
// module's namespace — and the walk keeps "reused from …" for it; its first-node gate judges that registered
|
||||||
|
// build as it judges one the walk built. A scratch reference is refused. Needs the store (MESH_TEST_POSTGRES), and
|
||||||
|
// is named in testdata/store-tests.
|
||||||
|
func TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
|
g := aGateMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := g.open.inventory
|
||||||
|
gateEvery = 0
|
||||||
|
|
||||||
|
digest := "sha256:" + strings.Repeat("c", 64)
|
||||||
|
reused := func(id, reference string) link.BuildResult {
|
||||||
|
manifest := mustJSON(t, catalogue.Manifest{Module: "app", Version: "c3", Resources: []map[string]any{
|
||||||
|
{"id": "svc", "type": "container", "name": "app", "image": reference}}})
|
||||||
|
return link.BuildResult{ID: id, Repository: "novox/mesh-catalog", Path: "modules/app", Ref: "main", On: "anchor",
|
||||||
|
Commit: "c3", Manifest: manifest, Trunk: "main", OnTrunk: true, Branches: []string{"main"},
|
||||||
|
Made: []link.MadeArtifact{{Name: "server", Kind: "image", Reference: reference}},
|
||||||
|
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"},
|
||||||
|
Reused: "reused from the check of novox/mesh-catalog#9 at head abcdefabcdef (merged into trunk 123456123456): server " + digest,
|
||||||
|
ReusedFrom: &link.ReusedFrom{Pull: link.PullRef{Owner: "novox", Repo: "mesh-catalog", Number: 9},
|
||||||
|
Head: "abcdefabcdef", Trunk: "123456123456", Key: "r1:" + strings.Repeat("1", 64)}}
|
||||||
|
}
|
||||||
|
if _, _, err := takeIn(ctx, inv, reused(link.NewBuildID(time.Now()), "store.internal:5000/scratch/app/server@"+digest)); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "scratch") {
|
||||||
|
t.Fatalf("a build in the scratch namespace was registered: %v", err)
|
||||||
|
}
|
||||||
|
id := link.NewBuildID(time.Now().Add(time.Second))
|
||||||
|
result := reused(id, "store.internal:5000/app/server@"+digest)
|
||||||
|
if _, _, err := takeIn(ctx, inv, result); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
current, err := inv.CurrentBuilds(ctx)
|
||||||
|
if err != nil || current["app"].Commit != "c3" {
|
||||||
|
t.Fatalf("the reused build is not what app is registered at: %+v %v", current["app"], err)
|
||||||
|
}
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(shelf["app"].Resources)
|
||||||
|
if !strings.Contains(string(raw), "app/server@"+digest) {
|
||||||
|
t.Fatalf("app is registered as %s, not by the check's digest in its own namespace", raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The walk asked it: the walk keeps what its build did, and its gate judges the registered build.
|
||||||
|
asked := time.Now().UTC()
|
||||||
|
p := g.plan(t)
|
||||||
|
p.Commit = "c3"
|
||||||
|
p.Modules["app"] = &inventory.PlanModule{State: "asked", AskedAt: &asked, Build: id}
|
||||||
|
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
planBuiltSaying(ctx, g.open, "app", "c3", "", asked, id, result.MadeWords())
|
||||||
|
p = g.plan(t)
|
||||||
|
if m := p.Modules["app"]; m.State != "built" || !strings.HasPrefix(m.Made, "reused from the check of novox/mesh-catalog#9") {
|
||||||
|
t.Fatalf("the walk keeps %+v", m)
|
||||||
|
}
|
||||||
|
for i := 0; i < 6; i++ {
|
||||||
|
advancePlans(ctx, g.open)
|
||||||
|
}
|
||||||
|
p = g.plan(t)
|
||||||
|
gate := p.Modules["app"].Gate
|
||||||
|
if gate == nil || gate.To != "c3" || gate.Verdict != inventory.GatePassed {
|
||||||
|
t.Fatalf("the gate judged %+v, not the reused build", gate)
|
||||||
|
}
|
||||||
|
if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c3" {
|
||||||
|
t.Fatalf("the first machine was sent %v, not the reused build", sent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A walk's leave names the builds its own pull requests' verdicts made at the head each merge merged — never an
|
||||||
|
// earlier head's — read once per walk, and none when they cannot be read: the builder takes nothing a verdict does
|
||||||
|
// not name (novox/hq ADR 0350 §2, issue 703).
|
||||||
|
func TestAWalksLeaveNamesTheBuildsItsPullRequestsVerdictsMade(t *testing.T) {
|
||||||
|
was := checkedBuildsOf
|
||||||
|
forgetWalkVerdicts()
|
||||||
|
t.Cleanup(func() { checkedBuildsOf = was; forgetWalkVerdicts() })
|
||||||
|
var asked [][]link.PullRef
|
||||||
|
pull := link.PullRef{Owner: "novox", Repo: "shop", Number: 7}
|
||||||
|
atHead := link.CheckedBuild{Pull: pull, Check: "build-2", Head: "bbbbbbbbbbbb", Built: link.CheckBuilt{Module: "app", Key: "r1:2"}}
|
||||||
|
earlier := link.CheckedBuild{Pull: pull, Check: "build-1", Head: "aaaaaaaaaaaa", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
||||||
|
checkedBuildsOf = func(_ context.Context, pulls []link.PullRef) ([]link.CheckedBuild, error) {
|
||||||
|
asked = append(asked, pulls)
|
||||||
|
return []link.CheckedBuild{atHead, earlier}, nil
|
||||||
|
}
|
||||||
|
pulls := []link.PullRef{pull}
|
||||||
|
heads := map[link.PullRef]string{pull: "bbbbbbbbbbbb0123456789"}
|
||||||
|
for range 3 {
|
||||||
|
if got := reuseWithVerdicts(t.Context(), "plan-1", &link.ReuseAsk{Pulls: pulls}, heads); !reflect.DeepEqual(got.Builds,
|
||||||
|
[]link.CheckedBuild{atHead}) {
|
||||||
|
t.Fatalf("the leave names %+v, not only the merged head's build", got.Builds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(asked, [][]link.PullRef{pulls}) {
|
||||||
|
t.Fatalf("the verdicts were read %d times for one walk: %v", len(asked), asked)
|
||||||
|
}
|
||||||
|
// A merge whose head was not said takes nothing.
|
||||||
|
if got := reuseWithVerdicts(t.Context(), "plan-1", &link.ReuseAsk{Pulls: pulls}, nil); got.Builds != nil {
|
||||||
|
t.Fatalf("a merge of no said head names %+v", got.Builds)
|
||||||
|
}
|
||||||
|
checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) {
|
||||||
|
return nil, errors.New("no bus")
|
||||||
|
}
|
||||||
|
if got := reuseWithVerdicts(t.Context(), "plan-2", &link.ReuseAsk{Pulls: pulls}, heads); got.Builds != nil {
|
||||||
|
t.Fatalf("verdicts that could not be read named %+v", got.Builds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A check is handed only the builds earlier checks of its own pull request at its own head made: another pull
|
||||||
|
// request's build, or an earlier head's, is never kept (novox/hq issue 703). None when they cannot be read.
|
||||||
|
func TestACheckIsHandedOnlyItsOwnHeadsBuilds(t *testing.T) {
|
||||||
|
was := checkedBuildsOf
|
||||||
|
t.Cleanup(func() { checkedBuildsOf = was })
|
||||||
|
pull := link.PullRef{Owner: "novox", Repo: "shop", Number: 7}
|
||||||
|
mine := link.CheckedBuild{Pull: pull, Check: "build-1", Head: "bbbbbbbbbbbb", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
||||||
|
earlier := link.CheckedBuild{Pull: pull, Check: "build-0", Head: "aaaaaaaaaaaa", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
||||||
|
another := link.CheckedBuild{Pull: link.PullRef{Owner: "novox", Repo: "shop", Number: 8}, Check: "build-9",
|
||||||
|
Head: "bbbbbbbbbbbb", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
||||||
|
var asked [][]link.PullRef
|
||||||
|
checkedBuildsOf = func(_ context.Context, pulls []link.PullRef) ([]link.CheckedBuild, error) {
|
||||||
|
asked = append(asked, pulls)
|
||||||
|
return []link.CheckedBuild{mine, earlier, another}, nil
|
||||||
|
}
|
||||||
|
p := link.PullUpdated{Owner: "novox", Repo: "shop", Number: 7, Commit: "bbbbbbbbbbbb0123456789"}
|
||||||
|
builds := []link.CheckBuild{{Module: "app"}}
|
||||||
|
if got := namedForTheCheck(t.Context(), p, builds); !reflect.DeepEqual(got, []link.CheckedBuild{mine}) ||
|
||||||
|
!reflect.DeepEqual(asked, [][]link.PullRef{{pull}}) {
|
||||||
|
t.Fatalf("the check is handed %+v, read for %v", got, asked)
|
||||||
|
}
|
||||||
|
checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) {
|
||||||
|
return nil, errors.New("no bus")
|
||||||
|
}
|
||||||
|
if got := namedForTheCheck(t.Context(), p, builds); got != nil {
|
||||||
|
t.Fatalf("verdicts that could not be read named %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func forgetWalkVerdicts() {
|
||||||
|
walkVerdicts.Lock()
|
||||||
|
defer walkVerdicts.Unlock()
|
||||||
|
walkVerdicts.read = map[string]verdictsRead{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A walk's merged heads are the forge's word for each merge; a merge announced without its head, or a pull request
|
||||||
|
// merged at two heads in one walk, has none, and its walk takes no check's build (novox/hq issue 703).
|
||||||
|
func TestAWalksMergedHeadsAreTheForgesWord(t *testing.T) {
|
||||||
|
merge := func(number int, head string) inventory.PlanMerge {
|
||||||
|
return inventory.PlanMerge{Repository: "novox/shop", Number: number, Head: head}
|
||||||
|
}
|
||||||
|
p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
|
||||||
|
merge(7, "aaaaaaaaaaaa1"), merge(8, ""), merge(9, "bbbbbbbbbbbb"), merge(9, "cccccccccccc")}}}
|
||||||
|
heads := mergedHeadsOf(p)
|
||||||
|
shop := func(n int) link.PullRef { return link.PullRef{Owner: "novox", Repo: "shop", Number: n} }
|
||||||
|
if heads[shop(7)] != "aaaaaaaaaaaa1" || heads[shop(8)] != "" || heads[shop(9)] != "" {
|
||||||
|
t.Fatalf("merged heads %v", heads)
|
||||||
|
}
|
||||||
|
}
|
||||||
+243
-25
@@ -3,6 +3,7 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"path"
|
"path"
|
||||||
"regexp"
|
"regexp"
|
||||||
@@ -188,6 +189,8 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
|||||||
if err := ask.Ask(ctx, request); err != nil {
|
if err := ask.Ask(ctx, request); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// A new head ends the check of the old one (novox/hq issue 726).
|
||||||
|
endingSupersededChecks(ctx, seat, p, request.ID, f.oneShot)
|
||||||
what := "its own merge-check.sh alone: " + noModuleTouched
|
what := "its own merge-check.sh alone: " + noModuleTouched
|
||||||
if scope.gated() {
|
if scope.gated() {
|
||||||
what = fmt.Sprintf("the gate over %s (a merge would build %d module(s) in %d tier(s))",
|
what = fmt.Sprintf("the gate over %s (a merge would build %d module(s) in %d tier(s))",
|
||||||
@@ -248,31 +251,16 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return link.BuildRequest{}, err
|
return link.BuildRequest{}, err
|
||||||
}
|
}
|
||||||
// Beside it, at what the mesh runs: each core repository by the module the mesh builds from it.
|
// **A pull request into a release branch is judged by that branch** (novox/hq ADR 0358): asked of the forge once.
|
||||||
beside := map[string]link.CheckedOut{}
|
release, err := releaseBranchOf(ctx, p, scope, entries)
|
||||||
for _, e := range entries {
|
if err != nil {
|
||||||
dir, core := coreModules[e.Manifest.Module]
|
return link.BuildRequest{}, err
|
||||||
if !core || e.Provided || e.Source.Repository == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
url, err := clone(e.Source)
|
|
||||||
if err != nil {
|
|
||||||
return link.BuildRequest{}, err
|
|
||||||
}
|
|
||||||
refs := besideRefs(dir, current[e.Manifest.Module].Commit)
|
|
||||||
beside[dir] = link.CheckedOut{Repository: url, Ref: refs[dir]}
|
|
||||||
if dir == "mesh-controller" {
|
|
||||||
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
|
|
||||||
if e.Source.Seat != "" {
|
|
||||||
for _, sibling := range []string{"mesh-lab", "mesh-sdk"} {
|
|
||||||
if url, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
|
||||||
sibling)}); err == nil {
|
|
||||||
beside[sibling] = link.CheckedOut{Repository: url, Ref: refs[sibling]}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
beside, err := besideOf(entries, func(module string) string { return current[module].Commit }, clone, release)
|
||||||
|
if err != nil {
|
||||||
|
return link.BuildRequest{}, err
|
||||||
|
}
|
||||||
|
builds := checkBuildsOf(p, scope, entries, clone)
|
||||||
return link.BuildRequest{
|
return link.BuildRequest{
|
||||||
ID: link.NewBuildID(time.Now()),
|
ID: link.NewBuildID(time.Now()),
|
||||||
Repository: repository,
|
Repository: repository,
|
||||||
@@ -282,10 +270,237 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
|||||||
Source: sourceOnSeat(source),
|
Source: sourceOnSeat(source),
|
||||||
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
|
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
|
||||||
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
|
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
|
||||||
Manifests: scope.Manifests, Judge: scope.Judge},
|
Manifests: scope.Manifests, Judge: scope.Judge, Builds: builds, Named: namedForTheCheck(ctx, p, builds),
|
||||||
|
Release: release, Running: runningBy(ctx)},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// besideOf is what a check reads beside its own repository, by the directory it is found under: each core repository
|
||||||
|
// by the module the mesh builds from it, at besideRefs — or, for a pull request into a release branch (novox/hq ADR
|
||||||
|
// 0358 §3), at its branch of that name where it has one and at its trunk otherwise, the controller's main kept for
|
||||||
|
// its judge, and mesh-lab at its trunk whatever the base: its replays run with the container runtime's socket. running
|
||||||
|
// is the commit the mesh runs of a module.
|
||||||
|
func besideOf(entries []inventory.Entry, running func(module string) string,
|
||||||
|
clone func(inventory.Source) (string, error), release string) (map[string]link.CheckedOut, error) {
|
||||||
|
beside := map[string]link.CheckedOut{}
|
||||||
|
for _, e := range entries {
|
||||||
|
dir, core := coreModules[e.Manifest.Module]
|
||||||
|
if !core || e.Provided || e.Source.Repository == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
url, err := clone(e.Source)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
refs := besideRefs(dir, running(e.Manifest.Module))
|
||||||
|
at := func(dir, trunk string) link.CheckedOut {
|
||||||
|
if release == "" {
|
||||||
|
return link.CheckedOut{Repository: url, Ref: refs[dir]}
|
||||||
|
}
|
||||||
|
if trunk == "" {
|
||||||
|
trunk = "main"
|
||||||
|
}
|
||||||
|
return link.CheckedOut{Repository: url, Ref: release, Else: trunk}
|
||||||
|
}
|
||||||
|
beside[dir] = at(dir, followedBranch(e.Source.Ref))
|
||||||
|
if dir == "mesh-controller" {
|
||||||
|
// The controller's main stays its main: it judges only a check the running controller predates.
|
||||||
|
beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]}
|
||||||
|
if release != "" {
|
||||||
|
// mesh-controller has the release branch: that is what makes it one (ADR 0358 §1).
|
||||||
|
beside[dir] = link.CheckedOut{Repository: url, Ref: release}
|
||||||
|
}
|
||||||
|
if e.Source.Seat != "" {
|
||||||
|
for _, sibling := range []string{"mesh-lab", "mesh-sdk"} {
|
||||||
|
if url, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository,
|
||||||
|
sibling)}); err == nil {
|
||||||
|
b := link.CheckedOut{Repository: url, Ref: refs[sibling]}
|
||||||
|
// **mesh-lab is read at its trunk, always** (ADR 0358): its replays run with the container
|
||||||
|
// runtime's socket, so they are only ever reviewed code — never a branch anyone may create.
|
||||||
|
if release != "" && sibling != "mesh-lab" {
|
||||||
|
b = link.CheckedOut{Repository: url, Ref: release, Else: "main"}
|
||||||
|
}
|
||||||
|
beside[sibling] = b
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return beside, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// releaseBranchOf is the release branch a pull request merges into (novox/hq ADR 0358 §1): its base, when that is not
|
||||||
|
// its repository's trunk and mesh-controller has a branch of that name; empty otherwise, and the pull request is
|
||||||
|
// checked as before it. **A base that is a trunk asks nothing**: the branch a module built from the repository follows
|
||||||
|
// (main when it records none), or mesh-controller's own trunk. Any other base is asked of the forge once, and an
|
||||||
|
// answer it cannot give is an error, never read as "no release branch".
|
||||||
|
func releaseBranchOf(ctx context.Context, p link.PullUpdated, scope checkScope, entries []inventory.Entry) (string, error) {
|
||||||
|
base := p.Base
|
||||||
|
if base == "" {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
if scope.From != nil && base == trunkOf(scope.From.Source) {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
var controller *inventory.Entry
|
||||||
|
for i, e := range entries {
|
||||||
|
if e.Manifest.Module == "mesh-controller" && !e.Provided && e.Source.Repository != "" {
|
||||||
|
controller = &entries[i]
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if controller == nil || base == trunkOf(controller.Source) {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
if !tryRef.MatchString(base) || strings.Contains(base, "..") {
|
||||||
|
return "", nil // no branch the forge could hold: checked as before
|
||||||
|
}
|
||||||
|
owner, repo := sourceOwner(controller.Source.Repository), sourceName(controller.Source.Repository)
|
||||||
|
if owner == "" || repo == "" {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
has, err := forgeHasBranch(ctx, owner, repo, base)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("whether %s/%s#%d merges into a release branch cannot be told — the forge did not say "+
|
||||||
|
"whether %s/%s has a branch %s (novox/hq ADR 0358): %w", p.Owner, p.Repo, p.Number, owner, repo, base, err)
|
||||||
|
}
|
||||||
|
if !has {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
return base, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// trunkOf is the branch a module's source follows: main when it records none, or records a commit.
|
||||||
|
func trunkOf(s inventory.Source) string {
|
||||||
|
if t := strings.TrimSpace(followedBranch(s.Ref)); t != "" {
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
return "main"
|
||||||
|
}
|
||||||
|
|
||||||
|
// sourceName is the name of a recorded repository, a path on the git seat or a URL: novox/mesh-host → mesh-host.
|
||||||
|
func sourceName(repository string) string {
|
||||||
|
parts := strings.Split(strings.Trim(strings.TrimSuffix(repository, ".git"), "/"), "/")
|
||||||
|
if len(parts) < 2 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return parts[len(parts)-1]
|
||||||
|
}
|
||||||
|
|
||||||
|
// forgeHasBranch asks the forge, through its module's tools on the bus, whether a repository has a branch. A
|
||||||
|
// variable so a test needs no forge.
|
||||||
|
var forgeHasBranch = func(ctx context.Context, owner, repo, branch string) (bool, error) {
|
||||||
|
js, err := aBus()
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
raw, _ := json.Marshal(map[string]any{"path": "/repos/" + owner + "/" + repo + "/branches/" + branch})
|
||||||
|
answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", "gitea_api", raw, 30*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return branchAnswered(answer, branch)
|
||||||
|
}
|
||||||
|
|
||||||
|
// branchAnswered reads the forge's answer about a branch: a 404 is "no such branch"; an answer naming the branch is
|
||||||
|
// "it has it"; anything else — another error, an unreadable answer, one naming no branch or another — is an error,
|
||||||
|
// never read as "no release branch".
|
||||||
|
func branchAnswered(answer link.Answer, branch string) (bool, error) {
|
||||||
|
if answer.Error != "" {
|
||||||
|
if strings.Contains(answer.Error, " 404 ") {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return false, errors.New(answer.Error)
|
||||||
|
}
|
||||||
|
var found struct {
|
||||||
|
Result struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
} `json:"result"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(answer.Result, &found); err != nil {
|
||||||
|
return false, fmt.Errorf("the forge's answer about branch %s cannot be read: %w", branch, err)
|
||||||
|
}
|
||||||
|
if found.Result.Name != branch {
|
||||||
|
return false, fmt.Errorf("the forge answered about branch %s naming %q", branch, found.Result.Name)
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// namedForTheCheck are the builds earlier checks of the same pull request at the same head named, read once for the
|
||||||
|
// check (novox/hq issue 703): asked again because only the trunk moved, the check keeps what they built of a key
|
||||||
|
// still the same. Another pull request's build, or one of an earlier head, is never kept. None when they cannot be
|
||||||
|
// read, and the check builds, as before.
|
||||||
|
func namedForTheCheck(ctx context.Context, p link.PullUpdated, builds []link.CheckBuild) []link.CheckedBuild {
|
||||||
|
if len(builds) == 0 || p.Commit == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
pull := link.PullRef{Owner: p.Owner, Repo: p.Repo, Number: p.Number}
|
||||||
|
read, err := checkedBuildsOf(ctx, []link.PullRef{pull})
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf(" the earlier checks' builds of %s/%s#%d could not be read, so the check builds: %v\n",
|
||||||
|
p.Owner, p.Repo, p.Number, err)
|
||||||
|
// empty-on-error: said above; a check handed no named build builds every module, as before issue 703
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var named []link.CheckedBuild
|
||||||
|
for _, b := range read {
|
||||||
|
if b.Pull == pull && link.SameHead(b.Head, p.Commit) {
|
||||||
|
named = append(named, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return named
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkBuildsOf are the modules a merge of the change would build, in the walk's order, as the check builds them
|
||||||
|
// for the walk to take (novox/hq ADR 0350 §1): a module built from the repository checked, at the branch it merges
|
||||||
|
// into, from the merge the check makes; any other — a module standing on one — from its own repository at the
|
||||||
|
// branch it follows, as the walk builds it. A module whose build reads the repository checked through a context,
|
||||||
|
// or that the merge deletes, cannot be built as the walk would: said, and nothing after it is built. A change that
|
||||||
|
// moves no module builds nothing.
|
||||||
|
func checkBuildsOf(p link.PullUpdated, scope checkScope, entries []inventory.Entry,
|
||||||
|
clone func(inventory.Source) (string, error)) []link.CheckBuild {
|
||||||
|
if len(scope.Reach.Plan.Tiers) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
m := link.SourceMoved{Owner: p.Owner, Repo: p.Repo, Base: p.Base, CloneURL: p.CloneURL}
|
||||||
|
byName := map[string]inventory.Entry{}
|
||||||
|
for _, e := range entries {
|
||||||
|
byName[e.Manifest.Module] = e
|
||||||
|
}
|
||||||
|
skip := map[string]string{}
|
||||||
|
for _, e := range scope.Reach.Packaging {
|
||||||
|
skip[e.Manifest.Module] = "its build reads " + p.Owner + "/" + p.Repo + " through a context, which a check " +
|
||||||
|
"cannot hand the merge"
|
||||||
|
}
|
||||||
|
for _, e := range scope.Reach.Deleted {
|
||||||
|
skip[e.Manifest.Module] = "the merge deletes it"
|
||||||
|
}
|
||||||
|
var out []link.CheckBuild
|
||||||
|
for tier, names := range scope.Reach.Plan.Tiers {
|
||||||
|
for _, name := range names {
|
||||||
|
e, known := byName[name]
|
||||||
|
b := link.CheckBuild{Module: name, Tier: tier, Skip: skip[name]}
|
||||||
|
switch {
|
||||||
|
case !known:
|
||||||
|
b.Skip = "the catalogue does not hold it"
|
||||||
|
case b.Skip != "":
|
||||||
|
case sourceIs(e.Source, m):
|
||||||
|
b.Merged, b.Path = true, e.Source.Path
|
||||||
|
default:
|
||||||
|
url, err := clone(e.Source)
|
||||||
|
if err != nil {
|
||||||
|
b.Skip = "its repository cannot be cloned: " + err.Error()
|
||||||
|
break
|
||||||
|
}
|
||||||
|
b.Repository, b.Path, b.Ref = url, e.Source.Path, followedBranch(e.Source.Ref)
|
||||||
|
}
|
||||||
|
out = append(out, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// besideRefs is the ref each repository is cloned at beside a check, by the directory it is found under:
|
// besideRefs is the ref each repository is cloned at beside a check, by the directory it is found under:
|
||||||
// a core repository at the commit the mesh runs of the module built from it (`running`) — but the
|
// a core repository at the commit the mesh runs of the module built from it (`running`) — but the
|
||||||
// catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge
|
// catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge
|
||||||
@@ -382,6 +597,9 @@ func checkedOf(result link.BuildResult) link.Checked {
|
|||||||
// What was judged (novox/hq ADR 0323 §3): the head merged into the trunk commit, and the commits of the
|
// What was judged (novox/hq ADR 0323 §3): the head merged into the trunk commit, and the commits of the
|
||||||
// repositories read beside it.
|
// repositories read beside it.
|
||||||
c.Trunk, c.Beside, c.Took = result.Check.Trunk, result.Check.Beside, result.Check.Took
|
c.Trunk, c.Beside, c.Took = result.Check.Trunk, result.Check.Beside, result.Check.Took
|
||||||
|
c.Built = result.Check.Built
|
||||||
|
c.Release, c.JudgedBy, c.BesideAt = result.Check.Release, result.Check.JudgedBy, result.Check.BesideAt
|
||||||
|
c.Validator = result.Check.Validator
|
||||||
if c.Gate == nil {
|
if c.Gate == nil {
|
||||||
// A build seat from before the layers: its verdict is the gate's.
|
// A build seat from before the layers: its verdict is the gate's.
|
||||||
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
||||||
|
|||||||
@@ -952,7 +952,7 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
|||||||
if *group == "" && len(members) == 1 {
|
if *group == "" && len(members) == 1 {
|
||||||
// One head, checked again as the forge's announcement would have it (a recheck): the controller's
|
// One head, checked again as the forge's announcement would have it (a recheck): the controller's
|
||||||
// own path for a pull request, run because the delivery's owner asked.
|
// own path for a pull request, run because the delivery's owner asked.
|
||||||
if err := sayingOnTheBus(ctx, inv, func() error { return (following{open}).PullUpdated(ctx, members[0].pull()) }); err != nil {
|
if err := sayingOnTheBus(ctx, inv, func() error { return (following{open: open, oneShot: true}).PullUpdated(ctx, members[0].pull()) }); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return answer(map[string]any{"rechecked": members[0].ID})
|
return answer(map[string]any{"rechecked": members[0].ID})
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ func foundDirectory(module string, since time.Time) inventory.ResourceHealth {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) {
|
func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
sent := now.Add(-time.Minute)
|
sent := now.Add(-time.Minute)
|
||||||
f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
|
f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
|
||||||
@@ -48,6 +49,7 @@ func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) {
|
|||||||
// The whole walk: a module whose directory was used as found long before still gets its fix to every machine,
|
// The whole walk: a module whose directory was used as found long before still gets its fix to every machine,
|
||||||
// its pass kept and the wait said; a directory this very send found holds it and puts it back.
|
// its pass kept and the wait said; a directory this very send found holds it and puts it back.
|
||||||
func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
|
func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
for _, c := range []struct {
|
for _, c := range []struct {
|
||||||
name string
|
name string
|
||||||
found time.Duration // when the directory was found, against now
|
found time.Duration // when the directory was found, against now
|
||||||
@@ -81,9 +83,13 @@ func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
|
|||||||
}
|
}
|
||||||
return f, err
|
return f, err
|
||||||
}
|
}
|
||||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||||
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||||
gateSettle, gateEvery, gateBound = 0, 0, 300*time.Millisecond
|
wasWindow := gateWitnessWindow
|
||||||
|
gateWitnessWindow = 0
|
||||||
|
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||||
|
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
|
||||||
|
gateWithin, gateEvery, gateBound = time.Hour, 0, 300*time.Millisecond
|
||||||
deadline := time.Now().Add(5 * time.Second)
|
deadline := time.Now().Add(5 * time.Second)
|
||||||
for time.Now().Before(deadline) {
|
for time.Now().Before(deadline) {
|
||||||
advancePlans(ctx, b.open)
|
advancePlans(ctx, b.open)
|
||||||
@@ -112,6 +118,7 @@ func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
|
|||||||
|
|
||||||
// The condition says the wait in its own kind: the operator's, never urgent, and cleared once handed over.
|
// The condition says the wait in its own kind: the operator's, never urgent, and cleared once handed over.
|
||||||
func TestADirectoryUsedAsFoundIsItsOwnCondition(t *testing.T) {
|
func TestADirectoryUsedAsFoundIsItsOwnCondition(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
k, _ := withConditionsInMemory(t)
|
k, _ := withConditionsInMemory(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
rs := map[string][]inventory.ResourceHealth{"notes": {foundDirectory("notes", time.Now().Add(-time.Hour))}}
|
rs := map[string][]inventory.ResourceHealth{"notes": {foundDirectory("notes", time.Now().Add(-time.Hour))}}
|
||||||
|
|||||||
+211
-36
@@ -28,9 +28,12 @@ import (
|
|||||||
// that machine reported it applied. A build that applies and then does nothing, crashes, serves no
|
// that machine reported it applied. A build that applies and then does nothing, crashes, serves no
|
||||||
// tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
|
// tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
|
||||||
// the component's health — the core's health definitions, or a module's own — within ten minutes of the
|
// the component's health — the core's health definitions, or a module's own — within ten minutes of the
|
||||||
// apply: a core module passing three times over at least two minutes, a leaf module once, as soon as the
|
// apply, each module by its rule (gateRuleOf): the node-engine and the mesh's resolver passing on their third
|
||||||
// node-engine states it healthy (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it). Only then
|
// healthy reading, and never later than two minutes after the send while healthy; the controller and the tool runner once, no earlier than the end of the
|
||||||
// are the rest sent.
|
// node-engine's witness window for it; mesh-delivery and a leaf module once, as soon as the node-engine states
|
||||||
|
// it healthy (novox/hq ADR 0351 and ADR 0352, changing ADR 0236 decision 3, ADR 0282 decision 4 and ADR 0304
|
||||||
|
// decision 3).
|
||||||
|
// Only then are the rest sent.
|
||||||
//
|
//
|
||||||
// **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
|
// **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
|
||||||
// at its gate — registration refuses it and nothing sends it again on its own — the module's registered
|
// at its gate — registration refuses it and nothing sends it again on its own — the module's registered
|
||||||
@@ -41,22 +44,30 @@ import (
|
|||||||
|
|
||||||
// The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes.
|
// The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes.
|
||||||
var (
|
var (
|
||||||
// gateSettle is how long the first machine must stay healthy, at the least.
|
// gateWithin is the longest a module under the three-reading rule that stays healthy is held: at it, the
|
||||||
gateSettle = 2 * time.Minute
|
// gate passes on the healthy readings it has, the newest healthy and none since the send not (novox/hq ADR
|
||||||
|
// 0352). Readings gateEvery apart, the first at the apply, put three inside it.
|
||||||
|
gateWithin = 2 * time.Minute
|
||||||
// gateBound is how long after the apply the build may take to become healthy.
|
// gateBound is how long after the apply the build may take to become healthy.
|
||||||
gateBound = 10 * time.Minute
|
gateBound = 10 * time.Minute
|
||||||
// gatePasses is how many consecutive judgings must find a core module healthy, gateEvery apart at the
|
// gatePasses is how many consecutive judgings must find such a core module healthy, gateEvery apart at
|
||||||
// least, after gateSettle. gateLeafPasses is a leaf module's: one, taken once the node-engine states every
|
// the least, counted from the send: it passes on the third (novox/hq ADR 0352). gateLeafPasses is a leaf module's: one, taken once the
|
||||||
// resource of it healthy by its own check, with no settle (novox/hq ADR 0282 decision 4 as ADR 0304
|
// node-engine states every resource of it healthy by its own check, with no settle (novox/hq ADR 0282
|
||||||
// decision 3 changed it, issue 524).
|
// decision 4 as ADR 0304 decision 3 changed it, issue 524).
|
||||||
gatePasses = 3
|
gatePasses = 3
|
||||||
gateLeafPasses = 1
|
gateLeafPasses = 1
|
||||||
gateLeafSettles = false
|
gateLeafSettles = false
|
||||||
gateEvery = 40 * time.Second
|
gateEvery = 40 * time.Second
|
||||||
|
// gateWitnessWindow is the node-engine's witness window for a new controller or tool runner, from its
|
||||||
|
// start (ADR 0236 decision 27, lease.ControllerWithin and lease.NodeToolsWithin): a reading of either
|
||||||
|
// counts only after it has ended, so the build is judged by the witness and the gate both (novox/hq ADR
|
||||||
|
// 0351 decisions 1 and 4).
|
||||||
|
gateWitnessWindow = max(lease.ControllerWithin, lease.NodeToolsWithin)
|
||||||
)
|
)
|
||||||
|
|
||||||
// coreModule is whether a module is judged as the core (novox/hq ADR 0282 decision 1): one on the controller's
|
// coreModule is whether a module is judged as the core (novox/hq ADR 0282 decision 1): one on the controller's
|
||||||
// own path, or one holding the mesh's resolver. Every other module is a leaf.
|
// own path, or one holding the mesh's resolver. Every other module is a leaf. Its class sets the module's
|
||||||
|
// delivery budget; how many readings its gate takes is gateRuleOf's.
|
||||||
func coreModule(module string, m catalogue.Manifest) bool {
|
func coreModule(module string, m catalogue.Manifest) bool {
|
||||||
if _, own := onTheControllersPath[module]; own {
|
if _, own := onTheControllersPath[module]; own {
|
||||||
return true
|
return true
|
||||||
@@ -69,12 +80,72 @@ func coreModule(module string, m catalogue.Manifest) bool {
|
|||||||
return coreComponent(module) != ""
|
return coreComponent(module) != ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// gateWants is how many passes a module needs, and whether it must stay healthy gateSettle first.
|
// gateRule is how a module's first-node gate judges it: passes healthy readings, gateEvery apart at the
|
||||||
func gateWants(module string, m catalogue.Manifest) (passes int, settles bool) {
|
// least, counted from the send or, where witnessed, from the end of the node-engine's witness window for it;
|
||||||
if coreModule(module, m) {
|
// and, where within is set, passed at within after the send on fewer, if it was healthy at every reading
|
||||||
return gatePasses, true
|
// (novox/hq ADR 0352). said names the rule in the gate's text.
|
||||||
|
type gateRule struct {
|
||||||
|
passes int
|
||||||
|
within time.Duration
|
||||||
|
witnessed bool
|
||||||
|
said string
|
||||||
|
}
|
||||||
|
|
||||||
|
// gateRuleOf is a module's gate rule (novox/hq ADR 0351): a core module keeps three readings only where its
|
||||||
|
// build, broken on every machine, could not receive its own fix — the node-engine, which applies every send,
|
||||||
|
// and the mesh's resolver, which gives every machine the bus's name (decisions 2, 3 and 6); the bus is never
|
||||||
|
// walked through a gate but kept so (decision 7). The controller, run on one machine, and the tool runner,
|
||||||
|
// whose fix the node-engine applies, are judged by one reading after the witness window (decisions 1 and 4);
|
||||||
|
// mesh-delivery, whose fix goes the controller's own path, by one (decision 5); a leaf module by one (ADR 0304
|
||||||
|
// decision 3).
|
||||||
|
func gateRuleOf(module string, m catalogue.Manifest) gateRule {
|
||||||
|
switch {
|
||||||
|
case coreComponent(module) == lease.ComponentController:
|
||||||
|
return gateRule{passes: 1, witnessed: true, said: "the controller's rule: one reading once it is ready, " +
|
||||||
|
"after the node-engine's witness window (novox/hq ADR 0351)"}
|
||||||
|
case coreComponent(module) == lease.ComponentNodeTools:
|
||||||
|
return gateRule{passes: 1, witnessed: true, said: "the tool runner's rule: one reading once it answers, " +
|
||||||
|
"after the node-engine's witness window (novox/hq ADR 0351)"}
|
||||||
|
case module == catalogue.DeliverySeat:
|
||||||
|
return gateRule{passes: 1, said: "mesh-delivery's rule: one reading (novox/hq ADR 0351)"}
|
||||||
|
case coreModule(module, m):
|
||||||
|
return gateRule{passes: gatePasses, within: gateWithin, said: fmt.Sprintf("the core's rule: %d readings, "+
|
||||||
|
"or healthy throughout %s after the send (novox/hq ADR 0352)", gatePasses, gateWithin)}
|
||||||
}
|
}
|
||||||
return gateLeafPasses, gateLeafSettles
|
r := gateRule{passes: gateLeafPasses, said: "a leaf module's rule: one reading (novox/hq ADR 0304)"}
|
||||||
|
if gateLeafSettles {
|
||||||
|
r.within = gateWithin
|
||||||
|
r.said = fmt.Sprintf("a leaf module's rule, judged as the core: %d readings, or healthy throughout %s after the send",
|
||||||
|
gateLeafPasses, gateWithin)
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// gateFloor is the earliest a module's reading counts by its rule: the send and, for a witnessed module, the
|
||||||
|
// end of the node-engine's witness window on every machine judged (novox/hq ADR 0351 decisions 1 and 4). A
|
||||||
|
// module under the three-reading rule counts from the send (novox/hq ADR 0352). The window opens when the node-engine starts the build, which is before the machine reports it
|
||||||
|
// applied (applied, as the gate first read it) and, for the controller, before the new controller took the
|
||||||
|
// lease (holder): the later of those, the send's time at the least, plus the window, is never before its end.
|
||||||
|
// A controller that starts again takes the lease again, and so moves its floor on. Pure.
|
||||||
|
func gateFloor(module string, rule gateRule, since time.Time, machines []string, applied map[string]time.Time,
|
||||||
|
holder *lease.Holder) time.Time {
|
||||||
|
floor := since
|
||||||
|
if !rule.witnessed {
|
||||||
|
return floor
|
||||||
|
}
|
||||||
|
for _, n := range machines {
|
||||||
|
start := since
|
||||||
|
if at, read := applied[n]; read && at.After(start) {
|
||||||
|
start = at
|
||||||
|
}
|
||||||
|
if coreComponent(module) == lease.ComponentController && holder != nil && holder.Taken.After(start) {
|
||||||
|
start = holder.Taken
|
||||||
|
}
|
||||||
|
if end := start.Add(gateWitnessWindow); end.After(floor) {
|
||||||
|
floor = end
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return floor
|
||||||
}
|
}
|
||||||
|
|
||||||
// gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions
|
// gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions
|
||||||
@@ -501,6 +572,13 @@ func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module stri
|
|||||||
return judgeMoves(ctx, open, g, pairs, now)
|
return judgeMoves(ctx, open, g, pairs, now)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// dueAtTheMost says a gate healthy since its send has reached gateWithin and not been judged since: the one
|
||||||
|
// judging that may come sooner than gateEvery after the last pass (novox/hq ADR 0352).
|
||||||
|
func dueAtTheMost(g *inventory.PlanGate, now time.Time) bool {
|
||||||
|
most := g.Since.Add(gateWithin)
|
||||||
|
return !g.Faulted && g.LastPass != nil && !now.Before(most) && g.LastPass.Before(most)
|
||||||
|
}
|
||||||
|
|
||||||
// judged is one module on one machine, as a gate judges it.
|
// judged is one module on one machine, as a gate judges it.
|
||||||
type judged struct{ module, node string }
|
type judged struct{ module, node string }
|
||||||
|
|
||||||
@@ -511,7 +589,9 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
if g.Verdict != "" {
|
if g.Verdict != "" {
|
||||||
return g.Verdict, nil
|
return g.Verdict, nil
|
||||||
}
|
}
|
||||||
if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery {
|
// A judging comes gateEvery after the last pass, and once at gateWithin after the send whatever the spacing:
|
||||||
|
// a module healthy throughout is never held past it (novox/hq ADR 0352).
|
||||||
|
if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery && !dueAtTheMost(g, now) {
|
||||||
return "", nil
|
return "", nil
|
||||||
}
|
}
|
||||||
for _, c := range g.Carried {
|
for _, c := range g.Carried {
|
||||||
@@ -544,6 +624,11 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
facts.sentBuilds[j.node] = builds
|
facts.sentBuilds[j.node] = builds
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// **A busy container on a judged machine holds the judging before it starts** (novox/hq ADR 0335 §4): the
|
||||||
|
// walk says it waits there, and its gate starts at the report that says the container replaced.
|
||||||
|
if busyWait(g, pairs, facts, now) {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
// **What is wrong with a machine itself is the machine's** (novox/hq issue 281): read once for each
|
// **What is wrong with a machine itself is the machine's** (novox/hq issue 281): read once for each
|
||||||
// machine judged, apart from what is wrong with a module there, and never pinned on the module the
|
// machine judged, apart from what is wrong with a module there, and never pinned on the module the
|
||||||
// gate happens to be kept on.
|
// gate happens to be kept on.
|
||||||
@@ -568,6 +653,9 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
for _, j := range pairs {
|
for _, j := range pairs {
|
||||||
w := words[j.node]
|
w := words[j.node]
|
||||||
h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, w.facts)
|
h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, w.facts)
|
||||||
|
if why, still := busyStill(j, facts); still && (h == healthGood || h == healthPerson) {
|
||||||
|
h, said = healthNotYet, why
|
||||||
|
}
|
||||||
if h == healthGood || h == healthPerson {
|
if h == healthGood || h == healthPerson {
|
||||||
if on, named := w.on[j.module]; named {
|
if on, named := w.on[j.module]; named {
|
||||||
h, said = healthNotYet, on
|
h, said = healthNotYet, on
|
||||||
@@ -608,6 +696,51 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
why = said
|
why = said
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// **Each module counts its readings by its own rule** (novox/hq ADR 0351): from when the gate first read
|
||||||
|
// each machine report its send applied, and who holds the controller lease, the earliest its reading counts.
|
||||||
|
if g.Applied == nil {
|
||||||
|
g.Applied = map[string]time.Time{}
|
||||||
|
}
|
||||||
|
for node := range byMachine {
|
||||||
|
r, said := facts.reports[node]
|
||||||
|
if _, read := g.Applied[node]; !read && said && r.At != nil && r.Outcome == inventory.OutcomeApplied &&
|
||||||
|
facts.reportedOn(node, r) {
|
||||||
|
g.Applied[node] = *r.At
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rules, floors := map[string]gateRule{}, map[string]time.Time{}
|
||||||
|
for _, m := range modules {
|
||||||
|
var on []string
|
||||||
|
for _, j := range pairs {
|
||||||
|
if j.module == m {
|
||||||
|
on = append(on, j.node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rules[m] = gateRuleOf(m, shelf[m])
|
||||||
|
floors[m] = gateFloor(m, rules[m], *g.Since, on, g.Applied, facts.holder)
|
||||||
|
}
|
||||||
|
tallyGate(g, gateJudging{modules: modules, reading: reading, waits: waits, worst: worst, why: why,
|
||||||
|
rules: rules, floors: floors}, now)
|
||||||
|
return g.Verdict, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// gateJudging is one judging's readings, as tallyGate counts them: the modules judged, each one's reading
|
||||||
|
// (the worst of its machines'), its waits for a person, the send's worst reading and why, and each module's
|
||||||
|
// rule and the earliest its reading counts.
|
||||||
|
type gateJudging struct {
|
||||||
|
modules []string
|
||||||
|
reading map[string]health
|
||||||
|
waits map[string]string
|
||||||
|
worst health
|
||||||
|
why string
|
||||||
|
rules map[string]gateRule
|
||||||
|
floors map[string]time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// tallyGate counts one judging into the gate: a pass counted, a pass missed (and why), or the verdict. Pure but
|
||||||
|
// for the gate it writes.
|
||||||
|
func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) {
|
||||||
|
modules, reading, waits, worst, why := j.modules, j.reading, j.waits, j.worst, j.why
|
||||||
if g.Healthy == nil {
|
if g.Healthy == nil {
|
||||||
g.Healthy = map[string]int{}
|
g.Healthy = map[string]int{}
|
||||||
}
|
}
|
||||||
@@ -623,38 +756,53 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
delete(g.HealthyAt, m)
|
delete(g.HealthyAt, m)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more
|
|
||||||
// often while another module beside it is not yet healthy counts no faster.
|
|
||||||
if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery {
|
|
||||||
g.Healthy[m]++
|
|
||||||
g.HealthyAt[m] = now
|
|
||||||
}
|
|
||||||
if w := waits[m]; w != "" {
|
if w := waits[m]; w != "" {
|
||||||
if g.Waits == nil {
|
if g.Waits == nil {
|
||||||
g.Waits = map[string]string{}
|
g.Waits = map[string]string{}
|
||||||
}
|
}
|
||||||
g.Waits[m] = w
|
g.Waits[m] = w
|
||||||
}
|
}
|
||||||
|
// **A reading counts only from the module's floor** (novox/hq ADR 0351 decisions 1 and 4): the
|
||||||
|
// controller's and the tool runner's after the witness window, any other module's from the send.
|
||||||
|
if now.Before(j.floors[m]) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more
|
||||||
|
// often while another module beside it is not yet healthy counts no faster.
|
||||||
|
if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery {
|
||||||
|
g.Healthy[m]++
|
||||||
|
g.HealthyAt[m] = now
|
||||||
|
}
|
||||||
}
|
}
|
||||||
settled := now.Sub(*g.Since) >= gateSettle
|
// What the send needs is what its most demanding module needs — the most readings, then the latest
|
||||||
// What the send needs is what its most demanding module needs: a send carrying a core module is judged
|
// floor: a send carrying the node-engine is judged by its three readings, or by being healthy throughout
|
||||||
// as the core.
|
// its two minutes (novox/hq ADR 0352).
|
||||||
wants, settles := 0, false
|
wants, floor, within, said := 0, time.Time{}, time.Duration(0), ""
|
||||||
for _, m := range modules {
|
for _, m := range modules {
|
||||||
p, s := gateWants(m, shelf[m])
|
r, f := j.rules[m], j.floors[m]
|
||||||
wants, settles = max(wants, p), settles || s
|
if r.passes > wants || (r.passes == wants && f.After(floor)) {
|
||||||
|
wants, within, said = r.passes, r.within, r.said
|
||||||
|
}
|
||||||
|
if f.After(floor) {
|
||||||
|
floor = f
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if len(modules) == 0 {
|
if len(modules) == 0 {
|
||||||
// A send judged on no machine of its modules (its first machine not running them) has no class to
|
// A send judged on no machine of its modules (its first machine not running them) has no class to
|
||||||
// be judged by: it is judged as the core, never passed at once without a reading.
|
// be judged by: it is judged as the core, never passed at once without a reading.
|
||||||
wants, settles = gatePasses, true
|
wants, floor, within = gatePasses, *g.Since, gateWithin
|
||||||
|
said = fmt.Sprintf("the core's rule: %d readings, or healthy throughout %s after the send", gatePasses, gateWithin)
|
||||||
|
}
|
||||||
|
// heldLongEnough is a rule's most: within after the send, healthy at every reading since it.
|
||||||
|
heldLongEnough := func(within time.Duration) bool {
|
||||||
|
return within > 0 && !g.Faulted && !now.Before(g.Since.Add(within))
|
||||||
}
|
}
|
||||||
g.Wants = wants
|
g.Wants = wants
|
||||||
// A module's own pass is its own class's: a leaf beside a core module that fails keeps its build after
|
// A module's own pass is its own rule's: a leaf beside a core module that fails keeps its build after
|
||||||
// its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524).
|
// its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524).
|
||||||
passedAlone := func(m string) bool {
|
passedAlone := func(m string) bool {
|
||||||
p, s := gateWants(m, shelf[m])
|
r := j.rules[m]
|
||||||
return (settled || !s) && reading[m] <= healthPerson && g.Healthy[m] >= p
|
return reading[m] <= healthPerson && (g.Healthy[m] >= r.passes || (g.Healthy[m] >= 1 && heldLongEnough(r.within)))
|
||||||
}
|
}
|
||||||
// fail decides the send failed: what passed on its own keeps its pass, everything else is put back.
|
// fail decides the send failed: what passed on its own keeps its pass, everything else is put back.
|
||||||
fail := func(why string) {
|
fail := func(why string) {
|
||||||
@@ -671,6 +819,7 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
pastBound := now.Sub(*g.Since) > gateBound
|
pastBound := now.Sub(*g.Since) > gateBound
|
||||||
switch {
|
switch {
|
||||||
case worst == healthBroken:
|
case worst == healthBroken:
|
||||||
|
g.Faulted = true
|
||||||
g.Read(now, false, g.BrokenWhy)
|
g.Read(now, false, g.BrokenWhy)
|
||||||
var judging []string
|
var judging []string
|
||||||
for _, m := range modules {
|
for _, m := range modules {
|
||||||
@@ -689,25 +838,36 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
|||||||
case worst == healthWaiting:
|
case worst == healthWaiting:
|
||||||
// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
|
// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
|
||||||
// the provider's own condition says what is wrong (ADR 0240 rule 5).
|
// the provider's own condition says what is wrong (ADR 0240 rule 5).
|
||||||
|
g.Faulted = g.Faulted || g.HealthySeen // not healthy after a healthy reading; before one, still starting
|
||||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||||
g.Read(now, false, why)
|
g.Read(now, false, why)
|
||||||
case worst == healthNotYet:
|
case worst == healthNotYet:
|
||||||
|
g.Faulted = g.Faulted || g.HealthySeen // not healthy after a healthy reading; before one, still starting
|
||||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||||
g.Read(now, false, why)
|
g.Read(now, false, why)
|
||||||
if pastBound {
|
if pastBound {
|
||||||
fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why))
|
fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why))
|
||||||
}
|
}
|
||||||
|
case now.Before(floor):
|
||||||
|
// Healthy before its rule lets a reading count: not a pass yet, and not a fault (novox/hq ADR 0351).
|
||||||
|
g.HealthySeen = true
|
||||||
|
// The bound still holds: a module that cannot be read healthy after its floor fails at it.
|
||||||
|
g.Last, g.Failing = fmt.Sprintf("healthy, and a reading counts from %s by %s",
|
||||||
|
floor.UTC().Format(time.RFC3339), said), nil
|
||||||
|
if pastBound {
|
||||||
|
fail(fmt.Sprintf("not judged within %s of its apply: %s", gateBound, g.Last))
|
||||||
|
}
|
||||||
default:
|
default:
|
||||||
// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
|
// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
|
||||||
|
g.HealthySeen = true
|
||||||
g.Passes++
|
g.Passes++
|
||||||
g.LastPass, g.Last, g.Failing = &now, "", nil
|
g.LastPass, g.Last, g.Failing = &now, "", nil
|
||||||
g.Read(now, true, "")
|
g.Read(now, true, "")
|
||||||
if g.Passes >= wants && (settled || !settles) {
|
if g.Passes >= wants || heldLongEnough(within) {
|
||||||
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
|
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s, by %s", g.Passes,
|
||||||
now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
|
now.Sub(*g.Since).Round(time.Second), said)+waitsSaid(g.Waits), now)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return g.Verdict, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// judgedCommits is the commit of each module a gate judges: the gate's own To for its module, and each
|
// judgedCommits is the commit of each module a gate judges: the gate's own To for its module, and each
|
||||||
@@ -1283,6 +1443,12 @@ func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error
|
|||||||
}
|
}
|
||||||
// A release held after a failed one, while builds still wait for a gate (ADR 0236).
|
// A release held after a failed one, while builds still wait for a gate (ADR 0236).
|
||||||
out = append(out, backlogObservation()...)
|
out = append(out, backlogObservation()...)
|
||||||
|
// And a rest that went silent (novox/hq ADR 0353 §4).
|
||||||
|
silent, err := silentObservation(ctx, d.open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = append(out, silent...)
|
||||||
return sortedFound(dedupeObservations(out)), nil
|
return sortedFound(dedupeObservations(out)), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1372,6 +1538,15 @@ func gateLine(g *inventory.PlanGate) string {
|
|||||||
case inventory.GateFailed:
|
case inventory.GateFailed:
|
||||||
what = "FAILED"
|
what = "FAILED"
|
||||||
}
|
}
|
||||||
|
switch {
|
||||||
|
case g.Verdict == "" && g.Pending:
|
||||||
|
what = "kept, not yet sent" // novox/hq ADR 0353 §8
|
||||||
|
case g.Verdict == "" && g.Silent:
|
||||||
|
what = "silent on its send, left" // ADR 0353 §4
|
||||||
|
}
|
||||||
|
if g.Verdict == "" && g.Waiting() {
|
||||||
|
what = "waits for a busy container on " + g.Busy.Machine
|
||||||
|
}
|
||||||
line := fmt.Sprintf("gate on %s: %s", strings.Join(g.Machines, ", "), what)
|
line := fmt.Sprintf("gate on %s: %s", strings.Join(g.Machines, ", "), what)
|
||||||
if g.Component != "" {
|
if g.Component != "" {
|
||||||
line += " (core: " + g.Component + ")"
|
line += " (core: " + g.Component + ")"
|
||||||
|
|||||||
@@ -21,11 +21,15 @@ import (
|
|||||||
// wait, and a provider waiting for a login says who waits on it.
|
// wait, and a provider waiting for a login says who waits on it.
|
||||||
|
|
||||||
// withGateBounds sets the gate's bounds for one test.
|
// withGateBounds sets the gate's bounds for one test.
|
||||||
func withGateBounds(t *testing.T, settle, every, bound time.Duration) {
|
func withGateBounds(t *testing.T, within, every, bound time.Duration) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||||
gateSettle, gateEvery, gateBound = settle, every, bound
|
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||||
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
wasWindow := gateWitnessWindow
|
||||||
|
gateWitnessWindow = 0
|
||||||
|
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||||
|
gateWithin, gateEvery, gateBound = within, every, bound
|
||||||
|
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
|
||||||
}
|
}
|
||||||
|
|
||||||
// factsOn is a judging's facts for one machine that applied its send: the node tools answer, and what it says
|
// factsOn is a judging's facts for one machine that applied its send: the node tools answer, and what it says
|
||||||
@@ -53,10 +57,11 @@ func healthyContainer(module string) inventory.ResourceHealth {
|
|||||||
// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass,
|
// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass,
|
||||||
// counted over the gate's spacing — before it says its own. The broken one alone is put back.
|
// counted over the gate's spacing — before it says its own. The broken one alone is put back.
|
||||||
func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
|
func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
judgeLeavesAsTheCore(t)
|
judgeLeavesAsTheCore(t)
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second)
|
withGateBounds(t, time.Hour, 20*time.Millisecond, 10*time.Second)
|
||||||
since := time.Now().Add(-time.Second)
|
since := time.Now().Add(-time.Second)
|
||||||
base := factsOn(t, "laptop", since, healthyContainer("app"))
|
base := factsOn(t, "laptop", since, healthyContainer("app"))
|
||||||
wasGather := gatherGateFacts
|
wasGather := gatherGateFacts
|
||||||
@@ -95,9 +100,10 @@ const broker_runtime = "node-tools"
|
|||||||
// **A pass is counted only the gate's spacing after the one before** (review (c)): a send judged every tick
|
// **A pass is counted only the gate's spacing after the one before** (review (c)): a send judged every tick
|
||||||
// while a module beside it is not yet healthy counts the healthy one once.
|
// while a module beside it is not yet healthy counts the healthy one once.
|
||||||
func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) {
|
func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
withGateBounds(t, 0, time.Hour, time.Hour)
|
withGateBounds(t, time.Hour, time.Hour, time.Hour)
|
||||||
since := time.Now().Add(-time.Minute)
|
since := time.Now().Add(-time.Minute)
|
||||||
base := factsOn(t, "laptop", since, healthyContainer("app"), inventory.ResourceHealth{Module: "late",
|
base := factsOn(t, "laptop", since, healthyContainer("app"), inventory.ResourceHealth{Module: "late",
|
||||||
Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, Reason: "down"})
|
Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, Reason: "down"})
|
||||||
@@ -119,6 +125,7 @@ func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) {
|
|||||||
|
|
||||||
// **A carried build's pass carries its own wait, never another's** (review (e)).
|
// **A carried build's pass carries its own wait, never another's** (review (e)).
|
||||||
func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) {
|
func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
b := aBacklog(t)
|
b := aBacklog(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := b.open.inventory
|
inv := b.open.inventory
|
||||||
@@ -143,6 +150,7 @@ func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) {
|
|||||||
// carried its own module, healthy, and a build waiting on that machine that never became healthy. At the
|
// carried its own module, healthy, and a build waiting on that machine that never became healthy. At the
|
||||||
// bound the waiting one is put back and marked; the plan's own module keeps its pass, so no walk waits on it.
|
// bound the waiting one is put back and marked; the plan's own module keeps its pass, so no walk waits on it.
|
||||||
func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) {
|
func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
g := aGateMesh(t)
|
g := aGateMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := g.open.inventory
|
inv := g.open.inventory
|
||||||
@@ -223,6 +231,7 @@ func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) {
|
|||||||
// **A provider waiting for a login says who waits on it** (review (g)): its consumer, failing a check that
|
// **A provider waiting for a login says who waits on it** (review (g)): its consumer, failing a check that
|
||||||
// needs it, is held under it, and the provider's relogin-needed condition lists it and is urgent.
|
// needs it, is held under it, and the provider's relogin-needed condition lists it and is urgent.
|
||||||
func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
|
func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
@@ -285,6 +294,7 @@ func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
|
|||||||
// tools are marked and put back in the very judging that found them broken — their registered build is the
|
// tools are marked and put back in the very judging that found them broken — their registered build is the
|
||||||
// one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass.
|
// one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass.
|
||||||
func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) {
|
func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
judgeLeavesAsTheCore(t)
|
judgeLeavesAsTheCore(t)
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
@@ -376,7 +386,7 @@ func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testi
|
|||||||
f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{app}}}
|
f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{app}}}
|
||||||
return f, nil
|
return f, nil
|
||||||
}
|
}
|
||||||
withGateBounds(t, 100*time.Millisecond, 0, 10*time.Second)
|
withGateBounds(t, time.Hour, 0, 10*time.Second)
|
||||||
|
|
||||||
release := func() inventory.Plan {
|
release := func() inventory.Plan {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
@@ -427,6 +437,7 @@ func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testi
|
|||||||
// **A move from a build not known excuses no wait** (issue 318 review): a plan's own module whose previous
|
// **A move from a build not known excuses no wait** (issue 318 review): a plan's own module whose previous
|
||||||
// build was not recorded, or whose previous manifest is not kept, cannot be shown to have added the group.
|
// build was not recorded, or whose previous manifest is not kept, cannot be shown to have added the group.
|
||||||
func TestAMoveFromAnUnknownBuildExcusesNoWait(t *testing.T) {
|
func TestAMoveFromAnUnknownBuildExcusesNoWait(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
|
|||||||
@@ -0,0 +1,338 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/lease"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The first-node gate's readings by module (novox/hq ADR 0351, ADR 0352): the controller and the tool runner pass
|
||||||
|
// on one reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's
|
||||||
|
// resolver on their third healthy reading and never later than two minutes while healthy; the bus is kept so.
|
||||||
|
|
||||||
|
// withTheGatesTimes sets the gate's production bounds for a test, whatever another test left.
|
||||||
|
func withTheGatesTimes(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
wasWithin, wasEvery, wasBound, wasWindow := gateWithin, gateEvery, gateBound, gateWitnessWindow
|
||||||
|
wasPasses, wasLeafPasses, wasLeafSettles := gatePasses, gateLeafPasses, gateLeafSettles
|
||||||
|
gateWithin, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second
|
||||||
|
gatePasses, gateLeafPasses, gateLeafSettles = 3, 1, false
|
||||||
|
t.Cleanup(func() {
|
||||||
|
gateWithin, gateEvery, gateBound, gateWitnessWindow = wasWithin, wasEvery, wasBound, wasWindow
|
||||||
|
gatePasses, gateLeafPasses, gateLeafSettles = wasPasses, wasLeafPasses, wasLeafSettles
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEachModuleHasItsGateRule(t *testing.T) {
|
||||||
|
withTheGatesTimes(t)
|
||||||
|
for _, c := range []struct {
|
||||||
|
module string
|
||||||
|
claims []catalogue.Claim
|
||||||
|
passes int
|
||||||
|
within time.Duration
|
||||||
|
witnessed bool
|
||||||
|
said string
|
||||||
|
}{
|
||||||
|
{"mesh-controller", nil, 1, 0, true, "the controller's rule"},
|
||||||
|
{"node-tools", nil, 1, 0, true, "the tool runner's rule"},
|
||||||
|
{catalogue.DeliverySeat, nil, 1, 0, false, "mesh-delivery's rule"},
|
||||||
|
{"mesh-host", nil, 3, 2 * time.Minute, false, "the core's rule"},
|
||||||
|
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, 2 * time.Minute, false, "the core's rule"},
|
||||||
|
{"nats", nil, 3, 2 * time.Minute, false, "the core's rule"},
|
||||||
|
{"app", nil, 1, 0, false, "a leaf module's rule"},
|
||||||
|
} {
|
||||||
|
r := gateRuleOf(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
|
||||||
|
if r.passes != c.passes || r.within != c.within || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) {
|
||||||
|
t.Errorf("%s: %+v; want %d readings, within %s, witnessed %v, said %q", c.module, r, c.passes, c.within,
|
||||||
|
c.witnessed, c.said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every module that was core stays core for its delivery budget and its walk: only its readings change.
|
||||||
|
func TestTheCoreStaysTheCore(t *testing.T) {
|
||||||
|
for _, m := range []string{"mesh-controller", "mesh-host", "node-tools", "nats", catalogue.DeliverySeat} {
|
||||||
|
if !coreModule(m, catalogue.Manifest{Module: m}) {
|
||||||
|
t.Errorf("%s is no longer judged as the core for its class", m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !coreModule("dnsmasq", catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}) {
|
||||||
|
t.Error("the resolver's holder is no longer judged as the core for its class")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// gateOn judges one module on one machine, healthy at every reading.
|
||||||
|
type gateOn struct {
|
||||||
|
module, node string
|
||||||
|
g *inventory.PlanGate
|
||||||
|
holder *lease.Holder
|
||||||
|
}
|
||||||
|
|
||||||
|
func aGateOn(module, node string, since time.Time) *gateOn {
|
||||||
|
return &gateOn{module: module, node: node, g: &inventory.PlanGate{Machines: []string{node}, Since: &since,
|
||||||
|
Applied: map[string]time.Time{}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *gateOn) read(at time.Time, h health, why string) {
|
||||||
|
r := gateRuleOf(o.module, catalogue.Manifest{Module: o.module})
|
||||||
|
floor := gateFloor(o.module, r, *o.g.Since, []string{o.node}, o.g.Applied, o.holder)
|
||||||
|
tallyGate(o.g, gateJudging{modules: []string{o.module}, reading: map[string]health{o.module: h},
|
||||||
|
worst: h, why: why, rules: map[string]gateRule{o.module: r}, floors: map[string]time.Time{o.module: floor}}, at)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheControllerPassesOnOneReadingAfterTheWitnessWindow(t *testing.T) {
|
||||||
|
withTheGatesTimes(t)
|
||||||
|
sent := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC)
|
||||||
|
o := aGateOn("mesh-controller", "novox", sent)
|
||||||
|
o.g.Applied["novox"] = sent.Add(15 * time.Second)
|
||||||
|
o.holder = &lease.Holder{Instance: "new", Taken: sent.Add(20 * time.Second)}
|
||||||
|
|
||||||
|
// Ready, but inside the witness window, which opened before it took the lease: not counted.
|
||||||
|
for _, after := range []time.Duration{30 * time.Second, 60 * time.Second, 79 * time.Second} {
|
||||||
|
o.read(sent.Add(after), healthGood, "")
|
||||||
|
if o.g.Verdict != "" || o.g.Passes != 0 || !strings.Contains(o.g.Last, "counts from") {
|
||||||
|
t.Fatalf("at %s: %+v; want no reading counted inside the witness window", after, o.g)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
o.read(sent.Add(80*time.Second), healthGood, "")
|
||||||
|
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || o.g.Wants != 1 {
|
||||||
|
t.Fatalf("after the witness window: %+v; want passed on one reading", o.g)
|
||||||
|
}
|
||||||
|
if !strings.Contains(o.g.Why, "healthy 1 times") || !strings.Contains(o.g.Why, "the controller's rule") ||
|
||||||
|
!strings.Contains(o.g.Why, "ADR 0351") {
|
||||||
|
t.Errorf("the verdict says %q; want the rule it applied", o.g.Why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A controller that starts again takes the lease again: its witness window starts again, and so does the
|
||||||
|
// gate's wait.
|
||||||
|
func TestAControllerThatStartsAgainWaitsOutItsNewWindow(t *testing.T) {
|
||||||
|
withTheGatesTimes(t)
|
||||||
|
sent := time.Now()
|
||||||
|
o := aGateOn("mesh-controller", "novox", sent)
|
||||||
|
o.g.Applied["novox"] = sent.Add(10 * time.Second)
|
||||||
|
o.holder = &lease.Holder{Instance: "again", Taken: sent.Add(70 * time.Second)}
|
||||||
|
o.read(sent.Add(90*time.Second), healthGood, "")
|
||||||
|
if o.g.Verdict != "" {
|
||||||
|
t.Fatalf("passed %s after a start the witness still judges: %+v", 20*time.Second, o.g)
|
||||||
|
}
|
||||||
|
o.read(sent.Add(130*time.Second), healthGood, "")
|
||||||
|
if o.g.Verdict != inventory.GatePassed {
|
||||||
|
t.Fatalf("not passed after the new window: %+v", o.g)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Without the lease's word (a controller carried by another gate), the window counts from the applied report,
|
||||||
|
// and from the send at the least.
|
||||||
|
func TestTheWitnessWindowCountsFromTheSendAtTheLeast(t *testing.T) {
|
||||||
|
withTheGatesTimes(t)
|
||||||
|
sent := time.Now()
|
||||||
|
r := gateRuleOf("mesh-controller", catalogue.Manifest{})
|
||||||
|
if f := gateFloor("mesh-controller", r, sent, []string{"novox"}, nil, nil); !f.Equal(sent.Add(time.Minute)) {
|
||||||
|
t.Errorf("floor %s after the send; want 1m", f.Sub(sent))
|
||||||
|
}
|
||||||
|
if f := gateFloor("node-tools", gateRuleOf("node-tools", catalogue.Manifest{}), sent, []string{"ace"},
|
||||||
|
map[string]time.Time{"ace": sent.Add(12 * time.Second)}, nil); !f.Equal(sent.Add(72 * time.Second)) {
|
||||||
|
t.Errorf("the tool runner's floor %s after the send; want 1m12s", f.Sub(sent))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A controller that a witness put back, or that never becomes ready, fails its gate as before.
|
||||||
|
func TestAControllerThatFailsIsStillFailed(t *testing.T) {
|
||||||
|
withTheGatesTimes(t)
|
||||||
|
sent := time.Now()
|
||||||
|
o := aGateOn("mesh-controller", "novox", sent)
|
||||||
|
o.read(sent.Add(30*time.Second), healthBroken, "the witness on novox judged the controller and rolled-back")
|
||||||
|
if o.g.Verdict != inventory.GateFailed {
|
||||||
|
t.Fatalf("a controller put back by its witness: %+v; want failed", o.g)
|
||||||
|
}
|
||||||
|
o = aGateOn("mesh-controller", "novox", sent)
|
||||||
|
o.read(sent.Add(5*time.Minute), healthNotYet, "the controller holding the lease does not say it is ready")
|
||||||
|
if o.g.Verdict != "" {
|
||||||
|
t.Fatalf("failed before the bound: %+v", o.g)
|
||||||
|
}
|
||||||
|
o.read(sent.Add(10*time.Minute+time.Second), healthNotYet, "the controller holding the lease does not say it is ready")
|
||||||
|
if o.g.Verdict != inventory.GateFailed || !strings.Contains(o.g.Why, "not healthy within") {
|
||||||
|
t.Fatalf("never ready: %+v; want failed at the bound", o.g)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// threeReadings judges a module under the three-reading rule on one machine, the send at sent.
|
||||||
|
func threeReadings(t *testing.T, module string, sent time.Time) (*inventory.PlanGate, func(time.Duration, health)) {
|
||||||
|
t.Helper()
|
||||||
|
g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent}
|
||||||
|
rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}})
|
||||||
|
floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil)
|
||||||
|
return g, func(after time.Duration, h health) {
|
||||||
|
why := ""
|
||||||
|
if h != healthGood {
|
||||||
|
why = "the node-engine reports another build"
|
||||||
|
}
|
||||||
|
tallyGate(g, gateJudging{modules: []string{module}, reading: map[string]health{module: h}, worst: h, why: why,
|
||||||
|
rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}}, sent.Add(after))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A healthy node-engine or resolver passes at its third healthy reading, 40 s apart from the apply: before two
|
||||||
|
// minutes (novox/hq ADR 0352).
|
||||||
|
func TestTheNodeEngineAndTheResolverPassAtTheirThirdReading(t *testing.T) {
|
||||||
|
withTheGatesTimes(t)
|
||||||
|
for _, module := range []string{"mesh-host", "dnsmasq"} {
|
||||||
|
g, judge := threeReadings(t, module, time.Now())
|
||||||
|
judge(15*time.Second, healthGood)
|
||||||
|
judge(55*time.Second, healthGood)
|
||||||
|
if g.Verdict != "" || g.Passes != 2 {
|
||||||
|
t.Fatalf("%s: after two readings %+v; want still judging", module, g)
|
||||||
|
}
|
||||||
|
judge(95*time.Second, healthGood)
|
||||||
|
if g.Verdict != inventory.GatePassed || g.Passes != 3 || g.Wants != 3 ||
|
||||||
|
!strings.Contains(g.Why, "over 1m35s") || !strings.Contains(g.Why, "the core's rule: 3 readings") {
|
||||||
|
t.Fatalf("%s: after three readings %+v; want passed at 1m35s by the core's rule", module, g)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// One healthy throughout whose readings came late passes at two minutes on the readings it has; the judging at
|
||||||
|
// two minutes comes sooner than the spacing after the last pass.
|
||||||
|
func TestAHealthyNodeEngineIsNeverHeldPastTwoMinutes(t *testing.T) {
|
||||||
|
withTheGatesTimes(t)
|
||||||
|
sent := time.Now()
|
||||||
|
last := sent.Add(100 * time.Second)
|
||||||
|
held := &inventory.PlanGate{Since: &sent, LastPass: &last}
|
||||||
|
if dueAtTheMost(held, sent.Add(119*time.Second)) || !dueAtTheMost(held, sent.Add(2*time.Minute)) {
|
||||||
|
t.Fatal("the judging at two minutes is not due, or is due before")
|
||||||
|
}
|
||||||
|
held.Faulted = true
|
||||||
|
if dueAtTheMost(held, sent.Add(2*time.Minute)) {
|
||||||
|
t.Fatal("a send that was not healthy at a reading is judged at two minutes")
|
||||||
|
}
|
||||||
|
|
||||||
|
g, judge := threeReadings(t, "mesh-host", time.Now())
|
||||||
|
judge(100*time.Second, healthGood)
|
||||||
|
judge(2*time.Minute, healthGood)
|
||||||
|
if g.Verdict != inventory.GatePassed || g.Passes != 2 {
|
||||||
|
t.Fatalf("late readings, healthy throughout, at two minutes: %+v; want passed on two", g)
|
||||||
|
}
|
||||||
|
g, judge = threeReadings(t, "mesh-host", time.Now())
|
||||||
|
judge(2*time.Minute, healthGood)
|
||||||
|
if g.Verdict != inventory.GatePassed || g.Passes != 1 {
|
||||||
|
t.Fatalf("one healthy reading, at two minutes: %+v; want passed on it", g)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A reading not yet healthy before the first healthy one is the build still starting: the two minutes still hold.
|
||||||
|
func TestANodeEngineStillStartingKeepsItsTwoMinutes(t *testing.T) {
|
||||||
|
withTheGatesTimes(t)
|
||||||
|
g, judge := threeReadings(t, "mesh-host", time.Now())
|
||||||
|
judge(10*time.Second, healthNotYet)
|
||||||
|
judge(100*time.Second, healthGood)
|
||||||
|
judge(2*time.Minute, healthGood)
|
||||||
|
if g.Verdict != inventory.GatePassed || g.Faulted {
|
||||||
|
t.Fatalf("%+v; want passed at two minutes", g)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A reading that is not healthy after a healthy one takes the two minutes away: the module needs its three readings, and one
|
||||||
|
// never healthy fails at the bound and is put back; one found broken fails.
|
||||||
|
func TestANodeEngineThatFailsAReadingIsNotPassedAtTwoMinutes(t *testing.T) {
|
||||||
|
withTheGatesTimes(t)
|
||||||
|
g, judge := threeReadings(t, "mesh-host", time.Now())
|
||||||
|
judge(20*time.Second, healthNotYet)
|
||||||
|
judge(60*time.Second, healthGood)
|
||||||
|
judge(100*time.Second, healthGood)
|
||||||
|
judge(2*time.Minute+10*time.Second, healthGood)
|
||||||
|
if g.Verdict != inventory.GatePassed || g.Passes != 3 {
|
||||||
|
t.Fatalf("%+v; want passed only on its third healthy reading", g)
|
||||||
|
}
|
||||||
|
g, judge = threeReadings(t, "mesh-host", time.Now())
|
||||||
|
judge(60*time.Second, healthGood)
|
||||||
|
judge(100*time.Second, healthNotYet)
|
||||||
|
judge(2*time.Minute, healthGood)
|
||||||
|
if g.Verdict != "" || !g.Faulted {
|
||||||
|
t.Fatalf("passed at two minutes after a reading that was not healthy: %+v", g)
|
||||||
|
}
|
||||||
|
judge(10*time.Minute+time.Second, healthNotYet)
|
||||||
|
if g.Verdict != inventory.GateFailed || !strings.Contains(g.Why, "not healthy within") {
|
||||||
|
t.Fatalf("never healthy again: %+v; want failed at the bound", g)
|
||||||
|
}
|
||||||
|
g, judge = threeReadings(t, "mesh-host", time.Now())
|
||||||
|
judge(60*time.Second, healthGood)
|
||||||
|
judge(100*time.Second, healthBroken)
|
||||||
|
if g.Verdict != inventory.GateFailed {
|
||||||
|
t.Fatalf("a broken reading: %+v; want failed, to be put back", g)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The tool runner passes on one reading after its witness window; mesh-delivery on one, at once.
|
||||||
|
func TestTheToolRunnerAndMeshDeliveryPassOnOneReading(t *testing.T) {
|
||||||
|
withTheGatesTimes(t)
|
||||||
|
sent := time.Now()
|
||||||
|
o := aGateOn("node-tools", "ace", sent)
|
||||||
|
o.g.Applied["ace"] = sent.Add(10 * time.Second)
|
||||||
|
o.read(sent.Add(30*time.Second), healthGood, "")
|
||||||
|
if o.g.Verdict != "" {
|
||||||
|
t.Fatalf("the tool runner passed inside its witness window: %+v", o.g)
|
||||||
|
}
|
||||||
|
o.read(sent.Add(70*time.Second), healthGood, "")
|
||||||
|
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "the tool runner's rule") {
|
||||||
|
t.Fatalf("the tool runner: %+v; want passed on one reading", o.g)
|
||||||
|
}
|
||||||
|
|
||||||
|
o = aGateOn(catalogue.DeliverySeat, "novox", sent)
|
||||||
|
o.read(sent.Add(5*time.Second), healthGood, "")
|
||||||
|
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "mesh-delivery's rule") {
|
||||||
|
t.Fatalf("mesh-delivery: %+v; want passed on one reading", o.g)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A dip after a healthy reading taken before the controller's witness floor is a fault: the send is not passed at
|
||||||
|
// two minutes (novox/hq ADR 0352 decision 3).
|
||||||
|
func TestADipAfterAnUncountedHealthyReadingIsAFault(t *testing.T) {
|
||||||
|
withTheGatesTimes(t)
|
||||||
|
sent := time.Now()
|
||||||
|
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
|
||||||
|
modules := []string{"mesh-controller", "mesh-host"}
|
||||||
|
rules, floors := map[string]gateRule{}, map[string]time.Time{}
|
||||||
|
for _, m := range modules {
|
||||||
|
rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m})
|
||||||
|
}
|
||||||
|
floors["mesh-host"] = sent
|
||||||
|
floors["mesh-controller"] = sent.Add(90 * time.Second)
|
||||||
|
judge := func(after time.Duration, h health) {
|
||||||
|
tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": h, "mesh-host": h},
|
||||||
|
worst: h, why: "not yet", rules: rules, floors: floors}, sent.Add(after))
|
||||||
|
}
|
||||||
|
judge(30*time.Second, healthGood)
|
||||||
|
if g.Passes != 0 {
|
||||||
|
t.Fatalf("a reading before the controller's floor was counted: %+v", g)
|
||||||
|
}
|
||||||
|
judge(70*time.Second, healthNotYet)
|
||||||
|
judge(2*time.Minute, healthGood)
|
||||||
|
if !g.Faulted || g.Verdict != "" {
|
||||||
|
t.Fatalf("%+v; want the dip a fault, and no pass at two minutes", g)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A send carrying the controller and the node-engine is judged by the node-engine's three readings.
|
||||||
|
func TestASendIsJudgedByItsMostDemandingRule(t *testing.T) {
|
||||||
|
withTheGatesTimes(t)
|
||||||
|
sent := time.Now()
|
||||||
|
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
|
||||||
|
modules := []string{"mesh-controller", "mesh-host"}
|
||||||
|
rules, floors := map[string]gateRule{}, map[string]time.Time{}
|
||||||
|
for _, m := range modules {
|
||||||
|
rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m})
|
||||||
|
floors[m] = gateFloor(m, rules[m], sent, []string{"novox"}, nil, nil)
|
||||||
|
}
|
||||||
|
for _, after := range []time.Duration{70 * time.Second, 110 * time.Second} {
|
||||||
|
tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": healthGood,
|
||||||
|
"mesh-host": healthGood}, worst: healthGood, rules: rules, floors: floors}, sent.Add(after))
|
||||||
|
}
|
||||||
|
if g.Verdict != "" || g.Wants != 3 || g.Passes != 2 || g.Healthy["mesh-controller"] != 2 {
|
||||||
|
t.Fatalf("%+v; want the send judged by the node-engine's three readings, the controller's own counted", g)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -36,6 +36,10 @@ func aGateMesh(t *testing.T) *gateMesh {
|
|||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
g := &gateMesh{open: open, health: map[string]health{}}
|
g := &gateMesh{open: open, health: map[string]health{}}
|
||||||
|
// The gate's own tests judge a first machine, as walks a controller before novox/hq ADR 0360 started still are.
|
||||||
|
wasGated := gateFirstMachine
|
||||||
|
gateFirstMachine = true
|
||||||
|
t.Cleanup(func() { gateFirstMachine = wasGated })
|
||||||
g.keeper, _ = withConditionsInMemory(t)
|
g.keeper, _ = withConditionsInMemory(t)
|
||||||
g.told = &conditions.Told{}
|
g.told = &conditions.Told{}
|
||||||
was := doctorFrom
|
was := doctorFrom
|
||||||
@@ -124,10 +128,14 @@ func aGateMesh(t *testing.T) *gateMesh {
|
|||||||
}
|
}
|
||||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||||
|
|
||||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||||
|
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||||
|
wasWindow := gateWitnessWindow
|
||||||
|
gateWitnessWindow = 0
|
||||||
|
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||||
// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
|
// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
|
||||||
gateSettle, gateEvery = 0, time.Hour
|
gateWithin, gateEvery = time.Hour, time.Hour
|
||||||
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
|
||||||
|
|
||||||
built := time.Now().UTC()
|
built := time.Now().UTC()
|
||||||
plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
||||||
@@ -267,12 +275,40 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A build rolls everywhere with no hand and no gate (novox/hq ADR 0360 §1): both machines in one send, no health
|
||||||
|
// judged, and the walk done once both reported on the send. Before, anchor was judged first, then laptop sent.
|
||||||
|
func TestABuildRollsToEveryMachineAtOnceWithNoGate(t *testing.T) {
|
||||||
|
g := aGateMesh(t)
|
||||||
|
gateFirstMachine = false
|
||||||
|
ctx := t.Context()
|
||||||
|
judged := false
|
||||||
|
wasGather := gatherGateFacts
|
||||||
|
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||||
|
judged = true
|
||||||
|
return wasGather(ctx, open, component)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||||
|
for i := 0; i < 6; i++ {
|
||||||
|
advancePlans(ctx, g.open)
|
||||||
|
}
|
||||||
|
p := g.plan(t)
|
||||||
|
if !reflect.DeepEqual(g.sent, [][]string{{"anchor", "laptop"}}) {
|
||||||
|
t.Fatalf("sent %v, not both machines in one send", g.sent)
|
||||||
|
}
|
||||||
|
if s := p.Modules["app"]; p.State != inventory.PlanDone || s.Gate != nil || s.FirstAt != nil || judged {
|
||||||
|
t.Fatalf("the plan is %s (%s), app %+v, judged %v", p.State, p.Note, s, judged)
|
||||||
|
}
|
||||||
|
if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 {
|
||||||
|
t.Fatalf("a rolled build left a condition: %+v %v", obs, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the
|
// A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the
|
||||||
// core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524).
|
// core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524).
|
||||||
func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
|
func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
|
||||||
g := aGateMesh(t)
|
g := aGateMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
gateSettle = time.Hour // a core module could not pass for an hour
|
gateWithin = time.Hour // a core module could not pass on fewer readings for an hour
|
||||||
advancePlans(ctx, g.open) // the first machine is sent the new build
|
advancePlans(ctx, g.open) // the first machine is sent the new build
|
||||||
advancePlans(ctx, g.open) // one healthy judging
|
advancePlans(ctx, g.open) // one healthy judging
|
||||||
p := g.plan(t)
|
p := g.plan(t)
|
||||||
@@ -285,31 +321,6 @@ func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Which modules are judged as the core: those on the controller's own path and the resolver's holder, three
|
|
||||||
// readings after the settle time; every other module one (novox/hq ADR 0282 decision 1, ADR 0304 decision 3).
|
|
||||||
func TestAModuleIsJudgedByItsClass(t *testing.T) {
|
|
||||||
for _, c := range []struct {
|
|
||||||
module string
|
|
||||||
claims []catalogue.Claim
|
|
||||||
passes int
|
|
||||||
settles bool
|
|
||||||
}{
|
|
||||||
{"app", nil, 1, false},
|
|
||||||
{"nzbget", nil, 1, false},
|
|
||||||
{"mesh-controller", nil, 3, true},
|
|
||||||
{"mesh-host", nil, 3, true},
|
|
||||||
{"node-tools", nil, 3, true},
|
|
||||||
{"nats", nil, 3, true},
|
|
||||||
{catalogue.DeliverySeat, nil, 3, true},
|
|
||||||
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, true},
|
|
||||||
} {
|
|
||||||
p, s := gateWants(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
|
|
||||||
if p != c.passes || s != c.settles {
|
|
||||||
t.Errorf("%s: %d passes, settles %v; want %d, %v", c.module, p, s, c.passes, c.settles)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A first machine that never becomes healthy fails its gate at the bound, and is put back.
|
// A first machine that never becomes healthy fails its gate at the bound, and is put back.
|
||||||
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
|
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
|
||||||
judgeLeavesAsTheCore(t)
|
judgeLeavesAsTheCore(t)
|
||||||
|
|||||||
@@ -55,7 +55,7 @@ func TestARepairByHandCarriesItsReason(t *testing.T) {
|
|||||||
want string
|
want string
|
||||||
}{
|
}{
|
||||||
{"push", map[string]any{"node": "anchor", "why": "stuck", "cause": "sent-not-reported"},
|
{"push", map[string]any{"node": "anchor", "why": "stuck", "cause": "sent-not-reported"},
|
||||||
"push anchor --wait 0 --why stuck --cause sent-not-reported"},
|
"push anchor --wait 0s --why stuck --cause sent-not-reported"},
|
||||||
{"plans", map[string]any{"close": "plan-1", "why": "the report will not come"},
|
{"plans", map[string]any{"close": "plan-1", "why": "the report will not come"},
|
||||||
"plans close plan-1 --why the report will not come"},
|
"plans close plan-1 --why the report will not come"},
|
||||||
{"plans", map[string]any{"retry": "plan-1"}, "plans retry plan-1"},
|
{"plans", map[string]any{"retry": "plan-1"}, "plans retry plan-1"},
|
||||||
|
|||||||
@@ -151,7 +151,7 @@ func heldMachines(ctx context.Context, open *stores, names []string) (map[string
|
|||||||
for _, mv := range f.moves(node, modules, same, known, false) {
|
for _, mv := range f.moves(node, modules, same, known, false) {
|
||||||
if planStillToSend(plans, mv.Module, node) == "" {
|
if planStillToSend(plans, mv.Module, node) == "" {
|
||||||
why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+
|
why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+
|
||||||
"plan sends it, one machine at a time, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
|
"plan sends it, one machine first and then the rest at once, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(why) > 0 {
|
if len(why) > 0 {
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ import (
|
|||||||
// another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module
|
// another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module
|
||||||
// to another build supersedes the judging: no verdict, nothing put back.
|
// to another build supersedes the judging: no verdict, nothing put back.
|
||||||
func TestReplay352(t *testing.T) {
|
func TestReplay352(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
judgeLeavesAsTheCore(t)
|
judgeLeavesAsTheCore(t)
|
||||||
t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn)
|
t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn)
|
||||||
t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging)
|
t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging)
|
||||||
@@ -67,6 +68,7 @@ func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) {
|
|||||||
|
|
||||||
// A plan's own first send waits while a release judges the same module on that machine with another build.
|
// A plan's own first send waits while a release judges the same module on that machine with another build.
|
||||||
func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
|
func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
judgeLeavesAsTheCore(t)
|
judgeLeavesAsTheCore(t)
|
||||||
b := aBacklog(t)
|
b := aBacklog(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
@@ -82,6 +84,7 @@ func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
|
|||||||
|
|
||||||
// A merge plan's judging is superseded the same way: another send moved its module on the first machine.
|
// A merge plan's judging is superseded the same way: another send moved its module on the first machine.
|
||||||
func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) {
|
func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
judgeLeavesAsTheCore(t)
|
judgeLeavesAsTheCore(t)
|
||||||
g := aGateMesh(t)
|
g := aGateMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
@@ -140,6 +143,7 @@ func testASendThatMovesTheModuleSupersedesTheJudging(t *testing.T) {
|
|||||||
// A report is on the gate's own send: the declaration itself, or one sequenced after it; a gate kept
|
// A report is on the gate's own send: the declaration itself, or one sequenced after it; a gate kept
|
||||||
// without its send reads the report against the send made last, as before. Pure.
|
// without its send reads the report against the send made last, as before. Pure.
|
||||||
func TestAReportIsHeldAgainstTheGatesOwnSend(t *testing.T) {
|
func TestAReportIsHeldAgainstTheGatesOwnSend(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
sent := inventory.SentDeclaration{Digest: "d-490", Sequence: 490}
|
sent := inventory.SentDeclaration{Digest: "d-490", Sequence: 490}
|
||||||
for _, c := range []struct {
|
for _, c := range []struct {
|
||||||
r inventory.Reported
|
r inventory.Reported
|
||||||
@@ -185,6 +189,7 @@ func TestAReportIsHeldAgainstTheGatesOwnSend(t *testing.T) {
|
|||||||
// A gate judges only the build the machine was last sent: last sent another build of the module, the
|
// A gate judges only the build the machine was last sent: last sent another build of the module, the
|
||||||
// judging is superseded, whatever the machine reports. Pure.
|
// judging is superseded, whatever the machine reports. Pure.
|
||||||
func TestAGateJudgesOnlyTheBuildTheMachineWasLastSent(t *testing.T) {
|
func TestAGateJudgesOnlyTheBuildTheMachineWasLastSent(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
at := time.Now()
|
at := time.Now()
|
||||||
f := gateFacts{now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor", Outcome: inventory.OutcomeApplied,
|
f := gateFacts{now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor", Outcome: inventory.OutcomeApplied,
|
||||||
At: &at, Current: true}}, engines: map[string]string{}, served: map[string]served{}, rolledBack: map[string][]lease.Rollback{},
|
At: &at, Current: true}}, engines: map[string]string{}, served: map[string]served{}, rolledBack: map[string][]lease.Rollback{},
|
||||||
@@ -212,6 +217,7 @@ func TestAGateJudgesOnlyTheBuildTheMachineWasLastSent(t *testing.T) {
|
|||||||
// A move of another build of a module to a machine where a release or a plan is judging that module
|
// A move of another build of a module to a machine where a release or a plan is judging that module
|
||||||
// waits for that judging; the same build to that machine is already there. Pure.
|
// waits for that judging; the same build to that machine is already there. Pure.
|
||||||
func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) {
|
func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
at := time.Now()
|
at := time.Now()
|
||||||
release := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{
|
release := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{
|
||||||
Gate: &inventory.PlanGate{Machines: []string{"novox"}, Carried: []inventory.CarriedMove{
|
Gate: &inventory.PlanGate{Machines: []string{"novox"}, Carried: []inventory.CarriedMove{
|
||||||
@@ -249,6 +255,7 @@ func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) {
|
|||||||
// The controller is never put back to a build that reaches less of the store's schema than the store
|
// The controller is never put back to a build that reaches less of the store's schema than the store
|
||||||
// has, or to one that never said what it reaches: the current build is kept, and the condition says so.
|
// has, or to one that never said what it reaches: the current build is kept, and the condition says so.
|
||||||
func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T) {
|
func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
@@ -345,6 +352,7 @@ func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T)
|
|||||||
|
|
||||||
// The lease's holder names the build the declaration told it it is, and the version stamp only without one.
|
// The lease's holder names the build the declaration told it it is, and the version stamp only without one.
|
||||||
func TestTheHolderNamesTheBuildTheDeclarationToldIt(t *testing.T) {
|
func TestTheHolderNamesTheBuildTheDeclarationToldIt(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
t.Setenv(RunningBuildVar, " ad62528c47c7 ")
|
t.Setenv(RunningBuildVar, " ad62528c47c7 ")
|
||||||
if h := holderOf("x"); h.Build != "ad62528c47c7" {
|
if h := holderOf("x"); h.Build != "ad62528c47c7" {
|
||||||
t.Fatalf("the holder's build is %q", h.Build)
|
t.Fatalf("the holder's build is %q", h.Build)
|
||||||
|
|||||||
@@ -0,0 +1,88 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A tier whose module is no move — made from the source every machine running it runs, so nothing is
|
||||||
|
// sent — is not waited on for a report after its marking (novox/hq issue 648). Its machines answered the
|
||||||
|
// send that last carried that build before the walk got there, and nothing will make them report again:
|
||||||
|
// the walk asks the next tier in the same step, rather than "waiting for app on anchor, laptop" for ever.
|
||||||
|
func TestANoMoveTierWaitsForNoNewReport(t *testing.T) {
|
||||||
|
judgeLeavesAsTheCore(t)
|
||||||
|
g := aGateMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := g.open.inventory
|
||||||
|
|
||||||
|
// app holds the build seat, so a module of the next tier is built by it; since novox/hq ADR 0360 §3 that gates
|
||||||
|
// nothing.
|
||||||
|
for _, m := range []catalogue.Manifest{
|
||||||
|
{Module: "app", Version: "c2", Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}},
|
||||||
|
{Module: "web", Version: "w1"},
|
||||||
|
} {
|
||||||
|
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
||||||
|
Path: "modules/" + m.Module, BuiltFrom: m.Version, Head: m.Version, Asked: time.Now().Add(-time.Minute)}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// What anchor and laptop were last sent (c1) was made from the source the plan's build of c2 is.
|
||||||
|
for _, b := range []inventory.Build{
|
||||||
|
{ID: "build-1s", Module: "app", Commit: "c1", SourceFingerprint: "src1:app",
|
||||||
|
Asked: time.Now().Add(-30 * time.Second), At: time.Now().Add(-30 * time.Second)},
|
||||||
|
{ID: "build-2s", Module: "app", Commit: "c2", SourceFingerprint: "src1:app", Asked: time.Now(), At: time.Now()},
|
||||||
|
} {
|
||||||
|
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
||||||
|
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Each machine answered that send, applied, before the walk's tier was built.
|
||||||
|
for _, n := range []string{"anchor", "laptop"} {
|
||||||
|
if _, err := inv.RecordDoing(ctx, nodeID(t, g.open, n), inventory.Doing{Node: n,
|
||||||
|
Outcome: inventory.OutcomeApplied, Declared: "d-" + n + "-c1", Applied: 1,
|
||||||
|
At: time.Now().Add(-10 * time.Second)}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
built := time.Now().UTC()
|
||||||
|
p := g.plan(t)
|
||||||
|
p.Tiers = [][]string{{"app"}, {"web"}}
|
||||||
|
p.Modules["app"].Build, p.Modules["app"].BuiltAt = "build-2s", &built
|
||||||
|
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
advancePlans(ctx, g.open)
|
||||||
|
|
||||||
|
if len(g.sent) != 0 {
|
||||||
|
t.Fatalf("a no-move build was sent: %v", g.sent)
|
||||||
|
}
|
||||||
|
p = g.plan(t)
|
||||||
|
if s := p.Modules["app"]; s == nil || !strings.HasPrefix(s.Why, "no move") {
|
||||||
|
t.Fatalf("the build was not marked no move: %+v", s)
|
||||||
|
}
|
||||||
|
if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") {
|
||||||
|
t.Fatalf("the walk waits on a no-move tier for a report nothing will cause: tier %d, %s %q", p.Tier, p.State, p.Note)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine that has not answered its last send is not waited on either: a built-by edge gates no tier
|
||||||
|
// (novox/hq ADR 0360 §3). Before, the walk waited for app on laptop.
|
||||||
|
p.Tier, p.State, p.Note = 0, inventory.PlanRolling, ""
|
||||||
|
if err := inv.RecordSent(ctx, nodeID(t, g.open, "laptop"), "d-laptop-later", map[string]string{"app": "c1"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
advancePlans(ctx, g.open)
|
||||||
|
p = g.plan(t)
|
||||||
|
if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") {
|
||||||
|
t.Fatalf("the walk waits on a built-by edge: tier %d, %q", p.Tier, p.Note)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,188 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq ADR 0360 §4, §5: a walk ends on its machines' reports — every machine sent has reported on its send,
|
||||||
|
// or is silent past ApplySilentAfter — and names a failed or refused apply; it never waits for a health reading.
|
||||||
|
func TestAWalkEndsOnItsMachinesReports(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
|
||||||
|
sent := now.Add(-time.Minute)
|
||||||
|
long := now.Add(-inventory.ApplySilentAfter - time.Minute)
|
||||||
|
at := now.Add(-10 * time.Second)
|
||||||
|
p := inventory.Plan{Modules: map[string]*inventory.PlanModule{
|
||||||
|
"app": {Rest: map[string]inventory.SentDeclaration{
|
||||||
|
"ace": {Digest: "d-ace", At: sent},
|
||||||
|
"novox": {Digest: "d-novox", At: sent},
|
||||||
|
"shanks": {Digest: "d-shanks", At: sent},
|
||||||
|
"laptop": {Digest: "d-laptop", At: long},
|
||||||
|
}},
|
||||||
|
"proxy": {Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d-ace", At: sent}}},
|
||||||
|
}}
|
||||||
|
reports := []inventory.Reported{
|
||||||
|
{Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"},
|
||||||
|
{Node: "novox", Outcome: inventory.OutcomeFailed, At: &at, Declared: "d-novox"},
|
||||||
|
{Node: "shanks", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-before"},
|
||||||
|
}
|
||||||
|
waiting, failed, silent := walkReports(p, reports, now)
|
||||||
|
if !slices.Equal(waiting, []string{"shanks"}) || !slices.Equal(failed, []string{"novox"}) ||
|
||||||
|
!slices.Equal(silent, []string{"laptop"}) {
|
||||||
|
t.Fatalf("waiting %v, failed %v, silent %v", waiting, failed, silent)
|
||||||
|
}
|
||||||
|
reports[2].Declared = "d-shanks"
|
||||||
|
if waiting, _, _ := walkReports(p, reports, now); len(waiting) != 0 {
|
||||||
|
t.Fatalf("the walk still waits once every machine reported: %v", waiting)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// withTheGate turns the first-machine gate on for a test of its own path: kept for the walks a controller before
|
||||||
|
// novox/hq ADR 0360 started.
|
||||||
|
func withTheGate(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
was := gateFirstMachine
|
||||||
|
gateFirstMachine = true
|
||||||
|
t.Cleanup(func() { gateFirstMachine = was })
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every machine heard from is sent at once and passed on its applied report, with no judging (novox/hq ADR 0360 §7);
|
||||||
|
// before, the first machine was judged before the rest were sent (ADR 0353 §1).
|
||||||
|
func TestAReleaseSendsEveryMachineAtOnce(t *testing.T) {
|
||||||
|
judgeLeavesAsTheCore(t)
|
||||||
|
b := aBacklogOnThree(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
said := events(t)
|
||||||
|
for i := 0; i < 12; i++ {
|
||||||
|
advancePlans(ctx, b.open)
|
||||||
|
}
|
||||||
|
p := b.release(t)
|
||||||
|
if p.State != inventory.PlanDone || len(p.Release.Done) != 3 {
|
||||||
|
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||||
|
}
|
||||||
|
for _, n := range []string{"desk", "laptop", "anchor"} {
|
||||||
|
if !slices.Contains(*said, "send "+n) {
|
||||||
|
t.Fatalf("%s was not sent: %v", n, *said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if slices.Contains(*said, "judge") {
|
||||||
|
t.Fatalf("a machine was judged by its health: %v", *said)
|
||||||
|
}
|
||||||
|
if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||||
|
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine that reports its apply failed is named and fails the release, with nothing put back, and every other
|
||||||
|
// machine goes on to its own report (novox/hq ADR 0360 §5, §7); before, it was put back (ADR 0353 §3).
|
||||||
|
func TestAMachineThatFailsItsApplyIsNamedAndNothingIsPutBack(t *testing.T) {
|
||||||
|
judgeLeavesAsTheCore(t)
|
||||||
|
b := aBacklogOnThree(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := b.open.inventory
|
||||||
|
b.broken["desk"] = true // desk fails what it is sent
|
||||||
|
advancePlans(ctx, b.open)
|
||||||
|
for i := 0; i < 12 && b.release(t).Open(); i++ {
|
||||||
|
advancePlans(ctx, b.open)
|
||||||
|
}
|
||||||
|
p := b.release(t)
|
||||||
|
if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) {
|
||||||
|
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||||
|
}
|
||||||
|
if !slices.Contains(p.Release.Done, "laptop") || !slices.Contains(p.Release.Done, "anchor") {
|
||||||
|
t.Fatalf("the others have no verdict of their own: %+v", p.Release)
|
||||||
|
}
|
||||||
|
if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||||
|
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
||||||
|
}
|
||||||
|
if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c2" {
|
||||||
|
t.Fatalf("desk was put back: %v", sent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A release's send kept before it was made, and made, is judged by the controller that takes over without being
|
||||||
|
// made again (ADR 0353 §8), with every machine sent at once (ADR 0360 §7).
|
||||||
|
func TestAReleaseSendKeptAndMadeIsNotMadeAgainAtOnce(t *testing.T) {
|
||||||
|
judgeLeavesAsTheCore(t)
|
||||||
|
b := aBacklog(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := b.open.inventory
|
||||||
|
p, err := releaseBacklog(ctx, b.open, "")
|
||||||
|
if err != nil || p == nil {
|
||||||
|
t.Fatalf("no release opened: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gates["anchor"] == nil || !p.Release.Gates["anchor"].Pending {
|
||||||
|
t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err)
|
||||||
|
}
|
||||||
|
if err := inv.SavePlan(ctx, p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// The send is made, and the controller is replaced before it keeps it.
|
||||||
|
if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
advancePlans(ctx, b.open)
|
||||||
|
if n := len(slices.DeleteFunc(slices.Clone(b.sent), func(s []string) bool { return !slices.Equal(s, []string{"anchor"}) })); n != 1 {
|
||||||
|
t.Fatalf("anchor was sent again: %v", b.sent)
|
||||||
|
}
|
||||||
|
g := b.release(t).Release.Gates["anchor"]
|
||||||
|
if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 {
|
||||||
|
t.Fatalf("anchor is not judged on the send made: %+v", g)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put
|
||||||
|
// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4); its step also sent anchor, so the release is done, not failed (ADR 0360 §7).
|
||||||
|
func TestAMachineSilentPastItsBoundIsLeftAndSaidAtOnce(t *testing.T) {
|
||||||
|
judgeLeavesAsTheCore(t)
|
||||||
|
b := aBacklog(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := b.open.inventory
|
||||||
|
reporting := sendRollout
|
||||||
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||||
|
if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing
|
||||||
|
b.sent = append(b.sent, names)
|
||||||
|
current, _ := open.inventory.CurrentBuilds(ctx)
|
||||||
|
return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep",
|
||||||
|
map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit})
|
||||||
|
}
|
||||||
|
return reporting(ctx, open, names)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { sendRollout = reporting })
|
||||||
|
for i := 0; i < 8; i++ {
|
||||||
|
advancePlans(ctx, b.open)
|
||||||
|
}
|
||||||
|
p := b.release(t)
|
||||||
|
g := p.Release.Gates["laptop"]
|
||||||
|
if !p.Open() || g == nil || g.Verdict != "" || g.Silent {
|
||||||
|
t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g)
|
||||||
|
}
|
||||||
|
since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute)
|
||||||
|
g.Since = &since
|
||||||
|
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
advancePlans(ctx, b.open)
|
||||||
|
p = b.release(t)
|
||||||
|
// anchor applied in the same step, so the release is done, laptop named silent (novox/hq ADR 0360 §7).
|
||||||
|
if p.State != inventory.PlanDone || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 {
|
||||||
|
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||||
|
}
|
||||||
|
if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed {
|
||||||
|
t.Fatal("a silent machine's build was marked failed")
|
||||||
|
}
|
||||||
|
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
|
||||||
|
t.Fatalf("laptop was put back: %v", sent)
|
||||||
|
}
|
||||||
|
obs, err := silentObservation(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) {
|
||||||
|
t.Fatalf("the silence is not said: %+v", obs)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
|
)
|
||||||
|
|
||||||
|
// localNetworksOf is a machine's attached networks judged (novox/hq ADR 0340): which count as its local
|
||||||
|
// network, from what it last said of itself and the links it said face outside it. A machine that has
|
||||||
|
// said none judges to nothing, and a port reached from `local-network` is then admitted from the private
|
||||||
|
// network only, and said so.
|
||||||
|
func localNetworksOf(ctx context.Context, inv *inventory.Inventory, node string) ([]catalogue.LocalNetwork, error) {
|
||||||
|
account, had, err := inv.MachineOf(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
outward, err := inv.OutwardLinksOf(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return judgedNetworks(account, had, outward, overlay.Interface), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgedNetworks is the judging alone, from what was read.
|
||||||
|
func judgedNetworks(account inventory.MachineAccount, had bool, outward []string, tunnel string) []catalogue.LocalNetwork {
|
||||||
|
if !had || account.Machine == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
attached := make([]catalogue.AttachedNetwork, 0, len(account.Machine.Networks))
|
||||||
|
for _, n := range account.Machine.Networks {
|
||||||
|
attached = append(attached, catalogue.AttachedNetwork{Link: n.Link, Network: n.Network})
|
||||||
|
}
|
||||||
|
return catalogue.JudgeLocalNetworks(attached, outward, tunnel)
|
||||||
|
}
|
||||||
|
|
||||||
|
// localNetworkLines is what `node` shows of a machine's attached networks (ADR 0340 §8): each one, with
|
||||||
|
// whether it counts as the local network and why, or why none is shown.
|
||||||
|
func localNetworkLines(account inventory.MachineAccount, had bool, judged []catalogue.LocalNetwork) []string {
|
||||||
|
out := []string{" its local network (what a port reached from local-network is admitted from):"}
|
||||||
|
switch {
|
||||||
|
case !had || account.Machine == nil:
|
||||||
|
return append(out, " not said — this machine has not reported what it is attached to")
|
||||||
|
case len(account.Machine.Networks) == 0:
|
||||||
|
why := account.Machine.Unknown["networks"]
|
||||||
|
if why == "" {
|
||||||
|
why = "its node-engine is older than the one that says it"
|
||||||
|
}
|
||||||
|
return append(out, " not said — "+why)
|
||||||
|
}
|
||||||
|
counted := 0
|
||||||
|
for _, j := range judged {
|
||||||
|
verdict := "not counted"
|
||||||
|
if j.Counts {
|
||||||
|
verdict = "counts"
|
||||||
|
counted++
|
||||||
|
}
|
||||||
|
out = append(out, fmt.Sprintf(" %-22s on %-10s %s: %s", j.Network, j.Link, verdict, j.Why))
|
||||||
|
}
|
||||||
|
if counted == 0 {
|
||||||
|
out = append(out, " none counts: a port reached from local-network is admitted from the private network only")
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **node shows what the local network resolved to** (novox/hq ADR 0340 §8): every attached network,
|
||||||
|
// whether it counts, and why the others do not — the public subnet named as never counted.
|
||||||
|
func TestNodeShowsWhichAttachedNetworksCountAndWhy(t *testing.T) {
|
||||||
|
account := inventory.MachineAccount{Architecture: "amd64", Kernel: "linux", Machine: &inventory.MadeOf{
|
||||||
|
Networks: []inventory.AttachedNetwork{
|
||||||
|
{Link: "eth0", Network: "192.168.1.10/24"},
|
||||||
|
{Link: "eth1", Network: "203.0.113.7/24"},
|
||||||
|
{Link: "wg-mesh", Network: "10.10.0.4/24"},
|
||||||
|
}}}
|
||||||
|
judged := judgedNetworks(account, true, []string{"eth0", "eth1"}, "wg-mesh")
|
||||||
|
text := strings.Join(localNetworkLines(account, true, judged), "\n")
|
||||||
|
for _, want := range []string{
|
||||||
|
"192.168.1.0/24", "counts: in a private range (RFC 1918, 192.168.0.0/16)",
|
||||||
|
"203.0.113.0/24", "not counted: public, never counted",
|
||||||
|
"10.10.0.0/24", "the private network's own link",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(text, want) {
|
||||||
|
t.Errorf("node does not say %q:\n%s", want, text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine that said none, or whose node-engine is older, is said as that — never as an empty list
|
||||||
|
// that reads like a machine on no network.
|
||||||
|
func TestNodeSaysWhenNoNetworkWasSaid(t *testing.T) {
|
||||||
|
older := inventory.MachineAccount{Machine: &inventory.MadeOf{}}
|
||||||
|
if text := strings.Join(localNetworkLines(older, true, nil), "\n"); !strings.Contains(text, "older than the one that says it") {
|
||||||
|
t.Errorf("an older node-engine is not said:\n%s", text)
|
||||||
|
}
|
||||||
|
public := inventory.MachineAccount{Machine: &inventory.MadeOf{
|
||||||
|
Networks: []inventory.AttachedNetwork{{Link: "eth0", Network: "198.51.100.9/24"}}}}
|
||||||
|
judged := judgedNetworks(public, true, []string{"eth0"}, "wg-mesh")
|
||||||
|
if text := strings.Join(localNetworkLines(public, true, judged), "\n"); !strings.Contains(text, "none counts") {
|
||||||
|
t.Errorf("a machine on a public network only is not said to have no local network:\n%s", text)
|
||||||
|
}
|
||||||
|
if text := strings.Join(localNetworkLines(inventory.MachineAccount{}, false, nil), "\n"); !strings.Contains(text, "not said") {
|
||||||
|
t.Errorf("a machine that never reported is not said:\n%s", text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The preview of a flip says what a port from the local network keeps: the local networks it stays
|
||||||
|
// open to, or that the machine reported none (ADR 0340 §6).
|
||||||
|
func TestTheFlipPreviewSaysWhatALocalNetworkPortKeeps(t *testing.T) {
|
||||||
|
rules := []catalogue.Rule{{Port: 1883, Protocol: "tcp", From: catalogue.FromLocalNetwork, Because: []string{"mosquitto"}}}
|
||||||
|
at := inventory.Reach{Protocol: "tcp", Address: "0.0.0.0", Port: 1883}
|
||||||
|
with := derivedFilter{rules: rules, mesh: []string{"10.10.0.1"}, outwardLinks: []string{"eth0"},
|
||||||
|
local: []string{"192.168.1.0/24"}}
|
||||||
|
if fate := with.fate(at); !strings.Contains(fate, "192.168.1.0/24") {
|
||||||
|
t.Errorf("the preview says %q, without the local network it stays open to", fate)
|
||||||
|
}
|
||||||
|
with.local = nil
|
||||||
|
if fate := with.fate(at); !strings.Contains(fate, "reported none") {
|
||||||
|
t.Errorf("the preview says %q, without saying no local network was reported", fate)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -104,6 +104,10 @@ func run() error {
|
|||||||
return replayCommand(ctx, args[1:])
|
return replayCommand(ctx, args[1:])
|
||||||
case "kill":
|
case "kill":
|
||||||
return killCommand(ctx, args[1:])
|
return killCommand(ctx, args[1:])
|
||||||
|
case "forget-checks":
|
||||||
|
return forgetChecksCommand(ctx, args[1:])
|
||||||
|
case "try":
|
||||||
|
return tryCommand(ctx, args[1:])
|
||||||
case "pause", "resume":
|
case "pause", "resume":
|
||||||
return pauseCommand(ctx, args[0], args[1:])
|
return pauseCommand(ctx, args[0], args[1:])
|
||||||
case "collection":
|
case "collection":
|
||||||
@@ -379,6 +383,13 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
|||||||
// **A merge check builds nothing** (novox/hq to-be 45 §9): its verdict is said, and nothing of it is
|
// **A merge check builds nothing** (novox/hq to-be 45 §9): its verdict is said, and nothing of it is
|
||||||
// recorded or registered.
|
// recorded or registered.
|
||||||
if result.Check != nil || result.Checked != nil {
|
if result.Check != nil || result.Checked != nil {
|
||||||
|
if isTry(result) {
|
||||||
|
// A branch tried before its pull request (novox/hq issue 686): its asker reads the outcome from the
|
||||||
|
// build's log. Never said as `checked`, so the forge's holder sets no status.
|
||||||
|
fmt.Printf("%s: a try of %s at %.8s, read with `try --id`: said to nobody, nothing taken in\n", result.ID,
|
||||||
|
result.Repository, result.Ref)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
checked(ctx, result)
|
checked(ctx, result)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -429,8 +440,14 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
|||||||
}
|
}
|
||||||
fmt.Printf("%s: %s %s registered, built on %s from %s\n",
|
fmt.Printf("%s: %s %s registered, built on %s from %s\n",
|
||||||
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
|
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
|
||||||
|
// What the walk's build did, kept on the walk whose ask it answers (novox/hq ADR 0350 §4); a build seat from
|
||||||
|
// before it says nothing, and that is what is kept.
|
||||||
|
made := result.MadeWords()
|
||||||
|
if result.Reused != "" || result.BuiltWhy != "" {
|
||||||
|
fmt.Printf("%s: %s %s\n", result.ID, manifest.Module, made)
|
||||||
|
}
|
||||||
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
|
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
|
||||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
planBuiltSaying(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID, made)
|
||||||
// A module registered may be one a machine is now behind: `status` is composed again.
|
// A module registered may be one a machine is now behind: `status` is composed again.
|
||||||
statusFrom.nudge()
|
statusFrom.nudge()
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -86,7 +86,7 @@ func owedToTheRecord(ctx context.Context, announced merges, m link.SourceMoved,
|
|||||||
|
|
||||||
// catchingUpOnMerges reads back the forge's announcements on a timer, until the context ends.
|
// catchingUpOnMerges reads back the forge's announcements on a timer, until the context ends.
|
||||||
func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
||||||
f := following{open}
|
f := following{open: open}
|
||||||
catalogued := func(ctx context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
catalogued := func(ctx context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
||||||
entries, err := open.inventory.Catalogued(ctx)
|
entries, err := open.inventory.Catalogued(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -75,7 +75,7 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
|
|||||||
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
|
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
|
||||||
Check: r.Check, Needs: r.Needs, Account: r.Account, Root: r.Root}
|
Check: r.Check, Needs: r.Needs, Account: r.Account, Root: r.Root}
|
||||||
for _, w := range r.Waits {
|
for _, w := range r.Waits {
|
||||||
kept.Waits = append(kept.Waits, inventory.Wait{Part: w.Part, Secret: w.Secret, Setting: w.Setting, What: w.What})
|
kept.Waits = append(kept.Waits, inventory.Wait{Part: w.Part, Secret: w.Secret, Setting: w.Setting, Provision: w.Provision, What: w.What})
|
||||||
}
|
}
|
||||||
resources = append(resources, kept)
|
resources = append(resources, kept)
|
||||||
}
|
}
|
||||||
@@ -240,6 +240,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
||||||
|
declaredSeverity(&o, hold, m)
|
||||||
if hold != nil {
|
if hold != nil {
|
||||||
if by, held := hold.heldWith(node, m, unhealthy[m]); held {
|
if by, held := hold.heldWith(node, m, unhealthy[m]); held {
|
||||||
// Held under the provider's condition: nothing of its own, and the provider's says it waits. The
|
// Held under the provider's condition: nothing of its own, and the provider's says it waits. The
|
||||||
@@ -347,6 +348,7 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata
|
|||||||
} else {
|
} else {
|
||||||
_, rest := besideAWait(p.Module, rs)
|
_, rest := besideAWait(p.Module, rs)
|
||||||
o = moduleUnhealthyObservation(p.Module, p.Node, rest)
|
o = moduleUnhealthyObservation(p.Module, p.Node, rest)
|
||||||
|
declaredSeverity(&o, hold, p.Module)
|
||||||
if said, waits := personWait(p.Module, p.Node, rest); waits {
|
if said, waits := personWait(p.Module, p.Node, rest); waits {
|
||||||
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rest)
|
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rest)
|
||||||
}
|
}
|
||||||
@@ -481,6 +483,19 @@ func sayWaitingOn(o *conditions.Observation, waiters []string) {
|
|||||||
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
|
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// declaredSeverity raises a module's unhealthy condition urgent when its manifest declares `unhealthy` urgent
|
||||||
|
// (novox/hq ADR 0355 §1). Only that condition changes (ADR 0355 §4): a wait for a new login, a module that needs
|
||||||
|
// the operator and a directory used as found keep their severities, and without the field, or without a manifest
|
||||||
|
// to read it from, it stays the warning it was.
|
||||||
|
func declaredSeverity(o *conditions.Observation, hold *holding, module string) {
|
||||||
|
if o.Kind != kindModuleUnhealthy || hold == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if m, ok := hold.manifestOf(module); ok && m.UnhealthyIsUrgent() {
|
||||||
|
o.Severity = conditions.Urgent
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
|
// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
|
||||||
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
|
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
|
||||||
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
|
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
|
||||||
|
|||||||
@@ -684,7 +684,7 @@ func artifactStoreOnNetwork(ctx context.Context, inv *inventory.Inventory,
|
|||||||
if !offers {
|
if !offers {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
serves, err := servedOnNode(ctx, inv, machine, m, catalogue.ArtifactStoreProvision)
|
serves, _, err := servedOnNode(ctx, inv, machine, m, catalogue.ArtifactStoreProvision)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", "", false, fmt.Errorf("reading where %s puts the artifact store: %w", machine, err)
|
return "", "", false, fmt.Errorf("reading where %s puts the artifact store: %w", machine, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ import (
|
|||||||
"github.com/novox/mesh-controller/internal/conditions"
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
"github.com/novox/mesh-controller/internal/token"
|
"github.com/novox/mesh-controller/internal/token"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -884,6 +885,16 @@ func showNode(ctx context.Context, stored *stores, name string) error {
|
|||||||
for _, line := range machineLines(account, had, time.Now()) {
|
for _, line := range machineLines(account, had, time.Now()) {
|
||||||
fmt.Println(line)
|
fmt.Println(line)
|
||||||
}
|
}
|
||||||
|
// And which of the networks it is attached to count as its local network, and why the others
|
||||||
|
// do not (novox/hq ADR 0340 §8).
|
||||||
|
outward, err := inv.OutwardLinksOf(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf(" its local network could NOT be judged: %v\n", err)
|
||||||
|
} else {
|
||||||
|
for _, line := range localNetworkLines(account, had, judgedNetworks(account, had, outward, overlay.Interface)) {
|
||||||
|
fmt.Println(line)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
held, err := inv.Profile(ctx, name)
|
held, err := inv.Profile(ctx, name)
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -44,6 +45,28 @@ func needsOperatorKey(module, node string) string {
|
|||||||
type operatorWaitFacts struct {
|
type operatorWaitFacts struct {
|
||||||
manifests map[string]catalogue.Manifest
|
manifests map[string]catalogue.Manifest
|
||||||
given map[string]map[string]time.Time
|
given map[string]map[string]time.Time
|
||||||
|
// outside is, per provision some module offers with a credential issued outside the mesh, that provider module
|
||||||
|
// and the machines it is assigned to (novox/hq ADR 0337); pairsGiven, per "<module>@<machine>", the provisions
|
||||||
|
// it holds a value a person gave for, sealed to the keys both ends have now. Absent is not known: no wait for a
|
||||||
|
// provision of it is excused.
|
||||||
|
outside map[string]outsideProvider
|
||||||
|
pairsGiven map[string]map[string]bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// outsideProvider is the module offering a provision whose credential only a person gives, and where it runs.
|
||||||
|
type outsideProvider struct {
|
||||||
|
Module string
|
||||||
|
Machines []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// acceptLine is the one line that gives an outside provision's credential for every consumer of its provider: on
|
||||||
|
// the provider's machine when it runs on one, named as <machine> when on several or none.
|
||||||
|
func (p outsideProvider) acceptLine(provision string) string {
|
||||||
|
machine := "<its machine>"
|
||||||
|
if len(p.Machines) == 1 {
|
||||||
|
machine = p.Machines[0]
|
||||||
|
}
|
||||||
|
return inventory.AcceptLine(machine, p.Module, provision)
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkWait is nil when a wait is excused, and otherwise why not, in words. Pure.
|
// checkWait is nil when a wait is excused, and otherwise why not, in words. Pure.
|
||||||
@@ -52,9 +75,18 @@ func checkWait(module, machine string, w inventory.Wait, f operatorWaitFacts) er
|
|||||||
if !known {
|
if !known {
|
||||||
return fmt.Errorf("says it waits for %s, and the mesh holds no manifest of %s to check it against", waitNames(w), module)
|
return fmt.Errorf("says it waits for %s, and the mesh holds no manifest of %s to check it against", waitNames(w), module)
|
||||||
}
|
}
|
||||||
|
named := 0
|
||||||
|
for _, n := range []string{w.Secret, w.Setting, w.Provision} {
|
||||||
|
if n != "" {
|
||||||
|
named++
|
||||||
|
}
|
||||||
|
}
|
||||||
switch {
|
switch {
|
||||||
case w.Secret != "" && w.Setting != "", w.Secret == "" && w.Setting == "":
|
case named != 1:
|
||||||
return fmt.Errorf("says it waits, naming %s, where a wait names one secret or one setting", waitNames(w))
|
return fmt.Errorf("says it waits, naming %s, where a wait names one secret, one setting or one provision",
|
||||||
|
waitNames(w))
|
||||||
|
case w.Provision != "":
|
||||||
|
return checkProvisionWait(module, machine, w, m, f)
|
||||||
case w.Setting != "":
|
case w.Setting != "":
|
||||||
if _, declared := m.Settings[w.Setting]; !declared {
|
if _, declared := m.Settings[w.Setting]; !declared {
|
||||||
return fmt.Errorf("says it waits for the setting %s, which %s does not declare", w.Setting, module)
|
return fmt.Errorf("says it waits for the setting %s, which %s does not declare", w.Setting, module)
|
||||||
@@ -80,20 +112,46 @@ func checkWait(module, machine string, w inventory.Wait, f operatorWaitFacts) er
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// checkProvisionWait excuses a wait for a provision's credential only when the store bears it out (novox/hq ADR 0337
|
||||||
|
// §6): the module requires the provision, a provider offers it with a credential issued outside the mesh, and the
|
||||||
|
// module on this machine holds no value a person gave for it under the keys both ends have now.
|
||||||
|
func checkProvisionWait(module, machine string, w inventory.Wait, m catalogue.Manifest, f operatorWaitFacts) error {
|
||||||
|
if !slices.Contains(m.Requires, w.Provision) {
|
||||||
|
return fmt.Errorf("says it waits for the provision %s, which %s does not require", w.Provision, module)
|
||||||
|
}
|
||||||
|
if _, outside := f.outside[w.Provision]; !outside {
|
||||||
|
return fmt.Errorf("says it waits for the provision %s, whose credential the mesh gives itself: only a "+
|
||||||
|
"credential issued outside the mesh waits for the operator", w.Provision)
|
||||||
|
}
|
||||||
|
given, readable := f.pairsGiven[module+"@"+machine]
|
||||||
|
if !readable {
|
||||||
|
return fmt.Errorf("says it waits for the provision %s, and what was given on %s could not be read", w.Provision, machine)
|
||||||
|
}
|
||||||
|
if given[w.Provision] {
|
||||||
|
return fmt.Errorf("says it waits for the provision %s, whose credential was given", w.Provision)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// waitRefusedPrefix opens every reason checkWait gives, so the words of a refused wait are told from a check's own.
|
// waitRefusedPrefix opens every reason checkWait gives, so the words of a refused wait are told from a check's own.
|
||||||
const waitRefusedPrefix = "says it waits"
|
const waitRefusedPrefix = "says it waits"
|
||||||
|
|
||||||
// waitNames is what a wait names, as "the secret x" or "the setting y".
|
// waitNames is what a wait names, as "the secret x" or "the setting y".
|
||||||
func waitNames(w inventory.Wait) string {
|
func waitNames(w inventory.Wait) string {
|
||||||
switch {
|
var names []string
|
||||||
case w.Secret != "" && w.Setting != "":
|
if w.Secret != "" {
|
||||||
return "the secret " + w.Secret + " and the setting " + w.Setting
|
names = append(names, "the secret "+w.Secret)
|
||||||
case w.Secret != "":
|
|
||||||
return "the secret " + w.Secret
|
|
||||||
case w.Setting != "":
|
|
||||||
return "the setting " + w.Setting
|
|
||||||
}
|
}
|
||||||
return "nothing"
|
if w.Setting != "" {
|
||||||
|
names = append(names, "the setting "+w.Setting)
|
||||||
|
}
|
||||||
|
if w.Provision != "" {
|
||||||
|
names = append(names, "the provision "+w.Provision)
|
||||||
|
}
|
||||||
|
if len(names) == 0 {
|
||||||
|
return "nothing"
|
||||||
|
}
|
||||||
|
return strings.Join(names, " and ")
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkWaiting reads one machine's resources against the facts: every waiting resource whose waits all check out is
|
// checkWaiting reads one machine's resources against the facts: every waiting resource whose waits all check out is
|
||||||
@@ -107,11 +165,17 @@ func checkWaiting(machine string, rs []inventory.ResourceHealth, f operatorWaitF
|
|||||||
if len(r.Waits) == 0 {
|
if len(r.Waits) == 0 {
|
||||||
why = fmt.Errorf("says it waits, and names nothing it waits for")
|
why = fmt.Errorf("says it waits, and names nothing it waits for")
|
||||||
}
|
}
|
||||||
|
waits := make([]inventory.Wait, 0, len(r.Waits))
|
||||||
for _, w := range r.Waits {
|
for _, w := range r.Waits {
|
||||||
if why == nil {
|
if why == nil {
|
||||||
why = checkWait(r.Module, machine, w, f)
|
why = checkWait(r.Module, machine, w, f)
|
||||||
}
|
}
|
||||||
|
if p, outside := f.outside[w.Provision]; outside && w.Provision != "" {
|
||||||
|
w.Act = p.acceptLine(w.Provision)
|
||||||
|
}
|
||||||
|
waits = append(waits, w)
|
||||||
}
|
}
|
||||||
|
r.Waits = waits
|
||||||
if why != nil {
|
if why != nil {
|
||||||
r.State, r.Reason = link.StateUnhealthy, why.Error()
|
r.State, r.Reason = link.StateUnhealthy, why.Error()
|
||||||
}
|
}
|
||||||
@@ -150,6 +214,10 @@ func operatorWaitSaid(module, machine string, waits []inventory.Wait) string {
|
|||||||
if w.Secret != "" {
|
if w.Secret != "" {
|
||||||
part += fmt.Sprintf(", given with `nox secret ask %s %s %s`", machine, module, w.Secret)
|
part += fmt.Sprintf(", given with `nox secret ask %s %s %s`", machine, module, w.Secret)
|
||||||
}
|
}
|
||||||
|
if w.Provision != "" && w.Act != "" {
|
||||||
|
// One line for the provider, which seals it to every consumer of it (ADR 0337 §3, §7).
|
||||||
|
part += fmt.Sprintf(", given for every consumer with `%s`", w.Act)
|
||||||
|
}
|
||||||
parts = append(parts, part+")")
|
parts = append(parts, part+")")
|
||||||
}
|
}
|
||||||
return fmt.Sprintf("%s on %s waits for the operator: %s", module, machine, strings.Join(parts, "; "))
|
return fmt.Sprintf("%s on %s waits for the operator: %s", module, machine, strings.Join(parts, "; "))
|
||||||
@@ -172,7 +240,7 @@ func needsOperatorObservation(module, node string, waits []inventory.Wait, rs []
|
|||||||
func needsOperatorWords(module, node string, waits []inventory.Wait) words {
|
func needsOperatorWords(module, node string, waits []inventory.Wait) words {
|
||||||
var acts []string
|
var acts []string
|
||||||
seen := map[string]bool{}
|
seen := map[string]bool{}
|
||||||
secret := false
|
secret, provision := false, false
|
||||||
for _, w := range waits {
|
for _, w := range waits {
|
||||||
var act string
|
var act string
|
||||||
switch {
|
switch {
|
||||||
@@ -180,6 +248,9 @@ func needsOperatorWords(module, node string, waits []inventory.Wait) words {
|
|||||||
act, secret = fmt.Sprintf("type %s at %s's desk prompt", w.What, node), true
|
act, secret = fmt.Sprintf("type %s at %s's desk prompt", w.What, node), true
|
||||||
case w.Setting != "":
|
case w.Setting != "":
|
||||||
act = fmt.Sprintf("approve %s of %s on %s when it is proposed to you", w.Setting, module, node)
|
act = fmt.Sprintf("approve %s of %s on %s when it is proposed to you", w.Setting, module, node)
|
||||||
|
case w.Provision != "":
|
||||||
|
// ADR 0337 §7: "give <what> of <provider> on <machine>"; the line is in the summary.
|
||||||
|
act, provision = "give "+w.What, true
|
||||||
}
|
}
|
||||||
if act != "" && !seen[act] {
|
if act != "" && !seen[act] {
|
||||||
seen[act] = true
|
seen[act] = true
|
||||||
@@ -197,6 +268,10 @@ func needsOperatorWords(module, node string, waits []inventory.Wait) words {
|
|||||||
explanation += " A hidden prompt opens at the desk when the secret is asked for, and what you type there " +
|
explanation += " A hidden prompt opens at the desk when the secret is asked for, and what you type there " +
|
||||||
"is sealed to the machine."
|
"is sealed to the machine."
|
||||||
}
|
}
|
||||||
|
if provision {
|
||||||
|
explanation += " A key only you can give is given once for the app that issued it, and reaches every " +
|
||||||
|
"module that uses it."
|
||||||
|
}
|
||||||
explanation += " Its update is in place and nothing was undone; it carries on by itself once it is given."
|
explanation += " Its update is in place and nothing was undone; it carries on by itself once it is given."
|
||||||
return words{
|
return words{
|
||||||
Headline: fmt.Sprintf("%s waits for you on %s", module, node),
|
Headline: fmt.Sprintf("%s waits for you on %s", module, node),
|
||||||
@@ -217,6 +292,12 @@ func readWaitFacts(ctx context.Context, inv *inventory.Inventory, machine string
|
|||||||
if f.given == nil {
|
if f.given == nil {
|
||||||
f.given = map[string]map[string]time.Time{}
|
f.given = map[string]map[string]time.Time{}
|
||||||
}
|
}
|
||||||
|
if f.outside == nil {
|
||||||
|
f.outside = map[string]outsideProvider{}
|
||||||
|
}
|
||||||
|
if f.pairsGiven == nil {
|
||||||
|
f.pairsGiven = map[string]map[string]bool{}
|
||||||
|
}
|
||||||
if inv == nil {
|
if inv == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -238,15 +319,59 @@ func readWaitFacts(ctx context.Context, inv *inventory.Inventory, machine string
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if _, read := f.given[module+"@"+machine]; read {
|
if _, read := f.given[module+"@"+machine]; !read {
|
||||||
|
if given, err := inv.GivenOwnSecrets(ctx, machine, module); err == nil {
|
||||||
|
f.given[module+"@"+machine] = given
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, read := f.pairsGiven[module+"@"+machine]; !read {
|
||||||
|
if given, err := inv.GivenPairsOf(ctx, machine, module); err == nil {
|
||||||
|
f.pairsGiven[module+"@"+machine] = given
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The provisions it requires whose credential only a person gives, and where their provider runs (ADR 0337).
|
||||||
|
m, known := f.manifests[module]
|
||||||
|
if !known {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if given, err := inv.GivenOwnSecrets(ctx, machine, module); err == nil {
|
for _, provision := range m.Requires {
|
||||||
f.given[module+"@"+machine] = given
|
if _, read := f.outside[provision]; read {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if shelf == nil {
|
||||||
|
var err error
|
||||||
|
if shelf, err = inv.Catalogue(ctx); err != nil {
|
||||||
|
shelf = map[string]catalogue.Manifest{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
provider, outside := catalogue.OffersIssuedOutside(shelf, provision)
|
||||||
|
if !outside {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
f.outside[provision] = outsideProvider{Module: provider, Machines: machinesRunning(ctx, inv, provider)}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// machinesRunning is every machine a module is assigned to, by name; a read that fails is left out.
|
||||||
|
func machinesRunning(ctx context.Context, inv *inventory.Inventory, module string) []string {
|
||||||
|
nodes, err := inv.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
// Only the summary's accept line reads it, which then names the provider's machine "<its machine>".
|
||||||
|
// empty-on-error: no wait is excused or refused by it
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for _, n := range nodes {
|
||||||
|
assigned, err := inv.Assigned(ctx, n.Name)
|
||||||
|
if err == nil && slices.Contains(assigned, module) {
|
||||||
|
out = append(out, n.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// waitingModules is every module with a waiting resource in a statement.
|
// waitingModules is every module with a waiting resource in a statement.
|
||||||
func waitingModules(rs []inventory.ResourceHealth) []string {
|
func waitingModules(rs []inventory.ResourceHealth) []string {
|
||||||
seen := map[string]bool{}
|
seen := map[string]bool{}
|
||||||
|
|||||||
@@ -55,8 +55,8 @@ func TestAWaitIsExcusedOnlyWhenItChecksOut(t *testing.T) {
|
|||||||
{"a secret the mesh makes", inventory.Wait{Part: "p", Secret: "made", What: "w"}, factsGiven(nil), "mesh makes itself"},
|
{"a secret the mesh makes", inventory.Wait{Part: "p", Secret: "made", What: "w"}, factsGiven(nil), "mesh makes itself"},
|
||||||
{"the bus account", inventory.Wait{Part: "p", Secret: "broker", What: "w"}, factsGiven(nil), "mesh makes itself"},
|
{"the bus account", inventory.Wait{Part: "p", Secret: "broker", What: "w"}, factsGiven(nil), "mesh makes itself"},
|
||||||
{"a setting not declared", inventory.Wait{Part: "p", Setting: "logins", What: "w"}, factsGiven(nil), "does not declare"},
|
{"a setting not declared", inventory.Wait{Part: "p", Setting: "logins", What: "w"}, factsGiven(nil), "does not declare"},
|
||||||
{"both", inventory.Wait{Part: "p", Secret: "licence", Setting: "smb-users", What: "w"}, factsGiven(nil), "one secret or one setting"},
|
{"both", inventory.Wait{Part: "p", Secret: "licence", Setting: "smb-users", What: "w"}, factsGiven(nil), "one secret, one setting or one provision"},
|
||||||
{"neither", inventory.Wait{Part: "p", What: "w"}, factsGiven(nil), "one secret or one setting"},
|
{"neither", inventory.Wait{Part: "p", What: "w"}, factsGiven(nil), "one secret, one setting or one provision"},
|
||||||
{"no manifest known", passwordWait, operatorWaitFacts{}, "no manifest"},
|
{"no manifest known", passwordWait, operatorWaitFacts{}, "no manifest"},
|
||||||
{"what was given cannot be read", passwordWait,
|
{"what was given cannot be read", passwordWait,
|
||||||
operatorWaitFacts{manifests: map[string]catalogue.Manifest{"mounts": mountsManifest}}, "could not be read"},
|
operatorWaitFacts{manifests: map[string]catalogue.Manifest{"mounts": mountsManifest}}, "could not be read"},
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// One accept for a provider reaches every consumer that binds the provision from it, and each consumer's send
|
||||||
|
// places it; the per-pair form naming a consumer is refused for such a provision, naming the provider's form; a
|
||||||
|
// value the mesh made before the rule is not placed (novox/hq ADR 0337 §2, §3, §4, issue 164, issue 165). Needs the
|
||||||
|
// store (MESH_TEST_POSTGRES), and is named in testdata/store-tests.
|
||||||
|
func TestOneAcceptForAProviderIsPlacedForEveryConsumer(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
// Before the rule: tv's offer said nothing of its credential, and the send made one per pair.
|
||||||
|
register(t, open, catalogue.Manifest{Module: "tv", Version: "1",
|
||||||
|
Provides: []catalogue.Offer{{Name: "tv-api", Scope: catalogue.ScopeMesh}}})
|
||||||
|
for _, m := range []string{"requests", "indexer"} {
|
||||||
|
register(t, open, catalogue.Manifest{Module: m, Version: "1", Requires: []string{"tv-api"},
|
||||||
|
Secrets: map[string]string{"tv-api": "/var/lib/mesh/" + m + "/tv-api.secret"}})
|
||||||
|
}
|
||||||
|
for _, a := range [][2]string{{"anchor", "tv"}, {"laptop", "requests"}, {"laptop", "indexer"}} {
|
||||||
|
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
placed := func(module string) bool {
|
||||||
|
t.Helper()
|
||||||
|
plan, settings, err := planFor(ctx, open, "laptop")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("laptop was refused: %v", err)
|
||||||
|
}
|
||||||
|
declared, err := declarationFor(ctx, open, "laptop", plan, settings)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("laptop did not compose: %v", err)
|
||||||
|
}
|
||||||
|
for _, r := range declared.Resources {
|
||||||
|
if r["path"] == "/var/lib/mesh/"+module+"/tv-api.secret" && r["sealed"] != nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
if !placed("requests") || !placed("indexer") {
|
||||||
|
t.Fatal("before the rule, the send made and placed a credential per pair")
|
||||||
|
}
|
||||||
|
// The offer now says only a person gives it: the values the mesh made are not placed.
|
||||||
|
register(t, open, catalogue.Manifest{Module: "tv", Version: "2",
|
||||||
|
Provides: []catalogue.Offer{{Name: "tv-api", Scope: catalogue.ScopeMesh,
|
||||||
|
Credential: &catalogue.OfferCredential{IssuedBy: catalogue.IssuedOutside}}}})
|
||||||
|
if placed("requests") || placed("indexer") {
|
||||||
|
t.Fatal("a credential nobody gave was placed")
|
||||||
|
}
|
||||||
|
|
||||||
|
// The per-pair form is refused, naming the provider's form, and reads nothing.
|
||||||
|
file := filepath.Join(t.TempDir(), "key")
|
||||||
|
if err := os.WriteFile(file, []byte("the app's key\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
err := secretCommand(ctx, []string{"accept", "laptop", "requests", "tv-api", "--provider", "anchor", "--from", file})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "secret accept anchor tv tv-api") {
|
||||||
|
t.Fatalf("the per-pair form was not refused naming the provider's: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
consumers, unresolved, err := consumersFrom(ctx, open, "anchor", "tv", "tv-api")
|
||||||
|
if err != nil || len(unresolved) != 0 {
|
||||||
|
t.Fatalf("the consumers of tv on anchor could not be found: %v %v", err, unresolved)
|
||||||
|
}
|
||||||
|
if len(consumers) != 2 {
|
||||||
|
t.Fatalf("the consumers of tv on anchor are %+v; want requests and indexer on laptop", consumers)
|
||||||
|
}
|
||||||
|
if _, err := open.inventory.AcceptSecretForProvision(ctx, "tv-api", "anchor", "tv", "the app's key", consumers); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !placed("requests") || !placed("indexer") {
|
||||||
|
t.Fatal("the value given once for the provider was not placed for every consumer")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,113 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A consumer may wait for a provision's credential only a person gives (novox/hq ADR 0337 §5–§7, issue 164).
|
||||||
|
|
||||||
|
var requestsManifest = catalogue.Manifest{Module: "requests", Version: "1", Requires: []string{"tv-api", "db"},
|
||||||
|
Secrets: map[string]string{"tv-api": "/s/tv-api", "db": "/s/db"}}
|
||||||
|
|
||||||
|
var tvKeyWait = inventory.Wait{Part: "the tv library", Provision: "tv-api", What: "the API key of tv"}
|
||||||
|
|
||||||
|
func outsideFacts(given map[string]bool) operatorWaitFacts {
|
||||||
|
return operatorWaitFacts{manifests: map[string]catalogue.Manifest{"requests": requestsManifest},
|
||||||
|
outside: map[string]outsideProvider{"tv-api": {Module: "tv", Machines: []string{"media"}}},
|
||||||
|
pairsGiven: map[string]map[string]bool{"requests@laptop": given}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// §6: excused only when the module requires the provision, its provider says it is issued outside, and no value a
|
||||||
|
// person gave is held for the pair under the keys both ends have now.
|
||||||
|
func TestAWaitForAProvisionIsExcusedOnlyWhenTheStoreBearsItOut(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
w inventory.Wait
|
||||||
|
f operatorWaitFacts
|
||||||
|
says string
|
||||||
|
}{
|
||||||
|
{"required, issued outside, not given", tvKeyWait, outsideFacts(nil), ""},
|
||||||
|
{"given", tvKeyWait, outsideFacts(map[string]bool{"tv-api": true}), "was given"},
|
||||||
|
{"not required", inventory.Wait{Part: "p", Provision: "music-api", What: "w"}, outsideFacts(nil), "does not require"},
|
||||||
|
{"a credential the mesh gives itself", inventory.Wait{Part: "p", Provision: "db", What: "w"}, outsideFacts(nil),
|
||||||
|
"mesh gives itself"},
|
||||||
|
{"what was given cannot be read", tvKeyWait, operatorWaitFacts{
|
||||||
|
manifests: map[string]catalogue.Manifest{"requests": requestsManifest},
|
||||||
|
outside: map[string]outsideProvider{"tv-api": {Module: "tv"}}}, "could not be read"},
|
||||||
|
{"a provision and a secret", inventory.Wait{Part: "p", Provision: "tv-api", Secret: "x", What: "w"},
|
||||||
|
outsideFacts(nil), "one secret, one setting or one provision"},
|
||||||
|
} {
|
||||||
|
err := checkWait("requests", "laptop", c.w, c.f)
|
||||||
|
switch {
|
||||||
|
case c.says == "" && err != nil:
|
||||||
|
t.Errorf("%s: refused: %v", c.name, err)
|
||||||
|
case c.says != "" && (err == nil || !strings.Contains(err.Error(), c.says)):
|
||||||
|
t.Errorf("%s: %v; want a refusal saying %q", c.name, err, c.says)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// §7: an excused wait is the consumer's needs-operator condition, in plain words, its summary naming the one line
|
||||||
|
// that gives the credential for every consumer of the provider; the wait as kept is not changed.
|
||||||
|
func TestAWaitForAProvisionNamesTheOneAcceptForItsProvider(t *testing.T) {
|
||||||
|
r := inventory.ResourceHealth{Module: "requests", Resource: "requests.app", Kind: "container", Target: "requests",
|
||||||
|
State: link.StateWaiting, Check: "tool", Waits: []inventory.Wait{tvKeyWait}}
|
||||||
|
kept := []inventory.ResourceHealth{r}
|
||||||
|
got := checkWaiting("laptop", kept, outsideFacts(nil))
|
||||||
|
if got[0].State != link.StateWaiting {
|
||||||
|
t.Fatalf("an excused wait for a provision was judged: %+v", got[0])
|
||||||
|
}
|
||||||
|
if kept[0].Waits[0].Act != "" {
|
||||||
|
t.Fatal("checking the wait changed the statement as kept")
|
||||||
|
}
|
||||||
|
waits, waiting := operatorWait("requests", got)
|
||||||
|
if !waiting {
|
||||||
|
t.Fatal("the module does not read as waiting for the operator")
|
||||||
|
}
|
||||||
|
o := needsOperatorObservation("requests", "laptop", waits, got)
|
||||||
|
if !strings.Contains(o.Summary, "secret accept media tv tv-api") {
|
||||||
|
t.Fatalf("the summary does not name the accept for the provider: %q", o.Summary)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(o.Needs, "give the API key of tv") {
|
||||||
|
t.Fatalf("the needs do not name the act: %q", o.Needs)
|
||||||
|
}
|
||||||
|
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation,
|
||||||
|
Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions}, "laptop"); !ok {
|
||||||
|
t.Fatalf("the condition is not in plain words: %s", why)
|
||||||
|
}
|
||||||
|
// A provider on several machines, or none: the line names its machine as a place to fill.
|
||||||
|
if line := (outsideProvider{Module: "tv", Machines: []string{"a", "b"}}).acceptLine("tv-api"); line !=
|
||||||
|
"secret accept <its machine> tv tv-api" {
|
||||||
|
t.Fatalf("a provider on two machines reads %q", line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// §3: the consumers one accept reaches are those whose resolution answers the provision from that provider.
|
||||||
|
func TestTheConsumersOfAProviderAreThoseItAnswers(t *testing.T) {
|
||||||
|
plan := catalogue.Resolution{Needs: []catalogue.Needed{
|
||||||
|
{Name: "tv-api", From: "media", Module: "tv", For: "requests"},
|
||||||
|
{Name: "tv-api", From: "media", Module: "tv", For: "indexer", Local: "main"},
|
||||||
|
{Name: "tv-api", From: "other", Module: "tv", For: "elsewhere"},
|
||||||
|
{Name: "tv-api", From: "media", Module: "another-tv", For: "not-this-module"},
|
||||||
|
{Name: "db", From: "media", Module: "tv", For: "requests"},
|
||||||
|
{Name: "tv-api", ByRecord: true, For: "by-record"},
|
||||||
|
}}
|
||||||
|
got := bindsFrom(plan, "laptop", "media", "tv", "tv-api")
|
||||||
|
if len(got) != 2 || got[0].ConsumerModule != "indexer" || got[0].Local != "main" || got[1].ConsumerModule != "requests" {
|
||||||
|
t.Fatalf("the consumers from tv on media are %+v", got)
|
||||||
|
}
|
||||||
|
said := acceptedSaid("media", "tv", "tv-api", inventory.ProvisionAccepted{
|
||||||
|
Sealed: []inventory.PairEnd{{Consumer: "laptop", ConsumerModule: "requests"}},
|
||||||
|
Unsealable: []inventory.PairEnd{{Consumer: "new", ConsumerModule: "indexer"}}}, []string{"down: refused"})
|
||||||
|
for _, want := range []string{"sealed to requests on laptop", "no sealing key", "down: refused"} {
|
||||||
|
if !strings.Contains(said, want) {
|
||||||
|
t.Errorf("what the accept says lacks %q:\n%s", want, said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -30,6 +30,7 @@ func userUnit(module, account string) inventory.ResourceHealth {
|
|||||||
// Only what depends on the new login alone is a wait; anything else unhealthy of the module is judged as
|
// Only what depends on the new login alone is a wait; anything else unhealthy of the module is judged as
|
||||||
// before, so that a fault is never excused.
|
// before, so that a fault is never excused.
|
||||||
func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) {
|
func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
notInGroup := relogin("lights", "operator")
|
notInGroup := relogin("lights", "operator")
|
||||||
notInGroup.Reason = "not in the group lights: the apply has not put operator there; its outcome says why"
|
notInGroup.Reason = "not in the group lights: the apply has not put operator there; its outcome says why"
|
||||||
systemUnit := userUnit("lights", "operator")
|
systemUnit := userUnit("lights", "operator")
|
||||||
@@ -74,6 +75,7 @@ func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) {
|
|||||||
// The gate reads a wait for a person as its own reading, never a fault, and passes with it; the same module
|
// The gate reads a wait for a person as its own reading, never a fault, and passes with it; the same module
|
||||||
// with a container down beside it is not yet healthy, as before.
|
// with a container down beside it is not yet healthy, as before.
|
||||||
func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) {
|
func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
since := now.Add(-time.Minute)
|
since := now.Add(-time.Minute)
|
||||||
account := relogin("lights", "operator")
|
account := relogin("lights", "operator")
|
||||||
@@ -102,6 +104,7 @@ func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) {
|
|||||||
// The module's condition says the wait in one sentence for a person, and clears on the first statement that
|
// The module's condition says the wait in one sentence for a person, and clears on the first statement that
|
||||||
// no longer says it — said as the module's own fault when its unit still fails after the new login.
|
// no longer says it — said as the module's own fault when its unit still fails after the new login.
|
||||||
func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) {
|
func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
k, _ := withConditionsInMemory(t)
|
k, _ := withConditionsInMemory(t)
|
||||||
@@ -158,6 +161,7 @@ func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) {
|
|||||||
// A module that is healthy on its own keeps its pass when another module of the same send fails at the
|
// A module that is healthy on its own keeps its pass when another module of the same send fails at the
|
||||||
// bound: it is neither put back nor left without a verdict for every other walk to wait on.
|
// bound: it is neither put back nor left without a verdict for every other walk to wait on.
|
||||||
func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) {
|
func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
b := aBacklog(t)
|
b := aBacklog(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := b.open.inventory
|
inv := b.open.inventory
|
||||||
@@ -207,6 +211,7 @@ func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) {
|
|||||||
// What the operator reads of the wait (ADR 0253, ADR 0254): it needs them, so it is never quiet, and it offers
|
// What the operator reads of the wait (ADR 0253, ADR 0254): it needs them, so it is never quiet, and it offers
|
||||||
// no button, since nothing but their own new login can do it.
|
// no button, since nothing but their own new login can do it.
|
||||||
func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) {
|
func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{
|
o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{
|
||||||
relogin("openrazer", "operator"), userUnit("openrazer", "operator")})
|
relogin("openrazer", "operator"), userUnit("openrazer", "operator")})
|
||||||
plainExample(t, o, "openrazer waits for a new login on g14",
|
plainExample(t, o, "openrazer waits for a new login on g14",
|
||||||
@@ -238,6 +243,7 @@ func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) {
|
|||||||
// When a module not working turns into one waiting for a new login, and back, the clearing line says what it
|
// When a module not working turns into one waiting for a new login, and back, the clearing line says what it
|
||||||
// became, never that it works again (issue 318 review).
|
// became, never that it works again (issue 318 review).
|
||||||
func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) {
|
func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
store := conditions.NewInMemory()
|
store := conditions.NewInMemory()
|
||||||
@@ -286,6 +292,7 @@ func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) {
|
|||||||
|
|
||||||
// Which moves add an account group, read from the builds' manifests (issue 318 review).
|
// Which moves add an account group, read from the builds' manifests (issue 318 review).
|
||||||
func TestOnlyAMoveThatAddsAnAccountGroupCanBringAWait(t *testing.T) {
|
func TestOnlyAMoveThatAddsAnAccountGroupCanBringAWait(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
user := func(groups ...any) catalogue.Manifest {
|
user := func(groups ...any) catalogue.Manifest {
|
||||||
return catalogue.Manifest{Module: "lights", Resources: []map[string]any{{"id": "operator", "type": "user",
|
return catalogue.Manifest{Module: "lights", Resources: []map[string]any{{"id": "operator", "type": "user",
|
||||||
"name": "operator", "groups": groups}}}
|
"name": "operator", "groups": groups}}}
|
||||||
|
|||||||
@@ -456,6 +456,9 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
|||||||
"release-held": worded(func(o conditions.Observation) words {
|
"release-held": worded(func(o conditions.Observation) words {
|
||||||
return releaseHeldWords(nil, o.Also)
|
return releaseHeldWords(nil, o.Also)
|
||||||
}),
|
}),
|
||||||
|
"release-silent": worded(func(o conditions.Observation) words {
|
||||||
|
return releaseSilentWords(o.Also)
|
||||||
|
}),
|
||||||
"facts-stale": worded(func(o conditions.Observation) words {
|
"facts-stale": worded(func(o conditions.Observation) words {
|
||||||
return words{Headline: "Merge checks use outdated facts",
|
return words{Headline: "Merge checks use outdated facts",
|
||||||
Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " +
|
Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " +
|
||||||
@@ -617,6 +620,18 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
|||||||
Explanation: "The bus reports a listener too slow to keep up, so messages to it are late.",
|
Explanation: "The bus reports a listener too slow to keep up, so messages to it are late.",
|
||||||
Resolved: "The listener keeps up again"}
|
Resolved: "The listener keeps up again"}
|
||||||
}),
|
}),
|
||||||
|
kindBusFaultsUnread: worded(func(o conditions.Observation) words {
|
||||||
|
return words{Headline: "Slow clients of the bus cannot be checked",
|
||||||
|
Explanation: "The controller could not read what the bus said about its clients' connections, so a " +
|
||||||
|
"client the bus cut off would go unnoticed. It tries again at every check.",
|
||||||
|
Resolved: "Resolved: the bus's clients are checked again"}
|
||||||
|
}),
|
||||||
|
"connection-closed": worded(func(o conditions.Observation) words {
|
||||||
|
return words{Headline: "The bus cut a client off",
|
||||||
|
Explanation: "The bus closed a client's connection because it broke the bus's rules, so what it was " +
|
||||||
|
"sending was lost.",
|
||||||
|
Resolved: "Resolved: the client keeps to the bus's rules"}
|
||||||
|
}),
|
||||||
kindBusReconnects: worded(func(o conditions.Observation) words {
|
kindBusReconnects: worded(func(o conditions.Observation) words {
|
||||||
return words{Headline: "A client keeps losing the bus",
|
return words{Headline: "A client keeps losing the bus",
|
||||||
Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. " +
|
Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. " +
|
||||||
@@ -841,6 +856,19 @@ func releaseHeldWords(modules, machines []string) words {
|
|||||||
Resolved: "Resolved: the held updates are released"}
|
Resolved: "Resolved: the held updates are released"}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// releaseSilentWords are the plain words of machines that said nothing on an update they were sent (ADR 0353 §4).
|
||||||
|
func releaseSilentWords(machines []string) words {
|
||||||
|
where := "Some machines"
|
||||||
|
if len(machines) > 0 {
|
||||||
|
where = conditions.Capital(namesWords(machines, 4))
|
||||||
|
}
|
||||||
|
return words{Headline: "Machines did not answer an update",
|
||||||
|
Needs: "check that they are awake and on the mesh; the next walk sends them what waits.",
|
||||||
|
Explanation: where + " were sent module updates and said nothing about them within fifteen minutes, so " +
|
||||||
|
"they were left as they were. What they run is not known to be the update.",
|
||||||
|
Resolved: "Resolved: the next walk reached them"}
|
||||||
|
}
|
||||||
|
|
||||||
// reloginNeeds is what an account waiting for its groups needs (ADR 0252).
|
// reloginNeeds is what an account waiting for its groups needs (ADR 0252).
|
||||||
func reloginNeeds(node string) string {
|
func reloginNeeds(node string) string {
|
||||||
return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node)
|
return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node)
|
||||||
|
|||||||
+114
-10
@@ -182,12 +182,21 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
|
|||||||
}
|
}
|
||||||
var secret inventory.Secret
|
var secret inventory.Secret
|
||||||
var err error
|
var err error
|
||||||
if n.SharedOwn != "" {
|
if outsideModule, outside := issuedOutsideBy(shelf, n); outside {
|
||||||
|
// Only a person can give it (novox/hq ADR 0337, issue 164): read as given, never made.
|
||||||
|
secret, err = inv.GivenPairSecret(ctx, n.Name, nodeName, n.For, n.From, outsideModule, n.Local)
|
||||||
|
} else if n.SharedOwn != "" {
|
||||||
// The provider's one credential, sealed to this consumer too (novox/hq ADR 0158).
|
// The provider's one credential, sealed to this consumer too (novox/hq ADR 0158).
|
||||||
secret, err = inv.SharedSecretFor(ctx, n.Name, nodeName, n.For, n.From, providerModuleOf(resolved, open, ctx, n), n.Local, n.SharedOwn)
|
secret, err = inv.SharedSecretFor(ctx, n.Name, nodeName, n.For, n.From, providerModuleOf(resolved, open, ctx, n), n.Local, n.SharedOwn)
|
||||||
} else {
|
} else {
|
||||||
secret, err = inv.SecretFor(ctx, n.Name, nodeName, n.For, n.From, n.Local)
|
secret, err = inv.SecretFor(ctx, n.Name, nodeName, n.For, n.From, n.Local)
|
||||||
}
|
}
|
||||||
|
if inventory.IsNotGiven(err) {
|
||||||
|
// **Not given, and the rest of the consumer runs** (ADR 0337 §4): no credential file, so the module can
|
||||||
|
// tell it was not given and says it waits for it; the machine is not refused for what only a person can
|
||||||
|
// give, and nothing the mesh made stands in for it.
|
||||||
|
continue
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Said rather than skipped. A machine that resolves cleanly and receives no
|
// Said rather than skipped. A machine that resolves cleanly and receives no
|
||||||
// credential is one that will fail to authenticate at some later, less obvious
|
// credential is one that will fail to authenticate at some later, less obvious
|
||||||
@@ -280,9 +289,14 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
caps, _ := inv.ProfileOf(ctx, n.Name)
|
caps, _ := inv.ProfileOf(ctx, n.Name)
|
||||||
|
// Its public domain too (novox/hq ADR 0346): a provider elsewhere serves its own composed names,
|
||||||
|
// and composes them as its own plan does, under its own domain.
|
||||||
|
// Read as its profile is: a machine whose domain cannot be read (removed since the list was read)
|
||||||
|
// composes no public name, which leaves the key out and says so, and fails no other machine's plan.
|
||||||
|
domain, _ := inv.PublicDomainOf(ctx, n.Name)
|
||||||
others = append(others, candidate{
|
others = append(others, candidate{
|
||||||
catalogue.Node{Name: n.Name, Site: siteOf[n.Name], Capabilities: caps,
|
catalogue.Node{Name: n.Name, Site: siteOf[n.Name], Capabilities: caps,
|
||||||
At: onNetwork[n.Name]}, theirs})
|
At: onNetwork[n.Name], PublicDomain: domain}, theirs})
|
||||||
}
|
}
|
||||||
|
|
||||||
offered := map[string][]catalogue.Provider{}
|
offered := map[string][]catalogue.Provider{}
|
||||||
@@ -301,12 +315,18 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
|||||||
// What that module says a consumer needs to know, with that node's settings on
|
// What that module says a consumer needs to know, with that node's settings on
|
||||||
// it: a port somebody moved on the provider is a port its consumers must be told
|
// it: a port somebody moved on the provider is a port its consumers must be told
|
||||||
// about, and the two coming from different places is how they come to disagree.
|
// about, and the two coming from different places is how they come to disagree.
|
||||||
serves, err := servedOnNode(ctx, inv, o.node.Name, m, name)
|
serves, layers, err := servedOnNode(ctx, inv, o.node.Name, m, name)
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.World{}, err
|
||||||
|
}
|
||||||
|
// And its own composed names, as that node composes them (novox/hq ADR 0346): a name it
|
||||||
|
// does not compose leaves the key out, said under each consumer's need.
|
||||||
|
serves, leftOut, err := got.ServedOwnNames(m, name, serves, catalogue.WithDefaults(m, layers))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return catalogue.World{}, err
|
return catalogue.World{}, err
|
||||||
}
|
}
|
||||||
offered[name] = append(offered[name], catalogue.Provider{
|
offered[name] = append(offered[name], catalogue.Provider{
|
||||||
Node: o.node.Name, At: o.node.At, Serves: serves, Module: m.Module})
|
Node: o.node.Name, At: o.node.At, Serves: serves, Module: m.Module, LeftOut: leftOut})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -442,6 +462,12 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return sendable{}, err
|
return sendable{}, err
|
||||||
}
|
}
|
||||||
|
// Which ports the filter admits from the local network, and from which networks, for plan to say
|
||||||
|
// (novox/hq ADR 0340 §8). The same rules the composition rendered, never sent.
|
||||||
|
rules, err := plan.Rules(with)
|
||||||
|
if err != nil {
|
||||||
|
return sendable{}, err
|
||||||
|
}
|
||||||
// And what was taken on it, said in every declaration it is sent from this one place.
|
// And what was taken on it, said in every declaration it is sent from this one place.
|
||||||
adoption, err := adoptionOf(ctx, open.inventory, record, plan, composed)
|
adoption, err := adoptionOf(ctx, open.inventory, record, plan, composed)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -451,7 +477,8 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
|||||||
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
||||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld,
|
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld,
|
||||||
unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced,
|
unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced,
|
||||||
modules: namedModules(plan, composed.LeftOut)}
|
modules: namedModules(plan, composed.LeftOut),
|
||||||
|
localNetwork: catalogue.LocalNetworkPorts(rules, with.LocalNetworks)}
|
||||||
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
||||||
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
||||||
if choosing == Allocating {
|
if choosing == Allocating {
|
||||||
@@ -960,6 +987,12 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return catalogue.Rendering{}, inventory.Node{}, err
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
}
|
}
|
||||||
|
// And which of the networks it is attached to count as its local network (novox/hq ADR 0340):
|
||||||
|
// private ranges only, on a link facing outside it. Reported by the machine, never set.
|
||||||
|
local, err := localNetworksOf(ctx, inv, node)
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
|
}
|
||||||
// Where this machine reaches each mesh seat's holder, for ${seat:<seat>:reach} (novox/hq ADR
|
// Where this machine reaches each mesh seat's holder, for ${seat:<seat>:reach} (novox/hq ADR
|
||||||
// 0222): the artifact store as this network reaches it, which the container runtime is told to
|
// 0222): the artifact store as this network reaches it, which the container runtime is told to
|
||||||
// trust (ADR 0082). The same address composed into every reference the mesh built.
|
// trust (ADR 0082). The same address composed into every reference the mesh built.
|
||||||
@@ -975,16 +1008,22 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return catalogue.Rendering{}, inventory.Node{}, err
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
}
|
}
|
||||||
|
readsBusy, err := engineReadsBusy(ctx, inv, node)
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
|
}
|
||||||
return catalogue.Rendering{
|
return catalogue.Rendering{
|
||||||
ReadsHealth: readsHealth,
|
ReadsHealth: readsHealth,
|
||||||
JudgesRoot: judgesRoot,
|
JudgesRoot: judgesRoot,
|
||||||
|
ReadsBusy: readsBusy,
|
||||||
BusMembership: memberships[node],
|
BusMembership: memberships[node],
|
||||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports,
|
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports,
|
||||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||||
Machines: machines, Zones: zones, Holders: replicas,
|
Machines: machines, Zones: zones, Holders: replicas,
|
||||||
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
|
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
|
||||||
Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
|
Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
|
||||||
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built,
|
LocalNetworks: catalogue.CountedLocalNetworks(local),
|
||||||
|
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built,
|
||||||
BusUsers: busUsers, Withheld: withheld, Unbound: unbound,
|
BusUsers: busUsers, Withheld: withheld, Unbound: unbound,
|
||||||
}, record, nil
|
}, record, nil
|
||||||
}
|
}
|
||||||
@@ -1010,6 +1049,35 @@ func engineJudgesRoot(ctx context.Context, inv *inventory.Inventory, node string
|
|||||||
return had && stated.Contract >= link.RootContract, nil
|
return had && stated.Contract >= link.RootContract, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// engineReadsBusy says whether a machine's node-engine reads a container's `busy` (novox/hq ADR 0335), by its
|
||||||
|
// own newest statement, for the same reason as engineReadsHealth: an older engine parses strictly.
|
||||||
|
func engineReadsBusy(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) {
|
||||||
|
stated, had, err := inv.HealthOf(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return had && stated.Contract >= link.BusyContract, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// busyNotSent is, for a node-engine that does not read a container's busy, each container whose module declares
|
||||||
|
// one: not sent it, so replaced at once while it may hold work (novox/hq ADR 0335). Nothing for one that reads it.
|
||||||
|
func busyNotSent(modules []catalogue.Manifest, reads bool) []string {
|
||||||
|
if reads {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for _, m := range modules {
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
if _, has := r[catalogue.BusyField]; has {
|
||||||
|
out = append(out, fmt.Sprintf("%s.%v declares busy, and this node-engine does not read it yet (its "+
|
||||||
|
"statement's contract is below %d): not sent, so the container is replaced at once, busy or not "+
|
||||||
|
"(novox/hq ADR 0335)", m.Module, r["id"], link.BusyContract))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR
|
// zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR
|
||||||
// 0199): the zone settled from that node's settings, the node's private address, the port the
|
// 0199): the zone settled from that node's settings, the node's private address, the port the
|
||||||
// answering listen is published on there.
|
// answering listen is published on there.
|
||||||
@@ -1403,10 +1471,29 @@ func planCommand(ctx context.Context, args []string) error {
|
|||||||
// in this output would have told anybody that.
|
// in this output would have told anybody that.
|
||||||
for _, n := range plan.Needs {
|
for _, n := range plan.Needs {
|
||||||
fmt.Printf(" needs %s from %s, for %s\n", n.Name, n.From, n.For)
|
fmt.Printf(" needs %s from %s, for %s\n", n.Name, n.From, n.For)
|
||||||
|
// And a served key the provider left out, because its machine composes no name it asks for
|
||||||
|
// (novox/hq ADR 0346 §3).
|
||||||
|
for _, line := range n.LeftOut {
|
||||||
|
fmt.Printf(" %s\n", line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// What the providers on this machine leave out of what they serve, and why: said where the remedy
|
||||||
|
// is (novox/hq ADR 0346 §3).
|
||||||
|
for _, line := range plan.ServedLeftOut(settings) {
|
||||||
|
fmt.Printf("%s: %s\n", args[0], line)
|
||||||
}
|
}
|
||||||
for _, line := range unboundLines(plan) {
|
for _, line := range unboundLines(plan) {
|
||||||
fmt.Println(line)
|
fmt.Println(line)
|
||||||
}
|
}
|
||||||
|
// And a container's busy that this machine's node-engine is too old to be sent (novox/hq ADR 0335): its
|
||||||
|
// container is replaced at once, as before. A statement that cannot be read is said as such.
|
||||||
|
if reads, err := engineReadsBusy(ctx, open.inventory, args[0]); err != nil {
|
||||||
|
fmt.Printf("%s: whether its node-engine reads a container's busy could not be read: %v\n", args[0], err)
|
||||||
|
} else {
|
||||||
|
for _, line := range busyNotSent(plan.Modules, reads) {
|
||||||
|
fmt.Printf("%s: %s\n", args[0], line)
|
||||||
|
}
|
||||||
|
}
|
||||||
declared, err := declarationFor(ctx, open, args[0], plan, settings)
|
declared, err := declarationFor(ctx, open, args[0], plan, settings)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -1417,6 +1504,11 @@ func planCommand(ctx context.Context, args []string) error {
|
|||||||
fmt.Printf(" %-20s settings from %s\n", module, layer.From)
|
fmt.Printf(" %-20s settings from %s\n", module, layer.From)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// What each port reached from the local network resolved to on this machine (novox/hq ADR 0340 §8),
|
||||||
|
// or that it resolved to none.
|
||||||
|
for _, p := range declared.localNetwork {
|
||||||
|
fmt.Println(" " + p.Said())
|
||||||
|
}
|
||||||
fmt.Printf("\n%d resource(s)\n", len(resources))
|
fmt.Printf("\n%d resource(s)\n", len(resources))
|
||||||
|
|
||||||
if *show {
|
if *show {
|
||||||
@@ -1538,19 +1630,20 @@ func managerPublicKeyFor(
|
|||||||
// artifact store's trust and the references composed through it (04-ISSUES/102). Unreadable
|
// artifact store's trust and the references composed through it (04-ISSUES/102). Unreadable
|
||||||
// given ports are that node's refusal to report, not this reader's.
|
// given ports are that node's refusal to report, not this reader's.
|
||||||
func servedOnNode(ctx context.Context, inv *inventory.Inventory, node string,
|
func servedOnNode(ctx context.Context, inv *inventory.Inventory, node string,
|
||||||
m catalogue.Manifest, provision string) (map[string]any, error) {
|
m catalogue.Manifest, provision string) (map[string]any, []catalogue.Layer, error) {
|
||||||
ports, layers, err := portsGivenOn(ctx, inv, node, m)
|
ports, layers, err := portsGivenOn(ctx, inv, node, m)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
serves := catalogue.ServedOn(m, provision, ports)
|
serves := catalogue.ServedOn(m, provision, ports)
|
||||||
if len(serves) > 0 {
|
if len(serves) > 0 {
|
||||||
serves, err = catalogue.Settle(serves, catalogue.WithDefaults(m, layers))
|
serves, err = catalogue.Settle(serves, catalogue.WithDefaults(m, layers))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return serves, nil
|
// The layers too, read once, for what is filled after (novox/hq ADR 0346).
|
||||||
|
return serves, layers, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// portsGivenOn is where a node puts a module's ports — assigned, then given over them — and the
|
// portsGivenOn is where a node puts a module's ports — assigned, then given over them — and the
|
||||||
@@ -1742,6 +1835,17 @@ func busUserList(ctx context.Context, inv *inventory.Inventory,
|
|||||||
|
|
||||||
// providerModuleOf is which module answers a need on the providing node: the one in this node's
|
// providerModuleOf is which module answers a need on the providing node: the one in this node's
|
||||||
// own set when the provider is here, else the one the catalogue says offers it.
|
// own set when the provider is here, else the one the catalogue says offers it.
|
||||||
|
// issuedOutsideBy is whether a need's provision is offered by its provider with a credential issued outside the
|
||||||
|
// mesh (novox/hq ADR 0337), and that provider module: the one answering it when known, else any that offers it so.
|
||||||
|
func issuedOutsideBy(shelf map[string]catalogue.Manifest, n catalogue.Needed) (string, bool) {
|
||||||
|
if n.Module != "" {
|
||||||
|
if m, known := shelf[n.Module]; known {
|
||||||
|
return n.Module, m.IssuedOutsideOf(n.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return catalogue.OffersIssuedOutside(shelf, n.Name)
|
||||||
|
}
|
||||||
|
|
||||||
func providerModuleOf(resolved catalogue.Resolution, open *stores, ctx context.Context, n catalogue.Needed) string {
|
func providerModuleOf(resolved catalogue.Resolution, open *stores, ctx context.Context, n catalogue.Needed) string {
|
||||||
for _, m := range resolved.Modules {
|
for _, m := range resolved.Modules {
|
||||||
if _, shared := m.SharedCredentialOf(n.Name); shared {
|
if _, shared := m.SharedCredentialOf(n.Name); shared {
|
||||||
|
|||||||
@@ -133,7 +133,7 @@ func TestASharedRepositoryIsPlannedFromTheRecordedBuildSources(t *testing.T) {
|
|||||||
{"the route proxy's program: the route proxy alone", "mesh-controller",
|
{"the route proxy's program: the route proxy alone", "mesh-controller",
|
||||||
[]string{"examples/route-proxy/main.go"}, "route-proxy", true},
|
[]string{"examples/route-proxy/main.go"}, "route-proxy", true},
|
||||||
{"a package all three build from: all three", "mesh-controller",
|
{"a package all three build from: all three", "mesh-controller",
|
||||||
[]string{"internal/broker/broker.go"}, "mesh-controller | build-agent | route-proxy", false},
|
[]string{"internal/broker/broker.go"}, "mesh-controller,route-proxy | build-agent", false},
|
||||||
{"the route proxy's directory in the catalogue: it alone", "mesh-catalog",
|
{"the route proxy's directory in the catalogue: it alone", "mesh-catalog",
|
||||||
[]string{"modules/route-proxy/module.json"}, "route-proxy", false},
|
[]string{"modules/route-proxy/module.json"}, "route-proxy", false},
|
||||||
{"the build agent's directory: it alone, nothing it builds", "mesh-catalog",
|
{"the build agent's directory: it alone, nothing it builds", "mesh-catalog",
|
||||||
|
|||||||
@@ -145,8 +145,9 @@ func TestAPlanIsWhatTheChangeTouchedAndWhatIsBuiltOnIt(t *testing.T) {
|
|||||||
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a,x"},
|
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a,x"},
|
||||||
{what: "built-by never widens", edges: []inventory.Edge{dep("x", builtBy, "a")},
|
{what: "built-by never widens", edges: []inventory.Edge{dep("x", builtBy, "a")},
|
||||||
repo: "one", paths: []string{"modules/a/x"}, want: "a"},
|
repo: "one", paths: []string{"modules/a/x"}, want: "a"},
|
||||||
{what: "built-by orders", edges: []inventory.Edge{dep("x", builtBy, "a")},
|
// Whichever builder runs builds it (novox/hq ADR 0360 §3): built-by orders nothing, as before it ordered.
|
||||||
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a | x"},
|
{what: "built-by does not order", edges: []inventory.Edge{dep("x", builtBy, "a")},
|
||||||
|
repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a,x"},
|
||||||
{what: "worker-of never widens", edges: []inventory.Edge{dep("x", workerOf, "a")},
|
{what: "worker-of never widens", edges: []inventory.Edge{dep("x", workerOf, "a")},
|
||||||
repo: "one", paths: []string{"modules/a/x"}, want: "a"},
|
repo: "one", paths: []string{"modules/a/x"}, want: "a"},
|
||||||
{what: "worker-of orders", edges: []inventory.Edge{dep("x", workerOf, "a")},
|
{what: "worker-of orders", edges: []inventory.Edge{dep("x", workerOf, "a")},
|
||||||
@@ -296,18 +297,18 @@ func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) {
|
|||||||
[]string{"internal/inventory/migrations/0088-a-build-says-its-build-source.sql"}, read,
|
[]string{"internal/inventory/migrations/0088-a-build-says-its-build-source.sql"}, read,
|
||||||
"mesh-controller | build-agent", ""},
|
"mesh-controller | build-agent", ""},
|
||||||
// A package all three build from: all three; the build seat's holder after the controller whose worker
|
// A package all three build from: all three; the build seat's holder after the controller whose worker
|
||||||
// it binds (worker-of, issue 206), the proxy after the holder that builds it (built-by).
|
// it binds (worker-of, issue 206), the proxy beside the controller: built-by orders nothing (ADR 0360 §3).
|
||||||
{"a package all three build from: all three", "mesh-controller",
|
{"a package all three build from: all three", "mesh-controller",
|
||||||
[]string{"internal/broker/broker.go"}, read, "mesh-controller | build-agent | route-proxy", ""},
|
[]string{"internal/broker/broker.go"}, read, "mesh-controller,route-proxy | build-agent", ""},
|
||||||
{"a vendored package all three build from: all three", "mesh-controller",
|
{"a vendored package all three build from: all three", "mesh-controller",
|
||||||
[]string{"vendor/github.com/nats-io/nats.go/nats.go"}, read, "mesh-controller | build-agent | route-proxy", ""},
|
[]string{"vendor/github.com/nats-io/nats.go/nats.go"}, read, "mesh-controller,route-proxy | build-agent", ""},
|
||||||
{"go.sum: all three", "mesh-controller",
|
{"go.sum: all three", "mesh-controller",
|
||||||
[]string{"go.sum"}, read, "mesh-controller | build-agent | route-proxy", ""},
|
[]string{"go.sum"}, read, "mesh-controller,route-proxy | build-agent", ""},
|
||||||
{"a file added to the proxy's package: the proxy", "mesh-controller",
|
{"a file added to the proxy's package: the proxy", "mesh-controller",
|
||||||
[]string{"examples/route-proxy/new.go"}, read, "route-proxy", ""},
|
[]string{"examples/route-proxy/new.go"}, read, "route-proxy", ""},
|
||||||
// Before any build source is said: as before.
|
// Before any build source is said: as before.
|
||||||
{"no build source said: a README moves all three, as before", "mesh-controller",
|
{"no build source said: a README moves all three, as before", "mesh-controller",
|
||||||
[]string{"README.md"}, unsaid, "mesh-controller | build-agent | route-proxy", ""},
|
[]string{"README.md"}, unsaid, "mesh-controller,route-proxy | build-agent", ""},
|
||||||
// In the catalogue, where they live, each by its own build source.
|
// In the catalogue, where they live, each by its own build source.
|
||||||
{"the route proxy's recipe: it alone", "mesh-catalog",
|
{"the route proxy's recipe: it alone", "mesh-catalog",
|
||||||
[]string{"modules/route-proxy/Dockerfile"}, read, "route-proxy", ""},
|
[]string{"modules/route-proxy/Dockerfile"}, read, "route-proxy", ""},
|
||||||
@@ -331,7 +332,7 @@ func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) {
|
|||||||
// Files not all said: everything the repository builds, as before.
|
// Files not all said: everything the repository builds, as before.
|
||||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Commit: "head",
|
m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Commit: "head",
|
||||||
Paths: []string{"README.md"}, PathsTruncated: true}
|
Paths: []string{"README.md"}, PathsTruncated: true}
|
||||||
if got := tiered(reachOfMerge(m, entries, read, edges).Plan.Tiers); got != "mesh-controller | build-agent | route-proxy" {
|
if got := tiered(reachOfMerge(m, entries, read, edges).Plan.Tiers); got != "mesh-controller,route-proxy | build-agent" {
|
||||||
t.Errorf("a merge whose files were not all said: planned %q, wanted all three", got)
|
t.Errorf("a merge whose files were not all said: planned %q, wanted all three", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -452,8 +453,9 @@ func TestAPlanIsTheTouchedModulesAndWhatIsReachableAlongTheWideningEdges(t *test
|
|||||||
kinds := []string{inventory.EdgeStandsOn, inventory.EdgeDeclared, inventory.EdgePackages,
|
kinds := []string{inventory.EdgeStandsOn, inventory.EdgeDeclared, inventory.EdgePackages,
|
||||||
inventory.EdgeBuiltBy, inventory.EdgeWorkerOf}
|
inventory.EdgeBuiltBy, inventory.EdgeWorkerOf}
|
||||||
widens := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true}
|
widens := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true}
|
||||||
|
// Built-by orders nothing since novox/hq ADR 0360 §3.
|
||||||
orders := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true,
|
orders := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true,
|
||||||
inventory.EdgeBuiltBy: true, inventory.EdgeWorkerOf: true}
|
inventory.EdgeWorkerOf: true}
|
||||||
// Directory names drawn from one pool, so two repositories hold directories of the same name, and one
|
// Directory names drawn from one pool, so two repositories hold directories of the same name, and one
|
||||||
// is a prefix of another.
|
// is a prefix of another.
|
||||||
dirs := []string{"a", "ab", "b", "c", "lib/x", "lib/y", "modules/a", "modules/a/sub"}
|
dirs := []string{"a", "ab", "b", "c", "lib/x", "lib/y", "modules/a", "modules/a/sub"}
|
||||||
|
|||||||
@@ -18,8 +18,9 @@ package main
|
|||||||
// it beside the layer (`settings` says it back), and records the warrant in the hand-act log on the bus, where a
|
// it beside the layer (`settings` says it back), and records the warrant in the hand-act log on the bus, where a
|
||||||
// person's decisions are read; the router edits the ask on every channel to its outcome. No seat event of its
|
// person's decisions are read; the router edits the ask on every channel to its outcome. No seat event of its
|
||||||
// own: nothing consumes one, and the installer's first user list in the node-engine's repository names every
|
// own: nothing consumes one, and the installer's first user list in the node-engine's repository names every
|
||||||
// controller event, so one would cost a node-engine change for a fact without a reader. The push afterwards is a
|
// controller event, so one would cost a node-engine change for a fact without a reader. No push is left for
|
||||||
// separate act, as it is for a layer set at the terminal.
|
// afterwards: the warrant that sets the layer opens the walk that sends it (novox/hq ADR 0344, settings_walk.go),
|
||||||
|
// so the operator's one Approve is the act that makes the change take effect.
|
||||||
//
|
//
|
||||||
// **What the operator reads** is the module, the machine, each key with its exact new value and, for a changed
|
// **What the operator reads** is the module, the machine, each key with its exact new value and, for a changed
|
||||||
// key, the value it replaces. A value that may not leave the mesh (an address, a path, a secret's shape: the
|
// key, the value it replaces. A value that may not leave the mesh (an address, a path, a secret's shape: the
|
||||||
@@ -27,6 +28,13 @@ package main
|
|||||||
// ask says so, and the whole is read with `settings proposals <id>` — whose fingerprint must be the one on the
|
// ask says so, and the whole is read with `settings proposals <id>` — whose fingerprint must be the one on the
|
||||||
// phone. Fail closed: a proposal nothing can carry to the operator is refused at once, in words, and never left
|
// phone. Fail closed: a proposal nothing can carry to the operator is refused at once, in words, and never left
|
||||||
// waiting for an answer that cannot come.
|
// waiting for an answer that cannot come.
|
||||||
|
//
|
||||||
|
// **On a channel that proves who answers, the values are shown WHOLE** (novox/hq issue 383, the operator's
|
||||||
|
// decision of 2026-10-10): a mount point, a share name or a private address is the thing being approved and must
|
||||||
|
// be readable, so the ask carries them whole beside the explanation (asks.Ask.Whole), the router shows that only
|
||||||
|
// on a kind that verifies its sender, and only a value shaped like a secret is withheld there. The message is the
|
||||||
|
// headline, one line per key, who proposed it and when; the fingerprint and how to read the proposal whole are
|
||||||
|
// the Details answer's (asks.Ask.Details). The masking stays for conditions and for channels that prove nothing.
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -47,6 +55,7 @@ import (
|
|||||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
"github.com/novox/mesh-controller/internal/outward"
|
"github.com/novox/mesh-controller/internal/outward"
|
||||||
)
|
)
|
||||||
@@ -154,47 +163,48 @@ func (p settingsProposal) ask(id string, machines []string) (asks.Ask, map[strin
|
|||||||
if len([]rune(headline)) > asks.HeadlineLength {
|
if len([]rune(headline)) > asks.HeadlineLength {
|
||||||
headline = verb + " settings?"
|
headline = verb + " settings?"
|
||||||
}
|
}
|
||||||
shown, whole := p.change(machines)
|
// The message reads at a glance (issues 383 and 639): a summary of what changes where, then at most three
|
||||||
var b strings.Builder
|
// points, a key each; every key with its whole value, who proposed it, the fingerprint and how to read it at the
|
||||||
if p.Clear {
|
// terminal are its Details, which the router notes after the answers.
|
||||||
fmt.Fprintf(&b, "Clear the settings of %s on %s, back to what the module says?\n\n", p.Module, p.where())
|
shown, shownAll, shownWhole := p.change(machines, false)
|
||||||
} else {
|
whole, wholeAll, _ := p.change(machines, true)
|
||||||
fmt.Fprintf(&b, "Set the settings of %s on %s to these values?\n\n", p.Module, p.where())
|
summary := p.summary()
|
||||||
|
compose := func(points []string) string {
|
||||||
|
return strings.Join(append([]string{summary}, points...), "\n")
|
||||||
}
|
}
|
||||||
fmt.Fprintf(&b, "Proposed by %s, at %s. ", sayable(p.From, machines), p.At.Local().Format("15:04 on 2 Jan"))
|
// The Details: every setting it sets, one line each, uncut — whole where the sender is proven (WholeDetails),
|
||||||
switch {
|
// masked elsewhere — then who proposed it, the fingerprint and how to read it at the terminal.
|
||||||
case p.Clear && p.HadLayer:
|
details := func(lines []string) string {
|
||||||
b.WriteString("The layer it removes:\n\n")
|
var d strings.Builder
|
||||||
case p.Clear:
|
d.WriteString(strings.Join(lines, "\n") + "\n")
|
||||||
b.WriteString("There is no layer to remove; approving changes nothing.")
|
d.WriteString(p.onApprove() + "\n")
|
||||||
case !p.HadLayer:
|
fmt.Fprintf(&d, "Proposed by %s at %s. Fingerprint %s.\n", sayable(p.From, machines), p.At.Local().Format("15:04 on 2 Jan"),
|
||||||
b.WriteString("There is no layer yet; this is the whole of it:\n\n")
|
fingerprint(p.Digest))
|
||||||
default:
|
if !shownWhole {
|
||||||
b.WriteString("The layer is replaced whole; what changes against it:\n\n")
|
d.WriteString("Where who answers is not proven, its values are shown as ‹address›, ‹path› or ‹withheld›.\n")
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&d, "Read it whole, with this fingerprint: settings proposals %s at the controller's terminal, or the "+
|
||||||
|
"controller's settings verb with proposal %s through the mesh MCP server.", id, id)
|
||||||
|
return d.String()
|
||||||
}
|
}
|
||||||
b.WriteString(shown)
|
q := asks.Ask{ID: id, Headline: headline, Explanation: compose(shown), Who: asks.Operator,
|
||||||
fmt.Fprintf(&b, "\n\nFingerprint %s.", fingerprint(p.Digest))
|
|
||||||
if !whole {
|
|
||||||
b.WriteString(" Parts shown as ‹address›, ‹path› or ‹withheld› may not leave the mesh: read it whole, with " +
|
|
||||||
"this fingerprint, through the mesh MCP server (mesh-controller.settings, proposal " + id + ") or with " +
|
|
||||||
"mesh-cli settings proposals " + id + ".")
|
|
||||||
}
|
|
||||||
if p.Clear {
|
|
||||||
b.WriteString(" Approved, the layer is removed at once and the machine takes it at its next push.")
|
|
||||||
} else {
|
|
||||||
b.WriteString(" Approved, the layer is set at once and the machine takes it at its next push.")
|
|
||||||
}
|
|
||||||
q := asks.Ask{ID: id, Headline: headline, Explanation: b.String(), Who: asks.Operator,
|
|
||||||
Expires: p.At.Add(askApproveFor), OnExpiry: "the proposal is discarded; nothing changes",
|
Expires: p.At.Add(askApproveFor), OnExpiry: "the proposal is discarded; nothing changes",
|
||||||
About: p.about()}
|
About: p.about(), Details: details(shownAll)}
|
||||||
|
// Only where a value was masked: an ask whose values all pass the content rule shows the same everywhere.
|
||||||
|
if w := compose(whole); w != q.Explanation {
|
||||||
|
q.Whole = w
|
||||||
|
}
|
||||||
|
if w := details(wholeAll); w != q.Details {
|
||||||
|
q.WholeDetails = w
|
||||||
|
}
|
||||||
options := map[string]int{}
|
options := map[string]int{}
|
||||||
for i, act := range p.actions(id) {
|
for i, act := range p.actions(id) {
|
||||||
binds, _ := asks.ActDigest(boundAct(act))
|
binds, _ := asks.ActDigest(boundAct(act))
|
||||||
oid := optionID(act.Label)
|
oid := optionID(act.Label)
|
||||||
options[oid] = i
|
options[oid] = i
|
||||||
does := "the settings are set; nothing is pushed yet"
|
does := "the settings are set and sent to " + p.reaches() + " at once, each machine judged"
|
||||||
if p.Clear {
|
if p.Clear {
|
||||||
does = "the layer is removed; nothing is pushed yet"
|
does = "the layer is removed and that is sent to " + p.reaches() + " at once, each machine judged"
|
||||||
}
|
}
|
||||||
if act.Arguments["answer"] == answerDecline {
|
if act.Arguments["answer"] == answerDecline {
|
||||||
does = "nothing changes; the proposal is discarded"
|
does = "nothing changes; the proposal is discarded"
|
||||||
@@ -205,55 +215,141 @@ func (p settingsProposal) ask(id string, machines []string) (asks.Ask, map[strin
|
|||||||
return q, options
|
return q, options
|
||||||
}
|
}
|
||||||
|
|
||||||
// shownMost is the most of a change the phone is shown, in bytes; the rest is read whole with `settings proposals`.
|
// summary is the proposal's first line, what changes where: how many settings, how many stay, and that Approve sends
|
||||||
const shownMost = 1400
|
// the layer at once by a walk, one machine at a time (novox/hq ADR 0344); the rest of what Approve does
|
||||||
|
// (onApprove) is the Details'.
|
||||||
|
func (p settingsProposal) summary() string {
|
||||||
|
if p.Clear {
|
||||||
|
if !p.HadLayer {
|
||||||
|
return "There is no layer to remove; approving changes nothing."
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("Removes the %d setting(s) of %s set on %s; approved, the layer is removed and that is sent at once "+
|
||||||
|
"by a walk to %s, one machine at a time, each judged.", len(leaves(p.Before, "")), p.Module, p.where(), p.reaches())
|
||||||
|
}
|
||||||
|
added, changed, removed := settingsChange(p.Before, p.Values)
|
||||||
|
n := len(added) + len(changed) + len(removed)
|
||||||
|
unchanged := len(leaves(p.Values, "")) - len(added) - len(changed)
|
||||||
|
if n == 0 {
|
||||||
|
return fmt.Sprintf("Nothing changes: the %d setting(s) of %s on %s are as they stand.", unchanged, p.Module, p.where())
|
||||||
|
}
|
||||||
|
stay := ""
|
||||||
|
if unchanged > 0 {
|
||||||
|
stay = fmt.Sprintf(", and %d stay as they are", unchanged)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("Changes %d setting(s) of %s on %s%s; approved, the layer is set and sent at once by a walk to %s, "+
|
||||||
|
"one machine at a time, each judged.", n, p.Module, p.where(), stay, p.reaches())
|
||||||
|
}
|
||||||
|
|
||||||
// change is what changes, line by line, each value shown under the content rule, and whether every value was
|
// change is what changes: as at most asks.MostPoints points for the glance, as every line whole for the note, and
|
||||||
// shown whole: "+ key: value" added, "~ key: value (was: old)" changed, "- key (was: old)" removed, and the
|
// whether every value was shown exactly: "+ key: value"
|
||||||
// count of keys unchanged.
|
// added, "~ key: value (was: old)" changed, "- key (was: old)" removed. Each value is shown under the content rule
|
||||||
func (p settingsProposal) change(machines []string) (string, bool) {
|
// (sayableValue), or — exact, for a channel that proves who answers — as it is, withheld only when shaped like a
|
||||||
|
// secret (wholeValue). More changes than points: the first two, and the keys of the rest; a point longer than a
|
||||||
|
// phone shows at a glance is cut and says so. The note carries every line uncut (the review of #304: Approve sets
|
||||||
|
// exactly what the operator can read).
|
||||||
|
func (p settingsProposal) change(machines []string, exact bool) ([]string, []string, bool) {
|
||||||
whole := true
|
whole := true
|
||||||
say := func(v any) string {
|
say := func(v any) string {
|
||||||
s, w := sayableValue(v, machines)
|
s, w := sayableValue(v, machines)
|
||||||
|
if exact {
|
||||||
|
s, w = wholeValue(v, machines)
|
||||||
|
}
|
||||||
whole = whole && w
|
whole = whole && w
|
||||||
return s
|
return s
|
||||||
}
|
}
|
||||||
var lines []string
|
type line struct{ key, text string }
|
||||||
|
var lines []line
|
||||||
if p.Clear {
|
if p.Clear {
|
||||||
was := leaves(p.Before, "")
|
was := leaves(p.Before, "")
|
||||||
for _, k := range layerKeys(was) {
|
for _, k := range layerKeys(was) {
|
||||||
lines = append(lines, fmt.Sprintf("- %s: %s", k, say(was[k])))
|
lines = append(lines, line{k, fmt.Sprintf("- %s: %s", k, say(was[k]))})
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
added, changed, removed := settingsChange(p.Before, p.Values)
|
added, changed, removed := settingsChange(p.Before, p.Values)
|
||||||
was, now := leaves(p.Before, ""), leaves(p.Values, "")
|
was, now := leaves(p.Before, ""), leaves(p.Values, "")
|
||||||
for _, k := range added {
|
for _, k := range added {
|
||||||
lines = append(lines, fmt.Sprintf("+ %s: %s", k, say(now[k])))
|
lines = append(lines, line{k, fmt.Sprintf("+ %s: %s", k, say(now[k]))})
|
||||||
}
|
}
|
||||||
for _, k := range changed {
|
for _, k := range changed {
|
||||||
lines = append(lines, fmt.Sprintf("~ %s: %s (was: %s)", k, say(now[k]), say(was[k])))
|
lines = append(lines, line{k, fmt.Sprintf("~ %s: %s (was: %s)", k, say(now[k]), say(was[k]))})
|
||||||
}
|
}
|
||||||
for _, k := range removed {
|
for _, k := range removed {
|
||||||
lines = append(lines, fmt.Sprintf("- %s (was: %s)", k, say(was[k])))
|
lines = append(lines, line{k, fmt.Sprintf("- %s (was: %s)", k, say(was[k]))})
|
||||||
}
|
|
||||||
unchanged := len(now) - len(added) - len(changed)
|
|
||||||
switch {
|
|
||||||
case len(lines) == 0 && unchanged > 0:
|
|
||||||
lines = append(lines, fmt.Sprintf("= nothing changes: the %d key(s) are as they stand", unchanged))
|
|
||||||
case unchanged > 0:
|
|
||||||
lines = append(lines, fmt.Sprintf("= %d key(s) unchanged", unchanged))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
out := strings.Join(lines, "\n")
|
// **Each line is judged as the router will judge it** (the review of #304, ADR 0347 §1 and §3): a line the
|
||||||
if len(out) > shownMost {
|
// content rule (masked) or the secret rule (whole) would refuse — a value given to a secret's name, say
|
||||||
cut := shownMost
|
// "+ password: hunter2" — is said with its key still named and its value withheld, never left for the
|
||||||
for cut > 0 && out[cut] != '\n' {
|
// router to withhold whole and unnamed. A line refused even so is the propose's refusal (judgeLines).
|
||||||
cut--
|
for i, l := range lines {
|
||||||
|
if lineLeaves(l.text, exact, machines) {
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
out = out[:cut] + fmt.Sprintf("\n… NOT SHOWN IN FULL here: %d more bytes", len(strings.Join(lines, "\n"))-cut)
|
lines[i].text = l.text[:2] + l.key + " (value withheld)"
|
||||||
whole = false
|
whole = false
|
||||||
}
|
}
|
||||||
return out, whole
|
all := make([]string, len(lines))
|
||||||
|
for i, l := range lines {
|
||||||
|
all[i] = l.text
|
||||||
|
}
|
||||||
|
// What the note holds of a value: whole where the glance is the whole one (shown only where the sender is
|
||||||
|
// proven, with the whole details beside it), under the content rule elsewhere — so the marker says no more than
|
||||||
|
// the note it points to (the review of ADR 0347).
|
||||||
|
inNote := "in the note"
|
||||||
|
if exact {
|
||||||
|
inNote = "whole in the note"
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for i, l := range lines {
|
||||||
|
if len(lines) > asks.MostPoints && i == asks.MostPoints-1 {
|
||||||
|
var rest []string
|
||||||
|
for _, r := range lines[i:] {
|
||||||
|
rest = append(rest, r.key)
|
||||||
|
}
|
||||||
|
out = append(out, fmt.Sprintf("and %d more, each %s: %s", len(rest), inNote, strings.Join(rest, ", ")))
|
||||||
|
break
|
||||||
|
}
|
||||||
|
out = append(out, l.text)
|
||||||
|
}
|
||||||
|
// A point too long for a glance is cut, and says so: its value is the note's.
|
||||||
|
cut := "… (cut: " + inNote + ")"
|
||||||
|
for i, l := range out {
|
||||||
|
if r := []rune(l); len(r) > asks.PointLength {
|
||||||
|
out[i] = string(r[:asks.PointLength-len([]rune(cut))]) + cut
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, all, whole
|
||||||
|
}
|
||||||
|
|
||||||
|
// lineLeaves says whether a line may leave the mesh as the router judges it: under the content rule for the masked
|
||||||
|
// form, the secret rule alone for the whole one.
|
||||||
|
func lineLeaves(line string, exact bool, machines []string) bool {
|
||||||
|
if exact {
|
||||||
|
_, ok := outward.Secret(line, machines...)
|
||||||
|
return ok
|
||||||
|
}
|
||||||
|
_, ok := outward.Check(line, machines...)
|
||||||
|
return ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgeLines is the first line of an ask's details the router would refuse, or withhold, and why: masked under
|
||||||
|
// the content rule line by line, whole under the secret rule. Empty when every line may leave.
|
||||||
|
func judgeLines(details string, exact bool, machines []string) (string, string) {
|
||||||
|
for _, line := range strings.Split(details, "\n") {
|
||||||
|
if strings.TrimSpace(line) == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if exact {
|
||||||
|
if refusal, ok := outward.Secret(line, machines...); !ok {
|
||||||
|
return line, refusal.String()
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if refusal, ok := outward.Check(line, machines...); !ok {
|
||||||
|
return line, refusal.String()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", ""
|
||||||
}
|
}
|
||||||
|
|
||||||
func layerKeys(m map[string]any) []string {
|
func layerKeys(m map[string]any) []string {
|
||||||
@@ -300,6 +396,27 @@ func sayableValue(v any, machines []string) (string, bool) {
|
|||||||
return out, out == text
|
return out, out == text
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// wholeValue is a value as a channel that proves who answers is shown it (asks.Ask.Whole), and whether it was
|
||||||
|
// shown whole: as it is, unless it is shaped like a secret (outward.Secret), which is withheld whole — never in
|
||||||
|
// part, so no half of a key reaches the phone.
|
||||||
|
func wholeValue(v any, machines []string) (string, bool) {
|
||||||
|
text, ok := v.(string)
|
||||||
|
if !ok {
|
||||||
|
raw, err := json.Marshal(v)
|
||||||
|
if err != nil {
|
||||||
|
return markWithheld, false
|
||||||
|
}
|
||||||
|
text = string(raw)
|
||||||
|
}
|
||||||
|
if text == "" {
|
||||||
|
return `""`, true
|
||||||
|
}
|
||||||
|
if _, ok := outward.Secret(text, machines...); !ok {
|
||||||
|
return markWithheld, false
|
||||||
|
}
|
||||||
|
return text, true
|
||||||
|
}
|
||||||
|
|
||||||
// sayable is a text with what may not leave the mesh replaced in place by a marker; what the markers cannot make
|
// sayable is a text with what may not leave the mesh replaced in place by a marker; what the markers cannot make
|
||||||
// pass is withheld whole.
|
// pass is withheld whole.
|
||||||
func sayable(text string, machines []string) string {
|
func sayable(text string, machines []string) string {
|
||||||
@@ -432,6 +549,23 @@ func (pr proposer) propose(ctx context.Context, in proposeInput) (string, error)
|
|||||||
return refuse("the ask's words would carry %s, which may not leave the mesh, and the change could not be "+
|
return refuse("the ask's words would carry %s, which may not leave the mesh, and the change could not be "+
|
||||||
"shown without it; %s", refusal, atTheTerminalInstead(in))
|
"shown without it; %s", refusal, atTheTerminalInstead(in))
|
||||||
}
|
}
|
||||||
|
// The whole words are shown only where the sender is proven, and never a secret's shape: wholeValue withholds
|
||||||
|
// one, and the router would refuse the ask if one were left, so it is refused here first, in words.
|
||||||
|
if refusal, ok := outward.Secret(q.Whole, machines...); !ok {
|
||||||
|
return refuse("the change shown whole would carry %s, which may not leave the mesh on any channel; %s",
|
||||||
|
refusal, atTheTerminalInstead(in))
|
||||||
|
}
|
||||||
|
// So are the details (ADR 0347 §1, §3): every line the router would withhold from the note, or refuse the ask
|
||||||
|
// for, is refused here, in the controller's own words, naming the line's key — Approve never sets a value the
|
||||||
|
// operator was shown nothing of.
|
||||||
|
if line, why := judgeLines(q.Details, false, machines); line != "" {
|
||||||
|
return refuse("the change's line %q would carry %s, which the operator's note may not show; %s",
|
||||||
|
keyOfLine(line), why, atTheTerminalInstead(in))
|
||||||
|
}
|
||||||
|
if line, why := judgeLines(q.WholeDetails, true, machines); line != "" {
|
||||||
|
return refuse("the change's line %q shown whole would carry %s, which may not leave the mesh on any channel; %s",
|
||||||
|
keyOfLine(line), why, atTheTerminalInstead(in))
|
||||||
|
}
|
||||||
// Fail closed, before anything is kept: no router, no grant, no channel means no ask.
|
// Fail closed, before anything is kept: no router, no grant, no channel means no ask.
|
||||||
if pr.routerHere != nil {
|
if pr.routerHere != nil {
|
||||||
here, err := pr.routerHere(ctx)
|
here, err := pr.routerHere(ctx)
|
||||||
@@ -519,15 +653,29 @@ func (pr proposer) propose(ctx context.Context, in proposeInput) (string, error)
|
|||||||
fingerprint(p.Digest))
|
fingerprint(p.Digest))
|
||||||
fmt.Fprintf(&b, " until %s nothing changes: the layer is set only on Approve, and discarded on Decline or at expiry\n",
|
fmt.Fprintf(&b, " until %s nothing changes: the layer is set only on Approve, and discarded on Decline or at expiry\n",
|
||||||
q.Expires.Local().Format("2006-01-02 15:04"))
|
q.Expires.Local().Format("2006-01-02 15:04"))
|
||||||
fmt.Fprintf(&b, " `settings proposals %s` shows it whole; once approved, `push %s` sends it", id, pushWord(p.Node))
|
fmt.Fprintf(&b, " `settings proposals %s` shows it whole; once approved, a walk sends it to %s at once, one machine at "+
|
||||||
|
"a time, each judged, and its end is told as every walk's is (novox/hq ADR 0344)", id, p.reaches())
|
||||||
return b.String(), nil
|
return b.String(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func pushWord(node string) string {
|
// reaches is the machines an approved layer is sent to, in words: the one it names, or every machine running the
|
||||||
if node == "" {
|
// module.
|
||||||
return "--behind"
|
func (p settingsProposal) reaches() string {
|
||||||
|
if p.Node == "" {
|
||||||
|
return "every machine running " + p.Module
|
||||||
}
|
}
|
||||||
return node
|
return p.Node
|
||||||
|
}
|
||||||
|
|
||||||
|
// onApprove is what the ask says Approve does (novox/hq ADR 0344 §4): the one Approve sets the layer and sends it.
|
||||||
|
func (p settingsProposal) onApprove() string {
|
||||||
|
what := "the layer is set"
|
||||||
|
if p.Clear {
|
||||||
|
what = "the layer is removed"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("Approved, %s and sent at once to %s, one machine at a time, each judged, every build there "+
|
||||||
|
"kept; its end is told as every walk's is. A module that reads its settings only when it starts takes them at "+
|
||||||
|
"its next start.", what, p.reaches())
|
||||||
}
|
}
|
||||||
|
|
||||||
func newProposalID() string {
|
func newProposalID() string {
|
||||||
@@ -788,6 +936,40 @@ func setLayerIn(open *stores) setOnWarrant {
|
|||||||
changed = "nothing changed"
|
changed = "nothing changed"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return changed + " (ask " + askID + ")", nil
|
// **And sends it** (novox/hq ADR 0344): the layer the operator approved is walked to the machines it reaches
|
||||||
|
// at once. A walk that cannot be opened is said as the act's failure — the layer is set and nothing sends it.
|
||||||
|
walked, err := openSettingsWalk(ctx, open, inventory.PlanSettings{Module: p.Module, Node: p.Node, Ask: askID,
|
||||||
|
Change: changed})
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("%s (ask %s) is set, and the walk that sends it could not be opened: %v — `push %s` sends it",
|
||||||
|
changed, askID, err, pushTo(p.Node))
|
||||||
|
}
|
||||||
|
return changed + " (ask " + askID + "); " + walked, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// pushTo is the push that sends a layer by hand when its walk could not: the machine, or every one behind.
|
||||||
|
func pushTo(node string) string {
|
||||||
|
if node == "" {
|
||||||
|
return "--behind"
|
||||||
|
}
|
||||||
|
return node
|
||||||
|
}
|
||||||
|
|
||||||
|
// keyOfLine is what a refusal names of a line: the key of a change's line ("+ key: …"), or the line's first words.
|
||||||
|
func keyOfLine(line string) string {
|
||||||
|
if len(line) > 2 && strings.ContainsRune("+~-=", rune(line[0])) && line[1] == ' ' {
|
||||||
|
rest := line[2:]
|
||||||
|
if i := strings.IndexAny(rest, ": "); i > 0 {
|
||||||
|
rest = rest[:i]
|
||||||
|
}
|
||||||
|
if _, ok := outward.Secret(rest); !ok {
|
||||||
|
return "whose key is shaped like a secret"
|
||||||
|
}
|
||||||
|
return rest
|
||||||
|
}
|
||||||
|
if r := []rune(line); len(r) > 40 {
|
||||||
|
return string(r[:40]) + "…"
|
||||||
|
}
|
||||||
|
return line
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"slices"
|
"slices"
|
||||||
@@ -116,24 +117,43 @@ func TestAProposalAsksAtTheLevelApproveWithTheExactChange(t *testing.T) {
|
|||||||
if q.Options[0].Binds == q.Options[1].Binds {
|
if q.Options[0].Binds == q.Options[1].Binds {
|
||||||
t.Error("Approve and Decline bind the same act")
|
t.Error("Approve and Decline bind the same act")
|
||||||
}
|
}
|
||||||
for _, line := range []string{
|
// The message reads at a glance (issues 383 and 639): a summary, then one point per key — and nothing else:
|
||||||
"Set the settings of mounts on shanks to these values?",
|
// who proposed it, the fingerprint and the how-to are the Details, which the router notes after the answers.
|
||||||
"Proposed by g14/claude-code, through the mesh-controller seat, at " + r.now.Local().Format("15:04 on 2 Jan") + ".",
|
summary := "Changes 3 setting(s) of mounts on shanks; approved, the layer is set and sent at once by a walk to shanks, one machine at a time, each judged."
|
||||||
"+ sources: recalbox=‹address›@‹path›:ro",
|
if q.Explanation != summary+"\n+ sources: recalbox=‹address›@‹path›:ro\n~ shares: library=‹path› (was: none)\n- old (was: x)" {
|
||||||
"~ shares: library=‹path› (was: none)",
|
t.Errorf("the explanation:\n%s", q.Explanation)
|
||||||
"- old (was: x)",
|
}
|
||||||
"Fingerprint " + fingerprint(layerDigest(mountsSources)) + ".",
|
if err := asks.Glance(q.Explanation); err != nil {
|
||||||
"read it whole, with this fingerprint",
|
t.Error(err)
|
||||||
} {
|
|
||||||
if !strings.Contains(q.Explanation, line) {
|
|
||||||
t.Errorf("the explanation lacks %q:\n%s", line, q.Explanation)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
for _, leak := range []string{"nas.lan", "/mnt/recalbox", "/mnt/library", "smb://"} {
|
for _, leak := range []string{"nas.lan", "/mnt/recalbox", "/mnt/library", "smb://"} {
|
||||||
if strings.Contains(q.Explanation, leak) {
|
if strings.Contains(q.Explanation, leak) {
|
||||||
t.Errorf("the explanation carries %q, which may not leave the mesh", leak)
|
t.Errorf("the explanation carries %q, which may not leave the mesh", leak)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Where the sender is proven, the values whole: the mount point and the share are what is approved.
|
||||||
|
if q.Whole != summary+"\n+ sources: recalbox=smb://nas.lan/recalbox@/mnt/recalbox:ro\n~ shares: library=/mnt/library (was: none)\n- old (was: x)" ||
|
||||||
|
asks.Glance(q.Whole) != nil {
|
||||||
|
t.Errorf("the whole words:\n%s", q.Whole)
|
||||||
|
}
|
||||||
|
for _, line := range []string{
|
||||||
|
"Proposed by g14/claude-code, through the mesh-controller seat at " + r.now.Local().Format("15:04 on 2 Jan") +
|
||||||
|
". Fingerprint " + fingerprint(layerDigest(mountsSources)) + ".",
|
||||||
|
"Where who answers is not proven, its values are shown as ‹address›, ‹path› or ‹withheld›.",
|
||||||
|
"Read it whole, with this fingerprint: settings proposals " + kept(r).ID + " at the controller's terminal",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(q.Details, line) {
|
||||||
|
t.Errorf("Details lack %q:\n%s", line, q.Details)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if refusal, ok := outward.Check(q.Details, "anchor", "laptop", "shanks"); !ok {
|
||||||
|
t.Errorf("the router would withhold a line of the details: %s", refusal)
|
||||||
|
}
|
||||||
|
for _, howTo := range []string{"ingerprint", "settings proposals", "settings verb", "not proven", "Proposed by"} {
|
||||||
|
if strings.Contains(q.Explanation, howTo) || strings.Contains(q.Whole, howTo) {
|
||||||
|
t.Errorf("the message carries the how-to %q", howTo)
|
||||||
|
}
|
||||||
|
}
|
||||||
all := []string{q.Headline, q.Explanation, q.OnExpiry}
|
all := []string{q.Headline, q.Explanation, q.OnExpiry}
|
||||||
for _, o := range q.Options {
|
for _, o := range q.Options {
|
||||||
all = append(all, o.Label, o.Does)
|
all = append(all, o.Label, o.Does)
|
||||||
@@ -141,6 +161,9 @@ func TestAProposalAsksAtTheLevelApproveWithTheExactChange(t *testing.T) {
|
|||||||
if refusal, ok := outward.Check(strings.Join(all, "\n"), "anchor", "laptop", "shanks"); !ok {
|
if refusal, ok := outward.Check(strings.Join(all, "\n"), "anchor", "laptop", "shanks"); !ok {
|
||||||
t.Errorf("the router would refuse the ask: %s", refusal)
|
t.Errorf("the router would refuse the ask: %s", refusal)
|
||||||
}
|
}
|
||||||
|
if refusal, ok := outward.Secret(q.Whole, "anchor", "laptop", "shanks"); !ok {
|
||||||
|
t.Errorf("the router would refuse the whole words: %s", refusal)
|
||||||
|
}
|
||||||
// Kept as the controller's own ask, about no condition, with the proposal whole and its digests.
|
// Kept as the controller's own ask, about no condition, with the proposal whole and its digests.
|
||||||
kept := r.theProposal(t)
|
kept := r.theProposal(t)
|
||||||
p := kept.Proposal
|
p := kept.Proposal
|
||||||
@@ -173,11 +196,98 @@ func TestAMeshWideLayerIsProposed(t *testing.T) {
|
|||||||
}
|
}
|
||||||
q := r.askSent(t)
|
q := r.askSent(t)
|
||||||
if q.Headline != "Set notes on the whole mesh?" || q.About != "settings.notes.mesh" ||
|
if q.Headline != "Set notes on the whole mesh?" || q.About != "settings.notes.mesh" ||
|
||||||
!strings.Contains(q.Explanation, "Set the settings of notes on the whole mesh to these values?") {
|
q.Explanation != "Changes 1 setting(s) of notes on the whole mesh; approved, the layer is set and sent at once by a walk to every machine running notes, one machine at a time, each judged.\n+ x: 1" ||
|
||||||
|
q.Whole != "" {
|
||||||
t.Errorf("%+v", q)
|
t.Errorf("%+v", q)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// kept is the one proposal the rig keeps.
|
||||||
|
func kept(r *proposerRig) asked {
|
||||||
|
for _, a := range r.store {
|
||||||
|
if a.Proposal != nil {
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return asked{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The switch between the masked and the whole change (issue 383): the same change, under the content rule and
|
||||||
|
// exact, differs in the masked values alone; a value shaped like a secret is withheld in both, whole, with its
|
||||||
|
// key still named; and a change no value of which is masked carries no whole words of its own.
|
||||||
|
func TestAProposalShowsItsValuesWholeOnlyWhereTheSenderIsProvenAndNeverASecret(t *testing.T) {
|
||||||
|
machines := []string{"anchor", "laptop", "shanks"}
|
||||||
|
r := newProposerRig(t)
|
||||||
|
values := map[string]any{"shares": "media=/storage/media", "sources": "recalbox=smb://nas.lan/recalbox@/mnt/recalbox", "font": "Inter 13"}
|
||||||
|
if _, err := r.pr.propose(context.Background(), proposeInput{module: "mounts", node: "shanks", values: values}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
q := r.askSent(t)
|
||||||
|
masked, _, _ := kept(r).Proposal.change(machines, false)
|
||||||
|
whole, _, _ := kept(r).Proposal.change(machines, true)
|
||||||
|
if !strings.HasSuffix(q.Explanation, "\n"+strings.Join(masked, "\n")) || !strings.HasSuffix(q.Whole, "\n"+strings.Join(whole, "\n")) {
|
||||||
|
t.Fatalf("the ask does not carry the change masked and whole:\n%s\n--\n%s", q.Explanation, q.Whole)
|
||||||
|
}
|
||||||
|
if want := []string{"+ font: Inter 13", "+ shares: media=‹path›", "+ sources: recalbox=‹address›@‹path›"}; strings.Join(masked, "\n") != strings.Join(want, "\n") {
|
||||||
|
t.Errorf("masked: %q", masked)
|
||||||
|
}
|
||||||
|
if want := []string{"+ font: Inter 13", "+ shares: media=/storage/media", "+ sources: recalbox=smb://nas.lan/recalbox@/mnt/recalbox"}; strings.Join(whole, "\n") != strings.Join(want, "\n") {
|
||||||
|
t.Errorf("whole: %q", whole)
|
||||||
|
}
|
||||||
|
if _, ok := outward.Secret(q.Whole, "shanks"); !ok {
|
||||||
|
t.Error("the router would refuse the whole words")
|
||||||
|
}
|
||||||
|
// Mutation: the masked and the whole change differ in exactly the lines whose value was masked.
|
||||||
|
differ := 0
|
||||||
|
for i := range masked {
|
||||||
|
if masked[i] != whole[i] {
|
||||||
|
differ++
|
||||||
|
if !strings.Contains(masked[i], "‹") || strings.Contains(whole[i], "‹") {
|
||||||
|
t.Errorf("the lines differ otherwise than by the mask:\n%s\n%s", masked[i], whole[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if differ != 2 {
|
||||||
|
t.Errorf("%d lines differ, and the mask covered 2", differ)
|
||||||
|
}
|
||||||
|
// A value shaped like a secret is withheld in both, whole, with its key still named.
|
||||||
|
r1 := newProposerRig(t)
|
||||||
|
secrets := map[string]any{"github": "ghp_abcdefghijklmnopqrstuvwxyz0123456789", "cert": "-----BEGIN CERTIFICATE-----"}
|
||||||
|
if _, err := r1.pr.propose(context.Background(), proposeInput{module: "mounts", node: "shanks", values: secrets}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
q1 := r1.askSent(t)
|
||||||
|
for _, line := range []string{"+ cert: ‹withheld›", "+ github: ‹withheld›"} {
|
||||||
|
if !strings.Contains(q1.Explanation, line) {
|
||||||
|
t.Errorf("lacks %q:\n%s", line, q1.Explanation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, secret := range []string{"ghp_", "BEGIN CERTIFICATE"} {
|
||||||
|
if strings.Contains(q1.Explanation, secret) || strings.Contains(q1.Whole, secret) || strings.Contains(q1.Details, secret) {
|
||||||
|
t.Errorf("the secret %q reaches the phone", secret)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// More keys than points (issue 639): the first two, then the keys of the rest, read whole at the terminal.
|
||||||
|
r3 := newProposerRig(t)
|
||||||
|
five := map[string]any{"a": "1", "b": "2", "c": "3", "d": "4", "e": "5"}
|
||||||
|
if _, err := r3.pr.propose(context.Background(), proposeInput{module: "notes", node: "laptop", values: five}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if q3 := r3.askSent(t); !strings.HasSuffix(q3.Explanation, "\n+ a: 1\n+ b: 2\nand 3 more, each in the note: c, d, e") || asks.Glance(q3.Explanation) != nil ||
|
||||||
|
!strings.Contains(q3.Details, "settings proposals") {
|
||||||
|
t.Errorf("five keys:\n%s", q3.Explanation)
|
||||||
|
}
|
||||||
|
// No value masked: the message is the same everywhere, and the ask says no whole words.
|
||||||
|
r2 := newProposerRig(t)
|
||||||
|
if _, err := r2.pr.propose(context.Background(), proposeInput{module: "dunst", node: "laptop", values: map[string]any{"font-size": 13, "width": 500}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if q2 := r2.askSent(t); q2.Whole != "" || !strings.HasSuffix(q2.Explanation, "\n+ font-size: 13\n+ width: 500") ||
|
||||||
|
strings.Contains(q2.Details, "not proven") {
|
||||||
|
t.Errorf("%+v", q2)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// A proposal expired unanswered is kept so by the reconciling, and a warrant for it afterwards sets nothing.
|
// A proposal expired unanswered is kept so by the reconciling, and a warrant for it afterwards sets nothing.
|
||||||
func TestAnExpiredProposalIsKeptExpired(t *testing.T) {
|
func TestAnExpiredProposalIsKeptExpired(t *testing.T) {
|
||||||
r := newAskerRig(t)
|
r := newAskerRig(t)
|
||||||
@@ -205,7 +315,8 @@ func TestAClearIsProposedAndShowsWhatItRemoves(t *testing.T) {
|
|||||||
}
|
}
|
||||||
q := r.askSent(t)
|
q := r.askSent(t)
|
||||||
if q.Headline != "Clear notes on laptop?" || !strings.Contains(q.Explanation, "- places.data.owner: 1001:1001") ||
|
if q.Headline != "Clear notes on laptop?" || !strings.Contains(q.Explanation, "- places.data.owner: 1001:1001") ||
|
||||||
!strings.Contains(q.Explanation, "- places.data.path: ‹path›") {
|
!strings.Contains(q.Explanation, "- places.data.path: ‹path›") || !strings.Contains(q.Whole, "- places.data.path: /srv/notes") ||
|
||||||
|
!strings.HasPrefix(q.Explanation, "Removes the 2 setting(s) of notes set on laptop; approved, the layer is removed and that is sent at once by a walk to laptop, one machine at a time") {
|
||||||
t.Errorf("%+v", q)
|
t.Errorf("%+v", q)
|
||||||
}
|
}
|
||||||
if p := r.theProposal(t).Proposal; !p.Clear || p.Digest != layerDigest(r.before) || len(r.judged) != 0 {
|
if p := r.theProposal(t).Proposal; !p.Clear || p.Digest != layerDigest(r.before) || len(r.judged) != 0 {
|
||||||
@@ -744,3 +855,125 @@ func captureStdout(t *testing.T, f func()) string {
|
|||||||
_ = w.Close()
|
_ = w.Close()
|
||||||
return <-done
|
return <-done
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The review of #304: Approve sets exactly what the operator can read. A proposal of five keys, one value 300
|
||||||
|
// characters long, shows two keys at a glance, says what it cut, and carries every key with its whole value in its
|
||||||
|
// Details for a channel that proves who answers (WholeDetails), and every key masked for one that does not.
|
||||||
|
func TestEveryKeyAndItsWholeValueReachTheOperator(t *testing.T) {
|
||||||
|
var long strings.Builder
|
||||||
|
long.WriteString("library=/storage")
|
||||||
|
for i := 0; long.Len() < 300; i++ {
|
||||||
|
long.WriteString(fmt.Sprintf("/album-%02d", i))
|
||||||
|
}
|
||||||
|
value := long.String()[:300]
|
||||||
|
values := map[string]any{"a": "1", "b": "2", "c": "3", "d": "4", "shares": value}
|
||||||
|
r := newProposerRig(t)
|
||||||
|
if _, err := r.pr.propose(context.Background(), proposeInput{module: "mounts", node: "shanks", values: values}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
q := r.askSent(t)
|
||||||
|
if err := q.Check(r.now); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// What the operator reads on a proving channel: the whole words, then the note's whole details.
|
||||||
|
visible := q.Whole + "\n" + q.WholeDetails
|
||||||
|
if q.Whole == "" {
|
||||||
|
visible = q.Explanation + "\n" + q.WholeDetails
|
||||||
|
}
|
||||||
|
for k, v := range values {
|
||||||
|
if !strings.Contains(visible, "+ "+k+": "+v.(string)) {
|
||||||
|
t.Errorf("the operator never reads %s whole:\n%s", k, visible)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The masked glance points to a note that holds the values masked, and says no more; the whole glance, shown only
|
||||||
|
// beside the whole details, says they are whole there.
|
||||||
|
if asks.Glance(q.Explanation) != nil || !strings.HasSuffix(q.Explanation, "\nand 3 more, each in the note: c, d, shares") ||
|
||||||
|
strings.Contains(q.Explanation, "whole in the note") || !strings.HasSuffix(q.Whole, "\nand 3 more, each whole in the note: c, d, shares") {
|
||||||
|
t.Errorf("the glance:\n%s", q.Explanation)
|
||||||
|
}
|
||||||
|
for k := range values {
|
||||||
|
if !strings.Contains(q.Details, "+ "+k+": ") {
|
||||||
|
t.Errorf("the masked details lack %s:\n%s", k, q.Details)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(q.Details, "/storage") {
|
||||||
|
t.Errorf("the masked details show a path:\n%s", q.Details)
|
||||||
|
}
|
||||||
|
if refusal, ok := outward.Check(q.Details, "shanks"); !ok {
|
||||||
|
t.Errorf("the router would withhold the masked details: %s", refusal)
|
||||||
|
}
|
||||||
|
if _, ok := outward.Secret(q.WholeDetails, "shanks"); !ok {
|
||||||
|
t.Error("the router would refuse the whole details")
|
||||||
|
}
|
||||||
|
// A value too long for a glance is cut there, and says so.
|
||||||
|
r2 := newProposerRig(t)
|
||||||
|
if _, err := r2.pr.propose(context.Background(), proposeInput{module: "mounts", node: "shanks", values: map[string]any{"shares": value}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
q2 := r2.askSent(t)
|
||||||
|
// The cut point is 160 characters, its marker inside the bound.
|
||||||
|
if last := q2.Whole[strings.LastIndex(q2.Whole, "\n")+1:]; len([]rune(last)) != asks.PointLength {
|
||||||
|
t.Errorf("the cut point is %d characters, not %d: %q", len([]rune(last)), asks.PointLength, last)
|
||||||
|
}
|
||||||
|
if !strings.HasSuffix(q2.Whole, "… (cut: whole in the note)") || !strings.HasSuffix(q2.Explanation, "‹path›") && !strings.HasSuffix(q2.Explanation, "… (cut: in the note)") || !strings.Contains(q2.WholeDetails, "+ shares: "+value) || asks.Glance(q2.Whole) != nil {
|
||||||
|
t.Errorf("the cut is not said, or the note lacks the whole:\n%s\n--\n%s", q2.Whole, q2.WholeDetails)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The review of #304: a value given to a secret's name, past the second key, is said with its key named and its
|
||||||
|
// value withheld, in the note on every channel; the router has nothing left to withhold unnamed or refuse. A line
|
||||||
|
// the router would refuse even so is refused at propose, in the controller's words.
|
||||||
|
func TestASecretNamedKeyIsNamedAndItsValueWithheld(t *testing.T) {
|
||||||
|
r := newProposerRig(t)
|
||||||
|
values := map[string]any{"a": "1", "b": "2", "c": "3", "password": "hunter2"}
|
||||||
|
if _, err := r.pr.propose(context.Background(), proposeInput{module: "notes", node: "laptop", values: values}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
q := r.askSent(t)
|
||||||
|
for _, form := range []string{q.Details, q.WholeDetails, q.Explanation, q.Whole} {
|
||||||
|
if strings.Contains(form, "hunter2") {
|
||||||
|
t.Errorf("the value reaches the phone:\n%s", form)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(q.Details, "+ password (value withheld)") {
|
||||||
|
t.Errorf("the key is not named with its value withheld:\n%s", q.Details)
|
||||||
|
}
|
||||||
|
if q.WholeDetails != "" && !strings.Contains(q.WholeDetails, "+ password (value withheld)") {
|
||||||
|
t.Errorf("the whole details do not name the key:\n%s", q.WholeDetails)
|
||||||
|
}
|
||||||
|
if line, why := judgeLines(q.Details, false, nil); line != "" {
|
||||||
|
t.Errorf("the router would withhold %q: %s", line, why)
|
||||||
|
}
|
||||||
|
if line, why := judgeLines(q.WholeDetails, true, nil); line != "" {
|
||||||
|
t.Errorf("the router would refuse %q: %s", line, why)
|
||||||
|
}
|
||||||
|
if !strings.HasSuffix(q.Explanation, "\nand 2 more, each in the note: c, password") {
|
||||||
|
t.Errorf("the glance:\n%s", q.Explanation)
|
||||||
|
}
|
||||||
|
// A key shaped like a secret cannot be named on any channel: refused at propose, in the controller's words, and
|
||||||
|
// the key itself not echoed.
|
||||||
|
r2 := newProposerRig(t)
|
||||||
|
secretKey := "ghp_abcdefghijklmnopqrstuvwxyz0123456789"
|
||||||
|
_, err := r2.pr.propose(context.Background(), proposeInput{module: "notes", node: "laptop",
|
||||||
|
values: map[string]any{"a": "1", "b": "2", secretKey: "3"}})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "Nothing was proposed") || strings.Contains(err.Error(), secretKey) {
|
||||||
|
t.Errorf("a secret-shaped key: %v", err)
|
||||||
|
}
|
||||||
|
if len(r2.sent) != 0 {
|
||||||
|
t.Errorf("the refused proposal was asked")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// keyOfLine names a change's key, and never one shaped like a secret.
|
||||||
|
func TestARefusalNamesTheKeyAndNeverASecret(t *testing.T) {
|
||||||
|
for line, want := range map[string]string{
|
||||||
|
"+ password (value withheld)": "password",
|
||||||
|
"~ shares: ‹path› (was: none)": "shares",
|
||||||
|
"+ ghp_abcdefghijklmnopqrstuvwxyz0123456789: 1": "whose key is shaped like a secret",
|
||||||
|
"Proposed by g14 at 15:04.": "Proposed by g14 at 15:04.",
|
||||||
|
} {
|
||||||
|
if got := keyOfLine(line); got != want {
|
||||||
|
t.Errorf("%q: %q, want %q", line, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+101
-5
@@ -17,6 +17,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
@@ -194,7 +195,7 @@ func serve(ctx context.Context) (err error) {
|
|||||||
// And what the catalogue decided a build meant. The builder's own result is already handled
|
// And what the catalogue decided a build meant. The builder's own result is already handled
|
||||||
// above; this is the other half — the control plane is the only one of the three that knows
|
// above; this is the other half — the control plane is the only one of the three that knows
|
||||||
// which machines run the thing, so it is the one that acts (novox/hq ADR 0072).
|
// which machines run the thing, so it is the one that acts (novox/hq ADR 0072).
|
||||||
if err := server.Follows(following{open}); err != nil {
|
if err := server.Follows(following{open: open}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// And the merges the bus announced and never handed over, read back on a timer and acted on late
|
// And the merges the bus announced and never handed over, read back on a timer and acted on late
|
||||||
@@ -202,7 +203,7 @@ func serve(ctx context.Context) (err error) {
|
|||||||
go catchingUpOnMerges(ctx, open, server)
|
go catchingUpOnMerges(ctx, open, server)
|
||||||
// And a catalogue that has just started, asking for what it missed. The same type answers
|
// And a catalogue that has just started, asking for what it missed. The same type answers
|
||||||
// both: what a build meant and what the builds were are two questions about one record.
|
// both: what a build meant and what the builds were are two questions about one record.
|
||||||
if err := server.Answers(following{open}); err != nil {
|
if err := server.Answers(following{open: open}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// And what providers say about consumers they keep failing, kept for `status` (novox/hq ADR
|
// And what providers say about consumers they keep failing, kept for `status` (novox/hq ADR
|
||||||
@@ -249,8 +250,14 @@ func serve(ctx context.Context) (err error) {
|
|||||||
// (novox/hq to-be 45 §9).
|
// (novox/hq to-be 45 §9).
|
||||||
checkEvents = bus
|
checkEvents = bus
|
||||||
// And every walk this controller keeps in a new state, for the delivery it walks (novox/hq ADR 0239).
|
// And every walk this controller keeps in a new state, for the delivery it walks (novox/hq ADR 0239).
|
||||||
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { sayPlanMoved(ctx, bus, open.inventory, p) }
|
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) {
|
||||||
if err := server.Checks(following{open}); err != nil {
|
sayPlanMoved(ctx, bus, open.inventory, p)
|
||||||
|
// A walk ended has taken what it reuses: its merges' check builds go (novox/hq ADR 0350 §5).
|
||||||
|
if p.WalkSaid == inventory.WalkEnded {
|
||||||
|
go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := server.Checks(following{open: open}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// Composed now and kept current, before the verb that answers from it is served.
|
// Composed now and kept current, before the verb that answers from it is served.
|
||||||
@@ -740,7 +747,14 @@ func waitForApplied(ctx context.Context, inv *inventory.Inventory, node, digest
|
|||||||
fmt.Printf("%s applied it\n", node)
|
fmt.Printf("%s applied it\n", node)
|
||||||
return nil
|
return nil
|
||||||
case inventory.OutcomeFailed:
|
case inventory.OutcomeFailed:
|
||||||
return fmt.Errorf("%s applied what it was sent but %d resource(s) failed", node, len(doing.Failed))
|
// Each failed resource by id with the machine's own words, so the answer says what to fix
|
||||||
|
// without a second look (novox/hq issue 629).
|
||||||
|
var each []string
|
||||||
|
for _, f := range doing.Failed {
|
||||||
|
each = append(each, f.ID+": "+f.Error)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%s applied what it was sent but %d resource(s) failed: %s", node, len(doing.Failed),
|
||||||
|
strings.Join(each, "; "))
|
||||||
case inventory.OutcomeRefused:
|
case inventory.OutcomeRefused:
|
||||||
return fmt.Errorf("%s refused what it was sent: %s", node, doing.Refused)
|
return fmt.Errorf("%s refused what it was sent: %s", node, doing.Refused)
|
||||||
}
|
}
|
||||||
@@ -1373,6 +1387,13 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
|
|||||||
if issued > 0 {
|
if issued > 0 {
|
||||||
fmt.Printf(" issued %d membership(s)\n", issued)
|
fmt.Printf(" issued %d membership(s)\n", issued)
|
||||||
}
|
}
|
||||||
|
// **And what every other machine hears of a module's stream** (novox/hq ADR 0363 §6): a grant the provider's
|
||||||
|
// setting changed reaches a consumer on a machine this send does not reach. Its membership is issued again
|
||||||
|
// as last issued, with only what it hears changed, so nothing else of it moves before its own send.
|
||||||
|
reheard := reissueHearing(ctx, bus, records, where, sent)
|
||||||
|
if reheard > 0 {
|
||||||
|
fmt.Printf(" issued %d membership(s) again for what they hear of a module's stream\n", reheard)
|
||||||
|
}
|
||||||
if len(refused) > 0 {
|
if len(refused) > 0 {
|
||||||
// Returned, never passed over (novox/hq issue 249): the declarations of the machines they are
|
// Returned, never passed over (novox/hq issue 249): the declarations of the machines they are
|
||||||
// for are not sent, and the rollout that asked is tried again rather than waiting for a push.
|
// for are not sent, and the rollout that asked is tried again rather than waiting for a push.
|
||||||
@@ -1731,3 +1752,78 @@ func schemaReach() (int, error) {
|
|||||||
}
|
}
|
||||||
return reach, nil
|
return reach, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// reissueHearing issues again, on every machine not just sent, the membership of each assignment whose
|
||||||
|
// `consumes` names a module owning a stream, when what it hears changed: read as last issued from the bus,
|
||||||
|
// its `hears` replaced, and published. One that was never issued is left to its own send. Failures are said
|
||||||
|
// and passed over: the send that carries the grant is not the consumer's, and its own next send issues it.
|
||||||
|
func reissueHearing(ctx context.Context, bus link.OverNATS, records broker.Records, where broker.Placements,
|
||||||
|
sent []readyNode) int {
|
||||||
|
reached := map[string]bool{}
|
||||||
|
for _, s := range sent {
|
||||||
|
reached[s.node] = true
|
||||||
|
}
|
||||||
|
issued := 0
|
||||||
|
for node, declared := range records.Assigned {
|
||||||
|
if reached[node] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, d := range declared {
|
||||||
|
if !d.HearsAStream {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
subject := broker.MembershipSubject(node, d.Module)
|
||||||
|
last, err := bus.JS.GetLastMsg(broker.AssignmentsStream, subject)
|
||||||
|
if err != nil {
|
||||||
|
if !errors.Is(err, nats.ErrMsgNotFound) {
|
||||||
|
fmt.Printf(" %s on %s: its membership could not be read to issue what it hears again: %v\n",
|
||||||
|
d.Module, node, err)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
body, changed, err := withHearing(last.Data, broker.HeardIssuedFor(node, d))
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf(" %s on %s: its membership could not be read to issue what it hears again: %v\n",
|
||||||
|
d.Module, node, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !changed {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := bus.PublishMembership(ctx, node, d.Module, body); err != nil {
|
||||||
|
fmt.Printf(" %s on %s: what it hears could not be issued again: %v\n", d.Module, node, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
issued++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return issued
|
||||||
|
}
|
||||||
|
|
||||||
|
// withHearing is a membership as issued with its `hears` replaced, and whether that changed it. Pure.
|
||||||
|
func withHearing(issued []byte, hears []broker.HeardIssued) ([]byte, bool, error) {
|
||||||
|
var fields map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(issued, &fields); err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
var had []broker.HeardIssued
|
||||||
|
if raw, ok := fields["hears"]; ok {
|
||||||
|
if err := json.Unmarshal(raw, &had); err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if slices.Equal(had, hears) {
|
||||||
|
return issued, false, nil
|
||||||
|
}
|
||||||
|
if len(hears) == 0 {
|
||||||
|
delete(fields, "hears")
|
||||||
|
} else {
|
||||||
|
raw, err := json.Marshal(hears)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
fields["hears"] = raw
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(fields)
|
||||||
|
return body, true, err
|
||||||
|
}
|
||||||
|
|||||||
@@ -340,7 +340,10 @@ func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat str
|
|||||||
return "", fmt.Errorf("%s is marked cancelled and could not be deleted from the queue (seq %d): %w — a "+
|
return "", fmt.Errorf("%s is marked cancelled and could not be deleted from the queue (seq %d): %w — a "+
|
||||||
"holder taking it ends it as cancelled", ask.ID, ask.Seq, err)
|
"holder taking it ends it as cancelled", ask.ID, ask.Seq, err)
|
||||||
}
|
}
|
||||||
recordCancelled(ctx, open, ask)
|
// A check ended because its pull request has a newer head records nothing (novox/hq issue 726): open is nil.
|
||||||
|
if open != nil {
|
||||||
|
recordCancelled(ctx, open, ask)
|
||||||
|
}
|
||||||
return cancelSaid(ask, seat, removed), nil
|
return cancelSaid(ask, seat, removed), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -616,6 +616,7 @@ func TestKillAndPauseAskTheHolderOnTheMachine(t *testing.T) {
|
|||||||
// A plan that stopped at its first machine is retried: the module sent to that machine again, the
|
// A plan that stopped at its first machine is retried: the module sent to that machine again, the
|
||||||
// send recorded as the first anew, and the plan goes on — unless a newer plan holds the module.
|
// send recorded as the first anew, and the plan goes on — unless a newer plan holds the module.
|
||||||
func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) {
|
func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
asked := asksRecorded(t)
|
asked := asksRecorded(t)
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"sort"
|
"sort"
|
||||||
|
|
||||||
@@ -83,6 +84,27 @@ func everyBuildKept(ctx context.Context, node string) bool {
|
|||||||
return on != "" && on == node
|
return on != "" && on == node
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type keepForSettingsKey struct{}
|
||||||
|
|
||||||
|
// keepingBuildsForSettings is a context whose sends compose every module of one machine — recorded or rolling out —
|
||||||
|
// at the build that machine was last sent, as keepingEveryBuild does, for a settings walk's send (novox/hq ADR 0344):
|
||||||
|
// the layer the operator approved, and nothing of any module's new code. A build held for a person, or waiting
|
||||||
|
// for a gate, stays where it is; the walk that is its own carries it.
|
||||||
|
func keepingBuildsForSettings(ctx context.Context, node string) context.Context {
|
||||||
|
return context.WithValue(ctx, keepForSettingsKey{}, node)
|
||||||
|
}
|
||||||
|
|
||||||
|
// errSettingsCannotKeep is a settings walk's send refused because a build of its machine cannot be kept: what the
|
||||||
|
// machine was last sent is not known, or a build it runs is gone from the records. It does not clear by itself, so
|
||||||
|
// the walk fails on it rather than trying again (ADR 0344).
|
||||||
|
var errSettingsCannotKeep = errors.New("the settings cannot be sent with every build kept")
|
||||||
|
|
||||||
|
// settingsKeptOn is whether this context is a settings walk's send to this machine.
|
||||||
|
func settingsKeptOn(ctx context.Context, node string) bool {
|
||||||
|
on, _ := ctx.Value(keepForSettingsKey{}).(string)
|
||||||
|
return on != "" && on == node
|
||||||
|
}
|
||||||
|
|
||||||
// recordedKept is, for a send under keepingRecorded, every recorded module the machine was last sent a
|
// recordedKept is, for a send under keepingRecorded, every recorded module the machine was last sent a
|
||||||
// build of that the mesh's build is not identical to: module → the commit it keeps. Nil when the context
|
// build of that the mesh's build is not identical to: module → the commit it keeps. Nil when the context
|
||||||
// keeps nothing, or when what the machine was last sent is not known (it is then held whole elsewhere —
|
// keeps nothing, or when what the machine was last sent is not known (it is then held whole elsewhere —
|
||||||
@@ -92,7 +114,7 @@ func everyBuildKept(ctx context.Context, node string) bool {
|
|||||||
// 490), and a machine whose last send is not known refuses the send: there is nothing to keep it at, and
|
// 490), and a machine whose last send is not known refuses the send: there is nothing to keep it at, and
|
||||||
// composing the mesh's builds would be the very move the grants step must not make.
|
// composing the mesh's builds would be the very move the grants step must not make.
|
||||||
func recordedKept(ctx context.Context, open *stores, node string) (map[string]string, error) {
|
func recordedKept(ctx context.Context, open *stores, node string) (map[string]string, error) {
|
||||||
every := everyBuildKept(ctx, node)
|
every := everyBuildKept(ctx, node) || settingsKeptOn(ctx, node)
|
||||||
skip, on := keptExcept(ctx)
|
skip, on := keptExcept(ctx)
|
||||||
if !on && !every {
|
if !on && !every {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
@@ -106,6 +128,9 @@ func recordedKept(ctx context.Context, open *stores, node string) (map[string]st
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if !known {
|
if !known {
|
||||||
|
if settingsKeptOn(ctx, node) {
|
||||||
|
return nil, fmt.Errorf("%w: what %s was last sent is not known (novox/hq ADR 0344)", errSettingsCannotKeep, node)
|
||||||
|
}
|
||||||
if every {
|
if every {
|
||||||
return nil, fmt.Errorf("what %s was last sent is not known, so the bus's user list cannot be sent "+
|
return nil, fmt.Errorf("what %s was last sent is not known, so the bus's user list cannot be sent "+
|
||||||
"there alone with every build kept (novox/hq issue 490)", node)
|
"there alone with every build kept (novox/hq issue 490)", node)
|
||||||
@@ -166,6 +191,10 @@ func keepRecorded(ctx context.Context, open *stores, node string, shelf map[stri
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if !found && settingsKeptOn(ctx, node) {
|
||||||
|
return nil, fmt.Errorf("%w: %s runs %s's build %s, which the build records no longer hold (novox/hq ADR 0344)",
|
||||||
|
errSettingsCannotKeep, node, m, short(kept[m]))
|
||||||
|
}
|
||||||
if !found && everyBuildKept(ctx, node) {
|
if !found && everyBuildKept(ctx, node) {
|
||||||
return nil, fmt.Errorf("%s runs %s's build %s, which the build records no longer hold: the bus's user "+
|
return nil, fmt.Errorf("%s runs %s's build %s, which the build records no longer hold: the bus's user "+
|
||||||
"list cannot be sent there alone with it kept (novox/hq issue 490)", node, m, short(kept[m]))
|
"list cannot be sent there alone with it kept (novox/hq issue 490)", node, m, short(kept[m]))
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A membership issued again for what it hears changes nothing else of it (novox/hq ADR 0363 §6).
|
||||||
|
func TestAMembershipIssuedAgainChangesOnlyWhatItHears(t *testing.T) {
|
||||||
|
issued := []byte(`{"node":"ace","module":"reader","receives":{"x":[1]},"emits":"mesh.mod.reader.event.{event}"}`)
|
||||||
|
hears := []broker.HeardIssued{{Stream: "MAIL", Consumer: "ace_reader"}}
|
||||||
|
body, changed, err := withHearing(issued, hears)
|
||||||
|
if err != nil || !changed {
|
||||||
|
t.Fatalf("changed %v, %v", changed, err)
|
||||||
|
}
|
||||||
|
var got map[string]any
|
||||||
|
if err := json.Unmarshal(body, &got); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got["receives"] == nil || got["emits"] != "mesh.mod.reader.event.{event}" || got["hears"] == nil {
|
||||||
|
t.Fatalf("issued again as %s", body)
|
||||||
|
}
|
||||||
|
if _, again, _ := withHearing(body, hears); again {
|
||||||
|
t.Fatal("the same hearing issued again counts as a change")
|
||||||
|
}
|
||||||
|
gone, changed, err := withHearing(body, nil)
|
||||||
|
if err != nil || !changed {
|
||||||
|
t.Fatalf("a withdrawal: %v, %v", changed, err)
|
||||||
|
}
|
||||||
|
got = nil
|
||||||
|
if err := json.Unmarshal(gone, &got); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, still := got["hears"]; still {
|
||||||
|
t.Fatalf("a withdrawn hearing is still issued: %s", gone)
|
||||||
|
}
|
||||||
|
}
|
||||||
+578
-43
@@ -39,8 +39,8 @@ import (
|
|||||||
// from the same manifest is no move.
|
// from the same manifest is no move.
|
||||||
//
|
//
|
||||||
// **The release plan** is what walks the waiting moves through: whenever moves wait for a gate that no
|
// **The release plan** is what walks the waiting moves through: whenever moves wait for a gate that no
|
||||||
// started plan is walking, the mesh opens one — every such machine, one at a time, the control node last,
|
// started plan is walking, the mesh opens one — every such machine, the first sent and judged, then the rest
|
||||||
// each sent and judged before the next. A release plan that fails stops the next one opening on its own:
|
// at once, each judged by its own gate (novox/hq ADR 0353, in place of one at a time). A release plan that fails stops the next one opening on its own:
|
||||||
// a person releases it again (`upgrade release-backlog --why`), after the condition says why.
|
// a person releases it again (`upgrade release-backlog --why`), after the condition says why.
|
||||||
|
|
||||||
// sendScope is what a send may carry that no gate has seen.
|
// sendScope is what a send may carry that no gate has seen.
|
||||||
@@ -81,6 +81,15 @@ type moveFacts struct {
|
|||||||
srcs map[string]map[string]string
|
srcs map[string]map[string]string
|
||||||
passed map[string]map[string]bool
|
passed map[string]map[string]bool
|
||||||
plans []inventory.Plan
|
plans []inventory.Plan
|
||||||
|
// self is the release walking these moves, whose own gates are no other walk's (novox/hq ADR 0353).
|
||||||
|
self string
|
||||||
|
}
|
||||||
|
|
||||||
|
type walkingKey struct{}
|
||||||
|
|
||||||
|
// withWalking says a release plan is the one sending: its own gates never hold its own sends back.
|
||||||
|
func withWalking(ctx context.Context, id string) context.Context {
|
||||||
|
return context.WithValue(ctx, walkingKey{}, id)
|
||||||
}
|
}
|
||||||
|
|
||||||
func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, error) {
|
func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, error) {
|
||||||
@@ -99,6 +108,7 @@ func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, er
|
|||||||
return f, err
|
return f, err
|
||||||
}
|
}
|
||||||
f.plans, err = inv.OpenPlans(ctx)
|
f.plans, err = inv.OpenPlans(ctx)
|
||||||
|
f.self, _ = ctx.Value(walkingKey{}).(string)
|
||||||
return f, err
|
return f, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -193,17 +203,24 @@ func (f moveFacts) moves(node string, modules []string, sent map[string]string,
|
|||||||
// feet. A release keeps no module records: its open gate's carried moves are its walk.
|
// feet. A release keeps no module records: its open gate's carried moves are its walk.
|
||||||
func (f moveFacts) walkedBy(module, node, to string) string {
|
func (f moveFacts) walkedBy(module, node, to string) string {
|
||||||
for _, p := range f.plans {
|
for _, p := range f.plans {
|
||||||
if !p.Open() {
|
if !p.Open() || (p.Release != nil && p.ID == f.self) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if p.Release != nil {
|
if p.Release != nil {
|
||||||
g := p.Release.Gate
|
// The first machine's gate, and every gate of the rest still judging (novox/hq ADR 0353).
|
||||||
if g == nil || g.Verdict != "" {
|
gates := []*inventory.PlanGate{p.Release.Gate}
|
||||||
continue
|
for _, g := range p.Release.Gates {
|
||||||
|
gates = append(gates, g)
|
||||||
}
|
}
|
||||||
for _, c := range g.Carried {
|
for _, g := range gates {
|
||||||
if c.Module == module && !(slices.Contains(g.Machines, node) && sameCommit(c.To, to)) {
|
// A send kept and not yet made walks nothing yet (ADR 0353 §8).
|
||||||
return p.ID
|
if g == nil || g.Verdict != "" || g.Silent || g.Pending {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, c := range g.Carried {
|
||||||
|
if c.Module == module && !(slices.Contains(g.Machines, node) && sameCommit(c.To, to)) {
|
||||||
|
return p.ID
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
@@ -303,8 +320,8 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(refused) > 0 {
|
if len(refused) > 0 {
|
||||||
return nil, fmt.Errorf("%w: %s — a walk sends them, one machine at a time, each judged (novox/hq ADR "+
|
return nil, fmt.Errorf("%w: %s — a walk sends them, one machine first and then the rest at once, each judged (novox/hq ADR "+
|
||||||
"0236); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))
|
"0236, ADR 0353); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))
|
||||||
}
|
}
|
||||||
return kept, nil
|
return kept, nil
|
||||||
}
|
}
|
||||||
@@ -320,19 +337,35 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
|
|||||||
// And the grants step first, when the send changes what the bus's user list must say (novox/hq issue 490):
|
// And the grants step first, when the send changes what the bus's user list must say (novox/hq issue 490):
|
||||||
// answered for the gate to keep, nil when there was none.
|
// answered for the gate to keep, nil when there was none.
|
||||||
func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, *inventory.GrantsStep, error) {
|
func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, *inventory.GrantsStep, error) {
|
||||||
inv := open.inventory
|
moves, err := gatedMoves(ctx, open, node, owns)
|
||||||
f, err := readMoveFacts(ctx, inv)
|
if err != nil || len(moves) == 0 {
|
||||||
|
return nil, nil, nil, err
|
||||||
|
}
|
||||||
|
sayRecreations(ctx, open, moves)
|
||||||
|
sent, grants, err := sendJudged(ctx, open, node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, err
|
return nil, nil, nil, err
|
||||||
}
|
}
|
||||||
|
return moves, sent, grants, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// gatedMoves is what a gated send to one machine would carry, with the build each moves to, made of nothing:
|
||||||
|
// gatedSend's reading, which a walk also keeps before it sends (novox/hq ADR 0353 §8). Refused the same way
|
||||||
|
// while a plan that has started walks one of those builds elsewhere.
|
||||||
|
func gatedMoves(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, error) {
|
||||||
|
inv := open.inventory
|
||||||
|
f, err := readMoveFacts(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
moves, err := machineMoves(ctx, open, f, node, true)
|
moves, err := machineMoves(ctx, open, f, node, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
// A move the send keeps is not made, and is no move for the gate to judge (novox/hq issue 201): a core module
|
// A move the send keeps is not made, and is no move for the gate to judge (novox/hq issue 201): a core module
|
||||||
// kept over an older record judged as moving back to it would read the gate as superseded by its own send.
|
// kept over an older record judged as moving back to it would read the gate as superseded by its own send.
|
||||||
if moves, err = withoutKept(ctx, open, node, moves); err != nil {
|
if moves, err = withoutKept(ctx, open, node, moves); err != nil {
|
||||||
return nil, nil, nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
own := func(module string) bool {
|
own := func(module string) bool {
|
||||||
return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module })
|
return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module })
|
||||||
@@ -342,7 +375,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if id := f.walkedBy(mv.Module, node, mv.To); id != "" {
|
if id := f.walkedBy(mv.Module, node, mv.To); id != "" {
|
||||||
return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
return nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||||
mv.Module, short(mv.To), node, id)
|
mv.Module, short(mv.To), node, id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -350,7 +383,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
|||||||
// plan, or a release, on this machine is not crossed with a newer build; this send waits for its gate.
|
// plan, or a release, on this machine is not crossed with a newer build; this send waits for its gate.
|
||||||
for _, o := range owns {
|
for _, o := range owns {
|
||||||
if id := f.walkedBy(o.Module, node, o.To); id != "" {
|
if id := f.walkedBy(o.Module, node, o.To); id != "" {
|
||||||
return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
return nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||||
o.Module, short(o.To), node, id)
|
o.Module, short(o.To), node, id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -363,30 +396,26 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
|||||||
moves[i].Build = o.Build
|
moves[i].Build = o.Build
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(moves) == 0 && len(owns) == 0 {
|
if len(moves) == 0 {
|
||||||
return nil, nil, nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
for i := range moves {
|
for i := range moves {
|
||||||
if moves[i].Build == "" {
|
if moves[i].Build == "" {
|
||||||
if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil {
|
if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil {
|
||||||
return nil, nil, nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module })
|
sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module })
|
||||||
sayRecreations(ctx, open, moves)
|
return moves, nil
|
||||||
sent, grants, err := sendJudged(ctx, open, node)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, nil, err
|
|
||||||
}
|
|
||||||
return moves, sent, grants, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// passCarried keeps a pass as the verdict of every build the gate judged beside its own module.
|
// passCarried keeps a pass as the verdict of every build the gate judged beside its own module.
|
||||||
func passCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) {
|
func passCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) {
|
||||||
seen := map[string]bool{}
|
seen := map[string]bool{}
|
||||||
for _, c := range g.Carried {
|
for _, c := range g.Carried {
|
||||||
if c.Module == except || c.Build == "" || seen[c.Build] {
|
// A build put back with another machine's failure is not this gate's to pass (novox/hq ADR 0353 §3).
|
||||||
|
if c.Module == except || c.Build == "" || seen[c.Build] || slices.Contains(g.Returned, c.Module) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
seen[c.Build] = true
|
seen[c.Build] = true
|
||||||
@@ -616,6 +645,10 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl
|
|||||||
}
|
}
|
||||||
for _, p := range plans {
|
for _, p := range plans {
|
||||||
if p.Release != nil {
|
if p.Release != nil {
|
||||||
|
if by != "" && p.Release.Settings != nil {
|
||||||
|
return nil, fmt.Errorf("%s is sending %s; what waits is released once it ends — `plans %s` says where it is",
|
||||||
|
p.ID, p.Repository, p.ID)
|
||||||
|
}
|
||||||
if by != "" {
|
if by != "" {
|
||||||
return nil, fmt.Errorf("%s is already releasing what waits; `plans %s` says where it is", p.ID, p.ID)
|
return nil, fmt.Errorf("%s is already releasing what waits; `plans %s` says where it is", p.ID, p.ID)
|
||||||
}
|
}
|
||||||
@@ -641,7 +674,9 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
for _, p := range recent {
|
for _, p := range recent {
|
||||||
if p.Release == nil {
|
// A settings walk that failed holds nothing back (novox/hq ADR 0344 §6): its failure is its module's
|
||||||
|
// settings on one machine, said on its own, not a build no gate passed.
|
||||||
|
if p.Release == nil || p.Release.Settings != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if p.State == inventory.PlanFailed {
|
if p.State == inventory.PlanFailed {
|
||||||
@@ -689,7 +724,7 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl
|
|||||||
p := inventory.Plan{ID: fmt.Sprintf("release-%d", now.UnixNano()), Repository: releaseRepository,
|
p := inventory.Plan{ID: fmt.Sprintf("release-%d", now.UnixNano()), Repository: releaseRepository,
|
||||||
Created: now, State: inventory.PlanRolling, Tiers: [][]string{names}, Modules: map[string]*inventory.PlanModule{},
|
Created: now, State: inventory.PlanRolling, Tiers: [][]string{names}, Modules: map[string]*inventory.PlanModule{},
|
||||||
Release: &inventory.PlanRelease{Order: order, By: by},
|
Release: &inventory.PlanRelease{Order: order, By: by},
|
||||||
Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine at a time, each judged", len(names),
|
Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine first, then the rest at once, each judged", len(names),
|
||||||
strings.Join(order, ", "))}
|
strings.Join(order, ", "))}
|
||||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -698,11 +733,28 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl
|
|||||||
return &p, nil
|
return &p, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// advanceRelease takes one step of a release plan: the next machine sent everything waiting there under a
|
// advanceRelease takes one step of a release plan: the first machine sent everything waiting there under a
|
||||||
// gate, or the machine being judged judged once more; the plan done when every machine is.
|
// gate and judged; once it passed, every other machine sent at once, each under its own gate (novox/hq ADR
|
||||||
|
// 0353 §1), and judged; the plan done when every machine is.
|
||||||
|
//
|
||||||
|
// **A send is kept before it is made** (ADR 0353 §8): the step that chooses a machine keeps its gate pending with
|
||||||
|
// what it will carry, and the next step makes the send. A controller replaced between the send and keeping it
|
||||||
|
// finds the gate pending, reads the machine's last send, and neither sends again nor leaves the machine unjudged.
|
||||||
func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) {
|
func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) {
|
||||||
r := p.Release
|
r := p.Release
|
||||||
|
if r.Settings != nil {
|
||||||
|
return advanceSettingsWalk(ctx, open, p)
|
||||||
|
}
|
||||||
|
ctx = withWalking(ctx, p.ID)
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
|
if r.Rest {
|
||||||
|
return advanceRest(ctx, open, p, now)
|
||||||
|
}
|
||||||
|
if r.Gate == nil && !gateFirstMachine {
|
||||||
|
// **No first machine** (novox/hq ADR 0360 §7): every machine heard from is sent at once, in one step.
|
||||||
|
r.Rest = true
|
||||||
|
return advanceRest(ctx, open, p, now)
|
||||||
|
}
|
||||||
if r.Gate == nil {
|
if r.Gate == nil {
|
||||||
heard, err := releaseHeard(ctx, open)
|
heard, err := releaseHeard(ctx, open)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -715,7 +767,7 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
|||||||
r.Next++
|
r.Next++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
moves, sent, grants, err := gatedSend(ctx, open, node, nil)
|
moves, err := gatedMoves(ctx, open, node, nil)
|
||||||
if errors.Is(err, errWalkedElsewhere) {
|
if errors.Is(err, errWalkedElsewhere) {
|
||||||
note := fmt.Sprintf("waiting before %s: %v", node, err)
|
note := fmt.Sprintf("waiting before %s: %v", node, err)
|
||||||
changed := p.Note != note
|
changed := p.Note != note
|
||||||
@@ -723,23 +775,15 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
|||||||
return changed, nil
|
return changed, nil
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("sending %s what waits there: %w", node, err)
|
return false, fmt.Errorf("reading what waits on %s: %w", node, err)
|
||||||
}
|
}
|
||||||
if len(moves) == 0 {
|
if len(moves) == 0 {
|
||||||
r.Done = append(r.Done, node)
|
r.Done = append(r.Done, node)
|
||||||
r.Next++
|
r.Next++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves, Sent: sentNow(ctx, open.inventory, sent),
|
r.Gate = &inventory.PlanGate{Machines: []string{node}, Carried: moves, Pending: true}
|
||||||
Grants: grants}
|
p.Note = fmt.Sprintf("sending %s %d build(s) that wait for a gate, first", node, len(moves))
|
||||||
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
|
|
||||||
if said := grantsSaid(grants); said != "" {
|
|
||||||
p.Note += "; " + said
|
|
||||||
}
|
|
||||||
if said := recreationsSaid(moves); said != "" {
|
|
||||||
p.Note += "; " + said
|
|
||||||
}
|
|
||||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
p.State, p.Tier = inventory.PlanDone, len(p.Tiers)
|
p.State, p.Tier = inventory.PlanDone, len(p.Tiers)
|
||||||
@@ -750,6 +794,19 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
|||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
g := r.Gate
|
g := r.Gate
|
||||||
|
if g.Pending {
|
||||||
|
node := firstOf(g.Machines)
|
||||||
|
out, err := sendKept(ctx, open, p, node, g, now)
|
||||||
|
if err != nil || out == keptWaiting {
|
||||||
|
return out == keptWaiting, err
|
||||||
|
}
|
||||||
|
if out == keptNothing {
|
||||||
|
r.Gate = nil
|
||||||
|
r.Done = append(r.Done, node)
|
||||||
|
r.Next++
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
verdict, err := judgeMoves(ctx, open, g, nil, now)
|
verdict, err := judgeMoves(ctx, open, g, nil, now)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
@@ -768,6 +825,11 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
|||||||
r.Done = append(r.Done, firstOf(g.Machines))
|
r.Done = append(r.Done, firstOf(g.Machines))
|
||||||
r.Next++
|
r.Next++
|
||||||
r.Gate = nil
|
r.Gate = nil
|
||||||
|
// **Then the rest at once** (novox/hq ADR 0353 §1): every other machine heard from, each judged by its own
|
||||||
|
// gate, in place of one machine at a time (ADR 0236 §22).
|
||||||
|
r.Rest = true
|
||||||
|
p.Note = fmt.Sprintf("passed on %s; the rest are sent at once, each judged", firstOf(g.Machines))
|
||||||
|
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||||
return true, nil
|
return true, nil
|
||||||
case inventory.GateSuperseded:
|
case inventory.GateSuperseded:
|
||||||
// Another send moved a judged module on the judged machine (novox/hq issue 352): no verdict on what
|
// Another send moved a judged module on the judged machine (novox/hq issue 352): no verdict on what
|
||||||
@@ -788,6 +850,479 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
|||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// What sendKept did with a kept send.
|
||||||
|
const (
|
||||||
|
keptSent = iota // sent now
|
||||||
|
keptAlready // the machine was already sent it: judged from that send, nothing sent
|
||||||
|
keptNothing // nothing waits there any more: no gate
|
||||||
|
keptWaiting // another walk judges a build it would carry: it waits
|
||||||
|
)
|
||||||
|
|
||||||
|
// sendKept makes a send a gate kept before it was made (novox/hq ADR 0353 §8) — unless the machine's last send
|
||||||
|
// already carries what the gate is for, made before a controller was replaced and before it was kept: then
|
||||||
|
// nothing is sent, and the gate judges from that send (§6).
|
||||||
|
func sendKept(ctx context.Context, open *stores, p *inventory.Plan, node string, g *inventory.PlanGate,
|
||||||
|
now time.Time) (int, error) {
|
||||||
|
inv := open.inventory
|
||||||
|
moves, sent, grants, err := gatedSend(ctx, open, node, nil)
|
||||||
|
if errors.Is(err, errWalkedElsewhere) {
|
||||||
|
p.Note = fmt.Sprintf("waiting before %s: %v", node, err)
|
||||||
|
return keptWaiting, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return keptSent, fmt.Errorf("sending %s what waits there: %w", node, err)
|
||||||
|
}
|
||||||
|
if len(moves) > 0 {
|
||||||
|
g.Machines, g.Carried, g.Since, g.Grants, g.Pending = sent, moves, &now, grants, false
|
||||||
|
g.Sent = sentNow(ctx, inv, sent)
|
||||||
|
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
|
||||||
|
if said := grantsSaid(grants); said != "" {
|
||||||
|
p.Note += "; " + said
|
||||||
|
}
|
||||||
|
if said := recreationsSaid(moves); said != "" {
|
||||||
|
p.Note += "; " + said
|
||||||
|
}
|
||||||
|
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||||
|
return keptSent, nil
|
||||||
|
}
|
||||||
|
if !alreadySent(ctx, inv, node, g.Carried) {
|
||||||
|
return keptNothing, nil
|
||||||
|
}
|
||||||
|
since := now
|
||||||
|
if s, found, err := inv.SentTo(ctx, node); err == nil && found {
|
||||||
|
if !s.At.IsZero() {
|
||||||
|
since = s.At
|
||||||
|
}
|
||||||
|
g.Sent = map[string]inventory.SentDeclaration{node: s}
|
||||||
|
}
|
||||||
|
g.Since, g.Pending = &since, false
|
||||||
|
p.Note = fmt.Sprintf("%s was already sent the %d build(s) kept for it, at %s: judged from that send, nothing sent again",
|
||||||
|
node, len(g.Carried), since.Local().Format("15:04:05"))
|
||||||
|
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||||
|
return keptAlready, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// alreadySent says a machine's last send carries every build of the moves: a send made and not kept.
|
||||||
|
func alreadySent(ctx context.Context, inv *inventory.Inventory, node string, moves []inventory.CarriedMove) bool {
|
||||||
|
if len(moves) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
sent, known, err := inv.SentBuilds(ctx, node)
|
||||||
|
if err != nil || !known {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, mv := range moves {
|
||||||
|
if !sameCommit(sent[mv.Module], mv.To) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// advanceRest takes one step of a release plan's rest (novox/hq ADR 0353): the sends kept for a step made, every
|
||||||
|
// machine of the step judged by its own gate, one that fails put back by itself while the others go on to their
|
||||||
|
// verdicts, and the next step — or the end — once every gate of the step has its verdict or is silent.
|
||||||
|
func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.Time) (bool, error) {
|
||||||
|
r := p.Release
|
||||||
|
// The sends kept for this step, in the plan's order: the control node, last in it, last (§1).
|
||||||
|
var pending []string
|
||||||
|
for _, n := range r.Order {
|
||||||
|
if g := r.Gates[n]; g != nil && g.Pending {
|
||||||
|
pending = append(pending, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(pending) > 0 {
|
||||||
|
// **A kept send carries only what was kept** (ADR 0353 §2): a build that passed no gate and came to wait on
|
||||||
|
// a machine since its step was chosen could reach a second machine in it; the step is chosen again.
|
||||||
|
if grew, err := keptStepGrew(ctx, open, r, pending); err != nil {
|
||||||
|
return false, err
|
||||||
|
} else if grew != "" {
|
||||||
|
for _, n := range pending {
|
||||||
|
delete(r.Gates, n)
|
||||||
|
}
|
||||||
|
opened, waiting, err := openRestStep(ctx, open, p)
|
||||||
|
if err != nil {
|
||||||
|
return true, err
|
||||||
|
}
|
||||||
|
p.Note = "chosen again: " + grew
|
||||||
|
if !opened && waiting != "" {
|
||||||
|
p.Note += "; " + waiting
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
var sent, already, waiting []string
|
||||||
|
for _, node := range pending {
|
||||||
|
out, err := sendKept(ctx, open, p, node, r.Gates[node], now)
|
||||||
|
if err != nil {
|
||||||
|
return true, err
|
||||||
|
}
|
||||||
|
switch out {
|
||||||
|
case keptSent:
|
||||||
|
sent = append(sent, node)
|
||||||
|
case keptAlready:
|
||||||
|
already = append(already, node)
|
||||||
|
case keptNothing:
|
||||||
|
delete(r.Gates, node)
|
||||||
|
r.Done = append(r.Done, node)
|
||||||
|
case keptWaiting:
|
||||||
|
waiting = append(waiting, node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var said []string
|
||||||
|
if len(sent) > 0 {
|
||||||
|
said = append(said, "sent "+strings.Join(sent, ", ")+" at once, each judged there")
|
||||||
|
}
|
||||||
|
if len(already) > 0 {
|
||||||
|
said = append(said, strings.Join(already, ", ")+" already sent it: judged from that send")
|
||||||
|
}
|
||||||
|
if len(waiting) > 0 {
|
||||||
|
said = append(said, "waiting before "+strings.Join(waiting, ", ")+": another walk judges a build it carries")
|
||||||
|
}
|
||||||
|
if len(said) > 0 {
|
||||||
|
p.Note = strings.Join(said, "; ")
|
||||||
|
}
|
||||||
|
return len(sent)+len(already) > 0 || len(waiting) < len(pending), nil
|
||||||
|
}
|
||||||
|
reports, err := open.inventory.LastReports(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
byNode := map[string]inventory.Reported{}
|
||||||
|
for _, rep := range reports {
|
||||||
|
byNode[rep.Node] = rep
|
||||||
|
}
|
||||||
|
changed, noteWas := false, p.Note
|
||||||
|
var notes, judging []string
|
||||||
|
batched, back := batchingRollbacks(ctx)
|
||||||
|
for _, node := range r.Order {
|
||||||
|
g := r.Gates[node]
|
||||||
|
if g == nil || g.Verdict != "" || g.Silent {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// What another machine's failure already put back is not this gate's to judge (ADR 0353 §3).
|
||||||
|
if len(g.Carried) > 0 && len(g.Returned) >= len(g.Carried) {
|
||||||
|
decide(g, inventory.GateFailed, "what it was sent was put back with another machine's failure", now)
|
||||||
|
r.Failed = append(r.Failed, node)
|
||||||
|
changed = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// **A machine that says nothing on its send is left silent, not failed** (ADR 0353 §4): it is not judged
|
||||||
|
// until it reports, and past ApplySilentAfter it is listed and left.
|
||||||
|
if !reportedOnSend(byNode[node], node, g) {
|
||||||
|
if now.Sub(*g.Since) > inventory.ApplySilentAfter {
|
||||||
|
g.Silent = true
|
||||||
|
r.Silent = append(r.Silent, node)
|
||||||
|
changed = true
|
||||||
|
notes = append(notes, fmt.Sprintf("%s said nothing on its send within %s: left silent", node,
|
||||||
|
inventory.ApplySilentAfter))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
judging = append(judging, node+": not yet reported on its send")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !gateFirstMachine {
|
||||||
|
// **Judged by its report alone** (novox/hq ADR 0360 §7): applied passes; failed or refused is named and
|
||||||
|
// fails the walk, with nothing put back — apply-failing says it.
|
||||||
|
switch rep := byNode[node]; rep.Outcome {
|
||||||
|
case inventory.OutcomeApplied:
|
||||||
|
decide(g, inventory.GatePassed, node+" reported it applied its send (novox/hq ADR 0360)", now)
|
||||||
|
passCarried(ctx, open, p, g, "")
|
||||||
|
r.Done = append(r.Done, node)
|
||||||
|
changed = true
|
||||||
|
case inventory.OutcomeFailed, inventory.OutcomeRefused:
|
||||||
|
decide(g, inventory.GateFailed, fmt.Sprintf("%s %s its send", node, rep.Outcome), now)
|
||||||
|
r.Failed = append(r.Failed, node)
|
||||||
|
changed = true
|
||||||
|
notes = append(notes, fmt.Sprintf("%s %s its send: nothing put back, apply-failing says it", node, rep.Outcome))
|
||||||
|
default:
|
||||||
|
judging = append(judging, node+": not yet reported on its send")
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
verdict, err := judgeMoves(ctx, open, g, nil, now)
|
||||||
|
if err != nil {
|
||||||
|
return changed, err
|
||||||
|
}
|
||||||
|
switch verdict {
|
||||||
|
case "":
|
||||||
|
if putBackBroken(ctx, open, p, g, "", nil) {
|
||||||
|
changed = true
|
||||||
|
notes = append(notes, p.Note)
|
||||||
|
}
|
||||||
|
judging = append(judging, node+": "+gateLine(g))
|
||||||
|
case inventory.GatePassed:
|
||||||
|
passCarried(ctx, open, p, g, "")
|
||||||
|
r.Done = append(r.Done, node)
|
||||||
|
changed = true
|
||||||
|
case inventory.GateSuperseded:
|
||||||
|
// Another send moved a judged module there (novox/hq issue 352): nothing judged, nothing put back on
|
||||||
|
// it; what still waits is released again.
|
||||||
|
r.Done = append(r.Done, node)
|
||||||
|
changed = true
|
||||||
|
notes = append(notes, fmt.Sprintf("superseded on %s: %s", node, g.Why))
|
||||||
|
default:
|
||||||
|
// **Put back by itself** (ADR 0353 §3): what was found wanting goes back at once, and every other
|
||||||
|
// machine of the step goes on to its own verdict.
|
||||||
|
stateWas := p.State
|
||||||
|
p.Note = ""
|
||||||
|
failCarried(batched, open, p, g, "")
|
||||||
|
p.State = stateWas
|
||||||
|
r.Failed = append(r.Failed, node)
|
||||||
|
changed = true
|
||||||
|
notes = append(notes, fmt.Sprintf("failed its gate on %s: %s — %s", node, g.Why, p.Note))
|
||||||
|
returnedElsewhere(ctx, open, r, node, g)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
p.Note = ""
|
||||||
|
sendRollbacks(ctx, open, p, back)
|
||||||
|
if p.Note != "" {
|
||||||
|
notes = append(notes, strings.TrimPrefix(p.Note, "; "))
|
||||||
|
}
|
||||||
|
if len(judging) > 0 {
|
||||||
|
note := strings.Join(append(notes, "judging "+strings.Join(judging, "; ")), "; ")
|
||||||
|
p.Note = note
|
||||||
|
return changed || noteWas != note, nil
|
||||||
|
}
|
||||||
|
// Every gate of the step has its verdict or is silent: the next step, unless a machine failed (§3).
|
||||||
|
if len(r.Failed) == 0 {
|
||||||
|
opened, waiting, err := openRestStep(ctx, open, p)
|
||||||
|
if err != nil {
|
||||||
|
return changed, err
|
||||||
|
}
|
||||||
|
if opened {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
if waiting != "" {
|
||||||
|
note := strings.Join(append(notes, waiting), "; ")
|
||||||
|
p.Note = note
|
||||||
|
return changed || noteWas != note, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
p.Tier = len(p.Tiers)
|
||||||
|
end := fmt.Sprintf("released on %s", orNone(strings.Join(r.Done, ", ")))
|
||||||
|
if len(r.Silent) > 0 {
|
||||||
|
end += "; silent on their send, left: " + strings.Join(r.Silent, ", ")
|
||||||
|
}
|
||||||
|
if len(r.Skipped) > 0 {
|
||||||
|
end += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ")
|
||||||
|
}
|
||||||
|
p.State = inventory.PlanDone
|
||||||
|
if len(r.Failed) > 0 {
|
||||||
|
p.State = inventory.PlanFailed
|
||||||
|
end = "failed on " + strings.Join(r.Failed, ", ") + "; " + end
|
||||||
|
} else if len(r.Silent) > 0 && len(r.Gates) == len(r.Silent) {
|
||||||
|
// **A rest that said nothing is no success** (ADR 0353 §4): failure is loud; the condition says it.
|
||||||
|
p.State = inventory.PlanFailed
|
||||||
|
end = "no machine of the rest reported on its send; " + end
|
||||||
|
}
|
||||||
|
p.Note = strings.Join(append(notes, end), "; ")
|
||||||
|
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// keptStepGrew says why a step's kept sends would now carry a build no gate passed that they were not kept with;
|
||||||
|
// empty when none would.
|
||||||
|
func keptStepGrew(ctx context.Context, open *stores, r *inventory.PlanRelease, pending []string) (string, error) {
|
||||||
|
if !gateFirstMachine {
|
||||||
|
return "", nil // every machine goes at once (novox/hq ADR 0360 §7): a send carries what waits when it is made
|
||||||
|
}
|
||||||
|
f, err := readMoveFacts(ctx, open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
shelf, err := open.inventory.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
for _, node := range pending {
|
||||||
|
moves, err := gatedMoves(ctx, open, node, nil)
|
||||||
|
if errors.Is(err, errWalkedElsewhere) {
|
||||||
|
continue // sendKept says it waits
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
kept := r.Gates[node].Carried
|
||||||
|
for _, mv := range moves {
|
||||||
|
// The node-engine and the resolver keep one machine at a time, passed or not (ADR 0353 §9).
|
||||||
|
if (f.gated(mv.Module, mv.To) && !oneAtATime(mv.Module, shelf[mv.Module])) || slices.ContainsFunc(kept, func(c inventory.CarriedMove) bool {
|
||||||
|
return c.Module == mv.Module && sameCommit(c.To, mv.To)
|
||||||
|
}) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s's build %s, which this step did not keep for it, came to wait on %s", mv.Module, short(mv.To), node), nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// reportedOnSend says a machine's last report is about the send its gate judges (novox/hq issue 352): the
|
||||||
|
// declaration kept on the gate, or one sequenced after it; a gate that kept none reads the send made last.
|
||||||
|
func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bool {
|
||||||
|
if r.At == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if s, kept := g.Sent[node]; kept {
|
||||||
|
return s.ReportsOn(r)
|
||||||
|
}
|
||||||
|
return r.Current
|
||||||
|
}
|
||||||
|
|
||||||
|
// returnedElsewhere marks, on every other gate of the rest still judging, the builds a failed machine's
|
||||||
|
// rollback put back wherever they were sent (ADR 0236 §5): no longer theirs to judge (ADR 0353 §3).
|
||||||
|
func returnedElsewhere(ctx context.Context, open *stores, r *inventory.PlanRelease, failed string, g *inventory.PlanGate) {
|
||||||
|
for _, c := range g.Carried {
|
||||||
|
if c.Build == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if marked, err := open.inventory.GateFailed(ctx, c.Build); err != nil || !marked {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for node, other := range r.Gates {
|
||||||
|
if node == failed || other.Verdict != "" || slices.Contains(other.Returned, c.Module) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, oc := range other.Carried {
|
||||||
|
if oc.Module == c.Module && sameCommit(oc.To, c.To) {
|
||||||
|
other.Returned = append(other.Returned, c.Module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// openRestStep chooses the next step of a release plan's rest and keeps its sends (novox/hq ADR 0353 §1, §2):
|
||||||
|
// every machine left that is heard from and has something waiting, except that a build no gate has passed goes
|
||||||
|
// to one machine of the step only. Answers whether a step was kept, or why the walk waits.
|
||||||
|
func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, string, error) {
|
||||||
|
r := p.Release
|
||||||
|
heard, err := releaseHeard(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
return false, "", err
|
||||||
|
}
|
||||||
|
f, err := readMoveFacts(ctx, open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
return false, "", err
|
||||||
|
}
|
||||||
|
waiting := map[string][]inventory.CarriedMove{}
|
||||||
|
var nodes, walked []string
|
||||||
|
for _, node := range r.Order[min(r.Next, len(r.Order)):] {
|
||||||
|
if r.Gates[node] != nil || slices.Contains(r.Done, node) || slices.Contains(r.Skipped, node) ||
|
||||||
|
slices.Contains(r.Failed, node) || slices.Contains(r.Silent, node) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !heard[node] {
|
||||||
|
r.Skipped = append(r.Skipped, node)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
moves, err := gatedMoves(ctx, open, node, nil)
|
||||||
|
if errors.Is(err, errWalkedElsewhere) {
|
||||||
|
walked = append(walked, fmt.Sprintf("%s (%v)", node, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return false, "", fmt.Errorf("reading what waits on %s: %w", node, err)
|
||||||
|
}
|
||||||
|
if len(moves) == 0 {
|
||||||
|
r.Done = append(r.Done, node)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
waiting[node] = moves
|
||||||
|
nodes = append(nodes, node)
|
||||||
|
}
|
||||||
|
shelf, err := open.inventory.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return false, "", err
|
||||||
|
}
|
||||||
|
// **Every machine at once** (novox/hq ADR 0360 §2, §7): no build waits for one machine to pass first.
|
||||||
|
step := nodes
|
||||||
|
if gateFirstMachine {
|
||||||
|
step = restStep(nodes, waiting, f.gated, func(module string) bool { return oneAtATime(module, shelf[module]) })
|
||||||
|
}
|
||||||
|
if len(step) == 0 {
|
||||||
|
if len(walked) > 0 {
|
||||||
|
return false, "waiting before " + strings.Join(walked, "; "), nil
|
||||||
|
}
|
||||||
|
return false, "", nil
|
||||||
|
}
|
||||||
|
if r.Gates == nil {
|
||||||
|
r.Gates = map[string]*inventory.PlanGate{}
|
||||||
|
}
|
||||||
|
for _, node := range step {
|
||||||
|
r.Gates[node] = &inventory.PlanGate{Machines: []string{node}, Carried: waiting[node], Pending: true}
|
||||||
|
}
|
||||||
|
p.Note = fmt.Sprintf("sending %s at once, each judged there", strings.Join(step, ", "))
|
||||||
|
if left := len(nodes) - len(step); left > 0 {
|
||||||
|
p.Note += fmt.Sprintf("; %d machine(s) wait for a build that passed no gate yet to pass on one", left)
|
||||||
|
}
|
||||||
|
return true, "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// oneAtATime is a module whose builds go to the rest one machine at a time (novox/hq ADR 0353 §9): the
|
||||||
|
// node-engine, which applies every send, and a module holding the mesh's resolver, which names the bus.
|
||||||
|
func oneAtATime(module string, m catalogue.Manifest) bool {
|
||||||
|
if module == hostModule {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == resolverSeat })
|
||||||
|
}
|
||||||
|
|
||||||
|
// restStep is the machines of one step of the rest (novox/hq ADR 0353 §2), in the order given: each machine, unless
|
||||||
|
// a build it would carry has passed no gate and an earlier machine of the step already carries it — that build's
|
||||||
|
// first machine. Pure.
|
||||||
|
//
|
||||||
|
// **The node-engine and the mesh's resolver still go one machine at a time** (ADR 0353 §9): a step carries a
|
||||||
|
// build of either to one machine only, judged before the next, since a fault that cuts a machine off the bus
|
||||||
|
// shows only as its silence and no rollback reaches it.
|
||||||
|
func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed func(module, commit string) bool,
|
||||||
|
alone func(module string) bool) []string {
|
||||||
|
first := map[string]string{}
|
||||||
|
var out []string
|
||||||
|
for _, n := range nodes {
|
||||||
|
var unpassed []string
|
||||||
|
for _, mv := range waiting[n] {
|
||||||
|
if !passed(mv.Module, mv.To) || alone(mv.Module) {
|
||||||
|
unpassed = append(unpassed, mv.Module+"@"+mv.To)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if slices.ContainsFunc(unpassed, func(k string) bool { _, taken := first[k]; return taken }) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, k := range unpassed {
|
||||||
|
first[k] = n
|
||||||
|
}
|
||||||
|
out = append(out, n)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// silentObservation is what the gate's probe says of the newest release whose rest went silent (novox/hq ADR 0353
|
||||||
|
// §4): machines that never reported on their send, left as they were — loud, never a success said by nothing.
|
||||||
|
func silentObservation(ctx context.Context, inv *inventory.Inventory) ([]conditions.Observation, error) {
|
||||||
|
recent, err := inv.RecentPlans(ctx, 20)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, p := range recent {
|
||||||
|
if p.Release == nil || p.Release.Settings != nil || p.Open() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(p.Release.Silent) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
machines := append([]string(nil), p.Release.Silent...)
|
||||||
|
sort.Strings(machines)
|
||||||
|
o := conditions.Observation{Scope: conditions.ScopeMesh, ID: "release", Token: "silent", Kind: "release-silent",
|
||||||
|
Severity: conditions.Warning, Resolver: conditions.ResolverOperator, Also: machines,
|
||||||
|
Summary: fmt.Sprintf("%s: %s said nothing on the send of the builds that waited for a gate within %s, and were "+
|
||||||
|
"left as they were", p.ID, strings.Join(machines, ", "), inventory.ApplySilentAfter)}
|
||||||
|
w := releaseSilentWords(machines)
|
||||||
|
o.Headline, o.Explanation, o.Resolved, o.Needs = w.Headline, w.Explanation, w.Resolved, w.Needs
|
||||||
|
return []conditions.Observation{o}, nil
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
// backlogObservation is what the gate's probe says of a release held after a failure.
|
// backlogObservation is what the gate's probe says of a release held after a failure.
|
||||||
func backlogObservation() []conditions.Observation {
|
func backlogObservation() []conditions.Observation {
|
||||||
if backlogNow.held == "" || len(backlogNow.waiting) == 0 {
|
if backlogNow.held == "" || len(backlogNow.waiting) == 0 {
|
||||||
|
|||||||
@@ -0,0 +1,273 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **A pull request into a release branch is judged by that branch** (novox/hq ADR 0358, issue 704): a base that is
|
||||||
|
// not its repository's trunk, where mesh-controller has a branch of that name, is a release branch; the check then
|
||||||
|
// reads every repository beside it at its branch of that name where it has one, else at its trunk. Any other base is
|
||||||
|
// checked exactly as before.
|
||||||
|
|
||||||
|
const aRelease = "feat/57-config-and-provisioning"
|
||||||
|
|
||||||
|
// theCore is the mesh's core as a check reads it beside: the controller, the node-engine, the tool runner and the
|
||||||
|
// catalogue's bus, on the git seat.
|
||||||
|
func theCore() []inventory.Entry {
|
||||||
|
onSeat := func(module, repository string) inventory.Entry {
|
||||||
|
e := fromRepo(module, repository, "")
|
||||||
|
e.Source.Seat = "git"
|
||||||
|
return e
|
||||||
|
}
|
||||||
|
nats := onSeat("nats", "novox/mesh-catalog")
|
||||||
|
nats.Source.Path = "modules/nats"
|
||||||
|
return []inventory.Entry{onSeat("mesh-controller", "novox/mesh-controller"), onSeat("mesh-host", "novox/mesh-host"),
|
||||||
|
onSeat("node-tools", "novox/mesh-tools"), nats, onSeat("gitea", "novox/mesh-catalog")}
|
||||||
|
}
|
||||||
|
|
||||||
|
// aForgeWithBranches stands in for the forge: which branches each repository has, and how often it was asked.
|
||||||
|
func aForgeWithBranches(t *testing.T, branches map[string][]string, failing error) *int {
|
||||||
|
t.Helper()
|
||||||
|
asked := 0
|
||||||
|
was := forgeHasBranch
|
||||||
|
t.Cleanup(func() { forgeHasBranch = was })
|
||||||
|
forgeHasBranch = func(_ context.Context, owner, repo, branch string) (bool, error) {
|
||||||
|
asked++
|
||||||
|
if failing != nil {
|
||||||
|
return false, failing
|
||||||
|
}
|
||||||
|
for _, b := range branches[owner+"/"+repo] {
|
||||||
|
if b == branch {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return &asked
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestABaseIsAReleaseBranchWhenTheControllerHasABranchOfThatName(t *testing.T) {
|
||||||
|
entries := theCore()
|
||||||
|
scopeOf := func(p link.PullUpdated) checkScope { return pullScope(p, entries, nil, nil) }
|
||||||
|
pull := func(repo, base string) link.PullUpdated {
|
||||||
|
return link.PullUpdated{Owner: "novox", Repo: repo, Number: 286, Base: base, Commit: "abc",
|
||||||
|
Paths: []string{"modules/gitea/index.ts"}}
|
||||||
|
}
|
||||||
|
|
||||||
|
asked := aForgeWithBranches(t, map[string][]string{"novox/mesh-controller": {"main", aRelease}}, nil)
|
||||||
|
for _, c := range []struct {
|
||||||
|
what string
|
||||||
|
p link.PullUpdated
|
||||||
|
release string
|
||||||
|
asks int
|
||||||
|
}{
|
||||||
|
{"a pull request into the trunk asks nothing and is no release", pull("mesh-catalog", "main"), "", 0},
|
||||||
|
{"a catalogue pull request into the release branch", pull("mesh-catalog", aRelease), aRelease, 1},
|
||||||
|
{"a controller pull request into the release branch", pull("mesh-controller", aRelease), aRelease, 1},
|
||||||
|
{"a base mesh-controller has no branch of is checked as today", pull("mesh-catalog", "feat/other"), "", 1},
|
||||||
|
{"no base: as today", pull("mesh-catalog", ""), "", 0},
|
||||||
|
} {
|
||||||
|
*asked = 0
|
||||||
|
got, err := releaseBranchOf(t.Context(), c.p, scopeOf(c.p), entries)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: %v", c.what, err)
|
||||||
|
}
|
||||||
|
if got != c.release || *asked != c.asks {
|
||||||
|
t.Errorf("%s: release %q after %d question(s) to the forge, not %q after %d", c.what, got, *asked,
|
||||||
|
c.release, c.asks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The branch a module built from the repository follows is its trunk: never asked about.
|
||||||
|
photos := fromRepo("photos", "novox/photos", "")
|
||||||
|
photos.Source.Ref = "nox-mesh"
|
||||||
|
withPhotos := append(theCore(), photos)
|
||||||
|
p := link.PullUpdated{Owner: "novox", Repo: "photos", Base: "nox-mesh", Commit: "abc", Paths: []string{"x.js"}}
|
||||||
|
*asked = 0
|
||||||
|
if got, err := releaseBranchOf(t.Context(), p, pullScope(p, withPhotos, nil, nil), withPhotos); err != nil ||
|
||||||
|
got != "" || *asked != 0 {
|
||||||
|
t.Fatalf("a module's own trunk read as release %q (%v) after %d question(s)", got, err, *asked)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A forge that cannot answer is an error, never "no release branch".
|
||||||
|
aForgeWithBranches(t, nil, errors.New("nothing serves gitea.gitea_api"))
|
||||||
|
q := pull("mesh-catalog", aRelease)
|
||||||
|
if _, err := releaseBranchOf(t.Context(), q, scopeOf(q), entries); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "cannot be told") {
|
||||||
|
t.Fatalf("an unanswered forge was read as an answer: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBesideAReleaseBranchCheckEachRepositoryIsReadAtThatBranchElseItsTrunk(t *testing.T) {
|
||||||
|
entries := theCore()
|
||||||
|
clone := func(s inventory.Source) (string, error) { return "http://git/" + s.Repository + ".git", nil }
|
||||||
|
running := func(module string) string { return "running-" + module }
|
||||||
|
|
||||||
|
beside, err := besideOf(entries, running, clone, aRelease)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
at := func(repo string, ref, orElse string) link.CheckedOut {
|
||||||
|
return link.CheckedOut{Repository: "http://git/novox/" + repo + ".git", Ref: ref, Else: orElse}
|
||||||
|
}
|
||||||
|
want := map[string]link.CheckedOut{
|
||||||
|
// The judge: mesh-controller's release branch, which it has.
|
||||||
|
"mesh-controller": at("mesh-controller", aRelease, ""),
|
||||||
|
"mesh-controller-main": at("mesh-controller", "main", ""),
|
||||||
|
// A controller pull request into the release branch reads mesh-host at that branch where it has one.
|
||||||
|
"mesh-host": at("mesh-host", aRelease, "main"),
|
||||||
|
"mesh-tools": at("mesh-tools", aRelease, "main"),
|
||||||
|
"mesh-catalog": at("mesh-catalog", aRelease, "main"),
|
||||||
|
// mesh-lab at its trunk, always: its replays run with the container runtime's socket.
|
||||||
|
"mesh-lab": at("mesh-lab", "main", ""),
|
||||||
|
"mesh-sdk": at("mesh-sdk", aRelease, "main"),
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(beside, want) {
|
||||||
|
t.Fatalf("beside a release-branch check:\n got %v\nwant %v", beside, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Any other base: exactly as before — what the mesh runs, the catalogue's and the lab's main.
|
||||||
|
beside, err = besideOf(entries, running, clone, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want = map[string]link.CheckedOut{
|
||||||
|
"mesh-controller": at("mesh-controller", "running-mesh-controller", ""),
|
||||||
|
"mesh-controller-main": at("mesh-controller", "main", ""),
|
||||||
|
"mesh-host": at("mesh-host", "running-mesh-host", ""),
|
||||||
|
"mesh-tools": at("mesh-tools", "running-node-tools", ""),
|
||||||
|
"mesh-catalog": at("mesh-catalog", "main", ""),
|
||||||
|
"mesh-lab": at("mesh-lab", "main", ""),
|
||||||
|
"mesh-sdk": at("mesh-sdk", sdkPinned(), ""),
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(beside, want) {
|
||||||
|
t.Fatalf("beside a trunk check:\n got %v\nwant %v", beside, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAReleaseBranchVerdictNamesTheBranchTheJudgeAndWhatItRead(t *testing.T) {
|
||||||
|
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 286, Base: aRelease, Release: aRelease}
|
||||||
|
c := checkedOf(link.BuildResult{ID: "b", Ref: "aaaaaaaaaaaa0000", Checked: asked, Check: &link.CheckOutcome{
|
||||||
|
Verdict: "pass", Head: "aaaaaaaaaaaa", Trunk: "111111111111", Gate: &link.CheckLayer{Verdict: "pass"},
|
||||||
|
RepoCheck: &link.CheckLayer{Verdict: "pass"}, Release: aRelease, JudgedBy: "fcdecb436117",
|
||||||
|
Beside: map[string]string{"mesh-host": "222222222222", "mesh-lab": "333333333333"},
|
||||||
|
BesideAt: map[string]string{"mesh-host": aRelease, "mesh-lab": "main"},
|
||||||
|
Validator: "mesh-host's validator of " + aRelease + " at 222222222222"}})
|
||||||
|
if c.Release != aRelease || c.JudgedBy != "fcdecb436117" {
|
||||||
|
t.Fatalf("the verdict names release %q judged by %q", c.Release, c.JudgedBy)
|
||||||
|
}
|
||||||
|
if c.BesideAt["mesh-host"] != aRelease || c.BesideAt["mesh-lab"] != "main" || c.Beside["mesh-host"] != "222222222222" ||
|
||||||
|
c.Validator != "mesh-host's validator of "+aRelease+" at 222222222222" {
|
||||||
|
t.Fatalf("the verdict names what it read beside it as %v at %v", c.Beside, c.BesideAt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Clause 5** (ADR 0358 §5): a release branch's own pull request into its trunk is no release-branch check: the forge
|
||||||
|
// is asked nothing, the request names no release branch, and the repositories beside it are read as for any trunk
|
||||||
|
// pull request. Its delivery group's composed check is asked as for any other (askGroupCheck, unchanged).
|
||||||
|
func TestAReleaseBranchsPullRequestIntoItsTrunkIsCheckedAsAnyTrunkPullRequest(t *testing.T) {
|
||||||
|
entries := theCore()
|
||||||
|
asked := aForgeWithBranches(t, map[string][]string{"novox/mesh-controller": {"main", aRelease}}, nil)
|
||||||
|
for _, repo := range []string{"mesh-catalog", "mesh-controller", "mesh-host"} {
|
||||||
|
p := link.PullUpdated{Owner: "novox", Repo: repo, Number: 400, Head: aRelease, Base: "main", Commit: "abc",
|
||||||
|
Paths: []string{"modules/gitea/index.ts"}}
|
||||||
|
got, err := releaseBranchOf(t.Context(), p, pullScope(p, entries, nil, nil), entries)
|
||||||
|
if err != nil || got != "" || *asked != 0 {
|
||||||
|
t.Fatalf("%s's release branch into main read as release %q (%v) after %d question(s)", repo, got, err, *asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
clone := func(s inventory.Source) (string, error) { return "http://git/" + s.Repository + ".git", nil }
|
||||||
|
beside, err := besideOf(entries, func(module string) string { return "running-" + module }, clone, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for dir, b := range beside {
|
||||||
|
if b.Ref == aRelease || b.Else != "" {
|
||||||
|
t.Fatalf("beside a trunk pull request %s is read at %+v", dir, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A forge answer that names no branch is an error**, never "no release branch"; a 404 is "no such branch".
|
||||||
|
func TestTheForgesAnswerAboutABranchIsReadStrictly(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
what string
|
||||||
|
answer link.Answer
|
||||||
|
has bool
|
||||||
|
fails bool
|
||||||
|
}{
|
||||||
|
{"the branch named", link.Answer{Result: []byte(`{"result":{"name":"` + aRelease + `"}}`)}, true, false},
|
||||||
|
{"not found", link.Answer{Error: "Gitea API /repos/novox/mesh-controller/branches/x: 404 {\"message\":\"not found\"}"},
|
||||||
|
false, false},
|
||||||
|
{"an empty branch name", link.Answer{Result: []byte(`{"result":{"name":""}}`)}, false, true},
|
||||||
|
{"no result", link.Answer{Result: []byte(`{}`)}, false, true},
|
||||||
|
{"another branch named", link.Answer{Result: []byte(`{"result":{"name":"main"}}`)}, false, true},
|
||||||
|
{"unreadable", link.Answer{Result: []byte(`not json`)}, false, true},
|
||||||
|
{"another error", link.Answer{Error: "Gitea API: 500 internal"}, false, true},
|
||||||
|
} {
|
||||||
|
has, err := branchAnswered(c.answer, aRelease)
|
||||||
|
if has != c.has || (err != nil) != c.fails {
|
||||||
|
t.Errorf("%s: has %v, error %v", c.what, has, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The trunk is the repository's own, not a name**: a repository whose trunk is not main, one that follows main, and
|
||||||
|
// a controller whose trunk is not main.
|
||||||
|
func TestATrunkIsTheBranchTheRepositoryFollows(t *testing.T) {
|
||||||
|
asked := aForgeWithBranches(t, map[string][]string{"novox/mesh-controller": {"main", aRelease, "master"}}, nil)
|
||||||
|
legacy := fromRepo("legacy", "novox/legacy", "")
|
||||||
|
legacy.Source.Ref = "master"
|
||||||
|
onMain := fromRepo("photos", "novox/photos", "")
|
||||||
|
onMain.Source.Ref = "main"
|
||||||
|
entries := append(theCore(), legacy, onMain)
|
||||||
|
for _, c := range []struct {
|
||||||
|
what, repo, base, release string
|
||||||
|
asks int
|
||||||
|
}{
|
||||||
|
{"a repository whose trunk is master, into master", "legacy", "master", "", 0},
|
||||||
|
{"a repository whose trunk is master, into the release branch", "legacy", aRelease, aRelease, 1},
|
||||||
|
{"a repository whose module follows main, into main", "photos", "main", "", 0},
|
||||||
|
{"a repository whose module follows main, into the release branch", "photos", aRelease, aRelease, 1},
|
||||||
|
} {
|
||||||
|
*asked = 0
|
||||||
|
p := link.PullUpdated{Owner: "novox", Repo: c.repo, Base: c.base, Commit: "abc", Paths: []string{"x"}}
|
||||||
|
got, err := releaseBranchOf(t.Context(), p, pullScope(p, entries, nil, nil), entries)
|
||||||
|
if err != nil || got != c.release || *asked != c.asks {
|
||||||
|
t.Errorf("%s: release %q (%v) after %d question(s), not %q after %d", c.what, got, err, *asked, c.release, c.asks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A controller whose trunk is not main: main may be a release branch there, and its own trunk never is.
|
||||||
|
core := theCore()
|
||||||
|
core[0].Source.Ref = "trunk"
|
||||||
|
for _, c := range []struct {
|
||||||
|
base, release string
|
||||||
|
asks int
|
||||||
|
}{{"trunk", "", 0}, {"main", "main", 1}} {
|
||||||
|
*asked = 0
|
||||||
|
p := link.PullUpdated{Owner: "novox", Repo: "hq", Base: c.base, Commit: "abc", Paths: []string{"x"}}
|
||||||
|
got, err := releaseBranchOf(t.Context(), p, pullScope(p, core, nil, nil), core)
|
||||||
|
if err != nil || got != c.release || *asked != c.asks {
|
||||||
|
t.Errorf("hq into %s, the controller following trunk: release %q (%v) after %d question(s)", c.base, got, err,
|
||||||
|
*asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// trunkOf: the branch a source follows, and main when it records none or records a commit (issue 215's rule).
|
||||||
|
func TestATrunkIsMainWhenTheSourceRecordsNone(t *testing.T) {
|
||||||
|
for ref, want := range map[string]string{"": "main", "main": "main", "master": "master", "nox-mesh": "nox-mesh",
|
||||||
|
"0123456789abcdef0123456789abcdef01234567": "main", " ": "main"} {
|
||||||
|
if got := trunkOf(inventory.Source{Ref: ref}); got != want {
|
||||||
|
t.Errorf("a source recording %q follows %q, not %q", ref, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -9,10 +9,13 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/builder"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
)
|
)
|
||||||
@@ -25,6 +28,10 @@ import (
|
|||||||
// alone cannot (a tier waiting for machines to report); a controller replaced mid-plan finds the
|
// alone cannot (a tier waiting for machines to report); a controller replaced mid-plan finds the
|
||||||
// plan where it left it.
|
// plan where it left it.
|
||||||
|
|
||||||
|
// gateFirstMachine sends a module that rolls out to one machine first and judges it there (ADR 0236). Off since
|
||||||
|
// novox/hq ADR 0360: the gate's code is kept for the walks a controller before it started, and its tests turn it on.
|
||||||
|
var gateFirstMachine = false
|
||||||
|
|
||||||
// planWaitBound is how long a plan may wait on one thing before `status` names it red.
|
// planWaitBound is how long a plan may wait on one thing before `status` names it red.
|
||||||
const planWaitBound = 30 * time.Minute
|
const planWaitBound = 30 * time.Minute
|
||||||
|
|
||||||
@@ -41,29 +48,15 @@ func tiersOf(set []string, edges []inventory.Edge) [][]string {
|
|||||||
for _, m := range set {
|
for _, m := range set {
|
||||||
deps[m] = map[string]bool{}
|
deps[m] = map[string]bool{}
|
||||||
}
|
}
|
||||||
// The build seat's holders follow the controller that defines their worker (EdgeWorkerOf,
|
|
||||||
// novox/hq issue 206), so the built-by edge from that controller to such a holder yields: the
|
|
||||||
// controller is built by whichever build machine is running, as the runtime image always was.
|
|
||||||
worker := map[string]map[string]bool{}
|
|
||||||
for _, e := range edges {
|
|
||||||
if e.Kind == inventory.EdgeWorkerOf && in[e.From] && in[e.To] {
|
|
||||||
if worker[e.To] == nil {
|
|
||||||
worker[e.To] = map[string]bool{}
|
|
||||||
}
|
|
||||||
worker[e.To][e.From] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, e := range edges {
|
for _, e := range edges {
|
||||||
// A code dependency — B packages A's source — rebuilds B with A, in the same tier: B's
|
// A code dependency — B packages A's source — rebuilds B with A, in the same tier: B's
|
||||||
// build needs nothing of A's first. The other kinds order: stands-on and declared after
|
// build needs nothing of A's first. Stands-on and declared order after the base is built, and the
|
||||||
// the base is built, built-by after the build machine is built and running — except for
|
// build seat's holder follows the controller that defines its worker (EdgeWorkerOf, issue 206).
|
||||||
// what the build machine itself stands on, and for the controller whose worker the build
|
|
||||||
// machine binds. The runtime image is built by the builder and the builder is built on the
|
|
||||||
// runtime image; the image comes first, built by the builder that is running.
|
|
||||||
if !in[e.From] || !in[e.To] || e.From == e.To || e.Kind == inventory.EdgePackages {
|
if !in[e.From] || !in[e.To] || e.From == e.To || e.Kind == inventory.EdgePackages {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if e.Kind == inventory.EdgeBuiltBy && (isBaseOf(e.From, e.To, edges, in) || worker[e.From][e.To]) {
|
// **Being built by the builder orders nothing** (novox/hq ADR 0360 §3): whichever builder runs builds it.
|
||||||
|
if e.Kind == inventory.EdgeBuiltBy {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
deps[e.From][e.To] = true
|
deps[e.From][e.To] = true
|
||||||
@@ -104,29 +97,6 @@ func tiersOf(set []string, edges []inventory.Edge) [][]string {
|
|||||||
return tiers
|
return tiers
|
||||||
}
|
}
|
||||||
|
|
||||||
// isBaseOf says whether `to` stands on `base`, directly or through other bases in the set, along
|
|
||||||
// the build edges alone.
|
|
||||||
func isBaseOf(base, to string, edges []inventory.Edge, in map[string]bool) bool {
|
|
||||||
seen := map[string]bool{}
|
|
||||||
var walk func(string) bool
|
|
||||||
walk = func(m string) bool {
|
|
||||||
if m == base {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
if seen[m] {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
seen[m] = true
|
|
||||||
for _, e := range edges {
|
|
||||||
if e.From == m && in[e.To] && (e.Kind == inventory.EdgeStandsOn || e.Kind == inventory.EdgeDeclared) && walk(e.To) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return walk(to)
|
|
||||||
}
|
|
||||||
|
|
||||||
// reachableFrom is the moved modules plus everything that depends on them, through every layer:
|
// reachableFrom is the moved modules plus everything that depends on them, through every layer:
|
||||||
// what a merge rebuilds. Along the code and build edges only: a module *built by* the build machine
|
// what a merge rebuilds. Along the code and build edges only: a module *built by* the build machine
|
||||||
// is not changed by a new build machine, so a built-by edge orders and gates a plan and never
|
// is not changed by a new build machine, so a built-by edge orders and gates a plan and never
|
||||||
@@ -162,7 +132,7 @@ func reachableFrom(moved []string, edges []inventory.Edge) []string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// hasCycle says whether the tiers' last tier holds modules that still depend on each other. A packages
|
// hasCycle says whether the tiers' last tier holds modules that still depend on each other. A packages
|
||||||
// edge orders nothing (tiersOf), so a module and what packages its source share a tier by rule: that is
|
// or built-by edge orders nothing (tiersOf), so a module and what packages its source share a tier by rule: that is
|
||||||
// no cycle, and saying one was is a false report in every such plan's log.
|
// no cycle, and saying one was is a false report in every such plan's log.
|
||||||
func hasCycle(tiers [][]string, edges []inventory.Edge) bool {
|
func hasCycle(tiers [][]string, edges []inventory.Edge) bool {
|
||||||
if len(tiers) == 0 {
|
if len(tiers) == 0 {
|
||||||
@@ -173,7 +143,7 @@ func hasCycle(tiers [][]string, edges []inventory.Edge) bool {
|
|||||||
last[m] = true
|
last[m] = true
|
||||||
}
|
}
|
||||||
for _, e := range edges {
|
for _, e := range edges {
|
||||||
if e.Kind == inventory.EdgePackages {
|
if e.Kind == inventory.EdgePackages || e.Kind == inventory.EdgeBuiltBy {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if last[e.From] && last[e.To] {
|
if last[e.From] && last[e.To] {
|
||||||
@@ -219,12 +189,6 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool)
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
inTier := map[string]bool{}
|
inTier := map[string]bool{}
|
||||||
all := map[string]bool{}
|
|
||||||
for _, tier := range p.Tiers {
|
|
||||||
for _, m := range tier {
|
|
||||||
all[m] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, m := range p.Tiers[p.Tier] {
|
for _, m := range p.Tiers[p.Tier] {
|
||||||
inTier[m] = true
|
inTier[m] = true
|
||||||
}
|
}
|
||||||
@@ -239,8 +203,8 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool)
|
|||||||
for _, e := range edges {
|
for _, e := range edges {
|
||||||
// A delivery group's order inside a walk (novox/hq ADR 0276 decision 5) gates as built-by does: the
|
// A delivery group's order inside a walk (novox/hq ADR 0276 decision 5) gates as built-by does: the
|
||||||
// member before is running on its machines before the member after is asked.
|
// member before is running on its machines before the member after is asked.
|
||||||
ordered := e.Kind == edgeGroupOrder ||
|
// A built-by edge waits for nothing since novox/hq ADR 0360 §3.
|
||||||
e.Kind == inventory.EdgeBuiltBy && !isBaseOf(e.From, e.To, edges, all)
|
ordered := e.Kind == edgeGroupOrder
|
||||||
if later[e.From] && inTier[e.To] && ordered && !seen[e.To] && rollsOut(e.To) {
|
if later[e.From] && inTier[e.To] && ordered && !seen[e.To] && rollsOut(e.To) {
|
||||||
seen[e.To] = true
|
seen[e.To] = true
|
||||||
out = append(out, e.To)
|
out = append(out, e.To)
|
||||||
@@ -250,6 +214,20 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool)
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// noMoveWhy is what a plan says of a module of its tier whose build was made from the source every machine
|
||||||
|
// running it runs (novox/hq issue 280): nothing is sent for it.
|
||||||
|
// noMovePrefix opens it, and is what noMove reads: one prefix for the writer and the reader.
|
||||||
|
const (
|
||||||
|
noMovePrefix = "no move"
|
||||||
|
noMoveWhy = noMovePrefix + ": made from the source every machine running it runs"
|
||||||
|
)
|
||||||
|
|
||||||
|
// noMove is whether the plan marked the module no move and sent it nothing. Read from what the plan says,
|
||||||
|
// so a plan kept before the mark had a name of its own is read the same (novox/hq issue 648).
|
||||||
|
func noMove(s *inventory.PlanModule) bool {
|
||||||
|
return s != nil && s.FirstAt == nil && s.Gate == nil && strings.HasPrefix(s.Why, noMovePrefix)
|
||||||
|
}
|
||||||
|
|
||||||
// applied says whether every machine running the module has reported since the module was built.
|
// applied says whether every machine running the module has reported since the module was built.
|
||||||
// appliedEach is applied with a moment of its own for each machine: the reports that count are the ones
|
// appliedEach is applied with a moment of its own for each machine: the reports that count are the ones
|
||||||
// after that machine was sent the build (novox/hq issue 256).
|
// after that machine was sent the build (novox/hq issue 256).
|
||||||
@@ -313,6 +291,144 @@ func init() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// reuseKey carries a walk's leave to take a check's build from askModule to the ask it makes (novox/hq ADR 0350 §2).
|
||||||
|
type reuseKey struct{}
|
||||||
|
|
||||||
|
// withReuse is ctx carrying a walk's leave to take a check's build.
|
||||||
|
func withReuse(ctx context.Context, r *link.ReuseAsk) context.Context {
|
||||||
|
return context.WithValue(ctx, reuseKey{}, r)
|
||||||
|
}
|
||||||
|
|
||||||
|
// reuseIn is the leave ctx carries; nil for every ask but a walk's.
|
||||||
|
func reuseIn(ctx context.Context) *link.ReuseAsk {
|
||||||
|
r, _ := ctx.Value(reuseKey{}).(*link.ReuseAsk)
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// reuseAsked is the leave an ask carries: a walk's, and never a dry run's — a dry run registers nothing, so it
|
||||||
|
// takes nothing either (novox/hq ADR 0350 §2, §6).
|
||||||
|
func reuseAsked(ctx context.Context, dryRun bool) *link.ReuseAsk {
|
||||||
|
if dryRun {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return reuseIn(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// reuseAskOf is a walk's leave to take a check's build: the pull requests of the merges it answers.
|
||||||
|
func reuseAskOf(p *inventory.Plan) *link.ReuseAsk {
|
||||||
|
r := &link.ReuseAsk{}
|
||||||
|
if p.Delivery == nil {
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
seen := map[link.PullRef]bool{}
|
||||||
|
for _, m := range p.Delivery.Merges {
|
||||||
|
pull, ok := pullOf(m)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !seen[pull] {
|
||||||
|
seen[pull] = true
|
||||||
|
r.Pulls = append(r.Pulls, pull)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkedBuildsOf are the builds the build seat's checks of those pull requests said they made for their merges,
|
||||||
|
// within the week check builds are kept (novox/hq ADR 0350 §2): read from the seat's own outcomes on the bus. A
|
||||||
|
// variable so a test needs no bus.
|
||||||
|
var checkedBuildsOf = func(ctx context.Context, pulls []link.PullRef) ([]link.CheckedBuild, error) {
|
||||||
|
if len(pulls) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
js, err := aBus()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
return link.ReadCheckBuilds(ctx, js, buildSeatHeld(ctx), pulls, time.Now().Add(-builder.CheckBuildsKept))
|
||||||
|
}
|
||||||
|
|
||||||
|
// mergedHeadsOf are the head commits the walk's merges merged, by pull request (novox/hq issue 703): a pull request
|
||||||
|
// merged twice in one walk, or whose head the announcement did not say, has none, and takes no check's build.
|
||||||
|
func mergedHeadsOf(p *inventory.Plan) map[link.PullRef]string {
|
||||||
|
heads := map[link.PullRef]string{}
|
||||||
|
if p.Delivery == nil {
|
||||||
|
return heads
|
||||||
|
}
|
||||||
|
seen := map[link.PullRef]bool{}
|
||||||
|
for _, m := range p.Delivery.Merges {
|
||||||
|
pull, ok := pullOf(m)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if seen[pull] && heads[pull] != m.Head {
|
||||||
|
heads[pull] = ""
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[pull] = true
|
||||||
|
heads[pull] = m.Head
|
||||||
|
}
|
||||||
|
return heads
|
||||||
|
}
|
||||||
|
|
||||||
|
// walkVerdicts are the checks' builds a walk read, by the walk's plan: read once per walk (novox/hq issue 649), not
|
||||||
|
// once per module it asks for. Kept a day; a walk lasts minutes.
|
||||||
|
var walkVerdicts = struct {
|
||||||
|
sync.Mutex
|
||||||
|
read map[string]verdictsRead
|
||||||
|
}{read: map[string]verdictsRead{}}
|
||||||
|
|
||||||
|
type verdictsRead struct {
|
||||||
|
at time.Time
|
||||||
|
builds []link.CheckedBuild
|
||||||
|
}
|
||||||
|
|
||||||
|
// reuseWithVerdicts is a walk's leave with the builds its pull requests' verdicts name: the only builds the builder
|
||||||
|
// may take (ADR 0350 §2), and **only those of a check of the head each merge merged** (novox/hq issue 703): a build
|
||||||
|
// a check made at an earlier head of the same pull request is never registered. Read once per walk. When they
|
||||||
|
// cannot be read the leave names none, and the walk builds, saying why.
|
||||||
|
func reuseWithVerdicts(ctx context.Context, walk string, r *link.ReuseAsk, heads map[link.PullRef]string) *link.ReuseAsk {
|
||||||
|
walkVerdicts.Lock()
|
||||||
|
read, known := walkVerdicts.read[walk]
|
||||||
|
walkVerdicts.Unlock()
|
||||||
|
if !known || walk == "" {
|
||||||
|
builds, err := checkedBuildsOf(ctx, r.Pulls)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf(" the checks' builds of %d pull request(s) could not be read, so the walk builds: %v\n", len(r.Pulls), err)
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
read = verdictsRead{at: time.Now(), builds: builds}
|
||||||
|
if walk != "" {
|
||||||
|
walkVerdicts.Lock()
|
||||||
|
for id, w := range walkVerdicts.read {
|
||||||
|
if time.Since(w.at) > 24*time.Hour {
|
||||||
|
delete(walkVerdicts.read, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
walkVerdicts.read[walk] = read
|
||||||
|
walkVerdicts.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
r.Builds = nil
|
||||||
|
for _, b := range read.builds {
|
||||||
|
if link.SameHead(b.Head, heads[b.Pull]) {
|
||||||
|
r.Builds = append(r.Builds, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// pullOf is a merge's pull request: owner and repository, the last two parts of however the repository is spelled
|
||||||
|
// (owner/repo, a path on the git seat, a URL), and its number. False when the merge names none.
|
||||||
|
func pullOf(m inventory.PlanMerge) (link.PullRef, bool) {
|
||||||
|
parts := strings.Split(strings.Trim(strings.TrimSuffix(m.Repository, ".git"), "/"), "/")
|
||||||
|
if len(parts) < 2 || m.Number <= 0 || parts[len(parts)-2] == "" || parts[len(parts)-1] == "" {
|
||||||
|
return link.PullRef{}, false
|
||||||
|
}
|
||||||
|
return link.PullRef{Owner: parts[len(parts)-2], Repo: parts[len(parts)-1], Number: m.Number}, true
|
||||||
|
}
|
||||||
|
|
||||||
// askModule asks the build machine for one module of a plan and marks it asked, with the id it was
|
// askModule asks the build machine for one module of a plan and marks it asked, with the id it was
|
||||||
// asked under (novox/hq ADR 0219) — or failed, with the plan, when it could not be asked.
|
// asked under (novox/hq ADR 0219) — or failed, with the plan, when it could not be asked.
|
||||||
func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[string]inventory.Entry) {
|
func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[string]inventory.Entry) {
|
||||||
@@ -340,7 +456,10 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
|
|||||||
if p.Delivery != nil && p.Delivery.Alone && carried != "" {
|
if p.Delivery != nil && p.Delivery.Alone && carried != "" {
|
||||||
ref = carried
|
ref = carried
|
||||||
}
|
}
|
||||||
id, err := askABuild(ctx, source, e.Source.Path, ref)
|
// **The walk takes the check's build of the same key** (novox/hq ADR 0350 §2): its ask says so, with the pull
|
||||||
|
// requests it answers, whose checks name the part of a differing key.
|
||||||
|
id, err := askABuild(withReuse(ctx, reuseWithVerdicts(ctx, p.ID, reuseAskOf(p), mergedHeadsOf(p))), source,
|
||||||
|
e.Source.Path, ref)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
state.State = "failed"
|
state.State = "failed"
|
||||||
state.Why = err.Error()
|
state.Why = err.Error()
|
||||||
@@ -359,7 +478,7 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
|
|||||||
state.State = "asked"
|
state.State = "asked"
|
||||||
state.AskedAt = &now
|
state.AskedAt = &now
|
||||||
state.Build = id
|
state.Build = id
|
||||||
state.Why, state.Commit, state.BuiltAt = "", "", nil
|
state.Why, state.Commit, state.BuiltAt, state.Made = "", "", nil, ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// planBuilt marks a module built (or failed) in every open plan whose current tier holds it, and
|
// planBuilt marks a module built (or failed) in every open plan whose current tier holds it, and
|
||||||
@@ -371,6 +490,12 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
|
|||||||
// would send machines, and the next tier, what the later merge replaced. asked is zero when the
|
// would send machines, and the next tier, what the later merge replaced. asked is zero when the
|
||||||
// build's request time is not known, and such an outcome is taken as before.
|
// build's request time is not known, and such an outcome is taken as before.
|
||||||
func planBuilt(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id string) {
|
func planBuilt(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id string) {
|
||||||
|
planBuiltSaying(ctx, open, module, commit, failed, asked, id, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// planBuiltSaying is planBuilt keeping what the build did — "reused from <check build>" or "built: <why>" (novox/hq
|
||||||
|
// ADR 0350 §4) — on the module of the walk it answers.
|
||||||
|
func planBuiltSaying(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id, made string) {
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
// One controller works the plans at a time (novox/hq issue 213); an outcome waits its turn rather
|
// One controller works the plans at a time (novox/hq issue 213); an outcome waits its turn rather
|
||||||
// than write over what the holder is about to save. Not taken, it is still in the build records,
|
// than write over what the holder is about to save. Not taken, it is still in the build records,
|
||||||
@@ -436,6 +561,7 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
|
|||||||
state.State = "built"
|
state.State = "built"
|
||||||
state.BuiltAt = &now
|
state.BuiltAt = &now
|
||||||
state.Commit = commit
|
state.Commit = commit
|
||||||
|
state.Made = made
|
||||||
}
|
}
|
||||||
if err := inv.SavePlan(ctx, p); err != nil {
|
if err := inv.SavePlan(ctx, p); err != nil {
|
||||||
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
||||||
@@ -505,7 +631,7 @@ func advancePlans(ctx context.Context, open *stores) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer release()
|
defer release()
|
||||||
// What waits for a gate, released one machine at a time (ADR 0236).
|
// What waits for a gate, released one machine first, then the rest at once (ADR 0236, ADR 0353).
|
||||||
if _, err := releaseBacklog(ctx, open, ""); err != nil {
|
if _, err := releaseBacklog(ctx, open, ""); err != nil {
|
||||||
fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err)
|
fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err)
|
||||||
}
|
}
|
||||||
@@ -715,7 +841,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
} else if same {
|
} else if same {
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
state.SentAt = &now
|
state.SentAt = &now
|
||||||
state.Why = "no move: made from the source every machine running it runs"
|
state.Why = noMoveWhy
|
||||||
fmt.Printf("%s: %s built from the source every machine running it runs: no move, nothing sent\n", p.ID, m)
|
fmt.Printf("%s: %s built from the source every machine running it runs: no move, nothing sent\n", p.ID, m)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -724,6 +850,12 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
|
// **Every machine at once** (novox/hq ADR 0360 §1, §2): a module that rolls out is sent to every machine
|
||||||
|
// running it in one step, with no first machine and no gate, whatever module it is. A module a controller
|
||||||
|
// before ADR 0360 already sent to its first machine finishes under its gate.
|
||||||
|
if !gateFirstMachine && state.FirstAt == nil {
|
||||||
|
policy.Together = true
|
||||||
|
}
|
||||||
var reports []inventory.Reported
|
var reports []inventory.Reported
|
||||||
if !policy.Together {
|
if !policy.Together {
|
||||||
// Read for the choice of the first machine as well as for its report.
|
// Read for the choice of the first machine as well as for its report.
|
||||||
@@ -905,6 +1037,25 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
if state.BuiltAt != nil {
|
if state.BuiltAt != nil {
|
||||||
since = *state.BuiltAt
|
since = *state.BuiltAt
|
||||||
}
|
}
|
||||||
|
// **A build that is no move is waited on by the send that last carried it** (novox/hq issue 648).
|
||||||
|
// Nothing was sent for it, so no report follows its marking or its build: a machine that does not
|
||||||
|
// happen to report again held the walk for ever. Each machine runs it once it answered its last
|
||||||
|
// send — the one whose builds made it no move — whenever that was.
|
||||||
|
if noMove(state) {
|
||||||
|
byNode := map[string]inventory.Reported{}
|
||||||
|
for _, r := range reports {
|
||||||
|
byNode[r.Node] = r
|
||||||
|
}
|
||||||
|
if ok, on := appliedEach(m, func(node string) time.Time {
|
||||||
|
if s := byNode[node].Sent; s != nil {
|
||||||
|
return *s
|
||||||
|
}
|
||||||
|
return time.Time{}
|
||||||
|
}, running, reports); !ok {
|
||||||
|
waiting = append(waiting, fmt.Sprintf("%s on %s", m, strings.Join(on, ", ")))
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
// **Each machine from its own send** (novox/hq issue 256). With one machine first (ADR 0218)
|
// **Each machine from its own send** (novox/hq issue 256). With one machine first (ADR 0218)
|
||||||
// a module is sent twice — the first machine, then the rest — and SentAt is the second.
|
// a module is sent twice — the first machine, then the rest — and SentAt is the second.
|
||||||
// Asked of every machine, the first machine's report, made between the two sends, read as
|
// Asked of every machine, the first machine's report, made between the two sends, read as
|
||||||
@@ -934,15 +1085,92 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
return changed, nil
|
return changed, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// **A walk ends on its machines' reports** (novox/hq ADR 0360 §4, §5): after its last tier, every machine it sent
|
||||||
|
// has reported on that send, or is silent past ApplySilentAfter; never a health reading.
|
||||||
|
end := ""
|
||||||
|
if p.Tier == len(p.Tiers)-1 {
|
||||||
|
reports, err := inv.LastReports(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
waiting, failed, silent := walkReports(*p, reports, time.Now().UTC())
|
||||||
|
if len(waiting) > 0 {
|
||||||
|
note := "sent; waiting for " + strings.Join(waiting, ", ") + " to report on their send"
|
||||||
|
changed := p.State != inventory.PlanRolling || p.Note != note
|
||||||
|
p.State = inventory.PlanRolling
|
||||||
|
p.Note = note
|
||||||
|
return changed, nil
|
||||||
|
}
|
||||||
|
var said []string
|
||||||
|
if len(failed) > 0 {
|
||||||
|
said = append(said, "failed or refused its send (apply-failing says it): "+strings.Join(failed, ", "))
|
||||||
|
}
|
||||||
|
if len(silent) > 0 {
|
||||||
|
said = append(said, fmt.Sprintf("said nothing on its send within %s: %s", inventory.ApplySilentAfter,
|
||||||
|
strings.Join(silent, ", ")))
|
||||||
|
}
|
||||||
|
end = strings.Join(said, "; ")
|
||||||
|
// **The new controller's user list follows its walk** (ADR 0236 §34, read without the gate since ADR 0360):
|
||||||
|
// once a walk that moved the controller ends, the bus's machine is sent the list it composes, if behind.
|
||||||
|
if s := p.Modules[catalogue.ControllerSeatName]; s != nil && s.SentAt != nil && s.Gate == nil && !noMove(s) {
|
||||||
|
carryUserList(ctx, open, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
p.Tier++
|
p.Tier++
|
||||||
p.State = inventory.PlanBuilding
|
p.State = inventory.PlanBuilding
|
||||||
p.Note = ""
|
p.Note = end
|
||||||
if p.Tier < len(p.Tiers) {
|
if p.Tier < len(p.Tiers) {
|
||||||
fmt.Printf("%s: tier %d done; asking tier %d: %s\n", p.ID, p.Tier-1, p.Tier, strings.Join(p.Tiers[p.Tier], ", "))
|
fmt.Printf("%s: tier %d done; asking tier %d: %s\n", p.ID, p.Tier-1, p.Tier, strings.Join(p.Tiers[p.Tier], ", "))
|
||||||
}
|
}
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// walkReports is, for every machine a walk sent its rest, whether it reported on that send (novox/hq ADR 0360 §4):
|
||||||
|
// those still waiting, those that reported a failed or refused apply, and those silent past ApplySilentAfter,
|
||||||
|
// each sorted. Pure.
|
||||||
|
func walkReports(p inventory.Plan, reports []inventory.Reported, now time.Time) (waiting, failed, silent []string) {
|
||||||
|
byNode := map[string]inventory.Reported{}
|
||||||
|
for _, r := range reports {
|
||||||
|
byNode[r.Node] = r
|
||||||
|
}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, s := range p.Modules {
|
||||||
|
if s == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for node, sent := range s.Rest {
|
||||||
|
if seen[node] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
r := byNode[node]
|
||||||
|
at := sent.At
|
||||||
|
if at.IsZero() && s.SentAt != nil {
|
||||||
|
at = *s.SentAt
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case r.At != nil && sent.ReportsOn(r):
|
||||||
|
seen[node] = true
|
||||||
|
if r.Outcome == inventory.OutcomeFailed || r.Outcome == inventory.OutcomeRefused {
|
||||||
|
failed = append(failed, node)
|
||||||
|
}
|
||||||
|
case !at.IsZero() && now.Sub(at) > inventory.ApplySilentAfter:
|
||||||
|
seen[node] = true
|
||||||
|
silent = append(silent, node)
|
||||||
|
default:
|
||||||
|
if !slices.Contains(waiting, node) {
|
||||||
|
waiting = append(waiting, node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A machine one module's send was reported on and another's not yet is the same send: reported.
|
||||||
|
waiting = slices.DeleteFunc(waiting, func(n string) bool { return seen[n] })
|
||||||
|
sort.Strings(waiting)
|
||||||
|
sort.Strings(failed)
|
||||||
|
sort.Strings(silent)
|
||||||
|
return waiting, failed, silent
|
||||||
|
}
|
||||||
|
|
||||||
// restOf is, for one module, every machine of its rest that the send reached and what it carried there.
|
// restOf is, for one module, every machine of its rest that the send reached and what it carried there.
|
||||||
func restOf(to, sent []string, what map[string]inventory.SentDeclaration) map[string]inventory.SentDeclaration {
|
func restOf(to, sent []string, what map[string]inventory.SentDeclaration) map[string]inventory.SentDeclaration {
|
||||||
out := map[string]inventory.SentDeclaration{}
|
out := map[string]inventory.SentDeclaration{}
|
||||||
@@ -966,10 +1194,36 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
|||||||
runningOf map[string][]string) error {
|
runningOf map[string][]string) error {
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
before, beforeKnown, _ := inv.SentBuilds(ctx, node)
|
before, beforeKnown, _ := inv.SentBuilds(ctx, node)
|
||||||
|
// **A machine already sent what this send is for is not sent again** (novox/hq ADR 0353 §6): sent by this plan
|
||||||
|
// before a controller was replaced and before it was kept, or by another walk. Judged from that send, and from
|
||||||
|
// the readings of a gate already judging it there.
|
||||||
|
// Only a send the machine reported it applied is one it has; one it never reported on is sent, as before
|
||||||
|
// (issue 280: what it ran before is then that build, and a failure leaves it as it was).
|
||||||
|
if beforeKnown && slices.IndexFunc(modules, func(m string) bool { return !sameCommit(before[m], p.Modules[m].Commit) }) < 0 &&
|
||||||
|
appliedLastSend(ctx, inv, node) {
|
||||||
|
if others, err := gatedMoves(ctx, open, node, nil); err == nil && len(others) == 0 {
|
||||||
|
adoptSend(ctx, open, p, node, modules, before, runningOf)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// **What the machine ran before is kept before the send is made** (novox/hq ADR 0353 §6): a controller replaced
|
||||||
|
// between the send and keeping it reads it here, and a failure still puts the machine back.
|
||||||
|
kept := false
|
||||||
|
for _, m := range modules {
|
||||||
|
s := p.Modules[m]
|
||||||
|
if beforeKnown && (s.Previous == "" || !sameCommit(before[m], s.Commit)) && s.Previous != before[m] {
|
||||||
|
s.Previous, kept = before[m], true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if kept {
|
||||||
|
if err := inv.SavePlan(ctx, p); err != nil {
|
||||||
|
return fmt.Errorf("what %s ran before is not kept, so nothing is sent: %w", node, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
owns := make([]inventory.CarriedMove, 0, len(modules))
|
owns := make([]inventory.CarriedMove, 0, len(modules))
|
||||||
for _, m := range modules {
|
for _, m := range modules {
|
||||||
s := p.Modules[m]
|
s := p.Modules[m]
|
||||||
owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: before[m], To: s.Commit, Build: s.Build})
|
owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: s.Previous, To: s.Commit, Build: s.Build})
|
||||||
}
|
}
|
||||||
// A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges
|
// A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges
|
||||||
// all of it there.
|
// all of it there.
|
||||||
@@ -984,10 +1238,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
|||||||
sentWhat := sentNow(ctx, inv, sent)
|
sentWhat := sentNow(ctx, inv, sent)
|
||||||
for _, m := range modules {
|
for _, m := range modules {
|
||||||
s := p.Modules[m]
|
s := p.Modules[m]
|
||||||
// What the first machine ran before: what a failed gate puts back (ADR 0236).
|
// What the first machine ran before (ADR 0236), kept above before the send.
|
||||||
if beforeKnown {
|
|
||||||
s.Previous = before[m]
|
|
||||||
}
|
|
||||||
s.First, s.FirstAt = sent, &now
|
s.First, s.FirstAt = sent, &now
|
||||||
s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning(sent, runningOf[m]),
|
s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning(sent, runningOf[m]),
|
||||||
From: s.Previous, To: s.Commit, Since: &now, Sent: sentWhat}
|
From: s.Previous, To: s.Commit, Since: &now, Sent: sentWhat}
|
||||||
@@ -1016,6 +1267,113 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// adoptSend records a first send the machine already had (novox/hq ADR 0353 §6, §7): nothing is sent; the gate counts
|
||||||
|
// from the machine's last send, and takes over the readings of an open gate of another walk judging the same build
|
||||||
|
// there — whose carried move also says what the machine ran before, for a rollback.
|
||||||
|
func adoptSend(ctx context.Context, open *stores, p *inventory.Plan, node string, modules []string,
|
||||||
|
before map[string]string, runningOf map[string][]string) {
|
||||||
|
inv := open.inventory
|
||||||
|
since := time.Now().UTC()
|
||||||
|
last, found, err := inv.SentTo(ctx, node)
|
||||||
|
if err == nil && found && !last.At.IsZero() {
|
||||||
|
since = last.At
|
||||||
|
}
|
||||||
|
lead := modules[0]
|
||||||
|
other, id, from := judgingThere(ctx, inv, p.ID, lead, node, p.Modules[lead].Commit, p.Modules[lead].Build)
|
||||||
|
for _, m := range modules {
|
||||||
|
s := p.Modules[m]
|
||||||
|
// What it ran before: kept by this walk before its send was made (§6); else what the other walk's send
|
||||||
|
// moved it from; else the build itself, so a failure leaves it as it was (issue 280).
|
||||||
|
switch {
|
||||||
|
case s.Previous != "":
|
||||||
|
case m == lead && from != "":
|
||||||
|
s.Previous = from
|
||||||
|
default:
|
||||||
|
s.Previous = before[m]
|
||||||
|
}
|
||||||
|
at := since
|
||||||
|
s.First, s.FirstAt = []string{node}, &at
|
||||||
|
s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning([]string{node}, runningOf[m]),
|
||||||
|
From: s.Previous, To: s.Commit, Since: &at}
|
||||||
|
if found {
|
||||||
|
s.Gate.Sent = map[string]inventory.SentDeclaration{node: last}
|
||||||
|
}
|
||||||
|
s.GatedBy = ""
|
||||||
|
if m == lead {
|
||||||
|
s.Gate.TakeReadings(other, modules, id)
|
||||||
|
} else {
|
||||||
|
s.GatedBy = lead
|
||||||
|
}
|
||||||
|
}
|
||||||
|
p.State = inventory.PlanRolling
|
||||||
|
p.Note = fmt.Sprintf("tier %d built; %s already runs %s's build: nothing sent again, judged from that send",
|
||||||
|
p.Tier, node, strings.Join(modules, ", "))
|
||||||
|
if id != "" {
|
||||||
|
p.Note += ", going on from the readings of " + id
|
||||||
|
}
|
||||||
|
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||||
|
}
|
||||||
|
|
||||||
|
// appliedLastSend says a machine reported it applied the declaration it was sent last.
|
||||||
|
func appliedLastSend(ctx context.Context, inv *inventory.Inventory, node string) bool {
|
||||||
|
reports, err := inv.LastReports(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, r := range reports {
|
||||||
|
if r.Node == node {
|
||||||
|
return r.Current && r.At != nil && r.Outcome == inventory.OutcomeApplied
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgingThere is an open gate of another walk judging a module's build on a machine (novox/hq ADR 0353 §7), the
|
||||||
|
// walk's id, and what the machine ran before; nil when none does.
|
||||||
|
func judgingThere(ctx context.Context, inv *inventory.Inventory, self, module, node, commit, build string) (*inventory.PlanGate, string, string) {
|
||||||
|
plans, err := inv.OpenPlans(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", ""
|
||||||
|
}
|
||||||
|
// The same build, by its id where both name one, else by its commit.
|
||||||
|
same := func(b, c string) bool {
|
||||||
|
if b != "" && build != "" {
|
||||||
|
return b == build
|
||||||
|
}
|
||||||
|
return sameCommit(c, commit)
|
||||||
|
}
|
||||||
|
usable := func(g *inventory.PlanGate) bool {
|
||||||
|
return g != nil && !g.Pending && g.Since != nil && (g.Verdict == "" || g.Verdict == inventory.GatePassed) &&
|
||||||
|
slices.Contains(g.Machines, node)
|
||||||
|
}
|
||||||
|
for _, q := range plans {
|
||||||
|
if q.ID == self {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if r := q.Release; r != nil {
|
||||||
|
gates := []*inventory.PlanGate{r.Gate}
|
||||||
|
for _, g := range r.Gates {
|
||||||
|
gates = append(gates, g)
|
||||||
|
}
|
||||||
|
for _, g := range gates {
|
||||||
|
if !usable(g) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, c := range g.Carried {
|
||||||
|
if c.Module == module && c.Node == node && same(c.Build, c.To) {
|
||||||
|
return g, q.ID, c.From
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if s := q.Modules[module]; s != nil && usable(s.Gate) && same(s.Build, s.Commit) {
|
||||||
|
return s.Gate, q.ID, s.Previous
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, "", ""
|
||||||
|
}
|
||||||
|
|
||||||
// passedWith keeps, on a module sent with others, the verdict of the gate that judged the send: the gate
|
// passedWith keeps, on a module sent with others, the verdict of the gate that judged the send: the gate
|
||||||
// on the first of them passed, and its pass was kept for every build it carried (passCarried).
|
// on the first of them passed, and its pass was kept for every build it carried (passCarried).
|
||||||
func passedWith(module string, s *inventory.PlanModule, lead string, g *inventory.PlanGate) {
|
func passedWith(module string, s *inventory.PlanModule, lead string, g *inventory.PlanGate) {
|
||||||
@@ -1504,9 +1862,19 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
// A release plan's walk (ADR 0236): machines done, the one judged, those to come.
|
// A release plan's walk (ADR 0236): machines done, the one judged, those to come.
|
||||||
fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "),
|
fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "),
|
||||||
orNone(strings.Join(r.Done, ", ")), orNone(strings.Join(r.Skipped, ", ")))
|
orNone(strings.Join(r.Done, ", ")), orNone(strings.Join(r.Skipped, ", ")))
|
||||||
if r.Gate != nil {
|
if len(r.Failed)+len(r.Silent) > 0 {
|
||||||
fmt.Printf(" %s\n", gateLine(r.Gate))
|
fmt.Printf(" failed: %s; silent: %s\n", orNone(strings.Join(r.Failed, ", ")), orNone(strings.Join(r.Silent, ", ")))
|
||||||
for _, c := range r.Gate.Carried {
|
}
|
||||||
|
gates := []*inventory.PlanGate{r.Gate}
|
||||||
|
for _, n := range r.Order {
|
||||||
|
gates = append(gates, r.Gates[n])
|
||||||
|
}
|
||||||
|
for _, g := range gates {
|
||||||
|
if g == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fmt.Printf(" %s\n", gateLine(g))
|
||||||
|
for _, c := range g.Carried {
|
||||||
fmt.Printf(" %-22s %s → %s\n", c.Module, short(c.From), short(c.To))
|
fmt.Printf(" %-22s %s → %s\n", c.Module, short(c.From), short(c.To))
|
||||||
if c.Recreates != "" {
|
if c.Recreates != "" {
|
||||||
fmt.Printf(" %-22s %s\n", "", c.Recreates)
|
fmt.Printf(" %-22s %s\n", "", c.Recreates)
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"slices"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -49,8 +50,10 @@ func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) {
|
|||||||
if pos["mesh-tools"] != 0 || pos["builder"] != 1 {
|
if pos["mesh-tools"] != 0 || pos["builder"] != 1 {
|
||||||
t.Fatalf("the runtime then the builder: %v", tiers)
|
t.Fatalf("the runtime then the builder: %v", tiers)
|
||||||
}
|
}
|
||||||
if !(pos["shop"] > pos["builder"] && pos["postgres"] > pos["builder"]) {
|
// Built-by orders nothing (novox/hq ADR 0360 §3): what the builder builds is beside it, after the runtime it
|
||||||
t.Fatalf("what the builder builds comes after the builder: %v", tiers)
|
// stands on.
|
||||||
|
if pos["shop"] != 1 || pos["postgres"] != 1 {
|
||||||
|
t.Fatalf("what the builder builds waits for the builder: %v", tiers)
|
||||||
}
|
}
|
||||||
if pos["shop-plugin"] <= pos["shop"] {
|
if pos["shop-plugin"] <= pos["shop"] {
|
||||||
t.Fatalf("a plugin after what it is declared on: %v", tiers)
|
t.Fatalf("a plugin after what it is declared on: %v", tiers)
|
||||||
@@ -67,24 +70,23 @@ func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) {
|
|||||||
// A change to a package all three build from moves all three. The builder holds
|
// A change to a package all three build from moves all three. The builder holds
|
||||||
// the build seat, whose worker the controller defines, so it follows the controller (worker-of,
|
// the build seat, whose worker the controller defines, so it follows the controller (worker-of,
|
||||||
// novox/hq issue 206), and the controller's built-by edge to it yields: the controller is built by the
|
// novox/hq issue 206), and the controller's built-by edge to it yields: the controller is built by the
|
||||||
// build machine that is running. The proxy is built by the new builder: the controller, the builder,
|
// build machine that is running. The proxy is built by whichever builder runs, so it goes beside the
|
||||||
// the proxy — the live plan of every controller merge. (This read "the builder, then the controller
|
// controller (novox/hq ADR 0360 §3; it read "the controller, the builder, the proxy" before).
|
||||||
// and the proxy together" before issue 206, and the fixture had no worker-of edge.)
|
|
||||||
small := reachableFrom([]string{"mesh-controller", "builder", "route-proxy"}, edges)
|
small := reachableFrom([]string{"mesh-controller", "builder", "route-proxy"}, edges)
|
||||||
smallTiers := tiersOf(small, edges)
|
smallTiers := tiersOf(small, edges)
|
||||||
if got := tiered(smallTiers); got != "mesh-controller | builder | route-proxy" {
|
if got := tiered(smallTiers); got != "mesh-controller,route-proxy | builder" {
|
||||||
t.Fatalf("the controller, then the builder, then the proxy: %v", smallTiers)
|
t.Fatalf("the controller and the proxy, then the builder: %v", smallTiers)
|
||||||
}
|
}
|
||||||
// The builder alone moved: the builder, and nothing it builds.
|
// The builder alone moved: the builder, and nothing it builds.
|
||||||
if only := reachableFrom([]string{"builder"}, edges); len(only) != 1 {
|
if only := reachableFrom([]string{"builder"}, edges); len(only) != 1 {
|
||||||
t.Fatalf("a build machine change rebuilds the build machine alone: %v", only)
|
t.Fatalf("a build machine change rebuilds the build machine alone: %v", only)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only a runtime dependency gates on deployment, and only when the module rolls out.
|
// Nothing waits on deployment for a built-by edge (novox/hq ADR 0360 §3): the builder gates no tier after it.
|
||||||
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"}, []string{"mesh-tools"}, edges)
|
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"}, []string{"mesh-tools"}, edges)
|
||||||
p.Tier = pos["builder"]
|
p.Tier = pos["builder"]
|
||||||
rollsOut := func(m string) bool { return m == "builder" }
|
rollsOut := func(m string) bool { return m == "builder" }
|
||||||
if g := gates(p, edges, rollsOut); len(g) != 1 || g[0] != "builder" {
|
if g := gates(p, edges, rollsOut); len(g) != 0 {
|
||||||
t.Fatalf("the builder gates the tier after it: %v", g)
|
t.Fatalf("the builder gates the tier after it: %v", g)
|
||||||
}
|
}
|
||||||
p.Tier = 0
|
p.Tier = 0
|
||||||
@@ -253,10 +255,10 @@ func TestAChangeToTheBuildAgentRebuildsTheBuildAgentAlone(t *testing.T) {
|
|||||||
t.Fatalf("%v planned %v: the build agent alone", paths, p.Tiers)
|
t.Fatalf("%v planned %v: the build agent alone", paths, p.Tiers)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// With what it builds moved beside it, the agent comes first and they follow: an order, not a widening.
|
// With what it builds moved beside it, both go in one tier (novox/hq ADR 0360 §3): built-by orders nothing.
|
||||||
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Commit: "abc"},
|
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Commit: "abc"},
|
||||||
[]string{"build-agent", "redis"}, edges)
|
[]string{"build-agent", "redis"}, edges)
|
||||||
if len(p.Tiers) != 2 || p.Tiers[0][0] != "build-agent" || p.Tiers[1][0] != "redis" || len(p.Modules) != 2 {
|
if len(p.Tiers) != 1 || !slices.Equal(p.Tiers[0], []string{"build-agent", "redis"}) || len(p.Modules) != 2 {
|
||||||
t.Fatalf("the agent, then what moved beside it: %v", p.Tiers)
|
t.Fatalf("the agent and what moved beside it, in one tier: %v", p.Tiers)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -111,8 +111,13 @@ func aBacklog(t *testing.T) *backlogMesh {
|
|||||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
|
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
// A broken machine reports its apply failed (novox/hq ADR 0360 §7: the report is the verdict).
|
||||||
|
outcome := inventory.OutcomeApplied
|
||||||
|
if b.broken[node] {
|
||||||
|
outcome = inventory.OutcomeFailed
|
||||||
|
}
|
||||||
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
||||||
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
Outcome: outcome, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -133,11 +138,15 @@ func aBacklog(t *testing.T) *backlogMesh {
|
|||||||
}
|
}
|
||||||
return f, nil
|
return f, nil
|
||||||
}
|
}
|
||||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||||
gateSettle, gateEvery = 0, time.Hour
|
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||||
|
wasWindow := gateWitnessWindow
|
||||||
|
gateWitnessWindow = 0
|
||||||
|
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||||
|
gateWithin, gateEvery = time.Hour, time.Hour
|
||||||
t.Cleanup(func() {
|
t.Cleanup(func() {
|
||||||
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
|
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
|
||||||
gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound
|
gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound
|
||||||
})
|
})
|
||||||
return b
|
return b
|
||||||
}
|
}
|
||||||
@@ -160,6 +169,7 @@ func (b *backlogMesh) release(t *testing.T) inventory.Plan {
|
|||||||
// The backlog goes out one machine at a time: the first judged before the second is sent, each build's
|
// The backlog goes out one machine at a time: the first judged before the second is sent, each build's
|
||||||
// pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused.
|
// pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused.
|
||||||
func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) {
|
func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
judgeLeavesAsTheCore(t)
|
judgeLeavesAsTheCore(t)
|
||||||
b := aBacklog(t)
|
b := aBacklog(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
@@ -215,6 +225,7 @@ func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) {
|
|||||||
// A release that fails on its first machine puts back what it carried there, goes no further, and the
|
// A release that fails on its first machine puts back what it carried there, goes no further, and the
|
||||||
// next one waits for a person, said as a condition; a person releases it with why.
|
// next one waits for a person, said as a condition; a person releases it with why.
|
||||||
func TestAFailedReleaseIsPutBackAndTheNextWaitsForAPerson(t *testing.T) {
|
func TestAFailedReleaseIsPutBackAndTheNextWaitsForAPerson(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
b := aBacklog(t)
|
b := aBacklog(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := b.open.inventory
|
inv := b.open.inventory
|
||||||
|
|||||||
@@ -0,0 +1,74 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestReplay164 replays novox/hq issue 164 (found provisioning the media machine's modules, re-checked
|
||||||
|
// 2026-10-08): a Servarr app's API key, an indexer's key and a downloader's password are values the provider's
|
||||||
|
// software holds and only a person can give, and for a consumer of such a provision the send made a random pair
|
||||||
|
// credential, sealed it to both ends and placed it; the provider refused it, and a consumer retrying it could be
|
||||||
|
// banned by the provider. The outcome asserted (ADR 0337): for a provision whose offer says its credential is
|
||||||
|
// issued outside the mesh, the send makes no value and places none, the consumer's machine still composes, and a
|
||||||
|
// value a person gave for the pair is placed. Written with only what the controller had before its fix —
|
||||||
|
// register, assign, planFor, declarationFor, AcceptSecretForPair and the inventory's secret reads — so it is laid
|
||||||
|
// over the commit before. Needs the store (MESH_TEST_POSTGRES), and is named in testdata/store-tests.
|
||||||
|
func TestReplay164(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
const secretFile = "/var/lib/mesh/requests/tv-api.secret"
|
||||||
|
register(t, open, catalogue.Manifest{Module: "tv", Version: "1",
|
||||||
|
Provides: []catalogue.Offer{{Name: "tv-api", Scope: catalogue.ScopeMesh,
|
||||||
|
Credential: &catalogue.OfferCredential{IssuedBy: catalogue.IssuedOutside}}},
|
||||||
|
Serves: map[string]map[string]any{"tv-api": {"port": 8989}}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "requests", Version: "1", Requires: []string{"tv-api"},
|
||||||
|
Secrets: map[string]string{"tv-api": secretFile}})
|
||||||
|
for _, a := range [][2]string{{"anchor", "tv"}, {"laptop", "requests"}} {
|
||||||
|
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sent := func() []map[string]any {
|
||||||
|
t.Helper()
|
||||||
|
plan, settings, err := planFor(ctx, open, "laptop")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the consumer's machine was refused for a credential only a person gives: %v", err)
|
||||||
|
}
|
||||||
|
declared, err := declarationFor(ctx, open, "laptop", plan, settings)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the send to laptop did not compose: %v", err)
|
||||||
|
}
|
||||||
|
return declared.Resources
|
||||||
|
}
|
||||||
|
placed := func(resources []map[string]any) map[string]any {
|
||||||
|
for _, r := range resources {
|
||||||
|
if r["path"] == secretFile && r["sealed"] != nil {
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := placed(sent()); got != nil {
|
||||||
|
t.Errorf("the send placed a credential for tv-api (%v), which only a person can give and nobody gave", got["id"])
|
||||||
|
}
|
||||||
|
held, err := open.inventory.SecretsOf(ctx, "laptop", "requests")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, s := range held {
|
||||||
|
if s.Name == "tv-api" {
|
||||||
|
t.Fatalf("the mesh holds a value for tv-api that nobody gave (origin %q)", s.Origin)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Given by a person for the pair, it is placed, as given.
|
||||||
|
if err := open.inventory.AcceptSecretForPair(ctx, "tv-api", "laptop", "requests", "anchor", "", "the app's key"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if placed(sent()) == nil {
|
||||||
|
t.Fatal("the value a person gave for tv-api was not placed")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -31,6 +31,7 @@ import (
|
|||||||
// nothing is put back, the walk goes on to the workstation, and openrazer's condition says, in one sentence
|
// nothing is put back, the walk goes on to the workstation, and openrazer's condition says, in one sentence
|
||||||
// for the operator, that a new login is needed on the laptop.
|
// for the operator, that a new login is needed on the laptop.
|
||||||
func TestReplay318(t *testing.T) {
|
func TestReplay318(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
@@ -112,10 +113,14 @@ func TestReplay318(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts
|
wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts
|
||||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||||
|
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||||
|
wasWindow := gateWitnessWindow
|
||||||
|
gateWitnessWindow = 0
|
||||||
|
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||||
t.Cleanup(func() {
|
t.Cleanup(func() {
|
||||||
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
|
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
|
||||||
gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound
|
gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound
|
||||||
})
|
})
|
||||||
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
|
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
|
||||||
modules, err := open.inventory.Assigned(ctx, node)
|
modules, err := open.inventory.Assigned(ctx, node)
|
||||||
@@ -184,7 +189,7 @@ func TestReplay318(t *testing.T) {
|
|||||||
return f, nil
|
return f, nil
|
||||||
}
|
}
|
||||||
// The bound is reached at the first judging that is not a pass: what happened at 11:21 happens at once.
|
// The bound is reached at the first judging that is not a pass: what happened at 11:21 happens at once.
|
||||||
gateSettle, gateEvery, gateBound = 0, 0, 0
|
gateWithin, gateEvery, gateBound = time.Hour, 0, 0
|
||||||
|
|
||||||
for i := 0; i < 12; i++ {
|
for i := 0; i < 12; i++ {
|
||||||
advancePlans(ctx, open)
|
advancePlans(ctx, open)
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import (
|
|||||||
// at its gate with the pass's own words, and put it back. Written with only what the controller had before
|
// at its gate with the pass's own words, and put it back. Written with only what the controller had before
|
||||||
// its fix, so it is laid over the commit before.
|
// its fix, so it is laid over the commit before.
|
||||||
func TestReplay335(t *testing.T) {
|
func TestReplay335(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
t.Run("the first machine's later reports", testAPassedGateIsNotJudgedAgainFromTheFirstMachinesLaterReports)
|
t.Run("the first machine's later reports", testAPassedGateIsNotJudgedAgainFromTheFirstMachinesLaterReports)
|
||||||
t.Run("a walk stopped after the pass", testAWalkStoppedAfterItsGatePassedKeepsThePass)
|
t.Run("a walk stopped after the pass", testAWalkStoppedAfterItsGatePassedKeepsThePass)
|
||||||
}
|
}
|
||||||
@@ -77,6 +78,7 @@ func testAWalkStoppedAfterItsGatePassedKeepsThePass(t *testing.T) {
|
|||||||
// the first send is older than the wait for a first machine's report. Read before the pass is taken over, the
|
// the first send is older than the wait for a first machine's report. Read before the pass is taken over, the
|
||||||
// carried module's step said the first machine never applied it, and the passed build was put back.
|
// carried module's step said the first machine never applied it, and the passed build was put back.
|
||||||
func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) {
|
func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
judgeLeavesAsTheCore(t)
|
judgeLeavesAsTheCore(t)
|
||||||
tm := aTierMesh(t, "m01", "m02")
|
tm := aTierMesh(t, "m01", "m02")
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ import (
|
|||||||
// had before its fix — the stores, register, assign, a plan saved and advanced, the watchdogs' gathering and
|
// had before its fix — the stores, register, assign, a plan saved and advanced, the watchdogs' gathering and
|
||||||
// seeing — so it is laid over the commit before.
|
// seeing — so it is laid over the commit before.
|
||||||
func TestReplay336(t *testing.T) {
|
func TestReplay336(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
|
|||||||
@@ -0,0 +1,71 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **A new SDK plans the toolchain, and then every bundle compiled in it** (novox/hq issue 614), reasoned
|
||||||
|
// as `mesh-controller.delivery-plan` reasons it: the modules registered as a build registers them —
|
||||||
|
// resolved, so carrying no `build` — their builds recorded with what they stood on and what they made,
|
||||||
|
// the graph read by theGraph and the change scoped by pullScope. On 2026-10-13 mesh-sdk 0.1.14 planned
|
||||||
|
// "builds mesh-sdk, sent nowhere": mesh-tools' `build.on` naming the SDK's package and the providers'
|
||||||
|
// TypeScript bundles were edges only while their manifests were unbuilt.
|
||||||
|
//
|
||||||
|
// **The replay of issue 614** (mesh-lab R614): written only with what the controller had before its fix — a
|
||||||
|
// store, modules registered and builds recorded, theGraph and pullScope — so it is laid over the commit before
|
||||||
|
// and fails there with the plan of that day, [[mesh-sdk]]. Needs a store: MESH_TEST_POSTGRES.
|
||||||
|
func TestReplay614ANewSDKPlansTheToolchainAndThenItsBundles(t *testing.T) {
|
||||||
|
inv := inventory.ForTest(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
asked := time.Now().Add(-time.Hour)
|
||||||
|
digest := func(c string) string { return "@sha256:" + strings.Repeat(c, 64) }
|
||||||
|
register := func(m catalogue.Manifest, repository, path string, against []string, made []inventory.Artifact) {
|
||||||
|
t.Helper()
|
||||||
|
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: repository, Seat: "git", Path: path,
|
||||||
|
Ref: "main", BuiltFrom: "old", Asked: asked}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + m.Module, Repository: repository, Ref: "main",
|
||||||
|
Module: m.Module, Commit: "old", On: "builder", Path: path, Against: against, Made: made, Asked: asked}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bundle := func(module, language string) catalogue.Manifest {
|
||||||
|
return catalogue.Manifest{Module: module, Version: "1", Bundles: []catalogue.Bundle{{
|
||||||
|
Name: "code", Language: language, Source: catalogue.ArtifactStoreScheme + module + "/code" + digest("2"),
|
||||||
|
Entrypoints: []string{"tools/index.js"}}}}
|
||||||
|
}
|
||||||
|
register(catalogue.Manifest{Module: "build-agent", Version: "1",
|
||||||
|
Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}},
|
||||||
|
"novox/mesh-catalog", "modules/build-agent", nil, nil)
|
||||||
|
register(catalogue.Manifest{Module: "mesh-sdk", Version: "1"}, "novox/mesh-sdk", "", nil,
|
||||||
|
[]inventory.Artifact{{Name: "lib", Kind: "package", Reference: "@novox/mesh-sdk@0.1.13"}})
|
||||||
|
register(catalogue.Manifest{Module: "mesh-tools", Version: "1"}, "novox/mesh-tools", "",
|
||||||
|
[]string{"@novox/mesh-sdk@0.1.13", catalogue.ArtifactStoreScheme + "upstream/docker.io/library/node" + digest("0")},
|
||||||
|
[]inventory.Artifact{{Name: "build", Kind: "image", Reference: catalogue.ArtifactStoreScheme + "mesh-tools/build" + digest("1")}})
|
||||||
|
register(catalogue.Manifest{Module: "mesh-tools-go", Version: "1"}, "novox/mesh-tools", "go", nil, nil)
|
||||||
|
register(bundle("minio", "typescript"), "novox/mesh-catalog", "modules/minio", nil, nil)
|
||||||
|
register(bundle("umami", "typescript"), "novox/mesh-catalog", "modules/umami", nil, nil)
|
||||||
|
register(bundle("dnsmasq", "go"), "novox/mesh-catalog", "modules/dnsmasq", nil, nil)
|
||||||
|
|
||||||
|
entries, read, edges, err := theGraph(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := orderMember{ID: "novox/mesh-sdk@new", Repository: "novox/mesh-sdk", Head: "new",
|
||||||
|
Paths: []string{"src/provisioner/index.ts", "package.json"}}
|
||||||
|
s := pullScope(m.pull(), entries, read, edges)
|
||||||
|
want := [][]string{{"mesh-sdk"}, {"mesh-tools"}, {"minio", "umami"}}
|
||||||
|
if !reflect.DeepEqual(s.Reach.Plan.Tiers, want) {
|
||||||
|
t.Fatalf("a new SDK plans %v, want %v (edges %v)", s.Reach.Plan.Tiers, want, edges)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(s.Dependents, []string{"mesh-tools", "minio", "umami"}) {
|
||||||
|
t.Errorf("dependents %v", s.Dependents)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -139,6 +139,7 @@ func TestReplay273AConsumerBesideItsStoreStaysBoundToIt(t *testing.T) {
|
|||||||
// one, kept below. Heard as bytes, so an older controller reads them as it reads any report — this file is
|
// one, kept below. Heard as bytes, so an older controller reads them as it reads any report — this file is
|
||||||
// written only with what the controller had before the judging, for the prover to lay over that commit.
|
// written only with what the controller had before the judging, for the prover to lay over that commit.
|
||||||
func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) {
|
func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
@@ -248,9 +249,13 @@ func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) {
|
|||||||
return names, nil
|
return names, nil
|
||||||
}
|
}
|
||||||
t.Cleanup(func() { sendRollout = wasSend })
|
t.Cleanup(func() { sendRollout = wasSend })
|
||||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||||
gateSettle, gateEvery = 0, 0
|
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||||
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
wasWindow := gateWitnessWindow
|
||||||
|
gateWitnessWindow = 0
|
||||||
|
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||||
|
gateWithin, gateEvery = time.Hour, 0
|
||||||
|
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
|
||||||
|
|
||||||
built := time.Now().UTC()
|
built := time.Now().UTC()
|
||||||
plan := inventory.Plan{ID: "plan-crashloop", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
plan := inventory.Plan{ID: "plan-crashloop", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
||||||
|
|||||||
@@ -0,0 +1,382 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq ADR 0353, kept behind gateFirstMachine for walks a controller before ADR 0360 started: after its first machine passes, a build goes to every other machine at once, each judged by
|
||||||
|
// its own gate and a failing one put back by itself; and a gate's readings outlive the controller that took them —
|
||||||
|
// a machine already sent what a send is for is not sent again, and its judging goes on from what was read.
|
||||||
|
//
|
||||||
|
// Seen 2026-10-14: release-1791930194184991192 sent ace, g14, shanks and the control node one after another in
|
||||||
|
// eleven minutes; the new controller its last send started sent the control node again the controller build it
|
||||||
|
// had just applied, and judged it from its first reading.
|
||||||
|
|
||||||
|
// aBacklogOnThree is the backlog fixture with a third machine, desk, running app at c1 like the others.
|
||||||
|
func aBacklogOnThree(t *testing.T) *backlogMesh {
|
||||||
|
t.Helper()
|
||||||
|
b := aBacklog(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := b.open.inventory
|
||||||
|
if _, err := inv.AddNode(ctx, "desk"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := inv.Assign(ctx, "desk", "app"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RecordSent(ctx, nodeID(t, b.open, "desk"), "d-desk", map[string]string{"app": "c1"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wasHeard := releaseHeard
|
||||||
|
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
|
||||||
|
return map[string]bool{"anchor": true, "laptop": true, "desk": true}, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { releaseHeard = wasHeard })
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
// events records every send and every judging, in order.
|
||||||
|
func events(t *testing.T) *[]string {
|
||||||
|
t.Helper()
|
||||||
|
var said []string
|
||||||
|
wasSend, wasGather := sendRollout, gatherGateFacts
|
||||||
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||||
|
said = append(said, "send "+strings.Join(names, ","))
|
||||||
|
return wasSend(ctx, open, names)
|
||||||
|
}
|
||||||
|
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||||
|
said = append(said, "judge")
|
||||||
|
return wasGather(ctx, open, component)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { sendRollout, gatherGateFacts = wasSend, wasGather })
|
||||||
|
return &said
|
||||||
|
}
|
||||||
|
|
||||||
|
// Once the first machine passed, the rest are both sent before either is judged (ADR 0353 §1); before, the second
|
||||||
|
// was judged to its verdict before the third was sent.
|
||||||
|
func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) {
|
||||||
|
withTheGate(t)
|
||||||
|
judgeLeavesAsTheCore(t)
|
||||||
|
b := aBacklogOnThree(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
said := events(t)
|
||||||
|
gateEvery = 0
|
||||||
|
for i := 0; i < 12; i++ {
|
||||||
|
advancePlans(ctx, b.open)
|
||||||
|
}
|
||||||
|
p := b.release(t)
|
||||||
|
if p.State != inventory.PlanDone || len(p.Release.Done) != 3 {
|
||||||
|
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||||
|
}
|
||||||
|
desk, laptop := slices.Index(*said, "send desk"), slices.Index(*said, "send laptop")
|
||||||
|
if desk < 0 || laptop < 0 || slices.Index(*said, "send anchor") > min(desk, laptop) {
|
||||||
|
t.Fatalf("sends and judgings: %v", *said)
|
||||||
|
}
|
||||||
|
if slices.Contains((*said)[min(desk, laptop):max(desk, laptop)], "judge") {
|
||||||
|
t.Fatalf("the rest were sent one at a time, each judged before the next: %v", *said)
|
||||||
|
}
|
||||||
|
if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||||
|
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine of the rest whose gate fails is put back at once, and the other machine of the step goes on to its own
|
||||||
|
// verdict (ADR 0353 §3); before, the walk stopped at the failure and the machine after it was never sent.
|
||||||
|
func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) {
|
||||||
|
withTheGate(t)
|
||||||
|
judgeLeavesAsTheCore(t)
|
||||||
|
b := aBacklogOnThree(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := b.open.inventory
|
||||||
|
gateEvery = 0
|
||||||
|
b.broken["desk"] = true // desk fails what it is sent in the rest's step
|
||||||
|
advancePlans(ctx, b.open)
|
||||||
|
for i := 0; i < 12 && b.release(t).Open(); i++ {
|
||||||
|
advancePlans(ctx, b.open)
|
||||||
|
}
|
||||||
|
p := b.release(t)
|
||||||
|
if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) {
|
||||||
|
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||||
|
}
|
||||||
|
if !slices.ContainsFunc(b.sent, func(s []string) bool { return slices.Equal(s, []string{"laptop"}) }) {
|
||||||
|
t.Fatalf("laptop was never sent: %v", b.sent)
|
||||||
|
}
|
||||||
|
if !slices.Contains(p.Release.Done, "laptop") {
|
||||||
|
t.Fatalf("laptop has no verdict of its own: %+v", p.Release)
|
||||||
|
}
|
||||||
|
// late waited on laptop alone and was healthy there: its pass is kept, whatever desk did.
|
||||||
|
if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||||
|
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
||||||
|
}
|
||||||
|
// app failed on desk: marked, and put back wherever it was sent (ADR 0236).
|
||||||
|
if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed {
|
||||||
|
t.Fatal("app's build is not marked failed")
|
||||||
|
}
|
||||||
|
if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c1" {
|
||||||
|
t.Fatalf("desk was not put back: %v", sent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A walk whose first machine already runs its build under another walk's gate sends nothing and goes on from that
|
||||||
|
// gate's readings (ADR 0353 §7); before, it sent the machine again and judged from its first reading.
|
||||||
|
func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) {
|
||||||
|
withTheGate(t)
|
||||||
|
judgeLeavesAsTheCore(t)
|
||||||
|
b := aBacklog(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := b.open.inventory
|
||||||
|
advancePlans(ctx, b.open) // the release sends anchor app c2, and reads it once
|
||||||
|
release := b.release(t)
|
||||||
|
judging := release.Release.Gate
|
||||||
|
if judging == nil || judging.Pending || judging.Passes == 0 {
|
||||||
|
t.Fatalf("the release is not judging anchor: %+v", judging)
|
||||||
|
}
|
||||||
|
built := time.Now().UTC()
|
||||||
|
plan := inventory.Plan{ID: "plan-app", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
||||||
|
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2",
|
||||||
|
Build: "build-app-c2"}}}
|
||||||
|
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
advancePlans(ctx, b.open)
|
||||||
|
if len(b.sent) != 1 {
|
||||||
|
t.Fatalf("anchor was sent again what it runs: %v", b.sent)
|
||||||
|
}
|
||||||
|
p, err := inv.PlanByID(ctx, "plan-app")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
g := p.Modules["app"].Gate
|
||||||
|
if g == nil || g.Since == nil || !g.Since.Equal(*judging.Since) || g.Passes < judging.Passes || g.TookOver != release.ID {
|
||||||
|
t.Fatalf("the walk's gate starts from zero: %+v (the release's: %+v)", g, judging)
|
||||||
|
}
|
||||||
|
if p.Modules["app"].Previous != "c1" {
|
||||||
|
t.Fatalf("what anchor ran before is %q, not c1", p.Modules["app"].Previous)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A walk's first send that was made and not kept — the controller replaced between the two — is not made again
|
||||||
|
// by the controller that takes over: the gate counts from that send (ADR 0353 §5, §6).
|
||||||
|
func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) {
|
||||||
|
withTheGate(t)
|
||||||
|
judgeLeavesAsTheCore(t)
|
||||||
|
g := aGateMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
// The old controller sends anchor app c2 and is replaced before it keeps the send.
|
||||||
|
made := sendRollout
|
||||||
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||||
|
if _, err := made(ctx, open, names); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return nil, context.Canceled
|
||||||
|
}
|
||||||
|
advancePlans(ctx, g.open)
|
||||||
|
sendRollout = made
|
||||||
|
if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil {
|
||||||
|
t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent)
|
||||||
|
}
|
||||||
|
last, _, err := g.open.inventory.SentTo(ctx, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
advancePlans(ctx, g.open)
|
||||||
|
if len(g.sent) != 1 {
|
||||||
|
t.Fatalf("the send was made again: %v", g.sent)
|
||||||
|
}
|
||||||
|
s := g.plan(t).Modules["app"]
|
||||||
|
if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) {
|
||||||
|
t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate)
|
||||||
|
}
|
||||||
|
// What anchor ran before was kept before the send, so a failure still puts it back (§5).
|
||||||
|
if s.Previous != "c1" || s.Gate.From != "c1" {
|
||||||
|
t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A release's send kept before it was made, and made, is judged by the controller that takes over without being
|
||||||
|
// made again (ADR 0353 §8).
|
||||||
|
func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) {
|
||||||
|
withTheGate(t)
|
||||||
|
judgeLeavesAsTheCore(t)
|
||||||
|
b := aBacklog(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := b.open.inventory
|
||||||
|
p, err := releaseBacklog(ctx, b.open, "")
|
||||||
|
if err != nil || p == nil {
|
||||||
|
t.Fatalf("no release opened: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gate == nil || !p.Release.Gate.Pending {
|
||||||
|
t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err)
|
||||||
|
}
|
||||||
|
if err := inv.SavePlan(ctx, p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// The send is made, and the controller is replaced before it keeps it.
|
||||||
|
if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
advancePlans(ctx, b.open)
|
||||||
|
if len(b.sent) != 1 {
|
||||||
|
t.Fatalf("anchor was sent again: %v", b.sent)
|
||||||
|
}
|
||||||
|
g := b.release(t).Release.Gate
|
||||||
|
if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 {
|
||||||
|
t.Fatalf("anchor is not judged on the send made: %+v", g)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A build no gate has passed goes to one machine of a step, its first; a machine that would be its second waits
|
||||||
|
// for the next step, and one whose every build passed goes now (ADR 0353 §2).
|
||||||
|
func TestABuildThatPassedNoGateGoesToOneMachineOfAStep(t *testing.T) {
|
||||||
|
withTheGate(t)
|
||||||
|
waiting := map[string][]inventory.CarriedMove{
|
||||||
|
"desk": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}},
|
||||||
|
"laptop": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}},
|
||||||
|
"shanks": {{Module: "app", To: "c2"}},
|
||||||
|
"novox": {{Module: "app", To: "c2"}, {Module: "ctl", To: "k1"}},
|
||||||
|
}
|
||||||
|
passed := func(module, commit string) bool { return module == "app" && commit == "c2" }
|
||||||
|
never := func(string) bool { return false }
|
||||||
|
step := restStep([]string{"desk", "laptop", "shanks", "novox"}, waiting, passed, never)
|
||||||
|
if !slices.Equal(step, []string{"desk", "shanks", "novox"}) {
|
||||||
|
t.Fatalf("the step is %v: laptop would be new's second machine before it passed", step)
|
||||||
|
}
|
||||||
|
if step := restStep([]string{"laptop"}, waiting, func(string, string) bool { return true }, never); !slices.Equal(step, []string{"laptop"}) {
|
||||||
|
t.Fatalf("a machine whose every build passed waits: %v", step)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The node-engine and the mesh's resolver go to the rest one machine at a time, even passed (ADR 0353 §9): a step
|
||||||
|
// carries either to one machine only, while what else passed goes everywhere at once.
|
||||||
|
func TestTheNodeEngineAndTheResolverGoToTheRestOneAtATime(t *testing.T) {
|
||||||
|
withTheGate(t)
|
||||||
|
waiting := map[string][]inventory.CarriedMove{
|
||||||
|
"desk": {{Module: hostModule, To: "h2"}},
|
||||||
|
"laptop": {{Module: hostModule, To: "h2"}, {Module: "app", To: "c2"}},
|
||||||
|
"shanks": {{Module: "app", To: "c2"}},
|
||||||
|
"novox": {{Module: "dns", To: "d2"}},
|
||||||
|
"ace": {{Module: "dns", To: "d2"}},
|
||||||
|
}
|
||||||
|
resolver := catalogue.Manifest{Module: "dns", Claims: []catalogue.Claim{{Name: resolverSeat}}}
|
||||||
|
alone := func(m string) bool {
|
||||||
|
if m == "dns" {
|
||||||
|
return oneAtATime(m, resolver)
|
||||||
|
}
|
||||||
|
return oneAtATime(m, catalogue.Manifest{Module: m})
|
||||||
|
}
|
||||||
|
step := restStep([]string{"desk", "laptop", "shanks", "novox", "ace"}, waiting,
|
||||||
|
func(string, string) bool { return true }, alone)
|
||||||
|
if !slices.Equal(step, []string{"desk", "shanks", "novox"}) {
|
||||||
|
t.Fatalf("the step is %v: the node-engine and the resolver each reach one machine of it", step)
|
||||||
|
}
|
||||||
|
if oneAtATime("app", catalogue.Manifest{Module: "app"}) {
|
||||||
|
t.Fatal("a leaf module is held to one machine at a time")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent.
|
||||||
|
func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) {
|
||||||
|
withTheGate(t)
|
||||||
|
b := aBacklogOnThree(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
gateEvery = 0
|
||||||
|
b.broken["anchor"] = true
|
||||||
|
advancePlans(ctx, b.open)
|
||||||
|
for i := 0; i < 6 && b.release(t).Open(); i++ {
|
||||||
|
advancePlans(ctx, b.open)
|
||||||
|
}
|
||||||
|
if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest {
|
||||||
|
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||||
|
}
|
||||||
|
for _, s := range b.sent {
|
||||||
|
if !slices.Equal(s, []string{"anchor"}) {
|
||||||
|
t.Fatalf("sent %v: the rest after a failed first machine", b.sent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The control node is sent last in a step of the rest (ADR 0353 §1): the machine holding the controller is
|
||||||
|
// ordered after every other.
|
||||||
|
func TestTheControlNodeIsSentLastInAStep(t *testing.T) {
|
||||||
|
withTheGate(t)
|
||||||
|
judgeLeavesAsTheCore(t)
|
||||||
|
b := aBacklogOnThree(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, b.open, catalogue.Manifest{Module: catalogue.ControllerSeatName, Version: "1"})
|
||||||
|
if _, err := b.open.inventory.Assign(ctx, "anchor", catalogue.ControllerSeatName); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
said := events(t)
|
||||||
|
gateEvery = 0
|
||||||
|
for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ {
|
||||||
|
advancePlans(ctx, b.open)
|
||||||
|
}
|
||||||
|
if p := b.release(t); !slices.Equal(p.Release.Order, []string{"desk", "laptop", "anchor"}) {
|
||||||
|
t.Fatalf("the order is %v", p.Release.Order)
|
||||||
|
}
|
||||||
|
laptop, anchor := slices.Index(*said, "send laptop"), slices.Index(*said, "send anchor")
|
||||||
|
if laptop < 0 || anchor < laptop || slices.Contains((*said)[laptop:anchor], "judge") {
|
||||||
|
t.Fatalf("the control node is not sent last in the rest's step: %v", *said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put
|
||||||
|
// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4).
|
||||||
|
func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) {
|
||||||
|
withTheGate(t)
|
||||||
|
judgeLeavesAsTheCore(t)
|
||||||
|
b := aBacklog(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := b.open.inventory
|
||||||
|
reporting := sendRollout
|
||||||
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||||
|
if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing
|
||||||
|
b.sent = append(b.sent, names)
|
||||||
|
current, _ := open.inventory.CurrentBuilds(ctx)
|
||||||
|
return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep",
|
||||||
|
map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit})
|
||||||
|
}
|
||||||
|
return reporting(ctx, open, names)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { sendRollout = reporting })
|
||||||
|
gateEvery = 0
|
||||||
|
for i := 0; i < 8; i++ {
|
||||||
|
advancePlans(ctx, b.open)
|
||||||
|
}
|
||||||
|
p := b.release(t)
|
||||||
|
g := p.Release.Gates["laptop"]
|
||||||
|
if !p.Open() || g == nil || g.Verdict != "" || g.Silent {
|
||||||
|
t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g)
|
||||||
|
}
|
||||||
|
since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute)
|
||||||
|
g.Since = &since
|
||||||
|
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
advancePlans(ctx, b.open)
|
||||||
|
p = b.release(t)
|
||||||
|
if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 {
|
||||||
|
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||||
|
}
|
||||||
|
if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed {
|
||||||
|
t.Fatal("a silent machine's build was marked failed")
|
||||||
|
}
|
||||||
|
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
|
||||||
|
t.Fatalf("laptop was put back: %v", sent)
|
||||||
|
}
|
||||||
|
obs, err := silentObservation(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) {
|
||||||
|
t.Fatalf("the silence is not said: %+v", obs)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,176 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **A module moves only when its program changes** (novox/hq issue 721). A merge check builds every module the
|
||||||
|
// delivery plan moves and compares each with the build the mesh holds of it; a walk is cut without a module whose
|
||||||
|
// check found its whole image the same as that build, while the mesh still holds it, and says so among the modules
|
||||||
|
// it left unchanged. A merge of one sentence in a package a program imports but never reaches moves no build of it.
|
||||||
|
|
||||||
|
// sameAsRunning are the modules of a walk its checks found the same as the build the mesh holds, each with what its
|
||||||
|
// check said. **Only when every pull request the walk answers was checked at the head it merged, and every one of
|
||||||
|
// those checks built the module and found it the same as the build the mesh still holds**: a merge the walk answers
|
||||||
|
// without a pull request, a check of another head, a check that built it differently, or a held build that moved
|
||||||
|
// since, leaves the module in the walk.
|
||||||
|
func sameAsRunning(plan *inventory.Plan, builds []link.CheckedBuild, held map[string]string) map[string]string {
|
||||||
|
if plan.Delivery == nil || len(plan.Delivery.Merges) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, m := range plan.Delivery.Merges {
|
||||||
|
if _, ok := pullOf(m); !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pulls := reuseAskOf(plan).Pulls
|
||||||
|
heads := mergedHeadsOf(plan)
|
||||||
|
out := map[string]string{}
|
||||||
|
for name := range plan.Modules {
|
||||||
|
said := ""
|
||||||
|
for _, pull := range pulls {
|
||||||
|
b, ok := checkedBuildOf(builds, pull, heads[pull], name)
|
||||||
|
if !ok || !stillHeld(b.Built, name, held) {
|
||||||
|
said = ""
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if said == "" {
|
||||||
|
said = b.Built.Same
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if said != "" {
|
||||||
|
out[name] = said
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(out) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkedBuildOf is the newest build a check of the pull request at head made of the module; false when none did.
|
||||||
|
func checkedBuildOf(builds []link.CheckedBuild, pull link.PullRef, head, module string) (link.CheckedBuild, bool) {
|
||||||
|
if head == "" {
|
||||||
|
return link.CheckedBuild{}, false
|
||||||
|
}
|
||||||
|
for _, b := range builds {
|
||||||
|
if b.Pull == pull && b.Built.Module == module && link.SameHead(b.Head, head) {
|
||||||
|
return b, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return link.CheckedBuild{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// stillHeld is whether a check found every artifact of the module the same as a build the mesh still holds.
|
||||||
|
func stillHeld(b link.CheckBuilt, module string, held map[string]string) bool {
|
||||||
|
if b.Same == "" || b.NotBuilt != "" || len(b.Made) == 0 || len(b.SameAs) != len(b.Made) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, m := range b.Made {
|
||||||
|
was := pinnedDigest(b.SameAs[m.Name])
|
||||||
|
if was == "" || was != pinnedDigest(held[module+"/"+m.Name]) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// pinnedDigest is a reference's pinned digest, empty when it is not pinned.
|
||||||
|
func pinnedDigest(ref string) string {
|
||||||
|
if i := strings.LastIndex(ref, "@"); i >= 0 {
|
||||||
|
return ref[i+1:]
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// leaveSameAsRunning takes from a walk being cut every module its checks found the same as the build the mesh holds
|
||||||
|
// (sameAsRunning), and every dependent in the walk only for one of those (withDependentsOf): what was taken, with why. **Anything not read leaves the walk as it was**, said.
|
||||||
|
func leaveSameAsRunning(ctx context.Context, inv *inventory.Inventory, plan *inventory.Plan, moved []string,
|
||||||
|
edges []inventory.Edge) map[string]string {
|
||||||
|
if plan.Delivery == nil || len(plan.Modules) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
pulls := reuseAskOf(plan).Pulls
|
||||||
|
if len(pulls) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
builds, err := checkedBuildsOf(ctx, pulls)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf(" the checks' builds could not be read, so every module moves: %v\n", err)
|
||||||
|
// empty-on-error: said above; nothing left out of the walk is the safe answer, every module moves as before
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
held, err := inv.RegisteredHeld(ctx)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf(" what the mesh holds could not be read, so every module moves: %v\n", err)
|
||||||
|
// empty-on-error: said above; nothing left out of the walk is the safe answer, every module moves as before
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
same := withDependentsOf(sameAsRunning(plan, builds, held), plan, moved, edges)
|
||||||
|
if len(same) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(same))
|
||||||
|
for n := range same {
|
||||||
|
names = append(names, n)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
withhold(plan, names, nil)
|
||||||
|
for _, n := range names {
|
||||||
|
fmt.Printf(" %s left out: %s\n", n, same[n])
|
||||||
|
}
|
||||||
|
return same
|
||||||
|
}
|
||||||
|
|
||||||
|
// withDependentsOf adds to the modules found the same every module of the walk that is in it only as a dependent of
|
||||||
|
// one of them (novox/hq ADR 0267 rule 3): **a dependent is left out when its base comes out the same**, since the
|
||||||
|
// base it stands on stays the build the mesh holds. A module a merge moves itself, or a dependent of a base that
|
||||||
|
// still moves, stays. moved are the modules the walk's merges move; edges the graph the walk was planned along.
|
||||||
|
func withDependentsOf(same map[string]string, plan *inventory.Plan, moved []string, edges []inventory.Edge) map[string]string {
|
||||||
|
if len(same) == 0 {
|
||||||
|
return same
|
||||||
|
}
|
||||||
|
var still []string
|
||||||
|
for _, n := range moved {
|
||||||
|
if same[n] == "" {
|
||||||
|
still = append(still, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
keep := map[string]bool{}
|
||||||
|
for _, n := range reachableFrom(still, edges) {
|
||||||
|
keep[n] = true
|
||||||
|
}
|
||||||
|
// Until nothing more is found: a dependent of a dependent left out is left out too.
|
||||||
|
for grew := true; grew; {
|
||||||
|
grew = false
|
||||||
|
for n := range plan.Modules {
|
||||||
|
if keep[n] || same[n] != "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var bases []string
|
||||||
|
for _, e := range edges {
|
||||||
|
ordering := e.Kind == inventory.EdgeBuiltBy || e.Kind == inventory.EdgeWorkerOf ||
|
||||||
|
e.Kind == inventory.EdgePackages || e.Kind == edgeGroupOrder
|
||||||
|
if !ordering && e.From == n && same[e.To] != "" && !slices.Contains(bases, e.To) {
|
||||||
|
bases = append(bases, e.To)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(bases)
|
||||||
|
if len(bases) == 0 {
|
||||||
|
// In the walk for no base found the same: whatever brought it in is not said here, so it moves.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
same[n] = "unchanged: it is in the walk only for " + strings.Join(bases, ", ") +
|
||||||
|
", which came out the same as the build the mesh holds (ADR 0267 rule 3)"
|
||||||
|
grew = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return same
|
||||||
|
}
|
||||||
@@ -0,0 +1,141 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestAWalkLeavesOutAModuleWhoseProgramDidNotChange is novox/hq issue 721: mesh-controller #335 changed one sentence
|
||||||
|
// that mesh-builder never reaches; its check found build-agent the same as the build the mesh holds, so the walk moves
|
||||||
|
// mesh-controller alone and says build-agent is unchanged.
|
||||||
|
func TestAWalkLeavesOutAModuleWhoseProgramDidNotChange(t *testing.T) {
|
||||||
|
pull := link.PullRef{Owner: "novox", Repo: "mesh-controller", Number: 335}
|
||||||
|
head := "34050dc6aaaabbbbccccdddd"
|
||||||
|
aPlan := func() *inventory.Plan {
|
||||||
|
return &inventory.Plan{Tiers: [][]string{{"mesh-controller", "build-agent"}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{"mesh-controller": {}, "build-agent": {}},
|
||||||
|
Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{{Repository: "novox/mesh-controller",
|
||||||
|
Commit: "c2ec0a60", Number: 335, Head: head}}}}
|
||||||
|
}
|
||||||
|
held := map[string]string{"build-agent/agent": "store/build-agent@sha256:aaaa",
|
||||||
|
"mesh-controller/controller": "store/mesh-controller@sha256:cccc"}
|
||||||
|
builds := []link.CheckedBuild{
|
||||||
|
{Pull: pull, Head: head[:12], Built: link.CheckBuilt{Module: "build-agent", Key: "k1",
|
||||||
|
Made: []link.MadeArtifact{{Name: "agent", Kind: "image", Reference: "scratch/build-agent@sha256:bbbb"}},
|
||||||
|
Same: "unchanged: its program is the same as running build agent aaaa",
|
||||||
|
SameAs: map[string]string{"agent": "10.0.0.1:5000/build-agent@sha256:aaaa"}}},
|
||||||
|
{Pull: pull, Head: head[:12], Built: link.CheckBuilt{Module: "mesh-controller", Key: "k2",
|
||||||
|
Made: []link.MadeArtifact{{Name: "controller", Kind: "image", Reference: "scratch/mesh-controller@sha256:dddd"}}}},
|
||||||
|
}
|
||||||
|
|
||||||
|
p := aPlan()
|
||||||
|
same := sameAsRunning(p, builds, held)
|
||||||
|
if len(same) != 1 || same["build-agent"] == "" {
|
||||||
|
t.Fatalf("the walk leaves out %v, want build-agent alone", same)
|
||||||
|
}
|
||||||
|
withhold(p, []string{"build-agent"}, nil)
|
||||||
|
if len(p.Tiers) != 1 || len(p.Tiers[0]) != 1 || p.Tiers[0][0] != "mesh-controller" {
|
||||||
|
t.Fatalf("the walk moves %v, want mesh-controller alone", p.Tiers)
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The held build moved since the check**: what it was compared with no longer runs.
|
||||||
|
moved := map[string]string{"build-agent/agent": "store/build-agent@sha256:eeee"}
|
||||||
|
if same := sameAsRunning(aPlan(), builds, moved); len(same) != 0 {
|
||||||
|
t.Fatalf("a module compared with a build the mesh no longer holds is left out: %v", same)
|
||||||
|
}
|
||||||
|
// **A check of another head** says nothing of this merge.
|
||||||
|
other := aPlan()
|
||||||
|
other.Delivery.Merges[0].Head = "ffffffffffffffff"
|
||||||
|
if same := sameAsRunning(other, builds, held); len(same) != 0 {
|
||||||
|
t.Fatalf("a check of another head left out %v", same)
|
||||||
|
}
|
||||||
|
// **Another pull request the walk answers** that did not find it the same leaves it in.
|
||||||
|
two := aPlan()
|
||||||
|
two.Delivery.Merges = append(two.Delivery.Merges, inventory.PlanMerge{Repository: "novox/mesh-catalog",
|
||||||
|
Commit: "1234", Number: 9, Head: "abcdefabcdefabcdef"})
|
||||||
|
if same := sameAsRunning(two, builds, held); len(same) != 0 {
|
||||||
|
t.Fatalf("a module another merge moves is left out: %v", same)
|
||||||
|
}
|
||||||
|
// **A merge with no pull request** leaves the walk as it is.
|
||||||
|
direct := aPlan()
|
||||||
|
direct.Delivery.Merges[0].Number = 0
|
||||||
|
if same := sameAsRunning(direct, builds, held); len(same) != 0 {
|
||||||
|
t.Fatalf("a walk answering a merge without a pull request left out %v", same)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestADependentOfABaseThatCameOutTheSameIsLeftOut is ADR 0267 rule 3 (novox/hq issue 607): a dependent is in a walk
|
||||||
|
// for its base; when the base comes out the same as the build the mesh holds, the dependent stands on that build
|
||||||
|
// still, and leaves the walk with it — and so does a dependent of that dependent. One a merge moves itself stays.
|
||||||
|
func TestADependentOfABaseThatCameOutTheSameIsLeftOut(t *testing.T) {
|
||||||
|
edges := []inventory.Edge{{From: "app", To: "base", Kind: inventory.EdgeDeclared},
|
||||||
|
{From: "app2", To: "app", Kind: inventory.EdgeDeclared}, {From: "own", To: "base", Kind: inventory.EdgeDeclared}}
|
||||||
|
plan := &inventory.Plan{Modules: map[string]*inventory.PlanModule{"base": {}, "app": {}, "app2": {}, "own": {}}}
|
||||||
|
same := withDependentsOf(map[string]string{"base": "unchanged: its program is the same"}, plan,
|
||||||
|
[]string{"base", "own"}, edges)
|
||||||
|
if !strings.Contains(same["app"], "only for base") || !strings.Contains(same["app2"], "only for app") {
|
||||||
|
t.Fatalf("the dependents of a base that came out the same are not left out: %v", same)
|
||||||
|
}
|
||||||
|
if _, left := same["own"]; left {
|
||||||
|
t.Fatalf("a module its merge moves itself is left out for its base: %v", same)
|
||||||
|
}
|
||||||
|
// A dependent of a base that still moves stays.
|
||||||
|
edges = append(edges, inventory.Edge{From: "app", To: "own", Kind: inventory.EdgeDeclared})
|
||||||
|
same = withDependentsOf(map[string]string{"base": "unchanged"}, plan, []string{"base", "own"}, edges)
|
||||||
|
if _, left := same["app"]; left {
|
||||||
|
t.Fatalf("a dependent of a base that still moves is left out: %v", same)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Against the store, the path a real merge takes (novox/hq issues 721 and 611): a merge whose check built its one
|
||||||
|
// module the same as the build the mesh holds is cut into a walk done with nothing to walk, whose record names the
|
||||||
|
// module unchanged and why — the walk mesh-delivery reads as delivered, never stranded.
|
||||||
|
func TestAWalkWhoseEveryBuildCameOutTheSameIsDone(t *testing.T) {
|
||||||
|
open := windowed(t)
|
||||||
|
endsItsLine(t)
|
||||||
|
asksRecorded(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-one", Repository: "novox/one", Ref: "main",
|
||||||
|
Module: "one", Commit: "c0", On: "builder", Asked: t0.Add(-time.Hour),
|
||||||
|
Made: []inventory.Artifact{{Name: "one", Kind: "image", Reference: "store/one@sha256:aaaa"}},
|
||||||
|
Sources: []inventory.BuildSource{{Paths: []string{"main.go", "go.mod"}}}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// What runs: the module registered with that build's image (RegisteredHeld reads the module table).
|
||||||
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "one", Version: "1", Resources: []map[string]any{
|
||||||
|
{"id": "app", "type": "container", "name": "one", "image": "store/one@sha256:aaaa"}}},
|
||||||
|
inventory.Source{Repository: "novox/one", Ref: "main", BuiltFrom: "c0", Asked: time.Now()}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
pull := link.PullRef{Owner: "novox", Repo: "one", Number: 7}
|
||||||
|
was := checkedBuildsOf
|
||||||
|
checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) {
|
||||||
|
return []link.CheckedBuild{{Pull: pull, Head: "headaaaaaaaa", Built: link.CheckBuilt{Module: "one", Key: "k",
|
||||||
|
Made: []link.MadeArtifact{{Name: "one", Kind: "image", Reference: "scratch/one@sha256:bbbb"}},
|
||||||
|
Same: "unchanged: its program is the same as running build one aaaa",
|
||||||
|
SameAs: map[string]string{"one": "10.0.0.1:5000/one@sha256:aaaa"}}}}, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { checkedBuildsOf = was })
|
||||||
|
code := repoMerge("one", "one1aaaaaaaa", t0)
|
||||||
|
code.Number, code.HeadSHA = 7, "headaaaaaaaa1111"
|
||||||
|
hear(t, open, code, t0.Add(time.Second))
|
||||||
|
cutAt(t, open, t0.Add(2*time.Minute))
|
||||||
|
ws, _ := walks(t, open)
|
||||||
|
if len(ws) != 1 {
|
||||||
|
t.Fatalf("walks: %+v", ws)
|
||||||
|
}
|
||||||
|
w := ws[0]
|
||||||
|
if w.State != inventory.PlanDone || len(w.Tiers) != 0 || len(w.Modules) != 0 {
|
||||||
|
t.Fatalf("the walk reads %s, tiers %v, modules %v; want done with nothing to walk", w.State, w.Tiers, w.Modules)
|
||||||
|
}
|
||||||
|
if w.Delivery == nil || !strings.Contains(w.Delivery.Unchanged["one"], "its program is the same as running build") {
|
||||||
|
t.Fatalf("the walk does not say one matched what runs: %+v", w.Delivery)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
@@ -451,6 +452,33 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
case "queue":
|
case "queue":
|
||||||
// As data, the table its output (novox/hq issue 455).
|
// As data, the table its output (novox/hq issue 455).
|
||||||
return []string{"queue", "--json"}, nil
|
return []string{"queue", "--json"}, nil
|
||||||
|
case "forget-checks":
|
||||||
|
if err := need("owner", "repo", "number"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return []string{"forget-checks", str("owner"), str("repo"), str("number")}, nil
|
||||||
|
case "try":
|
||||||
|
// A branch tried before its pull request (novox/hq issue 686): asked, or read by the id it answered.
|
||||||
|
if id := str("id"); id != "" {
|
||||||
|
return []string{"try", "--id", id}, nil
|
||||||
|
}
|
||||||
|
if a.given["repository"] == "" && a.given["ref"] == "" {
|
||||||
|
// Neither shape named: the command says its usage, which names both, as `build` does.
|
||||||
|
return []string{"try"}, nil
|
||||||
|
}
|
||||||
|
if err := need("repository", "ref"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
argv := []string{"try", str("repository"), str("ref")}
|
||||||
|
for _, flag := range []string{"base", "tests"} {
|
||||||
|
if v := str(flag); v != "" {
|
||||||
|
argv = append(argv, "--"+flag, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if on("store-only") {
|
||||||
|
argv = append(argv, "--store-only")
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
case "cancel", "kill":
|
case "cancel", "kill":
|
||||||
if err := need("id"); err != nil {
|
if err := need("id"); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -519,9 +547,13 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
}
|
}
|
||||||
return []string{"unpin", str("node"), str("provision")}, nil
|
return []string{"unpin", str("node"), str("provision")}, nil
|
||||||
case "push":
|
case "push":
|
||||||
// Sent and not waited for: the asker reads `status` for what the machine did, which is
|
// A push to a named machine waits for that machine's own answer, so the call's record says
|
||||||
// what a person at a shell does too. A tool call that blocked for a push's whole apply would
|
// how it went: applied, refused with the machine's reason, or which resources failed and why
|
||||||
// time out on every machine that takes a minute, and say nothing about the ones that did not.
|
// (novox/hq issue 629). How long is the call's `wait`, else the controller's setting `push-wait`;
|
||||||
|
// neither given waits for nothing, as before. The caller is answered before the push starts
|
||||||
|
// (answersFirst) and the machines are released once sent, so the wait holds nobody; it is what
|
||||||
|
// `calls` then shows. A sweep (--behind) does not wait: the slowest machine would hold back
|
||||||
|
// what is said of the others.
|
||||||
// A push through the seat is a push by hand, and says why (novox/hq to-be 45 §7).
|
// A push through the seat is a push by hand, and says why (novox/hq to-be 45 §7).
|
||||||
if err := need("why"); err != nil {
|
if err := need("why"); err != nil {
|
||||||
return nil, fmt.Errorf("%w: a push by hand is a repair, recorded in the hand-act log with why", err)
|
return nil, fmt.Errorf("%w: a push by hand is a repair, recorded in the hand-act log with why", err)
|
||||||
@@ -534,7 +566,11 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
// behind is not read here: given with a machine, it is refused as passed over — naming
|
// behind is not read here: given with a machine, it is refused as passed over — naming
|
||||||
// a machine and asking for every machine behind are two requests, and guessing one
|
// a machine and asking for every machine behind are two requests, and guessing one
|
||||||
// would push a machine nobody named, or not push one somebody did.
|
// would push a machine nobody named, or not push one somebody did.
|
||||||
argv := []string{"push", n, "--wait", "0"}
|
wait, err := pushWait(str("wait"))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
argv := []string{"push", n, "--wait", wait.String()}
|
||||||
// A running module's data moving is sent only when said, per module (novox/hq ADR 0217).
|
// A running module's data moving is sent only when said, per module (novox/hq ADR 0217).
|
||||||
if m := str("move"); m != "" {
|
if m := str("move"); m != "" {
|
||||||
argv = append(argv, "--move", m)
|
argv = append(argv, "--move", m)
|
||||||
@@ -1004,6 +1040,8 @@ var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "co
|
|||||||
"delivery": true,
|
"delivery": true,
|
||||||
// The build queue, each ask with its position among the waiting (novox/hq issue 455).
|
// The build queue, each ask with its position among the waiting (novox/hq issue 455).
|
||||||
"queue": true,
|
"queue": true,
|
||||||
|
// A branch tried before its pull request (novox/hq issue 686).
|
||||||
|
"try": true,
|
||||||
// A plan followed (novox/hq issue 229): `plans wait` answers JSON; its other forms say text, and so answer no data.
|
// A plan followed (novox/hq issue 229): `plans wait` answers JSON; its other forms say text, and so answer no data.
|
||||||
"plans": true}
|
"plans": true}
|
||||||
|
|
||||||
@@ -1294,6 +1332,10 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
|||||||
link.AnswersWithin(ctx, bound+planWaitStart)
|
link.AnswersWithin(ctx, bound+planWaitStart)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if verb == "try" && len(argv) > 1 && argv[1] == "--id" {
|
||||||
|
// Reading a try holds its answer for the verdict, up to its bound (novox/hq issue 686, 229).
|
||||||
|
link.AnswersWithin(ctx, tryWait+planWaitStart)
|
||||||
|
}
|
||||||
if answer, read := readHere(ctx, argv); read {
|
if answer, read := readHere(ctx, argv); read {
|
||||||
return answer, nil
|
return answer, nil
|
||||||
}
|
}
|
||||||
@@ -1328,6 +1370,47 @@ var inProcess = map[string]bool{"tools": true, "calls": true, "doctor": true,
|
|||||||
// 0259 §8): the router asks it before an approval.
|
// 0259 §8): the router asks it before an approval.
|
||||||
"root-free": true}
|
"root-free": true}
|
||||||
|
|
||||||
|
// pushFileVar names the file the controller's push settings are written to (module.json, `push-wait`).
|
||||||
|
const pushFileVar = "MESH_PUSH_FILE"
|
||||||
|
|
||||||
|
// pushWait is how long a push to a named machine waits for its answer (novox/hq issue 629): the call's own
|
||||||
|
// `wait` when given, else the controller's setting `push-wait`, else nothing. No length is written here: the
|
||||||
|
// setting's default is the module's, and a setting that cannot be read waits for nothing, as a push did
|
||||||
|
// before, and is said.
|
||||||
|
func pushWait(given string) (time.Duration, error) {
|
||||||
|
if strings.TrimSpace(given) != "" {
|
||||||
|
d, ok := durationSetting(given)
|
||||||
|
if !ok {
|
||||||
|
return 0, fmt.Errorf("push: wait %q is no duration: say it as \"90s\", \"2m\", or \"0\" to wait for nothing", given)
|
||||||
|
}
|
||||||
|
return d, nil
|
||||||
|
}
|
||||||
|
path := os.Getenv(pushFileVar)
|
||||||
|
if path == "" {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "the push settings cannot be read (%v); a push waits for nothing\n", err)
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
var values map[string]any
|
||||||
|
if err := json.Unmarshal(raw, &values); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "the push settings are not JSON (%v); a push waits for nothing\n", err)
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
v, has := values["push-wait"]
|
||||||
|
if !has || v == "" {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
d, ok := durationSetting(v)
|
||||||
|
if !ok {
|
||||||
|
fmt.Fprintf(os.Stderr, "the setting push-wait=%v is no duration; a push waits for nothing\n", v)
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
return d, nil
|
||||||
|
}
|
||||||
|
|
||||||
// answersFirst is a command line whose caller is answered before it runs: a push, by its verb or
|
// answersFirst is a command line whose caller is answered before it runs: a push, by its verb or
|
||||||
// through `command`. A push sends the machine holding the bus first when its user list changed, the
|
// through `command`. A push sends the machine holding the bus first when its user list changed, the
|
||||||
// broker reloads, and a reload forgets every answer the bus was about to permit — so an answer
|
// broker reloads, and a reload forgets every answer the bus was about to permit — so an answer
|
||||||
|
|||||||
@@ -146,7 +146,7 @@ func TestAnArgumentAVerbDoesNotDeclareIsRefused(t *testing.T) {
|
|||||||
// naming one beside behind is refused rather than one of the two guessed.
|
// naming one beside behind is refused rather than one of the two guessed.
|
||||||
func TestAPushIsOneMachineOrSaysItIsTheWholeMesh(t *testing.T) {
|
func TestAPushIsOneMachineOrSaysItIsTheWholeMesh(t *testing.T) {
|
||||||
argv, err := argvFor("push", map[string]any{"node": "g1", "why": "w"})
|
argv, err := argvFor("push", map[string]any{"node": "g1", "why": "w"})
|
||||||
if err != nil || strings.Join(argv, " ") != "push g1 --wait 0 --why w" {
|
if err != nil || strings.Join(argv, " ") != "push g1 --wait 0s --why w" {
|
||||||
t.Fatalf("a named push: %v %v", argv, err)
|
t.Fatalf("a named push: %v %v", argv, err)
|
||||||
}
|
}
|
||||||
for _, args := range []map[string]any{{"why": "w"}, {"behind": "true", "why": "w"}} {
|
for _, args := range []map[string]any{{"why": "w"}, {"behind": "true", "why": "w"}} {
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -164,12 +166,14 @@ func TestAVerbMissingWhatItNeedsIsRefused(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A push and a build are sent, not waited for: the asker reads status, or the build's log by its
|
// Acts do not block the call. A build is sent, not waited for: the asker reads the build's log by its id.
|
||||||
// id, for what happened. A repository given as a forge path is said to be one (issue 176).
|
// A push answers its caller before it starts (answersFirst), so a named push waits for the machine's own
|
||||||
|
// answer without holding anybody, and the call's record says how it went (novox/hq issue 629). A
|
||||||
|
// repository given as a forge path is said to be one (issue 176).
|
||||||
func TestActsDoNotBlockTheCall(t *testing.T) {
|
func TestActsDoNotBlockTheCall(t *testing.T) {
|
||||||
argv, _ := argvFor("push", map[string]any{"node": "one", "why": "w"})
|
argv, _ := argvFor("push", map[string]any{"node": "one", "why": "w"})
|
||||||
if strings.Join(argv, " ") != "push one --wait 0 --why w" {
|
if strings.Join(argv, " ") != "push one --wait 0s --why w" || !answersFirst(argv) {
|
||||||
t.Fatalf("push waits: %v", argv)
|
t.Fatalf("a named push with no setting waits, or holds its caller: %v", argv)
|
||||||
}
|
}
|
||||||
argv, _ = argvFor("build", map[string]any{"repository": "novox/x", "path": "modules/x"})
|
argv, _ = argvFor("build", map[string]any{"repository": "novox/x", "path": "modules/x"})
|
||||||
if strings.Join(argv, " ") != "build novox/x --wait 0 --self --path modules/x" {
|
if strings.Join(argv, " ") != "build novox/x --wait 0 --self --path modules/x" {
|
||||||
@@ -460,3 +464,40 @@ func TestAQuotedValueRoundTrips(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A named push waits as long as the call says, else as long as the controller's setting push-wait says, else
|
||||||
|
// not at all: no length is the code's own (novox/hq issue 629).
|
||||||
|
func TestAPushWaitsAsTheCallOrTheSettingSays(t *testing.T) {
|
||||||
|
file := filepath.Join(t.TempDir(), "push.json")
|
||||||
|
if err := os.WriteFile(file, []byte(`{"push-wait": "2m"}`), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Setenv(pushFileVar, file)
|
||||||
|
argv, err := argvFor("push", map[string]any{"node": "one", "why": "w"})
|
||||||
|
if err != nil || strings.Join(argv, " ") != "push one --wait 2m0s --why w" {
|
||||||
|
t.Fatalf("the setting was not the default: %v %v", argv, err)
|
||||||
|
}
|
||||||
|
argv, err = argvFor("push", map[string]any{"node": "one", "why": "w", "wait": "30s"})
|
||||||
|
if err != nil || strings.Join(argv, " ") != "push one --wait 30s --why w" {
|
||||||
|
t.Fatalf("the call's wait did not win over the setting: %v %v", argv, err)
|
||||||
|
}
|
||||||
|
argv, err = argvFor("push", map[string]any{"node": "one", "why": "w", "wait": "0"})
|
||||||
|
if err != nil || strings.Join(argv, " ") != "push one --wait 0s --why w" {
|
||||||
|
t.Fatalf("a wait of 0 did not wait for nothing: %v %v", argv, err)
|
||||||
|
}
|
||||||
|
if _, err := argvFor("push", map[string]any{"node": "one", "why": "w", "wait": "soon"}); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "no duration") {
|
||||||
|
t.Fatalf("a wait that is no duration was taken: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(file, []byte(`{"push-wait": "later"}`), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
argv, err = argvFor("push", map[string]any{"node": "one", "why": "w"})
|
||||||
|
if err != nil || strings.Join(argv, " ") != "push one --wait 0s --why w" {
|
||||||
|
t.Fatalf("an unreadable setting did not fall back to waiting for nothing: %v %v", argv, err)
|
||||||
|
}
|
||||||
|
argv, _ = argvFor("push", map[string]any{"why": "w"})
|
||||||
|
if strings.Join(argv, " ") != "push --behind --wait 0 --why w" {
|
||||||
|
t.Fatalf("a sweep waits: %v", argv)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -79,6 +79,37 @@ func secretCommand(ctx context.Context, args []string) error {
|
|||||||
return fmt.Errorf("a secret's value is given at the controller's terminal or at the desk (`give`), never "+
|
return fmt.Errorf("a secret's value is given at the controller's terminal or at the desk (`give`), never "+
|
||||||
"through a verb (this line came through %q): nothing was read or sealed", verb)
|
"through a verb (this line came through %q): nothing was read or sealed", verb)
|
||||||
}
|
}
|
||||||
|
// **A provision's credential issued outside the mesh is given once, for its provider** (novox/hq ADR 0337 §3,
|
||||||
|
// issue 165): `secret accept <provider's machine> <provider module> <provision>`, sealed to every consumer of
|
||||||
|
// that provider; the per-pair form naming a consumer is refused for it, naming this one.
|
||||||
|
outsideAt, err := outsideOffer(ctx, module, name, *provider)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if outsideAt == "" && *provider != "" {
|
||||||
|
if offered, err := offeredOutsideBy(ctx, name); err != nil {
|
||||||
|
return err
|
||||||
|
} else if offered != "" {
|
||||||
|
return fmt.Errorf("%s is issued outside the mesh and given once for its provider, which seals it to "+
|
||||||
|
"every consumer of it (ADR 0337): `secret accept %s %s %s`; nothing was read or sealed",
|
||||||
|
name, *provider, offered, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if outsideAt != "" {
|
||||||
|
if *desk != "" || *local != "" {
|
||||||
|
return errors.New("a provision's credential issued outside the mesh is given at the controller's " +
|
||||||
|
"terminal, for every consumer of its provider; it takes neither --at-desk nor --local")
|
||||||
|
}
|
||||||
|
value, err := valueFor(node, module, name, *from)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
value = asSupplied(value)
|
||||||
|
if value == "" {
|
||||||
|
return errors.New("there is nothing to seal")
|
||||||
|
}
|
||||||
|
return acceptForProvision(ctx, node, module, name, value)
|
||||||
|
}
|
||||||
if *desk != "" {
|
if *desk != "" {
|
||||||
if *from != "" || *provider != "" {
|
if *from != "" || *provider != "" {
|
||||||
return errors.New("--at-desk gives a module's own secret, and takes neither --from nor --provider")
|
return errors.New("--at-desk gives a module's own secret, and takes neither --from nor --provider")
|
||||||
@@ -173,6 +204,7 @@ const secretUsage = "secret rotate <node> <module> <name> [--why <text> [--cause
|
|||||||
"secret ask <node> <module> <name> [--at <machine>]\n" +
|
"secret ask <node> <module> <name> [--at <machine>]\n" +
|
||||||
"secret issued <provision>\n" +
|
"secret issued <provision>\n" +
|
||||||
"secret accept <node> <module> <name> [--from <file> | --at-desk <machine>] [--provider <node> [--local <name>]]\n" +
|
"secret accept <node> <module> <name> [--from <file> | --at-desk <machine>] [--provider <node> [--local <name>]]\n" +
|
||||||
|
"secret accept <provider's node> <provider module> <provision> [--from <file>] (a credential issued outside the mesh, for every consumer)\n" +
|
||||||
"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>] [--provider <node>]\n" +
|
"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>] [--provider <node>]\n" +
|
||||||
"secret export [--out <file>]"
|
"secret export [--out <file>]"
|
||||||
|
|
||||||
|
|||||||
@@ -40,6 +40,9 @@ type sendable struct {
|
|||||||
actingEpoch uint64
|
actingEpoch uint64
|
||||||
// modules are the modules this declaration names, for the record of the send; never on the wire.
|
// modules are the modules this declaration names, for the record of the send; never on the wire.
|
||||||
modules []string
|
modules []string
|
||||||
|
// localNetwork is every port admitted from the local network and what it resolved to (novox/hq ADR
|
||||||
|
// 0340 §8), for plan to say; never on the wire.
|
||||||
|
localNetwork []catalogue.LocalNetworkPort
|
||||||
// Adoption is nil for a converged node, and then the body is byte for byte what it was before
|
// Adoption is nil for a converged node, and then the body is byte for byte what it was before
|
||||||
// adoption existed: an older host parses the envelope strictly and would refuse the key.
|
// adoption existed: an older host parses the envelope strictly and would refuse the key.
|
||||||
Adoption *adoptionEnvelope
|
Adoption *adoptionEnvelope
|
||||||
|
|||||||
@@ -0,0 +1,60 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq ADR 0346, issue 624: a provider on another machine serves its consumers its own composed
|
||||||
|
// names, as that machine composes them; one it does not compose is left out, said under the need.
|
||||||
|
func TestAProviderElsewhereServesItsOwnComposedName(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
forge, err := catalogue.ParseManifest([]byte(`{"module":"forge","version":"1","requires":["route"],
|
||||||
|
"provides":[{"name":"npm-package-registry","scope":"mesh"}],
|
||||||
|
"contributes":{"route":{"web":{"label":"git","port":3000}}},
|
||||||
|
"serves":{"npm-package-registry":{"scheme":"http","port":3000,"npm-path":"/api/packages/novox/npm/",
|
||||||
|
"public-root":"https://${bound:route:name-web}/api/packages/novox/npm/"}}}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
register(t, open, catalogue.Manifest{Module: "proxy", Version: "1", Provides: catalogue.FromAnywhere("route"),
|
||||||
|
Receives: map[string]string{"route": "/var/lib/proxy/routes.json"}})
|
||||||
|
register(t, open, forge)
|
||||||
|
register(t, open, catalogue.Manifest{Module: "lab", Version: "1", Requires: []string{"npm-package-registry"},
|
||||||
|
Binds: map[string]string{"npm-package-registry": "/var/lib/lab/package-registry.json"}})
|
||||||
|
for _, a := range [][2]string{{"anchor", "proxy"}, {"anchor", "forge"}, {"laptop", "lab"}} {
|
||||||
|
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||||
|
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := open.inventory.SetPublicDomain(ctx, "anchor", "example.test"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
plan, _, err := planFor(ctx, open, "laptop")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
n := need(t, plan, "lab", "npm-package-registry")
|
||||||
|
if n.Serves["public-root"] != "https://git.example.test/api/packages/novox/npm/" || len(n.LeftOut) != 0 {
|
||||||
|
t.Fatalf("the consumer elsewhere is served the forge's public root: %v, left out %v", n.Serves, n.LeftOut)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := open.inventory.SetPublicDomain(ctx, "anchor", ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
plan, _, err = planFor(ctx, open, "laptop")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
n = need(t, plan, "lab", "npm-package-registry")
|
||||||
|
if _, there := n.Serves["public-root"]; there {
|
||||||
|
t.Fatalf("anchor composes no public name, so the key is left out: %v", n.Serves)
|
||||||
|
}
|
||||||
|
if len(n.LeftOut) != 1 || !strings.Contains(n.LeftOut[0], `without "public-root"`) {
|
||||||
|
t.Fatalf("and the need says so: %v", n.LeftOut)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,281 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// An approved settings change is sent by a walk at once, and the operator's one Approve is its push (novox/hq ADR
|
||||||
|
// 0344, issue 641).
|
||||||
|
//
|
||||||
|
// On 2026-10-13 the operator approved a betterbird proposal on Telegram and nothing happened: the warrant set the
|
||||||
|
// layer, and the push stayed a separate act (ADR 0277 §18) that nobody took. Now the warrant that sets the layer
|
||||||
|
// opens a **settings walk**: the walk ADR 0236 opens for builds waiting for a gate, with a layer in place of the
|
||||||
|
// builds. Its order is every machine running the module that the layer reaches, the control node last; each is
|
||||||
|
// sent and its module judged there, from the machine's report on that send, before the next is sent. The send is
|
||||||
|
// the layer alone: every module there stays at the build its machine was last sent (keepingBuildsForSettings), so a
|
||||||
|
// build held for a person (ADR 0242) or waiting for a gate stays where it is — a policy of `record` holds a build,
|
||||||
|
// never a setting — and a send that would move a running module's data stops the walk (ADR 0217 §3). The walk says its start and end on `plan-moved` (ADR 0308), and the holder of the operator's
|
||||||
|
// channel tells its end once. A failure stops the walk and leaves the layer as approved; it holds back nothing
|
||||||
|
// else.
|
||||||
|
|
||||||
|
// settingsWalkPrefix names a settings walk's id.
|
||||||
|
const settingsWalkPrefix = "settings-"
|
||||||
|
|
||||||
|
// settingsWalkFor is what a settings walk says it is for, where a merge's walk says its repository.
|
||||||
|
func settingsWalkFor(s inventory.PlanSettings) string {
|
||||||
|
where := s.Node
|
||||||
|
if where == "" {
|
||||||
|
where = "the whole mesh"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("the settings of %s on %s", s.Module, where)
|
||||||
|
}
|
||||||
|
|
||||||
|
// settingsWalkOrder is the machines a layer reaches, in the order a settings walk sends them: every machine
|
||||||
|
// running the module, or the one the layer names when it runs it there, sorted, the control node last. Pure.
|
||||||
|
func settingsWalkOrder(node string, running, controllers []string) []string {
|
||||||
|
var order, last []string
|
||||||
|
for _, n := range running {
|
||||||
|
if node != "" && n != node {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if slices.Contains(controllers, n) {
|
||||||
|
last = append(last, n)
|
||||||
|
} else {
|
||||||
|
order = append(order, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(order)
|
||||||
|
sort.Strings(last)
|
||||||
|
return append(order, last...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// openSettingsWalk opens the walk that sends an approved layer, and answers the words of what it does: the
|
||||||
|
// machines it sends, or that no machine runs the module and nothing is sent. Called on the operator's warrant,
|
||||||
|
// after the layer is set.
|
||||||
|
var openSettingsWalk = func(ctx context.Context, open *stores, s inventory.PlanSettings) (string, error) {
|
||||||
|
inv := open.inventory
|
||||||
|
running, err := inv.Running(ctx, s.Module)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
controllers, err := inv.Running(ctx, catalogue.ControllerSeatName)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
order := settingsWalkOrder(s.Node, running, controllers)
|
||||||
|
if len(order) == 0 {
|
||||||
|
return fmt.Sprintf("no machine runs %s there, so there is nothing to send", s.Module), nil
|
||||||
|
}
|
||||||
|
release, err := inv.HoldPlans(ctx, true)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
now := time.Now().UTC()
|
||||||
|
p := inventory.Plan{ID: fmt.Sprintf("%s%d", settingsWalkPrefix, now.UnixNano()), Repository: settingsWalkFor(s),
|
||||||
|
Created: now, State: inventory.PlanRolling, Tiers: [][]string{{s.Module}}, Modules: map[string]*inventory.PlanModule{},
|
||||||
|
Release: &inventory.PlanRelease{Order: order, Settings: &s},
|
||||||
|
Note: fmt.Sprintf("approved (ask %s): sent to %s, one machine at a time, each judged", s.Ask,
|
||||||
|
strings.Join(order, ", "))}
|
||||||
|
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s: %s — %s\n", p.ID, p.Repository, p.Note)
|
||||||
|
return fmt.Sprintf("%s sends it to %s, one machine at a time, each judged", p.ID, strings.Join(order, ", ")), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// settingsDataMoves is what a settings walk's send to a machine would move of a running module's data: composed as
|
||||||
|
// that send composes it, every build kept, and compared with what the machine was last sent (ADR 0217 §3).
|
||||||
|
var settingsDataMoves = func(ctx context.Context, open *stores, node string) ([]dataMove, error) {
|
||||||
|
keeping := keepingBuildsForSettings(keepingRecorded(ctx), node)
|
||||||
|
gens, err := generators(keeping, open)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
plan, settings, err := planFor(keeping, open, node)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
declared, err := declarationWith(keeping, open, node, plan, settings, gens, Reading)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
body, err := declared.Body()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return heldMoves(keeping, open.inventory, node, body, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgingOn is the open walk, other than the one named, whose gate is judging on a machine; empty when none is. A
|
||||||
|
// settings walk sends a machine only when no other walk judges there: its send would set aside the declaration
|
||||||
|
// that walk's gate reads the machine's report against.
|
||||||
|
var judgingOn = func(ctx context.Context, open *stores, self, node string) (string, error) {
|
||||||
|
plans, err := open.inventory.OpenPlans(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return judgingIn(plans, self, node), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgingIn is judgingOn over plans read. Pure.
|
||||||
|
func judgingIn(plans []inventory.Plan, self, node string) string {
|
||||||
|
judging := func(g *inventory.PlanGate) bool {
|
||||||
|
return g != nil && g.Verdict == "" && slices.Contains(g.Machines, node)
|
||||||
|
}
|
||||||
|
for _, p := range plans {
|
||||||
|
if p.ID == self || !p.Open() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if p.Release != nil {
|
||||||
|
if judging(p.Release.Gate) {
|
||||||
|
return p.ID
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, s := range p.Modules {
|
||||||
|
if s != nil && judging(s.Gate) {
|
||||||
|
return p.ID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// advanceSettingsWalk takes one step of a settings walk: the next machine sent and its module judged there, or the
|
||||||
|
// machine being judged judged once more; done when every machine is.
|
||||||
|
func advanceSettingsWalk(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) {
|
||||||
|
r, s := p.Release, p.Release.Settings
|
||||||
|
now := time.Now().UTC()
|
||||||
|
wait := func(note string) (bool, error) {
|
||||||
|
changed := p.Note != note
|
||||||
|
p.Note = note
|
||||||
|
return changed, nil
|
||||||
|
}
|
||||||
|
if r.Gate == nil {
|
||||||
|
heard, err := releaseHeard(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
for r.Next < len(r.Order) {
|
||||||
|
node := r.Order[r.Next]
|
||||||
|
if !heard[node] {
|
||||||
|
r.Skipped = append(r.Skipped, node)
|
||||||
|
r.Next++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if id, err := judgingOn(ctx, open, p.ID, node); err != nil {
|
||||||
|
return false, err
|
||||||
|
} else if id != "" {
|
||||||
|
return wait(fmt.Sprintf("waiting before %s: %s is judging there", node, id))
|
||||||
|
}
|
||||||
|
// **A running module's data that would move is a person's word** (ADR 0217 §3): the walk stops before
|
||||||
|
// that machine, failed and said, and `push --move` sends it.
|
||||||
|
held, err := settingsDataMoves(ctx, open, node)
|
||||||
|
if errors.Is(err, errSettingsCannotKeep) {
|
||||||
|
return failBefore(p, node, err.Error())
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("whether %s's data would move could not be read: %w", node, err)
|
||||||
|
}
|
||||||
|
if len(held) > 0 {
|
||||||
|
var said strings.Builder
|
||||||
|
sayDataHeld(&said, node, held)
|
||||||
|
return failBefore(p, node, "a running module's data would move there — "+
|
||||||
|
strings.Join(strings.Fields(said.String()), " "))
|
||||||
|
}
|
||||||
|
// A build that cannot be kept is refused by the composition above, which is the send's own, and fails the
|
||||||
|
// walk there: the send flattens its refusals into words, so it is not looked for again here.
|
||||||
|
sent, grants, err := sendJudged(keepingBuildsForSettings(ctx, node), open, node)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("sending %s %s: %w", node, settingsWalkFor(*s), err)
|
||||||
|
}
|
||||||
|
// **The module's build is the one the gate judges** (issue 641 review): kept by this send, it is what a
|
||||||
|
// crossing send — a merge's walk sending the machine a new build of the module — supersedes, so this gate
|
||||||
|
// never reads that send's report as its own.
|
||||||
|
kept := ""
|
||||||
|
if builds, known, err := open.inventory.SentBuilds(ctx, node); err == nil && known {
|
||||||
|
kept = builds[s.Module]
|
||||||
|
}
|
||||||
|
r.Gate = &inventory.PlanGate{Component: coreComponent(s.Module), Machines: sent, Since: &now,
|
||||||
|
From: kept, To: kept, Sent: sentNow(ctx, open.inventory, sent), Grants: grants}
|
||||||
|
p.Note = fmt.Sprintf("sent %s %s (ask %s), every build there kept; judging %s there", node,
|
||||||
|
settingsWalkFor(*s), s.Ask, s.Module)
|
||||||
|
if said := grantsSaid(grants); said != "" {
|
||||||
|
p.Note += "; " + said
|
||||||
|
}
|
||||||
|
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
p.State, p.Tier = inventory.PlanDone, len(p.Tiers)
|
||||||
|
p.Note = fmt.Sprintf("%s (ask %s) delivered on %s", settingsWalkFor(*s), s.Ask, strings.Join(r.Done, ", "))
|
||||||
|
if len(r.Done) == 0 {
|
||||||
|
// Sent nowhere is not delivered: said as the walk's failure, so the operator is not told it landed.
|
||||||
|
p.State = inventory.PlanFailed
|
||||||
|
p.Note = fmt.Sprintf("%s (ask %s) was sent to no machine", settingsWalkFor(*s), s.Ask)
|
||||||
|
}
|
||||||
|
if len(r.Skipped) > 0 {
|
||||||
|
p.Note += "; not heard from, left as they were, their next send carries it: " + strings.Join(r.Skipped, ", ")
|
||||||
|
}
|
||||||
|
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
g := r.Gate
|
||||||
|
node := r.Order[r.Next]
|
||||||
|
verdict, err := judgeMoves(ctx, open, g, []judged{{module: s.Module, node: node}}, now)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
switch verdict {
|
||||||
|
case "":
|
||||||
|
return wait(fmt.Sprintf("judging %s on %s: %s", s.Module, node, gateLine(g)))
|
||||||
|
case inventory.GatePassed:
|
||||||
|
r.Done = append(r.Done, node)
|
||||||
|
r.Next++
|
||||||
|
r.Gate = nil
|
||||||
|
return true, nil
|
||||||
|
case inventory.GateSuperseded:
|
||||||
|
p.State = inventory.PlanSuperseded
|
||||||
|
p.Note = fmt.Sprintf("superseded on %s: %s — nothing judged, nothing put back; the layer is set, and the "+
|
||||||
|
"machines not yet sent take it at their next send", node, g.Why)
|
||||||
|
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
p.State = inventory.PlanFailed
|
||||||
|
p.Note = fmt.Sprintf("%s failed its gate on %s: %s — the layer stays as the operator approved it, and the one it "+
|
||||||
|
"replaced is in `settings %s` history", settingsWalkFor(*s), node, g.Why, s.Module) + unreached(r)
|
||||||
|
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// failBefore ends a settings walk failed before it sent a machine, saying why, and that the layer stays.
|
||||||
|
func failBefore(p *inventory.Plan, node, why string) (bool, error) {
|
||||||
|
r, s := p.Release, p.Release.Settings
|
||||||
|
p.State = inventory.PlanFailed
|
||||||
|
p.Note = fmt.Sprintf("%s not sent to %s: %s; the layer stays as the operator approved it", settingsWalkFor(*s),
|
||||||
|
node, why) + unreached(r)
|
||||||
|
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// unreached is what a stopped settings walk left without the layer, in words: the machines after the one it stopped
|
||||||
|
// at, and those passed over as away (ADR 0344 §6). Empty when there are none.
|
||||||
|
func unreached(r *inventory.PlanRelease) string {
|
||||||
|
var out string
|
||||||
|
if r.Next+1 < len(r.Order) {
|
||||||
|
out += "; not sent: " + strings.Join(r.Order[r.Next+1:], ", ")
|
||||||
|
}
|
||||||
|
if len(r.Skipped) > 0 {
|
||||||
|
out += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ")
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,572 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/lease"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq ADR 0344, issue 641: the operator's one Approve on a settings proposal is the act that makes it take
|
||||||
|
// effect. The warrant sets the layer and opens a settings walk, which sends it to the machines it reaches one at a
|
||||||
|
// time, judging the module on each; the ask says so before the Approve, and nothing names a push.
|
||||||
|
|
||||||
|
// The order: every machine running the module, or the one the layer names, the control node last.
|
||||||
|
func TestASettingsWalkGoesToTheMachinesTheLayerReachesTheControlNodeLast(t *testing.T) {
|
||||||
|
running := []string{"shanks", "novox", "g14", "ace"}
|
||||||
|
for _, c := range []struct {
|
||||||
|
node string
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{"", []string{"ace", "g14", "shanks", "novox"}},
|
||||||
|
{"g14", []string{"g14"}},
|
||||||
|
{"novox", []string{"novox"}},
|
||||||
|
{"elsewhere", nil},
|
||||||
|
} {
|
||||||
|
if got := settingsWalkOrder(c.node, running, []string{"novox"}); !reflect.DeepEqual(got, c.want) {
|
||||||
|
t.Errorf("a layer on %q goes to %v, want %v", c.node, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A settings walk sends a machine only when no other walk judges there.
|
||||||
|
func TestASettingsWalkWaitsWhileAnotherWalkJudgesOnTheMachine(t *testing.T) {
|
||||||
|
judging := &inventory.PlanGate{Machines: []string{"g14"}}
|
||||||
|
passed := &inventory.PlanGate{Machines: []string{"g14"}, Verdict: inventory.GatePassed}
|
||||||
|
plans := []inventory.Plan{
|
||||||
|
{ID: "settings-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: judging}},
|
||||||
|
{ID: "release-2", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: passed}},
|
||||||
|
{ID: "plan-3", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{"app": {Gate: judging}}},
|
||||||
|
{ID: "plan-4", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{"app": {Gate: judging}}},
|
||||||
|
}
|
||||||
|
if id := judgingIn(plans, "settings-1", "g14"); id != "plan-3" {
|
||||||
|
t.Errorf("on g14 %q judges, want plan-3 (itself, a passed gate and an ended walk are not judging)", id)
|
||||||
|
}
|
||||||
|
if id := judgingIn(plans[1:2], "x", "g14"); id != "" {
|
||||||
|
t.Errorf("a passed gate judges: %q", id)
|
||||||
|
}
|
||||||
|
if id := judgingIn(plans, "x", "shanks"); id != "" {
|
||||||
|
t.Errorf("on shanks %q judges", id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The ask says, before the Approve, that the layer is sent at once and judged, and that its end is told as every
|
||||||
|
// walk's is; neither the ask nor the proposer's answer names a push any more.
|
||||||
|
func TestTheAskSaysAnApprovedLayerIsSentAtOnce(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
in proposeInput
|
||||||
|
where string
|
||||||
|
had bool
|
||||||
|
}{
|
||||||
|
{proposeInput{module: "mounts", node: "shanks", values: mountsSources}, "shanks", false},
|
||||||
|
{proposeInput{module: "notes", values: map[string]any{"x": "1"}}, "every machine running notes", true},
|
||||||
|
{proposeInput{module: "notes", node: "laptop", clear: true}, "laptop", true},
|
||||||
|
} {
|
||||||
|
r := newProposerRig(t)
|
||||||
|
if c.had {
|
||||||
|
r.before, r.had = map[string]any{"x": "0"}, true
|
||||||
|
}
|
||||||
|
words, err := r.pr.propose(context.Background(), c.in)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
q := r.askSent(t)
|
||||||
|
if err := q.Check(r.now); err != nil {
|
||||||
|
t.Fatalf("the ask is refused: %v", err)
|
||||||
|
}
|
||||||
|
// At a glance (ADR 0345): the summary says it is sent at once by a walk, one machine at a time; the rest of
|
||||||
|
// what Approve does is the Details', which the router notes after the answers.
|
||||||
|
if !strings.Contains(q.Explanation, "sent at once by a walk to "+c.where+", one machine at a time, each judged") ||
|
||||||
|
!strings.Contains(q.Details, "and sent at once to "+c.where+", one machine at a time, each judged, "+
|
||||||
|
"every build there kept; its end is told as every walk's is") {
|
||||||
|
t.Errorf("the ask does not say the layer is sent at once to %s:\n%s", c.where, q.Explanation)
|
||||||
|
}
|
||||||
|
approve := q.Options[0]
|
||||||
|
if approve.Label != "Approve" || !strings.Contains(approve.Does, "sent to "+c.where+" at once") {
|
||||||
|
t.Errorf("Approve does %q", approve.Does)
|
||||||
|
}
|
||||||
|
if !strings.Contains(words, "a walk sends it to "+c.where+" at once") {
|
||||||
|
t.Errorf("the proposer's answer:\n%s", words)
|
||||||
|
}
|
||||||
|
for _, said := range []string{q.Explanation, q.Whole, approve.Does, words} {
|
||||||
|
if strings.Contains(said, "push") {
|
||||||
|
t.Errorf("a push is still named, as if a second act were needed:\n%s", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- on the real stores -----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// The warrant sets the layer and opens the walk that sends it, naming the machines it reaches; a layer for a
|
||||||
|
// machine that does not run the module opens none and says so.
|
||||||
|
func TestAnApprovedLayerOpensTheWalkThatSendsIt(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, catalogue.Manifest{Module: "notes", Version: "1",
|
||||||
|
Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644",
|
||||||
|
"content": "x = ${setting:x}\n"}}})
|
||||||
|
if _, err := assign(ctx, open, "laptop", "notes"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// A layer set by hand opens no walk: it still waits for a push (ADR 0344 §7).
|
||||||
|
if err := atTheTerminal(t, "settings", "set", "notes", `{"x":"0"}`, "--node", "laptop"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if plans, _ := open.inventory.OpenPlans(ctx); len(plans) != 0 {
|
||||||
|
t.Fatalf("a layer set at the terminal opened %d walk(s)", len(plans))
|
||||||
|
}
|
||||||
|
set := setLayerIn(open)
|
||||||
|
now := time.Now()
|
||||||
|
values := map[string]any{"x": "1"}
|
||||||
|
p := settingsProposal{Module: "notes", Values: values, From: "g14/claude-code", At: now,
|
||||||
|
Digest: layerDigest(values), BeforeDigest: layerDigest(nil)}
|
||||||
|
words, err := set(ctx, p, "s9", "approved by the operator (ask s9)")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(words, "+ x (ask s9); settings-") || !strings.Contains(words, "sends it to laptop, one machine at a time") {
|
||||||
|
t.Errorf("the warrant's words: %q", words)
|
||||||
|
}
|
||||||
|
plans, err := open.inventory.OpenPlans(ctx)
|
||||||
|
if err != nil || len(plans) != 1 {
|
||||||
|
t.Fatalf("open walks: %+v %v", plans, err)
|
||||||
|
}
|
||||||
|
w := plans[0]
|
||||||
|
if w.Release == nil || w.Release.Settings == nil || w.Release.Settings.Module != "notes" || w.Release.Settings.Ask != "s9" ||
|
||||||
|
!reflect.DeepEqual(w.Release.Order, []string{"laptop"}) || w.Repository != "the settings of notes on the whole mesh" ||
|
||||||
|
!reflect.DeepEqual(w.Tiers, [][]string{{"notes"}}) {
|
||||||
|
t.Fatalf("the settings walk: %+v %+v", w, w.Release)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine's layer where the module does not run: set, and nothing to send.
|
||||||
|
q := settingsProposal{Module: "notes", Node: "anchor", Values: values, From: "g14/claude-code", At: now,
|
||||||
|
Digest: layerDigest(values), BeforeDigest: layerDigest(nil)}
|
||||||
|
words, err = set(ctx, q, "s10", "approved by the operator (ask s10)")
|
||||||
|
if err != nil || !strings.Contains(words, "no machine runs notes there, so there is nothing to send") {
|
||||||
|
t.Fatalf("a layer nobody runs: %q %v", words, err)
|
||||||
|
}
|
||||||
|
if plans, _ := open.inventory.OpenPlans(ctx); len(plans) != 1 {
|
||||||
|
t.Fatalf("a walk was opened for a machine that does not run the module: %d open", len(plans))
|
||||||
|
}
|
||||||
|
|
||||||
|
// A walk that cannot be opened is the act's failure, said with the push that sends it by hand.
|
||||||
|
was := openSettingsWalk
|
||||||
|
openSettingsWalk = func(context.Context, *stores, inventory.PlanSettings) (string, error) {
|
||||||
|
return "", fmt.Errorf("the plans are not to be had")
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { openSettingsWalk = was })
|
||||||
|
again := map[string]any{"x": "2"}
|
||||||
|
byHand := map[string]any{"x": "0"}
|
||||||
|
r := settingsProposal{Module: "notes", Node: "laptop", Values: again, Before: byHand, HadLayer: true,
|
||||||
|
From: "g14/claude-code", At: now, Digest: layerDigest(again), BeforeDigest: layerDigest(byHand)}
|
||||||
|
if _, err := set(ctx, r, "s11", "approved"); err == nil || !strings.Contains(err.Error(), "is set, and the walk that "+
|
||||||
|
"sends it could not be opened: the plans are not to be had — `push laptop` sends it") {
|
||||||
|
t.Errorf("an unopened walk: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// walkMesh is a mesh where `app` runs c1 on anchor and laptop and nothing waits for a gate: every send applied at
|
||||||
|
// once, recorded, and judged from the reports, with a machine named broken failing what it was sent.
|
||||||
|
type walkMesh struct {
|
||||||
|
open *stores
|
||||||
|
sent [][]string
|
||||||
|
broken map[string]bool
|
||||||
|
// kept says, per send, whether it kept every build of its machine; moves is the data a machine's send would move.
|
||||||
|
kept []bool
|
||||||
|
moves map[string][]dataMove
|
||||||
|
heard map[string]bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func aSettledMesh(t *testing.T) *walkMesh {
|
||||||
|
t.Helper()
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
w := &walkMesh{open: open, broken: map[string]bool{}, moves: map[string][]dataMove{},
|
||||||
|
heard: map[string]bool{"anchor": true, "laptop": true}}
|
||||||
|
withConditionsInMemory(t)
|
||||||
|
was := doctorFrom
|
||||||
|
doctorFrom = nil
|
||||||
|
t.Cleanup(func() { doctorFrom = was })
|
||||||
|
old := time.Now().Add(-2 * time.Hour)
|
||||||
|
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
||||||
|
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c1", Module: "app", Commit: "c1",
|
||||||
|
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: old, At: old,
|
||||||
|
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-1"}}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
|
||||||
|
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c1", Head: "c1", Asked: old}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, n := range []string{"anchor", "laptop"} {
|
||||||
|
if _, err := inv.Assign(ctx, n, "app"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, map[string]string{"app": "c1"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
wasMoves, wasHeard, wasSend, wasGather, wasData := machineMoves, releaseHeard, sendRollout, gatherGateFacts, settingsDataMoves
|
||||||
|
settingsDataMoves = func(_ context.Context, _ *stores, node string) ([]dataMove, error) { return w.moves[node], nil }
|
||||||
|
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
|
||||||
|
modules, err := open.inventory.Assigned(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
sent, known, err := open.inventory.SentBuilds(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return f.moves(node, modules, sent, known, all), nil
|
||||||
|
}
|
||||||
|
releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return w.heard, nil }
|
||||||
|
n := 0
|
||||||
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||||
|
w.sent = append(w.sent, append([]string(nil), names...))
|
||||||
|
w.kept = append(w.kept, len(names) == 1 && settingsKeptOn(ctx, names[0]))
|
||||||
|
current, err := open.inventory.CurrentBuilds(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, node := range names {
|
||||||
|
modules, _ := open.inventory.Assigned(ctx, node)
|
||||||
|
carried := map[string]string{}
|
||||||
|
for _, m := range modules {
|
||||||
|
carried[m] = current[m].Commit
|
||||||
|
}
|
||||||
|
n++
|
||||||
|
digest := fmt.Sprintf("d-%s-%d", node, n)
|
||||||
|
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
||||||
|
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return names, nil
|
||||||
|
}
|
||||||
|
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||||
|
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
||||||
|
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
|
||||||
|
reports, err := open.inventory.LastReports(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return f, err
|
||||||
|
}
|
||||||
|
for _, r := range reports {
|
||||||
|
if w.broken[r.Node] {
|
||||||
|
r.Outcome = inventory.OutcomeFailed
|
||||||
|
}
|
||||||
|
f.reports[r.Node] = r
|
||||||
|
}
|
||||||
|
return f, nil
|
||||||
|
}
|
||||||
|
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||||
|
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||||
|
wasWindow := gateWitnessWindow
|
||||||
|
gateWitnessWindow = 0
|
||||||
|
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||||
|
gateWithin, gateEvery = time.Hour, time.Hour
|
||||||
|
t.Cleanup(func() {
|
||||||
|
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
|
||||||
|
settingsDataMoves = wasData
|
||||||
|
gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound
|
||||||
|
})
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *walkMesh) walk(t *testing.T) inventory.Plan {
|
||||||
|
t.Helper()
|
||||||
|
plans, err := w.open.inventory.RecentPlans(t.Context(), 10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, p := range plans {
|
||||||
|
if p.Release != nil && p.Release.Settings != nil {
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatal("no settings walk")
|
||||||
|
return inventory.Plan{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The walk sends the first machine alone and judges the module there before it sends the next, and ends
|
||||||
|
// delivered on both; nothing waited for a gate, so each send carried the layer alone.
|
||||||
|
func TestASettingsWalkSendsOneMachineAtATimeEachJudged(t *testing.T) {
|
||||||
|
judgeLeavesAsTheCore(t)
|
||||||
|
w := aSettledMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
words, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s1", Change: "~ x"})
|
||||||
|
if err != nil || !strings.Contains(words, "sends it to anchor, laptop") {
|
||||||
|
t.Fatalf("%q %v", words, err)
|
||||||
|
}
|
||||||
|
advancePlans(ctx, w.open)
|
||||||
|
if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) {
|
||||||
|
t.Fatalf("sent %v: the first machine alone, before it is judged", w.sent)
|
||||||
|
}
|
||||||
|
p := w.walk(t)
|
||||||
|
if p.Release.Gate == nil || p.Release.Next != 0 || !strings.Contains(p.Note, "judging app on anchor") {
|
||||||
|
t.Fatalf("the walk after its first send: %s %+v", p.Note, p.Release)
|
||||||
|
}
|
||||||
|
gateEvery = 0
|
||||||
|
for i := 0; i < 6; i++ {
|
||||||
|
advancePlans(ctx, w.open)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}, {"laptop"}}) {
|
||||||
|
t.Fatalf("sent %v", w.sent)
|
||||||
|
}
|
||||||
|
p = w.walk(t)
|
||||||
|
if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, []string{"anchor", "laptop"}) ||
|
||||||
|
p.Outcome() != inventory.WalkDelivered || !strings.Contains(p.Note, "delivered on anchor, laptop") {
|
||||||
|
t.Fatalf("the walk is %s: %s %+v", p.State, p.Note, p.Release)
|
||||||
|
}
|
||||||
|
// It said its start and its end, so the operator is told once (ADR 0308).
|
||||||
|
if p.Times == nil || p.Times.Started == nil || p.Times.Ended == nil {
|
||||||
|
t.Fatalf("the walk's start and end are not marked: %+v", p.Times)
|
||||||
|
}
|
||||||
|
// Nothing waits, and no walk of waiting builds opened beside it.
|
||||||
|
if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r != nil {
|
||||||
|
t.Fatalf("%+v %v", r, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine that fails what it was sent stops the walk there: the next is not sent, the layer stays, and the
|
||||||
|
// failure holds back no walk of waiting builds.
|
||||||
|
func TestAFailedSettingsWalkStopsAndHoldsNothingBack(t *testing.T) {
|
||||||
|
w := aSettledMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := w.open.inventory
|
||||||
|
w.broken["anchor"] = true
|
||||||
|
if err := inv.KeepSettings(ctx, "", "app", map[string]any{"x": "1"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s2"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for i := 0; i < 4; i++ {
|
||||||
|
advancePlans(ctx, w.open)
|
||||||
|
}
|
||||||
|
p := w.walk(t)
|
||||||
|
if p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed ||
|
||||||
|
!strings.Contains(p.Note, "the settings of app on the whole mesh failed its gate on anchor") ||
|
||||||
|
!strings.Contains(p.Note, "the layer stays as the operator approved it") || !strings.Contains(p.Note, "not sent: laptop") {
|
||||||
|
t.Fatalf("the walk is %s: %s", p.State, p.Note)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) {
|
||||||
|
t.Fatalf("sent %v: anchor alone, nothing to laptop", w.sent)
|
||||||
|
}
|
||||||
|
if layer, has, err := inv.Layer(ctx, "", "app"); err != nil || !has || layer["x"] != "1" {
|
||||||
|
t.Fatalf("the layer after the failure: %v %v %v", layer, has, err)
|
||||||
|
}
|
||||||
|
// A build that waits for a gate is still walked: the failed settings walk holds nothing back.
|
||||||
|
w.broken["anchor"] = false
|
||||||
|
now := time.Now().Add(-time.Minute)
|
||||||
|
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
||||||
|
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2",
|
||||||
|
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now,
|
||||||
|
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
|
||||||
|
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r == nil {
|
||||||
|
t.Fatalf("the waiting build is held back by a failed settings walk: %+v %v", r, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Each send of a settings walk keeps every build of its machine: the layer goes, and a build held for a person or
|
||||||
|
// waiting for a gate stays where it is (ADR 0344 §3).
|
||||||
|
func TestASettingsWalkSendsTheLayerAloneEveryBuildKept(t *testing.T) {
|
||||||
|
w := aSettledMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := w.open.inventory
|
||||||
|
gateEvery = 0
|
||||||
|
if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s3"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for i := 0; i < 4; i++ {
|
||||||
|
advancePlans(ctx, w.open)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(w.kept, []bool{true, true}) || w.walk(t).State != inventory.PlanDone {
|
||||||
|
t.Fatalf("the sends kept every build: %v; the walk: %s", w.kept, w.walk(t).Note)
|
||||||
|
}
|
||||||
|
// What such a send keeps, on the store: a rolled-out build waiting there is kept at the one last sent.
|
||||||
|
now := time.Now().Add(-time.Minute)
|
||||||
|
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
||||||
|
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2",
|
||||||
|
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now,
|
||||||
|
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
|
||||||
|
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
kept, err := recordedKept(keepingBuildsForSettings(ctx, "anchor"), w.open, "anchor")
|
||||||
|
if err != nil || kept["app"] != "c1" {
|
||||||
|
t.Fatalf("a settings send to anchor keeps %v (%v), want app at c1", kept, err)
|
||||||
|
}
|
||||||
|
if kept, _ := recordedKept(ctx, w.open, "anchor"); kept["app"] != "" {
|
||||||
|
t.Fatalf("an ordinary send keeps a rolled-out build: %v", kept)
|
||||||
|
}
|
||||||
|
// And so the ungated build waiting on anchor is no move of the settings send's: nothing for a gate to see.
|
||||||
|
f, err := readMoveFacts(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
waiting, _ := machineMoves(ctx, w.open, f, "anchor", false)
|
||||||
|
if len(waiting) != 1 || waiting[0].Module != "app" {
|
||||||
|
t.Fatalf("anchor waits for %+v, want app's c2", waiting)
|
||||||
|
}
|
||||||
|
if moved, err := withoutKept(keepingBuildsForSettings(ctx, "anchor"), w.open, "anchor", waiting); err != nil || len(moved) != 0 {
|
||||||
|
t.Fatalf("a settings send to anchor would move %+v (%v)", moved, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine whose send would move a running module's data stops the walk there, failed and said with the push that
|
||||||
|
// sends it on a person's word (ADR 0217 §3); one not heard from is passed over and named; and a walk that reached
|
||||||
|
// no machine is not said delivered.
|
||||||
|
func TestASettingsWalkStopsBeforeADataMoveAndPassesOverAMachineAway(t *testing.T) {
|
||||||
|
w := aSettledMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
gateEvery = 0
|
||||||
|
w.moves["laptop"] = []dataMove{{Container: "app-server", Inside: "/data", From: "/srv/a", To: "/srv/b"}}
|
||||||
|
if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s4"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for i := 0; i < 4; i++ {
|
||||||
|
advancePlans(ctx, w.open)
|
||||||
|
}
|
||||||
|
p := w.walk(t)
|
||||||
|
if p.State != inventory.PlanFailed || !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) ||
|
||||||
|
!strings.Contains(p.Note, "not sent to laptop: a running module's data would move there") ||
|
||||||
|
!strings.Contains(p.Note, "push laptop --move app") {
|
||||||
|
t.Fatalf("sent %v; the walk is %s: %s", w.sent, p.State, p.Note)
|
||||||
|
}
|
||||||
|
|
||||||
|
v := aSettledMesh(t)
|
||||||
|
v.heard = map[string]bool{"laptop": true}
|
||||||
|
gateEvery = 0
|
||||||
|
if _, err := openSettingsWalk(ctx, v.open, inventory.PlanSettings{Module: "app", Ask: "s5"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for i := 0; i < 4; i++ {
|
||||||
|
advancePlans(ctx, v.open)
|
||||||
|
}
|
||||||
|
p = v.walk(t)
|
||||||
|
if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Skipped, []string{"anchor"}) ||
|
||||||
|
!strings.Contains(p.Note, "delivered on laptop; not heard from, left as they were, their next send carries it: anchor") {
|
||||||
|
t.Fatalf("the walk with anchor away is %s: %s", p.State, p.Note)
|
||||||
|
}
|
||||||
|
|
||||||
|
u := aSettledMesh(t)
|
||||||
|
u.heard = map[string]bool{}
|
||||||
|
if _, err := openSettingsWalk(ctx, u.open, inventory.PlanSettings{Module: "app", Ask: "s6"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
advancePlans(ctx, u.open)
|
||||||
|
if p := u.walk(t); p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed ||
|
||||||
|
!strings.Contains(p.Note, "was sent to no machine") || len(u.sent) != 0 {
|
||||||
|
t.Fatalf("a walk that reached nobody is %s: %s", p.State, p.Note)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The gate keeps the module's build its send kept, so a crossing send — a merge's walk sending the machine a new
|
||||||
|
// build of the module while the settings gate judges there — supersedes it, and its report is never read as the
|
||||||
|
// settings send's (issue 641 review, finding 1).
|
||||||
|
func TestACrossingSendSupersedesASettingsGate(t *testing.T) {
|
||||||
|
judgeLeavesAsTheCore(t)
|
||||||
|
w := aSettledMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := w.open.inventory
|
||||||
|
if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s7"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
advancePlans(ctx, w.open)
|
||||||
|
p := w.walk(t)
|
||||||
|
if g := p.Release.Gate; g == nil || g.To != "c1" || g.From != "c1" {
|
||||||
|
t.Fatalf("the settings gate judges no build: %+v", p.Release.Gate)
|
||||||
|
}
|
||||||
|
// Another walk sends anchor app's c2 while the settings gate judges there.
|
||||||
|
now := time.Now().Add(-time.Minute)
|
||||||
|
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
||||||
|
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2",
|
||||||
|
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now,
|
||||||
|
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RecordSent(ctx, nodeID(t, w.open, "anchor"), "d-merge", map[string]string{"app": "c2"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := inv.RecordDoing(ctx, nodeID(t, w.open, "anchor"), inventory.Doing{Node: "anchor",
|
||||||
|
Outcome: inventory.OutcomeApplied, Declared: "d-merge", Applied: 1, At: time.Now()}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
gateEvery = 0
|
||||||
|
advancePlans(ctx, w.open)
|
||||||
|
p = w.walk(t)
|
||||||
|
if p.State != inventory.PlanSuperseded || p.Release.Gate == nil || p.Release.Gate.Verdict != inventory.GateSuperseded ||
|
||||||
|
!strings.Contains(p.Note, "superseded on anchor") || len(p.Release.Done) != 0 {
|
||||||
|
t.Fatalf("the settings walk after a crossing send is %s: %s %+v", p.State, p.Note, p.Release)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) {
|
||||||
|
t.Fatalf("sent %v: laptop was sent after a superseded judging", w.sent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A send refusal that cannot clear by itself — what the machine was last sent is not known, or a build it runs is
|
||||||
|
// gone — fails the walk with why, rather than keeping it open and holding the walk of waiting builds; and a
|
||||||
|
// failure names the machines passed over as away (issue 641 review, findings 2 and 3).
|
||||||
|
func TestASettingsWalkFailsOnARefusalThatCannotClear(t *testing.T) {
|
||||||
|
w := aSettledMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := w.open.inventory
|
||||||
|
// What a send to a machine never sent can keep: nothing, and said as the refusal the walk fails on.
|
||||||
|
if _, err := inv.AddNode(ctx, "fresh"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := recordedKept(keepingBuildsForSettings(ctx, "fresh"), w.open, "fresh"); !errors.Is(err, errSettingsCannotKeep) {
|
||||||
|
t.Fatalf("a settings send to a machine never sent: %v", err)
|
||||||
|
}
|
||||||
|
w.heard = map[string]bool{"laptop": true}
|
||||||
|
settingsDataMoves = func(_ context.Context, _ *stores, node string) ([]dataMove, error) {
|
||||||
|
return nil, fmt.Errorf("%w: what %s was last sent is not known", errSettingsCannotKeep, node)
|
||||||
|
}
|
||||||
|
if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s8"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
advancePlans(ctx, w.open)
|
||||||
|
p := w.walk(t)
|
||||||
|
if p.State != inventory.PlanFailed || len(w.sent) != 0 ||
|
||||||
|
!strings.Contains(p.Note, "not sent to laptop: the settings cannot be sent with every build kept: what laptop was last sent is not known") ||
|
||||||
|
!strings.Contains(p.Note, "not heard from, left as they were: anchor") {
|
||||||
|
t.Fatalf("the walk is %s: %s", p.State, p.Note)
|
||||||
|
}
|
||||||
|
// It is not open, so a build waiting for a gate is walked.
|
||||||
|
now := time.Now().Add(-time.Minute)
|
||||||
|
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
||||||
|
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2",
|
||||||
|
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now,
|
||||||
|
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
|
||||||
|
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r == nil {
|
||||||
|
t.Fatalf("the waiting build is held by the failed settings walk: %+v %v", r, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -153,11 +153,15 @@ var signalsTable = []signalRow{
|
|||||||
return newestOf(f.standings, func(c conditions.Condition) time.Time { return c.LastObserved })
|
return newestOf(f.standings, func(c conditions.Condition) time.Time { return c.LastObserved })
|
||||||
}},
|
}},
|
||||||
{Row: "S9", Signal: "bus advisories: maximum deliveries, consumer deleted; the controller's own slow " +
|
{Row: "S9", Signal: "bus advisories: maximum deliveries, consumer deleted; the controller's own slow " +
|
||||||
"consumer and refused subjects", Emitter: "bus server's advisory subjects; the controller's connection",
|
"consumer and refused subjects; every user's connection the server closed as a slow consumer or for " +
|
||||||
|
"breaking the bus's rules (ADR 0357)",
|
||||||
|
Emitter: "bus server's advisory subjects; the controller's connection; the bus module's watch of its " +
|
||||||
|
"server (bus.slow-consumer, bus.connection-closed)",
|
||||||
Trigger: "any", Bound: "any occurrence; clears after an hour without another, a deleted consumer " +
|
Trigger: "any", Bound: "any occurrence; clears after an hour without another, a deleted consumer " +
|
||||||
"once it exists again or the mesh no longer expects it, and a message given up on once DEAD_LETTERS " +
|
"once it exists again or the mesh no longer expects it, and a message given up on once DEAD_LETTERS " +
|
||||||
"no longer holds it (novox/hq issue 330)",
|
"no longer holds it (novox/hq issue 330)",
|
||||||
Kind: "slow-consumer, max-deliveries, refused, consumer-lost", Severity: conditions.Warning, Phase: 1,
|
Kind: "slow-consumer, connection-closed, max-deliveries, refused, consumer-lost", Severity: conditions.Warning,
|
||||||
|
Phase: 1,
|
||||||
needs: func(f *signalFacts) error { return f.advisoriesErr }, watch: watchAdvisories,
|
needs: func(f *signalFacts) error { return f.advisoriesErr }, watch: watchAdvisories,
|
||||||
newest: func(f *signalFacts) time.Time {
|
newest: func(f *signalFacts) time.Time {
|
||||||
return newestOf(f.advisories, func(a link.Advisory) time.Time { return a.Last })
|
return newestOf(f.advisories, func(a link.Advisory) time.Time { return a.Last })
|
||||||
@@ -569,6 +573,9 @@ func watchAdvisories(f *signalFacts) []conditions.Observation {
|
|||||||
// The advisory is remembered and read again on every look for an hour: the condition counts
|
// The advisory is remembered and read again on every look for an hour: the condition counts
|
||||||
// what the bus said and when, not the looks (novox/hq issue 402).
|
// what the bus said and when, not the looks (novox/hq issue 402).
|
||||||
Happened: a.Last, Times: a.Count}
|
Happened: a.Last, Times: a.Count}
|
||||||
|
if a.User != "" && a.Reason != "" {
|
||||||
|
busUserClosed(&o, a, f.host)
|
||||||
|
}
|
||||||
if a.Kind == link.AdvisoryMaxDeliveries && a.Token == link.AdvisoryNotKept {
|
if a.Kind == link.AdvisoryMaxDeliveries && a.Token == link.AdvisoryNotKept {
|
||||||
o.Machine = consumerMachine(a.Stream, a.Consumer)
|
o.Machine = consumerMachine(a.Stream, a.Consumer)
|
||||||
o.Headline = clip(conditions.Capital(fmt.Sprintf("%s gave up on a message, not kept",
|
o.Headline = clip(conditions.Capital(fmt.Sprintf("%s gave up on a message, not kept",
|
||||||
@@ -579,9 +586,58 @@ func watchAdvisories(f *signalFacts) []conditions.Observation {
|
|||||||
}
|
}
|
||||||
out = append(out, o)
|
out = append(out, o)
|
||||||
}
|
}
|
||||||
|
if f.busFaultsErr != nil {
|
||||||
|
out = append(out, busFaultsUnread(f.busFaultsErr))
|
||||||
|
}
|
||||||
return append(out, watchDeadLetters(f)...)
|
return append(out, watchDeadLetters(f)...)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// kindBusFaultsUnread is the bus module's events not read back (novox/hq ADR 0357): every other user's
|
||||||
|
// slow consumer is unheard while it lasts, and the controller's own signals are still said.
|
||||||
|
const kindBusFaultsUnread = "bus-faults-unread"
|
||||||
|
|
||||||
|
func busFaultsUnread(err error) conditions.Observation {
|
||||||
|
return conditions.Observation{Scope: conditions.ScopeBus, ID: "bus-module", Kind: kindBusFaultsUnread,
|
||||||
|
Severity: conditions.Warning, Confirm: true,
|
||||||
|
Summary: "what the bus module said of its users' connections could not be read back, so a slow consumer " +
|
||||||
|
"or a connection the bus closed is not said until it can",
|
||||||
|
Said: firstLine(err.Error())}
|
||||||
|
}
|
||||||
|
|
||||||
|
// busUserClosed words an advisory about a bus user's connection the server closed (novox/hq ADR 0357): the
|
||||||
|
// user as the operator says it and the machine it is on, the dotted user name kept in the evidence only.
|
||||||
|
func busUserClosed(o *conditions.Observation, a link.Advisory, host string) {
|
||||||
|
who, machine := busUserWords(a.User)
|
||||||
|
if machine == "" && a.User == "controller" {
|
||||||
|
machine = host
|
||||||
|
}
|
||||||
|
o.Machine = machine
|
||||||
|
times, count := "once", "once"
|
||||||
|
if a.Count > 1 {
|
||||||
|
times = fmt.Sprintf("%d times since %s", a.Count, a.First.UTC().Format("15:04 MST"))
|
||||||
|
count = fmt.Sprintf("%d times", a.Count)
|
||||||
|
}
|
||||||
|
switch a.Kind {
|
||||||
|
case link.AdvisorySlowConsumer:
|
||||||
|
o.Summary = fmt.Sprintf("the bus closed the connection of %s as a slow consumer %s in the last hour, "+
|
||||||
|
"newest at %s: it could not take what it was sent fast enough, and what was on its way to it was lost",
|
||||||
|
who, times, a.Last.UTC().Format("15:04 MST"))
|
||||||
|
o.Headline = clip(conditions.Capital(who)+" was too slow for the bus", 60)
|
||||||
|
o.Explanation = conditions.Capital(fmt.Sprintf("%s could not keep up with what the bus sent it, so the bus "+
|
||||||
|
"closed its connection (%s in the last hour) and what was on its way was lost. It connects again "+
|
||||||
|
"by itself; this clears an hour after the last time.", who, count))
|
||||||
|
o.Resolved = "Resolved: " + who + " kept up for an hour"
|
||||||
|
default:
|
||||||
|
o.Summary = fmt.Sprintf("the bus closed the connection of %s for breaking its rules %s in the last hour, "+
|
||||||
|
"newest at %s: %s", who, times, a.Last.UTC().Format("15:04 MST"), strings.ToLower(a.Reason))
|
||||||
|
o.Headline = clip(conditions.Capital(who)+" was cut off by the bus", 60)
|
||||||
|
o.Explanation = conditions.Capital(fmt.Sprintf("The bus closed the connection of %s because it broke "+
|
||||||
|
"the bus's rules (%s, %s in the last hour), so what it was sending was lost. This clears an hour "+
|
||||||
|
"after the last time.", who, strings.ToLower(a.Reason), count))
|
||||||
|
o.Resolved = "Resolved: " + who + " kept to the bus's rules for an hour"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// watchDeadLetters says each consumer that DEAD_LETTERS holds a message for (novox/hq issue 330): open
|
// watchDeadLetters says each consumer that DEAD_LETTERS holds a message for (novox/hq issue 330): open
|
||||||
// while it holds any, so it clears when they are delivered again or dropped, never because the server
|
// while it holds any, so it clears when they are delivered again or dropped, never because the server
|
||||||
// stopped saying it.
|
// stopped saying it.
|
||||||
@@ -806,7 +862,7 @@ func watchLease(f *signalFacts) []conditions.Observation {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(lost) > 0 {
|
if len(lost) > 0 {
|
||||||
how := "lost it: its renewal was refused or could not be made"
|
how := "lost it: another holder wrote its key, or no renewal was made within the key's age"
|
||||||
if newest.How == inventory.EpochExpired {
|
if newest.How == inventory.EpochExpired {
|
||||||
how = "stopped renewing it without giving it back, and was taken over"
|
how = "stopped renewing it without giving it back, and was taken over"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,157 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A new push to a pull request ends the check of its old head (novox/hq issue 726).
|
||||||
|
//
|
||||||
|
// A check judges one head; once the pull request has another, its verdict answers nobody and its run holds a build
|
||||||
|
// machine for minutes the new head waits behind. So when a check is asked for pull request N of a repository at a
|
||||||
|
// head, **every check of that pull request at another head ends**: a waiting one is dropped from the queue as
|
||||||
|
// `cancel` drops it, one in flight is ended on its machine as `kill` ends it. Each is said in the log.
|
||||||
|
//
|
||||||
|
// **Never anything else**: a try (`try`, asked for no pull request), a walk's build (no check), a delivery group's
|
||||||
|
// composed check (it answers several pull requests), another pull request's check, a dead ask (left for `clear
|
||||||
|
// --dead`), and a check of the same head — a recheck of that head is not another head.
|
||||||
|
|
||||||
|
// **Only for the pull request's current head.** Announcements of a pull request are not ordered, one is heard again
|
||||||
|
// after a restart, and a recheck asks for the head it recorded: a check asked for a head the pull request no longer
|
||||||
|
// has must never end the check of the head it has. So the forge is asked first which head the pull request has; only
|
||||||
|
// when it names this one does anything end, and when it cannot be asked, or names another, nothing ends and that is
|
||||||
|
// said.
|
||||||
|
|
||||||
|
// forgePullHead asks the forge, through its module's tools on the bus, which commit a pull request's head is. A
|
||||||
|
// variable so a test needs no forge.
|
||||||
|
var forgePullHead = func(ctx context.Context, js *broker.JetStream, owner, repo string, number int) (string, error) {
|
||||||
|
raw, _ := json.Marshal(map[string]any{"path": fmt.Sprintf("/repos/%s/%s/pulls/%d", owner, repo, number)})
|
||||||
|
answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", "gitea_api", raw, 30*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if answer.Error != "" {
|
||||||
|
return "", errors.New(answer.Error)
|
||||||
|
}
|
||||||
|
var pull struct {
|
||||||
|
Result struct {
|
||||||
|
Head struct {
|
||||||
|
SHA string `json:"sha"`
|
||||||
|
} `json:"head"`
|
||||||
|
} `json:"result"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(answer.Result, &pull); err != nil {
|
||||||
|
return "", fmt.Errorf("the forge's answer about the pull request cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
if pull.Result.Head.SHA == "" {
|
||||||
|
return "", errors.New("the forge named no head for the pull request")
|
||||||
|
}
|
||||||
|
return pull.Result.Head.SHA, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// supersededChecks are the checks in the queue that a check of owner/repo#number at head ends: of the same pull
|
||||||
|
// request, at another head, waiting or in flight, and not the ask keep.
|
||||||
|
func supersededChecks(q link.Queue, owner, repo string, number int, head, keep string) []link.QueuedAsk {
|
||||||
|
if number <= 0 || head == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var out []link.QueuedAsk
|
||||||
|
for _, a := range q.Asks {
|
||||||
|
c := a.Request.Check
|
||||||
|
switch {
|
||||||
|
case c == nil, link.IsTry(a.Request), c.Group != "", c.Number != number, a.ID == keep,
|
||||||
|
!strings.EqualFold(c.Owner, owner), !strings.EqualFold(c.Repo, repo),
|
||||||
|
a.State != link.AskWaiting && a.State != link.AskInFlight:
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ref := a.Request.Ref
|
||||||
|
if ref == "" {
|
||||||
|
ref = a.Ref
|
||||||
|
}
|
||||||
|
if ref == "" || ref == head || link.SameHead(ref, head) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, a)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// endSupersededChecks ends the checks a check of p at its head supersedes, keep being the new check's own ask, and
|
||||||
|
// says each in the log. A waiting one is cancelled — nothing recorded: a check builds nothing, and the forge's status
|
||||||
|
// of the old head is no longer the pull request's — and one a machine builds is killed there.
|
||||||
|
func endSupersededChecks(ctx context.Context, js *broker.JetStream, seat string, p link.PullUpdated, keep string) []string {
|
||||||
|
if p.Number <= 0 || p.Commit == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
current, err := forgePullHead(ctx, js, p.Owner, p.Repo, p.Number)
|
||||||
|
if err != nil {
|
||||||
|
return []string{fmt.Sprintf("%s/%s#%d (%.8s): nothing ended — the forge could not say the pull request's "+
|
||||||
|
"current head, so checks of its other heads keep running: %v", p.Owner, p.Repo, p.Number, p.Commit, err)}
|
||||||
|
}
|
||||||
|
if current != p.Commit && !link.SameHead(current, p.Commit) {
|
||||||
|
return []string{fmt.Sprintf("%s/%s#%d (%.8s): nothing ended — the pull request's head is %.12s now, not "+
|
||||||
|
"this one", p.Owner, p.Repo, p.Number, p.Commit, current)}
|
||||||
|
}
|
||||||
|
q, err := link.ReadQueue(ctx, js, seat)
|
||||||
|
if err != nil {
|
||||||
|
return []string{fmt.Sprintf("%s/%s#%d (%.8s): could not read the %s queue for checks of its earlier heads: %v",
|
||||||
|
p.Owner, p.Repo, p.Number, p.Commit, seat, err)}
|
||||||
|
}
|
||||||
|
var said []string
|
||||||
|
for _, a := range supersededChecks(q, p.Owner, p.Repo, p.Number, p.Commit, keep) {
|
||||||
|
old := a.Request.Ref
|
||||||
|
if old == "" {
|
||||||
|
old = a.Ref
|
||||||
|
}
|
||||||
|
what := fmt.Sprintf("check of %s/%s#%d at %.12s", p.Owner, p.Repo, p.Number, old)
|
||||||
|
var err error
|
||||||
|
if a.State == link.AskInFlight && a.On != "" {
|
||||||
|
var answer link.Answer
|
||||||
|
answer, err = link.AskSeatTool(ctx, js.Conn(), seat, "kill", a.On, map[string]string{"id": a.ID}, killAnswer)
|
||||||
|
if err == nil && answer.Error != "" {
|
||||||
|
err = fmt.Errorf("%s refused: %s", a.On, answer.Error)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
_, err = cancelAsk(ctx, js, nil, seat, a)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
said = append(said, fmt.Sprintf("not ended: %s (%s) — superseded by %.12s, but: %v", what, a.ID, p.Commit, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
said = append(said, fmt.Sprintf("ended: %s — superseded by %.12s (%s, %s)", what, p.Commit, a.ID, a.State))
|
||||||
|
}
|
||||||
|
return said
|
||||||
|
}
|
||||||
|
|
||||||
|
// endingSupersededChecks ends them beside the announcement, so a kill waiting on its machine never holds the
|
||||||
|
// handler: on its own connection to the bus, within twice a kill's wait. A one-shot process (oneShot) ends them
|
||||||
|
// within the call instead, as its exit would cut them off.
|
||||||
|
func endingSupersededChecks(ctx context.Context, seat string, p link.PullUpdated, keep string, oneShot bool) {
|
||||||
|
end := func() {
|
||||||
|
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*killAnswer)
|
||||||
|
defer cancel()
|
||||||
|
js, err := dialTheBus()
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "%s/%s#%d (%.8s): could not reach the bus to end checks of its earlier heads: %v\n",
|
||||||
|
p.Owner, p.Repo, p.Number, p.Commit, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
for _, line := range endSupersededChecks(ctx, js, seat, p, keep) {
|
||||||
|
fmt.Println(line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if oneShot {
|
||||||
|
end()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
go end()
|
||||||
|
}
|
||||||
@@ -0,0 +1,192 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A new push to a pull request ends the check of its old head (novox/hq issue 726): the waiting one is dropped from
|
||||||
|
// the queue, the one in flight is killed on its machine, and each is said. A try, a walk's build, another pull
|
||||||
|
// request's check, a group's check and a recheck of the new head are left alone. On the bus alone: no store.
|
||||||
|
func TestANewHeadEndsTheChecksOfTheOldOnes(t *testing.T) {
|
||||||
|
js, _ := aBuildQueue(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
const (
|
||||||
|
oldHead = "1111111111111111111111111111111111111111"
|
||||||
|
olderHead = "2222222222222222222222222222222222222222"
|
||||||
|
newHead = "3333333333333333333333333333333333333333"
|
||||||
|
)
|
||||||
|
publish := func(r link.BuildRequest) {
|
||||||
|
t.Helper()
|
||||||
|
body, _ := json.Marshal(r)
|
||||||
|
if _, err := js.Context().Publish(link.BuildWork(), body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
check := func(id string, number int, head string) link.BuildRequest {
|
||||||
|
return link.BuildRequest{ID: id, Repository: "https://forge.example/novox/a.git", Ref: head,
|
||||||
|
Check: &link.CheckRequest{Owner: "novox", Repo: "a", Number: number}}
|
||||||
|
}
|
||||||
|
// In flight first: taken by a holder, started on ace.
|
||||||
|
publish(check("check-running", 7, olderHead))
|
||||||
|
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
|
||||||
|
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = sub.Unsubscribe() }()
|
||||||
|
if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
started, _ := json.Marshal(link.BuildStart{ID: "check-running", On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
|
||||||
|
if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
publish(check("check-old", 7, oldHead))
|
||||||
|
publish(check("check-other-pull", 8, oldHead))
|
||||||
|
publish(check("check-recheck", 7, newHead))
|
||||||
|
publish(check("check-new", 7, newHead))
|
||||||
|
try := check("try-old", 7, oldHead)
|
||||||
|
try.Check.Number, try.Check.Try = 0, true
|
||||||
|
publish(try)
|
||||||
|
group := check("group-old", 7, oldHead)
|
||||||
|
group.Check.Group = "g1"
|
||||||
|
publish(group)
|
||||||
|
publish(link.BuildRequest{ID: "walk-build", Repository: "https://forge.example/novox/a.git", Ref: oldHead})
|
||||||
|
|
||||||
|
var mu sync.Mutex
|
||||||
|
var killed []string
|
||||||
|
stop, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(link.TheBuildMachine, "ace", map[string]link.ToolHandler{
|
||||||
|
"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
|
||||||
|
var args struct{ ID string }
|
||||||
|
_ = json.Unmarshal(raw, &args)
|
||||||
|
mu.Lock()
|
||||||
|
killed = append(killed, args.ID)
|
||||||
|
mu.Unlock()
|
||||||
|
return map[string]any{"said": "killed " + args.ID}, nil
|
||||||
|
}}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
forgeSays(t, newHead, nil)
|
||||||
|
said := endSupersededChecks(ctx, js, link.TheBuildMachine,
|
||||||
|
link.PullUpdated{Owner: "novox", Repo: "a", Number: 7, Commit: newHead}, "check-new")
|
||||||
|
|
||||||
|
mu.Lock()
|
||||||
|
if len(killed) != 1 || killed[0] != "check-running" {
|
||||||
|
t.Errorf("the holder was asked to kill %v, want only the in-flight check of the older head", killed)
|
||||||
|
}
|
||||||
|
mu.Unlock()
|
||||||
|
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, found := q.Find("check-old"); found {
|
||||||
|
t.Error("the waiting check of the old head is still in the queue")
|
||||||
|
}
|
||||||
|
for _, id := range []string{"check-other-pull", "check-recheck", "check-new", "try-old", "group-old", "walk-build"} {
|
||||||
|
if _, found := q.Find(id); !found {
|
||||||
|
t.Errorf("%s was ended; only checks of the same pull request at another head end", id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
text := strings.Join(said, "\n")
|
||||||
|
for _, want := range []string{
|
||||||
|
"ended: check of novox/a#7 at " + oldHead[:12] + " — superseded by " + newHead[:12],
|
||||||
|
"ended: check of novox/a#7 at " + olderHead[:12] + " — superseded by " + newHead[:12],
|
||||||
|
} {
|
||||||
|
if !strings.Contains(text, want) {
|
||||||
|
t.Errorf("the log does not say %q:\n%s", want, text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(said) != 2 {
|
||||||
|
t.Errorf("said %d lines, want 2:\n%s", len(said), text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Which asks a check supersedes, read from the queue alone.
|
||||||
|
func TestSupersededChecksAreOnlyThePullRequestsOtherHeads(t *testing.T) {
|
||||||
|
ask := func(id, state, ref string, c *link.CheckRequest) link.QueuedAsk {
|
||||||
|
return link.QueuedAsk{ID: id, State: state, Ref: ref, Request: link.BuildRequest{ID: id, Ref: ref, Check: c}}
|
||||||
|
}
|
||||||
|
pr := func(n int) *link.CheckRequest { return &link.CheckRequest{Owner: "Novox", Repo: "A", Number: n} }
|
||||||
|
head := "abcdefabcdefabcdef0123456789abcdef012345"
|
||||||
|
q := link.Queue{Asks: []link.QueuedAsk{
|
||||||
|
ask("old", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", pr(7)),
|
||||||
|
ask("short-same", link.AskWaiting, head[:12], pr(7)),
|
||||||
|
ask("dead", link.AskDead, "0123456789ab0123456789ab0123456789ab0123", pr(7)),
|
||||||
|
ask("other", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", pr(8)),
|
||||||
|
ask("build", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", nil),
|
||||||
|
ask("keep", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", pr(7)),
|
||||||
|
}}
|
||||||
|
got := supersededChecks(q, "novox", "a", 7, head, "keep")
|
||||||
|
if len(got) != 1 || got[0].ID != "old" {
|
||||||
|
t.Fatalf("superseded %+v, want only old", got)
|
||||||
|
}
|
||||||
|
if got := supersededChecks(q, "novox", "a", 0, head, ""); got != nil {
|
||||||
|
t.Errorf("a check of no pull request superseded %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// forgeSays makes the forge name head as every pull request's current head, or fail with err.
|
||||||
|
func forgeSays(t *testing.T, head string, err error) {
|
||||||
|
t.Helper()
|
||||||
|
was := forgePullHead
|
||||||
|
forgePullHead = func(context.Context, *broker.JetStream, string, string, int) (string, error) { return head, err }
|
||||||
|
t.Cleanup(func() { forgePullHead = was })
|
||||||
|
}
|
||||||
|
|
||||||
|
// An older head heard after a newer one — an announcement out of order, one heard again after a restart, a recheck
|
||||||
|
// of a recorded head — ends nothing: the forge says the pull request's head is the newer one. Nor does a check whose
|
||||||
|
// pull request's head the forge cannot say.
|
||||||
|
func TestAnOlderHeadHeardLateEndsNothing(t *testing.T) {
|
||||||
|
js, _ := aBuildQueue(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
const (
|
||||||
|
oldHead = "1111111111111111111111111111111111111111"
|
||||||
|
newHead = "3333333333333333333333333333333333333333"
|
||||||
|
)
|
||||||
|
for _, id := range []string{"check-new", "check-old-late"} {
|
||||||
|
head := newHead
|
||||||
|
if id == "check-old-late" {
|
||||||
|
head = oldHead
|
||||||
|
}
|
||||||
|
body, _ := json.Marshal(link.BuildRequest{ID: id, Repository: "https://forge.example/novox/a.git", Ref: head,
|
||||||
|
Check: &link.CheckRequest{Owner: "novox", Repo: "a", Number: 7}})
|
||||||
|
if _, err := js.Context().Publish(link.BuildWork(), body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
late := link.PullUpdated{Owner: "novox", Repo: "a", Number: 7, Commit: oldHead}
|
||||||
|
forgeSays(t, newHead, nil)
|
||||||
|
said := endSupersededChecks(ctx, js, link.TheBuildMachine, late, "check-old-late")
|
||||||
|
if len(said) != 1 || !strings.Contains(said[0], "nothing ended") {
|
||||||
|
t.Errorf("an older head heard late said %q", said)
|
||||||
|
}
|
||||||
|
forgeSays(t, "", errors.New("nothing serves gitea.gitea_api"))
|
||||||
|
said = endSupersededChecks(ctx, js, link.TheBuildMachine,
|
||||||
|
link.PullUpdated{Owner: "novox", Repo: "a", Number: 7, Commit: newHead}, "check-new")
|
||||||
|
if len(said) != 1 || !strings.Contains(said[0], "nothing ended") {
|
||||||
|
t.Errorf("a head the forge could not confirm said %q", said)
|
||||||
|
}
|
||||||
|
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, id := range []string{"check-new", "check-old-late"} {
|
||||||
|
if _, found := q.Find(id); !found {
|
||||||
|
t.Errorf("%s was ended", id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -130,10 +130,14 @@ func aTierMesh(t *testing.T, modules ...string) *tierMesh {
|
|||||||
}
|
}
|
||||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||||
|
|
||||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||||
|
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||||
|
wasWindow := gateWitnessWindow
|
||||||
|
gateWitnessWindow = 0
|
||||||
|
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||||
// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
|
// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
|
||||||
gateSettle, gateEvery = 0, time.Hour
|
gateWithin, gateEvery = time.Hour, time.Hour
|
||||||
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
|
||||||
|
|
||||||
built := time.Now().UTC()
|
built := time.Now().UTC()
|
||||||
states := map[string]*inventory.PlanModule{}
|
states := map[string]*inventory.PlanModule{}
|
||||||
@@ -167,6 +171,7 @@ func (tm *tierMesh) raise(machine, kind, summary string) {
|
|||||||
// The incident's tier, small: a dozen modules whose first machine is the same are sent there in one
|
// The incident's tier, small: a dozen modules whose first machine is the same are sent there in one
|
||||||
// send, judged by one gate, and the rest of their machines are sent once — two sends, not two dozen.
|
// send, judged by one gate, and the rest of their machines are sent once — two sends, not two dozen.
|
||||||
func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) {
|
func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
judgeLeavesAsTheCore(t)
|
judgeLeavesAsTheCore(t)
|
||||||
var modules []string
|
var modules []string
|
||||||
for i := 1; i <= 12; i++ {
|
for i := 1; i <= 12; i++ {
|
||||||
@@ -219,9 +224,13 @@ func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) {
|
|||||||
// settling, is the node tools' — inside their settle window, the gate's bound — and no failure of the
|
// settling, is the node tools' — inside their settle window, the gate's bound — and no failure of the
|
||||||
// modules sent with them: at the bound only the node tools are put back.
|
// modules sent with them: at the bound only the node tools are put back.
|
||||||
func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) {
|
func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
tm := aTierMesh(t, "app1", "app2", broker.RuntimeModule)
|
tm := aTierMesh(t, "app1", "app2", broker.RuntimeModule)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := tm.open.inventory
|
inv := tm.open.inventory
|
||||||
|
// The node tools are held inside their witness window, which the node-engine keeps while they settle: their
|
||||||
|
// one reading does not count yet, so the core behind raised meanwhile is theirs (novox/hq ADR 0351 decision 4).
|
||||||
|
gateWitnessWindow = time.Hour
|
||||||
|
|
||||||
advancePlans(ctx, tm.open)
|
advancePlans(ctx, tm.open)
|
||||||
if len(tm.sent) != 1 {
|
if len(tm.sent) != 1 {
|
||||||
@@ -269,6 +278,7 @@ func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) {
|
|||||||
// A machine-level condition about anything else holds the machine back as a whole: everything the send
|
// A machine-level condition about anything else holds the machine back as a whole: everything the send
|
||||||
// moved there fails together, once, with that reason — not one module for it.
|
// moved there fails together, once, with that reason — not one module for it.
|
||||||
func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) {
|
func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
judgeLeavesAsTheCore(t)
|
judgeLeavesAsTheCore(t)
|
||||||
tm := aTierMesh(t, "app1", "app2", "app3")
|
tm := aTierMesh(t, "app1", "app2", "app3")
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
@@ -300,6 +310,7 @@ func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) {
|
|||||||
// The machine-level conditions, sorted: one naming a module is that module's; the core behind is the
|
// The machine-level conditions, sorted: one naming a module is that module's; the core behind is the
|
||||||
// core's when the send moved it and nobody's when it did not; anything else is the machine's as a whole.
|
// core's when the send moved it and nobody's when it did not; anything else is the machine's as a whole.
|
||||||
func TestTheConditionsAboutAMachineItself(t *testing.T) {
|
func TestTheConditionsAboutAMachineItself(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
since := time.Now().Add(-time.Minute)
|
since := time.Now().Add(-time.Minute)
|
||||||
about := func(kind, id string) conditions.Condition {
|
about := func(kind, id string) conditions.Condition {
|
||||||
return conditions.Condition{Key: "machine." + id + "." + kind, Kind: kind, Raised: time.Now(),
|
return conditions.Condition{Key: "machine." + id + "." + kind, Kind: kind, Raised: time.Now(),
|
||||||
@@ -332,6 +343,7 @@ func TestTheConditionsAboutAMachineItself(t *testing.T) {
|
|||||||
// rollback that failed; and `plans retry` asks it again rather than refusing a build that was never
|
// rollback that failed; and `plans retry` asks it again rather than refusing a build that was never
|
||||||
// judged (the incident's "NOT put back: no build kept").
|
// judged (the incident's "NOT put back: no build kept").
|
||||||
func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) {
|
func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
judgeLeavesAsTheCore(t)
|
judgeLeavesAsTheCore(t)
|
||||||
g := aGateMesh(t)
|
g := aGateMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
@@ -387,6 +399,7 @@ func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) {
|
|||||||
// them, they are registered back and marked in the judging that finds them broken, the plan goes on judging
|
// them, they are registered back and marked in the judging that finds them broken, the plan goes on judging
|
||||||
// the other module to its own pass, and only then fails.
|
// the other module to its own pass, and only then fails.
|
||||||
func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
|
func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
judgeLeavesAsTheCore(t)
|
judgeLeavesAsTheCore(t)
|
||||||
for _, order := range [][]string{{broker.RuntimeModule, "app1"}, {"app1", broker.RuntimeModule}} {
|
for _, order := range [][]string{{broker.RuntimeModule, "app1"}, {"app1", broker.RuntimeModule}} {
|
||||||
t.Run("gate kept on "+order[0], func(t *testing.T) {
|
t.Run("gate kept on "+order[0], func(t *testing.T) {
|
||||||
@@ -405,7 +418,10 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
|
|||||||
}
|
}
|
||||||
return f, err
|
return f, err
|
||||||
}
|
}
|
||||||
gateEvery, gateSettle, gateBound = 0, 100*time.Millisecond, 10*time.Second
|
// app1, judged as the core, counts a reading at most every 100 ms: it cannot have its three in the
|
||||||
|
// judgings that find the node tools broken, and is judged to its own pass after them (the hold the
|
||||||
|
// settle time gave before novox/hq ADR 0352 counted readings from the send).
|
||||||
|
gateEvery, gateWithin, gateBound = 100*time.Millisecond, time.Hour, 10*time.Second
|
||||||
for i := 0; i < 20 && len(seen) < 2; i++ {
|
for i := 0; i < 20 && len(seen) < 2; i++ {
|
||||||
advancePlans(ctx, tm.open)
|
advancePlans(ctx, tm.open)
|
||||||
}
|
}
|
||||||
@@ -440,15 +456,16 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// **A send carrying a core module waits for the core's readings** (issue 524 review): the leaf beside the node
|
// **A send carrying the tool runner waits for its witness window** (issue 524 review, novox/hq ADR 0351
|
||||||
// tools has its one reading at once, but the send is judged as the core: not before the settle time, and three
|
// decision 4): the leaf beside the node tools has its one reading at once, but the send is judged by the tool
|
||||||
// readings.
|
// runner's rule: one reading, not before the node-engine's witness window for it has ended.
|
||||||
func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) {
|
func TestASendCarryingTheToolRunnerWaitsForItsWitnessWindow(t *testing.T) {
|
||||||
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||||
tm := aTierMesh(t, broker.RuntimeModule, "app1")
|
tm := aTierMesh(t, broker.RuntimeModule, "app1")
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
// One reading counted per advance, as every test with a gate left open does: a gate judged with no time
|
// One reading counted per advance, as every test with a gate left open does: a gate judged with no time
|
||||||
// between readings would count a pass at every look and never let an advance rest.
|
// between readings would count a pass at every look and never let an advance rest.
|
||||||
gateEvery, gateSettle = time.Hour, time.Hour
|
gateEvery, gateWitnessWindow = time.Hour, time.Hour
|
||||||
for i := 0; i < 6; i++ {
|
for i := 0; i < 6; i++ {
|
||||||
advancePlans(ctx, tm.open)
|
advancePlans(ctx, tm.open)
|
||||||
}
|
}
|
||||||
@@ -457,10 +474,10 @@ func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) {
|
|||||||
if lead == nil {
|
if lead == nil {
|
||||||
lead = p.Modules["app1"].Gate
|
lead = p.Modules["app1"].Gate
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != gatePasses {
|
if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != 1 {
|
||||||
t.Fatalf("passed before the core's settle time: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead)
|
t.Fatalf("passed inside the witness window: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead)
|
||||||
}
|
}
|
||||||
gateEvery, gateSettle = 0, 0
|
gateEvery, gateWitnessWindow = 0, 0
|
||||||
for i := 0; i < 6; i++ {
|
for i := 0; i < 6; i++ {
|
||||||
advancePlans(ctx, tm.open)
|
advancePlans(ctx, tm.open)
|
||||||
}
|
}
|
||||||
@@ -469,8 +486,9 @@ func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) {
|
|||||||
t.Fatalf("the plan is %s (%s), sent %v", p.State, p.Note, tm.sent)
|
t.Fatalf("the plan is %s (%s), sent %v", p.State, p.Note, tm.sent)
|
||||||
}
|
}
|
||||||
for _, m := range []string{broker.RuntimeModule, "app1"} {
|
for _, m := range []string{broker.RuntimeModule, "app1"} {
|
||||||
if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < gatePasses) {
|
if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < 1 ||
|
||||||
t.Fatalf("%s's gate: %+v; want passed on %d readings", m, g, gatePasses)
|
!strings.Contains(g.Why, "the tool runner's rule")) {
|
||||||
|
t.Fatalf("%s's gate: %+v; want passed on one reading by the tool runner's rule", m, g)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,353 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A branch is tried before its pull request (novox/hq issue 686).
|
||||||
|
//
|
||||||
|
// A repository's store tests skip without a store, and no agent may start a database on its machine. So a
|
||||||
|
// builder pushed, a reviewer read, and only the pull request's merge check ran those tests for the first time:
|
||||||
|
// each failure there cost a round of fifteen to thirty minutes (mesh-controller #283, #310, #312).
|
||||||
|
//
|
||||||
|
// `try <owner>/<repository> <ref>` asks the build seat for the repository's own check of a pushed branch or
|
||||||
|
// commit, as the merge check runs it (internal/builder/check.go): the ref merged into the trunk's head, the
|
||||||
|
// trunk's merge-check.sh, each part in its toolchain's container, against a throwaway store and bus. It is a
|
||||||
|
// check and nothing more:
|
||||||
|
//
|
||||||
|
// - **no pull request, so no status**: its outcome is never said as `checked`, which is what the forge's
|
||||||
|
// holder sets statuses from (isTry, in Built);
|
||||||
|
// - **nothing registered, nothing built**: it names no module of the graph, so the merge gate does not run and
|
||||||
|
// no build is made for a merge (novox/hq ADR 0350's scratch builds are a pull request's check's alone);
|
||||||
|
// - **a check in the build queue**: after deliveries (link.KindOf, novox/hq issue 558), and bounded by the
|
||||||
|
// check's own bound (checkTimeout);
|
||||||
|
// - **the code a pull request's check would run anyway**: only a repository of the mesh, whose every pull
|
||||||
|
// request the build seat already checks, in containers that hold no runtime socket.
|
||||||
|
//
|
||||||
|
// A check outlasts a seat verb's one answer, so the ask answers its id at once and `try --id <id>` reads what
|
||||||
|
// came of it from the build's log, where the build seat says it whole (link.StepTried): holding its answer up
|
||||||
|
// to tryWait for a verdict, like `plans wait` (novox/hq issue 229). Nothing is kept beside the log.
|
||||||
|
|
||||||
|
// tryWait is how long reading a try holds its answer for the verdict. A variable so a test need not wait.
|
||||||
|
var tryWait = 20 * time.Second
|
||||||
|
|
||||||
|
// A ref as the forge is asked for it — a branch or a commit, never an option — and a `go test -run` pattern.
|
||||||
|
var (
|
||||||
|
tryRef = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]*$`)
|
||||||
|
tryTests = regexp.MustCompile(`^[A-Za-z0-9_^$|().*+?/\[\]-]+$`)
|
||||||
|
)
|
||||||
|
|
||||||
|
// tryAnswer is what `try` answers, asking or reading.
|
||||||
|
type tryAnswer struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
// State is asked (just now), waiting (nothing said of it yet: in the build queue), running, or the verdict:
|
||||||
|
// pass, warning (the repository declares no merge-check.sh), fail, or error (it could not run).
|
||||||
|
State string `json:"state"`
|
||||||
|
Said string `json:"said"`
|
||||||
|
Repository string `json:"repository,omitempty"`
|
||||||
|
Ref string `json:"ref,omitempty"`
|
||||||
|
Commit string `json:"commit,omitempty"`
|
||||||
|
Base string `json:"base,omitempty"`
|
||||||
|
Trunk string `json:"trunk,omitempty"`
|
||||||
|
On string `json:"on,omitempty"`
|
||||||
|
Took string `json:"took,omitempty"`
|
||||||
|
Failed []string `json:"failed,omitempty"`
|
||||||
|
// Last is the last thing the build seat said of one still running.
|
||||||
|
Last string `json:"last,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// isTry says a check's outcome is a try's: marked so, or — echoed by a build seat from before the mark — a
|
||||||
|
// check of no pull request and no group, which is nobody's status either.
|
||||||
|
func isTry(result link.BuildResult) bool {
|
||||||
|
c := result.Checked
|
||||||
|
return c != nil && (c.Try || (c.Number == 0 && c.Group == ""))
|
||||||
|
}
|
||||||
|
|
||||||
|
// asATry makes a check's ask a try's: the repository's own script alone, for no pull request.
|
||||||
|
func asATry(request link.BuildRequest, tests string, storeOnly bool) link.BuildRequest {
|
||||||
|
c := *request.Check
|
||||||
|
c.Number, c.Try, c.Tests, c.StoreOnly = 0, true, tests, storeOnly
|
||||||
|
c.Paths, c.Modules, c.New, c.Dependents, c.Manifests, c.Builds = nil, nil, nil, nil, nil, nil
|
||||||
|
c.Judge, c.Plan, c.Group, c.Members = "", nil, "", nil
|
||||||
|
request.Check = &c
|
||||||
|
return request
|
||||||
|
}
|
||||||
|
|
||||||
|
// maxTries is how many tries may wait or run at once, mesh-wide: each holds a build machine for minutes.
|
||||||
|
const maxTries = 4
|
||||||
|
|
||||||
|
// triesIn reads the build queue for a new try of one commit: the id of a try of that commit already waiting or
|
||||||
|
// running — the answer to a second ask — and whether there is room for another.
|
||||||
|
func triesIn(q link.Queue, owner, repo, commit string) (same string, room bool) {
|
||||||
|
n := 0
|
||||||
|
for _, a := range q.Asks {
|
||||||
|
if !link.IsTry(a.Request) || (a.State != link.AskWaiting && a.State != link.AskInFlight) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
n++
|
||||||
|
c := a.Request.Check
|
||||||
|
if same == "" && strings.EqualFold(c.Owner, owner) && strings.EqualFold(c.Repo, repo) && a.Request.Ref == commit {
|
||||||
|
same = a.ID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return same, n < maxTries
|
||||||
|
}
|
||||||
|
|
||||||
|
// forgeCommit asks the forge, through its module's tools on the bus, which commit a branch or commit of a
|
||||||
|
// repository is, and the repository's default branch. A variable so a test needs no forge.
|
||||||
|
var forgeCommit = func(ctx context.Context, owner, repo, ref string) (commit, trunk string, err error) {
|
||||||
|
js, err := aBus()
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
bus := link.OverNATS{Conn: js.Conn()}
|
||||||
|
ask := func(path string, into any) error {
|
||||||
|
raw, _ := json.Marshal(map[string]any{"path": path})
|
||||||
|
answer, err := link.Ask(ctx, bus, "gitea", "gitea_api", raw, 30*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if answer.Error != "" {
|
||||||
|
return errors.New(answer.Error)
|
||||||
|
}
|
||||||
|
return json.Unmarshal(answer.Result, into)
|
||||||
|
}
|
||||||
|
var commits struct {
|
||||||
|
Result []struct {
|
||||||
|
SHA string `json:"sha"`
|
||||||
|
} `json:"result"`
|
||||||
|
}
|
||||||
|
if err := ask("/repos/"+owner+"/"+repo+"/commits?limit=1&stat=false&verification=false&files=false&sha="+
|
||||||
|
url.QueryEscape(ref), &commits); err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
if len(commits.Result) == 0 || commits.Result[0].SHA == "" {
|
||||||
|
return "", "", errors.New("the forge named no commit for it")
|
||||||
|
}
|
||||||
|
var found struct {
|
||||||
|
Result struct {
|
||||||
|
DefaultBranch string `json:"default_branch"`
|
||||||
|
} `json:"result"`
|
||||||
|
}
|
||||||
|
if err := ask("/repos/"+owner+"/"+repo, &found); err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
return commits.Result[0].SHA, found.Result.DefaultBranch, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// tryCommand is `try <owner>/<repository> <ref> [--base <branch>] [--tests <pattern>] [--store-only]`, and
|
||||||
|
// `try --id <id>` to read what came of one.
|
||||||
|
func tryCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("try", flag.ContinueOnError)
|
||||||
|
base := set.String("base", "", "the branch it is merged into (the repository's default branch when none)")
|
||||||
|
tests := set.String("tests", "", "only the tests this `go test -run` pattern names")
|
||||||
|
storeOnly := set.Bool("store-only", false, "only the repository's store tests")
|
||||||
|
id := set.String("id", "", "a try's id: what came of it")
|
||||||
|
positionals, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
usage := errors.New("try <owner>/<repository> <branch or commit> [--base <branch>] [--tests <pattern>] " +
|
||||||
|
"[--store-only], or try --id <id>")
|
||||||
|
if *id != "" {
|
||||||
|
if len(positionals) != 0 {
|
||||||
|
return usage
|
||||||
|
}
|
||||||
|
// Before it is a token of a bus subject: an id, never a pattern.
|
||||||
|
if !link.SafeBuildID(*id) {
|
||||||
|
return fmt.Errorf("%q is not a try's id", *id)
|
||||||
|
}
|
||||||
|
return tryRead(ctx, *id)
|
||||||
|
}
|
||||||
|
if len(positionals) != 2 {
|
||||||
|
return usage
|
||||||
|
}
|
||||||
|
owner, repo, isPath := strings.Cut(positionals[0], "/")
|
||||||
|
ref := positionals[1]
|
||||||
|
switch {
|
||||||
|
case !isPath || !tryRef.MatchString(owner) || !tryRef.MatchString(repo) || strings.Contains(repo, "/"):
|
||||||
|
return fmt.Errorf("%q is not a repository on the forge, owner/name; nothing was asked", positionals[0])
|
||||||
|
case !tryRef.MatchString(ref):
|
||||||
|
return fmt.Errorf("%q is not a branch or a commit; nothing was asked", ref)
|
||||||
|
case *base != "" && !tryRef.MatchString(*base):
|
||||||
|
return fmt.Errorf("%q is not a branch; nothing was asked", *base)
|
||||||
|
case *tests != "" && !tryTests.MatchString(*tests):
|
||||||
|
return fmt.Errorf("%q is not a `go test -run` pattern of test names; nothing was asked", *tests)
|
||||||
|
}
|
||||||
|
// **An unknown ref is refused here, loudly** — not sent to a machine to fail at its clone minutes later.
|
||||||
|
commit, trunk, err := forgeCommit(ctx, owner, repo, ref)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the forge holds no branch or commit %s in %s/%s — is it pushed? — so nothing was asked: %w",
|
||||||
|
ref, owner, repo, err)
|
||||||
|
}
|
||||||
|
if *base != "" {
|
||||||
|
trunk = *base
|
||||||
|
}
|
||||||
|
if trunk == "" {
|
||||||
|
return fmt.Errorf("the forge named no default branch for %s/%s, and no --base says what %s is merged into; "+
|
||||||
|
"nothing was asked", owner, repo, ref)
|
||||||
|
}
|
||||||
|
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
entries, read, edges, err := theGraph(ctx, open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p := link.PullUpdated{Owner: owner, Repo: repo, Base: trunk, Commit: commit}
|
||||||
|
// Whose repository it is, as a pull request's check reckons it: only the mesh's is tried, the code the
|
||||||
|
// build seat runs for every pull request already.
|
||||||
|
mesh := pullScope(p, entries, read, edges)
|
||||||
|
if !mesh.Mesh {
|
||||||
|
return fmt.Errorf("%s/%s is no repository of the mesh — no module is built from it and its owner is not "+
|
||||||
|
"the core's — and only the mesh's are tried on the build seat; nothing was asked", owner, repo)
|
||||||
|
}
|
||||||
|
request, err := checkRequestFor(ctx, open, p, checkScope{Mesh: true, From: mesh.From}, entries)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
request = asATry(request, *tests, *storeOnly)
|
||||||
|
seat := buildSeatHeld(ctx)
|
||||||
|
// **Bounded in number** (novox/hq issue 686): one try of a commit at a time, and maxTries in all.
|
||||||
|
js, err := dialTheBus()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
queue, err := link.ReadQueue(ctx, js, seat)
|
||||||
|
js.Close()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
same, room := triesIn(queue, owner, repo, commit)
|
||||||
|
if same != "" {
|
||||||
|
return printTry(tryAnswer{ID: same, State: "asked", Repository: owner + "/" + repo, Ref: ref, Commit: commit,
|
||||||
|
On: seat, Said: fmt.Sprintf("a try of %s/%s at %.12s already waits or runs, as %s: nothing new was asked. "+
|
||||||
|
"Read it with try id=%s", owner, repo, commit, same, same)})
|
||||||
|
}
|
||||||
|
if !room {
|
||||||
|
return fmt.Errorf("%d tries already wait or run on the build seat, the most at once (`queue` lists them); "+
|
||||||
|
"nothing was asked — read one of yours, or ask again when one has ended", maxTries)
|
||||||
|
}
|
||||||
|
ask, err := askOverOn(seat)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer ask.Close()
|
||||||
|
if err := ask.Ask(ctx, request); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return printTry(tryAnswer{ID: request.ID, State: "asked", Repository: owner + "/" + repo, Ref: ref, Commit: commit,
|
||||||
|
Base: trunk, On: seat, Said: fmt.Sprintf("asked %s to try %s/%s at %.12s (%s) merged into %s: its own "+
|
||||||
|
"merge-check.sh, as a check in the build queue, bounded at %s; no status is set and nothing is registered. "+
|
||||||
|
"Read it with try id=%s — a full run of this kind takes minutes", seat, owner, repo, commit, ref, trunk,
|
||||||
|
checkTimeout, request.ID)})
|
||||||
|
}
|
||||||
|
|
||||||
|
// tryRead answers what came of a try, holding the answer up to tryWait for its verdict.
|
||||||
|
func tryRead(ctx context.Context, id string) error {
|
||||||
|
until := time.Now().Add(tryWait)
|
||||||
|
for {
|
||||||
|
lines, err := buildLines(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
a := tryAnswerOf(id, lines)
|
||||||
|
if (a.State != "waiting" && a.State != "running") || !time.Now().Add(2*time.Second).Before(until) {
|
||||||
|
return printTry(a)
|
||||||
|
}
|
||||||
|
if err := sleepOrDone(ctx, 2*time.Second); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// tryAnswerOf reads a try's answer from what the build seat said of it: its outcome under StepTried; `failed`
|
||||||
|
// for one that could not run — an error, never a pass; else how far it is.
|
||||||
|
func tryAnswerOf(id string, lines []link.BuildLine) tryAnswer {
|
||||||
|
a := tryAnswer{ID: id, State: "waiting", Said: "nothing is said of " + id + " yet: it waits in the build queue " +
|
||||||
|
"(`queue` says where), or no try has that id; ask again"}
|
||||||
|
for _, l := range lines {
|
||||||
|
switch l.Step {
|
||||||
|
case link.StepTried:
|
||||||
|
var t link.Tried
|
||||||
|
if err := json.Unmarshal([]byte(l.Message), &t); err != nil || t.Verdict == "" {
|
||||||
|
return tryAnswer{ID: id, State: "error", Said: "the build seat's word on it cannot be read: " + l.Message}
|
||||||
|
}
|
||||||
|
return tryAnswer{ID: id, State: t.Verdict, Said: t.Summary, Took: t.Took, Commit: t.Head, Trunk: t.Trunk,
|
||||||
|
Failed: t.Failed}
|
||||||
|
case "failed":
|
||||||
|
return tryAnswer{ID: id, State: "error", Said: "it could not run: " + l.Message}
|
||||||
|
case "checked":
|
||||||
|
// A check that is no try, or a build seat from before the verb: its verdict is not a try's.
|
||||||
|
return tryAnswer{ID: id, State: "error", Said: "the build seat checked it and said no try's outcome — " +
|
||||||
|
"it predates the verb (novox/hq issue 686), or " + id + " is no try: " + l.Message + "; `builds` with " +
|
||||||
|
"the id says every line"}
|
||||||
|
}
|
||||||
|
a.State, a.Last = "running", l.Message
|
||||||
|
a.Said = "still running; ask again. `builds` with the id says every line so far"
|
||||||
|
}
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
|
||||||
|
func printTry(a tryAnswer) error {
|
||||||
|
fmt.Fprintln(os.Stderr, a.Said)
|
||||||
|
for _, l := range a.Failed {
|
||||||
|
fmt.Fprintln(os.Stderr, " "+l)
|
||||||
|
}
|
||||||
|
out, err := json.MarshalIndent(a, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(string(out))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildLines is everything the build seat said of one build, as the bus keeps it, in order.
|
||||||
|
func buildLines(ctx context.Context, id string) ([]link.BuildLine, error) {
|
||||||
|
js, err := aBus()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot reach the bus to read a build's log: %w", err)
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
subject := link.BuildLogOf("*", id)
|
||||||
|
sub, err := js.Context().PullSubscribe(subject, "",
|
||||||
|
nats.BindStream(broker.EventsStream), nats.DeliverAll(), nats.AckNone())
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot read %s from the bus: %w", subject, err)
|
||||||
|
}
|
||||||
|
defer func() { _ = sub.Unsubscribe() }()
|
||||||
|
var lines []link.BuildLine
|
||||||
|
for {
|
||||||
|
batch, err := sub.Fetch(200, nats.MaxWait(2*time.Second))
|
||||||
|
if err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
return nil, fmt.Errorf("reading a build's log: %w", err)
|
||||||
|
}
|
||||||
|
for _, msg := range batch {
|
||||||
|
var line link.BuildLine
|
||||||
|
if json.Unmarshal(msg.Data, &line) == nil {
|
||||||
|
lines = append(lines, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(batch) < 200 {
|
||||||
|
return lines, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,194 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A branch is tried before its pull request (novox/hq issue 686): the repository's own check asked for a pushed
|
||||||
|
// branch or commit, with no pull request.
|
||||||
|
|
||||||
|
// What a try asks is a check — after deliveries in the build queue, bounded as one — of the repository's own
|
||||||
|
// script alone: no module of the graph, so no gate; nothing built for a merge, so no reusable image.
|
||||||
|
func TestATryAsksACheckOfTheRepositorysOwnScriptAlone(t *testing.T) {
|
||||||
|
request := link.BuildRequest{ID: "build-1", Ref: "c0ffee", Check: &link.CheckRequest{Owner: "novox", Repo: "a",
|
||||||
|
Number: 7, Base: "main", Modules: []string{"a"}, New: []string{"b"}, Manifests: []string{"module.json"},
|
||||||
|
Dependents: []string{"c"}, Builds: []link.CheckBuild{{Module: "a"}}, Judge: link.JudgeSelf,
|
||||||
|
Plan: &link.ChangePlan{}}}
|
||||||
|
tried := asATry(request, "^TestStore", true)
|
||||||
|
c := tried.Check
|
||||||
|
if !c.Try || c.Number != 0 || c.Tests != "^TestStore" || !c.StoreOnly {
|
||||||
|
t.Fatalf("not marked a try, or it kept a pull request's number: %+v", c)
|
||||||
|
}
|
||||||
|
if len(c.Modules)+len(c.New)+len(c.Manifests)+len(c.Dependents)+len(c.Builds) != 0 || c.Judge != "" || c.Plan != nil {
|
||||||
|
t.Fatalf("a try runs the gate or builds for a merge: %+v", c)
|
||||||
|
}
|
||||||
|
if c.Base != "main" || c.Owner != "novox" || tried.Ref != "c0ffee" {
|
||||||
|
t.Fatalf("a try lost what it merges into what: %+v", tried)
|
||||||
|
}
|
||||||
|
if link.KindOf(tried) != link.KindCheck {
|
||||||
|
t.Fatal("a try does not count as a check in the build queue")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A try's outcome is heard and left there: no `checked` is said, so the forge's holder sets no status, and
|
||||||
|
// nothing is recorded or registered (the handler holds no store here, and would fail reaching for one).
|
||||||
|
func TestATrysOutcomeSetsNoStatusAndRegistersNothing(t *testing.T) {
|
||||||
|
said := &saidOnTheBus{}
|
||||||
|
checkEvents = said
|
||||||
|
defer func() { checkEvents = nil }()
|
||||||
|
for name, result := range map[string]link.BuildResult{
|
||||||
|
"a try": {ID: "build-1", Ref: "c0ffee", Checked: &link.CheckRequest{Owner: "novox", Repo: "a", Try: true},
|
||||||
|
Check: &link.CheckOutcome{Verdict: "pass", RepoCheck: &link.CheckLayer{Verdict: "fail", Summary: "failed"}}},
|
||||||
|
"a try that could not run": {ID: "build-2", Checked: &link.CheckRequest{Owner: "novox", Repo: "a", Try: true},
|
||||||
|
Failed: "cannot clone"},
|
||||||
|
// A build seat from before the mark echoes the ask without it: a check of no pull request and no
|
||||||
|
// group is still nobody's status.
|
||||||
|
"a try echoed by an older build seat": {ID: "build-3", Checked: &link.CheckRequest{Owner: "novox", Repo: "a"},
|
||||||
|
Check: &link.CheckOutcome{Verdict: "pass"}},
|
||||||
|
} {
|
||||||
|
if err := (builds{}).Built(context.Background(), result); err != nil {
|
||||||
|
t.Fatalf("%s: %v", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(said.events) != 0 {
|
||||||
|
t.Fatalf("a try's outcome was said for the forge: %v", said.events)
|
||||||
|
}
|
||||||
|
// A pull request's check is still said.
|
||||||
|
_ = (builds{}).Built(context.Background(), link.BuildResult{ID: "build-4", Checked: &link.CheckRequest{
|
||||||
|
Owner: "novox", Repo: "a", Number: 7}, Check: &link.CheckOutcome{Verdict: "pass"}})
|
||||||
|
if len(said.events) != 1 {
|
||||||
|
t.Fatalf("a pull request's check was said %d time(s)", len(said.events))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// saidOnTheBus keeps the seat events said, and nothing else.
|
||||||
|
type saidOnTheBus struct {
|
||||||
|
link.Bus
|
||||||
|
events []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *saidOnTheBus) PublishSeatEvent(_ context.Context, seat, event string, _ []byte) error {
|
||||||
|
s.events = append(s.events, seat+" "+event)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// A ref the forge does not hold is refused before anything is asked, naming it.
|
||||||
|
func TestATryOfAnUnknownRefIsRefusedLoudly(t *testing.T) {
|
||||||
|
was := forgeCommit
|
||||||
|
defer func() { forgeCommit = was }()
|
||||||
|
forgeCommit = func(_ context.Context, owner, repo, ref string) (string, string, error) {
|
||||||
|
return "", "", errors.New("404 object does not exist [id: " + ref + "]")
|
||||||
|
}
|
||||||
|
err := tryCommand(context.Background(), []string{"novox/mesh-controller", "fix/no-such-branch"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a try of a ref the forge does not hold was asked")
|
||||||
|
}
|
||||||
|
for _, want := range []string{"fix/no-such-branch", "novox/mesh-controller", "nothing was asked", "404"} {
|
||||||
|
if !strings.Contains(err.Error(), want) {
|
||||||
|
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, args := range [][]string{{"mesh-controller", "main"}, {"novox/mesh-controller"}, {"novox/a", "--upload-pack=x"},
|
||||||
|
{"novox/a", "main", "--tests", "x; rm -rf /"}, {"novox/a", "main", "--id", "build-1"}} {
|
||||||
|
if err := tryCommand(context.Background(), args); err == nil {
|
||||||
|
t.Errorf("try %v was taken", args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The answer of a try carries the lines that name what failed (novox/hq issue 460's shape) and how long it took.
|
||||||
|
func TestATrysAnswerCarriesTheFailingLines(t *testing.T) {
|
||||||
|
tried, _ := json.Marshal(link.Tried{Verdict: "fail", Summary: "its merge-check.sh failed: TestX", Took: "5m3s",
|
||||||
|
Head: "c0ffee", Trunk: "ca3f43c", Failed: []string{"--- FAIL: TestX (0.01s)", " x_test.go:12: got 1, want 2",
|
||||||
|
"FAIL\tgithub.com/novox/a/cmd\t3.1s"}})
|
||||||
|
lines := []link.BuildLine{{Seq: 1, Step: "check", Message: "novox/a#0 at c0ffee"},
|
||||||
|
{Seq: 2, Step: "output", Message: "ok github.com/novox/a/internal 1.2s"},
|
||||||
|
{Seq: 3, Step: link.StepTried, Message: string(tried)}, {Seq: 4, Step: "checked", Message: "pass: x"}}
|
||||||
|
a := tryAnswerOf("build-1", lines)
|
||||||
|
if a.State != "fail" || a.Took != "5m3s" || len(a.Failed) != 3 || a.Failed[0] != "--- FAIL: TestX (0.01s)" ||
|
||||||
|
a.Trunk != "ca3f43c" || a.ID != "build-1" {
|
||||||
|
t.Fatalf("the answer lost the verdict, the time or the failing lines: %+v", a)
|
||||||
|
}
|
||||||
|
// Not answered yet: said as running, with how far it is — never as a pass.
|
||||||
|
if a := tryAnswerOf("build-1", lines[:2]); a.State != "running" || a.Last == "" {
|
||||||
|
t.Fatalf("a try still running: %+v", a)
|
||||||
|
}
|
||||||
|
if a := tryAnswerOf("build-1", nil); a.State != "waiting" {
|
||||||
|
t.Fatalf("a try nothing was said of: %+v", a)
|
||||||
|
}
|
||||||
|
// One that could not run is an error with the build seat's word, never a pass.
|
||||||
|
if a := tryAnswerOf("build-1", append(lines[:1:1], link.BuildLine{Seq: 2, Step: "failed",
|
||||||
|
Message: "cannot clone"})); a.State != "error" || !strings.Contains(a.Said, "cannot clone") {
|
||||||
|
t.Fatalf("a try that could not run: %+v", a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// One try of a commit at a time, and four in all (novox/hq issue 686): a second ask is answered the first's id,
|
||||||
|
// a fifth is refused; a try that ended, and a pull request's check, count for nothing.
|
||||||
|
func TestTriesAreBoundedInNumber(t *testing.T) {
|
||||||
|
ask := func(id, repo, ref, state string, try bool) link.QueuedAsk {
|
||||||
|
return link.QueuedAsk{ID: id, State: state, Request: link.BuildRequest{ID: id, Ref: ref,
|
||||||
|
Check: &link.CheckRequest{Owner: "novox", Repo: repo, Try: try}}}
|
||||||
|
}
|
||||||
|
q := link.Queue{Asks: []link.QueuedAsk{
|
||||||
|
ask("t1", "a", "c1", link.AskInFlight, true), ask("t2", "a", "c2", link.AskWaiting, true),
|
||||||
|
ask("t3", "b", "c1", link.AskDead, true), ask("p1", "a", "c9", link.AskWaiting, false)}}
|
||||||
|
if same, room := triesIn(q, "novox", "a", "c2"); same != "t2" || !room {
|
||||||
|
t.Fatalf("a second try of a commit: %q %v", same, room)
|
||||||
|
}
|
||||||
|
if same, room := triesIn(q, "novox", "b", "c1"); same != "" || !room {
|
||||||
|
t.Fatalf("a dead try, or another repository's, was taken for this one: %q %v", same, room)
|
||||||
|
}
|
||||||
|
if same, _ := triesIn(q, "novox", "a", "c9"); same != "" {
|
||||||
|
t.Fatalf("a pull request's check was taken for a try: %q", same)
|
||||||
|
}
|
||||||
|
q.Asks = append(q.Asks, ask("t4", "c", "c3", link.AskWaiting, true), ask("t5", "d", "c4", link.AskWaiting, true))
|
||||||
|
if _, room := triesIn(q, "novox", "e", "c5"); room {
|
||||||
|
t.Fatal("a fifth try was let in")
|
||||||
|
}
|
||||||
|
// An id is checked before it is a token of a bus subject.
|
||||||
|
for _, id := range []string{"x.>", "*", "a b"} {
|
||||||
|
if err := tryCommand(context.Background(), []string{"--id", id}); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "not a try's id") {
|
||||||
|
t.Errorf("try --id %q: %v", id, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The verb runs its command, and is declared with the shell habit it replaces (novox/hq ADR 0245).
|
||||||
|
func TestTheTryVerbRunsItsCommand(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
args map[string]any
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{map[string]any{"repository": "novox/a", "ref": "fix/x"}, []string{"try", "novox/a", "fix/x"}},
|
||||||
|
{map[string]any{"repository": "novox/a", "ref": "fix/x", "base": "main", "tests": "^TestX$", "store-only": "true"},
|
||||||
|
[]string{"try", "novox/a", "fix/x", "--base", "main", "--tests", "^TestX$", "--store-only"}},
|
||||||
|
{map[string]any{"id": "build-1"}, []string{"try", "--id", "build-1"}},
|
||||||
|
} {
|
||||||
|
got, err := argvFor("try", c.args)
|
||||||
|
if err != nil || !reflect.DeepEqual(got, c.want) {
|
||||||
|
t.Errorf("try %v: %v %v, want %v", c.args, got, err, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := argvFor("try", map[string]any{"repository": "novox/a"}); err == nil {
|
||||||
|
t.Error("a try of no ref was taken")
|
||||||
|
}
|
||||||
|
for _, v := range catalogue.ControllerVerbs {
|
||||||
|
if v.Name == "try" {
|
||||||
|
if len(v.Replaces) == 0 {
|
||||||
|
t.Error("try names no shell habit it replaces")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Error("try is not a verb of the controller seat")
|
||||||
|
}
|
||||||
@@ -160,7 +160,7 @@ func TestTheVerbsCarryReadReplaceAndMove(t *testing.T) {
|
|||||||
{"settings", map[string]any{"module": "plex", "history": "true"}, []string{"settings", "show", "plex", "--history"}},
|
{"settings", map[string]any{"module": "plex", "history": "true"}, []string{"settings", "show", "plex", "--history"}},
|
||||||
{"settings", map[string]any{"module": "plex", "values": "{}", "replace": "true"}, []string{"settings", "set", "plex", "{}", "--replace"}},
|
{"settings", map[string]any{"module": "plex", "values": "{}", "replace": "true"}, []string{"settings", "set", "plex", "{}", "--replace"}},
|
||||||
{"push", map[string]any{"node": "home", "move": "plex", "why": "w"},
|
{"push", map[string]any{"node": "home", "move": "plex", "why": "w"},
|
||||||
[]string{"push", "home", "--wait", "0", "--move", "plex", "--why", "w"}},
|
[]string{"push", "home", "--wait", "0s", "--move", "plex", "--why", "w"}},
|
||||||
{"push", map[string]any{"why": "w"}, []string{"push", "--behind", "--wait", "0", "--why", "w"}},
|
{"push", map[string]any{"why": "w"}, []string{"push", "--behind", "--wait", "0", "--why", "w"}},
|
||||||
{"plan", map[string]any{"node": "home", "diff": "true"}, []string{"plan", "home", "--diff"}},
|
{"plan", map[string]any{"node": "home", "diff": "true"}, []string{"plan", "home", "--diff"}},
|
||||||
} {
|
} {
|
||||||
|
|||||||
@@ -23,7 +23,12 @@ import (
|
|||||||
// **It holds the stores, not a copy of the decision.** What to do about an upgrade is read when
|
// **It holds the stores, not a copy of the decision.** What to do about an upgrade is read when
|
||||||
// one arrives, so changing it takes effect on the next upgrade rather than on the next restart of
|
// one arrives, so changing it takes effect on the next upgrade rather than on the next restart of
|
||||||
// the control plane.
|
// the control plane.
|
||||||
type following struct{ open *stores }
|
type following struct {
|
||||||
|
open *stores
|
||||||
|
// oneShot is a process that ends after the call (a recheck through `delivery check`): what it would do beside
|
||||||
|
// the call it does within it, or the process's exit would cut it off (novox/hq issue 726).
|
||||||
|
oneShot bool
|
||||||
|
}
|
||||||
|
|
||||||
// Upgraded sends the machines running a module the version the catalogue now considers current —
|
// Upgraded sends the machines running a module the version the catalogue now considers current —
|
||||||
// or records that they are behind, which is the default and needs no record.
|
// or records that they are behind, which is the default and needs no record.
|
||||||
@@ -79,7 +84,7 @@ func (f following) Upgraded(ctx context.Context, u link.Upgraded) error {
|
|||||||
// `rebuild`, a `replay` registered — was sent here one machine after another without any judging; it
|
// `rebuild`, a `replay` registered — was sent here one machine after another without any judging; it
|
||||||
// now waits for a gate like any other build, and the release plan walks it through the machines, one
|
// now waits for a gate like any other build, and the release plan walks it through the machines, one
|
||||||
// at a time, each judged.
|
// at a time, each judged.
|
||||||
fmt.Printf("%s moved to %s outside a plan; a release plan sends it to %s, one machine at a time, each judged "+
|
fmt.Printf("%s moved to %s outside a plan; a release plan sends it to %s, one machine first and then the rest at once, each judged "+
|
||||||
"(`upgrade backlog` lists what waits)\n", u.Module, shortCommit(u.Commit), readableList(on))
|
"(`upgrade backlog` lists what waits)\n", u.Module, shortCommit(u.Commit), readableList(on))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,91 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A module may declare its unhealthy urgent (novox/hq ADR 0355): its module-unhealthy condition is then raised
|
||||||
|
// urgent; without the field it stays a warning; a wait for a new login stays a warning whatever the module says.
|
||||||
|
|
||||||
|
// judgedTwice judges one machine's statement twice, over a catalogue in which mail declares its unhealthy
|
||||||
|
// urgent and notes says nothing, and returns what is open.
|
||||||
|
func judgedTwice(t *testing.T, unhealthy map[string][]inventory.ResourceHealth) []conditions.Condition {
|
||||||
|
t.Helper()
|
||||||
|
ctx := t.Context()
|
||||||
|
k, _ := withConditionsInMemory(t)
|
||||||
|
was := readHoldingFor
|
||||||
|
t.Cleanup(func() { readHoldingFor = was })
|
||||||
|
readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
||||||
|
return &holding{ctx: context.Background(), open: open, providers: map[string]providerLookup{},
|
||||||
|
shelf: map[string]catalogue.Manifest{
|
||||||
|
"mail": {Module: "mail", Version: "1", Unhealthy: &catalogue.UnhealthyPolicy{
|
||||||
|
Severity: catalogue.UnhealthyUrgent, Why: "mail is refused or stuck for everyone with a mailbox"}},
|
||||||
|
"notes": {Module: "notes", Version: "1"},
|
||||||
|
}}, nil
|
||||||
|
}
|
||||||
|
streaks := map[string]int{}
|
||||||
|
for look := 1; look <= 2; look++ {
|
||||||
|
for m := range unhealthy {
|
||||||
|
streaks[m] = look
|
||||||
|
}
|
||||||
|
if err := judgeModuleHealth(ctx, nil, k, "anchor", unhealthy, streaks, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
open, err := k.Open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return open
|
||||||
|
}
|
||||||
|
|
||||||
|
func containerDown(module string) inventory.ResourceHealth {
|
||||||
|
return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
|
||||||
|
State: link.StateUnhealthy, Reason: "down"}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAModuleThatDeclaresItsUnhealthyUrgentIsRaisedUrgent(t *testing.T) {
|
||||||
|
open := judgedTwice(t, map[string][]inventory.ResourceHealth{"mail": {containerDown("mail")}})
|
||||||
|
c := conditionOf(open, "module.mail.anchor.unhealthy")
|
||||||
|
if c == nil {
|
||||||
|
t.Fatalf("mail's unhealthy condition was not raised: %+v", open)
|
||||||
|
}
|
||||||
|
if c.Severity != conditions.Urgent {
|
||||||
|
t.Errorf("mail declares its unhealthy urgent, and it was raised %s", c.Severity)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAModuleThatSaysNothingOfItsUnhealthyStaysAWarning(t *testing.T) {
|
||||||
|
open := judgedTwice(t, map[string][]inventory.ResourceHealth{"notes": {containerDown("notes")}})
|
||||||
|
c := conditionOf(open, "module.notes.anchor.unhealthy")
|
||||||
|
if c == nil {
|
||||||
|
t.Fatalf("notes' unhealthy condition was not raised: %+v", open)
|
||||||
|
}
|
||||||
|
if c.Severity != conditions.Warning {
|
||||||
|
t.Errorf("notes says nothing of its unhealthy, and it was raised %s", c.Severity)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only the module-unhealthy condition changes (ADR 0355 §4): a wait for a new login (ADR 0254) of a module that
|
||||||
|
// declares its unhealthy urgent stays a warning.
|
||||||
|
func TestAWaitForANewLoginStaysAWarningWhenTheModuleDeclaresItsUnhealthyUrgent(t *testing.T) {
|
||||||
|
open := judgedTwice(t, map[string][]inventory.ResourceHealth{"mail": {relogin("mail", "operator"),
|
||||||
|
userUnit("mail", "operator")}})
|
||||||
|
c := conditionOf(open, "module.mail.anchor.relogin-needed")
|
||||||
|
if c == nil {
|
||||||
|
t.Fatalf("mail's relogin-needed condition was not raised: %+v", open)
|
||||||
|
}
|
||||||
|
if c.Severity != conditions.Warning {
|
||||||
|
t.Errorf("a wait for a new login was raised %s; want a warning", c.Severity)
|
||||||
|
}
|
||||||
|
if conditionOf(open, "module.mail.anchor.unhealthy") != nil {
|
||||||
|
t.Errorf("a wait for a new login raised mail's unhealthy condition beside it: %+v", open)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -177,6 +177,12 @@ func walkReached(p inventory.Plan, running func(string) []string) map[string][]s
|
|||||||
reached = appendNew(reached, r.Gate.Machines...)
|
reached = appendNew(reached, r.Gate.Machines...)
|
||||||
carried(r.Gate)
|
carried(r.Gate)
|
||||||
}
|
}
|
||||||
|
for _, g := range r.Gates {
|
||||||
|
if g != nil && !g.Pending {
|
||||||
|
reached = appendNew(reached, g.Machines...)
|
||||||
|
carried(g)
|
||||||
|
}
|
||||||
|
}
|
||||||
for _, tier := range p.Tiers {
|
for _, tier := range p.Tiers {
|
||||||
for _, m := range tier {
|
for _, m := range tier {
|
||||||
for _, n := range running(m) {
|
for _, n := range running(m) {
|
||||||
|
|||||||
@@ -92,6 +92,8 @@ type signalFacts struct {
|
|||||||
// consumers: the condition counts the messages given up on, not the looks (novox/hq issue 440).
|
// consumers: the condition counts the messages given up on, not the looks (novox/hq issue 440).
|
||||||
deadLettersNewest map[string]time.Time
|
deadLettersNewest map[string]time.Time
|
||||||
advisoriesErr error
|
advisoriesErr error
|
||||||
|
// busFaultsErr is why the bus module's events could not be read this tick (novox/hq ADR 0357).
|
||||||
|
busFaultsErr error
|
||||||
|
|
||||||
selfCheck selfCheckFacts
|
selfCheck selfCheckFacts
|
||||||
|
|
||||||
@@ -358,6 +360,15 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
|
|||||||
}
|
}
|
||||||
f.advisories = link.Advisories.Since(now.Add(-advisoryQuiet))
|
f.advisories = link.Advisories.Since(now.Add(-advisoryQuiet))
|
||||||
f.lostConsumers, f.advisoriesErr = w.lostConsumers(ctx, f.advisories)
|
f.lostConsumers, f.advisoriesErr = w.lostConsumers(ctx, f.advisories)
|
||||||
|
// And what the bus did to every other user's connection, as the bus module said it (novox/hq ADR 0357):
|
||||||
|
// read back for the same hour, and one advisory with the controller's own where both are its.
|
||||||
|
// A read that fails is its own condition, never S9 blind: the controller's own signals are still said.
|
||||||
|
if w.server != nil && w.js != nil {
|
||||||
|
var faults []link.Advisory
|
||||||
|
if faults, f.busFaultsErr = w.server.BusFaults(ctx, now.Add(-advisoryQuiet)); f.busFaultsErr == nil {
|
||||||
|
f.advisories = link.MergeAdvisories(f.advisories, faults)
|
||||||
|
}
|
||||||
|
}
|
||||||
if f.advisoriesErr == nil && w.js != nil {
|
if f.advisoriesErr == nil && w.js != nil {
|
||||||
f.deadLetters, f.advisoriesErr = link.HeldDeadLetters(w.js.Context())
|
f.deadLetters, f.advisoriesErr = link.HeldDeadLetters(w.js.Context())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -38,8 +38,9 @@ func TestTheBuildSeatsHolderFollowsTheControllerThatDefinesItsWorker(t *testing.
|
|||||||
if pos["build-agent"] <= pos["mesh-controller"] {
|
if pos["build-agent"] <= pos["mesh-controller"] {
|
||||||
t.Fatalf("the build machine after the controller that defines its worker: %v", tiers)
|
t.Fatalf("the build machine after the controller that defines its worker: %v", tiers)
|
||||||
}
|
}
|
||||||
if pos["route-proxy"] <= pos["build-agent"] {
|
// Built-by orders nothing (novox/hq ADR 0360 §3): whichever builder runs builds the proxy, beside the controller.
|
||||||
t.Fatalf("what the build machine builds comes after it: %v", tiers)
|
if pos["route-proxy"] != 0 {
|
||||||
|
t.Fatalf("what the builder builds waits for a new one: %v", tiers)
|
||||||
}
|
}
|
||||||
if hasCycle(tiers, edges) {
|
if hasCycle(tiers, edges) {
|
||||||
t.Fatalf("no cycle here: %v", tiers)
|
t.Fatalf("no cycle here: %v", tiers)
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ module github.com/novox/mesh-controller
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44
|
||||||
github.com/jackc/pgx/v5 v5.10.0
|
github.com/jackc/pgx/v5 v5.10.0
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
@@ -36,4 +36,4 @@ require (
|
|||||||
// committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and
|
// committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and
|
||||||
// fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without
|
// fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without
|
||||||
// `go mod vendor` fails loudly, at once, everywhere.
|
// `go mod vendor` fails loudly, at once, everywhere.
|
||||||
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f
|
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261013220009-b9a293f8c8bc
|
||||||
|
|||||||
@@ -1,7 +1,13 @@
|
|||||||
git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f h1:8N5OW2mdTNIck2pe4EciTYX5NsrPsHrTLENGNIWYNTU=
|
git.novox.be/novox/mesh-host v0.0.0-20261013220009-b9a293f8c8bc h1:h5v2AbdjM/clMY1YxS9p7VuyYtIjkcw85w9Oz1cOYQc=
|
||||||
git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c=
|
git.novox.be/novox/mesh-host v0.0.0-20261013220009-b9a293f8c8bc/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013141520-9588871d6629 h1:7TYFV1Q6Mcz+lsLP/sds87tysET4rKCn3rRIQV8gNmA=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013141520-9588871d6629/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013145622-f4745447ba6d h1:HEJyqorKktrkJPeeJdwEo0kdrJjH3fm/YVS9MGftCqY=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013145622-f4745447ba6d/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44 h1:r5xIo5qWOXQdCSMxvqJy8kV4U+P/QWXFK6971Ha6wU8=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
|||||||
@@ -0,0 +1,91 @@
|
|||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Listing and letting go of tagged documents: the check builds the scratch namespace keeps (novox/hq ADR 0350
|
||||||
|
// §5). A check build is found by its tag, and the build seat's holder lets each go when its pull request closes
|
||||||
|
// or it is seven days old, so the store keeps a week of check builds and no more.
|
||||||
|
|
||||||
|
// Tags is every tag repository holds, in the store's order. None for a repository the store does not hold.
|
||||||
|
func (s Store) Tags(ctx context.Context, repository string) ([]string, error) {
|
||||||
|
if s.Address == "" {
|
||||||
|
return nil, fmt.Errorf("this mesh has no artifact store on its network to list %s in", repository)
|
||||||
|
}
|
||||||
|
var tags []string
|
||||||
|
next := "http://" + s.Address + "/v2/" + repository + "/tags/list?n=1000"
|
||||||
|
for next != "" {
|
||||||
|
request, err := http.NewRequestWithContext(ctx, http.MethodGet, next, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
response, err := s.client().Do(request)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
|
||||||
|
}
|
||||||
|
switch response.StatusCode {
|
||||||
|
case http.StatusOK:
|
||||||
|
case http.StatusNotFound:
|
||||||
|
response.Body.Close()
|
||||||
|
return tags, nil
|
||||||
|
default:
|
||||||
|
response.Body.Close()
|
||||||
|
return nil, fmt.Errorf("the artifact store answered %s for the tags of %s", response.Status, repository)
|
||||||
|
}
|
||||||
|
var page struct {
|
||||||
|
Tags []string `json:"tags"`
|
||||||
|
}
|
||||||
|
err = json.NewDecoder(io.LimitReader(response.Body, 8<<20)).Decode(&page)
|
||||||
|
link := response.Header.Get("Link")
|
||||||
|
response.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the tags of %s could not be read: %w", repository, err)
|
||||||
|
}
|
||||||
|
tags = append(tags, page.Tags...)
|
||||||
|
next = nextPage(link, s.Address)
|
||||||
|
}
|
||||||
|
return tags, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// nextPage is the registry's `Link: </v2/…>; rel="next"`, made whole; empty when there is no next page.
|
||||||
|
func nextPage(link, address string) string {
|
||||||
|
if !strings.Contains(link, `rel="next"`) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
start, end := strings.Index(link, "<"), strings.Index(link, ">")
|
||||||
|
if start < 0 || end <= start+1 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
target := link[start+1 : end]
|
||||||
|
if strings.HasPrefix(target, "/") {
|
||||||
|
return "http://" + address + target
|
||||||
|
}
|
||||||
|
return target
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteTagged lets go of the document tag names in repository: its manifest, by digest, so the next
|
||||||
|
// collection takes its layer. Gone when the tag names nothing.
|
||||||
|
func (s Store) DeleteTagged(ctx context.Context, repository, tag string) error {
|
||||||
|
if s.Address == "" {
|
||||||
|
return fmt.Errorf("this mesh has no artifact store on its network to let go of %s:%s in", repository, tag)
|
||||||
|
}
|
||||||
|
digest, err := s.manifestDigest(ctx, repository, tag)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if digest == "" {
|
||||||
|
return Gone
|
||||||
|
}
|
||||||
|
err = s.remove(ctx, s.url(repository, "manifests", digest), repository+":"+tag)
|
||||||
|
if errors.Is(err, Gone) {
|
||||||
|
return Gone
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **The bus module's watch speaks through the runtime that carries it** (novox/hq ADR 0357): the node's
|
||||||
|
// runtime is granted the events the bus module declares, under the module's name, while the bus's own
|
||||||
|
// account keeps the snapshot API and nothing else (ADR 0235).
|
||||||
|
func TestTheRuntimeCarryingTheBusModuleMayPublishItsEventsAndTheBussAccountMayNot(t *testing.T) {
|
||||||
|
events := []string{"bus.slow-consumer", "bus.connection-closed"}
|
||||||
|
runtime := Principal{Kind: KindNodeTools, Node: "anchor", Module: RuntimeModule,
|
||||||
|
Carries: []Declared{{Module: "nats", Emits: events}}}
|
||||||
|
p, err := PermissionsFor(runtime)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, e := range events {
|
||||||
|
if !slices.Contains(p.Publish, "mesh.mod.nats.event."+e) {
|
||||||
|
t.Errorf("the runtime carrying the bus module may not publish %s: %v", e, p.Publish)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
account := Principal{Kind: KindModule, Node: "anchor", Module: "nats", Emits: events, SnapshotsTheBus: true}
|
||||||
|
a, err := PermissionsFor(account)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, s := range a.Publish {
|
||||||
|
if s == "mesh.mod.nats.event.bus.slow-consumer" || s == "mesh.mod.nats.event.bus.connection-closed" {
|
||||||
|
t.Errorf("the bus's own account is granted %s", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -128,7 +128,13 @@ func ConsumerFor(p Principal) (Consumer, bool) {
|
|||||||
// connected, and its graph stayed empty with nothing anywhere reporting why.
|
// connected, and its graph stayed empty with nothing anywhere reporting why.
|
||||||
// And one that hears its own answers on a seat it uses (novox/hq ADR 0259 §3): an asker's warrants.
|
// And one that hears its own answers on a seat it uses (novox/hq ADR 0259 §3): an asker's warrants.
|
||||||
hearsItsOwn := len(SeatTrafficOf(p.Module, nil, p.Uses, nil).Subscribe) > 0
|
hearsItsOwn := len(SeatTrafficOf(p.Module, nil, p.Uses, nil).Subscribe) > 0
|
||||||
if p.Kind != KindModule || (len(p.Consumes) == 0 && len(p.Watches) == 0 && !hearsItsOwn) {
|
hearsAStream := false
|
||||||
|
for _, h := range p.Hears {
|
||||||
|
if len(h.Filters) > 0 {
|
||||||
|
hearsAStream = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if p.Kind != KindModule || (len(p.Consumes) == 0 && len(p.Watches) == 0 && !hearsItsOwn && !hearsAStream) {
|
||||||
return Consumer{}, false
|
return Consumer{}, false
|
||||||
}
|
}
|
||||||
perms, err := PermissionsFor(p)
|
perms, err := PermissionsFor(p)
|
||||||
@@ -140,11 +146,14 @@ func ConsumerFor(p Principal) (Consumer, bool) {
|
|||||||
// business, which is why this is a filter and not the whole list.
|
// business, which is why this is a filter and not the whole list.
|
||||||
var filters []string
|
var filters []string
|
||||||
for _, s := range perms.Subscribe {
|
for _, s := range perms.Subscribe {
|
||||||
if strings.Contains(s, ".event.") {
|
// Not a module stream's, which its own consumer on that stream hears (novox/hq ADR 0363).
|
||||||
|
if strings.Contains(s, ".event.") && !strings.HasPrefix(s, moduleStreamPrefix) {
|
||||||
filters = append(filters, s)
|
filters = append(filters, s)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(filters) == 0 {
|
// **A module that hears only a module's stream still has one here** (novox/hq ADR 0363 §7): what it
|
||||||
|
// gives up on there is delivered again under `mesh.again.<consumer>.` on EVENTS, and only this filters it.
|
||||||
|
if len(filters) == 0 && !hearsAStream {
|
||||||
return Consumer{}, false
|
return Consumer{}, false
|
||||||
}
|
}
|
||||||
sort.Strings(filters)
|
sort.Strings(filters)
|
||||||
@@ -181,6 +190,13 @@ func ConsumersOf(users []Principal) []ModuleConsumer {
|
|||||||
}
|
}
|
||||||
seen[p.Node+"/"+p.Module] = true
|
seen[p.Node+"/"+p.Module] = true
|
||||||
out = append(out, ModuleConsumer{Node: p.Node, Module: p.Module, Consumer: c})
|
out = append(out, ModuleConsumer{Node: p.Node, Module: p.Module, Consumer: c})
|
||||||
|
// And one on each module stream it hears (novox/hq ADR 0363 §5).
|
||||||
|
for _, h := range p.Hears {
|
||||||
|
if len(h.Filters) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, ModuleConsumer{Node: p.Node, Module: p.Module, Consumer: heardConsumer(p, h)})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
for _, p := range users {
|
for _, p := range users {
|
||||||
if p.Kind == KindModule {
|
if p.Kind == KindModule {
|
||||||
@@ -193,7 +209,8 @@ func ConsumersOf(users []Principal) []ModuleConsumer {
|
|||||||
}
|
}
|
||||||
for _, d := range p.Carries {
|
for _, d := range p.Carries {
|
||||||
add(Principal{Kind: KindModule, Node: p.Node, Module: d.Module, Emits: d.Emits,
|
add(Principal{Kind: KindModule, Node: p.Node, Module: d.Module, Emits: d.Emits,
|
||||||
Consumes: d.Consumes, Serves: d.Serves, Holds: d.Holds, Uses: d.Uses, Watches: d.Watches})
|
Consumes: d.Consumes, Serves: d.Serves, Holds: d.Holds, Uses: d.Uses, Watches: d.Watches,
|
||||||
|
Streams: d.Streams, Hears: d.Hears})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
|
|||||||
@@ -486,3 +486,25 @@ func (j *JetStream) LogStreams() ([]string, error) {
|
|||||||
}
|
}
|
||||||
return out, lister.Err()
|
return out, lister.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ConsumerNames is every durable consumer on a stream; none for a stream that does not exist yet.
|
||||||
|
func (j *JetStream) ConsumerNames(stream string) ([]string, error) {
|
||||||
|
if _, err := j.js.StreamInfo(stream); errors.Is(err, nats.ErrStreamNotFound) {
|
||||||
|
return nil, nil
|
||||||
|
} else if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for name := range j.js.ConsumerNames(stream) {
|
||||||
|
out = append(out, name)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// RemoveConsumer deletes one durable consumer (novox/hq ADR 0363 §6): one the composition no longer asks for.
|
||||||
|
func (j *JetStream) RemoveConsumer(stream, name string) error {
|
||||||
|
if err := j.js.DeleteConsumer(stream, name); err != nil && !errors.Is(err, nats.ErrConsumerNotFound) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -61,6 +61,11 @@ type Membership struct {
|
|||||||
// over every module on the machine, so one module's code reaching another's name or kind through
|
// over every module on the machine, so one module's code reaching another's name or kind through
|
||||||
// the runtime is the runtime's to refuse.
|
// the runtime is the runtime's to refuse.
|
||||||
SeatTraffic *SeatTraffic `json:"seat-traffic,omitempty"`
|
SeatTraffic *SeatTraffic `json:"seat-traffic,omitempty"`
|
||||||
|
// Streams are this module's own streams (novox/hq ADR 0363 §2): an event one of them carries is published
|
||||||
|
// on that stream's subject, never on Emits. Hears are the module streams it hears, each through a durable
|
||||||
|
// consumer on that stream the runtime binds beside the one on EVENTS (§8).
|
||||||
|
Streams []StreamIssued `json:"streams,omitempty"`
|
||||||
|
Hears []HeardIssued `json:"hears,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Served is one address a tool is answered on.
|
// Served is one address a tool is answered on.
|
||||||
@@ -126,6 +131,8 @@ func MembershipFor(node string, d Declared, where Placements) Membership {
|
|||||||
}
|
}
|
||||||
m.State = stateIssuedFor(d, node)
|
m.State = stateIssuedFor(d, node)
|
||||||
m.Logs = logsIssuedFor(d)
|
m.Logs = logsIssuedFor(d)
|
||||||
|
m.Streams = streamsIssuedFor(d)
|
||||||
|
m.Hears = HeardIssuedFor(node, d)
|
||||||
t := SeatTrafficOf(d.Module, d.Holds, d.Uses, d.Watches)
|
t := SeatTrafficOf(d.Module, d.Holds, d.Uses, d.Watches)
|
||||||
for _, s := range append(append([]Seat{}, d.Uses...), d.Watches...) {
|
for _, s := range append(append([]Seat{}, d.Uses...), d.Watches...) {
|
||||||
if !s.Kinded {
|
if !s.Kinded {
|
||||||
|
|||||||
@@ -0,0 +1,243 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A module's own stream on the bus (novox/hq ADR 0363).
|
||||||
|
//
|
||||||
|
// A module declares a stream as it declares a log: named locally, created by the controller from the
|
||||||
|
// catalogue on every raise and every send, never removed by the mesh. Its events are published on
|
||||||
|
// `mesh.stream.<module>.event.<event>`, a subject space the shared EVENTS stream (`mesh.mod.*.event.>`)
|
||||||
|
// does not take — the bus refuses two streams claiming one subject, so a module's stream cannot share
|
||||||
|
// EVENTS' — and a consumer hears it through a durable consumer of its own on that stream, named as on
|
||||||
|
// EVENTS. Pure; the controller's raise asserts what is composed here.
|
||||||
|
|
||||||
|
// moduleStreamPrefix starts every subject a module's own stream takes.
|
||||||
|
const moduleStreamPrefix = "mesh.stream."
|
||||||
|
|
||||||
|
// ModuleStreamSubject is where a module's event on its own stream lands: `mesh.stream.<module>.event.<event>`.
|
||||||
|
// The token before `.event.` is the emitter, as on EVENTS, so a runtime reads the same key from it.
|
||||||
|
func ModuleStreamSubject(module, event string) string {
|
||||||
|
return moduleStreamPrefix + module + ".event." + event
|
||||||
|
}
|
||||||
|
|
||||||
|
// OnAModuleStream says a subject is one a module's own stream takes.
|
||||||
|
func OnAModuleStream(subject string) bool { return strings.HasPrefix(subject, moduleStreamPrefix) }
|
||||||
|
|
||||||
|
// WirePattern is a local event pattern as the bus spells it: `**` is `>`.
|
||||||
|
func WirePattern(local string) string {
|
||||||
|
parts := strings.Split(local, ".")
|
||||||
|
for i, p := range parts {
|
||||||
|
if p == "**" {
|
||||||
|
parts[i] = ">"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.Join(parts, ".")
|
||||||
|
}
|
||||||
|
|
||||||
|
// A ModuleStream is one module's declared stream as the bus holds it.
|
||||||
|
type ModuleStream struct {
|
||||||
|
Module string
|
||||||
|
// Name is its name on the bus, `MAIL`.
|
||||||
|
Name string
|
||||||
|
// Events are the events it carries, as the bus spells them (WirePattern).
|
||||||
|
Events []string
|
||||||
|
MaxAgeDays int
|
||||||
|
MaxMiB int
|
||||||
|
// Restricted says a trusted config value of the module grants who hears it (ADR 0363 §4).
|
||||||
|
Restricted bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Subjects are the subjects the stream takes.
|
||||||
|
func (s ModuleStream) Subjects() []string {
|
||||||
|
var out []string
|
||||||
|
for _, e := range s.Events {
|
||||||
|
out = append(out, ModuleStreamSubject(s.Module, e))
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stream is the stream the controller asserts: kept by limits, its age and size the module's, dropping
|
||||||
|
// its oldest when full — a consumer that was away catches up within the age, and the module's own record
|
||||||
|
// answers for anything older.
|
||||||
|
func (s ModuleStream) Stream() Stream {
|
||||||
|
why := fmt.Sprintf("%s's own stream (novox/hq ADR 0363): its events, kept %d days and at most %d MiB so a "+
|
||||||
|
"consumer that was away catches up", s.Module, s.MaxAgeDays, s.MaxMiB)
|
||||||
|
if s.Restricted {
|
||||||
|
why += "; heard only by the consumers its trusted config value grants"
|
||||||
|
}
|
||||||
|
return Stream{
|
||||||
|
Name: s.Name,
|
||||||
|
Subjects: s.Subjects(),
|
||||||
|
Retention: RetentionLimits,
|
||||||
|
MaxAge: s.MaxAgeDays * 24 * 60 * 60,
|
||||||
|
MaxBytes: int64(s.MaxMiB) * 1024 * 1024,
|
||||||
|
Why: why,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ModuleStreams is every declared module stream as the controller asserts it, in name order.
|
||||||
|
func ModuleStreams(declared []ModuleStream) []Stream {
|
||||||
|
sorted := append([]ModuleStream(nil), declared...)
|
||||||
|
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[j].Name })
|
||||||
|
out := make([]Stream, 0, len(sorted))
|
||||||
|
for _, s := range sorted {
|
||||||
|
out = append(out, s.Stream())
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Heard is a module's stream another module hears, and the subjects it hears of it: its `consumes`
|
||||||
|
// within the stream and, for a restricted stream, within what the provider's trusted config value grants it
|
||||||
|
// (ADR 0363 §5). Composed only with at least one subject.
|
||||||
|
type Heard struct {
|
||||||
|
Stream string
|
||||||
|
Emitter string
|
||||||
|
Filters []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// heardConsumer is the durable consumer a module hears a module's stream through: named as its consumer
|
||||||
|
// on EVENTS, on that stream, with the subjects it was granted as its filters.
|
||||||
|
func heardConsumer(p Principal, h Heard) Consumer {
|
||||||
|
filters := append([]string(nil), h.Filters...)
|
||||||
|
sort.Strings(filters)
|
||||||
|
return Consumer{
|
||||||
|
Name: consumerDurable(p),
|
||||||
|
Stream: h.Stream,
|
||||||
|
Filters: filters,
|
||||||
|
AckWaitSeconds: 30,
|
||||||
|
MaxDeliver: 5,
|
||||||
|
Why: "what " + p.Module + " consumes of " + h.Emitter + "'s stream " + h.Stream + " and is granted " +
|
||||||
|
"(novox/hq ADR 0363); after max-deliver it gives an event up, and the controller keeps it in " +
|
||||||
|
"DEAD_LETTERS until a person delivers it again or drops it",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// heardGrants are what a principal publishes to read and acknowledge the consumers it hears module streams
|
||||||
|
// through: asking about each, pulling from it, acknowledging it — its own name on each stream, no other's.
|
||||||
|
func heardGrants(p Principal, hears []Heard) []string {
|
||||||
|
var out []string
|
||||||
|
durable := consumerDurable(p)
|
||||||
|
for _, h := range hears {
|
||||||
|
if !safeSubject.MatchString(h.Stream) || len(h.Filters) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out,
|
||||||
|
"$JS.API.CONSUMER.INFO."+h.Stream+"."+durable,
|
||||||
|
"$JS.API.CONSUMER.MSG.NEXT."+h.Stream+"."+durable,
|
||||||
|
"$JS.ACK."+h.Stream+"."+durable+".>")
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// ownStreamGrants are the subjects a module publishes its own streams' events on, and no other's.
|
||||||
|
func ownStreamGrants(module string, streams []ModuleStream) []string {
|
||||||
|
var out []string
|
||||||
|
for _, s := range streams {
|
||||||
|
if s.Module != module {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, s.Subjects()...)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// StreamIssued is one of a module's own streams as its membership names it (ADR 0363 §2): the events it
|
||||||
|
// carries, as the bus spells them, and the subject an event of them is published on, `{event}` the event.
|
||||||
|
type StreamIssued struct {
|
||||||
|
Stream string `json:"stream"`
|
||||||
|
Events []string `json:"events"`
|
||||||
|
Subject string `json:"subject"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// HeardIssued is one module stream an assignment hears, as its membership names it (ADR 0363 §8): the
|
||||||
|
// stream and the durable consumer on it the runtime binds.
|
||||||
|
type HeardIssued struct {
|
||||||
|
Stream string `json:"stream"`
|
||||||
|
Consumer string `json:"consumer"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// streamsIssuedFor is every stream of its own a module's membership names.
|
||||||
|
func streamsIssuedFor(d Declared) []StreamIssued {
|
||||||
|
var out []StreamIssued
|
||||||
|
for _, s := range d.Streams {
|
||||||
|
if s.Module != d.Module || !safeSubject.MatchString(d.Module) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, StreamIssued{Stream: s.Name, Events: append([]string(nil), s.Events...),
|
||||||
|
Subject: ModuleStreamSubject(d.Module, "{event}")})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// HeardIssuedFor is every module stream an assignment hears, as its membership names it.
|
||||||
|
func HeardIssuedFor(node string, d Declared) []HeardIssued {
|
||||||
|
var out []HeardIssued
|
||||||
|
for _, h := range d.Hears {
|
||||||
|
if len(h.Filters) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, HeardIssued{Stream: h.Stream, Consumer: node + "_" + d.Module})
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool { return out[i].Stream < out[j].Stream })
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pruner is the part of a JetStream connection that removes a module's consumer on a module's stream it
|
||||||
|
// may no longer hear (ADR 0363 §6). Optional: what only records what it is asked to assert has none.
|
||||||
|
type Pruner interface {
|
||||||
|
ConsumerNames(stream string) ([]string, error)
|
||||||
|
RemoveConsumer(stream, name string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// PruneHeard removes, from each module stream, every durable consumer the composition no longer asks for,
|
||||||
|
// and answers what it removed as `<stream>/<consumer>`. A consumer that loses its grant loses what it reads
|
||||||
|
// through, whatever its runtime still holds; a stream that does not exist yet has nothing to remove.
|
||||||
|
func PruneHeard(p Pruner, streams []ModuleStream, wanted []Consumer) ([]string, error) {
|
||||||
|
keep := map[string]bool{}
|
||||||
|
for _, c := range wanted {
|
||||||
|
keep[c.Stream+"/"+c.Name] = true
|
||||||
|
}
|
||||||
|
var removed []string
|
||||||
|
for _, s := range streams {
|
||||||
|
names, err := p.ConsumerNames(s.Name)
|
||||||
|
if err != nil {
|
||||||
|
return removed, fmt.Errorf("listing the consumers on %s: %w", s.Name, err)
|
||||||
|
}
|
||||||
|
for _, n := range names {
|
||||||
|
if keep[s.Name+"/"+n] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := p.RemoveConsumer(s.Name, n); err != nil {
|
||||||
|
return removed, fmt.Errorf("removing %s's consumer on %s, which is no longer granted: %w", n, s.Name, err)
|
||||||
|
}
|
||||||
|
removed = append(removed, s.Name+"/"+n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return removed, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// EnsureHeard asserts a consumer on a module's stream (ADR 0363 §5, §6). **On a restricted stream, a consumer
|
||||||
|
// the bus will not bring to match is removed**: a pull consumer delivers to whoever pulls it, so one left with
|
||||||
|
// its wider filters after a narrowing was refused would keep handing over what the grant no longer names,
|
||||||
|
// whatever the user list now says. The next assertion makes it again with the granted filters. The refusal is
|
||||||
|
// returned either way, saying what was done, so the send says it and raises it.
|
||||||
|
func EnsureHeard(r Raiser, c Consumer, restricted bool) error {
|
||||||
|
err := r.EnsureConsumer(c)
|
||||||
|
if err == nil || !restricted {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p, prunes := r.(Pruner)
|
||||||
|
if !prunes {
|
||||||
|
return fmt.Errorf("%w; it could not be removed here, and may still deliver more than it is granted", err)
|
||||||
|
}
|
||||||
|
if rerr := p.RemoveConsumer(c.Stream, c.Name); rerr != nil {
|
||||||
|
return fmt.Errorf("%w; REMOVING IT FAILED TOO, so it may still deliver more than it is granted: %v", err, rerr)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%w; REMOVED, so it delivers nothing it is no longer granted: the next assertion makes it "+
|
||||||
|
"again with its granted filters (novox/hq ADR 0363 §6)", err)
|
||||||
|
}
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/testbus"
|
||||||
|
)
|
||||||
|
|
||||||
|
// On a real bus: a module's stream is made beside EVENTS without a clash, keeps what the module publishes on
|
||||||
|
// its own subjects, a granted consumer hears only its part, a narrowed grant narrows it, and a withdrawn one
|
||||||
|
// is removed (novox/hq ADR 0363).
|
||||||
|
//
|
||||||
|
// go test ./internal/broker/ -run TestAModuleStreamOnARealBus
|
||||||
|
func TestAModuleStreamOnARealBus(t *testing.T) {
|
||||||
|
js, err := Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
for _, s := range []Stream{MeshStreams()[3], mailStream.Stream()} {
|
||||||
|
_ = js.js.DeleteStream(s.Name)
|
||||||
|
if err := js.EnsureStream(s); err != nil {
|
||||||
|
t.Fatalf("%s: %v", s.Name, err)
|
||||||
|
}
|
||||||
|
defer func(name string) { _ = js.js.DeleteStream(name) }(s.Name)
|
||||||
|
}
|
||||||
|
for _, account := range []string{"mesh", "work"} {
|
||||||
|
if _, err := js.js.Publish(ModuleStreamSubject("mesh-mail", "mail."+account+".message.added"), []byte("{}")); err != nil {
|
||||||
|
t.Fatalf("publishing %s's mail: %v", account, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if info, _ := js.js.StreamInfo(EventsStream); info.State.Msgs != 0 {
|
||||||
|
t.Fatalf("EVENTS kept %d of the module stream's events", info.State.Msgs)
|
||||||
|
}
|
||||||
|
|
||||||
|
reader := Principal{Kind: KindModule, Node: "ace", Module: "reader", Hears: []Heard{mailHeard}}
|
||||||
|
c := heardConsumer(reader, mailHeard)
|
||||||
|
if err := js.EnsureConsumer(c); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
if info, _ := js.js.ConsumerInfo("MAIL", c.Name); info.NumPending != 1 {
|
||||||
|
t.Fatalf("the reader granted `mesh` has %d pending, want 1", info.NumPending)
|
||||||
|
}
|
||||||
|
narrowed := c
|
||||||
|
narrowed.Filters = []string{"mesh.stream.mesh-mail.event.mail.mesh.message.changed"}
|
||||||
|
if err := EnsureHeard(js, narrowed, true); err != nil {
|
||||||
|
t.Fatalf("narrowing the grant: %v", err)
|
||||||
|
}
|
||||||
|
if info, _ := js.js.ConsumerInfo("MAIL", c.Name); info.Config.FilterSubject != narrowed.Filters[0] || len(info.Config.FilterSubjects) != 0 {
|
||||||
|
t.Fatalf("narrowed to %q %v, want only %s", info.Config.FilterSubject, info.Config.FilterSubjects, narrowed.Filters[0])
|
||||||
|
}
|
||||||
|
if info, _ := js.js.ConsumerInfo("MAIL", c.Name); info.NumPending != 0 {
|
||||||
|
t.Fatalf("after narrowing, %d of the reader's former grant are still pending", info.NumPending)
|
||||||
|
}
|
||||||
|
removed, err := PruneHeard(js, []ModuleStream{mailStream}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !slices.Equal(removed, []string{"MAIL/ace_reader"}) {
|
||||||
|
t.Fatalf("removed %v", removed)
|
||||||
|
}
|
||||||
|
if names, _ := js.ConsumerNames("MAIL"); len(names) != 0 {
|
||||||
|
t.Fatalf("still on MAIL: %v", names)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,210 @@
|
|||||||
|
package broker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// mailStream is mesh-mail's own stream as the bus holds it (novox/hq ADR 0363).
|
||||||
|
var mailStream = ModuleStream{Module: "mesh-mail", Name: "MAIL",
|
||||||
|
Events: []string{"mail.*.message.added", "mail.*.message.changed", "mail.*.message.removed"},
|
||||||
|
MaxAgeDays: 7, MaxMiB: 1024, Restricted: true}
|
||||||
|
|
||||||
|
// mailHeard is what a reader granted the account `mesh` hears of it.
|
||||||
|
var mailHeard = Heard{Stream: "MAIL", Emitter: "mesh-mail", Filters: []string{
|
||||||
|
"mesh.stream.mesh-mail.event.mail.mesh.message.added", "mesh.stream.mesh-mail.event.mail.mesh.message.changed"}}
|
||||||
|
|
||||||
|
// A module's stream is its own: subjects EVENTS does not claim, its age and size, kept by limits (ADR 0363 §1, §2).
|
||||||
|
func TestAModuleStreamTakesSubjectsEventsDoesNot(t *testing.T) {
|
||||||
|
s := mailStream.Stream()
|
||||||
|
if s.Name != "MAIL" || s.Retention != RetentionLimits || s.MaxAge != 7*24*60*60 || s.MaxBytes != 1024<<20 || s.DiscardNew {
|
||||||
|
t.Fatalf("composed as %+v", s)
|
||||||
|
}
|
||||||
|
want := []string{"mesh.stream.mesh-mail.event.mail.*.message.added", "mesh.stream.mesh-mail.event.mail.*.message.changed",
|
||||||
|
"mesh.stream.mesh-mail.event.mail.*.message.removed"}
|
||||||
|
if !reflect.DeepEqual(s.Subjects, want) {
|
||||||
|
t.Fatalf("subjects %v", s.Subjects)
|
||||||
|
}
|
||||||
|
for _, own := range MeshStreams() {
|
||||||
|
for _, a := range own.Subjects {
|
||||||
|
for _, b := range s.Subjects {
|
||||||
|
if SubjectsOverlap(a, b) {
|
||||||
|
t.Fatalf("%s claims %s, which overlaps %s of %s: the bus would refuse the stream", s.Name, b, a, own.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if WirePattern("mail.**") != "mail.>" {
|
||||||
|
t.Fatal("`**` is spelled `>` on the wire")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The provider publishes its stream's events on their subjects and nowhere on EVENTS for them; a consumer
|
||||||
|
// granted a part hears it through its own consumer on the stream, and one not granted has nothing there
|
||||||
|
// (ADR 0363 §2, §5).
|
||||||
|
func TestTheProviderPublishesItsStreamAndOnlyAGrantedConsumerHearsIt(t *testing.T) {
|
||||||
|
provider, err := PermissionsFor(Principal{Kind: KindModule, Node: "novox", Module: "mesh-mail",
|
||||||
|
Streams: []ModuleStream{mailStream}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !MayPublish(provider, "mesh.stream.mesh-mail.event.mail.work.message.added") {
|
||||||
|
t.Fatal("the provider may not publish its own stream's event")
|
||||||
|
}
|
||||||
|
if MayPublish(provider, "mesh.mod.mesh-mail.event.mail.work.message.added") {
|
||||||
|
t.Fatal("the provider may publish its stream's event on EVENTS")
|
||||||
|
}
|
||||||
|
if MayPublish(provider, "mesh.stream.other.event.mail.work.message.added") {
|
||||||
|
t.Fatal("the provider may publish on another module's stream")
|
||||||
|
}
|
||||||
|
|
||||||
|
reader := Principal{Kind: KindModule, Node: "ace", Module: "reader", Hears: []Heard{mailHeard}}
|
||||||
|
perms, err := PermissionsFor(reader)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, s := range []string{"$JS.API.CONSUMER.MSG.NEXT.MAIL.ace_reader", "$JS.API.CONSUMER.INFO.MAIL.ace_reader",
|
||||||
|
"$JS.ACK.MAIL.ace_reader.1"} {
|
||||||
|
if !MayPublish(perms, s) {
|
||||||
|
t.Errorf("the granted reader may not publish %s", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !MaySubscribe(perms, "mesh.stream.mesh-mail.event.mail.mesh.message.added") ||
|
||||||
|
MaySubscribe(perms, "mesh.stream.mesh-mail.event.mail.work.message.added") {
|
||||||
|
t.Fatal("the reader's subscription is not what it was granted")
|
||||||
|
}
|
||||||
|
if MayPublish(perms, "$JS.API.CONSUMER.MSG.NEXT.MAIL.ace_other") {
|
||||||
|
t.Fatal("the reader may pull another's consumer")
|
||||||
|
}
|
||||||
|
|
||||||
|
stranger, err := PermissionsFor(Principal{Kind: KindModule, Node: "ace", Module: "stranger",
|
||||||
|
Consumes: []string{"billing.paid"}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if MayPublish(stranger, "$JS.API.CONSUMER.MSG.NEXT.MAIL.ace_stranger") ||
|
||||||
|
MaySubscribe(stranger, "mesh.stream.mesh-mail.event.mail.mesh.message.added") {
|
||||||
|
t.Fatal("a module not granted reaches the stream")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A granted consumer has a durable consumer on the stream, with its granted subjects as filters, and an EVENTS
|
||||||
|
// consumer that hears what it gives up on delivered again (ADR 0363 §5, §7); the runtime carrying it may read both.
|
||||||
|
func TestAGrantedConsumerHasAConsumerOnTheStream(t *testing.T) {
|
||||||
|
records := Records{Nodes: []string{"ace", "novox"}, Assigned: map[string][]Declared{
|
||||||
|
"novox": {{Module: "mesh-mail", Streams: []ModuleStream{mailStream}}},
|
||||||
|
"ace": {{Module: RuntimeModule}, {Module: "reader", Hears: []Heard{mailHeard}, HearsAStream: true}, {Module: "stranger", HearsAStream: true}},
|
||||||
|
}}
|
||||||
|
users, err := Users(records)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var onMail, onEvents []Consumer
|
||||||
|
for _, c := range ConsumersOf(users) {
|
||||||
|
switch c.Consumer.Stream {
|
||||||
|
case "MAIL":
|
||||||
|
onMail = append(onMail, c.Consumer)
|
||||||
|
case EventsStream:
|
||||||
|
onEvents = append(onEvents, c.Consumer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(onMail) != 1 || onMail[0].Name != "ace_reader" || !slices.Equal(onMail[0].Filters, mailHeard.Filters) {
|
||||||
|
t.Fatalf("on MAIL: %+v", onMail)
|
||||||
|
}
|
||||||
|
if len(onEvents) != 1 || onEvents[0].Name != "ace_reader" || !slices.Equal(onEvents[0].Filters, []string{AgainFilter("ace_reader")}) {
|
||||||
|
t.Fatalf("on EVENTS: %+v", onEvents)
|
||||||
|
}
|
||||||
|
for _, u := range users {
|
||||||
|
if u.Kind != KindNodeTools || u.Node != "ace" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
perms, err := PermissionsFor(u)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !MayPublish(perms, "$JS.API.CONSUMER.MSG.NEXT.MAIL.ace_reader") || !MayPublish(perms, "$JS.ACK.MAIL.ace_reader.1") {
|
||||||
|
t.Fatal("the runtime carrying the reader may not read its consumer on MAIL")
|
||||||
|
}
|
||||||
|
if MayPublish(perms, "$JS.API.CONSUMER.MSG.NEXT.MAIL.ace_stranger") {
|
||||||
|
t.Fatal("the runtime may read a consumer no module it carries is granted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The membership says where a module's stream events go and which consumers on module streams to bind
|
||||||
|
// (ADR 0363 §2, §8).
|
||||||
|
func TestTheMembershipNamesTheStreamAndWhatIsHeard(t *testing.T) {
|
||||||
|
where := Placements{Nodes: map[string][]string{}}
|
||||||
|
provider := MembershipFor("novox", Declared{Module: "mesh-mail", Streams: []ModuleStream{mailStream}}, where)
|
||||||
|
if len(provider.Streams) != 1 || provider.Streams[0].Stream != "MAIL" ||
|
||||||
|
provider.Streams[0].Subject != "mesh.stream.mesh-mail.event.{event}" ||
|
||||||
|
!slices.Equal(provider.Streams[0].Events, mailStream.Events) {
|
||||||
|
t.Fatalf("the provider's membership: %+v", provider.Streams)
|
||||||
|
}
|
||||||
|
reader := MembershipFor("ace", Declared{Module: "reader", Hears: []Heard{mailHeard}}, where)
|
||||||
|
if !reflect.DeepEqual(reader.Hears, []HeardIssued{{Stream: "MAIL", Consumer: "ace_reader"}}) {
|
||||||
|
t.Fatalf("the reader's membership: %+v", reader.Hears)
|
||||||
|
}
|
||||||
|
if none := MembershipFor("ace", Declared{Module: "stranger", HearsAStream: true}, where); none.Hears != nil {
|
||||||
|
t.Fatalf("a module not granted is told to bind %+v", none.Hears)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakePruner struct {
|
||||||
|
on map[string][]string
|
||||||
|
removed []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakePruner) ConsumerNames(stream string) ([]string, error) { return f.on[stream], nil }
|
||||||
|
func (f *fakePruner) RemoveConsumer(stream, name string) error {
|
||||||
|
f.removed = append(f.removed, stream+"/"+name)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// A consumer that lost its grant has its consumer on the stream removed by the next assertion (ADR 0363 §6).
|
||||||
|
func TestAConsumerThatLostItsGrantIsRemoved(t *testing.T) {
|
||||||
|
f := &fakePruner{on: map[string][]string{"MAIL": {"ace_reader", "ace_former"}}}
|
||||||
|
wanted := []Consumer{{Name: "ace_reader", Stream: "MAIL"}, {Name: "ace_former", Stream: EventsStream}}
|
||||||
|
removed, err := PruneHeard(f, []ModuleStream{mailStream}, wanted)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !slices.Equal(removed, []string{"MAIL/ace_former"}) || !slices.Equal(f.removed, removed) {
|
||||||
|
t.Fatalf("removed %v", removed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type refusingRaiser struct {
|
||||||
|
fakePruner
|
||||||
|
refuse bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *refusingRaiser) EnsureStream(Stream) error { return nil }
|
||||||
|
func (r *refusingRaiser) EnsureConsumer(Consumer) error {
|
||||||
|
if r.refuse {
|
||||||
|
return errors.New("consumer update refused")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// A narrowing the bus refuses on a restricted stream removes the consumer, so its wider filters deliver nothing
|
||||||
|
// more; on an unrestricted stream the refusal is only said (review of PR 338, ADR 0363 §6).
|
||||||
|
func TestARefusedNarrowingOnARestrictedStreamRemovesTheConsumer(t *testing.T) {
|
||||||
|
c := Consumer{Name: "ace_reader", Stream: "MAIL", Filters: mailHeard.Filters[:1]}
|
||||||
|
r := &refusingRaiser{refuse: true}
|
||||||
|
err := EnsureHeard(r, c, true)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "REMOVED") || !slices.Equal(r.removed, []string{"MAIL/ace_reader"}) {
|
||||||
|
t.Fatalf("restricted: %v, removed %v", err, r.removed)
|
||||||
|
}
|
||||||
|
r = &refusingRaiser{refuse: true}
|
||||||
|
if err := EnsureHeard(r, c, false); err == nil || len(r.removed) != 0 {
|
||||||
|
t.Fatalf("unrestricted: %v, removed %v", err, r.removed)
|
||||||
|
}
|
||||||
|
r = &refusingRaiser{}
|
||||||
|
if err := EnsureHeard(r, c, true); err != nil || len(r.removed) != 0 {
|
||||||
|
t.Fatalf("accepted: %v, removed %v", err, r.removed)
|
||||||
|
}
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user