Compare commits
12
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
454a3db69a | ||
|
|
7b2a9eae80 | ||
|
|
5795e03fad | ||
|
|
cbac703e81 | ||
|
|
18221c4079 | ||
|
|
db0cb1e15e | ||
|
|
ee1d7e5cae | ||
|
|
43cab56452 | ||
|
|
56d6313262 | ||
|
|
7f3ff32361 | ||
|
|
2819915e8e | ||
|
|
7892019d9e |
@@ -64,6 +64,11 @@ type actor struct {
|
||||
// the highest epoch issued, and what was said of it; zero when it was not (S12).
|
||||
reset time.Time
|
||||
resetSaid string
|
||||
// renewEvery and poll, when set, are how often the serving controller renews the lease it holds and
|
||||
// looks again at one another holds, in place of the lease's own: a test's, so two controllers hand the
|
||||
// lease over in a second rather than in a renewal of five (novox/hq issue 591). A renewal is given
|
||||
// the lease's own RenewEvery to answer however often it is made, so a slow bus does not lose it.
|
||||
renewEvery, poll time.Duration
|
||||
}
|
||||
|
||||
// theLease is this process's standing to the lease.
|
||||
@@ -223,7 +228,7 @@ func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory
|
||||
}
|
||||
say := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
|
||||
l, err := lease.Open(ctx, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
|
||||
Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) {
|
||||
RenewEvery: a.renewEvery, RenewWithin: lease.RenewEvery, Poll: a.poll, Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.reset = time.Now()
|
||||
|
||||
@@ -75,6 +75,16 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
failed += len(wrong)
|
||||
faulted[m.Module] = true
|
||||
}
|
||||
// A module that declares a setting names the keys of its mergeable files (novox/hq ADR 0326 §3,
|
||||
// issue 574): judged where a definition is written, as at registration, never where a stored
|
||||
// build is read again.
|
||||
if wrong := catalogue.WrittenSettingsProblems(m); len(wrong) > 0 {
|
||||
for _, p := range wrong {
|
||||
fmt.Fprintf(out, "%s: %s\n", path, p)
|
||||
}
|
||||
failed += len(wrong)
|
||||
faulted[m.Module] = true
|
||||
}
|
||||
if named := catalogue.InstallationProblems(m); len(named) > 0 {
|
||||
for _, p := range named {
|
||||
fmt.Fprintf(out, "%s: %s\n", path, p)
|
||||
|
||||
@@ -429,6 +429,20 @@ func checkedOf(result link.BuildResult) link.Checked {
|
||||
l.Verdict = "error"
|
||||
}
|
||||
}
|
||||
// **A check run again says so** (novox/hq issue 594). The summaries are the lines the pull request
|
||||
// shows, clipped by the forge to a few words, so they get only a short marker after the verdict's own
|
||||
// words; the report gets the whole sentence. Layers copied, so the build's own outcome is left as it came.
|
||||
if mark := link.RunMarker(result.Delivery); mark != "" {
|
||||
c.Summary += " " + mark
|
||||
for _, l := range []**link.CheckLayer{&c.Gate, &c.RepoCheck} {
|
||||
if *l != nil {
|
||||
layer := **l
|
||||
layer.Summary += " " + mark
|
||||
*l = &layer
|
||||
}
|
||||
}
|
||||
c.Report = link.Restarted(result.Delivery) + "\n\n" + c.Report
|
||||
}
|
||||
if len(c.Report) > maxCheckReport {
|
||||
c.Report = "…" + c.Report[len(c.Report)-maxCheckReport:]
|
||||
}
|
||||
|
||||
@@ -55,8 +55,14 @@ func TestTwoControllersOneActs(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
ctx := t.Context()
|
||||
|
||||
// Each renews five times and looks again ten times as often as a serving controller does, so the
|
||||
// handover and the loss below are seen in a second, not in a renewal of five (novox/hq issue 591): what
|
||||
// is checked is the order of events, which does not change with the interval. Each renewal still has a
|
||||
// serving controller's five seconds to answer (acting.go), so a slow bus does not lose A the lease.
|
||||
const renewEvery, poll = lease.RenewEvery / 5, lease.Poll / 10
|
||||
|
||||
// Controller A takes the lease.
|
||||
a := &actor{}
|
||||
a := &actor{renewEvery: renewEvery, poll: poll}
|
||||
aCtx, stopA := context.WithCancel(ctx)
|
||||
defer stopA()
|
||||
lostA, err := a.serveUnderTheLease(aCtx, inv, url)
|
||||
@@ -69,7 +75,7 @@ func TestTwoControllersOneActs(t *testing.T) {
|
||||
}
|
||||
|
||||
// Controller B starts while A holds it, and waits — acting on nothing meanwhile.
|
||||
b := &actor{}
|
||||
b := &actor{renewEvery: renewEvery, poll: poll}
|
||||
bCtx, stopB := context.WithCancel(ctx)
|
||||
defer stopB()
|
||||
tookB := make(chan (<-chan struct{}), 1)
|
||||
@@ -82,10 +88,11 @@ func TestTwoControllersOneActs(t *testing.T) {
|
||||
}
|
||||
tookB <- lost
|
||||
}()
|
||||
// B looks fifteen times, across one of A's renewals at least.
|
||||
select {
|
||||
case <-tookB:
|
||||
t.Fatal("B took the lease while A held it")
|
||||
case <-time.After(3 * time.Second):
|
||||
case <-time.After(15 * poll):
|
||||
}
|
||||
if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) {
|
||||
t.Fatalf("B, waiting, may act: %v", err)
|
||||
@@ -125,7 +132,7 @@ func TestTwoControllersOneActs(t *testing.T) {
|
||||
}
|
||||
select {
|
||||
case <-lostB:
|
||||
case <-time.After(2 * lease.RenewEvery):
|
||||
case <-time.After(2 * renewEvery):
|
||||
t.Fatal("B was not told it lost the lease")
|
||||
}
|
||||
if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) {
|
||||
|
||||
@@ -1051,6 +1051,11 @@ func namesNoInstallation(m catalogue.Manifest) error {
|
||||
if problems := catalogue.TrustProblems(m); len(problems) > 0 {
|
||||
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
||||
}
|
||||
// And a module that declares a setting names the keys of its mergeable files (novox/hq ADR 0326
|
||||
// §3, issue 574): judged where a definition is written, never where a stored build is read again.
|
||||
if problems := catalogue.WrittenSettingsProblems(m); len(problems) > 0 {
|
||||
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
||||
}
|
||||
named := catalogue.InstallationProblems(m)
|
||||
if len(named) == 0 {
|
||||
return nil
|
||||
|
||||
@@ -30,8 +30,9 @@ var maxPlanWait = link.LongestAnswer - planWaitStart
|
||||
// answered that the call is still running, and `calls` with its id says what came of it.
|
||||
const planWaitStart = 5 * time.Second
|
||||
|
||||
// planWaitEvery is how often a wait reads the plan again.
|
||||
const planWaitEvery = time.Second
|
||||
// planWaitEvery is how often a wait reads the plan again. A variable so a test reads it more often
|
||||
// (novox/hq issue 591).
|
||||
var planWaitEvery = time.Second
|
||||
|
||||
// The things a wait waits until.
|
||||
var planWaitUntil = []string{"moved", "tier", "done"}
|
||||
|
||||
@@ -138,6 +138,12 @@ func TestThePlansVerbComposesAWait(t *testing.T) {
|
||||
func TestAPlanFollowedFromTheStoreAnswersWhenItMoves(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
// Read every 50ms rather than every second, so the wait answers as soon as the plan moves and the bound
|
||||
// below passes in a fraction of a second (novox/hq issue 591); the move still waits 1.5s, long enough
|
||||
// for the wait to have read the plan before it.
|
||||
was := planWaitEvery
|
||||
planWaitEvery = 50 * time.Millisecond
|
||||
t.Cleanup(func() { planWaitEvery = was })
|
||||
now := time.Now().UTC()
|
||||
walk := inventory.Plan{ID: "plan-229", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c229c229",
|
||||
Created: now, State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
||||
@@ -169,11 +175,11 @@ func TestAPlanFollowedFromTheStoreAnswersWhenItMoves(t *testing.T) {
|
||||
if got.Came != "tier" || got.Plan.Tier != 1 || got.Plan.State != inventory.PlanRolling {
|
||||
t.Fatalf("the wait answered %+v", got)
|
||||
}
|
||||
if waited := time.Since(began); waited > 10*time.Second {
|
||||
if waited := time.Since(began); waited > 3*time.Second {
|
||||
t.Errorf("it answered after %s, not when the plan moved", waited)
|
||||
}
|
||||
out = printed(t, func() error {
|
||||
return plansCommand(ctx, []string{"wait", walk.ID, "--bound", "2"})
|
||||
return plansCommand(ctx, []string{"wait", walk.ID, "--bound", "300ms"})
|
||||
})
|
||||
if err := json.Unmarshal([]byte(out), &got); err != nil || got.Came != "bound" || got.Until != "moved" {
|
||||
t.Fatalf("a plan that did not move answered %+v (%v)", got, err)
|
||||
|
||||
@@ -141,6 +141,9 @@ type queuedAnswer struct {
|
||||
// Position is its place among the waiting, 1 being handed out next; 0 for an ask not waiting.
|
||||
Position int `json:"position,omitempty"`
|
||||
Seq uint64 `json:"seq"`
|
||||
// Delivery is the how-manyth time an ask in flight was handed out; more than 1 is a restart
|
||||
// (novox/hq issue 594).
|
||||
Delivery int `json:"delivery,omitempty"`
|
||||
}
|
||||
|
||||
// aCommitRef is a ref that names a commit by its full hash (SHA-1 or SHA-256).
|
||||
@@ -164,7 +167,7 @@ func queueAnswerOf(q link.Queue, now time.Time) queueAnswer {
|
||||
}
|
||||
return queuedAnswer{ID: a.ID, State: a.State, Kind: kind, Repository: link.WithoutPassword(a.Repository), Path: a.Path,
|
||||
Ref: a.Ref, Commit: commit, Machine: a.On, WasOn: a.Was, AskedAt: a.AskedAt, Since: since,
|
||||
Position: position, Seq: a.Seq}
|
||||
Position: position, Seq: a.Seq, Delivery: a.Delivery}
|
||||
}
|
||||
for _, a := range running {
|
||||
out.Asks = append(out.Asks, one(a, 0))
|
||||
@@ -207,6 +210,10 @@ func queueText(q link.Queue, now time.Time) string {
|
||||
switch {
|
||||
case a.On != "":
|
||||
where = fmt.Sprintf("on %s for %s", a.On, now.Sub(a.Started).Round(time.Second))
|
||||
// Started again after an earlier taker left it (novox/hq issue 594).
|
||||
if a.Delivery > 1 {
|
||||
where += fmt.Sprintf(", handed out %d times", a.Delivery)
|
||||
}
|
||||
case a.Was != "":
|
||||
where = fmt.Sprintf("handed back by %s, to be handed out again", a.Was)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A check that ran again says so on the pull request** (novox/hq issue 594): a build agent replaced
|
||||
// mid-check leaves the ask to be handed out again, and the verdict that finally comes is from a second
|
||||
// run. Each layer's summary — what the forge shows as the check's line — names the restart.
|
||||
func TestARestartedCheckSaysSoOnEveryLayer(t *testing.T) {
|
||||
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-controller", Number: 279}
|
||||
gate := &link.CheckLayer{Verdict: "pass", Summary: "every node composes"}
|
||||
repo := &link.CheckLayer{Verdict: "pass", Summary: "merge-check.sh passed"}
|
||||
c := checkedOf(link.BuildResult{ID: "b", Ref: "abc", On: "shanks", Checked: asked, Delivery: 2,
|
||||
Check: &link.CheckOutcome{Verdict: "pass", Summary: "every node composes", Gate: gate, RepoCheck: repo}})
|
||||
for name, said := range map[string]string{"summary": c.Summary, "gate": c.Gate.Summary,
|
||||
"repository check": c.RepoCheck.Summary} {
|
||||
// The verdict's own words first, then the marker: the forge clips a status to a few words.
|
||||
if !strings.HasSuffix(said, " (run 2)") || strings.HasPrefix(said, "(run") {
|
||||
t.Errorf("the %s says %q: a check run twice reads as run once, or its marker pushes its words out", name, said)
|
||||
}
|
||||
}
|
||||
if !strings.HasPrefix(c.Report, "restarted: handed out 2 times") {
|
||||
t.Errorf("the report does not open with the restart: %q", c.Report)
|
||||
}
|
||||
if gate.Summary != "every node composes" {
|
||||
t.Errorf("the build's own outcome was changed under it: %q", gate.Summary)
|
||||
}
|
||||
|
||||
// Taken once, nothing is added.
|
||||
once := checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Delivery: 1,
|
||||
Check: &link.CheckOutcome{Verdict: "pass", Summary: "every node composes"}})
|
||||
if strings.Contains(once.Summary, "(run") || strings.Contains(once.Report, "restarted") {
|
||||
t.Errorf("a check run once says %q", once.Summary)
|
||||
}
|
||||
}
|
||||
|
||||
// And the queue says it of the ask in flight.
|
||||
func TestTheQueueSaysAnAskInFlightRestarted(t *testing.T) {
|
||||
now := time.Date(2026, 10, 13, 0, 16, 0, 0, time.UTC)
|
||||
q := link.Queue{Seat: link.TheBuildMachine, Delivered: 1, AckPending: 1, MaxDeliver: 5, Asks: []link.QueuedAsk{
|
||||
{Seq: 1, ID: "build-279", Repository: "ssh://forge/novox/mesh-controller", AskedAt: now.Add(-7 * time.Minute),
|
||||
State: link.AskInFlight, On: "shanks", Started: now.Add(-time.Minute), Delivery: 2},
|
||||
}}
|
||||
text := queueText(q, now)
|
||||
if !strings.Contains(text, "handed out 2 times") {
|
||||
t.Fatalf("the queue does not say the ask was handed out again:\n%s", text)
|
||||
}
|
||||
if a := queueAnswerOf(q, now).Asks[0]; a.Delivery != 2 {
|
||||
t.Errorf("the queue's data says delivery %d, want 2", a.Delivery)
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,14 @@ import (
|
||||
// shape fails the suite, naming its source. The keeper would say such a summary in words at run time;
|
||||
// this is where the producer is made to say it rightly in the first place.
|
||||
func TestMain(m *testing.M) {
|
||||
// **A command line this suite runs ends when it is done** (novox/hq issue 591). Many tests run this test
|
||||
// binary as a verb's command line, and a binary built with the race detector sleeps a second before it
|
||||
// exits (GORACE's atexit_sleep_ms), which was more than the rest of each such test. The children still
|
||||
// run under the race detector and still fail on a race they find; they only stop waiting at their end.
|
||||
// Set before any child starts, and only where nobody chose GORACE.
|
||||
if _, chosen := os.LookupEnv("GORACE"); !chosen {
|
||||
_ = os.Setenv("GORACE", "atexit_sleep_ms=0")
|
||||
}
|
||||
// The process a mesh-cli test runs as a command line: it says the verb and the caller it was given, and
|
||||
// ends (meshcli_test.go).
|
||||
// The process a mesh-cli test runs as `secret accept`: it reads its value exactly as `secret accept` does
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// ADR 0326 §3 is judged where a definition is written (novox/hq issue 574): `module check`, and
|
||||
// namesNoInstallation, which `module add`, a build taken in and the merge gate's registration run.
|
||||
func TestWhereADefinitionIsWrittenADeclaringModuleNamesItsFilesKeys(t *testing.T) {
|
||||
const silent = `{"module":"mail-client","version":"1",
|
||||
"settings":{"prefs":{"kind":"preference","default":"// none","why":"its own defaults until a machine names one"}},
|
||||
"resources":[
|
||||
{"id":"user-js","type":"file","path":"/var/lib/mail-client/user.js","mode":"0644","content":"${setting:prefs}\n"},
|
||||
{"id":"accounts","type":"file","path":"/var/lib/mail-client/accounts.json","mode":"0600","merge":"json",
|
||||
"content":"{}\n"%s}]}`
|
||||
dir := t.TempDir()
|
||||
bad := filepath.Join(dir, "bad.json")
|
||||
good := filepath.Join(dir, "good.json")
|
||||
os.WriteFile(bad, []byte(strings.Replace(silent, "%s", "", 1)), 0o600)
|
||||
os.WriteFile(good, []byte(strings.Replace(silent, "%s", `,"takes-settings":["accounts"]`, 1)), 0o600)
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := moduleCheck([]string{bad}, &out); err == nil ||
|
||||
!strings.Contains(out.String(), "mergeable accounts names no takes-settings") {
|
||||
t.Fatalf("module check passed a declaring module's silent mergeable file: %v\n%s", err, out.String())
|
||||
}
|
||||
out.Reset()
|
||||
if err := moduleCheck([]string{good}, &out); err != nil {
|
||||
t.Fatalf("the file naming its keys was refused: %v\n%s", err, out.String())
|
||||
}
|
||||
|
||||
raw, _ := os.ReadFile(bad)
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("reading it is not where it is judged: %v", err)
|
||||
}
|
||||
if err := namesNoInstallation(m); err == nil || !strings.Contains(err.Error(), "names no takes-settings") {
|
||||
t.Fatalf("registration passed it: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -82,7 +82,7 @@ func theCarriedAccounts(t *testing.T) string {
|
||||
// the commit the mesh runs, or as captured in testdata/beside (internal/beside, novox/hq issue 432).
|
||||
func theTemplate(t *testing.T) []map[string]any {
|
||||
t.Helper()
|
||||
path := filepath.Join(beside.Dir(t, "mesh-host"), "examples", "foundation-first-node-nats.lock")
|
||||
path := filepath.Join(beside.Dir(t, "mesh-host"), "examples", "foundation-first-node.lock")
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -226,6 +226,13 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
||||
say("manifest", "INVALID: %v", err)
|
||||
return Result{}, err
|
||||
}
|
||||
// What is judged only where a definition is written (novox/hq ADR 0326 §3, issue 574): a build is
|
||||
// where a changed definition is first read, so it is refused here before anything is made of it.
|
||||
if problems := catalogue.WrittenSettingsProblems(manifest); len(problems) > 0 {
|
||||
err := fmt.Errorf("%s", strings.Join(problems, "; "))
|
||||
say("manifest", "INVALID: %v", err)
|
||||
return Result{}, err
|
||||
}
|
||||
say("manifest", "%s v%s — %d artifact(s)", manifest.Module, manifest.Version, artifactCount(manifest))
|
||||
|
||||
// What it is made from, for its source fingerprint: the module's own tree first.
|
||||
|
||||
@@ -70,12 +70,42 @@ func namesItsKeys(m Manifest) bool {
|
||||
// reads itself, one the file keeps for itself, or one mergeable file naming its keys beside another
|
||||
// that names none.
|
||||
func TakenSettingsProblems(m Manifest) []string {
|
||||
// Not yet here: refusing a module that declares a setting and owns a mergeable file naming no keys
|
||||
// (ADR 0326 §3). betterbird is one, and the catalogue cannot name its keys before this controller
|
||||
// runs, so that refusal follows once the catalogue does (novox/hq issue 574).
|
||||
return takenSettingsProblems(m)
|
||||
}
|
||||
|
||||
// WrittenSettingsProblems is ADR 0326 §3, judged only where a definition is written — the module
|
||||
// check, `module add`, a build taken in, the builder and the merge gate's registration — and never
|
||||
// where a stored one is read again (ParseManifest, a build's manifest at a commit, a module put back):
|
||||
// a module that declares a setting names the keys of every mergeable file it owns (novox/hq issue
|
||||
// 574, ADR 0164 §11 and §15). A build stored before the rule, such as betterbird's before its accounts
|
||||
// file named its keys, must still compose and still be put back; only its next change is held to it.
|
||||
//
|
||||
// Said only when none of the module's mergeable files names its keys: when one does, §2 already
|
||||
// names every file that does not, and a file is not reported twice.
|
||||
func WrittenSettingsProblems(m Manifest) []string {
|
||||
if len(m.Settings) == 0 {
|
||||
return nil
|
||||
}
|
||||
var silent []string
|
||||
for _, r := range m.Resources {
|
||||
how, _ := r["merge"].(string)
|
||||
if how == "" {
|
||||
continue
|
||||
}
|
||||
if _, said := r[TakesSettingsField]; said {
|
||||
return nil
|
||||
}
|
||||
silent = append(silent, fmt.Sprint(r["id"]))
|
||||
}
|
||||
var problems []string
|
||||
for _, id := range silent {
|
||||
problems = append(problems, fmt.Sprintf("%s declares settings, and its mergeable %s names no %s: "+
|
||||
"a module that declares a setting names the keys each mergeable file takes, an empty list "+
|
||||
"for none (novox/hq ADR 0326 §3, ADR 0164 §11)", m.Module, id, TakesSettingsField))
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// takenSettingsProblems is every rule of ADR 0326 §2 and §4, judged where a definition is checked and
|
||||
// where a stored one is composed.
|
||||
func takenSettingsProblems(m Manifest) []string {
|
||||
|
||||
@@ -194,3 +194,50 @@ func containsText(list []string, text string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestAModuleThatDeclaresASettingNamesItsFilesKeys(t *testing.T) {
|
||||
// ADR 0326 §3, ADR 0164 §15, novox/hq issue 574: only a module that declares no settings keeps
|
||||
// today's form. One that declares a preference and owns a silent mergeable file is refused where
|
||||
// a definition is written — and read and composed where a stored one is read again, so a build kept
|
||||
// from before the rule is still sent and still put back.
|
||||
declaring := func(app, sidecar []any) Manifest {
|
||||
m := twoFiles(app, sidecar)
|
||||
m.Resources[2]["content"] = "WORKERS=${setting:workers}\n"
|
||||
m.Settings = map[string]SettingDeclaration{"workers": {Kind: KindPreference, Default: 4.0, Why: "a number"}}
|
||||
return m
|
||||
}
|
||||
m := declaring(nil, nil)
|
||||
problems := WrittenSettingsProblems(m)
|
||||
if len(problems) != 2 || !containsText(problems, "mergeable app names no takes-settings") ||
|
||||
!containsText(problems, "mergeable sidecar names no takes-settings") {
|
||||
t.Errorf("where it is written: %v", problems)
|
||||
}
|
||||
if problems := TakenSettingsProblems(m); len(problems) != 0 {
|
||||
t.Errorf("what a stored definition is read by refuses it: %v", problems)
|
||||
}
|
||||
raw, err := json.Marshal(map[string]any{"module": m.Module, "version": "1", "resources": m.Resources,
|
||||
"settings": map[string]any{"workers": map[string]any{"kind": KindPreference, "default": 4, "why": "a number"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
read, err := ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("a stored build from before the rule is not read again: %v", err)
|
||||
}
|
||||
if err := JudgeSettings(read, nil, false); err != nil {
|
||||
t.Errorf("a stored definition is composed as before, and was refused: %v", err)
|
||||
}
|
||||
|
||||
if problems := WrittenSettingsProblems(declaring([]any{"server"}, []any{})); len(problems) != 0 {
|
||||
t.Errorf("every mergeable file names its keys, and was refused: %v", problems)
|
||||
}
|
||||
if problems := WrittenSettingsProblems(twoFiles(nil, nil)); len(problems) != 0 {
|
||||
t.Errorf("a module declaring no setting was refused: %v", problems)
|
||||
}
|
||||
// One naming file and one silent: §2 names the silent one, and §3 does not name it again.
|
||||
mixed := declaring([]any{"server"}, nil)
|
||||
if written, taken := WrittenSettingsProblems(mixed), TakenSettingsProblems(mixed); len(written) != 0 || len(taken) != 1 {
|
||||
t.Errorf("the silent file is reported %d times by §3 and %d by §2; want once, by §2: %v %v",
|
||||
len(written), len(taken), written, taken)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,14 +29,13 @@ func ForTest(t *testing.T) *Identity {
|
||||
if len(name) > 60 {
|
||||
name = name[:60]
|
||||
}
|
||||
conn, err := pgx.Connect(t.Context(), admin)
|
||||
// Created already migrated, a copy of a template migrated once (novox/hq issue 591); the Migrate
|
||||
// below then finds nothing pending, which it does only if the copy's record matches every migration.
|
||||
migrations, err := Migrations()
|
||||
if err != nil {
|
||||
t.Fatalf("cannot reach the test PostgreSQL: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := conn.Exec(t.Context(), "create database "+name); err != nil {
|
||||
t.Fatalf("cannot create %s: %v", name, err)
|
||||
}
|
||||
conn.Close(t.Context())
|
||||
store.CreateMigratedForTest(t, admin, name, migrations)
|
||||
|
||||
cut := strings.LastIndex(admin, "/")
|
||||
t.Setenv(store.Variable(Name), admin[:cut]+"/"+name+"?sslmode=disable")
|
||||
@@ -57,10 +56,6 @@ func ForTest(t *testing.T) *Identity {
|
||||
if err := ident.Ready(t.Context(), 20*time.Second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
migrations, err := Migrations()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ident.store.Migrate(t.Context(), migrations); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
@@ -23,6 +24,25 @@ import (
|
||||
// access change would blank every module's password if the mesh kept nothing (design 25 §4, and the
|
||||
// migration beside this).
|
||||
|
||||
// busPasswordCostForTest, when set, is the bcrypt cost a bus password is hashed at instead of the
|
||||
// library's default. Only ForTest sets it, to bcrypt's least (novox/hq issue 591): at the default
|
||||
// cost one hash takes most of a second under the race detector, and a hash at the least cost is the
|
||||
// same kind of hash, checked by the same comparison and by the bus the same way.
|
||||
var busPasswordCostForTest atomic.Int32
|
||||
|
||||
// busPasswordCost is the bcrypt cost a bus password is hashed at: the library's default, but in a test.
|
||||
func busPasswordCost() int {
|
||||
if c := busPasswordCostForTest.Load(); c != 0 {
|
||||
return int(c)
|
||||
}
|
||||
return bcrypt.DefaultCost
|
||||
}
|
||||
|
||||
// hashBusPassword is the hash a bus password is kept as, minted, recorded or seeded alike.
|
||||
func hashBusPassword(password string) ([]byte, error) {
|
||||
return bcrypt.GenerateFromPassword([]byte(password), busPasswordCost())
|
||||
}
|
||||
|
||||
// BusUser is one user of the bus, as the mesh records it.
|
||||
type BusUser struct {
|
||||
Username string
|
||||
@@ -67,7 +87,7 @@ func (i *Inventory) RecordBusPassword(ctx context.Context, u BusUser, password s
|
||||
if password == "" {
|
||||
return errors.New("a bus user needs a password")
|
||||
}
|
||||
hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
|
||||
hash, err := hashBusPassword(password)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot hash a bus password: %w", err)
|
||||
}
|
||||
@@ -101,7 +121,7 @@ func (i *Inventory) MintBusPassword(ctx context.Context, u BusUser) (string, err
|
||||
// The cost the server will pay on every connection. Left at the library's default rather than
|
||||
// raised: a node reconnecting after a network blip pays it, and the mesh's own links reconnect
|
||||
// far more often than a person logs in anywhere.
|
||||
hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
|
||||
hash, err := hashBusPassword(password)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot hash a bus password: %w", err)
|
||||
}
|
||||
@@ -183,7 +203,7 @@ func (i *Inventory) SeedBusUser(ctx context.Context, u BusUser, password string)
|
||||
if u.Username == "" || u.Kind == "" || password == "" {
|
||||
return errors.New("a bus user needs a username, a kind and the credential it is using")
|
||||
}
|
||||
hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
|
||||
hash, err := hashBusPassword(password)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot hash a bus password: %w", err)
|
||||
}
|
||||
|
||||
@@ -158,3 +158,17 @@ func TestACredentialTheMeshDidNotMintIsRecordedOnceAndNotOverwritten(t *testing.
|
||||
t.Fatal("a restart put the bootstrap credential back over a rotated one")
|
||||
}
|
||||
}
|
||||
|
||||
// Outside a test a bus password is hashed at bcrypt's default cost: only ForTest lowers it (novox/hq issue 591).
|
||||
func TestABusPasswordIsHashedAtTheDefaultCostOutsideATest(t *testing.T) {
|
||||
was := busPasswordCostForTest.Load()
|
||||
t.Cleanup(func() { busPasswordCostForTest.Store(was) })
|
||||
busPasswordCostForTest.Store(0)
|
||||
hash, err := hashBusPassword("a password")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := bcrypt.Cost(hash); err != nil || got != bcrypt.DefaultCost {
|
||||
t.Fatalf("a bus password is hashed at cost %d (%v), not bcrypt's default %d", got, err, bcrypt.DefaultCost)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/store"
|
||||
)
|
||||
@@ -30,14 +31,14 @@ func ForTest(t *testing.T) *Inventory {
|
||||
name = name[:60]
|
||||
}
|
||||
|
||||
conn, err := pgx.Connect(t.Context(), admin)
|
||||
// Created already migrated, a copy of a template migrated once (novox/hq issue 591); the Migrate
|
||||
// below then finds nothing pending, which it does only if the copy's record matches every migration.
|
||||
migrations, err := Migrations()
|
||||
if err != nil {
|
||||
t.Fatalf("cannot reach the test PostgreSQL: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := conn.Exec(t.Context(), "create database "+name); err != nil {
|
||||
t.Fatalf("cannot create %s: %v", name, err)
|
||||
}
|
||||
conn.Close(t.Context())
|
||||
store.CreateMigratedForTest(t, admin, name, migrations)
|
||||
busPasswordCostForTest.Store(int32(bcrypt.MinCost))
|
||||
|
||||
cut := strings.LastIndex(admin, "/")
|
||||
t.Setenv(store.Variable(Name), admin[:cut]+"/"+name+"?sslmode=disable")
|
||||
@@ -59,10 +60,6 @@ func ForTest(t *testing.T) *Inventory {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
migrations, err := Migrations()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.store.Migrate(t.Context(), migrations); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// A build kept from before ADR 0326 §3 is still read and composed (novox/hq issue 574). betterbird's
|
||||
// builds from before mesh-catalog 8f80cd4 declare a preference and own a mergeable file that names no
|
||||
// keys: refused where a definition is written, but a machine still runs that build and a rollback may
|
||||
// put it back, so reading it again — ManifestAt, which a walk, the gate and a release compose from —
|
||||
// must not refuse it.
|
||||
func TestAStoredBuildWithASilentMergeableFileIsStillReadAndComposed(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
ctx := context.Background()
|
||||
manifest, err := json.Marshal(map[string]any{
|
||||
"module": "mail-client", "version": "1",
|
||||
"settings": map[string]any{"prefs": map[string]any{
|
||||
"kind": catalogue.KindPreference, "default": "// none", "why": "its own defaults until a machine names one"}},
|
||||
"resources": []any{
|
||||
map[string]any{"id": "user-js", "type": "file", "path": "/var/lib/mail-client/user.js", "mode": "0644",
|
||||
"content": "${setting:prefs}\n"},
|
||||
map[string]any{"id": "accounts", "type": "file", "path": "/var/lib/mail-client/accounts.json",
|
||||
"mode": "0600", "merge": "json", "content": "{\"accounts\": {}}\n"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := aBuild("before-574", "mail-client", "")
|
||||
b.Manifest = manifest
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
m, found, err := inv.ManifestAt(ctx, "mail-client", b.Commit)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("the stored build is not read again: found %v, %v", found, err)
|
||||
}
|
||||
if written := catalogue.WrittenSettingsProblems(m); len(written) != 1 {
|
||||
t.Fatalf("the rule this build predates should name its accounts file where it is written: %v", written)
|
||||
}
|
||||
|
||||
// And composed, in today's form: the accounts file still takes its layer's keys.
|
||||
with := catalogue.Rendering{Settings: catalogue.SettingsBy{"mail-client": {{From: "workstation",
|
||||
Values: map[string]any{"accounts": map[string]any{"work": map[string]any{"email": "a@example.invalid"}}}}}}}
|
||||
if err := catalogue.JudgeSettings(m, with.Settings["mail-client"], false); err != nil {
|
||||
t.Fatalf("the stored build's settings do not compose: %v", err)
|
||||
}
|
||||
out, err := catalogue.Resolution{Node: "workstation", Modules: []catalogue.Manifest{m}}.Declaration(with)
|
||||
if err != nil {
|
||||
t.Fatalf("the stored build is not composed: %v", err)
|
||||
}
|
||||
for _, r := range out {
|
||||
if r["id"] == "mail-client.accounts" {
|
||||
if content, _ := r["content"].(string); !strings.Contains(content, "a@example.invalid") {
|
||||
t.Fatalf("the accounts file did not take its layer: %v", content)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("the accounts file was not composed: %v", out)
|
||||
}
|
||||
@@ -75,6 +75,10 @@ type Options struct {
|
||||
Holder Holder
|
||||
// RenewEvery, Margin and Poll default to the package's.
|
||||
RenewEvery, Margin, Poll time.Duration
|
||||
// RenewWithin is how long one renewal may take before it counts as failed: RenewEvery when zero. Apart
|
||||
// from it so a test that renews often still gives a slow bus a serving controller's time to answer
|
||||
// (novox/hq issue 591).
|
||||
RenewWithin time.Duration
|
||||
// Floor is the highest epoch this mesh has issued, from the controller's own store. **An epoch is
|
||||
// never issued twice**: a lease bucket raised again from nothing — a bus whose data was replaced —
|
||||
// starts its revisions over, and every machine that heard epoch 57 would refuse the next controller
|
||||
@@ -138,6 +142,9 @@ func Open(ctx context.Context, js jetstream.JetStream, bucket string, o Options)
|
||||
if o.RenewEvery <= 0 {
|
||||
o.RenewEvery = RenewEvery
|
||||
}
|
||||
if o.RenewWithin <= 0 {
|
||||
o.RenewWithin = o.RenewEvery
|
||||
}
|
||||
if o.Margin <= 0 {
|
||||
o.Margin = Margin
|
||||
}
|
||||
@@ -406,7 +413,7 @@ func (l *Lease) Renew(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return l.lose(err)
|
||||
}
|
||||
renewing, cancel := context.WithTimeout(ctx, l.o.RenewEvery)
|
||||
renewing, cancel := context.WithTimeout(ctx, l.o.RenewWithin)
|
||||
defer cancel()
|
||||
next, err := l.kv.Update(renewing, Key, value, revision)
|
||||
if err != nil {
|
||||
|
||||
@@ -32,14 +32,13 @@ func ForTest(t *testing.T) *Licences {
|
||||
name = name[:60]
|
||||
}
|
||||
|
||||
conn, err := pgx.Connect(t.Context(), admin)
|
||||
// Created already migrated, a copy of a template migrated once (novox/hq issue 591); the Migrate
|
||||
// below then finds nothing pending, which it does only if the copy's record matches every migration.
|
||||
migrations, err := Migrations()
|
||||
if err != nil {
|
||||
t.Fatalf("cannot reach the test PostgreSQL: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := conn.Exec(t.Context(), "create database "+name); err != nil {
|
||||
t.Fatalf("cannot create %s: %v", name, err)
|
||||
}
|
||||
conn.Close(t.Context())
|
||||
store.CreateMigratedForTest(t, admin, name, migrations)
|
||||
|
||||
cut := strings.LastIndex(admin, "/")
|
||||
t.Setenv(store.Variable(Name), admin[:cut]+"/"+name+"?sslmode=disable")
|
||||
@@ -61,10 +60,6 @@ func ForTest(t *testing.T) *Licences {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
migrations, err := Migrations()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := held.store.Migrate(t.Context(), migrations); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -257,6 +257,10 @@ type BuildResult struct {
|
||||
// build; then only identical artifacts make a rebuild no move.
|
||||
SourceFingerprint string `json:"source-fingerprint,omitempty"`
|
||||
|
||||
// Delivery is the how-manyth time the queue handed this ask out (BuildStart.Delivery): more than 1
|
||||
// says the outcome is from a run started again after an earlier taker left it (novox/hq issue 594).
|
||||
Delivery int `json:"delivery,omitempty"`
|
||||
|
||||
// Failed is why, when it did.
|
||||
Failed string `json:"failed,omitempty"`
|
||||
|
||||
|
||||
@@ -92,6 +92,31 @@ type BuildStart struct {
|
||||
Ref string `json:"ref,omitempty"`
|
||||
On string `json:"on"`
|
||||
At string `json:"at"`
|
||||
// Delivery is the how-manyth time the queue handed this ask out: 1 for a first start, more for one
|
||||
// an earlier taker left unfinished — most often a build agent replaced mid-build — and started
|
||||
// again from the beginning (novox/hq issue 594). 0 from a machine that predates it.
|
||||
Delivery int `json:"delivery,omitempty"`
|
||||
}
|
||||
|
||||
// Restarted is how a build handed out again says so (novox/hq issue 594), in its log and in its check's
|
||||
// report, so a second run never reads as the first. Neutral on why: a holder replaced mid-build, a paused
|
||||
// holder handing an ask back as it arrived, and a bus that held its acknowledgements past the wait all
|
||||
// count as a hand-out. Empty for a first start.
|
||||
func Restarted(delivery int) string {
|
||||
if delivery < 2 {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("restarted: handed out %d times; an earlier hand-out did not finish here, so this run "+
|
||||
"started from the beginning", delivery)
|
||||
}
|
||||
|
||||
// RunMarker is the short form of Restarted, for a line clipped to a few words — a check's status on the
|
||||
// forge (novox/hq issue 594): "(run 2)". Empty for a first start.
|
||||
func RunMarker(delivery int) string {
|
||||
if delivery < 2 {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("(run %d)", delivery)
|
||||
}
|
||||
|
||||
// BuildLine is one thing a build said while building.
|
||||
|
||||
@@ -310,7 +310,7 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
// built. A set that cannot be read is said and the ask built: a cancel is a person's
|
||||
// exception, and a holder that refused every build while its grant was missing would
|
||||
// stop the mesh building for it.
|
||||
build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat}
|
||||
build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat, delivery: deliveryOf(msg)}
|
||||
if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err)
|
||||
} else if cancelled {
|
||||
@@ -336,6 +336,18 @@ type natsBuild struct {
|
||||
// seat is the role this build was taken from; what the machine says about it is that role's.
|
||||
seat string
|
||||
seq int
|
||||
// delivery is the how-manyth time the queue handed this ask out (novox/hq issue 594); 0 when the
|
||||
// server did not say.
|
||||
delivery int
|
||||
}
|
||||
|
||||
// deliveryOf is how many times the queue has handed this message out, this time included.
|
||||
func deliveryOf(msg *nats.Msg) int {
|
||||
meta, err := msg.Metadata()
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return int(meta.NumDelivered)
|
||||
}
|
||||
|
||||
func (b *natsBuild) Request() BuildRequest { return b.request }
|
||||
@@ -356,6 +368,11 @@ func (b *natsBuild) Announce(ctx context.Context, result BuildResult) error {
|
||||
if result.Failed == "" && result.Commit != "" {
|
||||
result.Module = ModuleOf(result.Manifest)
|
||||
}
|
||||
// **Which run this is** (novox/hq issue 594): stamped by the transport, which alone knows, so every
|
||||
// outcome carries it whatever the machine's code filled in.
|
||||
if result.Delivery == 0 {
|
||||
result.Delivery = b.delivery
|
||||
}
|
||||
body, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -375,11 +392,16 @@ func (b *natsBuild) Done() error { return b.msg.Ack() }
|
||||
func (b *natsBuild) Began(ctx context.Context) error {
|
||||
body, err := json.Marshal(BuildStart{
|
||||
ID: b.request.ID, Repository: b.request.Repository, Path: b.request.Path,
|
||||
Ref: b.request.Ref, On: b.on, At: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
Ref: b.request.Ref, On: b.on, At: time.Now().UTC().Format(time.RFC3339Nano), Delivery: b.delivery,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// **A start that is a restart says so in the build's log** (novox/hq issue 594): a reader following
|
||||
// the build sees why it began again, never a silent second run.
|
||||
if said := Restarted(b.delivery); said != "" {
|
||||
b.Say("restarted", said)
|
||||
}
|
||||
publish, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
if _, err := b.js.Context().Publish(BuildStartedOf(b.seat), body, nats.Context(publish)); err != nil {
|
||||
|
||||
@@ -401,3 +401,97 @@ func TestNatsAHolderSaysItTakesWorkOnceItIsBound(t *testing.T) {
|
||||
t.Fatal("a machine that could not bind said it takes work")
|
||||
}
|
||||
}
|
||||
|
||||
// **An ask handed out again says so** (novox/hq issue 594). A build agent replaced mid-build leaves its
|
||||
// ask unsettled, and the queue hands it out again to start over from the beginning. That second start
|
||||
// must not look like a first one: the machine that takes it says it restarted — in its started event,
|
||||
// in the build's log and in its outcome — so the queue and the pull request can say it too.
|
||||
func TestNatsAnAskHandedOutAgainSaysItRestarted(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
|
||||
started, err := js.Conn().SubscribeSync(BuildStarted())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = started.Unsubscribe() }()
|
||||
lines, err := js.Conn().SubscribeSync(BuildLog("b-594"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = lines.Unsubscribe() }()
|
||||
outcomes, err := js.Conn().SubscribeSync(BuildOutcome())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = outcomes.Unsubscribe() }()
|
||||
_ = js.Conn().Flush()
|
||||
|
||||
body, _ := json.Marshal(BuildRequest{ID: "b-594", Repository: "/r"})
|
||||
if _, err := js.Context().Publish(BuildWork(), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
machine := MachineOverNATS(js, "anchor")
|
||||
defer machine.Close()
|
||||
var first sync.Once
|
||||
go func() {
|
||||
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
||||
_ = work.Began(ctx)
|
||||
handedBack := false
|
||||
// The first time it is left unfinished — an agent replaced mid-build — and comes back.
|
||||
first.Do(func() { _ = work.Hold(100 * time.Millisecond); handedBack = true })
|
||||
if handedBack {
|
||||
return
|
||||
}
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
|
||||
var starts []BuildStart
|
||||
for len(starts) < 2 {
|
||||
msg, err := started.NextMsg(10 * time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("heard %d start(s) of an ask handed out twice: %v", len(starts), err)
|
||||
}
|
||||
var s BuildStart
|
||||
if err := json.Unmarshal(msg.Data, &s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
starts = append(starts, s)
|
||||
}
|
||||
if starts[0].Delivery != 1 || starts[1].Delivery != 2 {
|
||||
t.Fatalf("the starts say deliveries %d and %d, want 1 and 2: a restart reads as a first start",
|
||||
starts[0].Delivery, starts[1].Delivery)
|
||||
}
|
||||
|
||||
msg, err := outcomes.NextMsg(10 * time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var result BuildResult
|
||||
if err := json.Unmarshal(msg.Data, &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Delivery != 2 {
|
||||
t.Fatalf("the outcome says delivery %d, want 2: the pull request cannot say the check restarted",
|
||||
result.Delivery)
|
||||
}
|
||||
|
||||
sawRestarted := false
|
||||
for {
|
||||
msg, err := lines.NextMsg(time.Second)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
var line BuildLine
|
||||
if json.Unmarshal(msg.Data, &line) == nil && line.Step == "restarted" {
|
||||
sawRestarted = true
|
||||
}
|
||||
}
|
||||
if !sawRestarted {
|
||||
t.Fatal("the build's log never said it restarted")
|
||||
}
|
||||
}
|
||||
|
||||
+19
-7
@@ -73,6 +73,9 @@ type QueuedAsk struct {
|
||||
On string `json:"on,omitempty"`
|
||||
Was string `json:"was-on,omitempty"`
|
||||
Started time.Time `json:"started,omitempty"`
|
||||
// Delivery is the how-manyth time the ask was handed out, as its latest start said: more than 1 is
|
||||
// a build started again after an earlier taker left it (novox/hq issue 594).
|
||||
Delivery int `json:"delivery,omitempty"`
|
||||
}
|
||||
|
||||
// Queue is the seat's work queue as it stands.
|
||||
@@ -162,7 +165,7 @@ func ClassifyQueue(asks []QueuedAsk, delivered uint64, ackPending int, heard Bui
|
||||
}
|
||||
a.State = AskDead
|
||||
if s, ok := current[a.ID]; ok {
|
||||
a.On, a.Started = s.On, startedAt(s)
|
||||
a.On, a.Started, a.Delivery = s.On, startedAt(s), s.Delivery
|
||||
running = append(running, i)
|
||||
continue
|
||||
}
|
||||
@@ -183,7 +186,7 @@ func ClassifyQueue(asks []QueuedAsk, delivered uint64, ackPending int, heard Bui
|
||||
for _, i := range group {
|
||||
if slots == 0 {
|
||||
// Past what the worker counts: handed out as often as it may be.
|
||||
out[i].On, out[i].Started = "", time.Time{}
|
||||
out[i].On, out[i].Started, out[i].Delivery = "", time.Time{}, 0
|
||||
continue
|
||||
}
|
||||
out[i].State = AskInFlight
|
||||
@@ -321,11 +324,23 @@ func ReadBuildEvents(ctx context.Context, js *broker.JetStream, seat string, sin
|
||||
return heard, fmt.Errorf("cannot read %s from the bus: %w", subject, err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
for {
|
||||
batch, err := sub.Fetch(500, nats.MaxWait(2*time.Second))
|
||||
// **Read what is there, then stop** (novox/hq issue 591): the consumer says how many events it holds
|
||||
// from the moment asked, and the reading ends once they are read. A fetch for a full batch waits out
|
||||
// its whole wait when fewer are there, so every reading of the queue took two seconds and more.
|
||||
info, err := sub.ConsumerInfo()
|
||||
if err != nil {
|
||||
return heard, fmt.Errorf("cannot read %s from the bus: %w", subject, err)
|
||||
}
|
||||
for remaining := info.NumPending; remaining > 0; {
|
||||
batch, err := sub.Fetch(int(min(remaining, 500)), nats.MaxWait(2*time.Second))
|
||||
if err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.DeadlineExceeded) {
|
||||
return heard, fmt.Errorf("reading %s: %w", subject, err)
|
||||
}
|
||||
// None in the whole wait: what was counted is gone from the stream since (its limits), not coming.
|
||||
if len(batch) == 0 {
|
||||
break
|
||||
}
|
||||
remaining -= uint64(len(batch))
|
||||
for _, msg := range batch {
|
||||
switch msg.Subject {
|
||||
case BuildStartedOf(seat):
|
||||
@@ -342,9 +357,6 @@ func ReadBuildEvents(ctx context.Context, js *broker.JetStream, seat string, sin
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(batch) < 500 {
|
||||
break
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return heard, ctx.Err()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// CreateMigratedForTest creates the database name on the test PostgreSQL that admin reaches, holding
|
||||
// every one of migrations already applied and recorded, as a fresh database migrated by Migrate would.
|
||||
//
|
||||
// **A copy of a template, migrated once** (novox/hq issue 591): a store test used to create an empty
|
||||
// database and run every migration into it, some 90 for the inventory, and with hundreds of store tests
|
||||
// that alone was most of the controller's check. The template is migrated by Migrate itself, once for
|
||||
// each set of migrations, and every test's database is copied from it by PostgreSQL. The caller still
|
||||
// runs Migrate on the copy, which finds nothing pending only if the copy's record matches the migrations
|
||||
// exactly, so a template that disagreed with them would fail the test rather than pass it unseen.
|
||||
//
|
||||
// The template is named after a digest of the migrations, so a changed or added migration is a new
|
||||
// template, never an old one reused. Packages tested at once in separate processes share one: it is
|
||||
// built under an advisory lock, and a template a crashed build left half done is not marked a template
|
||||
// and is built again. It takes a *testing.T, so nothing that is not a test can call it.
|
||||
func CreateMigratedForTest(t *testing.T, admin, name string, migrations []Migration) {
|
||||
t.Helper()
|
||||
template, err := templateFor(admin, migrations)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot raise the migrated template: %v", err)
|
||||
}
|
||||
conn, err := pgx.Connect(t.Context(), admin)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot reach the test PostgreSQL: %v", err)
|
||||
}
|
||||
defer conn.Close(context.Background())
|
||||
// A copy is refused while anything is connected to its template; the template allows no connection
|
||||
// once built, so only the builder's last connection closing can still be in the way.
|
||||
for deadline := time.Now().Add(10 * time.Second); ; {
|
||||
_, err = conn.Exec(t.Context(), "create database "+name+" template "+template)
|
||||
var pgErr *pgconn.PgError
|
||||
if err == nil || !errors.As(err, &pgErr) || pgErr.Code != "55006" || time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("cannot create %s from %s: %v", name, template, err)
|
||||
}
|
||||
}
|
||||
|
||||
// templates are the templates this process has raised or found, by name.
|
||||
var templates sync.Map // name → *templateOnce
|
||||
|
||||
type templateOnce struct {
|
||||
once sync.Once
|
||||
err error
|
||||
}
|
||||
|
||||
// templateFor is the name of the template holding migrations, raised if no process has yet.
|
||||
func templateFor(admin string, migrations []Migration) (string, error) {
|
||||
digest := sha256.New()
|
||||
for _, m := range migrations {
|
||||
fmt.Fprintf(digest, "%d %s %s\n", m.Number, m.Name, m.Checksum)
|
||||
}
|
||||
sum := digest.Sum(nil)
|
||||
name := "tpl_" + hex.EncodeToString(sum[:12])
|
||||
held, _ := templates.LoadOrStore(name, &templateOnce{})
|
||||
once := held.(*templateOnce)
|
||||
once.once.Do(func() {
|
||||
// Not the asking test's context: the template outlives that test, and is raised once for all.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
once.err = raiseTemplate(ctx, admin, name, int64(binary.BigEndian.Uint64(sum[12:20])), migrations)
|
||||
})
|
||||
return name, once.err
|
||||
}
|
||||
|
||||
// raiseTemplate builds the template name under the advisory lock key, unless it stands built.
|
||||
func raiseTemplate(ctx context.Context, admin, name string, key int64, migrations []Migration) error {
|
||||
conn, err := pgx.Connect(ctx, admin)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Close(context.Background())
|
||||
if _, err := conn.Exec(ctx, `select pg_advisory_lock($1)`, key); err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _, _ = conn.Exec(context.Background(), `select pg_advisory_unlock($1)`, key) }()
|
||||
|
||||
var built bool
|
||||
if err := conn.QueryRow(ctx, `select exists (select 1 from pg_database where datname = $1 and datistemplate)`,
|
||||
name).Scan(&built); err != nil {
|
||||
return err
|
||||
}
|
||||
if built {
|
||||
return nil
|
||||
}
|
||||
if _, err := conn.Exec(ctx, "drop database if exists "+name+" with (force)"); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := conn.Exec(ctx, "create database "+name); err != nil {
|
||||
return err
|
||||
}
|
||||
cut := strings.LastIndex(admin, "/")
|
||||
rest := ""
|
||||
if q := strings.Index(admin[cut:], "?"); q >= 0 {
|
||||
rest = admin[cut+q:]
|
||||
}
|
||||
pool, err := pgxpool.New(ctx, admin[:cut]+"/"+name+rest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = (&Store{context: "template", pool: pool}).Migrate(ctx, migrations)
|
||||
pool.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Marked a template only once whole, and closed to connections, so a copy is never refused for one.
|
||||
for _, sql := range []string{
|
||||
"alter database " + name + " with allow_connections false",
|
||||
"select pg_terminate_backend(pid) from pg_stat_activity where datname = '" + name + "'",
|
||||
"alter database " + name + " with is_template true",
|
||||
} {
|
||||
if _, err := conn.Exec(ctx, sql); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Vendored
+2
-2
@@ -8,7 +8,7 @@ private addresses. That is acceptable while mesh-controller is private; if it ev
|
||||
a second place to clean besides the catalogue itself.
|
||||
|
||||
mesh-catalog b9de001833b1b61b297182b8e3bcdae1cbdeace9 modules/*/module.json, modules/nats/Dockerfile
|
||||
mesh-host a76baa4b47f07a8b2bab065603414530bd02b53a examples/foundation-first-node-nats.lock
|
||||
mesh-host 7a41e120635b3f7b6cb94cca8b3fc4093e4e6110 examples/foundation-first-node.lock
|
||||
mesh-sdk 33917f91ac3cc213b879b32f3c70e2aa70395f98 conformance/events/module-event.json
|
||||
|
||||
To move them, from this repository's root, with the two repositories checked out beside it:
|
||||
@@ -18,7 +18,7 @@ To move them, from this repository's root, with the two repositories checked out
|
||||
git -C ../mesh-catalog archive <commit> modules | tar -x -C testdata/beside/mesh-catalog --wildcards \
|
||||
'modules/*/module.json' 'modules/nats/Dockerfile'
|
||||
find testdata/beside -name module.json -exec mv {} {}.captured \;
|
||||
git -C ../mesh-host archive <commit> examples/foundation-first-node-nats.lock | tar -x -C testdata/beside/mesh-host
|
||||
git -C ../mesh-host archive <commit> examples/foundation-first-node.lock | tar -x -C testdata/beside/mesh-host
|
||||
|
||||
and write the commits here. The SDK is captured at the commit go.mod pins for git.novox.be/novox/mesh-sdk/go
|
||||
(the last part of its pseudo-version; novox/hq issue 449), which internal/link's conformance test holds it to:
|
||||
|
||||
+25
-8
@@ -1,11 +1,17 @@
|
||||
// foundation-first-node-nats.lock — what a machine must be before a mesh exists, on the bus being
|
||||
// built (novox/hq ADR 0106, design 25).
|
||||
// foundation-first-node.lock — what a machine must be before a mesh exists: the one genesis bundle
|
||||
// (novox/hq ADR 0331 §2), on the bus the mesh runs on (ADR 0106, ADR 0131, design 25).
|
||||
//
|
||||
// The same twelve steps as foundation-first-node.lock, with one difference that matters: **the mesh
|
||||
// writes its own user list, and at genesis there is no mesh yet to write it.** So this carries the
|
||||
// first one — the controller's own account, at a well-known bootstrap password, exactly as the store
|
||||
// is reached at `postgres:bootstrap` and the old bus at `guest:guest`. It is rotated with those, and
|
||||
// from the controller's first composition onward the file is the controller's to write.
|
||||
// The whole bootstrap (novox/hq 03-DESIGN/01-to-be/07-the-foundation.md): a container runtime, a
|
||||
// filter, a store, a database per context, those contexts' schemas, the bus with its certificate
|
||||
// and its first user list, and the control plane running on top of them.
|
||||
//
|
||||
// **The mesh writes its own user list, and at genesis there is no mesh yet to write it.** So this
|
||||
// carries the first one — the controller's own account — and the control plane's address names the
|
||||
// same account. Both say `bootstrap`, and both are placeholders: the installer makes the account's
|
||||
// password, puts it in the address and its hash in the list (mesh-host internal/bootstrap
|
||||
// rootsecrets.go), and refuses a bundle whose bus connection it does not recognise. Until the first
|
||||
// node has enrolled the installer places the list the controller composes; from the bus's adoption
|
||||
// as the `nats` module on, the module receives it like any module (ADR 0331 §1, §5).
|
||||
//
|
||||
// The accounts file is its own file beside the server's configuration, because the server's own
|
||||
// settings belong to whoever raises it and the users belong to the mesh (design 25 §4). Both live in
|
||||
@@ -15,7 +21,18 @@
|
||||
//
|
||||
// No `verify` on the TLS block, deliberately — that setting makes the server demand a *client*
|
||||
// certificate, and nothing in the mesh presents one: a host pins this server's exact certificate and
|
||||
// authenticates with a password (ADR 0004, design 25 §4).
|
||||
// authenticates with a password (ADR 0004, design 25 §4). The control plane pins it too, through
|
||||
// MESH_BROKER_CERTIFICATE, and dials the bus on the machine's own loopback.
|
||||
//
|
||||
// PINNED BY DIGEST, and the digest is not decoration: a tag can be made to point at a different
|
||||
// image, and this file is applied on a machine with no mesh to ask about anything (ADR 0006). The
|
||||
// registry prefix is the lab's; the digests are upstream's.
|
||||
//
|
||||
// The store's data is a NAMED VOLUME, not a directory on the machine. A directory the host
|
||||
// creates is owned by root, and the database runs as somebody else inside the container — so it
|
||||
// could not write, and the container crash-looped. A named volume lets the image set up its own
|
||||
// ownership, and outlives the container, which is what you want for the thing holding the mesh's
|
||||
// state.
|
||||
|
||||
{
|
||||
"declaration": 1,
|
||||
@@ -523,6 +523,20 @@ func refusedPlaces(resources []declaration.Resource) map[string]error {
|
||||
switch res := r.(type) {
|
||||
case *declaration.Directory:
|
||||
err = refusePath(res.Path, placingDirectory, moduleOfID(res.ID), res.Owner)
|
||||
// And outside every home, the walk's judgement before anything is applied (novox/hq issue 358): a
|
||||
// link another account could make in its path refuses it, and with it what is below it, whatever
|
||||
// order they are declared in. The apply walks again at the moment of use. A directory handed over
|
||||
// keeps its own, stricter walk (issue 356), which refuses any link at all.
|
||||
if err == nil && meshDirsByDescriptor && homeAbove(res.Path) == "" &&
|
||||
!HandedOver()[handOverKey(res.ID, res.Path)] {
|
||||
if judged := judgeMeshDir(res.Path); judged != nil {
|
||||
err = judged
|
||||
var link *LinkInPathError
|
||||
if errors.As(judged, &link) {
|
||||
err = &PlacementRefusedError{Path: res.Path, Why: "it " + link.why()}
|
||||
}
|
||||
}
|
||||
}
|
||||
case *declaration.Access:
|
||||
err = refusePath(res.Path, placingAccess, moduleOfID(res.ID), "")
|
||||
default:
|
||||
|
||||
Vendored
+1
@@ -37,3 +37,4 @@
|
||||
./cmd/mesh-controller TestReplay401
|
||||
./cmd/mesh-controller TestReplay201ThePlansSend
|
||||
./internal/inventory TestAWalksRestIsReadFromEachMachinesOwnSend
|
||||
./internal/inventory TestAStoredBuildWithASilentMergeableFileIsStillReadAndComposed
|
||||
|
||||
Reference in New Issue
Block a user