Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0173d9c70e | ||
|
|
89abce5df9 | ||
|
|
185d0294eb | ||
|
|
4d4d51eb01 | ||
|
|
b7d996878b |
@@ -218,6 +218,15 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan, onPath map[string]st
|
||||
}
|
||||
return oneWalkAtATime(p, plans, onPath)
|
||||
}
|
||||
// **A walk lost to silence is resumed** (novox/hq ADR 0531, issue 1886): its tier was sent whole and only the gate after it failed it, every
|
||||
// machine it waited on silent; once they reported, it goes on from that gate.
|
||||
if len(stopped) == 0 && lostToSilence(p) {
|
||||
if q, found := newerOpenPlan(p, plans); found {
|
||||
return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and resuming %s would send "+
|
||||
"what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID)
|
||||
}
|
||||
return oneWalkAtATime(p, plans, onPath)
|
||||
}
|
||||
if len(stopped) == 0 {
|
||||
return fmt.Errorf("nothing in tier %d of %s failed to build or stopped rolling out — it stopped at: %s",
|
||||
p.Tier, p.ID, p.Note)
|
||||
@@ -412,6 +421,20 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||
}
|
||||
return said, nil
|
||||
}
|
||||
if len(failed) == 0 && lostToSilence(p) {
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
said, err := resumeLost(&p, silentNodes(p, reports, time.Now().UTC(), provLookup(ctx)))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return said, nil
|
||||
}
|
||||
if len(failed) == 0 {
|
||||
return retryRollouts(ctx, open, &p)
|
||||
}
|
||||
@@ -693,3 +716,51 @@ func resumeStopped(p *inventory.Plan) string {
|
||||
return fmt.Sprintf("%s resumed at tier %d of %d, stopped by %s: %s; this verb asks nothing — the next look carries it "+
|
||||
"on as any walk", p.ID, p.Tier+1, len(p.Tiers), by, waits)
|
||||
}
|
||||
|
||||
// silentGateWords say, in a plan's note, that the gate between two tiers found every machine it waits on silent past
|
||||
// ApplySilentAfter, and failed the plan as lost (issue 1160, ADR 0531): one text for the writer and the reader (lostToSilence).
|
||||
const silentGateWords = "every machine it waits on said nothing on its send within"
|
||||
|
||||
// lostToSilence is whether a plan failed only at the gate after its tier, every machine it waited on silent (ADR 0531,
|
||||
// issue 1886): nothing in its tier failed to build, stopped rolling out or is still to ask — the look had sent the whole tier.
|
||||
func lostToSilence(p inventory.Plan) bool {
|
||||
if p.State != inventory.PlanFailed || p.Tier >= len(p.Tiers) || !strings.Contains(p.Note, silentGateWords) ||
|
||||
len(failedIn(p)) > 0 || len(stoppedRollouts(p)) > 0 {
|
||||
return false
|
||||
}
|
||||
asked, _, unasked := waitingIn(p)
|
||||
return asked == 0 && unasked == 0
|
||||
}
|
||||
|
||||
// resumeLost sets a walk lost to silence building again at the tier it failed after (ADR 0531, issue 1886), for the next look to
|
||||
// judge its gate again on the machines' reports, unless every machine its tier was sent to is still silent: then
|
||||
// nothing it needs is delivered anywhere yet, and the retry is refused as the gate would fail it again.
|
||||
func resumeLost(p *inventory.Plan, silent map[string]bool) (string, error) {
|
||||
var sent, still []string
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil {
|
||||
for n := range s.Rest {
|
||||
if !slices.Contains(sent, n) {
|
||||
sent = append(sent, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, n := range sent {
|
||||
if silent[n] {
|
||||
still = append(still, n)
|
||||
}
|
||||
}
|
||||
sort.Strings(still)
|
||||
if len(sent) > 0 && len(still) == len(sent) {
|
||||
return "", fmt.Errorf("%s failed because every machine its tier %d waits on said nothing on its send, and %s "+
|
||||
"still said nothing: retry once it reported", p.ID, p.Tier+1, strings.Join(still, ", "))
|
||||
}
|
||||
was := p.Note
|
||||
p.State = inventory.PlanBuilding
|
||||
goesOnAgain(p)
|
||||
p.Note = fmt.Sprintf("tier %d resumed by hand after it failed as lost (%s): its gate is judged again on the "+
|
||||
"machines' reports", p.Tier, was)
|
||||
return fmt.Sprintf("%s resumed at tier %d of %d, failed as lost to silence (%s); this verb asks and sends nothing — "+
|
||||
"the next look judges its gate again and carries it on as any walk", p.ID, p.Tier+1, len(p.Tiers), was), nil
|
||||
}
|
||||
|
||||
@@ -260,7 +260,7 @@ type gateNeed struct {
|
||||
}
|
||||
|
||||
// silentNodes is the machines a walk sent and heard nothing from past ApplySilentAfter, on whichever tier it sent
|
||||
// them (novox/hq ADR 0360 §5, issue 1160): the one rule both the gates between tiers and the walk's end read, through
|
||||
// them (novox/hq ADR 0360 §7, issue 1160): the one rule both the gates between tiers and the walk's end read, through
|
||||
// walkReports. A machine that reports on its send is not silent from then on.
|
||||
func silentNodes(p inventory.Plan, reports []inventory.Reported, now time.Time, prov inventory.ProvLookup) map[string]bool {
|
||||
_, _, silent := walkReports(p, reports, now, prov)
|
||||
@@ -272,7 +272,7 @@ func silentNodes(p inventory.Plan, reports []inventory.Reported, now time.Time,
|
||||
}
|
||||
|
||||
// withoutSilent is the machines of running that are not silent: a later tier is not sent to a machine the walk heard
|
||||
// nothing from, which keeps its want of the plan's builds for the send it gets when it is heard again (ADR 0360 §5).
|
||||
// nothing from, which keeps its want of the plan's builds for the send it gets when it is heard again (ADR 0360 §7).
|
||||
func withoutSilent(running []string, silent map[string]bool) []string {
|
||||
return slices.DeleteFunc(slices.Clone(running), func(n string) bool { return silent[n] })
|
||||
}
|
||||
@@ -1129,7 +1129,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
// being behind. Now every module of the tier whose first machine is the same is sent there in one
|
||||
// send, judged there by one gate — kept on the first of them, the others gated by it — and the rest
|
||||
// of the tier's machines are sent once, together, when the gates have passed.
|
||||
// **A machine silent past the bound is left as it was, on every tier** (novox/hq ADR 0360 §5, issue 1160): the
|
||||
// **A machine silent past the bound is left as it was, on every tier** (novox/hq ADR 0360 §7, issue 1160): the
|
||||
// walk's end names it, and the gates between tiers and the sends of the tiers after do not wait for it or go to it.
|
||||
allReports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
@@ -1457,8 +1457,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
// Every machine the next tier waits on is silent: nothing it needs was delivered anywhere, so the plan does
|
||||
// not go on as if it had been (issue 1160).
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("tier %d built; every machine it waits on said nothing on its send within %s: %s",
|
||||
p.Tier, applySilentAfter, strings.Join(quiet, ", "))
|
||||
p.Note = fmt.Sprintf("tier %d built; %s %s: %s", p.Tier, silentGateWords, applySilentAfter, strings.Join(quiet, ", "))
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return true, nil
|
||||
}
|
||||
@@ -1476,7 +1475,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
return changed, nil
|
||||
}
|
||||
}
|
||||
// **A walk ends on its machines' reports** (novox/hq ADR 0360 §4, §5): after its last tier, every machine it sent
|
||||
// **A walk ends on its machines' reports** (novox/hq ADR 0360 §4, §7): after its last tier, every machine it sent
|
||||
// has reported on that send, or is silent past ApplySilentAfter; never a health reading.
|
||||
end := ""
|
||||
if p.Tier == len(p.Tiers)-1 {
|
||||
@@ -1543,50 +1542,62 @@ func walkReports(p inventory.Plan, reports []inventory.Reported, now time.Time,
|
||||
carried[node] = append(carried[node], m)
|
||||
}
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
// **Each machine on the walk's last send to it** (novox/hq ADR 0531, issue 1886): a send carries the machine's whole declaration (ADR
|
||||
// 0221), so a report on the last one answers every earlier one, and the bound runs from it. Judged on every send
|
||||
// it had made, a walk whose tiers were sent hours apart — R4e, stopped at tier 1 and resumed — held tier 0's send of
|
||||
// 00:23 against tier 1's templates and named its machines silent 35 seconds after tier 1 was sent.
|
||||
last := map[string]inventory.SentDeclaration{}
|
||||
lastAt := map[string]time.Time{}
|
||||
for _, s := range p.Modules {
|
||||
if s == nil {
|
||||
continue
|
||||
}
|
||||
for node, sent := range s.Rest {
|
||||
sent = sent.Only(carried[node]...)
|
||||
if seen[node] {
|
||||
continue
|
||||
}
|
||||
r := byNode[node]
|
||||
at := sent.At
|
||||
if at.IsZero() && s.SentAt != nil {
|
||||
at = *s.SentAt
|
||||
}
|
||||
// A module handed to the provisioner is applied only once its client said so too (ADR 0362 §1-5).
|
||||
outcome, reported := provisionedOutcome(node, sent, r, r.At != nil && sent.ReportsOn(r), prov)
|
||||
switch {
|
||||
case reported:
|
||||
seen[node] = true
|
||||
switch {
|
||||
case outcome == inventory.OutcomeFailed || outcome == inventory.OutcomeRefused:
|
||||
failed = append(failed, node)
|
||||
case strings.HasPrefix(outcome, inventory.OutcomeProvisioningFailed):
|
||||
failed = append(failed, node+" ("+outcome+")")
|
||||
}
|
||||
case !at.IsZero() && now.Sub(at) > applySilentAfter:
|
||||
seen[node] = true
|
||||
silent = append(silent, node)
|
||||
default:
|
||||
if !slices.Contains(waiting, node) {
|
||||
waiting = append(waiting, node)
|
||||
}
|
||||
was, ok := last[node]
|
||||
if !ok || laterSend(sent, at, was, lastAt[node]) {
|
||||
last[node], lastAt[node] = sent, at
|
||||
}
|
||||
}
|
||||
}
|
||||
// A machine one module's send was reported on and another's not yet is the same send: reported.
|
||||
waiting = slices.DeleteFunc(waiting, func(n string) bool { return seen[n] })
|
||||
for node, sent := range last {
|
||||
at := lastAt[node]
|
||||
sent = sent.Only(carried[node]...)
|
||||
r := byNode[node]
|
||||
// A module handed to the provisioner is applied only once its client said so too (ADR 0362 §1-5).
|
||||
outcome, reported := provisionedOutcome(node, sent, r, r.At != nil && sent.ReportsOn(r), prov)
|
||||
switch {
|
||||
case reported:
|
||||
switch {
|
||||
case outcome == inventory.OutcomeFailed || outcome == inventory.OutcomeRefused:
|
||||
failed = append(failed, node)
|
||||
case strings.HasPrefix(outcome, inventory.OutcomeProvisioningFailed):
|
||||
failed = append(failed, node+" ("+outcome+")")
|
||||
}
|
||||
case !at.IsZero() && now.Sub(at) > applySilentAfter:
|
||||
silent = append(silent, node)
|
||||
default:
|
||||
waiting = append(waiting, node)
|
||||
}
|
||||
}
|
||||
sort.Strings(waiting)
|
||||
sort.Strings(failed)
|
||||
sort.Strings(silent)
|
||||
return waiting, failed, silent
|
||||
}
|
||||
|
||||
// laterSend is whether a, sent at aAt, is a later send to a machine than b, sent at bAt: by its sequence when both carry
|
||||
// one, else by when it went.
|
||||
func laterSend(a inventory.SentDeclaration, aAt time.Time, b inventory.SentDeclaration, bAt time.Time) bool {
|
||||
if a.Sequence > 0 && b.Sequence > 0 && a.Sequence != b.Sequence {
|
||||
return a.Sequence > b.Sequence
|
||||
}
|
||||
return aAt.After(bAt)
|
||||
}
|
||||
|
||||
// restOf is, for one module, every machine of its rest that the send reached and what it carried there.
|
||||
func restOf(to, sent []string, what map[string]inventory.SentDeclaration) map[string]inventory.SentDeclaration {
|
||||
out := map[string]inventory.SentDeclaration{}
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// Issue 1886, the resume of R4e (plan-1792707174627079192): tier 0 was sent at 00:23:28, the walk was stopped at
|
||||
// 02:01 and resumed at 07:43, and tier 1 was sent at 07:49:54. 35 seconds later the gate named ace silent "within
|
||||
// 15m0s", and at 07:51:30 shanks, in the same second shanks reported it applied: the plan failed as lost. The walk
|
||||
// judged each machine on every send it had made it, tier 0's of 00:23 too, narrowed to every module the walk carried
|
||||
// there — so tier 0's send was held against the templates of tier 1's modules it never carried, and silence was
|
||||
// measured from 00:23. A machine is judged on the walk's last send to it: that send carries its whole declaration.
|
||||
|
||||
func r4eResumed() (inventory.Plan, []inventory.Reported, inventory.ProvLookup, time.Time) {
|
||||
tier0 := time.Date(2026, 10, 22, 22, 23, 28, 0, time.UTC)
|
||||
tier1 := time.Date(2026, 10, 23, 5, 49, 54, 0, time.UTC)
|
||||
now := time.Date(2026, 10, 23, 5, 51, 30, 0, time.UTC)
|
||||
t0, t1 := tier0, tier1
|
||||
p := inventory.Plan{ID: "plan-1792707174627079192", State: inventory.PlanRolling, Tier: 1,
|
||||
Tiers: [][]string{{"pacman"}, {"mesh-build-image"}, {"gitea"}},
|
||||
Modules: map[string]*inventory.PlanModule{
|
||||
"pacman": {State: "built", SentAt: &t0, Rest: map[string]inventory.SentDeclaration{
|
||||
"shanks": {Digest: "d0-shanks", Sequence: 900, At: tier0,
|
||||
Provisioned: map[string]string{"pacman": "pac-1", "mesh-build-image": "img-old"}},
|
||||
"ace": {Digest: "d0-ace", Sequence: 1100, At: tier0,
|
||||
Provisioned: map[string]string{"pacman": "pac-1", "mesh-build-image": "img-old"}}}},
|
||||
"mesh-build-image": {State: "built", SentAt: &t1, Rest: map[string]inventory.SentDeclaration{
|
||||
"shanks": {Digest: "d1-shanks", Sequence: 971, At: tier1,
|
||||
Provisioned: map[string]string{"pacman": "pac-1", "mesh-build-image": "img-new"}},
|
||||
"ace": {Digest: "d1-ace", Sequence: 1212, At: tier1,
|
||||
Provisioned: map[string]string{"pacman": "pac-1", "mesh-build-image": "img-new"}}}},
|
||||
"gitea": {}}}
|
||||
shanksAt := now
|
||||
aceAt := time.Date(2026, 10, 23, 0, 57, 50, 0, time.UTC)
|
||||
reports := []inventory.Reported{
|
||||
// shanks reported on tier 1's send in the second the gate looked.
|
||||
{Node: "shanks", Outcome: inventory.OutcomeApplied, At: &shanksAt, Declared: "d1-shanks", Current: true,
|
||||
ReportedSequence: 971},
|
||||
// ace's last report is from before tier 1's send, on a declaration sequenced after tier 0's.
|
||||
{Node: "ace", Outcome: inventory.OutcomeApplied, At: &aceAt, Declared: "d-ace-0257", ReportedSequence: 1150},
|
||||
}
|
||||
says := map[string]inventory.ProvReport{
|
||||
"shanks/pacman": {Status: inventory.ProvApplied, Templates: "pac-1", At: now},
|
||||
"shanks/mesh-build-image": {Status: inventory.ProvApplied, Templates: "img-new", At: now},
|
||||
"ace/pacman": {Status: inventory.ProvApplied, Templates: "pac-1", At: aceAt},
|
||||
// ace's client had not placed the old image's files (issue 1884), and has not yet the new one's.
|
||||
"ace/mesh-build-image": {Status: inventory.ProvWaiting, Templates: "img-old", At: aceAt},
|
||||
}
|
||||
prov := func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok }
|
||||
return p, reports, prov, now
|
||||
}
|
||||
|
||||
func TestAResumedWalkJudgesEachMachineOnItsLastSend(t *testing.T) {
|
||||
p, reports, prov, now := r4eResumed()
|
||||
waiting, failed, silent := walkReports(p, reports, now, prov)
|
||||
if len(silent) != 0 {
|
||||
t.Fatalf("1886: %v named silent within %s of tier 1's send, from tier 0's send of 00:23", silent, applySilentAfter)
|
||||
}
|
||||
if len(failed) != 0 || !slices.Equal(waiting, []string{"ace"}) {
|
||||
t.Fatalf("1886: waiting %v, failed %v; want ace waited for (sent 96 s ago) and shanks reported", waiting, failed)
|
||||
}
|
||||
// The gate the plan failed at: mesh-build-image runs on ace and shanks, which tier 2 needs.
|
||||
needs := []gateNeed{{module: "mesh-build-image", running: []string{"ace", "shanks"},
|
||||
since: func(string) time.Time { return *p.Modules["mesh-build-image"].SentAt }}}
|
||||
w, quiet, lost := settleTierGate(needs, reports, silentNodes(p, reports, now, prov))
|
||||
if lost || len(quiet) != 0 || !slices.Equal(w, []string{"mesh-build-image on ace"}) {
|
||||
t.Fatalf("1886: the gate says waiting %v, silent %v, lost %v; want mesh-build-image on ace waited for", w, quiet, lost)
|
||||
}
|
||||
}
|
||||
|
||||
// The machine silent on the walk's last send to it is still silent once the bound has passed since that send.
|
||||
func TestAMachineSilentOnItsLastSendIsStillSilent(t *testing.T) {
|
||||
p, reports, prov, now := r4eResumed()
|
||||
later := now.Add(applySilentAfter + time.Minute)
|
||||
_, _, silent := walkReports(p, reports, later, prov)
|
||||
if !slices.Equal(silent, []string{"ace"}) {
|
||||
t.Fatalf("1886: silent %v past the bound from tier 1's send; want ace", silent)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk failed only because every machine its gate waited on was silent, whose tiers are all sent, is carried on by
|
||||
// `plans retry` once those machines reported (issue 1886): it failed on no build and no rollout, so the retry of failed
|
||||
// builds and of stopped rollouts refused it with "nothing in tier 1 … failed to build or stopped rolling out".
|
||||
func r4eLost() inventory.Plan {
|
||||
p, _, _, _ := r4eResumed()
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = "tier 1 built; " + silentGateWords + " 15m0s: shanks"
|
||||
// mesh-tools is built and recorded, never sent, as R4e's was.
|
||||
p.Tiers[1] = append(p.Tiers[1], "mesh-tools")
|
||||
p.Modules["mesh-tools"] = &inventory.PlanModule{State: "built"}
|
||||
return p
|
||||
}
|
||||
|
||||
func TestRetryCarriesOnAWalkLostToSilenceOnceItsMachinesReported(t *testing.T) {
|
||||
p, reports, prov, now := r4eResumed()
|
||||
p = r4eLost()
|
||||
if !lostToSilence(p) {
|
||||
t.Fatalf("1886: the plan R4e failed as is not read as lost to silence: %q", p.Note)
|
||||
}
|
||||
if err := retryRefusal(p, []inventory.Plan{p}, ownPathOf(nil)); err != nil {
|
||||
t.Fatalf("1886: a walk lost to silence was refused a retry: %v", err)
|
||||
}
|
||||
reports[1].Current, reports[1].Declared, reports[1].ReportedSequence = true, "d1-ace", 1212
|
||||
reports[1].At = &now
|
||||
says := map[string]inventory.ProvReport{}
|
||||
for _, n := range []string{"ace", "shanks"} {
|
||||
says[n+"/pacman"] = inventory.ProvReport{Status: inventory.ProvApplied, Templates: "pac-1", At: now}
|
||||
says[n+"/mesh-build-image"] = inventory.ProvReport{Status: inventory.ProvApplied, Templates: "img-new", At: now}
|
||||
}
|
||||
prov = func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok }
|
||||
said, err := resumeLost(&p, silentNodes(p, reports, now.Add(time.Hour), prov))
|
||||
if err != nil {
|
||||
t.Fatalf("1886: the resume of a walk whose machines reported refused: %v", err)
|
||||
}
|
||||
if p.State != inventory.PlanBuilding || p.Tier != 1 {
|
||||
t.Fatalf("1886: resumed walk is %s at tier %d; want building at tier 1, for the next look to pass its gate", p.State, p.Tier)
|
||||
}
|
||||
if !strings.Contains(p.Note, "resumed by hand") || !strings.Contains(said, "resumed at tier 2 of 3") {
|
||||
t.Fatalf("1886: note %q, said %q", p.Note, said)
|
||||
}
|
||||
}
|
||||
|
||||
// Every machine its tier was sent to still silent: the gate would fail it again, so the retry says so instead.
|
||||
func TestRetryRefusesAWalkLostToSilenceWhileEveryMachineIsStillSilent(t *testing.T) {
|
||||
p := r4eLost()
|
||||
if _, err := resumeLost(&p, map[string]bool{"ace": true, "shanks": true}); err == nil || !strings.Contains(err.Error(), "ace, shanks") {
|
||||
t.Fatalf("1886: a walk whose machine is still silent was resumed: %v", err)
|
||||
}
|
||||
if p.State != inventory.PlanFailed {
|
||||
t.Fatalf("1886: a refused resume changed the plan to %s", p.State)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk lost to silence with something of its tier never sent is not this: its send is the look's to make.
|
||||
func TestALostWalkWithATierUnsentIsNotLostToSilence(t *testing.T) {
|
||||
p := r4eLost()
|
||||
p.Modules["mesh-build-image"].State = "asked"
|
||||
if lostToSilence(p) {
|
||||
t.Fatal("1886: a tier with a module still asked was read as lost to silence")
|
||||
}
|
||||
p = r4eLost()
|
||||
p.Note = "tier 1 built; mesh-build-image failed to build"
|
||||
if lostToSilence(p) {
|
||||
t.Fatal("1886: a plan failed by a build was read as lost to silence")
|
||||
}
|
||||
}
|
||||
|
||||
// A walk lost to silence is refused its resume beside a newer open plan of its repository and branch, which supersedes
|
||||
// it, and beside a related open plan: one plan at a time (ADR 0531).
|
||||
func TestRetryRefusesAWalkLostToSilenceBesideANewerOrARelatedOpenPlan(t *testing.T) {
|
||||
p := r4eLost()
|
||||
created := time.Date(2026, 10, 22, 22, 15, 9, 0, time.UTC)
|
||||
p.Repository, p.Branch, p.Commit, p.Created = "novox/mesh-sdk", "main", "7aacbc16", created
|
||||
newerOpen := inventory.Plan{ID: "plan-newer", Repository: "novox/mesh-sdk", Branch: "main", Commit: "d00dfeed",
|
||||
Created: created.Add(time.Hour), State: inventory.PlanBuilding, Tiers: [][]string{{"zsh"}},
|
||||
Modules: map[string]*inventory.PlanModule{"zsh": {State: "asked"}}}
|
||||
if err := retryRefusal(p, []inventory.Plan{p, newerOpen}, ownPathOf(nil)); err == nil ||
|
||||
!strings.Contains(err.Error(), "plan-newer supersedes it") {
|
||||
t.Errorf("1886: a walk lost to silence was resumed under a newer open plan of its branch: %v", err)
|
||||
}
|
||||
beside := inventory.Plan{ID: "plan-beside", Repository: "novox/mesh-sdk", Branch: "release", Commit: "bbadbb2a",
|
||||
Created: created.Add(time.Hour), State: inventory.PlanBuilding, Tiers: [][]string{{"mesh-build-image"}},
|
||||
Modules: map[string]*inventory.PlanModule{"mesh-build-image": {State: "asked"}}}
|
||||
if err := retryRefusal(p, []inventory.Plan{p, beside}, ownPathOf(nil)); err == nil ||
|
||||
!strings.Contains(err.Error(), "one walk at a time") {
|
||||
t.Errorf("1886: a walk lost to silence was resumed beside a related open plan: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -37,8 +37,7 @@ func ForTest(t *testing.T) *Identity {
|
||||
}
|
||||
store.CreateMigratedForTest(t, admin, name, migrations)
|
||||
|
||||
cut := strings.LastIndex(admin, "/")
|
||||
t.Setenv(store.Variable(Name), admin[:cut]+"/"+name+"?sslmode=disable")
|
||||
t.Setenv(store.Variable(Name), store.DatabaseForTest(admin, name))
|
||||
|
||||
ident, err := Open(t.Context())
|
||||
if err != nil {
|
||||
|
||||
@@ -7,61 +7,19 @@ import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/novox/mesh-controller/internal/store"
|
||||
)
|
||||
|
||||
// fresh is the package's own name for ForTest: one fixture for every store test of identity, so the way
|
||||
// a test's database is made and addressed is kept true in one place (novox/hq issue 1889: this copy
|
||||
// dropped the store address's query and reached no server on the checker's socket-addressed store).
|
||||
func fresh(t *testing.T) *Identity {
|
||||
t.Helper()
|
||||
admin := os.Getenv("MESH_TEST_POSTGRES")
|
||||
if admin == "" {
|
||||
t.Skip("no MESH_TEST_POSTGRES; run `make check` to raise one")
|
||||
}
|
||||
name := fmt.Sprintf("ident_%d", time.Now().UnixNano()%10_000_000)
|
||||
|
||||
conn, err := pgx.Connect(t.Context(), admin)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot reach the test PostgreSQL: %v", err)
|
||||
}
|
||||
if _, err := conn.Exec(t.Context(), "create database "+name); err != nil {
|
||||
t.Fatalf("cannot create %s: %v", name, err)
|
||||
}
|
||||
conn.Close(t.Context())
|
||||
|
||||
cut := strings.LastIndex(admin, "/")
|
||||
t.Setenv(store.Variable(Name), admin[:cut]+"/"+name+"?sslmode=disable")
|
||||
|
||||
ident, err := Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
ident.Close()
|
||||
c, err := pgx.Connect(context.Background(), admin)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer c.Close(context.Background())
|
||||
_, _ = c.Exec(context.Background(), "drop database if exists "+name+" with (force)")
|
||||
})
|
||||
if err := ident.Ready(t.Context(), 20*time.Second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
migrations, err := Migrations()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ident.store.Migrate(t.Context(), migrations); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return ident
|
||||
return ForTest(t)
|
||||
}
|
||||
|
||||
func TestNoKeyIsAnErrorRatherThanAnEmptyKey(t *testing.T) {
|
||||
|
||||
@@ -675,6 +675,12 @@ func tierLeftOf(old *Plan, p Plan, now time.Time) (time.Time, *Duration) {
|
||||
since = old.Created
|
||||
}
|
||||
wasOpen := old.State == PlanBuilding || old.State == PlanRolling
|
||||
if !wasOpen && p.Open() {
|
||||
// **A walk set open again enters its tier anew** (issue 1886): a resume or a retry at the tier it ended at. Kept,
|
||||
// the entry of before its stop measured the tier from hours before the resume — R4e read "waiting … for 5h48m53s"
|
||||
// seven minutes after it was resumed — and the watchdog of its progress (S3) found it late at once.
|
||||
return now, nil
|
||||
}
|
||||
if !wasOpen || (old.Tier == p.Tier && p.Open()) {
|
||||
return since, nil // still in the tier, or already ended
|
||||
}
|
||||
|
||||
@@ -42,8 +42,7 @@ func ForTest(t *testing.T) *Inventory {
|
||||
store.CreateMigratedForTest(t, admin, name, migrations)
|
||||
busPasswordCostForTest.Store(int32(bcrypt.MinCost))
|
||||
|
||||
cut := strings.LastIndex(admin, "/")
|
||||
t.Setenv(store.Variable(Name), admin[:cut]+"/"+name+"?sslmode=disable")
|
||||
t.Setenv(store.Variable(Name), store.DatabaseForTest(admin, name))
|
||||
|
||||
inv, err := Open(t.Context())
|
||||
if err != nil {
|
||||
@@ -113,8 +112,7 @@ func ForTestBefore(t *testing.T, number int) (*Inventory, func()) {
|
||||
}
|
||||
store.CreateMigratedForTest(t, admin, name, before)
|
||||
busPasswordCostForTest.Store(int32(bcrypt.MinCost))
|
||||
cut := strings.LastIndex(admin, "/")
|
||||
t.Setenv(store.Variable(Name), admin[:cut]+"/"+name+"?sslmode=disable")
|
||||
t.Setenv(store.Variable(Name), store.DatabaseForTest(admin, name))
|
||||
inv, err := Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Issue 1886: R4e entered tier 1 at 02:01, was stopped, and was resumed at 07:43 at the same tier. Its save kept the
|
||||
// tier's entry of 02:01, so at 07:50 the plan read "waiting for mesh-build-image on shanks to be applied for 5h48m53s"
|
||||
// and its progress watchdog measured the tier from before the stop. A walk set open again at the tier it ended at
|
||||
// enters that tier anew: its time in the tier runs from the resume.
|
||||
func TestAWalkResumedAtItsTierEntersItAnew(t *testing.T) {
|
||||
entered := time.Date(2026, 10, 23, 0, 1, 7, 0, time.UTC)
|
||||
resumed := time.Date(2026, 10, 23, 5, 43, 30, 0, time.UTC)
|
||||
old := &Plan{ID: "plan-r4e", State: PlanFailed, Tier: 1, Tiers: [][]string{{"a"}, {"b"}, {"c"}},
|
||||
Created: entered.Add(-2 * time.Hour), TierEntered: entered}
|
||||
p := *old
|
||||
p.State = PlanBuilding
|
||||
got, d := tierLeftOf(old, p, resumed)
|
||||
if !got.Equal(resumed) || d != nil {
|
||||
t.Fatalf("1886: a walk resumed at tier 1 entered it at %s (duration %v); want %s, no tier left", got, d, resumed)
|
||||
}
|
||||
// A walk still in its tier keeps its entry; one that ended keeps it too.
|
||||
open := *old
|
||||
open.State = PlanRolling
|
||||
if got, _ := tierLeftOf(&open, open, resumed); !got.Equal(entered) {
|
||||
t.Fatalf("an open walk still in its tier was stamped anew: %s", got)
|
||||
}
|
||||
if got, _ := tierLeftOf(old, *old, resumed); !got.Equal(entered) {
|
||||
t.Fatalf("a failed walk saved again was stamped anew: %s", got)
|
||||
}
|
||||
}
|
||||
@@ -40,8 +40,7 @@ func ForTest(t *testing.T) *Licences {
|
||||
}
|
||||
store.CreateMigratedForTest(t, admin, name, migrations)
|
||||
|
||||
cut := strings.LastIndex(admin, "/")
|
||||
t.Setenv(store.Variable(Name), admin[:cut]+"/"+name+"?sslmode=disable")
|
||||
t.Setenv(store.Variable(Name), store.DatabaseForTest(admin, name))
|
||||
|
||||
held, err := Open(t.Context())
|
||||
if err != nil {
|
||||
|
||||
@@ -110,12 +110,7 @@ func raiseTemplate(ctx context.Context, admin, name string, key int64, migration
|
||||
if _, err := conn.Exec(ctx, "create database "+name); err != nil {
|
||||
return err
|
||||
}
|
||||
cut := strings.LastIndex(admin, "/")
|
||||
rest := ""
|
||||
if q := strings.Index(admin[cut:], "?"); q >= 0 {
|
||||
rest = admin[cut+q:]
|
||||
}
|
||||
pool, err := pgxpool.New(ctx, admin[:cut]+"/"+name+rest)
|
||||
pool, err := pgxpool.New(ctx, DatabaseForTest(admin, name))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -136,3 +131,19 @@ func raiseTemplate(ctx context.Context, admin, name string, key int64, migration
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DatabaseForTest is the address of the database name on the same test PostgreSQL that admin reaches: admin
|
||||
// with its database replaced and its query kept.
|
||||
//
|
||||
// **The query is kept, never replaced** (novox/hq issue 1889): the checker's integration store answers on a
|
||||
// unix socket named in the query (`?host=<socket directory>&sslmode=disable`), and an address that drops it
|
||||
// reaches pgx's default socket, where no server is, so every store test waited out its readiness bound and
|
||||
// failed. Every store fixture builds its database's address here, so none can drop it again.
|
||||
func DatabaseForTest(admin, name string) string {
|
||||
cut := strings.LastIndex(admin, "/")
|
||||
query := ""
|
||||
if q := strings.Index(admin[cut:], "?"); q >= 0 {
|
||||
query = admin[cut+q:]
|
||||
}
|
||||
return admin[:cut] + "/" + name + query
|
||||
}
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package store
|
||||
|
||||
import "testing"
|
||||
|
||||
// 1889/AC1: the checker's integration store answers on a unix socket named in the address's query
|
||||
// (mesh-build-go's store). A test database's address that drops the query reaches pgx's default socket,
|
||||
// where no server is, and every store test waits out its 20 s and fails.
|
||||
func TestATestDatabaseIsOnTheSocketItsAdminAddressNames(t *testing.T) {
|
||||
admin := "postgres://postgres@/postgres?host=%2Fwork%2Frun-1%2Fpg%2Fsocket&sslmode=disable"
|
||||
got := DatabaseForTest(admin, "ident_1_x")
|
||||
want := "postgres://postgres@/ident_1_x?host=%2Fwork%2Frun-1%2Fpg%2Fsocket&sslmode=disable"
|
||||
if got != want {
|
||||
t.Fatalf("the test database's address is %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// 1889/AC2: the address `make check` gives (TCP, with sslmode) builds what the fixtures always built.
|
||||
func TestATestDatabaseOnTCPIsAddressedAsBefore(t *testing.T) {
|
||||
admin := "postgres://postgres:check@127.0.0.1:55566/postgres?sslmode=disable"
|
||||
got := DatabaseForTest(admin, "inv_2_y")
|
||||
want := "postgres://postgres:check@127.0.0.1:55566/inv_2_y?sslmode=disable"
|
||||
if got != want {
|
||||
t.Fatalf("the test database's address is %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
@@ -55,7 +55,7 @@ func freshStore(t *testing.T) *Store {
|
||||
}
|
||||
conn.Close(t.Context())
|
||||
|
||||
t.Setenv(Variable("testing"), replaceDatabase(dsn, name))
|
||||
t.Setenv(Variable("testing"), DatabaseForTest(dsn, name))
|
||||
s, err := Open(t.Context(), "testing")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -75,15 +75,6 @@ func freshStore(t *testing.T) *Store {
|
||||
return s
|
||||
}
|
||||
|
||||
func replaceDatabase(dsn, name string) string {
|
||||
cut := strings.LastIndex(dsn, "/")
|
||||
rest := ""
|
||||
if q := strings.Index(dsn[cut:], "?"); q >= 0 {
|
||||
rest = dsn[cut+q:]
|
||||
}
|
||||
return dsn[:cut] + "/" + name + rest
|
||||
}
|
||||
|
||||
func TestTheSchemaIsAppliedAndRecorded(t *testing.T) {
|
||||
s := freshStore(t)
|
||||
migrations := []Migration{{Number: 1, Name: "people", SQL: "create table person (id int)", Checksum: "a"}}
|
||||
|
||||
Reference in New Issue
Block a user