Send at once what is raised on a failed walk's machines in the two minutes after it (ADR 0505 §6)
This commit is contained in:
@@ -209,3 +209,39 @@ func TestAWalkThatSettlesSurvivesARouterRestart(t *testing.T) {
|
||||
t.Fatalf("its late start opened it again: %+v", again.walks)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0505 §6: a condition raised on a touched subject 30 seconds after a failed walk is sent at once, even while
|
||||
// another open walk touches the same machine; the same condition after a delivered walk is held.
|
||||
func TestAConditionJustAfterAFailedWalkIsSentAtOnceAndAfterADeliveredOneHeld(t *testing.T) {
|
||||
for _, outcome := range []string{WalkFailed, WalkStopped, WalkDelivered} {
|
||||
h, tg, dt, c, _ := newHolder(t)
|
||||
walk(t, h, "plan-f", WalkStarted, "", c.now(), 600, novox("gitea"))
|
||||
walk(t, h, "plan-o", WalkStarted, "", c.now(), 1200, novox("mesh-issues"))
|
||||
runFor(h, c, 2*time.Minute)
|
||||
walk(t, h, "plan-f", WalkEnded, outcome, c.now(), 0, novox("gitea"))
|
||||
runFor(h, c, 30*time.Second)
|
||||
k := cond("machine.novox.uncomposable", Urgent, "novox cannot get new instructions")
|
||||
k.Machine, k.Raised = "novox", c.now()
|
||||
h.Condition(EventRaised, k)
|
||||
runFor(h, c, 30*time.Second)
|
||||
sent := heard(headlineOf(k.Key), tg, dt)
|
||||
if outcome != WalkDelivered && !sent {
|
||||
t.Fatalf("after a walk that %s, held: %v", outcome, outcomes(h, k.Key))
|
||||
}
|
||||
if outcome == WalkDelivered && sent {
|
||||
t.Fatalf("after a delivered walk, sent at once: %+v", dt.sends)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0505 §6: an urgent condition on a machine no open or settling walk touches is sent at once during a walk.
|
||||
func TestAnUrgentConditionOnAnUntouchedMachineIsSentAtOnceDuringAWalk(t *testing.T) {
|
||||
h, tg, dt, c, _ := newHolder(t)
|
||||
walk(t, h, "plan-u", WalkStarted, "", c.now(), 1200, novox("gitea"))
|
||||
k := cond("machine.ace.silent", Urgent, "ace has not been heard")
|
||||
k.Machine, k.Raised = "ace", c.now()
|
||||
h.Condition(EventRaised, k)
|
||||
if !heard(headlineOf(k.Key), tg, dt) {
|
||||
t.Fatalf("held, while no walk touches ace: %v", outcomes(h, k.Key))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -392,7 +392,11 @@ func (h *Holder) Walked(ev WalkEvent) {
|
||||
w.Started = ev.StartedAt
|
||||
}
|
||||
}
|
||||
tomb := &Walk{ID: ev.Walk, Started: ev.StartedAt, Ended: at, EndedSaid: ev.At}
|
||||
tomb := &Walk{ID: ev.Walk, Started: ev.StartedAt, Ended: at, EndedSaid: ev.At, Outcome: ev.Outcome,
|
||||
Touches: mergeTouches(ev.Touches, nil)}
|
||||
if w != nil {
|
||||
tomb.Touches, tomb.SettleSeconds = w.Touches, w.SettleSeconds
|
||||
}
|
||||
if prev := h.ended[ev.Walk]; prev != nil && prev.Ended.After(at) {
|
||||
tomb = prev
|
||||
}
|
||||
@@ -944,6 +948,13 @@ func (h *Holder) hold(r *Record, what string) bool {
|
||||
now := h.Now()
|
||||
h.mu.Lock()
|
||||
s := subjectOf(r)
|
||||
if f := h.failedJustNow(s, now); f != nil {
|
||||
// A failure is the case the operator most needs the cause of: sent at once, with its delivery line (ADR 0505 §6).
|
||||
h.mu.Unlock()
|
||||
h.record(Sent{At: now, Channel: "-", Key: r.Key, What: what + ", " + roughly(now.Sub(f.Ended)) + " after walk " + f.ID + " " + f.Outcome,
|
||||
Outcome: "not held: its walk " + f.Outcome})
|
||||
return false
|
||||
}
|
||||
var by []*Walk
|
||||
var ids []string
|
||||
fresh := false
|
||||
@@ -984,6 +995,32 @@ func (h *Holder) hold(r *Record, what string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// failedJustNow is the walk that failed or stopped within its settle and touched a subject (ADR 0505 §6): nothing
|
||||
// on it is held then. Under the lock.
|
||||
func (h *Holder) failedJustNow(s subject, now time.Time) *Walk {
|
||||
var newest *Walk
|
||||
for _, e := range h.ended {
|
||||
if e.Outcome != WalkFailed && e.Outcome != WalkStopped {
|
||||
continue
|
||||
}
|
||||
if now.Before(e.Ended) || !now.Before(e.Ended.Add(h.settleOf(e))) || !e.touches(s) {
|
||||
continue
|
||||
}
|
||||
if newest == nil || e.Ended.After(newest.Ended) {
|
||||
newest = e
|
||||
}
|
||||
}
|
||||
return newest
|
||||
}
|
||||
|
||||
// settleOf is the settle a walk took at its start, or the setting's for one kept before it was.
|
||||
func (h *Holder) settleOf(w *Walk) time.Duration {
|
||||
if w.SettleSeconds > 0 {
|
||||
return w.settleFor()
|
||||
}
|
||||
return h.settle()
|
||||
}
|
||||
|
||||
// heldBy is the walks holding a record now.
|
||||
func (h *Holder) heldBy(key string, now time.Time) []string {
|
||||
var ids []string
|
||||
|
||||
Reference in New Issue
Block a user