Answer the review of hq issues 348 and 349: gaps, parts, the newest merge in any state
- A reopened fault keeps its gaps: it was there at a send unless the send fell in one, so a send that breaks a machine recovered before it still fails its gate (A2). - An undecided part holds only the conditions that name it (A4). - D2 holds a silent resolver for the next run again, refused or not: a burst of refusals is also a restart (A3). - A late merge is planned at the newest planned merge of its branch in any state, not only an open one (A1); merge times to the nanosecond (A5).
This commit is contained in:
@@ -146,10 +146,12 @@ type Condition struct {
|
||||
Raised time.Time `json:"raised"`
|
||||
LastObserved time.Time `json:"last-observed"`
|
||||
// First is when this fault was first raised, a reopening within ReopenWithin counted as the same
|
||||
// fault: Raised of the first raising. Absent on one written before it was kept, and on a first
|
||||
// raising, where it is Raised (Began). A reopening is the same fault, so what asks when a fault began
|
||||
// — the gate, asking whether it began after a send — reads Began, never Raised (novox/hq issue 348).
|
||||
// fault; Gaps are the stretches between, each from a clearing to the reopening after it. Absent on a
|
||||
// first raising, and on one written before they were kept. What asks whether the fault was there at a
|
||||
// moment — the gate, at a send — reads OpenAt, never Raised (novox/hq issue 348): a fault cleared and
|
||||
// reopened after a send was there at the send unless the send fell in one of its gaps.
|
||||
First time.Time `json:"first,omitzero"`
|
||||
Gaps []Gap `json:"gaps,omitempty"`
|
||||
// Observations is how many times it was observed since raised.
|
||||
Observations int `json:"observations"`
|
||||
// Count is how many times it has been raised, a reopening within ReopenWithin counted.
|
||||
@@ -280,13 +282,50 @@ func Order(list []Condition) {
|
||||
})
|
||||
}
|
||||
|
||||
// Gap is a stretch in which a fault was cleared, between its clearing and its reopening.
|
||||
type Gap struct {
|
||||
Cleared time.Time `json:"cleared"`
|
||||
Reopened time.Time `json:"reopened"`
|
||||
}
|
||||
|
||||
// KeptGaps is how many gaps a condition keeps. Past it the oldest go, and the fault is said to have begun
|
||||
// at the reopening after the newest of them: earlier history forgotten, never a fault said older than known.
|
||||
const KeptGaps = 8
|
||||
|
||||
// Began is when this fault began as the mesh knows it: its first raising, a reopening within
|
||||
// ReopenWithin being the same fault again (novox/hq issue 348). A condition that cleared on a statement
|
||||
// that judged nothing yet — a node-engine just restarted — and was raised again a minute later began
|
||||
// when it was first raised, not at the reopening.
|
||||
// ReopenWithin being the same fault again (novox/hq issue 348).
|
||||
func (c Condition) Began() time.Time {
|
||||
if !c.First.IsZero() && c.First.Before(c.Raised) {
|
||||
return c.First
|
||||
}
|
||||
return c.Raised
|
||||
}
|
||||
|
||||
// OpenAt says the fault was there at t: it began at or before t, and t fell in none of its gaps. A fault
|
||||
// that cleared before a send and came back after it was not there at the send — that is the send's to
|
||||
// answer for — while one that flapped after the send was (novox/hq issue 348).
|
||||
func (c Condition) OpenAt(t time.Time) bool {
|
||||
if t.Before(c.Began()) {
|
||||
return false
|
||||
}
|
||||
for _, g := range c.Gaps {
|
||||
if !t.Before(g.Cleared) && t.Before(g.Reopened) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// reopen is c raised again at now after a clearing at cleared, of a fault that began at first with gaps:
|
||||
// the same fault, with one more gap.
|
||||
func (c *Condition) reopen(first time.Time, gaps []Gap, cleared, now time.Time) {
|
||||
if first.IsZero() || !first.Before(now) {
|
||||
return
|
||||
}
|
||||
c.First = first
|
||||
c.Gaps = append(append([]Gap(nil), gaps...), Gap{Cleared: cleared, Reopened: now})
|
||||
if over := len(c.Gaps) - KeptGaps; over > 0 {
|
||||
c.First = c.Gaps[over-1].Reopened
|
||||
c.Gaps = c.Gaps[over:]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -79,8 +79,10 @@ type Keeper struct {
|
||||
type clearing struct {
|
||||
at time.Time
|
||||
count int
|
||||
// began is when the fault that cleared began, so its reopening keeps it (Condition.Began).
|
||||
// began is when the fault that cleared began, and gaps its earlier gaps, so its reopening keeps
|
||||
// them (Condition.OpenAt).
|
||||
began time.Time
|
||||
gaps []Gap
|
||||
silenced *Silence
|
||||
// tried is what healers tried before it cleared: a reopening is the same fault, and what was
|
||||
// tried on it is still what was tried.
|
||||
@@ -122,7 +124,7 @@ func NewKeeper(ctx context.Context, o Options) *Keeper {
|
||||
if recent, err := k.history.Since(ctx, k.now().Add(-ReopenWithin)); err == nil {
|
||||
for _, e := range recent {
|
||||
if e.Change == ChangeCleared {
|
||||
k.cleared[e.Key] = clearing{at: e.At, count: e.Condition.Count, began: e.Condition.Began(),
|
||||
k.cleared[e.Key] = clearing{at: e.At, count: e.Condition.Count, began: e.Condition.Began(), gaps: e.Condition.Gaps,
|
||||
silenced: e.Condition.Silenced, tried: e.Condition.Tried}
|
||||
}
|
||||
}
|
||||
@@ -199,9 +201,7 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
|
||||
k.mu.Lock()
|
||||
if before, ok := k.cleared[key]; ok && now.Sub(before.at) <= ReopenWithin {
|
||||
c.Count, change = before.count+1, ChangeReopened
|
||||
if !before.began.IsZero() && before.began.Before(now) {
|
||||
c.First = before.began
|
||||
}
|
||||
c.reopen(before.began, before.gaps, before.at, now)
|
||||
// A silence a person gave the condition before it cleared still holds: they said
|
||||
// they knew, and the same fault again ten minutes later is what they knew about.
|
||||
if before.silenced != nil && now.Before(before.silenced.Until) {
|
||||
@@ -318,7 +318,7 @@ func (k *Keeper) ClearSaying(ctx context.Context, key, why, resolved string) (bo
|
||||
}
|
||||
now := k.now().UTC()
|
||||
k.mu.Lock()
|
||||
k.cleared[key] = clearing{at: now, count: c.Count, began: c.Began(), silenced: c.Silenced, tried: c.Tried}
|
||||
k.cleared[key] = clearing{at: now, count: c.Count, began: c.Began(), gaps: c.Gaps, silenced: c.Silenced, tried: c.Tried}
|
||||
k.mu.Unlock()
|
||||
k.tell(Event{Condition: c, At: now, Change: ChangeCleared, Why: why, Cleared: &now})
|
||||
return true, nil
|
||||
|
||||
@@ -392,6 +392,7 @@ func TestAReopeningKeepsWhenTheFaultBegan(t *testing.T) {
|
||||
if !first.Began().Equal(first.Raised) || !first.First.IsZero() {
|
||||
t.Fatalf("a first raising began at %s, raised %s, first %s", first.Began(), first.Raised, first.First)
|
||||
}
|
||||
c.pass(30 * time.Second)
|
||||
if _, err := k.Clear(ctx, "machine.ace.silent", "heard again"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -400,8 +401,13 @@ func TestAReopeningKeepsWhenTheFaultBegan(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !again.Raised.After(first.Raised) || !again.Began().Equal(first.Raised) {
|
||||
t.Fatalf("reopened: raised %s, began %s; first raised %s", again.Raised, again.Began(), first.Raised)
|
||||
if !again.Raised.After(first.Raised) || !again.Began().Equal(first.Raised) || len(again.Gaps) != 1 ||
|
||||
!again.Gaps[0].Reopened.Equal(again.Raised) || !again.Gaps[0].Cleared.Before(again.Raised) {
|
||||
t.Fatalf("reopened: raised %s, began %s, gaps %+v; first raised %s", again.Raised, again.Began(), again.Gaps, first.Raised)
|
||||
}
|
||||
if !again.OpenAt(first.Raised) || again.OpenAt(again.Gaps[0].Cleared) || !again.OpenAt(again.Raised) ||
|
||||
again.OpenAt(first.Raised.Add(-time.Second)) {
|
||||
t.Fatalf("open at the wrong moments: %+v", again)
|
||||
}
|
||||
// Through a restarted controller, reading the clearing from the history.
|
||||
if _, err := k.Clear(ctx, "machine.ace.silent", "heard again"); err != nil {
|
||||
@@ -416,8 +422,8 @@ func TestAReopeningKeepsWhenTheFaultBegan(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !third.Began().Equal(first.Raised) {
|
||||
t.Fatalf("after a restart the reopening began at %s, not %s", third.Began(), first.Raised)
|
||||
if !third.Began().Equal(first.Raised) || len(third.Gaps) != 2 {
|
||||
t.Fatalf("after a restart the reopening began at %s, not %s, with gaps %+v", third.Began(), first.Raised, third.Gaps)
|
||||
}
|
||||
if _, err := next.Clear(ctx, "machine.ace.silent", "heard again"); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -427,7 +433,29 @@ func TestAReopeningKeepsWhenTheFaultBegan(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !fourth.Began().Equal(fourth.Raised) {
|
||||
t.Fatalf("past the window the fault began at %s, raised %s", fourth.Began(), fourth.Raised)
|
||||
if !fourth.Began().Equal(fourth.Raised) || len(fourth.Gaps) != 0 {
|
||||
t.Fatalf("past the window the fault began at %s, raised %s, gaps %+v", fourth.Began(), fourth.Raised, fourth.Gaps)
|
||||
}
|
||||
}
|
||||
|
||||
// Past KeptGaps the oldest gaps go, and the fault is said to have begun after them, never before.
|
||||
func TestAFaultKeepsItsNewestGapsAndForgetsWhatWasBefore(t *testing.T) {
|
||||
t0 := time.Date(2026, 10, 9, 10, 0, 0, 0, time.UTC)
|
||||
c := Condition{Raised: t0}
|
||||
first, gaps := t0, []Gap(nil)
|
||||
for i := 1; i <= KeptGaps+2; i++ {
|
||||
cleared, now := t0.Add(time.Duration(2*i)*time.Minute), t0.Add(time.Duration(2*i+1)*time.Minute)
|
||||
c = Condition{Raised: now}
|
||||
c.reopen(first, gaps, cleared, now)
|
||||
first, gaps = c.Began(), c.Gaps
|
||||
}
|
||||
if len(c.Gaps) != KeptGaps || !c.Began().Equal(t0.Add(5*time.Minute)) || c.OpenAt(t0.Add(time.Minute)) {
|
||||
t.Fatalf("after %d gaps: began %s, %d gaps", KeptGaps+2, c.Began(), len(c.Gaps))
|
||||
}
|
||||
// A first time not before the reopening is no earlier fault.
|
||||
d := Condition{Raised: t0}
|
||||
d.reopen(t0, nil, t0.Add(-time.Minute), t0)
|
||||
if !d.First.IsZero() || len(d.Gaps) != 0 {
|
||||
t.Fatalf("a fault reopened at its own first raising: %+v", d)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -313,6 +313,18 @@ func (i *Inventory) RecentPlans(ctx context.Context, limit int) ([]Plan, error)
|
||||
return i.plans(ctx, fmt.Sprintf(`order by created desc limit %d`, limit))
|
||||
}
|
||||
|
||||
// NewestMergeOf is the plan, in any state, of the newest merge into a repository's branch that the mesh
|
||||
// planned: the branch's newest commit the mesh knows of (novox/hq issue 349). False when no plan of it
|
||||
// recorded when its merge was made.
|
||||
func (i *Inventory) NewestMergeOf(ctx context.Context, repository, branch string) (Plan, bool, error) {
|
||||
plans, err := i.plans(ctx, `where lower(repository) = lower($1) and branch = $2 and merged_at is not null
|
||||
and release is null order by merged_at desc, created desc limit 1`, repository, branch)
|
||||
if err != nil || len(plans) == 0 {
|
||||
return Plan{}, false, err
|
||||
}
|
||||
return plans[0], true, nil
|
||||
}
|
||||
|
||||
// PlanByID is one plan.
|
||||
func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
|
||||
plans, err := i.plans(ctx, `where id = '`+id+`'`)
|
||||
@@ -325,11 +337,11 @@ func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
|
||||
return plans[0], nil
|
||||
}
|
||||
|
||||
func (i *Inventory) plans(ctx context.Context, tail string) ([]Plan, error) {
|
||||
func (i *Inventory) plans(ctx context.Context, tail string, args ...any) ([]Plan, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch,
|
||||
coalesce(tier_entered, created), revision, coalesce(epoch, 0), release, delivery, merged_at
|
||||
from release_plan `+tail)
|
||||
from release_plan `+tail, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user