Answer the review of hq issues 348 and 349: gaps, parts, the newest merge in any state
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery delivered

- A reopened fault keeps its gaps: it was there at a send unless the send
  fell in one, so a send that breaks a machine recovered before it still
  fails its gate (A2).
- An undecided part holds only the conditions that name it (A4).
- D2 holds a silent resolver for the next run again, refused or not: a
  burst of refusals is also a restart (A3).
- A late merge is planned at the newest planned merge of its branch in any
  state, not only an open one (A1); merge times to the nanosecond (A5).
This commit is contained in:
2026-10-09 15:00:56 +02:00
committed by jochen
parent c3c56a69e2
commit 58cb586c37
11 changed files with 347 additions and 179 deletions
+45 -6
View File
@@ -146,10 +146,12 @@ type Condition struct {
Raised time.Time `json:"raised"`
LastObserved time.Time `json:"last-observed"`
// First is when this fault was first raised, a reopening within ReopenWithin counted as the same
// fault: Raised of the first raising. Absent on one written before it was kept, and on a first
// raising, where it is Raised (Began). A reopening is the same fault, so what asks when a fault began
// — the gate, asking whether it began after a send — reads Began, never Raised (novox/hq issue 348).
// fault; Gaps are the stretches between, each from a clearing to the reopening after it. Absent on a
// first raising, and on one written before they were kept. What asks whether the fault was there at a
// moment — the gate, at a send — reads OpenAt, never Raised (novox/hq issue 348): a fault cleared and
// reopened after a send was there at the send unless the send fell in one of its gaps.
First time.Time `json:"first,omitzero"`
Gaps []Gap `json:"gaps,omitempty"`
// Observations is how many times it was observed since raised.
Observations int `json:"observations"`
// Count is how many times it has been raised, a reopening within ReopenWithin counted.
@@ -280,13 +282,50 @@ func Order(list []Condition) {
})
}
// Gap is a stretch in which a fault was cleared, between its clearing and its reopening.
type Gap struct {
Cleared time.Time `json:"cleared"`
Reopened time.Time `json:"reopened"`
}
// KeptGaps is how many gaps a condition keeps. Past it the oldest go, and the fault is said to have begun
// at the reopening after the newest of them: earlier history forgotten, never a fault said older than known.
const KeptGaps = 8
// Began is when this fault began as the mesh knows it: its first raising, a reopening within
// ReopenWithin being the same fault again (novox/hq issue 348). A condition that cleared on a statement
// that judged nothing yet — a node-engine just restarted — and was raised again a minute later began
// when it was first raised, not at the reopening.
// ReopenWithin being the same fault again (novox/hq issue 348).
func (c Condition) Began() time.Time {
if !c.First.IsZero() && c.First.Before(c.Raised) {
return c.First
}
return c.Raised
}
// OpenAt says the fault was there at t: it began at or before t, and t fell in none of its gaps. A fault
// that cleared before a send and came back after it was not there at the send — that is the send's to
// answer for — while one that flapped after the send was (novox/hq issue 348).
func (c Condition) OpenAt(t time.Time) bool {
if t.Before(c.Began()) {
return false
}
for _, g := range c.Gaps {
if !t.Before(g.Cleared) && t.Before(g.Reopened) {
return false
}
}
return true
}
// reopen is c raised again at now after a clearing at cleared, of a fault that began at first with gaps:
// the same fault, with one more gap.
func (c *Condition) reopen(first time.Time, gaps []Gap, cleared, now time.Time) {
if first.IsZero() || !first.Before(now) {
return
}
c.First = first
c.Gaps = append(append([]Gap(nil), gaps...), Gap{Cleared: cleared, Reopened: now})
if over := len(c.Gaps) - KeptGaps; over > 0 {
c.First = c.Gaps[over-1].Reopened
c.Gaps = c.Gaps[over:]
}
}
+6 -6
View File
@@ -79,8 +79,10 @@ type Keeper struct {
type clearing struct {
at time.Time
count int
// began is when the fault that cleared began, so its reopening keeps it (Condition.Began).
// began is when the fault that cleared began, and gaps its earlier gaps, so its reopening keeps
// them (Condition.OpenAt).
began time.Time
gaps []Gap
silenced *Silence
// tried is what healers tried before it cleared: a reopening is the same fault, and what was
// tried on it is still what was tried.
@@ -122,7 +124,7 @@ func NewKeeper(ctx context.Context, o Options) *Keeper {
if recent, err := k.history.Since(ctx, k.now().Add(-ReopenWithin)); err == nil {
for _, e := range recent {
if e.Change == ChangeCleared {
k.cleared[e.Key] = clearing{at: e.At, count: e.Condition.Count, began: e.Condition.Began(),
k.cleared[e.Key] = clearing{at: e.At, count: e.Condition.Count, began: e.Condition.Began(), gaps: e.Condition.Gaps,
silenced: e.Condition.Silenced, tried: e.Condition.Tried}
}
}
@@ -199,9 +201,7 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
k.mu.Lock()
if before, ok := k.cleared[key]; ok && now.Sub(before.at) <= ReopenWithin {
c.Count, change = before.count+1, ChangeReopened
if !before.began.IsZero() && before.began.Before(now) {
c.First = before.began
}
c.reopen(before.began, before.gaps, before.at, now)
// A silence a person gave the condition before it cleared still holds: they said
// they knew, and the same fault again ten minutes later is what they knew about.
if before.silenced != nil && now.Before(before.silenced.Until) {
@@ -318,7 +318,7 @@ func (k *Keeper) ClearSaying(ctx context.Context, key, why, resolved string) (bo
}
now := k.now().UTC()
k.mu.Lock()
k.cleared[key] = clearing{at: now, count: c.Count, began: c.Began(), silenced: c.Silenced, tried: c.Tried}
k.cleared[key] = clearing{at: now, count: c.Count, began: c.Began(), gaps: c.Gaps, silenced: c.Silenced, tried: c.Tried}
k.mu.Unlock()
k.tell(Event{Condition: c, At: now, Change: ChangeCleared, Why: why, Cleared: &now})
return true, nil
+34 -6
View File
@@ -392,6 +392,7 @@ func TestAReopeningKeepsWhenTheFaultBegan(t *testing.T) {
if !first.Began().Equal(first.Raised) || !first.First.IsZero() {
t.Fatalf("a first raising began at %s, raised %s, first %s", first.Began(), first.Raised, first.First)
}
c.pass(30 * time.Second)
if _, err := k.Clear(ctx, "machine.ace.silent", "heard again"); err != nil {
t.Fatal(err)
}
@@ -400,8 +401,13 @@ func TestAReopeningKeepsWhenTheFaultBegan(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if !again.Raised.After(first.Raised) || !again.Began().Equal(first.Raised) {
t.Fatalf("reopened: raised %s, began %s; first raised %s", again.Raised, again.Began(), first.Raised)
if !again.Raised.After(first.Raised) || !again.Began().Equal(first.Raised) || len(again.Gaps) != 1 ||
!again.Gaps[0].Reopened.Equal(again.Raised) || !again.Gaps[0].Cleared.Before(again.Raised) {
t.Fatalf("reopened: raised %s, began %s, gaps %+v; first raised %s", again.Raised, again.Began(), again.Gaps, first.Raised)
}
if !again.OpenAt(first.Raised) || again.OpenAt(again.Gaps[0].Cleared) || !again.OpenAt(again.Raised) ||
again.OpenAt(first.Raised.Add(-time.Second)) {
t.Fatalf("open at the wrong moments: %+v", again)
}
// Through a restarted controller, reading the clearing from the history.
if _, err := k.Clear(ctx, "machine.ace.silent", "heard again"); err != nil {
@@ -416,8 +422,8 @@ func TestAReopeningKeepsWhenTheFaultBegan(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if !third.Began().Equal(first.Raised) {
t.Fatalf("after a restart the reopening began at %s, not %s", third.Began(), first.Raised)
if !third.Began().Equal(first.Raised) || len(third.Gaps) != 2 {
t.Fatalf("after a restart the reopening began at %s, not %s, with gaps %+v", third.Began(), first.Raised, third.Gaps)
}
if _, err := next.Clear(ctx, "machine.ace.silent", "heard again"); err != nil {
t.Fatal(err)
@@ -427,7 +433,29 @@ func TestAReopeningKeepsWhenTheFaultBegan(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if !fourth.Began().Equal(fourth.Raised) {
t.Fatalf("past the window the fault began at %s, raised %s", fourth.Began(), fourth.Raised)
if !fourth.Began().Equal(fourth.Raised) || len(fourth.Gaps) != 0 {
t.Fatalf("past the window the fault began at %s, raised %s, gaps %+v", fourth.Began(), fourth.Raised, fourth.Gaps)
}
}
// Past KeptGaps the oldest gaps go, and the fault is said to have begun after them, never before.
func TestAFaultKeepsItsNewestGapsAndForgetsWhatWasBefore(t *testing.T) {
t0 := time.Date(2026, 10, 9, 10, 0, 0, 0, time.UTC)
c := Condition{Raised: t0}
first, gaps := t0, []Gap(nil)
for i := 1; i <= KeptGaps+2; i++ {
cleared, now := t0.Add(time.Duration(2*i)*time.Minute), t0.Add(time.Duration(2*i+1)*time.Minute)
c = Condition{Raised: now}
c.reopen(first, gaps, cleared, now)
first, gaps = c.Began(), c.Gaps
}
if len(c.Gaps) != KeptGaps || !c.Began().Equal(t0.Add(5*time.Minute)) || c.OpenAt(t0.Add(time.Minute)) {
t.Fatalf("after %d gaps: began %s, %d gaps", KeptGaps+2, c.Began(), len(c.Gaps))
}
// A first time not before the reopening is no earlier fault.
d := Condition{Raised: t0}
d.reopen(t0, nil, t0.Add(-time.Minute), t0)
if !d.First.IsZero() || len(d.Gaps) != 0 {
t.Fatalf("a fault reopened at its own first raising: %+v", d)
}
}
+14 -2
View File
@@ -313,6 +313,18 @@ func (i *Inventory) RecentPlans(ctx context.Context, limit int) ([]Plan, error)
return i.plans(ctx, fmt.Sprintf(`order by created desc limit %d`, limit))
}
// NewestMergeOf is the plan, in any state, of the newest merge into a repository's branch that the mesh
// planned: the branch's newest commit the mesh knows of (novox/hq issue 349). False when no plan of it
// recorded when its merge was made.
func (i *Inventory) NewestMergeOf(ctx context.Context, repository, branch string) (Plan, bool, error) {
plans, err := i.plans(ctx, `where lower(repository) = lower($1) and branch = $2 and merged_at is not null
and release is null order by merged_at desc, created desc limit 1`, repository, branch)
if err != nil || len(plans) == 0 {
return Plan{}, false, err
}
return plans[0], true, nil
}
// PlanByID is one plan.
func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
plans, err := i.plans(ctx, `where id = '`+id+`'`)
@@ -325,11 +337,11 @@ func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
return plans[0], nil
}
func (i *Inventory) plans(ctx context.Context, tail string) ([]Plan, error) {
func (i *Inventory) plans(ctx context.Context, tail string, args ...any) ([]Plan, error) {
rows, err := i.store.Pool().Query(ctx,
`select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch,
coalesce(tier_entered, created), revision, coalesce(epoch, 0), release, delivery, merged_at
from release_plan `+tail)
from release_plan `+tail, args...)
if err != nil {
return nil, err
}