Files
mesh-delivery/modules/mesh-delivery-checker/internal/check/awaiting.go
T
jschoubben 290e1696b2 mesh-delivery-checker: a check runs every member, level and the composition to its end and names every failure
The operator, 2026-11-02: "A failed check shows only its first fault". Group feat/1126-removal's
check failed four times, each verdict naming one fault, so three faults took three runs of 20-40 minutes.

The checker no longer returns at a member's first failure: every member is asked, every level's
failures and every member's refused step or conflict are joined in judging order, the manifests are
judged and the composition is read also when the tests failed, and a failed composition is added after
them. The status line says how many failed and names the first, cut so "; and N more" stays whole.
checker_verdicts and checker_pull answer the whole list (failed). Needs mesh-sdk's buildstep change
(every level runs after a failure) pinned at the group's head.

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
2026-11-02 18:56:19 +01:00

169 lines
6.1 KiB
Go

package check
import (
"context"
"errors"
"fmt"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// ErrAssignerNotRunning is what a wait for a composition answers when no instance of the assigner is online: nobody is
// making it, so the check does not wait for it (issue 1297). The verdict says the composition is not judged, and when the
// assigner starts and writes it, the checker takes it in then (Checker.Composed).
var ErrAssignerNotRunning = errors.New(delivery.AssignerNotRunning)
// PresenceEvent is a change of the watchman's presence bucket, or Settled once what stands was handed.
type PresenceEvent struct {
Key string
Online bool
Settled bool
}
// CompositionEvent is a change of the composition's key (its value, nil for a delete), or Settled once what stands was
// handed.
type CompositionEvent struct {
Value []byte
Settled bool
}
// AwaitComposition answers the assigner's composition once one is written that does not wait, while an instance of the
// assigner is online by the watchman's presence (`part.mesh-delivery-assigner.<instance>`). Once both watches handed
// what stands, no instance online is ErrAssignerNotRunning at once, and so is the last instance going offline during the
// wait. No timer: every answer follows an event; ctx ending is ctx's error.
func AwaitComposition(ctx context.Context, presence <-chan PresenceEvent, comps <-chan CompositionEvent) (delivery.CompositionVerdict, error) {
online := map[string]bool{}
presenceSettled, compSettled := false, false
for {
select {
case <-ctx.Done():
return delivery.CompositionVerdict{}, ctx.Err()
case e, ok := <-presence:
if !ok {
return delivery.CompositionVerdict{}, errors.New("the watch of the assigner's presence ended")
}
if e.Settled {
presenceSettled = true
break
}
if _, part, instance, err := delivery.ParsePresenceKey(e.Key); err == nil && part == delivery.AssignerModule {
if e.Online {
online[instance] = true
} else {
delete(online, instance)
}
}
case e, ok := <-comps:
if !ok {
return delivery.CompositionVerdict{}, errors.New("the watch of the composition ended")
}
if e.Settled {
compSettled = true
break
}
if e.Value == nil {
break
}
v, err := delivery.DecodeCompositionVerdict(e.Value)
if err != nil {
return delivery.CompositionVerdict{}, fmt.Errorf("the composition cannot be read: %w", err)
}
if v.State != delivery.CompositionWaits {
return v, nil
}
}
if presenceSettled && compSettled && len(online) == 0 {
return delivery.CompositionVerdict{}, ErrAssignerNotRunning
}
}
}
// withComposition is v with the assigner's composition taken in: a failed one fails the check, naming its machine.
func withComposition(v *delivery.Verdict, comp delivery.CompositionVerdict) {
v.Composition = comp.Composition()
v.Took.Composition = comp.Took
if comp.State == delivery.CompositionFail {
machine := comp.Machine
if machine == "" {
machine = "the composition"
}
f := delivery.FailedFirst{Machine: machine, Words: comp.Words}
if v.State == delivery.VerdictFail {
addFailure(v, f) // the tests failed too: the composition is named after them (the operator, 2026-11-02)
return
}
v.State, v.Failed = delivery.VerdictFail, f
}
}
// notJudged says whether a verdict's composition was not judged because the assigner was not running.
func notJudged(v delivery.Verdict) bool {
return v.Composition != nil && v.Composition.State == delivery.CompositionNotRun && v.Composition.Words == delivery.AssignerNotRunning
}
// unanswered says whether a verdict failed only on its composition, and that only because nodes the change places did
// not answer its placement in time (delivery.CompositionUnanswered, ADR 0559 §5): its tests passed (the composition is
// its first failure, and a composition is named after every other failure), so a later composition of the same heads is
// taken in on its event, with no new check (ADR 0548 §1: a part of the mesh not heard is judged again when it is heard).
func unanswered(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && v.Composition != nil && v.Composition.State == delivery.CompositionFail &&
strings.Contains(v.Composition.Words, delivery.CompositionUnanswered) && v.Failed.Words == v.Composition.Words
}
// Composed takes in a composition the assigner wrote after the check's verdict said it was not judged because the
// assigner was not running (the assigner started since), or after it failed only because placed nodes had not answered
// (unanswered): the verdict and its status are written again with it. A composition of heads no candidate holds now, or
// of a verdict that judged its composition otherwise, is left.
func (ch *Checker) Composed(ctx context.Context, comp delivery.CompositionVerdict) {
ch.init()
if comp.State == delivery.CompositionWaits {
return
}
ckey, err := delivery.CandidatesKey(comp.Group)
if err != nil {
return
}
ch.mu.Lock()
c, ok := ch.known[ckey]
ch.mu.Unlock()
if !ok {
return
}
cs := CaseOf(c, ch.heads())
if cs == nil || cs.HeadsHash != comp.Heads {
return
}
cur, rev, found, err := ch.Verdicts.Get(ctx, cs.Key)
if err != nil || !found {
return
}
v, err := delivery.DecodeVerdict(cur)
if err != nil {
return
}
switch {
case v.State == delivery.VerdictPass && notJudged(v):
case unanswered(v) && comp.Words != v.Composition.Words:
// The nodes answered since (or another failure stands now): the verdict is its tests' pass with this composition.
v.State, v.Failed, v.Failures = delivery.VerdictPass, delivery.FailedFirst{}, nil
default:
return
}
withComposition(&v, comp)
if v.Composition != nil && v.Composition.State != delivery.CompositionNotRun {
v.Composition.FPSet = cs.FPSet
}
data, err := delivery.EncodeVerdict(v)
if err != nil {
ch.logf("%s: the verdict with its composition cannot be written: %v", cs.Key, err)
return
}
if _, err := ch.Verdicts.Update(ctx, cs.Key, data, rev); err != nil {
ch.logf("%s: the verdict with its composition could not be written: %v", cs.Key, err)
return
}
ch.writeOwed(ctx, cs, v, Outcome{Verdict: v})
ch.logf("%s: the composition came after the verdict: %s", cs.Key, comp.State)
}