The operator, 2026-11-02: "A failed check shows only its first fault". Group feat/1126-removal's check failed four times, each verdict naming one fault, so three faults took three runs of 20-40 minutes. The checker no longer returns at a member's first failure: every member is asked, every level's failures and every member's refused step or conflict are joined in judging order, the manifests are judged and the composition is read also when the tests failed, and a failed composition is added after them. The status line says how many failed and names the first, cut so "; and N more" stays whole. checker_verdicts and checker_pull answer the whole list (failed). Needs mesh-sdk's buildstep change (every level runs after a failure) pinned at the group's head. Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
169 lines
6.1 KiB
Go
169 lines
6.1 KiB
Go
package check
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
)
|
|
|
|
// ErrAssignerNotRunning is what a wait for a composition answers when no instance of the assigner is online: nobody is
|
|
// making it, so the check does not wait for it (issue 1297). The verdict says the composition is not judged, and when the
|
|
// assigner starts and writes it, the checker takes it in then (Checker.Composed).
|
|
var ErrAssignerNotRunning = errors.New(delivery.AssignerNotRunning)
|
|
|
|
// PresenceEvent is a change of the watchman's presence bucket, or Settled once what stands was handed.
|
|
type PresenceEvent struct {
|
|
Key string
|
|
Online bool
|
|
Settled bool
|
|
}
|
|
|
|
// CompositionEvent is a change of the composition's key (its value, nil for a delete), or Settled once what stands was
|
|
// handed.
|
|
type CompositionEvent struct {
|
|
Value []byte
|
|
Settled bool
|
|
}
|
|
|
|
// AwaitComposition answers the assigner's composition once one is written that does not wait, while an instance of the
|
|
// assigner is online by the watchman's presence (`part.mesh-delivery-assigner.<instance>`). Once both watches handed
|
|
// what stands, no instance online is ErrAssignerNotRunning at once, and so is the last instance going offline during the
|
|
// wait. No timer: every answer follows an event; ctx ending is ctx's error.
|
|
func AwaitComposition(ctx context.Context, presence <-chan PresenceEvent, comps <-chan CompositionEvent) (delivery.CompositionVerdict, error) {
|
|
online := map[string]bool{}
|
|
presenceSettled, compSettled := false, false
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return delivery.CompositionVerdict{}, ctx.Err()
|
|
case e, ok := <-presence:
|
|
if !ok {
|
|
return delivery.CompositionVerdict{}, errors.New("the watch of the assigner's presence ended")
|
|
}
|
|
if e.Settled {
|
|
presenceSettled = true
|
|
break
|
|
}
|
|
if _, part, instance, err := delivery.ParsePresenceKey(e.Key); err == nil && part == delivery.AssignerModule {
|
|
if e.Online {
|
|
online[instance] = true
|
|
} else {
|
|
delete(online, instance)
|
|
}
|
|
}
|
|
case e, ok := <-comps:
|
|
if !ok {
|
|
return delivery.CompositionVerdict{}, errors.New("the watch of the composition ended")
|
|
}
|
|
if e.Settled {
|
|
compSettled = true
|
|
break
|
|
}
|
|
if e.Value == nil {
|
|
break
|
|
}
|
|
v, err := delivery.DecodeCompositionVerdict(e.Value)
|
|
if err != nil {
|
|
return delivery.CompositionVerdict{}, fmt.Errorf("the composition cannot be read: %w", err)
|
|
}
|
|
if v.State != delivery.CompositionWaits {
|
|
return v, nil
|
|
}
|
|
}
|
|
if presenceSettled && compSettled && len(online) == 0 {
|
|
return delivery.CompositionVerdict{}, ErrAssignerNotRunning
|
|
}
|
|
}
|
|
}
|
|
|
|
// withComposition is v with the assigner's composition taken in: a failed one fails the check, naming its machine.
|
|
func withComposition(v *delivery.Verdict, comp delivery.CompositionVerdict) {
|
|
v.Composition = comp.Composition()
|
|
v.Took.Composition = comp.Took
|
|
if comp.State == delivery.CompositionFail {
|
|
machine := comp.Machine
|
|
if machine == "" {
|
|
machine = "the composition"
|
|
}
|
|
f := delivery.FailedFirst{Machine: machine, Words: comp.Words}
|
|
if v.State == delivery.VerdictFail {
|
|
addFailure(v, f) // the tests failed too: the composition is named after them (the operator, 2026-11-02)
|
|
return
|
|
}
|
|
v.State, v.Failed = delivery.VerdictFail, f
|
|
}
|
|
}
|
|
|
|
// notJudged says whether a verdict's composition was not judged because the assigner was not running.
|
|
func notJudged(v delivery.Verdict) bool {
|
|
return v.Composition != nil && v.Composition.State == delivery.CompositionNotRun && v.Composition.Words == delivery.AssignerNotRunning
|
|
}
|
|
|
|
// unanswered says whether a verdict failed only on its composition, and that only because nodes the change places did
|
|
// not answer its placement in time (delivery.CompositionUnanswered, ADR 0559 §5): its tests passed (the composition is
|
|
// its first failure, and a composition is named after every other failure), so a later composition of the same heads is
|
|
// taken in on its event, with no new check (ADR 0548 §1: a part of the mesh not heard is judged again when it is heard).
|
|
func unanswered(v delivery.Verdict) bool {
|
|
return v.State == delivery.VerdictFail && v.Composition != nil && v.Composition.State == delivery.CompositionFail &&
|
|
strings.Contains(v.Composition.Words, delivery.CompositionUnanswered) && v.Failed.Words == v.Composition.Words
|
|
}
|
|
|
|
// Composed takes in a composition the assigner wrote after the check's verdict said it was not judged because the
|
|
// assigner was not running (the assigner started since), or after it failed only because placed nodes had not answered
|
|
// (unanswered): the verdict and its status are written again with it. A composition of heads no candidate holds now, or
|
|
// of a verdict that judged its composition otherwise, is left.
|
|
func (ch *Checker) Composed(ctx context.Context, comp delivery.CompositionVerdict) {
|
|
ch.init()
|
|
if comp.State == delivery.CompositionWaits {
|
|
return
|
|
}
|
|
ckey, err := delivery.CandidatesKey(comp.Group)
|
|
if err != nil {
|
|
return
|
|
}
|
|
ch.mu.Lock()
|
|
c, ok := ch.known[ckey]
|
|
ch.mu.Unlock()
|
|
if !ok {
|
|
return
|
|
}
|
|
cs := CaseOf(c, ch.heads())
|
|
if cs == nil || cs.HeadsHash != comp.Heads {
|
|
return
|
|
}
|
|
cur, rev, found, err := ch.Verdicts.Get(ctx, cs.Key)
|
|
if err != nil || !found {
|
|
return
|
|
}
|
|
v, err := delivery.DecodeVerdict(cur)
|
|
if err != nil {
|
|
return
|
|
}
|
|
switch {
|
|
case v.State == delivery.VerdictPass && notJudged(v):
|
|
case unanswered(v) && comp.Words != v.Composition.Words:
|
|
// The nodes answered since (or another failure stands now): the verdict is its tests' pass with this composition.
|
|
v.State, v.Failed, v.Failures = delivery.VerdictPass, delivery.FailedFirst{}, nil
|
|
default:
|
|
return
|
|
}
|
|
withComposition(&v, comp)
|
|
if v.Composition != nil && v.Composition.State != delivery.CompositionNotRun {
|
|
v.Composition.FPSet = cs.FPSet
|
|
}
|
|
data, err := delivery.EncodeVerdict(v)
|
|
if err != nil {
|
|
ch.logf("%s: the verdict with its composition cannot be written: %v", cs.Key, err)
|
|
return
|
|
}
|
|
if _, err := ch.Verdicts.Update(ctx, cs.Key, data, rev); err != nil {
|
|
ch.logf("%s: the verdict with its composition could not be written: %v", cs.Key, err)
|
|
return
|
|
}
|
|
ch.writeOwed(ctx, cs, v, Outcome{Verdict: v})
|
|
ch.logf("%s: the composition came after the verdict: %s", cs.Key, comp.State)
|
|
}
|