Compare commits

..
Author SHA1 Message Date
jschoubben db55190cd4 Pin the checker to mesh-sdk 979f2cd, the group's head after the cache layer's review fix (issue 1921) 2026-10-29 01:05:40 +01:00
jschoubben 10c7eaa67e Pin the checker to mesh-sdk 62a50eb, the group's head, which adds the check ask's cache layer (issue 1921) 2026-10-29 00:54:06 +01:00
jschoubben cbc53b0e50 Name every group member that does not merge in one verdict, not only the first
After a member's check fails, the checker asks each other member its merge alone (mesh-sdk CheckJob.MergeOnly), so a hidden conflict no longer costs a whole check round and an Approve (issue 1921's kind; no open issue covers it; not issue 1276).
2026-10-29 00:07:40 +01:00
28 changed files with 220 additions and 457 deletions
@@ -40,9 +40,6 @@ type passer struct {
// noWant is, by node and module, the reason last said why the node gets no want for the module (issue 1945): a reason is
// said in the journal when it appears or changes, and its clearing when the node has a want again; one line per change.
noWant map[string]string
// unreached is, by node and module, the words last said why a wanted module may not be reached there (issue 1960: its
// provider is down, which holds no want): said once per words, and once more when it is reached again.
unreached map[string]string
// nodeFailed is, by node and module, the words of the condition last raised for a failure that is the node's own (ADR 0557
// §1, issue 1496): raised once per words, cleared once when it ends.
nodeFailed map[string]string
@@ -199,7 +196,6 @@ func (p *passer) wants(ctx context.Context) {
gatherNeither(neither, name, out)
leftNeither(left, m, wanted[name], out)
p.sayNoWant(name, m.Nodes, wanted[name], out)
p.sayUnreached(name, m.Nodes, out)
p.sayNodeFailed(name, fp, m.Nodes, nodes, out, first)
if first {
startClears(stale, m, out)
@@ -278,26 +274,6 @@ func (p *passer) sayNoWant(module string, nodes []string, before map[string]deli
}
}
// sayUnreached says in the journal, once per words, that a module is wanted on a node and may not be reached there, since
// no provider of what it requires runs healthy (rule 5, which holds no want: issue 1960), and once when it is reached again.
func (p *passer) sayUnreached(module string, nodes []string, out assign.Outcome) {
if p.unreached == nil {
p.unreached = map[string]string{}
}
for _, node := range nodes {
key := node + "." + module
why, last := out.Unreached[node], p.unreached[key]
switch {
case why != "" && why != last:
p.logf("%s on %s is wanted and may not be reached: %s", module, node, why)
p.unreached[key] = why
case why == "" && last != "":
p.logf("%s on %s is reached: its providers run healthy", module, node)
delete(p.unreached, key)
}
}
}
// sayNodeFailed raises, once per node and words, the condition of a failed health check of a module's newest fingerprint fp
// that is the node's own, since fp runs healthy elsewhere and is not held (ADR 0557 §1, issue 1496), and says it in the journal;
// it clears the condition once when the node no longer fails alone, or is no longer one of the module's nodes. The first pass
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028212605-77b48a4f1aff
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 h1:Tt16HJk8JZRce1i3JzzDSNdCWOrv7dw8FQ/bckNQ5Ig=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028212605-77b48a4f1aff h1:EHuM+ed+/M9woMZsMou6/2xm6hIdb1YCFINprrP54Ys=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028212605-77b48a4f1aff/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -89,15 +89,14 @@ func Canary(nodes []string, online map[string]bool) string {
// Holds is whether every rule holds for moving node's want for m to m.FP, and if not, the first rule that does not,
// with why (ADR 0429 §6). Rule 4, at most `step` others ahead, always holds: the mesh keeps no step, and step 0 is no
// limit (ADR 0528 clause 3). Rule 5, every provider healthy, holds no want (issue 1960): a module whose provider is down
// is wanted all the same, and said as not reached (Unreached), since a site runs while the proxy in front of it is
// down and one module never blocks another.
// limit (ADR 0528 clause 3).
func Holds(m Module, node string, n Nodes) (bool, int, string) {
return holds(m, node, n, true)
}
// holds is Holds; with ordering false it leaves out the canary (3), which orders a move among nodes, and keeps those
// without which there is nothing to want: the build done, not held, the grant and the composition.
// holds is Holds; with ordering false it leaves out the two rules that order a move among nodes and modules, the canary (3)
// and the providers (5), and keeps those without which there is nothing to want: the build done, not held, the grant and the
// composition.
func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
switch {
case !m.Built:
@@ -113,6 +112,11 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
}
for _, provision := range m.Manifest.Requires {
if ok, why := providedHealthy(provision, node, n); !ok {
return false, delivery.RuleProviders, why
}
}
}
if !m.Granted {
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
@@ -153,17 +157,6 @@ func providedHealthy(provision, node string, n Nodes) (bool, string) {
return false, fmt.Sprintf("no provider of %s runs healthy on %s or at its canary", provision, node)
}
// unreached is why m on node may be wanted and not reached: the first provision it requires that no provider runs
// healthy on node or at its canary (rule 5, said and never holding a want; issue 1960); empty when all do.
func unreached(m Module, node string, n Nodes) string {
for _, provision := range m.Manifest.Requires {
if ok, why := providedHealthy(provision, node, n); !ok {
return why
}
}
return ""
}
// keepsData says whether a module holds data a set-back would lose: an own item of a class that keeps (irreplaceable,
// valuable or rebuildable; cache and none do not).
func keepsData(m catalogue.Manifest) bool {
@@ -193,9 +186,6 @@ type Outcome struct {
Restore []string
// Rules is, by node, the rule holding the node back from m.FP and why.
Rules map[string]Holding
// Unreached is, by node, why the module may be wanted there and not reached: no provider of a provision it requires runs
// healthy (rule 5, issue 1960). It holds no want back; the pass says it.
Unreached map[string]string
// Neither is, by node, the nodes the module is assigned to that run it (or whose have names it), hold no want for it after this
// pass and are not held (a held module is on the held list), with the rule holding the first want back: the node's own composition does not hold a module with a fingerprint of the new form, so such a module is on
// neither list and nothing applies it on that node (issue 1945). A module with no fingerprint of the new form goes with the node's
@@ -311,34 +301,27 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
}
ok, rule, words := Holds(mm, node, n)
target := m.FP
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && rule == delivery.RuleCanary {
// The canary orders a move, never the first want of a node that does not run the module (issue 1960: de-spiegel,
// link2pay and photos on novox were left with no want). A node that runs an older fingerprint of the new form,
// composed and granted, gets its first want at what it runs, and rule 3 then orders its move to F as from any want.
// A node that runs an old-form build, which no want can name, stays under the canary (issue 1945, a design limit):
// its first want names F once the canary runs F healthy. Any other node that runs it (F itself, or an older
// fingerprint with no composition or grant) gets F by the rules without which there is nothing to want.
older := ran && runs(have) && have.FP != m.FP && strings.HasPrefix(have.FP, delivery.SourcePrefix2)
if older {
_, composed := m.composition(node, have.FP)
older = composed && m.GrantedAt != nil && m.GrantedAt(have.FP)
}
oldForm := ran && runs(have) && have.FP != m.FP && !strings.HasPrefix(have.FP, delivery.SourcePrefix2)
switch {
case older:
ok, target = true, have.FP
case oldForm:
// held back by rule 3 as written: on neither list until the canary runs F healthy
default:
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && (rule == delivery.RuleCanary || rule == delivery.RuleProviders) && runs(have) {
// A node that runs the module and has no want for it has no want the ordering rules could hold it back to: holding its first
// want back leaves the module on neither list, since the node's own composition gives up every module with a fingerprint of
// the new form (issue 1945: on g14 the bundles of clipmenu, screen-lock and every module requiring x11-display were removed
// and never made again, because xorg's health is unknown and screen-lock's canary never ran it healthy). Its first want names
// what it runs, never a newer fingerprint, so the canary is never passed over: F when it runs F (by the rules without which
// there is nothing to want), or an older fingerprint of the new form whose composition and grant exist. Rules 3 and 5 then
// order the move from it as from any want.
switch _, composed := m.composition(node, have.FP); {
case !ran:
case have.FP == m.FP:
ok, rule, words = holds(mm, node, n, false)
case !strings.HasPrefix(have.FP, delivery.SourcePrefix2):
// An old-form build (not src2:) has no composition, so no want can ever name it: its first want names F, and only by every
// rule as usual, the canary (3) and the providers (5) included, which here hold it back. Nothing is written and the module
// stays on neither list until the canary runs F healthy; then Holds passes and the want names F (issue 1945, a design
// limit set by the lead: the canary stays in force and old-form modules leave neither list by themselves).
case composed && m.GrantedAt != nil && m.GrantedAt(have.FP):
ok, target = true, have.FP
}
}
if why := unreached(m, node, n); why != "" {
if out.Unreached == nil {
out.Unreached = map[string]string{}
}
out.Unreached[node] = why
}
if ok && target == m.FP && len(out.NodeFailed) > 0 && !(had && !before.Removed && before.FP == m.FP) &&
n.Haves[node][m.Name].FP != m.FP {
// While a node fails F alone, F reaches no further node: a node that neither wants nor runs F gets no new want
@@ -65,6 +65,9 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
delivery.RuleCanary: func(_ *Module, n *Nodes) {
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
},
delivery.RuleProviders: func(_ *Module, n *Nodes) {
n.Haves["ace"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthWaits}
},
delivery.RuleGrant: func(m *Module, _ *Nodes) { m.Granted = false },
delivery.RuleComposed: func(m *Module, _ *Nodes) {
m.Composed = func(string, string) (delivery.ComposedDigest, bool) { return delivery.ComposedDigest{}, false }
@@ -83,50 +86,6 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
t.Errorf("rule %d broken: g14's want is %+v", rule, w)
}
}
// 1960: rule 5 holds no want, and no module is left without a first want for a provider down or a node offline; the
// provider down is said (Unreached). novox on 2026-10-29: route-proxy down held de-spiegel and link2pay, minio not
// healthy held photos, and novox offline left a module with no canary.
for _, c := range []struct {
name, module, provision, provider string
have *delivery.Have // what novox runs of the module; nil: a new assignment
before map[string]delivery.Want
online bool
want string
}{
{"de-spiegel, route-proxy down, old form", "de-spiegel", "route", "route-proxy",
&delivery.Have{FP: "build-1", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}, nil, true, fNew},
{"link2pay, route-proxy down, new assignment", "link2pay", "route", "route-proxy", nil, nil, true, fNew},
{"photos, minio not healthy, moving its want", "photos", "s3-bucket", "minio",
&delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK},
map[string]delivery.Want{"novox": {FP: fOld, Composed: dig(fOld)}}, true, fNew},
{"novox offline, no canary, new assignment", "link2pay", "route", "route-proxy", nil, nil, false, fNew},
{"novox offline, no canary, moving its want", "photos", "s3-bucket", "minio",
&delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK},
map[string]delivery.Want{"novox": {FP: fOld, Composed: dig(fOld)}}, false, fOld},
} {
m := Module{Name: c.module, FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
Manifest: catalogue.Manifest{Module: c.module, Requires: []string{c.provision}}}
n := Nodes{Online: map[string]bool{"novox": c.online}, Haves: map[string]map[string]delivery.Have{"novox": {
c.provider: {FP: fpOf('p'), Phase: delivery.PhaseFailed, Health: delivery.HealthFailed}}},
Providers: map[string]map[string][]string{c.provision: {c.provider: {"novox"}}}}
if c.have != nil {
n.Haves["novox"][c.module] = *c.have
}
out := Wants(m, n, c.before, nil)
w, written := out.Wants["novox"]
if !written {
w = c.before["novox"]
}
if w.FP != c.want || w.Removed {
t.Errorf("%s: novox's want = %+v, want %s", c.name, w, short(c.want))
}
if _, on := out.Neither["novox"]; on {
t.Errorf("%s: on neither list: %+v", c.name, out.Neither)
}
if c.online && !strings.Contains(out.Unreached["novox"], c.provision) {
t.Errorf("%s: the provider down is not said: %q", c.name, out.Unreached["novox"])
}
}
// 1126/AC-SW1, AC-SW2, AC-SW3: a want written Self while g14 was placed, kept by a rule after the placement is gone, names
// the controller's composition of the fingerprint the rule keeps (on 2026-10-27 g14's systemd-resolved, held, and forticlient,
@@ -149,7 +108,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
{name: "rule 5 at F", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
n.Haves["ace"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthWaits}
}, want: delivery.Want{FP: fNew, Composed: dig(fNew)}},
}, want: delivery.Want{FP: fNew, Composed: dig(fNew)}, rule: delivery.RuleProviders},
{name: "rule 3 at F", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
@@ -1003,12 +962,10 @@ func x11() (Module, Nodes) {
func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T) {
m, n := x11()
out := Wants(m, n, nil, nil)
// 1960: the providers rule holds no want: the canary g14 gets F, its provider's health unknown said
if w := out.Wants["g14"]; w.FP != fNew || out.Unreached["g14"] == "" {
t.Errorf("g14's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["g14"])
}
if w := out.Wants["shanks"]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != delivery.RuleCanary {
t.Errorf("shanks's want = %+v, want its first want at the fingerprint it runs, held back from F by the canary", w)
for _, node := range []string{"g14", "shanks"} {
if w := out.Wants[node]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule == 0 {
t.Errorf("%s's want = %+v, want its first want at the fingerprint it runs, held back from F", node, w)
}
}
if len(out.Neither) != 0 {
t.Errorf("left on neither list: %+v", out.Neither)
@@ -1046,22 +1003,11 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
m, n := x11()
change(&m, &n)
out := Wants(m, n, nil, nil)
if what == "an old-form build" {
// the canary g14 does not run F healthy: no want, on neither list by rule 3 (issue 1945)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("%s: shanks got a want: %+v", what, w)
}
if h, ok := out.Neither["shanks"]; !ok || h.Rule != delivery.RuleCanary {
t.Errorf("%s: shanks on neither list = %+v (named %v), want rule 3", what, h, ok)
}
continue
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("%s: shanks got a want: %+v", what, w)
}
// 1960: no want can name what it runs, so its first want names F rather than none: never on neither list
if w := out.Wants["shanks"]; w.FP != fNew {
t.Errorf("%s: shanks's want = %+v, want F", what, w)
}
if h, ok := out.Neither["shanks"]; ok {
t.Errorf("%s: shanks on neither list = %+v", what, h)
if h, ok := out.Neither["shanks"]; !ok || h.Rule == 0 {
t.Errorf("%s: shanks on neither list = %+v (named %v)", what, h, ok)
}
}
m, n := x11()
@@ -1070,12 +1016,10 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
if len(out.Neither) != 0 {
t.Errorf("nodes holding a want named on neither list: %+v", out.Neither)
}
// the canary g14 moves to F (its provider's health no rule); shanks waits for it
if w := out.Wants["shanks"]; w.FP != "" && w.FP != fOld {
t.Errorf("shanks's want moved from its want past the canary: %+v", w)
}
if w := out.Wants["g14"]; w.FP != fNew {
t.Errorf("the canary g14's want = %+v, want F", w)
for node, w := range out.Wants {
if w.FP != fOld {
t.Errorf("%s's want moved from its want past the ordering rules: %+v", node, w)
}
}
}
@@ -1086,12 +1030,11 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
m, n := x11()
delete(n.Haves["shanks"], "screen-lock")
out := Wants(m, n, nil, nil)
// 1960: the canary orders moves, never a first want: the new assignment is wanted at F
if w := out.Wants["shanks"]; w.FP != fNew {
t.Errorf("a new assignment while the canary does not run F: %+v, want F", w)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("a new assignment waiting for the canary got a want: %+v", w)
}
if _, ok := out.Neither["shanks"]; ok {
t.Errorf("a new assignment is named on neither list: %+v", out.Neither)
t.Errorf("a new assignment waiting for the canary is named on neither list: %+v", out.Neither)
}
m.Built = false
if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != delivery.RuleBuildDone {
@@ -1133,7 +1076,7 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
// 1945/AC21: a node whose have names an old-form build (no fingerprint of the new form, so no composition of it) can never be
// named in a want at what it runs; its first want names F, under the canary rule as usual and with everything F needs and the
// providers rule. Given the canary runs F healthy, shanks's first want names F; given it does not yet, shanks gets no want and
// stays on neither list; the providers hold no want (issue 1960). The old-form node that is itself the canary gets F once its providers run healthy.
// stays on neither list. The old-form node that is itself the canary gets F once its providers run healthy.
func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
oldForm := func(n Nodes, node string) {
n.Haves[node]["screen-lock"] = delivery.Have{FP: "build-1792260876956849831", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
@@ -1167,16 +1110,16 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
if h, ok := out.Neither["shanks"]; !ok || h.Rule != delivery.RuleCanary {
t.Errorf("canary not healthy on F: shanks on neither list = %+v (named %v), want rule 3", h, ok)
}
// 1960: the canary healthy on F but the providers not healthy on shanks: F all the same, the provider down said
// the canary healthy on F but the providers not healthy on shanks: rule 5 still holds it back
m, n = x11()
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
out = Wants(m, n, nil, nil)
if w := out.Wants["shanks"]; w.FP != fNew || out.Unreached["shanks"] == "" {
t.Errorf("providers not healthy: shanks's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["shanks"])
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("providers not healthy: shanks got a want: %+v", w)
}
if _, on := out.Neither["shanks"]; on {
t.Errorf("providers not healthy: shanks on neither list: %+v", out.Neither)
if h := out.Neither["shanks"]; h.Rule != delivery.RuleProviders {
t.Errorf("providers not healthy: shanks on neither list = %+v, want rule 5", h)
}
// F not granted: nothing, even with the canary healthy
m, n = x11()
@@ -94,34 +94,6 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
return map[string]any{"key": key, "revision": r.Revision, "asks": r.Asks, "answer": r.Answer,
"next": build.StuckTool + " shows the key and its asks while they wait"}, nil
}},
{Name: build.AnnounceAgainTool,
Description: "Announce a done build again (take in again, compose again): the builder writes the done key of module and fingerprint " +
"again, unchanged, as a new revision of the same key (its content, artifacts and manifest stay identical), so the controller's " +
"take-in hears it and composes the module and writes its grant again. Use it when a done build was taken in but its composition " +
"or grant is gone and every want is refused for it. Answers the new revision. Refuses a key that is not done: " + build.BuildAgainTool +
" builds a failed key again, " + build.AskAgainTool + " asks a waiting key's asks again.",
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the module's name"},
"fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>"},
"why": map[string]any{"type": "string", "description": "why it is announced again, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := build.NewAnnounceAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
raw, err := buildAgain(key, value)
if err != nil {
return nil, err
}
r, err := build.DecodeReAsk(raw)
if err != nil {
return nil, err
}
if r.Refused || r.Revision == 0 {
return nil, errors.New(r.Answer)
}
return map[string]any{"key": key, "revision": r.Revision, "answer": r.Answer}, nil
}},
{Name: build.SeatQueueTool,
Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_<LANGUAGE>), as the builder reads it: the stream's message count and " +
"sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " +
@@ -19,7 +19,6 @@ import (
type askAgainer interface {
AskAgain(key, why string) []string
BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) build.BuildAgainResult
AnnounceAgain(ctx context.Context, module, fp, why, by string) build.BuildAgainResult
}
// buildAgainBound is how long the builder waits for the claim and the asks of a key built again by hand; the tool waits
@@ -46,10 +45,6 @@ func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstre
go builtAgain(ctx, b, kv, e, r, logf)
return
}
if r.Mode == build.ModeAnnounce {
announcedAgain(ctx, b, kv, e, r, logf)
return
}
asked := b.AskAgain(e.Key(), r.Why)
if len(asked) == 0 {
r.Answer = "no ask of this key waits in this builder now (it is not building, or its build has not reached its ask yet): nothing was asked; builder_stuck shows where it stands"
@@ -119,22 +114,3 @@ func builtAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jets
logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
}
}
// announcedAgain answers a reasks value of mode announce (the tool builder_announce_again): the done key is written again
// unchanged, and the value is written back with the new revision, or why nothing was done.
func announcedAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) {
res := b.AnnounceAgain(ctx, r.Module, r.FP, r.Why, r.By)
if res.Refused != "" {
r.Answer, r.Refused = res.Refused, true
} else {
r.Revision, r.Answer = res.Revision, fmt.Sprintf("announced again: the done key is written again, unchanged, as revision %d", res.Revision)
}
r.Answered = time.Now().UTC()
data, err := json.Marshal(r)
if err != nil {
return
}
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
}
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-builder
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028193502-f91340fe82ef
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 h1:Tt16HJk8JZRce1i3JzzDSNdCWOrv7dw8FQ/bckNQ5Ig=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028193502-f91340fe82ef h1:ggwHXCvAOhBCKAY+ErehKEhu/IQTS4poJBRDokGYnDc=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028193502-f91340fe82ef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -1,53 +0,0 @@
package build
import (
"context"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// **A done build is announced again by hand** (recovery of photos and link2pay on novox, 2026-10-29). The controller takes a
// build in when a write of the build key reaches its take-in consumer, and composes then; a done key is never written again
// by the builder, so a composition or a grant lost afterwards could not be made again by any tool. AnnounceAgain writes the
// done key again with its own bytes, as a new revision of the same key: its content, state, artifacts and manifest stay
// identical, and only the write is new. Who asked and why is in the builder's log and in the reasks answer.
// AnnounceAgain rewrites the done key of module at fp unchanged, as a new revision, and answers the revision. A key that is
// not done is refused with why: failed (builder_build_again builds it) or waiting (builder_ask_again).
func (b *Builder) AnnounceAgain(ctx context.Context, module, fp, why, by string) BuildAgainResult {
b.init()
key, err := delivery.BuilderBuildKey(module, fp)
if err != nil {
return BuildAgainResult{Refused: err.Error()}
}
value, rev, found, err := b.Claimer.Store.Get(ctx, key)
if err != nil {
return BuildAgainResult{Refused: fmt.Sprintf("%s cannot be read: %v", key, err)}
}
if !found {
return BuildAgainResult{Refused: fmt.Sprintf("%s has no build key: nothing to announce", key)}
}
cur, err := delivery.DecodeBuilderBuild(value)
if err != nil {
return BuildAgainResult{Refused: fmt.Sprintf("%s holds what is not a build: %v", key, err)}
}
switch cur.State {
case delivery.BuildDone:
case delivery.BuildFailed:
return BuildAgainResult{Refused: fmt.Sprintf("%s is failed, not done: %s builds it again", key, BuildAgainTool)}
default:
return BuildAgainResult{Refused: fmt.Sprintf("%s is %q, not done: %s asks its waiting asks again", key, cur.State, AskAgainTool)}
}
// Compare-and-set at the revision read: a build written meanwhile is not overwritten.
nrev, err := b.Claimer.Store.Update(ctx, key, value, rev)
if err != nil {
if errors.Is(err, ErrConflict) {
return BuildAgainResult{Refused: fmt.Sprintf("%s changed while it was announced: read it again", key)}
}
return BuildAgainResult{Refused: fmt.Sprintf("%s could not be written: %v", key, err)}
}
b.logf("%s: done, announced again by %s (revision %d): %s", key, by, nrev, why)
return BuildAgainResult{Revision: nrev}
}
@@ -1,49 +0,0 @@
package build
import (
"bytes"
"context"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func TestAnnounceAgainRewritesOnlyADoneKeyUnchanged(t *testing.T) { // recovery of photos and link2pay, 2026-10-29
for _, c := range []struct {
name string
key *delivery.BuilderBuild
fp string
refuse string
}{
{name: "a done key is written again, unchanged, as a new revision", key: &delivery.BuilderBuild{State: delivery.BuildDone, Step: "built"}},
{name: "a failed key is refused, builder_build_again is for it", refuse: "builder_build_again", key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "x"}},
{name: "a building key is refused, builder_ask_again is for it", refuse: "builder_ask_again", key: &delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Step: "asked"}},
{name: "a key with no build is refused", refuse: "has no build key"},
{name: "a fingerprint that is not one is refused", refuse: "fingerprint", fp: "nope"},
} {
t.Run(c.name, func(t *testing.T) {
r := newRig(t, goBundle)
var rev uint64
if c.key != nil {
c.key.Manifest = manifestD
rev = putBuild(t, r.st, r.key, *c.key)
}
before := bytes.Clone(r.st.vals[r.key])
fp := fp1
if c.fp != "" {
fp = c.fp
}
res := r.b.AnnounceAgain(context.Background(), "m", fp, "grant lost", "shanks.lead")
if c.refuse != "" {
if !strings.Contains(res.Refused, c.refuse) || res.Revision != 0 || r.st.revs[r.key] != rev || !bytes.Equal(r.st.vals[r.key], before) {
t.Fatalf("answer %+v, key at revision %d (was %d); want refused with %q and nothing written", res, r.st.revs[r.key], rev, c.refuse)
}
return
}
if res.Refused != "" || res.Revision <= rev || r.st.revs[r.key] != res.Revision || !bytes.Equal(r.st.vals[r.key], before) {
t.Fatalf("answer %+v, key at revision %d (was %d); want a new revision and the same bytes", res, r.st.revs[r.key], rev)
}
})
}
}
@@ -24,9 +24,6 @@ const (
AskAgainTool = "builder_ask_again"
// BuildAgainTool builds a failed key again by hand (issue 1126 note 72): it writes a reasks value of Mode ModeBuild.
BuildAgainTool = "builder_build_again"
// AnnounceAgainTool writes a done key again unchanged, so the controller takes it in and composes again: a reasks value
// of Mode ModeAnnounce.
AnnounceAgainTool = "builder_announce_again"
// ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key.
ReAsksState = "reasks"
ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState
@@ -59,9 +56,6 @@ type ReAsk struct {
// ModeBuild is the Mode of a reasks value that asks the key to be built again, not its waiting asks to be asked again.
const ModeBuild = "build"
// ModeAnnounce is the Mode of a reasks value that asks a done key to be announced again.
const ModeAnnounce = "announce"
// DecodeReAsk reads a reasks value.
func DecodeReAsk(data []byte) (ReAsk, error) {
var r ReAsk
@@ -100,19 +94,6 @@ func NewBuildAgain(module, fp, why, by string, at time.Time) (string, []byte, er
return key, data, err
}
// NewAnnounceAgain is the value builder_announce_again writes for a module's fingerprint: its key and the value.
func NewAnnounceAgain(module, fp, why, by string, at time.Time) (string, []byte, error) {
key, err := delivery.BuilderBuildKey(module, fp)
if err != nil {
return "", nil, err
}
if strings.TrimSpace(why) == "" {
return "", nil, errors.New("say why the done build is announced again (why)")
}
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeAnnounce, Module: module, FP: fp})
return key, data, err
}
// StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known).
type StateRead struct {
Keys func() ([]string, error)
@@ -116,7 +116,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
t.Errorf("secrets-owner = %q", m.SecretsOwner)
}
// 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again.
if want := []string{"builder_announce_again", "builder_ask_again", "builder_build_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) {
if want := []string{"builder_ask_again", "builder_build_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) {
@@ -3,7 +3,6 @@
"version": "1",
"slug": "builder",
"tools": [
"builder_announce_again",
"builder_ask_again",
"builder_build_again",
"builder_seat_queue",
@@ -229,8 +229,7 @@ func Tools(r check.Reader) []stdio.Tool {
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). held_by_turn names the candidates whose check an approved " +
"merge's turn holds off (novox/hq ADR 0548 §3), with the repository and the merge that waits; the gate's turns tool reads every turn and its " +
"hand-now tool steers one. dead_letters says whether the checker hears the asks the build seats give up on (issue 734), and why not: " +
"deaf, a try or check whose ask is given up waits until its bound. judging names the candidates whose newest step ended and whose end " +
"the checker judges now (levels, kept verdicts, composition): their verdict follows by itself; lost names those nothing will judge. Read-only.",
"deaf, a try or check whose ask is given up waits until its bound. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) { return r.Status() }},
}
@@ -244,9 +243,8 @@ func RecoveryTools(r check.Reader, w check.VerdictEntry) []stdio.Tool {
Description: "Ask a candidate's check again by hand, whose verdict stays running while nothing runs for it (" + check.StatusReadTool + " names it " +
"under lost): its verdict is ended as asked again, saying why, and the checker, which hears its verdicts, stops the wait that judged nothing " +
"and asks the check anew, whole. Refused for a group with no candidate, a verdict that is not running (a pass or failure stands until a push or " +
"a trunk move), while a step of its pull requests can still run on a build module (its end judges it), and while the checker judges the end " +
"of its newest step (" + check.StatusReadTool + " names it under judging: its verdict follows). The checker reads a step that ended while " +
"its own watch was deaf by itself at the next step end of any step of its build module; this is for what that does not reach.",
"a trunk move), and while a step of its pull requests can still run on a build module (its end judges it). The checker reads a step that " +
"ended while its watch was deaf by itself at the next step end of its build module; this is for what that does not reach.",
Input: map[string]any{"type": "object", "required": []string{"group", "why"}, "properties": map[string]any{
"group": map[string]any{"type": "string", "description": "the delivery group's branch name"},
"why": map[string]any{"type": "string", "description": "why it is asked again, in words; kept in the verdict it ends"}}},
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-checker
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029000420-979f2cd71819
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+6 -2
View File
@@ -1,5 +1,9 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 h1:Tt16HJk8JZRce1i3JzzDSNdCWOrv7dw8FQ/bckNQ5Ig=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028230718-261f41aec1ae h1:bwcj9NnfGk/DP0sbTRInPAA4aMUnvr8+CmQLhnDvV9w=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028230718-261f41aec1ae/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028235320-62a50ebcf822 h1:od9tMSw9yHRLe4c03/u9EVr7TpXZ+pEJa2XY5evx6Ho=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028235320-62a50ebcf822/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029000420-979f2cd71819 h1:ypKERyJAbbsydMKLlPmWKU++wU2YCPLzGC0nzdw6QvQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029000420-979f2cd71819/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -6,7 +6,6 @@ import (
"errors"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -30,18 +29,6 @@ const (
// CheckAsks are the asks a check put, in order, as the state asks keeps them while it runs; its end deletes the key.
type CheckAsks struct {
Asks []string `json:"asks"`
// Judging names the step whose end the check heard and judges now (its levels, kept verdicts, the composition), until
// its next ask or its verdict: between a step's end and its verdict no step of the check runs, and without it the read
// tools took that judging for a check nothing will judge (issue 1721: mesh-catalog #562 was named lost ten seconds
// after its step passed, and checker_ask_again ended the judging that would have passed it). Nil while an ask waits.
Judging *Judging `json:"judging,omitempty"`
}
// Judging is the step whose end a running check heard and judges now, and since when.
type Judging struct {
Step string `json:"step"`
Node string `json:"node"`
Since time.Time `json:"since"`
}
// AsksKV is the state asks: a KV whose key a check's end deletes, so the state holds only the checks that run.
@@ -58,14 +45,13 @@ func (ch *Checker) forgetAsks(ctx context.Context, key string) {
}
}
// recordAsks keeps the asks a check put under its verdict key, with the step whose end it judges now (nil while an ask
// waits); nil Asks keeps nothing. A record that cannot be written is said: checker_ask_again then refuses the check, as it
// cannot tell whether its ask waits in a stream.
func (ch *Checker) recordAsks(ctx context.Context, key string, asks []string, judging *Judging) {
// recordAsks keeps the asks a check put under its verdict key; nil Asks keeps nothing. A record that cannot be written
// is said: checker_ask_again then refuses the check, as it cannot tell whether its ask waits in a stream.
func (ch *Checker) recordAsks(ctx context.Context, key string, asks []string) {
if ch.Asks == nil {
return
}
data, err := json.Marshal(CheckAsks{Asks: asks, Judging: judging})
data, err := json.Marshal(CheckAsks{Asks: asks})
if err == nil {
_, err = putOnce(ctx, ch.Asks, key, data, func(cur []byte) bool { return string(cur) == string(data) })
}
@@ -153,9 +139,6 @@ func (r Reader) AskAgain(w VerdictEntry, group, why string) (AskedAgain, error)
if err != nil || len(put.Asks) == 0 {
return AskedAgain{}, fmt.Errorf("the checker names no ask of %s (%v), so whether one waits in a build seat's stream cannot be told; asking again could run it twice", key, err)
}
if why := r.judgingWhy(put.Judging); why != "" {
return AskedAgain{}, fmt.Errorf("%s: asking again would end that judging and run the check twice", why)
}
taken := map[string]bool{}
for _, s := range steps.all {
taken[s.step.ID] = true
@@ -749,11 +749,8 @@ func (ch *Checker) try(ctx, run context.Context, key string, cs *Case) {
began := ch.now()
if cs.Watch == nil && ch.Asks != nil {
cs.Watch = func(_ string, step *delivery.Step) {
switch {
case step == nil: // put: kept before its wait starts
ch.recordAsks(ctx, cs.Key, cs.Asks(), nil)
case step.State != delivery.StepRunning: // its end, heard: judged from now until the next ask or the verdict
ch.recordAsks(ctx, cs.Key, cs.Asks(), &Judging{Step: step.ID, Node: step.Node, Since: ch.now().UTC()})
if step == nil { // put: kept before its wait starts
ch.recordAsks(ctx, cs.Key, cs.Asks())
}
}
defer ch.forgetAsks(ctx, cs.Key)
@@ -14,21 +14,10 @@ type fakeCompositions struct {
v delivery.CompositionVerdict
err error
asked []string
// hold, when set, is handed a value when Await is entered and holds it until it is handed one back: a check judging
// its step's end, its verdict not written yet.
hold chan struct{}
}
func (f *fakeCompositions) Await(ctx context.Context, key string) (delivery.CompositionVerdict, error) {
func (f *fakeCompositions) Await(_ context.Context, key string) (delivery.CompositionVerdict, error) {
f.asked = append(f.asked, key)
if f.hold != nil {
f.hold <- struct{}{}
select {
case <-f.hold:
case <-ctx.Done():
return delivery.CompositionVerdict{}, ctx.Err()
}
}
return f.v, f.err
}
@@ -484,6 +484,40 @@ func TestAHeadThatDoesNotMergeEndsFailedNamingTheFilesAndRunsNoTest(t *testing.T
if n := len(b.steps.tool("digest")) + len(b.steps.tool("test")); n != 0 {
t.Errorf("%d digest and test steps ran after a conflict", n)
}
// A group: after the first member's failure every other member is asked its merge alone, so one verdict names each
// member that does not merge, with its files, and runs no test of the others.
for name, c := range map[string]struct {
conflicts map[string]bool // by repository
want []string // the members the verdict names in conflict
listed int // the verdict's Failures: one each when more than one failed, none for one alone
}{
"both conflict": {map[string]bool{repoSDK: true, repoA: true}, []string{repoSDK + "#12 ", repoA + "#7 "}, 2},
"the second merges": {map[string]bool{repoSDK: true}, []string{repoSDK + "#12 "}, 0},
} {
b := newBench(t)
conflict := captured[Fetched](t, "go-fetch-conflict")
b.steps.handlers["build_go_fetch"] = func(sc stepCall) (any, error) {
if c.conflicts[sc.args["repository"].(string)] {
return conflict, nil
}
return b.fetch, nil
}
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))).Verdict
if v.State != delivery.VerdictFail || !IsConflictFailure(v) || len(v.Failures) != c.listed {
t.Fatalf("%s: verdict = %+v", name, v)
}
for n, w := range c.want {
if !strings.Contains(v.Failed.Words, w+ConflictWords+"a.go, sub/b.go") || len(v.Failures) > 0 && !strings.HasPrefix(v.Failures[n].Words, w) {
t.Errorf("%s: the verdict does not name %s in conflict: %+v", name, w, v)
}
}
if n := len(b.steps.tool("fetch")); n != 2 {
t.Errorf("%s: %d fetches, want each member's merge tried", name, n)
}
if n := len(b.steps.tool("digest")) + len(b.steps.tool("test")); n != 0 {
t.Errorf("%s: %d digest and test steps ran after a conflict", name, n)
}
}
}
func TestALevelWhoseDigestIsNotNamedRunsWholeAndKeepsNothing(t *testing.T) { // 1287/AC4
@@ -333,9 +333,104 @@ func (st *state) levelsOf() []string {
}
// judgeAll puts one check ask per pull request and build module the repository needs, the mesh-sdk member first, and
// judges each answer. A head that does not merge, a pin that is not allowed or a level that fails ends the check at once
// (ADR 0484 clause 13).
// judges each answer. A head that does not merge, a pin that is not allowed or a level that fails ends the check (ADR 0484
// clause 13); the members not yet judged are then asked their merge alone, so the verdict names every member that does not
// merge into its trunk, each with its files, and not only the first (a hidden conflict costs a whole check round).
func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
pulls, v := st.judgeUntilFailed(ctx)
if v == nil {
return nil
}
var more []delivery.FailedFirst
for _, p := range pulls {
if f, ok := st.mergeOnly(ctx, p); ok {
more = append(more, f)
}
}
return withMergeFailures(v, more)
}
// withMergeFailures adds to the verdict v the members asked their merge alone after it failed: what failed first stays
// first; a conflict's words name every member in conflict on one line, so the status line names them all, and the
// verdict's Failures list each.
func withMergeFailures(v *delivery.Verdict, more []delivery.FailedFirst) *delivery.Verdict {
if len(more) == 0 {
return v
}
if IsConflictFailure(*v) {
words := []string{v.Failed.Words}
for _, f := range more {
if strings.Contains(f.Words, ConflictWords) {
words = append(words, f.Words)
}
}
first := v.Failed
v.Failed.Words = strings.Join(words, "; ")
v.Failures = []delivery.FailedFirst{first}
} else if len(v.Failures) == 0 {
v.Failures = []delivery.FailedFirst{v.Failed}
}
all := append(slices.Clone(v.Failures), more...)
if n := len(all); n > delivery.MaxFailures+1 {
all = append(all[:delivery.MaxFailures], delivery.FailedFirst{Test: fmt.Sprintf("and %d more", n-delivery.MaxFailures)})
}
v.Failures = all
return v
}
// mergeOnly asks the merge of a member's head onto its trunk alone, of its repository's first build module, and answers the
// failure it names when the head does not merge or the ask could not be answered. A build module from before
// CheckJob.MergeOnly runs the levels too; only the fetch of its answer is read.
func (st *state) mergeOnly(ctx context.Context, p Pull) (delivery.FailedFirst, bool) {
bms := st.j.Modules.BuildModules(p.Repository)
if len(bms) == 0 || ctx.Err() != nil {
return delivery.FailedFirst{}, false
}
bm := bms[0]
args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk}
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: askLanguage[bm], Asker: delivery.CheckerModule,
At: st.j.now().UTC(), Check: &delivery.CheckJob{Fetch: args, Levels: st.levelsOf(), MergeOnly: true}}
st.c.Progress.Set(fmt.Sprintf("the merge alone of %s#%d (ask %s), after the check failed", p.Repository, p.Number, a.ID), "")
st.c.Asked(a.ID)
failed := func(machine, words string) (delivery.FailedFirst, bool) {
return delivery.FailedFirst{Machine: machine, Words: words}, true
}
if err := st.j.Asks.Put(ctx, a); err != nil {
return failed(bm, fmt.Sprintf("the merge of %s#%d onto its trunk could not be asked: %v", p.Repository, p.Number, err))
}
waiting := ctx
if watch := st.c.Watch; watch != nil {
id := a.ID
watch(id, nil)
waiting = buildrun.WithStepSeen(ctx, func(s delivery.Step) { watch(id, &s) })
}
step, err := st.j.Asks.Wait(waiting, a)
if err != nil {
if ctx.Err() != nil {
return delivery.FailedFirst{}, false // the check was ended: what failed first is its verdict
}
return failed(bm, fmt.Sprintf("the merge of %s#%d onto its trunk: %v", p.Repository, p.Number, err))
}
st.nodes[step.Node] = true
if step.State != delivery.StepPassed {
return failed(step.Node, fmt.Sprintf("the merge of %s#%d onto its trunk %s: %s", p.Repository, p.Number, step.State, step.Reason))
}
var answer delivery.CheckAnswer
var f Fetched
if err := json.Unmarshal(step.Result, &answer); err != nil || json.Unmarshal(answer.Fetch, &f) != nil {
return failed(step.Node, fmt.Sprintf("the merge of %s#%d onto its trunk answered what is not a check: %s", p.Repository, p.Number, step.Result))
}
if len(f.Conflicts) == 0 {
return delivery.FailedFirst{}, false
}
if st.c.Try == nil {
st.j.writePreview(ctx, p, []Fetched{f})
}
return failed(step.Node, fmt.Sprintf("%s#%d %s%s", p.Repository, p.Number, ConflictWords, strings.Join(f.Conflicts, ", ")))
}
// judgeUntilFailed is judgeAll up to the first failure, answering it and the members after the one that failed.
func (st *state) judgeUntilFailed(ctx context.Context) ([]Pull, *delivery.Verdict) {
// The mesh-sdk member first: whether its merge changes the TypeScript package's paths decides whether the other members'
// Node fetches take the group's package (ADR 0500 §5).
pulls := slices.Clone(st.c.Pulls)
@@ -349,7 +444,8 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
return 0
})
var groupPackage map[string]any
for _, p := range pulls {
for n, p := range pulls {
rest := pulls[n+1:]
var merged []Fetched
for _, bm := range st.j.Modules.BuildModules(p.Repository) {
t := &target{pull: p, module: bm}
@@ -363,7 +459,7 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
if bm == buildrun.GoBuildModule {
beside, err := st.j.besideFor(ctx, st.c, p)
if err != nil {
return fail(forgeMachine(err, bm), err.Error()+unheardWords(err))
return rest, fail(forgeMachine(err, bm), err.Error()+unheardWords(err))
}
if len(beside) > 0 {
args["beside"] = beside
@@ -377,7 +473,7 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
t.fetchArgs = args
st.targets = append(st.targets, t)
if v := st.ask(ctx, t, &merged); v != nil {
return v
return rest, v
}
}
if st.c.Try == nil { // a try owes the indexer no preview: no pull request holds its head
@@ -387,7 +483,7 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
groupPackage = map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number)}
}
}
return nil
return nil, nil
}
// DefaultDeclineBound is an Examiner's DeclineBound when it names none: longer than a check runs, so a node with the
@@ -266,8 +266,6 @@ func TestACheckWhoseAskJudgedNothingIsAskedAgainOrRead(t *testing.T) {
{name: "its watch deaf, a step of its build module ends", deaf: 1, how: "nudge", state: delivery.VerdictPass, asks: 1},
{name: "its watch deaf, asked again by hand", deaf: 1, how: "hand", state: delivery.VerdictPass, asks: 2},
{name: "its watch deaf, asked again by hand after its build module replaced it", deaf: 1, restarts: 1, how: "hand", state: delivery.VerdictPass, asks: 2},
// issue 1721: mesh-catalog #562 was named lost while the checker judged its step's end, and asked again by hand
{name: "its step's end heard and being judged", how: "judging", state: delivery.VerdictPass, asks: 1},
} {
t.Run(row.name, func(t *testing.T) {
b := newBench(t)
@@ -284,11 +282,6 @@ func TestACheckWhoseAskJudgedNothingIsAskedAgainOrRead(t *testing.T) {
b.ex.Asks = buildrun.Asker{Publisher: tr, Waiter: heard, Bound: 5 * time.Second}
asked = func() int { return len(tr.Asks()) }
}
var held *fakeCompositions
if row.how == "judging" {
held = &fakeCompositions{err: ErrAssignerNotRunning, hold: make(chan struct{})}
b.ex.Compositions = held
}
r := newRig(t, b.ex)
asksKept := newMemKV()
r.ch.Asks = asksKept
@@ -304,24 +297,6 @@ func TestACheckWhoseAskJudgedNothingIsAskedAgainOrRead(t *testing.T) {
}
}
switch row.how {
case "judging":
select {
case <-held.hold: // its step ended and was heard: the checker judges it, its verdict not written yet
case <-time.After(5 * time.Second):
t.Fatal("the check never came to judge its step's end")
}
cands := newMemKV()
ckey, _ := delivery.CandidatesKey("feat/x")
_, _ = cands.Create(ctx, ckey, cand)
rd := Reader{CandidateState: kvRead(cands), VerdictState: kvRead(r.verdicts), AskState: kvRead(asksKept),
StepStates: map[string]StateRead{buildrun.GoBuildModule: tr.Steps(false)}}
if st, _ := rd.Status(); st.Lost["feat/x"] != "" || !strings.Contains(st.Judging["feat/x"], "judges the end of its step") {
t.Errorf("status while its step's end is judged: lost %v, judging %v; want judging, not lost", st.Lost, st.Judging)
}
if _, err := rd.AskAgain(asVerdicts{r.verdicts}, "feat/x", "named lost"); err == nil || !strings.Contains(err.Error(), "judges the end of its step") {
t.Errorf("asked again while its step's end is judged: %v", err)
}
held.hold <- struct{}{}
case "nudge":
heard.Nudge(buildrun.NodeBuildModule) // another build module's step: not this one's to read
heard.Nudge(buildrun.GoBuildModule)
@@ -338,10 +338,6 @@ type Status struct {
// Lost are the candidates whose verdict is running while no step of theirs can still run and their newest step ended, by
// group, with that step: nothing will judge them (the lost check of 2026-10-28); checker_ask_again asks one again.
Lost map[string]string `json:"lost"`
// Judging are the candidates whose verdict is running while the checker judges the end of their newest step (its
// levels, kept verdicts, the composition), by group, with that step and since when: their verdict follows by itself,
// and checker_ask_again refuses them (issue 1721).
Judging map[string]string `json:"judging"`
// DeadLetters says whether the checker hears the asks the build seats give up on (issue 734): deaf, a try or check
// whose ask is given up waits until its bound.
DeadLetters DeadLettersHeard `json:"dead_letters"`
@@ -405,7 +401,7 @@ func heldBy(t *Turns, c delivery.Candidate, heads map[string]string) string {
// Status reads the build modules' steps and the candidates against their verdicts.
func (r Reader) Status() (Status, error) {
st := Status{RunningByNode: map[string]int{}, Lost: map[string]string{}, Judging: map[string]string{}}
st := Status{RunningByNode: map[string]int{}, Lost: map[string]string{}}
read := r.readSteps()
st.Running, st.LeftRunning, st.Notes = read.running, read.left, read.notes
for _, rs := range st.Running {
@@ -491,9 +487,7 @@ func (r Reader) Status() (Status, error) {
switch {
case err != nil:
case v.State == delivery.VerdictRunning:
if judging := r.judgingWhy(r.judgingOf(c)); judging != "" {
st.Judging[c.Group] = judging
} else if why := read.lostWhy(c); why != "" {
if why := read.lostWhy(c); why != "" {
st.Lost[c.Group] = why
}
case IsNodesFailure(v):
@@ -628,41 +622,6 @@ func (s stepsRead) lostWhy(c delivery.Candidate) string {
last.ID, last.Node, last.State, last.Ended.UTC().Format(time.RFC3339), AskAgainTool)
}
// judgingOf is the step whose end the candidate's running check judges now, as the checker's state asks keeps it; nil
// for none, or when the state cannot be read.
func (r Reader) judgingOf(c delivery.Candidate) *Judging {
key, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
if err != nil || r.AskState.Get == nil {
return nil
}
raw, err := r.AskState.Get(key)
var put CheckAsks
if err != nil || len(raw) == 0 || json.Unmarshal(raw, &put) != nil {
return nil
}
return put.Judging
}
// judgingWhy says that the checker judges a step's end now, or "" for none: a judging older than the step bound is not
// counted, as the checker that heard that end stopped before its verdict (one that starts again takes the check anew and
// writes its asks again), so the check is lost.
func (r Reader) judgingWhy(j *Judging) string {
if j == nil {
return ""
}
now, bound := time.Now(), r.StepBound
if r.Now != nil {
now = r.Now()
}
if bound <= 0 {
bound = DefaultStepBound
}
if now.Sub(j.Since) > bound {
return ""
}
return fmt.Sprintf("the checker judges the end of its step %s on %s since %s; its verdict follows", j.Step, j.Node, j.Since.UTC().Format(time.RFC3339))
}
// DefaultStepBound is how long one check ask's step may run once a build node took it (its key written running; the fetch,
// the digests and the levels), never counting its wait in the stream; past it the check fails naming the node.
const DefaultStepBound = 30 * time.Minute
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 h1:Tt16HJk8JZRce1i3JzzDSNdCWOrv7dw8FQ/bckNQ5Ig=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-grants
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028191251-5228a5782d5c
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 h1:Tt16HJk8JZRce1i3JzzDSNdCWOrv7dw8FQ/bckNQ5Ig=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028191251-5228a5782d5c h1:ngcItoLuh2uP24toZvkCXGA/3+TBrLTwTbE28XpYDWY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028191251-5228a5782d5c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=