10 Commits
Author SHA1 Message Date
jschoubben a5b763f8a2 Merge remote-tracking branch 'origin/main' into feat/try-speed 2026-10-25 03:17:48 +01:00
jschoubben 50a2e4c4e2 mesh-delivery-checker: let a try asked by a bare commit take a mesh-sdk pin that is the head of a mesh-sdk branch, naming it, as a try asked by branch does (issue 1921, issue 1624, try speed S12) 2026-10-25 02:03:26 +02:00
jschoubben 4169faf3f0 mesh-delivery-checker: report a lint finding as the lint step's failure (ADR 0542 §2); pin mesh-sdk's try speed-up head with main in it in the checker and the indexer (issue 1921) 2026-10-25 01:31:53 +02:00
jschoubben 103eec8855 Merge remote-tracking branch 'origin/main' into feat/try-speed 2026-10-25 01:29:44 +02:00
jschoubben 8a62f23afe mesh-delivery-indexer: pin mesh-sdk at the try speed-up's head, beside the checker (issue 1921) 2026-10-25 01:27:18 +02:00
jschoubben 84395420ee mesh-delivery-checker: take an ask's longest level off its time for the clone, as the levels now run side by side, and pin mesh-sdk's side-by-side levels (issue 1921, try speed S10) 2026-10-25 01:24:37 +02:00
jschoubben 9740842af7 mesh-delivery-checker: stop a check whose heads its group no longer holds and end it failed, superseded; judge a group's open members by the verdict it had once others merged, so their merge is not refused (issue 1921, issue 1791, try speed S9) 2026-10-25 01:20:29 +02:00
jschoubben 588cfd54bb mesh-delivery-indexer: keep a merged member in its group's candidate until every member merged, so the heads the Approve covers hold and no new check starts for the members left (issue 1921, issue 1791, try speed S9) 2026-10-25 01:18:20 +02:00
jschoubben 4ccd78c9ec mesh-delivery-checker: put every member's check ask of a group at once and judge the answers in member order, so a group's members run side by side (issue 1921, try speed S8) 2026-10-25 01:15:28 +02:00
jschoubben 6fe7b7dd6c Name every failed test in a check's verdict and a try's failed list (issue 1921)
The checker stopped reading a level's test results at the first that did
not pass and kept only its first failure, so a check named one fault per
round. It now reads and counts every result and writes every failed test
with its first line into the verdict's Failures and the try's failed
list (at most 50, then how many more); Failed keeps the first one's
words whole (issue 1901). Pins mesh-sdk fix/1921-every-failure.
2026-10-24 23:44:46 +02:00
34 changed files with 761 additions and 1138 deletions
@@ -341,14 +341,12 @@ func TestAHeldFingerprintIsAskedAbout(t *testing.T) {
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`)) b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`))
b.start(t) b.start(t)
hkey, _ := delivery.HeldKey("web", fNew) hkey, _ := delivery.HeldKey("web", fNew)
// 1906/H3: the held record names the machine and keeps the health, when and the node's words. if h, err := delivery.DecodeHeld(b.get(t, delivery.AssignerWantsBucket, hkey)); err != nil || h.Node != "ace" {
if h, err := delivery.DecodeHeld(b.get(t, delivery.AssignerWantsBucket, hkey)); err != nil || h.Node != "ace" ||
!strings.HasPrefix(h.Words, "health failed, phase running, seen at ") || !strings.HasSuffix(h.Words, ": connection refused") {
t.Fatalf("held = %+v, %v", h, err) t.Fatalf("held = %+v, %v", h, err)
} }
akey, _ := delivery.HoldAskKey("web", fNew) akey, _ := delivery.HoldAskKey("web", fNew)
a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey)) a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
if err != nil || a.Asking <= 0 || a.Ask.ID != delivery.AskingID(akey, a.Asking) || len(a.Ask.Options) != 2 { if err != nil || a.Ask.ID != delivery.AskingID(akey, 0) || len(a.Ask.Options) != 2 {
t.Fatalf("the ask = %+v, %v", a, err) t.Fatalf("the ask = %+v, %v", a, err)
} }
} }
@@ -582,15 +580,8 @@ func (b *bed) wantNow(t *testing.T, node, module, what string, ok func(delivery.
return last return last
} }
// answer says the teller wrote the operator's answer to the ask that stands at key (its warrant names that asking's id). // answer says the teller wrote the operator's answer to the ask at key.
func (b *bed) answer(t *testing.T, key, option string) { func (b *bed) answer(t *testing.T, key, option string) { b.answerTo(t, key, option, "r1") }
t.Helper()
a, err := delivery.DecodeAssignerAsk(key, b.get(t, delivery.AssignerAsksBucket, key))
if err != nil {
t.Fatal(err)
}
b.answerTo(t, key, option, a.Ask.ID)
}
// answerTo is answer to the ask with this id (the router's warrant names it). // answerTo is answer to the ask with this id (the router's warrant names it).
func (b *bed) answerTo(t *testing.T, key, option, askID string) { func (b *bed) answerTo(t *testing.T, key, option, askID string) {
@@ -782,81 +773,3 @@ func TestAUserListRevisionOfTheBusIsWrittenAtOnce(t *testing.T) {
t.Errorf("the user-list revision was asked") t.Errorf("the user-list revision was asked")
} }
} }
// 1906/H2, whole on the bus: given a fingerprint held after one failed check, when every node that runs the module then runs
// it healthy, the assigner lifts the hold by itself: the held record and its ask are taken away, the held condition is
// cleared saying why, and the want stays at the fingerprint.
func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
b := startBed(t, "")
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
b.start(t)
hkey, _ := delivery.HeldKey("web", fNew)
akey, _ := delivery.HoldAskKey("web", fNew)
b.get(t, delivery.AssignerWantsBucket, hkey)
b.get(t, delivery.AssignerAsksBucket, akey)
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
if b.hasKey(delivery.AssignerWantsBucket, hkey) || b.hasKey(delivery.AssignerAsksBucket, akey) {
continue
}
for _, c := range b.conditionsSaid(t) {
if c.Subject.ID == "web" && c.State == asks.StateCleared && strings.Contains(c.Resolved, "lifted its hold by itself") {
if w := b.wantNow(t, "ace", "web", "the want at the lifted fingerprint", func(w delivery.Want) bool { return w.FP == fNew }); w.Rule != 0 {
t.Errorf("the want = %+v", w)
}
return
}
}
}
t.Fatalf("the hold was not lifted: held %v, ask %v", b.hasKey(delivery.AssignerWantsBucket, hkey), b.hasKey(delivery.AssignerAsksBucket, akey))
}
// 1906/H2, whole on the bus: a hold lifted by itself and written again after the assigner was started again is asked under a
// new id, so the teller raises it; the answer to the first hold does not carry over (1906/H4).
func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) {
b := startBed(t, "")
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
_, stop := b.startStoppable(t)
hkey, _ := delivery.HeldKey("web", fNew)
akey, _ := delivery.HoldAskKey("web", fNew)
first, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.AssignerWantsBucket, hkey); time.Sleep(100 * time.Millisecond) {
if time.Now().After(deadline) {
t.Fatal("the hold was not lifted")
}
}
stop()
time.Sleep(1100 * time.Millisecond) // the next hold is written in another second
b.answerTo(t, akey, "keep", first.Ask.ID) // an answer to the first hold, which the later hold must not take
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
b.start(t)
again, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
if err != nil {
t.Fatal(err)
}
if again.Ask.ID == first.Ask.ID {
t.Fatalf("the hold written again is asked under the first id %s", first.Ask.ID)
}
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.AssignerWantsBucket, hkey); time.Sleep(100 * time.Millisecond) {
if time.Now().After(deadline) {
t.Fatal("the stale \"keep\" of the first hold kept the second one held")
}
}
}
@@ -28,15 +28,6 @@ type passer struct {
// answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want. // answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want.
answeredAt map[string]uint64 answeredAt map[string]uint64
asks *composeAsks asks *composeAsks
// now is the clock; nil is time.Now.
now func() time.Time
}
func (p *passer) clock() time.Time {
if p.now != nil {
return p.now()
}
return time.Now()
} }
func (p *passer) pass(ctx context.Context) { func (p *passer) pass(ctx context.Context) {
@@ -97,7 +88,7 @@ func (p *passer) wants(ctx context.Context) {
for name := range names { for name := range names {
fp := trunk[name].FP2 fp := trunk[name].FP2
name := name name := name
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name], Now: time.Now(), m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name],
Composed: func(node, fp string) (delivery.ComposedDigest, bool) { return p.w.composed(node, name, fp) }, Composed: func(node, fp string) (delivery.ComposedDigest, bool) { return p.w.composed(node, name, fp) },
Restored: func(node string) bool { Restored: func(node string) bool {
key, err := delivery.RestoreAskKey(node, name, fp) key, err := delivery.RestoreAskKey(node, name, fp)
@@ -112,7 +103,6 @@ func (p *passer) wants(ctx context.Context) {
grant, _ := delivery.ControllerGrantKey(name, fp) grant, _ := delivery.ControllerGrantKey(name, fp)
hold, _ := delivery.HoldAskKey(name, fp) hold, _ := delivery.HoldAskKey(name, fp)
m.Released = p.answeredWith(hold, "release") m.Released = p.answeredWith(hold, "release")
m.Kept = p.answeredWith(hold, "keep")
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(delivery.ControllerGrantsBucket, grant) m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(delivery.ControllerGrantsBucket, grant)
if m.Released { if m.Released {
p.clearHeld(name, fp) p.clearHeld(name, fp)
@@ -130,11 +120,8 @@ func (p *passer) wants(ctx context.Context) {
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeHeld(held) }) p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeHeld(held) })
} }
} }
if out.Lifted { if out.Held != nil || m.Held {
p.lift(ctx, name, fp) p.holdAsk(ctx, name, fp)
}
if out.Held != nil || (m.Held && !out.Lifted) {
p.holdAsk(ctx, name, fp, out.Held)
p.heldCondition(ctx, name, fp, out.Held, len(out.Restore) > 0) p.heldCondition(ctx, name, fp, out.Held, len(out.Restore) > 0)
} }
for _, node := range out.Restore { for _, node := range out.Restore {
@@ -223,62 +210,15 @@ func (p *passer) put(ctx context.Context, bucket, key string, encode func() ([]b
// holdAsk writes, once, the ask about a held fingerprint (ADR 0528 clause 6, ADR 0437 clause 10): the first ask that needs // holdAsk writes, once, the ask about a held fingerprint (ADR 0528 clause 6, ADR 0437 clause 10): the first ask that needs
// no proof of who answered, which the view may answer and the teller fills. In release 5 an answer changes nothing on a // no proof of who answered, which the view may answer and the teller fills. In release 5 an answer changes nothing on a
// node (wants are only compared, ADR 0429 §7), and the options say so. // node (wants are only compared, ADR 0429 §7), and the options say so.
func (p *passer) holdAsk(ctx context.Context, module, fp string, fresh *delivery.Held) { func (p *passer) holdAsk(ctx context.Context, module, fp string) {
key, err := delivery.HoldAskKey(module, fp) key, err := delivery.HoldAskKey(module, fp)
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) { if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
return return
} }
// The asking is the hold's time in seconds: a hold of the same fingerprint written again after a lift is another asking, under a := assign.HoldAsk(key, module, fp, time.Now())
// an id of its own, also after a restart, since the router drops a second ask under an id it keeps (issue 1906).
now := p.clock()
asking := int(now.Unix())
h := p.heldHold(module, fp, fresh)
a := assign.HoldAsk(key, module, fp, h.Node, h.Words, asking, now)
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) }) p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
} }
// lift takes away a hold the assigner lifted by itself, since every node that runs the module runs the held fingerprint healthy
// (issue 1906): the held record and its ask (the teller then takes the ask back from the operator), and the held condition is
// cleared saying why, once. A hold of the same fingerprint written later is said and asked again.
func (p *passer) lift(ctx context.Context, module, fp string) {
held, err := delivery.HeldKey(module, fp)
if err != nil {
return
}
ask, err := delivery.HoldAskKey(module, fp)
if err != nil {
return
}
c := assign.LiftedCondition(module, fp)
if !p.said[c.Key+".lifted"] {
if p.say != nil {
if err := p.say(c); err != nil {
// The hold stays written, so the next pass lifts it again and says so then.
p.logf("the condition of the lifted hold of %s could not be cleared, and is cleared at the next pass: %v", module, err)
return
}
}
p.logf("%s", assign.LiftedWords(module, fp))
p.said[c.Key+".lifted"] = true
delete(p.said, c.Key+".raised") // a hold written again is said again
}
p.remove(ctx, delivery.AssignerWantsBucket, held)
p.remove(ctx, delivery.AssignerAsksBucket, ask)
}
// remove deletes one key of the assigner's own bucket when it holds it; a delete that fails is said and made again at the next
// pass.
func (p *passer) remove(ctx context.Context, bucket, key string) {
if !p.w.has(bucket, key) {
return
}
if err := p.kvs[bucket].Delete(ctx, key); err != nil && !errors.Is(err, context.Canceled) {
p.logf("%s %s could not be taken away: %v", bucket, key, err)
return
}
p.w.put(bucket, key, nil)
}
// heldHold is the hold of fp as written: the node whose health check failed and its words. // heldHold is the hold of fp as written: the node whose health check failed and its words.
func (p *passer) heldHold(module, fp string, fresh *delivery.Held) delivery.Held { func (p *passer) heldHold(module, fp string, fresh *delivery.Held) delivery.Held {
if fresh != nil { if fresh != nil {
@@ -305,7 +245,6 @@ func (p *passer) heldCondition(ctx context.Context, module, fp string, fresh *de
return return
} }
p.said[c.Key+".raised"] = true p.said[c.Key+".raised"] = true
delete(p.said, c.Key+".lifted") // a hold raised again and lifted again is said lifted again
} }
// restoreAsk writes, once, the ask about a module holding data whose held fingerprint failed its health check on node. // restoreAsk writes, once, the ask about a module holding data whose held fingerprint failed its health check on node.
@@ -319,21 +258,11 @@ func (p *passer) restoreAsk(ctx context.Context, module, node, fp string) {
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) }) p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
} }
// answeredWith says whether the teller wrote an answer to the ask at key as it stands now and the option chosen is option. The // answeredWith says whether the teller wrote an answer to the ask at key and the option chosen is option. The answer is the
// answer is the operator's, given in the view or on the phone, and stays under its key (ADR 0539 §14, §16, §19); it counts only // operator's, given in the view or on the phone, and stays under its key (ADR 0539 §14, §16, §19).
// for the asking it answered (its warrant names that asking's id), so an answer to an ask taken away, or to an earlier hold of
// the same fingerprint, does not carry over to a later one (issue 1906).
func (p *passer) answeredWith(key, option string) bool { func (p *passer) answeredWith(key, option string) bool {
a, ok := p.w.answer(key) a, ok := p.w.answer(key)
if !ok || a.Option != option { return ok && a.Option == option
return false
}
ask, err := delivery.DecodeAssignerAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
if err != nil {
return false
}
id, _, _ := strings.Cut(a.Warrant, "@")
return id == ask.Ask.ID
} }
// clearHeld clears, once, the condition of a hold the operator released. // clearHeld clears, once, the condition of a hold the operator released.
@@ -1,100 +0,0 @@
package main
import (
"context"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-assigner/internal/assign"
)
// memKV is the part of a bucket a pass writes: puts and deletes.
type memKV struct {
jetstream.KeyValue
values map[string][]byte
}
func (m *memKV) Put(_ context.Context, key string, value []byte) (uint64, error) {
m.values[key] = value
return uint64(len(m.values)), nil
}
func (m *memKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
delete(m.values, key)
return nil
}
// newPasser is a pass on a fresh world, as after a start, writing to the buckets given, at the clock given.
func newPasser(kvs buckets, at time.Time) *passer {
return &passer{kvs: kvs, w: newWorld(), logf: func(string, ...any) {}, said: map[string]bool{}, answeredAt: map[string]uint64{},
now: func() time.Time { return at }}
}
var fpTest = delivery.SourcePrefix2 + strings.Repeat("b", 64)
// 1906/H2: given a hold lifted by itself, when the assigner is started again and the same fingerprint is held again, then the
// hold ask is written under another asking, with an id of its own, so the teller raises it (the router drops a second ask under
// an id it keeps).
func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
kvs := buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}
key, _ := delivery.HoldAskKey("web", fpTest)
held := &delivery.Held{Node: "ace", Words: "health failed"}
first := newPasser(kvs, time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC))
first.holdAsk(context.Background(), "web", fpTest, held)
a1, err := delivery.DecodeAssignerAsk(key, asksKV.values[key])
if err != nil {
t.Fatal(err)
}
first.lift(context.Background(), "web", fpTest)
if _, there := asksKV.values[key]; there {
t.Fatal("the lift left the hold ask")
}
again := newPasser(kvs, time.Date(2026, 10, 25, 2, 0, 0, 0, time.UTC)) // started again: it remembers nothing
again.holdAsk(context.Background(), "web", fpTest, held)
a2, err := delivery.DecodeAssignerAsk(key, asksKV.values[key])
if err != nil {
t.Fatal(err)
}
if a2.Ask.ID == a1.Ask.ID {
t.Errorf("the ask written again after a lift and a restart has the id %s of the first", a1.Ask.ID)
}
}
// 1906/H4: an answer counts only for the asking it answered: an answer to an earlier hold of the same fingerprint, or to an ask
// taken away, is not taken for the hold that stands now.
func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
key, _ := delivery.HoldAskKey("web", fpTest)
p := newPasser(buckets{}, time.Now())
put := func(asking int) string {
a := assign.HoldAsk(key, "web", fpTest, "ace", "health failed", asking, time.Now())
v, err := delivery.EncodeAssignerAsk(key, a)
if err != nil {
t.Fatal(err)
}
p.w.put(delivery.AssignerAsksBucket, key, v)
return a.Ask.ID
}
first := put(100)
v, _ := delivery.EncodeAnswer(delivery.Answer{Option: "release", Warrant: first + "@2026-10-25T01:00:00Z", Words: "the operator"})
p.w.put(delivery.AnswersBucket, key, v)
if !p.answeredWith(key, "release") {
t.Fatal("the answer to the ask that stands is not taken")
}
if p.answeredWith(key, "keep") {
t.Error("an answer was taken for an option not chosen")
}
put(200) // the same fingerprint held again, after a lift
if p.answeredWith(key, "release") {
t.Error("the answer to an earlier hold carried over to a later one")
}
p.w.put(delivery.AssignerAsksBucket, key, nil) // the ask taken away
if p.answeredWith(key, "release") {
t.Error("an answer to an ask taken away was taken")
}
}
@@ -326,13 +326,6 @@ func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[stri
return n return n
} }
// value is what a bucket holds at a key now, nil when nothing.
func (w *world) value(bucket, key string) []byte {
w.mu.Lock()
defer w.mu.Unlock()
return w.values[bucket][key]
}
// has says whether a bucket holds a key now. // has says whether a bucket holds a key now.
func (w *world) has(bucket, key string) bool { func (w *world) has(bucket, key string) bool {
w.mu.Lock() w.mu.Lock()
@@ -10,17 +10,14 @@ import (
"git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery"
) )
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey): keep it held or // HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey), first asking: keep
// release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19). node and words are the held // it held or release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19).
// record's: the machine whose health check failed and the hold's words (issue 1906). asking numbers the writing: 0 the first, a func HoldAsk(key, module, fp string, now time.Time) delivery.AssignerAsk {
// larger number an ask written again after a hold lifted by itself, so the router takes it as new. return delivery.AssignerAsk{Ask: asks.Ask{
func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) delivery.AssignerAsk { ID: delivery.AskingID(key, 0),
return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
ID: delivery.AskingID(key, asking),
Headline: fmt.Sprintf("%s is held at %s", module, short(fp)), Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
Explanation: fmt.Sprintf("The health check of %s failed on %s at %s (%s), so the assigner holds that version and wants "+ Explanation: fmt.Sprintf("A node's health check of %s failed at %s, so the assigner holds that version and wants "+
"no node to move to it. It lifts the hold by itself once every node that runs %s runs it healthy; releasing it "+ "no node to move to it. Releasing it lets the wants move to it again.", module, short(fp)),
"lets the wants move to it now.", module, node, short(fp), words, module),
Options: []asks.Option{ Options: []asks.Option{
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge}, {ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge},
{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: asks.Acknowledge}, {ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: asks.Acknowledge},
@@ -81,13 +78,6 @@ func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.C
return c return c
} }
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
func LiftedCondition(module, fp string) asks.Condition {
c := HeldCondition(module, fp, "", "", false, false)
c.Resolved = LiftedWords(module, fp) + "."
return c
}
// The options of the record ask (ADR 0539 §14) and of the backup ask (§16). // The options of the record ask (ADR 0539 §14) and of the backup ask (§16).
const ( const (
OptionYes = "yes" OptionYes = "yes"
@@ -17,25 +17,13 @@ func TestTheHoldAskIsValid(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
now := time.Now() now := time.Now()
a := HoldAsk(key, "mail", fNew, "ace", "health failed, phase running: connection refused", 0, now) a := HoldAsk(key, "mail", fNew, now)
if err := a.Ask.Check(now); err != nil { if err := a.Ask.Check(now); err != nil {
t.Errorf("the router refuses the ask: %v", err) t.Errorf("the router refuses the ask: %v", err)
} }
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil { if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
t.Errorf("the ask cannot be written: %v", err) t.Errorf("the ask cannot be written: %v", err)
} }
// 1906/H3: the ask names the machine whose check failed and the hold's words.
if !strings.Contains(a.Ask.Explanation, "on ace") || !strings.Contains(a.Ask.Explanation, "connection refused") {
t.Errorf("the ask does not name the node and the hold's words: %q", a.Ask.Explanation)
}
// 1906/H2: an ask written again after a lift is another asking, under an id of its own.
again := HoldAsk(key, "mail", fNew, "ace", "x", 7, now)
if again.Asking != 7 || again.Ask.ID != delivery.AskingID(key, 7) || again.Ask.ID == a.Ask.ID {
t.Errorf("the asking after a lift = %d %s", again.Asking, again.Ask.ID)
}
if _, err := delivery.EncodeAssignerAsk(key, again); err != nil {
t.Errorf("the ask asked again cannot be written: %v", err)
}
if _, err := delivery.ViewAnswerKey(key); err != nil { if _, err := delivery.ViewAnswerKey(key); err != nil {
t.Errorf("the view may not answer it: %v", err) t.Errorf("the view may not answer it: %v", err)
} }
@@ -86,17 +74,6 @@ func TestTheHeldConditionNamesModuleNodeAndWords(t *testing.T) {
} }
} }
// 1906/H2: a hold lifted by itself clears the held condition under its key, saying why.
func TestALiftedHoldClearsItsConditionSayingWhy(t *testing.T) {
c := LiftedCondition("mail", fNew)
if _, err := asks.EncodeCondition(c); err != nil {
t.Fatalf("the condition is refused: %v", err)
}
if c.State != asks.StateCleared || c.Key != HeldCondition("mail", fNew, "ace", "x", true, false).Key || !strings.Contains(c.Resolved, "lifted its hold by itself") {
t.Errorf("condition = %+v", c)
}
}
// 1906/D4: the record ask and the backup ask are valid asks of the router and of mesh-sdk's layout at their keys, answerable in // 1906/D4: the record ask and the backup ask are valid asks of the router and of mesh-sdk's layout at their keys, answerable in
// the view, and name the move and the node. // the view, and name the move and the node.
func TestTheRecordAndBackupAsksAreValid(t *testing.T) { func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
@@ -4,8 +4,6 @@ import (
"fmt" "fmt"
"slices" "slices"
"sort" "sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery"
@@ -33,10 +31,6 @@ type Module struct {
// Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back // Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back
// as any module is, losing the writes made since (ADR 0539 §19). Nil: none did. // as any module is, losing the writes made since (ADR 0539 §19). Nil: none did.
Restored func(node string) bool Restored func(node string) bool
// Kept says the operator answered "keep" to the hold ask of FP: a hold kept so is not lifted by itself.
Kept bool
// Now is when this pass reads the haves: the held record names it as when the health check was seen to fail. Zero: not said.
Now time.Time
} }
func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) { func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
@@ -144,9 +138,6 @@ type Outcome struct {
Wants map[string]delivery.Want Wants map[string]delivery.Want
// Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet. // Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet.
Held *delivery.Held Held *delivery.Held
// Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held
// record and its ask are taken away, and the wants move by the rules again in this pass.
Lifted bool
// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14). // Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
Record []RecordNeed Record []RecordNeed
// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set // Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
@@ -179,13 +170,10 @@ type Holding struct {
func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[string]string) Outcome { func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[string]string) Outcome {
out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}} out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}}
held := m.Held held := m.Held
if held && !m.Kept && m.FP != "" && allRunHealthy(m, n) { if !held && !m.Released && m.FP != "" {
out.Lifted, held = true, false
}
if !held && !out.Lifted && !m.Released && m.FP != "" {
for _, node := range slices.Sorted(slices.Values(m.Nodes)) { for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && h.Health == delivery.HealthFailed { if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && h.Health == delivery.HealthFailed {
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)} out.Held = &delivery.Held{Node: node, Words: h.Detail}
held = true held = true
break break
} }
@@ -264,43 +252,6 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
return out return out
} }
// allRunHealthy says whether every node of m that runs the module (has a have of it) runs m.FP healthy, and at least one does:
// a hold whose fingerprint runs healthy wherever the module runs holds nothing back any more (issue 1906: seven modules stayed
// held on every machine after one transient failed check). A node set back to another fingerprint does not run m.FP, so its
// hold stays.
func allRunHealthy(m Module, n Nodes) bool {
ran := 0
for _, node := range m.Nodes {
h, ok := n.Haves[node][m.Name]
if !ok {
continue
}
if !healthy(h, m.FP) {
return false
}
ran++
}
return ran > 0
}
// HeldWords are the words a hold keeps of its cause (issue 1906): the health and phase the node reported, when the assigner saw
// it, and the node's own words. The node is the held record's own field.
func HeldWords(h delivery.Have, at time.Time) string {
words := fmt.Sprintf("health %s, phase %s", h.Health, h.Phase)
if !at.IsZero() {
words += ", seen at " + at.UTC().Format(time.RFC3339)
}
if d := strings.TrimSpace(h.Detail); d != "" {
return words + ": " + d
}
return words + ": the node gave no words"
}
// LiftedWords say why a hold was lifted by itself.
func LiftedWords(module, fp string) string {
return fmt.Sprintf("every node that runs %s runs %s healthy, so the assigner lifted its hold by itself", module, short(fp))
}
// normal is a want without its version, as a decoded and a composed one are compared. // normal is a want without its version, as a decoded and a composed one are compared.
func normal(w delivery.Want) delivery.Want { func normal(w delivery.Want) delivery.Want {
w.V = 0 w.V = 0
@@ -4,7 +4,6 @@ import (
"slices" "slices"
"strings" "strings"
"testing" "testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery"
@@ -111,10 +110,8 @@ func TestAnOfflineCanaryIsPassedOver(t *testing.T) {
func TestAFailedHealthHoldsAndSetsBack(t *testing.T) { func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
m, n := ready() m, n := ready()
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"} n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"}
m.Now = time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC)
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld}) out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld})
// 1906/H3: the held record keeps its cause in words: which machine (its node), which health, when, and the node's words. if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "connection refused" {
if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "health failed, phase running, seen at 2026-10-25T01:00:00Z: connection refused" {
t.Fatalf("held = %+v", out.Held) t.Fatalf("held = %+v", out.Held)
} }
if w := out.Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != delivery.RuleNotHeld { if w := out.Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != delivery.RuleNotHeld {
@@ -523,59 +520,3 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) {
} }
} }
} }
// 1906/H3: a failed check whose node gave no words is held saying so, never with empty words.
func TestAHoldOfAFailureWithoutWordsSaysSo(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}}, map[string]string{"ace": fOld})
if out.Held == nil || out.Held.Words != "health failed, phase running: the node gave no words" {
t.Fatalf("held = %+v", out.Held)
}
}
// 1906/H2: given a held fingerprint that every node running the module now runs healthy, when the assigner passes, then the
// hold is lifted by itself and the wants move to it by the rules again. A node set back to another fingerprint, a node whose
// check fails, or a hold the operator answered "keep" leaves it held; a node that does not run the module at all does not count.
func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
everywhere := func() (Module, Nodes) {
m, n := ready()
m.Held = true
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
return m, n
}
m, n := everywhere()
out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld})
if !out.Lifted || out.Held != nil {
t.Fatalf("lifted = %v, held = %+v: want the hold lifted", out.Lifted, out.Held)
}
for _, node := range []string{"ace", "g14"} {
if w := out.Wants[node]; w.FP != fNew || w.Rule != 0 {
t.Errorf("%s's want = %+v, want it moved to the lifted fingerprint", node, w)
}
}
m, n = everywhere()
delete(n.Haves["g14"], "mail")
if out := Wants(m, n, wantedOld(), nil); !out.Lifted {
t.Error("a node that does not run the module kept the hold")
}
for name, change := range map[string]func(*Module, *Nodes){
"set back": func(_ *Module, n *Nodes) {
n.Haves["g14"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
},
"failing": func(_ *Module, n *Nodes) {
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
},
"kept": func(m *Module, _ *Nodes) { m.Kept = true },
"no haves": func(_ *Module, n *Nodes) { delete(n.Haves["g14"], "mail"); delete(n.Haves["ace"], "mail") },
"not held": func(m *Module, _ *Nodes) { m.Held = false },
} {
m, n := everywhere()
change(&m, &n)
if out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld}); out.Lifted {
t.Errorf("%s: the hold was lifted", name)
}
}
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-checker
go 1.26.0 go 1.26.0
require ( require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95
github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0 github.com/nats-io/nats.go v1.54.0
) )
+8 -2
View File
@@ -1,5 +1,11 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93 h1:FAiFY2KBKEoZSH49OZTt7/zRBIQxyl5g/KhYrpggbN4= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024214154-bc16740feb0d h1:oJREw0ggKqJWJ5DEISSL0i2ZC5XqdXAidxo9NJ74TCc=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024214154-bc16740feb0d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024231716-571ac521f053 h1:8Kbp3aJG3jGndDOfmGdMSDgJVEcc1hyu921hVnnlnlQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024231716-571ac521f053/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232250-9b408db24a58 h1:4Damtdqlf6u6nJhvMA99peDy1jmjixtcwFXJ4bWEkLk=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232250-9b408db24a58/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95 h1:I8zbgGzRjmgYydPTWo8hmaZW/P/g+qGDUfu3E32/pkU=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -0,0 +1,96 @@
package check
import (
"context"
"fmt"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// countingAsks puts asks through inner, and answers no wait until want asks are put (or a bound passes, when it says
// how many were): a group whose asks are put one after another, each waited for, never gets past its first.
type countingAsks struct {
inner Asks
want int
mu sync.Mutex
puts int
all chan struct{}
}
func (c *countingAsks) Put(ctx context.Context, a delivery.StepAsk) error {
if err := c.inner.Put(ctx, a); err != nil {
return err
}
c.mu.Lock()
c.puts++
if c.puts == c.want {
close(c.all)
}
c.mu.Unlock()
return nil
}
func (c *countingAsks) Wait(ctx context.Context, a delivery.StepAsk) (delivery.Step, error) {
select {
case <-c.all:
case <-time.After(time.Second):
c.mu.Lock()
defer c.mu.Unlock()
return delivery.Step{}, fmt.Errorf("%d of %d asks were put before the first was waited for", c.puts, c.want)
}
return c.inner.Wait(ctx, a)
}
// 1921/S8: a group's asks are all put before any is waited for, so its members run side by side on the build nodes;
// the verdict is judged in member order as before.
func TestAGroupPutsEveryMembersAskBeforeTheFirstAnswer(t *testing.T) {
b := newBench(t)
counting := &countingAsks{inner: b.ex.Asks, want: 3, all: make(chan struct{})}
b.ex.Asks = counting
cs := caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2), pullOf("novox/mesh-controller", 3, head1))
v := b.judge(t, cs).Verdict
if v.State != delivery.VerdictPass {
t.Fatalf("verdict = %+v", v)
}
if counting.puts != 3 {
t.Errorf("%d asks were put, want 3", counting.puts)
}
}
// 1921/S10: the levels of an ask run side by side, so what of the ask's time is its fetch (Took.Clone) is the ask's time
// less its longest level, not less the sum of its levels.
func TestTheCloneTimeTakesTheLongestLevelOffNotTheSum(t *testing.T) {
b := newBench(t)
b.test = func(c stepCall) (any, error) {
r := captured[Tested](t, "go-test-slow")
r.Level, _ = c.args["level"].(string)
r.Elapsed, r.Setup, r.Lint = 1.5, 0, 0 // the ask's step takes 2 s (bench.answer)
return r, nil
}
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
if v.State != delivery.VerdictPass || v.Took.Clone != 500 {
t.Errorf("clone = %d ms, want 500 (2 s less the longest level's 1.5 s): %+v", v.Took.Clone, v)
}
}
// ADR 0542 §2: a lint finding, which the Go build module marks as the lint step's, is reported as the lint step's failure,
// not as the unit level's.
func TestALintFindingIsTheLintStepsFailure(t *testing.T) {
b := newBench(t)
b.test = func(c stepCall) (any, error) {
r := captured[Tested](t, "go-test-slow")
r.Level, _ = c.args["level"].(string)
if r.Level == delivery.LevelUnit {
r.Passed = false
r.Failed = []Failure{{Package: ".", Test: "gofmt", Step: "lint", Output: "not as gofmt writes them (run gofmt -w): a.go"}}
}
return r, nil
}
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
if v.State != delivery.VerdictFail || v.Failed.Test != "lint: ..gofmt" {
t.Errorf("failed = %+v, want the lint step's failure", v.Failed)
}
}
@@ -82,6 +82,9 @@ func CaseOf(c delivery.Candidate, trunk map[string]string) *Case {
} }
cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}} cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}}
for _, p := range c.Pulls { for _, p := range c.Pulls {
if p.Merged {
continue // merged while the group waits for its other members: judged by the verdict the group had (issue 1791)
}
switch head, known := trunk[p.Repository]; { switch head, known := trunk[p.Repository]; {
case !known || head == "": case !known || head == "":
cs.Left = append(cs.Left, fmt.Sprintf("%s#%d: the trunk head of %s is not known", p.Repository, p.Number, p.Repository)) cs.Left = append(cs.Left, fmt.Sprintf("%s#%d: the trunk head of %s is not known", p.Repository, p.Number, p.Repository))
@@ -119,7 +122,18 @@ func GateHeads(c delivery.Candidate) string {
} }
// sameTrunk says whether two verdicts' trunk heads are the same. // sameTrunk says whether two verdicts' trunk heads are the same.
func sameTrunk(a, b map[string]string) bool { return maps.Equal(a, b) } // sameTrunk says whether a verdict judged on the trunk heads judged stands for a case on the trunk heads now: every
// repository of the case at the head it was judged on. A repository the verdict names and the case does not is a member
// that merged since (issue 1791): its own merge moved that trunk, and the members left are judged by the verdict the
// group had.
func sameTrunk(judged, now map[string]string) bool {
for repo, head := range now {
if judged[repo] != head {
return false
}
}
return len(now) > 0 || len(judged) == 0
}
// Trunks holds the trunk head of each repository, as the gate's source.main says (a watch puts it in). // Trunks holds the trunk head of each repository, as the gate's source.main says (a watch puts it in).
type Trunks struct { type Trunks struct {
@@ -83,12 +83,24 @@ type Checker struct {
tried map[string]bool // by verdict key: looked at since this process started tried map[string]bool // by verdict key: looked at since this process started
epoch uint64 // changes of the nodes seen since this process started epoch uint64 // changes of the nodes seen since this process started
nodesFail map[string]uint64 // by verdict key: the epoch a failure of the nodes was written at nodesFail map[string]uint64 // by verdict key: the epoch a failure of the nodes was written at
// current is, by candidates key, the verdict key of the heads the group holds now; running holds, by candidates key,
// the stop of each check that runs, by its verdict key (issue 1791; issue 1921, try speed S9).
current map[string]string
running map[string]map[string]context.CancelCauseFunc
}
// superseded is why a check whose heads the group no longer holds was stopped: the heads it holds now.
type superseded struct{ by string }
func (s superseded) Error() string {
return "superseded: the group holds other heads now (" + s.by + "), whose check judges it"
} }
func (ch *Checker) init() { func (ch *Checker) init() {
ch.once.Do(func() { ch.once.Do(func() {
ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{} ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
ch.nodesFail = map[string]uint64{} ch.nodesFail = map[string]uint64{}
ch.current, ch.running = map[string]string{}, map[string]map[string]context.CancelCauseFunc{}
}) })
} }
@@ -126,6 +138,15 @@ func (ch *Checker) Candidate(ctx context.Context, key string, value []byte) {
} }
ch.mu.Lock() ch.mu.Lock()
ch.known[key] = c ch.known[key] = c
if vk, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash); err == nil {
ch.current[key] = vk
// A check of heads the group no longer holds is stopped: its asks are not put, and it ends superseded.
for running, stop := range ch.running[key] {
if running != vk {
stop(superseded{by: c.HeadsHash})
}
}
}
ch.mu.Unlock() ch.mu.Unlock()
ch.consider(ctx, key, c) ch.consider(ctx, key, c)
} }
@@ -329,12 +350,48 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) {
if cl == nil { if cl == nil {
return return
} }
run, stop := context.WithCancelCause(ctx)
defer stop(nil)
ch.mu.Lock()
now := ch.current[key]
if ch.running[key] == nil {
ch.running[key] = map[string]context.CancelCauseFunc{}
}
ch.running[key][cs.Key] = stop
ch.mu.Unlock()
defer func() {
ch.mu.Lock()
delete(ch.running[key], cs.Key)
if len(ch.running[key]) == 0 {
delete(ch.running, key)
}
ch.mu.Unlock()
}()
if now != "" && now != cs.Key {
stop(superseded{by: now}) // the group moved on before the check began
}
began := ch.now() began := ch.now()
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{}) if run.Err() == nil {
out := ch.judge(ctx, cs) ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
}
out := ch.judge(run, cs)
if ctx.Err() != nil { if ctx.Err() != nil {
return return
} }
var sup superseded
if errors.As(context.Cause(run), &sup) {
// Stopped for newer heads: the verdict ends failed, not running, and no status is put, as the newer check owns the
// status of the heads they share.
v := delivery.Verdict{Group: cs.Group, Heads: cs.Heads, Trunk: cs.Trunk, State: delivery.VerdictFail,
Failed: delivery.FailedFirst{Machine: "checker", Words: sup.Error()}}
if data, err := delivery.EncodeVerdict(v); err == nil {
if _, err := ch.Verdicts.Update(ctx, cs.Key, data, cl.rev); err != nil {
ch.logf("%s: the superseded verdict could not be written: %v", cs.Key, err)
}
}
ch.logf("%s: %s", cs.Key, sup.Error())
return
}
v := out.Verdict v := out.Verdict
v.Group, v.Heads, v.Trunk = cs.Group, cs.Heads, cs.Trunk v.Group, v.Heads, v.Trunk = cs.Group, cs.Heads, cs.Trunk
if v.Took.Queue == 0 && !cs.Queued.IsZero() && began.After(cs.Queued) { if v.Took.Queue == 0 && !cs.Queued.IsZero() && began.After(cs.Queued) {
@@ -0,0 +1,90 @@
package check
import (
"encoding/json"
"fmt"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// failingResult is a level's test result in which the named tests failed, each in its own package.
func failingResult(t *testing.T, tests int, failed ...Failure) json.RawMessage {
t.Helper()
raw, err := json.Marshal(Tested{Level: delivery.LevelIntegration, Tests: tests, Elapsed: 2, Failed: failed, Passed: len(failed) == 0})
if err != nil {
t.Fatal(err)
}
return raw
}
func judgeLevel(t *testing.T, tests ...json.RawMessage) (*delivery.Verdict, *state) {
t.Helper()
st := &state{j: &Examiner{}, c: &Case{}, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
v, _ := st.level(t.Context(), &target{pull: Pull{Repository: "novox/x"}}, delivery.LevelAnswer{Level: delivery.LevelIntegration, Tests: tests})
return v, st
}
// Issue 1921: two failing tests in two packages are two entries of the verdict's Failures and two lines of a try's
// failed list, each with its own first line (its cause, issue 1901), whether the build module answered them in one test
// result (one go test run over both packages) or in two (the changed packages first, then the rest); every result is
// counted, the ones after a failure too; and Failed keeps the first one's words whole.
func TestTwoFailingTestsInTwoPackagesAreTwoEntries(t *testing.T) {
a := Failure{Package: "x/a", Test: "TestA", Cause: "a_test.go:9: want 1, got 2", Output: " a_test.go:3: logged\n a_test.go:9: want 1, got 2\n--- FAIL: TestA (0.00s)"}
b := Failure{Package: "x/b", Test: "TestB", Cause: "b_test.go:4: boom", Output: " b_test.go:4: boom\n--- FAIL: TestB (0.00s)"}
for name, tests := range map[string][]json.RawMessage{
"one result": {failingResult(t, 10, a, b)},
"two results": {failingResult(t, 4, a), failingResult(t, 3), failingResult(t, 3, b)},
} {
v, st := judgeLevel(t, tests...)
if v == nil {
t.Fatalf("%s: the level passed", name)
}
want := []string{"integration: x/a.TestA: a_test.go:9: want 1, got 2", "integration: x/b.TestB: b_test.go:4: boom"}
if got := failedLines(*v); strings.Join(got, "\n") != strings.Join(want, "\n") {
t.Errorf("%s: failed lines\n%s\nwant\n%s", name, strings.Join(got, "\n"), strings.Join(want, "\n"))
}
if len(v.Failures) != 2 || v.Failed.Test != "integration: x/a.TestA" || !strings.Contains(v.Failed.Words, "a_test.go:3: logged") {
t.Errorf("%s: verdict %+v", name, v)
}
if run := st.levels[delivery.LevelIntegration].run; run != 10 {
t.Errorf("%s: %d tests counted, want all 10", name, run)
}
if _, err := delivery.EncodeVerdict(delivery.Verdict{Group: "g", Heads: "0123456789ab", State: v.State, Failed: v.Failed, Failures: v.Failures}); err != nil {
t.Errorf("%s: the verdict is not written: %v", name, err)
}
}
}
// Issue 1921: past delivery.MaxFailures the list names the first fifty and how many more, the status line says how many
// failed, and each entry's line is cut to MaxFailureLine.
func TestFailuresAreCappedWithHowManyMore(t *testing.T) {
var fs []Failure
for i := range 60 {
fs = append(fs, Failure{Package: fmt.Sprintf("x/p%d", i), Test: "TestT", Cause: strings.Repeat("y", 500)})
}
v, _ := judgeLevel(t, failingResult(t, 60, fs...))
if len(v.Failures) != delivery.MaxFailures+1 || v.Failures[delivery.MaxFailures].Test != "and 10 more" {
t.Fatalf("%d entries, last %+v", len(v.Failures), v.Failures[len(v.Failures)-1])
}
if len(v.Failures[0].Words) > MaxFailureLine {
t.Errorf("an entry's line is %d bytes", len(v.Failures[0].Words))
}
if n := failedCount(*v); n != 60 {
t.Errorf("failedCount = %d, want 60", n)
}
cs := &Case{Pulls: []Pull{{Repository: "novox/x", Head: "0123456789abcdef", Trunk: "fedcba9876543210"}}}
if line := Line(*v, cs, cs.Pulls[0], nil, nil, nil); !strings.HasPrefix(line, "failed (60 tests): integration: x/p0.TestT") {
t.Errorf("line %q", line)
}
}
// A verdict that names no list (a conflict, a refused step, a verdict written before issue 1921) is one failed line, as
// before.
func TestAVerdictWithoutFailuresIsOneLine(t *testing.T) {
v := delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "g14", Words: "files in conflict: go.mod\nmore"}}
if got := failedLines(v); len(got) != 1 || got[0] != "g14: files in conflict: go.mod" {
t.Fatalf("%q", got)
}
}
@@ -74,6 +74,18 @@ func (f *fakeSteps) tool(name string) []stepCall {
return out return out
} }
// toolAt is tool's calls of one level: a check's levels run side by side (issue 1921, try speed S10), so only the calls of
// one level keep an order.
func (f *fakeSteps) toolAt(name, level string) []stepCall {
var out []stepCall
for _, c := range f.tool(name) {
if c.args["level"] == level {
out = append(out, c)
}
}
return out
}
type entries []delivery.NodeToolchain type entries []delivery.NodeToolchain
func (e entries) Entries(language string) []delivery.NodeToolchain { func (e entries) Entries(language string) []delivery.NodeToolchain {
@@ -269,9 +281,9 @@ func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1
t.Fatal(err) t.Fatal(err)
} }
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
tests := b.steps.tool("test") tests := b.steps.toolAt("test", "unit")
if len(tests) != 2 || strings.Join(packagesOf(tests[0]), ",") != "fixture/slow" || tests[0].args["level"] != "unit" { if len(tests) != 1 || strings.Join(packagesOf(tests[0]), ",") != "fixture/slow" || len(b.steps.toolAt("test", "integration")) != 1 {
t.Fatalf("test calls = %+v, want the unit level of fixture/slow alone, then the integration level", tests) t.Fatalf("test calls = %+v, want the unit level of fixture/slow alone, and the integration level", b.steps.tool("test"))
} }
if u := out.Verdict.Levels[delivery.LevelUnit]; u.Run != 2 || u.Kept != 40 { if u := out.Verdict.Levels[delivery.LevelUnit]; u.Run != 2 || u.Kept != 40 {
t.Errorf("unit = %+v, want 2 run and 40 kept", u) t.Errorf("unit = %+v, want 2 run and 40 kept", u)
@@ -302,7 +314,7 @@ func TestAReAskRunsOnlyWhatFailedAndWhatChanged(t *testing.T) { // 1288/AC2, AC3
// Asked again with nothing changed: only the failed package runs; the good one is taken. // Asked again with nothing changed: only the failed package runs; the good one is taken.
b.steps.calls = nil b.steps.calls = nil
b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
tests := b.steps.tool("test") tests := b.steps.toolAt("test", "unit")
if len(tests) == 0 || strings.Join(packagesOf(tests[0]), ",") != "fixture/failing/bad" { if len(tests) == 0 || strings.Join(packagesOf(tests[0]), ",") != "fixture/failing/bad" {
t.Fatalf("the re-ask ran %+v, want only fixture/failing/bad", tests) t.Fatalf("the re-ask ran %+v, want only fixture/failing/bad", tests)
} }
@@ -310,7 +322,7 @@ func TestAReAskRunsOnlyWhatFailedAndWhatChanged(t *testing.T) { // 1288/AC2, AC3
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"fixture/failing/bad": digA, "fixture/failing/good": digD}} b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"fixture/failing/bad": digA, "fixture/failing/good": digD}}
b.steps.calls = nil b.steps.calls = nil
b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
tests = b.steps.tool("test") tests = b.steps.toolAt("test", "unit")
if got := strings.Join(packagesOf(tests[0]), ","); got != "fixture/failing/bad,fixture/failing/good" && got != "fixture/failing/good,fixture/failing/bad" { if got := strings.Join(packagesOf(tests[0]), ","); got != "fixture/failing/bad,fixture/failing/good" && got != "fixture/failing/good,fixture/failing/bad" {
t.Errorf("after a digest changed the run was %q, want the failed package and the changed one", got) t.Errorf("after a digest changed the run was %q, want the failed package and the changed one", got)
} }
@@ -341,7 +353,9 @@ func TestANodePackageIsNeverKeptAndRunsEveryCheck(t *testing.T) { // 1288/AC4, 1
} }
} }
func TestTheChangesOwnFailingTestRunsFirstAndNothingElseRuns(t *testing.T) { // 1289/AC1, AC2 // 1289/AC1, AC2 as issue 1921 changed it: the change's own failing test runs first, the level's other packages still run
// after it (so one check names every failure); the later level ran beside it (S10) and is not judged.
func TestTheChangesOwnFailingTestRunsFirstThenTheRestAndNoLaterLevel(t *testing.T) {
b := newBench(t) b := newBench(t)
b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"} b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"}
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}} b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}}
@@ -356,12 +370,12 @@ func TestTheChangesOwnFailingTestRunsFirstAndNothingElseRuns(t *testing.T) { //
if out.Verdict.State != delivery.VerdictFail || !strings.Contains(out.Verdict.Failed.Test, "TestThreeIsFour") { if out.Verdict.State != delivery.VerdictFail || !strings.Contains(out.Verdict.Failed.Test, "TestThreeIsFour") {
t.Fatalf("verdict = %+v", out.Verdict) t.Fatalf("verdict = %+v", out.Verdict)
} }
tests := b.steps.tool("test") tests := b.steps.toolAt("test", "unit")
if len(tests) != 1 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" { if len(tests) != 2 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" || strings.Join(packagesOf(tests[1]), ",") != "m/internal/a" {
t.Fatalf("test calls = %+v, want only m/internal/b, once", tests) t.Fatalf("test calls = %+v, want m/internal/b, then m/internal/a, each once", tests)
} }
if n := len(b.steps.tool("digest")); n != 1 { if out.Verdict.Levels[delivery.LevelIntegration].Run != 0 {
t.Errorf("%d digest calls, want the unit level's alone (the check stopped)", n) t.Errorf("the integration level was judged after the unit level failed: %+v", out.Verdict.Levels)
} }
if out.Verdict.Composition == nil || out.Verdict.Composition.State != delivery.CompositionNotRun { if out.Verdict.Composition == nil || out.Verdict.Composition.State != delivery.CompositionNotRun {
t.Errorf("composition = %+v", out.Verdict.Composition) t.Errorf("composition = %+v", out.Verdict.Composition)
@@ -159,6 +159,9 @@ type target struct {
runner string runner string
// judged is what the line says of the program that judged a change to the build module; empty for any other change. // judged is what the line says of the program that judged a change to the build module; empty for any other change.
judged string judged string
// ask is the check ask put for it, and failed why it could not be asked (issue 1921, try speed S8).
ask delivery.StepAsk
failed *delivery.Verdict
} }
// judgedBy takes what a step under runner workspace answered of the program that ran it (judged_by): the candidate, or the // judgedBy takes what a step under runner workspace answered of the program that ran it (judged_by): the candidate, or the
@@ -307,12 +310,13 @@ func (st *state) levelsOf() []string {
return []string{delivery.LevelUnit, delivery.LevelIntegration} return []string{delivery.LevelUnit, delivery.LevelIntegration}
} }
// judgeAll puts one check ask per pull request and build module the repository needs, the mesh-sdk member first, and // judgeAll puts one check ask per pull request and build module the repository needs, every ask of the group at once so
// judges each answer. A head that does not merge, a pin that is not allowed or a level that fails ends the check at once // the members run side by side on the build nodes (issue 1921, try speed S8), and judges the answers in member order, the
// (ADR 0484 clause 13). // mesh-sdk member first. A head that does not merge, a pin that is not allowed or a level that fails ends the check at once
// (ADR 0484 clause 13): the verdict is the first failure in member order, as when the asks were put one after another.
func (st *state) judgeAll(ctx context.Context) *delivery.Verdict { func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
// The mesh-sdk member first: whether its merge changes the TypeScript package's paths decides whether the other members' // The mesh-sdk member first: whether its merge changes the TypeScript package's paths decides whether the other members'
// Node fetches take the group's package (ADR 0500 §5). // Node fetches take the group's package (ADR 0500 §5). Those Node asks alone wait for the mesh-sdk member's answer.
pulls := slices.Clone(st.c.Pulls) pulls := slices.Clone(st.c.Pulls)
slices.SortStableFunc(pulls, func(a, b Pull) int { slices.SortStableFunc(pulls, func(a, b Pull) int {
if (a.Repository == SDKRepository) != (b.Repository == SDKRepository) { if (a.Repository == SDKRepository) != (b.Repository == SDKRepository) {
@@ -323,34 +327,32 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
} }
return 0 return 0
}) })
var groupPackage map[string]any sdkGroup := len(st.c.Pulls) > 1 && len(pulls) > 0 && pulls[0].Repository == SDKRepository
for _, p := range pulls { members := make([][]*target, len(pulls))
var merged []Fetched var waiting []*target // Node asks of the other members, put once the mesh-sdk member is judged
for i, p := range pulls {
for _, bm := range st.j.Modules.BuildModules(p.Repository) { for _, bm := range st.j.Modules.BuildModules(p.Repository) {
t := &target{pull: p, module: bm} t := st.target(ctx, p, bm)
if tc, none := delivery.AgreedToolchain(st.j.Toolchains.Entries(buildrun.LanguageOf[bm])); none == "" { members[i] = append(members[i], t)
t.toolchain = tc.Version
}
args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk}
if bm == buildrun.NodeBuildModule && groupPackage != nil && p.Repository != SDKRepository {
args["package"] = groupPackage
}
if bm == buildrun.GoBuildModule {
beside, err := st.j.besideFor(ctx, st.c, p)
if err != nil {
return fail(forgeMachine(err, bm), err.Error()+unheardWords(err))
}
if len(beside) > 0 {
args["beside"] = beside
t.beside = besideLines(beside)
}
if p.Repository == catalogueRepository {
t.runner = RunnerWorkspace // used by the build module when the merge changes it (ADR 0535 §1)
}
}
t.fetchArgs = args
st.targets = append(st.targets, t) st.targets = append(st.targets, t)
if v := st.ask(ctx, t, &merged); v != nil { if t.failed != nil {
continue // said when the judging reaches it, so the verdict stays the first failure in member order
}
if sdkGroup && bm == buildrun.NodeBuildModule && p.Repository != SDKRepository {
waiting = append(waiting, t)
continue
}
t.failed = st.put(ctx, t)
}
}
var groupPackage map[string]any
for i, p := range pulls {
var merged []Fetched
for _, t := range members[i] {
if t.failed != nil {
return t.failed
}
if v := st.judgeAsk(ctx, t, &merged); v != nil {
return v return v
} }
} }
@@ -360,14 +362,51 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
if p.Repository == SDKRepository && len(st.c.Pulls) > 1 && st.j.changesPackage(p, merged) { if p.Repository == SDKRepository && len(st.c.Pulls) > 1 && st.j.changesPackage(p, merged) {
groupPackage = map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number)} groupPackage = map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number)}
} }
if i == 0 && sdkGroup {
for _, t := range waiting {
if groupPackage != nil {
t.fetchArgs["package"] = groupPackage
}
t.failed = st.put(ctx, t)
}
}
} }
return nil return nil
} }
// ask puts the target's check ask, waits for the step of the node that pulled it and judges its answer: the fetch, the // target is the check of one pull request by one build module: its fetch's arguments, its toolchain, the siblings its
// pins of its go.mod files, then each level's digest, kept verdicts and tests. // tests read and the runner it is asked with; failed says why it cannot be asked.
func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *delivery.Verdict { func (st *state) target(ctx context.Context, p Pull, bm string) *target {
t := &target{pull: p, module: bm}
if tc, none := delivery.AgreedToolchain(st.j.Toolchains.Entries(buildrun.LanguageOf[bm])); none == "" {
t.toolchain = tc.Version
}
args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk}
if bm == buildrun.GoBuildModule {
beside, err := st.j.besideFor(ctx, st.c, p)
if err != nil {
t.failed = fail(forgeMachine(err, bm), err.Error()+unheardWords(err))
return t
}
if len(beside) > 0 {
args["beside"] = beside
t.beside = besideLines(beside)
}
if p.Repository == catalogueRepository {
t.runner = RunnerWorkspace // used by the build module when the merge changes it (ADR 0535 §1)
}
}
t.fetchArgs = args
return t
}
// put puts the target's check ask on its build seat's stream; a check that was stopped (its group holds other heads now,
// issue 1791) puts none.
func (st *state) put(ctx context.Context, t *target) *delivery.Verdict {
p := t.pull p := t.pull
if err := context.Cause(ctx); err != nil {
return fail(t.module, fmt.Sprintf("the check of %s#%d was not asked: %v", p.Repository, p.Number, err))
}
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: st.levelsOf(), Kept: true} job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: st.levelsOf(), Kept: true}
if t.runner != "" { if t.runner != "" {
job.Runner, job.RunnerWhen = t.runner, buildGoDir job.Runner, job.RunnerWhen = t.runner, buildGoDir
@@ -379,13 +418,20 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
if st.c.Kind == KindTrunk { if st.c.Kind == KindTrunk {
kind = delivery.AskRecheck // a re-judge after a trunk move goes after every new head's check (ADR 0536 §4) kind = delivery.AskRecheck // a re-judge after a trunk move goes after every new head's check (ADR 0536 §4)
} }
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: askLanguage[t.module], Asker: delivery.CheckerModule, t.ask = delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: askLanguage[t.module], Asker: delivery.CheckerModule,
At: st.j.now().UTC(), Check: job} At: st.j.now().UTC(), Check: job}
st.c.Progress.Set(fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number), "") st.c.Progress.Set(fmt.Sprintf("the %s ask %s of %s#%d", kind, t.ask.ID, p.Repository, p.Number), "")
if err := st.j.Asks.Put(ctx, a); err != nil { if err := st.j.Asks.Put(ctx, t.ask); err != nil {
return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err)) return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
} }
step, err := st.j.Asks.Wait(ctx, a) return nil
}
// judgeAsk waits for the step of the node that pulled the target's ask and judges its answer: the fetch, the pins of its
// go.mod files, then each level's digest, kept verdicts and tests.
func (st *state) judgeAsk(ctx context.Context, t *target, merged *[]Fetched) *delivery.Verdict {
p := t.pull
step, err := st.j.Asks.Wait(ctx, t.ask)
if err != nil { if err != nil {
if ctx.Err() != nil { if ctx.Err() != nil {
return fail(t.module, ctx.Err().Error()) return fail(t.module, ctx.Err().Error())
@@ -427,7 +473,7 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
ran := 0.0 ran := 0.0
for _, la := range answer.Levels { for _, la := range answer.Levels {
v, elapsed := st.level(ctx, t, la) v, elapsed := st.level(ctx, t, la)
ran += elapsed ran = max(ran, elapsed) // the levels run side by side (issue 1921, try speed S10): the longest is what they took
if v != nil { if v != nil {
st.clone += max(0, took-time.Duration(ran*float64(time.Second))) st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
return v return v
@@ -438,8 +484,9 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
} }
// level judges one level of a check ask's answer as the tools' answers were judged: the digest keys the kept verdicts // level judges one level of a check ask's answer as the tools' answers were judged: the digest keys the kept verdicts
// of the packages that passed, the kept ones taken are counted, and the first test result that did not pass is the // of the packages that passed, the kept ones taken are counted, and every test result is read and counted, also after
// verdict. It answers the level's running time (its tests' and setup's). // one that did not pass, so the verdict names every failed test of the level (issue 1921). It answers the level's
// running time (its tests' and setup's).
func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer) (*delivery.Verdict, float64) { func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer) (*delivery.Verdict, float64) {
level := la.Level level := la.Level
a := st.levels[level] a := st.levels[level]
@@ -461,6 +508,7 @@ func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer)
a.kept += k.Tests a.kept += k.Tests
} }
elapsed := 0.0 elapsed := 0.0
var failedResults []Tested
for _, raw := range la.Tests { for _, raw := range la.Tests {
var res Tested var res Tested
if err := json.Unmarshal(raw, &res); err != nil { if err := json.Unmarshal(raw, &res); err != nil {
@@ -489,9 +537,12 @@ func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer)
} }
} }
if !res.Passed { if !res.Passed {
return levelFailure(t, level, res), elapsed failedResults = append(failedResults, res)
} }
} }
if len(failedResults) > 0 {
return levelFailure(t, level, failedResults...), elapsed
}
return nil, elapsed return nil, elapsed
} }
@@ -558,23 +609,64 @@ func firstLine(s string) string {
return "" return ""
} }
// failure turns a level that did not pass into a verdict naming what failed first. // MaxFailureLine is the most bytes of a failure's line in a verdict's Failures: the first line of its words, which is
func levelFailure(t *target, level string, res Tested) *delivery.Verdict { // its cause where the build module told one apart (issue 1901). Failed keeps the first failure's words whole.
const MaxFailureLine = 300
// levelFailure turns a level that did not pass into a verdict: Failed names what failed first with its words whole,
// and Failures every failed test of the level's results with its first line, at most delivery.MaxFailures and an entry
// saying how many more (issue 1921).
func levelFailure(t *target, level string, results ...Tested) *delivery.Verdict {
v := delivery.Verdict{State: delivery.VerdictFail} v := delivery.Verdict{State: delivery.VerdictFail}
switch { var all []delivery.FailedFirst
case len(res.Failed) > 0: for _, res := range results {
f := res.Failed[0] if len(res.Failed) == 0 {
name := failedName(f) why := res.Why
if t.module == buildrun.NodeBuildModule && f.Package == "" { if why == "" {
name = res.Module why = "the " + level + " level did not pass"
}
all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: why})
continue
} }
v.Failed = delivery.FailedFirst{Test: level + ": " + name, Words: failureWords(f)} for _, f := range res.Failed {
default: name := failedName(f)
why := res.Why if t.module == buildrun.NodeBuildModule && f.Package == "" {
if why == "" { name = res.Module
why = "the " + level + " level did not pass" }
step := level
if f.Step != "" {
step = f.Step // the lint step's finding, not the level's (ADR 0542 §2)
}
all = append(all, delivery.FailedFirst{Test: step + ": " + name, Words: failureWords(f)})
} }
v.Failed = delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: why}
} }
if len(all) == 0 { // no result: a level said failed with nothing to name
all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: "the " + level + " level did not pass"})
}
v.Failed = all[0]
v.Failures = failureLines(all)
return &v return &v
} }
// failureLines is every failure with the first line of its words, cut to MaxFailureLine, at most delivery.MaxFailures
// and then one entry saying how many more failed.
func failureLines(all []delivery.FailedFirst) []delivery.FailedFirst {
n := min(len(all), delivery.MaxFailures)
out := make([]delivery.FailedFirst, 0, n+1)
for _, f := range all[:n] {
out = append(out, delivery.FailedFirst{Test: f.Test, Machine: f.Machine, Words: clipLine(firstLine(f.Words), MaxFailureLine)})
}
if more := len(all) - n; more > 0 {
out = append(out, delivery.FailedFirst{Test: fmt.Sprintf("and %d more", more)})
}
return out
}
// clipLine cuts a line to at most n bytes on a whole character, marking the cut.
func clipLine(s string, n int) string {
if len(s) <= n {
return s
}
s = strings.ToValidUTF8(s[:n-3], "")
return s + "..."
}
@@ -177,8 +177,8 @@ func TestACandidateThatDoesNotCompileFailsTheCheckWithTheCompilersWords(t *testi
if out.Verdict.State != delivery.VerdictFail || !strings.Contains(out.Verdict.Failed.Words, "undefined: x") { if out.Verdict.State != delivery.VerdictFail || !strings.Contains(out.Verdict.Failed.Words, "undefined: x") {
t.Fatalf("verdict = %+v, want a failure naming the compile error", out.Verdict) t.Fatalf("verdict = %+v, want a failure naming the compile error", out.Verdict)
} }
if n := len(b.steps.tool("test")); n != 1 { if n := len(b.steps.toolAt("test", "unit")); n != 1 {
t.Errorf("%d test steps, want the first one's failure to end the check", n) t.Errorf("%d unit test steps, want the first one's failure to end the level", n)
} }
} }
@@ -49,6 +49,34 @@ func firstFailed(v delivery.Verdict) string {
return firstWordsLine(f.Words) return firstWordsLine(f.Words)
} }
// failedLines is everything a verdict names as failed, one line each: every failed test with its first line (issue
// 1921), or what failed first when the verdict names no list (a conflict, a refused step, an older verdict).
func failedLines(v delivery.Verdict) []string {
if len(v.Failures) == 0 {
if f := firstFailed(v); f != "" {
return []string{f}
}
return nil
}
lines := make([]string, 0, len(v.Failures))
for _, f := range v.Failures {
lines = append(lines, firstFailed(delivery.Verdict{Failed: f}))
}
return lines
}
// failedCount is how many tests a verdict names as failed, the ones after delivery.MaxFailures counted too.
func failedCount(v delivery.Verdict) int {
n := len(v.Failures)
if n > delivery.MaxFailures {
var more int
if _, err := fmt.Sscanf(v.Failures[n-1].Test, "and %d more", &more); err == nil {
return delivery.MaxFailures + more
}
}
return n
}
// firstWordsLine is the first line of a failure's words: the words are several lines (issue 1901) and the status line one. // firstWordsLine is the first line of a failure's words: the words are several lines (issue 1901) and the status line one.
func firstWordsLine(w string) string { return firstLine(w) } func firstWordsLine(w string) string { return firstLine(w) }
@@ -155,6 +183,9 @@ func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []s
pc.head = "pass " + commits pc.head = "pass " + commits
default: default:
pc.head = "failed: " + firstFailed(v) pc.head = "failed: " + firstFailed(v)
if n := failedCount(v); n > 1 {
pc.head = fmt.Sprintf("failed (%d tests): %s", n, firstFailed(v))
}
pc.commits = commits pc.commits = commits
} }
var counts []string var counts []string
@@ -118,8 +118,10 @@ func (st *state) checkPinsOf(ctx context.Context, t *target) verdictPtr {
return nil return nil
} }
// sameBranchPin names the branch of the tried repository that points at the tried head and that mesh-sdk also has, at the // sameBranchPin names the mesh-sdk branch whose head the pin is, for a try: the branch of the tried repository that points
// commit the pin names; empty when there is none. // at the tried head and that mesh-sdk also has, at the commit the pin names, first; else any branch of mesh-sdk but its
// trunk whose head the pin is, for a try asked by a bare commit that no branch of the tried repository points at (issue
// 1624; try speed S12). Empty when there is none.
func (st *state) sameBranchPin(ctx context.Context, t *target, pin string) (string, error) { func (st *state) sameBranchPin(ctx context.Context, t *target, pin string) (string, error) {
mine, err := st.j.Forge.BranchHeads(ctx, t.pull.Repository) mine, err := st.j.Forge.BranchHeads(ctx, t.pull.Repository)
if err != nil { if err != nil {
@@ -131,9 +133,6 @@ func (st *state) sameBranchPin(ctx context.Context, t *target, pin string) (stri
same = append(same, name) same = append(same, name)
} }
} }
if len(same) == 0 {
return "", nil
}
sdk, err := st.j.Forge.BranchHeads(ctx, SDKRepository) sdk, err := st.j.Forge.BranchHeads(ctx, SDKRepository)
if err != nil { if err != nil {
return "", err return "", err
@@ -144,6 +143,16 @@ func (st *state) sameBranchPin(ctx context.Context, t *target, pin string) (stri
return name, nil return name, nil
} }
} }
var any []string
for name, h := range sdk {
if name != "main" && strings.HasPrefix(h, pin) {
any = append(any, name)
}
}
sort.Strings(any)
if len(any) > 0 {
return any[0], nil
}
return "", nil return "", nil
} }
@@ -408,3 +408,22 @@ func TestAMergeCheckWithTheSameBranchPinIsStillRefusedWithoutAGroupPullRequest(t
t.Fatalf("verdict = %+v", v) t.Fatalf("verdict = %+v", v)
} }
} }
// Issue 1624 (met again on 2026-10-25; try speed S12): a try asked by a bare commit, which no branch of the tried
// repository points at, still takes a pin that is the head of a mesh-sdk branch, naming that branch; a pin that is the head
// of no mesh-sdk branch is still refused.
func TestATryByCommitTakesAPinThatIsTheHeadOfAnSDKBranch(t *testing.T) {
b, c := sameBranchBench(t, head2[:12])
b.ex.Forge.(*fakeForge).branches[repoA] = map[string]string{"main": trunkA, "feat/r6": trunkA2} // the branch moved on
if v := b.judge(t, c).Verdict; v.State != delivery.VerdictPass {
t.Fatalf("verdict = %+v", v)
}
if n := c.Notes(); len(n) != 1 || n[0] != "pin accepted: head of mesh-sdk feat/r6" {
t.Errorf("notes = %q", n)
}
b, c = sameBranchBench(t, "9aaa5c80ac5c")
b.ex.Forge.(*fakeForge).branches[repoA] = map[string]string{"main": trunkA}
if v := b.judge(t, c).Verdict; v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "neither on mesh-sdk's trunk") {
t.Errorf("a pin no mesh-sdk branch holds: verdict = %+v", v)
}
}
@@ -61,6 +61,9 @@ type Failure struct {
Output string `json:"output,omitempty"` Output string `json:"output,omitempty"`
// Cause is the first line of what the test reported as its error, when the build module told it apart (issue 1901). // Cause is the first line of what the test reported as its error, when the build module told it apart (issue 1901).
Cause string `json:"cause,omitempty"` Cause string `json:"cause,omitempty"`
// Step is "lint" for a finding of the lint step (gofmt, go vet), which runs beside the unit level and is reported as
// its own failure, not the level's (ADR 0542 §2).
Step string `json:"step,omitempty"`
} }
// TestTime is one test and the seconds it took. // TestTime is one test and the seconds it took.
@@ -0,0 +1,102 @@
package check
import (
"context"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
const repoC = "novox/mesh-controller"
func statusOf(t *testing.T, r *rig, repo string, n int, head string) string {
t.Helper()
key, _ := delivery.CheckerOwedKey(repo, n, head)
raw, _, found, _ := r.owed.Get(context.Background(), key)
if !found {
return ""
}
o, err := delivery.DecodeCheckerOwed(raw)
if err != nil {
t.Fatal(err)
}
return o.State
}
// 1791 note 9 (issue 1921, try speed S9): two of a group's three members merge. The candidate keeps them, marked merged,
// with the same heads; their trunks moved by their own merges. No new check is asked, and the member left keeps its
// passed status, so the merge of the rest is not refused.
func TestTwoMembersMergingAskNoNewCheckAndTheThirdKeepsItsPass(t *testing.T) {
ctx, j := context.Background(), passJudge()
r := newRig(t, j)
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, repoC: trunkA})
a, s, c := pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2), pullOf(repoC, 3, head1)
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", a, s, c))
r.ch.Wait()
if j.count() != 1 || statusOf(t, r, repoC, 3, head1) != "success" {
t.Fatalf("judged %d times, the third's status %q", j.count(), statusOf(t, r, repoC, 3, head1))
}
a.Merged, s.Merged = true, true
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", a, s, c))
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS2, repoC: trunkA})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
if j.count() != 1 {
t.Errorf("a new check was asked after two members merged: judged %d times", j.count())
}
if got := statusOf(t, r, repoC, 3, head1); got != "success" {
t.Errorf("the third member's status is %q, want success", got)
}
// The trunk of the member left moves: that is a re-judge as before.
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS2, repoC: trunkA2})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
if j.count() != 2 {
t.Errorf("the member left's trunk moved and it was judged %d times, want again", j.count())
}
}
// 1791 note 8 (issue 1921, try speed S9): a group opened member by member writes a candidate for each partial set. When
// the group's candidate moves to new heads, the check of the older heads is stopped and ends failed, superseded, instead
// of running on or staying running; it puts no status, as the newer check owns the heads they share.
func TestACheckWhoseHeadsTheGroupNoLongerHoldsIsStoppedAndEndsSuperseded(t *testing.T) {
ctx := context.Background()
started := make(chan struct{})
j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) {
if len(c.Pulls) == 1 {
close(started)
select {
case <-ctx.Done():
return Outcome{Verdict: failure("checker", ctx.Err().Error())}, nil
case <-time.After(3 * time.Second):
return Outcome{Verdict: passing()}, nil // never stopped
}
}
return Outcome{Verdict: passing()}, nil
}}
r := newRig(t, j)
partial := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
whole := candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))
r.ch.Candidate(ctx, "feat%2Fx", partial)
<-started
r.ch.Candidate(ctx, "feat%2Fx", whole)
done := make(chan struct{})
go func() { r.ch.Wait(); close(done) }()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("the check of the older heads was not stopped")
}
old := r.verdict(t, keyOf(t, "feat/x", partial))
if old.State != delivery.VerdictFail || !strings.Contains(old.Failed.Words, "superseded") {
t.Errorf("the older heads' verdict = %+v, want failed, superseded", old)
}
if v := r.verdict(t, keyOf(t, "feat/x", whole)); v.State != delivery.VerdictPass {
t.Errorf("the newer heads' verdict = %+v", v)
}
if got := statusOf(t, r, repoA, 7, head1); got != "success" {
t.Errorf("the shared head's status is %q, want the newer check's success", got)
}
}
@@ -67,7 +67,9 @@ type Try struct {
// Said is the try's report in words: what failed first, the tests run, kept and skipped per level, the time of each part. // Said is the try's report in words: what failed first, the tests run, kept and skipped per level, the time of each part.
Said string `json:"said,omitempty"` Said string `json:"said,omitempty"`
// Took is the time from its start to its end, as Go writes a duration. // Took is the time from its start to its end, as Go writes a duration.
Took string `json:"took,omitempty"` Took string `json:"took,omitempty"`
// Failed is every test that failed, each with its first line, at most delivery.MaxFailures and then how many more
// (issue 1921); or what else failed first (a conflict, a refused step).
Failed []string `json:"failed,omitempty"` Failed []string `json:"failed,omitempty"`
Levels map[string]delivery.LevelResult `json:"levels,omitempty"` Levels map[string]delivery.LevelResult `json:"levels,omitempty"`
Machines []string `json:"machines,omitempty"` Machines []string `json:"machines,omitempty"`
@@ -339,9 +341,7 @@ func (tr *Tryer) run(ctx context.Context, key string) {
t.State = TryPass t.State = TryPass
default: default:
t.State = TryFail t.State = TryFail
if f := firstFailed(v); f != "" { t.Failed = failedLines(v)
t.Failed = []string{f}
}
} }
if t.Tests != "" { if t.Tests != "" {
t.Said += " | only the tests " + t.Tests + " (Go); nothing kept" t.Said += " | only the tests " + t.Tests + " (Go); nothing kept"
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
go 1.26.0 go 1.26.0
require ( require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95
github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0 github.com/nats-io/nats.go v1.54.0
) )
+6
View File
@@ -1,5 +1,11 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024231716-571ac521f053 h1:8Kbp3aJG3jGndDOfmGdMSDgJVEcc1hyu921hVnnlnlQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024231716-571ac521f053/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232250-9b408db24a58 h1:4Damtdqlf6u6nJhvMA99peDy1jmjixtcwFXJ4bWEkLk=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232250-9b408db24a58/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95 h1:I8zbgGzRjmgYydPTWo8hmaZW/P/g+qGDUfu3E32/pkU=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -71,7 +71,7 @@ func (ix *Indexer) logf(format string, args ...any) {
} }
// Apply takes one message of the bridge's stream FORGE into the group of its branch. A pull message adds or moves the // Apply takes one message of the bridge's stream FORGE into the group of its branch. A pull message adds or moves the
// pull request, a closed one or a merged message takes it out; a group left with none is deleted. A change is written // pull request, a closed one takes it out, a merged one marks it merged; a group left with none open is deleted. A change is written
// by compare-and-set on the revision read, and read again on a conflict. A lone pull request is a group of one. A // by compare-and-set on the revision read, and read again on a conflict. A lone pull request is a group of one. A
// merged message also writes the fingerprint of each module of its repository (modules.go). // merged message also writes the fingerprint of each module of its repository (modules.go).
func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error { func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
@@ -154,12 +154,13 @@ func (ix *Indexer) applyOpen(ctx context.Context, f delivery.Forge) error {
return nil, false, fmt.Errorf("candidates.%s holds what is not a candidate: %w", key, err) return nil, false, fmt.Errorf("candidates.%s holds what is not a candidate: %w", key, err)
} }
kept := slices.DeleteFunc(slices.Clone(c.Pulls), func(p delivery.CandidatePull) bool { kept := slices.DeleteFunc(slices.Clone(c.Pulls), func(p delivery.CandidatePull) bool {
return !named[delivery.OpenPull{Repository: p.Repository, Number: p.Number}] // a member that merged is open no more, and stays while another member is (issue 1791)
return !p.Merged && !named[delivery.OpenPull{Repository: p.Repository, Number: p.Number}]
}) })
switch { switch {
case len(kept) == len(c.Pulls): case len(kept) == len(c.Pulls):
return nil, false, nil return nil, false, nil
case len(kept) == 0: case !anyOpen(kept):
ix.logf("%s: no pull request of the group is open, so no candidate", key) ix.logf("%s: no pull request of the group is open, so no candidate", key)
return nil, true, nil return nil, true, nil
} }
@@ -232,7 +233,16 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[s
if at < 0 { if at < 0 {
return cur return cur
} }
pulls = slices.Delete(pulls, at, at+1) if f.Kind == delivery.ForgeMerged {
// A member that merges stays, marked merged, with the head it merged at: the heads hash the group's Approve
// covers holds, so the checker starts no new check for the members still open (issue 1791; issue 1921 S9).
pulls[at].Merged = true
} else {
pulls = slices.Delete(pulls, at, at+1)
}
if !anyOpen(pulls) {
return nil // every member merged or closed: the group is done
}
} else { } else {
entry := delivery.CandidatePull{Head: delivery.Head{Repository: f.Repository, Number: f.Number, Head: f.Head}, Modules: []string{}} entry := delivery.CandidatePull{Head: delivery.Head{Repository: f.Repository, Number: f.Number, Head: f.Head}, Modules: []string{}}
switch { switch {
@@ -276,6 +286,11 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[s
return &delivery.Candidate{V: delivery.Version, Group: f.Branch, Base: base, HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)} return &delivery.Candidate{V: delivery.Version, Group: f.Branch, Base: base, HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)}
} }
// anyOpen says whether a group's pull requests hold one that has not merged.
func anyOpen(pulls []delivery.CandidatePull) bool {
return slices.ContainsFunc(pulls, func(p delivery.CandidatePull) bool { return !p.Merged })
}
// NoneNoPreview is why a candidate's touched module has no fingerprint of the new form: the merge preview is not // NoneNoPreview is why a candidate's touched module has no fingerprint of the new form: the merge preview is not
// computed in this release (novox/hq ADR 0465 clause 3; the checker comes with issue 1445). // computed in this release (novox/hq ADR 0465 clause 3; the checker comes with issue 1445).
const NoneNoPreview = "no merge preview is computed" const NoneNoPreview = "no merge preview is computed"
@@ -206,8 +206,8 @@ func TestDirsNotKnownKeepTheModulesOfTheSameHead(t *testing.T) { // 1238/AC4
} }
} }
func TestClosedAndMergedTakeAPullOutAndTheLastDeletesTheKey(t *testing.T) { // 1238/AC5 func TestClosedTakesAPullOutAndTheLastDeletesTheKey(t *testing.T) { // 1238/AC5 (a merge: TestAMergedMemberStays...)
for _, how := range []string{"closed", "merged"} { for _, how := range []string{"closed"} {
ix, st, _ := newIndexer(t) ix, st, _ := newIndexer(t)
ctx := context.Background() ctx := context.Background()
_ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a")) _ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
@@ -387,3 +387,70 @@ func TestAnOpenMessageDropsEveryCandidateItDoesNotName(t *testing.T) { // 1279/A
t.Errorf("after an empty open message the candidates hold %v", st.vals) t.Errorf("after an empty open message the candidates hold %v", st.vals)
} }
} }
// 1791 (issue 1921, try speed S9): a member of a group that merges while another is open stays in the candidate, marked
// merged, with its head: the heads hash the Approve covers holds, so no new check starts for the members left. An open
// message does not drop it, a close of the last open member deletes the group, and the last merge does too.
func TestAMergedMemberStaysInItsGroupUntilEveryMemberMerged(t *testing.T) {
merged := func(repo string, n int, head string) delivery.Forge {
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repo, Branch: "feat/x", Number: n, Head: head, Merge: h3, Root: "r", Trees: map[string]string{"a": "t"}}
}
ix, st, _ := newIndexer(t)
ctx := context.Background()
for _, f := range []delivery.Forge{pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"), pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c"),
pull("novox/mesh-sdk", 7, "feat/x", h3)} {
if err := ix.Apply(ctx, f); err != nil {
t.Fatal(err)
}
}
before, _ := read(t, st, "feat/x")
if err := ix.Apply(ctx, merged("novox/mesh-sdk", 7, h3)); err != nil {
t.Fatal(err)
}
if err := ix.Apply(ctx, merged("novox/mesh-delivery", 12, h1)); err != nil {
t.Fatal(err)
}
c, ok := read(t, st, "feat/x")
if !ok || len(c.Pulls) != 3 || c.HeadsHash != before.HeadsHash {
t.Fatalf("after two merges the candidate = %+v (kept %v), want the three members and the same heads hash", c, ok)
}
for _, p := range c.Pulls {
if want := p.Repository != "novox/mesh-catalog"; p.Merged != want {
t.Errorf("%s#%d merged = %v, want %v", p.Repository, p.Number, p.Merged, want)
}
}
// The open message names only the member still open: the merged ones stay.
if err := ix.Apply(ctx, open(delivery.OpenPull{Repository: "novox/mesh-catalog", Number: 300})); err != nil {
t.Fatal(err)
}
if c, _ := read(t, st, "feat/x"); len(c.Pulls) != 3 || c.HeadsHash != before.HeadsHash {
t.Errorf("the open message dropped a merged member: %+v", c)
}
// The last open member merges: the group is done.
if err := ix.Apply(ctx, merged("novox/mesh-catalog", 300, h2)); err != nil {
t.Fatal(err)
}
if _, ok := read(t, st, "feat/x"); ok {
t.Error("every member merged and the group is still a candidate")
}
// A close of the last open member deletes the group too.
ix, st, _ = newIndexer(t)
_ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
_ = ix.Apply(ctx, pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c"))
_ = ix.Apply(ctx, merged("novox/mesh-delivery", 12, h1))
closed := pull("novox/mesh-catalog", 300, "feat/x", h2)
closed.Action = delivery.PullClosed
if err := ix.Apply(ctx, closed); err != nil {
t.Fatal(err)
}
if _, ok := read(t, st, "feat/x"); ok {
t.Error("the last open member closed and the group, holding only a merged one, is still a candidate")
}
// A lone pull request that merges leaves no group.
ix, st, _ = newIndexer(t)
_ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
_ = ix.Apply(ctx, merged("novox/mesh-delivery", 12, h1))
if _, ok := read(t, st, "feat/x"); ok {
t.Error("a lone merged pull request is still a candidate")
}
}
@@ -1,213 +0,0 @@
package main
import (
"encoding/json"
"fmt"
"slices"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The assigner's asks reach the operator (issue 1906). The assigner writes its asks (the hold of a fingerprint, the restore
// of a data module, the backup before an apply and a move of a module of policy record) in its bucket asks; until now the
// teller only filled them from an answer given in the view, so an ask nobody opened the view for reached nobody: on
// 2026-10-25 seven modules stayed held on every machine and the operator was never asked. Now the teller raises each on the
// operator channel as it raises the gate's merge asks: as the asker mesh-delivery-teller, once per router id, through the same
// records, the same wait for room and the same warrants. The operator's answer, from the phone or from the view, is written
// under the ask's own key in the bucket answers, which the assigner reads (mesh-delivery-assigner pass.go answeredWith); none
// of these asks needs proof of who answered (mesh-sdk go/delivery AskNeedsProof).
// mostAssignerOpen is how many of the assigner's asks the teller keeps open at the router at once. The router keeps at most
// asks.MostOpen open asks of one asker, and the teller asks both the gate's merge asks and the assigner's as one asker, so a burst
// of holds (seven at once on 2026-10-25) would take all the room and a merge Approve could not be asked; one place is always
// left for a merge ask. An assigner ask beyond these waits in the wait for room and is asked when one of them ends.
const mostAssignerOpen = asks.MostOpen - 1
// assignerOpenRecord is the key of the router ids of the assigner's asks the teller asked.
const assignerOpenRecord = "assigner.open"
// assignerOpen is the router ids of the assigner's asks open now: those asked whose ask is neither answered nor ended. An id
// that ended is dropped from the record as it is read.
func (t *Teller) assignerOpen() ([]string, *stdio.StateEntry) {
e, err := t.Store.Get(assignerOpenRecord)
if err != nil || e == nil {
return nil, e
}
var ids []string
if err := json.Unmarshal(e.Value, &ids); err != nil {
t.logf("the record %s cannot be read: %v", assignerOpenRecord, err)
return nil, e
}
return slices.DeleteFunc(ids, func(id string) bool {
k, _, err := t.keptOf(id)
return err == nil && !open(k)
}), e
}
// assignerRoom says whether another of the assigner's asks may be asked now.
func (t *Teller) assignerRoom() bool {
ids, _ := t.assignerOpen()
return len(ids) < mostAssignerOpen
}
// assignerAsked keeps that the assigner's ask with this router id is asked.
func (t *Teller) assignerAsked(id string) {
ids, e := t.assignerOpen()
if !slices.Contains(ids, id) {
ids = append(ids, id)
}
var err error
if e == nil {
_, err = t.Store.Create(assignerOpenRecord, ids)
} else {
_, err = t.Store.Update(assignerOpenRecord, ids, e.Revision)
}
if err != nil {
t.logf("the record %s could not be kept: %v", assignerOpenRecord, err)
}
}
// waitForAssignerRoom puts an assigner ask in the wait for room, once.
func (t *Teller) waitForAssignerRoom(key, id string) {
ws, rec, err := t.waiting()
if err != nil {
t.logf("%v", err)
return
}
if slices.ContainsFunc(ws, func(x waitingAsk) bool { return x.ID == id }) {
return
}
if t.keepWaiting(append(ws, waitingAsk{Key: key, ID: id}), rec) == nil {
t.logf("%s: the assigner's ask %s waits: %d of the assigner's asks are open, the most the teller keeps open so a merge ask always has room", key, id, mostAssignerOpen)
}
}
// isAssignerKey says whether an ask key is one of the assigner's, not the gate's merge ask.
func isAssignerKey(key string) bool {
k, err := delivery.ParseAnyAskKey(key)
return err == nil && k.Kind != delivery.AskKindMerge
}
// assignerAsk reads the assigner's ask at key: found false when the key holds none.
func (t *Teller) assignerAsk(key string) (delivery.AssignerAsk, bool, error) {
if t.Assigner == nil {
return delivery.AssignerAsk{}, false, fmt.Errorf("the teller reads no assigner asks")
}
raw, found, err := t.Assigner.Get(key)
if err != nil || !found {
return delivery.AssignerAsk{}, false, err
}
a, err := delivery.DecodeAssignerAsk(key, raw)
if err != nil {
return delivery.AssignerAsk{}, false, fmt.Errorf("the assigner's key %s holds no ask: %w", key, err)
}
return a, true, nil
}
// raiseAssignerAsk asks the operator the assigner's ask at key, once per router id: an ask answered already (from the view or
// the phone) is not asked; an earlier asking still open is taken back first; an asking already made is only caught up with
// at the router, and one that ended is not asked again, for the router drops a second ask under an id it keeps.
func (t *Teller) raiseAssignerAsk(key string) {
a, found, err := t.assignerAsk(key)
if err != nil {
t.logf("%s: the ask is not raised: %v", key, err)
return
}
if !found {
return
}
id := a.Ask.ID
if raw, _, has, err := t.Answers.Get(delivery.AnswerKey(key)); err != nil {
t.logf("%s: the answers could not be read, so the ask is not raised: %v", key, err)
return
} else if has && answeredAsking(raw) == id {
return
}
prior, grec, err := t.seenAt(gateRecordKey(key))
if err != nil {
t.logf("%s: %v", key, err)
return
}
if grec != nil && prior.ID != id {
t.cancelOpen(key, prior.ID) // a new asking of the same key: the earlier one is taken back
}
k, _, err := t.keptOf(id)
if err != nil {
t.logf("%s: %v", key, err)
return
}
if k != nil {
if grec == nil || prior.ID != id {
_ = t.note(id, key, 0, grec)
}
if open(k) {
t.catchUp(id)
}
return
}
if err := t.note(id, key, 0, grec); err != nil {
t.logf("%s: the record of the ask %s could not be kept, so it is not raised: %v", key, id, err)
return
}
if !t.assignerRoom() {
t.waitForAssignerRoom(key, id)
return
}
if err := t.Asker.Ask(a.Ask); err != nil {
t.logf("%s: the ask %s was not asked: %v", key, id, err)
if kept, _, _ := t.keptOf(id); open(kept) {
_ = t.Store.Delete(keptKey(id)) // kept but not published: forgotten, so the next writing of the key asks it
}
return
}
t.assignerAsked(id)
t.logf("%s: the assigner's ask %s is asked of the operator (%s)", key, id, a.Ask.Headline)
}
// OnAssignerAskGone handles a key the assigner took away from its asks (a hold it lifted by itself): the open router ask of
// its latest asking is taken back, so the operator is not asked about what no longer holds.
func (t *Teller) OnAssignerAskGone(key string) {
t.mu.Lock()
defer t.mu.Unlock()
if !isAssignerKey(key) {
t.logf("the assigner's key %s was taken away and is no ask key of the assigner, and is left", key)
return
}
prior, grec, err := t.seenAt(gateRecordKey(key))
if err != nil || grec == nil {
if err != nil {
t.logf("%s: %v", key, err)
}
return
}
t.cancelOpen(key, prior.ID)
}
// answerAssigner is the act of a warrant for one of the assigner's asks: the ask must still stand at its key under the asking
// and the words the operator answered, and then the answer is written under the ask's key, once. Its options bind no act:
// the assigner reads the option chosen (asker.Client.Take has checked the option, its level and the ask's digest).
func (t *Teller) answerAssigner(key string, o asks.Option, w asks.Warrant) error {
a, found, err := t.assignerAsk(key)
if err != nil {
return err
}
if !found {
t.logf("%s: the ask %s was answered and the assigner has taken it away: the answer changes nothing", key, w.Ask)
return nil
}
if a.Ask.ID != w.Ask {
t.logf("%s: the ask %s was answered, and the assigner has asked again under %s: an answer to an earlier asking changes nothing", key, w.Ask, a.Ask.ID)
return nil
}
if a.Ask.Digest() != w.AskDigest {
return fmt.Errorf("the assigner's ask %s is no longer the ask the operator answered (its digest is %s, the warrant's %s)", key, a.Ask.Digest(), w.AskDigest)
}
body, err := delivery.EncodeAnswer(delivery.Answer{Option: o.ID, Warrant: warrantID(w), Words: w.Says()})
if err != nil {
return err
}
return t.writeAnswer(key, w.Ask, body, w.Says())
}
@@ -1,209 +0,0 @@
package main
import (
"encoding/json"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// 1906/H1: given the assigner's hold ask in its bucket and no answer, when the teller hears the key, then it raises the ask on
// the operator channel as itself, unchanged and once (the key heard again, as a watch replays it, asks nothing more).
func TestAnAssignerAskIsRaisedOnTheOperatorChannelOnce(t *testing.T) {
b := newBed(t)
assigner, _ := withView(b)
key, value, ask := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
b.t.OnAssignerAsk(key)
asked := b.askedOf(asks.AskSubject(delivery.TellerModule))
if len(asked) != 1 {
t.Fatalf("published %d asks, want one; log:\n%s", len(asked), b.logs.String())
}
var got asks.Ask
if err := json.Unmarshal(asked[0].body, &got); err != nil {
t.Fatal(err)
}
if got.ID != ask.ID || got.Digest() != ask.Digest() {
t.Errorf("the router's ask %+v differs from the assigner's", got)
}
}
// 1906/H1: without the view's bucket the assigner's asks are still raised.
func TestAnAssignerAskIsRaisedWithoutTheView(t *testing.T) {
b := newBed(t)
assigner := memGate{}
b.t.Assigner = assigner
key, value, _ := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
if n := len(b.askedOf(asks.AskSubject(delivery.TellerModule))); n != 1 {
t.Fatalf("published %d asks, want one; log:\n%s", n, b.logs.String())
}
}
// 1906/H1: the operator's answer on the phone comes back by the same path as the view's: written under the ask's own key in
// the bucket answers, naming the asking, once; a second warrant writes nothing more and nothing is said as a fault.
func TestTheOperatorsAnswerToAnAssignerAskIsWrittenUnderItsKey(t *testing.T) {
b := newBed(t)
assigner, _ := withView(b)
key, value, ask := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
w := ackFor(ask, "release")
b.deliver(t, w)
raw, ok := b.answers.written[delivery.AnswerKey(key)]
if !ok {
t.Fatalf("no answer under %s; log:\n%s", key, b.logs.String())
}
if a, err := delivery.DecodeAnswer(raw); err != nil || a.Option != "release" || answeredAsking(raw) != ask.ID {
t.Errorf("answer = %+v, %v", a, err)
}
b.deliver(t, w)
if b.answers.writes != 1 || len(b.conds) != 0 {
t.Errorf("%d writes, conditions %+v", b.answers.writes, b.conds)
}
}
// 1906/H1: an answer to an asking the assigner has since taken away or asked again changes nothing.
func TestAnAnswerToAnAssignerAskTakenAwayWritesNothing(t *testing.T) {
b := newBed(t)
assigner, _ := withView(b)
key, value, ask := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
delete(assigner, key)
b.deliver(t, ackFor(ask, "release"))
if b.answers.writes != 0 {
t.Errorf("an answer to a withdrawn ask was written")
}
}
// 1906/H1, H2: an ask answered in the view is taken back from the phone, and an ask the assigner took away (a hold it lifted by
// itself) is taken back too; an ask answered before it is heard is never raised.
func TestAnAssignerAskAnsweredInTheViewOrTakenAwayIsTakenBack(t *testing.T) {
cancelled := func(b *bed, id string) bool {
for _, p := range b.askedOf(asks.CancelSubject(delivery.TellerModule)) {
if string(p.body) == `{"id":"`+id+`"}` {
return true
}
}
return false
}
b := newBed(t)
assigner, view := withView(b)
key, value, ask := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
view[key+".view"] = viewAnswer(t, "release", "")
b.t.OnViewAnswer(key + ".view")
if !cancelled(b, ask.ID) {
t.Errorf("the ask answered in the view was not taken back; log:\n%s", b.logs.String())
}
b = newBed(t)
assigner, _ = withView(b)
assigner[key] = value
b.t.OnAssignerAsk(key)
delete(assigner, key)
b.t.OnAssignerAskGone(key)
if !cancelled(b, ask.ID) {
t.Errorf("the ask taken away was not taken back; log:\n%s", b.logs.String())
}
b = newBed(t)
assigner, view = withView(b)
assigner[key] = value
view[key+".view"] = viewAnswer(t, "keep", "")
b.t.OnAssignerAsk(key)
if n := len(b.askedOf(asks.AskSubject(delivery.TellerModule))); n != 0 {
t.Errorf("an ask answered in the view was raised %d times", n)
}
}
// 1906/H1: an assigner ask refused for want of room waits and is asked again when another of the teller's asks ends, as a merge
// ask does; a refusal for its words is said as a condition about a delivery ask, not a merge.
func TestAnAssignerAskWaitsForRoomAndARefusalIsSaidAsADeliveryAsk(t *testing.T) {
b := newBed(t)
assigner, _ := withView(b)
_, first := b.askOf(t, 1, 1)
b.askOf(t, 2, 2)
b.askOf(t, 3, 3)
key, value, ask := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
b.refuse(t, ask.ID, roomRefusal)
b.deliver(t, warrantFor(first.Ask, "approve"))
got := b.askedIDs()
if len(got) != 5 || got[4] != ask.ID {
t.Fatalf("asks = %v, want the assigner's %s asked again", got, ask.ID)
}
b = newBed(t)
assigner, _ = withView(b)
assigner[key] = value
b.t.OnAssignerAsk(key)
b.refuse(t, ask.ID, "the ask has no headline")
if len(b.conds) != 1 || b.conds[0].Subject.Scope != "delivery" || strings.Contains(b.conds[0].Explanation, "merge") {
t.Errorf("conditions = %+v", b.conds)
}
}
// ackFor is the router's warrant for an option of level acknowledge, as the assigner's asks offer.
func ackFor(a asks.Ask, option string) asks.Warrant {
w := warrantFor(a, option)
w.Level, w.Label = asks.Acknowledge, option
return w
}
// holdAskOf is the assigner's hold ask of module plex at the n-th fingerprint, shaped as holdAsk.
func holdAskOf(t *testing.T, n int) (string, []byte, asks.Ask) {
t.Helper()
key, err := delivery.HoldAskKey("plex", "src2:"+strings.Repeat(string(rune('0'+n)), 64))
if err != nil {
t.Fatal(err)
}
_, _, ask := holdAsk(t)
ask.ID = delivery.AskID(key)
value, err := delivery.EncodeAssignerAsk(key, delivery.AssignerAsk{Ask: ask})
if err != nil {
t.Fatal(err)
}
return key, value, ask
}
// 1906/H5: given seven holds at once (as on 2026-10-25), when they are raised, then at most two are open at the router, a merge
// ask is still asked, and each hold answered lets the next waiting one be asked.
func TestABurstOfHoldsLeavesRoomForAMergeAsk(t *testing.T) {
b := newBed(t)
assigner, _ := withView(b)
var held []asks.Ask
for n := 0; n < 7; n++ {
key, value, ask := holdAskOf(t, n)
assigner[key] = value
b.t.OnAssignerAsk(key)
held = append(held, ask)
}
if got := b.askedIDs(); len(got) != mostAssignerOpen || got[0] != held[0].ID || got[1] != held[1].ID {
t.Fatalf("asked %v, want the first %d holds only; log:\n%s", got, mostAssignerOpen, b.logs.String())
}
_, merge := b.askOf(t, 1, 1)
if got := b.askedIDs(); len(got) != mostAssignerOpen+1 || got[mostAssignerOpen] != merge.Ask.ID {
t.Fatalf("asked %v, want the merge ask asked beside the holds", got)
}
if len(b.dropped) != 0 || len(b.conds) != 0 {
t.Errorf("router dropped %v, conditions %+v", b.dropped, b.conds)
}
b.deliver(t, ackFor(held[0], "release"))
got := b.askedIDs()
if len(got) != mostAssignerOpen+2 || got[len(got)-1] != held[2].ID {
t.Fatalf("asked %v after a hold was answered, want the third hold asked next; log:\n%s", got, b.logs.String())
}
// The merge ask answered frees its own place, and no hold takes it.
b.deliver(t, warrantFor(merge.Ask, "approve"))
if got := b.askedIDs(); len(got) != mostAssignerOpen+2 {
t.Errorf("asked %v after the merge ask was answered, want no further hold", got)
}
}
@@ -545,99 +545,3 @@ func TestTellerFillsAnAskFromTheViewsAnswerOnce(t *testing.T) { // 1297/B10 AC
t.Error("the merge's ask was answered from a view key") t.Error("the merge's ask was answered from a view key")
} }
} }
// TestTellerRaisesAnAssignerAskAndWritesTheOperatorsAnswer starts the program against the bus: the assigner's hold ask, put in
// its bucket, is on the operator channel's queue as the teller's ask; the router's warrant for "release" is written under the
// ask's key, where the assigner reads it; a second ask the assigner takes away (a hold it lifted) is cancelled at the router
// (1906/H1, H2 on the real bus).
func TestTellerRaisesAnAssignerAskAndWritesTheOperatorsAnswer(t *testing.T) {
now = time.Now().UTC()
b := startFull(t)
assigner := b.as(t, "n1.mesh-delivery-assigner")
broker := filepath.Join(t.TempDir(), "broker")
cred, _ := json.Marshal(credential{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-teller", User: "n1.mesh-delivery-teller", Password: "pw"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
env := map[string]string{"MESH_BROKER_FILE": broker}
logs := &syncBuffer{}
rn := startRunner(b.ctx, func(c context.Context) error { return run(c, func(k string) string { return env[k] }, logs) })
t.Cleanup(func() { rn.finish(t, 10*time.Second) })
key, value, ask := holdAsk(t)
askKV, err := assigner.KeyValue(b.ctx, assignerAsks)
if err != nil {
t.Fatal(err)
}
time.Sleep(500 * time.Millisecond) // the program hears warrants and watches
if _, err := askKV.Put(b.ctx, key, value); err != nil {
t.Fatal(err)
}
seat, err := b.js.Stream(b.ctx, "SEAT_OPERATOR_CHANNEL")
if err != nil {
t.Fatal(err)
}
var raw *jetstream.RawStreamMsg
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if raw, err = seat.GetLastMsgForSubject(b.ctx, seatAsk); err == nil {
break
}
}
if raw == nil {
t.Fatalf("the assigner's ask is not on the seat's queue: %v\n%s", err, logs.String())
}
var asked asks.Ask
if err := json.Unmarshal(raw.Data, &asked); err != nil || asked.ID != ask.ID || asked.Digest() != ask.Digest() {
t.Fatalf("the ask on the queue is %+v (%v), want the assigner's", asked, err)
}
w, _ := json.Marshal(ackFor(ask, "release"))
if err := b.ctl.Publish(seatDecided, w); err != nil {
t.Fatal(err)
}
answers, err := b.js.KeyValue(b.ctx, tellerAnswers)
if err != nil {
t.Fatal(err)
}
var e jetstream.KeyValueEntry
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err = answers.Get(b.ctx, delivery.AnswerKey(key)); err == nil {
break
}
}
if e == nil {
t.Fatalf("no answer under %s: %v\n%s", key, err, logs.String())
}
if a, err := delivery.DecodeAnswer(e.Value()); err != nil || a.Option != "release" || answeredAsking(e.Value()) != ask.ID {
t.Fatalf("answer %+v err %v", a, err)
}
key2, err := delivery.HoldAskKey("plex", "src2:"+strings.Repeat("3", 64))
if err != nil {
t.Fatal(err)
}
ask2 := ask
ask2.ID = delivery.AskID(key2)
value2, err := delivery.EncodeAssignerAsk(key2, delivery.AssignerAsk{Ask: ask2})
if err != nil {
t.Fatal(err)
}
if _, err := askKV.Put(b.ctx, key2, value2); err != nil {
t.Fatal(err)
}
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if strings.Contains(logs.String(), ask2.ID+" is asked") {
break
}
}
if err := askKV.Delete(b.ctx, key2); err != nil {
t.Fatal(err)
}
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if raw, err = seat.GetLastMsgForSubject(b.ctx, seatCancel); err == nil {
break
}
}
if err != nil || string(raw.Data) != `{"id":"`+ask2.ID+`"}` {
t.Errorf("the ask taken away was not cancelled: %v %v\n%s", raw, err, logs.String())
}
}
@@ -141,20 +141,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// there is said and leaves the teller without that path rather than without the gate's asks (a wait for it needs a timer). // there is said and leaves the teller without that path rather than without the gate's asks (a wait for it needs a timer).
failed := make(chan error, 2) failed := make(chan error, 2)
assignerKV, viewKV := optionalBucket(ctx, js, delivery.AssignerAsksBucket, logger), optionalBucket(ctx, js, delivery.ViewAnswersBucket, logger) assignerKV, viewKV := optionalBucket(ctx, js, delivery.AssignerAsksBucket, logger), optionalBucket(ctx, js, delivery.ViewAnswersBucket, logger)
// The assigner's asks are raised on the operator channel whether or not the view's answers can be read (issue 1906).
if viewKV != nil {
t.View = kvGate{ctx: ctx, kv: viewKV}
}
if assignerKV != nil {
t.Assigner = kvGate{ctx: ctx, kv: assignerKV}
go func() {
failed <- watchAsks(ctx, assignerKV, func(key string, _ uint64, _ []byte) { t.OnAssignerAsk(key) }, t.OnAssignerAskGone, nil)
}()
}
if assignerKV != nil && viewKV != nil { if assignerKV != nil && viewKV != nil {
t.Assigner, t.View = kvGate{ctx: ctx, kv: assignerKV}, kvGate{ctx: ctx, kv: viewKV}
go func() { go func() {
failed <- watchAsks(ctx, viewKV, func(key string, _ uint64, _ []byte) { t.OnViewAnswer(key) }, func(string) {}, nil) failed <- watchAsks(ctx, viewKV, func(key string, _ uint64, _ []byte) { t.OnViewAnswer(key) }, func(string) {}, nil)
}() }()
go func() {
failed <- watchAsks(ctx, assignerKV, func(key string, _ uint64, _ []byte) { t.OnAssignerAsk(key) }, func(string) {}, nil)
}()
} }
go func() { go func() {
if err := watchAsks(ctx, gate, t.OnAsk, t.OnAskGone, t.OnCurrent); err != nil { if err := watchAsks(ctx, gate, t.OnAsk, t.OnAskGone, t.OnCurrent); err != nil {
@@ -176,7 +170,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
func optionalBucket(ctx context.Context, js jetstream.JetStream, name string, logger *log.Logger) jetstream.KeyValue { func optionalBucket(ctx context.Context, js jetstream.JetStream, name string, logger *log.Logger) jetstream.KeyValue {
kv, err := js.KeyValue(ctx, name) kv, err := js.KeyValue(ctx, name)
if err != nil { if err != nil {
logger.Printf("the bucket %s cannot be read (%v): the asks and answers it holds are not carried until the teller is started again with it", name, err) logger.Printf("the bucket %s cannot be read (%v): the view's answers fill no ask until the teller is started again with it", name, err)
return nil return nil
} }
return kv return kv
@@ -340,9 +340,6 @@ func (t *Teller) Answer(o asks.Option, w asks.Warrant) error {
if srec == nil { if srec == nil {
return fmt.Errorf("the teller holds no gate key for the ask %s", w.Ask) return fmt.Errorf("the teller holds no gate key for the ask %s", w.Ask)
} }
if isAssignerKey(s.Key) {
return t.answerAssigner(s.Key, o, w)
}
raw, found, err := t.Gate.Get(s.Key) raw, found, err := t.Gate.Get(s.Key)
if err != nil { if err != nil {
return fmt.Errorf("the gate's ask %s could not be read: %w", s.Key, err) return fmt.Errorf("the gate's ask %s could not be read: %w", s.Key, err)
@@ -370,31 +367,26 @@ func (t *Teller) Answer(o asks.Option, w asks.Warrant) error {
if err != nil { if err != nil {
return err return err
} }
return t.writeAnswer(s.Key, w.Ask, body, w.Says()) // Compare-and-set: created when the key has no answer; updated at the revision read when it holds an earlier asking's
} // answer (the gate ignores that one); left when it holds this asking's.
akey := delivery.AnswerKey(s.Key)
// writeAnswer writes the answer to the asking askID under the ask's key by compare-and-set: created when the key has no answer;
// updated at the revision read when it holds an earlier asking's answer (the reader ignores that one); left when it holds this
// asking's.
func (t *Teller) writeAnswer(key, askID string, body []byte, words string) error {
akey := delivery.AnswerKey(key)
cur, crev, found, err := t.Answers.Get(akey) cur, crev, found, err := t.Answers.Get(akey)
if err != nil { if err != nil {
return fmt.Errorf("the answer under %s could not be read: %w", key, err) return fmt.Errorf("the answer under %s could not be read: %w", s.Key, err)
} }
if found && answeredAsking(cur) == askID { if found && answeredAsking(cur) == w.Ask {
t.logf("%s: an answer is there already, and is not written again", key) t.logf("%s: an answer is there already, and is not written again", s.Key)
return nil return nil
} }
written, err := t.Answers.Write(akey, body, crev, found) written, err := t.Answers.Write(akey, body, crev, found)
if err != nil { if err != nil {
return fmt.Errorf("the answer under %s could not be written: %w", key, err) return fmt.Errorf("the answer under %s could not be written: %w", s.Key, err)
} }
if !written { if !written {
t.logf("%s: another writer answered first, and the answer is not written again", key) t.logf("%s: another writer answered first, and the answer is not written again", s.Key)
return nil return nil
} }
t.logf("%s: %s", key, words) t.logf("%s: %s", s.Key, w.Says())
return nil return nil
} }
@@ -431,10 +423,10 @@ func (t *Teller) saidOnce(h asker.Handled, w asks.Warrant, err error) {
t.clear(w.Ask) t.clear(w.Ask)
case h == asker.Refused || (h == asker.Acted && err != nil): case h == asker.Refused || (h == asker.Acted && err != nil):
t.logf("the warrant of %s does not hold, and no answer is written: %v", w.Ask, err) t.logf("the warrant of %s does not hold, and no answer is written: %v", w.Ask, err)
t.raise(w.Ask, t.kindOf(w.Ask).answerHeadline, "its answer did not hold here: "+oneLine(err)) t.raise(w.Ask, "A merge Approve did not hold", "its answer did not hold here: "+oneLine(err))
case h == asker.Recorded && w.Outcome == asks.OutcomeRefused: case h == asker.Recorded && w.Outcome == asks.OutcomeRefused:
t.logf("the router refused the ask %s: %s", w.Ask, w.Words) t.logf("the router refused the ask %s: %s", w.Ask, w.Words)
t.raise(w.Ask, t.kindOf(w.Ask).refusedHeadline, "the router refused it: "+w.Words) t.raise(w.Ask, "A merge ask was refused", "the router refused it: "+w.Words)
case err != nil: case err != nil:
t.logf("the warrant of %s: %v", w.Ask, err) t.logf("the warrant of %s: %v", w.Ask, err)
} }
@@ -454,30 +446,6 @@ func oneLine(err error) string {
// conditionKey is the key of the condition about an ask. // conditionKey is the key of the condition about an ask.
func conditionKey(id string) string { return delivery.TellerModule + ".merge-ask." + id } func conditionKey(id string) string { return delivery.TellerModule + ".merge-ask." + id }
// askWords are the words the teller's conditions use for one kind of ask: the gate's merge ask, or one of the assigner's.
type askWords struct {
scope, noun, needs, nothing, resolved string
answerHeadline, refusedHeadline, unaskedHeadline string
}
var (
mergeWords = askWords{scope: "merge", noun: "The merge ask ", needs: "to ask for the Approve again", nothing: "Nothing was merged.",
resolved: "A good answer to the merge ask was written.", answerHeadline: "A merge Approve did not hold",
refusedHeadline: "A merge ask was refused", unaskedHeadline: "A merge ask could not be asked"}
assignerWords = askWords{scope: "delivery", noun: "The delivery ask ", needs: "to answer it in the delivery view",
nothing: "Nothing was changed on a machine.", resolved: "A good answer to the delivery ask was written.",
answerHeadline: "An answer to a delivery ask did not hold", refusedHeadline: "A delivery ask was refused",
unaskedHeadline: "A delivery ask could not be asked"}
)
// kindOf is the words for the ask with this router id, by the key it was asked for.
func (t *Teller) kindOf(askID string) askWords {
if isAssignerKey(t.condID(askID)) {
return assignerWords
}
return mergeWords
}
func (t *Teller) raise(askID, headline, why string) { func (t *Teller) raise(askID, headline, why string) {
if t.Say == nil { if t.Say == nil {
return return
@@ -489,12 +457,11 @@ func (t *Teller) raise(askID, headline, why string) {
} }
t.logf("the condition of %s could not be kept: %v", id, err) t.logf("the condition of %s could not be kept: %v", id, err)
} }
k := t.kindOf(askID) needs := "to ask for the Approve again"
needs := k.needs c := asks.Condition{Key: conditionKey(id), Severity: asks.SeverityWarning, Subject: asks.Scope{Scope: "merge", ID: id}, State: asks.StateRaised,
c := asks.Condition{Key: conditionKey(id), Severity: asks.SeverityWarning, Subject: asks.Scope{Scope: k.scope, ID: id}, State: asks.StateRaised, Summary: "The merge ask " + id + ": " + why, Headline: headline, Needs: needs,
Summary: k.noun + id + ": " + why, Headline: headline, Needs: needs, Explanation: asks.NeedsYou + needs + ". The merge ask " + id + ", " + why + ". Nothing was merged.",
Explanation: asks.NeedsYou + needs + ". " + k.noun + id + ", " + why + ". " + k.nothing, Resolved: "A good answer to the merge ask was written."}
Resolved: k.resolved}
if err := t.Say(c); err != nil { if err := t.Say(c); err != nil {
t.logf("the condition of %s could not be said: %v", id, err) t.logf("the condition of %s could not be said: %v", id, err)
_ = t.Store.Delete(condKey(id)) _ = t.Store.Delete(condKey(id))
@@ -510,7 +477,7 @@ func (t *Teller) clear(askID string) {
if err != nil || e == nil { if err != nil || e == nil {
return return
} }
c := asks.Condition{Key: conditionKey(id), Subject: asks.Scope{Scope: t.kindOf(askID).scope, ID: id}, State: asks.StateCleared} c := asks.Condition{Key: conditionKey(id), Subject: asks.Scope{Scope: "merge", ID: id}, State: asks.StateCleared}
if err := t.Say(c); err != nil { if err := t.Say(c); err != nil {
t.logf("the condition of %s could not be cleared: %v", id, err) t.logf("the condition of %s could not be cleared: %v", id, err)
return return
@@ -611,7 +578,7 @@ func (t *Teller) roomRefused(w asks.Warrant) {
return return
} }
t.logf("the router refused the ask %s for want of room %d times (%s): it is asked no more, and said", w.Ask, n, w.Words) t.logf("the router refused the ask %s for want of room %d times (%s): it is asked no more, and said", w.Ask, n, w.Words)
t.raise(w.Ask, t.kindOf(w.Ask).unaskedHeadline, fmt.Sprintf("the router refused it %d times for want of room: %s", n, w.Words)) t.raise(w.Ask, "A merge ask could not be asked", fmt.Sprintf("the router refused it %d times for want of room: %s", n, w.Words))
return return
} }
if t.keepWaiting(ws, rec) != nil { if t.keepWaiting(ws, rec) != nil {
@@ -708,11 +675,6 @@ func (t *Teller) askWaiting(n int) {
i++ i++
continue continue
} }
assigner := isAssignerKey(x.Key)
if assigner && !t.assignerRoom() {
i++ // the assigner's asks hold their most open: it waits, and a merge ask after it may be asked
continue
}
a, why := t.stillAsked(x) a, why := t.stillAsked(x)
if why != "" { if why != "" {
t.logf("%s: the ask %s, waiting for room, is not asked again: %s", x.Key, x.ID, why) t.logf("%s: the ask %s, waiting for room, is not asked again: %s", x.Key, x.ID, why)
@@ -725,9 +687,6 @@ func (t *Teller) askWaiting(n int) {
return return
} }
ws[i].Asked = true ws[i].Asked = true
if assigner {
t.assignerAsked(x.ID)
}
t.logf("%s: the ask %s is asked of the operator again (refused %d times for want of room)", x.Key, x.ID, x.Refusals) t.logf("%s: the ask %s is asked of the operator again (refused %d times for want of room)", x.Key, x.ID, x.Refusals)
if n--; n <= 0 { if n--; n <= 0 {
return return
@@ -742,9 +701,6 @@ func (t *Teller) stillAsked(x waitingAsk) (asks.Ask, string) {
if x.Key == "" { if x.Key == "" {
return asks.Ask{}, "the teller holds no gate key for it" return asks.Ask{}, "the teller holds no gate key for it"
} }
if isAssignerKey(x.Key) {
return t.stillAskedByAssigner(x)
}
raw, found, err := t.Gate.Get(x.Key) raw, found, err := t.Gate.Get(x.Key)
if err != nil { if err != nil {
return asks.Ask{}, "the gate's ask could not be read: " + err.Error() return asks.Ask{}, "the gate's ask could not be read: " + err.Error()
@@ -768,26 +724,6 @@ func (t *Teller) stillAsked(x waitingAsk) (asks.Ask, string) {
return ga.Ask, "" return ga.Ask, ""
} }
// stillAskedByAssigner is stillAsked for one of the assigner's asks.
func (t *Teller) stillAskedByAssigner(x waitingAsk) (asks.Ask, string) {
a, found, err := t.assignerAsk(x.Key)
switch {
case err != nil:
return asks.Ask{}, err.Error()
case !found:
return asks.Ask{}, "the assigner has taken it away"
case a.Ask.ID != x.ID:
return asks.Ask{}, "the assigner asks again under " + a.Ask.ID
}
if cur, _, has, err := t.Answers.Get(delivery.AnswerKey(x.Key)); err == nil && has && answeredAsking(cur) == x.ID {
return asks.Ask{}, "it is answered"
}
if err := a.Ask.Check(t.now()); err != nil {
return asks.Ask{}, "it no longer holds: " + err.Error()
}
return a.Ask, ""
}
// askAgain keeps the ask afresh as the asker would (its record of the refusal is replaced, so its warrant is taken) and // askAgain keeps the ask afresh as the asker would (its record of the refusal is replaced, so its warrant is taken) and
// publishes it under the bus message id given. // publishes it under the bus message id given.
func (t *Teller) askAgain(a asks.Ask, msgID string) error { func (t *Teller) askAgain(a asks.Ask, msgID string) error {
@@ -25,8 +25,7 @@ func (t *Teller) OnViewAnswer(key string) {
t.fillFromView(askKey) t.fillFromView(askKey)
} }
// OnAssignerAsk handles a key put in the assigner's bucket asks: a view answer that came before its ask fills it now, and an // OnAssignerAsk handles a key put in the assigner's bucket asks: a view answer that came before its ask fills it now.
// ask not answered yet is raised on the operator channel (issue 1906; assignerasks.go).
func (t *Teller) OnAssignerAsk(key string) { func (t *Teller) OnAssignerAsk(key string) {
t.mu.Lock() t.mu.Lock()
defer t.mu.Unlock() defer t.mu.Unlock()
@@ -35,7 +34,6 @@ func (t *Teller) OnAssignerAsk(key string) {
return return
} }
t.fillFromView(key) t.fillFromView(key)
t.raiseAssignerAsk(key)
} }
// fillFromView writes the answer to the assigner's ask at askKey from the view's answer, when both are there and readable, the // fillFromView writes the answer to the assigner's ask at askKey from the view's answer, when both are there and readable, the
@@ -105,6 +103,5 @@ func (t *Teller) fillFromView(askKey string) {
t.logf("%s: another writer answered first, and the view's answer is not written", askKey) t.logf("%s: another writer answered first, and the view's answer is not written", askKey)
default: default:
t.logf("%s: %s", askKey, words) t.logf("%s: %s", askKey, words)
t.cancelOpen(askKey, ask.Ask.ID) // answered in the view: the same ask on the phone is taken back
} }
} }
@@ -10,9 +10,8 @@ import (
const fpHeld = "src2:2222222222222222222222222222222222222222222222222222222222222222" const fpHeld = "src2:2222222222222222222222222222222222222222222222222222222222222222"
// holdAsk is a key of the assigner's asks: the ask about a held fingerprint, shaped as the assigner composes it // holdAsk is a key of the assigner's asks: the ask about a held fingerprint, as a fixture (in release 5 the assigner writes
// (mesh-delivery-assigner internal/assign asks.go HoldAsk: two options of level acknowledge that bind no act, so the router // none yet; the value is `{"v":1,"ask":<the router's ask>}`).
// takes it; the value is `{"v":1,"ask":<the router's ask>}`).
func holdAsk(t *testing.T) (key string, value []byte, ask asks.Ask) { func holdAsk(t *testing.T) (key string, value []byte, ask asks.Ask) {
t.Helper() t.Helper()
key, err := delivery.HoldAskKey("plex", fpHeld) key, err := delivery.HoldAskKey("plex", fpHeld)
@@ -21,7 +20,7 @@ func holdAsk(t *testing.T) (key string, value []byte, ask asks.Ask) {
} }
ask = asks.Ask{ID: delivery.AskID(key), Headline: "plex is held after a failed health check", Explanation: "ace failed the check.", Who: asks.Operator, ask = asks.Ask{ID: delivery.AskID(key), Headline: "plex is held after a failed health check", Explanation: "ace failed the check.", Who: asks.Operator,
Expires: now.Add(asks.ApproveLasts), OnExpiry: "the fingerprint stays held", Expires: now.Add(asks.ApproveLasts), OnExpiry: "the fingerprint stays held",
Options: []asks.Option{{ID: "keep", Label: "Keep it held", Does: "keeps it held", Level: asks.Acknowledge}, {ID: "release", Label: "Release it", Does: "lets the move go on", Level: asks.Acknowledge}}} Options: []asks.Option{{ID: "release", Label: "Release", Does: "lets the move go on", Level: asks.Approve}, {ID: "keep", Label: "Keep held", Does: "keeps it held", Level: asks.Approve}}}
value, err = delivery.EncodeAssignerAsk(key, delivery.AssignerAsk{Ask: ask}) value, err = delivery.EncodeAssignerAsk(key, delivery.AssignerAsk{Ask: ask})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)