Compare commits
10
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a5b763f8a2 | ||
|
|
50a2e4c4e2 | ||
|
|
4169faf3f0 | ||
|
|
103eec8855 | ||
|
|
8a62f23afe | ||
|
|
84395420ee | ||
|
|
9740842af7 | ||
|
|
588cfd54bb | ||
|
|
4ccd78c9ec | ||
|
|
6fe7b7dd6c |
@@ -341,14 +341,12 @@ func TestAHeldFingerprintIsAskedAbout(t *testing.T) {
|
|||||||
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`))
|
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`))
|
||||||
b.start(t)
|
b.start(t)
|
||||||
hkey, _ := delivery.HeldKey("web", fNew)
|
hkey, _ := delivery.HeldKey("web", fNew)
|
||||||
// 1906/H3: the held record names the machine and keeps the health, when and the node's words.
|
if h, err := delivery.DecodeHeld(b.get(t, delivery.AssignerWantsBucket, hkey)); err != nil || h.Node != "ace" {
|
||||||
if h, err := delivery.DecodeHeld(b.get(t, delivery.AssignerWantsBucket, hkey)); err != nil || h.Node != "ace" ||
|
|
||||||
!strings.HasPrefix(h.Words, "health failed, phase running, seen at ") || !strings.HasSuffix(h.Words, ": connection refused") {
|
|
||||||
t.Fatalf("held = %+v, %v", h, err)
|
t.Fatalf("held = %+v, %v", h, err)
|
||||||
}
|
}
|
||||||
akey, _ := delivery.HoldAskKey("web", fNew)
|
akey, _ := delivery.HoldAskKey("web", fNew)
|
||||||
a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
|
a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
|
||||||
if err != nil || a.Asking <= 0 || a.Ask.ID != delivery.AskingID(akey, a.Asking) || len(a.Ask.Options) != 2 {
|
if err != nil || a.Ask.ID != delivery.AskingID(akey, 0) || len(a.Ask.Options) != 2 {
|
||||||
t.Fatalf("the ask = %+v, %v", a, err)
|
t.Fatalf("the ask = %+v, %v", a, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -582,15 +580,8 @@ func (b *bed) wantNow(t *testing.T, node, module, what string, ok func(delivery.
|
|||||||
return last
|
return last
|
||||||
}
|
}
|
||||||
|
|
||||||
// answer says the teller wrote the operator's answer to the ask that stands at key (its warrant names that asking's id).
|
// answer says the teller wrote the operator's answer to the ask at key.
|
||||||
func (b *bed) answer(t *testing.T, key, option string) {
|
func (b *bed) answer(t *testing.T, key, option string) { b.answerTo(t, key, option, "r1") }
|
||||||
t.Helper()
|
|
||||||
a, err := delivery.DecodeAssignerAsk(key, b.get(t, delivery.AssignerAsksBucket, key))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
b.answerTo(t, key, option, a.Ask.ID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// answerTo is answer to the ask with this id (the router's warrant names it).
|
// answerTo is answer to the ask with this id (the router's warrant names it).
|
||||||
func (b *bed) answerTo(t *testing.T, key, option, askID string) {
|
func (b *bed) answerTo(t *testing.T, key, option, askID string) {
|
||||||
@@ -782,81 +773,3 @@ func TestAUserListRevisionOfTheBusIsWrittenAtOnce(t *testing.T) {
|
|||||||
t.Errorf("the user-list revision was asked")
|
t.Errorf("the user-list revision was asked")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1906/H2, whole on the bus: given a fingerprint held after one failed check, when every node that runs the module then runs
|
|
||||||
// it healthy, the assigner lifts the hold by itself: the held record and its ask are taken away, the held condition is
|
|
||||||
// cleared saying why, and the want stays at the fingerprint.
|
|
||||||
func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
|
|
||||||
b := startBed(t, "")
|
|
||||||
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
|
||||||
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
|
||||||
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
|
||||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
|
||||||
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
|
||||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
|
|
||||||
b.start(t)
|
|
||||||
hkey, _ := delivery.HeldKey("web", fNew)
|
|
||||||
akey, _ := delivery.HoldAskKey("web", fNew)
|
|
||||||
b.get(t, delivery.AssignerWantsBucket, hkey)
|
|
||||||
b.get(t, delivery.AssignerAsksBucket, akey)
|
|
||||||
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
|
||||||
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
|
|
||||||
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
|
||||||
if b.hasKey(delivery.AssignerWantsBucket, hkey) || b.hasKey(delivery.AssignerAsksBucket, akey) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, c := range b.conditionsSaid(t) {
|
|
||||||
if c.Subject.ID == "web" && c.State == asks.StateCleared && strings.Contains(c.Resolved, "lifted its hold by itself") {
|
|
||||||
if w := b.wantNow(t, "ace", "web", "the want at the lifted fingerprint", func(w delivery.Want) bool { return w.FP == fNew }); w.Rule != 0 {
|
|
||||||
t.Errorf("the want = %+v", w)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.Fatalf("the hold was not lifted: held %v, ask %v", b.hasKey(delivery.AssignerWantsBucket, hkey), b.hasKey(delivery.AssignerAsksBucket, akey))
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1906/H2, whole on the bus: a hold lifted by itself and written again after the assigner was started again is asked under a
|
|
||||||
// new id, so the teller raises it; the answer to the first hold does not carry over (1906/H4).
|
|
||||||
func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) {
|
|
||||||
b := startBed(t, "")
|
|
||||||
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
|
||||||
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
|
||||||
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
|
||||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
|
||||||
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
|
||||||
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
|
||||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
|
|
||||||
_, stop := b.startStoppable(t)
|
|
||||||
hkey, _ := delivery.HeldKey("web", fNew)
|
|
||||||
akey, _ := delivery.HoldAskKey("web", fNew)
|
|
||||||
first, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
|
|
||||||
for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.AssignerWantsBucket, hkey); time.Sleep(100 * time.Millisecond) {
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatal("the hold was not lifted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
stop()
|
|
||||||
time.Sleep(1100 * time.Millisecond) // the next hold is written in another second
|
|
||||||
b.answerTo(t, akey, "keep", first.Ask.ID) // an answer to the first hold, which the later hold must not take
|
|
||||||
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
|
||||||
b.start(t)
|
|
||||||
again, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if again.Ask.ID == first.Ask.ID {
|
|
||||||
t.Fatalf("the hold written again is asked under the first id %s", first.Ask.ID)
|
|
||||||
}
|
|
||||||
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
|
|
||||||
for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.AssignerWantsBucket, hkey); time.Sleep(100 * time.Millisecond) {
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatal("the stale \"keep\" of the first hold kept the second one held")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -28,15 +28,6 @@ type passer struct {
|
|||||||
// answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want.
|
// answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want.
|
||||||
answeredAt map[string]uint64
|
answeredAt map[string]uint64
|
||||||
asks *composeAsks
|
asks *composeAsks
|
||||||
// now is the clock; nil is time.Now.
|
|
||||||
now func() time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *passer) clock() time.Time {
|
|
||||||
if p.now != nil {
|
|
||||||
return p.now()
|
|
||||||
}
|
|
||||||
return time.Now()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *passer) pass(ctx context.Context) {
|
func (p *passer) pass(ctx context.Context) {
|
||||||
@@ -97,7 +88,7 @@ func (p *passer) wants(ctx context.Context) {
|
|||||||
for name := range names {
|
for name := range names {
|
||||||
fp := trunk[name].FP2
|
fp := trunk[name].FP2
|
||||||
name := name
|
name := name
|
||||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name], Now: time.Now(),
|
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name],
|
||||||
Composed: func(node, fp string) (delivery.ComposedDigest, bool) { return p.w.composed(node, name, fp) },
|
Composed: func(node, fp string) (delivery.ComposedDigest, bool) { return p.w.composed(node, name, fp) },
|
||||||
Restored: func(node string) bool {
|
Restored: func(node string) bool {
|
||||||
key, err := delivery.RestoreAskKey(node, name, fp)
|
key, err := delivery.RestoreAskKey(node, name, fp)
|
||||||
@@ -112,7 +103,6 @@ func (p *passer) wants(ctx context.Context) {
|
|||||||
grant, _ := delivery.ControllerGrantKey(name, fp)
|
grant, _ := delivery.ControllerGrantKey(name, fp)
|
||||||
hold, _ := delivery.HoldAskKey(name, fp)
|
hold, _ := delivery.HoldAskKey(name, fp)
|
||||||
m.Released = p.answeredWith(hold, "release")
|
m.Released = p.answeredWith(hold, "release")
|
||||||
m.Kept = p.answeredWith(hold, "keep")
|
|
||||||
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(delivery.ControllerGrantsBucket, grant)
|
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(delivery.ControllerGrantsBucket, grant)
|
||||||
if m.Released {
|
if m.Released {
|
||||||
p.clearHeld(name, fp)
|
p.clearHeld(name, fp)
|
||||||
@@ -130,11 +120,8 @@ func (p *passer) wants(ctx context.Context) {
|
|||||||
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeHeld(held) })
|
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeHeld(held) })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if out.Lifted {
|
if out.Held != nil || m.Held {
|
||||||
p.lift(ctx, name, fp)
|
p.holdAsk(ctx, name, fp)
|
||||||
}
|
|
||||||
if out.Held != nil || (m.Held && !out.Lifted) {
|
|
||||||
p.holdAsk(ctx, name, fp, out.Held)
|
|
||||||
p.heldCondition(ctx, name, fp, out.Held, len(out.Restore) > 0)
|
p.heldCondition(ctx, name, fp, out.Held, len(out.Restore) > 0)
|
||||||
}
|
}
|
||||||
for _, node := range out.Restore {
|
for _, node := range out.Restore {
|
||||||
@@ -223,62 +210,15 @@ func (p *passer) put(ctx context.Context, bucket, key string, encode func() ([]b
|
|||||||
// holdAsk writes, once, the ask about a held fingerprint (ADR 0528 clause 6, ADR 0437 clause 10): the first ask that needs
|
// holdAsk writes, once, the ask about a held fingerprint (ADR 0528 clause 6, ADR 0437 clause 10): the first ask that needs
|
||||||
// no proof of who answered, which the view may answer and the teller fills. In release 5 an answer changes nothing on a
|
// no proof of who answered, which the view may answer and the teller fills. In release 5 an answer changes nothing on a
|
||||||
// node (wants are only compared, ADR 0429 §7), and the options say so.
|
// node (wants are only compared, ADR 0429 §7), and the options say so.
|
||||||
func (p *passer) holdAsk(ctx context.Context, module, fp string, fresh *delivery.Held) {
|
func (p *passer) holdAsk(ctx context.Context, module, fp string) {
|
||||||
key, err := delivery.HoldAskKey(module, fp)
|
key, err := delivery.HoldAskKey(module, fp)
|
||||||
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
|
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
// The asking is the hold's time in seconds: a hold of the same fingerprint written again after a lift is another asking, under
|
a := assign.HoldAsk(key, module, fp, time.Now())
|
||||||
// an id of its own, also after a restart, since the router drops a second ask under an id it keeps (issue 1906).
|
|
||||||
now := p.clock()
|
|
||||||
asking := int(now.Unix())
|
|
||||||
h := p.heldHold(module, fp, fresh)
|
|
||||||
a := assign.HoldAsk(key, module, fp, h.Node, h.Words, asking, now)
|
|
||||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
||||||
}
|
}
|
||||||
|
|
||||||
// lift takes away a hold the assigner lifted by itself, since every node that runs the module runs the held fingerprint healthy
|
|
||||||
// (issue 1906): the held record and its ask (the teller then takes the ask back from the operator), and the held condition is
|
|
||||||
// cleared saying why, once. A hold of the same fingerprint written later is said and asked again.
|
|
||||||
func (p *passer) lift(ctx context.Context, module, fp string) {
|
|
||||||
held, err := delivery.HeldKey(module, fp)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
ask, err := delivery.HoldAskKey(module, fp)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
c := assign.LiftedCondition(module, fp)
|
|
||||||
if !p.said[c.Key+".lifted"] {
|
|
||||||
if p.say != nil {
|
|
||||||
if err := p.say(c); err != nil {
|
|
||||||
// The hold stays written, so the next pass lifts it again and says so then.
|
|
||||||
p.logf("the condition of the lifted hold of %s could not be cleared, and is cleared at the next pass: %v", module, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
p.logf("%s", assign.LiftedWords(module, fp))
|
|
||||||
p.said[c.Key+".lifted"] = true
|
|
||||||
delete(p.said, c.Key+".raised") // a hold written again is said again
|
|
||||||
}
|
|
||||||
p.remove(ctx, delivery.AssignerWantsBucket, held)
|
|
||||||
p.remove(ctx, delivery.AssignerAsksBucket, ask)
|
|
||||||
}
|
|
||||||
|
|
||||||
// remove deletes one key of the assigner's own bucket when it holds it; a delete that fails is said and made again at the next
|
|
||||||
// pass.
|
|
||||||
func (p *passer) remove(ctx context.Context, bucket, key string) {
|
|
||||||
if !p.w.has(bucket, key) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err := p.kvs[bucket].Delete(ctx, key); err != nil && !errors.Is(err, context.Canceled) {
|
|
||||||
p.logf("%s %s could not be taken away: %v", bucket, key, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
p.w.put(bucket, key, nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
// heldHold is the hold of fp as written: the node whose health check failed and its words.
|
// heldHold is the hold of fp as written: the node whose health check failed and its words.
|
||||||
func (p *passer) heldHold(module, fp string, fresh *delivery.Held) delivery.Held {
|
func (p *passer) heldHold(module, fp string, fresh *delivery.Held) delivery.Held {
|
||||||
if fresh != nil {
|
if fresh != nil {
|
||||||
@@ -305,7 +245,6 @@ func (p *passer) heldCondition(ctx context.Context, module, fp string, fresh *de
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
p.said[c.Key+".raised"] = true
|
p.said[c.Key+".raised"] = true
|
||||||
delete(p.said, c.Key+".lifted") // a hold raised again and lifted again is said lifted again
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// restoreAsk writes, once, the ask about a module holding data whose held fingerprint failed its health check on node.
|
// restoreAsk writes, once, the ask about a module holding data whose held fingerprint failed its health check on node.
|
||||||
@@ -319,21 +258,11 @@ func (p *passer) restoreAsk(ctx context.Context, module, node, fp string) {
|
|||||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
||||||
}
|
}
|
||||||
|
|
||||||
// answeredWith says whether the teller wrote an answer to the ask at key as it stands now and the option chosen is option. The
|
// answeredWith says whether the teller wrote an answer to the ask at key and the option chosen is option. The answer is the
|
||||||
// answer is the operator's, given in the view or on the phone, and stays under its key (ADR 0539 §14, §16, §19); it counts only
|
// operator's, given in the view or on the phone, and stays under its key (ADR 0539 §14, §16, §19).
|
||||||
// for the asking it answered (its warrant names that asking's id), so an answer to an ask taken away, or to an earlier hold of
|
|
||||||
// the same fingerprint, does not carry over to a later one (issue 1906).
|
|
||||||
func (p *passer) answeredWith(key, option string) bool {
|
func (p *passer) answeredWith(key, option string) bool {
|
||||||
a, ok := p.w.answer(key)
|
a, ok := p.w.answer(key)
|
||||||
if !ok || a.Option != option {
|
return ok && a.Option == option
|
||||||
return false
|
|
||||||
}
|
|
||||||
ask, err := delivery.DecodeAssignerAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
|
|
||||||
if err != nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
id, _, _ := strings.Cut(a.Warrant, "@")
|
|
||||||
return id == ask.Ask.ID
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// clearHeld clears, once, the condition of a hold the operator released.
|
// clearHeld clears, once, the condition of a hold the operator released.
|
||||||
|
|||||||
@@ -1,100 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
||||||
|
|
||||||
"mesh-delivery-assigner/internal/assign"
|
|
||||||
)
|
|
||||||
|
|
||||||
// memKV is the part of a bucket a pass writes: puts and deletes.
|
|
||||||
type memKV struct {
|
|
||||||
jetstream.KeyValue
|
|
||||||
values map[string][]byte
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *memKV) Put(_ context.Context, key string, value []byte) (uint64, error) {
|
|
||||||
m.values[key] = value
|
|
||||||
return uint64(len(m.values)), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *memKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
|
|
||||||
delete(m.values, key)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// newPasser is a pass on a fresh world, as after a start, writing to the buckets given, at the clock given.
|
|
||||||
func newPasser(kvs buckets, at time.Time) *passer {
|
|
||||||
return &passer{kvs: kvs, w: newWorld(), logf: func(string, ...any) {}, said: map[string]bool{}, answeredAt: map[string]uint64{},
|
|
||||||
now: func() time.Time { return at }}
|
|
||||||
}
|
|
||||||
|
|
||||||
var fpTest = delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
|
||||||
|
|
||||||
// 1906/H2: given a hold lifted by itself, when the assigner is started again and the same fingerprint is held again, then the
|
|
||||||
// hold ask is written under another asking, with an id of its own, so the teller raises it (the router drops a second ask under
|
|
||||||
// an id it keeps).
|
|
||||||
func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
|
|
||||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
|
||||||
kvs := buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}
|
|
||||||
key, _ := delivery.HoldAskKey("web", fpTest)
|
|
||||||
held := &delivery.Held{Node: "ace", Words: "health failed"}
|
|
||||||
first := newPasser(kvs, time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC))
|
|
||||||
first.holdAsk(context.Background(), "web", fpTest, held)
|
|
||||||
a1, err := delivery.DecodeAssignerAsk(key, asksKV.values[key])
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
first.lift(context.Background(), "web", fpTest)
|
|
||||||
if _, there := asksKV.values[key]; there {
|
|
||||||
t.Fatal("the lift left the hold ask")
|
|
||||||
}
|
|
||||||
again := newPasser(kvs, time.Date(2026, 10, 25, 2, 0, 0, 0, time.UTC)) // started again: it remembers nothing
|
|
||||||
again.holdAsk(context.Background(), "web", fpTest, held)
|
|
||||||
a2, err := delivery.DecodeAssignerAsk(key, asksKV.values[key])
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if a2.Ask.ID == a1.Ask.ID {
|
|
||||||
t.Errorf("the ask written again after a lift and a restart has the id %s of the first", a1.Ask.ID)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1906/H4: an answer counts only for the asking it answered: an answer to an earlier hold of the same fingerprint, or to an ask
|
|
||||||
// taken away, is not taken for the hold that stands now.
|
|
||||||
func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
|
|
||||||
key, _ := delivery.HoldAskKey("web", fpTest)
|
|
||||||
p := newPasser(buckets{}, time.Now())
|
|
||||||
put := func(asking int) string {
|
|
||||||
a := assign.HoldAsk(key, "web", fpTest, "ace", "health failed", asking, time.Now())
|
|
||||||
v, err := delivery.EncodeAssignerAsk(key, a)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
p.w.put(delivery.AssignerAsksBucket, key, v)
|
|
||||||
return a.Ask.ID
|
|
||||||
}
|
|
||||||
first := put(100)
|
|
||||||
v, _ := delivery.EncodeAnswer(delivery.Answer{Option: "release", Warrant: first + "@2026-10-25T01:00:00Z", Words: "the operator"})
|
|
||||||
p.w.put(delivery.AnswersBucket, key, v)
|
|
||||||
if !p.answeredWith(key, "release") {
|
|
||||||
t.Fatal("the answer to the ask that stands is not taken")
|
|
||||||
}
|
|
||||||
if p.answeredWith(key, "keep") {
|
|
||||||
t.Error("an answer was taken for an option not chosen")
|
|
||||||
}
|
|
||||||
put(200) // the same fingerprint held again, after a lift
|
|
||||||
if p.answeredWith(key, "release") {
|
|
||||||
t.Error("the answer to an earlier hold carried over to a later one")
|
|
||||||
}
|
|
||||||
p.w.put(delivery.AssignerAsksBucket, key, nil) // the ask taken away
|
|
||||||
if p.answeredWith(key, "release") {
|
|
||||||
t.Error("an answer to an ask taken away was taken")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -326,13 +326,6 @@ func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[stri
|
|||||||
return n
|
return n
|
||||||
}
|
}
|
||||||
|
|
||||||
// value is what a bucket holds at a key now, nil when nothing.
|
|
||||||
func (w *world) value(bucket, key string) []byte {
|
|
||||||
w.mu.Lock()
|
|
||||||
defer w.mu.Unlock()
|
|
||||||
return w.values[bucket][key]
|
|
||||||
}
|
|
||||||
|
|
||||||
// has says whether a bucket holds a key now.
|
// has says whether a bucket holds a key now.
|
||||||
func (w *world) has(bucket, key string) bool {
|
func (w *world) has(bucket, key string) bool {
|
||||||
w.mu.Lock()
|
w.mu.Lock()
|
||||||
|
|||||||
@@ -10,17 +10,14 @@ import (
|
|||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey): keep it held or
|
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey), first asking: keep
|
||||||
// release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19). node and words are the held
|
// it held or release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19).
|
||||||
// record's: the machine whose health check failed and the hold's words (issue 1906). asking numbers the writing: 0 the first, a
|
func HoldAsk(key, module, fp string, now time.Time) delivery.AssignerAsk {
|
||||||
// larger number an ask written again after a hold lifted by itself, so the router takes it as new.
|
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||||
func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) delivery.AssignerAsk {
|
ID: delivery.AskingID(key, 0),
|
||||||
return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
|
|
||||||
ID: delivery.AskingID(key, asking),
|
|
||||||
Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
|
Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
|
||||||
Explanation: fmt.Sprintf("The health check of %s failed on %s at %s (%s), so the assigner holds that version and wants "+
|
Explanation: fmt.Sprintf("A node's health check of %s failed at %s, so the assigner holds that version and wants "+
|
||||||
"no node to move to it. It lifts the hold by itself once every node that runs %s runs it healthy; releasing it "+
|
"no node to move to it. Releasing it lets the wants move to it again.", module, short(fp)),
|
||||||
"lets the wants move to it now.", module, node, short(fp), words, module),
|
|
||||||
Options: []asks.Option{
|
Options: []asks.Option{
|
||||||
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge},
|
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge},
|
||||||
{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: asks.Acknowledge},
|
{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: asks.Acknowledge},
|
||||||
@@ -81,13 +78,6 @@ func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.C
|
|||||||
return c
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
|
|
||||||
func LiftedCondition(module, fp string) asks.Condition {
|
|
||||||
c := HeldCondition(module, fp, "", "", false, false)
|
|
||||||
c.Resolved = LiftedWords(module, fp) + "."
|
|
||||||
return c
|
|
||||||
}
|
|
||||||
|
|
||||||
// The options of the record ask (ADR 0539 §14) and of the backup ask (§16).
|
// The options of the record ask (ADR 0539 §14) and of the backup ask (§16).
|
||||||
const (
|
const (
|
||||||
OptionYes = "yes"
|
OptionYes = "yes"
|
||||||
|
|||||||
@@ -17,25 +17,13 @@ func TestTheHoldAskIsValid(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
a := HoldAsk(key, "mail", fNew, "ace", "health failed, phase running: connection refused", 0, now)
|
a := HoldAsk(key, "mail", fNew, now)
|
||||||
if err := a.Ask.Check(now); err != nil {
|
if err := a.Ask.Check(now); err != nil {
|
||||||
t.Errorf("the router refuses the ask: %v", err)
|
t.Errorf("the router refuses the ask: %v", err)
|
||||||
}
|
}
|
||||||
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
|
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
|
||||||
t.Errorf("the ask cannot be written: %v", err)
|
t.Errorf("the ask cannot be written: %v", err)
|
||||||
}
|
}
|
||||||
// 1906/H3: the ask names the machine whose check failed and the hold's words.
|
|
||||||
if !strings.Contains(a.Ask.Explanation, "on ace") || !strings.Contains(a.Ask.Explanation, "connection refused") {
|
|
||||||
t.Errorf("the ask does not name the node and the hold's words: %q", a.Ask.Explanation)
|
|
||||||
}
|
|
||||||
// 1906/H2: an ask written again after a lift is another asking, under an id of its own.
|
|
||||||
again := HoldAsk(key, "mail", fNew, "ace", "x", 7, now)
|
|
||||||
if again.Asking != 7 || again.Ask.ID != delivery.AskingID(key, 7) || again.Ask.ID == a.Ask.ID {
|
|
||||||
t.Errorf("the asking after a lift = %d %s", again.Asking, again.Ask.ID)
|
|
||||||
}
|
|
||||||
if _, err := delivery.EncodeAssignerAsk(key, again); err != nil {
|
|
||||||
t.Errorf("the ask asked again cannot be written: %v", err)
|
|
||||||
}
|
|
||||||
if _, err := delivery.ViewAnswerKey(key); err != nil {
|
if _, err := delivery.ViewAnswerKey(key); err != nil {
|
||||||
t.Errorf("the view may not answer it: %v", err)
|
t.Errorf("the view may not answer it: %v", err)
|
||||||
}
|
}
|
||||||
@@ -86,17 +74,6 @@ func TestTheHeldConditionNamesModuleNodeAndWords(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1906/H2: a hold lifted by itself clears the held condition under its key, saying why.
|
|
||||||
func TestALiftedHoldClearsItsConditionSayingWhy(t *testing.T) {
|
|
||||||
c := LiftedCondition("mail", fNew)
|
|
||||||
if _, err := asks.EncodeCondition(c); err != nil {
|
|
||||||
t.Fatalf("the condition is refused: %v", err)
|
|
||||||
}
|
|
||||||
if c.State != asks.StateCleared || c.Key != HeldCondition("mail", fNew, "ace", "x", true, false).Key || !strings.Contains(c.Resolved, "lifted its hold by itself") {
|
|
||||||
t.Errorf("condition = %+v", c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1906/D4: the record ask and the backup ask are valid asks of the router and of mesh-sdk's layout at their keys, answerable in
|
// 1906/D4: the record ask and the backup ask are valid asks of the router and of mesh-sdk's layout at their keys, answerable in
|
||||||
// the view, and name the move and the node.
|
// the view, and name the move and the node.
|
||||||
func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
||||||
|
|||||||
@@ -4,8 +4,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"slices"
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
@@ -33,10 +31,6 @@ type Module struct {
|
|||||||
// Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back
|
// Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back
|
||||||
// as any module is, losing the writes made since (ADR 0539 §19). Nil: none did.
|
// as any module is, losing the writes made since (ADR 0539 §19). Nil: none did.
|
||||||
Restored func(node string) bool
|
Restored func(node string) bool
|
||||||
// Kept says the operator answered "keep" to the hold ask of FP: a hold kept so is not lifted by itself.
|
|
||||||
Kept bool
|
|
||||||
// Now is when this pass reads the haves: the held record names it as when the health check was seen to fail. Zero: not said.
|
|
||||||
Now time.Time
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
|
func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
|
||||||
@@ -144,9 +138,6 @@ type Outcome struct {
|
|||||||
Wants map[string]delivery.Want
|
Wants map[string]delivery.Want
|
||||||
// Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet.
|
// Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet.
|
||||||
Held *delivery.Held
|
Held *delivery.Held
|
||||||
// Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held
|
|
||||||
// record and its ask are taken away, and the wants move by the rules again in this pass.
|
|
||||||
Lifted bool
|
|
||||||
// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
|
// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
|
||||||
Record []RecordNeed
|
Record []RecordNeed
|
||||||
// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
|
// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
|
||||||
@@ -179,13 +170,10 @@ type Holding struct {
|
|||||||
func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[string]string) Outcome {
|
func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[string]string) Outcome {
|
||||||
out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}}
|
out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}}
|
||||||
held := m.Held
|
held := m.Held
|
||||||
if held && !m.Kept && m.FP != "" && allRunHealthy(m, n) {
|
if !held && !m.Released && m.FP != "" {
|
||||||
out.Lifted, held = true, false
|
|
||||||
}
|
|
||||||
if !held && !out.Lifted && !m.Released && m.FP != "" {
|
|
||||||
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
|
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
|
||||||
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && h.Health == delivery.HealthFailed {
|
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && h.Health == delivery.HealthFailed {
|
||||||
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
|
out.Held = &delivery.Held{Node: node, Words: h.Detail}
|
||||||
held = true
|
held = true
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
@@ -264,43 +252,6 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// allRunHealthy says whether every node of m that runs the module (has a have of it) runs m.FP healthy, and at least one does:
|
|
||||||
// a hold whose fingerprint runs healthy wherever the module runs holds nothing back any more (issue 1906: seven modules stayed
|
|
||||||
// held on every machine after one transient failed check). A node set back to another fingerprint does not run m.FP, so its
|
|
||||||
// hold stays.
|
|
||||||
func allRunHealthy(m Module, n Nodes) bool {
|
|
||||||
ran := 0
|
|
||||||
for _, node := range m.Nodes {
|
|
||||||
h, ok := n.Haves[node][m.Name]
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !healthy(h, m.FP) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
ran++
|
|
||||||
}
|
|
||||||
return ran > 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// HeldWords are the words a hold keeps of its cause (issue 1906): the health and phase the node reported, when the assigner saw
|
|
||||||
// it, and the node's own words. The node is the held record's own field.
|
|
||||||
func HeldWords(h delivery.Have, at time.Time) string {
|
|
||||||
words := fmt.Sprintf("health %s, phase %s", h.Health, h.Phase)
|
|
||||||
if !at.IsZero() {
|
|
||||||
words += ", seen at " + at.UTC().Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
if d := strings.TrimSpace(h.Detail); d != "" {
|
|
||||||
return words + ": " + d
|
|
||||||
}
|
|
||||||
return words + ": the node gave no words"
|
|
||||||
}
|
|
||||||
|
|
||||||
// LiftedWords say why a hold was lifted by itself.
|
|
||||||
func LiftedWords(module, fp string) string {
|
|
||||||
return fmt.Sprintf("every node that runs %s runs %s healthy, so the assigner lifted its hold by itself", module, short(fp))
|
|
||||||
}
|
|
||||||
|
|
||||||
// normal is a want without its version, as a decoded and a composed one are compared.
|
// normal is a want without its version, as a decoded and a composed one are compared.
|
||||||
func normal(w delivery.Want) delivery.Want {
|
func normal(w delivery.Want) delivery.Want {
|
||||||
w.V = 0
|
w.V = 0
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
@@ -111,10 +110,8 @@ func TestAnOfflineCanaryIsPassedOver(t *testing.T) {
|
|||||||
func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
|
func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
|
||||||
m, n := ready()
|
m, n := ready()
|
||||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"}
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"}
|
||||||
m.Now = time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC)
|
|
||||||
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld})
|
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld})
|
||||||
// 1906/H3: the held record keeps its cause in words: which machine (its node), which health, when, and the node's words.
|
if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "connection refused" {
|
||||||
if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "health failed, phase running, seen at 2026-10-25T01:00:00Z: connection refused" {
|
|
||||||
t.Fatalf("held = %+v", out.Held)
|
t.Fatalf("held = %+v", out.Held)
|
||||||
}
|
}
|
||||||
if w := out.Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != delivery.RuleNotHeld {
|
if w := out.Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != delivery.RuleNotHeld {
|
||||||
@@ -523,59 +520,3 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1906/H3: a failed check whose node gave no words is held saying so, never with empty words.
|
|
||||||
func TestAHoldOfAFailureWithoutWordsSaysSo(t *testing.T) {
|
|
||||||
m, n := ready()
|
|
||||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
|
|
||||||
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}}, map[string]string{"ace": fOld})
|
|
||||||
if out.Held == nil || out.Held.Words != "health failed, phase running: the node gave no words" {
|
|
||||||
t.Fatalf("held = %+v", out.Held)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1906/H2: given a held fingerprint that every node running the module now runs healthy, when the assigner passes, then the
|
|
||||||
// hold is lifted by itself and the wants move to it by the rules again. A node set back to another fingerprint, a node whose
|
|
||||||
// check fails, or a hold the operator answered "keep" leaves it held; a node that does not run the module at all does not count.
|
|
||||||
func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
|
|
||||||
everywhere := func() (Module, Nodes) {
|
|
||||||
m, n := ready()
|
|
||||||
m.Held = true
|
|
||||||
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
||||||
return m, n
|
|
||||||
}
|
|
||||||
m, n := everywhere()
|
|
||||||
out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld})
|
|
||||||
if !out.Lifted || out.Held != nil {
|
|
||||||
t.Fatalf("lifted = %v, held = %+v: want the hold lifted", out.Lifted, out.Held)
|
|
||||||
}
|
|
||||||
for _, node := range []string{"ace", "g14"} {
|
|
||||||
if w := out.Wants[node]; w.FP != fNew || w.Rule != 0 {
|
|
||||||
t.Errorf("%s's want = %+v, want it moved to the lifted fingerprint", node, w)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
m, n = everywhere()
|
|
||||||
delete(n.Haves["g14"], "mail")
|
|
||||||
if out := Wants(m, n, wantedOld(), nil); !out.Lifted {
|
|
||||||
t.Error("a node that does not run the module kept the hold")
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, change := range map[string]func(*Module, *Nodes){
|
|
||||||
"set back": func(_ *Module, n *Nodes) {
|
|
||||||
n.Haves["g14"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
||||||
},
|
|
||||||
"failing": func(_ *Module, n *Nodes) {
|
|
||||||
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
|
|
||||||
},
|
|
||||||
"kept": func(m *Module, _ *Nodes) { m.Kept = true },
|
|
||||||
"no haves": func(_ *Module, n *Nodes) { delete(n.Haves["g14"], "mail"); delete(n.Haves["ace"], "mail") },
|
|
||||||
"not held": func(m *Module, _ *Nodes) { m.Held = false },
|
|
||||||
} {
|
|
||||||
m, n := everywhere()
|
|
||||||
change(&m, &n)
|
|
||||||
if out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld}); out.Lifted {
|
|
||||||
t.Errorf("%s: the hold was lifted", name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ module mesh-delivery-checker
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,5 +1,11 @@
|
|||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93 h1:FAiFY2KBKEoZSH49OZTt7/zRBIQxyl5g/KhYrpggbN4=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024214154-bc16740feb0d h1:oJREw0ggKqJWJ5DEISSL0i2ZC5XqdXAidxo9NJ74TCc=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024214154-bc16740feb0d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024231716-571ac521f053 h1:8Kbp3aJG3jGndDOfmGdMSDgJVEcc1hyu921hVnnlnlQ=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024231716-571ac521f053/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232250-9b408db24a58 h1:4Damtdqlf6u6nJhvMA99peDy1jmjixtcwFXJ4bWEkLk=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232250-9b408db24a58/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95 h1:I8zbgGzRjmgYydPTWo8hmaZW/P/g+qGDUfu3E32/pkU=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||||
|
|||||||
@@ -0,0 +1,96 @@
|
|||||||
|
package check
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// countingAsks puts asks through inner, and answers no wait until want asks are put (or a bound passes, when it says
|
||||||
|
// how many were): a group whose asks are put one after another, each waited for, never gets past its first.
|
||||||
|
type countingAsks struct {
|
||||||
|
inner Asks
|
||||||
|
want int
|
||||||
|
mu sync.Mutex
|
||||||
|
puts int
|
||||||
|
all chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *countingAsks) Put(ctx context.Context, a delivery.StepAsk) error {
|
||||||
|
if err := c.inner.Put(ctx, a); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
c.puts++
|
||||||
|
if c.puts == c.want {
|
||||||
|
close(c.all)
|
||||||
|
}
|
||||||
|
c.mu.Unlock()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *countingAsks) Wait(ctx context.Context, a delivery.StepAsk) (delivery.Step, error) {
|
||||||
|
select {
|
||||||
|
case <-c.all:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
return delivery.Step{}, fmt.Errorf("%d of %d asks were put before the first was waited for", c.puts, c.want)
|
||||||
|
}
|
||||||
|
return c.inner.Wait(ctx, a)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1921/S8: a group's asks are all put before any is waited for, so its members run side by side on the build nodes;
|
||||||
|
// the verdict is judged in member order as before.
|
||||||
|
func TestAGroupPutsEveryMembersAskBeforeTheFirstAnswer(t *testing.T) {
|
||||||
|
b := newBench(t)
|
||||||
|
counting := &countingAsks{inner: b.ex.Asks, want: 3, all: make(chan struct{})}
|
||||||
|
b.ex.Asks = counting
|
||||||
|
cs := caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2), pullOf("novox/mesh-controller", 3, head1))
|
||||||
|
v := b.judge(t, cs).Verdict
|
||||||
|
if v.State != delivery.VerdictPass {
|
||||||
|
t.Fatalf("verdict = %+v", v)
|
||||||
|
}
|
||||||
|
if counting.puts != 3 {
|
||||||
|
t.Errorf("%d asks were put, want 3", counting.puts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1921/S10: the levels of an ask run side by side, so what of the ask's time is its fetch (Took.Clone) is the ask's time
|
||||||
|
// less its longest level, not less the sum of its levels.
|
||||||
|
func TestTheCloneTimeTakesTheLongestLevelOffNotTheSum(t *testing.T) {
|
||||||
|
b := newBench(t)
|
||||||
|
b.test = func(c stepCall) (any, error) {
|
||||||
|
r := captured[Tested](t, "go-test-slow")
|
||||||
|
r.Level, _ = c.args["level"].(string)
|
||||||
|
r.Elapsed, r.Setup, r.Lint = 1.5, 0, 0 // the ask's step takes 2 s (bench.answer)
|
||||||
|
return r, nil
|
||||||
|
}
|
||||||
|
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
|
||||||
|
if v.State != delivery.VerdictPass || v.Took.Clone != 500 {
|
||||||
|
t.Errorf("clone = %d ms, want 500 (2 s less the longest level's 1.5 s): %+v", v.Took.Clone, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ADR 0542 §2: a lint finding, which the Go build module marks as the lint step's, is reported as the lint step's failure,
|
||||||
|
// not as the unit level's.
|
||||||
|
func TestALintFindingIsTheLintStepsFailure(t *testing.T) {
|
||||||
|
b := newBench(t)
|
||||||
|
b.test = func(c stepCall) (any, error) {
|
||||||
|
r := captured[Tested](t, "go-test-slow")
|
||||||
|
r.Level, _ = c.args["level"].(string)
|
||||||
|
if r.Level == delivery.LevelUnit {
|
||||||
|
r.Passed = false
|
||||||
|
r.Failed = []Failure{{Package: ".", Test: "gofmt", Step: "lint", Output: "not as gofmt writes them (run gofmt -w): a.go"}}
|
||||||
|
}
|
||||||
|
return r, nil
|
||||||
|
}
|
||||||
|
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
|
||||||
|
if v.State != delivery.VerdictFail || v.Failed.Test != "lint: ..gofmt" {
|
||||||
|
t.Errorf("failed = %+v, want the lint step's failure", v.Failed)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -82,6 +82,9 @@ func CaseOf(c delivery.Candidate, trunk map[string]string) *Case {
|
|||||||
}
|
}
|
||||||
cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}}
|
cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}}
|
||||||
for _, p := range c.Pulls {
|
for _, p := range c.Pulls {
|
||||||
|
if p.Merged {
|
||||||
|
continue // merged while the group waits for its other members: judged by the verdict the group had (issue 1791)
|
||||||
|
}
|
||||||
switch head, known := trunk[p.Repository]; {
|
switch head, known := trunk[p.Repository]; {
|
||||||
case !known || head == "":
|
case !known || head == "":
|
||||||
cs.Left = append(cs.Left, fmt.Sprintf("%s#%d: the trunk head of %s is not known", p.Repository, p.Number, p.Repository))
|
cs.Left = append(cs.Left, fmt.Sprintf("%s#%d: the trunk head of %s is not known", p.Repository, p.Number, p.Repository))
|
||||||
@@ -119,7 +122,18 @@ func GateHeads(c delivery.Candidate) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// sameTrunk says whether two verdicts' trunk heads are the same.
|
// sameTrunk says whether two verdicts' trunk heads are the same.
|
||||||
func sameTrunk(a, b map[string]string) bool { return maps.Equal(a, b) }
|
// sameTrunk says whether a verdict judged on the trunk heads judged stands for a case on the trunk heads now: every
|
||||||
|
// repository of the case at the head it was judged on. A repository the verdict names and the case does not is a member
|
||||||
|
// that merged since (issue 1791): its own merge moved that trunk, and the members left are judged by the verdict the
|
||||||
|
// group had.
|
||||||
|
func sameTrunk(judged, now map[string]string) bool {
|
||||||
|
for repo, head := range now {
|
||||||
|
if judged[repo] != head {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(now) > 0 || len(judged) == 0
|
||||||
|
}
|
||||||
|
|
||||||
// Trunks holds the trunk head of each repository, as the gate's source.main says (a watch puts it in).
|
// Trunks holds the trunk head of each repository, as the gate's source.main says (a watch puts it in).
|
||||||
type Trunks struct {
|
type Trunks struct {
|
||||||
|
|||||||
@@ -83,12 +83,24 @@ type Checker struct {
|
|||||||
tried map[string]bool // by verdict key: looked at since this process started
|
tried map[string]bool // by verdict key: looked at since this process started
|
||||||
epoch uint64 // changes of the nodes seen since this process started
|
epoch uint64 // changes of the nodes seen since this process started
|
||||||
nodesFail map[string]uint64 // by verdict key: the epoch a failure of the nodes was written at
|
nodesFail map[string]uint64 // by verdict key: the epoch a failure of the nodes was written at
|
||||||
|
// current is, by candidates key, the verdict key of the heads the group holds now; running holds, by candidates key,
|
||||||
|
// the stop of each check that runs, by its verdict key (issue 1791; issue 1921, try speed S9).
|
||||||
|
current map[string]string
|
||||||
|
running map[string]map[string]context.CancelCauseFunc
|
||||||
|
}
|
||||||
|
|
||||||
|
// superseded is why a check whose heads the group no longer holds was stopped: the heads it holds now.
|
||||||
|
type superseded struct{ by string }
|
||||||
|
|
||||||
|
func (s superseded) Error() string {
|
||||||
|
return "superseded: the group holds other heads now (" + s.by + "), whose check judges it"
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ch *Checker) init() {
|
func (ch *Checker) init() {
|
||||||
ch.once.Do(func() {
|
ch.once.Do(func() {
|
||||||
ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
|
ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
|
||||||
ch.nodesFail = map[string]uint64{}
|
ch.nodesFail = map[string]uint64{}
|
||||||
|
ch.current, ch.running = map[string]string{}, map[string]map[string]context.CancelCauseFunc{}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -126,6 +138,15 @@ func (ch *Checker) Candidate(ctx context.Context, key string, value []byte) {
|
|||||||
}
|
}
|
||||||
ch.mu.Lock()
|
ch.mu.Lock()
|
||||||
ch.known[key] = c
|
ch.known[key] = c
|
||||||
|
if vk, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash); err == nil {
|
||||||
|
ch.current[key] = vk
|
||||||
|
// A check of heads the group no longer holds is stopped: its asks are not put, and it ends superseded.
|
||||||
|
for running, stop := range ch.running[key] {
|
||||||
|
if running != vk {
|
||||||
|
stop(superseded{by: c.HeadsHash})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
ch.mu.Unlock()
|
ch.mu.Unlock()
|
||||||
ch.consider(ctx, key, c)
|
ch.consider(ctx, key, c)
|
||||||
}
|
}
|
||||||
@@ -329,12 +350,48 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) {
|
|||||||
if cl == nil {
|
if cl == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
run, stop := context.WithCancelCause(ctx)
|
||||||
|
defer stop(nil)
|
||||||
|
ch.mu.Lock()
|
||||||
|
now := ch.current[key]
|
||||||
|
if ch.running[key] == nil {
|
||||||
|
ch.running[key] = map[string]context.CancelCauseFunc{}
|
||||||
|
}
|
||||||
|
ch.running[key][cs.Key] = stop
|
||||||
|
ch.mu.Unlock()
|
||||||
|
defer func() {
|
||||||
|
ch.mu.Lock()
|
||||||
|
delete(ch.running[key], cs.Key)
|
||||||
|
if len(ch.running[key]) == 0 {
|
||||||
|
delete(ch.running, key)
|
||||||
|
}
|
||||||
|
ch.mu.Unlock()
|
||||||
|
}()
|
||||||
|
if now != "" && now != cs.Key {
|
||||||
|
stop(superseded{by: now}) // the group moved on before the check began
|
||||||
|
}
|
||||||
began := ch.now()
|
began := ch.now()
|
||||||
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
if run.Err() == nil {
|
||||||
out := ch.judge(ctx, cs)
|
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
||||||
|
}
|
||||||
|
out := ch.judge(run, cs)
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
var sup superseded
|
||||||
|
if errors.As(context.Cause(run), &sup) {
|
||||||
|
// Stopped for newer heads: the verdict ends failed, not running, and no status is put, as the newer check owns the
|
||||||
|
// status of the heads they share.
|
||||||
|
v := delivery.Verdict{Group: cs.Group, Heads: cs.Heads, Trunk: cs.Trunk, State: delivery.VerdictFail,
|
||||||
|
Failed: delivery.FailedFirst{Machine: "checker", Words: sup.Error()}}
|
||||||
|
if data, err := delivery.EncodeVerdict(v); err == nil {
|
||||||
|
if _, err := ch.Verdicts.Update(ctx, cs.Key, data, cl.rev); err != nil {
|
||||||
|
ch.logf("%s: the superseded verdict could not be written: %v", cs.Key, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ch.logf("%s: %s", cs.Key, sup.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
v := out.Verdict
|
v := out.Verdict
|
||||||
v.Group, v.Heads, v.Trunk = cs.Group, cs.Heads, cs.Trunk
|
v.Group, v.Heads, v.Trunk = cs.Group, cs.Heads, cs.Trunk
|
||||||
if v.Took.Queue == 0 && !cs.Queued.IsZero() && began.After(cs.Queued) {
|
if v.Took.Queue == 0 && !cs.Queued.IsZero() && began.After(cs.Queued) {
|
||||||
|
|||||||
@@ -0,0 +1,90 @@
|
|||||||
|
package check
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// failingResult is a level's test result in which the named tests failed, each in its own package.
|
||||||
|
func failingResult(t *testing.T, tests int, failed ...Failure) json.RawMessage {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := json.Marshal(Tested{Level: delivery.LevelIntegration, Tests: tests, Elapsed: 2, Failed: failed, Passed: len(failed) == 0})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
func judgeLevel(t *testing.T, tests ...json.RawMessage) (*delivery.Verdict, *state) {
|
||||||
|
t.Helper()
|
||||||
|
st := &state{j: &Examiner{}, c: &Case{}, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
|
||||||
|
v, _ := st.level(t.Context(), &target{pull: Pull{Repository: "novox/x"}}, delivery.LevelAnswer{Level: delivery.LevelIntegration, Tests: tests})
|
||||||
|
return v, st
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue 1921: two failing tests in two packages are two entries of the verdict's Failures and two lines of a try's
|
||||||
|
// failed list, each with its own first line (its cause, issue 1901), whether the build module answered them in one test
|
||||||
|
// result (one go test run over both packages) or in two (the changed packages first, then the rest); every result is
|
||||||
|
// counted, the ones after a failure too; and Failed keeps the first one's words whole.
|
||||||
|
func TestTwoFailingTestsInTwoPackagesAreTwoEntries(t *testing.T) {
|
||||||
|
a := Failure{Package: "x/a", Test: "TestA", Cause: "a_test.go:9: want 1, got 2", Output: " a_test.go:3: logged\n a_test.go:9: want 1, got 2\n--- FAIL: TestA (0.00s)"}
|
||||||
|
b := Failure{Package: "x/b", Test: "TestB", Cause: "b_test.go:4: boom", Output: " b_test.go:4: boom\n--- FAIL: TestB (0.00s)"}
|
||||||
|
for name, tests := range map[string][]json.RawMessage{
|
||||||
|
"one result": {failingResult(t, 10, a, b)},
|
||||||
|
"two results": {failingResult(t, 4, a), failingResult(t, 3), failingResult(t, 3, b)},
|
||||||
|
} {
|
||||||
|
v, st := judgeLevel(t, tests...)
|
||||||
|
if v == nil {
|
||||||
|
t.Fatalf("%s: the level passed", name)
|
||||||
|
}
|
||||||
|
want := []string{"integration: x/a.TestA: a_test.go:9: want 1, got 2", "integration: x/b.TestB: b_test.go:4: boom"}
|
||||||
|
if got := failedLines(*v); strings.Join(got, "\n") != strings.Join(want, "\n") {
|
||||||
|
t.Errorf("%s: failed lines\n%s\nwant\n%s", name, strings.Join(got, "\n"), strings.Join(want, "\n"))
|
||||||
|
}
|
||||||
|
if len(v.Failures) != 2 || v.Failed.Test != "integration: x/a.TestA" || !strings.Contains(v.Failed.Words, "a_test.go:3: logged") {
|
||||||
|
t.Errorf("%s: verdict %+v", name, v)
|
||||||
|
}
|
||||||
|
if run := st.levels[delivery.LevelIntegration].run; run != 10 {
|
||||||
|
t.Errorf("%s: %d tests counted, want all 10", name, run)
|
||||||
|
}
|
||||||
|
if _, err := delivery.EncodeVerdict(delivery.Verdict{Group: "g", Heads: "0123456789ab", State: v.State, Failed: v.Failed, Failures: v.Failures}); err != nil {
|
||||||
|
t.Errorf("%s: the verdict is not written: %v", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue 1921: past delivery.MaxFailures the list names the first fifty and how many more, the status line says how many
|
||||||
|
// failed, and each entry's line is cut to MaxFailureLine.
|
||||||
|
func TestFailuresAreCappedWithHowManyMore(t *testing.T) {
|
||||||
|
var fs []Failure
|
||||||
|
for i := range 60 {
|
||||||
|
fs = append(fs, Failure{Package: fmt.Sprintf("x/p%d", i), Test: "TestT", Cause: strings.Repeat("y", 500)})
|
||||||
|
}
|
||||||
|
v, _ := judgeLevel(t, failingResult(t, 60, fs...))
|
||||||
|
if len(v.Failures) != delivery.MaxFailures+1 || v.Failures[delivery.MaxFailures].Test != "and 10 more" {
|
||||||
|
t.Fatalf("%d entries, last %+v", len(v.Failures), v.Failures[len(v.Failures)-1])
|
||||||
|
}
|
||||||
|
if len(v.Failures[0].Words) > MaxFailureLine {
|
||||||
|
t.Errorf("an entry's line is %d bytes", len(v.Failures[0].Words))
|
||||||
|
}
|
||||||
|
if n := failedCount(*v); n != 60 {
|
||||||
|
t.Errorf("failedCount = %d, want 60", n)
|
||||||
|
}
|
||||||
|
cs := &Case{Pulls: []Pull{{Repository: "novox/x", Head: "0123456789abcdef", Trunk: "fedcba9876543210"}}}
|
||||||
|
if line := Line(*v, cs, cs.Pulls[0], nil, nil, nil); !strings.HasPrefix(line, "failed (60 tests): integration: x/p0.TestT") {
|
||||||
|
t.Errorf("line %q", line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A verdict that names no list (a conflict, a refused step, a verdict written before issue 1921) is one failed line, as
|
||||||
|
// before.
|
||||||
|
func TestAVerdictWithoutFailuresIsOneLine(t *testing.T) {
|
||||||
|
v := delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "g14", Words: "files in conflict: go.mod\nmore"}}
|
||||||
|
if got := failedLines(v); len(got) != 1 || got[0] != "g14: files in conflict: go.mod" {
|
||||||
|
t.Fatalf("%q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -74,6 +74,18 @@ func (f *fakeSteps) tool(name string) []stepCall {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// toolAt is tool's calls of one level: a check's levels run side by side (issue 1921, try speed S10), so only the calls of
|
||||||
|
// one level keep an order.
|
||||||
|
func (f *fakeSteps) toolAt(name, level string) []stepCall {
|
||||||
|
var out []stepCall
|
||||||
|
for _, c := range f.tool(name) {
|
||||||
|
if c.args["level"] == level {
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
type entries []delivery.NodeToolchain
|
type entries []delivery.NodeToolchain
|
||||||
|
|
||||||
func (e entries) Entries(language string) []delivery.NodeToolchain {
|
func (e entries) Entries(language string) []delivery.NodeToolchain {
|
||||||
@@ -269,9 +281,9 @@ func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
||||||
tests := b.steps.tool("test")
|
tests := b.steps.toolAt("test", "unit")
|
||||||
if len(tests) != 2 || strings.Join(packagesOf(tests[0]), ",") != "fixture/slow" || tests[0].args["level"] != "unit" {
|
if len(tests) != 1 || strings.Join(packagesOf(tests[0]), ",") != "fixture/slow" || len(b.steps.toolAt("test", "integration")) != 1 {
|
||||||
t.Fatalf("test calls = %+v, want the unit level of fixture/slow alone, then the integration level", tests)
|
t.Fatalf("test calls = %+v, want the unit level of fixture/slow alone, and the integration level", b.steps.tool("test"))
|
||||||
}
|
}
|
||||||
if u := out.Verdict.Levels[delivery.LevelUnit]; u.Run != 2 || u.Kept != 40 {
|
if u := out.Verdict.Levels[delivery.LevelUnit]; u.Run != 2 || u.Kept != 40 {
|
||||||
t.Errorf("unit = %+v, want 2 run and 40 kept", u)
|
t.Errorf("unit = %+v, want 2 run and 40 kept", u)
|
||||||
@@ -302,7 +314,7 @@ func TestAReAskRunsOnlyWhatFailedAndWhatChanged(t *testing.T) { // 1288/AC2, AC3
|
|||||||
// Asked again with nothing changed: only the failed package runs; the good one is taken.
|
// Asked again with nothing changed: only the failed package runs; the good one is taken.
|
||||||
b.steps.calls = nil
|
b.steps.calls = nil
|
||||||
b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
||||||
tests := b.steps.tool("test")
|
tests := b.steps.toolAt("test", "unit")
|
||||||
if len(tests) == 0 || strings.Join(packagesOf(tests[0]), ",") != "fixture/failing/bad" {
|
if len(tests) == 0 || strings.Join(packagesOf(tests[0]), ",") != "fixture/failing/bad" {
|
||||||
t.Fatalf("the re-ask ran %+v, want only fixture/failing/bad", tests)
|
t.Fatalf("the re-ask ran %+v, want only fixture/failing/bad", tests)
|
||||||
}
|
}
|
||||||
@@ -310,7 +322,7 @@ func TestAReAskRunsOnlyWhatFailedAndWhatChanged(t *testing.T) { // 1288/AC2, AC3
|
|||||||
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"fixture/failing/bad": digA, "fixture/failing/good": digD}}
|
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"fixture/failing/bad": digA, "fixture/failing/good": digD}}
|
||||||
b.steps.calls = nil
|
b.steps.calls = nil
|
||||||
b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
||||||
tests = b.steps.tool("test")
|
tests = b.steps.toolAt("test", "unit")
|
||||||
if got := strings.Join(packagesOf(tests[0]), ","); got != "fixture/failing/bad,fixture/failing/good" && got != "fixture/failing/good,fixture/failing/bad" {
|
if got := strings.Join(packagesOf(tests[0]), ","); got != "fixture/failing/bad,fixture/failing/good" && got != "fixture/failing/good,fixture/failing/bad" {
|
||||||
t.Errorf("after a digest changed the run was %q, want the failed package and the changed one", got)
|
t.Errorf("after a digest changed the run was %q, want the failed package and the changed one", got)
|
||||||
}
|
}
|
||||||
@@ -341,7 +353,9 @@ func TestANodePackageIsNeverKeptAndRunsEveryCheck(t *testing.T) { // 1288/AC4, 1
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTheChangesOwnFailingTestRunsFirstAndNothingElseRuns(t *testing.T) { // 1289/AC1, AC2
|
// 1289/AC1, AC2 as issue 1921 changed it: the change's own failing test runs first, the level's other packages still run
|
||||||
|
// after it (so one check names every failure); the later level ran beside it (S10) and is not judged.
|
||||||
|
func TestTheChangesOwnFailingTestRunsFirstThenTheRestAndNoLaterLevel(t *testing.T) {
|
||||||
b := newBench(t)
|
b := newBench(t)
|
||||||
b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"}
|
b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"}
|
||||||
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}}
|
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}}
|
||||||
@@ -356,12 +370,12 @@ func TestTheChangesOwnFailingTestRunsFirstAndNothingElseRuns(t *testing.T) { //
|
|||||||
if out.Verdict.State != delivery.VerdictFail || !strings.Contains(out.Verdict.Failed.Test, "TestThreeIsFour") {
|
if out.Verdict.State != delivery.VerdictFail || !strings.Contains(out.Verdict.Failed.Test, "TestThreeIsFour") {
|
||||||
t.Fatalf("verdict = %+v", out.Verdict)
|
t.Fatalf("verdict = %+v", out.Verdict)
|
||||||
}
|
}
|
||||||
tests := b.steps.tool("test")
|
tests := b.steps.toolAt("test", "unit")
|
||||||
if len(tests) != 1 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" {
|
if len(tests) != 2 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" || strings.Join(packagesOf(tests[1]), ",") != "m/internal/a" {
|
||||||
t.Fatalf("test calls = %+v, want only m/internal/b, once", tests)
|
t.Fatalf("test calls = %+v, want m/internal/b, then m/internal/a, each once", tests)
|
||||||
}
|
}
|
||||||
if n := len(b.steps.tool("digest")); n != 1 {
|
if out.Verdict.Levels[delivery.LevelIntegration].Run != 0 {
|
||||||
t.Errorf("%d digest calls, want the unit level's alone (the check stopped)", n)
|
t.Errorf("the integration level was judged after the unit level failed: %+v", out.Verdict.Levels)
|
||||||
}
|
}
|
||||||
if out.Verdict.Composition == nil || out.Verdict.Composition.State != delivery.CompositionNotRun {
|
if out.Verdict.Composition == nil || out.Verdict.Composition.State != delivery.CompositionNotRun {
|
||||||
t.Errorf("composition = %+v", out.Verdict.Composition)
|
t.Errorf("composition = %+v", out.Verdict.Composition)
|
||||||
|
|||||||
@@ -159,6 +159,9 @@ type target struct {
|
|||||||
runner string
|
runner string
|
||||||
// judged is what the line says of the program that judged a change to the build module; empty for any other change.
|
// judged is what the line says of the program that judged a change to the build module; empty for any other change.
|
||||||
judged string
|
judged string
|
||||||
|
// ask is the check ask put for it, and failed why it could not be asked (issue 1921, try speed S8).
|
||||||
|
ask delivery.StepAsk
|
||||||
|
failed *delivery.Verdict
|
||||||
}
|
}
|
||||||
|
|
||||||
// judgedBy takes what a step under runner workspace answered of the program that ran it (judged_by): the candidate, or the
|
// judgedBy takes what a step under runner workspace answered of the program that ran it (judged_by): the candidate, or the
|
||||||
@@ -307,12 +310,13 @@ func (st *state) levelsOf() []string {
|
|||||||
return []string{delivery.LevelUnit, delivery.LevelIntegration}
|
return []string{delivery.LevelUnit, delivery.LevelIntegration}
|
||||||
}
|
}
|
||||||
|
|
||||||
// judgeAll puts one check ask per pull request and build module the repository needs, the mesh-sdk member first, and
|
// judgeAll puts one check ask per pull request and build module the repository needs, every ask of the group at once so
|
||||||
// judges each answer. A head that does not merge, a pin that is not allowed or a level that fails ends the check at once
|
// the members run side by side on the build nodes (issue 1921, try speed S8), and judges the answers in member order, the
|
||||||
// (ADR 0484 clause 13).
|
// mesh-sdk member first. A head that does not merge, a pin that is not allowed or a level that fails ends the check at once
|
||||||
|
// (ADR 0484 clause 13): the verdict is the first failure in member order, as when the asks were put one after another.
|
||||||
func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
||||||
// The mesh-sdk member first: whether its merge changes the TypeScript package's paths decides whether the other members'
|
// The mesh-sdk member first: whether its merge changes the TypeScript package's paths decides whether the other members'
|
||||||
// Node fetches take the group's package (ADR 0500 §5).
|
// Node fetches take the group's package (ADR 0500 §5). Those Node asks alone wait for the mesh-sdk member's answer.
|
||||||
pulls := slices.Clone(st.c.Pulls)
|
pulls := slices.Clone(st.c.Pulls)
|
||||||
slices.SortStableFunc(pulls, func(a, b Pull) int {
|
slices.SortStableFunc(pulls, func(a, b Pull) int {
|
||||||
if (a.Repository == SDKRepository) != (b.Repository == SDKRepository) {
|
if (a.Repository == SDKRepository) != (b.Repository == SDKRepository) {
|
||||||
@@ -323,34 +327,32 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
|||||||
}
|
}
|
||||||
return 0
|
return 0
|
||||||
})
|
})
|
||||||
var groupPackage map[string]any
|
sdkGroup := len(st.c.Pulls) > 1 && len(pulls) > 0 && pulls[0].Repository == SDKRepository
|
||||||
for _, p := range pulls {
|
members := make([][]*target, len(pulls))
|
||||||
var merged []Fetched
|
var waiting []*target // Node asks of the other members, put once the mesh-sdk member is judged
|
||||||
|
for i, p := range pulls {
|
||||||
for _, bm := range st.j.Modules.BuildModules(p.Repository) {
|
for _, bm := range st.j.Modules.BuildModules(p.Repository) {
|
||||||
t := &target{pull: p, module: bm}
|
t := st.target(ctx, p, bm)
|
||||||
if tc, none := delivery.AgreedToolchain(st.j.Toolchains.Entries(buildrun.LanguageOf[bm])); none == "" {
|
members[i] = append(members[i], t)
|
||||||
t.toolchain = tc.Version
|
|
||||||
}
|
|
||||||
args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk}
|
|
||||||
if bm == buildrun.NodeBuildModule && groupPackage != nil && p.Repository != SDKRepository {
|
|
||||||
args["package"] = groupPackage
|
|
||||||
}
|
|
||||||
if bm == buildrun.GoBuildModule {
|
|
||||||
beside, err := st.j.besideFor(ctx, st.c, p)
|
|
||||||
if err != nil {
|
|
||||||
return fail(forgeMachine(err, bm), err.Error()+unheardWords(err))
|
|
||||||
}
|
|
||||||
if len(beside) > 0 {
|
|
||||||
args["beside"] = beside
|
|
||||||
t.beside = besideLines(beside)
|
|
||||||
}
|
|
||||||
if p.Repository == catalogueRepository {
|
|
||||||
t.runner = RunnerWorkspace // used by the build module when the merge changes it (ADR 0535 §1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.fetchArgs = args
|
|
||||||
st.targets = append(st.targets, t)
|
st.targets = append(st.targets, t)
|
||||||
if v := st.ask(ctx, t, &merged); v != nil {
|
if t.failed != nil {
|
||||||
|
continue // said when the judging reaches it, so the verdict stays the first failure in member order
|
||||||
|
}
|
||||||
|
if sdkGroup && bm == buildrun.NodeBuildModule && p.Repository != SDKRepository {
|
||||||
|
waiting = append(waiting, t)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
t.failed = st.put(ctx, t)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var groupPackage map[string]any
|
||||||
|
for i, p := range pulls {
|
||||||
|
var merged []Fetched
|
||||||
|
for _, t := range members[i] {
|
||||||
|
if t.failed != nil {
|
||||||
|
return t.failed
|
||||||
|
}
|
||||||
|
if v := st.judgeAsk(ctx, t, &merged); v != nil {
|
||||||
return v
|
return v
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -360,14 +362,51 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
|||||||
if p.Repository == SDKRepository && len(st.c.Pulls) > 1 && st.j.changesPackage(p, merged) {
|
if p.Repository == SDKRepository && len(st.c.Pulls) > 1 && st.j.changesPackage(p, merged) {
|
||||||
groupPackage = map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number)}
|
groupPackage = map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number)}
|
||||||
}
|
}
|
||||||
|
if i == 0 && sdkGroup {
|
||||||
|
for _, t := range waiting {
|
||||||
|
if groupPackage != nil {
|
||||||
|
t.fetchArgs["package"] = groupPackage
|
||||||
|
}
|
||||||
|
t.failed = st.put(ctx, t)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ask puts the target's check ask, waits for the step of the node that pulled it and judges its answer: the fetch, the
|
// target is the check of one pull request by one build module: its fetch's arguments, its toolchain, the siblings its
|
||||||
// pins of its go.mod files, then each level's digest, kept verdicts and tests.
|
// tests read and the runner it is asked with; failed says why it cannot be asked.
|
||||||
func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *delivery.Verdict {
|
func (st *state) target(ctx context.Context, p Pull, bm string) *target {
|
||||||
|
t := &target{pull: p, module: bm}
|
||||||
|
if tc, none := delivery.AgreedToolchain(st.j.Toolchains.Entries(buildrun.LanguageOf[bm])); none == "" {
|
||||||
|
t.toolchain = tc.Version
|
||||||
|
}
|
||||||
|
args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk}
|
||||||
|
if bm == buildrun.GoBuildModule {
|
||||||
|
beside, err := st.j.besideFor(ctx, st.c, p)
|
||||||
|
if err != nil {
|
||||||
|
t.failed = fail(forgeMachine(err, bm), err.Error()+unheardWords(err))
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
if len(beside) > 0 {
|
||||||
|
args["beside"] = beside
|
||||||
|
t.beside = besideLines(beside)
|
||||||
|
}
|
||||||
|
if p.Repository == catalogueRepository {
|
||||||
|
t.runner = RunnerWorkspace // used by the build module when the merge changes it (ADR 0535 §1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.fetchArgs = args
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
|
||||||
|
// put puts the target's check ask on its build seat's stream; a check that was stopped (its group holds other heads now,
|
||||||
|
// issue 1791) puts none.
|
||||||
|
func (st *state) put(ctx context.Context, t *target) *delivery.Verdict {
|
||||||
p := t.pull
|
p := t.pull
|
||||||
|
if err := context.Cause(ctx); err != nil {
|
||||||
|
return fail(t.module, fmt.Sprintf("the check of %s#%d was not asked: %v", p.Repository, p.Number, err))
|
||||||
|
}
|
||||||
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: st.levelsOf(), Kept: true}
|
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: st.levelsOf(), Kept: true}
|
||||||
if t.runner != "" {
|
if t.runner != "" {
|
||||||
job.Runner, job.RunnerWhen = t.runner, buildGoDir
|
job.Runner, job.RunnerWhen = t.runner, buildGoDir
|
||||||
@@ -379,13 +418,20 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
|
|||||||
if st.c.Kind == KindTrunk {
|
if st.c.Kind == KindTrunk {
|
||||||
kind = delivery.AskRecheck // a re-judge after a trunk move goes after every new head's check (ADR 0536 §4)
|
kind = delivery.AskRecheck // a re-judge after a trunk move goes after every new head's check (ADR 0536 §4)
|
||||||
}
|
}
|
||||||
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: askLanguage[t.module], Asker: delivery.CheckerModule,
|
t.ask = delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: askLanguage[t.module], Asker: delivery.CheckerModule,
|
||||||
At: st.j.now().UTC(), Check: job}
|
At: st.j.now().UTC(), Check: job}
|
||||||
st.c.Progress.Set(fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number), "")
|
st.c.Progress.Set(fmt.Sprintf("the %s ask %s of %s#%d", kind, t.ask.ID, p.Repository, p.Number), "")
|
||||||
if err := st.j.Asks.Put(ctx, a); err != nil {
|
if err := st.j.Asks.Put(ctx, t.ask); err != nil {
|
||||||
return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
|
return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
|
||||||
}
|
}
|
||||||
step, err := st.j.Asks.Wait(ctx, a)
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgeAsk waits for the step of the node that pulled the target's ask and judges its answer: the fetch, the pins of its
|
||||||
|
// go.mod files, then each level's digest, kept verdicts and tests.
|
||||||
|
func (st *state) judgeAsk(ctx context.Context, t *target, merged *[]Fetched) *delivery.Verdict {
|
||||||
|
p := t.pull
|
||||||
|
step, err := st.j.Asks.Wait(ctx, t.ask)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return fail(t.module, ctx.Err().Error())
|
return fail(t.module, ctx.Err().Error())
|
||||||
@@ -427,7 +473,7 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
|
|||||||
ran := 0.0
|
ran := 0.0
|
||||||
for _, la := range answer.Levels {
|
for _, la := range answer.Levels {
|
||||||
v, elapsed := st.level(ctx, t, la)
|
v, elapsed := st.level(ctx, t, la)
|
||||||
ran += elapsed
|
ran = max(ran, elapsed) // the levels run side by side (issue 1921, try speed S10): the longest is what they took
|
||||||
if v != nil {
|
if v != nil {
|
||||||
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
|
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
|
||||||
return v
|
return v
|
||||||
@@ -438,8 +484,9 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
|
|||||||
}
|
}
|
||||||
|
|
||||||
// level judges one level of a check ask's answer as the tools' answers were judged: the digest keys the kept verdicts
|
// level judges one level of a check ask's answer as the tools' answers were judged: the digest keys the kept verdicts
|
||||||
// of the packages that passed, the kept ones taken are counted, and the first test result that did not pass is the
|
// of the packages that passed, the kept ones taken are counted, and every test result is read and counted, also after
|
||||||
// verdict. It answers the level's running time (its tests' and setup's).
|
// one that did not pass, so the verdict names every failed test of the level (issue 1921). It answers the level's
|
||||||
|
// running time (its tests' and setup's).
|
||||||
func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer) (*delivery.Verdict, float64) {
|
func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer) (*delivery.Verdict, float64) {
|
||||||
level := la.Level
|
level := la.Level
|
||||||
a := st.levels[level]
|
a := st.levels[level]
|
||||||
@@ -461,6 +508,7 @@ func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer)
|
|||||||
a.kept += k.Tests
|
a.kept += k.Tests
|
||||||
}
|
}
|
||||||
elapsed := 0.0
|
elapsed := 0.0
|
||||||
|
var failedResults []Tested
|
||||||
for _, raw := range la.Tests {
|
for _, raw := range la.Tests {
|
||||||
var res Tested
|
var res Tested
|
||||||
if err := json.Unmarshal(raw, &res); err != nil {
|
if err := json.Unmarshal(raw, &res); err != nil {
|
||||||
@@ -489,9 +537,12 @@ func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !res.Passed {
|
if !res.Passed {
|
||||||
return levelFailure(t, level, res), elapsed
|
failedResults = append(failedResults, res)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if len(failedResults) > 0 {
|
||||||
|
return levelFailure(t, level, failedResults...), elapsed
|
||||||
|
}
|
||||||
return nil, elapsed
|
return nil, elapsed
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -558,23 +609,64 @@ func firstLine(s string) string {
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// failure turns a level that did not pass into a verdict naming what failed first.
|
// MaxFailureLine is the most bytes of a failure's line in a verdict's Failures: the first line of its words, which is
|
||||||
func levelFailure(t *target, level string, res Tested) *delivery.Verdict {
|
// its cause where the build module told one apart (issue 1901). Failed keeps the first failure's words whole.
|
||||||
|
const MaxFailureLine = 300
|
||||||
|
|
||||||
|
// levelFailure turns a level that did not pass into a verdict: Failed names what failed first with its words whole,
|
||||||
|
// and Failures every failed test of the level's results with its first line, at most delivery.MaxFailures and an entry
|
||||||
|
// saying how many more (issue 1921).
|
||||||
|
func levelFailure(t *target, level string, results ...Tested) *delivery.Verdict {
|
||||||
v := delivery.Verdict{State: delivery.VerdictFail}
|
v := delivery.Verdict{State: delivery.VerdictFail}
|
||||||
switch {
|
var all []delivery.FailedFirst
|
||||||
case len(res.Failed) > 0:
|
for _, res := range results {
|
||||||
f := res.Failed[0]
|
if len(res.Failed) == 0 {
|
||||||
name := failedName(f)
|
why := res.Why
|
||||||
if t.module == buildrun.NodeBuildModule && f.Package == "" {
|
if why == "" {
|
||||||
name = res.Module
|
why = "the " + level + " level did not pass"
|
||||||
|
}
|
||||||
|
all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: why})
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
v.Failed = delivery.FailedFirst{Test: level + ": " + name, Words: failureWords(f)}
|
for _, f := range res.Failed {
|
||||||
default:
|
name := failedName(f)
|
||||||
why := res.Why
|
if t.module == buildrun.NodeBuildModule && f.Package == "" {
|
||||||
if why == "" {
|
name = res.Module
|
||||||
why = "the " + level + " level did not pass"
|
}
|
||||||
|
step := level
|
||||||
|
if f.Step != "" {
|
||||||
|
step = f.Step // the lint step's finding, not the level's (ADR 0542 §2)
|
||||||
|
}
|
||||||
|
all = append(all, delivery.FailedFirst{Test: step + ": " + name, Words: failureWords(f)})
|
||||||
}
|
}
|
||||||
v.Failed = delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: why}
|
|
||||||
}
|
}
|
||||||
|
if len(all) == 0 { // no result: a level said failed with nothing to name
|
||||||
|
all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: "the " + level + " level did not pass"})
|
||||||
|
}
|
||||||
|
v.Failed = all[0]
|
||||||
|
v.Failures = failureLines(all)
|
||||||
return &v
|
return &v
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// failureLines is every failure with the first line of its words, cut to MaxFailureLine, at most delivery.MaxFailures
|
||||||
|
// and then one entry saying how many more failed.
|
||||||
|
func failureLines(all []delivery.FailedFirst) []delivery.FailedFirst {
|
||||||
|
n := min(len(all), delivery.MaxFailures)
|
||||||
|
out := make([]delivery.FailedFirst, 0, n+1)
|
||||||
|
for _, f := range all[:n] {
|
||||||
|
out = append(out, delivery.FailedFirst{Test: f.Test, Machine: f.Machine, Words: clipLine(firstLine(f.Words), MaxFailureLine)})
|
||||||
|
}
|
||||||
|
if more := len(all) - n; more > 0 {
|
||||||
|
out = append(out, delivery.FailedFirst{Test: fmt.Sprintf("and %d more", more)})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// clipLine cuts a line to at most n bytes on a whole character, marking the cut.
|
||||||
|
func clipLine(s string, n int) string {
|
||||||
|
if len(s) <= n {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
s = strings.ToValidUTF8(s[:n-3], "")
|
||||||
|
return s + "..."
|
||||||
|
}
|
||||||
|
|||||||
@@ -177,8 +177,8 @@ func TestACandidateThatDoesNotCompileFailsTheCheckWithTheCompilersWords(t *testi
|
|||||||
if out.Verdict.State != delivery.VerdictFail || !strings.Contains(out.Verdict.Failed.Words, "undefined: x") {
|
if out.Verdict.State != delivery.VerdictFail || !strings.Contains(out.Verdict.Failed.Words, "undefined: x") {
|
||||||
t.Fatalf("verdict = %+v, want a failure naming the compile error", out.Verdict)
|
t.Fatalf("verdict = %+v, want a failure naming the compile error", out.Verdict)
|
||||||
}
|
}
|
||||||
if n := len(b.steps.tool("test")); n != 1 {
|
if n := len(b.steps.toolAt("test", "unit")); n != 1 {
|
||||||
t.Errorf("%d test steps, want the first one's failure to end the check", n)
|
t.Errorf("%d unit test steps, want the first one's failure to end the level", n)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -49,6 +49,34 @@ func firstFailed(v delivery.Verdict) string {
|
|||||||
return firstWordsLine(f.Words)
|
return firstWordsLine(f.Words)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// failedLines is everything a verdict names as failed, one line each: every failed test with its first line (issue
|
||||||
|
// 1921), or what failed first when the verdict names no list (a conflict, a refused step, an older verdict).
|
||||||
|
func failedLines(v delivery.Verdict) []string {
|
||||||
|
if len(v.Failures) == 0 {
|
||||||
|
if f := firstFailed(v); f != "" {
|
||||||
|
return []string{f}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
lines := make([]string, 0, len(v.Failures))
|
||||||
|
for _, f := range v.Failures {
|
||||||
|
lines = append(lines, firstFailed(delivery.Verdict{Failed: f}))
|
||||||
|
}
|
||||||
|
return lines
|
||||||
|
}
|
||||||
|
|
||||||
|
// failedCount is how many tests a verdict names as failed, the ones after delivery.MaxFailures counted too.
|
||||||
|
func failedCount(v delivery.Verdict) int {
|
||||||
|
n := len(v.Failures)
|
||||||
|
if n > delivery.MaxFailures {
|
||||||
|
var more int
|
||||||
|
if _, err := fmt.Sscanf(v.Failures[n-1].Test, "and %d more", &more); err == nil {
|
||||||
|
return delivery.MaxFailures + more
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
// firstWordsLine is the first line of a failure's words: the words are several lines (issue 1901) and the status line one.
|
// firstWordsLine is the first line of a failure's words: the words are several lines (issue 1901) and the status line one.
|
||||||
func firstWordsLine(w string) string { return firstLine(w) }
|
func firstWordsLine(w string) string { return firstLine(w) }
|
||||||
|
|
||||||
@@ -155,6 +183,9 @@ func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []s
|
|||||||
pc.head = "pass " + commits
|
pc.head = "pass " + commits
|
||||||
default:
|
default:
|
||||||
pc.head = "failed: " + firstFailed(v)
|
pc.head = "failed: " + firstFailed(v)
|
||||||
|
if n := failedCount(v); n > 1 {
|
||||||
|
pc.head = fmt.Sprintf("failed (%d tests): %s", n, firstFailed(v))
|
||||||
|
}
|
||||||
pc.commits = commits
|
pc.commits = commits
|
||||||
}
|
}
|
||||||
var counts []string
|
var counts []string
|
||||||
|
|||||||
@@ -118,8 +118,10 @@ func (st *state) checkPinsOf(ctx context.Context, t *target) verdictPtr {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// sameBranchPin names the branch of the tried repository that points at the tried head and that mesh-sdk also has, at the
|
// sameBranchPin names the mesh-sdk branch whose head the pin is, for a try: the branch of the tried repository that points
|
||||||
// commit the pin names; empty when there is none.
|
// at the tried head and that mesh-sdk also has, at the commit the pin names, first; else any branch of mesh-sdk but its
|
||||||
|
// trunk whose head the pin is, for a try asked by a bare commit that no branch of the tried repository points at (issue
|
||||||
|
// 1624; try speed S12). Empty when there is none.
|
||||||
func (st *state) sameBranchPin(ctx context.Context, t *target, pin string) (string, error) {
|
func (st *state) sameBranchPin(ctx context.Context, t *target, pin string) (string, error) {
|
||||||
mine, err := st.j.Forge.BranchHeads(ctx, t.pull.Repository)
|
mine, err := st.j.Forge.BranchHeads(ctx, t.pull.Repository)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -131,9 +133,6 @@ func (st *state) sameBranchPin(ctx context.Context, t *target, pin string) (stri
|
|||||||
same = append(same, name)
|
same = append(same, name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(same) == 0 {
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
sdk, err := st.j.Forge.BranchHeads(ctx, SDKRepository)
|
sdk, err := st.j.Forge.BranchHeads(ctx, SDKRepository)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
@@ -144,6 +143,16 @@ func (st *state) sameBranchPin(ctx context.Context, t *target, pin string) (stri
|
|||||||
return name, nil
|
return name, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
var any []string
|
||||||
|
for name, h := range sdk {
|
||||||
|
if name != "main" && strings.HasPrefix(h, pin) {
|
||||||
|
any = append(any, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(any)
|
||||||
|
if len(any) > 0 {
|
||||||
|
return any[0], nil
|
||||||
|
}
|
||||||
return "", nil
|
return "", nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -408,3 +408,22 @@ func TestAMergeCheckWithTheSameBranchPinIsStillRefusedWithoutAGroupPullRequest(t
|
|||||||
t.Fatalf("verdict = %+v", v)
|
t.Fatalf("verdict = %+v", v)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Issue 1624 (met again on 2026-10-25; try speed S12): a try asked by a bare commit, which no branch of the tried
|
||||||
|
// repository points at, still takes a pin that is the head of a mesh-sdk branch, naming that branch; a pin that is the head
|
||||||
|
// of no mesh-sdk branch is still refused.
|
||||||
|
func TestATryByCommitTakesAPinThatIsTheHeadOfAnSDKBranch(t *testing.T) {
|
||||||
|
b, c := sameBranchBench(t, head2[:12])
|
||||||
|
b.ex.Forge.(*fakeForge).branches[repoA] = map[string]string{"main": trunkA, "feat/r6": trunkA2} // the branch moved on
|
||||||
|
if v := b.judge(t, c).Verdict; v.State != delivery.VerdictPass {
|
||||||
|
t.Fatalf("verdict = %+v", v)
|
||||||
|
}
|
||||||
|
if n := c.Notes(); len(n) != 1 || n[0] != "pin accepted: head of mesh-sdk feat/r6" {
|
||||||
|
t.Errorf("notes = %q", n)
|
||||||
|
}
|
||||||
|
b, c = sameBranchBench(t, "9aaa5c80ac5c")
|
||||||
|
b.ex.Forge.(*fakeForge).branches[repoA] = map[string]string{"main": trunkA}
|
||||||
|
if v := b.judge(t, c).Verdict; v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "neither on mesh-sdk's trunk") {
|
||||||
|
t.Errorf("a pin no mesh-sdk branch holds: verdict = %+v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -61,6 +61,9 @@ type Failure struct {
|
|||||||
Output string `json:"output,omitempty"`
|
Output string `json:"output,omitempty"`
|
||||||
// Cause is the first line of what the test reported as its error, when the build module told it apart (issue 1901).
|
// Cause is the first line of what the test reported as its error, when the build module told it apart (issue 1901).
|
||||||
Cause string `json:"cause,omitempty"`
|
Cause string `json:"cause,omitempty"`
|
||||||
|
// Step is "lint" for a finding of the lint step (gofmt, go vet), which runs beside the unit level and is reported as
|
||||||
|
// its own failure, not the level's (ADR 0542 §2).
|
||||||
|
Step string `json:"step,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestTime is one test and the seconds it took.
|
// TestTime is one test and the seconds it took.
|
||||||
|
|||||||
@@ -0,0 +1,102 @@
|
|||||||
|
package check
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
const repoC = "novox/mesh-controller"
|
||||||
|
|
||||||
|
func statusOf(t *testing.T, r *rig, repo string, n int, head string) string {
|
||||||
|
t.Helper()
|
||||||
|
key, _ := delivery.CheckerOwedKey(repo, n, head)
|
||||||
|
raw, _, found, _ := r.owed.Get(context.Background(), key)
|
||||||
|
if !found {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
o, err := delivery.DecodeCheckerOwed(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return o.State
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1791 note 9 (issue 1921, try speed S9): two of a group's three members merge. The candidate keeps them, marked merged,
|
||||||
|
// with the same heads; their trunks moved by their own merges. No new check is asked, and the member left keeps its
|
||||||
|
// passed status, so the merge of the rest is not refused.
|
||||||
|
func TestTwoMembersMergingAskNoNewCheckAndTheThirdKeepsItsPass(t *testing.T) {
|
||||||
|
ctx, j := context.Background(), passJudge()
|
||||||
|
r := newRig(t, j)
|
||||||
|
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, repoC: trunkA})
|
||||||
|
a, s, c := pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2), pullOf(repoC, 3, head1)
|
||||||
|
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", a, s, c))
|
||||||
|
r.ch.Wait()
|
||||||
|
if j.count() != 1 || statusOf(t, r, repoC, 3, head1) != "success" {
|
||||||
|
t.Fatalf("judged %d times, the third's status %q", j.count(), statusOf(t, r, repoC, 3, head1))
|
||||||
|
}
|
||||||
|
a.Merged, s.Merged = true, true
|
||||||
|
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", a, s, c))
|
||||||
|
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS2, repoC: trunkA})
|
||||||
|
r.ch.TrunkMoved(ctx)
|
||||||
|
r.ch.Wait()
|
||||||
|
if j.count() != 1 {
|
||||||
|
t.Errorf("a new check was asked after two members merged: judged %d times", j.count())
|
||||||
|
}
|
||||||
|
if got := statusOf(t, r, repoC, 3, head1); got != "success" {
|
||||||
|
t.Errorf("the third member's status is %q, want success", got)
|
||||||
|
}
|
||||||
|
// The trunk of the member left moves: that is a re-judge as before.
|
||||||
|
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS2, repoC: trunkA2})
|
||||||
|
r.ch.TrunkMoved(ctx)
|
||||||
|
r.ch.Wait()
|
||||||
|
if j.count() != 2 {
|
||||||
|
t.Errorf("the member left's trunk moved and it was judged %d times, want again", j.count())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1791 note 8 (issue 1921, try speed S9): a group opened member by member writes a candidate for each partial set. When
|
||||||
|
// the group's candidate moves to new heads, the check of the older heads is stopped and ends failed, superseded, instead
|
||||||
|
// of running on or staying running; it puts no status, as the newer check owns the heads they share.
|
||||||
|
func TestACheckWhoseHeadsTheGroupNoLongerHoldsIsStoppedAndEndsSuperseded(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
started := make(chan struct{})
|
||||||
|
j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) {
|
||||||
|
if len(c.Pulls) == 1 {
|
||||||
|
close(started)
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return Outcome{Verdict: failure("checker", ctx.Err().Error())}, nil
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
return Outcome{Verdict: passing()}, nil // never stopped
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Outcome{Verdict: passing()}, nil
|
||||||
|
}}
|
||||||
|
r := newRig(t, j)
|
||||||
|
partial := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||||
|
whole := candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))
|
||||||
|
r.ch.Candidate(ctx, "feat%2Fx", partial)
|
||||||
|
<-started
|
||||||
|
r.ch.Candidate(ctx, "feat%2Fx", whole)
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { r.ch.Wait(); close(done) }()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("the check of the older heads was not stopped")
|
||||||
|
}
|
||||||
|
old := r.verdict(t, keyOf(t, "feat/x", partial))
|
||||||
|
if old.State != delivery.VerdictFail || !strings.Contains(old.Failed.Words, "superseded") {
|
||||||
|
t.Errorf("the older heads' verdict = %+v, want failed, superseded", old)
|
||||||
|
}
|
||||||
|
if v := r.verdict(t, keyOf(t, "feat/x", whole)); v.State != delivery.VerdictPass {
|
||||||
|
t.Errorf("the newer heads' verdict = %+v", v)
|
||||||
|
}
|
||||||
|
if got := statusOf(t, r, repoA, 7, head1); got != "success" {
|
||||||
|
t.Errorf("the shared head's status is %q, want the newer check's success", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -67,7 +67,9 @@ type Try struct {
|
|||||||
// Said is the try's report in words: what failed first, the tests run, kept and skipped per level, the time of each part.
|
// Said is the try's report in words: what failed first, the tests run, kept and skipped per level, the time of each part.
|
||||||
Said string `json:"said,omitempty"`
|
Said string `json:"said,omitempty"`
|
||||||
// Took is the time from its start to its end, as Go writes a duration.
|
// Took is the time from its start to its end, as Go writes a duration.
|
||||||
Took string `json:"took,omitempty"`
|
Took string `json:"took,omitempty"`
|
||||||
|
// Failed is every test that failed, each with its first line, at most delivery.MaxFailures and then how many more
|
||||||
|
// (issue 1921); or what else failed first (a conflict, a refused step).
|
||||||
Failed []string `json:"failed,omitempty"`
|
Failed []string `json:"failed,omitempty"`
|
||||||
Levels map[string]delivery.LevelResult `json:"levels,omitempty"`
|
Levels map[string]delivery.LevelResult `json:"levels,omitempty"`
|
||||||
Machines []string `json:"machines,omitempty"`
|
Machines []string `json:"machines,omitempty"`
|
||||||
@@ -339,9 +341,7 @@ func (tr *Tryer) run(ctx context.Context, key string) {
|
|||||||
t.State = TryPass
|
t.State = TryPass
|
||||||
default:
|
default:
|
||||||
t.State = TryFail
|
t.State = TryFail
|
||||||
if f := firstFailed(v); f != "" {
|
t.Failed = failedLines(v)
|
||||||
t.Failed = []string{f}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if t.Tests != "" {
|
if t.Tests != "" {
|
||||||
t.Said += " | only the tests " + t.Tests + " (Go); nothing kept"
|
t.Said += " | only the tests " + t.Tests + " (Go); nothing kept"
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,5 +1,11 @@
|
|||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024231716-571ac521f053 h1:8Kbp3aJG3jGndDOfmGdMSDgJVEcc1hyu921hVnnlnlQ=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024231716-571ac521f053/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232250-9b408db24a58 h1:4Damtdqlf6u6nJhvMA99peDy1jmjixtcwFXJ4bWEkLk=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232250-9b408db24a58/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95 h1:I8zbgGzRjmgYydPTWo8hmaZW/P/g+qGDUfu3E32/pkU=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||||
|
|||||||
@@ -71,7 +71,7 @@ func (ix *Indexer) logf(format string, args ...any) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Apply takes one message of the bridge's stream FORGE into the group of its branch. A pull message adds or moves the
|
// Apply takes one message of the bridge's stream FORGE into the group of its branch. A pull message adds or moves the
|
||||||
// pull request, a closed one or a merged message takes it out; a group left with none is deleted. A change is written
|
// pull request, a closed one takes it out, a merged one marks it merged; a group left with none open is deleted. A change is written
|
||||||
// by compare-and-set on the revision read, and read again on a conflict. A lone pull request is a group of one. A
|
// by compare-and-set on the revision read, and read again on a conflict. A lone pull request is a group of one. A
|
||||||
// merged message also writes the fingerprint of each module of its repository (modules.go).
|
// merged message also writes the fingerprint of each module of its repository (modules.go).
|
||||||
func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
|
func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
|
||||||
@@ -154,12 +154,13 @@ func (ix *Indexer) applyOpen(ctx context.Context, f delivery.Forge) error {
|
|||||||
return nil, false, fmt.Errorf("candidates.%s holds what is not a candidate: %w", key, err)
|
return nil, false, fmt.Errorf("candidates.%s holds what is not a candidate: %w", key, err)
|
||||||
}
|
}
|
||||||
kept := slices.DeleteFunc(slices.Clone(c.Pulls), func(p delivery.CandidatePull) bool {
|
kept := slices.DeleteFunc(slices.Clone(c.Pulls), func(p delivery.CandidatePull) bool {
|
||||||
return !named[delivery.OpenPull{Repository: p.Repository, Number: p.Number}]
|
// a member that merged is open no more, and stays while another member is (issue 1791)
|
||||||
|
return !p.Merged && !named[delivery.OpenPull{Repository: p.Repository, Number: p.Number}]
|
||||||
})
|
})
|
||||||
switch {
|
switch {
|
||||||
case len(kept) == len(c.Pulls):
|
case len(kept) == len(c.Pulls):
|
||||||
return nil, false, nil
|
return nil, false, nil
|
||||||
case len(kept) == 0:
|
case !anyOpen(kept):
|
||||||
ix.logf("%s: no pull request of the group is open, so no candidate", key)
|
ix.logf("%s: no pull request of the group is open, so no candidate", key)
|
||||||
return nil, true, nil
|
return nil, true, nil
|
||||||
}
|
}
|
||||||
@@ -232,7 +233,16 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[s
|
|||||||
if at < 0 {
|
if at < 0 {
|
||||||
return cur
|
return cur
|
||||||
}
|
}
|
||||||
pulls = slices.Delete(pulls, at, at+1)
|
if f.Kind == delivery.ForgeMerged {
|
||||||
|
// A member that merges stays, marked merged, with the head it merged at: the heads hash the group's Approve
|
||||||
|
// covers holds, so the checker starts no new check for the members still open (issue 1791; issue 1921 S9).
|
||||||
|
pulls[at].Merged = true
|
||||||
|
} else {
|
||||||
|
pulls = slices.Delete(pulls, at, at+1)
|
||||||
|
}
|
||||||
|
if !anyOpen(pulls) {
|
||||||
|
return nil // every member merged or closed: the group is done
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
entry := delivery.CandidatePull{Head: delivery.Head{Repository: f.Repository, Number: f.Number, Head: f.Head}, Modules: []string{}}
|
entry := delivery.CandidatePull{Head: delivery.Head{Repository: f.Repository, Number: f.Number, Head: f.Head}, Modules: []string{}}
|
||||||
switch {
|
switch {
|
||||||
@@ -276,6 +286,11 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[s
|
|||||||
return &delivery.Candidate{V: delivery.Version, Group: f.Branch, Base: base, HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)}
|
return &delivery.Candidate{V: delivery.Version, Group: f.Branch, Base: base, HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// anyOpen says whether a group's pull requests hold one that has not merged.
|
||||||
|
func anyOpen(pulls []delivery.CandidatePull) bool {
|
||||||
|
return slices.ContainsFunc(pulls, func(p delivery.CandidatePull) bool { return !p.Merged })
|
||||||
|
}
|
||||||
|
|
||||||
// NoneNoPreview is why a candidate's touched module has no fingerprint of the new form: the merge preview is not
|
// NoneNoPreview is why a candidate's touched module has no fingerprint of the new form: the merge preview is not
|
||||||
// computed in this release (novox/hq ADR 0465 clause 3; the checker comes with issue 1445).
|
// computed in this release (novox/hq ADR 0465 clause 3; the checker comes with issue 1445).
|
||||||
const NoneNoPreview = "no merge preview is computed"
|
const NoneNoPreview = "no merge preview is computed"
|
||||||
|
|||||||
@@ -206,8 +206,8 @@ func TestDirsNotKnownKeepTheModulesOfTheSameHead(t *testing.T) { // 1238/AC4
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestClosedAndMergedTakeAPullOutAndTheLastDeletesTheKey(t *testing.T) { // 1238/AC5
|
func TestClosedTakesAPullOutAndTheLastDeletesTheKey(t *testing.T) { // 1238/AC5 (a merge: TestAMergedMemberStays...)
|
||||||
for _, how := range []string{"closed", "merged"} {
|
for _, how := range []string{"closed"} {
|
||||||
ix, st, _ := newIndexer(t)
|
ix, st, _ := newIndexer(t)
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
_ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
|
_ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
|
||||||
@@ -387,3 +387,70 @@ func TestAnOpenMessageDropsEveryCandidateItDoesNotName(t *testing.T) { // 1279/A
|
|||||||
t.Errorf("after an empty open message the candidates hold %v", st.vals)
|
t.Errorf("after an empty open message the candidates hold %v", st.vals)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 1791 (issue 1921, try speed S9): a member of a group that merges while another is open stays in the candidate, marked
|
||||||
|
// merged, with its head: the heads hash the Approve covers holds, so no new check starts for the members left. An open
|
||||||
|
// message does not drop it, a close of the last open member deletes the group, and the last merge does too.
|
||||||
|
func TestAMergedMemberStaysInItsGroupUntilEveryMemberMerged(t *testing.T) {
|
||||||
|
merged := func(repo string, n int, head string) delivery.Forge {
|
||||||
|
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repo, Branch: "feat/x", Number: n, Head: head, Merge: h3, Root: "r", Trees: map[string]string{"a": "t"}}
|
||||||
|
}
|
||||||
|
ix, st, _ := newIndexer(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
for _, f := range []delivery.Forge{pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"), pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c"),
|
||||||
|
pull("novox/mesh-sdk", 7, "feat/x", h3)} {
|
||||||
|
if err := ix.Apply(ctx, f); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
before, _ := read(t, st, "feat/x")
|
||||||
|
if err := ix.Apply(ctx, merged("novox/mesh-sdk", 7, h3)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := ix.Apply(ctx, merged("novox/mesh-delivery", 12, h1)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
c, ok := read(t, st, "feat/x")
|
||||||
|
if !ok || len(c.Pulls) != 3 || c.HeadsHash != before.HeadsHash {
|
||||||
|
t.Fatalf("after two merges the candidate = %+v (kept %v), want the three members and the same heads hash", c, ok)
|
||||||
|
}
|
||||||
|
for _, p := range c.Pulls {
|
||||||
|
if want := p.Repository != "novox/mesh-catalog"; p.Merged != want {
|
||||||
|
t.Errorf("%s#%d merged = %v, want %v", p.Repository, p.Number, p.Merged, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The open message names only the member still open: the merged ones stay.
|
||||||
|
if err := ix.Apply(ctx, open(delivery.OpenPull{Repository: "novox/mesh-catalog", Number: 300})); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if c, _ := read(t, st, "feat/x"); len(c.Pulls) != 3 || c.HeadsHash != before.HeadsHash {
|
||||||
|
t.Errorf("the open message dropped a merged member: %+v", c)
|
||||||
|
}
|
||||||
|
// The last open member merges: the group is done.
|
||||||
|
if err := ix.Apply(ctx, merged("novox/mesh-catalog", 300, h2)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, ok := read(t, st, "feat/x"); ok {
|
||||||
|
t.Error("every member merged and the group is still a candidate")
|
||||||
|
}
|
||||||
|
// A close of the last open member deletes the group too.
|
||||||
|
ix, st, _ = newIndexer(t)
|
||||||
|
_ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
|
||||||
|
_ = ix.Apply(ctx, pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c"))
|
||||||
|
_ = ix.Apply(ctx, merged("novox/mesh-delivery", 12, h1))
|
||||||
|
closed := pull("novox/mesh-catalog", 300, "feat/x", h2)
|
||||||
|
closed.Action = delivery.PullClosed
|
||||||
|
if err := ix.Apply(ctx, closed); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, ok := read(t, st, "feat/x"); ok {
|
||||||
|
t.Error("the last open member closed and the group, holding only a merged one, is still a candidate")
|
||||||
|
}
|
||||||
|
// A lone pull request that merges leaves no group.
|
||||||
|
ix, st, _ = newIndexer(t)
|
||||||
|
_ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
|
||||||
|
_ = ix.Apply(ctx, merged("novox/mesh-delivery", 12, h1))
|
||||||
|
if _, ok := read(t, st, "feat/x"); ok {
|
||||||
|
t.Error("a lone merged pull request is still a candidate")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,213 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"slices"
|
|
||||||
|
|
||||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The assigner's asks reach the operator (issue 1906). The assigner writes its asks (the hold of a fingerprint, the restore
|
|
||||||
// of a data module, the backup before an apply and a move of a module of policy record) in its bucket asks; until now the
|
|
||||||
// teller only filled them from an answer given in the view, so an ask nobody opened the view for reached nobody: on
|
|
||||||
// 2026-10-25 seven modules stayed held on every machine and the operator was never asked. Now the teller raises each on the
|
|
||||||
// operator channel as it raises the gate's merge asks: as the asker mesh-delivery-teller, once per router id, through the same
|
|
||||||
// records, the same wait for room and the same warrants. The operator's answer, from the phone or from the view, is written
|
|
||||||
// under the ask's own key in the bucket answers, which the assigner reads (mesh-delivery-assigner pass.go answeredWith); none
|
|
||||||
// of these asks needs proof of who answered (mesh-sdk go/delivery AskNeedsProof).
|
|
||||||
|
|
||||||
// mostAssignerOpen is how many of the assigner's asks the teller keeps open at the router at once. The router keeps at most
|
|
||||||
// asks.MostOpen open asks of one asker, and the teller asks both the gate's merge asks and the assigner's as one asker, so a burst
|
|
||||||
// of holds (seven at once on 2026-10-25) would take all the room and a merge Approve could not be asked; one place is always
|
|
||||||
// left for a merge ask. An assigner ask beyond these waits in the wait for room and is asked when one of them ends.
|
|
||||||
const mostAssignerOpen = asks.MostOpen - 1
|
|
||||||
|
|
||||||
// assignerOpenRecord is the key of the router ids of the assigner's asks the teller asked.
|
|
||||||
const assignerOpenRecord = "assigner.open"
|
|
||||||
|
|
||||||
// assignerOpen is the router ids of the assigner's asks open now: those asked whose ask is neither answered nor ended. An id
|
|
||||||
// that ended is dropped from the record as it is read.
|
|
||||||
func (t *Teller) assignerOpen() ([]string, *stdio.StateEntry) {
|
|
||||||
e, err := t.Store.Get(assignerOpenRecord)
|
|
||||||
if err != nil || e == nil {
|
|
||||||
return nil, e
|
|
||||||
}
|
|
||||||
var ids []string
|
|
||||||
if err := json.Unmarshal(e.Value, &ids); err != nil {
|
|
||||||
t.logf("the record %s cannot be read: %v", assignerOpenRecord, err)
|
|
||||||
return nil, e
|
|
||||||
}
|
|
||||||
return slices.DeleteFunc(ids, func(id string) bool {
|
|
||||||
k, _, err := t.keptOf(id)
|
|
||||||
return err == nil && !open(k)
|
|
||||||
}), e
|
|
||||||
}
|
|
||||||
|
|
||||||
// assignerRoom says whether another of the assigner's asks may be asked now.
|
|
||||||
func (t *Teller) assignerRoom() bool {
|
|
||||||
ids, _ := t.assignerOpen()
|
|
||||||
return len(ids) < mostAssignerOpen
|
|
||||||
}
|
|
||||||
|
|
||||||
// assignerAsked keeps that the assigner's ask with this router id is asked.
|
|
||||||
func (t *Teller) assignerAsked(id string) {
|
|
||||||
ids, e := t.assignerOpen()
|
|
||||||
if !slices.Contains(ids, id) {
|
|
||||||
ids = append(ids, id)
|
|
||||||
}
|
|
||||||
var err error
|
|
||||||
if e == nil {
|
|
||||||
_, err = t.Store.Create(assignerOpenRecord, ids)
|
|
||||||
} else {
|
|
||||||
_, err = t.Store.Update(assignerOpenRecord, ids, e.Revision)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
t.logf("the record %s could not be kept: %v", assignerOpenRecord, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// waitForAssignerRoom puts an assigner ask in the wait for room, once.
|
|
||||||
func (t *Teller) waitForAssignerRoom(key, id string) {
|
|
||||||
ws, rec, err := t.waiting()
|
|
||||||
if err != nil {
|
|
||||||
t.logf("%v", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if slices.ContainsFunc(ws, func(x waitingAsk) bool { return x.ID == id }) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if t.keepWaiting(append(ws, waitingAsk{Key: key, ID: id}), rec) == nil {
|
|
||||||
t.logf("%s: the assigner's ask %s waits: %d of the assigner's asks are open, the most the teller keeps open so a merge ask always has room", key, id, mostAssignerOpen)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// isAssignerKey says whether an ask key is one of the assigner's, not the gate's merge ask.
|
|
||||||
func isAssignerKey(key string) bool {
|
|
||||||
k, err := delivery.ParseAnyAskKey(key)
|
|
||||||
return err == nil && k.Kind != delivery.AskKindMerge
|
|
||||||
}
|
|
||||||
|
|
||||||
// assignerAsk reads the assigner's ask at key: found false when the key holds none.
|
|
||||||
func (t *Teller) assignerAsk(key string) (delivery.AssignerAsk, bool, error) {
|
|
||||||
if t.Assigner == nil {
|
|
||||||
return delivery.AssignerAsk{}, false, fmt.Errorf("the teller reads no assigner asks")
|
|
||||||
}
|
|
||||||
raw, found, err := t.Assigner.Get(key)
|
|
||||||
if err != nil || !found {
|
|
||||||
return delivery.AssignerAsk{}, false, err
|
|
||||||
}
|
|
||||||
a, err := delivery.DecodeAssignerAsk(key, raw)
|
|
||||||
if err != nil {
|
|
||||||
return delivery.AssignerAsk{}, false, fmt.Errorf("the assigner's key %s holds no ask: %w", key, err)
|
|
||||||
}
|
|
||||||
return a, true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// raiseAssignerAsk asks the operator the assigner's ask at key, once per router id: an ask answered already (from the view or
|
|
||||||
// the phone) is not asked; an earlier asking still open is taken back first; an asking already made is only caught up with
|
|
||||||
// at the router, and one that ended is not asked again, for the router drops a second ask under an id it keeps.
|
|
||||||
func (t *Teller) raiseAssignerAsk(key string) {
|
|
||||||
a, found, err := t.assignerAsk(key)
|
|
||||||
if err != nil {
|
|
||||||
t.logf("%s: the ask is not raised: %v", key, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
id := a.Ask.ID
|
|
||||||
if raw, _, has, err := t.Answers.Get(delivery.AnswerKey(key)); err != nil {
|
|
||||||
t.logf("%s: the answers could not be read, so the ask is not raised: %v", key, err)
|
|
||||||
return
|
|
||||||
} else if has && answeredAsking(raw) == id {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
prior, grec, err := t.seenAt(gateRecordKey(key))
|
|
||||||
if err != nil {
|
|
||||||
t.logf("%s: %v", key, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if grec != nil && prior.ID != id {
|
|
||||||
t.cancelOpen(key, prior.ID) // a new asking of the same key: the earlier one is taken back
|
|
||||||
}
|
|
||||||
k, _, err := t.keptOf(id)
|
|
||||||
if err != nil {
|
|
||||||
t.logf("%s: %v", key, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if k != nil {
|
|
||||||
if grec == nil || prior.ID != id {
|
|
||||||
_ = t.note(id, key, 0, grec)
|
|
||||||
}
|
|
||||||
if open(k) {
|
|
||||||
t.catchUp(id)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err := t.note(id, key, 0, grec); err != nil {
|
|
||||||
t.logf("%s: the record of the ask %s could not be kept, so it is not raised: %v", key, id, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !t.assignerRoom() {
|
|
||||||
t.waitForAssignerRoom(key, id)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err := t.Asker.Ask(a.Ask); err != nil {
|
|
||||||
t.logf("%s: the ask %s was not asked: %v", key, id, err)
|
|
||||||
if kept, _, _ := t.keptOf(id); open(kept) {
|
|
||||||
_ = t.Store.Delete(keptKey(id)) // kept but not published: forgotten, so the next writing of the key asks it
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
t.assignerAsked(id)
|
|
||||||
t.logf("%s: the assigner's ask %s is asked of the operator (%s)", key, id, a.Ask.Headline)
|
|
||||||
}
|
|
||||||
|
|
||||||
// OnAssignerAskGone handles a key the assigner took away from its asks (a hold it lifted by itself): the open router ask of
|
|
||||||
// its latest asking is taken back, so the operator is not asked about what no longer holds.
|
|
||||||
func (t *Teller) OnAssignerAskGone(key string) {
|
|
||||||
t.mu.Lock()
|
|
||||||
defer t.mu.Unlock()
|
|
||||||
if !isAssignerKey(key) {
|
|
||||||
t.logf("the assigner's key %s was taken away and is no ask key of the assigner, and is left", key)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
prior, grec, err := t.seenAt(gateRecordKey(key))
|
|
||||||
if err != nil || grec == nil {
|
|
||||||
if err != nil {
|
|
||||||
t.logf("%s: %v", key, err)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
t.cancelOpen(key, prior.ID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// answerAssigner is the act of a warrant for one of the assigner's asks: the ask must still stand at its key under the asking
|
|
||||||
// and the words the operator answered, and then the answer is written under the ask's key, once. Its options bind no act:
|
|
||||||
// the assigner reads the option chosen (asker.Client.Take has checked the option, its level and the ask's digest).
|
|
||||||
func (t *Teller) answerAssigner(key string, o asks.Option, w asks.Warrant) error {
|
|
||||||
a, found, err := t.assignerAsk(key)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
t.logf("%s: the ask %s was answered and the assigner has taken it away: the answer changes nothing", key, w.Ask)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if a.Ask.ID != w.Ask {
|
|
||||||
t.logf("%s: the ask %s was answered, and the assigner has asked again under %s: an answer to an earlier asking changes nothing", key, w.Ask, a.Ask.ID)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if a.Ask.Digest() != w.AskDigest {
|
|
||||||
return fmt.Errorf("the assigner's ask %s is no longer the ask the operator answered (its digest is %s, the warrant's %s)", key, a.Ask.Digest(), w.AskDigest)
|
|
||||||
}
|
|
||||||
body, err := delivery.EncodeAnswer(delivery.Answer{Option: o.ID, Warrant: warrantID(w), Words: w.Says()})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return t.writeAnswer(key, w.Ask, body, w.Says())
|
|
||||||
}
|
|
||||||
@@ -1,209 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
// 1906/H1: given the assigner's hold ask in its bucket and no answer, when the teller hears the key, then it raises the ask on
|
|
||||||
// the operator channel as itself, unchanged and once (the key heard again, as a watch replays it, asks nothing more).
|
|
||||||
func TestAnAssignerAskIsRaisedOnTheOperatorChannelOnce(t *testing.T) {
|
|
||||||
b := newBed(t)
|
|
||||||
assigner, _ := withView(b)
|
|
||||||
key, value, ask := holdAsk(t)
|
|
||||||
assigner[key] = value
|
|
||||||
b.t.OnAssignerAsk(key)
|
|
||||||
b.t.OnAssignerAsk(key)
|
|
||||||
asked := b.askedOf(asks.AskSubject(delivery.TellerModule))
|
|
||||||
if len(asked) != 1 {
|
|
||||||
t.Fatalf("published %d asks, want one; log:\n%s", len(asked), b.logs.String())
|
|
||||||
}
|
|
||||||
var got asks.Ask
|
|
||||||
if err := json.Unmarshal(asked[0].body, &got); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got.ID != ask.ID || got.Digest() != ask.Digest() {
|
|
||||||
t.Errorf("the router's ask %+v differs from the assigner's", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1906/H1: without the view's bucket the assigner's asks are still raised.
|
|
||||||
func TestAnAssignerAskIsRaisedWithoutTheView(t *testing.T) {
|
|
||||||
b := newBed(t)
|
|
||||||
assigner := memGate{}
|
|
||||||
b.t.Assigner = assigner
|
|
||||||
key, value, _ := holdAsk(t)
|
|
||||||
assigner[key] = value
|
|
||||||
b.t.OnAssignerAsk(key)
|
|
||||||
if n := len(b.askedOf(asks.AskSubject(delivery.TellerModule))); n != 1 {
|
|
||||||
t.Fatalf("published %d asks, want one; log:\n%s", n, b.logs.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1906/H1: the operator's answer on the phone comes back by the same path as the view's: written under the ask's own key in
|
|
||||||
// the bucket answers, naming the asking, once; a second warrant writes nothing more and nothing is said as a fault.
|
|
||||||
func TestTheOperatorsAnswerToAnAssignerAskIsWrittenUnderItsKey(t *testing.T) {
|
|
||||||
b := newBed(t)
|
|
||||||
assigner, _ := withView(b)
|
|
||||||
key, value, ask := holdAsk(t)
|
|
||||||
assigner[key] = value
|
|
||||||
b.t.OnAssignerAsk(key)
|
|
||||||
w := ackFor(ask, "release")
|
|
||||||
b.deliver(t, w)
|
|
||||||
raw, ok := b.answers.written[delivery.AnswerKey(key)]
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("no answer under %s; log:\n%s", key, b.logs.String())
|
|
||||||
}
|
|
||||||
if a, err := delivery.DecodeAnswer(raw); err != nil || a.Option != "release" || answeredAsking(raw) != ask.ID {
|
|
||||||
t.Errorf("answer = %+v, %v", a, err)
|
|
||||||
}
|
|
||||||
b.deliver(t, w)
|
|
||||||
if b.answers.writes != 1 || len(b.conds) != 0 {
|
|
||||||
t.Errorf("%d writes, conditions %+v", b.answers.writes, b.conds)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1906/H1: an answer to an asking the assigner has since taken away or asked again changes nothing.
|
|
||||||
func TestAnAnswerToAnAssignerAskTakenAwayWritesNothing(t *testing.T) {
|
|
||||||
b := newBed(t)
|
|
||||||
assigner, _ := withView(b)
|
|
||||||
key, value, ask := holdAsk(t)
|
|
||||||
assigner[key] = value
|
|
||||||
b.t.OnAssignerAsk(key)
|
|
||||||
delete(assigner, key)
|
|
||||||
b.deliver(t, ackFor(ask, "release"))
|
|
||||||
if b.answers.writes != 0 {
|
|
||||||
t.Errorf("an answer to a withdrawn ask was written")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1906/H1, H2: an ask answered in the view is taken back from the phone, and an ask the assigner took away (a hold it lifted by
|
|
||||||
// itself) is taken back too; an ask answered before it is heard is never raised.
|
|
||||||
func TestAnAssignerAskAnsweredInTheViewOrTakenAwayIsTakenBack(t *testing.T) {
|
|
||||||
cancelled := func(b *bed, id string) bool {
|
|
||||||
for _, p := range b.askedOf(asks.CancelSubject(delivery.TellerModule)) {
|
|
||||||
if string(p.body) == `{"id":"`+id+`"}` {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
b := newBed(t)
|
|
||||||
assigner, view := withView(b)
|
|
||||||
key, value, ask := holdAsk(t)
|
|
||||||
assigner[key] = value
|
|
||||||
b.t.OnAssignerAsk(key)
|
|
||||||
view[key+".view"] = viewAnswer(t, "release", "")
|
|
||||||
b.t.OnViewAnswer(key + ".view")
|
|
||||||
if !cancelled(b, ask.ID) {
|
|
||||||
t.Errorf("the ask answered in the view was not taken back; log:\n%s", b.logs.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
b = newBed(t)
|
|
||||||
assigner, _ = withView(b)
|
|
||||||
assigner[key] = value
|
|
||||||
b.t.OnAssignerAsk(key)
|
|
||||||
delete(assigner, key)
|
|
||||||
b.t.OnAssignerAskGone(key)
|
|
||||||
if !cancelled(b, ask.ID) {
|
|
||||||
t.Errorf("the ask taken away was not taken back; log:\n%s", b.logs.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
b = newBed(t)
|
|
||||||
assigner, view = withView(b)
|
|
||||||
assigner[key] = value
|
|
||||||
view[key+".view"] = viewAnswer(t, "keep", "")
|
|
||||||
b.t.OnAssignerAsk(key)
|
|
||||||
if n := len(b.askedOf(asks.AskSubject(delivery.TellerModule))); n != 0 {
|
|
||||||
t.Errorf("an ask answered in the view was raised %d times", n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1906/H1: an assigner ask refused for want of room waits and is asked again when another of the teller's asks ends, as a merge
|
|
||||||
// ask does; a refusal for its words is said as a condition about a delivery ask, not a merge.
|
|
||||||
func TestAnAssignerAskWaitsForRoomAndARefusalIsSaidAsADeliveryAsk(t *testing.T) {
|
|
||||||
b := newBed(t)
|
|
||||||
assigner, _ := withView(b)
|
|
||||||
_, first := b.askOf(t, 1, 1)
|
|
||||||
b.askOf(t, 2, 2)
|
|
||||||
b.askOf(t, 3, 3)
|
|
||||||
key, value, ask := holdAsk(t)
|
|
||||||
assigner[key] = value
|
|
||||||
b.t.OnAssignerAsk(key)
|
|
||||||
b.refuse(t, ask.ID, roomRefusal)
|
|
||||||
b.deliver(t, warrantFor(first.Ask, "approve"))
|
|
||||||
got := b.askedIDs()
|
|
||||||
if len(got) != 5 || got[4] != ask.ID {
|
|
||||||
t.Fatalf("asks = %v, want the assigner's %s asked again", got, ask.ID)
|
|
||||||
}
|
|
||||||
|
|
||||||
b = newBed(t)
|
|
||||||
assigner, _ = withView(b)
|
|
||||||
assigner[key] = value
|
|
||||||
b.t.OnAssignerAsk(key)
|
|
||||||
b.refuse(t, ask.ID, "the ask has no headline")
|
|
||||||
if len(b.conds) != 1 || b.conds[0].Subject.Scope != "delivery" || strings.Contains(b.conds[0].Explanation, "merge") {
|
|
||||||
t.Errorf("conditions = %+v", b.conds)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ackFor is the router's warrant for an option of level acknowledge, as the assigner's asks offer.
|
|
||||||
func ackFor(a asks.Ask, option string) asks.Warrant {
|
|
||||||
w := warrantFor(a, option)
|
|
||||||
w.Level, w.Label = asks.Acknowledge, option
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
|
|
||||||
// holdAskOf is the assigner's hold ask of module plex at the n-th fingerprint, shaped as holdAsk.
|
|
||||||
func holdAskOf(t *testing.T, n int) (string, []byte, asks.Ask) {
|
|
||||||
t.Helper()
|
|
||||||
key, err := delivery.HoldAskKey("plex", "src2:"+strings.Repeat(string(rune('0'+n)), 64))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_, _, ask := holdAsk(t)
|
|
||||||
ask.ID = delivery.AskID(key)
|
|
||||||
value, err := delivery.EncodeAssignerAsk(key, delivery.AssignerAsk{Ask: ask})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return key, value, ask
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1906/H5: given seven holds at once (as on 2026-10-25), when they are raised, then at most two are open at the router, a merge
|
|
||||||
// ask is still asked, and each hold answered lets the next waiting one be asked.
|
|
||||||
func TestABurstOfHoldsLeavesRoomForAMergeAsk(t *testing.T) {
|
|
||||||
b := newBed(t)
|
|
||||||
assigner, _ := withView(b)
|
|
||||||
var held []asks.Ask
|
|
||||||
for n := 0; n < 7; n++ {
|
|
||||||
key, value, ask := holdAskOf(t, n)
|
|
||||||
assigner[key] = value
|
|
||||||
b.t.OnAssignerAsk(key)
|
|
||||||
held = append(held, ask)
|
|
||||||
}
|
|
||||||
if got := b.askedIDs(); len(got) != mostAssignerOpen || got[0] != held[0].ID || got[1] != held[1].ID {
|
|
||||||
t.Fatalf("asked %v, want the first %d holds only; log:\n%s", got, mostAssignerOpen, b.logs.String())
|
|
||||||
}
|
|
||||||
_, merge := b.askOf(t, 1, 1)
|
|
||||||
if got := b.askedIDs(); len(got) != mostAssignerOpen+1 || got[mostAssignerOpen] != merge.Ask.ID {
|
|
||||||
t.Fatalf("asked %v, want the merge ask asked beside the holds", got)
|
|
||||||
}
|
|
||||||
if len(b.dropped) != 0 || len(b.conds) != 0 {
|
|
||||||
t.Errorf("router dropped %v, conditions %+v", b.dropped, b.conds)
|
|
||||||
}
|
|
||||||
b.deliver(t, ackFor(held[0], "release"))
|
|
||||||
got := b.askedIDs()
|
|
||||||
if len(got) != mostAssignerOpen+2 || got[len(got)-1] != held[2].ID {
|
|
||||||
t.Fatalf("asked %v after a hold was answered, want the third hold asked next; log:\n%s", got, b.logs.String())
|
|
||||||
}
|
|
||||||
// The merge ask answered frees its own place, and no hold takes it.
|
|
||||||
b.deliver(t, warrantFor(merge.Ask, "approve"))
|
|
||||||
if got := b.askedIDs(); len(got) != mostAssignerOpen+2 {
|
|
||||||
t.Errorf("asked %v after the merge ask was answered, want no further hold", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -524,9 +524,8 @@ func TestTellerFillsAnAskFromTheViewsAnswerOnce(t *testing.T) { // 1297/B10 AC
|
|||||||
if a, err := delivery.DecodeAnswer(e.Value()); err != nil || a.Option != "release" || answeredAsking(e.Value()) != ask.ID {
|
if a, err := delivery.DecodeAnswer(e.Value()); err != nil || a.Option != "release" || answeredAsking(e.Value()) != ask.ID {
|
||||||
t.Fatalf("answer %+v err %v", a, err)
|
t.Fatalf("answer %+v err %v", a, err)
|
||||||
}
|
}
|
||||||
// The view's answer written again with the same option changes nothing under the ask (another option, later, is the
|
// The view's answer written again changes nothing under the ask.
|
||||||
// operator changing their mind: TestTellerTakesAChangedHoldAnswerFromTheView, issue 1906 note 23).
|
if _, err := viewKV.Put(b.ctx, key+".view", viewAnswer(t, "keep", "")); err != nil {
|
||||||
if _, err := viewKV.Put(b.ctx, key+".view", viewAnswer(t, "release", "")); err != nil {
|
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
time.Sleep(time.Second)
|
time.Sleep(time.Second)
|
||||||
@@ -546,148 +545,3 @@ func TestTellerFillsAnAskFromTheViewsAnswerOnce(t *testing.T) { // 1297/B10 AC
|
|||||||
t.Error("the merge's ask was answered from a view key")
|
t.Error("the merge's ask was answered from a view key")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestTellerRaisesAnAssignerAskAndWritesTheOperatorsAnswer starts the program against the bus: the assigner's hold ask, put in
|
|
||||||
// its bucket, is on the operator channel's queue as the teller's ask; the router's warrant for "release" is written under the
|
|
||||||
// ask's key, where the assigner reads it; a second ask the assigner takes away (a hold it lifted) is cancelled at the router
|
|
||||||
// (1906/H1, H2 on the real bus).
|
|
||||||
func TestTellerRaisesAnAssignerAskAndWritesTheOperatorsAnswer(t *testing.T) {
|
|
||||||
now = time.Now().UTC()
|
|
||||||
b := startFull(t)
|
|
||||||
assigner := b.as(t, "n1.mesh-delivery-assigner")
|
|
||||||
broker := filepath.Join(t.TempDir(), "broker")
|
|
||||||
cred, _ := json.Marshal(credential{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-teller", User: "n1.mesh-delivery-teller", Password: "pw"})
|
|
||||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
env := map[string]string{"MESH_BROKER_FILE": broker}
|
|
||||||
logs := &syncBuffer{}
|
|
||||||
rn := startRunner(b.ctx, func(c context.Context) error { return run(c, func(k string) string { return env[k] }, logs) })
|
|
||||||
t.Cleanup(func() { rn.finish(t, 10*time.Second) })
|
|
||||||
|
|
||||||
key, value, ask := holdAsk(t)
|
|
||||||
askKV, err := assigner.KeyValue(b.ctx, assignerAsks)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
time.Sleep(500 * time.Millisecond) // the program hears warrants and watches
|
|
||||||
if _, err := askKV.Put(b.ctx, key, value); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
seat, err := b.js.Stream(b.ctx, "SEAT_OPERATOR_CHANNEL")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var raw *jetstream.RawStreamMsg
|
|
||||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
|
||||||
if raw, err = seat.GetLastMsgForSubject(b.ctx, seatAsk); err == nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if raw == nil {
|
|
||||||
t.Fatalf("the assigner's ask is not on the seat's queue: %v\n%s", err, logs.String())
|
|
||||||
}
|
|
||||||
var asked asks.Ask
|
|
||||||
if err := json.Unmarshal(raw.Data, &asked); err != nil || asked.ID != ask.ID || asked.Digest() != ask.Digest() {
|
|
||||||
t.Fatalf("the ask on the queue is %+v (%v), want the assigner's", asked, err)
|
|
||||||
}
|
|
||||||
w, _ := json.Marshal(ackFor(ask, "release"))
|
|
||||||
if err := b.ctl.Publish(seatDecided, w); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
answers, err := b.js.KeyValue(b.ctx, tellerAnswers)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var e jetstream.KeyValueEntry
|
|
||||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
|
||||||
if e, err = answers.Get(b.ctx, delivery.AnswerKey(key)); err == nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if e == nil {
|
|
||||||
t.Fatalf("no answer under %s: %v\n%s", key, err, logs.String())
|
|
||||||
}
|
|
||||||
if a, err := delivery.DecodeAnswer(e.Value()); err != nil || a.Option != "release" || answeredAsking(e.Value()) != ask.ID {
|
|
||||||
t.Fatalf("answer %+v err %v", a, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
key2, err := delivery.HoldAskKey("plex", "src2:"+strings.Repeat("3", 64))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ask2 := ask
|
|
||||||
ask2.ID = delivery.AskID(key2)
|
|
||||||
value2, err := delivery.EncodeAssignerAsk(key2, delivery.AssignerAsk{Ask: ask2})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := askKV.Put(b.ctx, key2, value2); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
|
||||||
if strings.Contains(logs.String(), ask2.ID+" is asked") {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := askKV.Delete(b.ctx, key2); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
|
||||||
if raw, err = seat.GetLastMsgForSubject(b.ctx, seatCancel); err == nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err != nil || string(raw.Data) != `{"id":"`+ask2.ID+`"}` {
|
|
||||||
t.Errorf("the ask taken away was not cancelled: %v %v\n%s", raw, err, logs.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTellerTakesAChangedHoldAnswerFromTheView starts the program against the bus: a hold ask answered "keep" earlier, and the
|
|
||||||
// view's answer "release" written now, replace the answer of that asking with "release" (1906/N23 on the real bus).
|
|
||||||
func TestTellerTakesAChangedHoldAnswerFromTheView(t *testing.T) {
|
|
||||||
now = time.Now().UTC()
|
|
||||||
b := startFull(t)
|
|
||||||
assigner, view := b.as(t, "n1.mesh-delivery-assigner"), b.as(t, "n1.mesh-delivery-view")
|
|
||||||
broker := filepath.Join(t.TempDir(), "broker")
|
|
||||||
cred, _ := json.Marshal(credential{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-teller", User: "n1.mesh-delivery-teller", Password: "pw"})
|
|
||||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
key, value, ask := holdAsk(t)
|
|
||||||
answers, err := b.js.KeyValue(b.ctx, tellerAnswers)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
kept, _ := delivery.EncodeAnswer(delivery.Answer{Option: "keep", Warrant: ask.ID + "@" + time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano), Words: "the operator chose keep"})
|
|
||||||
if _, err := answers.Put(b.ctx, delivery.AnswerKey(key), kept); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
askKV, err := assigner.KeyValue(b.ctx, assignerAsks)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := askKV.Put(b.ctx, key, value); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
env := map[string]string{"MESH_BROKER_FILE": broker}
|
|
||||||
logs := &syncBuffer{}
|
|
||||||
rn := startRunner(b.ctx, func(c context.Context) error { return run(c, func(k string) string { return env[k] }, logs) })
|
|
||||||
t.Cleanup(func() { rn.finish(t, 10*time.Second) })
|
|
||||||
viewKV, err := view.KeyValue(b.ctx, viewAnswers)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
time.Sleep(500 * time.Millisecond) // the program is watching
|
|
||||||
if _, err := viewKV.Put(b.ctx, key+".view", viewAnswer(t, "release", "it was a flake")); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
|
||||||
if e, err := answers.Get(b.ctx, delivery.AnswerKey(key)); err == nil {
|
|
||||||
if a, err := delivery.DecodeAnswer(e.Value()); err == nil && a.Option == "release" && answeredAsking(e.Value()) == ask.ID {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.Fatalf("the kept hold's answer was not changed to release\n%s", logs.String())
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
@@ -108,15 +107,3 @@ func (g kvGate) Get(key string) ([]byte, bool, error) {
|
|||||||
}
|
}
|
||||||
return e.Value(), true, nil
|
return e.Value(), true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetAt is Get with the time the value was written.
|
|
||||||
func (g kvGate) GetAt(key string) ([]byte, time.Time, bool, error) {
|
|
||||||
e, err := g.kv.Get(g.ctx, key)
|
|
||||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
|
||||||
return nil, time.Time{}, false, nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, time.Time{}, false, err
|
|
||||||
}
|
|
||||||
return e.Value(), e.Created(), true, nil
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -141,20 +141,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
// there is said and leaves the teller without that path rather than without the gate's asks (a wait for it needs a timer).
|
// there is said and leaves the teller without that path rather than without the gate's asks (a wait for it needs a timer).
|
||||||
failed := make(chan error, 2)
|
failed := make(chan error, 2)
|
||||||
assignerKV, viewKV := optionalBucket(ctx, js, delivery.AssignerAsksBucket, logger), optionalBucket(ctx, js, delivery.ViewAnswersBucket, logger)
|
assignerKV, viewKV := optionalBucket(ctx, js, delivery.AssignerAsksBucket, logger), optionalBucket(ctx, js, delivery.ViewAnswersBucket, logger)
|
||||||
// The assigner's asks are raised on the operator channel whether or not the view's answers can be read (issue 1906).
|
|
||||||
if viewKV != nil {
|
|
||||||
t.View = kvGate{ctx: ctx, kv: viewKV}
|
|
||||||
}
|
|
||||||
if assignerKV != nil {
|
|
||||||
t.Assigner = kvGate{ctx: ctx, kv: assignerKV}
|
|
||||||
go func() {
|
|
||||||
failed <- watchAsks(ctx, assignerKV, func(key string, _ uint64, _ []byte) { t.OnAssignerAsk(key) }, t.OnAssignerAskGone, nil)
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
if assignerKV != nil && viewKV != nil {
|
if assignerKV != nil && viewKV != nil {
|
||||||
|
t.Assigner, t.View = kvGate{ctx: ctx, kv: assignerKV}, kvGate{ctx: ctx, kv: viewKV}
|
||||||
go func() {
|
go func() {
|
||||||
failed <- watchAsks(ctx, viewKV, func(key string, _ uint64, _ []byte) { t.OnViewAnswer(key) }, func(string) {}, nil)
|
failed <- watchAsks(ctx, viewKV, func(key string, _ uint64, _ []byte) { t.OnViewAnswer(key) }, func(string) {}, nil)
|
||||||
}()
|
}()
|
||||||
|
go func() {
|
||||||
|
failed <- watchAsks(ctx, assignerKV, func(key string, _ uint64, _ []byte) { t.OnAssignerAsk(key) }, func(string) {}, nil)
|
||||||
|
}()
|
||||||
}
|
}
|
||||||
go func() {
|
go func() {
|
||||||
if err := watchAsks(ctx, gate, t.OnAsk, t.OnAskGone, t.OnCurrent); err != nil {
|
if err := watchAsks(ctx, gate, t.OnAsk, t.OnAskGone, t.OnCurrent); err != nil {
|
||||||
@@ -176,7 +170,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
func optionalBucket(ctx context.Context, js jetstream.JetStream, name string, logger *log.Logger) jetstream.KeyValue {
|
func optionalBucket(ctx context.Context, js jetstream.JetStream, name string, logger *log.Logger) jetstream.KeyValue {
|
||||||
kv, err := js.KeyValue(ctx, name)
|
kv, err := js.KeyValue(ctx, name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Printf("the bucket %s cannot be read (%v): the asks and answers it holds are not carried until the teller is started again with it", name, err)
|
logger.Printf("the bucket %s cannot be read (%v): the view's answers fill no ask until the teller is started again with it", name, err)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return kv
|
return kv
|
||||||
|
|||||||
@@ -340,9 +340,6 @@ func (t *Teller) Answer(o asks.Option, w asks.Warrant) error {
|
|||||||
if srec == nil {
|
if srec == nil {
|
||||||
return fmt.Errorf("the teller holds no gate key for the ask %s", w.Ask)
|
return fmt.Errorf("the teller holds no gate key for the ask %s", w.Ask)
|
||||||
}
|
}
|
||||||
if isAssignerKey(s.Key) {
|
|
||||||
return t.answerAssigner(s.Key, o, w)
|
|
||||||
}
|
|
||||||
raw, found, err := t.Gate.Get(s.Key)
|
raw, found, err := t.Gate.Get(s.Key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("the gate's ask %s could not be read: %w", s.Key, err)
|
return fmt.Errorf("the gate's ask %s could not be read: %w", s.Key, err)
|
||||||
@@ -370,31 +367,26 @@ func (t *Teller) Answer(o asks.Option, w asks.Warrant) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return t.writeAnswer(s.Key, w.Ask, body, w.Says())
|
// Compare-and-set: created when the key has no answer; updated at the revision read when it holds an earlier asking's
|
||||||
}
|
// answer (the gate ignores that one); left when it holds this asking's.
|
||||||
|
akey := delivery.AnswerKey(s.Key)
|
||||||
// writeAnswer writes the answer to the asking askID under the ask's key by compare-and-set: created when the key has no answer;
|
|
||||||
// updated at the revision read when it holds an earlier asking's answer (the reader ignores that one); left when it holds this
|
|
||||||
// asking's.
|
|
||||||
func (t *Teller) writeAnswer(key, askID string, body []byte, words string) error {
|
|
||||||
akey := delivery.AnswerKey(key)
|
|
||||||
cur, crev, found, err := t.Answers.Get(akey)
|
cur, crev, found, err := t.Answers.Get(akey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("the answer under %s could not be read: %w", key, err)
|
return fmt.Errorf("the answer under %s could not be read: %w", s.Key, err)
|
||||||
}
|
}
|
||||||
if found && answeredAsking(cur) == askID {
|
if found && answeredAsking(cur) == w.Ask {
|
||||||
t.logf("%s: an answer is there already, and is not written again", key)
|
t.logf("%s: an answer is there already, and is not written again", s.Key)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
written, err := t.Answers.Write(akey, body, crev, found)
|
written, err := t.Answers.Write(akey, body, crev, found)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("the answer under %s could not be written: %w", key, err)
|
return fmt.Errorf("the answer under %s could not be written: %w", s.Key, err)
|
||||||
}
|
}
|
||||||
if !written {
|
if !written {
|
||||||
t.logf("%s: another writer answered first, and the answer is not written again", key)
|
t.logf("%s: another writer answered first, and the answer is not written again", s.Key)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
t.logf("%s: %s", key, words)
|
t.logf("%s: %s", s.Key, w.Says())
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -431,10 +423,10 @@ func (t *Teller) saidOnce(h asker.Handled, w asks.Warrant, err error) {
|
|||||||
t.clear(w.Ask)
|
t.clear(w.Ask)
|
||||||
case h == asker.Refused || (h == asker.Acted && err != nil):
|
case h == asker.Refused || (h == asker.Acted && err != nil):
|
||||||
t.logf("the warrant of %s does not hold, and no answer is written: %v", w.Ask, err)
|
t.logf("the warrant of %s does not hold, and no answer is written: %v", w.Ask, err)
|
||||||
t.raise(w.Ask, t.kindOf(w.Ask).answerHeadline, "its answer did not hold here: "+oneLine(err))
|
t.raise(w.Ask, "A merge Approve did not hold", "its answer did not hold here: "+oneLine(err))
|
||||||
case h == asker.Recorded && w.Outcome == asks.OutcomeRefused:
|
case h == asker.Recorded && w.Outcome == asks.OutcomeRefused:
|
||||||
t.logf("the router refused the ask %s: %s", w.Ask, w.Words)
|
t.logf("the router refused the ask %s: %s", w.Ask, w.Words)
|
||||||
t.raise(w.Ask, t.kindOf(w.Ask).refusedHeadline, "the router refused it: "+w.Words)
|
t.raise(w.Ask, "A merge ask was refused", "the router refused it: "+w.Words)
|
||||||
case err != nil:
|
case err != nil:
|
||||||
t.logf("the warrant of %s: %v", w.Ask, err)
|
t.logf("the warrant of %s: %v", w.Ask, err)
|
||||||
}
|
}
|
||||||
@@ -454,30 +446,6 @@ func oneLine(err error) string {
|
|||||||
// conditionKey is the key of the condition about an ask.
|
// conditionKey is the key of the condition about an ask.
|
||||||
func conditionKey(id string) string { return delivery.TellerModule + ".merge-ask." + id }
|
func conditionKey(id string) string { return delivery.TellerModule + ".merge-ask." + id }
|
||||||
|
|
||||||
// askWords are the words the teller's conditions use for one kind of ask: the gate's merge ask, or one of the assigner's.
|
|
||||||
type askWords struct {
|
|
||||||
scope, noun, needs, nothing, resolved string
|
|
||||||
answerHeadline, refusedHeadline, unaskedHeadline string
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
mergeWords = askWords{scope: "merge", noun: "The merge ask ", needs: "to ask for the Approve again", nothing: "Nothing was merged.",
|
|
||||||
resolved: "A good answer to the merge ask was written.", answerHeadline: "A merge Approve did not hold",
|
|
||||||
refusedHeadline: "A merge ask was refused", unaskedHeadline: "A merge ask could not be asked"}
|
|
||||||
assignerWords = askWords{scope: "delivery", noun: "The delivery ask ", needs: "to answer it in the delivery view",
|
|
||||||
nothing: "Nothing was changed on a machine.", resolved: "A good answer to the delivery ask was written.",
|
|
||||||
answerHeadline: "An answer to a delivery ask did not hold", refusedHeadline: "A delivery ask was refused",
|
|
||||||
unaskedHeadline: "A delivery ask could not be asked"}
|
|
||||||
)
|
|
||||||
|
|
||||||
// kindOf is the words for the ask with this router id, by the key it was asked for.
|
|
||||||
func (t *Teller) kindOf(askID string) askWords {
|
|
||||||
if isAssignerKey(t.condID(askID)) {
|
|
||||||
return assignerWords
|
|
||||||
}
|
|
||||||
return mergeWords
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *Teller) raise(askID, headline, why string) {
|
func (t *Teller) raise(askID, headline, why string) {
|
||||||
if t.Say == nil {
|
if t.Say == nil {
|
||||||
return
|
return
|
||||||
@@ -489,12 +457,11 @@ func (t *Teller) raise(askID, headline, why string) {
|
|||||||
}
|
}
|
||||||
t.logf("the condition of %s could not be kept: %v", id, err)
|
t.logf("the condition of %s could not be kept: %v", id, err)
|
||||||
}
|
}
|
||||||
k := t.kindOf(askID)
|
needs := "to ask for the Approve again"
|
||||||
needs := k.needs
|
c := asks.Condition{Key: conditionKey(id), Severity: asks.SeverityWarning, Subject: asks.Scope{Scope: "merge", ID: id}, State: asks.StateRaised,
|
||||||
c := asks.Condition{Key: conditionKey(id), Severity: asks.SeverityWarning, Subject: asks.Scope{Scope: k.scope, ID: id}, State: asks.StateRaised,
|
Summary: "The merge ask " + id + ": " + why, Headline: headline, Needs: needs,
|
||||||
Summary: k.noun + id + ": " + why, Headline: headline, Needs: needs,
|
Explanation: asks.NeedsYou + needs + ". The merge ask " + id + ", " + why + ". Nothing was merged.",
|
||||||
Explanation: asks.NeedsYou + needs + ". " + k.noun + id + ", " + why + ". " + k.nothing,
|
Resolved: "A good answer to the merge ask was written."}
|
||||||
Resolved: k.resolved}
|
|
||||||
if err := t.Say(c); err != nil {
|
if err := t.Say(c); err != nil {
|
||||||
t.logf("the condition of %s could not be said: %v", id, err)
|
t.logf("the condition of %s could not be said: %v", id, err)
|
||||||
_ = t.Store.Delete(condKey(id))
|
_ = t.Store.Delete(condKey(id))
|
||||||
@@ -510,7 +477,7 @@ func (t *Teller) clear(askID string) {
|
|||||||
if err != nil || e == nil {
|
if err != nil || e == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
c := asks.Condition{Key: conditionKey(id), Subject: asks.Scope{Scope: t.kindOf(askID).scope, ID: id}, State: asks.StateCleared}
|
c := asks.Condition{Key: conditionKey(id), Subject: asks.Scope{Scope: "merge", ID: id}, State: asks.StateCleared}
|
||||||
if err := t.Say(c); err != nil {
|
if err := t.Say(c); err != nil {
|
||||||
t.logf("the condition of %s could not be cleared: %v", id, err)
|
t.logf("the condition of %s could not be cleared: %v", id, err)
|
||||||
return
|
return
|
||||||
@@ -611,7 +578,7 @@ func (t *Teller) roomRefused(w asks.Warrant) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
t.logf("the router refused the ask %s for want of room %d times (%s): it is asked no more, and said", w.Ask, n, w.Words)
|
t.logf("the router refused the ask %s for want of room %d times (%s): it is asked no more, and said", w.Ask, n, w.Words)
|
||||||
t.raise(w.Ask, t.kindOf(w.Ask).unaskedHeadline, fmt.Sprintf("the router refused it %d times for want of room: %s", n, w.Words))
|
t.raise(w.Ask, "A merge ask could not be asked", fmt.Sprintf("the router refused it %d times for want of room: %s", n, w.Words))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if t.keepWaiting(ws, rec) != nil {
|
if t.keepWaiting(ws, rec) != nil {
|
||||||
@@ -708,11 +675,6 @@ func (t *Teller) askWaiting(n int) {
|
|||||||
i++
|
i++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
assigner := isAssignerKey(x.Key)
|
|
||||||
if assigner && !t.assignerRoom() {
|
|
||||||
i++ // the assigner's asks hold their most open: it waits, and a merge ask after it may be asked
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
a, why := t.stillAsked(x)
|
a, why := t.stillAsked(x)
|
||||||
if why != "" {
|
if why != "" {
|
||||||
t.logf("%s: the ask %s, waiting for room, is not asked again: %s", x.Key, x.ID, why)
|
t.logf("%s: the ask %s, waiting for room, is not asked again: %s", x.Key, x.ID, why)
|
||||||
@@ -725,9 +687,6 @@ func (t *Teller) askWaiting(n int) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
ws[i].Asked = true
|
ws[i].Asked = true
|
||||||
if assigner {
|
|
||||||
t.assignerAsked(x.ID)
|
|
||||||
}
|
|
||||||
t.logf("%s: the ask %s is asked of the operator again (refused %d times for want of room)", x.Key, x.ID, x.Refusals)
|
t.logf("%s: the ask %s is asked of the operator again (refused %d times for want of room)", x.Key, x.ID, x.Refusals)
|
||||||
if n--; n <= 0 {
|
if n--; n <= 0 {
|
||||||
return
|
return
|
||||||
@@ -742,9 +701,6 @@ func (t *Teller) stillAsked(x waitingAsk) (asks.Ask, string) {
|
|||||||
if x.Key == "" {
|
if x.Key == "" {
|
||||||
return asks.Ask{}, "the teller holds no gate key for it"
|
return asks.Ask{}, "the teller holds no gate key for it"
|
||||||
}
|
}
|
||||||
if isAssignerKey(x.Key) {
|
|
||||||
return t.stillAskedByAssigner(x)
|
|
||||||
}
|
|
||||||
raw, found, err := t.Gate.Get(x.Key)
|
raw, found, err := t.Gate.Get(x.Key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return asks.Ask{}, "the gate's ask could not be read: " + err.Error()
|
return asks.Ask{}, "the gate's ask could not be read: " + err.Error()
|
||||||
@@ -768,26 +724,6 @@ func (t *Teller) stillAsked(x waitingAsk) (asks.Ask, string) {
|
|||||||
return ga.Ask, ""
|
return ga.Ask, ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// stillAskedByAssigner is stillAsked for one of the assigner's asks.
|
|
||||||
func (t *Teller) stillAskedByAssigner(x waitingAsk) (asks.Ask, string) {
|
|
||||||
a, found, err := t.assignerAsk(x.Key)
|
|
||||||
switch {
|
|
||||||
case err != nil:
|
|
||||||
return asks.Ask{}, err.Error()
|
|
||||||
case !found:
|
|
||||||
return asks.Ask{}, "the assigner has taken it away"
|
|
||||||
case a.Ask.ID != x.ID:
|
|
||||||
return asks.Ask{}, "the assigner asks again under " + a.Ask.ID
|
|
||||||
}
|
|
||||||
if cur, _, has, err := t.Answers.Get(delivery.AnswerKey(x.Key)); err == nil && has && answeredAsking(cur) == x.ID {
|
|
||||||
return asks.Ask{}, "it is answered"
|
|
||||||
}
|
|
||||||
if err := a.Ask.Check(t.now()); err != nil {
|
|
||||||
return asks.Ask{}, "it no longer holds: " + err.Error()
|
|
||||||
}
|
|
||||||
return a.Ask, ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// askAgain keeps the ask afresh as the asker would (its record of the refusal is replaced, so its warrant is taken) and
|
// askAgain keeps the ask afresh as the asker would (its record of the refusal is replaced, so its warrant is taken) and
|
||||||
// publishes it under the bus message id given.
|
// publishes it under the bus message id given.
|
||||||
func (t *Teller) askAgain(a asks.Ask, msgID string) error {
|
func (t *Teller) askAgain(a asks.Ask, msgID string) error {
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
@@ -26,8 +25,7 @@ func (t *Teller) OnViewAnswer(key string) {
|
|||||||
t.fillFromView(askKey)
|
t.fillFromView(askKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
// OnAssignerAsk handles a key put in the assigner's bucket asks: a view answer that came before its ask fills it now, and an
|
// OnAssignerAsk handles a key put in the assigner's bucket asks: a view answer that came before its ask fills it now.
|
||||||
// ask not answered yet is raised on the operator channel (issue 1906; assignerasks.go).
|
|
||||||
func (t *Teller) OnAssignerAsk(key string) {
|
func (t *Teller) OnAssignerAsk(key string) {
|
||||||
t.mu.Lock()
|
t.mu.Lock()
|
||||||
defer t.mu.Unlock()
|
defer t.mu.Unlock()
|
||||||
@@ -36,14 +34,10 @@ func (t *Teller) OnAssignerAsk(key string) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
t.fillFromView(key)
|
t.fillFromView(key)
|
||||||
t.raiseAssignerAsk(key)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// fillFromView writes the answer to the assigner's ask at askKey from the view's answer, when both are there and readable, the
|
// fillFromView writes the answer to the assigner's ask at askKey from the view's answer, when both are there and readable, the
|
||||||
// ask is of a kind that needs no proof, the option is one the ask offers and the asking is not answered yet. A hold ask answered
|
// ask is of a kind that needs no proof, the option is one the ask offers and the asking is not answered yet.
|
||||||
// already takes a later answer of the view that chose another option: the operator changed their mind (issue 1906 note 23: a
|
|
||||||
// "keep" given on the phone could not be undone). Later means the view's answer was written after the answer that stands, so a
|
|
||||||
// view answer read again at a start never replaces a newer answer from the phone.
|
|
||||||
func (t *Teller) fillFromView(askKey string) {
|
func (t *Teller) fillFromView(askKey string) {
|
||||||
if t.Assigner == nil || t.View == nil {
|
if t.Assigner == nil || t.View == nil {
|
||||||
return
|
return
|
||||||
@@ -53,7 +47,7 @@ func (t *Teller) fillFromView(askKey string) {
|
|||||||
t.logf("%s is not answered from the view: %v", askKey, err)
|
t.logf("%s is not answered from the view: %v", askKey, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
rawAnswer, at, found, err := t.viewAnswer(viewKey)
|
rawAnswer, found, err := t.View.Get(viewKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.logf("%s: the view's answer could not be read, so the ask is left: %v", askKey, err)
|
t.logf("%s: the view's answer could not be read, so the ask is left: %v", askKey, err)
|
||||||
return
|
return
|
||||||
@@ -89,14 +83,14 @@ func (t *Teller) fillFromView(askKey string) {
|
|||||||
t.logf("%s: the answer could not be read: %v", askKey, err)
|
t.logf("%s: the answer could not be read: %v", askKey, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if has && answeredAsking(cur) == ask.Ask.ID && !t.changesMind(askKey, cur, va, at) {
|
if has && answeredAsking(cur) == ask.Ask.ID {
|
||||||
return // this asking is answered, from the view or elsewhere
|
return // this asking is answered, from the view or elsewhere
|
||||||
}
|
}
|
||||||
words := "answered in the view (mesh-delivery-view): " + va.Option
|
words := "answered in the view (mesh-delivery-view): " + va.Option
|
||||||
if w := strings.TrimSpace(va.Words); w != "" {
|
if w := strings.TrimSpace(va.Words); w != "" {
|
||||||
words += ": " + w
|
words += ": " + w
|
||||||
}
|
}
|
||||||
body, err := delivery.EncodeAnswer(delivery.Answer{Option: va.Option, Warrant: ask.Ask.ID + "@" + at.UTC().Format(time.RFC3339Nano), Words: words})
|
body, err := delivery.EncodeAnswer(delivery.Answer{Option: va.Option, Warrant: ask.Ask.ID + "@" + t.now().UTC().Format("2006-01-02T15:04:05.999999999Z07:00"), Words: words})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.logf("%s: the answer could not be made: %v", askKey, err)
|
t.logf("%s: the answer could not be made: %v", askKey, err)
|
||||||
return
|
return
|
||||||
@@ -109,48 +103,5 @@ func (t *Teller) fillFromView(askKey string) {
|
|||||||
t.logf("%s: another writer answered first, and the view's answer is not written", askKey)
|
t.logf("%s: another writer answered first, and the view's answer is not written", askKey)
|
||||||
default:
|
default:
|
||||||
t.logf("%s: %s", askKey, words)
|
t.logf("%s: %s", askKey, words)
|
||||||
t.cancelOpen(askKey, ask.Ask.ID) // answered in the view: the same ask on the phone is taken back
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// timedAnswers is the view's bucket as the program reads it: each answer with the time it was written.
|
|
||||||
type timedAnswers interface {
|
|
||||||
GetAt(key string) (value []byte, at time.Time, found bool, err error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// viewAnswer is the view's answer at key and when it was written; a bucket that keeps no time gives the teller's now.
|
|
||||||
func (t *Teller) viewAnswer(key string) ([]byte, time.Time, bool, error) {
|
|
||||||
if tv, ok := t.View.(timedAnswers); ok {
|
|
||||||
v, at, found, err := tv.GetAt(key)
|
|
||||||
if err == nil && !at.IsZero() {
|
|
||||||
return v, at, found, nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, time.Time{}, false, err
|
|
||||||
}
|
|
||||||
return v, t.now(), found, nil
|
|
||||||
}
|
|
||||||
v, found, err := t.View.Get(key)
|
|
||||||
return v, t.now(), found, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// changesMind says whether the view's answer va, written at at, replaces the answer cur of the same asking of the hold ask at
|
|
||||||
// askKey: it chose another option and was written after cur's warrant.
|
|
||||||
func (t *Teller) changesMind(askKey string, cur []byte, va delivery.ViewAnswer, at time.Time) bool {
|
|
||||||
k, err := delivery.ParseAnyAskKey(askKey)
|
|
||||||
if err != nil || k.Kind != delivery.AskKindHold {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
a, err := delivery.DecodeAnswer(cur)
|
|
||||||
if err != nil || a.Option == va.Option {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
_, when, _ := strings.Cut(a.Warrant, "@")
|
|
||||||
before, err := time.Parse(time.RFC3339Nano, when)
|
|
||||||
if err != nil || !at.After(before) {
|
|
||||||
t.logf("%s: the view's answer %s is not newer than the answer %s that stands (%s), and is not taken", askKey, va.Option, a.Option, when)
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
t.logf("%s: the operator changed the answer from %s to %s in the view", askKey, a.Option, va.Option)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package main
|
|||||||
import (
|
import (
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
@@ -11,9 +10,8 @@ import (
|
|||||||
|
|
||||||
const fpHeld = "src2:2222222222222222222222222222222222222222222222222222222222222222"
|
const fpHeld = "src2:2222222222222222222222222222222222222222222222222222222222222222"
|
||||||
|
|
||||||
// holdAsk is a key of the assigner's asks: the ask about a held fingerprint, shaped as the assigner composes it
|
// holdAsk is a key of the assigner's asks: the ask about a held fingerprint, as a fixture (in release 5 the assigner writes
|
||||||
// (mesh-delivery-assigner internal/assign asks.go HoldAsk: two options of level acknowledge that bind no act, so the router
|
// none yet; the value is `{"v":1,"ask":<the router's ask>}`).
|
||||||
// takes it; the value is `{"v":1,"ask":<the router's ask>}`).
|
|
||||||
func holdAsk(t *testing.T) (key string, value []byte, ask asks.Ask) {
|
func holdAsk(t *testing.T) (key string, value []byte, ask asks.Ask) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
key, err := delivery.HoldAskKey("plex", fpHeld)
|
key, err := delivery.HoldAskKey("plex", fpHeld)
|
||||||
@@ -22,7 +20,7 @@ func holdAsk(t *testing.T) (key string, value []byte, ask asks.Ask) {
|
|||||||
}
|
}
|
||||||
ask = asks.Ask{ID: delivery.AskID(key), Headline: "plex is held after a failed health check", Explanation: "ace failed the check.", Who: asks.Operator,
|
ask = asks.Ask{ID: delivery.AskID(key), Headline: "plex is held after a failed health check", Explanation: "ace failed the check.", Who: asks.Operator,
|
||||||
Expires: now.Add(asks.ApproveLasts), OnExpiry: "the fingerprint stays held",
|
Expires: now.Add(asks.ApproveLasts), OnExpiry: "the fingerprint stays held",
|
||||||
Options: []asks.Option{{ID: "keep", Label: "Keep it held", Does: "keeps it held", Level: asks.Acknowledge}, {ID: "release", Label: "Release it", Does: "lets the move go on", Level: asks.Acknowledge}}}
|
Options: []asks.Option{{ID: "release", Label: "Release", Does: "lets the move go on", Level: asks.Approve}, {ID: "keep", Label: "Keep held", Does: "keeps it held", Level: asks.Approve}}}
|
||||||
value, err = delivery.EncodeAssignerAsk(key, delivery.AssignerAsk{Ask: ask})
|
value, err = delivery.EncodeAssignerAsk(key, delivery.AssignerAsk{Ask: ask})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -119,11 +117,8 @@ func TestAViewAnswerOfAnOptionTheAskDoesNotOfferFillsNothing(t *testing.T) { //
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A view answer written before the answer that stands never replaces it (1906/N23: only a later one may, for a hold). The
|
|
||||||
// view's time is fixed here, not the package's clock, which the integration tests set to the wall clock.
|
|
||||||
func TestAViewAnswerNeverReplacesTheAnswerOfTheSameAsking(t *testing.T) { // 1297/B10
|
func TestAViewAnswerNeverReplacesTheAnswerOfTheSameAsking(t *testing.T) { // 1297/B10
|
||||||
b := newBed(t)
|
b := newBed(t)
|
||||||
b.t.Now = func() time.Time { return time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) }
|
|
||||||
assigner, view := withView(b)
|
assigner, view := withView(b)
|
||||||
key, value, ask := holdAsk(t)
|
key, value, ask := holdAsk(t)
|
||||||
assigner[key] = value
|
assigner[key] = value
|
||||||
@@ -179,53 +174,3 @@ func TestAStaleViewAnswerOfAnotherAskingFillsNothing(t *testing.T) { // 1297/B10
|
|||||||
t.Errorf("writes %d; log:\n%s", b.answers.writes, b.logs.String())
|
t.Errorf("writes %d; log:\n%s", b.answers.writes, b.logs.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// timedView is the view's bucket with the time each answer was written.
|
|
||||||
type timedView struct {
|
|
||||||
memGate
|
|
||||||
at map[string]time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v timedView) GetAt(key string) ([]byte, time.Time, bool, error) {
|
|
||||||
b, ok := v.memGate[key]
|
|
||||||
return b, v.at[key], ok, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1906/N23-AC4: given a hold ask answered "keep" on the phone, when the operator answers "release" in the view later, then the
|
|
||||||
// teller replaces the answer of that asking with "release", whose warrant names the asking and the view's time; a view answer
|
|
||||||
// written before the phone's answer, the same option, or an ask of another kind replaces nothing.
|
|
||||||
func TestALaterViewAnswerChangesAHoldsAnswer(t *testing.T) {
|
|
||||||
setup := func(viewAt time.Time, option string) (*bed, string, asks.Ask) {
|
|
||||||
b := newBed(t)
|
|
||||||
assigner := memGate{}
|
|
||||||
view := timedView{memGate: memGate{}, at: map[string]time.Time{}}
|
|
||||||
b.t.Assigner, b.t.View = assigner, view
|
|
||||||
key, value, ask := holdAsk(t)
|
|
||||||
assigner[key] = value
|
|
||||||
kept, _ := delivery.EncodeAnswer(delivery.Answer{Option: "keep", Warrant: ask.ID + "@" + now.Add(time.Hour).UTC().Format(time.RFC3339Nano), Words: "the operator, via telegram, chose keep"})
|
|
||||||
b.answers.written[key], b.answers.revs[key] = kept, 3
|
|
||||||
view.memGate[key+".view"] = viewAnswer(t, option, "it was a flake")
|
|
||||||
view.at[key+".view"] = viewAt
|
|
||||||
b.t.OnViewAnswer(key + ".view")
|
|
||||||
return b, key, ask
|
|
||||||
}
|
|
||||||
later := now.Add(2 * time.Hour)
|
|
||||||
b, key, ask := setup(later, "release")
|
|
||||||
a, err := delivery.DecodeAnswer(b.answers.written[key])
|
|
||||||
if err != nil || b.answers.writes != 1 || a.Option != "release" || a.Warrant != ask.ID+"@"+later.UTC().Format(time.RFC3339Nano) {
|
|
||||||
t.Fatalf("writes %d, answer %+v, %v; log:\n%s", b.answers.writes, a, err, b.logs.String())
|
|
||||||
}
|
|
||||||
for name, b := range map[string]*bed{
|
|
||||||
"older": func() *bed { b, _, _ := setup(now, "release"); return b }(),
|
|
||||||
"same": func() *bed { b, _, _ := setup(later, "keep"); return b }(),
|
|
||||||
} {
|
|
||||||
if b.answers.writes != 0 {
|
|
||||||
t.Errorf("%s: the answer was replaced", name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// A restore ask answered stays answered: only a hold's answer may be changed.
|
|
||||||
bb := newBed(t)
|
|
||||||
if bb.t.changesMind("restore.ace.plex.2222222222222222222222222222222222222222222222222222222222222222", b.answers.written[key], delivery.ViewAnswer{Option: "keep"}, later.Add(time.Hour)) {
|
|
||||||
t.Error("a restore ask's answer was changed")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,8 +1,6 @@
|
|||||||
// Command mesh-delivery-cli is the operator's command for the delivery view (novox/hq ADR 0429 §3). It asks the
|
// Command mesh-delivery-cli is the operator's command for the delivery view (novox/hq ADR 0429 §3). It asks the
|
||||||
// view's local socket for the rows and prints them; it holds no credential. `answer <ask key> <option> [words]` asks the view
|
// view's local socket for the rows and prints them; it holds no credential. `answer <ask key> <option> [words]` asks the view
|
||||||
// to write the operator's answer to an ask that needs no proof of who answered (the view refuses the merge's Approve). `holds`
|
// to write the operator's answer to an ask that needs no proof of who answered (the view refuses the merge's Approve).
|
||||||
// lists every fingerprint the assigner holds with its node, cause and answer; `hold-release <module> <fingerprint> [words]` and
|
|
||||||
// `hold-keep <module> <fingerprint> [words]` answer its hold ask, again when the operator changes their mind (issue 1906).
|
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -12,7 +10,6 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"text/tabwriter"
|
|
||||||
|
|
||||||
"mesh-delivery-view/internal/view"
|
"mesh-delivery-view/internal/view"
|
||||||
)
|
)
|
||||||
@@ -34,26 +31,9 @@ func run(args []string, socket string, out, errOut io.Writer) int {
|
|||||||
if err := fs.Parse(args); err != nil {
|
if err := fs.Parse(args); err != nil {
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
if fs.NArg() > 0 && fs.Arg(0) == "holds" {
|
|
||||||
return holds(socket, *asJSON, out, errOut)
|
|
||||||
}
|
|
||||||
if fs.NArg() > 0 && (fs.Arg(0) == "hold-release" || fs.Arg(0) == "hold-keep") {
|
|
||||||
if fs.NArg() < 3 {
|
|
||||||
fmt.Fprintf(errOut, "mesh-delivery-cli %s <module> <fingerprint> [words]\n", fs.Arg(0))
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
option := strings.TrimPrefix(fs.Arg(0), "hold-")
|
|
||||||
r, err := view.AnswerHoldOn(socket, fs.Arg(1), fs.Arg(2), option, strings.Join(fs.Args()[3:], " "))
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintln(errOut, "mesh-delivery-cli:", err)
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
fmt.Fprintf(out, "answered the hold of %s at %s with %s; the teller fills the ask from it and the assigner reads it\n", r.Module, r.FP, option)
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
if fs.NArg() > 0 {
|
if fs.NArg() > 0 {
|
||||||
if fs.Arg(0) != "answer" || fs.NArg() < 3 {
|
if fs.Arg(0) != "answer" || fs.NArg() < 3 {
|
||||||
fmt.Fprintln(errOut, "mesh-delivery-cli shows the rows, or answers: mesh-delivery-cli answer <ask key> <option> [words]; holds; hold-release|hold-keep <module> <fingerprint> [words]")
|
fmt.Fprintln(errOut, "mesh-delivery-cli shows the rows, or answers: mesh-delivery-cli answer <ask key> <option> [words]")
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
key, option := fs.Arg(1), fs.Arg(2)
|
key, option := fs.Arg(1), fs.Arg(2)
|
||||||
@@ -81,53 +61,3 @@ func run(args []string, socket string, out, errOut io.Writer) int {
|
|||||||
view.Table(out, s)
|
view.Table(out, s)
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
// holds prints every hold: module, fingerprint, node, since, answer, words.
|
|
||||||
func holds(socket string, asJSON bool, out, errOut io.Writer) int {
|
|
||||||
rows, err := view.AskHolds(socket)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintln(errOut, "mesh-delivery-cli:", err)
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
if asJSON {
|
|
||||||
enc := json.NewEncoder(out)
|
|
||||||
enc.SetIndent("", " ")
|
|
||||||
if err := enc.Encode(rows); err != nil {
|
|
||||||
fmt.Fprintln(errOut, "mesh-delivery-cli:", err)
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
if len(rows) == 0 {
|
|
||||||
fmt.Fprintln(out, "the assigner holds nothing")
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
tw := tabwriter.NewWriter(out, 0, 2, 2, ' ', 0)
|
|
||||||
fmt.Fprintln(tw, "MODULE\tFINGERPRINT\tNODE\tSINCE\tANSWER\tWORDS")
|
|
||||||
for _, r := range rows {
|
|
||||||
answer := r.Answer
|
|
||||||
switch {
|
|
||||||
case r.AskKey == "":
|
|
||||||
answer = "no ask"
|
|
||||||
case r.Pending != "":
|
|
||||||
answer = r.Pending + " (from the view, the teller fills it)"
|
|
||||||
case answer == "":
|
|
||||||
answer = "not answered"
|
|
||||||
}
|
|
||||||
fp := strings.TrimPrefix(r.FP, "src2:")
|
|
||||||
if len(fp) > 12 {
|
|
||||||
fp = fp[:12]
|
|
||||||
}
|
|
||||||
since := ""
|
|
||||||
if !r.Since.IsZero() {
|
|
||||||
since = r.Since.Local().Format("2006-01-02 15:04")
|
|
||||||
}
|
|
||||||
words := r.Words
|
|
||||||
if r.Error != "" {
|
|
||||||
words = "unreadable: " + r.Error
|
|
||||||
}
|
|
||||||
fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n", r.Module, fp, r.Node, since, answer, words)
|
|
||||||
}
|
|
||||||
tw.Flush()
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -240,46 +240,3 @@ func TestTheAnswerCommandAnswersAnAskNeedingNoProof(t *testing.T) { // 1297/B9b
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1906/N23-AC3: `holds` lists the holds with node, cause and answer; `hold-release <module> <fingerprint>` and `hold-keep` answer
|
|
||||||
// the hold ask through the view, also one answered already.
|
|
||||||
func TestTheHoldCommandsListAndAnswerHolds(t *testing.T) {
|
|
||||||
fp := "src2:04efd1a1aa" + strings.Repeat("2", 54)
|
|
||||||
key, err := delivery.HoldAskKey("systemd-resolved", fp)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
hold := asks.Ask{ID: delivery.AskID(key), Headline: "systemd-resolved is held", Explanation: "ace failed the check.", Who: asks.Operator, Expires: time.Now().Add(time.Hour), OnExpiry: "stays held",
|
|
||||||
Options: []asks.Option{{ID: "keep", Label: "Keep", Does: "stays", Level: asks.Acknowledge}, {ID: "release", Label: "Release", Does: "goes on", Level: asks.Acknowledge}}}
|
|
||||||
value, _ := delivery.EncodeAssignerAsk(key, delivery.AssignerAsk{Ask: hold})
|
|
||||||
hkey, _ := delivery.HeldKey("systemd-resolved", fp)
|
|
||||||
held, _ := delivery.EncodeHeld(delivery.Held{Node: "ace", Words: "health failed, phase running: dns timeout"})
|
|
||||||
ans, _ := delivery.EncodeAnswer(delivery.Answer{Option: "keep", Warrant: hold.ID + "@2026-10-25T04:00:00Z", Words: "the operator chose keep"})
|
|
||||||
m, w := view.New(), &putter{puts: map[string][]byte{}}
|
|
||||||
m.SetWriter(w)
|
|
||||||
m.Apply(view.Entry{Bucket: delivery.AssignerAsksBucket, Key: key, Value: value})
|
|
||||||
m.Apply(view.Entry{Bucket: delivery.AssignerWantsBucket, Key: hkey, Value: held})
|
|
||||||
m.Apply(view.Entry{Bucket: delivery.AnswersBucket, Key: key, Value: ans})
|
|
||||||
socket := serveWith(t, m)
|
|
||||||
|
|
||||||
var out, errOut bytes.Buffer
|
|
||||||
if code := run([]string{"holds"}, socket, &out, &errOut); code != 0 {
|
|
||||||
t.Fatalf("exit %d: %s", code, errOut.String())
|
|
||||||
}
|
|
||||||
for _, want := range []string{"systemd-resolved", "04efd1a1aa22", "ace", "keep", "dns timeout"} {
|
|
||||||
if !strings.Contains(out.String(), want) {
|
|
||||||
t.Errorf("holds printed %q, missing %q", out.String(), want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
out.Reset()
|
|
||||||
if code := run([]string{"hold-release", "systemd-resolved", "04efd1a1", "a", "flake"}, socket, &out, &errOut); code != 0 {
|
|
||||||
t.Fatalf("exit %d: %s", code, errOut.String())
|
|
||||||
}
|
|
||||||
if a, err := delivery.DecodeViewAnswer(w.puts[key+".view"]); err != nil || a.Option != "release" || a.Asking != hold.ID || a.Words != "a flake" {
|
|
||||||
t.Errorf("the view's answer = %+v, %v", a, err)
|
|
||||||
}
|
|
||||||
errOut.Reset()
|
|
||||||
if code := run([]string{"hold-keep", "systemd-resolved"}, socket, &out, &errOut); code != 2 {
|
|
||||||
t.Errorf("hold-keep without a fingerprint: exit %d", code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,188 +0,0 @@
|
|||||||
package view
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"net"
|
|
||||||
"slices"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The assigner's holds, listed and answered from the operator's command (issue 1906 note 23). A hold ask answered once (on the
|
|
||||||
// phone or in the view) ended with that answer, so nothing listed it any more and an answer "keep" could not be changed: the
|
|
||||||
// operator kept systemd-resolved held and then could not release it, and a kept hold is never lifted by itself. Now the view
|
|
||||||
// lists every hold with its node, its cause and its answer, and answers the hold ask that stands, again if the operator changes
|
|
||||||
// their mind, through the view's one write (`<ask key>.view`), which the teller takes into the ask's answer. Nothing else writes:
|
|
||||||
// the answer still reaches the assigner only through the teller (mesh-delivery-teller viewanswers.go).
|
|
||||||
|
|
||||||
// The options of the hold ask (mesh-delivery-assigner internal/assign asks.go HoldAsk).
|
|
||||||
const (
|
|
||||||
HoldRelease = "release"
|
|
||||||
HoldKeep = "keep"
|
|
||||||
)
|
|
||||||
|
|
||||||
type heldEntry struct {
|
|
||||||
module, fp string
|
|
||||||
value delivery.Held
|
|
||||||
err string
|
|
||||||
at time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// applyHeld takes a change of a held key of the assigner's wants, and says whether the key was one.
|
|
||||||
func (m *Model) applyHeld(e Entry) bool {
|
|
||||||
module, fp, err := delivery.ParseHeldKey(e.Key)
|
|
||||||
if err != nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if e.Deleted {
|
|
||||||
delete(m.holds, e.Key)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
h, err := delivery.DecodeHeld(e.Value)
|
|
||||||
m.holds[e.Key] = heldEntry{module: module, fp: fp, value: h, err: errText(err), at: e.At}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// HoldRow is one fingerprint the assigner holds: the module, the fingerprint, the node whose health check failed, the hold's
|
|
||||||
// words (its cause) and since when; the ask about it (its key and the asking's id, empty when the assigner asks none), and the
|
|
||||||
// answer that stands for that asking (the option and its words, empty when none), with the view's own answer when it waits for
|
|
||||||
// the teller.
|
|
||||||
type HoldRow struct {
|
|
||||||
Module string `json:"module"`
|
|
||||||
FP string `json:"fp"`
|
|
||||||
Node string `json:"node,omitempty"`
|
|
||||||
Words string `json:"words,omitempty"`
|
|
||||||
Since time.Time `json:"since,omitempty"`
|
|
||||||
AskKey string `json:"ask-key,omitempty"`
|
|
||||||
Asking string `json:"asking,omitempty"`
|
|
||||||
Answer string `json:"answer,omitempty"`
|
|
||||||
Answered string `json:"answered,omitempty"`
|
|
||||||
Pending string `json:"pending,omitempty"`
|
|
||||||
Error string `json:"error,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Holds is every hold, ordered by module and fingerprint.
|
|
||||||
func (m *Model) Holds() []HoldRow {
|
|
||||||
m.mu.Lock()
|
|
||||||
defer m.mu.Unlock()
|
|
||||||
out := []HoldRow{}
|
|
||||||
for _, h := range m.holds {
|
|
||||||
row := HoldRow{Module: h.module, FP: h.fp, Node: h.value.Node, Words: h.value.Words, Since: h.at, Error: h.err}
|
|
||||||
if key, err := delivery.HoldAskKey(h.module, h.fp); err == nil {
|
|
||||||
if a, ok := m.aasks[key]; ok && a.err == "" {
|
|
||||||
row.AskKey, row.Asking = key, a.ask.ID
|
|
||||||
if ans, ok := m.answerOf[key]; ok && m.answers[key] == a.ask.ID {
|
|
||||||
row.Answer, row.Answered = ans.Option, ans.Words
|
|
||||||
}
|
|
||||||
if c := m.chosenOf(key, a.ask); c != "" && c != row.Answer {
|
|
||||||
row.Pending = c
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
out = append(out, row)
|
|
||||||
}
|
|
||||||
sort.Slice(out, func(i, j int) bool {
|
|
||||||
if out[i].Module != out[j].Module {
|
|
||||||
return out[i].Module < out[j].Module
|
|
||||||
}
|
|
||||||
return out[i].FP < out[j].FP
|
|
||||||
})
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// AnswerHold writes the view's answer to the hold ask of module at the fingerprint fp: release or keep, with the operator's
|
|
||||||
// words. fp is the whole fingerprint, or the start of its hex (as the rows print it), naming one hold of the module. Unlike
|
|
||||||
// Answer it answers an ask answered already, so the operator can change their mind; an answer equal to the one that stands is
|
|
||||||
// refused, as is a hold the assigner asks nothing about.
|
|
||||||
func (m *Model) AnswerHold(module, fp, option, words string) (HoldRow, error) {
|
|
||||||
option = strings.TrimSpace(option)
|
|
||||||
if option != HoldRelease && option != HoldKeep {
|
|
||||||
return HoldRow{}, fmt.Errorf("a hold is answered %s or %s, not %q", HoldRelease, HoldKeep, option)
|
|
||||||
}
|
|
||||||
var match []HoldRow
|
|
||||||
want := strings.TrimPrefix(strings.TrimSpace(fp), delivery.SourcePrefix2)
|
|
||||||
for _, r := range m.Holds() {
|
|
||||||
if r.Module == module && want != "" && strings.HasPrefix(strings.TrimPrefix(r.FP, delivery.SourcePrefix2), want) {
|
|
||||||
match = append(match, r)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
switch {
|
|
||||||
case len(match) == 0:
|
|
||||||
return HoldRow{}, fmt.Errorf("the assigner holds no fingerprint %q of %s (mesh-delivery-cli holds lists them)", fp, module)
|
|
||||||
case len(match) > 1:
|
|
||||||
return HoldRow{}, fmt.Errorf("%q names %d holds of %s: give more of the fingerprint", fp, len(match), module)
|
|
||||||
}
|
|
||||||
r := match[0]
|
|
||||||
if r.AskKey == "" {
|
|
||||||
return r, fmt.Errorf("the assigner asks nothing about the hold of %s at %s, so there is nothing to answer", module, r.FP)
|
|
||||||
}
|
|
||||||
if r.Answer == option && r.Pending == "" {
|
|
||||||
return r, fmt.Errorf("the hold of %s at %s is answered %s already", module, r.FP, option)
|
|
||||||
}
|
|
||||||
m.mu.Lock()
|
|
||||||
a, w := m.aasks[r.AskKey], m.writer
|
|
||||||
m.mu.Unlock()
|
|
||||||
if !slices.ContainsFunc(a.ask.Options, func(o asks.Option) bool { return o.ID == option }) {
|
|
||||||
return r, fmt.Errorf("the hold ask %s offers no option %q", r.AskKey, option)
|
|
||||||
}
|
|
||||||
if w == nil {
|
|
||||||
return r, fmt.Errorf("the view cannot write its answers here")
|
|
||||||
}
|
|
||||||
key, err := delivery.ViewAnswerKey(r.AskKey)
|
|
||||||
if err != nil {
|
|
||||||
return r, err
|
|
||||||
}
|
|
||||||
body, err := delivery.EncodeViewAnswer(delivery.ViewAnswer{Asking: a.ask.ID, Option: option, Words: strings.TrimSpace(words)})
|
|
||||||
if err != nil {
|
|
||||||
return r, err
|
|
||||||
}
|
|
||||||
if err := w.Put(key, body); err != nil {
|
|
||||||
return r, err
|
|
||||||
}
|
|
||||||
return r, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// askOn sends one ask to the socket at path and decodes the reply into out; the view's refusal is the error.
|
|
||||||
func askOn(path string, req ask, out any) error {
|
|
||||||
c, err := net.DialTimeout("unix", path, 5*time.Second)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("cannot reach the view at %s: %w", path, err)
|
|
||||||
}
|
|
||||||
defer c.Close()
|
|
||||||
c.SetDeadline(time.Now().Add(10 * time.Second))
|
|
||||||
raw, _ := json.Marshal(req)
|
|
||||||
if _, err := c.Write(append(raw, '\n')); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
var reply json.RawMessage
|
|
||||||
if err := json.NewDecoder(c).Decode(&reply); err != nil {
|
|
||||||
return fmt.Errorf("the view's reply cannot be read: %w", err)
|
|
||||||
}
|
|
||||||
var refusal struct {
|
|
||||||
Error string `json:"error"`
|
|
||||||
}
|
|
||||||
if json.Unmarshal(reply, &refusal) == nil && refusal.Error != "" {
|
|
||||||
return errors.New(refusal.Error)
|
|
||||||
}
|
|
||||||
return json.Unmarshal(reply, out)
|
|
||||||
}
|
|
||||||
|
|
||||||
// AskHolds asks the socket at path for every hold.
|
|
||||||
func AskHolds(path string) ([]HoldRow, error) {
|
|
||||||
var out []HoldRow
|
|
||||||
err := askOn(path, ask{Ask: "holds"}, &out)
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// AnswerHoldOn asks the socket at path to answer the hold ask of module at fp with option.
|
|
||||||
func AnswerHoldOn(path, module, fp, option, words string) (HoldRow, error) {
|
|
||||||
var out HoldRow
|
|
||||||
err := askOn(path, ask{Ask: "hold", Module: module, FP: fp, Option: option, Words: words}, &out)
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
@@ -1,91 +0,0 @@
|
|||||||
package view
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
// heldE is the assigner's hold of module at fp, as it writes it in its wants.
|
|
||||||
func heldE(t *testing.T, module, fp, node, words string) Entry {
|
|
||||||
t.Helper()
|
|
||||||
key, err := delivery.HeldKey(module, fp)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
v, err := delivery.EncodeHeld(delivery.Held{Node: node, Words: words})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return Entry{Bucket: delivery.AssignerWantsBucket, Key: key, Value: v, At: t0}
|
|
||||||
}
|
|
||||||
|
|
||||||
func answered(t *testing.T, key, id, option string) Entry {
|
|
||||||
t.Helper()
|
|
||||||
v, err := delivery.EncodeAnswer(delivery.Answer{Option: option, Warrant: id + "@2026-10-25T04:00:00Z", Words: "the operator, via telegram, chose " + option})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return Entry{Bucket: delivery.AnswersBucket, Key: key, Value: v, At: t0}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1906/N23-AC1: given holds in the assigner's wants, the view lists each with its node, its cause in words, its hold ask and
|
|
||||||
// the answer that stands for that asking; a hold without an ask says so, and a held key is never taken for a want.
|
|
||||||
func TestTheViewListsEveryHoldWithItsNodeCauseAndAnswer(t *testing.T) {
|
|
||||||
m := New()
|
|
||||||
e, a := holdAsk(t)
|
|
||||||
fill(m, e, heldE(t, "plex", fpNew, "ace", "health failed, phase running: connection refused"), answered(t, e.Key, a.ID, "keep"),
|
|
||||||
heldE(t, "web", fpNew, "g14", "health failed"))
|
|
||||||
got := m.Holds()
|
|
||||||
if len(got) != 2 {
|
|
||||||
t.Fatalf("holds = %+v", got)
|
|
||||||
}
|
|
||||||
p := got[0]
|
|
||||||
if p.Module != "plex" || p.FP != fpNew || p.Node != "ace" || !strings.Contains(p.Words, "connection refused") || p.AskKey != e.Key || p.Asking != a.ID || p.Answer != "keep" {
|
|
||||||
t.Errorf("plex = %+v", p)
|
|
||||||
}
|
|
||||||
if w := got[1]; w.Module != "web" || w.Node != "g14" || w.AskKey != "" || w.Answer != "" {
|
|
||||||
t.Errorf("web = %+v", w)
|
|
||||||
}
|
|
||||||
if len(m.Snapshot().Wants) != 0 {
|
|
||||||
t.Errorf("a hold was taken for a want: %+v", m.Snapshot().Wants)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1906/N23-AC2: given a hold ask answered "keep" (the ask ended with it), when the operator releases the hold, then the view
|
|
||||||
// writes its answer "release" to that asking under the ask's view key, though the ask was answered; the fingerprint may be the
|
|
||||||
// start of its hex. The same answer again, an unknown hold and another option are refused, and write nothing.
|
|
||||||
func TestAKeptHoldCanBeReleased(t *testing.T) {
|
|
||||||
m, w := New(), &memWriter{}
|
|
||||||
m.SetWriter(w)
|
|
||||||
e, a := holdAsk(t)
|
|
||||||
fill(m, e, heldE(t, "plex", fpNew, "ace", "health failed"), answered(t, e.Key, a.ID, "keep"))
|
|
||||||
short := strings.TrimPrefix(fpNew, delivery.SourcePrefix2)[:8]
|
|
||||||
if _, err := m.AnswerHold("plex", short, HoldRelease, "it was a flake"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
va, err := delivery.DecodeViewAnswer(w.puts[e.Key+".view"])
|
|
||||||
if err != nil || va.Asking != a.ID || va.Option != HoldRelease || va.Words != "it was a flake" {
|
|
||||||
t.Fatalf("the view's answer = %+v, %v", va, err)
|
|
||||||
}
|
|
||||||
w.puts = nil
|
|
||||||
fill(m, answered(t, e.Key, a.ID, HoldRelease))
|
|
||||||
for name, try := range map[string]func() error{
|
|
||||||
"again": func() error { _, err := m.AnswerHold("plex", fpNew, HoldRelease, ""); return err },
|
|
||||||
"unknown": func() error { _, err := m.AnswerHold("plex", "ffff", HoldKeep, ""); return err },
|
|
||||||
"option": func() error { _, err := m.AnswerHold("plex", fpNew, "restore", ""); return err },
|
|
||||||
"module": func() error { _, err := m.AnswerHold("web", fpNew, HoldKeep, ""); return err },
|
|
||||||
} {
|
|
||||||
if err := try(); err == nil {
|
|
||||||
t.Errorf("%s: not refused", name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(w.puts) != 0 {
|
|
||||||
t.Errorf("a refused answer wrote %v", w.puts)
|
|
||||||
}
|
|
||||||
// And kept again: the operator may change their mind both ways.
|
|
||||||
if _, err := m.AnswerHold("plex", fpNew, HoldKeep, ""); err != nil {
|
|
||||||
t.Errorf("keep after release: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -46,8 +46,6 @@ type ask struct {
|
|||||||
Key string `json:"key,omitempty"`
|
Key string `json:"key,omitempty"`
|
||||||
Option string `json:"option,omitempty"`
|
Option string `json:"option,omitempty"`
|
||||||
Words string `json:"words,omitempty"`
|
Words string `json:"words,omitempty"`
|
||||||
Module string `json:"module,omitempty"`
|
|
||||||
FP string `json:"fp,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ServeSocket answers {"ask":"rows"} on l, line-delimited JSON, until ctx ends, and {"ask":"answer","key":…,"option":…,"words":…}
|
// ServeSocket answers {"ask":"rows"} on l, line-delimited JSON, until ctx ends, and {"ask":"answer","key":…,"option":…,"words":…}
|
||||||
@@ -74,21 +72,8 @@ func answer(c net.Conn, m *Model) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
var a ask
|
var a ask
|
||||||
if json.Unmarshal(line, &a) != nil || (a.Ask != "rows" && a.Ask != "answer" && a.Ask != "holds" && a.Ask != "hold") {
|
if json.Unmarshal(line, &a) != nil || (a.Ask != "rows" && a.Ask != "answer") {
|
||||||
json.NewEncoder(c).Encode(map[string]string{"error": `refused: this socket answers {"ask":"rows"}, {"ask":"answer"}, {"ask":"holds"} and {"ask":"hold"} and nothing else`})
|
json.NewEncoder(c).Encode(map[string]string{"error": `refused: this socket answers {"ask":"rows"} and {"ask":"answer"} and nothing else`})
|
||||||
return
|
|
||||||
}
|
|
||||||
switch a.Ask {
|
|
||||||
case "holds":
|
|
||||||
json.NewEncoder(c).Encode(m.Holds())
|
|
||||||
return
|
|
||||||
case "hold":
|
|
||||||
r, err := m.AnswerHold(a.Module, a.FP, a.Option, a.Words)
|
|
||||||
if err != nil {
|
|
||||||
json.NewEncoder(c).Encode(map[string]string{"error": err.Error()})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
json.NewEncoder(c).Encode(r)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if a.Ask == "answer" {
|
if a.Ask == "answer" {
|
||||||
|
|||||||
@@ -231,10 +231,8 @@ type Model struct {
|
|||||||
chosen map[string]delivery.ViewAnswer // the answers the view wrote, by ask key
|
chosen map[string]delivery.ViewAnswer // the answers the view wrote, by ask key
|
||||||
writer Writer
|
writer Writer
|
||||||
backlog *Backlog
|
backlog *Backlog
|
||||||
answers map[string]string // by key: the router id of the asking the teller's answer is for ("" when unreadable)
|
answers map[string]string // by key: the router id of the asking the teller's answer is for ("" when unreadable)
|
||||||
answerOf map[string]delivery.Answer // by key: the teller's answer whole, when readable (holds.go)
|
newest map[string]string // module -> id
|
||||||
holds map[string]heldEntry // the assigner's holds, by held key (holds.go)
|
|
||||||
newest map[string]string // module -> id
|
|
||||||
haves map[string]haveEntry
|
haves map[string]haveEntry
|
||||||
nodes map[string]nodeEntry
|
nodes map[string]nodeEntry
|
||||||
// the watchman's buckets, by key
|
// the watchman's buckets, by key
|
||||||
@@ -346,7 +344,7 @@ type nodeEntry struct {
|
|||||||
|
|
||||||
// New is an empty model.
|
// New is an empty model.
|
||||||
func New() *Model {
|
func New() *Model {
|
||||||
return &Model{assigns: map[string]assignment{}, builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, keys: map[string]keyEntry{}, asks: map[string]askRecord{}, wants: map[string]wantEntry{}, told: map[string]bool{}, initial: map[string]bool{}, composed: map[string]composedEntry{}, aasks: map[string]assignerAskRecord{}, chosen: map[string]delivery.ViewAnswer{}, answers: map[string]string{}, answerOf: map[string]delivery.Answer{}, holds: map[string]heldEntry{}, newest: map[string]string{},
|
return &Model{assigns: map[string]assignment{}, builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, keys: map[string]keyEntry{}, asks: map[string]askRecord{}, wants: map[string]wantEntry{}, told: map[string]bool{}, initial: map[string]bool{}, composed: map[string]composedEntry{}, aasks: map[string]assignerAskRecord{}, chosen: map[string]delivery.ViewAnswer{}, answers: map[string]string{}, newest: map[string]string{},
|
||||||
haves: map[string]haveEntry{}, nodes: map[string]nodeEntry{},
|
haves: map[string]haveEntry{}, nodes: map[string]nodeEntry{},
|
||||||
presence: map[string]presenceEntry{}, stuck: map[string]stuckEntry{}}
|
presence: map[string]presenceEntry{}, stuck: map[string]stuckEntry{}}
|
||||||
}
|
}
|
||||||
@@ -488,12 +486,9 @@ func (m *Model) apply(e Entry) {
|
|||||||
m.chosen[askKey] = a
|
m.chosen[askKey] = a
|
||||||
}
|
}
|
||||||
case delivery.AssignerWantsBucket:
|
case delivery.AssignerWantsBucket:
|
||||||
if m.applyHeld(e) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
node, module, err := delivery.ParseWantKey(e.Key)
|
node, module, err := delivery.ParseWantKey(e.Key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return // the node's own key is not a module's want
|
return // a hold, and the node's own key, are not a module's want
|
||||||
}
|
}
|
||||||
key, kerr := delivery.HaveKey(node, module)
|
key, kerr := delivery.HaveKey(node, module)
|
||||||
if kerr != nil {
|
if kerr != nil {
|
||||||
@@ -525,7 +520,6 @@ func (m *Model) apply(e Entry) {
|
|||||||
case delivery.AnswersBucket:
|
case delivery.AnswersBucket:
|
||||||
if e.Deleted {
|
if e.Deleted {
|
||||||
delete(m.answers, e.Key)
|
delete(m.answers, e.Key)
|
||||||
delete(m.answerOf, e.Key)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
// The answer's warrant is `<router id>@<time>`: it ends that asking, and only that one (a later asking of the key is
|
// The answer's warrant is `<router id>@<time>`: it ends that asking, and only that one (a later asking of the key is
|
||||||
@@ -533,9 +527,6 @@ func (m *Model) apply(e Entry) {
|
|||||||
id := ""
|
id := ""
|
||||||
if a, err := delivery.DecodeAnswer(e.Value); err == nil {
|
if a, err := delivery.DecodeAnswer(e.Value); err == nil {
|
||||||
id, _, _ = strings.Cut(a.Warrant, "@")
|
id, _, _ = strings.Cut(a.Warrant, "@")
|
||||||
m.answerOf[e.Key] = a
|
|
||||||
} else {
|
|
||||||
delete(m.answerOf, e.Key)
|
|
||||||
}
|
}
|
||||||
m.answers[e.Key] = id
|
m.answers[e.Key] = id
|
||||||
case delivery.BuilderBuildsBucket:
|
case delivery.BuilderBuildsBucket:
|
||||||
|
|||||||
Reference in New Issue
Block a user