diff --git a/modules/mesh-delivery-assigner/internal/assign/wants.go b/modules/mesh-delivery-assigner/internal/assign/wants.go index ab020fca..a7f19d62 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants.go @@ -153,8 +153,14 @@ func (m Module) provisionWaiting(node string) (string, bool) { return fmt.Sprintf("provision %s has no key yet for %s on %s", missing[0], m.Name, node), true } keyed := m.Provisions(node) + // Only a provision the newest manifest still requires is waited on (ADR 0603 §9): a key of an older revision's edge, + // asked or retiring for the operator's answer (ADR 0570 §5), holds nothing. + needed := m.Manifest.Requires + if m.Answering != nil { + needed = catalogue.Awaited(m.Manifest, m.Answering) + } for _, key := range slices.Sorted(maps.Keys(keyed)) { - if p := keyed[key]; p.State != provisions.Provided { + if p := keyed[key]; p.State != provisions.Provided && slices.Contains(needed, p.Provision) { words := fmt.Sprintf("the provision %s of %s on %s is %s, not provided", key, m.Name, node, p.State) if p.Why != "" { words += ": " + p.Why diff --git a/modules/mesh-delivery-assigner/internal/assign/wants_test.go b/modules/mesh-delivery-assigner/internal/assign/wants_test.go index ba188a94..00c91352 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants_test.go @@ -108,14 +108,17 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { // asked for and requires oidc instead; the seat's keys follow the newest manifest, so the older edge is no wait: a provided // oidc moves the want, an asked one holds it and names oidc and never database. for _, c := range []struct { - name string - state string - moves bool - }{{"the replaced edge provided", provisions.Provided, true}, {"the replaced edge asked", provisions.Asked, false}} { + name string + state, older string + moves bool + }{{"the replaced edge provided, the older edge retiring", provisions.Provided, provisions.Retiring, true}, + {"the replaced edge provided, the older edge asked", provisions.Provided, provisions.Asked, true}, + {"the replaced edge asked, the older edge retiring", provisions.Asked, provisions.Retiring, false}} { m, n := ready() m.Manifest.Requires = []string{"oidc"} m.Provisions = func(node string) map[string]provisions.Provision { - return map[string]provisions.Provision{node + ".mail.oidc": {Provision: "oidc", State: c.state}} + return map[string]provisions.Provision{node + ".mail.oidc": {Provision: "oidc", State: c.state}, + node + ".mail.database": {Provision: "database", State: c.older, Why: "the operator's answer"}} } out := Wants(m, n, wantedOld(), nil) if got := out.Wants["g14"].FP == fNew; got != c.moves || (!c.moves && (!strings.Contains(out.Rules["g14"].Words, "oidc") || strings.Contains(out.Rules["g14"].Words, "database"))) { diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index b0a04e78..40e308b2 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -476,6 +476,14 @@ func TestTheWatchmansFindings(t *testing.T) { if len(o.pulses) != 3 || len(o.findings) != 0 { t.Fatalf("pulses %v findings %v", o.pulses, o.findings) } + // The keys are written again as their parts move on, with new words and a new time: the wait still began at t0. + c.t = c.t.Add(20 * time.Minute) + held2, _ := delivery.EncodeWant(delivery.Want{FP: fpOld, Self: true, Newest: fpNew, Rule: delivery.RuleCanary, Words: "the canary ace runs it, not healthy yet"}) + asked2, _ := provisions.Encode(provisions.Provision{Provision: "oidc", Consumer: provisions.End{Module: "photos", Node: "ace"}, + Provider: provisions.End{Module: "keycloak", Node: "novox"}, State: provisions.Asked, Why: "the provider is applying", At: c.t}) + w.Want("want.ace.web", held2, Put, c.t) + w.Provision("ace.photos.oidc", asked2, Put) + w.Build(bkey, queued, Put, c.t) for bucket, key := range waits { expire(w, c, pulseKeyOf(t, bucket, key), 31*time.Minute) } diff --git a/modules/mesh-delivery-watchman/internal/watch/mirror.go b/modules/mesh-delivery-watchman/internal/watch/mirror.go index b2f0e180..76fd2cf5 100644 --- a/modules/mesh-delivery-watchman/internal/watch/mirror.go +++ b/modules/mesh-delivery-watchman/internal/watch/mirror.go @@ -102,13 +102,16 @@ func (w *Watchman) Want(key string, value []byte, op Op, at time.Time) { at = w.now() } w.held[key] = true - w.mirrorWithin(w.stepBound, delivery.AssignerWantsBucket, key, + w.mirrorKept(delivery.AssignerWantsBucket, key, describe("the hold on %s on %s at %s (rule %d) to end, bound %s: %s", module, node, shortFP(want.FP), want.Rule, w.stepBound, want.Words), - "held|"+want.FP+"|"+want.Newest, at, 0) + "held|"+want.FP+"|"+want.Newest, at) return } if w.held[key] { delete(w.held, key) + if pk, err := delivery.PulseKey(delivery.AssignerWantsBucket, key); err == nil { + delete(w.starts, pk) + } w.stuckMoved(delivery.AssignerWantsBucket, key) // the hold ended: its finding is cleared } if w.have(node, module) == want.FP { @@ -156,8 +159,8 @@ func (w *Watchman) Build(key string, value []byte, op Op, at time.Time) { if at.IsZero() { at = w.now() } - w.mirrorWithin(w.stepBound, delivery.BuilderBuildsBucket, key, - describe("the package %s to be published before the build of %s at %s, bound %s", b.Waits, module, shortFP(fp), w.stepBound), b.Waits, at, 0) + w.mirrorKept(delivery.BuilderBuildsBucket, key, + describe("the package %s to be published before the build of %s at %s, bound %s", b.Waits, module, shortFP(fp), w.stepBound), b.Waits, at) } // Have is a change of a key of the node-engine's have bucket: the fingerprint a node runs of a module. One that names the diff --git a/modules/mesh-delivery-watchman/internal/watch/signals.go b/modules/mesh-delivery-watchman/internal/watch/signals.go index 9c2e8501..042a5321 100644 --- a/modules/mesh-delivery-watchman/internal/watch/signals.go +++ b/modules/mesh-delivery-watchman/internal/watch/signals.go @@ -632,7 +632,7 @@ func (w *Watchman) askedWait(key string, value []byte, op Op) { if p.Why != "" { words += ": " + p.Why } - w.mirrorWithin(w.stepBound, bucket, key, words, "asked", since, 0) + w.mirrorKept(bucket, key, words, "asked", since) } func sortedKeys(m map[string]string) []string { diff --git a/modules/mesh-delivery-watchman/internal/watch/watch.go b/modules/mesh-delivery-watchman/internal/watch/watch.go index ef16ebbd..0e8b7925 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch.go @@ -64,8 +64,10 @@ type Watchman struct { keptSince map[string]time.Time wants map[string]string // the fingerprint of each want that stands, by want key haves map[string]string // the fingerprint each node runs of a module, by have key - held map[string]bool // each want key held by a rule (ADR 0603 §8), mirrored as the hold's wait - bound time.Duration + // starts is, by pulse key, the first start of a wait whose mark still stands (mirrorKept). + starts map[string]startMark + held map[string]bool // each want key held by a rule (ADR 0603 §8), mirrored as the hold's wait + bound time.Duration // findings is every finding written, by key, as written; seededFindings those read at the start and not yet observed // again. findings map[string][]byte @@ -104,7 +106,7 @@ type Watchman struct { // New is a watchman writing to out, reading the time from now. func New(out Out, now func() time.Time) *Watchman { return &Watchman{out: out, now: now, counts: map[string]int{}, presence: map[string]presence{}, - pulses: map[string]*pulse{}, stuck: map[string]delivery.Stuck{}, wants: map[string]string{}, haves: map[string]string{}, held: map[string]bool{}, bound: delivery.PulseMoveBound, stepBound: StepBound, + pulses: map[string]*pulse{}, stuck: map[string]delivery.Stuck{}, wants: map[string]string{}, haves: map[string]string{}, held: map[string]bool{}, starts: map[string]startMark{}, bound: delivery.PulseMoveBound, stepBound: StepBound, findings: map[string][]byte{}, seededFindings: map[string]bool{}, signals: map[string]signalDue{}, away: map[string]bool{}, haveState: map[string]delivery.Have{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{}, control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{}, @@ -162,6 +164,25 @@ func (w *Watchman) mirrorWithin(bound time.Duration, bucket, key, waited, head s w.pulses[pk] = &pulse{bucket: bucket, key: key, waited: waited, since: since, head: head, token: token} } +// startMark is when a wait began, for the mark that names it. +type startMark struct { + mark string + since time.Time +} + +// mirrorKept is mirrorWithin for a wait that is the key's value standing as it is (ADR 0603 §8): while the same mark stands +// (a provision asked, a want held at the same fingerprints, a build waiting for the same package), a rewrite of the key (new +// words, a new time) does not start the bound again; the earliest start is kept. A different mark is a new wait. +func (w *Watchman) mirrorKept(bucket, key, waited, mark string, since time.Time) { + if pk, err := delivery.PulseKey(bucket, key); err == nil { + if cur, ok := w.starts[pk]; ok && cur.mark == mark && cur.since.Before(since) { + since = cur.since + } + w.starts[pk] = startMark{mark: mark, since: since} + } + w.mirrorWithin(w.stepBound, bucket, key, waited, mark, since, 0) +} + func (w *Watchman) deletePulse(pk string) { delete(w.pulses, pk) if err := w.out.DeletePulse(pk); err != nil { @@ -176,6 +197,7 @@ func (w *Watchman) moved(bucket, key string) { if err != nil { return } + delete(w.starts, pk) if _, ok := w.pulses[pk]; ok { w.deletePulse(pk) }