Bind the have bucket once per pass, bound by the queue's timeout, and stop the pass at the first error (issue 1132)

internal/link/have.go

@@ -14,13 +14,20 @@ import (
 // ErrHaveAbsent is what HaveGet returns for a key the bucket does not hold.
 var ErrHaveAbsent = errors.New("link: no such key")
 
-// HaveBus is a link that can read and write the node-engine's own keys in the have bucket (novox/hq ADR 0429 §4). Separate
-// from Bus, as HealthBus is: a link that cannot is still a link, and the write waits for the next apply or look.
+// HaveBus is a link that can bind the have bucket (novox/hq ADR 0429 §4). Separate from Bus, as HealthBus is: a link that
+// cannot is still a link, and the write waits for the next apply or look.
 type HaveBus interface {
-	// HaveGet is a key's value and last revision, or ErrHaveAbsent.
-	HaveGet(ctx context.Context, key string) (value []byte, revision uint64, err error)
-	// HaveSet writes a key over the revision read: zero creates it, and a key that moved since is refused.
-	HaveSet(ctx context.Context, key string, value []byte, revision uint64) error
+	// HaveBind binds the bucket the controller made, once for a pass of writes, within ctx. A node whose grant is not there
+	// yet gets an error here (or the context's end), and writes nothing.
+	HaveBind(ctx context.Context) (HaveBucket, error)
+}
+
+// HaveBucket is the bound bucket.
+type HaveBucket interface {
+	// Get is a key's value and last revision, or ErrHaveAbsent.
+	Get(ctx context.Context, key string) (value []byte, revision uint64, err error)
+	// Set writes a key over the revision read: zero creates it, and a key that moved since is refused.
+	Set(ctx context.Context, key string, value []byte, revision uint64) error
 }
 
 // haveState is what the node-engine last learned of each module of the declaration it applied, and of each module's health
@@ -151,8 +158,15 @@ func (q *Queue) writeHaves(ctx context.Context, bus Bus) {
 		return
 	}
 	node := q.Membership.Node
+	// The whole pass is bounded by the queue's timeout, and it stops at the first error: a node whose grant is not there
+	// yet would otherwise make the worker wait the bound again for every key (issue 1132).
 	write, cancel := context.WithTimeout(context.WithoutCancel(ctx), q.timeout())
 	defer cancel()
+	bucket, err := hb.HaveBind(write)
+	if err != nil {
+		q.sayHave(fmt.Sprintf("could not bind the have bucket, and wrote nothing there; the next apply or look writes again: %v", err))
+		return
+	}
 	modules := make([]string, 0, len(s.modules))
 	for module := range s.modules {
 		modules = append(modules, module)
@@ -178,10 +192,12 @@ func (q *Queue) writeHaves(ctx context.Context, bus Bus) {
 			q.sayHave(fmt.Sprintf("cannot encode %s: %v", key, err))
 			continue
 		}
-		q.putHave(write, hb, key, raw, func(current []byte) bool {
+		if !q.putHave(write, bucket, key, raw, func(current []byte) bool {
 			c, err := delivery.DecodeHave(current)
 			return err == nil && delivery.SameHave(c, h)
-		})
+		}) {
+			return
+		}
 	}
 	if s.node != nil {
 		key, err := delivery.HaveNodeKey(node)
@@ -189,7 +205,7 @@ func (q *Queue) writeHaves(ctx context.Context, bus Bus) {
 			var raw []byte
 			if raw, err = delivery.EncodeHaveNode(*s.node); err == nil {
 				want := *s.node
-				q.putHave(write, hb, key, raw, func(current []byte) bool {
+				q.putHave(write, bucket, key, raw, func(current []byte) bool {
 					c, err := delivery.DecodeHaveNode(current)
 					return err == nil && delivery.SameHaveNode(c, want)
 				})
@@ -201,19 +217,26 @@ func (q *Queue) writeHaves(ctx context.Context, bus Bus) {
 	}
 }
 
-// putHave reads a key and writes it unless same says it already holds the value.
-func (q *Queue) putHave(ctx context.Context, hb HaveBus, key string, value []byte, same func([]byte) bool) {
-	current, revision, err := hb.HaveGet(ctx, key)
+// putHave reads a key and writes it unless same says it already holds the value. False when the pass should stop: the
+// bucket failed, the error is said once, and the rest wait for the next pass.
+func (q *Queue) putHave(ctx context.Context, b HaveBucket, key string, value []byte, same func([]byte) bool) bool {
+	if ctx.Err() != nil {
+		q.sayHave(fmt.Sprintf("the time for writing the have bucket ran out before %s; the next apply or look writes it", key))
+		return false
+	}
+	current, revision, err := b.Get(ctx, key)
 	switch {
 	case errors.Is(err, ErrHaveAbsent):
 		revision = 0
 	case err != nil:
-		q.sayHave(fmt.Sprintf("could not read %s in the have bucket, and wrote nothing there: %v", key, err))
-		return
+		q.sayHave(fmt.Sprintf("could not read %s in the have bucket, and wrote nothing more there: %v", key, err))
+		return false
 	case same(current):
-		return
+		return true
 	}
-	if err := hb.HaveSet(ctx, key, value, revision); err != nil {
+	if err := b.Set(ctx, key, value, revision); err != nil {
 		q.sayHave(fmt.Sprintf("could not write %s in the have bucket; the next apply or look writes it again: %v", key, err))
+		return false
 	}
+	return true
 }