ENG-8, ENG-9, ENG-11: a failed pull from a store is tried again once per settled of the store's holder, heard on every node, and the failed have says whom it waits on and what triggered the last retry (design 51, issue 1966; needs mesh-sdk request R1, R2)

Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig

cmd/mesh-host/backup_event_test.go

@@ -6,10 +6,13 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/agent"
+	"github.com/novox/mesh-host/internal/apply"
 	"github.com/novox/mesh-host/internal/link"
 )
 
@@ -85,4 +88,62 @@ func TestABackupEndsWhenItsEventSaysSo(t *testing.T) {
 			t.Errorf("%s: asked %d times, %v", row.name, asked, err)
 		}
 	}
+
+	// The events of the same kind (ENG-11): the status view's settled is told to the agent by the module that settled, on
+	// whichever node; a body with no module, or no JSON, tells nothing; the end of the link ends the hearing.
+	told := &settledRecorder{}
+	events := make(chan []byte, 4)
+	for _, body := range []string{`{"node":"n2","module":"distribution"}`, `{"node":"n1"}`, `not json`, `{"module":"gitea"}`} {
+		events <- []byte(body)
+	}
+	linked, end := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() { defer close(done); listen(linked, events, func() settledTold { return told }, func(string) {}) }()
+	deadline := time.Now().Add(2 * time.Second)
+	for len(told.heard()) < 2 && time.Now().Before(deadline) {
+		time.Sleep(5 * time.Millisecond)
+	}
+	end()
+	<-done
+	if got := strings.Join(told.heard(), ","); got != "distribution,gitea" {
+		t.Errorf("the agent was told %q settled, want distribution,gitea", got)
+	}
+	// And the images an apply could not pull are said with the holder the declaration names beside each, none for a public
+	// one (ENG-8, ENG-10).
+	d, err := declaration.ParseFileTrusted([]byte(`{"declaration":1,"resources":[` +
+		`{"id":"a.first","type":"container","name":"first","image":"reg/x@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","image-seat":"mesh-artifact-store","image-holder":"distribution"},` +
+		`{"id":"a.second","type":"container","name":"second","image":"public/y@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	pull := func(image string) error {
+		return &apply.Error{Resource: "x", Err: &apply.PullError{Image: image, Err: errors.New("manifest unknown")}}
+	}
+	if got := pullsOf(d, pull("reg/x@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")); len(got) != 1 || got[0].Seat != "mesh-artifact-store" || got[0].Holder != "distribution" {
+		t.Errorf("a store's image is said as %+v", got)
+	}
+	if got := pullsOf(d, pull("public/y@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")); len(got) != 1 || got[0].Holder != "" || got[0].Seat != "" {
+		t.Errorf("a public image is said as %+v", got)
+	}
+	if got := pullsOf(d, errors.New("another failure")); len(got) != 0 {
+		t.Errorf("a failure that is no pull is said as %+v", got)
+	}
+}
+
+// settledRecorder is the agent a hearing tells.
+type settledRecorder struct {
+	mu   sync.Mutex
+	list []string
+}
+
+func (r *settledRecorder) Settled(holder string) {
+	r.mu.Lock()
+	r.list = append(r.list, holder)
+	r.mu.Unlock()
+}
+
+func (r *settledRecorder) heard() []string {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return append([]string(nil), r.list...)
 }

cmd/mesh-host/main.go

@@ -1064,6 +1064,14 @@ func runLink(ctx context.Context, opts options) error {
 	// And what it trusts, followed on the bus, said in its record whenever it changes (ADR 0550 §2).
 	trust.onChange(func() { factsOfNow(aside, current.get()) })
 	go followTrust(aside, queue, trust, say)
+	// And the status view's `settled`, for a failed pull from a store to be tried again once the store's holder has settled
+	// (design 51, ENG-11).
+	go followSettled(aside, queue, func() settledTold {
+		if ag := current.get(); ag != nil {
+			return ag
+		}
+		return nil
+	}, say)
 	// And the login it dials as, which the vault's node-logins names for this node (novox/hq ADR 0583 §1).
 	rotator := newRotator(queue, opts.state, opts.timeout, say)
 	go followLogins(aside, queue, mine.Node, func(nl delivery.NodeLogin) {

cmd/mesh-host/nodeagent.go

@@ -731,12 +731,19 @@ func applyComposition(ctx context.Context, opts options, c delivery.Composed, ot
 	if got.Refused != "" {
 		return failedWith(got.Refused)
 	}
+	// An image that could not be pulled from a store is said with its holder, for the engine to try the apply again once the
+	// holder has settled (design 51, ENG-8).
+	failedOn := func(why string) agent.Result {
+		res := failedWith(why)
+		res.Pulls = pullsOf(d, got.Err)
+		return res
+	}
 	results := apply.ModuleResults(d, got.Outcome, got.Err)
 	if r, ok := results[name]; ok && r.Failed {
-		return failedWith(r.Detail)
+		return failedOn(r.Detail)
 	}
 	if got.Err != nil {
-		return failedWith(got.Err.Error())
+		return failedOn(got.Err.Error())
 	}
 	if got.Probe != nil {
 		return failedWith(got.Probe.partial().Error())

cmd/mesh-host/pulls.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+
+	"github.com/novox/mesh-host/internal/agent"
+	"github.com/novox/mesh-host/internal/apply"
+)
+
+// pullsOf is the images an apply could not find or pull, each with the seat it came through and that seat's holder as the
+// declaration names them beside the container (design 51 *distribution's first delivery*, ENG-8, ENG-10). An image the
+// declaration names no holder for (a public registry's) carries none: its pull is not tried again on a store settling.
+func pullsOf(d *declaration.Declaration, err error) []agent.PullFailure {
+	var out []agent.PullFailure
+	for _, p := range apply.PullErrors(err) {
+		f := agent.PullFailure{Image: p.Image, Started: p.Started}
+		for _, r := range d.Resources {
+			if c, ok := r.(*declaration.Container); ok && c.Image == p.Image && c.ImageHolder != "" {
+				f.Seat, f.Holder = c.ImageSeat, c.ImageHolder
+				break
+			}
+		}
+		out = append(out, f)
+	}
+	return out
+}

cmd/mesh-host/settled.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/novox/mesh-host/internal/link"
+)
+
+// settledEvent is where the status view says a module has settled on a node: its have reached its want, running and healthy
+// (design 47; the view's `settled`, said once for a module on a node at one composition). The engine hears it for every module
+// and tells its agent, which tries again, once per settled, a failed apply that waits on that module as the holder of the store
+// its image comes from (design 51 *distribution's first delivery*, issue 1966 ENG-11). The grants part grants the engine this
+// subject (NodeAgentHears); a refused subscription is said, and made again when the next link opens.
+const settledEvent = "mesh.mod.mesh-delivery-view.event.settled"
+
+// settledBody is what of the event the engine reads: the module that settled, on whichever node.
+type settledBody struct {
+	Module string `json:"module"`
+}
+
+// settledTold is who is told a holder has settled.
+type settledTold interface{ Settled(holder string) }
+
+// followSettled hears the status view's `settled` on every link, never on a timer, until ctx ends. now names the agent running
+// now (nil while the node agent begins again): a settled heard while there is none is not lost for it, since a new agent applies
+// again every have that failed at its start.
+func followSettled(ctx context.Context, session *link.Session, now func() settledTold, say link.Announce) {
+	said := ""
+	for ctx.Err() == nil {
+		linked, cancel, err := session.WhenLinked(ctx)
+		if err != nil {
+			return
+		}
+		events, stop, herr := session.Hear(settledEvent)
+		if herr != nil {
+			if line := fmt.Sprintf("a store settling is not heard yet (%s: %v): a failed pull waits for the next link, or for compose", settledEvent, herr); line != said {
+				said = line
+				say(line)
+			}
+		} else {
+			said = ""
+			listen(linked, events, now, say)
+			stop()
+		}
+		<-linked.Done()
+		cancel()
+	}
+}
+
+// listen tells the agent each settled it hears, until the link ends.
+func listen(linked context.Context, events <-chan []byte, now func() settledTold, say link.Announce) {
+	for {
+		select {
+		case <-linked.Done():
+			return
+		case raw, ok := <-events:
+			if !ok {
+				return
+			}
+			var b settledBody
+			if json.Unmarshal(raw, &b) != nil || b.Module == "" {
+				continue
+			}
+			if told := now(); told != nil {
+				told.Settled(b.Module)
+			}
+		}
+	}
+}