Design 47 H1: the node-engine forgets a resolution whose fetch the store refused (H-A-146, H-A-150) #240

Merged
mesh-tools merged 3 commits from fix/47-H1-engine-forgets-failed-resolution into main 2026-11-20 15:47:41 +00:00
9 changed files with 237 additions and 6 deletions
+6
View File
@@ -555,6 +555,12 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
if kf := keptFetch.Load(); kf != nil {
actx = apply.WithKept(actx, *kf)
}
// A bundle's fetch the store answered and that failed forgets the resolution of its digest (H-A-150).
actx = apply.WithFetchFailed(actx, func(digest, why string) {
if c := composerNow.Load(); c != nil {
c.ForgetFetched(digest, why)
}
})
// While the grants part's list is followed, the controller's list in the bus module's composition is not
// written: the apply keeps the list on the machine (ADR 0547 clause 8, never both).
held, herr := users.Hold(c)
+6
View File
@@ -75,12 +75,18 @@ func fetchFirst(ctx context.Context, ordered []declaration.Resource, d *declarat
if !seen && !kept {
got.body, got.err = fetchMinding(ctx, source, in.away)
bySource[cacheKey] = got
if ff := fetchFailedFrom(ctx); ff != nil && StoreAnswered(got.err) {
ff(digest, fmt.Sprintf("the apply's fetch of %s failed: %v", source, got.err)) // H-A-150
}
}
if got.err == nil {
sum := sha256.Sum256(got.body)
if have := "sha256:" + hex.EncodeToString(sum[:]); have != digest {
got = fetchedArtifact{err: fmt.Errorf("%s was declared as %s and what arrived is %s; nothing was unpacked",
source, digest, have)}
if ff := fetchFailedFrom(ctx); ff != nil && !seen && !kept {
ff(digest, "the apply's fetch failed: "+got.err.Error()) // H-A-150
}
}
}
if got.err == nil {
+20
View File
@@ -27,3 +27,23 @@ func keptFetchFrom(ctx context.Context) KeptFetch {
f, _ := ctx.Value(keptFetchKey{}).(KeptFetch)
return f
}
// FetchFailed is told the digest of an artifact the store answered for and that failed (an answer other than the bytes, or
// bytes not of the digest), and why: the node's composer then forgets the resolution that named it (design 47 H, the lead's
// answers H-A-146 and H-A-150). A fetch nothing answered for is never told.
type FetchFailed func(digest, why string)
type fetchFailedKey struct{}
// WithFetchFailed is ctx carrying whom an apply tells of a failed fetch.
func WithFetchFailed(ctx context.Context, f FetchFailed) context.Context {
if f == nil {
return ctx
}
return context.WithValue(ctx, fetchFailedKey{}, f)
}
func fetchFailedFrom(ctx context.Context) FetchFailed {
f, _ := ctx.Value(fetchFailedKey{}).(FetchFailed)
return f
}
+12
View File
@@ -63,6 +63,18 @@ type statusError struct {
func (e statusError) Error() string { return fmt.Sprintf("%s answered %s", e.source, e.status) }
// StoreAnswered says the store answered this fetch, other than with the bytes; false for a gateway saying the store is away
// (storeAwayErr). A reader that cannot import this package asks it by this method (H-A-150).
func (e statusError) StoreAnswered() bool { return !storeAwayErr(e) }
// StoreAnswered is whether a fetch failed with the store's own answer (a status other than the bytes, and not a gateway
// saying the store is away): a fetch the store answered no to, which forgets the resolution of its digest (H-A-150). A fetch
// nothing answered for is tried again as before.
func StoreAnswered(err error) bool {
var s statusError
return errors.As(err, &s) && s.StoreAnswered()
}
// storeAwayErr is whether a fetch failed because nothing answered for the store, rather than because
// the store answered no.
func storeAwayErr(err error) bool {
+27 -2
View File
@@ -277,8 +277,11 @@ func TestAStoreThatStaysAwayCostsOneBoundedWait(t *testing.T) {
`","digest":"`+aDigest+`","path":"`+dir+`/`+m+`"}`)
}
d := declare(t, strings.Join(resources, ","))
// H-A-150: the composer is told of a fetch the store answered and that failed, never of one nothing answered for.
var told []string
ctx := WithFetchFailed(context.Background(), func(digest, why string) { told = append(told, digest+" "+why) })
began := time.Now()
_, _, err := ApplyMindingWindows(context.Background(), archHost(t), d, store.State{},
_, _, err := ApplyMindingWindows(ctx, archHost(t), d, store.State{},
store.OriginDeclared, noServices, nil, nil, nil, nil)
if err == nil {
t.Fatal("a store that never came back was said fetched")
@@ -289,6 +292,9 @@ func TestAStoreThatStaysAwayCostsOneBoundedWait(t *testing.T) {
if !strings.Contains(err.Error(), "did not answer") {
t.Fatalf("the failure does not say it waited: %v", err)
}
if len(told) != 0 {
t.Fatalf("a store nothing answered for forgot a resolution: %q", told)
}
// An answer is not an absence: 404 fails at once.
patiently(t, 10*time.Second)
@@ -297,11 +303,30 @@ func TestAStoreThatStaysAwayCostsOneBoundedWait(t *testing.T) {
d = declare(t, `{"id":"x.bundle-code","type":"archive","source":"`+missing.url("x/code")+
`","digest":"`+aDigest+`","path":"`+dir+`/x"}`)
began = time.Now()
if _, _, err := ApplyMindingWindows(context.Background(), archHost(t), d, store.State{},
if _, _, err := ApplyMindingWindows(ctx, archHost(t), d, store.State{},
store.OriginDeclared, noServices, nil, nil, nil, nil); err == nil || !strings.Contains(err.Error(), "404") {
t.Fatalf("a missing blob: %v", err)
}
if len(told) != 1 || !strings.HasPrefix(told[0], aDigest+" the apply's fetch of ") || !strings.Contains(told[0], "404") {
t.Fatalf("the apply's 404 told %q: want its digest forgotten once", told)
}
if time.Since(began) > time.Second || missing.fetches() != 1 {
t.Fatalf("a store that answered no was asked %d times", missing.fetches())
}
// H-A-150: a store that serves other bytes than the declared digest's answered, so the resolution is forgotten: told once
// with the declared digest, though two modules name the same source.
other := newPretendStore(t, []byte("the manifest, not the blob"))
told = nil
d = declare(t, `{"id":"a.bundle-code","type":"archive","source":"`+other.url("a/code")+
`","digest":"`+aDigest+`","path":"`+dir+`/a"},{"id":"b.bundle-code","type":"archive","source":"`+other.url("a/code")+
`","digest":"`+aDigest+`","path":"`+dir+`/b"}`)
if _, _, err := ApplyMindingWindows(ctx, archHost(t), d, store.State{},
store.OriginDeclared, noServices, nil, nil, nil, nil); err == nil || !strings.Contains(err.Error(), "what arrived is") {
t.Fatalf("bytes not of the digest: %v", err)
}
if len(told) != 1 || !strings.HasPrefix(told[0], aDigest+" the apply's fetch failed: ") ||
!strings.Contains(told[0], "what arrived is") {
t.Fatalf("other bytes told %q: want the declared digest forgotten once", told)
}
}
+17 -1
View File
@@ -460,6 +460,7 @@ func (c *Composer) ComposeNow(ctx context.Context) (Pass, error) {
return Pass{}, errors.New("the node's composer runs no loop now: it composes nothing until its agent starts again")
}
c.mu.Lock()
c.dropForgottenLocked("") // a compose by hand asks again what a failed fetch made it forget (H-A-146)
last := c.last
c.mu.Unlock()
if !last.At.IsZero() && time.Since(last.At) < askedWithin {
@@ -857,6 +858,7 @@ func (c *Composer) takeWantLocked(u agent.Update) string {
}
delete(c.notTaken, module)
delete(c.keptWants, module)
c.dropForgottenLocked(module) // its next want asks again what a failed fetch made it forget (H-A-146)
w, err := meshwants.DecodeWantsByWantNodeModuleValue(u.Value)
if u.Deleted || err != nil || w.Removed || w.Fp == "" {
delete(c.wants, module)
@@ -1691,16 +1693,30 @@ func (r reader) Archive(ctx context.Context, url, digest string) ([]byte, error)
defer cancel()
body, err := r.c.env.Fetch(fetching, url)
if r.c.note(err) != nil {
// A resolution whose fetch the store answered and failed is forgotten, and versions asked again (H-A-146); a fetch
// nothing answered for is tried again as before, the module not held for it (H-A-150).
if storeAnswered(err) {
r.c.forgetFetched(digest, fmt.Sprintf("the fetch of %s failed: %v", url, err))
}
return nil, err
}
sum := sha256.Sum256(body)
if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest {
return nil, fmt.Errorf("%s was recorded as %s and what arrived is %s", url, digest, got)
err := fmt.Errorf("%s was recorded as %s and what arrived is %s", url, digest, got)
r.c.forgetFetched(digest, "its fetch failed: "+err.Error())
return nil, err
}
r.c.keepRead(func(k *Composer) { k.archives[digest] = body })
return body, nil
}
// storeAnswered is whether a fetch failed with the store's own answer (the apply's fetch says so by StoreAnswered: a 404, never a
// store nothing answered for), H-A-150.
func storeAnswered(err error) bool {
var a interface{ StoreAnswered() bool }
return errors.As(err, &a) && a.StoreAnswered()
}
// Served reads, of another node's module, the keys named of mesh-config's resolved values, one key each: the node is
// granted those keys alone (buscompose Principal.ServedReads, issue 1126).
func (r reader) Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) {
+1
View File
@@ -212,6 +212,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
t.Run("it reads the catalog half by the fingerprints its wants name", composeFromEntries)
t.Run("a running module without an entry stands as the newest build", runningModuleWithoutEntryStandsAsTheNewestBuild)
t.Run("a want naming its version is resolved through the registry seats", aWantNamingItsVersionIsResolvedThroughTheRegistrySeats)
t.Run("a resolution whose fetch failed is forgotten", aResolutionWhoseFetchFailedIsForgotten)
b := newFakeBus()
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
digest := meshcatalogclient.ManifestDigest([]byte(lemurs))
+51 -3
View File
@@ -3,6 +3,8 @@ package selfcompose
import (
"encoding/json"
"fmt"
"maps"
"slices"
"sort"
"strings"
"time"
@@ -33,10 +35,13 @@ const versionsVerb = "versions"
type resolvedKey struct{ module, fp, version string }
// versionParts is a module's build at a version as resolved: its digests by artifact reference, or why not, and when that was found.
// forgotten marks a resolution whose fetch failed (H-A-146): it stands as a failed one, and is asked again at the module's next want,
// at the next compose by hand, or after resolveAgain.
type versionParts struct {
parts map[string]string
failed string
at time.Time
parts map[string]string
failed string
at time.Time
forgotten bool
}
// registryVersions is the answer of either seat's verb `versions`: the two seats answer the same shape.
@@ -125,6 +130,49 @@ func (c *Composer) resolve(module string, e meshcatalog.CatalogEntry, w meshwant
return parts, ""
}
// forgetFetched forgets every resolution that names digest, as its fetch failed for why (design 47 H, the lead's answer H-A-146; A-430a:
// the bytes are fetched by the digest the versions answer gives): the store's answer may have been wrong, and a resolution kept for
// the process's life would never see it fixed. Each is said in the journal; the module is held until the seat's verb versions is
// asked again, at its next want, at the next compose by hand, or after resolveAgain. A resolution whose fetch succeeded is kept.
func (c *Composer) forgetFetched(digest, why string) {
if digest == "" {
return
}
now := c.now()
var lines []string
c.mu.Lock()
for k, r := range c.resolved {
if r.failed != "" || !slices.Contains(slices.Collect(maps.Values(r.parts)), digest) {
continue
}
line := fmt.Sprintf("resolution of %s %s forgotten: %s", k.module, k.version, why)
c.resolved[k] = versionParts{failed: line + "; the seats' verb versions is asked again at its next want, at the next compose or in " + resolveAgain.String(),
at: now, forgotten: true}
lines = append(lines, line)
}
c.mu.Unlock()
sort.Strings(lines)
for _, line := range lines {
c.env.Say(line)
}
if len(lines) > 0 {
time.AfterFunc(resolveAgain, c.askRehand)
}
}
// ForgetFetched is forgetFetched for the apply's fetch of a bundle the store answered and that failed (H-A-150, apply.FetchFailed).
func (c *Composer) ForgetFetched(digest, why string) { c.forgetFetched(digest, why) }
// dropForgottenLocked ends the standing of every forgotten resolution (H-A-146), of module alone or, for "", of all: the next
// pass asks the seats' verb versions again. Under mu.
func (c *Composer) dropForgottenLocked(module string) {
for k, r := range c.resolved {
if r.forgotten && (module == "" || k.module == module) {
delete(c.resolved, k)
}
}
}
// lookUp asks each artifact's digest at w's version of the seat of its kind. Every failure is said, naming the module, the version
// and the seat; any failure resolves nothing of the build.
func (c *Composer) lookUp(module string, e meshcatalog.CatalogEntry, w meshwants.WantsByWantNodeModuleValue) (map[string]string, string) {
+97
View File
@@ -2,11 +2,16 @@ package selfcompose
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"strings"
"sync"
"testing"
"time"
"github.com/novox/mesh-host/internal/agent"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
@@ -130,3 +135,95 @@ func TestAModulesClaimsAreReadFromTheEntryOfItsWantsBuild(t *testing.T) {
t.Errorf("a module with no want has no manifest read: claims %v, known %v", claims, known)
}
}
// Design 47 H, the lead's answer H-A-146 (A-430a: the bytes are fetched by the digest the versions answer gives): a resolution
// whose fetch failed is forgotten and said so in the journal; the seats' verb versions is asked again at the module's next want, at
// the next compose by hand, and after a bound while the module is held, never before; a resolution whose fetch succeeded is kept.
func aResolutionWhoseFetchFailedIsForgotten(t *testing.T) {
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
body := []byte("the files")
sum := sha256.Sum256(body)
good := "sha256:" + hex.EncodeToString(sum[:])
wrong := "sha256:" + strings.Repeat("e", 64) // the store's answer of the manifest's digest where the blob's was due
entry := meshcatalog.CatalogEntry{Module: "mail", Fp: fp, Manifest: map[string]any{"module": "mail", "version": "1"},
Artifacts: []meshcatalog.Artifact{{Ref: "mail:files", Kind: "archive"}}}
want := meshwants.WantsByWantNodeModuleValue{Fp: fp, Version: "1.2.3", Self: true}
for name, row := range map[string]struct {
answer string // the digest versions answers
fetch error // the store's answer to the fetch
then string // what follows the fetch: "" (a pass within the bound), want, compose, bound
asked bool // versions is asked again
forgets bool // the journal says it forgot
failed bool // the fetch fails though nothing is forgotten
}{
"a fetch that succeeded keeps its resolution": {answer: good, then: "bound"},
"a 404, the next pass within the bound": {answer: wrong, fetch: notFound{}, forgets: true},
"a 404, then the module's next want": {answer: wrong, fetch: notFound{}, then: "want", asked: true, forgets: true},
"a 404, then a compose by hand": {answer: wrong, fetch: notFound{}, then: "compose", asked: true, forgets: true},
"a 404, then the bound while it is held": {answer: wrong, fetch: notFound{}, then: "bound", asked: true, forgets: true},
"bytes that are not of the digest": {answer: wrong, then: "want", asked: true, forgets: true},
// H-A-150: a fetch nothing answered for keeps its resolution, and the module is not held for it.
"a bus that did not answer keeps it": {answer: good, fetch: errors.New("nats: timeout"), failed: true},
"a store away keeps it": {answer: good, fetch: storeAway{}, failed: true},
} {
now := time.Date(2026, 11, 20, 16, 0, 0, 0, time.UTC)
var said []string
env := Env{Node: "desk", Say: func(line string) { said = append(said, line) }, Now: func() time.Time { return now }}
calls := 0
env.Registry = EntryCall(func(context.Context, string, string, any) (json.RawMessage, error) {
calls++
return json.RawMessage(`{"versions":[{"digest":"` + row.answer + `","version":"1.2.3"}]}`), nil
})
env.Fetch = func(context.Context, string) ([]byte, error) { return body, row.fetch }
env.Entry = EntryCall(func(context.Context, string, string, any) (json.RawMessage, error) {
return nil, errors.New("not asked")
})
c := New(env)
c.wants["mail"] = want
c.keptEnt[fp] = keptEntry{entry: entry}
parts, _ := c.partsOf("mail", entry, want)
_, err := reader{c: c}.Archive(context.Background(), referenceOf("mail", "files", parts["mail:files"]), parts["mail:files"])
if (err != nil) != (row.forgets || row.failed) {
t.Errorf("%s: the fetch: %v", name, err)
}
switch row.then {
case "want":
raw, _ := json.Marshal(want)
c.takeWant(agent.Update{Key: meshwants.WantsByWantNodeModuleKey("desk", "mail"), Value: raw})
case "compose":
c.watching.Store(true)
ended, cancel := context.WithCancel(context.Background())
cancel()
c.ComposeNow(ended)
case "bound":
now = now.Add(resolveAgain)
}
before := calls
again, missing := c.partsOf("mail", entry, want)
if asked := calls > before; asked != row.asked {
t.Errorf("%s: versions asked again %v, want %v", name, asked, row.asked)
}
if row.forgets && !row.asked && (again != nil || !strings.Contains(missing, "forgotten")) {
t.Errorf("%s: parts %v, missing %q: want the module held, saying the resolution was forgotten", name, again, missing)
}
if !row.forgets && again["mail:files"] != good {
t.Errorf("%s: parts %v: want the resolution kept", name, again)
}
line := "resolution of mail 1.2.3 forgotten: "
if got := strings.Contains(strings.Join(said, "\n"), line); got != row.forgets {
t.Errorf("%s: said %q; want %q said: %v", name, said, line, row.forgets)
}
}
}
// notFound is the artifact store's answer that it holds no such blob, as the apply's fetch says it (StoreAnswered).
type notFound struct{}
func (notFound) Error() string { return "artifact-store answered 404 Not Found" }
func (notFound) StoreAnswered() bool { return true }
// storeAway is a gateway's answer that the store is away (a 503): no answer of the store's own.
type storeAway struct{}
func (storeAway) Error() string { return "artifact-store answered 503 Service Unavailable" }
func (storeAway) StoreAnswered() bool { return false }