diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 0b5e488c..68b29564 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -555,6 +555,12 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus if kf := keptFetch.Load(); kf != nil { actx = apply.WithKept(actx, *kf) } + // A bundle's fetch the store answered and that failed forgets the resolution of its digest (H-A-150). + actx = apply.WithFetchFailed(actx, func(digest, why string) { + if c := composerNow.Load(); c != nil { + c.ForgetFetched(digest, why) + } + }) // While the grants part's list is followed, the controller's list in the bus module's composition is not // written: the apply keeps the list on the machine (ADR 0547 clause 8, never both). held, herr := users.Hold(c) diff --git a/internal/apply/fetch_first.go b/internal/apply/fetch_first.go index e4645f80..02f651dd 100644 --- a/internal/apply/fetch_first.go +++ b/internal/apply/fetch_first.go @@ -75,12 +75,18 @@ func fetchFirst(ctx context.Context, ordered []declaration.Resource, d *declarat if !seen && !kept { got.body, got.err = fetchMinding(ctx, source, in.away) bySource[cacheKey] = got + if ff := fetchFailedFrom(ctx); ff != nil && StoreAnswered(got.err) { + ff(digest, fmt.Sprintf("the apply's fetch of %s failed: %v", source, got.err)) // H-A-150 + } } if got.err == nil { sum := sha256.Sum256(got.body) if have := "sha256:" + hex.EncodeToString(sum[:]); have != digest { got = fetchedArtifact{err: fmt.Errorf("%s was declared as %s and what arrived is %s; nothing was unpacked", source, digest, have)} + if ff := fetchFailedFrom(ctx); ff != nil && !seen && !kept { + ff(digest, "the apply's fetch failed: "+got.err.Error()) // H-A-150 + } } } if got.err == nil { diff --git a/internal/apply/kept_artifact.go b/internal/apply/kept_artifact.go index 3585470a..75893b67 100644 --- a/internal/apply/kept_artifact.go +++ b/internal/apply/kept_artifact.go @@ -27,3 +27,23 @@ func keptFetchFrom(ctx context.Context) KeptFetch { f, _ := ctx.Value(keptFetchKey{}).(KeptFetch) return f } + +// FetchFailed is told the digest of an artifact the store answered for and that failed (an answer other than the bytes, or +// bytes not of the digest), and why: the node's composer then forgets the resolution that named it (design 47 H, the lead's +// answers H-A-146 and H-A-150). A fetch nothing answered for is never told. +type FetchFailed func(digest, why string) + +type fetchFailedKey struct{} + +// WithFetchFailed is ctx carrying whom an apply tells of a failed fetch. +func WithFetchFailed(ctx context.Context, f FetchFailed) context.Context { + if f == nil { + return ctx + } + return context.WithValue(ctx, fetchFailedKey{}, f) +} + +func fetchFailedFrom(ctx context.Context) FetchFailed { + f, _ := ctx.Value(fetchFailedKey{}).(FetchFailed) + return f +} diff --git a/internal/apply/store_away.go b/internal/apply/store_away.go index 89a78b74..5b50ce24 100644 --- a/internal/apply/store_away.go +++ b/internal/apply/store_away.go @@ -63,6 +63,18 @@ type statusError struct { func (e statusError) Error() string { return fmt.Sprintf("%s answered %s", e.source, e.status) } +// StoreAnswered says the store answered this fetch, other than with the bytes; false for a gateway saying the store is away +// (storeAwayErr). A reader that cannot import this package asks it by this method (H-A-150). +func (e statusError) StoreAnswered() bool { return !storeAwayErr(e) } + +// StoreAnswered is whether a fetch failed with the store's own answer (a status other than the bytes, and not a gateway +// saying the store is away): a fetch the store answered no to, which forgets the resolution of its digest (H-A-150). A fetch +// nothing answered for is tried again as before. +func StoreAnswered(err error) bool { + var s statusError + return errors.As(err, &s) && s.StoreAnswered() +} + // storeAwayErr is whether a fetch failed because nothing answered for the store, rather than because // the store answered no. func storeAwayErr(err error) bool { diff --git a/internal/apply/store_away_test.go b/internal/apply/store_away_test.go index e21d19e0..58bd4b22 100644 --- a/internal/apply/store_away_test.go +++ b/internal/apply/store_away_test.go @@ -277,8 +277,11 @@ func TestAStoreThatStaysAwayCostsOneBoundedWait(t *testing.T) { `","digest":"`+aDigest+`","path":"`+dir+`/`+m+`"}`) } d := declare(t, strings.Join(resources, ",")) + // H-A-150: the composer is told of a fetch the store answered and that failed, never of one nothing answered for. + var told []string + ctx := WithFetchFailed(context.Background(), func(digest, why string) { told = append(told, digest+" "+why) }) began := time.Now() - _, _, err := ApplyMindingWindows(context.Background(), archHost(t), d, store.State{}, + _, _, err := ApplyMindingWindows(ctx, archHost(t), d, store.State{}, store.OriginDeclared, noServices, nil, nil, nil, nil) if err == nil { t.Fatal("a store that never came back was said fetched") @@ -289,6 +292,9 @@ func TestAStoreThatStaysAwayCostsOneBoundedWait(t *testing.T) { if !strings.Contains(err.Error(), "did not answer") { t.Fatalf("the failure does not say it waited: %v", err) } + if len(told) != 0 { + t.Fatalf("a store nothing answered for forgot a resolution: %q", told) + } // An answer is not an absence: 404 fails at once. patiently(t, 10*time.Second) @@ -297,10 +303,13 @@ func TestAStoreThatStaysAwayCostsOneBoundedWait(t *testing.T) { d = declare(t, `{"id":"x.bundle-code","type":"archive","source":"`+missing.url("x/code")+ `","digest":"`+aDigest+`","path":"`+dir+`/x"}`) began = time.Now() - if _, _, err := ApplyMindingWindows(context.Background(), archHost(t), d, store.State{}, + if _, _, err := ApplyMindingWindows(ctx, archHost(t), d, store.State{}, store.OriginDeclared, noServices, nil, nil, nil, nil); err == nil || !strings.Contains(err.Error(), "404") { t.Fatalf("a missing blob: %v", err) } + if len(told) != 1 || !strings.HasPrefix(told[0], aDigest+" the apply's fetch of ") || !strings.Contains(told[0], "404") { + t.Fatalf("the apply's 404 told %q: want its digest forgotten once", told) + } if time.Since(began) > time.Second || missing.fetches() != 1 { t.Fatalf("a store that answered no was asked %d times", missing.fetches()) } diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 64feb327..8ff12e5a 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -1693,8 +1693,11 @@ func (r reader) Archive(ctx context.Context, url, digest string) ([]byte, error) defer cancel() body, err := r.c.env.Fetch(fetching, url) if r.c.note(err) != nil { - // A resolution whose fetch failed is forgotten, and versions asked again (H-A-146). - r.c.forgetFetched(digest, fmt.Sprintf("the fetch of %s failed: %v", url, err)) + // A resolution whose fetch the store answered and failed is forgotten, and versions asked again (H-A-146); a fetch + // nothing answered for is tried again as before, the module not held for it (H-A-150). + if storeAnswered(err) { + r.c.forgetFetched(digest, fmt.Sprintf("the fetch of %s failed: %v", url, err)) + } return nil, err } sum := sha256.Sum256(body) @@ -1707,6 +1710,13 @@ func (r reader) Archive(ctx context.Context, url, digest string) ([]byte, error) return body, nil } +// storeAnswered is whether a fetch failed with the store's own answer (the apply's fetch says so by StoreAnswered: a 404, never a +// store nothing answered for), H-A-150. +func storeAnswered(err error) bool { + var a interface{ StoreAnswered() bool } + return errors.As(err, &a) && a.StoreAnswered() +} + // Served reads, of another node's module, the keys named of mesh-config's resolved values, one key each: the node is // granted those keys alone (buscompose Principal.ServedReads, issue 1126). func (r reader) Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) { diff --git a/internal/selfcompose/versions.go b/internal/selfcompose/versions.go index 8baa6fa0..f60e1688 100644 --- a/internal/selfcompose/versions.go +++ b/internal/selfcompose/versions.go @@ -160,6 +160,9 @@ func (c *Composer) forgetFetched(digest, why string) { } } +// ForgetFetched is forgetFetched for the apply's fetch of a bundle the store answered and that failed (H-A-150, apply.FetchFailed). +func (c *Composer) ForgetFetched(digest, why string) { c.forgetFetched(digest, why) } + // dropForgottenLocked ends the standing of every forgotten resolution (H-A-146), of module alone or, for "", of all: the next // pass asks the seats' verb versions again. Under mu. func (c *Composer) dropForgottenLocked(module string) { diff --git a/internal/selfcompose/versions_test.go b/internal/selfcompose/versions_test.go index 2f15afa7..c77bc647 100644 --- a/internal/selfcompose/versions_test.go +++ b/internal/selfcompose/versions_test.go @@ -154,13 +154,17 @@ func aResolutionWhoseFetchFailedIsForgotten(t *testing.T) { then string // what follows the fetch: "" (a pass within the bound), want, compose, bound asked bool // versions is asked again forgets bool // the journal says it forgot + failed bool // the fetch fails though nothing is forgotten }{ "a fetch that succeeded keeps its resolution": {answer: good, then: "bound"}, - "a 404, the next pass within the bound": {answer: wrong, fetch: errors.New("404 Not Found"), forgets: true}, - "a 404, then the module's next want": {answer: wrong, fetch: errors.New("404 Not Found"), then: "want", asked: true, forgets: true}, - "a 404, then a compose by hand": {answer: wrong, fetch: errors.New("404 Not Found"), then: "compose", asked: true, forgets: true}, - "a 404, then the bound while it is held": {answer: wrong, fetch: errors.New("404 Not Found"), then: "bound", asked: true, forgets: true}, + "a 404, the next pass within the bound": {answer: wrong, fetch: notFound{}, forgets: true}, + "a 404, then the module's next want": {answer: wrong, fetch: notFound{}, then: "want", asked: true, forgets: true}, + "a 404, then a compose by hand": {answer: wrong, fetch: notFound{}, then: "compose", asked: true, forgets: true}, + "a 404, then the bound while it is held": {answer: wrong, fetch: notFound{}, then: "bound", asked: true, forgets: true}, "bytes that are not of the digest": {answer: wrong, then: "want", asked: true, forgets: true}, + // H-A-150: a fetch nothing answered for keeps its resolution, and the module is not held for it. + "a bus that did not answer keeps it": {answer: good, fetch: errors.New("nats: timeout"), failed: true}, + "a store away keeps it": {answer: good, fetch: storeAway{}, failed: true}, } { now := time.Date(2026, 11, 20, 16, 0, 0, 0, time.UTC) var said []string @@ -179,7 +183,7 @@ func aResolutionWhoseFetchFailedIsForgotten(t *testing.T) { c.keptEnt[fp] = keptEntry{entry: entry} parts, _ := c.partsOf("mail", entry, want) _, err := reader{c: c}.Archive(context.Background(), referenceOf("mail", "files", parts["mail:files"]), parts["mail:files"]) - if (err != nil) != row.forgets { + if (err != nil) != (row.forgets || row.failed) { t.Errorf("%s: the fetch: %v", name, err) } switch row.then { @@ -211,3 +215,15 @@ func aResolutionWhoseFetchFailedIsForgotten(t *testing.T) { } } } + +// notFound is the artifact store's answer that it holds no such blob, as the apply's fetch says it (StoreAnswered). +type notFound struct{} + +func (notFound) Error() string { return "artifact-store answered 404 Not Found" } +func (notFound) StoreAnswered() bool { return true } + +// storeAway is a gateway's answer that the store is away (a 503): no answer of the store's own. +type storeAway struct{} + +func (storeAway) Error() string { return "artifact-store answered 503 Service Unavailable" } +func (storeAway) StoreAnswered() bool { return false }