H-A-150: forget only on the store's own answer, and on the apply's fetch too
A fetch nothing answered for (a bus or store that did not answer, a gateway saying the store is away) proves nothing about the digest, so it is tried again as before and the module is not held for it; only an answer of the store's (a 404, bytes not of the digest) forgets the resolution. The apply's fetch of a bundle meets the same wrong digest as the take-in's, so it tells the composer too (apply.WithFetchFailed, set in the node agent's apply). Builds: design 47 H H-A-150, H-A-146, G A-430a
This commit is contained in:
@@ -555,6 +555,12 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
|
|||||||
if kf := keptFetch.Load(); kf != nil {
|
if kf := keptFetch.Load(); kf != nil {
|
||||||
actx = apply.WithKept(actx, *kf)
|
actx = apply.WithKept(actx, *kf)
|
||||||
}
|
}
|
||||||
|
// A bundle's fetch the store answered and that failed forgets the resolution of its digest (H-A-150).
|
||||||
|
actx = apply.WithFetchFailed(actx, func(digest, why string) {
|
||||||
|
if c := composerNow.Load(); c != nil {
|
||||||
|
c.ForgetFetched(digest, why)
|
||||||
|
}
|
||||||
|
})
|
||||||
// While the grants part's list is followed, the controller's list in the bus module's composition is not
|
// While the grants part's list is followed, the controller's list in the bus module's composition is not
|
||||||
// written: the apply keeps the list on the machine (ADR 0547 clause 8, never both).
|
// written: the apply keeps the list on the machine (ADR 0547 clause 8, never both).
|
||||||
held, herr := users.Hold(c)
|
held, herr := users.Hold(c)
|
||||||
|
|||||||
@@ -75,12 +75,18 @@ func fetchFirst(ctx context.Context, ordered []declaration.Resource, d *declarat
|
|||||||
if !seen && !kept {
|
if !seen && !kept {
|
||||||
got.body, got.err = fetchMinding(ctx, source, in.away)
|
got.body, got.err = fetchMinding(ctx, source, in.away)
|
||||||
bySource[cacheKey] = got
|
bySource[cacheKey] = got
|
||||||
|
if ff := fetchFailedFrom(ctx); ff != nil && StoreAnswered(got.err) {
|
||||||
|
ff(digest, fmt.Sprintf("the apply's fetch of %s failed: %v", source, got.err)) // H-A-150
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if got.err == nil {
|
if got.err == nil {
|
||||||
sum := sha256.Sum256(got.body)
|
sum := sha256.Sum256(got.body)
|
||||||
if have := "sha256:" + hex.EncodeToString(sum[:]); have != digest {
|
if have := "sha256:" + hex.EncodeToString(sum[:]); have != digest {
|
||||||
got = fetchedArtifact{err: fmt.Errorf("%s was declared as %s and what arrived is %s; nothing was unpacked",
|
got = fetchedArtifact{err: fmt.Errorf("%s was declared as %s and what arrived is %s; nothing was unpacked",
|
||||||
source, digest, have)}
|
source, digest, have)}
|
||||||
|
if ff := fetchFailedFrom(ctx); ff != nil && !seen && !kept {
|
||||||
|
ff(digest, "the apply's fetch failed: "+got.err.Error()) // H-A-150
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if got.err == nil {
|
if got.err == nil {
|
||||||
|
|||||||
@@ -27,3 +27,23 @@ func keptFetchFrom(ctx context.Context) KeptFetch {
|
|||||||
f, _ := ctx.Value(keptFetchKey{}).(KeptFetch)
|
f, _ := ctx.Value(keptFetchKey{}).(KeptFetch)
|
||||||
return f
|
return f
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FetchFailed is told the digest of an artifact the store answered for and that failed (an answer other than the bytes, or
|
||||||
|
// bytes not of the digest), and why: the node's composer then forgets the resolution that named it (design 47 H, the lead's
|
||||||
|
// answers H-A-146 and H-A-150). A fetch nothing answered for is never told.
|
||||||
|
type FetchFailed func(digest, why string)
|
||||||
|
|
||||||
|
type fetchFailedKey struct{}
|
||||||
|
|
||||||
|
// WithFetchFailed is ctx carrying whom an apply tells of a failed fetch.
|
||||||
|
func WithFetchFailed(ctx context.Context, f FetchFailed) context.Context {
|
||||||
|
if f == nil {
|
||||||
|
return ctx
|
||||||
|
}
|
||||||
|
return context.WithValue(ctx, fetchFailedKey{}, f)
|
||||||
|
}
|
||||||
|
|
||||||
|
func fetchFailedFrom(ctx context.Context) FetchFailed {
|
||||||
|
f, _ := ctx.Value(fetchFailedKey{}).(FetchFailed)
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|||||||
@@ -63,6 +63,18 @@ type statusError struct {
|
|||||||
|
|
||||||
func (e statusError) Error() string { return fmt.Sprintf("%s answered %s", e.source, e.status) }
|
func (e statusError) Error() string { return fmt.Sprintf("%s answered %s", e.source, e.status) }
|
||||||
|
|
||||||
|
// StoreAnswered says the store answered this fetch, other than with the bytes; false for a gateway saying the store is away
|
||||||
|
// (storeAwayErr). A reader that cannot import this package asks it by this method (H-A-150).
|
||||||
|
func (e statusError) StoreAnswered() bool { return !storeAwayErr(e) }
|
||||||
|
|
||||||
|
// StoreAnswered is whether a fetch failed with the store's own answer (a status other than the bytes, and not a gateway
|
||||||
|
// saying the store is away): a fetch the store answered no to, which forgets the resolution of its digest (H-A-150). A fetch
|
||||||
|
// nothing answered for is tried again as before.
|
||||||
|
func StoreAnswered(err error) bool {
|
||||||
|
var s statusError
|
||||||
|
return errors.As(err, &s) && s.StoreAnswered()
|
||||||
|
}
|
||||||
|
|
||||||
// storeAwayErr is whether a fetch failed because nothing answered for the store, rather than because
|
// storeAwayErr is whether a fetch failed because nothing answered for the store, rather than because
|
||||||
// the store answered no.
|
// the store answered no.
|
||||||
func storeAwayErr(err error) bool {
|
func storeAwayErr(err error) bool {
|
||||||
|
|||||||
@@ -277,8 +277,11 @@ func TestAStoreThatStaysAwayCostsOneBoundedWait(t *testing.T) {
|
|||||||
`","digest":"`+aDigest+`","path":"`+dir+`/`+m+`"}`)
|
`","digest":"`+aDigest+`","path":"`+dir+`/`+m+`"}`)
|
||||||
}
|
}
|
||||||
d := declare(t, strings.Join(resources, ","))
|
d := declare(t, strings.Join(resources, ","))
|
||||||
|
// H-A-150: the composer is told of a fetch the store answered and that failed, never of one nothing answered for.
|
||||||
|
var told []string
|
||||||
|
ctx := WithFetchFailed(context.Background(), func(digest, why string) { told = append(told, digest+" "+why) })
|
||||||
began := time.Now()
|
began := time.Now()
|
||||||
_, _, err := ApplyMindingWindows(context.Background(), archHost(t), d, store.State{},
|
_, _, err := ApplyMindingWindows(ctx, archHost(t), d, store.State{},
|
||||||
store.OriginDeclared, noServices, nil, nil, nil, nil)
|
store.OriginDeclared, noServices, nil, nil, nil, nil)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("a store that never came back was said fetched")
|
t.Fatal("a store that never came back was said fetched")
|
||||||
@@ -289,6 +292,9 @@ func TestAStoreThatStaysAwayCostsOneBoundedWait(t *testing.T) {
|
|||||||
if !strings.Contains(err.Error(), "did not answer") {
|
if !strings.Contains(err.Error(), "did not answer") {
|
||||||
t.Fatalf("the failure does not say it waited: %v", err)
|
t.Fatalf("the failure does not say it waited: %v", err)
|
||||||
}
|
}
|
||||||
|
if len(told) != 0 {
|
||||||
|
t.Fatalf("a store nothing answered for forgot a resolution: %q", told)
|
||||||
|
}
|
||||||
|
|
||||||
// An answer is not an absence: 404 fails at once.
|
// An answer is not an absence: 404 fails at once.
|
||||||
patiently(t, 10*time.Second)
|
patiently(t, 10*time.Second)
|
||||||
@@ -297,10 +303,13 @@ func TestAStoreThatStaysAwayCostsOneBoundedWait(t *testing.T) {
|
|||||||
d = declare(t, `{"id":"x.bundle-code","type":"archive","source":"`+missing.url("x/code")+
|
d = declare(t, `{"id":"x.bundle-code","type":"archive","source":"`+missing.url("x/code")+
|
||||||
`","digest":"`+aDigest+`","path":"`+dir+`/x"}`)
|
`","digest":"`+aDigest+`","path":"`+dir+`/x"}`)
|
||||||
began = time.Now()
|
began = time.Now()
|
||||||
if _, _, err := ApplyMindingWindows(context.Background(), archHost(t), d, store.State{},
|
if _, _, err := ApplyMindingWindows(ctx, archHost(t), d, store.State{},
|
||||||
store.OriginDeclared, noServices, nil, nil, nil, nil); err == nil || !strings.Contains(err.Error(), "404") {
|
store.OriginDeclared, noServices, nil, nil, nil, nil); err == nil || !strings.Contains(err.Error(), "404") {
|
||||||
t.Fatalf("a missing blob: %v", err)
|
t.Fatalf("a missing blob: %v", err)
|
||||||
}
|
}
|
||||||
|
if len(told) != 1 || !strings.HasPrefix(told[0], aDigest+" the apply's fetch of ") || !strings.Contains(told[0], "404") {
|
||||||
|
t.Fatalf("the apply's 404 told %q: want its digest forgotten once", told)
|
||||||
|
}
|
||||||
if time.Since(began) > time.Second || missing.fetches() != 1 {
|
if time.Since(began) > time.Second || missing.fetches() != 1 {
|
||||||
t.Fatalf("a store that answered no was asked %d times", missing.fetches())
|
t.Fatalf("a store that answered no was asked %d times", missing.fetches())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1693,8 +1693,11 @@ func (r reader) Archive(ctx context.Context, url, digest string) ([]byte, error)
|
|||||||
defer cancel()
|
defer cancel()
|
||||||
body, err := r.c.env.Fetch(fetching, url)
|
body, err := r.c.env.Fetch(fetching, url)
|
||||||
if r.c.note(err) != nil {
|
if r.c.note(err) != nil {
|
||||||
// A resolution whose fetch failed is forgotten, and versions asked again (H-A-146).
|
// A resolution whose fetch the store answered and failed is forgotten, and versions asked again (H-A-146); a fetch
|
||||||
|
// nothing answered for is tried again as before, the module not held for it (H-A-150).
|
||||||
|
if storeAnswered(err) {
|
||||||
r.c.forgetFetched(digest, fmt.Sprintf("the fetch of %s failed: %v", url, err))
|
r.c.forgetFetched(digest, fmt.Sprintf("the fetch of %s failed: %v", url, err))
|
||||||
|
}
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
sum := sha256.Sum256(body)
|
sum := sha256.Sum256(body)
|
||||||
@@ -1707,6 +1710,13 @@ func (r reader) Archive(ctx context.Context, url, digest string) ([]byte, error)
|
|||||||
return body, nil
|
return body, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// storeAnswered is whether a fetch failed with the store's own answer (the apply's fetch says so by StoreAnswered: a 404, never a
|
||||||
|
// store nothing answered for), H-A-150.
|
||||||
|
func storeAnswered(err error) bool {
|
||||||
|
var a interface{ StoreAnswered() bool }
|
||||||
|
return errors.As(err, &a) && a.StoreAnswered()
|
||||||
|
}
|
||||||
|
|
||||||
// Served reads, of another node's module, the keys named of mesh-config's resolved values, one key each: the node is
|
// Served reads, of another node's module, the keys named of mesh-config's resolved values, one key each: the node is
|
||||||
// granted those keys alone (buscompose Principal.ServedReads, issue 1126).
|
// granted those keys alone (buscompose Principal.ServedReads, issue 1126).
|
||||||
func (r reader) Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) {
|
func (r reader) Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) {
|
||||||
|
|||||||
@@ -160,6 +160,9 @@ func (c *Composer) forgetFetched(digest, why string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ForgetFetched is forgetFetched for the apply's fetch of a bundle the store answered and that failed (H-A-150, apply.FetchFailed).
|
||||||
|
func (c *Composer) ForgetFetched(digest, why string) { c.forgetFetched(digest, why) }
|
||||||
|
|
||||||
// dropForgottenLocked ends the standing of every forgotten resolution (H-A-146), of module alone or, for "", of all: the next
|
// dropForgottenLocked ends the standing of every forgotten resolution (H-A-146), of module alone or, for "", of all: the next
|
||||||
// pass asks the seats' verb versions again. Under mu.
|
// pass asks the seats' verb versions again. Under mu.
|
||||||
func (c *Composer) dropForgottenLocked(module string) {
|
func (c *Composer) dropForgottenLocked(module string) {
|
||||||
|
|||||||
@@ -154,13 +154,17 @@ func aResolutionWhoseFetchFailedIsForgotten(t *testing.T) {
|
|||||||
then string // what follows the fetch: "" (a pass within the bound), want, compose, bound
|
then string // what follows the fetch: "" (a pass within the bound), want, compose, bound
|
||||||
asked bool // versions is asked again
|
asked bool // versions is asked again
|
||||||
forgets bool // the journal says it forgot
|
forgets bool // the journal says it forgot
|
||||||
|
failed bool // the fetch fails though nothing is forgotten
|
||||||
}{
|
}{
|
||||||
"a fetch that succeeded keeps its resolution": {answer: good, then: "bound"},
|
"a fetch that succeeded keeps its resolution": {answer: good, then: "bound"},
|
||||||
"a 404, the next pass within the bound": {answer: wrong, fetch: errors.New("404 Not Found"), forgets: true},
|
"a 404, the next pass within the bound": {answer: wrong, fetch: notFound{}, forgets: true},
|
||||||
"a 404, then the module's next want": {answer: wrong, fetch: errors.New("404 Not Found"), then: "want", asked: true, forgets: true},
|
"a 404, then the module's next want": {answer: wrong, fetch: notFound{}, then: "want", asked: true, forgets: true},
|
||||||
"a 404, then a compose by hand": {answer: wrong, fetch: errors.New("404 Not Found"), then: "compose", asked: true, forgets: true},
|
"a 404, then a compose by hand": {answer: wrong, fetch: notFound{}, then: "compose", asked: true, forgets: true},
|
||||||
"a 404, then the bound while it is held": {answer: wrong, fetch: errors.New("404 Not Found"), then: "bound", asked: true, forgets: true},
|
"a 404, then the bound while it is held": {answer: wrong, fetch: notFound{}, then: "bound", asked: true, forgets: true},
|
||||||
"bytes that are not of the digest": {answer: wrong, then: "want", asked: true, forgets: true},
|
"bytes that are not of the digest": {answer: wrong, then: "want", asked: true, forgets: true},
|
||||||
|
// H-A-150: a fetch nothing answered for keeps its resolution, and the module is not held for it.
|
||||||
|
"a bus that did not answer keeps it": {answer: good, fetch: errors.New("nats: timeout"), failed: true},
|
||||||
|
"a store away keeps it": {answer: good, fetch: storeAway{}, failed: true},
|
||||||
} {
|
} {
|
||||||
now := time.Date(2026, 11, 20, 16, 0, 0, 0, time.UTC)
|
now := time.Date(2026, 11, 20, 16, 0, 0, 0, time.UTC)
|
||||||
var said []string
|
var said []string
|
||||||
@@ -179,7 +183,7 @@ func aResolutionWhoseFetchFailedIsForgotten(t *testing.T) {
|
|||||||
c.keptEnt[fp] = keptEntry{entry: entry}
|
c.keptEnt[fp] = keptEntry{entry: entry}
|
||||||
parts, _ := c.partsOf("mail", entry, want)
|
parts, _ := c.partsOf("mail", entry, want)
|
||||||
_, err := reader{c: c}.Archive(context.Background(), referenceOf("mail", "files", parts["mail:files"]), parts["mail:files"])
|
_, err := reader{c: c}.Archive(context.Background(), referenceOf("mail", "files", parts["mail:files"]), parts["mail:files"])
|
||||||
if (err != nil) != row.forgets {
|
if (err != nil) != (row.forgets || row.failed) {
|
||||||
t.Errorf("%s: the fetch: %v", name, err)
|
t.Errorf("%s: the fetch: %v", name, err)
|
||||||
}
|
}
|
||||||
switch row.then {
|
switch row.then {
|
||||||
@@ -211,3 +215,15 @@ func aResolutionWhoseFetchFailedIsForgotten(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// notFound is the artifact store's answer that it holds no such blob, as the apply's fetch says it (StoreAnswered).
|
||||||
|
type notFound struct{}
|
||||||
|
|
||||||
|
func (notFound) Error() string { return "artifact-store answered 404 Not Found" }
|
||||||
|
func (notFound) StoreAnswered() bool { return true }
|
||||||
|
|
||||||
|
// storeAway is a gateway's answer that the store is away (a 503): no answer of the store's own.
|
||||||
|
type storeAway struct{}
|
||||||
|
|
||||||
|
func (storeAway) Error() string { return "artifact-store answered 503 Service Unavailable" }
|
||||||
|
func (storeAway) StoreAnswered() bool { return false }
|
||||||
|
|||||||
Reference in New Issue
Block a user