diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 0b5e488c..68b29564 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -555,6 +555,12 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus if kf := keptFetch.Load(); kf != nil { actx = apply.WithKept(actx, *kf) } + // A bundle's fetch the store answered and that failed forgets the resolution of its digest (H-A-150). + actx = apply.WithFetchFailed(actx, func(digest, why string) { + if c := composerNow.Load(); c != nil { + c.ForgetFetched(digest, why) + } + }) // While the grants part's list is followed, the controller's list in the bus module's composition is not // written: the apply keeps the list on the machine (ADR 0547 clause 8, never both). held, herr := users.Hold(c) diff --git a/internal/apply/fetch_first.go b/internal/apply/fetch_first.go index e4645f80..02f651dd 100644 --- a/internal/apply/fetch_first.go +++ b/internal/apply/fetch_first.go @@ -75,12 +75,18 @@ func fetchFirst(ctx context.Context, ordered []declaration.Resource, d *declarat if !seen && !kept { got.body, got.err = fetchMinding(ctx, source, in.away) bySource[cacheKey] = got + if ff := fetchFailedFrom(ctx); ff != nil && StoreAnswered(got.err) { + ff(digest, fmt.Sprintf("the apply's fetch of %s failed: %v", source, got.err)) // H-A-150 + } } if got.err == nil { sum := sha256.Sum256(got.body) if have := "sha256:" + hex.EncodeToString(sum[:]); have != digest { got = fetchedArtifact{err: fmt.Errorf("%s was declared as %s and what arrived is %s; nothing was unpacked", source, digest, have)} + if ff := fetchFailedFrom(ctx); ff != nil && !seen && !kept { + ff(digest, "the apply's fetch failed: "+got.err.Error()) // H-A-150 + } } } if got.err == nil { diff --git a/internal/apply/kept_artifact.go b/internal/apply/kept_artifact.go index 3585470a..75893b67 100644 --- a/internal/apply/kept_artifact.go +++ b/internal/apply/kept_artifact.go @@ -27,3 +27,23 @@ func keptFetchFrom(ctx context.Context) KeptFetch { f, _ := ctx.Value(keptFetchKey{}).(KeptFetch) return f } + +// FetchFailed is told the digest of an artifact the store answered for and that failed (an answer other than the bytes, or +// bytes not of the digest), and why: the node's composer then forgets the resolution that named it (design 47 H, the lead's +// answers H-A-146 and H-A-150). A fetch nothing answered for is never told. +type FetchFailed func(digest, why string) + +type fetchFailedKey struct{} + +// WithFetchFailed is ctx carrying whom an apply tells of a failed fetch. +func WithFetchFailed(ctx context.Context, f FetchFailed) context.Context { + if f == nil { + return ctx + } + return context.WithValue(ctx, fetchFailedKey{}, f) +} + +func fetchFailedFrom(ctx context.Context) FetchFailed { + f, _ := ctx.Value(fetchFailedKey{}).(FetchFailed) + return f +} diff --git a/internal/apply/store_away.go b/internal/apply/store_away.go index 89a78b74..5b50ce24 100644 --- a/internal/apply/store_away.go +++ b/internal/apply/store_away.go @@ -63,6 +63,18 @@ type statusError struct { func (e statusError) Error() string { return fmt.Sprintf("%s answered %s", e.source, e.status) } +// StoreAnswered says the store answered this fetch, other than with the bytes; false for a gateway saying the store is away +// (storeAwayErr). A reader that cannot import this package asks it by this method (H-A-150). +func (e statusError) StoreAnswered() bool { return !storeAwayErr(e) } + +// StoreAnswered is whether a fetch failed with the store's own answer (a status other than the bytes, and not a gateway +// saying the store is away): a fetch the store answered no to, which forgets the resolution of its digest (H-A-150). A fetch +// nothing answered for is tried again as before. +func StoreAnswered(err error) bool { + var s statusError + return errors.As(err, &s) && s.StoreAnswered() +} + // storeAwayErr is whether a fetch failed because nothing answered for the store, rather than because // the store answered no. func storeAwayErr(err error) bool { diff --git a/internal/apply/store_away_test.go b/internal/apply/store_away_test.go index e21d19e0..8dcea3ad 100644 --- a/internal/apply/store_away_test.go +++ b/internal/apply/store_away_test.go @@ -277,8 +277,11 @@ func TestAStoreThatStaysAwayCostsOneBoundedWait(t *testing.T) { `","digest":"`+aDigest+`","path":"`+dir+`/`+m+`"}`) } d := declare(t, strings.Join(resources, ",")) + // H-A-150: the composer is told of a fetch the store answered and that failed, never of one nothing answered for. + var told []string + ctx := WithFetchFailed(context.Background(), func(digest, why string) { told = append(told, digest+" "+why) }) began := time.Now() - _, _, err := ApplyMindingWindows(context.Background(), archHost(t), d, store.State{}, + _, _, err := ApplyMindingWindows(ctx, archHost(t), d, store.State{}, store.OriginDeclared, noServices, nil, nil, nil, nil) if err == nil { t.Fatal("a store that never came back was said fetched") @@ -289,6 +292,9 @@ func TestAStoreThatStaysAwayCostsOneBoundedWait(t *testing.T) { if !strings.Contains(err.Error(), "did not answer") { t.Fatalf("the failure does not say it waited: %v", err) } + if len(told) != 0 { + t.Fatalf("a store nothing answered for forgot a resolution: %q", told) + } // An answer is not an absence: 404 fails at once. patiently(t, 10*time.Second) @@ -297,11 +303,30 @@ func TestAStoreThatStaysAwayCostsOneBoundedWait(t *testing.T) { d = declare(t, `{"id":"x.bundle-code","type":"archive","source":"`+missing.url("x/code")+ `","digest":"`+aDigest+`","path":"`+dir+`/x"}`) began = time.Now() - if _, _, err := ApplyMindingWindows(context.Background(), archHost(t), d, store.State{}, + if _, _, err := ApplyMindingWindows(ctx, archHost(t), d, store.State{}, store.OriginDeclared, noServices, nil, nil, nil, nil); err == nil || !strings.Contains(err.Error(), "404") { t.Fatalf("a missing blob: %v", err) } + if len(told) != 1 || !strings.HasPrefix(told[0], aDigest+" the apply's fetch of ") || !strings.Contains(told[0], "404") { + t.Fatalf("the apply's 404 told %q: want its digest forgotten once", told) + } if time.Since(began) > time.Second || missing.fetches() != 1 { t.Fatalf("a store that answered no was asked %d times", missing.fetches()) } + + // H-A-150: a store that serves other bytes than the declared digest's answered, so the resolution is forgotten: told once + // with the declared digest, though two modules name the same source. + other := newPretendStore(t, []byte("the manifest, not the blob")) + told = nil + d = declare(t, `{"id":"a.bundle-code","type":"archive","source":"`+other.url("a/code")+ + `","digest":"`+aDigest+`","path":"`+dir+`/a"},{"id":"b.bundle-code","type":"archive","source":"`+other.url("a/code")+ + `","digest":"`+aDigest+`","path":"`+dir+`/b"}`) + if _, _, err := ApplyMindingWindows(ctx, archHost(t), d, store.State{}, + store.OriginDeclared, noServices, nil, nil, nil, nil); err == nil || !strings.Contains(err.Error(), "what arrived is") { + t.Fatalf("bytes not of the digest: %v", err) + } + if len(told) != 1 || !strings.HasPrefix(told[0], aDigest+" the apply's fetch failed: ") || + !strings.Contains(told[0], "what arrived is") { + t.Fatalf("other bytes told %q: want the declared digest forgotten once", told) + } } diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 1ad36a7f..8ff12e5a 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -460,6 +460,7 @@ func (c *Composer) ComposeNow(ctx context.Context) (Pass, error) { return Pass{}, errors.New("the node's composer runs no loop now: it composes nothing until its agent starts again") } c.mu.Lock() + c.dropForgottenLocked("") // a compose by hand asks again what a failed fetch made it forget (H-A-146) last := c.last c.mu.Unlock() if !last.At.IsZero() && time.Since(last.At) < askedWithin { @@ -857,6 +858,7 @@ func (c *Composer) takeWantLocked(u agent.Update) string { } delete(c.notTaken, module) delete(c.keptWants, module) + c.dropForgottenLocked(module) // its next want asks again what a failed fetch made it forget (H-A-146) w, err := meshwants.DecodeWantsByWantNodeModuleValue(u.Value) if u.Deleted || err != nil || w.Removed || w.Fp == "" { delete(c.wants, module) @@ -1691,16 +1693,30 @@ func (r reader) Archive(ctx context.Context, url, digest string) ([]byte, error) defer cancel() body, err := r.c.env.Fetch(fetching, url) if r.c.note(err) != nil { + // A resolution whose fetch the store answered and failed is forgotten, and versions asked again (H-A-146); a fetch + // nothing answered for is tried again as before, the module not held for it (H-A-150). + if storeAnswered(err) { + r.c.forgetFetched(digest, fmt.Sprintf("the fetch of %s failed: %v", url, err)) + } return nil, err } sum := sha256.Sum256(body) if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest { - return nil, fmt.Errorf("%s was recorded as %s and what arrived is %s", url, digest, got) + err := fmt.Errorf("%s was recorded as %s and what arrived is %s", url, digest, got) + r.c.forgetFetched(digest, "its fetch failed: "+err.Error()) + return nil, err } r.c.keepRead(func(k *Composer) { k.archives[digest] = body }) return body, nil } +// storeAnswered is whether a fetch failed with the store's own answer (the apply's fetch says so by StoreAnswered: a 404, never a +// store nothing answered for), H-A-150. +func storeAnswered(err error) bool { + var a interface{ StoreAnswered() bool } + return errors.As(err, &a) && a.StoreAnswered() +} + // Served reads, of another node's module, the keys named of mesh-config's resolved values, one key each: the node is // granted those keys alone (buscompose Principal.ServedReads, issue 1126). func (r reader) Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) { diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index 0a924f67..d06377de 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -212,6 +212,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { t.Run("it reads the catalog half by the fingerprints its wants name", composeFromEntries) t.Run("a running module without an entry stands as the newest build", runningModuleWithoutEntryStandsAsTheNewestBuild) t.Run("a want naming its version is resolved through the registry seats", aWantNamingItsVersionIsResolvedThroughTheRegistrySeats) + t.Run("a resolution whose fetch failed is forgotten", aResolutionWhoseFetchFailedIsForgotten) b := newFakeBus() fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64) digest := meshcatalogclient.ManifestDigest([]byte(lemurs)) diff --git a/internal/selfcompose/versions.go b/internal/selfcompose/versions.go index f3bf6060..f60e1688 100644 --- a/internal/selfcompose/versions.go +++ b/internal/selfcompose/versions.go @@ -3,6 +3,8 @@ package selfcompose import ( "encoding/json" "fmt" + "maps" + "slices" "sort" "strings" "time" @@ -33,10 +35,13 @@ const versionsVerb = "versions" type resolvedKey struct{ module, fp, version string } // versionParts is a module's build at a version as resolved: its digests by artifact reference, or why not, and when that was found. +// forgotten marks a resolution whose fetch failed (H-A-146): it stands as a failed one, and is asked again at the module's next want, +// at the next compose by hand, or after resolveAgain. type versionParts struct { - parts map[string]string - failed string - at time.Time + parts map[string]string + failed string + at time.Time + forgotten bool } // registryVersions is the answer of either seat's verb `versions`: the two seats answer the same shape. @@ -125,6 +130,49 @@ func (c *Composer) resolve(module string, e meshcatalog.CatalogEntry, w meshwant return parts, "" } +// forgetFetched forgets every resolution that names digest, as its fetch failed for why (design 47 H, the lead's answer H-A-146; A-430a: +// the bytes are fetched by the digest the versions answer gives): the store's answer may have been wrong, and a resolution kept for +// the process's life would never see it fixed. Each is said in the journal; the module is held until the seat's verb versions is +// asked again, at its next want, at the next compose by hand, or after resolveAgain. A resolution whose fetch succeeded is kept. +func (c *Composer) forgetFetched(digest, why string) { + if digest == "" { + return + } + now := c.now() + var lines []string + c.mu.Lock() + for k, r := range c.resolved { + if r.failed != "" || !slices.Contains(slices.Collect(maps.Values(r.parts)), digest) { + continue + } + line := fmt.Sprintf("resolution of %s %s forgotten: %s", k.module, k.version, why) + c.resolved[k] = versionParts{failed: line + "; the seats' verb versions is asked again at its next want, at the next compose or in " + resolveAgain.String(), + at: now, forgotten: true} + lines = append(lines, line) + } + c.mu.Unlock() + sort.Strings(lines) + for _, line := range lines { + c.env.Say(line) + } + if len(lines) > 0 { + time.AfterFunc(resolveAgain, c.askRehand) + } +} + +// ForgetFetched is forgetFetched for the apply's fetch of a bundle the store answered and that failed (H-A-150, apply.FetchFailed). +func (c *Composer) ForgetFetched(digest, why string) { c.forgetFetched(digest, why) } + +// dropForgottenLocked ends the standing of every forgotten resolution (H-A-146), of module alone or, for "", of all: the next +// pass asks the seats' verb versions again. Under mu. +func (c *Composer) dropForgottenLocked(module string) { + for k, r := range c.resolved { + if r.forgotten && (module == "" || k.module == module) { + delete(c.resolved, k) + } + } +} + // lookUp asks each artifact's digest at w's version of the seat of its kind. Every failure is said, naming the module, the version // and the seat; any failure resolves nothing of the build. func (c *Composer) lookUp(module string, e meshcatalog.CatalogEntry, w meshwants.WantsByWantNodeModuleValue) (map[string]string, string) { diff --git a/internal/selfcompose/versions_test.go b/internal/selfcompose/versions_test.go index 65aab52a..c77bc647 100644 --- a/internal/selfcompose/versions_test.go +++ b/internal/selfcompose/versions_test.go @@ -2,11 +2,16 @@ package selfcompose import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "strings" "sync" "testing" + "time" + + "github.com/novox/mesh-host/internal/agent" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" @@ -130,3 +135,95 @@ func TestAModulesClaimsAreReadFromTheEntryOfItsWantsBuild(t *testing.T) { t.Errorf("a module with no want has no manifest read: claims %v, known %v", claims, known) } } + +// Design 47 H, the lead's answer H-A-146 (A-430a: the bytes are fetched by the digest the versions answer gives): a resolution +// whose fetch failed is forgotten and said so in the journal; the seats' verb versions is asked again at the module's next want, at +// the next compose by hand, and after a bound while the module is held, never before; a resolution whose fetch succeeded is kept. +func aResolutionWhoseFetchFailedIsForgotten(t *testing.T) { + fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64) + body := []byte("the files") + sum := sha256.Sum256(body) + good := "sha256:" + hex.EncodeToString(sum[:]) + wrong := "sha256:" + strings.Repeat("e", 64) // the store's answer of the manifest's digest where the blob's was due + entry := meshcatalog.CatalogEntry{Module: "mail", Fp: fp, Manifest: map[string]any{"module": "mail", "version": "1"}, + Artifacts: []meshcatalog.Artifact{{Ref: "mail:files", Kind: "archive"}}} + want := meshwants.WantsByWantNodeModuleValue{Fp: fp, Version: "1.2.3", Self: true} + for name, row := range map[string]struct { + answer string // the digest versions answers + fetch error // the store's answer to the fetch + then string // what follows the fetch: "" (a pass within the bound), want, compose, bound + asked bool // versions is asked again + forgets bool // the journal says it forgot + failed bool // the fetch fails though nothing is forgotten + }{ + "a fetch that succeeded keeps its resolution": {answer: good, then: "bound"}, + "a 404, the next pass within the bound": {answer: wrong, fetch: notFound{}, forgets: true}, + "a 404, then the module's next want": {answer: wrong, fetch: notFound{}, then: "want", asked: true, forgets: true}, + "a 404, then a compose by hand": {answer: wrong, fetch: notFound{}, then: "compose", asked: true, forgets: true}, + "a 404, then the bound while it is held": {answer: wrong, fetch: notFound{}, then: "bound", asked: true, forgets: true}, + "bytes that are not of the digest": {answer: wrong, then: "want", asked: true, forgets: true}, + // H-A-150: a fetch nothing answered for keeps its resolution, and the module is not held for it. + "a bus that did not answer keeps it": {answer: good, fetch: errors.New("nats: timeout"), failed: true}, + "a store away keeps it": {answer: good, fetch: storeAway{}, failed: true}, + } { + now := time.Date(2026, 11, 20, 16, 0, 0, 0, time.UTC) + var said []string + env := Env{Node: "desk", Say: func(line string) { said = append(said, line) }, Now: func() time.Time { return now }} + calls := 0 + env.Registry = EntryCall(func(context.Context, string, string, any) (json.RawMessage, error) { + calls++ + return json.RawMessage(`{"versions":[{"digest":"` + row.answer + `","version":"1.2.3"}]}`), nil + }) + env.Fetch = func(context.Context, string) ([]byte, error) { return body, row.fetch } + env.Entry = EntryCall(func(context.Context, string, string, any) (json.RawMessage, error) { + return nil, errors.New("not asked") + }) + c := New(env) + c.wants["mail"] = want + c.keptEnt[fp] = keptEntry{entry: entry} + parts, _ := c.partsOf("mail", entry, want) + _, err := reader{c: c}.Archive(context.Background(), referenceOf("mail", "files", parts["mail:files"]), parts["mail:files"]) + if (err != nil) != (row.forgets || row.failed) { + t.Errorf("%s: the fetch: %v", name, err) + } + switch row.then { + case "want": + raw, _ := json.Marshal(want) + c.takeWant(agent.Update{Key: meshwants.WantsByWantNodeModuleKey("desk", "mail"), Value: raw}) + case "compose": + c.watching.Store(true) + ended, cancel := context.WithCancel(context.Background()) + cancel() + c.ComposeNow(ended) + case "bound": + now = now.Add(resolveAgain) + } + before := calls + again, missing := c.partsOf("mail", entry, want) + if asked := calls > before; asked != row.asked { + t.Errorf("%s: versions asked again %v, want %v", name, asked, row.asked) + } + if row.forgets && !row.asked && (again != nil || !strings.Contains(missing, "forgotten")) { + t.Errorf("%s: parts %v, missing %q: want the module held, saying the resolution was forgotten", name, again, missing) + } + if !row.forgets && again["mail:files"] != good { + t.Errorf("%s: parts %v: want the resolution kept", name, again) + } + line := "resolution of mail 1.2.3 forgotten: " + if got := strings.Contains(strings.Join(said, "\n"), line); got != row.forgets { + t.Errorf("%s: said %q; want %q said: %v", name, said, line, row.forgets) + } + } +} + +// notFound is the artifact store's answer that it holds no such blob, as the apply's fetch says it (StoreAnswered). +type notFound struct{} + +func (notFound) Error() string { return "artifact-store answered 404 Not Found" } +func (notFound) StoreAnswered() bool { return true } + +// storeAway is a gateway's answer that the store is away (a 503): no answer of the store's own. +type storeAway struct{} + +func (storeAway) Error() string { return "artifact-store answered 503 Service Unavailable" } +func (storeAway) StoreAnswered() bool { return false }