Design 47 H1: the node-engine forgets a resolution whose fetch the store refused (H-A-146, H-A-150) #240
@@ -555,6 +555,12 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
|
||||
if kf := keptFetch.Load(); kf != nil {
|
||||
actx = apply.WithKept(actx, *kf)
|
||||
}
|
||||
// A bundle's fetch the store answered and that failed forgets the resolution of its digest (H-A-150).
|
||||
actx = apply.WithFetchFailed(actx, func(digest, why string) {
|
||||
if c := composerNow.Load(); c != nil {
|
||||
c.ForgetFetched(digest, why)
|
||||
}
|
||||
})
|
||||
// While the grants part's list is followed, the controller's list in the bus module's composition is not
|
||||
// written: the apply keeps the list on the machine (ADR 0547 clause 8, never both).
|
||||
held, herr := users.Hold(c)
|
||||
|
||||
@@ -75,12 +75,18 @@ func fetchFirst(ctx context.Context, ordered []declaration.Resource, d *declarat
|
||||
if !seen && !kept {
|
||||
got.body, got.err = fetchMinding(ctx, source, in.away)
|
||||
bySource[cacheKey] = got
|
||||
if ff := fetchFailedFrom(ctx); ff != nil && StoreAnswered(got.err) {
|
||||
ff(digest, fmt.Sprintf("the apply's fetch of %s failed: %v", source, got.err)) // H-A-150
|
||||
}
|
||||
}
|
||||
if got.err == nil {
|
||||
sum := sha256.Sum256(got.body)
|
||||
if have := "sha256:" + hex.EncodeToString(sum[:]); have != digest {
|
||||
got = fetchedArtifact{err: fmt.Errorf("%s was declared as %s and what arrived is %s; nothing was unpacked",
|
||||
source, digest, have)}
|
||||
if ff := fetchFailedFrom(ctx); ff != nil && !seen && !kept {
|
||||
ff(digest, "the apply's fetch failed: "+got.err.Error()) // H-A-150
|
||||
}
|
||||
}
|
||||
}
|
||||
if got.err == nil {
|
||||
|
||||
@@ -27,3 +27,23 @@ func keptFetchFrom(ctx context.Context) KeptFetch {
|
||||
f, _ := ctx.Value(keptFetchKey{}).(KeptFetch)
|
||||
return f
|
||||
}
|
||||
|
||||
// FetchFailed is told the digest of an artifact the store answered for and that failed (an answer other than the bytes, or
|
||||
// bytes not of the digest), and why: the node's composer then forgets the resolution that named it (design 47 H, the lead's
|
||||
// answers H-A-146 and H-A-150). A fetch nothing answered for is never told.
|
||||
type FetchFailed func(digest, why string)
|
||||
|
||||
type fetchFailedKey struct{}
|
||||
|
||||
// WithFetchFailed is ctx carrying whom an apply tells of a failed fetch.
|
||||
func WithFetchFailed(ctx context.Context, f FetchFailed) context.Context {
|
||||
if f == nil {
|
||||
return ctx
|
||||
}
|
||||
return context.WithValue(ctx, fetchFailedKey{}, f)
|
||||
}
|
||||
|
||||
func fetchFailedFrom(ctx context.Context) FetchFailed {
|
||||
f, _ := ctx.Value(fetchFailedKey{}).(FetchFailed)
|
||||
return f
|
||||
}
|
||||
|
||||
@@ -63,6 +63,18 @@ type statusError struct {
|
||||
|
||||
func (e statusError) Error() string { return fmt.Sprintf("%s answered %s", e.source, e.status) }
|
||||
|
||||
// StoreAnswered says the store answered this fetch, other than with the bytes; false for a gateway saying the store is away
|
||||
// (storeAwayErr). A reader that cannot import this package asks it by this method (H-A-150).
|
||||
func (e statusError) StoreAnswered() bool { return !storeAwayErr(e) }
|
||||
|
||||
// StoreAnswered is whether a fetch failed with the store's own answer (a status other than the bytes, and not a gateway
|
||||
// saying the store is away): a fetch the store answered no to, which forgets the resolution of its digest (H-A-150). A fetch
|
||||
// nothing answered for is tried again as before.
|
||||
func StoreAnswered(err error) bool {
|
||||
var s statusError
|
||||
return errors.As(err, &s) && s.StoreAnswered()
|
||||
}
|
||||
|
||||
// storeAwayErr is whether a fetch failed because nothing answered for the store, rather than because
|
||||
// the store answered no.
|
||||
func storeAwayErr(err error) bool {
|
||||
|
||||
@@ -277,8 +277,11 @@ func TestAStoreThatStaysAwayCostsOneBoundedWait(t *testing.T) {
|
||||
`","digest":"`+aDigest+`","path":"`+dir+`/`+m+`"}`)
|
||||
}
|
||||
d := declare(t, strings.Join(resources, ","))
|
||||
// H-A-150: the composer is told of a fetch the store answered and that failed, never of one nothing answered for.
|
||||
var told []string
|
||||
ctx := WithFetchFailed(context.Background(), func(digest, why string) { told = append(told, digest+" "+why) })
|
||||
began := time.Now()
|
||||
_, _, err := ApplyMindingWindows(context.Background(), archHost(t), d, store.State{},
|
||||
_, _, err := ApplyMindingWindows(ctx, archHost(t), d, store.State{},
|
||||
store.OriginDeclared, noServices, nil, nil, nil, nil)
|
||||
if err == nil {
|
||||
t.Fatal("a store that never came back was said fetched")
|
||||
@@ -289,6 +292,9 @@ func TestAStoreThatStaysAwayCostsOneBoundedWait(t *testing.T) {
|
||||
if !strings.Contains(err.Error(), "did not answer") {
|
||||
t.Fatalf("the failure does not say it waited: %v", err)
|
||||
}
|
||||
if len(told) != 0 {
|
||||
t.Fatalf("a store nothing answered for forgot a resolution: %q", told)
|
||||
}
|
||||
|
||||
// An answer is not an absence: 404 fails at once.
|
||||
patiently(t, 10*time.Second)
|
||||
@@ -297,11 +303,30 @@ func TestAStoreThatStaysAwayCostsOneBoundedWait(t *testing.T) {
|
||||
d = declare(t, `{"id":"x.bundle-code","type":"archive","source":"`+missing.url("x/code")+
|
||||
`","digest":"`+aDigest+`","path":"`+dir+`/x"}`)
|
||||
began = time.Now()
|
||||
if _, _, err := ApplyMindingWindows(context.Background(), archHost(t), d, store.State{},
|
||||
if _, _, err := ApplyMindingWindows(ctx, archHost(t), d, store.State{},
|
||||
store.OriginDeclared, noServices, nil, nil, nil, nil); err == nil || !strings.Contains(err.Error(), "404") {
|
||||
t.Fatalf("a missing blob: %v", err)
|
||||
}
|
||||
if len(told) != 1 || !strings.HasPrefix(told[0], aDigest+" the apply's fetch of ") || !strings.Contains(told[0], "404") {
|
||||
t.Fatalf("the apply's 404 told %q: want its digest forgotten once", told)
|
||||
}
|
||||
if time.Since(began) > time.Second || missing.fetches() != 1 {
|
||||
t.Fatalf("a store that answered no was asked %d times", missing.fetches())
|
||||
}
|
||||
|
||||
// H-A-150: a store that serves other bytes than the declared digest's answered, so the resolution is forgotten: told once
|
||||
// with the declared digest, though two modules name the same source.
|
||||
other := newPretendStore(t, []byte("the manifest, not the blob"))
|
||||
told = nil
|
||||
d = declare(t, `{"id":"a.bundle-code","type":"archive","source":"`+other.url("a/code")+
|
||||
`","digest":"`+aDigest+`","path":"`+dir+`/a"},{"id":"b.bundle-code","type":"archive","source":"`+other.url("a/code")+
|
||||
`","digest":"`+aDigest+`","path":"`+dir+`/b"}`)
|
||||
if _, _, err := ApplyMindingWindows(ctx, archHost(t), d, store.State{},
|
||||
store.OriginDeclared, noServices, nil, nil, nil, nil); err == nil || !strings.Contains(err.Error(), "what arrived is") {
|
||||
t.Fatalf("bytes not of the digest: %v", err)
|
||||
}
|
||||
if len(told) != 1 || !strings.HasPrefix(told[0], aDigest+" the apply's fetch failed: ") ||
|
||||
!strings.Contains(told[0], "what arrived is") {
|
||||
t.Fatalf("other bytes told %q: want the declared digest forgotten once", told)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -460,6 +460,7 @@ func (c *Composer) ComposeNow(ctx context.Context) (Pass, error) {
|
||||
return Pass{}, errors.New("the node's composer runs no loop now: it composes nothing until its agent starts again")
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.dropForgottenLocked("") // a compose by hand asks again what a failed fetch made it forget (H-A-146)
|
||||
last := c.last
|
||||
c.mu.Unlock()
|
||||
if !last.At.IsZero() && time.Since(last.At) < askedWithin {
|
||||
@@ -857,6 +858,7 @@ func (c *Composer) takeWantLocked(u agent.Update) string {
|
||||
}
|
||||
delete(c.notTaken, module)
|
||||
delete(c.keptWants, module)
|
||||
c.dropForgottenLocked(module) // its next want asks again what a failed fetch made it forget (H-A-146)
|
||||
w, err := meshwants.DecodeWantsByWantNodeModuleValue(u.Value)
|
||||
if u.Deleted || err != nil || w.Removed || w.Fp == "" {
|
||||
delete(c.wants, module)
|
||||
@@ -1691,16 +1693,30 @@ func (r reader) Archive(ctx context.Context, url, digest string) ([]byte, error)
|
||||
defer cancel()
|
||||
body, err := r.c.env.Fetch(fetching, url)
|
||||
if r.c.note(err) != nil {
|
||||
// A resolution whose fetch the store answered and failed is forgotten, and versions asked again (H-A-146); a fetch
|
||||
// nothing answered for is tried again as before, the module not held for it (H-A-150).
|
||||
if storeAnswered(err) {
|
||||
r.c.forgetFetched(digest, fmt.Sprintf("the fetch of %s failed: %v", url, err))
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest {
|
||||
return nil, fmt.Errorf("%s was recorded as %s and what arrived is %s", url, digest, got)
|
||||
err := fmt.Errorf("%s was recorded as %s and what arrived is %s", url, digest, got)
|
||||
r.c.forgetFetched(digest, "its fetch failed: "+err.Error())
|
||||
return nil, err
|
||||
}
|
||||
r.c.keepRead(func(k *Composer) { k.archives[digest] = body })
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// storeAnswered is whether a fetch failed with the store's own answer (the apply's fetch says so by StoreAnswered: a 404, never a
|
||||
// store nothing answered for), H-A-150.
|
||||
func storeAnswered(err error) bool {
|
||||
var a interface{ StoreAnswered() bool }
|
||||
return errors.As(err, &a) && a.StoreAnswered()
|
||||
}
|
||||
|
||||
// Served reads, of another node's module, the keys named of mesh-config's resolved values, one key each: the node is
|
||||
// granted those keys alone (buscompose Principal.ServedReads, issue 1126).
|
||||
func (r reader) Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) {
|
||||
|
||||
@@ -212,6 +212,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
t.Run("it reads the catalog half by the fingerprints its wants name", composeFromEntries)
|
||||
t.Run("a running module without an entry stands as the newest build", runningModuleWithoutEntryStandsAsTheNewestBuild)
|
||||
t.Run("a want naming its version is resolved through the registry seats", aWantNamingItsVersionIsResolvedThroughTheRegistrySeats)
|
||||
t.Run("a resolution whose fetch failed is forgotten", aResolutionWhoseFetchFailedIsForgotten)
|
||||
b := newFakeBus()
|
||||
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
digest := meshcatalogclient.ManifestDigest([]byte(lemurs))
|
||||
|
||||
@@ -3,6 +3,8 @@ package selfcompose
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -33,10 +35,13 @@ const versionsVerb = "versions"
|
||||
type resolvedKey struct{ module, fp, version string }
|
||||
|
||||
// versionParts is a module's build at a version as resolved: its digests by artifact reference, or why not, and when that was found.
|
||||
// forgotten marks a resolution whose fetch failed (H-A-146): it stands as a failed one, and is asked again at the module's next want,
|
||||
// at the next compose by hand, or after resolveAgain.
|
||||
type versionParts struct {
|
||||
parts map[string]string
|
||||
failed string
|
||||
at time.Time
|
||||
parts map[string]string
|
||||
failed string
|
||||
at time.Time
|
||||
forgotten bool
|
||||
}
|
||||
|
||||
// registryVersions is the answer of either seat's verb `versions`: the two seats answer the same shape.
|
||||
@@ -125,6 +130,49 @@ func (c *Composer) resolve(module string, e meshcatalog.CatalogEntry, w meshwant
|
||||
return parts, ""
|
||||
}
|
||||
|
||||
// forgetFetched forgets every resolution that names digest, as its fetch failed for why (design 47 H, the lead's answer H-A-146; A-430a:
|
||||
// the bytes are fetched by the digest the versions answer gives): the store's answer may have been wrong, and a resolution kept for
|
||||
// the process's life would never see it fixed. Each is said in the journal; the module is held until the seat's verb versions is
|
||||
// asked again, at its next want, at the next compose by hand, or after resolveAgain. A resolution whose fetch succeeded is kept.
|
||||
func (c *Composer) forgetFetched(digest, why string) {
|
||||
if digest == "" {
|
||||
return
|
||||
}
|
||||
now := c.now()
|
||||
var lines []string
|
||||
c.mu.Lock()
|
||||
for k, r := range c.resolved {
|
||||
if r.failed != "" || !slices.Contains(slices.Collect(maps.Values(r.parts)), digest) {
|
||||
continue
|
||||
}
|
||||
line := fmt.Sprintf("resolution of %s %s forgotten: %s", k.module, k.version, why)
|
||||
c.resolved[k] = versionParts{failed: line + "; the seats' verb versions is asked again at its next want, at the next compose or in " + resolveAgain.String(),
|
||||
at: now, forgotten: true}
|
||||
lines = append(lines, line)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
sort.Strings(lines)
|
||||
for _, line := range lines {
|
||||
c.env.Say(line)
|
||||
}
|
||||
if len(lines) > 0 {
|
||||
time.AfterFunc(resolveAgain, c.askRehand)
|
||||
}
|
||||
}
|
||||
|
||||
// ForgetFetched is forgetFetched for the apply's fetch of a bundle the store answered and that failed (H-A-150, apply.FetchFailed).
|
||||
func (c *Composer) ForgetFetched(digest, why string) { c.forgetFetched(digest, why) }
|
||||
|
||||
// dropForgottenLocked ends the standing of every forgotten resolution (H-A-146), of module alone or, for "", of all: the next
|
||||
// pass asks the seats' verb versions again. Under mu.
|
||||
func (c *Composer) dropForgottenLocked(module string) {
|
||||
for k, r := range c.resolved {
|
||||
if r.forgotten && (module == "" || k.module == module) {
|
||||
delete(c.resolved, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// lookUp asks each artifact's digest at w's version of the seat of its kind. Every failure is said, naming the module, the version
|
||||
// and the seat; any failure resolves nothing of the build.
|
||||
func (c *Composer) lookUp(module string, e meshcatalog.CatalogEntry, w meshwants.WantsByWantNodeModuleValue) (map[string]string, string) {
|
||||
|
||||
@@ -2,11 +2,16 @@ package selfcompose
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
@@ -130,3 +135,95 @@ func TestAModulesClaimsAreReadFromTheEntryOfItsWantsBuild(t *testing.T) {
|
||||
t.Errorf("a module with no want has no manifest read: claims %v, known %v", claims, known)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H, the lead's answer H-A-146 (A-430a: the bytes are fetched by the digest the versions answer gives): a resolution
|
||||
// whose fetch failed is forgotten and said so in the journal; the seats' verb versions is asked again at the module's next want, at
|
||||
// the next compose by hand, and after a bound while the module is held, never before; a resolution whose fetch succeeded is kept.
|
||||
func aResolutionWhoseFetchFailedIsForgotten(t *testing.T) {
|
||||
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
body := []byte("the files")
|
||||
sum := sha256.Sum256(body)
|
||||
good := "sha256:" + hex.EncodeToString(sum[:])
|
||||
wrong := "sha256:" + strings.Repeat("e", 64) // the store's answer of the manifest's digest where the blob's was due
|
||||
entry := meshcatalog.CatalogEntry{Module: "mail", Fp: fp, Manifest: map[string]any{"module": "mail", "version": "1"},
|
||||
Artifacts: []meshcatalog.Artifact{{Ref: "mail:files", Kind: "archive"}}}
|
||||
want := meshwants.WantsByWantNodeModuleValue{Fp: fp, Version: "1.2.3", Self: true}
|
||||
for name, row := range map[string]struct {
|
||||
answer string // the digest versions answers
|
||||
fetch error // the store's answer to the fetch
|
||||
then string // what follows the fetch: "" (a pass within the bound), want, compose, bound
|
||||
asked bool // versions is asked again
|
||||
forgets bool // the journal says it forgot
|
||||
failed bool // the fetch fails though nothing is forgotten
|
||||
}{
|
||||
"a fetch that succeeded keeps its resolution": {answer: good, then: "bound"},
|
||||
"a 404, the next pass within the bound": {answer: wrong, fetch: notFound{}, forgets: true},
|
||||
"a 404, then the module's next want": {answer: wrong, fetch: notFound{}, then: "want", asked: true, forgets: true},
|
||||
"a 404, then a compose by hand": {answer: wrong, fetch: notFound{}, then: "compose", asked: true, forgets: true},
|
||||
"a 404, then the bound while it is held": {answer: wrong, fetch: notFound{}, then: "bound", asked: true, forgets: true},
|
||||
"bytes that are not of the digest": {answer: wrong, then: "want", asked: true, forgets: true},
|
||||
// H-A-150: a fetch nothing answered for keeps its resolution, and the module is not held for it.
|
||||
"a bus that did not answer keeps it": {answer: good, fetch: errors.New("nats: timeout"), failed: true},
|
||||
"a store away keeps it": {answer: good, fetch: storeAway{}, failed: true},
|
||||
} {
|
||||
now := time.Date(2026, 11, 20, 16, 0, 0, 0, time.UTC)
|
||||
var said []string
|
||||
env := Env{Node: "desk", Say: func(line string) { said = append(said, line) }, Now: func() time.Time { return now }}
|
||||
calls := 0
|
||||
env.Registry = EntryCall(func(context.Context, string, string, any) (json.RawMessage, error) {
|
||||
calls++
|
||||
return json.RawMessage(`{"versions":[{"digest":"` + row.answer + `","version":"1.2.3"}]}`), nil
|
||||
})
|
||||
env.Fetch = func(context.Context, string) ([]byte, error) { return body, row.fetch }
|
||||
env.Entry = EntryCall(func(context.Context, string, string, any) (json.RawMessage, error) {
|
||||
return nil, errors.New("not asked")
|
||||
})
|
||||
c := New(env)
|
||||
c.wants["mail"] = want
|
||||
c.keptEnt[fp] = keptEntry{entry: entry}
|
||||
parts, _ := c.partsOf("mail", entry, want)
|
||||
_, err := reader{c: c}.Archive(context.Background(), referenceOf("mail", "files", parts["mail:files"]), parts["mail:files"])
|
||||
if (err != nil) != (row.forgets || row.failed) {
|
||||
t.Errorf("%s: the fetch: %v", name, err)
|
||||
}
|
||||
switch row.then {
|
||||
case "want":
|
||||
raw, _ := json.Marshal(want)
|
||||
c.takeWant(agent.Update{Key: meshwants.WantsByWantNodeModuleKey("desk", "mail"), Value: raw})
|
||||
case "compose":
|
||||
c.watching.Store(true)
|
||||
ended, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
c.ComposeNow(ended)
|
||||
case "bound":
|
||||
now = now.Add(resolveAgain)
|
||||
}
|
||||
before := calls
|
||||
again, missing := c.partsOf("mail", entry, want)
|
||||
if asked := calls > before; asked != row.asked {
|
||||
t.Errorf("%s: versions asked again %v, want %v", name, asked, row.asked)
|
||||
}
|
||||
if row.forgets && !row.asked && (again != nil || !strings.Contains(missing, "forgotten")) {
|
||||
t.Errorf("%s: parts %v, missing %q: want the module held, saying the resolution was forgotten", name, again, missing)
|
||||
}
|
||||
if !row.forgets && again["mail:files"] != good {
|
||||
t.Errorf("%s: parts %v: want the resolution kept", name, again)
|
||||
}
|
||||
line := "resolution of mail 1.2.3 forgotten: "
|
||||
if got := strings.Contains(strings.Join(said, "\n"), line); got != row.forgets {
|
||||
t.Errorf("%s: said %q; want %q said: %v", name, said, line, row.forgets)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// notFound is the artifact store's answer that it holds no such blob, as the apply's fetch says it (StoreAnswered).
|
||||
type notFound struct{}
|
||||
|
||||
func (notFound) Error() string { return "artifact-store answered 404 Not Found" }
|
||||
func (notFound) StoreAnswered() bool { return true }
|
||||
|
||||
// storeAway is a gateway's answer that the store is away (a 503): no answer of the store's own.
|
||||
type storeAway struct{}
|
||||
|
||||
func (storeAway) Error() string { return "artifact-store answered 503 Service Unavailable" }
|
||||
func (storeAway) StoreAnswered() bool { return false }
|
||||
|
||||
Reference in New Issue
Block a user