Merge pull request 'Design 47 G start: node-engine fixes placed by hand (A-226b/c, A-237b/c, manifest digest)' (#230) from fix/47-G-start-2-host into main

This commit was merged in pull request #230.
This commit is contained in:
2026-11-16 14:15:10 +00:00
24 changed files with 986 additions and 64 deletions
+38 -8
View File
@@ -6,6 +6,7 @@ import (
"encoding/json"
"errors"
"fmt"
"log"
"os"
"path/filepath"
"strings"
@@ -17,6 +18,7 @@ import (
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"github.com/novox/mesh-host/internal/accounts"
"github.com/novox/mesh-host/internal/agent"
"github.com/novox/mesh-host/internal/apply"
@@ -378,9 +380,9 @@ func packagesFrom(node string, session *link.Session, composer *atomic.Pointer[s
// nec.AnswerBound, selfcompose candidateWithin). The answer is
// the seat's shape (nodeengine.ComposeCandidateAnswer), checked as it is encoded; a node whose shadow does not run yet
// answers unready, which the checker takes as no answer.
func composeCandidateFrom(node string, shadow *atomic.Pointer[selfcompose.Shadow]) func(candidate string, input bool) (any, error) {
return func(candidate string, input bool) (any, error) {
ask := nodeengine.ComposeCandidateInput{Candidate: candidate, Input: input}
func composeCandidateFrom(node string, shadow *atomic.Pointer[selfcompose.Shadow]) func(candidate string, input bool, call string, line buscalls.ChunkLine) (any, error) {
return func(candidate string, input bool, call string, line buscalls.ChunkLine) (any, error) {
ask := nodeengine.ComposeCandidateInput{Candidate: candidate, Input: input, Call: call}
if _, err := nec.EncodeAsk(ask); err != nil {
return nil, err
}
@@ -394,11 +396,7 @@ func composeCandidateFrom(node string, shadow *atomic.Pointer[selfcompose.Shadow
defer cancel()
a = s.ComposeCandidate(ctx, ask)
}
raw, err := nec.EncodeAnswer(a)
if err != nil {
return nil, fmt.Errorf("this node's answer to %s does not fit the seat's shape: %w", candidate, err)
}
return json.RawMessage(raw), nil
return answerComposeCandidate(a, candidate, input, call, line)
}
}
@@ -425,3 +423,35 @@ func entryAgainFrom(composer *atomic.Pointer[selfcompose.Composer]) func(ctx con
return c.EntryAgain(ctx, fingerprint)
}
}
// answerComposeCandidate encodes the answer of compose-candidate. The verb is marked chunks (A-237b, A-237c): asked with input and a
// call token, the composition input goes in chunks over the link's connection, whatever its size, and the answer names them;
// asked with no token (the checker's, until G removes it, G-6 §321, A-237c) it stays inline in the answer.
func answerComposeCandidate(a nodeengine.ComposeCandidateAnswer, candidate string, input bool, call string, line buscalls.ChunkLine) (any, error) {
var chunked []byte
if input && call != "" && a.Input != nil {
if line == nil {
return nil, errors.New("this engine has no bus connection to send the composition input on")
}
body, err := json.Marshal(a.Input)
if err != nil {
return nil, fmt.Errorf("the composition input of %s cannot be encoded: %w", candidate, err)
}
chunked, a.Input = body, nil
a.Chunks = int64((len(body) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes)
}
raw, err := nec.EncodeAnswer(a)
if err != nil {
return nil, fmt.Errorf("this node's answer to %s does not fit the seat's shape: %w", candidate, err)
}
if chunked != nil {
send := buscalls.SeatChunks{Seat: nodeengine.Seat, Verb: nodeengine.VerbComposeCandidate, Call: call}
// The answer names the chunks first; the send runs after it and ends at a chunk the caller does not acknowledge.
go func() {
if err := send.Send(line, chunked, buscalls.SeatAckWithin, log.Printf); err != nil {
log.Printf("the send %s of the composition input of %s: %v", call, candidate, err)
}
}()
}
return json.RawMessage(raw), nil
}
+20
View File
@@ -992,6 +992,11 @@ func runLink(ctx context.Context, opts options) error {
// The engine's own module, first from its applied record of the build whose fingerprint is its own (design 47 G §527,
// §634, A-149); on a first start with no such record, from its first composing.
own.Set(ownModuleApplied(appliedStoreFor(opts.state), os.Getenv(compositions.FingerprintVar)))
// The live start's recovery (A-404): a first start whose composing never ends is named by its unit's environment, so it
// serves its seat (root-free); the name the assignments place on this node. Ended by the lookup from the bus after G.
if own.Get() == "" {
own.Set(os.Getenv("MESH_ENGINE_OWN_MODULE"))
}
queue := &link.Session{
Own: own,
Membership: membership,
@@ -1257,6 +1262,7 @@ const (
// anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240
// rule 6) — but for going on with a take act stopped at its bound (ADR 0328 §7), which runs beside it.
func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce, reapply, reconcile func(modules ...string)) {
say("the looks of what runs and of the accounts start (A-405)")
var missingBefore, inactiveBefore map[string]bool
ticker := time.NewTicker(liveness.LookEvery)
defer ticker.Stop()
@@ -1279,8 +1285,16 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
case <-nudged:
owed = true
}
first := lastAccounts.Load() == nil
began := time.Now()
if first {
say("the first look of what runs begins (A-405)")
}
st, changed := j.Look(ctx)
owed = owed || changed
if first {
say(fmt.Sprintf("the first look of what runs took %s; units and accounts next (A-405)", time.Since(began).Round(time.Millisecond)))
}
var unitSt *units.Statement
if u := unitJudge; u != nil {
us, unitsChanged := u.Look(ctx)
@@ -1294,9 +1308,15 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
}
var accountSt *accounts.Statement
if a := accountJudge; a != nil {
if first {
say(fmt.Sprintf("the first look of the accounts begins at %s (A-405)", time.Since(began).Round(time.Millisecond)))
}
as, accountsChanged := a.Look(ctx)
accountSt = &as
lastAccounts.Store(&as)
if first {
say(fmt.Sprintf("the first look of the accounts ended at %s: %d accounts (A-405)", time.Since(began).Round(time.Millisecond), len(as.Accounts)))
}
owed = owed || accountsChanged
if accountsChanged {
for _, v := range as.Accounts {
+2
View File
@@ -1200,6 +1200,8 @@ func composerEnv(session *link.Session, node string, say func(string), hooks com
// And the shadow too: the catalog half read by the fingerprints its wants and runnings name, over its own calls of the seat mesh-catalog's
// verb entry on its link session (design 47 G §421): neither the indexer's nor the builder's states are read.
env.Entry = selfcompose.EntryCall(session.AskMeshSeat)
// And the registry seats' verb versions, to resolve a want that names its version to its artifacts' digests (A-430a).
env.Registry = selfcompose.EntryCall(session.AskMeshSeat)
return env
}
+74 -2
View File
@@ -29,6 +29,7 @@ import (
"time"
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/declaration"
"git.novox.be/novox/mesh-sdk/go/seatlist"
@@ -702,7 +703,10 @@ func h8candidate(t *testing.T, b *itBed, fp, running string) {
t.Fatal(err)
}
path := filepath.Join(b.dir, "web.conf")
man := `{"module":"web","version":"2","config":{"greeting":{"type":"string","description":"what the file says","default":"hello"}},` +
// A description past what one answer carries (900 KiB, and under the bus's 1 MiB for one value), so the composition input this
// node sends is over the inline bound (A-237b, A-237c).
pad := strings.Repeat("x", 930_000)
man := `{"module":"web","version":"2","config":{"greeting":{"type":"string","description":"what the file says ` + pad + `","default":"hello"}},` +
`"resources":[{"id":"conf","type":"file","path":"` + path + `","mode":"0640","content":"greeting=${config:greeting}\n"},` +
`{"id":"tools","type":"package","package":"web-tools"}]}`
digest := meshcatalogclient.ManifestDigest([]byte(man))
@@ -728,7 +732,43 @@ func h8candidate(t *testing.T, b *itBed, fp, running string) {
t.Fatal(err)
}
before, _ := b.composed.Status()
got, err := composeCandidateFrom(itNode, &shadowNow)(key, false)
// A-237c: asked with input and no call token (the checker's, until G removes it, G-6 §321) an input past what one answer carries
// is refused; asked with a token it is sent in chunks through the engine's call, whatever its size (A-237b).
if over, err := composeCandidateFrom(itNode, &shadowNow)(key, true, "", nil); err != nil {
t.Fatal(err)
} else if a, err := nec.DecodeAnswer(over.(json.RawMessage)); err != nil || a.Verdict != nec.Refused || !strings.Contains(a.Refused, "an answer on the bus carries") || a.Input != nil {
t.Errorf("an input over the inline bound asked with no token answered %+v (%v)", a, err)
}
line := &ackingLine{chunks: map[string][]byte{}, done: make(chan struct{})}
const call = "0123456789abcdef0123456789abcdef"
acts := &link.HandActs{Node: itNode, ComposeCandidate: composeCandidateFrom(itNode, &shadowNow)}
chunked, err := acts.Call(link.EngineCall{Verb: link.ActComposeCandidate, Line: line,
Args: map[string]any{"candidate": key, "input": true, "call": call}})
if err != nil {
t.Fatal(err)
}
ca, err := nec.DecodeAnswer(chunked.(json.RawMessage))
if err != nil || ca.Verdict != nec.Composed || ca.Chunks < 2 || ca.Input != nil {
t.Fatalf("the input asked with a token answered %+v (%v), want the chunks and no inline input", ca, err)
}
line.want = int(ca.Chunks)
select {
case <-line.done:
case <-time.After(10 * time.Second):
t.Fatalf("%d of %d chunks were sent", line.sent(), ca.Chunks)
}
var file []byte
for n := int64(1); n <= ca.Chunks; n++ {
var c buscalls.SeatChunk
if err := json.Unmarshal(line.chunks[buscalls.SeatChunks{Seat: nodeengine.Seat, Verb: nodeengine.VerbComposeCandidate, Call: call}.Chunk(n)], &c); err != nil {
t.Fatal(err)
}
file = append(file, c.Data...)
}
if len(file) <= 900<<10 || !json.Valid(file) {
t.Errorf("the chunks held %d bytes, want a JSON input over 900 KiB", len(file))
}
got, err := composeCandidateFrom(itNode, &shadowNow)(key, false, "", nil)
if err != nil {
t.Fatal(err)
}
@@ -1237,3 +1277,35 @@ func TestNatsTheFirstReadOfTheSeatListEndsTheNodesWait(t *testing.T) {
t.Error("the list was read by the reader of a run begun since, not by the one that met no responders")
}
}
// ackingLine is a bus connection that acknowledges every chunk published on it, as a caller does, and keeps the chunks.
type ackingLine struct {
mu sync.Mutex
onAck nats.MsgHandler
chunks map[string][]byte
done chan struct{}
want int
closed bool
}
func (l *ackingLine) Subscribe(_ string, cb nats.MsgHandler) (*nats.Subscription, error) {
l.mu.Lock()
l.onAck = cb
l.mu.Unlock()
return &nats.Subscription{}, nil
}
func (l *ackingLine) sent() int { l.mu.Lock(); defer l.mu.Unlock(); return len(l.chunks) }
func (l *ackingLine) Publish(subject string, data []byte) error {
l.mu.Lock()
l.chunks[subject] = data
cb := l.onAck
if l.want > 0 && len(l.chunks) >= l.want && !l.closed {
l.closed = true
close(l.done)
}
l.mu.Unlock()
go cb(&nats.Msg{Subject: strings.Replace(subject, ".chunk.", ".ack.", 1)})
return nil
}
+1 -1
View File
@@ -1530,7 +1530,7 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
a.mu.Unlock()
return false
}
if h.Phase == string(nodeengineclient.PhaseHeld) {
if h.Phase == string(nodeengineclient.PhaseHeld) && !judgedAgain(h.Detail) {
// Held for a failed backup, and the operator's answer is still to come: the hold is kept, nothing is
// backed up again or applied, and the next revision of the want is the answer (ADR 0568 §4). Said, so
// why it holds is read from the journal and not guessed (issue 1585: this branch was silent).
+9
View File
@@ -366,6 +366,11 @@ func TestAWantAndItsCompositionApplyTheModuleAndTheHaveNamesBoth(t *testing.T) {
if got := partsDifference(map[string]string{"web:c": d3}, []json.RawMessage{container}); got != "" {
t.Fatalf("a container whose digest is in the parts differs: %q", got)
}
// A-432: a runtime of its own runs the tool runner's program, which the module's parts do not name.
ownRuntime := json.RawMessage(`{"id":"web.own-runtime","type":"process","name":"web-runtime","source":"store/node-tools","digest":"` + d1 + `","run":["./node-tools"],"module":"web"}`)
if got := partsDifference(map[string]string{"web:c": d3}, []json.RawMessage{container, ownRuntime}); got != "" {
t.Fatalf("a runtime of its own differs from the module's parts: %q", got)
}
}
// R6-A AC2: given a kept declaration and wants and compositions equal to it, when the agent starts, nothing is applied
@@ -829,6 +834,10 @@ func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) {
{"restic: the repository is locked", meshwants.WantsByWantNodeModuleValueBackupAgain, 1, 1, nodeengineclient.PhaseRunning, false},
{"restic: the repository is locked", meshwants.WantsByWantNodeModuleValueBackupSkip, 0, 1, nodeengineclient.PhaseRunning, false},
{"restic: the repository is locked", "", 0, 0, nodeengineclient.PhaseHeld, false},
// A hold the node-engine judges again (novox's indexer, 2026-11-16): its parts differed or its entry was not
// held, which no answer of the operator's releases; at a start it is judged again, here backed up and applied.
{PartsDiffer + " fetched and not in the want's parts: sha256:a03131c6", "", 1, 1, nodeengineclient.PhaseRunning, false},
{EntryMissing + " the catalog half holds no entry of the build", "", 1, 1, nodeengineclient.PhaseRunning, false},
} {
r := newBed(t)
r.comps.before(ck1, craw1)
+11 -1
View File
@@ -6,6 +6,7 @@ import (
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/declaration"
)
@@ -21,6 +22,14 @@ const (
EntryMissing = "entry-missing:"
)
// judgedAgain says a held have's detail is a hold the node-engine judges again at a start, not one that waits for the
// operator's answer: its want's parts differ from its composition (§524) or its entry is not held (§421). Only a failed
// backup or restore point waits for an answer (ADR 0568 §4); novox, 2026-11-16: the indexer held "parts-differ" was kept
// at a start as "held for a failed backup", waiting for an answer nobody was asked.
func judgedAgain(why string) bool {
return strings.HasPrefix(why, PartsDiffer) || heldForEntry(why)
}
// heldForEntry says whether why is a hold for the entry a want names.
func heldForEntry(why string) bool {
return strings.HasPrefix(why, EntryAsked) || strings.HasPrefix(why, EntryMissing)
@@ -47,7 +56,8 @@ func partsDifference(parts map[string]string, resources []json.RawMessage) strin
fetched[res.Digest] = true
}
case *declaration.Process:
if res.Source != "" {
// A runtime of its own runs the tool runner's program, which the tool runner's want checks (A-432).
if res.Source != "" && !strings.HasSuffix(res.ID, "."+catalogue.OwnRuntimeID()) {
fetched[res.Digest] = true
}
case *declaration.Container:
+51 -1
View File
@@ -161,7 +161,7 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli
}
kept := make(map[string]catalogue.Manifest, len(n.Kept))
for m, man := range n.Kept {
if man.Stubbed && man.RunsAs == "" {
if man.Stubbed && man.RunsAs == "" && !keepsProgram(man) {
man.Bundles = runningBundles(n.RunsTools[m], reading.Catalogue[m])
}
kept[m] = man
@@ -489,6 +489,56 @@ func Stub(m catalogue.Manifest) catalogue.Manifest {
// run). What it provides, requires, listens on and grants is its manifest's. A module whose resources runs holds none
// of is its Stub, as before.
func Carried(m catalogue.Manifest, module string, runs *engcomp.Composed) catalogue.Manifest {
stub := carried(m, module, runs)
if catalogue.IsToolRunner(m) && (len(stub.Bundles) != 1 || stub.Bundles[0].Binary == "") {
if b, ok := runningProgram(module, runs); ok {
stub.Bundles = []catalogue.Bundle{b}
}
}
return stub
}
// keepsProgram says a held tool runner carries the program it runs (Carried, runningProgram): its one bundle is that
// program, which the tools bundles it serves never replace, or the modules of their own account it runs are held for
// want of it (lead-47 2026-11-16: 40ed063 kept novox's, and the loop over what the runtime serves dropped it again).
func keepsProgram(m catalogue.Manifest) bool {
return catalogue.IsToolRunner(m) && len(m.Bundles) == 1 && m.Bundles[0].Binary != ""
}
// runningProgram is the program bundle of the tool runner as it runs on the node: its process's source, digest and binary.
// A held tool runner keeps it, so a module of its own account is run by the same program the node runs (novox/hq ADR 0259
// §8) and is not held for want of a runtime that runs (lead-47 2026-11-16: novox's tool runner was held for a build the
// catalog half held no entry of, and held the new indexer with it).
func runningProgram(module string, runs *engcomp.Composed) (catalogue.Bundle, bool) {
if runs == nil {
return catalogue.Bundle{}, false
}
for _, raw := range runs.Resources {
var r struct {
ID string `json:"id"`
Module string `json:"module"`
Type string `json:"type"`
Source string `json:"source"`
Digest string `json:"digest"`
Run []string `json:"run"`
}
if json.Unmarshal(raw, &r) != nil || r.Type != "process" || r.Source == "" || r.Digest == "" || len(r.Run) == 0 {
continue
}
owner := r.Module
if owner == "" {
owner, _ = declaration.ModuleGuess(r.ID)
}
binary, ok := strings.CutPrefix(r.Run[0], "./")
if owner != module || !ok || binary == "" || strings.Contains(binary, "/") {
continue
}
return catalogue.Bundle{Name: "program", Source: registryforms.Forms.Recorded(r.Source), Digest: r.Digest, Binary: binary}, true
}
return catalogue.Bundle{}, false
}
func carried(m catalogue.Manifest, module string, runs *engcomp.Composed) catalogue.Manifest {
if runs == nil {
return Stub(m)
}
+52
View File
@@ -19,6 +19,7 @@ func fpFor(c byte) string { return meshcatalogclient.SourcePrefix2 + strings.Rep
// to: every resource once, each in its module's composition at the fingerprint given, the module's ownership said on
// it; a module given no fingerprint is named as skipped and composed nowhere.
func TestCompositionsAreTheDeclarationSplitByModule(t *testing.T) {
t.Run("a held tool runner keeps its program for the modules of their own account", aHeldToolRunnerKeepsItsProgramForTheModulesOfTheirOwnAccount)
in := aSmallMesh(t)
d, _, err := Compose(in, "laptop")
if err != nil {
@@ -191,3 +192,54 @@ func TestCompositionsAreTheDeclarationSplitByModule(t *testing.T) {
}
}
}
// lead-47 2026-11-16: a held tool runner keeps the program it runs (Carried, runningProgram) after what the runtime serves
// is laid on the stubs (Node.RunsTools), so a module of its own account on its node is run by that program, not held
// "the tool runner is not here to run it from" (novox, the new indexer).
func aHeldToolRunnerKeepsItsProgramForTheModulesOfTheirOwnAccount(t *testing.T) {
dg := "sha256:" + strings.Repeat("d", 64)
source := catalogue.ArtifactStoreScheme + "node-tools/runtime/blobs/" + dg
runtime := catalogue.Manifest{Module: "node-tools", Version: "1", ToolRunner: true,
OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh/node-tools/broker"}}}
own := catalogue.Manifest{Module: "mesh-self-check", Version: "1", RunsAs: "mesh-self-check",
OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh/mesh-self-check/broker", Taken: catalogue.TakenAtStart}},
Bundles: []catalogue.Bundle{{Name: "tools", Source: catalogue.ArtifactStoreScheme + "mesh-self-check/tools/blobs/" + dg,
Digest: dg, Language: "go", Loads: []string{"mesh-self-check-tools"}}}}
store := catalogue.Manifest{Module: "artifacts", Version: "1",
Claims: []catalogue.Claim{{Name: "container-image-registry", Scope: catalogue.ScopeMesh}},
Listens: []catalogue.Listening{{Name: "registry", Port: 5100, From: catalogue.FromMesh}}}
runs := engcomp.Composed{Node: "anchor", Module: "node-tools", Resources: []json.RawMessage{json.RawMessage(
`{"id":"node-tools.runtime","module":"node-tools","type":"process","name":"node-tools","source":"` + source +
`","digest":"` + dg + `","run":["./node-tools"],"env":{"MESH_TOOL_MODULES":""}}`)}}
in := aMesh(t)
in.Catalogue["node-tools"], in.Catalogue["artifacts"], in.Catalogue["mesh-self-check"] = runtime, store, own
anchor := nodeOf(t, &in, "anchor")
anchor.Assigned = append(anchor.Assigned, "artifacts", "node-tools", "mesh-self-check")
in.BusMinted = []string{"anchor.node-tools", "anchor.mesh-self-check"}
anchor.SealingKey = keyOf(t, "anchor")
anchor.Held = map[string]bool{"node-tools": true}
anchor.Kept = map[string]catalogue.Manifest{"node-tools": Carried(runtime, "node-tools", &runs)}
anchor.RunsTools = ToolsRunning([]engcomp.Composed{runs})
fps := map[string]string{overlay.Name: fpFor('a'), "artifacts": fpFor('5'), "node-tools": fpFor('e'), "mesh-self-check": fpFor('6')}
out, _, _, err := CompositionsIn(in, "anchor", fps, Foreseeing)
if err != nil {
t.Fatalf("a module of its own account beside a held tool runner: %v", err)
}
ran := false
for _, c := range out {
for _, raw := range c.Resources {
var r struct {
Type string `json:"type"`
Digest string `json:"digest"`
Run []string `json:"run"`
User string `json:"user"`
}
if json.Unmarshal(raw, &r) == nil && r.Type == "process" && r.User == "mesh-self-check" {
ran = r.Digest == dg && len(r.Run) == 1 && r.Run[0] == "./node-tools"
}
}
}
if !ran {
t.Errorf("mesh-self-check is not run by the program node-tools runs: %v", out)
}
}
+39
View File
@@ -141,6 +141,45 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st
a.Differs = slices.DeleteFunc(cmp.Differs, func(d nodeengine.CandidateDiffers) bool { return !notFailed(d.Module) })
cmp.Added = slices.DeleteFunc(cmp.Added, func(d nodeengine.CandidateAdded) bool { return !notFailed(d.Module) })
skipped = slices.DeleteFunc(skipped, func(m string) bool { return !notFailed(m) })
// A module the node runs and an assignment names, but that no build can be given to compose at (what runs of it carries
// no fingerprint, it having started before the node-engine wrote them, and the declaration gives none), is held: kept as
// it runs, said as held with why, never removed and never stopped for it. Only a module no assignment names is removed
// (A-410).
for _, m := range skipped {
if !runs[m] {
continue
}
a.Removed = slices.DeleteFunc(a.Removed, func(x string) bool { return x == m })
a.Held = append(a.Held, nodeengine.CandidateRefusal{Module: m,
Why: "what runs of " + m + " has no fingerprint and none is given to compose it at, so it is kept as it runs",
Today: "running, kept as it runs"})
}
skipped = slices.DeleteFunc(skipped, func(m string) bool { return runs[m] })
// And whatever the candidate's assignment of this node names that runs and was composed to nothing for any other reason
// (the module is in no catalogue the composition reads, so it has no resources and is not even skipped): held the same
// way. Only a module no assignment names is removed (A-410).
placedHere := map[string]bool{}
if p, given := s.Assignments[s.Node]; given && p != nil {
for _, m := range p.Modules {
placedHere[m.Module] = true
}
} else if !given {
for _, m := range trunk[s.Node].Modules {
placedHere[m.Module] = true
}
}
a.Removed = slices.DeleteFunc(a.Removed, func(m string) bool {
if !placedHere[m] {
return false
}
a.Held = append(a.Held, nodeengine.CandidateRefusal{Module: m,
Why: "the assignment names " + m + " on this node, and the composition made nothing of it, so it is kept as it runs",
Today: "running, kept as it runs"})
return true
})
if len(a.Removed) == 0 {
a.Removed = nil
}
// What the proposal adds, by node and module: placed by a candidate and not by the trunk on that node.
added := map[string]bool{}
@@ -12,6 +12,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
engcomp "github.com/novox/mesh-host/internal/compositions"
"maps"
"os"
"reflect"
"slices"
@@ -193,6 +194,47 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) {
if a = Answer(ctx, b, Mesh{}, Scope{Node: "n1", Assignments: given(b), InOwn: []string{"spiegel"}}, fps, withMembership, "p-h1", 3); a.Verdict != nodeengine.CandidateAnswerVerdictSame {
t.Errorf("the node's own with its bus membership is answered %+v, want the same", a)
}
// (C) A module the node runs and an assignment names, given no fingerprint to compose at (it started before the node
// wrote them): held as it runs, never removed (A-410). One no assignment names is removed.
noFp := maps.Clone(fps)
delete(noFp, "lemurs")
unmarked := make([]engcomp.Composed, 0, len(running))
for _, c := range running {
if c.Module == "lemurs" {
c.FP = ""
}
unmarked = append(unmarked, c)
}
unmarked = append(unmarked, engcomp.Composed{Node: "n1", Module: "gone", FP: fp("1")})
a = Answer(ctx, b, Mesh{}, Scope{Node: "n1", Assignments: given(b), InOwn: []string{"spiegel"}}, noFp, unmarked, "p-h1", 3)
if !reflect.DeepEqual(a.Removed, []string{"gone"}) {
t.Errorf("removed %v, want only the module no assignment names", a.Removed)
}
heldIt := false
for _, h := range a.Held {
heldIt = heldIt || h.Module == "lemurs" && h.Why != ""
}
if !heldIt || len(a.Added) != 0 {
t.Errorf("a running module with no fingerprint is answered held %+v added %+v, want held with why and not added", a.Held, a.Added)
}
// (D) One the assignment names whose manifest says its name alone (no resources: what stands in for a module whose entry
// is not held) composes to nothing, and is not even skipped: held too, not removed (A-410).
nameOnly := []byte(`{"module":"ghost","version":"0"}`)
ghostDigest := meshcatalogclient.ManifestDigest(nameOnly)
b.manifests[ghostDigest] = nameOnly
b.modules["ghost"] = meshcatalog.ModulesByModuleValue{Manifest: ghostDigest}
pGhost := b.assignments["n1"]
trunkMods := pGhost.Modules
pGhost.Modules = append(append([]meshmoduleassigner.PlacedModule{}, trunkMods...), meshmoduleassigner.PlacedModule{Module: "ghost"})
b.assignments["n1"] = pGhost
ghostRuns := append(append([]engcomp.Composed{}, unmarked...), engcomp.Composed{Node: "n1", Module: "ghost", FP: fp("2")})
a = Answer(ctx, b, Mesh{}, Scope{Node: "n1", Assignments: given(b), InOwn: []string{"spiegel"}}, noFp, ghostRuns, "p-h1", 3)
if !reflect.DeepEqual(a.Removed, []string{"gone"}) || !slices.ContainsFunc(a.Held, func(h nodeengine.CandidateRefusal) bool { return h.Module == "ghost" }) {
t.Errorf("a named module the composition made nothing of: removed %v held %+v, want it held and only gone removed (%.600s)", a.Removed, a.Held, fmt.Sprintf("%+v", a))
}
pGhost.Modules = trunkMods
b.assignments["n1"] = pGhost
delete(b.modules, "ghost")
// Without what runs, the fold composes spiegel from its manifest, as before, and the composition refuses it.
if _, _, held, err := Compose(ctx, b, Mesh{}, Scope{Node: "n1", InOwn: []string{"spiegel"}}, fps); err != nil ||
!strings.Contains(held["spiegel"], "spiegel has not been built for this mesh") {
+5
View File
@@ -271,6 +271,11 @@ func taken(name string, raw []byte, m meshcatalog.ModulesByModuleValue) (man cat
func input(ctx context.Context, r Reader, mesh Mesh, s Scope, refuse func(module string, err error), wanted map[string]string) (compose.Input, error) {
b, err := readBase(ctx, r, mesh, s)
b.refuseOwn(refuse)
// The engine is named from the catalogue the base read, before the rest of the input, so a node whose input fails
// still serves its seat (root-free): live on novox 2026-11-14 no pass ended, so the seat was never served (A-403).
if s.Engine != nil && len(b.in.Catalogue) > 0 {
s.Engine(EngineModule(b.in, s.Node, s.OwnFingerprint))
}
if err != nil {
return compose.Input{}, err
}
+12 -1
View File
@@ -7,6 +7,7 @@ import (
"encoding/json"
"errors"
"fmt"
"log"
"sort"
"strings"
"time"
@@ -14,6 +15,7 @@ import (
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/busretry"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/seatlist"
@@ -52,6 +54,9 @@ type EngineCall struct {
Verb string
Caller string
Args map[string]any
// Line is the link's bus connection, which a verb marked chunks sends its answer's file on (buscalls.SeatChunks.Send); nil
// where the call did not come over a live link.
Line buscalls.ChunkLine
}
// EngineFunc answers one call: its result, or why it is refused.
@@ -157,12 +162,17 @@ func (l *natsLink) serveEngineSeat(handle EngineFunc, module string) {
Description: "the node seat node-engine of " + l.node + " (novox/hq ADR 0563)", QueueGroupDisabled: true,
Metadata: map[string]string{"seat": EngineSeat, "scope": "node", "node": l.node}})
if err != nil {
log.Printf("the node seat %s of %s is not served: %v (A-405)", EngineSeat, l.node, err)
return
}
apart := make(chan struct{}, apartAtOnce)
withLine := func(c EngineCall) (any, error) {
c.Line = l.conn // a verb marked chunks sends its file on the link's own connection
return handle(c)
}
for _, v := range ServedEngineSeat().Serves {
verb := v.Name
answer := EngineAnswer(handle, verb, l.node, apart)
answer := EngineAnswer(withLine, verb, l.node, apart)
meta := EngineEndpointMetadata(v, l.node, module)
_ = svc.AddEndpoint(verb, answer, micro.WithEndpointSubject(EngineToolSubject(verb, l.node)),
micro.WithEndpointMetadata(meta))
@@ -172,6 +182,7 @@ func (l *natsLink) serveEngineSeat(handle EngineFunc, module string) {
l.engineMu.Lock()
l.engine = svc
l.engineMu.Unlock()
log.Printf("the node seat %s of %s is served as %s (A-405)", EngineSeat, l.node, module)
}
// answeredApart are the verbs answered off the subscription that delivered them: each endpoint's subscription hands its
+4 -2
View File
@@ -8,6 +8,7 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/buscalls"
sdkkeys "git.novox.be/novox/mesh-sdk/go/keys"
"sort"
"strconv"
@@ -175,7 +176,8 @@ type HandActs struct {
// ComposeCandidate composes a check's candidate (the indexer's candidate key) on this node in shadow and answers per
// module, applying and writing nothing; with input, also the composition input it composed from (the verb
// compose-candidate, ADR 0593 §7).
ComposeCandidate func(candidate string, input bool) (any, error)
// call is the caller's token for the chunks the input is sent in (A-237b) and line the connection they go on; with no token the input is inline.
ComposeCandidate func(candidate string, input bool, call string, line buscalls.ChunkLine) (any, error)
// ManifestReads is what this engine states it reads of a manifest and when its record last carried them (the verb
// manifest-reads, novox/hq ADR 0618 §3).
ManifestReads func() ([]string, time.Time)
@@ -289,7 +291,7 @@ func (h *HandActs) Call(c EngineCall) (any, error) {
if b, ok := c.Args["input"].(bool); ok {
input = b
}
return h.ComposeCandidate(arg("candidate"), input)
return h.ComposeCandidate(arg("candidate"), input, arg("call"), c.Line)
case ActManifestReads:
if h.ManifestReads == nil {
return nil, errors.New("this engine states no manifest reads yet: its node agent has not started")
+117 -23
View File
@@ -9,11 +9,14 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/keys"
"github.com/novox/mesh-host/internal/buckets"
engcomp "github.com/novox/mesh-host/internal/compositions"
"maps"
"slices"
"sort"
"strings"
"sync"
"time"
@@ -143,13 +146,7 @@ func (s *Shadow) composeCandidate(ctx context.Context, c *Composer, ask nodeengi
a.Verdict = nec.Unready
return a
}
group, heads, err := nec.ParseCompositionKey(ask.Candidate)
if err != nil {
return refuse(err.Error())
}
// The ask names the check (the composition key, `<group>.<heads hash>`); the indexer keys the group's candidate by
// the group alone (buckets.CandidateKey) and holds its heads hash in the value.
ckey, err := buckets.CandidateKey(group)
key, err := nec.ParseCandidateKey(ask.Candidate)
if err != nil {
return refuse(err.Error())
}
@@ -157,21 +154,48 @@ func (s *Shadow) composeCandidate(ctx context.Context, c *Composer, ask nodeengi
return unready(errNotReady.Error())
}
base := run.add(c.scratch())
raw, err := base.get(ctx, buckets.Candidates, ckey)
if err != nil {
return unready(fmt.Sprintf("the indexer's candidate %s cannot be read: %v", ask.Candidate, err))
}
if raw == nil {
return refuse("the indexer holds no candidate " + ask.Candidate)
}
cand, err := meshcatalog.DecodeCandidatesByGroupValue(raw)
if err != nil {
return refuse(fmt.Sprintf("the indexer's candidate %s cannot be read: %v", ask.Candidate, err))
}
if cand.Heads != heads {
// The group moved to other heads: the check asked is replaced, and this node composes not a candidate it was
// not asked about. No answer (ADR 0593 §9), never a verdict on the wrong heads.
return unready(fmt.Sprintf("the indexer's candidate of the group %s is at heads %s, not %s: the check asked is replaced", group, cand.Heads, heads))
var cand meshcatalog.CandidatesByGroupValue
var group, heads string
if key.Kind == nec.CandidateRevision {
// A branch pipeline's catalog graph revision (A-226b): its entries are read through the seat mesh-catalog's verb
// revision and composed as the candidate, each module at the revision's fingerprint.
var answered bool
cand, answered, err = s.revisionCandidate(ctx, base, key)
if err != nil {
return unready(err.Error())
}
if !answered {
return refuse(fmt.Sprintf("the catalog holds no revision %d of the branch %s", key.Revision, key.Branch))
}
} else {
group, heads = key.Group, key.HeadsHash
if key.Kind == nec.CandidateRun {
// A run on request's candidate is the indexer's `candidates.run-<id>`, read by its group as a check's is, with no
// heads to compare: the run is asked for by its id alone (A-226c).
group, heads = "run-"+key.Run, ""
}
// The ask names the check (the composition key, `<group>.<heads hash>`); the indexer keys the group's candidate by
// the group alone (buckets.CandidateKey) and holds its heads hash in the value.
ckey, err := buckets.CandidateKey(group)
if err != nil {
return refuse(err.Error())
}
raw, err := base.get(ctx, buckets.Candidates, ckey)
if err != nil {
return unready(fmt.Sprintf("the indexer's candidate %s cannot be read: %v", ask.Candidate, err))
}
if raw == nil {
return refuse("the indexer holds no candidate " + ask.Candidate)
}
cand, err = meshcatalog.DecodeCandidatesByGroupValue(raw)
if err != nil {
return refuse(fmt.Sprintf("the indexer's candidate %s cannot be read: %v", ask.Candidate, err))
}
if key.Kind == nec.CandidateCheck && cand.Heads != heads {
// The group moved to other heads: the check asked is replaced, and this node composes not a candidate it was
// not asked about. No answer (ADR 0593 §9), never a verdict on the wrong heads.
return unready(fmt.Sprintf("the indexer's candidate of the group %s is at heads %s, not %s: the check asked is replaced", group, cand.Heads, heads))
}
}
// Every module's manifest is read before either composition, several at once, into the shadow's own: a composition
// reads every manifest in the mesh one by one, which off the control-node took about 35 s each (issue 1126).
@@ -214,7 +238,9 @@ func (s *Shadow) composeCandidate(ctx context.Context, c *Composer, ask nodeengi
}
}
})
if sentSize > maxSentInput {
// An ask with a call token takes the input in chunks, whatever its size (A-237b); one with none is the checker's, answered
// inline until G removes the checker (G-6 §321, A-237c), so past what the bus carries in one answer it is refused.
if sentSize > maxSentInput && ask.Call == "" {
// Past what the bus carries in one answer, the checker would hear nothing: refused, saying so.
size := sentSize
a.Input = nil
@@ -481,3 +507,71 @@ func (c *Composer) composeShadow(ctx context.Context, running []engcomp.Composed
// candidateNoneNoModule is the words the indexer writes in a candidate's module that the merge preview names no manifest
// for: the change removes the module. It is the indexer's text, kept here until the catalogue client names it.
const candidateNoneNoModule = "the merge preview names no manifest for the module's directory"
// revisionCandidate reads the catalog graph revision key names, whole, page by page, through the seat mesh-catalog's verb
// revision (the Entry call, the engine's own name), and makes the candidate it composes as: one pull per repository and
// commit with each module's fingerprint and manifest digest, and a module the base holds that the revision does not marked as
// removed. answered is false when the catalog holds no such revision.
func (s *Shadow) revisionCandidate(ctx context.Context, base *Composer, key nec.Candidate) (cand meshcatalog.CandidatesByGroupValue, answered bool, err error) {
call := base.env.Entry
if call == nil {
return cand, false, errors.New("this node's engine cannot ask the catalog for a revision")
}
in := meshcatalog.RevisionInput{Branch: keys.EscapeName(key.Branch), Revision: key.Revision}
type group struct{ repository, commit string }
pulls := map[group]map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{}
seen := map[string]bool{}
for {
raw, err := call(meshcatalogclient.RevisionVerb, in)
if err != nil {
return cand, false, fmt.Errorf("the catalog's revision %d of %s cannot be read: %w", key.Revision, key.Branch, err)
}
body, said, ok := buscalls.Unwrap(raw)
if !ok {
return cand, false, fmt.Errorf("the catalog's revision %d of %s cannot be read: %s", key.Revision, key.Branch, said)
}
var a meshcatalog.RevisionAnswer
if err := json.Unmarshal(body, &a); err != nil {
return cand, false, fmt.Errorf("the catalog's revision %d of %s is not a revision answer: %w", key.Revision, key.Branch, err)
}
if a.NoneYet || a.Revision == nil || a.Revision.Revision != key.Revision {
return cand, false, nil
}
for _, e := range a.Entries {
seen[e.Module] = true
if e.Fp == "" {
continue // no build at the revision: composed as the trunk holds it
}
g := group{e.Repository, e.Commit}
if pulls[g] == nil {
pulls[g] = map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{}
}
// The catalog entry's manifest-digest is the `sha256:`-prefixed form (the seat's definition, entryManifest); a candidate's
// module and the manifests and verdicts keys carry the 64 hex digits alone (ADR 0429 clause 16, meshcatalogclient).
pulls[g][e.Module] = meshcatalog.CandidatesByGroupValuePullsItemFpsValue{Fp2: e.Fp, Manifest: strings.TrimPrefix(e.ManifestDigest, "sha256:")}
}
if a.Next == "" {
break
}
in.After = a.Next
}
removed := map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{}
base.mu.Lock()
for module := range base.modules {
if !seen[module] {
removed[module] = meshcatalog.CandidatesByGroupValuePullsItemFpsValue{None2: candidateNoneNoModule}
}
}
base.mu.Unlock()
cand.Group = key.Branch
for g, fps := range pulls {
cand.Pulls = append(cand.Pulls, meshcatalog.CandidatesByGroupValuePullsItem{Repository: g.repository, Head: g.commit, Fps: fps})
}
sort.Slice(cand.Pulls, func(i, j int) bool {
return cand.Pulls[i].Repository+cand.Pulls[i].Head < cand.Pulls[j].Repository+cand.Pulls[j].Head
})
if len(removed) > 0 {
cand.Pulls = append(cand.Pulls, meshcatalog.CandidatesByGroupValuePullsItem{Fps: removed})
}
return cand, true, nil
}
+33 -1
View File
@@ -85,7 +85,18 @@ func TestTheNodeComposesACandidateOnAsk(t *testing.T) {
t.Fatal(err)
}
entry := meshcatalog.CatalogEntry{Module: "lemurs", Fp: fp, Manifest: manifest, Repository: "novox/x", Commit: "c", ManifestDigest: digest}
env.Entry = EntryCall(func(_ context.Context, _, _ string, _ any) (json.RawMessage, error) {
// A-226b: the revision a branch pipeline names is read through the verb revision: revision 362 of main holds lemurs at its
// new build and manifest; no other revision is held.
env.Entry = EntryCall(func(_ context.Context, _, verb string, args any) (json.RawMessage, error) {
if verb == meshcatalog.VerbRevision {
in, _ := args.(meshcatalog.RevisionInput)
if in.Revision != 362 || in.Branch != "main" {
return json.Marshal(meshcatalog.RevisionAnswer{NoneYet: true})
}
moved := entry
moved.Fp, moved.ManifestDigest = fpNew, "sha256:"+digest2 // the entry carries the prefixed form
return json.Marshal(meshcatalog.RevisionAnswer{Revision: &meshcatalog.CatalogRevision{Branch: "main", Revision: 362}, Entries: []meshcatalog.CatalogEntry{moved}})
}
return json.Marshal(meshcatalog.EntryAnswer{Entry: entry})
})
var handed atomic.Int64
@@ -141,6 +152,27 @@ func TestTheNodeComposesACandidateOnAsk(t *testing.T) {
if _, err := nec.EncodeAnswer(r); err != nil {
t.Errorf("the refused answer does not fit the seat's shape: %v", err)
}
// A-226b: a branch pipeline's revision composes as the candidate, each module at the revision's build; one the catalog does
// not hold, and a run on request's key, are refused in words.
revKey, _ := nec.RevisionKey("main", 362)
rv := s.ComposeCandidate(ctx, nodeengine.ComposeCandidateInput{Candidate: revKey})
if rv.Verdict != nec.Composed || len(rv.Modules) != 1 || rv.Modules[0].Fp != fpNew || !rv.Modules[0].Changed {
t.Errorf("a catalog revision answered %+v", rv)
}
noRev, _ := nec.RevisionKey("main", 999)
if m := s.ComposeCandidate(ctx, nodeengine.ComposeCandidateInput{Candidate: noRev}); m.Verdict != nec.Refused || !strings.Contains(m.Refused, "holds no revision 999") {
t.Errorf("a revision the catalog does not hold answered %+v", m)
}
// A-226c: a run's key reads the indexer's candidates.run-<id> by its group, with no heads compared; one it holds none of is refused.
candidateOf("run-r1", digest2)
runKey, _ := nec.RunKey("r1")
if m := s.ComposeCandidate(ctx, nodeengine.ComposeCandidateInput{Candidate: runKey}); m.Verdict != nec.Composed || len(m.Modules) != 1 || m.Modules[0].Fp != fpNew || !m.Modules[0].Changed {
t.Errorf("a run's key answered %+v", m)
}
noRun, _ := nec.RunKey("r2")
if m := s.ComposeCandidate(ctx, nodeengine.ComposeCandidateInput{Candidate: noRun}); m.Verdict != nec.Refused || !strings.Contains(m.Refused, "holds no candidate") {
t.Errorf("a run the indexer holds none of answered %+v", m)
}
missing, _ := nec.CompositionKey("novox/x#8", "0123456789ab")
if m := s.ComposeCandidate(ctx, nodeengine.ComposeCandidateInput{Candidate: missing}); !strings.Contains(m.Refused, "holds no candidate") {
t.Errorf("a candidate the indexer does not hold answered %+v", m)
+55 -18
View File
@@ -228,6 +228,12 @@ func (c *Composer) wantedBuilds() map[string]bool {
if module != agent.NodeModule && strings.HasPrefix(w.Fp, meshcatalogclient.SourcePrefix2) {
out[w.Fp] = true
}
// The build its want says is being built describes, for resolution alone, a module whose wanted build has no entry (A-405, A-407).
if module != agent.NodeModule && strings.HasPrefix(w.Newest, meshcatalogclient.SourcePrefix2) {
if _, named := out[w.Newest]; !named {
out[w.Newest] = false
}
}
}
for _, mods := range c.heard {
for _, run := range mods {
@@ -309,6 +315,12 @@ func (c *Composer) syncEntries(ctx context.Context) {
// layModules lays c.modules and c.manifests from the entries held: what each node was heard to run first, then each module this
// node's want names, at the want's fingerprint (a want wins).
func (c *Composer) layModules() {
var said []string
defer func() {
for _, line := range said {
c.sayOnceEach(line)
}
}()
c.mu.Lock()
defer c.mu.Unlock()
modules := map[string]meshcatalog.ModulesByModuleValue{}
@@ -343,9 +355,26 @@ func (c *Composer) layModules() {
}
}
for module, w := range c.wants {
if module != agent.NodeModule && w.Fp != "" {
lay(module, w.Fp)
if module == agent.NodeModule || w.Fp == "" {
continue
}
if k, ok := c.keptEnt[w.Fp]; ok && k.entry.Module != "" {
lay(module, w.Fp)
continue
}
// A module the node runs whose wanted build has no entry stands as the build its want says is being built, for resolution
// alone: what that entry provides, claims and requires stands for the running build. Its want stays held (entryHolds), at
// the want's fingerprint, and nothing new of it is composed (A-405, A-407).
if k, ok := c.keptEnt[w.Newest]; ok && w.Newest != "" && k.entry.Module != "" && c.runsHere(module) {
lay(module, w.Newest)
if m, laid := modules[module]; laid {
m.Fp2 = w.Fp
modules[module] = m
said = append(said, fmt.Sprintf("%s runs a build whose entry the catalog half does not hold, so the node resolves around it as the build %s its want says is being built; its want stays held", module, w.Newest))
}
continue
}
lay(module, w.Fp)
}
// A module an assignment places whose entry is not held (not wanted here, not heard running, or its entry asked and not
// answered) stands in the catalogue as its name alone, so the placement still finds it: a module held for its entry offers
@@ -404,7 +433,7 @@ func (c *Composer) entryHolds() map[string]string {
out[module] = fmt.Sprintf("%s the catalog half holds no entry of the build %s, so the node composes nothing new of %s",
agent.EntryMissing, w.Fp, module)
default:
if missing := partsMissing(e, w); missing != "" {
if _, missing := c.partsOf(module, e, w); missing != "" {
out[module] = fmt.Sprintf("the want of %s names no digest for %s, so the node composes nothing new of it", module, missing)
}
}
@@ -412,19 +441,7 @@ func (c *Composer) entryHolds() map[string]string {
return out
}
// partsMissing names the artifacts of e whose digest w's parts do not give; "" when each has one.
func partsMissing(e meshcatalog.CatalogEntry, w meshwants.WantsByWantNodeModuleValue) string {
var out []string
for _, a := range e.Artifacts {
if w.Parts[a.Ref] == "" {
out = append(out, a.Ref)
}
}
sort.Strings(out)
return strings.Join(out, ", ")
}
// referenceOf is an artifact's reference made from the digest its want's part names, in the form the artifact store is
// referenceOf is an artifact's reference made from the digest its want's part names, or its version resolves (A-430a), in the form the artifact store is
// addressed by (the lead's answer A-50): artifact-store://<module>/<artifact>/blobs/<digest>.
func referenceOf(module, artifact, digest string) string {
return "artifact-store://" + module + "/" + artifact + "/blobs/" + digest
@@ -439,7 +456,11 @@ func (c *Composer) buildOf(module, fp string) *builds.BuilderBuild {
w, wanted := c.wants[module]
k, ok := c.keptEnt[fp]
c.mu.Unlock()
if !wanted || w.Fp != fp || !ok || partsMissing(k.entry, w) != "" {
if !wanted || w.Fp != fp || !ok {
return nil
}
parts, missing := c.partsOf(module, k.entry, w)
if missing != "" {
return nil
}
digest, raw, err := entryManifest(k.entry)
@@ -452,7 +473,7 @@ func (c *Composer) buildOf(module, fp string) *builds.BuilderBuild {
b := builds.BuilderBuild{State: builds.BuildDone, Manifest: digest}
for _, a := range k.entry.Artifacts {
_, name, _ := strings.Cut(a.Ref, ":")
digest := w.Parts[a.Ref]
digest := parts[a.Ref]
b.Artifacts = append(b.Artifacts, builds.BuiltArtifact{Name: name, Kind: a.Kind, Digest: digest,
Reference: referenceOf(module, name, digest)})
}
@@ -651,3 +672,19 @@ func (c *Composer) askBuildsEntries(fps map[string]string) (pending string) {
}
return pending
}
// runsHere says whether this node runs the module: heard to, or in what the node runs now (c.mu held; env.Running reads the
// agent's own record).
func (c *Composer) runsHere(module string) bool {
if run, ok := c.heard[c.env.Node][module]; ok && !run.stopped {
return true
}
if c.env.Running != nil {
for _, r := range c.env.Running() {
if r.Module == module {
return true
}
}
}
return false
}
+52
View File
@@ -259,3 +259,55 @@ func (c *Composer) holdEntry(t *testing.T, module, fp, body string) {
}
c.keptEnt[fp] = keptEntry{entry: half[fp]}
}
// A-405, A-407: a module the node runs whose wanted build has no entry stands, for resolution alone, as the build its want says
// is being built, at the want's fingerprint, so its want stays held; one not running here, or whose want names no newer build the
// half holds, stands as its name alone.
func runningModuleWithoutEntryStandsAsTheNewestBuild(t *testing.T) {
old := meshcatalogclient.SourcePrefix2 + strings.Repeat("1", 64)
newest := meshcatalogclient.SourcePrefix2 + strings.Repeat("2", 64)
body := `{"module":"postgres","version":"2"}`
for _, tc := range []struct {
name string
runs bool
newest string
holds bool
laid bool
wantFp string
}{
{"runs, newest entry held", true, newest, true, true, old},
{"not running here", false, newest, true, false, old},
{"names no newer build", true, "", false, false, old},
{"newest entry not held", true, newest, false, false, old},
} {
t.Run(tc.name, func(t *testing.T) {
env := newFakeBus().env()
env.Node = "n1"
c := New(env)
c.mu.Lock()
c.wants["postgres"] = meshwants.WantsByWantNodeModuleValue{Fp: old, Newest: tc.newest}
c.assignments["n1"] = meshmoduleassigner.Assignments{Node: "n1", Modules: []meshmoduleassigner.PlacedModule{{Module: "postgres"}}}
if tc.runs {
c.heard = map[string]map[string]heardRun{"n1": {"postgres": {fingerprint: old}}}
}
c.mu.Unlock()
if tc.holds {
c.holdEntry(t, "postgres", newest, body)
}
if got := c.wantedBuilds()[newest]; tc.newest != "" && got {
t.Errorf("the newest build is named by a want, want only asked")
}
c.layModules()
c.mu.Lock()
m, ok := c.modules["postgres"]
raw := c.manifests[m.Manifest]
c.mu.Unlock()
if !ok || m.Fp2 != tc.wantFp {
t.Fatalf("modules[postgres] = %+v (%v), want fingerprint %s", m, ok, tc.wantFp)
}
if got := strings.Contains(string(raw), `"version":"2"`); got != tc.laid {
t.Errorf("manifest of the newest build laid = %v, want %v (%s)", got, tc.laid, raw)
}
})
}
}
+13 -2
View File
@@ -16,9 +16,20 @@ func (c *Composer) KeptBuildOf(_ context.Context, module, digest string) (node,
if !ok || w.Fp == "" || w.KeptOn == "" {
return "", "", false, nil
}
if len(w.Parts) > 0 {
parts := w.Parts
if len(parts) == 0 && w.Version != "" {
// A want naming its version names its digests through the registry seats (A-430a): the resolution the composing made.
c.mu.Lock()
r := c.resolved[resolvedKey{module, w.Fp, w.Version}]
c.mu.Unlock()
if r.parts == nil {
return "", "", false, nil
}
parts = r.parts
}
if len(parts) > 0 {
named := false
for _, d := range w.Parts {
for _, d := range parts {
if strings.TrimPrefix(d, "sha256:") == strings.TrimPrefix(digest, "sha256:") {
named = true
}
+13 -3
View File
@@ -136,6 +136,9 @@ type Env struct {
// Set, the composer reads the catalog half by the fingerprints its wants and the runnings it heard name and nothing else of
// the indexer's or the builder's states (design 47 G §421, §628); a composer is made with it.
Entry meshcatalogclient.Call
// Registry asks the registry seats' verb versions (oci-artifact-registry, container-image-registry) in the engine's own
// name: a want naming its version is resolved to its artifacts' digests through it (design 47 G A-430a).
Registry meshcatalogclient.Call
}
// ManifestOf is the digest of the manifest of the build a module runs at fp, from the composer's picture of the catalog half;
@@ -193,7 +196,9 @@ type Composer struct {
entries *meshcatalogclient.Entries
keptEnt map[string]keptEntry
// notFound is each fingerprint whose entry-not-found the composer told, so it is told once and cleared once.
notFound map[string]bool
notFound map[string]bool
// resolved is each wanted build's digests as its version resolved them through the registry seats (versions.go, A-430a).
resolved map[resolvedKey]versionParts
keptWantVal map[string]keptWant
keptWants map[string]bool
keptWrite sync.Mutex
@@ -718,7 +723,9 @@ func (c *Composer) entry(bucket, key string) (v any, has, known bool) {
defer c.mu.Unlock()
switch bucket {
case buckets.Wants:
// A want is composed by its fingerprint alone (fingerprints); its words, rule and newest change nothing composed.
// A want is composed by its fingerprint, version, parts and keeping node (design 47 G §525); its words, rule and newest
// change nothing composed. Judged by its fingerprint alone, a want that gained its parts at the same fingerprint asked no
// pass, and the agent waited for a composition never made (lead-47 2026-11-16, the indexer on novox).
module := agent.NodeModule
if _, ok := meshwants.ParseWantsByWantNodeNodeKey(key); !ok {
node, m, ok := meshwants.ParseWantsByWantNodeModuleKey(key)
@@ -728,7 +735,10 @@ func (c *Composer) entry(bucket, key string) (v any, has, known bool) {
module = m
}
w, ok := c.wants[module]
return w.Fp, ok, true
return struct {
Fp, Version, KeptOn string
Parts map[string]string
}{w.Fp, w.Version, w.KeptOn, w.Parts}, ok, true
case buckets.Assignments:
// Any key of a node, its node key, a module key or its legacy key, stands for the node's whole assignment: a
// module key written before its node key names it changes nothing composed.
+2
View File
@@ -158,6 +158,8 @@ func quiet(t *testing.T, out <-chan agent.Update) {
// names any more is taken back; a node that cannot compose says so once and hands nothing.
func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
t.Run("it reads the catalog half by the fingerprints its wants name", composeFromEntries)
t.Run("a running module without an entry stands as the newest build", runningModuleWithoutEntryStandsAsTheNewestBuild)
t.Run("a want naming its version is resolved through the registry seats", aWantNamingItsVersionIsResolvedThroughTheRegistrySeats)
b := newFakeBus()
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
digest := meshcatalogclient.ManifestDigest([]byte(lemurs))
+1 -1
View File
@@ -630,7 +630,7 @@ func (s *Shadow) answerProposal(ctx context.Context, proposal string) (nodeengin
}
// The module is built from its entry and the parts the candidate gives (A-316); an artifact it gives no part for holds it.
w := meshwants.WantsByWantNodeModuleValue{Fp: fp, Parts: own.c.Added[pm.Module].Parts, Version: own.c.Added[pm.Module].Version}
if missing := partsMissing(entry, w); missing != "" {
if _, missing := c.partsOf(pm.Module, entry, w); missing != "" {
heldAdded[pm.Module] = fmt.Sprintf("the candidate gives no part for %s of %s's build %s, so the node composes nothing new of it", missing, pm.Module, fp)
continue
}
+217
View File
@@ -0,0 +1,217 @@
package selfcompose
import (
"encoding/json"
"fmt"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// **A want names its version; the node-engine resolves the digests** (design 47 G A-430, A-430a, the lead's answer A-430b).
// A want of a released build names the module's version and no parts: each artifact of the build's entry is looked up at that
// version through the verb `versions` of the registry seat of its kind (oci-artifact-registry for a bundle or an archive,
// container-image-registry for an image), under the name the builder numbered it by: `<module>/<artifact>`, or the entry's
// package name for a package. An upstream copy is not numbered (it is mirrored by its pinned digest): its digest is the one its
// manifest's `from` pins. A want that names parts (a build no version is known of) is read by its parts as before. A lookup that
// fails is said in the journal, naming the module, the version and the seat, and the module is held whole: never composed partly.
// resolveAgain is how long a failed lookup stands before it is asked again; a pass is asked then.
const resolveAgain = time.Minute
// versionsVerb is the verb a lookup asks.
const versionsVerb = "versions"
// resolvedKey names one resolution: a module's build at a version.
type resolvedKey struct{ module, fp, version string }
// versionParts is a module's build at a version as resolved: its digests by artifact reference, or why not, and when that was found.
type versionParts struct {
parts map[string]string
failed string
at time.Time
}
// registryVersions is the answer of either seat's verb `versions`: the two seats answer the same shape.
type registryVersions struct {
Versions []struct {
Digest string `json:"digest"`
Tag string `json:"tag,omitempty"`
Version string `json:"version,omitempty"`
} `json:"versions"`
}
// seatOfKind is the registry seat an artifact of kind is numbered in, and whether one is (design 51 *The seats*).
func seatOfKind(kind string) (string, bool) {
switch kind {
case catalogue.ArtifactImage:
return containerimageregistry.Seat, true
case catalogue.ArtifactBundle, catalogue.ArtifactArchive:
return ociartifactregistry.Seat, true
}
return "", false
}
// registryName is the name an artifact of module is numbered under: `<module>/<artifact>`, or a package's name in its
// ecosystem as the entry gives it.
func registryName(module string, a meshcatalog.Artifact) string {
_, artifact, _ := strings.Cut(a.Ref, ":")
if a.Kind == catalogue.ArtifactPackage && a.Package != nil && a.Package.Name != "" {
return a.Package.Name
}
return module + "/" + artifact
}
// partsOf is the digest of each artifact of e for want w, by artifact reference, and the artifacts it names no digest for ("" when
// each has one). A want with parts gives them; a want with a version and no parts is resolved through the registry seats.
func (c *Composer) partsOf(module string, e meshcatalog.CatalogEntry, w meshwants.WantsByWantNodeModuleValue) (map[string]string, string) {
parts := w.Parts
if len(parts) == 0 && w.Version != "" && len(e.Artifacts) > 0 {
var why string
if parts, why = c.resolve(module, e, w); why != "" {
return nil, why
}
}
return parts, partsMissing(e, parts)
}
// partsMissing names the artifacts of e whose digest parts does not give; "" when each has one.
func partsMissing(e meshcatalog.CatalogEntry, parts map[string]string) string {
var out []string
for _, a := range e.Artifacts {
if parts[a.Ref] == "" {
out = append(out, a.Ref)
}
}
sort.Strings(out)
return strings.Join(out, ", ")
}
// resolve is the digests of module's build at w's version, from what is resolved already or asked of the seats now. why names
// what failed; a failure stands resolveAgain, then is asked again.
func (c *Composer) resolve(module string, e meshcatalog.CatalogEntry, w meshwants.WantsByWantNodeModuleValue) (map[string]string, string) {
key := resolvedKey{module, w.Fp, w.Version}
now := c.now()
c.mu.Lock()
r, ok := c.resolved[key]
c.mu.Unlock()
if ok && (r.failed == "" || now.Sub(r.at) < resolveAgain) {
return r.parts, r.failed
}
parts, failed := c.lookUp(module, e, w)
c.mu.Lock()
if c.resolved == nil {
c.resolved = map[resolvedKey]versionParts{}
}
// Only the resolutions of the builds a want names now are kept.
for k := range c.resolved {
if cur, wanted := c.wants[k.module]; k.module != module && (!wanted || cur.Fp != k.fp || cur.Version != k.version) {
delete(c.resolved, k)
}
}
c.resolved[key] = versionParts{parts: parts, failed: failed, at: now}
c.mu.Unlock()
if failed != "" {
time.AfterFunc(resolveAgain, c.askRehand)
return nil, failed
}
return parts, ""
}
// lookUp asks each artifact's digest at w's version of the seat of its kind. Every failure is said, naming the module, the version
// and the seat; any failure resolves nothing of the build.
func (c *Composer) lookUp(module string, e meshcatalog.CatalogEntry, w meshwants.WantsByWantNodeModuleValue) (map[string]string, string) {
parts := map[string]string{}
var failed []string
fail := func(seat, ref, why string) {
line := fmt.Sprintf("%s at version %s: the seat %s's verb versions resolves no digest of %s: %s, so the node holds %s and composes nothing of it",
module, w.Version, seat, ref, why, module)
c.sayOnceEach(line)
failed = append(failed, fmt.Sprintf("%s (seat %s: %s)", ref, seat, why))
}
var manifest *catalogue.Manifest
for _, a := range e.Artifacts {
_, artifact, _ := strings.Cut(a.Ref, ":")
if a.Kind == catalogue.ArtifactUpstream {
if manifest == nil {
_, raw, err := entryManifest(e)
m, derr := catalogue.Decode(raw)
if err != nil || derr != nil {
fail("(none)", a.Ref, "the entry's manifest cannot be read for the digest its upstream copy pins")
continue
}
manifest = &m
}
if d := pinnedDigest(*manifest, artifact); d != "" {
parts[a.Ref] = d
} else {
fail("(none)", a.Ref, "the manifest pins no digest of the upstream copy")
}
continue
}
seat, numbered := seatOfKind(a.Kind)
if !numbered {
fail("(none)", a.Ref, "no registry seat the node-engine reads numbers an artifact of kind "+a.Kind)
continue
}
digest, why := c.versionDigest(seat, registryName(module, a), w)
if why != "" {
fail(seat, a.Ref, why)
continue
}
parts[a.Ref] = digest
}
if len(failed) > 0 {
sort.Strings(failed)
return nil, "no digest is resolved at version " + w.Version + " for " + strings.Join(failed, ", ")
}
return parts, ""
}
// versionDigest is the digest the seat's verb versions answers for name at w's version (its version or its tag), the newest
// first: a version names one build (A-430a), so the version alone is matched.
func (c *Composer) versionDigest(seat, name string, w meshwants.WantsByWantNodeModuleValue) (string, string) {
if c.env.Registry == nil {
return "", "this node-engine reads no registry seat"
}
raw, err := c.env.Registry("seat:"+seat+"."+versionsVerb, map[string]string{"name": name})
if err != nil {
return "", fmt.Sprintf("the call for %s failed: %v", name, err)
}
answer, said, ok := buscalls.Unwrap(raw)
if !ok {
return "", fmt.Sprintf("the seat refused %s: %s", name, said)
}
var v registryVersions
if err := json.Unmarshal(answer, &v); err != nil {
return "", fmt.Sprintf("its answer for %s cannot be read: %v", name, err)
}
for _, a := range v.Versions {
if (a.Version == w.Version || a.Tag == w.Version) && a.Digest != "" {
return a.Digest, ""
}
}
return "", fmt.Sprintf("it holds no version %s of %s", w.Version, name)
}
// pinnedDigest is the digest the manifest's upstream artifact pins in its from (`<image>@sha256:<hex>`); "" when none.
func pinnedDigest(m catalogue.Manifest, artifact string) string {
if m.Build == nil {
return ""
}
for _, a := range m.Build.Artifacts {
if a.Name == artifact {
if _, d, ok := strings.Cut(a.From, "@"); ok && strings.HasPrefix(d, "sha256:") {
return d
}
}
}
return ""
}
+123
View File
@@ -0,0 +1,123 @@
package selfcompose
import (
"context"
"encoding/json"
"errors"
"strings"
"sync"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// Design 47 G A-430, A-430a (the lead's answer A-430b): a want naming its version and no parts is resolved to its artifacts'
// digests through the verb versions of the registry seat of each artifact's kind, under `<module>/<artifact>`; an upstream copy
// by the digest its manifest pins. The build is made from those digests. A lookup that fails is said, naming the module, the
// version and the seat, and holds the module whole; a want with parts is read by its parts as before.
func aWantNamingItsVersionIsResolvedThroughTheRegistrySeats(t *testing.T) {
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
d := func(c string) string { return "sha256:" + strings.Repeat(c, 64) }
manifest := map[string]any{"module": "mail", "version": "1", "build": map[string]any{"artifacts": []any{
map[string]any{"name": "server", "kind": "upstream", "from": "alpine@" + d("9")},
map[string]any{"name": "files", "kind": "archive", "from": "files"},
map[string]any{"name": "web", "kind": "image", "from": "Dockerfile"},
}}}
entry := meshcatalog.CatalogEntry{Module: "mail", Fp: fp, Manifest: manifest, Artifacts: []meshcatalog.Artifact{
{Ref: "mail:server", Kind: "upstream"}, {Ref: "mail:files", Kind: "archive"}, {Ref: "mail:web", Kind: "image"}}}
// The seats hold mail/files and mail/web; versions newest first.
held := map[string]string{
"oci-artifact-registry.mail/files": `{"name":"mail/files","versions":[{"name":"mail/files","kind":"archive","digest":"` + d("2") + `","version":"1.3.0"},{"name":"mail/files","kind":"archive","digest":"` + d("1") + `","version":"1.2.3"}]}`,
"container-image-registry.mail/web": `{"name":"mail/web","versions":[{"name":"mail/web","kind":"image","digest":"` + d("3") + `","version":"1.2.3"}]}`,
"container-image-registry.mail/other": `{"name":"mail/other","versions":[]}`,
}
for name, row := range map[string]struct {
want meshwants.WantsByWantNodeModuleValue
down string // a seat whose call fails
parts map[string]string
missing string // a word the reason holds
said []string
}{
"resolved": {want: meshwants.WantsByWantNodeModuleValue{Fp: fp, Version: "1.2.3", Self: true},
parts: map[string]string{"mail:server": d("9"), "mail:files": d("1"), "mail:web": d("3")}},
"a version a seat does not hold": {want: meshwants.WantsByWantNodeModuleValue{Fp: fp, Version: "2.0.0", Self: true},
missing: "2.0.0", said: []string{"mail at version 2.0.0", "oci-artifact-registry", "container-image-registry"}},
"a seat that does not answer": {want: meshwants.WantsByWantNodeModuleValue{Fp: fp, Version: "1.2.3", Self: true}, down: "container-image-registry",
missing: "mail:web", said: []string{"mail at version 1.2.3: the seat container-image-registry"}},
"parts as before": {want: meshwants.WantsByWantNodeModuleValue{Fp: fp, Version: "1.2.3", Self: true,
Parts: map[string]string{"mail:server": d("9"), "mail:files": d("5"), "mail:web": d("6")}},
parts: map[string]string{"mail:server": d("9"), "mail:files": d("5"), "mail:web": d("6")}},
} {
var mu sync.Mutex
var said []string
env := Env{Say: func(line string) { mu.Lock(); said = append(said, line); mu.Unlock() }}
calls := 0
env.Registry = EntryCall(func(_ context.Context, seat, verb string, args any) (json.RawMessage, error) {
calls++
if verb != "versions" {
t.Errorf("%s: asked %s.%s", name, seat, verb)
}
if seat == row.down {
return nil, errors.New("no responders")
}
answer, ok := held[seat+"."+args.(map[string]string)["name"]]
if !ok {
return nil, errors.New("not held")
}
return json.RawMessage(answer), nil
})
env.Entry = EntryCall(func(context.Context, string, string, any) (json.RawMessage, error) {
return nil, errors.New("not asked")
})
c := New(env)
c.wants["mail"] = row.want
c.keptEnt[fp] = keptEntry{entry: entry}
parts, missing := c.partsOf("mail", entry, row.want)
b := c.buildOf("mail", fp)
if row.missing != "" {
if parts != nil || b != nil || !strings.Contains(missing, row.missing) {
t.Errorf("%s: parts %v, build %v, missing %q: want the module held whole, naming %s", name, parts, b, missing, row.missing)
}
if c.entryHolds()["mail"] == "" {
t.Errorf("%s: the module is not held", name)
}
for _, w := range row.said {
if !strings.Contains(strings.Join(said, "\n"), w) {
t.Errorf("%s: said %q, want a line naming %q", name, said, w)
}
}
continue
}
if missing != "" || len(parts) != len(row.parts) {
t.Fatalf("%s: parts %v, missing %q", name, parts, missing)
}
for ref, digest := range row.parts {
if parts[ref] != digest {
t.Errorf("%s: %s at %s, want %s", name, ref, parts[ref], digest)
}
}
if b == nil || len(b.Artifacts) != 3 {
t.Fatalf("%s: build %+v", name, b)
}
for _, a := range b.Artifacts {
if a.Digest != row.parts["mail:"+a.Name] || a.Reference != referenceOf("mail", a.Name, a.Digest) {
t.Errorf("%s: artifact %+v", name, a)
}
}
// A resolution is asked once, and it names the node keeping the build its digests are of.
before := calls
c.partsOf("mail", entry, row.want)
if calls != before {
t.Errorf("%s: resolved again: %d calls, then %d", name, before, calls)
}
c.wants["mail"] = meshwants.WantsByWantNodeModuleValue{Fp: fp, Version: row.want.Version, Parts: row.want.Parts, KeptOn: "g14", Self: true}
if node, _, kept, _ := c.KeptBuildOf(context.Background(), "mail", row.parts["mail:files"]); !kept || node != "g14" {
t.Errorf("%s: kept on %q %v, want g14", name, node, kept)
}
if _, _, kept, _ := c.KeptBuildOf(context.Background(), "mail", d("7")); kept {
t.Errorf("%s: a digest the want does not resolve to is said kept", name)
}
}
}