From c1bc21c35f36fa53138446a392caaf99dba74fe3 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 30 Oct 2026 19:43:46 +0100 Subject: [PATCH 1/3] Say what a node held on the way when it is refused whole, so a held module's own refusal is never hidden (issue 774) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At 19:30 on 2026-10-30 ace, g14 and shanks refused every module with "node-tools … its build produced 0 bundle(s)": the refusal of node-tools' stand-in, while why node-tools could not be composed was dropped with the hold. Quick fix: log held in mesh-host before marking. Thorough fix: Compose's node-wide error carries "(held before: : ; …)", so the engine's journal, every have's detail and any caller see the cause. Chose thorough. --- go/compose/frombus/frombus.go | 11 ++++++++ go/compose/frombus/frombus_test.go | 2 ++ go/compose/frombus/held_rows_test.go | 40 ++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+) create mode 100644 go/compose/frombus/held_rows_test.go diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index ac832ac..eabb180 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -429,6 +429,17 @@ func input(ctx context.Context, r Reader, mesh Mesh, s Scope, refuse func(module // elsewhere, a stand-in for the node's own) refuses the node, err. func Compose(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]string) (out map[string]delivery.Composed, skipped []string, held map[string]string, err error) { held = map[string]string{} + // A node refused whole says what it held on the way, since the refusal can be of a held module's stub (issue 774, + // 2026-10-30 19:30: node-tools held, then "its build produced 0 bundle(s)" with its own refusal unsaid). + defer func() { + if err != nil && len(held) > 0 { + var said []string + for _, m := range sortedKeys(held) { + said = append(said, m+": "+held[m]) + } + err = fmt.Errorf("%w (held before: %s)", err, strings.Join(said, "; ")) + } + }() in, err := input(ctx, r, mesh, s, func(module string, err error) { held[module] = err.Error() }) if err != nil { return nil, nil, held, err diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go index bb53330..09627ec 100644 --- a/go/compose/frombus/frombus_test.go +++ b/go/compose/frombus/frombus_test.go @@ -138,6 +138,8 @@ func TestInput(t *testing.T) { // saying why (answer_rows_test.go). t.Run("every refusal, and an old build left to the node", answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode) t.Run("no store reached says why", noStoreReachedSaysWhy) + // Issue 774: a node refused whole says what it held on the way. + t.Run("a node refused whole says what it held", aNodeRefusedWholeSaysWhatItHeld) t.Run("another node not resolved is named", otherNodeNotResolvedIsNamed) ctx := context.Background() xterm, err := os.ReadFile("../../catalogue/testdata/catalogue-835bf22b-xterm.module.json") diff --git a/go/compose/frombus/held_rows_test.go b/go/compose/frombus/held_rows_test.go new file mode 100644 index 0000000..aa8b79e --- /dev/null +++ b/go/compose/frombus/held_rows_test.go @@ -0,0 +1,40 @@ +package frombus + +import ( + "context" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// aNode is a bus with n1 the hub, each of manifests placed on it and held at the indexer, in the order given. +func aNode(t *testing.T, manifests ...string) (*bus, map[string]string) { + t.Helper() + p := delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"}} + b := &bus{records: map[string]delivery.NodeRecord{"n1": {Node: "n1", TunnelKey: publicKey(t), SealingKey: "sk", Capabilities: []string{"systemd"}}}, + modules: map[string]delivery.Module{}, manifests: map[string][]byte{}, builds: map[string]*delivery.BuilderBuild{}} + fps := map[string]string{} + for i, raw := range manifests { + module := strings.SplitN(strings.TrimPrefix(raw, `{"module":"`), `"`, 2)[0] + p.Modules = append(p.Modules, delivery.PlacedModule{Module: module}) + b.modules[module] = delivery.Module{Manifest: delivery.ManifestDigest([]byte(raw))} + b.manifests[delivery.ManifestDigest([]byte(raw))] = []byte(raw) + fps[module] = "src2:" + strings.Repeat(string(rune('1'+i)), 64) + } + b.placements = map[string]delivery.Placement{"n1": p} + return b, fps +} + +// Issue 774, 2026-10-30 19:30: a node refused whole after holding a module says what it held and why, since the last +// refusal can be of the held module's stub (node-tools held, then "its build produced 0 bundle(s)", its own why unsaid). +func aNodeRefusedWholeSaysWhatItHeld(t *testing.T) { + refused := `{"module":"power","version":"1","items":[{"collection":"contribution:node-bar:block","attributes":{"shows":"state"},"template":"files/a"}]}` + needs := `{"module":"lamp","version":"1","requires":["nothing-provides-this"]}` + b, fps := aNode(t, refused, needs) + _, _, held, err := Compose(context.Background(), b, Mesh{}, Scope{Node: "n1"}, fps) + if err == nil || !strings.Contains(err.Error(), "nothing provides") || !strings.Contains(err.Error(), `held before: power: `) || + !strings.Contains(err.Error(), `"shows" is none of them`) { + t.Errorf("the node refused whole with %v (held %v), want lamp's refusal and power held before, with its why", err, held) + } +} From d539d7e4eb2571e9bf32709f257f6e9806cbe455 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 30 Oct 2026 19:44:08 +0100 Subject: [PATCH 2/3] Take the runtime as a culprit only where its stub composes, so node-tools is never held for an error that names it (issue 774) The culprit rule held a module whose stub made the composition fail otherwise. The runtime's stub carries no bundle, so it always fails otherwise: any refusal naming node-tools (or a module whose name holds it) held the runtime, and the node was then refused whole for the runtime's "0 bundle(s)". Quick fix: never take node-tools as a culprit. Thorough fix: blames, the rule as one function, takes the runtime only where its stub composes. Chose thorough: a runtime whose stub composes may still be held alone. --- go/compose/frombus/answer.go | 12 +++++++++++- go/compose/frombus/frombus_test.go | 1 + go/compose/frombus/held_rows_test.go | 23 +++++++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/go/compose/frombus/answer.go b/go/compose/frombus/answer.go index fc57937..74f7337 100644 --- a/go/compose/frombus/answer.go +++ b/go/compose/frombus/answer.go @@ -373,6 +373,16 @@ func composeEach(in compose.Input, node string, fps map[string]string, applied b return out, skipped, stood, nil } +// blames says the composition with m's stub (again) makes m the culprit of err: it composes, or fails otherwise. The +// runtime's stub is never a runtime (no build is taken in, so it carries no bundle) and always fails otherwise: it is +// the culprit only where its stub composes (issue 774, 2026-10-30 19:30: node-tools held for any error naming it). +func blames(m string, again, err error) bool { + if m == catalogue.RuntimeModule { + return again == nil + } + return again == nil || again.Error() != err.Error() +} + // leaveAll leaves out the whole of a held module's offer and its own secrets (a bare stub); any other word leaves out // what it contributes to that provision alone. const leaveAll = "*" @@ -419,7 +429,7 @@ func culprit(in compose.Input, node string, fps map[string]string, mode compose. } changes := func(m, leave string) bool { _, _, _, again := compose.CompositionsIn(stubbed(in, node, m, leave), node, fps, mode) - return again == nil || again.Error() != err.Error() + return blames(m, again, err) } for _, m := range order { if _, done := failed[m]; done { diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go index 09627ec..7dc9ab5 100644 --- a/go/compose/frombus/frombus_test.go +++ b/go/compose/frombus/frombus_test.go @@ -140,6 +140,7 @@ func TestInput(t *testing.T) { t.Run("no store reached says why", noStoreReachedSaysWhy) // Issue 774: a node refused whole says what it held on the way. t.Run("a node refused whole says what it held", aNodeRefusedWholeSaysWhatItHeld) + t.Run("the runtime is blamed only where its stub composes", theRuntimeIsBlamedOnlyWhereItsStubComposes) t.Run("another node not resolved is named", otherNodeNotResolvedIsNamed) ctx := context.Background() xterm, err := os.ReadFile("../../catalogue/testdata/catalogue-835bf22b-xterm.module.json") diff --git a/go/compose/frombus/held_rows_test.go b/go/compose/frombus/held_rows_test.go index aa8b79e..fc3438e 100644 --- a/go/compose/frombus/held_rows_test.go +++ b/go/compose/frombus/held_rows_test.go @@ -2,9 +2,11 @@ package frombus import ( "context" + "errors" "strings" "testing" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -38,3 +40,24 @@ func aNodeRefusedWholeSaysWhatItHeld(t *testing.T) { t.Errorf("the node refused whole with %v (held %v), want lamp's refusal and power held before, with its why", err, held) } } + +// Issue 774: the culprit rule takes a module whose stub composes or fails otherwise; the runtime only where its stub +// composes, since its stub always fails otherwise. +func theRuntimeIsBlamedOnlyWhereItsStubComposes(t *testing.T) { + err := errors.New("lamp refuses") + for _, row := range []struct { + module string + again error + blamed bool + }{ + {"lamp", nil, true}, + {"lamp", errors.New("other"), true}, + {"lamp", errors.New("lamp refuses"), false}, + {catalogue.RuntimeModule, nil, true}, + {catalogue.RuntimeModule, errors.New("node-tools is assigned to n1 and its build produced 0 bundle(s)"), false}, + } { + if got := blames(row.module, row.again, err); got != row.blamed { + t.Errorf("%s with %v: blamed %v, want %v", row.module, row.again, got, row.blamed) + } + } +} From 9685990a15ed4d37931a176f5aa7e97b5210ee6c Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 30 Oct 2026 19:45:32 +0100 Subject: [PATCH 3/3] Compose a module at the build its want names and hold a running one whose build is not done, so the runtime never loses its bundle (issue 774) At 19:30 on 2026-10-30 node-tools' trunk fingerprint moved to a build not yet done; fromBuilds then composed it from the indexer's manifest, which names no bundle, and ace, g14 and shanks refused every module ("its build produced 0 bundle(s)") until another input moved after the build landed. Quick fix: skip the runtime's check while its build is not done. Thorough fix: Compose reads each placed module at the build its want names (as Scope.KeptAt), and a module that runs whose build is not done is held, keeping what runs (a held runtime is composed as what runs of it, its running bundle), named with why; a module nothing runs of still composes from the indexer's manifest as before. Input, PlansHeld and Answer read the trunk's builds as before. Chose thorough: no running module is ever composed from a source manifest that names no build. --- go/compose/frombus/answer.go | 2 +- go/compose/frombus/frombus.go | 46 ++++++++++++++---- go/compose/frombus/frombus_test.go | 1 + go/compose/frombus/held_rows_test.go | 70 ++++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 11 deletions(-) diff --git a/go/compose/frombus/answer.go b/go/compose/frombus/answer.go index 74f7337..dcdb2f2 100644 --- a/go/compose/frombus/answer.go +++ b/go/compose/frombus/answer.go @@ -41,7 +41,7 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st // Every module it cannot compose is named, not the first only (issue 1126): a build it cannot read or take in, and // below, a module the composition refuses, each composed as a stub so the rest is composed and compared. failed := map[string]string{} - in, err := input(ctx, r, mesh, s, func(module string, err error) { failed[module] = err.Error() }) + in, err := input(ctx, r, mesh, s, func(module string, err error) { failed[module] = err.Error() }, nil) if err != nil { return refuse(err) } diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index eabb180..315d20c 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -34,6 +34,7 @@ import ( "context" "encoding/json" "fmt" + "maps" "net/netip" "slices" "sort" @@ -177,7 +178,7 @@ func InOwn(running []delivery.Composed, p delivery.Placement) []string { // Input is the composition input for s, read through r. func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, error) { - return input(ctx, r, mesh, s, nil) + return input(ctx, r, mesh, s, nil, nil) } // input is Input; with refuse set, a module placed on the composing node whose build cannot be read or taken in, or whose @@ -185,7 +186,7 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er // rather than ending the read, so a node's answer names every module it cannot compose (issue 1126, issue 774); a // module placed elsewhere whose manifest the mesh refuses is read for its offer alone (unusable). A manifest the bus does // not answer for still ends the read: that is the bus's, not the module's. -func input(ctx context.Context, r Reader, mesh Mesh, s Scope, refuse func(module string, err error)) (compose.Input, error) { +func input(ctx context.Context, r Reader, mesh Mesh, s Scope, refuse func(module string, err error), wanted map[string]string) (compose.Input, error) { placements, err := r.Placements(ctx) if err != nil { return compose.Input{}, fmt.Errorf("frombus: placement: %w", err) @@ -397,7 +398,7 @@ func input(ctx context.Context, r Reader, mesh Mesh, s Scope, refuse func(module if _, ok := records[s.Node]; !ok { return compose.Input{}, fmt.Errorf("frombus: %s has written no record of itself yet", s.Node) } - if err := fromBuilds(ctx, r, &in, s, placements[s.Node], modules, read, parse, refuse); err != nil { + if err := fromBuilds(ctx, r, &in, s, placements[s.Node], modules, read, parse, refuse, wanted); err != nil { return compose.Input{}, err } // Its own modules' secrets, sealed to its own key by the vault: a composition carries them sealed (ADR 0550 §5). @@ -440,7 +441,7 @@ func Compose(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]s err = fmt.Errorf("%w (held before: %s)", err, strings.Join(said, "; ")) } }() - in, err := input(ctx, r, mesh, s, func(module string, err error) { held[module] = err.Error() }) + in, err := input(ctx, r, mesh, s, func(module string, err error) { held[module] = err.Error() }, fps) if err != nil { return nil, nil, held, err } @@ -478,7 +479,7 @@ func Plans(ctx context.Context, r Reader, mesh Mesh, s Scope) (map[string]catalo // err. The seat mesh-provisioner's holder composes so: a node with one bad manifest still has every other module's plan. func PlansHeld(ctx context.Context, r Reader, mesh Mesh, s Scope) (plans map[string]catalogue.Plan, held map[string]string, err error) { held = map[string]string{} - in, err := input(ctx, r, mesh, s, func(module string, err error) { held[module] = err.Error() }) + in, err := input(ctx, r, mesh, s, func(module string, err error) { held[module] = err.Error() }, nil) if err != nil { return nil, held, err } @@ -618,7 +619,7 @@ func ranOf(ctx context.Context, r Reader, mesh Mesh, running []delivery.Composed // (issue 1126). func fromBuilds(ctx context.Context, r Reader, in *compose.Input, s Scope, p delivery.Placement, modules map[string]delivery.Module, read func(module, digest string) (catalogue.Manifest, error), parse func([]byte) (catalogue.Manifest, error), - refuse func(module string, err error)) error { + refuse func(module string, err error), wanted map[string]string) error { // stub keeps a refused module in the composition as a stub for this node alone (compose.Stub): what it provides // and requires still shapes the rest. stub := func(module string, err error) { @@ -678,9 +679,27 @@ func fromBuilds(ctx context.Context, r Reader, in *compose.Input, s Scope, p del } return resolved, nil } + // A module wanted at another build than the trunk's is composed from the build its want names (issue 774), as one + // the scope keeps at another build is. + keptAt := maps.Clone(s.KeptAt) + for _, pm := range p.Modules { + if m, ok := modules[pm.Module]; ok && m.FP2 != "" && wanted[pm.Module] != "" && wanted[pm.Module] != m.FP2 && + keptAt[pm.Module] == "" && !slices.Contains(s.InOwn, pm.Module) { + if keptAt == nil { + keptAt = map[string]string{} + } + keptAt[pm.Module] = wanted[pm.Module] + } + } + runs := map[string]bool{} + for _, c := range s.Running { + if c.Module != "" { + runs[c.Module] = true + } + } for _, pm := range p.Modules { m, ok := modules[pm.Module] - if !ok || m.FP2 == "" || s.KeptAt[pm.Module] != "" || slices.Contains(s.InOwn, pm.Module) { + if !ok || m.FP2 == "" || keptAt[pm.Module] != "" || slices.Contains(s.InOwn, pm.Module) { // A module it runs inside its own composition is composed there as before: no build of the new form is what // it runs (Scope.InOwn). continue @@ -698,7 +717,14 @@ func fromBuilds(ctx context.Context, r Reader, in *compose.Input, s Scope, p del continue } if b == nil || b.State != delivery.BuildDone { - continue // no done build at the trunk's: composed from the indexer's manifest, as before + if refuse != nil && runs[pm.Module] { + // A module that runs is never composed from the indexer's manifest, which names no build's artifacts (no + // runtime's bundle): it is held, keeping what runs, until its build lands (issue 774, 2026-10-30 19:30: + // node-tools on ace, g14 and shanks, "its build produced 0 bundle(s)", and every module refused). + stub(pm.Module, fmt.Errorf("frombus: %s is wanted at %s, of which the builder holds no done build yet; it "+ + "keeps running what it runs", pm.Module, m.FP2)) + } + continue // nothing of it runs: composed from the indexer's manifest, as before } resolved, err := built(pm.Module, m.FP2, b) if err != nil { @@ -714,8 +740,8 @@ func fromBuilds(ctx context.Context, r Reader, in *compose.Input, s Scope, p del if in.Nodes[i].Name != s.Node { continue } - for _, module := range sortedKeys(s.KeptAt) { - fp := s.KeptAt[module] + for _, module := range sortedKeys(keptAt) { + fp := keptAt[module] b, err := r.Build(ctx, module, fp) switch { case err != nil: diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go index 7dc9ab5..b77fa2f 100644 --- a/go/compose/frombus/frombus_test.go +++ b/go/compose/frombus/frombus_test.go @@ -141,6 +141,7 @@ func TestInput(t *testing.T) { // Issue 774: a node refused whole says what it held on the way. t.Run("a node refused whole says what it held", aNodeRefusedWholeSaysWhatItHeld) t.Run("the runtime is blamed only where its stub composes", theRuntimeIsBlamedOnlyWhereItsStubComposes) + t.Run("a running module waits for its build", aRunningModuleWaitsForItsBuild) t.Run("another node not resolved is named", otherNodeNotResolvedIsNamed) ctx := context.Background() xterm, err := os.ReadFile("../../catalogue/testdata/catalogue-835bf22b-xterm.module.json") diff --git a/go/compose/frombus/held_rows_test.go b/go/compose/frombus/held_rows_test.go index fc3438e..4deb0ce 100644 --- a/go/compose/frombus/held_rows_test.go +++ b/go/compose/frombus/held_rows_test.go @@ -2,11 +2,14 @@ package frombus import ( "context" + "encoding/json" "errors" + "fmt" "strings" "testing" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -61,3 +64,70 @@ func theRuntimeIsBlamedOnlyWhereItsStubComposes(t *testing.T) { } } } + +// Issue 774, 2026-10-30 19:30: the trunk's node-tools moved to a build not yet done, and ace, g14 and shanks composed +// it from the indexer's manifest, which names no bundle: "its build produced 0 bundle(s)", every module refused. A +// module that runs is now held until its build lands, the runtime composed as what runs of it; and a module wanted at +// another build than the trunk's is composed from the build its want names. +func aRunningModuleWaitsForItsBuild(t *testing.T) { + dg := func(c string) string { return "sha256:" + strings.Repeat(c, 64) } + tools := `{"module":"node-tools","version":"1","own-secrets":{"broker":"/var/lib/mesh/node-tools/broker"},` + + `"build":{"artifacts":[{"name":"runtime","kind":"bundle","language":"go","system":"arch","from":"cmd/node-tools"}]}}` + lamp := `{"module":"lamp","version":"1","resources":[{"id":"service","type":"service","unit":"lamp.service","state":"running","boot":"enabled"}]}` + store := `{"module":"artifacts","version":"1","provides":[{"name":"artifact-store","scope":"mesh"}],"serves":{"artifact-store":{"port":5100}}}` + old, next := "src2:"+strings.Repeat("a", 64), "src2:"+strings.Repeat("b", 64) + running := []delivery.Composed{{Node: "n1", Module: catalogue.RuntimeModule, FP: old, Resources: []json.RawMessage{ + json.RawMessage(`{"id":"node-tools.runtime","module":"node-tools","type":"process","name":"node-tools","env":{"MESH_TOOL_MODULES":""}}`), + json.RawMessage(`{"id":"node-tools.bundle-runtime","module":"node-tools","type":"archive","source":"artifact-store://node-tools/runtime/blobs/` + + dg("1") + `","digest":"` + dg("1") + `","path":"` + catalogue.BundlePath("node-tools", "runtime") + `"}`)}}} + for _, row := range []struct { + name string + want string // the build node-tools' want names + done bool // the trunk's build is done + held string // words node-tools' hold says; empty: composed + digest string // the runtime bundle's digest composed + }{ + {name: "the trunk's build not done yet", want: next, held: "no done build yet"}, + {name: "wanted at the build that runs, the trunk's not done", want: old, digest: dg("1")}, + {name: "the trunk's build done", want: next, done: true, digest: dg("2")}, + } { + b, fps := aNode(t, store, tools, lamp) + b.own = []compose.OwnSecretRow{{Node: "n1", Module: "node-tools", Name: "broker", Origin: "made", SealedTo: "sk", Sealed: `"box"`}} + mod := b.modules["node-tools"] + mod.FP2 = next + b.modules["node-tools"] = mod + b.builds["node-tools "+old] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mod.Manifest, + Artifacts: []delivery.BuiltArtifact{{Name: "runtime", Kind: "bundle", Reference: "artifact-store://node-tools/runtime/blobs/" + dg("1"), Digest: dg("1")}}} + b.builds["node-tools "+next] = &delivery.BuilderBuild{State: "building", Manifest: mod.Manifest} + if row.done { + b.builds["node-tools "+next] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mod.Manifest, + Artifacts: []delivery.BuiltArtifact{{Name: "runtime", Kind: "bundle", Reference: "artifact-store://node-tools/runtime/blobs/" + dg("2"), Digest: dg("2")}}} + } + fps["node-tools"] = row.want + out, _, held, err := Compose(context.Background(), b, Mesh{}, Scope{Node: "n1", Running: running}, fps) + if err != nil { + t.Errorf("%s: the node is refused: %v", row.name, err) + continue + } + if lampKey, _ := delivery.ComposedKey("n1", "lamp", fps["lamp"]); out[lampKey].Module != "lamp" { + t.Errorf("%s: lamp is not composed (held %v)", row.name, held) + } + if why := held["node-tools"]; (row.held == "") != (why == "") || !strings.Contains(why, row.held) { + t.Errorf("%s: node-tools held %q, want %q", row.name, why, row.held) + } + if row.digest != "" { + key, _ := delivery.ComposedKey("n1", "node-tools", row.want) + if c := out[key]; !strings.Contains(fmt.Sprint(rawStrings(c.Resources)), row.digest) { + t.Errorf("%s: node-tools is composed as %v, want its bundle %s", row.name, rawStrings(c.Resources), row.digest) + } + } + } +} + +func rawStrings(rs []json.RawMessage) []string { + var out []string + for _, r := range rs { + out = append(out, string(r)) + } + return out +}