jschoubben
eff81f1220
On 2026-10-30 at 21:04 mesh-self-check (ace) and mesh-conditions, mesh-healers (novox) were placed before
their bus accounts were issued. Each was held for "needs a secret called broker and none was made"; its bare
stub then lost its own secrets and was refused by ownRuntimes ("runs as its own account and declares no own
secret broker"), a refusal no culprit takes, so 74 modules on ace and 79 on novox failed. A held runtime's stub
met the same kind earlier today ("its build produced 0 bundle(s)").
Quick fix: let the bare stub keep its broker own secret, for this one rule.
Thorough fix (chosen, the fourth of the kind today): a held module's stand-in (Stubbed) has no runtime
composed at all, so no rule about its runtime, bundle or broker can refuse it: ownRuntimes passes it over and
runtimeProcess composes none for a held runtime (what runs of either is carried, as before). A module of its
own account left without its runtime by a held runtime is refused with a typed NoRuntimeToRunError, which makes
the held runtime the culprit, so those modules are held after it and the node composes the rest.
224 lines
12 KiB
Go
224 lines
12 KiB
Go
package frombus
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
"git.novox.be/novox/mesh-sdk/go/compose"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
)
|
|
|
|
// aNode is a bus with n1 the hub, each of manifests placed on it and held at the indexer, in the order given.
|
|
func aNode(t *testing.T, manifests ...string) (*bus, map[string]string) {
|
|
t.Helper()
|
|
p := delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"}}
|
|
b := &bus{records: map[string]delivery.NodeRecord{"n1": {Node: "n1", TunnelKey: publicKey(t), SealingKey: "sk", Capabilities: []string{"systemd"}}},
|
|
modules: map[string]delivery.Module{}, manifests: map[string][]byte{}, builds: map[string]*delivery.BuilderBuild{}}
|
|
fps := map[string]string{}
|
|
for i, raw := range manifests {
|
|
module := strings.SplitN(strings.TrimPrefix(raw, `{"module":"`), `"`, 2)[0]
|
|
p.Modules = append(p.Modules, delivery.PlacedModule{Module: module})
|
|
b.modules[module] = delivery.Module{Manifest: delivery.ManifestDigest([]byte(raw))}
|
|
b.manifests[delivery.ManifestDigest([]byte(raw))] = []byte(raw)
|
|
fps[module] = "src2:" + strings.Repeat(string(rune('1'+i)), 64)
|
|
}
|
|
b.placements = map[string]delivery.Placement{"n1": p}
|
|
return b, fps
|
|
}
|
|
|
|
// Issue 774, 2026-10-30 19:30: a node refused whole after holding a module says what it held and why, since the last
|
|
// refusal can be of the held module's stub (node-tools held, then "its build produced 0 bundle(s)", its own why unsaid).
|
|
func aNodeRefusedWholeSaysWhatItHeld(t *testing.T) {
|
|
refused := `{"module":"power","version":"1","items":[{"collection":"contribution:node-bar:block","attributes":{"shows":"state"},"template":"files/a"}]}`
|
|
needs := `{"module":"lamp","version":"1","requires":["nothing-provides-this"]}`
|
|
b, fps := aNode(t, refused, needs)
|
|
_, _, held, err := Compose(context.Background(), b, Mesh{}, Scope{Node: "n1"}, fps)
|
|
if err == nil || !strings.Contains(err.Error(), "nothing provides") || !strings.Contains(err.Error(), `held before: power: `) ||
|
|
!strings.Contains(err.Error(), `"shows" is none of them`) {
|
|
t.Errorf("the node refused whole with %v (held %v), want lamp's refusal and power held before, with its why", err, held)
|
|
}
|
|
}
|
|
|
|
// Issue 774: the culprit rule takes a module whose stub composes or fails otherwise; the runtime only where its stub
|
|
// composes, since its stub always fails otherwise.
|
|
func theRuntimeIsBlamedOnlyWhereItsStubComposes(t *testing.T) {
|
|
err := errors.New("lamp refuses")
|
|
for _, row := range []struct {
|
|
module string
|
|
again error
|
|
blamed bool
|
|
}{
|
|
{"lamp", nil, true},
|
|
{"lamp", errors.New("other"), true},
|
|
{"lamp", errors.New("lamp refuses"), false},
|
|
{catalogue.RuntimeModule, nil, true},
|
|
{catalogue.RuntimeModule, errors.New("node-tools is assigned to n1 and its build produced 0 bundle(s)"), false},
|
|
{catalogue.RuntimeModule, fmt.Errorf("compose: %w", &catalogue.NoRuntimeToRunError{Held: true}), true},
|
|
{catalogue.RuntimeModule, &catalogue.NoRuntimeToRunError{}, false},
|
|
} {
|
|
if got := blames(row.module, row.again, err); got != row.blamed {
|
|
t.Errorf("%s with %v: blamed %v, want %v", row.module, row.again, got, row.blamed)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Issue 774, 2026-10-30 19:30: the trunk's node-tools moved to a build not yet done, and ace, g14 and shanks composed
|
|
// it from the indexer's manifest, which names no bundle: "its build produced 0 bundle(s)", every module refused. A
|
|
// module that runs is now held until its build lands, the runtime composed as what runs of it; and a module wanted at
|
|
// another build than the trunk's is composed from the build its want names.
|
|
func aRunningModuleWaitsForItsBuild(t *testing.T) {
|
|
dg := func(c string) string { return "sha256:" + strings.Repeat(c, 64) }
|
|
tools := `{"module":"node-tools","version":"1","own-secrets":{"broker":"/var/lib/mesh/node-tools/broker"},` +
|
|
`"build":{"artifacts":[{"name":"runtime","kind":"bundle","language":"go","system":"arch","from":"cmd/node-tools"}]}}`
|
|
lamp := `{"module":"lamp","version":"1","resources":[{"id":"service","type":"service","unit":"lamp.service","state":"running","boot":"enabled"}]}`
|
|
store := `{"module":"artifacts","version":"1","provides":[{"name":"artifact-store","scope":"mesh"}],"serves":{"artifact-store":{"port":5100}}}`
|
|
old, next := "src2:"+strings.Repeat("a", 64), "src2:"+strings.Repeat("b", 64)
|
|
running := []delivery.Composed{{Node: "n1", Module: catalogue.RuntimeModule, FP: old, Resources: []json.RawMessage{
|
|
json.RawMessage(`{"id":"node-tools.runtime","module":"node-tools","type":"process","name":"node-tools","env":{"MESH_TOOL_MODULES":""}}`),
|
|
json.RawMessage(`{"id":"node-tools.bundle-runtime","module":"node-tools","type":"archive","source":"artifact-store://node-tools/runtime/blobs/` +
|
|
dg("1") + `","digest":"` + dg("1") + `","path":"` + catalogue.BundlePath("node-tools", "runtime") + `"}`)}}}
|
|
for _, row := range []struct {
|
|
name string
|
|
want string // the build node-tools' want names
|
|
done bool // the trunk's build is done
|
|
held string // words node-tools' hold says; empty: composed
|
|
digest string // the runtime bundle's digest composed
|
|
}{
|
|
{name: "the trunk's build not done yet", want: next, held: "no done build yet"},
|
|
{name: "wanted at the build that runs, the trunk's not done", want: old, digest: dg("1")},
|
|
{name: "the trunk's build done", want: next, done: true, digest: dg("2")},
|
|
} {
|
|
b, fps := aNode(t, store, tools, lamp)
|
|
b.own = []compose.OwnSecretRow{{Node: "n1", Module: "node-tools", Name: "broker", Origin: "made", SealedTo: "sk", Sealed: `"box"`}}
|
|
mod := b.modules["node-tools"]
|
|
mod.FP2 = next
|
|
b.modules["node-tools"] = mod
|
|
b.builds["node-tools "+old] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mod.Manifest,
|
|
Artifacts: []delivery.BuiltArtifact{{Name: "runtime", Kind: "bundle", Reference: "artifact-store://node-tools/runtime/blobs/" + dg("1"), Digest: dg("1")}}}
|
|
b.builds["node-tools "+next] = &delivery.BuilderBuild{State: "building", Manifest: mod.Manifest}
|
|
if row.done {
|
|
b.builds["node-tools "+next] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mod.Manifest,
|
|
Artifacts: []delivery.BuiltArtifact{{Name: "runtime", Kind: "bundle", Reference: "artifact-store://node-tools/runtime/blobs/" + dg("2"), Digest: dg("2")}}}
|
|
}
|
|
fps["node-tools"] = row.want
|
|
out, _, held, err := Compose(context.Background(), b, Mesh{}, Scope{Node: "n1", Running: running}, fps)
|
|
if err != nil {
|
|
t.Errorf("%s: the node is refused: %v", row.name, err)
|
|
continue
|
|
}
|
|
if lampKey, _ := delivery.ComposedKey("n1", "lamp", fps["lamp"]); out[lampKey].Module != "lamp" {
|
|
t.Errorf("%s: lamp is not composed (held %v)", row.name, held)
|
|
}
|
|
if why := held["node-tools"]; (row.held == "") != (why == "") || !strings.Contains(why, row.held) {
|
|
t.Errorf("%s: node-tools held %q, want %q", row.name, why, row.held)
|
|
}
|
|
if row.digest != "" {
|
|
key, _ := delivery.ComposedKey("n1", "node-tools", row.want)
|
|
if c := out[key]; !strings.Contains(fmt.Sprint(rawStrings(c.Resources)), row.digest) {
|
|
t.Errorf("%s: node-tools is composed as %v, want its bundle %s", row.name, rawStrings(c.Resources), row.digest)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func rawStrings(rs []json.RawMessage) []string {
|
|
var out []string
|
|
for _, r := range rs {
|
|
out = append(out, string(r))
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Issue 774: a refusal of a held module's stand-in leads with why the module was held, its first reason, as mesh-self-check's
|
|
// on ace should have; a refusal naming no held module is as it was.
|
|
func aHeldModulesFirstReasonLeads(t *testing.T) {
|
|
held := map[string]string{"mesh-self-check": "mesh-self-check on ace has no bus credential: nothing was issued for ace.mesh-self-check"}
|
|
for _, row := range []struct {
|
|
err, want string
|
|
}{
|
|
{"mesh-self-check runs as its own account and declares no own secret broker",
|
|
"mesh-self-check is held: mesh-self-check on ace has no bus credential: nothing was issued for ace.mesh-self-check; " +
|
|
"its stand-in is then refused: mesh-self-check runs as its own account and declares no own secret broker"},
|
|
{"ace is the hub and has no endpoint", "ace is the hub and has no endpoint"},
|
|
} {
|
|
err := errors.New(row.err)
|
|
got := firstReason(err, held)
|
|
if got.Error() != row.want || !errors.Is(got, err) {
|
|
t.Errorf("refused %q, want %q", got, row.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Issue 774, 2026-10-30 21:04: mesh-conditions and mesh-healers on novox and mesh-self-check on ace were placed before their
|
|
// bus accounts were issued. Held for "needs a secret called broker and none was made", each one's bare stub was then
|
|
// refused as "runs as its own account and declares no own secret broker", and that refusal failed every module of the
|
|
// node (79 on novox, 74 on ace). A held module's stand-in is never refused by a rule about its own runtime or its own
|
|
// secrets: the node composes every other module, and the held one keeps the words it was first held for.
|
|
func aHeldModuleOfItsOwnAccountNeverRefusesTheNode(t *testing.T) {
|
|
dg := "sha256:" + strings.Repeat("1", 64)
|
|
tools := `{"module":"node-tools","version":"1","own-secrets":{"broker":"/var/lib/mesh/node-tools/broker"},` +
|
|
`"build":{"artifacts":[{"name":"runtime","kind":"bundle","language":"go","system":"arch","from":"cmd/node-tools"}]}}`
|
|
own := func(name string) string {
|
|
return `{"module":"` + name + `","version":"1","runs-as":"` + name + `","secrets-owner":"` + name + `","tools":["status"],` +
|
|
`"own-secrets":{"broker":"/var/lib/` + name + `/broker"},` +
|
|
`"resources":[{"id":"account","type":"user","name":"` + name + `"}],` +
|
|
`"build":{"artifacts":[{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["tools/index.js"]}]}}`
|
|
}
|
|
lamp := `{"module":"lamp","version":"1","resources":[{"id":"service","type":"service","unit":"lamp.service","state":"running","boot":"enabled"}]}`
|
|
store := `{"module":"artifacts","version":"1","provides":[{"name":"artifact-store","scope":"mesh"}],"serves":{"artifact-store":{"port":5100}}}`
|
|
for _, row := range []struct {
|
|
name string
|
|
issued []string // the own-account modules whose broker is issued
|
|
building bool // node-tools' build is not done, and nothing of it runs
|
|
held map[string]string // the modules held, each with words of the reason it was first held for
|
|
}{
|
|
{name: "none issued", held: map[string]string{"check": "broker", "heal": "broker"}},
|
|
{name: "one issued", issued: []string{"heal"}, held: map[string]string{"check": "broker"}},
|
|
{name: "both issued", issued: []string{"check", "heal"}},
|
|
// The runtime held: its stand-in is not refused for its bundle; the modules it would run are held, not the node.
|
|
{name: "the runtime's build not done", issued: []string{"check", "heal"}, building: true,
|
|
held: map[string]string{"node-tools": "its build produced 0 bundle(s)", "check": "not here to run it from", "heal": "not here to run it from"}},
|
|
} {
|
|
b, fps := aNode(t, store, tools, own("check"), lamp, own("heal"))
|
|
b.own = []compose.OwnSecretRow{{Node: "n1", Module: "node-tools", Name: "broker", Origin: "made", SealedTo: "sk", Sealed: `"box"`}}
|
|
for _, m := range row.issued {
|
|
b.own = append(b.own, compose.OwnSecretRow{Node: "n1", Module: m, Name: "broker", Origin: "made", SealedTo: "sk", Sealed: `"box"`})
|
|
}
|
|
for _, m := range []string{"node-tools", "check", "heal"} {
|
|
name := "tools"
|
|
if m == "node-tools" {
|
|
name = "runtime"
|
|
}
|
|
mod := b.modules[m]
|
|
mod.FP2 = fps[m]
|
|
b.modules[m] = mod
|
|
b.builds[m+" "+fps[m]] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mod.Manifest,
|
|
Artifacts: []delivery.BuiltArtifact{{Name: name, Kind: "bundle", Reference: "artifact-store://" + m + "/" + name + "/blobs/" + dg, Digest: dg}}}
|
|
if m == "node-tools" && row.building {
|
|
b.builds[m+" "+fps[m]] = &delivery.BuilderBuild{State: "building", Manifest: mod.Manifest}
|
|
}
|
|
}
|
|
out, _, held, err := Compose(context.Background(), b, Mesh{}, Scope{Node: "n1"}, fps)
|
|
if err != nil {
|
|
t.Errorf("%s: the node is refused: %v", row.name, err)
|
|
continue
|
|
}
|
|
for _, m := range []string{"node-tools", "check", "lamp", "heal"} {
|
|
key, _ := delivery.ComposedKey("n1", m, fps[m])
|
|
_, composed := out[key]
|
|
why, isHeld := held[m]
|
|
want, wantHeld := row.held[m]
|
|
if isHeld != wantHeld || composed == wantHeld {
|
|
t.Errorf("%s: %s held %v (%q), composed %v; want held %v", row.name, m, isHeld, why, composed, wantHeld)
|
|
} else if isHeld && !strings.Contains(why, want) {
|
|
t.Errorf("%s: %s is held for %q, want its first reason, %q", row.name, m, why, want)
|
|
}
|
|
}
|
|
}
|
|
}
|