jschoubben
7631a7d5de
Issue 1585, S2: one refused manifest silenced disk-load for 45 hours. frombus decodes leniently, takes the indexer's verdict, stands a refused module in as its stub and names it in Input.Refused; DataItemsHeld names it. Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
299 lines
16 KiB
Go
299 lines
16 KiB
Go
package frombus
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
"git.novox.be/novox/mesh-sdk/go/compose"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
)
|
|
|
|
// aNode is a bus with n1 the hub, each of manifests placed on it and held at the indexer, in the order given.
|
|
func aNode(t *testing.T, manifests ...string) (*bus, map[string]string) {
|
|
t.Helper()
|
|
p := delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"}}
|
|
b := &bus{records: map[string]delivery.NodeRecord{"n1": {Node: "n1", TunnelKey: publicKey(t), SealingKey: "sk", Capabilities: []string{"systemd"}}},
|
|
modules: map[string]delivery.Module{}, manifests: map[string][]byte{}, builds: map[string]*delivery.BuilderBuild{}}
|
|
fps := map[string]string{}
|
|
for i, raw := range manifests {
|
|
module := strings.SplitN(strings.TrimPrefix(raw, `{"module":"`), `"`, 2)[0]
|
|
p.Modules = append(p.Modules, delivery.PlacedModule{Module: module})
|
|
b.modules[module] = delivery.Module{Manifest: delivery.ManifestDigest([]byte(raw))}
|
|
b.manifests[delivery.ManifestDigest([]byte(raw))] = []byte(raw)
|
|
fps[module] = "src2:" + strings.Repeat(string(rune('1'+i)), 64)
|
|
}
|
|
b.placements = map[string]delivery.Placement{"n1": p}
|
|
return b, fps
|
|
}
|
|
|
|
// Issue 774, 2026-10-30 19:30: a node refused whole after holding a module says what it held and why, since the last
|
|
// refusal can be of the held module's stub (node-tools held, then "its build produced 0 bundle(s)", its own why unsaid).
|
|
func aNodeRefusedWholeSaysWhatItHeld(t *testing.T) {
|
|
refused := `{"module":"power","version":"1","items":[{"collection":"contribution:node-bar:block","attributes":{"shows":"state"},"template":"files/a"}]}`
|
|
needs := `{"module":"lamp","version":"1","requires":["nothing-provides-this"]}`
|
|
b, fps := aNode(t, refused, needs)
|
|
refuse(b, "power", `"shows" is none of them`)
|
|
_, _, held, err := Compose(context.Background(), b, Mesh{}, Scope{Node: "n1"}, fps)
|
|
if err == nil || !strings.Contains(err.Error(), "nothing provides") || !strings.Contains(err.Error(), `held before: power: `) ||
|
|
!strings.Contains(err.Error(), `"shows" is none of them`) {
|
|
t.Errorf("the node refused whole with %v (held %v), want lamp's refusal and power held before, with its why", err, held)
|
|
}
|
|
}
|
|
|
|
// refuse writes the indexer's refusal of module's manifest into its modules value (novox/hq ADR 0578 §1).
|
|
func refuse(b *bus, module string, problems ...string) {
|
|
m := b.modules[module]
|
|
m.Verdict, m.Problems = delivery.VerdictRefused, problems
|
|
b.modules[module] = m
|
|
}
|
|
|
|
// ADR 0578 §3: the bus reader never ends a whole read on a refused manifest. Input, Plans and DataItems compose the rest
|
|
// and hold the refused module, named with the indexer's words in Refused; a field the decoder does not know is set aside
|
|
// and named, and its module composed.
|
|
func aRefusedManifestNeverEndsTheRead(t *testing.T) {
|
|
ok := `{"module":"lamp","version":"1","resources":[{"id":"d","type":"directory","path":"/srv/lamp","mode":"0700"}],` +
|
|
`"data":{"own":[{"id":"d","path":"${dir:d}","class":"valuable"}]}}`
|
|
bad := `{"module":"power","version":"1","resources":[{"id":"d","type":"directory","path":"/srv/power","mode":"0700"}],` +
|
|
`"data":{"own":[{"id":"d","path":"${dir:d}","class":"valuable"}]}}`
|
|
later := `{"module":"later","version":"1","a-field-from-later":1}`
|
|
b, _ := aNode(t, ok, bad, later)
|
|
refuse(b, "power", "removes a stream it may not")
|
|
ctx := context.Background()
|
|
in, err := Input(ctx, b, Mesh{}, Scope{Node: "n1"})
|
|
if err != nil {
|
|
t.Fatalf("one refused manifest ended the read: %v", err)
|
|
}
|
|
if why := in.Refused["power"]; !strings.Contains(why, "refused the manifest of power: removes a stream it may not") ||
|
|
len(in.Refused) != 1 {
|
|
t.Fatalf("refused: %v", in.Refused)
|
|
}
|
|
if aside := in.SetAside["later"]; !strings.Contains(aside, "a-field-from-later") {
|
|
t.Fatalf("set aside: %v", in.SetAside)
|
|
}
|
|
if !in.Catalogue["power"].Stubbed || in.Catalogue["lamp"].Stubbed || in.Catalogue["later"].Module != "later" {
|
|
t.Fatalf("the catalogue: %+v", in.Catalogue)
|
|
}
|
|
if _, err := Plans(ctx, b, Mesh{}, Scope{Node: "n1"}); err != nil {
|
|
t.Fatalf("plans: %v", err)
|
|
}
|
|
items, _, held, err := DataItemsHeld(ctx, b, Mesh{}, Scope{Node: "n1"})
|
|
if err != nil || len(items["lamp"]) != 1 || items["power"] != nil || !strings.Contains(held["power"], "removes a stream") {
|
|
t.Fatalf("data items %v held %v: %v", items, held, err)
|
|
}
|
|
}
|
|
|
|
// Issue 774: the culprit rule takes a module whose stub composes or fails otherwise; the runtime only where its stub
|
|
// composes, since its stub always fails otherwise.
|
|
func theRuntimeIsBlamedOnlyWhereItsStubComposes(t *testing.T) {
|
|
err := errors.New("lamp refuses")
|
|
for _, row := range []struct {
|
|
module string
|
|
again error
|
|
blamed bool
|
|
}{
|
|
{"lamp", nil, true},
|
|
{"lamp", errors.New("other"), true},
|
|
{"lamp", errors.New("lamp refuses"), false},
|
|
{catalogue.RuntimeModule, nil, true},
|
|
{catalogue.RuntimeModule, errors.New("node-tools is assigned to n1 and its build produced 0 bundle(s)"), false},
|
|
{catalogue.RuntimeModule, fmt.Errorf("compose: %w", &catalogue.NoRuntimeToRunError{Held: true}), true},
|
|
{catalogue.RuntimeModule, &catalogue.NoRuntimeToRunError{}, false},
|
|
} {
|
|
if got := blames(row.module, row.again, err); got != row.blamed {
|
|
t.Errorf("%s with %v: blamed %v, want %v", row.module, row.again, got, row.blamed)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Issue 774, 2026-10-30 19:30: the trunk's node-tools moved to a build not yet done, and ace, g14 and shanks composed
|
|
// it from the indexer's manifest, which names no bundle: "its build produced 0 bundle(s)", every module refused. A
|
|
// module that runs is now held until its build lands, the runtime composed as what runs of it; and a module wanted at
|
|
// another build than the trunk's is composed from the build its want names.
|
|
func aRunningModuleWaitsForItsBuild(t *testing.T) {
|
|
dg := func(c string) string { return "sha256:" + strings.Repeat(c, 64) }
|
|
tools := `{"module":"node-tools","version":"1","own-secrets":{"broker":"/var/lib/mesh/node-tools/broker"},` +
|
|
`"build":{"artifacts":[{"name":"runtime","kind":"bundle","language":"go","system":"arch","from":"cmd/node-tools"}]}}`
|
|
lamp := `{"module":"lamp","version":"1","resources":[{"id":"service","type":"service","unit":"lamp.service","state":"running","boot":"enabled"}]}`
|
|
store := `{"module":"artifacts","version":"1","provides":[{"name":"artifact-store","scope":"mesh"}],"serves":{"artifact-store":{"port":5100}},` +
|
|
`"receives":{"artifact-store":"/var/lib/artifacts/received.json"}}`
|
|
old, next := "src2:"+strings.Repeat("a", 64), "src2:"+strings.Repeat("b", 64)
|
|
running := []delivery.Composed{{Node: "n1", Module: catalogue.RuntimeModule, FP: old, Resources: []json.RawMessage{
|
|
json.RawMessage(`{"id":"node-tools.runtime","module":"node-tools","type":"process","name":"node-tools","env":{"MESH_TOOL_MODULES":""}}`),
|
|
json.RawMessage(`{"id":"node-tools.bundle-runtime","module":"node-tools","type":"archive","source":"artifact-store://node-tools/runtime/blobs/` +
|
|
dg("1") + `","digest":"` + dg("1") + `","path":"` + catalogue.BundlePath("node-tools", "runtime") + `"}`)}}}
|
|
for _, row := range []struct {
|
|
name string
|
|
want string // the build node-tools' want names
|
|
done bool // the trunk's build is done
|
|
held string // words node-tools' hold says; empty: composed
|
|
digest string // the runtime bundle's digest composed
|
|
}{
|
|
{name: "the trunk's build not done yet", want: next, held: "no done build yet"},
|
|
{name: "wanted at the build that runs, the trunk's not done", want: old, digest: dg("1")},
|
|
{name: "the trunk's build done", want: next, done: true, digest: dg("2")},
|
|
} {
|
|
b, fps := aNode(t, store, tools, lamp)
|
|
b.held = true // every grant held, so a contribution's credential stands in for nothing
|
|
b.own = []compose.OwnSecretRow{{Node: "n1", Module: "node-tools", Name: "broker", Origin: "made", SealedTo: "sk", Sealed: `"box"`}}
|
|
mod := b.modules["node-tools"]
|
|
mod.FP2 = next
|
|
b.modules["node-tools"] = mod
|
|
b.builds["node-tools "+old] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mod.Manifest,
|
|
Artifacts: []delivery.BuiltArtifact{{Name: "runtime", Kind: "bundle", Reference: "artifact-store://node-tools/runtime/blobs/" + dg("1"), Digest: dg("1")}}}
|
|
b.builds["node-tools "+next] = &delivery.BuilderBuild{State: "building", Manifest: mod.Manifest}
|
|
if row.done {
|
|
b.builds["node-tools "+next] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mod.Manifest,
|
|
Artifacts: []delivery.BuiltArtifact{{Name: "runtime", Kind: "bundle", Reference: "artifact-store://node-tools/runtime/blobs/" + dg("2"), Digest: dg("2")}}}
|
|
}
|
|
fps["node-tools"] = row.want
|
|
out, _, held, err := Compose(context.Background(), b, Mesh{}, Scope{Node: "n1", Running: running}, fps)
|
|
if err != nil {
|
|
t.Errorf("%s: the node is refused: %v", row.name, err)
|
|
continue
|
|
}
|
|
if lampKey, _ := delivery.ComposedKey("n1", "lamp", fps["lamp"]); out[lampKey].Module != "lamp" {
|
|
t.Errorf("%s: lamp is not composed (held %v)", row.name, held)
|
|
}
|
|
if why := held["node-tools"]; (row.held == "") != (why == "") || !strings.Contains(why, row.held) {
|
|
t.Errorf("%s: node-tools held %q, want %q", row.name, why, row.held)
|
|
}
|
|
if row.digest != "" {
|
|
key, _ := delivery.ComposedKey("n1", "node-tools", row.want)
|
|
if c := out[key]; !strings.Contains(fmt.Sprint(rawStrings(c.Resources)), row.digest) {
|
|
t.Errorf("%s: node-tools is composed as %v, want its bundle %s", row.name, rawStrings(c.Resources), row.digest)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func rawStrings(rs []json.RawMessage) []string {
|
|
var out []string
|
|
for _, r := range rs {
|
|
out = append(out, string(r))
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Issue 774: a refusal of a held module's stand-in leads with why the module was held, its first reason, as mesh-self-check's
|
|
// on ace should have; a refusal naming no held module is as it was.
|
|
func aHeldModulesFirstReasonLeads(t *testing.T) {
|
|
held := map[string]string{"mesh-self-check": "mesh-self-check on ace has no bus credential: nothing was issued for ace.mesh-self-check"}
|
|
for _, row := range []struct {
|
|
err, want string
|
|
}{
|
|
{"mesh-self-check runs as its own account and declares no own secret broker",
|
|
"mesh-self-check is held: mesh-self-check on ace has no bus credential: nothing was issued for ace.mesh-self-check; " +
|
|
"its stand-in is then refused: mesh-self-check runs as its own account and declares no own secret broker"},
|
|
{"ace is the hub and has no endpoint", "ace is the hub and has no endpoint"},
|
|
} {
|
|
err := errors.New(row.err)
|
|
got := firstReason(err, held)
|
|
if got.Error() != row.want || !errors.Is(got, err) {
|
|
t.Errorf("refused %q, want %q", got, row.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Issue 774, 2026-10-30 21:04: mesh-conditions and mesh-healers on novox and mesh-self-check on ace were placed before their
|
|
// bus accounts were issued. Held for "needs a secret called broker and none was made", each one's bare stub was then
|
|
// refused as "runs as its own account and declares no own secret broker", and that refusal failed every module of the
|
|
// node (79 on novox, 74 on ace). A held module's stand-in is never refused by a rule about its own runtime or its own
|
|
// secrets: the node composes every other module, and the held one keeps the words it was first held for.
|
|
func aHeldModuleOfItsOwnAccountNeverRefusesTheNode(t *testing.T) {
|
|
dg := "sha256:" + strings.Repeat("1", 64)
|
|
tools := `{"module":"node-tools","version":"1","own-secrets":{"broker":"/var/lib/mesh/node-tools/broker"},` +
|
|
`"build":{"artifacts":[{"name":"runtime","kind":"bundle","language":"go","system":"arch","from":"cmd/node-tools"}]}}`
|
|
own := func(name string) string {
|
|
return `{"module":"` + name + `","version":"1","runs-as":"` + name + `","secrets-owner":"` + name + `","tools":["status"],` +
|
|
`"own-secrets":{"broker":"/var/lib/` + name + `/broker"},"contributes":{"artifact-store":{"name":"` + name + `"}},` +
|
|
`"resources":[{"id":"account","type":"user","name":"` + name + `"}],` +
|
|
`"build":{"artifacts":[{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["tools/index.js"]}]}}`
|
|
}
|
|
lamp := `{"module":"lamp","version":"1","resources":[{"id":"service","type":"service","unit":"lamp.service","state":"running","boot":"enabled"}]}`
|
|
store := `{"module":"artifacts","version":"1","provides":[{"name":"artifact-store","scope":"mesh"}],"serves":{"artifact-store":{"port":5100}},` +
|
|
`"receives":{"artifact-store":"/var/lib/artifacts/received.json"}}`
|
|
for _, row := range []struct {
|
|
name string
|
|
issued []string // the own-account modules whose broker is issued
|
|
building bool // node-tools' build is not done, and nothing of it runs
|
|
held map[string]string // the modules held, each with words of the reason it was first held for
|
|
// ungranted gives the provider a grants file and no consumer's grant is issued yet (a newly placed module's
|
|
// usual state): a held module's contribution then stands in for its grant, and is left out (review round 1).
|
|
ungranted bool
|
|
}{
|
|
{name: "none issued", held: map[string]string{"check": "broker", "heal": "broker"}},
|
|
{name: "none issued, no grant issued", ungranted: true, held: map[string]string{"check": "broker", "heal": "broker"}},
|
|
{name: "one issued", issued: []string{"heal"}, held: map[string]string{"check": "broker"}},
|
|
{name: "both issued", issued: []string{"check", "heal"}},
|
|
// The runtime held: its stand-in is not refused for its bundle; the modules it would run are held, not the node.
|
|
{name: "the runtime's build not done", issued: []string{"check", "heal"}, building: true,
|
|
held: map[string]string{"node-tools": "its build produced 0 bundle(s)", "check": "not here to run it from", "heal": "not here to run it from"}},
|
|
} {
|
|
provider := store
|
|
if row.ungranted {
|
|
provider = strings.Replace(store, `"receives":`, `"grants":{"artifact-store":"/var/lib/artifacts/grants"},"receives":`, 1)
|
|
}
|
|
b, fps := aNode(t, provider, tools, own("check"), lamp, own("heal"))
|
|
// Every grant held, so the held modules' contributions are composed whole; the row ungranted issues none.
|
|
b.held = !row.ungranted
|
|
b.own = []compose.OwnSecretRow{{Node: "n1", Module: "node-tools", Name: "broker", Origin: "made", SealedTo: "sk", Sealed: `"box"`}}
|
|
for _, m := range row.issued {
|
|
b.own = append(b.own, compose.OwnSecretRow{Node: "n1", Module: m, Name: "broker", Origin: "made", SealedTo: "sk", Sealed: `"box"`})
|
|
}
|
|
for _, m := range []string{"node-tools", "check", "heal"} {
|
|
name := "tools"
|
|
if m == "node-tools" {
|
|
name = "runtime"
|
|
}
|
|
mod := b.modules[m]
|
|
mod.FP2 = fps[m]
|
|
b.modules[m] = mod
|
|
b.builds[m+" "+fps[m]] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mod.Manifest,
|
|
Artifacts: []delivery.BuiltArtifact{{Name: name, Kind: "bundle", Reference: "artifact-store://" + m + "/" + name + "/blobs/" + dg, Digest: dg}}}
|
|
if m == "node-tools" && row.building {
|
|
b.builds[m+" "+fps[m]] = &delivery.BuilderBuild{State: "building", Manifest: mod.Manifest}
|
|
}
|
|
}
|
|
out, _, held, err := Compose(context.Background(), b, Mesh{}, Scope{Node: "n1"}, fps)
|
|
if err != nil {
|
|
t.Errorf("%s: the node is refused: %v", row.name, err)
|
|
continue
|
|
}
|
|
// A held module keeps what it contributes (issue 774, 2026-10-31: nextcloud held for its admin secret dropped out
|
|
// of postgres's received file, and postgres's helper would have locked its login): only its own secrets are left
|
|
// out of its stand-in, never its contributions, so the provider's received file names every consumer.
|
|
storeKey, _ := delivery.ComposedKey("n1", "artifacts", fps["artifacts"])
|
|
// And never a stand-in for a grant not yet issued: the provider is applied, so a held module whose grant is
|
|
// not issued is composed without that contribution, never with "foreseen: made by the next send".
|
|
for key, c := range out {
|
|
if got, _ := json.Marshal(c.Resources); strings.Contains(string(got), "foreseen") {
|
|
t.Errorf("%s: %s applies a stand-in: %s", row.name, key, got)
|
|
}
|
|
}
|
|
for _, m := range []string{"check", "heal"} {
|
|
if row.ungranted {
|
|
break
|
|
}
|
|
if got, _ := json.Marshal(out[storeKey].Resources); !strings.Contains(string(got), `\"from\": \"`+m+`\"`) {
|
|
t.Errorf("%s: the provider's received file drops %s's contribution: %s", row.name, m, got)
|
|
}
|
|
}
|
|
for _, m := range []string{"node-tools", "check", "lamp", "heal"} {
|
|
key, _ := delivery.ComposedKey("n1", m, fps[m])
|
|
_, composed := out[key]
|
|
why, isHeld := held[m]
|
|
want, wantHeld := row.held[m]
|
|
if isHeld != wantHeld || composed == wantHeld {
|
|
t.Errorf("%s: %s held %v (%q), composed %v; want held %v", row.name, m, isHeld, why, composed, wantHeld)
|
|
} else if isHeld && !strings.Contains(why, want) {
|
|
t.Errorf("%s: %s is held for %q, want its first reason, %q", row.name, m, why, want)
|
|
}
|
|
}
|
|
}
|
|
}
|