diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index 3a1e413..c0bd6c0 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -131,6 +131,8 @@ func grantsAsTheControllers(t *testing.T, got, want string) { } } +const SignersBucketName = "mesh-vault_signers" + func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) { raw, err := os.ReadFile(filepath.Join("testdata", "controller-user-lists.json")) if err != nil { @@ -237,6 +239,21 @@ func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) { if !slices.Contains(perms.Subscribe, VaultSecretChanged) { t.Errorf("node one does not hear the vault's secret-changed: %v", perms.Subscribe) } + // L3 s6 (ADR 0550 §2): while the vault serves `sign`, a node asks it in its own name alone, and reads the vault's + // `signers` and the lease key, never writing either. + signs, _ := PermissionsFor(Principal{Kind: KindNode, Node: "one", AsksTheVaultToSign: true}) + signersGet := "$JS.API.DIRECT.GET.KV_" + SignersBucketName + ".$KV." + SignersBucketName + ".key" + leaseGet := "$JS.API.DIRECT.GET.KV_" + LeaseBucket + ".$KV." + LeaseBucket + ".holder" + for subject, want := range map[string]bool{VaultSignCall(one): true, signersGet: true, leaseGet: true, + VaultSignCall(Principal{Kind: KindNode, Node: "two"}): false, "$KV." + SignersBucketName + ".key": false, + "$KV." + LeaseBucket + ".holder": false} { + if MayPublish(signs, subject) != want { + t.Errorf("a node asking sign may publish %s: %v, want %v", subject, !want, want) + } + } + if MayPublish(perms, VaultSignCall(one)) || MayPublish(perms, signersGet) { + t.Error("a node asks sign or reads signers while the vault does not serve sign") + } // And what a module on another node serves from its settings, that key alone (mailu's smtp domain, issue 1126). users, err := Users(Records{Nodes: []string{"one", "two"}, Assigned: map[string][]Declared{ "two": {{Module: "mailer", Served: []string{"domain", "not.a.token"}}}}}) diff --git a/go/buscompose/nats.go b/go/buscompose/nats.go index 682c37e..fb5f6a5 100644 --- a/go/buscompose/nats.go +++ b/go/buscompose/nats.go @@ -217,6 +217,11 @@ type Principal struct { // ADR 0547 §2, ADR 0550 §5, issue 1126): its modules' own secrets sealed to its key, for the compositions it makes // itself. Set while the vault's holder serves it. AsksItsOwnSecrets bool + // AsksTheVaultToSign is a machine principal whose node-engine may ask the vault's `sign` in its own name, read the + // vault's public signing halves (`signers`) and the controller's lease key (novox/hq ADR 0550 §2): its own node's + // certificate, the keys it trusts, and the lease write that drops the controller's key. Set while the vault's holder + // serves `sign`. + AsksTheVaultToSign bool // ServedReads are, for a machine principal, the keys of mesh-config's resolved values that other nodes' modules serve // from (`..`, catalogue.ServedSettings): what every consumer is told, and so what a node composing // its own modules reads of a provider elsewhere (issue 1126), each key alone. Sorted, none twice. @@ -360,6 +365,10 @@ var VaultNodeKey = SeatVerb{Seat: "mesh-vault", Verb: "node-key"} // key (issue 1126, ADR 0550 §5); granted as Principal.AsksItsOwnSecrets says. var VaultOwn = SeatVerb{Seat: "mesh-vault", Verb: "own"} +// VaultSign is the vault's verb a machine's node-engine asks in its own name for its own node's certificate (novox/hq +// ADR 0550 §2); granted as Principal.AsksTheVaultToSign says. +var VaultSign = SeatVerb{Seat: "mesh-vault", Verb: delivery.SignVerb} + // VerbsTheControllerActsOnAWarrant are the other seat verbs the controller calls when the operator's warrant // chooses them (novox/hq ADR 0259): a machine's service restarted, and a walk started or stopped through the // controller's own `plans`, a machine's held builds sent by its own `push` and the builds held after a failed release @@ -953,6 +962,14 @@ func PermissionsFor(p Principal) (Permissions, error) { // on it rather than at the next unrelated change (ADR 0547 §2). sub = append(sub, VaultSecretChanged) } + // And, while the vault serves `sign`, its own node's certificate in its own name, the vault's public signing + // halves, read and watched, and the controller's lease key, read and watched, which says when the controller's key + // is no longer trusted (ADR 0550 §2). Read only. + if p.AsksTheVaultToSign { + pub = append(pub, VaultSignCall(p)) + pub = append(pub, readGrants(delivery.SignersBucket, "")...) + pub = append(pub, readGrants(LeaseBucket, leaseKey)...) + } // And, on the node holding mesh-broker, the grants part's sealed user list it follows from take-over (novox/hq ADR // 0547 clause 7): one key, and an object by its name. if p.FollowsGrantsUserList { @@ -1845,6 +1862,11 @@ func VaultOwnCall(p Principal) string { return "mesh.seat." + VaultOwn.Seat + "." + CallKind + "." + VaultOwn.Verb + "." + CallerToken(p.Username()) } +// VaultSignCall is the subject a machine principal's node-engine asks the vault's `sign` on, in its own name. +func VaultSignCall(p Principal) string { + return "mesh.seat." + VaultSign.Seat + "." + CallKind + "." + VaultSign.Verb + "." + CallerToken(p.Username()) +} + // VaultSecretChanged is the vault's event that a secret changed (a name and its generation, never a value), which a // node composing its own hears to compose again (issue 1126, ADR 0547 §2). var VaultSecretChanged = "mesh.mod." + VaultOwn.Seat + ".event.secret-changed" diff --git a/go/buscompose/users.go b/go/buscompose/users.go index 15cbd14..2424fcf 100644 --- a/go/buscompose/users.go +++ b/go/buscompose/users.go @@ -165,7 +165,7 @@ func Users(r Records) ([]Principal, error) { } out = append(out, Principal{Kind: KindNode, Node: node, WitnessesController: witness, Checks: checks, MovesIntoTheVault: servesMoveIn(r), StatesItsKeyToTheVault: servesVerb(r, VaultNodeKey), - AsksItsOwnSecrets: servesVerb(r, VaultOwn), + AsksItsOwnSecrets: servesVerb(r, VaultOwn), AsksTheVaultToSign: servesVerb(r, VaultSign), FollowsGrantsUserList: bus, ServedReads: servedOn(served, node)}) // **And the machine's provisioner client, on an account of its own** (novox/hq ADR 0349 clause 11): one // per machine that runs it, delivered to the module running it. diff --git a/go/catalogue/seats.go b/go/catalogue/seats.go index 0417420..ba5ff8f 100644 --- a/go/catalogue/seats.go +++ b/go/catalogue/seats.go @@ -228,6 +228,14 @@ var defaultSeats = append([]Seat{ "ADR 0561 §9): only while the controller's lease is handed away, never a node's account or the " + "controller's, never anew for one the vault holds a value or an adopted hash for. Never the value.", Input: schema(map[string]string{"user": "the bus user, e.g. ace.mesh-build-go"}, []string{"user"})}, + {Name: "sign", Optional: true, Description: "Sign with the vault's key or authority (novox/hq ADR 0550 §2): " + + "kind node-certificate signs the certificate of the calling machine's internal name for the serving key " + + "its record states, built from bus state alone and asked by that machine's node-engine only; kind " + + "enrolment signs an enrolment reply under its own context, for a caller the adoption record names. " + + "Answers the certificate or the signature and the signer's fingerprint, and says signed; never a key.", + Input: schema(map[string]string{"kind": "node-certificate or enrolment", + "node": "the machine, for a node certificate: the caller's own", + "reply": "the enrolment reply's bytes in base64, for an enrolment"}, []string{"kind"})}, }}, // The owner of every config value (novox/hq ADR 0348 §9, design 57 K3): its holder alone writes the buckets // mesh-config_values, _resolved, _predefined and _proposals, and serves the one verb `config` (§1, §10). diff --git a/go/compose/composed.go b/go/compose/composed.go index 2c6313c..a6a8bea 100644 --- a/go/compose/composed.go +++ b/go/compose/composed.go @@ -8,6 +8,7 @@ import ( "sort" "strings" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/declaration" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/overlay" @@ -76,6 +77,42 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli if !found { return nil, nil, nil, fmt.Errorf("compose: the mesh has no node %q", node) } + // A folded module built by a build of the old form (Node.RunsOwn) is left to the node: composed as a stub, so what + // it provides and requires still shapes the rest, and its resources are then the ones the node runs. + carried := map[string]bool{} + var runsOwn *delivery.Composed + for i := range reading.Nodes { + n := &reading.Nodes[i] + if n.Name != node || n.RunsOwn == nil { + continue + } + runsOwn = n.RunsOwn + for m := range inOwn { + if !folds(m) { + continue + } + man, ok := n.Kept[m] + if !ok { + man, ok = reading.Catalogue[m] + } + if !ok || !NamesBuild(man) { + continue + } + carried[m] = true + kept := make(map[string]catalogue.Manifest, len(n.Kept)+1) + for k, v := range n.Kept { + kept[k] = v + } + kept[m] = Stub(man) + n.Kept = kept + } + } + isCarried := func(module, id string) bool { + if module == "" { + module, _ = declaration.ModuleGuess(id) + } + return carried[module] + } d, problems, err := ComposeIn(reading, node, mode) if err != nil { return nil, nil, nil, err @@ -118,6 +155,7 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli left := map[string]bool{} for _, raw := range envelope.Resources { var head struct { + ID string `json:"id"` Module string `json:"module"` Type string `json:"type"` Path string `json:"path"` @@ -125,6 +163,9 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli if err := json.Unmarshal(raw, &head); err != nil { return nil, nil, nil, err } + if isCarried(head.Module, head.ID) { + continue // what the stub composed of it: the node's own carries what runs of it instead + } if folds(head.Module) { // Into the node's own, in its place among the node's resources, its module taken off (mesh-controller's // compositionsOf): the node's own composition is then the controller's, byte for byte. @@ -151,6 +192,18 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli } kept = append(kept, raw) } + // What runs of each carried module, in the node's own composition, each after the resource it runs after. + if len(carried) > 0 { + var err error + if kept, foldedPaths, err = carry(kept, foldedPaths, runsOwn, isCarried); err != nil { + return nil, nil, nil, err + } + for _, p := range runsOwn.Filled { + if foldedPaths[p] && !slices.Contains(envelope.Filled, p) { + envelope.Filled = append(envelope.Filled, p) + } + } + } // A filled file of a module's own composition goes with that module (SplitComposed); one of a folded module goes with // the node's own, after them, in the declaration's order; one of a module left out is dropped. var filled, ownFilled []string @@ -188,3 +241,83 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli sort.Strings(skipped) return out, skipped, stood, nil } + +// NamesBuild says a manifest names what a build makes: its build, or a resource naming an artifact. A node cannot +// compose such a module folded into its own composition: the build of the old form it runs is held by the controller +// alone (issue 1126). +func NamesBuild(m catalogue.Manifest) bool { + if m.Build != nil { + return true + } + for _, r := range m.Resources { + if _, ok := r["artifact"]; ok { + return true + } + } + return false +} + +// Stub is m with nothing to build and no resource but its directories: what it provides, requires, serves and grants +// still shape the rest of the composition, the places its own secrets, data and grants name (${dir:…}) still resolve, +// and nothing else of its own is composed (issue 1126: nats's ${dir:mesh-state} on novox). +func Stub(m catalogue.Manifest) catalogue.Manifest { + m.Build = nil + var dirs []map[string]any + for _, r := range m.Resources { + if r["type"] == "directory" { + dirs = append(dirs, r) + } + } + m.Resources = dirs + return m +} + +// carry puts into the node's resources (kept, in the declaration's order) the resources of the node's own composition as +// it runs (runs) that belong to a carried module, each after the resource it runs after, or first when that one is +// not among them; a file among them is a folded path. +func carry(kept []json.RawMessage, foldedPaths map[string]bool, runs *delivery.Composed, carried func(module, id string) bool) ([]json.RawMessage, map[string]bool, error) { + idOf := func(raw json.RawMessage) (string, string, string, string, error) { + var head struct { + ID string `json:"id"` + Module string `json:"module"` + Type string `json:"type"` + Path string `json:"path"` + } + err := json.Unmarshal(raw, &head) + return head.ID, head.Module, head.Type, head.Path, err + } + present := map[string]bool{} + for _, raw := range kept { + if id, module, _, _, err := idOf(raw); err == nil && module == "" { + present[id] = true + } + } + // By the id of the resource of the node's own it runs after, "" for first: the last one before it that is composed. + after := map[string][]json.RawMessage{} + anchor := "" + for _, raw := range runs.Resources { + id, module, typ, path, err := idOf(raw) + if err != nil { + return nil, nil, fmt.Errorf("compose: a resource the node runs in its own composition: %w", err) + } + if !carried(module, id) { + if module == "" && present[id] { + anchor = id + } + continue + } + after[anchor] = append(after[anchor], raw) + if typ == string(declaration.TypeFile) { + foldedPaths[path] = true + } + } + out := make([]json.RawMessage, 0, len(kept)+len(runs.Resources)) + out = append(out, after[""]...) + for _, raw := range kept { + out = append(out, raw) + if id, module, _, _, err := idOf(raw); err == nil && module == "" { + out = append(out, after[id]...) + } + } + return out, foldedPaths, nil +} diff --git a/go/compose/frombus/answer.go b/go/compose/frombus/answer.go index 20115be..078dbbc 100644 --- a/go/compose/frombus/answer.go +++ b/go/compose/frombus/answer.go @@ -4,9 +4,11 @@ import ( "context" "fmt" "regexp" + "slices" "sort" "strings" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/compose" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -32,7 +34,13 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st if err != nil { return refuse(fmt.Errorf("frombus: placement: %w", err)) } - in, err := Input(ctx, r, mesh, s) + if s.Running == nil { + s.Running = running + } + // Every module it cannot compose is named, not the first only (issue 1126): a build it cannot read or take in, and + // below, a module the composition refuses, each composed as a stub so the rest is composed and compared. + failed := map[string]string{} + in, err := input(ctx, r, mesh, s, func(module string, err error) { failed[module] = err.Error() }) if err != nil { return refuse(err) } @@ -63,8 +71,20 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st } } composed, skipped, stood, err := compose.CompositionsShadow(in, s.Node, fps) - if err != nil { - return refuse(err) + for err != nil { + module := culprit(in, s.Node, fps, failed, err) + if module == "" { + // Not one module's: the node's own (a value served elsewhere, the mesh's network), said once as the node's. + if len(failed) == 0 { + return refuse(err) + } + failed[delivery.HaveNodeWord] = err.Error() + composed, skipped, stood = nil, nil, nil + break + } + failed[module] = err.Error() + in = stubbed(in, s.Node, module) + composed, skipped, stood, err = compose.CompositionsShadow(in, s.Node, fps) } // The node's own composition is judged only where the node runs one: one that runs none applies it only once a want // names it (the node agent's own composer hands it on that want alone), so a candidate does not add it. @@ -79,8 +99,49 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st } } } - cmp := delivery.CompareComposed(composed, running) - a.Same, a.Differs, a.Removed = cmp.Same, cmp.Differs, cmp.Removed + if _, whole := failed[delivery.HaveNodeWord]; whole { + // Nothing composed to compare: every refusal is said, and nothing else. + for _, m := range sortedKeys(failed) { + a.Refusals = append(a.Refusals, delivery.ModuleRefused{Module: m, Why: failed[m]}) + } + a.Verdict = delivery.VerdictOf(a) + return a + } + // A module the composition folds inside the node's own is judged there: the agent also says it runs by module, from + // the declaration it kept before the switch (agent.Everything), which is no removal (issue 1126, seed #531). + if len(s.InOwn) > 0 { + var judged []delivery.Composed + for _, c := range running { + if c.Module == "" || !slices.Contains(s.InOwn, c.Module) { + judged = append(judged, c) + } + } + running = judged + } + cmp := delivery.CompareComposedWith(composed, running, s.SameValue) + // The answer rides in the node's have key: its shown bodies are bounded, the rest named only. + budget := MaxShown + for i := range cmp.Differs { + kept := cmp.Differs[i].Bodies[:0] + for _, b := range cmp.Differs[i].Bodies { + if n := len(b.Composed) + len(b.Running); n <= budget { + budget -= n + kept = append(kept, b) + } else { + kept = append(kept, delivery.ResourceBodies{ID: b.ID, Why: b.Why}) + a.Note = "some differing resources are named without their bodies: the answer holds at most " + + fmt.Sprint(MaxShown) + " bytes of them" + } + } + cmp.Differs[i].Bodies = kept + } + // A refused module was composed as a stub: what it composed is no difference; its refusal is said. + notFailed := func(m string) bool { _, ok := failed[m]; return !ok } + a.Same = slices.DeleteFunc(cmp.Same, func(m string) bool { return !notFailed(m) }) + a.Removed = slices.DeleteFunc(cmp.Removed, func(m string) bool { return !notFailed(m) }) + a.Differs = slices.DeleteFunc(cmp.Differs, func(d delivery.ModuleDiffers) bool { return !notFailed(d.Module) }) + cmp.Added = slices.DeleteFunc(cmp.Added, func(d delivery.ModuleAdded) bool { return !notFailed(d.Module) }) + skipped = slices.DeleteFunc(skipped, func(m string) bool { return !notFailed(m) }) // What the group adds, by node and module: placed by a candidate and not by the trunk on that node. added := map[string]bool{} @@ -138,13 +199,25 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st if _, isAdded := addedHere[m]; isAdded { continue // a module the candidate adds here: what it lacks is said under Added } + if _, isFailed := failed[m]; isFailed { + continue // said below, as why it could not be composed + } a.Refusals = append(a.Refusals, delivery.ModuleRefused{Module: m, Why: "stands in for " + strings.Join(refusals[m], ", ")}) } + for _, m := range sortedKeys(failed) { + a.Refusals = append(a.Refusals, delivery.ModuleRefused{Module: m, Why: failed[m]}) + } sort.Slice(a.Refusals, func(i, j int) bool { return a.Refusals[i].Module < a.Refusals[j].Module }) a.Verdict = delivery.VerdictOf(a) return a } +// maxCulprits bounds the modules one refusal is tried on, each a whole composition. +const maxCulprits = 8 + +// MaxShown bounds the bytes of shown resource bodies in one answer (delivery.ModuleDiffers.Bodies). +const MaxShown = 256 << 10 + var ( grantStandIn = regexp.MustCompile(`^the grant of \S+ for (\S+) on (\S+)$`) needsStandIn = regexp.MustCompile(`^(\S+) on (\S+) needs `) @@ -170,3 +243,57 @@ func standsInFor(x, node string) (string, string) { } return node, "" } + +// culprit is the module placed on node whose composition the error is of: the first, in placement order, whose stub +// (compose.Stub) makes the composition succeed or fail otherwise; empty when none does (the error is not one module's). +// Modules already refused are passed over. +func culprit(in compose.Input, node string, fps map[string]string, failed map[string]string, err error) string { + var assigned []string + for _, n := range in.Nodes { + if n.Name == node { + assigned = n.Assigned + } + } + // Only the modules the error names: the words of a module's refusal name it. Each try is a whole composition, so a + // refusal naming none is no module's, answered at once rather than after a composition per module (issue 1126: ace's + // answer took 7m46s). + var order []string + for _, m := range assigned { + if strings.Contains(err.Error(), m) && len(order) < maxCulprits { + order = append(order, m) + } + } + for _, m := range order { + if _, done := failed[m]; done { + continue + } + if _, _, _, again := compose.CompositionsShadow(stubbed(in, node, m), node, fps); again == nil || again.Error() != err.Error() { + return m + } + } + return "" +} + +// stubbed is in with module composed as a stub on node alone (compose.Stub, as node's Kept). +func stubbed(in compose.Input, node, module string) compose.Input { + out := in + out.Nodes = append([]compose.Node(nil), in.Nodes...) + for i := range out.Nodes { + if out.Nodes[i].Name != node { + continue + } + man, ok := out.Nodes[i].Kept[module] + if !ok { + if man, ok = in.Catalogue[module]; !ok { + return in + } + } + kept := make(map[string]catalogue.Manifest, len(out.Nodes[i].Kept)+1) + for k, v := range out.Nodes[i].Kept { + kept[k] = v + } + kept[module] = compose.Stub(man) + out.Nodes[i].Kept = kept + } + return out +} diff --git a/go/compose/frombus/answer_rows_test.go b/go/compose/frombus/answer_rows_test.go new file mode 100644 index 0000000..74b1601 --- /dev/null +++ b/go/compose/frombus/answer_rows_test.go @@ -0,0 +1,209 @@ +package frombus + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/overlay" +) + +// Issue 1126 (the seed's shadow, round 2): a node's answer names every module it cannot compose, not the first only; +// and a module folded into its own composition that names a build of the old form (de-spiegel on novox, nats, photos, +// systemd-resolved: their haves name a build-... id and no fingerprint) is left to the node, carried as it runs. +func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { + ctx := context.Background() + image := "alpine@sha256:" + strings.Repeat("a", 64) + manifest := func(module string) []byte { + // Its data names its directory, as nats's names ${dir:mesh-state} (issue 1126: novox's own composition refused + // "nats says ${dir:mesh-state}, and nats declares no directory mesh-state" when its stub held no directory). + return []byte(`{"module":"` + module + `","version":"1","capabilities":["container-runtime"],` + + `"own-secrets":{"token":"${dir:state}/token"},` + + `"resources":[{"id":"state","type":"directory","path":"/var/lib/` + module + `","mode":"0700"},` + + `{"id":"server","type":"container","name":"` + module + `","artifact":"site"}],` + + `"build":{"artifacts":[{"name":"site","kind":"upstream","from":"` + image + `"}]}}`) + } + lemurs := []byte(`{"module":"lemurs","version":"1","resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`) + fp := func(c string) string { return "src2:" + strings.Repeat(c, 64) } + mk := func(placed ...string) *bus { + p := delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"}} + for _, m := range placed { + p.Modules = append(p.Modules, delivery.PlacedModule{Module: m}) + } + b := &bus{placements: map[string]delivery.Placement{"n1": p}, + records: map[string]delivery.NodeRecord{"n1": {Node: "n1", TunnelKey: publicKey(t), SealingKey: "sk", Capabilities: []string{"systemd", "container-runtime"}}}, + modules: map[string]delivery.Module{"lemurs": {Manifest: delivery.ManifestDigest(lemurs)}}, + manifests: map[string][]byte{delivery.ManifestDigest(lemurs): lemurs}, + builds: map[string]*delivery.BuilderBuild{}} + for _, m := range []string{"web", "api", "spiegel"} { + b.own = append(b.own, compose.OwnSecretRow{Node: "n1", Module: m, Name: "token", Origin: "made", SealedTo: "sk", Sealed: `"box"`}) + } + for _, m := range []string{"web", "api", "spiegel"} { + raw := manifest(m) + b.manifests[delivery.ManifestDigest(raw)] = raw + b.modules[m] = delivery.Module{Manifest: delivery.ManifestDigest(raw)} + } + // web and api are built, their keys done and waiting for the reference of their image; spiegel has no build of + // the new form at all. + for i, m := range []string{"web", "api"} { + mod := b.modules[m] + mod.FP2 = fp(string(rune('1' + i))) + b.modules[m] = mod + b.builds[m+" "+mod.FP2] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mod.Manifest, + Artifacts: []delivery.BuiltArtifact{{Name: "site", Kind: "image", Digest: "sha256:" + strings.Repeat("b", 64)}}} + } + return b + } + given := func(b *bus) map[string]*delivery.Placement { + p := b.placements["n1"] + return map[string]*delivery.Placement{"n1": &p} + } + + // Every refusal: two builds taken in, and a module the composition refuses (its image never built), each named. + b := mk("lemurs", "web", "api", "spiegel") + a := Answer(ctx, b, Mesh{}, Scope{Node: "n1", Placements: given(b)}, + map[string]string{"lemurs": fp("e"), "web": fp("1"), "api": fp("2"), "spiegel": fp("3")}, nil, "h1", 3) + var refused []string + for _, r := range a.Refusals { + refused = append(refused, r.Module) + } + if a.Verdict != delivery.CandidateRefused || a.Refused != "" || !reflect.DeepEqual(refused, []string{"api", "spiegel", "web"}) { + t.Errorf("every refusal: %+v, want api, spiegel and web refused, each named", a) + } + for _, r := range a.Refusals { + want := map[string]string{"api": "waits for the reference of site", "web": "waits for the reference of site", + "spiegel": "spiegel has not been built for this mesh"}[r.Module] + if !strings.Contains(r.Why, want) { + t.Errorf("%s refused as %q, want %q", r.Module, r.Why, want) + } + } + if raw, err := delivery.EncodeHaveNode(delivery.HaveNode{Candidates: map[string]delivery.CandidateAnswer{"g": a}}); err != nil { + t.Errorf("the answer is not one a have holds: %v (%s)", err, raw) + } + if lemursOK := slices.ContainsFunc(a.Added, func(m delivery.ModuleAdded) bool { return m.Module == "lemurs" }); !lemursOK { + t.Errorf("lemurs, which composes, is not answered beside the refusals: %+v", a) + } + + // The fold: spiegel runs inside the node's own composition at a build of the old form. Its resources are carried as + // they run, each after the one it runs after: the node's own is the one it runs, and the node composes itself. + b = mk("lemurs", overlay.Name, "spiegel") + p := b.placements["n1"] + p.Decided.Endpoint = "n1.example:51820" + b.placements["n1"] = p + fps := map[string]string{"lemurs": fp("e")} + stub := Scope{Node: "n1"} + in, err := Input(ctx, b, Mesh{}, stub) + if err != nil { + t.Fatal(err) + } + in.Nodes[0].Kept = map[string]catalogue.Manifest{"spiegel": compose.Stub(in.Catalogue["spiegel"])} + base, _, err := compose.Compositions(in, "n1", fps, compose.Reading) + if err != nil { + t.Fatal(err) + } + // A refusal naming no module placed here is no module's: no compose is spent stubbing each in turn (issue 1126: ace's + // answer took 7m46s, one compose per module), though here a stub would compose. + if m := culprit(in, "n1", fps, map[string]string{}, fmt.Errorf("nothing in this mesh provides \"artifact-store\"")); m != "" { + t.Errorf("a refusal naming no placed module was laid on %s", m) + } + ownKey, _ := delivery.ComposedNodeKey("n1") + own := base[ownKey] + carriedRes := []json.RawMessage{ + json.RawMessage(`{"id":"spiegel.state","type":"directory","path":"/var/lib/spiegel","mode":"0700"}`), + json.RawMessage(`{"id":"spiegel.server","type":"container","name":"spiegel","image":"registry.example/spiegel@sha256:` + strings.Repeat("c", 64) + `"}`)} + // They run after the node's first own resource. + own.Resources = append(append(append([]json.RawMessage{}, own.Resources[:1]...), carriedRes...), own.Resources[1:]...) + var running []delivery.Composed + for k, c := range base { + if k == ownKey { + c = own + } + running = append(running, c) + } + scope := Scope{Node: "n1", InOwn: []string{"spiegel"}, Running: running} + got, _, err := Compose(ctx, b, Mesh{}, scope, fps) + if err != nil { + t.Fatalf("the node does not compose itself with spiegel folded at a build of the old form: %v", err) + } + gotOwn, _ := json.Marshal(got[ownKey].Resources) + wantOwn, _ := json.Marshal(own.Resources) + if string(gotOwn) != string(wantOwn) { + t.Errorf("the node's own composition is\n%s\nwant what runs\n%s", gotOwn, wantOwn) + } + a = Answer(ctx, b, Mesh{}, Scope{Node: "n1", Placements: given(b), InOwn: []string{"spiegel"}}, fps, running, "h1", 3) + if a.Verdict != delivery.CandidateSame { + t.Errorf("the fold's answer: %+v, want the same", a) + } + // (A) The agent also says spiegel runs by module, from the declaration kept before the switch (agent.Everything): a + // module the composition folds is judged inside the node's own, not as removed (issue 1126, seed #531's answer). + keptToo := append(append([]delivery.Composed{}, running...), delivery.Composed{Node: "n1", Module: "spiegel", Resources: carriedRes}) + if a = Answer(ctx, b, Mesh{}, Scope{Node: "n1", Placements: given(b), InOwn: []string{"spiegel"}}, fps, keptToo, "h1", 3); a.Verdict != delivery.CandidateSame || len(a.Removed) != 0 { + t.Errorf("a folded module running from the kept declaration is answered %+v, want the same", a) + } + // (B) The node's own composition carries its bus membership as it runs it (the controller's, from its store): composed + // the same, not 'in what runs, not composed' (seed #531, novox). + membership := json.RawMessage(`{"id":"bus-membership","type":"file","path":"` + catalogue.BusMembershipPath + `","sealed":"{\"box\":\"m\"}","mode":"0600"}`) + withMembership := make([]delivery.Composed, 0, len(running)) + for _, c := range running { + if c.Module == "" { + c.Resources = append(append([]json.RawMessage{}, c.Resources...), membership) + } + withMembership = append(withMembership, c) + } + if a = Answer(ctx, b, Mesh{}, Scope{Node: "n1", Placements: given(b), InOwn: []string{"spiegel"}}, fps, withMembership, "h1", 3); a.Verdict != delivery.CandidateSame { + t.Errorf("the node's own with its bus membership is answered %+v, want the same", a) + } + // Without what runs, the fold composes spiegel from its manifest, as before, and the composition refuses it. + if _, _, err := Compose(ctx, b, Mesh{}, Scope{Node: "n1", InOwn: []string{"spiegel"}}, fps); err == nil || + !strings.Contains(err.Error(), "spiegel has not been built for this mesh") { + t.Errorf("composed spiegel without what runs: %v", err) + } +} + +// A module another node cannot run in this composition offers nothing in it: the refusal names it and why +// (issue 1126, probe #532: ace and g14 answered "nothing in this mesh provides artifact-store … assign distribution to a +// node" while novox runs it). And a refusal no module's stub changes is answered at once, not after a compose per module +// (ace took 7m46s). +func otherNodeNotResolvedIsNamed(t *testing.T) { + ctx := context.Background() + prov := []byte(`{"module":"prov","version":"1","capabilities":["container-runtime"],"provides":[{"name":"thing","scope":"mesh"}]}`) + cons := []byte(`{"module":"cons","version":"1","requires":["thing"]}`) + // n2 runs prov, which needs a container runtime n2's record does not say it has: n2's set does not resolve here. + b := &bus{placements: map[string]delivery.Placement{ + "n1": {Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1", Endpoint: "n1.example:51820"}, Modules: []delivery.PlacedModule{{Module: "cons"}, {Module: overlay.Name}}}, + "n2": {Node: "n2", Decided: delivery.NodeDecided{Address: "10.77.0.2"}, Modules: []delivery.PlacedModule{{Module: "prov"}, {Module: overlay.Name}}}}, + records: map[string]delivery.NodeRecord{"n1": {Node: "n1", TunnelKey: publicKey(t)}, "n2": {Node: "n2", TunnelKey: publicKey(t)}}, + modules: map[string]delivery.Module{"prov": {Manifest: delivery.ManifestDigest(prov)}, "cons": {Manifest: delivery.ManifestDigest(cons)}}, + manifests: map[string][]byte{delivery.ManifestDigest(prov): prov, delivery.ManifestDigest(cons): cons}} + given := map[string]*delivery.Placement{} + for n, p := range b.placements { + p := p + given[n] = &p + } + a := Answer(ctx, b, Mesh{}, Scope{Node: "n1", Placements: given}, map[string]string{"cons": "src2:" + strings.Repeat("1", 64)}, nil, "h", 1) + if a.Verdict != delivery.CandidateRefused || !strings.Contains(a.Refused, "prov on n2 lacks container-runtime") { + t.Errorf("a refusal for want of n2's provider = %+v", a) + } +} + +// A node that reaches no artifact store says what its composition holds of one (issue 1126: ace and g14 answered +// "reaches no artifact store", and nothing said why). +func noStoreReachedSaysWhy(t *testing.T) { + in := compose.Input{Catalogue: map[string]catalogue.Manifest{ + "distribution": {Module: "distribution", Serves: map[string]map[string]any{catalogue.ArtifactStoreProvision: {"port": 5000}}}}, + Nodes: []compose.Node{{Name: "g14", Assigned: []string{"adwaita"}}, {Name: "novox", Assigned: []string{"distribution"}}}} + if got := storeSought(in); got != "the composition holds g14, novox; artifact-store is served by distribution on novox, none of it on the private network with a port" { + t.Errorf("said %q", got) + } + in.Nodes = in.Nodes[:1] + if got := storeSought(in); got != "the composition holds g14; no module placed there serves artifact-store" { + t.Errorf("said %q", got) + } +} diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index 0c1c28f..c5676c4 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -83,6 +83,16 @@ type Reader interface { Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) } +// SignersReader is a Reader that also reads the vault's public authority from its state `signers` (delivery.SignersBucket, +// key delivery.SignersAuthorityKey; ADR 0550 §2). A Reader that has it composes the mesh's authority as a bundle, the +// controller's and the vault's; one that has not, or that answers none, composes the controller's alone, as before. +type SignersReader interface { + // VaultAuthority is the vault's authority certificate as PEM, checked (delivery.DecodeSignerAuthority); empty when the + // vault has published none. A read that fails is said by the reader and answered empty: a node that may not read the + // state yet composes as before rather than not at all. + VaultAuthority(ctx context.Context) string +} + // Mesh is what is true of the whole mesh and is no part's state on the bus yet: the private network's range and the // internal names' suffix as genesis was told them, and the mesh's authority certificate (public). Empty fields take // compose's defaults. @@ -113,12 +123,21 @@ type Mesh struct { // // InOwn is the modules the composing node runs inside its own composition (InOwn of what it runs), composed there again // (compose.Node.InOwn) rather than cut out as modules no build is given for (issue 1126). +// +// Running is what the composing node runs; of it, its own composition (`.node`) is given to the composition +// (compose.Node.RunsOwn), so a module folded there that names a build of the old form is left to the node, carried +// as it runs (issue 1126). Nil composes every folded module from its manifest. type Scope struct { Node string Every bool KeptAt map[string]string Placements map[string]*delivery.Placement InOwn []string + Running []delivery.Composed + // SameValue, in an answer, says whether two sealed values open to the same value (the node opens them with its own + // key), so a differing resource shows a value sealed again as such (delivery.CompareComposedWith). Nil shows them + // as differing. + SameValue func(a, b string) (same, known bool) } // InOwn is, of the modules p places on its node, those whose resources the node's own composition it runs @@ -154,6 +173,13 @@ func InOwn(running []delivery.Composed, p delivery.Placement) []string { // Input is the composition input for s, read through r. func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, error) { + return input(ctx, r, mesh, s, nil) +} + +// input is Input; with refuse set, a module placed on the composing node whose build cannot be read or taken in is +// handed to refuse and composed as a stub (compose.Stub, kept for that node alone) rather than ending the read, so a +// node's answer names every module it cannot compose (issue 1126). +func input(ctx context.Context, r Reader, mesh Mesh, s Scope, refuse func(module string, err error)) (compose.Input, error) { placements, err := r.Placements(ctx) if err != nil { return compose.Input{}, fmt.Errorf("frombus: placement: %w", err) @@ -203,6 +229,13 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er } } } + // And the vault's authority beside it, once the vault publishes one (ADR 0550 §2): a certificate either signed checks + // against the bundle. The controller's stays while a certificate it issued is in a node's record. + if sr, ok := r.(SignersReader); ok { + if vault := sr.VaultAuthority(ctx); vault != "" { + in.Authority = delivery.AuthorityBundle(in.Authority, vault) + } + } parse := func(raw []byte) (catalogue.Manifest, error) { if mesh.Predefined != nil { return catalogue.ParseManifestWith(raw, mesh.Predefined) @@ -280,6 +313,22 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er } n.InOwn[module] = true } + for i := range s.Running { + if s.Running[i].Module == "" { + own := s.Running[i] + n.RunsOwn = &own + // Its bus membership as it runs it: the controller composes it from its store, which the node does + // not read; until the grants part hands memberships sealed to the node, the node's own composition + // carries the one it runs (issue 1126, seed #531 on novox: "bus-membership in what runs, not + // composed", which applied would have taken the node's bus credential away). + if sealed := membershipOf(own); sealed != "" { + if in.BusMemberships == nil { + in.BusMemberships = map[string]string{} + } + in.BusMemberships[s.Node] = sealed + } + } + } } in.Nodes = append(in.Nodes, n) in.Holdings = append(in.Holdings, p.Held()...) @@ -309,7 +358,7 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er if _, ok := records[s.Node]; !ok { return compose.Input{}, fmt.Errorf("frombus: %s has written no record of itself yet", s.Node) } - if err := fromBuilds(ctx, r, &in, s, placements[s.Node], modules, read, parse); err != nil { + if err := fromBuilds(ctx, r, &in, s, placements[s.Node], modules, read, parse, refuse); err != nil { return compose.Input{}, err } // Its own modules' secrets, sealed to its own key by the vault: a composition carries them sealed (ADR 0550 §5). @@ -358,7 +407,29 @@ func Compose(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]s // by the one take-in rule (takein.Manifest), a templates archive read in the artifact store as the node reaches it // (issue 1126). func fromBuilds(ctx context.Context, r Reader, in *compose.Input, s Scope, p delivery.Placement, modules map[string]delivery.Module, - read func(module, digest string) (catalogue.Manifest, error), parse func([]byte) (catalogue.Manifest, error)) error { + read func(module, digest string) (catalogue.Manifest, error), parse func([]byte) (catalogue.Manifest, error), + refuse func(module string, err error)) error { + // stub keeps a refused module in the composition as a stub for this node alone (compose.Stub): what it provides + // and requires still shapes the rest. + stub := func(module string, err error) { + refuse(module, err) + for i := range in.Nodes { + if in.Nodes[i].Name != s.Node { + continue + } + man, ok := in.Nodes[i].Kept[module] + if !ok { + man, ok = in.Catalogue[module] + } + if !ok { + return + } + if in.Nodes[i].Kept == nil { + in.Nodes[i].Kept = map[string]catalogue.Manifest{} + } + in.Nodes[i].Kept[module] = compose.Stub(man) + } + } address, addressErr := "", error(nil) addressed := false archive := func(ctx context.Context, reference, digest string) ([]byte, error) { @@ -370,7 +441,7 @@ func fromBuilds(ctx context.Context, r Reader, in *compose.Input, s Scope, p del case addressErr != nil: return nil, addressErr case address == "": - return nil, fmt.Errorf("%s reaches no artifact store", s.Node) + return nil, fmt.Errorf("%s reaches no artifact store (%s)", s.Node, storeSought(*in)) } return r.Archive(ctx, catalogue.Routed(reference, address), digest) } @@ -409,14 +480,23 @@ func fromBuilds(ctx context.Context, r Reader, in *compose.Input, s Scope, p del } b, err := r.Build(ctx, pm.Module, m.FP2) if err != nil { - return fmt.Errorf("frombus: the build of %s at %s: %w", pm.Module, m.FP2, err) + err = fmt.Errorf("frombus: the build of %s at %s: %w", pm.Module, m.FP2, err) + if refuse == nil { + return err + } + stub(pm.Module, err) + continue } if b == nil || b.State != delivery.BuildDone { continue // no done build at the trunk's: composed from the indexer's manifest, as before } resolved, err := built(pm.Module, m.FP2, b) if err != nil { - return err + if refuse == nil { + return err + } + stub(pm.Module, err) + continue } in.Catalogue[pm.Module] = resolved } @@ -427,15 +507,22 @@ func fromBuilds(ctx context.Context, r Reader, in *compose.Input, s Scope, p del for _, module := range sortedKeys(s.KeptAt) { fp := s.KeptAt[module] b, err := r.Build(ctx, module, fp) - if err != nil { - return fmt.Errorf("frombus: the build of %s at %s: %w", module, fp, err) + switch { + case err != nil: + err = fmt.Errorf("frombus: the build of %s at %s: %w", module, fp, err) + case b == nil || b.State != delivery.BuildDone: + err = fmt.Errorf("frombus: %s is wanted at %s, of which the builder holds no done build", module, fp) } - if b == nil || b.State != delivery.BuildDone { - return fmt.Errorf("frombus: %s is wanted at %s, of which the builder holds no done build", module, fp) + var resolved catalogue.Manifest + if err == nil { + resolved, err = built(module, fp, b) } - resolved, err := built(module, fp, b) if err != nil { - return err + if refuse == nil { + return err + } + stub(module, err) + continue } if in.Nodes[i].Kept == nil { in.Nodes[i].Kept = map[string]catalogue.Manifest{} @@ -613,3 +700,42 @@ func sortedKeys[V any](m map[string]V) []string { sort.Strings(out) return out } + +// storeSought says, for a node that reaches no artifact store, what the composition holds of one: the nodes it holds and +// where each module serving the artifact store is placed, so the refusal names why (issue 1126). +func storeSought(in compose.Input) string { + var nodes, where []string + for _, n := range in.Nodes { + nodes = append(nodes, n.Name) + for _, a := range n.Assigned { + if m, ok := in.Catalogue[a]; ok { + if _, serves := m.Serves[catalogue.ArtifactStoreProvision]; serves { + where = append(where, a+" on "+n.Name) + } + } + } + } + sort.Strings(nodes) + sort.Strings(where) + said := "the composition holds " + strings.Join(nodes, ", ") + if len(where) == 0 { + return said + "; no module placed there serves " + catalogue.ArtifactStoreProvision + } + return said + "; " + catalogue.ArtifactStoreProvision + " is served by " + strings.Join(where, ", ") + + ", none of it on the private network with a port" +} + +// membershipOf is the sealed value of the bus membership resource (catalogue.BusMembershipID) of a node's own +// composition, or "". +func membershipOf(own delivery.Composed) string { + for _, raw := range own.Resources { + var r struct { + ID string `json:"id"` + Sealed string `json:"sealed"` + } + if json.Unmarshal(raw, &r) == nil && r.ID == catalogue.BusMembershipID() { + return r.Sealed + } + } + return "" +} diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go index 20a5ae5..4a6ee0e 100644 --- a/go/compose/frombus/frombus_test.go +++ b/go/compose/frombus/frombus_test.go @@ -25,14 +25,15 @@ import ( // bus is a Reader over values held in memory, counting whose config values were read. type bus struct { - placements map[string]delivery.Placement - records map[string]delivery.NodeRecord - modules map[string]delivery.Module - manifests map[string][]byte - values map[string]map[string]any // by node.module - valuesRead []string - own []compose.OwnSecretRow - ownErr error + vaultAuthority string + placements map[string]delivery.Placement + records map[string]delivery.NodeRecord + modules map[string]delivery.Module + manifests map[string][]byte + values map[string]map[string]any // by node.module + valuesRead []string + own []compose.OwnSecretRow + ownErr error // asked is every consumer credential asked; the fake refuses the whole ask, as the vault's own does, for a consuming // module its placement does not put on the node. asked []compose.CredentialKey @@ -74,6 +75,7 @@ func (b *bus) Served(_ context.Context, node, module string, keys []string) (map func (b *bus) Placements(context.Context) (map[string]delivery.Placement, error) { return b.placements, nil } +func (b *bus) VaultAuthority(context.Context) string { return b.vaultAuthority } func (b *bus) Records(context.Context) (map[string]delivery.NodeRecord, error) { return b.records, nil } func (b *bus) Modules(context.Context) (map[string]delivery.Module, error) { return b.modules, nil } func (b *bus) Manifest(_ context.Context, digest string) ([]byte, error) { @@ -119,6 +121,11 @@ func publicKey(t *testing.T) string { // controller's store fills for compose's own two-machine mesh (compose's aMesh: anchor the hub, laptop), and that Input // composes; a node reads only its own config values; what cannot be composed is refused, naming it. func TestInput(t *testing.T) { + // Issue 1126, the seed's shadow: every refusal named, an old-form module left to the node, and no store reached + // saying why (answer_rows_test.go). + t.Run("every refusal, and an old build left to the node", answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode) + t.Run("no store reached says why", noStoreReachedSaysWhy) + t.Run("another node not resolved is named", otherNodeNotResolvedIsNamed) ctx := context.Background() xterm, err := os.ReadFile("../../catalogue/testdata/catalogue-835bf22b-xterm.module.json") if err != nil { @@ -211,6 +218,14 @@ func TestInput(t *testing.T) { wj, _ := json.MarshalIndent(want, "", " ") t.Fatalf("Input from the bus differs from the store's:\n got %s\nwant %s", gj, wj) } + // L3 s8 (ADR 0550 §2): with the vault's authority published, the mesh's authority is the bundle of the controller's + // and the vault's; with none, the controller's alone, as above. + signing := mk() + signing.vaultAuthority = "the vault's certificate" + if bundled, err := Input(ctx, signing, Mesh{}, Scope{Node: "laptop"}); err != nil || + bundled.Authority != "the mesh's certificate\nthe vault's certificate\n" { + t.Errorf("the authority with the vault's published: %q, %v", bundled.Authority, err) + } if !reflect.DeepEqual(b.valuesRead, []string{"laptop." + overlay.Name, "laptop.xterm", "laptop.media"}) { t.Errorf("laptop read config values %v; a node reads its own slice alone", b.valuesRead) } diff --git a/go/compose/input.go b/go/compose/input.go index bf06c3c..2ca4f25 100644 --- a/go/compose/input.go +++ b/go/compose/input.go @@ -23,6 +23,7 @@ import ( "strings" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" ) // Input is everything the composition reads. Each field is what one read of the controller's inventory answers, named @@ -194,6 +195,11 @@ type Node struct { // resources into the node's own, as the controller's composed writer does, unless fps gives one of the new form. // The private network's module (overlay.Name), which nobody builds, is folded whether or not it is named here. InOwn map[string]bool + // RunsOwn is the node's own composition (`.node`) as it runs, when the node composes itself: a module folded + // there (InOwn) whose manifest names a build (NamesBuild) runs a build of the old form only the controller holds, + // so it is left to the node: Compositions composes it as a stub (Stub) and carries its resources as they run, each + // after the resource it runs after (issue 1126). Nil composes every folded module from its manifest, as before. + RunsOwn *delivery.Composed } // Place is where a node is, as its placement says (inv.Overlay). diff --git a/go/compose/network.go b/go/compose/network.go index afc8363..f51d6d7 100644 --- a/go/compose/network.go +++ b/go/compose/network.go @@ -24,6 +24,9 @@ type composer struct { in Input said []string seen map[string]bool + // unhostable is, from the last look at the rest of the mesh, each module assigned to another node that this + // composition finds that node cannot run: what it would have offered is missing here (issue 1126, probe #532). + unhostable []string } func newComposer(in Input) *composer { return &composer{in: in, seen: map[string]bool{}} } diff --git a/go/compose/resolve.go b/go/compose/resolve.go index d2fffd8..a481909 100644 --- a/go/compose/resolve.go +++ b/go/compose/resolve.go @@ -3,6 +3,7 @@ package compose import ( "fmt" "slices" + "strings" "git.novox.be/novox/mesh-sdk/go/catalogue" ) @@ -203,7 +204,7 @@ func (c *composer) resolution(nodeName string) (catalogue.Resolution, error) { return catalogue.Resolution{}, err } shelf := c.shelfFor(node) - world, _, _, err := c.theRestOfTheMesh(nodeName) + world, _, left, err := c.theRestOfTheMesh(nodeName) if err != nil { return catalogue.Resolution{}, err } @@ -226,6 +227,19 @@ func (c *composer) resolution(nodeName string) (catalogue.Resolution, error) { Account: node.Account, AccountHome: node.AccountHome, AgentAccount: node.AgentAccount, AgentAccountHome: node.AgentAccountHome}, world) if err != nil { + // What another node's set would have offered is missing when that set does not resolve here: said with the + // refusal, so "nothing provides" is read beside why (issue 1126, probe #532). + if len(left) > 0 { + var why []string + for _, l := range left { + why = append(why, l.String()) + } + err = fmt.Errorf("%w\n\nand %d node(s) offer nothing here, as they could not be resolved: %s", err, len(left), strings.Join(why, "; ")) + } + if len(c.unhostable) > 0 { + err = fmt.Errorf("%w\n\nand these modules offer nothing here, as their node cannot run them in this composition: %s", + err, strings.Join(c.unhostable, "; ")) + } // The node's own set does not compose: the only failure a mesh-wide gatherer may pass over. return catalogue.Resolution{}, notResolvable{err} } diff --git a/go/compose/rest.go b/go/compose/rest.go index f7509e2..a5caf70 100644 --- a/go/compose/rest.go +++ b/go/compose/rest.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "sort" + "strings" "git.novox.be/novox/mesh-sdk/go/catalogue" ) @@ -100,6 +101,7 @@ func (c *composer) theRestOfTheMesh(exclude string) (catalogue.World, Passes, [] var left []LeftOut offered := map[string][]catalogue.Provider{} var firstHeld []catalogue.Held + c.unhostable = nil for _, o := range others { got, err := catalogue.Resolve(shelf, o.assigned, o.node, catalogue.World{Unchecked: true, Holdings: holdings}) if err != nil { @@ -107,6 +109,9 @@ func (c *composer) theRestOfTheMesh(exclude string) (catalogue.World, Passes, [] left = append(left, LeftOut{Node: o.node.Name, Err: err}) continue } + for _, u := range got.Unhostable { + c.unhostable = append(c.unhostable, fmt.Sprintf("%s on %s lacks %s", u.Module, o.node.Name, strings.Join(u.Missing, ", "))) + } firstHeld = append(firstHeld, got.Claims...) for _, m := range got.Modules { for _, name := range m.OffersAt(catalogue.ScopeMesh) { diff --git a/go/delivery/composed_compare.go b/go/delivery/composed_compare.go index ecb4ca0..4faee55 100644 --- a/go/delivery/composed_compare.go +++ b/go/delivery/composed_compare.go @@ -28,6 +28,13 @@ type Comparison struct { // resource is equal it says what else differs (their order, the files filled, the data, the node's own parts, a file // carried by digest). func CompareComposed(composed map[string]Composed, running []Composed) Comparison { + return CompareComposedWith(composed, running, nil) +} + +// CompareComposedWith is CompareComposed showing, for each module that differs, every differing resource with both +// bodies masked (MaskedPair); sameValue, when given, says whether two sealed values open to the same value (the node +// opens them with its own key), so a value sealed again reads as such. +func CompareComposedWith(composed map[string]Composed, running []Composed, sameValue func(a, b string) (same, known bool)) Comparison { name := func(c Composed) string { if c.Module == "" { return HaveNodeWord @@ -60,7 +67,7 @@ func CompareComposed(composed map[string]Composed, running []Composed) Compariso case !composedHere: out.Removed = append(out.Removed, m) default: - if d, ok := compareOne(m, a, b); ok { + if d, ok := compareOne(m, a, b, sameValue); ok { out.Same = append(out.Same, m) } else { out.Differs = append(out.Differs, d) @@ -71,7 +78,7 @@ func CompareComposed(composed map[string]Composed, running []Composed) Compariso } // compareOne is one module's comparison: true when the digests agree, else what differs first. -func compareOne(module string, composed, running Composed) (ModuleDiffers, bool) { +func compareOne(module string, composed, running Composed, sameValue func(a, b string) (bool, bool)) (ModuleDiffers, bool) { composed, running = withoutBusUsers(composed), withoutBusUsers(running) dc, errC := ComposedDigestOf(composed) dr, errR := ComposedDigestOf(running) @@ -99,17 +106,35 @@ func compareOne(module string, composed, running Composed) (ModuleDiffers, bool) for id := range rb { ids[id] = true } + var first ModuleDiffers + found := false + var bodies []ResourceBodies for _, id := range sortedKeys(ids) { x, inA := ra[id] y, inB := rb[id] + why := "" switch { case !inB: - return differs(id, "composed, and not in what runs") + why = "composed, and not in what runs" case !inA: - return differs(id, "in what runs, and not composed") + why = "in what runs, and not composed" case !bytes.Equal(x, y): - return differs(id, "the resource differs") + why = "the resource differs" + default: + continue } + if !found { + first, _ = differs(id, why) + found = true + } + if len(bodies) < MaxBodies { + c, r := MaskedPair(x, y, sameValue) + bodies = append(bodies, ResourceBodies{ID: id, Why: why, Composed: cut(c), Running: cut(r)}) + } + } + if found { + first.Bodies = bodies + return first, false } switch { case !jsonEqual(a.Filled, b.Filled): @@ -165,3 +190,162 @@ func resourcesByID(resources []json.RawMessage) map[string][]byte { } return out } + +// cut is s cut at MaxBody bytes, saying so. +func cut(s string) string { + if len(s) <= MaxBody { + return s + } + return s[:MaxBody] + "… (cut at " + strconv.Itoa(MaxBody) + " bytes)" +} + +// secretWords are the words of a key whose value may hold a secret or a sealed one: masked in a shown body. +var secretWords = []string{"sealed", "secret", "password", "passwd", "token", "private", "key", "credential", "jwt", "salt", "hash"} + +func secretKey(k string) bool { + k = strings.ToLower(k) + for _, w := range secretWords { + if strings.Contains(k, w) { + return true + } + } + return false +} + +// MaskedPair is two bodies of one resource (either may be nil) as JSON text with every value under a key a secret could +// be in (secretWords, at any depth) masked: "", "" or "" when sameValue says so, "" otherwise, "" on one side +// alone. A body that is not JSON is shown as its length only. +func MaskedPair(composed, running json.RawMessage, sameValue func(a, b string) (same, known bool)) (string, string) { + var a, b any + okA := composed != nil && json.Unmarshal(composed, &a) == nil + okB := running != nil && json.Unmarshal(running, &b) == nil + if okA && okB { + a, b = maskBoth(a, b, sameValue) + } else if okA { + a, _ = maskBoth(a, nil, nil) + } else if okB { + _, b = maskBoth(nil, b, nil) + } + show := func(v any, raw json.RawMessage, ok bool) string { + if raw == nil { + return "" + } + if !ok { + return "" + } + var out bytes.Buffer + enc := json.NewEncoder(&out) + enc.SetEscapeHTML(false) + _ = enc.Encode(v) + return strings.TrimSuffix(out.String(), "\n") + } + return show(a, composed, okA), show(b, running, okB) +} + +func maskBoth(a, b any, sameValue func(a, b string) (bool, bool)) (any, any) { + ma, isA := a.(map[string]any) + mb, isB := b.(map[string]any) + if isA || isB { + outA, outB := map[string]any{}, map[string]any{} + keys := map[string]bool{} + for k := range ma { + keys[k] = true + } + for k := range mb { + keys[k] = true + } + for k := range keys { + va, inA := ma[k] + vb, inB := mb[k] + if secretKey(k) { + word := maskWord(va, vb, inA && inB, sameValue) + if inA { + outA[k] = word + } + if inB { + outB[k] = word + } + continue + } + x, y := maskBoth(va, vb, sameValue) + if inA { + outA[k] = x + } + if inB { + outB[k] = y + } + } + var ra, rb any + if isA { + ra = outA + } else { + ra = a + } + if isB { + rb = outB + } else { + rb = b + } + return ra, rb + } + la, isLA := a.([]any) + lb, isLB := b.([]any) + if isLA || isLB { + outA := make([]any, len(la)) + outB := make([]any, len(lb)) + for i := range max(len(la), len(lb)) { + var x, y any + if i < len(la) { + x = la[i] + } + if i < len(lb) { + y = lb[i] + } + x, y = maskBoth(x, y, sameValue) + if i < len(la) { + outA[i] = x + } + if i < len(lb) { + outB[i] = y + } + } + var ra, rb any = a, b + if isLA { + ra = outA + } + if isLB { + rb = outB + } + return ra, rb + } + return a, b +} + +func maskWord(a, b any, both bool, sameValue func(a, b string) (bool, bool)) string { + if !both { + return "" + } + x, _ := json.Marshal(a) + y, _ := json.Marshal(b) + if bytes.Equal(x, y) { + return "" + } + if sameValue != nil { + sa, okA := a.(string) + sb, okB := b.(string) + if !okA { + sa = string(x) + } + if !okB { + sb = string(y) + } + if same, known := sameValue(sa, sb); known { + if same { + return "" + } + return "" + } + } + return "" +} diff --git a/go/delivery/composed_signature_test.go b/go/delivery/composed_signature_test.go index 816e8a8..3098e27 100644 --- a/go/delivery/composed_signature_test.go +++ b/go/delivery/composed_signature_test.go @@ -3,9 +3,12 @@ package delivery import ( "crypto/ed25519" "crypto/rand" + "crypto/x509" "encoding/json" + "encoding/pem" "strings" "testing" + "time" ) func aSigningKey(t *testing.T) (ed25519.PublicKey, ed25519.PrivateKey) { @@ -100,4 +103,88 @@ func TestACompositionIsSignedByAKeptKey(t *testing.T) { if _, err := SignComposedBy(c, pub, func(m []byte) ([]byte, error) { return ed25519.Sign(other, m), nil }); err == nil { t.Errorf("a signature by another key than the one named was accepted") } + + // L3 s1 (ADR 0550 §2): a message the vault or the controller signed under a kind's context verifies against the + // signers a node trusts and names which; an unknown key and another kind's context are refused. + vaultPub, vault := aSigningKey(t) + controllerPub, controller := aSigningKey(t) + trusted := []Signer{{Name: SignerController, Key: controllerPub}, {Name: SignerVault, Key: vaultPub}} + enrol, _ := SignContext(KindEnrolment) + certCtx, _ := SignContext(KindNodeCertificate) + msg := []byte("the reply") + for _, row := range []struct { + name string + key ed25519.PrivateKey + signed string + checked string + want string + }{ + {"the vault's key verifies", vault, enrol, enrol, SignerVault}, + {"the controller's key verifies", controller, enrol, enrol, SignerController}, + {"an unknown key is refused", other, enrol, enrol, ""}, + {"the wrong context is refused", vault, certCtx, enrol, ""}, + } { + got, err := VerifyBy(trusted, row.checked, msg, SignMessage(row.key, row.signed, msg)) + if row.want == "" && err == nil { + t.Errorf("%s: verified as %s", row.name, got.Name) + } + if row.want != "" && (err != nil || got.Name != row.want) { + t.Errorf("%s: %+v, %v", row.name, got.Name, err) + } + } + if _, err := SignContext("hand-over"); err == nil { + t.Error("a kind the vault does not sign has a context") + } + // The signers' key reads back only when its fingerprint is its key's, and what a node trusts names both holders. + k := NewSignerKey(vaultPub, time.Unix(0, 0)) + raw, _ := json.Marshal(k) + if back, err := DecodeSignerKey(raw); err != nil || back.Fingerprint != SignerFingerprint(vaultPub) { + t.Errorf("the signers' key: %+v, %v", back, err) + } + k.Fingerprint = SignerFingerprint(controllerPub) + if raw, _ := json.Marshal(k); true { + if _, err := DecodeSignerKey(raw); err == nil { + t.Error("a signers' key naming another fingerprint was taken") + } + } + if got := Trusts(trusted, nil); len(got) != 2 || !TrustsKey(got, SignerVault, SignerFingerprint(vaultPub)) || + !TrustsKey(got, SignerController, SignerFingerprint(controllerPub)) { + t.Errorf("trusts: %v", got) + } + // A certificate the vault's authority signs for a node's name and serving key checks against a bundle holding that + // authority, and not for another name, another key, or against the controller's alone; a bundle given again adds no + // certificate twice. + authority := func(pub ed25519.PublicKey, priv ed25519.PrivateKey) string { + tpl := AuthorityTemplate(pub, time.Now().Add(-time.Hour)) + der, err := x509.CreateCertificate(nil, tpl, tpl, pub, priv) + if err != nil { + t.Fatal(err) + } + return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})) + } + vaultCA, controllerCA := authority(vaultPub, vault), authority(controllerPub, controller) + parent, _ := ParseCertificate(vaultCA) + serving, _ := aSigningKey(t) + der, err := x509.CreateCertificate(nil, NodeCertificateTemplate("ace.internal", serving, time.Now().Add(-time.Hour)), parent, serving, vault) + if err != nil { + t.Fatal(err) + } + leaf := string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})) + bundle := AuthorityBundle(controllerCA, AuthorityBundle(controllerCA, vaultCA)) + if n := strings.Count(bundle, "BEGIN CERTIFICATE"); n != 2 { + t.Errorf("the bundle holds %d certificates, want 2", n) + } + now := time.Now() + if err := CheckNodeCertificate(leaf, []string{bundle}, "ace.internal", serving, now); err != nil { + t.Errorf("the vault's certificate does not check: %v", err) + } + for name, err := range map[string]error{ + "another name": CheckNodeCertificate(leaf, []string{vaultCA}, "g14.internal", serving, now), + "another key": CheckNodeCertificate(leaf, []string{vaultCA}, "ace.internal", controllerPub, now), + "the controller's alone": CheckNodeCertificate(leaf, []string{controllerCA}, "ace.internal", serving, now), + } { + if err == nil { + t.Errorf("%s: checked", name) + } + } } diff --git a/go/delivery/delivery.go b/go/delivery/delivery.go index 1b2306f..6e51c60 100644 --- a/go/delivery/delivery.go +++ b/go/delivery/delivery.go @@ -410,8 +410,20 @@ type HaveNode struct { // its shadow composer whenever an answer changes, removed when the candidate key goes. The node applies nothing of a // candidate. Candidates map[string]CandidateAnswer `json:"candidates,omitempty"` + // Composes is whose compositions the node applies: its own (ComposesOwn, the catalogue places it) or the + // controller's (ComposesController); ComposesWhy says why in words. Written by the node agent at each start, so a + // node returning to the controller's compositions when its placement key is deleted says so (issue 1126). Empty + // from an engine older than the field. + Composes string `json:"composes,omitempty"` + ComposesWhy string `json:"composes-why,omitempty"` } +// Whose compositions a node applies (HaveNode.Composes). +const ( + ComposesOwn = "own" + ComposesController = "controller" +) + // Presence is whether a node or a part's instance is online; the entry's own time says since when. The watchman // writes it. type Presence struct { @@ -506,6 +518,11 @@ func DecodeHave(data []byte) (Have, error) { } func (n HaveNode) check() error { + switch n.Composes { + case "", ComposesOwn, ComposesController: + default: + return fmt.Errorf("delivery: a node composes %q, neither %s nor %s", n.Composes, ComposesOwn, ComposesController) + } for group, a := range n.Candidates { if group == "" { return fmt.Errorf("delivery: a node's candidate answer names no group") @@ -590,7 +607,8 @@ func SameHaveNode(a, b HaveNode) bool { return a.V == b.V && a.Composed == b.Composed && sameJSON(a.Windows, b.Windows) && sameJSON(a.Busy, b.Busy) && sameJSON(a.Strays, b.Strays) && sameJSON(a.Filters, b.Filters) && sameJSON(a.Profile, b.Profile) && (a.Record == nil) == (b.Record == nil) && (a.Record == nil || SameNodeRecord(*a.Record, *b.Record)) && - jsonEqual(a.Sealed, b.Sealed) && jsonEqual(a.Candidates, b.Candidates) + jsonEqual(a.Sealed, b.Sealed) && jsonEqual(a.Candidates, b.Candidates) && a.Composes == b.Composes && + a.ComposesWhy == b.ComposesWhy } // SamePresence says whether two presences are the same value. diff --git a/go/delivery/noderecord.go b/go/delivery/noderecord.go index c22f8b2..423cf6b 100644 --- a/go/delivery/noderecord.go +++ b/go/delivery/noderecord.go @@ -55,6 +55,10 @@ type NodeRecord struct { // Authority is the mesh's own certificate as this machine holds it (public): the one its declaration last wrote where // a module asked for it. A node composing its own composes a certificate with it (compose.Input.Authority). Authority string `json:"authority,omitempty"` + // Trusts is what this machine trusts a mesh signature from now (Trusts in signers.go, ADR 0550 §2): the controller's + // key until the lease is handed away, and the vault's key and authority once it reads them from `signers`. The gate + // before the lease write reads every node's from the bus. + Trusts []string `json:"trusts,omitempty"` } func (r NodeRecord) check() error { diff --git a/go/delivery/noderecord_test.go b/go/delivery/noderecord_test.go index e49b327..1e985e9 100644 --- a/go/delivery/noderecord_test.go +++ b/go/delivery/noderecord_test.go @@ -18,7 +18,7 @@ var secretNamed = regexp.MustCompile(`(?i)(secret|passw|token|api[-_]?key|privat func TestNodeRecord(t *testing.T) { hub := NodeRecord{Node: "novox", TunnelKey: "k", Capabilities: []string{"docker"}, SealingKey: "age1x", ServingKey: "pub", Certificate: "cert", OutwardLinks: []string{"eth0"}, Networks: []catalogue.AttachedNetwork{{Link: "eth0", Network: "192.0.2.0/24"}}, - NetworksSaid: true, Contract: 3} + NetworksSaid: true, Contract: 3, Trusts: []string{"controller key sha256:aa", "vault key sha256:bb"}} raw, err := EncodeNodeRecord(hub) if err != nil { t.Fatal(err) diff --git a/go/delivery/placement_candidate.go b/go/delivery/placement_candidate.go index edf257b..64a8d47 100644 --- a/go/delivery/placement_candidate.go +++ b/go/delivery/placement_candidate.go @@ -132,8 +132,20 @@ type CandidateAnswer struct { Refusals []ModuleRefused `json:"refusals,omitempty"` // Refused is why the node could not compose with the candidate at all; Refusals is then empty. Refused string `json:"refused,omitempty"` + // Returns is, for a head that deletes the node's placement file, what the node does then: it returns to the + // controller's compositions (ReturnsToController), as a node the catalogue does not place composes none of its own + // (issue 1126: the seed's revert). Same, Differs, Added and Removed then compare the controller's compositions at + // the node's wants with what it runs, and the verdict is theirs; a want the controller holds no composition for is + // refused (Refused, without Returns). + Returns string `json:"returns,omitempty"` + // Note is what the node says of how it answered, in words, beside the verdict: a pass that ran over a minute, inputs + // it waited for (issue 1126: the shadow shows its work). It changes no verdict. + Note string `json:"note,omitempty"` } +// ReturnsToController is what a node answers a head that deletes its placement file. +const ReturnsToController = "the head deletes this node's placement: it returns to the controller's compositions" + // ModuleDiffers is one module the node runs and the candidate still places whose composition is not the one it runs: // both digests (Composed with the candidate, Running what runs), the first resource that differs by its id, or empty // when every resource is equal, and why in words. @@ -143,8 +155,26 @@ type ModuleDiffers struct { Running string `json:"running,omitempty"` Resource string `json:"resource,omitempty"` Why string `json:"why"` + // Bodies is every resource that differs, both sides, each value a secret could be in masked (issue 1126: the shadow + // shows its work, so why a module differs is read, not guessed). At most MaxBodies, each side cut at MaxBody bytes. + Bodies []ResourceBodies `json:"bodies,omitempty"` } +// ResourceBodies is one differing resource of a module: its id, why, and its body composed and as it runs, masked +// (MaskedPair); a side that does not hold it is empty. +type ResourceBodies struct { + ID string `json:"id"` + Why string `json:"why"` + Composed string `json:"composed,omitempty"` + Running string `json:"running,omitempty"` +} + +// The bounds of a module's shown bodies. +const ( + MaxBodies = 20 + MaxBody = 8192 +) + // ModuleAdded is one module the candidate places on the node and the node does not run: its composition's digest when // it composed, and the values not yet issued for it (its own secrets, its grants, a host port), each in the words the // composition stood in for it with. @@ -187,6 +217,10 @@ func (a CandidateAnswer) check() error { if a.Refused != "" && len(a.Same)+len(a.Differs)+len(a.Added)+len(a.Removed)+len(a.Refusals) > 0 { return fmt.Errorf("delivery: a candidate the node could not compose names modules") } + if a.Returns != "" && (a.Refused != "" || len(a.Refusals) > 0) { + return fmt.Errorf("delivery: a candidate the node returns to the controller's compositions on is compared, never refused: " + + "a composition the controller holds none of is refused without returning") + } for _, d := range a.Differs { if d.Module == "" || d.Why == "" { return fmt.Errorf("delivery: a candidate difference names no module or no why") diff --git a/go/delivery/placement_test.go b/go/delivery/placement_test.go index ae77b5f..dd906b2 100644 --- a/go/delivery/placement_test.go +++ b/go/delivery/placement_test.go @@ -229,19 +229,33 @@ func TestPlacement(t *testing.T) { "refused": {Head: "h1", Revision: 3, Verdict: CandidateRefused, Refused: "unbound postgres"}, "added": {Head: "h1", Verdict: CandidateDiffers, Same: []string{"node"}, Added: []ModuleAdded{{Module: "app", Unissued: []string{"app/token"}}}, Removed: []string{"old"}}, "a value": {Head: "h1", Verdict: CandidateRefused, Same: []string{"node"}, Refusals: []ModuleRefused{{Module: "db", Why: "stands in for db/password"}}}, + // A head that deletes the node's placement file: the node returns to the controller's compositions (issue 1126, + // the seed's revert, mesh-catalog #526). + "returns": {Head: "h1", Verdict: CandidateSame, Returns: ReturnsToController, Same: []string{"app", "node"}}, + "returns, the controller's differs": {Head: "h1", Verdict: CandidateDiffers, Returns: ReturnsToController, Same: []string{"node"}, + Differs: []ModuleDiffers{{Module: "app", Resource: "file:/etc/app", Why: "the resource differs"}}, Removed: []string{"old"}}, } - n := HaveNode{Composed: "sha256:" + strings.Repeat("a", 64), Sealed: []string{"box"}, Candidates: answers} + n := HaveNode{Composed: "sha256:" + strings.Repeat("a", 64), Sealed: []string{"box"}, Candidates: answers, + Composes: ComposesOwn, ComposesWhy: "the catalogue places it (placement.n1)"} raw, err := EncodeHaveNode(n) if err != nil { t.Fatal(err) } back, err := DecodeHaveNode(raw) - if err != nil || !SameHaveNode(back, HaveNode{V: Version, Composed: n.Composed, Sealed: n.Sealed, Candidates: answers}) { + if err != nil || !SameHaveNode(back, HaveNode{V: Version, Composed: n.Composed, Sealed: n.Sealed, Candidates: answers, + Composes: ComposesOwn, ComposesWhy: n.ComposesWhy}) { t.Fatalf("a node's answers changed on the way: %+v, %v", back, err) } - if SameHaveNode(back, HaveNode{V: Version, Composed: n.Composed, Sealed: n.Sealed}) { + if SameHaveNode(back, HaveNode{V: Version, Composed: n.Composed, Sealed: n.Sealed, Composes: ComposesOwn, ComposesWhy: n.ComposesWhy}) { t.Error("a node's have without its answers counts as the same") } + if SameHaveNode(back, HaveNode{V: Version, Composed: n.Composed, Sealed: n.Sealed, Candidates: answers, + Composes: ComposesController, ComposesWhy: n.ComposesWhy}) { + t.Error("a node's have composing otherwise counts as the same") + } + if _, err := EncodeHaveNode(HaveNode{Composes: "both"}); err == nil { + t.Error("a node composing neither its own nor the controller's was taken") + } // CompareComposed: the bus's user list is left out on both sides; anything else that differs is named. file := func(id, content string) json.RawMessage { return json.RawMessage(`{"id":"` + id + `","type":"file","module":"nats","path":"/etc/` + id + `","mode":"0600","content":"` + content + `"}`) @@ -269,6 +283,39 @@ func TestPlacement(t *testing.T) { t.Errorf("%s: differs %+v, want %s: %s with both digests", name, c.Differs, tc.resource, tc.why) } } + // Issue 1126, the shadow shows its work: every differing resource with both bodies, each value a secret could be + // in masked (never a secret in an answer), saying whether the two sides hold the same value. + sealedFile := func(id, sealed, mode string) json.RawMessage { + return json.RawMessage(`{"id":"` + id + `","type":"file","module":"nats","path":"/etc/` + id + `","mode":"` + mode + `","sealed":"` + sealed + `"}`) + } + shown := CompareComposedWith(map[string]Composed{"k": comp(sealedFile("a", "box-1", "0600"), sealedFile("b", "box-2", "0600"), file("c", "x"))}, + []Composed{comp(sealedFile("a", "box-1", "0644"), sealedFile("b", "box-3", "0600"), file("c", "y"))}, + func(a, b string) (same, known bool) { return a == "box-2" && b == "box-3", true }) + if len(shown.Differs) != 1 || len(shown.Differs[0].Bodies) != 3 { + t.Fatalf("every differing resource is not shown: %+v", shown.Differs) + } + for _, row := range []struct{ id, has, hasNot string }{ + {"a", `"mode":"0600"`, "box-1"}, + {"b", "another seal of the same value", "box-2"}, + {"c", `"content":"x"`, ""}, + } { + var body ResourceBodies + for _, b := range shown.Differs[0].Bodies { + if b.ID == row.id { + body = b + } + } + both := body.Composed + body.Running + if !strings.Contains(both, row.has) || (row.hasNot != "" && strings.Contains(both, row.hasNot)) { + t.Errorf("%s shown as %+v", row.id, body) + } + } + if c, r := MaskedPair(json.RawMessage(`{"env":{"DB_PASSWORD":"hunter2","PORT":"80"},"list":[{"token":"t1"}]}`), + json.RawMessage(`{"env":{"DB_PASSWORD":"hunter3","PORT":"81"},"list":[{"token":"t1"}]}`), nil); strings.Contains(c+r, "hunter") || + strings.Contains(c+r, `"t1"`) || !strings.Contains(r, `"PORT":"81"`) || !strings.Contains(c, "") || + !strings.Contains(c, "") { + t.Errorf("masked as %s and %s", c, r) + } if c := CompareComposed(nil, []Composed{{Node: "novox", Resources: []json.RawMessage{}}}); len(c.Removed) != 1 || c.Removed[0] != "node" { t.Errorf("the node's own composition run and not composed is removed: %+v", c) } @@ -285,6 +332,8 @@ func TestPlacement(t *testing.T) { "same with an added": {Head: "h", Verdict: CandidateSame, Added: []ModuleAdded{{Module: "a"}}}, "differs refusing": {Head: "h", Verdict: CandidateDiffers, Removed: []string{"a"}, Refusals: []ModuleRefused{{Module: "b", Why: "x"}}}, "refused and modules": {Head: "h", Verdict: CandidateRefused, Refused: "x", Same: []string{"a"}}, + "returns refusing": {Head: "h", Verdict: CandidateRefused, Returns: ReturnsToController, Refusals: []ModuleRefused{{Module: "b", Why: "x"}}}, + "returns refused": {Head: "h", Verdict: CandidateRefused, Returns: ReturnsToController, Refused: "x"}, } { if _, err := EncodeHaveNode(HaveNode{Candidates: map[string]CandidateAnswer{"g": a}}); err == nil { t.Errorf("%s: taken", name) diff --git a/go/delivery/signers.go b/go/delivery/signers.go new file mode 100644 index 0000000..811ea13 --- /dev/null +++ b/go/delivery/signers.go @@ -0,0 +1,350 @@ +package delivery + +import ( + "crypto/ed25519" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/hex" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "math/big" + "slices" + "strings" + "time" +) + +// **The vault signs** (novox/hq ADR 0550 §2; design 47's L3 item d). The vault makes an ed25519 signing key and an x509 +// authority once, keeps both as its own secrets with their operator copies, and publishes their public halves in its +// state `signers`, which only the vault writes and every node principal and composing part reads. Its verb `sign` signs a +// node's certificate for that node's own node-engine, from bus state alone (the name from the node's placement, the +// serving key from its record), and says `signed` naming the caller, the kind, the node and the digest, never the key. +// +// A node trusts the vault's key through the bus grants alone (the lead's design limit, 2026-10-28): only the vault may +// write `signers`, the same trust the wants and the placement rest on. Until the controller's lease is handed away the +// node trusts the controller's key beside it; from then on the vault's alone (LeaseHandedAway). What a node trusts now +// it states in its record, NodeRecord.Trusts, so the gate before the lease write reads it from the bus. + +const ( + // VaultModule is the module holding the seat mesh-vault, whose state `signers` is. + VaultModule = "mesh-vault" + // SignersState is the vault's state of its public signing halves, and SignersBucket its bucket. + SignersState = "signers" + SignersBucket = VaultModule + "_" + SignersState + // SignersRead is the state as a manifest's `reads` names it. + SignersRead = VaultModule + "." + SignersState + // SignersKeyKey holds the signing key's public half (SignerKey); SignersAuthorityKey the authority (SignerAuthority). + SignersKeyKey = "key" + SignersAuthorityKey = "authority" + + // SignVerb is the vault's verb that signs. + SignVerb = "sign" + // SignedEvent is what the vault says when it signed (Signed), on SignedSubject. + SignedEvent = "signed" + SignedSubject = "mesh.mod." + VaultModule + ".event." + SignedEvent + + // KindNodeCertificate is a node's certificate for its internal name, asked by that node's node-engine alone. + KindNodeCertificate = "node-certificate" + // KindEnrolment is an enrolment reply, signed under its own context; no caller is named until adoption's record + // (ADR 0561 §5). + KindEnrolment = "enrolment" + // KindAuthority is the vault's own authority certificate, which the vault signs for itself. + KindAuthority = "authority" + + // SignerController and SignerVault name the two signers a node may trust. + SignerController = "controller" + SignerVault = "vault" + + // CertificateLife is how long a certificate the vault signs for a node lasts: the controller's ten years + // (identity.forever), renewed by asking again, never by a clock running out unseen. + CertificateLife = 10 * 365 * 24 * time.Hour +) + +// SignKinds are the kinds `sign` knows. +var SignKinds = []string{KindNodeCertificate, KindEnrolment, KindAuthority} + +// SignContext is the line a message of kind is signed after, so a signature made for one kind never verifies as +// another's (as the composition's and the asks' contexts are kept apart). +func SignContext(kind string) (string, error) { + if !slices.Contains(SignKinds, kind) { + return "", fmt.Errorf("delivery: %q is not a kind the vault signs (%s)", kind, strings.Join(SignKinds, ", ")) + } + return "novox " + kind + " v1\n", nil +} + +// SignerKey is the key `key` of `signers`: the vault's signing key's public half in standard base64, its fingerprint, +// and when it was made. +type SignerKey struct { + Key string `json:"key"` + Fingerprint string `json:"fingerprint"` + Made time.Time `json:"made"` +} + +// SignerAuthority is the key `authority` of `signers`: the vault's authority certificate as PEM, its fingerprint, and +// when it was made. +type SignerAuthority struct { + Certificate string `json:"certificate"` + Fingerprint string `json:"fingerprint"` + Made time.Time `json:"made"` +} + +// SignerFingerprint is an ed25519 public key's fingerprint, `sha256:` of its 32 bytes. +func SignerFingerprint(key ed25519.PublicKey) string { + sum := sha256.Sum256(key) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// AuthorityFingerprint is an authority certificate's fingerprint, `sha256:` of its DER; refused when certificate is +// not one PEM certificate. +func AuthorityFingerprint(certificate string) (string, error) { + c, err := ParseCertificate(certificate) + if err != nil { + return "", err + } + sum := sha256.Sum256(c.Raw) + return "sha256:" + hex.EncodeToString(sum[:]), nil +} + +// ParseCertificate is the first certificate of a PEM text. +func ParseCertificate(text string) (*x509.Certificate, error) { + block, _ := pem.Decode([]byte(strings.TrimSpace(text))) + if block == nil || block.Type != "CERTIFICATE" { + return nil, errors.New("delivery: not a PEM certificate") + } + return x509.ParseCertificate(block.Bytes) +} + +// NewSignerKey is the value of `key` for pub. +func NewSignerKey(pub ed25519.PublicKey, made time.Time) SignerKey { + return SignerKey{Key: base64.StdEncoding.EncodeToString(pub), Fingerprint: SignerFingerprint(pub), Made: made.UTC()} +} + +// PublicKey is k's key, checked: an ed25519 public key whose fingerprint is the one k names. +func (k SignerKey) PublicKey() (ed25519.PublicKey, error) { + raw, err := base64.StdEncoding.DecodeString(k.Key) + if err != nil || len(raw) != ed25519.PublicKeySize { + return nil, errors.New("delivery: the signers' key is not an ed25519 public key in base64") + } + if fp := SignerFingerprint(raw); fp != k.Fingerprint { + return nil, fmt.Errorf("delivery: the signers' key has the fingerprint %s, and names %s", fp, k.Fingerprint) + } + return ed25519.PublicKey(raw), nil +} + +// DecodeSignerKey reads the key `key` of `signers`, refused when it does not check. +func DecodeSignerKey(raw []byte) (SignerKey, error) { + var k SignerKey + if err := json.Unmarshal(raw, &k); err != nil { + return SignerKey{}, fmt.Errorf("delivery: %s %s: %w", SignersRead, SignersKeyKey, err) + } + if _, err := k.PublicKey(); err != nil { + return SignerKey{}, err + } + return k, nil +} + +// DecodeSignerAuthority reads the key `authority` of `signers`: one authority certificate (a CA that signs leaves +// alone) whose fingerprint is the one named. +func DecodeSignerAuthority(raw []byte) (SignerAuthority, error) { + var a SignerAuthority + if err := json.Unmarshal(raw, &a); err != nil { + return SignerAuthority{}, fmt.Errorf("delivery: %s %s: %w", SignersRead, SignersAuthorityKey, err) + } + c, err := ParseCertificate(a.Certificate) + if err != nil { + return SignerAuthority{}, err + } + if !c.IsCA { + return SignerAuthority{}, errors.New("delivery: the signers' authority is not an authority certificate") + } + if fp, _ := AuthorityFingerprint(a.Certificate); fp != a.Fingerprint { + return SignerAuthority{}, fmt.Errorf("delivery: the signers' authority has the fingerprint %s, and names %s", fp, a.Fingerprint) + } + return a, nil +} + +// Signer is one key a node trusts, by who holds it (SignerController, SignerVault, or the operator's). +type Signer struct { + Name string + Key ed25519.PublicKey +} + +// Fingerprint is the signer's key's fingerprint. +func (s Signer) Fingerprint() string { return SignerFingerprint(s.Key) } + +// VerifyBy is the trusted signer whose key verifies sig over context and msg: refused when no trusted key does, saying +// so, and when a key is not an ed25519 key. A signature made under another context never verifies. +func VerifyBy(trusted []Signer, context string, msg, sig []byte) (Signer, error) { + if len(sig) != ed25519.SignatureSize { + return Signer{}, errors.New("delivery: not an ed25519 signature") + } + message := append([]byte(context), msg...) + for _, s := range trusted { + if len(s.Key) != ed25519.PublicKeySize { + continue + } + if ed25519.Verify(s.Key, message, sig) { + return s, nil + } + } + names := make([]string, 0, len(trusted)) + for _, s := range trusted { + names = append(names, s.Name) + } + return Signer{}, fmt.Errorf("delivery: the signature verifies against no trusted key (%s)", strings.Join(names, ", ")) +} + +// SignMessage is sig over context and msg by key: what the vault's `sign` does for a kind whose bytes are given. +func SignMessage(key ed25519.PrivateKey, context string, msg []byte) []byte { + return ed25519.Sign(key, append([]byte(context), msg...)) +} + +// Signed is the vault's `signed`: who asked, the kind, the node, and the sha256 digest of what was signed. Never a key. +type Signed struct { + Caller string `json:"caller"` + Kind string `json:"kind"` + Node string `json:"node,omitempty"` + Digest string `json:"digest"` + Signer string `json:"signer"` + At time.Time `json:"at"` +} + +// SignAnswer is what `sign` answers: for a node certificate the certificate as PEM and the authority that signed it, +// for an enrolment the signature in base64; always the signer's fingerprint and the digest. Never a key's private half. +type SignAnswer struct { + Kind string `json:"kind"` + Node string `json:"node,omitempty"` + Certificate string `json:"certificate,omitempty"` + Authority string `json:"authority,omitempty"` + Signature string `json:"signature,omitempty"` + Signer string `json:"signer"` + Digest string `json:"digest"` +} + +// DigestOf is the digest a `signed` names: `sha256:` of the bytes signed. +func DigestOf(b []byte) string { + sum := sha256.Sum256(b) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// NodeCertificateTemplate is the certificate the vault signs for node's internal name, as the controller's authority +// shaped it (identity.Certify): the name in the subject and the subject alternative names, server and client use, the +// serial derived from the name and the key so the same ask gives the same certificate, and its life from notBefore. +func NodeCertificateTemplate(name string, servingKey ed25519.PublicKey, notBefore time.Time) *x509.Certificate { + sum := sha256.Sum256(append([]byte(name+"\n"), servingKey...)) + return &x509.Certificate{ + SerialNumber: new(big.Int).SetBytes(sum[:16]), + Subject: pkix.Name{CommonName: name}, + DNSNames: []string{name}, + NotBefore: notBefore.UTC().Truncate(time.Second), + NotAfter: notBefore.UTC().Truncate(time.Second).Add(CertificateLife), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + } +} + +// AuthorityTemplate is the vault's authority certificate for its key, made once: an authority that signs leaves and no +// other authority, as the controller's. +func AuthorityTemplate(key ed25519.PublicKey, made time.Time) *x509.Certificate { + sum := sha256.Sum256(append([]byte("authority\n"), key...)) + return &x509.Certificate{ + SerialNumber: new(big.Int).SetBytes(sum[:16]), + Subject: pkix.Name{CommonName: "the mesh's vault"}, + NotBefore: made.UTC().Truncate(time.Second), + NotAfter: made.UTC().Truncate(time.Second).Add(2 * CertificateLife), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + BasicConstraintsValid: true, + MaxPathLen: 0, + MaxPathLenZero: true, + } +} + +// CheckNodeCertificate says why certificate is not a certificate of name for servingKey by one of the authorities, or +// nil: a node uses it to tell whether its record holds a certificate from an authority it trusts, and another node to +// check one the vault signed (ADR 0550 §2's live check). +func CheckNodeCertificate(certificate string, authorities []string, name string, servingKey ed25519.PublicKey, at time.Time) error { + c, err := ParseCertificate(certificate) + if err != nil { + return err + } + pool := x509.NewCertPool() + for _, a := range authorities { + pool.AppendCertsFromPEM([]byte(a)) // a bundle is every certificate it holds + } + if _, err := c.Verify(x509.VerifyOptions{Roots: pool, DNSName: name, CurrentTime: at, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}}); err != nil { + return fmt.Errorf("delivery: the certificate of %s is not one a trusted authority signed for it: %w", name, err) + } + held, ok := c.PublicKey.(ed25519.PublicKey) + if !ok || !held.Equal(servingKey) { + return fmt.Errorf("delivery: the certificate of %s binds another key than the node serves with", name) + } + return nil +} + +// AuthorityBundle is the authorities given as one PEM text, each certificate once, in the order given, blanks left out: +// what a module asking for the mesh's authority is given once the vault signs beside the controller (ADR 0550 §2). An +// authority given is split into its certificates, so a bundle given again (the one a node kept from its last +// composition) adds nothing twice; a text that holds no PEM block is taken whole. +func AuthorityBundle(authorities ...string) string { + var out []string + for _, a := range authorities { + for _, piece := range pemPieces(a) { + if !slices.Contains(out, piece) { + out = append(out, piece) + } + } + } + if len(out) == 0 { + return "" + } + return strings.Join(out, "\n") + "\n" +} + +// pemPieces is each PEM block of text re-encoded, or the trimmed text whole when it holds none. +func pemPieces(text string) []string { + text = strings.TrimSpace(text) + if text == "" { + return nil + } + var out []string + rest := []byte(text) + for { + block, more := pem.Decode(rest) + if block == nil { + break + } + out = append(out, strings.TrimSpace(string(pem.EncodeToMemory(block)))) + rest = more + } + if len(out) == 0 { + return []string{text} + } + return out +} + +// Trusts is what a node states it trusts now (NodeRecord.Trusts): ` key ` for each signer and +// ` authority ` for each authority certificate, by who holds it, each once, sorted. The gate before the +// lease write reads `vault key ` in every node's record. +func Trusts(signers []Signer, authorities map[string]string) []string { + var out []string + for _, s := range signers { + if len(s.Key) == ed25519.PublicKeySize { + out = append(out, s.Name+" key "+s.Fingerprint()) + } + } + for who, a := range authorities { + if fp, err := AuthorityFingerprint(a); err == nil { + out = append(out, who+" authority "+fp) + } + } + slices.Sort(out) + return slices.Compact(out) +} + +// TrustsKey says whether trusts names the key with fingerprint fp held by who. +func TrustsKey(trusts []string, who, fp string) bool { return slices.Contains(trusts, who+" key "+fp) }