From 16b8d8a1ee7fbda0252d4d3d6c1d849074dbcc45 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 28 Oct 2026 09:36:36 +0100 Subject: [PATCH 1/3] Compose as the controller does, so a node's shadow can answer same (issue 1126, C1 C2 C3 C5 C6, T1) Seals are reused and compared by slot (resource, field), never across resources; one rule says which resolved config values are laid; a seat path that steps back is owned by the seat it steps back for, whatever the seat order; a folded module of an old-form build is composed from what runs of it; the operator export's file is carried as it runs; a differing resource names its differing JSON paths, found before its bodies are cut, and an answer is fitted to the node's key. --- go/catalogue/declaration.go | 11 ++ go/catalogue/files.go | 18 ++- go/catalogue/files_test.go | 31 +++++ go/catalogue/laid_values.go | 44 +++++++ go/catalogue/manifest.go | 5 + go/catalogue/seat_paths.go | 5 + go/compose/composed.go | 64 +++++++++- go/compose/frombus/answer_rows_test.go | 86 +++++++++++++ go/compose/frombus/frombus.go | 26 ++++ go/compose/input.go | 4 + go/compose/rendering.go | 3 + go/delivery/answer_paths.go | 160 +++++++++++++++++++++++++ go/delivery/composed_compare.go | 8 +- go/delivery/composed_seals.go | 150 +++++++++++++++++++++++ go/delivery/delivery.go | 12 +- go/delivery/placement_candidate.go | 22 ++-- go/delivery/placement_test.go | 60 ++++++++-- 17 files changed, 683 insertions(+), 26 deletions(-) create mode 100644 go/catalogue/laid_values.go create mode 100644 go/delivery/answer_paths.go create mode 100644 go/delivery/composed_seals.go diff --git a/go/catalogue/declaration.go b/go/catalogue/declaration.go index 35895c8..efa24a3 100644 --- a/go/catalogue/declaration.go +++ b/go/catalogue/declaration.go @@ -138,6 +138,9 @@ type Rendering struct { // Kept is every operator-sealed secret in the mesh, for a module that `keeps` them. Nil when // nothing on this node keeps them, or the mesh has no operator key. Kept *KeptExport + // KeptRuns is, by module, the file of the export as the node runs it, for a composer that holds no export (a node + // composing its own: the export is the controller's store's, issue 1126, C6): carried as it runs when Kept is nil. + KeptRuns map[string]map[string]any // OutwardLinks is the links this machine reported as facing outside it, which the filter is // written around (novox/hq ADR 0140). Empty means the machine has not said, and the mesh @@ -929,6 +932,14 @@ func (r Resolution) compose(with Rendering, owner map[string]string, } // Read by the module as its secrets' owner when it names one: the vault, as its own account (issue 723). first = append(first, ownedBy(m.SecretsOwner, file)) + } else if runs, ok := with.KeptRuns[m.Module]; m.Keeps != "" && ok { + file := make(map[string]any, len(runs)) + for k, v := range runs { + file[k] = v + } + file["id"] = KeptID() + delete(file, "module") + first = append(first, file) } if m.Computed != "" { mine, part := generated[m.Module] diff --git a/go/catalogue/files.go b/go/catalogue/files.go index a1231a3..81b7839 100644 --- a/go/catalogue/files.go +++ b/go/catalogue/files.go @@ -121,7 +121,7 @@ func (r Resolution) ownedPaths() map[string]string { for _, s := range Seats() { for _, sp := range s.Paths { for _, m := range r.Modules { - if m.ClaimsSeat(s.Name) && declaresPath(m, path.Clean(sp.Path)) { + if m.ClaimsSeat(s.Name) && declaresPath(m, path.Clean(sp.Path)) && !r.stepsBackHere(m, sp) { owned[sp.Path] = m.Module + " (" + s.Name + ")" } } @@ -130,6 +130,22 @@ func (r Resolution) ownedPaths() map[string]string { return owned } +// stepsBackHere says m's seat path steps back on this machine (novox/hq ADR 0287): it names a seat whose holder here +// declares the path too, so that holder owns it, as stepsBack leaves m's own rendering of it out. Without it the owner +// told was the last seat in the order of the seat table, which the controller reads sorted by name and a node in the +// order compiled (issue 1126, C3: /etc/resolv.conf, node-uplink against node-resolver). +func (r Resolution) stepsBackHere(m Manifest, sp SeatPath) bool { + if sp.StepsBackFor == "" { + return false + } + for _, o := range r.Modules { + if o.Module != m.Module && holdsSeat(o, sp.StepsBackFor) && declaresPath(o, path.Clean(sp.Path)) { + return true + } + } + return false +} + // filesWords is what the holder of node-files is given beside its own words: the node's secret places and the // paths modules own here. func (r Resolution) filesWords(m Manifest, words map[string]string, with Rendering) (map[string]string, error) { diff --git a/go/catalogue/files_test.go b/go/catalogue/files_test.go index 6cd1742..dcbf535 100644 --- a/go/catalogue/files_test.go +++ b/go/catalogue/files_test.go @@ -2,6 +2,7 @@ package catalogue import ( "encoding/json" + "slices" "sort" "strings" "testing" @@ -86,6 +87,36 @@ func TestTheHolderOfNodeFilesIsToldSecretPlacesAndOwnedPaths(t *testing.T) { if !strings.HasPrefix(owned["/etc/app/app.conf"], "app") { t.Errorf("a placed file is owned by %q, want app", owned["/etc/app/app.conf"]) } + // Issue 1126, C3: a seat path that steps back for a seat held here is that holder's, whatever the order of the seat + // table (the controller reads it sorted by name, a node in the order compiled). + resolver := Manifest{Module: "systemd-resolved", Version: "1", Claims: []Claim{{Name: ResolverSeat}}, + Resources: []map[string]any{{"id": "stub", "type": "file", "path": ResolverFile}}} + uplink := Manifest{Module: "networkmanager", Version: "1", Claims: []Claim{{Name: "node-uplink"}}, + Resources: []map[string]any{{"id": "resolv", "type": "file", "path": ResolverFile}}} + compiled := seats + t.Cleanup(func() { seats = compiled }) + for _, order := range []string{"compiled", "reversed"} { + if order == "reversed" { + seats = append([]Seat(nil), compiled...) + slices.Reverse(seats) + } + both := Resolution{Node: "anchor", Modules: []Manifest{files, uplink, resolver}} + w, err := both.filesWords(files, nil, Rendering{}) + if err != nil { + t.Fatal(err) + } + var o map[string]string + if err := json.Unmarshal([]byte(w[FilesOwnedPaths]), &o); err != nil || o[ResolverFile] != "systemd-resolved ("+ResolverSeat+")" { + t.Errorf("seat table %s: %s is owned by %q, want systemd-resolved (%s)", order, ResolverFile, o[ResolverFile], ResolverSeat) + } + } + seats = compiled + // Issue 1126, C5: a module carried as it runs (compose.Carried) owns the paths it runs at, beside its resources'. + carried := Manifest{Module: "de-spiegel", Version: "1", RunsAt: []string{"/var/lib/de-spiegel/server.env"}} + w, err := Resolution{Node: "anchor", Modules: []Manifest{files, carried}}.filesWords(files, nil, Rendering{}) + if err != nil || !strings.Contains(w[FilesOwnedPaths], `"/var/lib/de-spiegel/server.env":"de-spiegel (its own tools)"`) { + t.Errorf("the paths a carried module runs at are not owned: %v %v", w[FilesOwnedPaths], err) + } if w, _ := r.filesWords(app, nil, Rendering{}); w[FilesOwnedPaths] != "" { t.Error("a module not holding node-files is told the owned paths") } diff --git a/go/catalogue/laid_values.go b/go/catalogue/laid_values.go new file mode 100644 index 0000000..caf1792 --- /dev/null +++ b/go/catalogue/laid_values.go @@ -0,0 +1,44 @@ +package catalogue + +import "encoding/json" + +// The sources mesh-config says a resolved value came from, in mesh-config_resolved (novox/hq ADR 0348 §9): a key of +// it holds {"value": …, "source": …}. +const ( + SourceMesh = "mesh" + SourceNode = "node" + SourceModule = "module" + SourceDefault = "default" + SourceUnset = "unset" +) + +// LaidSource says whether a value resolved from source is laid over a module's files: mesh config, node config and the +// module's own value are; a manifest's default is not, since it is the composition's already and a taken key is laid +// only when a layer sets it, never from a default (TakesSettingsField); an unset value is not, since a required one is +// refused by the composition by name. **The one rule for every composer** (issue 1126, C2): the controller's config +// read and a node composing its own lay the same values, so the two compose the same files. +func LaidSource(source string) bool { + switch source { + case SourceMesh, SourceNode, SourceModule: + return true + } + return false +} + +// LaidValue is one key of mesh-config_resolved read as a composer lays it: its value, and true when its source is laid +// (LaidSource) and it holds a value; false for a default, an unset value, a source not known and a value that cannot be +// read. +func LaidValue(raw []byte) (any, bool) { + var r struct { + Value json.RawMessage `json:"value"` + Source string `json:"source"` + } + if json.Unmarshal(raw, &r) != nil || !LaidSource(r.Source) || len(r.Value) == 0 { + return nil, false + } + var v any + if json.Unmarshal(r.Value, &v) != nil { + return nil, false + } + return v, true +} diff --git a/go/catalogue/manifest.go b/go/catalogue/manifest.go index 0c721c7..2f0be1e 100644 --- a/go/catalogue/manifest.go +++ b/go/catalogue/manifest.go @@ -864,6 +864,11 @@ type Manifest struct { // that could only see its own ports would write a rule set that closed everything else. Filtering *Filtering `json:"filtering,omitempty"` + // RunsAt is, for a module composed from what runs of it (compose.Carried: folded into its node's own composition + // at a build of the old form, issue 1126, C5), the paths its running resources are at: paths it owns beside its + // resources' (declaredPaths), so the holder of node-files is told them. Never in a manifest's JSON. + RunsAt []string `json:"-"` + // Jails are the fail2ban jails this module declares for its own service (novox/hq to-be 31). // Written into whichever node runs the module, the same way `listens` become that node's rules. Jails []Jail `json:"jails,omitempty"` diff --git a/go/catalogue/seat_paths.go b/go/catalogue/seat_paths.go index 1b74e6f..b80dd4d 100644 --- a/go/catalogue/seat_paths.go +++ b/go/catalogue/seat_paths.go @@ -133,6 +133,11 @@ func declaredPaths(m Manifest) map[string]bool { out[path.Clean(f.Path)] = true } } + for _, p := range m.RunsAt { + if strings.HasPrefix(p, "/") { + out[path.Clean(p)] = true + } + } return out } diff --git a/go/compose/composed.go b/go/compose/composed.go index a6a8bea..f6b9bb0 100644 --- a/go/compose/composed.go +++ b/go/compose/composed.go @@ -77,8 +77,9 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli if !found { return nil, nil, nil, fmt.Errorf("compose: the mesh has no node %q", node) } - // A folded module built by a build of the old form (Node.RunsOwn) is left to the node: composed as a stub, so what - // it provides and requires still shapes the rest, and its resources are then the ones the node runs. + // A folded module built by a build of the old form (Node.RunsOwn) is left to the node: composed from what runs of it + // (Carried), so what it provides and requires still shapes the rest and what the others read of it (its published + // ports, the paths it owns) is what runs, and its resources are then the ones the node runs. carried := map[string]bool{} var runsOwn *delivery.Composed for i := range reading.Nodes { @@ -103,7 +104,7 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli for k, v := range n.Kept { kept[k] = v } - kept[m] = Stub(man) + kept[m] = Carried(man, m, runsOwn) n.Kept = kept } } @@ -259,7 +260,8 @@ func NamesBuild(m catalogue.Manifest) bool { // Stub is m with nothing to build and no resource but its directories: what it provides, requires, serves and grants // still shape the rest of the composition, the places its own secrets, data and grants name (${dir:…}) still resolve, -// and nothing else of its own is composed (issue 1126: nats's ${dir:mesh-state} on novox). +// and nothing else of its own is composed (issue 1126: nats's ${dir:mesh-state} on novox). A module refused in a +// node's answer is composed so; a folded one the node runs is Carried. func Stub(m catalogue.Manifest) catalogue.Manifest { m.Build = nil var dirs []map[string]any @@ -272,6 +274,60 @@ func Stub(m catalogue.Manifest) catalogue.Manifest { return m } +// Carried is m, folded into its node's own composition at a build of the old form, as the node runs it (issue 1126, +// C5): nothing to build; its containers and directories those the node's own composition (runs) carries of it, their +// module's word taken off their ids, with the manifest's directories it does not run; and the path of every resource it +// runs (catalogue.Manifest.RunsAt). So what other modules read of it is what runs: the machine ports its containers +// publish (the filter's rules, the routes reaching it: MachineSide) and the paths it owns (the holder of node-files is +// told them). Its other resources are not taken, since the composition makes some of them again from its manifest (its +// secrets' files) and one id twice is refused; none of them is composed into what is applied either way (compositions +// leaves a carried module's out, and carries the ones that run). What it provides, requires, listens on and grants is +// its manifest's. A module whose resources runs holds none of is its Stub, as before. +func Carried(m catalogue.Manifest, module string, runs *delivery.Composed) catalogue.Manifest { + if runs == nil { + return Stub(m) + } + var taken []map[string]any + var at []string + ids := map[string]bool{} + for _, raw := range runs.Resources { + var r map[string]any + if json.Unmarshal(raw, &r) != nil { + continue + } + id, _ := r["id"].(string) + owner, _ := r["module"].(string) + if owner == "" { + owner, _ = declaration.ModuleGuess(id) + } + if owner != module { + continue + } + if p, ok := r["path"].(string); ok && p != "" { + at = append(at, p) + } + if typ := r["type"]; typ != "container" && typ != "directory" { + continue + } + r["id"] = strings.TrimPrefix(id, module+".") + delete(r, "module") + ids[r["id"].(string)] = true + taken = append(taken, r) + } + if len(at) == 0 && len(taken) == 0 { + return Stub(m) + } + stub := Stub(m) + for _, r := range stub.Resources { + if id, _ := r["id"].(string); !ids[id] { + taken = append(taken, r) + } + } + stub.Resources = taken + stub.RunsAt = at + return stub +} + // carry puts into the node's resources (kept, in the declaration's order) the resources of the node's own composition as // it runs (runs) that belong to a carried module, each after the resource it runs after, or first when that one is // not among them; a file among them is a folded path. diff --git a/go/compose/frombus/answer_rows_test.go b/go/compose/frombus/answer_rows_test.go index 74b1601..d39da77 100644 --- a/go/compose/frombus/answer_rows_test.go +++ b/go/compose/frombus/answer_rows_test.go @@ -165,6 +165,92 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { !strings.Contains(err.Error(), "spiegel has not been built for this mesh") { t.Errorf("composed spiegel without what runs: %v", err) } + + // (C) Issue 1126, C5: what the others read of a module carried as it runs is what runs, never its stub's: the filter + // opens the machine port its running container publishes (de-spiegel's 20005 on novox, not its software's 8080; a + // stub had no container, so snake's 80 on ace was opened beside the route proxy's). And C6: the operator export's file + // a module that keeps it runs is carried as it runs (mesh-vault.kept on novox), since the node holds no export. + listening := []byte(`{"module":"spiegel","version":"1","capabilities":["container-runtime"],` + + `"listens":[{"port":8080,"from":"mesh","why":"the site"}],"own-secrets":{"token":"${dir:state}/token"},` + + `"resources":[{"id":"state","type":"directory","path":"/var/lib/spiegel","mode":"0700"},` + + `{"id":"server","type":"container","name":"spiegel","artifact":"site","ports":["8080"]}],` + + `"build":{"artifacts":[{"name":"site","kind":"upstream","from":"` + image + `"}]}}`) + nft := []byte(`{"module":"nft","version":"1","capabilities":["systemd"],"filtering":{"into":"/etc/nftables.conf"}}`) + vault := []byte(`{"module":"vault","version":"1","capabilities":["systemd"],"keeps":"/var/lib/vault/root",` + + `"resources":[{"id":"service","type":"service","unit":"vault.service","state":"running","boot":"enabled"}]}`) + b = mk("lemurs", overlay.Name, "spiegel", "nft", "vault") + for m, raw := range map[string][]byte{"spiegel": listening, "nft": nft, "vault": vault} { + b.manifests[delivery.ManifestDigest(raw)] = raw + b.modules[m] = delivery.Module{Manifest: delivery.ManifestDigest(raw)} + } + p = b.placements["n1"] + p.Decided.Endpoint = "n1.example:51820" + for i := range p.Modules { + if p.Modules[i].Module == "spiegel" { + p.Modules[i].Ports = map[string]int{"8080": 20005} + } + } + b.placements["n1"] = p + rec := b.records["n1"] + rec.OutwardLinks = []string{"eth0"} + b.records["n1"] = rec + fps = map[string]string{"lemurs": fp("e"), "nft": fp("f"), "vault": fp("9")} + if in, err = Input(ctx, b, Mesh{}, stub); err != nil { + t.Fatal(err) + } + in.Nodes[0].Kept = map[string]catalogue.Manifest{"spiegel": compose.Stub(in.Catalogue["spiegel"])} + if base, _, err = compose.Compositions(in, "n1", fps, compose.Reading); err != nil { + t.Fatal(err) + } + keptFile := json.RawMessage(`{"digest":"sha256:` + strings.Repeat("d", 64) + `","id":"vault.kept","mode":"0600","module":"vault","path":"/var/lib/vault/root/export.json","size":143604,"type":"file"}`) + running = nil + for _, c := range base { + switch c.Module { + case "": + c.Resources = append(append([]json.RawMessage{}, c.Resources...), + json.RawMessage(`{"id":"spiegel.state","type":"directory","path":"/var/lib/spiegel","mode":"0700"}`), + json.RawMessage(`{"id":"spiegel.server","type":"container","name":"spiegel","image":"registry.example/spiegel@sha256:`+strings.Repeat("c", 64)+`","ports":["10.77.0.1:20005:8080"]}`), + json.RawMessage(`{"id":"spiegel.server-env","type":"file","path":"/var/lib/spiegel/server.env","mode":"0600","content":"A=1\n"}`)) + case "vault": + c.Resources = append(append([]json.RawMessage{}, c.Resources...), keptFile) + } + running = append(running, c) + } + got, _, err = Compose(ctx, b, Mesh{}, Scope{Node: "n1", InOwn: []string{"spiegel"}, Running: running}, fps) + if err != nil { + t.Fatalf("the node does not compose itself with spiegel carried: %v", err) + } + var rules string + var keptComposed json.RawMessage + for _, c := range got { + for _, raw := range c.Resources { + var r struct { + ID string `json:"id"` + Content string `json:"content"` + } + _ = json.Unmarshal(raw, &r) + switch r.ID { + case "nft." + catalogue.FilteringID(): + rules = r.Content + case "vault." + catalogue.KeptID(): + keptComposed = raw + } + } + } + if !strings.Contains(rules, "# spiegel — the site") || !strings.Contains(rules, "dport 20005 accept") || strings.Contains(rules, "dport 8080") { + t.Errorf("the filter does not open the machine port spiegel runs at:\n%s", rules) + } + for _, c := range running { + if c.Module == "" && !slices.Contains(compose.Carried(in.Catalogue["spiegel"], "spiegel", &c).RunsAt, "/var/lib/spiegel/server.env") { + t.Errorf("spiegel carried does not own the path its running file is at") + } + } + var k1, k2 any + _ = json.Unmarshal(keptComposed, &k1) + _ = json.Unmarshal(keptFile, &k2) + if !reflect.DeepEqual(k1, k2) { + t.Errorf("the operator export is composed as %s, want the file that runs %s", keptComposed, keptFile) + } } // A module another node cannot run in this composition offers nothing in it: the refusal names it and why diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index 735a831..7101fa0 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -297,6 +297,21 @@ func input(ctx context.Context, r Reader, mesh Mesh, s Scope, refuse func(module n.InOwn[module] = true } for i := range s.Running { + // The operator export as the vault runs it: the controller composes it from its store, which the node + // does not read; until the export is on the bus, the node carries the file it runs (issue 1126, C6: + // mesh-vault.kept on novox, "in what runs, and not composed"). + if m := s.Running[i].Module; m != "" { + if kept := keptOf(s.Running[i]); kept != nil { + if in.KeptRuns == nil { + in.KeptRuns = map[string]map[string]map[string]any{} + } + if in.KeptRuns[s.Node] == nil { + in.KeptRuns[s.Node] = map[string]map[string]any{} + } + in.KeptRuns[s.Node][m] = kept + } + continue + } if s.Running[i].Module == "" { own := s.Running[i] n.RunsOwn = &own @@ -708,6 +723,17 @@ func storeSought(in compose.Input) string { ", none of it on the private network with a port" } +// keptOf is the operator export's file (catalogue.KeptID) of a module's composition as it runs, or nil. +func keptOf(c delivery.Composed) map[string]any { + for _, raw := range c.Resources { + var r map[string]any + if json.Unmarshal(raw, &r) == nil && r["id"] == c.Module+"."+catalogue.KeptID() { + return r + } + } + return nil +} + // membershipOf is the sealed value of the bus membership resource (catalogue.BusMembershipID) of a node's own // composition, or "". func membershipOf(own delivery.Composed) string { diff --git a/go/compose/input.go b/go/compose/input.go index 2ca4f25..5b54123 100644 --- a/go/compose/input.go +++ b/go/compose/input.go @@ -68,6 +68,10 @@ type Input struct { // OperatorExport is every operator-sealed secret in the mesh, for a module that keeps them (inv.OperatorExport): sealed // values the composition carries without opening. Nil when the mesh has no operator key. OperatorExport *catalogue.KeptExport + // KeptRuns is, by node and module, the file of the export as that node runs it (its resource of a module's running + // composition): what a node composing its own carries where it holds no OperatorExport (catalogue.Rendering.KeptRuns, + // issue 1126, C6). Used only while OperatorExport is nil. + KeptRuns map[string]map[string]map[string]any // Issued is the credentials made for the consumers of each provider (inv.SecretsFrom): one row for each pair credential // that exists, by the provider node it is for. The grants a provider must create are read from these, so a credential not diff --git a/go/compose/rendering.go b/go/compose/rendering.go index 4f0e648..ee619cf 100644 --- a/go/compose/rendering.go +++ b/go/compose/rendering.go @@ -128,6 +128,9 @@ func (c *composer) places(nodeName string, resolved Resolved) (Placed, error) { continue } r.Kept = c.in.OperatorExport + if r.Kept == nil { + r.KeptRuns = c.in.KeptRuns[node.Name] + } break } diff --git a/go/delivery/answer_paths.go b/go/delivery/answer_paths.go new file mode 100644 index 0000000..e2d3a72 --- /dev/null +++ b/go/delivery/answer_paths.go @@ -0,0 +1,160 @@ +package delivery + +import ( + "bytes" + "encoding/json" + "sort" + "strconv" + "strings" +) + +// DifferingPaths is every JSON path at which two bodies of one resource differ, sorted (issue 1126, T1: the shadow +// shows its work): object keys and list places joined by dots (`content.url`, `links.0`), a string that holds a JSON +// object or list read as that JSON and walked on (a runtime's config carried as text), a key or place one side alone +// holds named as it is. A key a secret could be under (secretWords) is named and not walked into. A body that is not +// JSON differs at "" (the whole), and a resource one side alone holds has none. At most MaxPaths: the last then says how many more there are. A path names keys +// alone, never a value, so it holds no secret. +func DifferingPaths(composed, running json.RawMessage) []string { + if composed == nil || running == nil { + return nil // one side alone holds it: its why says so + } + var a, b any + okA := unmarshalNumbers(composed, &a) == nil + okB := unmarshalNumbers(running, &b) == nil + if !okA || !okB { + if bytes.Equal(composed, running) { + return nil + } + return []string{""} + } + var out []string + walkDiffering(a, b, "", &out) + sort.Strings(out) + if len(out) > MaxPaths { + more := len(out) - (MaxPaths - 1) + out = append(out[:MaxPaths-1], "… "+strconv.Itoa(more)+" more") + } + return out +} + +func unmarshalNumbers(raw []byte, into *any) error { + d := json.NewDecoder(bytes.NewReader(raw)) + d.UseNumber() + return d.Decode(into) +} + +// asJSON is a string holding a JSON object or list read as that value; v itself otherwise. +func asJSON(v any) any { + s, ok := v.(string) + if !ok { + return v + } + t := strings.TrimSpace(s) + if t == "" || (t[0] != '{' && t[0] != '[') { + return v + } + var parsed any + if unmarshalNumbers([]byte(t), &parsed) != nil { + return v + } + return parsed +} + +func join(at, key string) string { + if at == "" { + return key + } + return at + "." + key +} + +func walkDiffering(a, b any, at string, out *[]string) { + a, b = asJSON(a), asJSON(b) + ma, isMA := a.(map[string]any) + mb, isMB := b.(map[string]any) + if isMA && isMB { + keys := map[string]bool{} + for k := range ma { + keys[k] = true + } + for k := range mb { + keys[k] = true + } + for k := range keys { + x, inA := ma[k] + y, inB := mb[k] + switch { + case !inA || !inB: + *out = append(*out, join(at, k)) + case secretKey(k): + if !sameAny(x, y) { + *out = append(*out, join(at, k)) + } + default: + walkDiffering(x, y, join(at, k), out) + } + } + return + } + la, isLA := a.([]any) + lb, isLB := b.([]any) + if isLA && isLB { + for i := range max(len(la), len(lb)) { + if i >= len(la) || i >= len(lb) { + *out = append(*out, join(at, strconv.Itoa(i))) + continue + } + walkDiffering(la[i], lb[i], join(at, strconv.Itoa(i)), out) + } + return + } + if !sameAny(a, b) { + *out = append(*out, at) + } +} + +func sameAny(a, b any) bool { + x, errA := json.Marshal(a) + y, errB := json.Marshal(b) + return errA == nil && errB == nil && bytes.Equal(x, y) +} + +// FitAnswer is a with its shown bodies cut until its JSON is within limit bytes (MaxAnswerBytes for the node's key): +// first the bodies' texts, from the last differing module back, keeping their paths; then the paths too. What is cut +// is said in the answer's note. The verdict and every list of modules are never cut. +func FitAnswer(a CandidateAnswer, limit int) CandidateAnswer { + size := func() int { + raw, err := json.Marshal(a) + if err != nil { + return 0 + } + return len(raw) + } + if size() <= limit { + return a + } + a.Differs = append([]ModuleDiffers(nil), a.Differs...) + for i := range a.Differs { + a.Differs[i].Bodies = append([]ResourceBodies(nil), a.Differs[i].Bodies...) + } + cutTexts, cutPaths := false, false + for i := len(a.Differs) - 1; i >= 0 && size() > limit; i-- { + for j := range a.Differs[i].Bodies { + a.Differs[i].Bodies[j].Composed, a.Differs[i].Bodies[j].Running = "", "" + } + cutTexts = true + } + for i := len(a.Differs) - 1; i >= 0 && size() > limit; i-- { + for j := range a.Differs[i].Bodies { + a.Differs[i].Bodies[j].Paths = nil + } + cutPaths = true + } + why := "the bodies shown were cut to keep the answer within " + strconv.Itoa(limit>>10) + " KiB" + if cutPaths { + why += ", their paths too" + } + if cutTexts { + a.Note = strings.TrimPrefix(a.Note+"; ", "; ") + why + } + return a +} diff --git a/go/delivery/composed_compare.go b/go/delivery/composed_compare.go index 4faee55..3d2a609 100644 --- a/go/delivery/composed_compare.go +++ b/go/delivery/composed_compare.go @@ -80,6 +80,9 @@ func CompareComposedWith(composed map[string]Composed, running []Composed, sameV // compareOne is one module's comparison: true when the digests agree, else what differs first. func compareOne(module string, composed, running Composed, sameValue func(a, b string) (bool, bool)) (ModuleDiffers, bool) { composed, running = withoutBusUsers(composed), withoutBusUsers(running) + // A sealed value is judged by the value it opens to, slot by slot (issue 1126, C1): a value sealed again at the same + // field of the same resource is the box that runs, never a difference; and a box is never matched across resources. + composed = KeepSeals(composed, SealsIn(running), sameValue) dc, errC := ComposedDigestOf(composed) dr, errR := ComposedDigestOf(running) if errC != nil { @@ -128,8 +131,11 @@ func compareOne(module string, composed, running Composed, sameValue func(a, b s found = true } if len(bodies) < MaxBodies { + // The paths are found on the whole bodies, before either is cut (issue 1126, T1: three resources over + // MaxBody could not be told apart). c, r := MaskedPair(x, y, sameValue) - bodies = append(bodies, ResourceBodies{ID: id, Why: why, Composed: cut(c), Running: cut(r)}) + bodies = append(bodies, ResourceBodies{ID: id, Why: why, Paths: DifferingPaths(x, y), + Composed: cut(c), Running: cut(r)}) } } if found { diff --git a/go/delivery/composed_seals.go b/go/delivery/composed_seals.go new file mode 100644 index 0000000..af3ca54 --- /dev/null +++ b/go/delivery/composed_seals.go @@ -0,0 +1,150 @@ +package delivery + +import ( + "bytes" + "encoding/json" + "strconv" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Seals is, by slot, each value one composition carries sealed to its node's key (issue 1126, C1): a file's `sealed` +// and a vault box (catalogue.VaultSealedField), as their JSON text (a box is carried as its JSON, a sealed value as +// the string it is). A slot is one field of one resource of the composition's module (SealSlot): a box is reused only +// at the slot that holds it, never for another resource whose value opens to the same (a provider's grant and its +// consumer's secret on one node hold one value). +type Seals map[string]string + +// SealedFields are the fields of a resource that carry a value sealed to the node's key. +var SealedFields = []string{"sealed", catalogue.VaultSealedField} + +// SealSlot is the slot of one sealed field of one resource, by the resource's id (its place, `#`, when it has none). +// The module is the composition's: a slot is never compared across compositions. +func SealSlot(resource, field string) string { return resource + "#" + field } + +// SealsIn is every sealed value c carries, by slot; nil when it carries none. +func SealsIn(c Composed) Seals { + var out Seals + for i, raw := range c.Resources { + id, fields, ok := sealedFieldsOf(raw, i) + if !ok { + continue + } + for _, field := range SealedFields { + f, has := fields[field] + if !has { + continue + } + if text := sealedText(f); text != "" { + if out == nil { + out = Seals{} + } + out[SealSlot(id, field)] = text + } + } + } + return out +} + +// KeepSeals is c with each sealed value replaced by the one kept at the same slot, where same says the two open to the +// same value (known and same): a value the vault sealed again is then the box that runs, so the composition is the +// one that runs (ADR 0547 §3, issue 1126). A slot kept holds nothing for, or whose value differs or cannot be opened, +// keeps c's. Every other byte of a resource is kept as it is; c itself is not changed. +func KeepSeals(c Composed, kept Seals, same func(a, b string) (same, known bool)) Composed { + if len(kept) == 0 || same == nil { + return c + } + out := c + out.Resources = nil + for i, raw := range c.Resources { + id, fields, ok := sealedFieldsOf(raw, i) + if ok { + for _, field := range SealedFields { + f, has := fields[field] + if !has { + continue + } + ours := sealedText(f) + theirs, held := kept[SealSlot(id, field)] + if !held || ours == "" || theirs == ours { + continue + } + if s, known := same(ours, theirs); !known || !s { + continue + } + value := json.RawMessage(theirs) + if f[0] == '"' { + value, _ = json.Marshal(theirs) + } + if swapped, ok := withField(raw, field, value); ok { + raw = swapped + } + } + } + out.Resources = append(out.Resources, raw) + } + return out +} + +// sealedFieldsOf is a resource's id (`#` when it has none) and its top-level fields, false when it is no object. +func sealedFieldsOf(raw json.RawMessage, i int) (string, map[string]json.RawMessage, bool) { + var fields map[string]json.RawMessage + if json.Unmarshal(raw, &fields) != nil { + return "", nil, false + } + var id string + if f, ok := fields["id"]; !ok || json.Unmarshal(f, &id) != nil || id == "" { + id = "#" + strconv.Itoa(i) + } + return id, fields, true +} + +// sealedText is a sealed field as its text: a string as it is, a box as its JSON. +func sealedText(f json.RawMessage) string { + var text string + if json.Unmarshal(f, &text) == nil { + return text + } + var compact bytes.Buffer + if json.Compact(&compact, f) != nil { + return string(f) + } + return compact.String() +} + +// withField is the object raw with its top-level field replaced by value, every other byte as it was (so a composition +// whose sealed values are those that run is the one that runs, byte for byte); false when raw is no object holding it. +func withField(raw json.RawMessage, field string, value json.RawMessage) (json.RawMessage, bool) { + d := json.NewDecoder(bytes.NewReader(raw)) + if t, err := d.Token(); err != nil || t != json.Delim('{') { + return nil, false + } + for d.More() { + t, err := d.Token() + if err != nil { + return nil, false + } + key, _ := t.(string) + before := d.InputOffset() + var skip json.RawMessage + if err := d.Decode(&skip); err != nil { + return nil, false + } + if key != field { + continue + } + after := d.InputOffset() + // The span holds the colon, any space and the value: kept up to the colon, the value replaced. + span := raw[before:after] + colon := bytes.IndexByte(span, ':') + if colon < 0 { + return nil, false + } + out := make([]byte, 0, len(raw)-len(span)+colon+1+len(value)) + out = append(out, raw[:int(before)+colon+1]...) + out = append(out, value...) + out = append(out, raw[after:]...) + return out, true + } + return nil, false +} diff --git a/go/delivery/delivery.go b/go/delivery/delivery.go index 6e51c60..10df852 100644 --- a/go/delivery/delivery.go +++ b/go/delivery/delivery.go @@ -48,6 +48,7 @@ import ( "encoding/json" "errors" "fmt" + "maps" "reflect" "regexp" "slices" @@ -355,6 +356,10 @@ type Have struct { // after a restart while it opens to the vault's value, so a value sealed again moves no composition. Openable by the // node alone, as the vault hands them over the bus. Empty on every other have. Sealed []string `json:"sealed,omitempty"` + // SealedAt is Sealed by slot (Seals: a field of a resource, issue 1126, C1): the composer reuses a box only at the + // slot that holds it, so a provider and a consumer of one value on one node each keep their own. Written in Sealed's + // place by a node-engine that knows it; Sealed is read only from a have written before. + SealedAt Seals `json:"sealed-at,omitempty"` } // UserListWritten is the bus's user list as the node-engine of the bus node wrote it to the bus module's file, applied @@ -406,6 +411,8 @@ type HaveNode struct { // applied (Have.Sealed for the node's own, issue 1126): a value the vault seals again after a restart moves no // composition, so a restart applies nothing again. Sealed []string `json:"sealed,omitempty"` + // SealedAt is Sealed by slot (Have.SealedAt). + SealedAt Seals `json:"sealed-at,omitempty"` // Candidates is the node's answer to each placement candidate naming it, by group (PlacementCandidate): written by // its shadow composer whenever an answer changes, removed when the candidate key goes. The node applies nothing of a // candidate. @@ -586,10 +593,11 @@ func DecodePulse(data []byte) (Pulse, error) { func SameHave(a, b Have) bool { ua, ub := a.UserList, b.UserList a.UserList, b.UserList = nil, nil - if !slices.Equal(a.Sealed, b.Sealed) { + if !slices.Equal(a.Sealed, b.Sealed) || !maps.Equal(a.SealedAt, b.SealedAt) { return false } a.Sealed, b.Sealed = nil, nil + a.SealedAt, b.SealedAt = nil, nil if !reflect.DeepEqual(a, b) || (ua == nil) != (ub == nil) { return false } @@ -607,7 +615,7 @@ func SameHaveNode(a, b HaveNode) bool { return a.V == b.V && a.Composed == b.Composed && sameJSON(a.Windows, b.Windows) && sameJSON(a.Busy, b.Busy) && sameJSON(a.Strays, b.Strays) && sameJSON(a.Filters, b.Filters) && sameJSON(a.Profile, b.Profile) && (a.Record == nil) == (b.Record == nil) && (a.Record == nil || SameNodeRecord(*a.Record, *b.Record)) && - jsonEqual(a.Sealed, b.Sealed) && jsonEqual(a.Candidates, b.Candidates) && a.Composes == b.Composes && + jsonEqual(a.Sealed, b.Sealed) && maps.Equal(a.SealedAt, b.SealedAt) && jsonEqual(a.Candidates, b.Candidates) && a.Composes == b.Composes && a.ComposesWhy == b.ComposesWhy } diff --git a/go/delivery/placement_candidate.go b/go/delivery/placement_candidate.go index 64a8d47..12f276e 100644 --- a/go/delivery/placement_candidate.go +++ b/go/delivery/placement_candidate.go @@ -163,16 +163,24 @@ type ModuleDiffers struct { // ResourceBodies is one differing resource of a module: its id, why, and its body composed and as it runs, masked // (MaskedPair); a side that does not hold it is empty. type ResourceBodies struct { - ID string `json:"id"` - Why string `json:"why"` - Composed string `json:"composed,omitempty"` - Running string `json:"running,omitempty"` + ID string `json:"id"` + Why string `json:"why"` + // Paths is every JSON path at which the two bodies differ (DifferingPaths: `sealed`, `content.url`, `links.0`), found + // on the whole bodies before they are cut, a JSON value written in a string read as JSON; a path names keys alone, + // never a value. At most MaxPaths, the last then saying how many more. + Paths []string `json:"paths,omitempty"` + Composed string `json:"composed,omitempty"` + Running string `json:"running,omitempty"` } -// The bounds of a module's shown bodies. +// The bounds of a module's shown bodies, and of one answer as the node's own key holds it: an answer over +// MaxAnswerBytes is fitted (FitAnswer). The node's key (HaveBucket) takes a value of about 1 MiB +// (buscompose.ChainValueBytes) and holds every group's answer beside the node's facts, so one answer keeps to a sixth. const ( - MaxBodies = 20 - MaxBody = 8192 + MaxBodies = 20 + MaxBody = 8192 + MaxPaths = 64 + MaxAnswerBytes = 160 << 10 ) // ModuleAdded is one module the candidate places on the node and the node does not run: its composition's digest when diff --git a/go/delivery/placement_test.go b/go/delivery/placement_test.go index dd906b2..cde72a4 100644 --- a/go/delivery/placement_test.go +++ b/go/delivery/placement_test.go @@ -3,7 +3,9 @@ package delivery import ( "encoding/json" "errors" + "fmt" "reflect" + "slices" "strings" "testing" @@ -288,16 +290,30 @@ func TestPlacement(t *testing.T) { sealedFile := func(id, sealed, mode string) json.RawMessage { return json.RawMessage(`{"id":"` + id + `","type":"file","module":"nats","path":"/etc/` + id + `","mode":"` + mode + `","sealed":"` + sealed + `"}`) } - shown := CompareComposedWith(map[string]Composed{"k": comp(sealedFile("a", "box-1", "0600"), sealedFile("b", "box-2", "0600"), file("c", "x"))}, - []Composed{comp(sealedFile("a", "box-1", "0644"), sealedFile("b", "box-3", "0600"), file("c", "y"))}, - func(a, b string) (same, known bool) { return a == "box-2" && b == "box-3", true }) - if len(shown.Differs) != 1 || len(shown.Differs[0].Bodies) != 3 { - t.Fatalf("every differing resource is not shown: %+v", shown.Differs) + // Issue 1126, C1: a sealed value is judged by the value it opens to at its own slot: b sealed again (box-2 and + // box-3 open to one value) is no difference; d holds at its slot a box of another value, though that value is + // the one b holds, and differs at its path `sealed`. + sameBox := func(a, b string) (same, known bool) { + return (a == "box-2" && b == "box-3") || a == b, true } - for _, row := range []struct{ id, has, hasNot string }{ - {"a", `"mode":"0600"`, "box-1"}, - {"b", "another seal of the same value", "box-2"}, - {"c", `"content":"x"`, ""}, + shown := CompareComposedWith(map[string]Composed{"k": comp(sealedFile("a", "box-1", "0600"), sealedFile("b", "box-2", "0600"), + file("c", "x"), sealedFile("d", "box-2", "0600"))}, + []Composed{comp(sealedFile("a", "box-1", "0644"), sealedFile("b", "box-3", "0600"), file("c", "y"), sealedFile("d", "box-4", "0600"))}, + sameBox) + if len(shown.Differs) != 1 || len(shown.Differs[0].Bodies) != 3 { + t.Fatalf("every differing resource is not shown, or b sealed again is shown: %+v", shown.Differs) + } + if c := CompareComposedWith(map[string]Composed{"k": comp(sealedFile("b", "box-2", "0600"))}, + []Composed{comp(sealedFile("b", "box-3", "0600"))}, sameBox); len(c.Same) != 1 { + t.Errorf("a value sealed again at its own slot is not the same: %+v", c) + } + for _, row := range []struct { + id, has, hasNot string + paths []string + }{ + {"a", `"mode":"0600"`, "box-1", []string{"mode"}}, + {"c", `"content":"x"`, "", []string{"content"}}, + {"d", "", "box-2", []string{"sealed"}}, } { var body ResourceBodies for _, b := range shown.Differs[0].Bodies { @@ -306,10 +322,32 @@ func TestPlacement(t *testing.T) { } } both := body.Composed + body.Running - if !strings.Contains(both, row.has) || (row.hasNot != "" && strings.Contains(both, row.hasNot)) { - t.Errorf("%s shown as %+v", row.id, body) + if !strings.Contains(both, row.has) || (row.hasNot != "" && strings.Contains(both, row.hasNot)) || + !slices.Equal(body.Paths, row.paths) { + t.Errorf("%s shown as %+v, want paths %v", row.id, body, row.paths) } } + // Issue 1126, T1: the paths are found on the whole bodies, before they are cut; JSON in a string is walked into, a + // key a secret could be under is named and never shown. + long := strings.Repeat("x", MaxBody) + conf := func(link, password string) json.RawMessage { + inner, _ := json.Marshal(map[string]any{"pad": long, "links": []string{"a", link}, "password": password}) + outer, _ := json.Marshal(map[string]any{"id": "conf", "type": "file", "module": "nats", "path": "/etc/conf", + "mode": "0600", "content": string(inner)}) + return outer + } + far := CompareComposedWith(map[string]Composed{"k": comp(conf("b", "hunter2"))}, []Composed{comp(conf("c", "hunter3"))}, nil) + if len(far.Differs) != 1 || len(far.Differs[0].Bodies) != 1 || + !slices.Equal(far.Differs[0].Bodies[0].Paths, []string{"content.links.1", "content.password"}) || + !strings.Contains(far.Differs[0].Bodies[0].Composed, "cut at") || strings.Contains(fmt.Sprint(far), "hunter") { + t.Errorf("a difference past the cut: %+v", far.Differs) + } + // And an answer too large for the node's key is fitted: the texts go first, the verdict and the paths stay. + big := CandidateAnswer{Head: "h", Verdict: CandidateDiffers, Differs: far.Differs} + if fitted := FitAnswer(big, 4<<10); len(fitted.Differs[0].Bodies[0].Composed) != 0 || len(fitted.Differs[0].Bodies[0].Paths) != 2 || + fitted.Verdict != CandidateDiffers || !strings.Contains(fitted.Note, "cut") || big.Differs[0].Bodies[0].Composed == "" { + t.Errorf("fitted as %+v", fitted) + } if c, r := MaskedPair(json.RawMessage(`{"env":{"DB_PASSWORD":"hunter2","PORT":"80"},"list":[{"token":"t1"}]}`), json.RawMessage(`{"env":{"DB_PASSWORD":"hunter3","PORT":"81"},"list":[{"token":"t1"}]}`), nil); strings.Contains(c+r, "hunter") || strings.Contains(c+r, `"t1"`) || !strings.Contains(r, `"PORT":"81"`) || !strings.Contains(c, "") || -- 2.54.0 From 1062b33d32c526c34b47ae8b0b715c6d2278e32f Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 28 Oct 2026 09:49:03 +0100 Subject: [PATCH 2/3] A node's shadow answer counts the values that drifted from the vault's, and names the own secrets among them (issue 1126, cause C4) delivery.OwnValuesDiffer, ValuesDiffer and WithValuesDiffer read the shown bodies masked as another value; a candidate answer carries values_differ. The node agent hands the vault those own secrets' running values through move-in's replace_running, so no seat verb and no bus grant changes here. --- go/delivery/own_drift.go | 84 ++++++++++++++++++++++++++++++ go/delivery/placement_candidate.go | 4 ++ go/delivery/placement_test.go | 17 ++++++ 3 files changed, 105 insertions(+) create mode 100644 go/delivery/own_drift.go diff --git a/go/delivery/own_drift.go b/go/delivery/own_drift.go new file mode 100644 index 0000000..fac57fa --- /dev/null +++ b/go/delivery/own_drift.go @@ -0,0 +1,84 @@ +package delivery + +// The own secrets a node's shadow answer finds the vault holds another value for than the node runs (novox/hq issue 1126, +// cause C4): the vault's `move-in` never replaces a value it holds, so a value the controller made anew later (a rotation +// at start, a re-issue) runs while the vault keeps the first one it was handed. The node opens both seals with its own +// key and its answer's shown bodies mask the value as MaskedAnotherValue; this reads which own secrets those are, for the +// node agent (which hands the vault what runs) and for the vault's `own_drift` (which lists them mesh-wide), so both read +// the answer one way. + +import ( + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// MaskedAnotherValue is the word a shown body masks a value with when the node opened both sides with its key and they +// are two values (MaskedPair, maskWord). +const MaskedAnotherValue = "" + +// OwnValueDiffers is one own secret of a module whose value composed (the vault's) is another than the one that runs. +type OwnValueDiffers struct { + Module string `json:"module"` + Name string `json:"name"` +} + +// ownNeedPrefix starts the id of the file a module's own secret is placed in (catalogue.NeedID). +var ownNeedPrefix = catalogue.NeedID("") + +// OwnValuesDiffer is every own secret a's shown bodies say holds another value composed than running: a body of a +// module's own secret's file (catalogue.NeedID) masked MaskedAnotherValue on either side. Sorted by module and name, each +// once. A body the answer did not show (beyond MaxBodies) is not read. +func OwnValuesDiffer(a CandidateAnswer) []OwnValueDiffers { + seen := map[OwnValueDiffers]bool{} + var out []OwnValueDiffers + for _, d := range a.Differs { + for _, b := range d.Bodies { + name, own := strings.CutPrefix(b.ID, ownNeedPrefix) + if !own || name == "" || !anotherValue(b) { + continue + } + o := OwnValueDiffers{Module: d.Module, Name: name} + if !seen[o] { + seen[o] = true + out = append(out, o) + } + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].Module != out[j].Module { + return out[i].Module < out[j].Module + } + return out[i].Name < out[j].Name + }) + return out +} + +// ValuesDiffer is how many shown bodies of a hold a value masked MaskedAnotherValue, own secrets and every other +// resource alike: the count a node's answer carries as CandidateAnswer.ValuesDiffer. +func ValuesDiffer(a CandidateAnswer) int { + n := 0 + for _, d := range a.Differs { + for _, b := range d.Bodies { + if anotherValue(b) { + n++ + } + } + } + return n +} + +// WithValuesDiffer is answers with each one's ValuesDiffer counted. +func WithValuesDiffer(answers map[string]CandidateAnswer) map[string]CandidateAnswer { + out := make(map[string]CandidateAnswer, len(answers)) + for g, a := range answers { + a.ValuesDiffer = ValuesDiffer(a) + out[g] = a + } + return out +} + +func anotherValue(b ResourceBodies) bool { + return strings.Contains(b.Composed, MaskedAnotherValue) || strings.Contains(b.Running, MaskedAnotherValue) +} diff --git a/go/delivery/placement_candidate.go b/go/delivery/placement_candidate.go index 12f276e..d6ae907 100644 --- a/go/delivery/placement_candidate.go +++ b/go/delivery/placement_candidate.go @@ -141,6 +141,10 @@ type CandidateAnswer struct { // Note is what the node says of how it answered, in words, beside the verdict: a pass that ran over a minute, inputs // it waited for (issue 1126: the shadow shows its work). It changes no verdict. Note string `json:"note,omitempty"` + // ValuesDiffer is how many shown bodies hold a value the node opened on both sides to two values (ValuesDiffer, + // MaskedAnotherValue): the vault's value and the one that runs drifted apart (issue 1126, cause C4). It changes no + // verdict. + ValuesDiffer int `json:"values_differ,omitempty"` } // ReturnsToController is what a node answers a head that deletes its placement file. diff --git a/go/delivery/placement_test.go b/go/delivery/placement_test.go index cde72a4..8f43377 100644 --- a/go/delivery/placement_test.go +++ b/go/delivery/placement_test.go @@ -354,6 +354,23 @@ func TestPlacement(t *testing.T) { !strings.Contains(c, "") { t.Errorf("masked as %s and %s", c, r) } + // Issue 1126, cause C4: the own secrets whose two sides open to two values are read from the shown bodies, and + // every body so masked is counted, own or not. + drift := CompareComposedWith(map[string]Composed{"k": comp(sealedFile("needs-jwt", "box-1", "0600"), + sealedFile("needs-salt", "box-4", "0600"), sealedFile("conf", "box-6", "0600"))}, + []Composed{comp(sealedFile("needs-jwt", "box-2", "0600"), sealedFile("needs-salt", "box-5", "0600"), + sealedFile("conf", "box-7", "0600"))}, + func(a, b string) (same, known bool) { return a == "box-4", true }) + got := CandidateAnswer{Head: "h", Differs: drift.Differs} + if own := OwnValuesDiffer(got); len(own) != 1 || own[0] != (OwnValueDiffers{Module: "nats", Name: "jwt"}) { + t.Errorf("the own secrets with another value are %+v, want nats jwt alone", own) + } + if n := ValuesDiffer(got); n != 2 { + t.Errorf("%d bodies hold another value, want 2 (jwt and conf)", n) + } + if w := WithValuesDiffer(map[string]CandidateAnswer{"g": got}); w["g"].ValuesDiffer != 2 { + t.Errorf("the answer carries values_differ %d", w["g"].ValuesDiffer) + } if c := CompareComposed(nil, []Composed{{Node: "novox", Resources: []json.RawMessage{}}}); len(c.Removed) != 1 || c.Removed[0] != "node" { t.Errorf("the node's own composition run and not composed is removed: %+v", c) } -- 2.54.0 From 3028c7ec4ed21b14e58d5043c22d54c3bbcbb2bc Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 28 Oct 2026 10:00:29 +0100 Subject: [PATCH 3/3] A drifted own secret is read from a resource id as a node runs it, its module's name first (only-office.needs-jwt), so the node agent finds what to hand the vault (issue 1126, cause C4; review round 2) delivery.OwnNeedName reads .needs- and the bare needs-; OwnValuesDiffer and the node agent use it. The fixtures carry the id shape captured from novox's shadow answer. --- go/delivery/own_drift.go | 17 ++++++++++++++--- go/delivery/placement_test.go | 23 ++++++++++++++++------- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/go/delivery/own_drift.go b/go/delivery/own_drift.go index fac57fa..546a4ee 100644 --- a/go/delivery/own_drift.go +++ b/go/delivery/own_drift.go @@ -27,16 +27,27 @@ type OwnValueDiffers struct { // ownNeedPrefix starts the id of the file a module's own secret is placed in (catalogue.NeedID). var ownNeedPrefix = catalogue.NeedID("") +// OwnNeedName is the name of module's own secret whose file a resource id names, and whether it names one. On a node a +// resource's id carries its module's name (`only-office.needs-jwt`, as a live shadow answer shows it); the bare form a +// declaration composes (`needs-jwt`, catalogue.NeedID) is read too. An id of another module's is not this module's. +func OwnNeedName(module, id string) (string, bool) { + if module != "" { + id = strings.TrimPrefix(id, module+".") + } + name, own := strings.CutPrefix(id, ownNeedPrefix) + return name, own && name != "" && !strings.Contains(name, ".") +} + // OwnValuesDiffer is every own secret a's shown bodies say holds another value composed than running: a body of a -// module's own secret's file (catalogue.NeedID) masked MaskedAnotherValue on either side. Sorted by module and name, each +// module's own secret's file (OwnNeedName: `.needs-` on a node) masked MaskedAnotherValue on either side. Sorted by module and name, each // once. A body the answer did not show (beyond MaxBodies) is not read. func OwnValuesDiffer(a CandidateAnswer) []OwnValueDiffers { seen := map[OwnValueDiffers]bool{} var out []OwnValueDiffers for _, d := range a.Differs { for _, b := range d.Bodies { - name, own := strings.CutPrefix(b.ID, ownNeedPrefix) - if !own || name == "" || !anotherValue(b) { + name, own := OwnNeedName(d.Module, b.ID) + if !own || !anotherValue(b) { continue } o := OwnValueDiffers{Module: d.Module, Name: name} diff --git a/go/delivery/placement_test.go b/go/delivery/placement_test.go index 8f43377..1799d06 100644 --- a/go/delivery/placement_test.go +++ b/go/delivery/placement_test.go @@ -356,19 +356,28 @@ func TestPlacement(t *testing.T) { } // Issue 1126, cause C4: the own secrets whose two sides open to two values are read from the shown bodies, and // every body so masked is counted, own or not. - drift := CompareComposedWith(map[string]Composed{"k": comp(sealedFile("needs-jwt", "box-1", "0600"), - sealedFile("needs-salt", "box-4", "0600"), sealedFile("conf", "box-6", "0600"))}, - []Composed{comp(sealedFile("needs-jwt", "box-2", "0600"), sealedFile("needs-salt", "box-5", "0600"), - sealedFile("conf", "box-7", "0600"))}, + // The ids are as a node runs them, its module's name first (captured from novox's shadow answer, 2026-10-28: + // `only-office.needs-jwt`); another module's need, named here, is not nats's. + drift := CompareComposedWith(map[string]Composed{"k": comp(sealedFile("nats.needs-jwt", "box-1", "0600"), + sealedFile("nats.needs-salt", "box-4", "0600"), sealedFile("nats.conf", "box-6", "0600"), + sealedFile("other.needs-key", "box-8", "0600"))}, + []Composed{comp(sealedFile("nats.needs-jwt", "box-2", "0600"), sealedFile("nats.needs-salt", "box-5", "0600"), + sealedFile("nats.conf", "box-7", "0600"), sealedFile("other.needs-key", "box-9", "0600"))}, func(a, b string) (same, known bool) { return a == "box-4", true }) got := CandidateAnswer{Head: "h", Differs: drift.Differs} if own := OwnValuesDiffer(got); len(own) != 1 || own[0] != (OwnValueDiffers{Module: "nats", Name: "jwt"}) { t.Errorf("the own secrets with another value are %+v, want nats jwt alone", own) } - if n := ValuesDiffer(got); n != 2 { - t.Errorf("%d bodies hold another value, want 2 (jwt and conf)", n) + if n := ValuesDiffer(got); n != 3 { + t.Errorf("%d bodies hold another value, want 3 (jwt, conf and other's key)", n) } - if w := WithValuesDiffer(map[string]CandidateAnswer{"g": got}); w["g"].ValuesDiffer != 2 { + for id, want := range map[string]string{"nats.needs-jwt": "jwt", "needs-jwt": "jwt", "other.needs-jwt": "", + "nats.conf": "", "nats.needs-": ""} { + if name, own := OwnNeedName("nats", id); name != want && (own || want != "") { + t.Errorf("%s names nats's own secret %q (%v), want %q", id, name, own, want) + } + } + if w := WithValuesDiffer(map[string]CandidateAnswer{"g": got}); w["g"].ValuesDiffer != 3 { t.Errorf("the answer carries values_differ %d", w["g"].ValuesDiffer) } if c := CompareComposed(nil, []Composed{{Node: "novox", Resources: []json.RawMessage{}}}); len(c.Removed) != 1 || c.Removed[0] != "node" { -- 2.54.0