Node-engine: composition comparison, seals and differing paths are the engine's own package, internal/compositions, with the tests moved from mesh-sdk's delivery (H-1, ADR 0634 §1; issue 1966)
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
This commit is contained in:
@@ -0,0 +1,137 @@
|
||||
package compositions
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DifferingPaths is every JSON path at which two bodies of one resource differ, sorted (issue 1126, T1: the shadow
|
||||
// shows its work): object keys and list places joined by dots (`content.url`, `links.0`), a string that holds a JSON
|
||||
// object or list read as that JSON and walked on (a runtime's config carried as text), a key or place one side alone
|
||||
// holds named as it is. A key a secret could be under (secretWords) is named and not walked into. A body that is not
|
||||
// JSON differs at "" (the whole), and a resource one side alone holds has none. At most MaxPaths: the last then says how many more there are. A path names keys
|
||||
// alone, never a value, so it holds no secret.
|
||||
func DifferingPaths(composed, running json.RawMessage) []string {
|
||||
if composed == nil || running == nil {
|
||||
return nil // one side alone holds it: its why says so
|
||||
}
|
||||
var a, b any
|
||||
okA := unmarshalNumbers(composed, &a) == nil
|
||||
okB := unmarshalNumbers(running, &b) == nil
|
||||
if !okA || !okB {
|
||||
if bytes.Equal(composed, running) {
|
||||
return nil
|
||||
}
|
||||
return []string{""}
|
||||
}
|
||||
var out []string
|
||||
walkDiffering(a, b, "", &out)
|
||||
sort.Strings(out)
|
||||
if len(out) > delivery.MaxPaths {
|
||||
more := len(out) - (delivery.MaxPaths - 1)
|
||||
out = append(out[:delivery.MaxPaths-1], "… "+strconv.Itoa(more)+" more")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func unmarshalNumbers(raw []byte, into *any) error {
|
||||
d := json.NewDecoder(bytes.NewReader(raw))
|
||||
d.UseNumber()
|
||||
return d.Decode(into)
|
||||
}
|
||||
|
||||
// asJSON is a string holding a JSON object or list read as that value; v itself otherwise.
|
||||
func asJSON(v any) any {
|
||||
s, ok := v.(string)
|
||||
if !ok {
|
||||
return v
|
||||
}
|
||||
t := strings.TrimSpace(s)
|
||||
if t == "" || (t[0] != '{' && t[0] != '[') {
|
||||
return v
|
||||
}
|
||||
var parsed any
|
||||
if unmarshalNumbers([]byte(t), &parsed) != nil {
|
||||
return v
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func join(at, key string) string {
|
||||
if at == "" {
|
||||
return key
|
||||
}
|
||||
return at + "." + key
|
||||
}
|
||||
|
||||
func walkDiffering(a, b any, at string, out *[]string) {
|
||||
a, b = asJSON(a), asJSON(b)
|
||||
ma, isMA := a.(map[string]any)
|
||||
mb, isMB := b.(map[string]any)
|
||||
if isMA && isMB {
|
||||
keys := map[string]bool{}
|
||||
for k := range ma {
|
||||
keys[k] = true
|
||||
}
|
||||
for k := range mb {
|
||||
keys[k] = true
|
||||
}
|
||||
for k := range keys {
|
||||
x, inA := ma[k]
|
||||
y, inB := mb[k]
|
||||
switch {
|
||||
case !inA || !inB:
|
||||
*out = append(*out, join(at, k))
|
||||
case secretKey(k):
|
||||
// A file's secrets (SecretsField) are named entry by entry, each entry never entered; any other
|
||||
// object under a secret key, a vault box included, is one path.
|
||||
sx, isSX := x.(map[string]any)
|
||||
sy, isSY := y.(map[string]any)
|
||||
if k != SecretsField || !isSX || !isSY {
|
||||
if !sameAny(x, y) {
|
||||
*out = append(*out, join(at, k))
|
||||
}
|
||||
continue
|
||||
}
|
||||
for name := range sx {
|
||||
if v, in := sy[name]; !in || !sameAny(sx[name], v) {
|
||||
*out = append(*out, join(join(at, k), name))
|
||||
}
|
||||
}
|
||||
for name := range sy {
|
||||
if _, in := sx[name]; !in {
|
||||
*out = append(*out, join(join(at, k), name))
|
||||
}
|
||||
}
|
||||
default:
|
||||
walkDiffering(x, y, join(at, k), out)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
la, isLA := a.([]any)
|
||||
lb, isLB := b.([]any)
|
||||
if isLA && isLB {
|
||||
for i := range max(len(la), len(lb)) {
|
||||
if i >= len(la) || i >= len(lb) {
|
||||
*out = append(*out, join(at, strconv.Itoa(i)))
|
||||
continue
|
||||
}
|
||||
walkDiffering(la[i], lb[i], join(at, strconv.Itoa(i)), out)
|
||||
}
|
||||
return
|
||||
}
|
||||
if !sameAny(a, b) {
|
||||
*out = append(*out, at)
|
||||
}
|
||||
}
|
||||
|
||||
func sameAny(a, b any) bool {
|
||||
x, errA := json.Marshal(a)
|
||||
y, errB := json.Marshal(b)
|
||||
return errA == nil && errB == nil && bytes.Equal(x, y)
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package compositions
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// What the node-engine finds when it compares the compositions it made with the ones it runs (issue 1126): the bus's user
|
||||
// list left out, a resource that differs named with both digests, a value sealed again judged by what it opens to, every
|
||||
// differing body shown with its secrets masked, and the paths found on the whole bodies. Moved from mesh-sdk's delivery
|
||||
// package with the code (ADR 0634 §1: only the node-engine compares compositions).
|
||||
func TestComparingCompositions(t *testing.T) {
|
||||
// CompareComposed: the bus's user list is left out on both sides; anything else that differs is named.
|
||||
file := func(id, content string) json.RawMessage {
|
||||
return json.RawMessage(`{"id":"` + id + `","type":"file","module":"nats","path":"/etc/` + id + `","mode":"0600","content":"` + content + `"}`)
|
||||
}
|
||||
fp := "src2:" + strings.Repeat("a", 64)
|
||||
comp := func(rs ...json.RawMessage) delivery.Composed {
|
||||
return delivery.Composed{Node: "novox", Module: "nats", FP: fp, Resources: rs}
|
||||
}
|
||||
for name, tc := range map[string]struct {
|
||||
composed, running delivery.Composed
|
||||
resource, why string
|
||||
}{
|
||||
"user list alone": {comp(file("conf", "a"), file("nats.bus-users", "stand-in")), comp(file("conf", "a"), file("nats.bus-users", "real")), "", ""},
|
||||
"a resource": {comp(file("conf", "a")), comp(file("conf", "b")), "conf", "the resource differs"},
|
||||
"one side": {comp(file("conf", "a"), file("more", "x")), comp(file("conf", "a")), "more", "composed, and not in what runs"},
|
||||
"order": {comp(file("conf", "a"), file("more", "x")), comp(file("more", "x"), file("conf", "a")), "", "the same resources in another order"},
|
||||
} {
|
||||
c := CompareComposed(map[string]delivery.Composed{"novox.nats." + fp: tc.composed}, []delivery.Composed{tc.running})
|
||||
if tc.why == "" {
|
||||
if len(c.Differs) != 0 || len(c.Same) != 1 {
|
||||
t.Errorf("%s: %+v", name, c)
|
||||
}
|
||||
} else if len(c.Differs) != 1 || c.Differs[0].Resource != tc.resource || c.Differs[0].Why != tc.why ||
|
||||
c.Differs[0].Composed == "" || c.Differs[0].Running == "" || c.Differs[0].Composed == c.Differs[0].Running {
|
||||
t.Errorf("%s: differs %+v, want %s: %s with both digests", name, c.Differs, tc.resource, tc.why)
|
||||
}
|
||||
}
|
||||
// Issue 1126, the shadow shows its work: every differing resource with both bodies, each value a secret could be
|
||||
// in masked (never a secret in an answer), saying whether the two sides hold the same value.
|
||||
sealedFile := func(id, sealed, mode string) json.RawMessage {
|
||||
return json.RawMessage(`{"id":"` + id + `","type":"file","module":"nats","path":"/etc/` + id + `","mode":"` + mode + `","sealed":"` + sealed + `"}`)
|
||||
}
|
||||
// Issue 1126, C1: a sealed value is judged by the value it opens to at its own slot: b sealed again (box-2 and
|
||||
// box-3 open to one value) is no difference; d holds at its slot a box of another value, though that value is
|
||||
// the one b holds, and differs at its path `sealed`.
|
||||
sameBox := func(a, b string) (same, known bool) {
|
||||
return (a == "box-2" && b == "box-3") || a == b, true
|
||||
}
|
||||
shown := CompareComposedWith(map[string]delivery.Composed{"k": comp(sealedFile("a", "box-1", "0600"), sealedFile("b", "box-2", "0600"),
|
||||
file("c", "x"), sealedFile("d", "box-2", "0600"))},
|
||||
[]delivery.Composed{comp(sealedFile("a", "box-1", "0644"), sealedFile("b", "box-3", "0600"), file("c", "y"), sealedFile("d", "box-4", "0600"))},
|
||||
sameBox)
|
||||
if len(shown.Differs) != 1 || len(shown.Differs[0].Bodies) != 3 {
|
||||
t.Fatalf("every differing resource is not shown, or b sealed again is shown: %+v", shown.Differs)
|
||||
}
|
||||
if c := CompareComposedWith(map[string]delivery.Composed{"k": comp(sealedFile("b", "box-2", "0600"))},
|
||||
[]delivery.Composed{comp(sealedFile("b", "box-3", "0600"))}, sameBox); len(c.Same) != 1 {
|
||||
t.Errorf("a value sealed again at its own slot is not the same: %+v", c)
|
||||
}
|
||||
for _, row := range []struct {
|
||||
id, has, hasNot string
|
||||
paths []string
|
||||
}{
|
||||
{"a", `"mode":"0600"`, "box-1", []string{"mode"}},
|
||||
{"c", `"content":"x"`, "", []string{"content"}},
|
||||
{"d", "<masked: another value>", "box-2", []string{"sealed"}},
|
||||
} {
|
||||
var body delivery.ResourceBodies
|
||||
for _, b := range shown.Differs[0].Bodies {
|
||||
if b.ID == row.id {
|
||||
body = b
|
||||
}
|
||||
}
|
||||
both := body.Composed + body.Running
|
||||
if !strings.Contains(both, row.has) || (row.hasNot != "" && strings.Contains(both, row.hasNot)) ||
|
||||
!slices.Equal(body.Paths, row.paths) {
|
||||
t.Errorf("%s shown as %+v, want paths %v", row.id, body, row.paths)
|
||||
}
|
||||
}
|
||||
// Issue 1126, C1, the seed-7 probe: a file naming secrets carries each sealed value in its `secrets`, and each entry
|
||||
// is a slot of its own. The shape is intoFile's output, captured from catalogue for a file naming two secrets
|
||||
// (2026-10-28), with the module a composition names each resource of; only the boxes are replaced.
|
||||
withSecrets := func(db, key string) json.RawMessage {
|
||||
return json.RawMessage(`{"content":"[database]\npassword = ${secret:postgres-database}\n[security]\nsecret_key = ${secret:secret-key}\n",` +
|
||||
`"id":"grafana.ini","mode":"0600","module":"nats","path":"/etc/grafana/grafana.ini","secrets":{"postgres-database":"` + db +
|
||||
`","secret-key":"` + key + `"},"type":"file"}`)
|
||||
}
|
||||
if c := CompareComposedWith(map[string]delivery.Composed{"k": comp(withSecrets("box-2", "box-1"))},
|
||||
[]delivery.Composed{comp(withSecrets("box-3", "box-1"))}, sameBox); len(c.Same) != 1 {
|
||||
t.Errorf("a secret sealed again at its own entry is not the same: %+v", c)
|
||||
}
|
||||
if kept := KeepSeals(comp(withSecrets("box-2", "box-1")), SealsIn(comp(withSecrets("box-3", "box-1"))), sameBox); !bytes.Equal(kept.Resources[0], withSecrets("box-3", "box-1")) {
|
||||
t.Errorf("the box that runs is not kept byte for byte: %s", kept.Resources[0])
|
||||
}
|
||||
if s := SealsIn(comp(withSecrets("box-2", "box-1"))); s[SealSlot("grafana.ini", SecretSlotField("postgres-database"))] != "box-2" || len(s) != 2 {
|
||||
t.Errorf("the secrets' slots are %v", s)
|
||||
}
|
||||
// Another value at one entry differs there alone, each entry masked by what it opens to.
|
||||
// box-2 and box-4 open to one value, so the box that runs is kept there; box-5 and box-6 are two values.
|
||||
mixed := CompareComposedWith(map[string]delivery.Composed{"k": comp(withSecrets("box-2", "box-5"))},
|
||||
[]delivery.Composed{comp(withSecrets("box-4", "box-6"))}, func(a, b string) (bool, bool) {
|
||||
return (a == "box-2" && b == "box-4") || a == b, true
|
||||
})
|
||||
if len(mixed.Differs) != 1 || len(mixed.Differs[0].Bodies) != 1 {
|
||||
t.Fatalf("a secret of another value is not shown: %+v", mixed)
|
||||
}
|
||||
if body := mixed.Differs[0].Bodies[0]; !slices.Equal(body.Paths, []string{"secrets.secret-key"}) ||
|
||||
!strings.Contains(body.Composed, `"secret-key":"<masked: another value>"`) || strings.Contains(body.Composed+body.Running, "box-") ||
|
||||
!strings.Contains(body.Running, `"postgres-database":"<masked: the same on both sides>"`) {
|
||||
t.Errorf("a file's secrets shown as %+v", body)
|
||||
}
|
||||
if c, _ := MaskedPair(withSecrets("box-2", "box-5"), withSecrets("box-4", "box-6"), func(a, b string) (bool, bool) {
|
||||
return a == "box-2", true
|
||||
}); !strings.Contains(c, `"postgres-database":"<masked: another seal of the same value>"`) || !strings.Contains(c, `"secret-key":"<masked: another value>"`) {
|
||||
t.Errorf("each secret is not masked by what it opens to: %s", c)
|
||||
}
|
||||
// Only a file's secrets are split by entry: a vault box (an object under `vault-sealed`) is one masked word and one
|
||||
// path, judged whole by what it opens to (rev-seed-same-3, round 1).
|
||||
box := func(ct string) json.RawMessage {
|
||||
return json.RawMessage(`{"id":"core-box","type":"file","module":"nats","path":"/etc/core","mode":"0600","vault-sealed":{"v":1,"eph":"e-` +
|
||||
ct + `","ct":"` + ct + `"}}`)
|
||||
}
|
||||
vault := CompareComposedWith(map[string]delivery.Composed{"k": comp(box("c1"))}, []delivery.Composed{comp(box("c2"))},
|
||||
func(a, b string) (bool, bool) {
|
||||
return false, strings.Contains(a, `"ct":"c1"`) && strings.Contains(b, `"ct":"c2"`)
|
||||
})
|
||||
if len(vault.Differs) != 1 || len(vault.Differs[0].Bodies) != 1 {
|
||||
t.Fatalf("a box of another value is not shown: %+v", vault)
|
||||
}
|
||||
if body := vault.Differs[0].Bodies[0]; !slices.Equal(body.Paths, []string{"vault-sealed"}) ||
|
||||
!strings.Contains(body.Composed, `"vault-sealed":"<masked: another value>"`) || strings.Contains(body.Composed+body.Running, "c1") {
|
||||
t.Errorf("a vault box shown as %+v", body)
|
||||
}
|
||||
// Issue 1126, T1: the paths are found on the whole bodies, before they are cut; JSON in a string is walked into, a
|
||||
// key a secret could be under is named and never shown.
|
||||
long := strings.Repeat("x", delivery.MaxBody)
|
||||
conf := func(link, password string) json.RawMessage {
|
||||
inner, _ := json.Marshal(map[string]any{"pad": long, "links": []string{"a", link}, "password": password})
|
||||
outer, _ := json.Marshal(map[string]any{"id": "conf", "type": "file", "module": "nats", "path": "/etc/conf",
|
||||
"mode": "0600", "content": string(inner)})
|
||||
return outer
|
||||
}
|
||||
far := CompareComposedWith(map[string]delivery.Composed{"k": comp(conf("b", "hunter2"))}, []delivery.Composed{comp(conf("c", "hunter3"))}, nil)
|
||||
if len(far.Differs) != 1 || len(far.Differs[0].Bodies) != 1 ||
|
||||
!slices.Equal(far.Differs[0].Bodies[0].Paths, []string{"content.links.1", "content.password"}) ||
|
||||
!strings.Contains(far.Differs[0].Bodies[0].Composed, "cut at") || strings.Contains(fmt.Sprint(far), "hunter") {
|
||||
t.Errorf("a difference past the cut: %+v", far.Differs)
|
||||
}
|
||||
if c, r := MaskedPair(json.RawMessage(`{"env":{"DB_PASSWORD":"hunter2","PORT":"80"},"list":[{"token":"t1"}]}`),
|
||||
json.RawMessage(`{"env":{"DB_PASSWORD":"hunter3","PORT":"81"},"list":[{"token":"t1"}]}`), nil); strings.Contains(c+r, "hunter") ||
|
||||
strings.Contains(c+r, `"t1"`) || !strings.Contains(r, `"PORT":"81"`) || !strings.Contains(c, "<masked: differs>") ||
|
||||
!strings.Contains(c, "<masked: the same on both sides>") {
|
||||
t.Errorf("masked as %s and %s", c, r)
|
||||
}
|
||||
if c := CompareComposed(nil, []delivery.Composed{{Node: "novox", Resources: []json.RawMessage{}}}); len(c.Removed) != 1 || c.Removed[0] != "node" {
|
||||
t.Errorf("the node's own composition run and not composed is removed: %+v", c)
|
||||
}
|
||||
if c := CompareComposed(map[string]delivery.Composed{"k": comp(file("conf", "a"))}, nil); len(c.Added) != 1 || c.Added[0].Module != "nats" || c.Added[0].Composed == "" {
|
||||
t.Errorf("a composition not run is added: %+v", c)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,412 @@
|
||||
package compositions
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"io"
|
||||
"maps"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// Comparison is what CompareComposed finds, module by module: the same, differing, added by what was composed (not run
|
||||
// now) and removed by it (run now, not composed). Each list is sorted by module.
|
||||
type Comparison struct {
|
||||
Same []string
|
||||
Differs []delivery.ModuleDiffers
|
||||
Added []delivery.ModuleAdded
|
||||
Removed []string
|
||||
}
|
||||
|
||||
// CompareComposed compares the compositions a node composed (by key, as compose.Compositions answers them) with the ones
|
||||
// it runs, as its agent would (ADR 0559 §4: a node's answer to a placement candidate). A module is the same when its
|
||||
// composition's digest is the one it runs (ComposedDigestOf, the signature left out), so the agent would apply nothing
|
||||
// of it; it differs when the digests differ (both named); it is added when composed and not run, removed when run and
|
||||
// not composed. The node's own composition is named HaveNodeWord. The bus's user list (catalogue.BusUsersID) is left out
|
||||
// on both sides: the bus node follows it alone and never applies it from a composition it made (ADR 0547 §2, §8). A
|
||||
// difference names the first resource, by its id, whose content differs or that one side alone holds; when every
|
||||
// resource is equal it says what else differs (their order, the files filled, the data, the node's own parts, a file
|
||||
// carried by digest).
|
||||
func CompareComposed(composed map[string]delivery.Composed, running []delivery.Composed) Comparison {
|
||||
return CompareComposedWith(composed, running, nil)
|
||||
}
|
||||
|
||||
// CompareComposedWith is CompareComposed showing, for each module that differs, every differing resource with both
|
||||
// bodies masked (MaskedPair); sameValue, when given, says whether two sealed values open to the same value (the node
|
||||
// opens them with its own key), so a value sealed again reads as such.
|
||||
func CompareComposedWith(composed map[string]delivery.Composed, running []delivery.Composed, sameValue func(a, b string) (same, known bool)) Comparison {
|
||||
name := func(c delivery.Composed) string {
|
||||
if c.Module == "" {
|
||||
return delivery.HaveNodeWord
|
||||
}
|
||||
return c.Module
|
||||
}
|
||||
ours := map[string]delivery.Composed{}
|
||||
for _, c := range composed {
|
||||
ours[name(c)] = c
|
||||
}
|
||||
theirs := map[string]delivery.Composed{}
|
||||
for _, c := range running {
|
||||
theirs[name(c)] = c
|
||||
}
|
||||
modules := map[string]bool{}
|
||||
for m := range ours {
|
||||
modules[m] = true
|
||||
}
|
||||
for m := range theirs {
|
||||
modules[m] = true
|
||||
}
|
||||
var out Comparison
|
||||
for _, m := range slices.Sorted(maps.Keys(modules)) {
|
||||
a, composedHere := ours[m]
|
||||
b, runs := theirs[m]
|
||||
switch {
|
||||
case !runs:
|
||||
dig, _ := delivery.ComposedDigestOf(withoutBusUsers(a))
|
||||
out.Added = append(out.Added, delivery.ModuleAdded{Module: m, Composed: dig})
|
||||
case !composedHere:
|
||||
out.Removed = append(out.Removed, m)
|
||||
default:
|
||||
if d, ok := compareOne(m, a, b, sameValue); ok {
|
||||
out.Same = append(out.Same, m)
|
||||
} else {
|
||||
out.Differs = append(out.Differs, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// compareOne is one module's comparison: true when the digests agree, else what differs first.
|
||||
func compareOne(module string, composed, running delivery.Composed, sameValue func(a, b string) (bool, bool)) (delivery.ModuleDiffers, bool) {
|
||||
composed, running = withoutBusUsers(composed), withoutBusUsers(running)
|
||||
// A sealed value is judged by the value it opens to, slot by slot (issue 1126, C1): a value sealed again at the same
|
||||
// field of the same resource is the box that runs, never a difference; and a box is never matched across resources.
|
||||
composed = KeepSeals(composed, SealsIn(running), sameValue)
|
||||
dc, errC := delivery.ComposedDigestOf(composed)
|
||||
dr, errR := delivery.ComposedDigestOf(running)
|
||||
if errC != nil {
|
||||
return delivery.ModuleDiffers{Module: module, Running: dr, Why: "the composition has no digest: " + errC.Error()}, false
|
||||
}
|
||||
if errR == nil && dc == dr {
|
||||
return delivery.ModuleDiffers{}, true
|
||||
}
|
||||
differs := func(resource, why string) (delivery.ModuleDiffers, bool) {
|
||||
return delivery.ModuleDiffers{Module: module, Composed: dc, Running: dr, Resource: resource, Why: why}, false
|
||||
}
|
||||
// Words for the difference: resource by resource, each side with its large files by digest, so a file carried whole
|
||||
// and the same file carried by digest read as one.
|
||||
a, _, errA := composed.ByDigest()
|
||||
b, _, errB := running.ByDigest()
|
||||
if errA != nil || errB != nil {
|
||||
return differs("", "the compositions differ and cannot be read resource by resource")
|
||||
}
|
||||
ra, rb := resourcesByID(a.Resources), resourcesByID(b.Resources)
|
||||
ids := map[string]bool{}
|
||||
for id := range ra {
|
||||
ids[id] = true
|
||||
}
|
||||
for id := range rb {
|
||||
ids[id] = true
|
||||
}
|
||||
var first delivery.ModuleDiffers
|
||||
found := false
|
||||
var bodies []delivery.ResourceBodies
|
||||
for _, id := range slices.Sorted(maps.Keys(ids)) {
|
||||
x, inA := ra[id]
|
||||
y, inB := rb[id]
|
||||
why := ""
|
||||
switch {
|
||||
case !inB:
|
||||
why = "composed, and not in what runs"
|
||||
case !inA:
|
||||
why = "in what runs, and not composed"
|
||||
case !bytes.Equal(x, y):
|
||||
why = "the resource differs"
|
||||
default:
|
||||
continue
|
||||
}
|
||||
if !found {
|
||||
first, _ = differs(id, why)
|
||||
found = true
|
||||
}
|
||||
if len(bodies) < delivery.MaxBodies {
|
||||
// The paths are found on the whole bodies, before either is cut (issue 1126, T1: three resources over
|
||||
// MaxBody could not be told apart).
|
||||
c, r := MaskedPair(x, y, sameValue)
|
||||
bodies = append(bodies, delivery.ResourceBodies{ID: id, Why: why, Paths: DifferingPaths(x, y),
|
||||
Composed: cut(c), Running: cut(r)})
|
||||
}
|
||||
}
|
||||
if found {
|
||||
first.Bodies = bodies
|
||||
return first, false
|
||||
}
|
||||
switch {
|
||||
case !jsonEqual(a.Filled, b.Filled):
|
||||
return differs("", "the files the provisioner fills differ")
|
||||
case a.Data != b.Data:
|
||||
return differs("", "whether it declares data differs")
|
||||
case !jsonEqual(a.Own, b.Own):
|
||||
return differs("", "the node's own parts differ")
|
||||
case a.FP != b.FP:
|
||||
return differs("", "the fingerprint differs")
|
||||
}
|
||||
if dA, err := delivery.ComposedDigestOf(a); err == nil {
|
||||
if dB, err := delivery.ComposedDigestOf(b); err == nil && dA == dB {
|
||||
return differs("", "the same content, a large file carried whole on one side and by digest on the other")
|
||||
}
|
||||
}
|
||||
return differs("", "the same resources in another order")
|
||||
}
|
||||
|
||||
// withoutBusUsers is c without the bus's user list resource (its id `bus-users`, or `<module>.bus-users`).
|
||||
func withoutBusUsers(c delivery.Composed) delivery.Composed {
|
||||
out := c
|
||||
out.Resources = nil
|
||||
for _, raw := range c.Resources {
|
||||
var head struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if json.Unmarshal(raw, &head) == nil && (head.ID == catalogue.BusUsersID() || strings.HasSuffix(head.ID, "."+catalogue.BusUsersID())) {
|
||||
continue
|
||||
}
|
||||
out.Resources = append(out.Resources, raw)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// resourcesByID is each resource canonical, by its id (a resource without one by its place).
|
||||
func resourcesByID(resources []json.RawMessage) map[string][]byte {
|
||||
out := make(map[string][]byte, len(resources))
|
||||
for i, raw := range resources {
|
||||
var head struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &head)
|
||||
id := head.ID
|
||||
if id == "" {
|
||||
id = "#" + strconv.Itoa(i)
|
||||
}
|
||||
canon, err := canonicalJSON(raw)
|
||||
if err != nil {
|
||||
canon = raw
|
||||
}
|
||||
out[id] = canon
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// cut is s cut at MaxBody bytes, saying so.
|
||||
func cut(s string) string {
|
||||
if len(s) <= delivery.MaxBody {
|
||||
return s
|
||||
}
|
||||
return s[:delivery.MaxBody] + "… (cut at " + strconv.Itoa(delivery.MaxBody) + " bytes)"
|
||||
}
|
||||
|
||||
// secretWords are the words of a key whose value may hold a secret or a sealed one: masked in a shown body.
|
||||
var secretWords = []string{"sealed", "secret", "password", "passwd", "token", "private", "key", "credential", "jwt", "salt", "hash"}
|
||||
|
||||
func secretKey(k string) bool {
|
||||
k = strings.ToLower(k)
|
||||
for _, w := range secretWords {
|
||||
if strings.Contains(k, w) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// MaskedPair is two bodies of one resource (either may be nil) as JSON text with every value under a key a secret could
|
||||
// be in (secretWords, at any depth) masked: "<masked: the same on both sides>", "<masked: another seal of the same
|
||||
// value>" or "<masked: another value>" when sameValue says so, "<masked: differs>" otherwise, "<masked>" on one side
|
||||
// alone. A body that is not JSON is shown as its length only.
|
||||
func MaskedPair(composed, running json.RawMessage, sameValue func(a, b string) (same, known bool)) (string, string) {
|
||||
var a, b any
|
||||
okA := composed != nil && json.Unmarshal(composed, &a) == nil
|
||||
okB := running != nil && json.Unmarshal(running, &b) == nil
|
||||
if okA && okB {
|
||||
a, b = maskBoth(a, b, sameValue)
|
||||
} else if okA {
|
||||
a, _ = maskBoth(a, nil, nil)
|
||||
} else if okB {
|
||||
_, b = maskBoth(nil, b, nil)
|
||||
}
|
||||
show := func(v any, raw json.RawMessage, ok bool) string {
|
||||
if raw == nil {
|
||||
return ""
|
||||
}
|
||||
if !ok {
|
||||
return "<not JSON: " + strconv.Itoa(len(raw)) + " bytes>"
|
||||
}
|
||||
var out bytes.Buffer
|
||||
enc := json.NewEncoder(&out)
|
||||
enc.SetEscapeHTML(false)
|
||||
_ = enc.Encode(v)
|
||||
return strings.TrimSuffix(out.String(), "\n")
|
||||
}
|
||||
return show(a, composed, okA), show(b, running, okB)
|
||||
}
|
||||
|
||||
func maskBoth(a, b any, sameValue func(a, b string) (bool, bool)) (any, any) {
|
||||
ma, isA := a.(map[string]any)
|
||||
mb, isB := b.(map[string]any)
|
||||
if isA || isB {
|
||||
outA, outB := map[string]any{}, map[string]any{}
|
||||
keys := map[string]bool{}
|
||||
for k := range ma {
|
||||
keys[k] = true
|
||||
}
|
||||
for k := range mb {
|
||||
keys[k] = true
|
||||
}
|
||||
for k := range keys {
|
||||
va, inA := ma[k]
|
||||
vb, inB := mb[k]
|
||||
if secretKey(k) {
|
||||
x, y := maskAll(k, va, vb, inA && inB, sameValue)
|
||||
if inA {
|
||||
outA[k] = x
|
||||
}
|
||||
if inB {
|
||||
outB[k] = y
|
||||
}
|
||||
continue
|
||||
}
|
||||
x, y := maskBoth(va, vb, sameValue)
|
||||
if inA {
|
||||
outA[k] = x
|
||||
}
|
||||
if inB {
|
||||
outB[k] = y
|
||||
}
|
||||
}
|
||||
var ra, rb any
|
||||
if isA {
|
||||
ra = outA
|
||||
} else {
|
||||
ra = a
|
||||
}
|
||||
if isB {
|
||||
rb = outB
|
||||
} else {
|
||||
rb = b
|
||||
}
|
||||
return ra, rb
|
||||
}
|
||||
la, isLA := a.([]any)
|
||||
lb, isLB := b.([]any)
|
||||
if isLA || isLB {
|
||||
outA := make([]any, len(la))
|
||||
outB := make([]any, len(lb))
|
||||
for i := range max(len(la), len(lb)) {
|
||||
var x, y any
|
||||
if i < len(la) {
|
||||
x = la[i]
|
||||
}
|
||||
if i < len(lb) {
|
||||
y = lb[i]
|
||||
}
|
||||
x, y = maskBoth(x, y, sameValue)
|
||||
if i < len(la) {
|
||||
outA[i] = x
|
||||
}
|
||||
if i < len(lb) {
|
||||
outB[i] = y
|
||||
}
|
||||
}
|
||||
var ra, rb any = a, b
|
||||
if isLA {
|
||||
ra = outA
|
||||
}
|
||||
if isLB {
|
||||
rb = outB
|
||||
}
|
||||
return ra, rb
|
||||
}
|
||||
return a, b
|
||||
}
|
||||
|
||||
// maskAll is a value under a key a secret could be under, masked as one word (maskWord); a file's secrets
|
||||
// (SecretsField) alone entry by entry, each its own sealed value: "another seal of the same value" or "another value"
|
||||
// per secret, its name shown (issue 1126, C1). Any other object, a vault box included, is one word, judged whole.
|
||||
func maskAll(key string, a, b any, both bool, sameValue func(a, b string) (bool, bool)) (any, any) {
|
||||
ma, isA := a.(map[string]any)
|
||||
mb, isB := b.(map[string]any)
|
||||
if key != SecretsField || (!isA && !isB) {
|
||||
w := maskWord(a, b, both, sameValue)
|
||||
return w, w
|
||||
}
|
||||
outA, outB := map[string]any{}, map[string]any{}
|
||||
for k, va := range ma {
|
||||
vb, inB := mb[k]
|
||||
outA[k] = maskWord(va, vb, isB && inB, sameValue)
|
||||
}
|
||||
for k, vb := range mb {
|
||||
va, inA := ma[k]
|
||||
outB[k] = maskWord(va, vb, isA && inA, sameValue)
|
||||
}
|
||||
// A map on one side and another value on the other: the other is one word.
|
||||
var ra, rb any = outA, outB
|
||||
if !isA {
|
||||
ra = maskWord(a, b, both, sameValue)
|
||||
}
|
||||
if !isB {
|
||||
rb = maskWord(a, b, both, sameValue)
|
||||
}
|
||||
return ra, rb
|
||||
}
|
||||
|
||||
func maskWord(a, b any, both bool, sameValue func(a, b string) (bool, bool)) string {
|
||||
if !both {
|
||||
return "<masked>"
|
||||
}
|
||||
x, _ := json.Marshal(a)
|
||||
y, _ := json.Marshal(b)
|
||||
if bytes.Equal(x, y) {
|
||||
return "<masked: the same on both sides>"
|
||||
}
|
||||
if sameValue != nil {
|
||||
sa, okA := a.(string)
|
||||
sb, okB := b.(string)
|
||||
if !okA {
|
||||
sa = string(x)
|
||||
}
|
||||
if !okB {
|
||||
sb = string(y)
|
||||
}
|
||||
if same, known := sameValue(sa, sb); known {
|
||||
if same {
|
||||
return "<masked: another seal of the same value>"
|
||||
}
|
||||
return "<masked: another value>"
|
||||
}
|
||||
}
|
||||
return "<masked: differs>"
|
||||
}
|
||||
|
||||
// canonicalJSON is raw read as one JSON value (numbers kept as written) and written again, so equal values compare equal.
|
||||
func canonicalJSON(raw json.RawMessage) (json.RawMessage, error) {
|
||||
var v any
|
||||
d := json.NewDecoder(bytes.NewReader(raw))
|
||||
d.UseNumber()
|
||||
if err := d.Decode(&v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := d.Token(); err != io.EOF {
|
||||
return nil, errors.New("more after the value")
|
||||
}
|
||||
return json.Marshal(v)
|
||||
}
|
||||
|
||||
func jsonEqual(a, b any) bool {
|
||||
ja, _ := json.Marshal(a)
|
||||
jb, _ := json.Marshal(b)
|
||||
return string(ja) == string(jb)
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package compositions
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"sort"
|
||||
"strconv"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// Seals is, by slot, each value one composition carries sealed to its node's key (issue 1126, C1): a file's `sealed`
|
||||
// a vault box (catalogue.VaultSealedField) and each entry of a file's secrets (SecretsField), as their JSON text (a box is carried as its JSON, a sealed value as
|
||||
// the string it is). A slot is one field of one resource of the composition's module (SealSlot): a box is reused only
|
||||
// at the slot that holds it, never for another resource whose value opens to the same (a provider's grant and its
|
||||
// consumer's secret on one node hold one value).
|
||||
type Seals = map[string]string
|
||||
|
||||
// SealedFields are the fields of a resource that carry a value sealed to the node's key.
|
||||
var SealedFields = []string{"sealed", catalogue.VaultSealedField}
|
||||
|
||||
// SealSlot is the slot of one sealed field of one resource, by the resource's id (its place, `#<i>`, when it has none).
|
||||
// The module is the composition's: a slot is never compared across compositions.
|
||||
func SealSlot(resource, field string) string { return resource + "#" + field }
|
||||
|
||||
// SecretsField is the field of a file that carries, by name, each sealed value its content asks for
|
||||
// (${secret:<name>}, catalogue's intoFile): each entry is a slot of its own (SecretSlotField), so a secret sealed again
|
||||
// is judged by the value it opens to, entry by entry, like a file's `sealed` (issue 1126, C1; the seed-7 probe: about 45
|
||||
// files on ace and novox differed by their secrets' bytes alone).
|
||||
const SecretsField = "secrets"
|
||||
|
||||
// SecretSlotField is the field part of the slot of one entry of a file's secrets (SealSlot(id, SecretSlotField(name))).
|
||||
func SecretSlotField(name string) string { return SecretsField + "." + name }
|
||||
|
||||
// sealAt is one sealed value of a resource: its slot's field, its place (a top-level field, or an entry of the
|
||||
// secrets), and its raw JSON.
|
||||
type sealAt struct {
|
||||
field, entry string
|
||||
raw json.RawMessage
|
||||
}
|
||||
|
||||
// sealsAt is every sealed value of one resource's fields, in a fixed order: SealedFields, then the secrets by name.
|
||||
func sealsAt(fields map[string]json.RawMessage) []sealAt {
|
||||
var out []sealAt
|
||||
for _, field := range SealedFields {
|
||||
if f, has := fields[field]; has {
|
||||
out = append(out, sealAt{field: field, raw: f})
|
||||
}
|
||||
}
|
||||
var secrets map[string]json.RawMessage
|
||||
if f, has := fields[SecretsField]; has && json.Unmarshal(f, &secrets) == nil {
|
||||
names := make([]string, 0, len(secrets))
|
||||
for name := range secrets {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
out = append(out, sealAt{field: SecretSlotField(name), entry: name, raw: secrets[name]})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// SealsIn is every sealed value c carries, by slot; nil when it carries none.
|
||||
func SealsIn(c delivery.Composed) Seals {
|
||||
var out Seals
|
||||
for i, raw := range c.Resources {
|
||||
id, fields, ok := sealedFieldsOf(raw, i)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, s := range sealsAt(fields) {
|
||||
if text := sealedText(s.raw); text != "" {
|
||||
if out == nil {
|
||||
out = Seals{}
|
||||
}
|
||||
out[SealSlot(id, s.field)] = text
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// KeepSeals is c with each sealed value replaced by the one kept at the same slot, where same says the two open to the
|
||||
// same value (known and same): a value the vault sealed again is then the box that runs, so the composition is the
|
||||
// one that runs (ADR 0547 §3, issue 1126). A slot kept holds nothing for, or whose value differs or cannot be opened,
|
||||
// keeps c's. Every other byte of a resource is kept as it is; c itself is not changed.
|
||||
func KeepSeals(c delivery.Composed, kept Seals, same func(a, b string) (same, known bool)) delivery.Composed {
|
||||
if len(kept) == 0 || same == nil {
|
||||
return c
|
||||
}
|
||||
out := c
|
||||
out.Resources = nil
|
||||
for i, raw := range c.Resources {
|
||||
id, fields, ok := sealedFieldsOf(raw, i)
|
||||
if ok {
|
||||
for _, s := range sealsAt(fields) {
|
||||
ours := sealedText(s.raw)
|
||||
theirs, held := kept[SealSlot(id, s.field)]
|
||||
if !held || ours == "" || theirs == ours {
|
||||
continue
|
||||
}
|
||||
if opens, known := same(ours, theirs); !known || !opens {
|
||||
continue
|
||||
}
|
||||
value := json.RawMessage(theirs)
|
||||
if s.raw[0] == '"' {
|
||||
value, _ = json.Marshal(theirs)
|
||||
}
|
||||
if s.entry == "" {
|
||||
if swapped, ok := withField(raw, s.field, value); ok {
|
||||
raw = swapped
|
||||
}
|
||||
continue
|
||||
}
|
||||
// An entry of the secrets: swapped inside the secrets as they stand in raw, then the secrets in raw.
|
||||
var now map[string]json.RawMessage
|
||||
if json.Unmarshal(raw, &now) != nil {
|
||||
continue
|
||||
}
|
||||
if secrets, ok := withField(now[SecretsField], s.entry, value); ok {
|
||||
if swapped, ok := withField(raw, SecretsField, secrets); ok {
|
||||
raw = swapped
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out.Resources = append(out.Resources, raw)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sealedFieldsOf is a resource's id (`#<i>` when it has none) and its top-level fields, false when it is no object.
|
||||
func sealedFieldsOf(raw json.RawMessage, i int) (string, map[string]json.RawMessage, bool) {
|
||||
var fields map[string]json.RawMessage
|
||||
if json.Unmarshal(raw, &fields) != nil {
|
||||
return "", nil, false
|
||||
}
|
||||
var id string
|
||||
if f, ok := fields["id"]; !ok || json.Unmarshal(f, &id) != nil || id == "" {
|
||||
id = "#" + strconv.Itoa(i)
|
||||
}
|
||||
return id, fields, true
|
||||
}
|
||||
|
||||
// sealedText is a sealed field as its text: a string as it is, a box as its JSON.
|
||||
func sealedText(f json.RawMessage) string {
|
||||
var text string
|
||||
if json.Unmarshal(f, &text) == nil {
|
||||
return text
|
||||
}
|
||||
var compact bytes.Buffer
|
||||
if json.Compact(&compact, f) != nil {
|
||||
return string(f)
|
||||
}
|
||||
return compact.String()
|
||||
}
|
||||
|
||||
// withField is the object raw with its top-level field replaced by value, every other byte as it was (so a composition
|
||||
// whose sealed values are those that run is the one that runs, byte for byte); false when raw is no object holding it.
|
||||
func withField(raw json.RawMessage, field string, value json.RawMessage) (json.RawMessage, bool) {
|
||||
d := json.NewDecoder(bytes.NewReader(raw))
|
||||
if t, err := d.Token(); err != nil || t != json.Delim('{') {
|
||||
return nil, false
|
||||
}
|
||||
for d.More() {
|
||||
t, err := d.Token()
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
key, _ := t.(string)
|
||||
before := d.InputOffset()
|
||||
var skip json.RawMessage
|
||||
if err := d.Decode(&skip); err != nil {
|
||||
return nil, false
|
||||
}
|
||||
if key != field {
|
||||
continue
|
||||
}
|
||||
after := d.InputOffset()
|
||||
// The span holds the colon, any space and the value: kept up to the colon, the value replaced.
|
||||
span := raw[before:after]
|
||||
colon := bytes.IndexByte(span, ':')
|
||||
if colon < 0 {
|
||||
return nil, false
|
||||
}
|
||||
out := make([]byte, 0, len(raw)-len(span)+colon+1+len(value))
|
||||
out = append(out, raw[:int(before)+colon+1]...)
|
||||
out = append(out, value...)
|
||||
out = append(out, raw[after:]...)
|
||||
return out, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
Reference in New Issue
Block a user