jschoubben
a9d96b30be
mesh/delivery merged: source.main moved to novox/mesh-catalog@85bbeb37e3e0
mesh/delivery-group group fix/1126-vault-own merged: every member is merged
mesh/check pass a9d96b30be8d on 69056948bfa8 | unit 1639 run, 4201 kept, 46 skipped | integration 1674 run, 4215 kept, 47 skipped | over bound: unit, …
A watch that failed partway kept what it had taken; begun again, the runtime does not hand a key deleted meanwhile, so a removed assignment read as standing and credentialRefusal could hand a consumer credential to a node no longer running its module. Each attempt now begins from nothing held, and a late delivery to an older attempt changes nothing. A 'not on the node' refusal is confirmed by a direct read first, since a watched mirror can be behind.
239 lines
12 KiB
Go
239 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
"git.novox.be/novox/mesh-sdk/go/seal"
|
|
)
|
|
|
|
// Issue 1126 (ADR 0547 §2, ADR 0550 §5): `own` answers a machine's node-engine the own secrets of its modules, sealed to
|
|
// the key that machine's engine stated, about that machine alone; another caller, another machine's secrets and a
|
|
// machine with no stated key get nothing.
|
|
func TestOwnAnswersAMachineItsModulesSecretsSealedToItsKey(t *testing.T) {
|
|
plans := assignments{"anchor.web": `{"node":"anchor","module":"web"}`, "laptop.web": `{"node":"laptop","module":"web"}`}
|
|
v := newVault(t, nil)
|
|
v.Askers = Declared{Read: plans.read, Store: v.Store}
|
|
anchorPub, anchorPriv := aSealingKey(t)
|
|
laptopPub, _ := aSealingKey(t)
|
|
for _, m := range []struct {
|
|
caller string
|
|
of MoveOf
|
|
value string
|
|
}{
|
|
{"node.anchor", MoveOf{Node: "anchor", Module: "web", Name: "password", Origin: "made", Generation: "g-1", SealingKey: anchorPub}, "anchor-web-pw"},
|
|
{"node.anchor", MoveOf{Node: "anchor", Module: "web", Name: "api-key", Origin: "accepted", Generation: "g-1", SealingKey: anchorPub}, "anchor-web-key"},
|
|
{"node.laptop", MoveOf{Node: "laptop", Module: "web", Name: "password", Origin: "made", Generation: "g-1", SealingKey: laptopPub}, "laptop-web-pw"},
|
|
} {
|
|
if _, err := moveIn(t, v, m.caller, m.of, m.value); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
got, err := v.OwnOfNode("node.anchor")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got.Node != "anchor" || got.Key != anchorPub || len(got.Own) != 2 {
|
|
t.Fatalf("anchor was answered %+v", got)
|
|
}
|
|
want := map[string][2]string{"api-key": {"accepted", "anchor-web-key"}, "password": {"made", "anchor-web-pw"}}
|
|
for _, s := range got.Own {
|
|
w, ok := want[s.Name]
|
|
if !ok || s.Module != "web" || s.Origin != w[0] || s.Generation < 1 {
|
|
t.Errorf("anchor's %s: %+v", s.Name, s)
|
|
continue
|
|
}
|
|
value, err := seal.Open(s.Sealed, anchorPriv)
|
|
if err != nil || string(value) != w[1] {
|
|
t.Errorf("anchor's %s does not open to its value with anchor's key: %q, %v", s.Name, value, err)
|
|
}
|
|
}
|
|
for caller, says := range map[string]string{
|
|
"controller": "is not a node-engine",
|
|
"anchor.web": "is not a node-engine",
|
|
"": "an unproven caller",
|
|
"node.unknown": "no secret named own/unknown yet",
|
|
} {
|
|
if _, err := v.OwnOfNode(caller); err == nil || !strings.Contains(err.Error(), says) {
|
|
t.Errorf("%q: %v, want a refusal saying %q", caller, err, says)
|
|
}
|
|
}
|
|
if laptop, err := v.OwnOfNode("node.laptop"); err != nil || len(laptop.Own) != 1 || laptop.Own[0].Name != "password" {
|
|
t.Errorf("laptop was answered %+v, %v; want its one secret alone", laptop, err)
|
|
}
|
|
|
|
// A pair credential: laptop's web consumes anchor's postgres. Each end is answered the one value sealed to its own key;
|
|
// a credential the vault holds none of is named missing; a machine at neither end is refused.
|
|
if _, err := moveIn(t, v, "node.laptop", MoveOf{Provider: "anchor", Provision: "postgres", Consumer: "laptop",
|
|
ConsumerModule: "web", Origin: "made", Generation: "g-1", SealingKey: laptopPub}, "pg-pair"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
pair := CredentialAsked{Provision: "postgres", Consumer: "laptop", ConsumerModule: "web", Provider: "anchor"}
|
|
asConsumer, asProvider := pair, pair
|
|
asConsumer.Role, asProvider.Role = "consumer", "provider"
|
|
none := CredentialAsked{Role: "consumer", Provision: "redis", Consumer: "laptop", ConsumerModule: "web", Provider: "anchor"}
|
|
laptop, err := v.OwnOfNode("node.laptop", asConsumer, none)
|
|
if err != nil || len(laptop.Credentials) != 1 || len(laptop.Missing) != 1 || laptop.Missing[0] != none {
|
|
t.Fatalf("laptop as consumer: %+v, %v", laptop, err)
|
|
}
|
|
if !strings.HasPrefix(laptop.Credentials[0].Name, "pair/anchor/postgres/laptop/web") {
|
|
t.Errorf("laptop's credential is kept as %s", laptop.Credentials[0].Name)
|
|
}
|
|
anchor, err := v.OwnOfNode("node.anchor", asProvider)
|
|
if err != nil || len(anchor.Credentials) != 1 {
|
|
t.Fatalf("anchor as provider: %+v, %v", anchor, err)
|
|
}
|
|
if value, err := seal.Open(anchor.Credentials[0].Sealed, anchorPriv); err != nil || string(value) != "pg-pair" {
|
|
t.Errorf("anchor's copy does not open to the value with its key: %q, %v", value, err)
|
|
}
|
|
for who, c := range map[string]CredentialAsked{"node.anchor": asConsumer, "node.laptop": asProvider} {
|
|
if _, err := v.OwnOfNode(who, c); err == nil || !strings.Contains(err.Error(), "is not that machine") {
|
|
t.Errorf("%s asking %+v: %v", who, c, err)
|
|
}
|
|
}
|
|
|
|
// Review of L2 (C1, C2): `own` fails closed. A provision's one shared value is handed only to a consumer the record
|
|
// of its holders names, never to a machine that names its provider; assignments that cannot be read, or a vault
|
|
// that reads none, refuse rather than hand.
|
|
if _, err := v.keep("provision/anchor/redis", "shared-redis", "test", "", OriginMade); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := v.Store.WriteShared(Shared{Name: "provision/anchor/redis", Provider: "anchor", Provision: "redis",
|
|
Consumers: []string{"anchor/web"}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got, err := v.OwnOfNode("node.laptop", none); err != nil || len(got.Credentials) != 0 || len(got.Missing) != 1 {
|
|
t.Errorf("laptop, whose web the shared value's record does not name, got %+v, %v", got, err)
|
|
}
|
|
anchorRedis := CredentialAsked{Role: "consumer", Provision: "redis", Consumer: "anchor", ConsumerModule: "web", Provider: "anchor"}
|
|
if got, err := v.OwnOfNode("node.anchor", anchorRedis); err != nil || len(got.Credentials) != 1 {
|
|
t.Errorf("anchor's web, a holder the record names, got %+v, %v", got, err)
|
|
}
|
|
v.Askers = Declared{Read: func(string) ([]byte, error) { return nil, errors.New("the bucket did not answer") }, Store: v.Store}
|
|
if _, err := v.OwnOfNode("node.anchor"); err == nil || !strings.Contains(err.Error(), "could not be read") {
|
|
t.Errorf("own with the assignments unread: %v", err)
|
|
}
|
|
v.Askers = nil
|
|
if _, err := v.OwnOfNode("node.anchor", anchorRedis); err == nil || !strings.Contains(err.Error(), "cannot read the assignments") {
|
|
t.Errorf("own with no reader of the assignments: %v", err)
|
|
}
|
|
}
|
|
|
|
// Issue 1126, 2026-10-28 (seed-9's probe: novox's 57 own secrets and 146 credentials took 26 s, past the node-engine's
|
|
// 15 s): one answer of `own` reads each module's assignment once, and records every hand-out in one synced write.
|
|
func TestOwnReadsEachAssignmentOnceAndSyncsItsRecordOnce(t *testing.T) {
|
|
plans := assignments{"anchor.web": `{"node":"anchor","module":"web"}`, "anchor.db": `{"node":"anchor","module":"db"}`}
|
|
reads := map[string]int{}
|
|
v := newVault(t, nil)
|
|
v.Askers = Declared{Read: func(key string) ([]byte, error) { reads[key]++; return plans.read(key) }, Store: v.Store}
|
|
pub, _ := aSealingKey(t)
|
|
for _, m := range []MoveOf{
|
|
{Node: "anchor", Module: "web", Name: "password", Origin: "made", Generation: "g-1", SealingKey: pub},
|
|
{Node: "anchor", Module: "web", Name: "api-key", Origin: "made", Generation: "g-1", SealingKey: pub},
|
|
{Node: "anchor", Module: "web", Name: "cookie", Origin: "made", Generation: "g-1", SealingKey: pub},
|
|
{Node: "anchor", Module: "db", Name: "root", Origin: "made", Generation: "g-1", SealingKey: pub},
|
|
{Provider: "anchor", Provision: "postgres", Consumer: "anchor", ConsumerModule: "web", Origin: "made", Generation: "g-1", SealingKey: pub},
|
|
{Provider: "anchor", Provision: "redis", Consumer: "anchor", ConsumerModule: "web", Origin: "made", Generation: "g-1", SealingKey: pub},
|
|
} {
|
|
if _, err := moveIn(t, v, "node.anchor", m, "v-"+m.Name+m.Provision); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for k := range reads {
|
|
delete(reads, k)
|
|
}
|
|
syncs, handed := v.Store.syncs, len(mustHanded(t, v))
|
|
got, err := v.OwnOfNode("node.anchor",
|
|
CredentialAsked{Role: "consumer", Provision: "postgres", Consumer: "anchor", ConsumerModule: "web", Provider: "anchor"},
|
|
CredentialAsked{Role: "consumer", Provision: "redis", Consumer: "anchor", ConsumerModule: "web", Provider: "anchor"})
|
|
if err != nil || len(got.Own) != 4 || len(got.Credentials) != 2 {
|
|
t.Fatalf("anchor was answered %+v, %v", got, err)
|
|
}
|
|
for key, n := range reads {
|
|
if n != 1 {
|
|
t.Errorf("the assignment %s was read %d times in one answer", key, n)
|
|
}
|
|
}
|
|
if len(reads) != 2 {
|
|
t.Errorf("the answer read %v, want each of anchor's two modules once", reads)
|
|
}
|
|
if n := v.Store.syncs - syncs; n != 1 {
|
|
t.Errorf("one answer synced the record %d times", n)
|
|
}
|
|
if n := len(mustHanded(t, v)) - handed; n != 6 {
|
|
t.Errorf("one answer recorded %d hand-outs, want 6", n)
|
|
}
|
|
// Review round 1, note (b): a consumer module the mirror does not hold yet (its watch behind) is read directly before
|
|
// the credential is refused.
|
|
v.Askers = Declared{Read: func(string) ([]byte, error) { return nil, nil }, Confirm: plans.read, Store: v.Store}
|
|
if got, err := v.OwnOfNode("node.anchor", CredentialAsked{Role: "consumer", Provision: "postgres", Consumer: "anchor",
|
|
ConsumerModule: "web", Provider: "anchor"}); err != nil || len(got.Credentials) != 1 {
|
|
t.Errorf("a module the mirror does not hold yet, read directly: %+v, %v", got, err)
|
|
}
|
|
v.Askers = Declared{Read: plans.read, Store: v.Store}
|
|
// A record that cannot be written hands nothing, the batch as one.
|
|
if err := os.Remove(filepath.Join(v.dir, "store", "handed.jsonl")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.Mkdir(filepath.Join(v.dir, "store", "handed.jsonl"), 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if a, err := v.OwnOfNode("node.anchor"); err == nil {
|
|
t.Errorf("handed with no record of it: %+v", a)
|
|
}
|
|
}
|
|
|
|
func mustHanded(t *testing.T, v testVault) []Handing {
|
|
t.Helper()
|
|
h, err := v.Store.Handed(1000)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return h
|
|
}
|
|
|
|
// Issue 1126, 2026-10-28: the vault keeps the assignments mirror as its watch hands it over, so `own` and `get` read
|
|
// no assignment over the bus once the watch has caught up; before that a read goes to the bus.
|
|
func TestTheWatchedAssignmentsAnswerWithoutTheBusOnceCaughtUp(t *testing.T) {
|
|
asked := 0
|
|
w := &WatchedAssignments{Fallback: func(key string) ([]byte, error) { asked++; return []byte(`{"from":"bus"}`), nil }}
|
|
if raw, err := w.Read("anchor.web"); err != nil || string(raw) != `{"from":"bus"}` || asked != 1 {
|
|
t.Errorf("before the watch caught up: %s, %v, %d asks", raw, err, asked)
|
|
}
|
|
// A first watch takes two values and fails before it caught up; a late value still arrives on it.
|
|
first := w.Begin()
|
|
for _, c := range []stdio.StateChange{
|
|
{Key: "anchor.web", Op: "put", Value: []byte(`{"node":"anchor","module":"web"}`), Current: true},
|
|
{Key: "anchor.db", Op: "put", Value: []byte(`{"node":"anchor","module":"db"}`), Current: true},
|
|
} {
|
|
if err := first.Take(c); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
// Review round 1: the watch begun again holds anchor.db no more (removed meanwhile); what the first took is gone.
|
|
second := w.Begin()
|
|
_ = second.Take(stdio.StateChange{Key: "anchor.web", Op: "put", Value: []byte(`{"node":"anchor","module":"web"}`), Current: true})
|
|
_ = first.Take(stdio.StateChange{Key: "anchor.db", Op: "put", Value: []byte(`{"node":"anchor","module":"db"}`)})
|
|
first.Caught()
|
|
if raw, _ := w.Read("anchor.db"); string(raw) != `{"from":"bus"}` {
|
|
t.Errorf("the failed watch's caught up counted: %s", raw)
|
|
}
|
|
second.Caught()
|
|
_ = first.Take(stdio.StateChange{Key: "anchor.db", Op: "put", Value: []byte(`{"node":"anchor","module":"db"}`)})
|
|
_ = second.Take(stdio.StateChange{Key: "anchor.web", Op: "put", Value: []byte(`{"node":"anchor","module":"web","v":2}`)})
|
|
asked = 0
|
|
for key, want := range map[string]string{"anchor.web": `{"node":"anchor","module":"web","v":2}`, "anchor.db": "", "laptop.x": ""} {
|
|
raw, err := w.Read(key)
|
|
if err != nil || string(raw) != want || (want == "") != (raw == nil) {
|
|
t.Errorf("%s read %q, %v; want %q", key, raw, err, want)
|
|
}
|
|
}
|
|
if asked != 0 {
|
|
t.Errorf("caught up, the mirror asked the bus %d times", asked)
|
|
}
|
|
}
|