Prove the compositions through serve() whole, on a bus running the composed user list: the start, a done key, a real rotation (issue 1945; the review's G1)
The tests no longer call the writer on a bus without permissions: the controller runs as it serves, the builder and the indexer write their own keys as their own accounts, and the operator's rotate is the real verb, so a subject the user list does not grant fails them as it would fail the mesh.
This commit is contained in:
@@ -0,0 +1,374 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||
"github.com/novox/mesh-controller/internal/identity"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The compositions through the controller whole (novox/hq ADR 0539; the review's G1): the real store, the controller's
|
||||
// signing key, and a bus that runs the user list the controller composes from that store, so a subject release 6 uses
|
||||
// and the list does not grant is refused as it would be on the mesh. serve() runs whole on it; the builder and the
|
||||
// indexer are their own accounts writing their own keys; a rotation is the operator's verb.
|
||||
type servedMesh struct {
|
||||
open *stores
|
||||
controller jetstream.JetStream
|
||||
builder jetstream.JetStream
|
||||
indexer jetstream.JetStream
|
||||
}
|
||||
|
||||
// keyOfManifest is a done key of a module at fp with no artifacts, its manifest the given one, and the indexer's trunk
|
||||
// value naming it in the catalogue's repository at commit.
|
||||
func keyOfManifest(t *testing.T, m catalogue.Manifest, fp, commit string) (raw []byte, k sdkdelivery.BuilderBuild, trunk sdkdelivery.Module) {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := sdkdelivery.ManifestDigest(raw)
|
||||
return raw, sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, By: "anchor", Manifest: digest},
|
||||
sdkdelivery.Module{FP2: fp, Manifest: digest, Repository: "novox/mesh-catalog", Commit: commit, Path: "modules/" + m.Module}
|
||||
}
|
||||
|
||||
func aServedMesh(t *testing.T) *servedMesh {
|
||||
t.Helper()
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
withTrunkList(t, true)
|
||||
ident := identity.ForTest(t)
|
||||
if _, err := ident.Establish(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A provider and its consumer (bindings_test.go's store and board), registered from done keys of the catalogue's
|
||||
// repository, so each has a fingerprint of the new form; the store holds the mesh's seat on anchor, the board on
|
||||
// laptop is bound to it.
|
||||
shelf := map[string]catalogue.Manifest{}
|
||||
for _, m := range storeManifests() {
|
||||
shelf[m.Module] = m
|
||||
}
|
||||
for i, module := range []string{"store", "board"} {
|
||||
raw, k, trunk := keyOfManifest(t, shelf[module], fpOf(byte('a'+i)), "c0ffee0123456789")
|
||||
keys := trunkKeys{keyBuckets: keyBuckets{keys: map[string]builderkeys.Key{module: aBuilderKey(module, trunk.FP2, k)},
|
||||
manifests: map[string]string{trunk.Manifest: string(raw)}}, trunk: trunk}
|
||||
if _, _, err := takeInKey(ctx, inv, keys, keyTake{Module: module, Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}); err != nil {
|
||||
t.Fatalf("%s is not registered from its key: %v", module, err)
|
||||
}
|
||||
}
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.HoldSeat(ctx, "mesh-store", catalogue.ScopeMesh, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "laptop", "board"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The bus, running the user list the controller composes from the store, with the builder and the indexer as the
|
||||
// accounts their own manifests make (each writes its own state).
|
||||
records, err := inv.BusRecords(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var controller broker.Principal
|
||||
for _, p := range users {
|
||||
if p.Kind == broker.KindController {
|
||||
controller = p
|
||||
}
|
||||
}
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: controller.Username(), Kind: inventory.BusController})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
builder := broker.Principal{Kind: broker.KindModule, Node: "anchor", Module: sdkdelivery.BuilderModule, State: []string{"builds"}}
|
||||
indexer := broker.Principal{Kind: broker.KindModule, Node: "anchor", Module: sdkdelivery.IndexerModule, State: []string{"modules", "manifests"}}
|
||||
kept, err := inv.BusUsers(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hashes := map[string]string{}
|
||||
for name, u := range kept {
|
||||
hashes[name] = u.PasswordHash
|
||||
}
|
||||
fixture, _ := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
|
||||
hashes[builder.Username()], hashes[indexer.Username()] = string(fixture), string(fixture)
|
||||
filled, missing := broker.WithPasswords([]broker.Principal{controller, builder, indexer}, hashes)
|
||||
if len(missing) > 0 {
|
||||
t.Fatalf("no password for %v", missing)
|
||||
}
|
||||
accounts, err := broker.ComposeAccounts(filled)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conf := filepath.Join(t.TempDir(), "bus.conf")
|
||||
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts, err := server.ProcessConfigFile(conf)
|
||||
if err != nil {
|
||||
t.Fatalf("the composed user list does not parse: %v", err)
|
||||
}
|
||||
opts.NoLog, opts.NoSigs = true, true
|
||||
s, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(s.Shutdown)
|
||||
u, _ := url.Parse(s.ClientURL())
|
||||
u.User = url.UserPassword(controller.Username(), password)
|
||||
t.Setenv(broker.NATSVar, u.String())
|
||||
|
||||
connect := func(user, pw string) jetstream.JetStream {
|
||||
t.Helper()
|
||||
nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, pw), nats.CustomInboxPrefix("_INBOX."+user))
|
||||
if err != nil {
|
||||
t.Fatalf("%s cannot connect: %v", user, err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return js
|
||||
}
|
||||
m := &servedMesh{open: open, controller: connect(controller.Username(), password),
|
||||
builder: connect(builder.Username(), "pw"), indexer: connect(indexer.Username(), "pw")}
|
||||
|
||||
// The builder's and the indexer's buckets as their own parts make them, there before the controller starts (its
|
||||
// watch of their keys begins at its start); and the plans' buckets a controller of release 5 left.
|
||||
for _, b := range []string{sdkdelivery.BuilderBuildsBucket, sdkdelivery.ModulesBucket, sdkdelivery.ManifestsBucket} {
|
||||
if _, err := m.controller.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5}); err != nil {
|
||||
t.Fatalf("the bucket %s: %v", b, err)
|
||||
}
|
||||
}
|
||||
for _, b := range broker.PlanStateBuckets {
|
||||
if _, err := m.controller.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
serving, stop := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- serve(serving) }()
|
||||
t.Cleanup(func() {
|
||||
stop()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Log("the controller did not stop within 30 s")
|
||||
}
|
||||
})
|
||||
// Served once both nodes have their own composition: the start's pass ran.
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
own, _ := sdkdelivery.ComposedNodeKey(node)
|
||||
m.waitFor(t, done, node, own, 0)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// aBuilderKey is the builder's done key of module at fp, as the take-in reads it.
|
||||
func aBuilderKey(module, fp string, b sdkdelivery.BuilderBuild) builderkeys.Key {
|
||||
return builderkeys.Key{Module: module, Fingerprint: fp, Found: true, Revision: 1, Build: b}
|
||||
}
|
||||
|
||||
// waitFor waits until the node's composition bucket holds key at a revision above after, failing when the controller
|
||||
// stops or a minute passes: a test's wait for what an event made, never a loop of the code under test.
|
||||
func (m *servedMesh) waitFor(t *testing.T, done chan error, node, key string, after uint64) uint64 {
|
||||
t.Helper()
|
||||
bucket, _ := sdkdelivery.ComposedBucketFor(node)
|
||||
deadline := time.Now().Add(time.Minute)
|
||||
for time.Now().Before(deadline) {
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("the controller stopped: %v", err)
|
||||
default:
|
||||
}
|
||||
if kv, err := m.controller.KeyValue(t.Context(), bucket); err == nil {
|
||||
if e, err := kv.Get(t.Context(), key); err == nil && e.Revision() > after {
|
||||
return e.Revision()
|
||||
}
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("%s in %s was not written within a minute", key, bucket)
|
||||
return 0
|
||||
}
|
||||
|
||||
// revisions are a node's composition keys and their revisions.
|
||||
func (m *servedMesh) revisions(t *testing.T, node string) map[string]uint64 {
|
||||
t.Helper()
|
||||
bucket, _ := sdkdelivery.ComposedBucketFor(node)
|
||||
kv, err := m.controller.KeyValue(t.Context(), bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lister, err := kv.ListKeys(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := map[string]uint64{}
|
||||
for k := range lister.Keys() {
|
||||
if e, err := kv.Get(t.Context(), k); err == nil {
|
||||
out[k] = e.Revision()
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// declarationsSent is how many declarations the bus's stream of them holds: none is ever sent in release 6.
|
||||
func (m *servedMesh) declarationsSent(t *testing.T) uint64 {
|
||||
t.Helper()
|
||||
s, err := m.controller.Stream(t.Context(), broker.DeclarationsStream)
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return 0
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := s.Info(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return info.State.Msgs
|
||||
}
|
||||
|
||||
// 1945/AC6, AC14: started whole on a bus running its own user list, the controller composes every node and every
|
||||
// assigned module, deletes the plans' buckets and says each as bucket-deleted, and sends no declaration.
|
||||
func TestServedTheStartComposesEveryNodeAndDeletesThePlansBuckets(t *testing.T) {
|
||||
m := aServedMesh(t)
|
||||
for _, k := range []struct{ node, module string }{{"anchor", "store"}, {"laptop", "board"}} {
|
||||
fp := map[string]string{"store": fpOf('a'), "board": fpOf('b')}[k.module]
|
||||
key, _ := sdkdelivery.ComposedKey(k.node, k.module, fp)
|
||||
m.waitFor(t, nil, k.node, key, 0)
|
||||
}
|
||||
for _, b := range broker.PlanStateBuckets {
|
||||
if _, err := m.controller.KeyValue(t.Context(), b); !errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
t.Errorf("%s is still there: %v", b, err)
|
||||
}
|
||||
}
|
||||
if n := m.declarationsSent(t); n != 0 {
|
||||
t.Errorf("%d declarations were sent", n)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC3: the builder writes the done key of a new fingerprint of an assigned module and the indexer its trunk
|
||||
// value: the controller, hearing the keys, registers it, writes its composition on the node it is assigned to and
|
||||
// `grants.<module>.<fp>`; nothing is sent.
|
||||
func TestServedADoneKeyIsComposedAndGranted(t *testing.T) {
|
||||
m := aServedMesh(t)
|
||||
ctx := t.Context()
|
||||
shelf := map[string]catalogue.Manifest{}
|
||||
for _, mf := range storeManifests() {
|
||||
shelf[mf.Module] = mf
|
||||
}
|
||||
next := shelf["board"]
|
||||
next.Version = "2"
|
||||
fp := fpOf('n')
|
||||
raw, k, trunk := keyOfManifest(t, next, fp, "d00d0123456789ab")
|
||||
manifests, err := m.indexer.KeyValue(ctx, sdkdelivery.ManifestsBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, _ := sdkdelivery.EncodeManifest(sdkdelivery.Manifest{Module: "board", JSON: string(raw)})
|
||||
if _, err := manifests.Put(ctx, trunk.Manifest, value); err != nil {
|
||||
t.Fatalf("the indexer cannot write its manifest: %v", err)
|
||||
}
|
||||
builds, err := m.builder.KeyValue(ctx, sdkdelivery.BuilderBuildsBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := sdkdelivery.BuilderBuildKey("board", fp)
|
||||
value, _ = sdkdelivery.EncodeBuilderBuild(k)
|
||||
if _, err := builds.Put(ctx, key, value); err != nil {
|
||||
t.Fatalf("the builder cannot write its key: %v", err)
|
||||
}
|
||||
modules, err := m.indexer.KeyValue(ctx, sdkdelivery.ModulesBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mk, _ := sdkdelivery.ModulesKey("board")
|
||||
value, _ = sdkdelivery.EncodeModule(trunk)
|
||||
if _, err := modules.Put(ctx, mk, value); err != nil {
|
||||
t.Fatalf("the indexer cannot write its trunk value: %v", err)
|
||||
}
|
||||
composed, _ := sdkdelivery.ComposedKey("laptop", "board", fp)
|
||||
m.waitFor(t, nil, "laptop", composed, 0)
|
||||
grants, err := m.controller.KeyValue(ctx, broker.GrantsBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gk, _ := sdkdelivery.ControllerGrantKey("board", fp)
|
||||
if _, err := grants.Get(ctx, gk); err != nil {
|
||||
t.Errorf("grants.%s is not written: %v", gk, err)
|
||||
}
|
||||
if n := m.declarationsSent(t); n != 0 {
|
||||
t.Errorf("%d declarations were sent", n)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC2: the operator rotates the store's credential, the real verb: the compositions that carry it (the board's on
|
||||
// laptop, the store's grants on anchor) get a new revision, every other composition keeps its own, and nothing is sent.
|
||||
func TestServedARotationRewritesOnlyTheCompositionsThatUseIt(t *testing.T) {
|
||||
m := aServedMesh(t)
|
||||
before := map[string]map[string]uint64{"anchor": m.revisions(t, "anchor"), "laptop": m.revisions(t, "laptop")}
|
||||
if err := rotateCommand(t.Context(), []string{"postgres-database"}); err != nil {
|
||||
t.Fatalf("rotate: %v", err)
|
||||
}
|
||||
boardKey, _ := sdkdelivery.ComposedKey("laptop", "board", fpOf('b'))
|
||||
m.waitFor(t, nil, "laptop", boardKey, before["laptop"][boardKey])
|
||||
moved := map[string]bool{}
|
||||
for node, was := range before {
|
||||
for k, rev := range m.revisions(t, node) {
|
||||
if rev != was[k] {
|
||||
moved[k] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
storeKey, _ := sdkdelivery.ComposedKey("anchor", "store", fpOf('a'))
|
||||
for k := range moved {
|
||||
if k != boardKey && k != storeKey {
|
||||
t.Errorf("%s moved: only the compositions carrying the credential do", k)
|
||||
}
|
||||
}
|
||||
if !moved[boardKey] {
|
||||
t.Errorf("the board's composition did not move: %v", moved)
|
||||
}
|
||||
if n := m.declarationsSent(t); n != 0 {
|
||||
t.Errorf("%d declarations were sent", n)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user