Files
mesh-catalog/modules/mesh-provisioner/cmd/mesh-provisioner/holder_integration_test.go
T
jschoubben c9d8950fd0 Every module follows the assigner's new places: the contracts' go/ path, mesh-wants in the delivery contracts, the state and tool assignments (issue 1966)
The legacy key form placement.<node> is gone from the client, so the readers and their fixtures lose it. The check noplacement
now also refuses the assigner's retired names in a manifest.

Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
2026-11-05 22:29:07 +01:00

294 lines
9.8 KiB
Go

//go:build integration
package main
// The holder against a real bus (novox/hq ADR 0570 clauses 1-3, 5, as changed by ADR 0593 §8; issue 1954 H5): its
// states and every input it reads are a real nats-server's key-value buckets, read through the watched copy of those
// buckets (ADR 0580 clause 2); the plans come from n1's published compositions in `composed-n1` (ADR 0593 §6), as
// captured from the composition of mesh-sdk's frombustest mesh (restic on n1, requiring postgres-database from postgres
// there), and nothing is composed. An assignment, a want and a published composition make the key; the providers'
// outcomes (the vault's entry for the credential, postgres's own have) turn it provided; restic leaving the node makes
// it retiring and raises the ask.
import (
"context"
"encoding/json"
"errors"
assignerclient "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
meshseat "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
catalogclient "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/testbus"
)
// kvStore is the Store over a real bus: this module's states by their local names, another's as `<module>.<name>`.
type kvStore struct {
ctx context.Context
js jetstream.JetStream
}
func bucketOf(state string) string {
if state == ComposedRead { // the node-engine's named state: here n1's bucket, the one node that composes
return "composed-n1"
}
if owner, name, ok := strings.Cut(state, "."); ok {
return owner + "_" + name
}
return Module + "_" + state
}
func (s kvStore) kv(state string) (jetstream.KeyValue, error) {
kv, err := s.js.KeyValue(s.ctx, bucketOf(state))
if errors.Is(err, jetstream.ErrBucketNotFound) {
return s.js.CreateKeyValue(s.ctx, jetstream.KeyValueConfig{Bucket: bucketOf(state)})
}
return kv, err
}
func (s kvStore) Get(state, key string) (*Entry, error) {
kv, err := s.kv(state)
if err != nil {
return nil, err
}
e, err := kv.Get(s.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, nil
}
if err != nil {
return nil, err
}
return &Entry{Key: key, Value: e.Value(), Revision: e.Revision()}, nil
}
func (s kvStore) Put(state, key string, value any) error {
kv, err := s.kv(state)
if err != nil {
return err
}
raw, err := json.Marshal(value)
if err != nil {
return err
}
_, err = kv.Put(s.ctx, key, raw)
return err
}
func (s kvStore) Delete(state, key string) error {
kv, err := s.kv(state)
if err != nil {
return err
}
return kv.Delete(s.ctx, key)
}
func (s kvStore) Keys(state string) ([]string, error) {
kv, err := s.kv(state)
if err != nil {
return nil, err
}
keys, err := kv.Keys(s.ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
return keys, err
}
type asked struct{ ids []string }
func (a *asked) Publish(_ string, body any, id string) error {
a.ids = append(a.ids, id)
_ = body.(oc.Ask)
return nil
}
func (a *asked) Record(string) ([]byte, bool, error) { return nil, false, nil }
// 1954/H5.
func TestTheHolderPlansProvidesAndAsksOnTheBus(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
nc, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
s := kvStore{ctx: ctx, js: js}
var c today
readJSON(t, "testdata/today-composed.json", &c)
node := c.Node
// The bus as the holder reads it: the indexer's modules and manifests, n1's assignments and the vault's on n2, the
// want of restic on n1, and what n1's engine published.
mem := goldenMesh(t, c)
published(t, mem, c)
vaultManifest := `{"module":"mesh-vault","version":"1","state":["bound"]}` // the vault keeps its outcomes as entries
vaultDigest := catalogclient.ManifestDigest([]byte(vaultManifest))
vaultValue, _ := catalogclient.EncodeManifest(catalog.ManifestsByDigestValue{Module: vaultModule, Manifest: vaultManifest})
mem.write(t, ManifestsRead, mustKey(catalogclient.ManifestsKey(vaultDigest)), string(vaultValue))
mem.write(t, ModulesRead, vaultModule, `{"v":1,"none2":"the test's","manifest":"`+vaultDigest+`"}`)
for state, keys := range mem.states {
for key, raw := range keys {
if err := s.Put(state, key, json.RawMessage(raw)); err != nil {
t.Fatal(err)
}
}
}
// The watched copy over the real buckets: each state by one watch, a manifest by key.
ws := stdio.WatchedStatesOver(func(state string, handle func(stdio.StateChange) error) error {
kv, err := s.kv(state)
if err != nil {
return err
}
w, err := kv.WatchAll(ctx)
if err != nil {
return err
}
hand := func(e jetstream.KeyValueEntry) {
op := stdio.OpPut
if e.Operation() != jetstream.KeyValuePut {
op = stdio.OpDelete
}
_ = handle(stdio.StateChange{State: state, Key: e.Key(), Op: op, Value: e.Value(), Revision: e.Revision()})
}
for e := range w.Updates() {
if e == nil { // the current values are all handed over
break
}
hand(e)
}
go func() {
for e := range w.Updates() {
hand(e)
}
}()
return nil
}, func(state, key string) ([]byte, bool, error) {
e, err := s.Get(state, key)
if err != nil || e == nil {
return nil, false, err
}
return e.Value, true, nil
})
// settled waits until the copy holds a key as the bus does: in the mesh the holder is woken by that same watch.
settled := func(state, key string) {
t.Helper()
for deadline := time.Now().Add(5 * time.Second); ; time.Sleep(10 * time.Millisecond) {
e, _ := s.Get(state, key)
v, found, _ := ws.Get(state, key)
if (e == nil && !found) || (e != nil && found && string(e.Value) == string(v)) {
return
}
if time.Now().After(deadline) {
t.Fatalf("the copy of %s never took %s", state, key)
}
}
}
a := &asked{}
h := &Holder{Store: Watching(s, ws), Asks: a, Now: time.Now, Logf: t.Logf}
h.Compose = FromCompositions(h.Store)
state := func(key string) meshprovisionerclient.Provision {
e, _ := s.Get(ProvisionsState, key)
if e == nil {
return meshprovisionerclient.Provision{}
}
p, _ := meshprovisionerclient.Decode(e.Value)
return p
}
key := node + ".restic.postgres-database"
if _, err := h.ComposeNode(ctx, node); err != nil {
t.Fatal(err)
}
if p := state(key); p.State != meshprovisionerclient.Asked || p.Provider.Module != "postgres" {
t.Fatalf("the assignments and the want composed %+v", p)
}
if e, _ := s.Get(PlansState, node+".restic"); e == nil {
t.Fatal("restic's plan is not on the bus")
}
// The providers' outcomes: the vault provides the credential where restic has one, postgres runs healthy.
for _, dep := range state(key).DependsOn {
if err := s.Put(boundRead(vaultModule), dep, map[string]any{"state": "provided"}); err != nil {
t.Fatal(err)
}
}
if err := s.Put(HaveRead, node+".postgres", map[string]any{"v": 1, "fp": fpsOf["postgres"], "phase": "running",
"health": "ok"}); err != nil {
t.Fatal(err)
}
settled(HaveRead, node+".postgres")
// The running build's manifest, as the builder keyed it: one that does not offer the provision keeps the key asked
// (ADR 0603 clause 5), one that does lets the have count.
buildsKey := mustKey(builds.BuilderBuildKey("postgres", fpsOf["postgres"]))
offered := func(provides string) {
raw := `{"module":"postgres","version":"1","provides":` + provides + `}`
digest := catalogclient.ManifestDigest([]byte(raw))
value, _ := catalogclient.EncodeManifest(catalog.ManifestsByDigestValue{Module: "postgres", Manifest: raw})
if err := s.Put(ManifestsRead, mustKey(catalogclient.ManifestsKey(digest)), json.RawMessage(value)); err != nil {
t.Fatal(err)
}
build, err := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: digest})
if err != nil {
t.Fatal(err)
}
if err := s.Put(BuildsRead, buildsKey, json.RawMessage(build)); err != nil {
t.Fatal(err)
}
}
offered(`["other"]`)
for i := 0; i < 2; i++ {
if _, err := h.ComposeNode(ctx, node); err != nil {
t.Fatal(err)
}
}
if p := state(key); p.State != meshprovisionerclient.Asked {
t.Fatalf("a build not offering the provision turned it %s", p.State)
}
offered(`["postgres-database"]`)
for i := 0; i < 2; i++ { // the credential judged first, then what depends on it
if _, err := h.ComposeNode(ctx, node); err != nil {
t.Fatal(err)
}
}
if p := state(key); p.State != meshprovisionerclient.Provided {
t.Fatalf("the outcomes did not turn it provided: %+v", p)
}
// restic leaves the node and its want goes: retiring, one ask. Its compositions stay until its engine deletes them.
left := meshseat.Assignments{V: assignerclient.Version, Node: node, Modules: []meshseat.PlacedModule{{Module: "store"}, {Module: "postgres"},
{Module: "mesh-provisioner-client"}}}
values, err := assignerclient.AssignmentsValues(left)
if err != nil {
t.Fatal(err)
}
for k, raw := range values {
if err := s.Put(AssignmentsRead, k, json.RawMessage(raw)); err != nil {
t.Fatal(err)
}
}
if err := s.Delete(AssignmentsRead, node+".restic"); err != nil {
t.Fatal(err)
}
if err := s.Delete(WantsRead, "want."+node+".restic"); err != nil {
t.Fatal(err)
}
settled(AssignmentsRead, node+".node")
settled(WantsRead, "want."+node+".restic")
if _, err := h.ComposeNode(ctx, node); err != nil {
t.Fatal(err)
}
if r := state(key); r.State != meshprovisionerclient.Retiring || r.Ask == "" || len(a.ids) == 0 {
t.Fatalf("removing the want did not ask: %+v, asked %v", r, a.ids)
}
}