The view read the gate's state source and its history, fixed to the branch main. Each watched branch's source is now the seat mesh-delivery's verb source given that branch, asked again on its source-moved; the source row and the event delivered are per branch, a repository's reach is from the source of its modules' registration's first branch, laid over the reach the view's own state kept at its start, and view_tiers given no revision answers per branch. The manifest drops its read of mesh-delivery-gate.source and names the source answer's fields it relies on. Builds: design 47 G §507, §512, §532, §558, A-173, A-225 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
1001 lines
41 KiB
Go
1001 lines
41 KiB
Go
//go:build integration
|
|
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"io"
|
|
"maps"
|
|
"mesh-delivery-view/internal/view"
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
|
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
|
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
|
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
|
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient"
|
|
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
|
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate"
|
|
"git.novox.be/novox/mesh-sdk/go/process"
|
|
"git.novox.be/novox/mesh-sdk/go/views"
|
|
"git.novox.be/novox/mesh-sdk/go/views/viewstest"
|
|
"github.com/nats-io/nats-server/v2/server"
|
|
"github.com/nats-io/nats.go"
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
)
|
|
|
|
// serveOwners serves the six states of the indexer and the assigner the view's process follows by their first views
|
|
// (ADR 0615 §1, DLV-M4): each owner's read tool and views family, with the view's durable consumer (`n_mesh-delivery-view`).
|
|
// A bucket the test made is kept; the others are made empty.
|
|
func serveOwners(t *testing.T, ctx context.Context, js jetstream.JetStream, nc *nats.Conn) {
|
|
t.Helper()
|
|
for _, o := range ownersFollowed {
|
|
viewstest.Serve(t, ctx, js, nc, o.owner, o.state, o.tool, "n_"+viewstate.ViewModule)
|
|
}
|
|
}
|
|
|
|
// catalogHolder is the seat mesh-catalog's holder as the view's process needs it (W16.1): its verb revision answering the
|
|
// trunk's newest revision whole, and its event catalog-revision said on the shared events stream at each change, which the
|
|
// view's durable consumer hears.
|
|
type catalogHolder struct {
|
|
t testing.TB
|
|
js jetstream.JetStream
|
|
mu sync.Mutex
|
|
// rev is the revision's number; entries are its entries by module.
|
|
rev int64
|
|
entries map[string]meshcatalog.CatalogEntry
|
|
// src is the gate's source revision of catalogBranch, as its verb source answers it.
|
|
src meshdelivery.SourceAnswer
|
|
ctx context.Context
|
|
}
|
|
|
|
func serveCatalog(t testing.TB, ctx context.Context, js jetstream.JetStream, nc *nats.Conn) *catalogHolder {
|
|
t.Helper()
|
|
h := &catalogHolder{t: t, js: js, ctx: ctx, rev: 1, entries: map[string]meshcatalog.CatalogEntry{}}
|
|
events, err := js.Stream(ctx, process.EventsStream)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
info := events.CachedInfo().Config
|
|
if !slices.Contains(info.Subjects, "mesh.seat.*.event.>") {
|
|
info.Subjects = append(info.Subjects, "mesh.seat.*.event.>")
|
|
if _, err := js.UpdateStream(ctx, info); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
answer := func(m *nats.Msg) {
|
|
h.mu.Lock()
|
|
a := meshcatalog.RevisionAnswer{Revision: &meshcatalog.CatalogRevision{Revision: h.rev, Kind: meshcatalogclient.KindTrunk}}
|
|
for _, e := range h.entries {
|
|
a.Entries = append(a.Entries, e)
|
|
}
|
|
h.mu.Unlock()
|
|
raw, _ := json.Marshal(map[string]any{"result": a, "node": "n0"})
|
|
_ = m.Respond(raw)
|
|
}
|
|
// The branch follower's other two first views (design 47 G §614): the registrations, each module on catalogBranch, and the
|
|
// gate's source, which keeps catalogBranch.
|
|
registrations := func(m *nats.Msg) {
|
|
h.mu.Lock()
|
|
a := meshcatalog.RegistrationsAnswer{Registrations: []meshcatalog.Registration{}}
|
|
for module := range h.entries {
|
|
a.Registrations = append(a.Registrations, meshcatalog.Registration{Module: module, Version: 1, Branches: []string{catalogBranch}})
|
|
}
|
|
h.mu.Unlock()
|
|
raw, _ := json.Marshal(map[string]any{"result": a, "node": "n0"})
|
|
_ = m.Respond(raw)
|
|
}
|
|
// The gate's verb source: given no branch the branches it keeps, given catalogBranch its source revision (A-173).
|
|
source := func(m *nats.Msg) {
|
|
var in meshdelivery.SourceInput
|
|
_ = json.Unmarshal(m.Data, &in)
|
|
var a any = map[string]any{"branches": []string{catalogBranch}}
|
|
if in.Branch != "" {
|
|
h.mu.Lock()
|
|
a = meshdelivery.SourceAnswer{Branch: in.Branch, Revision: h.src.Revision, Group: h.src.Group, Repositories: h.src.Repositories}
|
|
h.mu.Unlock()
|
|
}
|
|
raw, _ := json.Marshal(map[string]any{"result": a, "node": "n0"})
|
|
_ = m.Respond(raw)
|
|
}
|
|
for subject, handle := range map[string]nats.MsgHandler{"mesh.seat.mesh-catalog.call.revision": answer,
|
|
"mesh.seat.mesh-catalog.call.registrations": registrations, "mesh.seat.mesh-delivery.call.source": source} {
|
|
for _, s := range []string{subject + ".>", subject} {
|
|
sub, err := nc.Subscribe(s, handle)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
|
}
|
|
}
|
|
return h
|
|
}
|
|
|
|
// catalogBranch is the one watched branch of these tests: no branch is named in the view's code (design 47 G §532).
|
|
const catalogBranch = "release"
|
|
|
|
// moveSource is the gate's next source revision of catalogBranch, said as source-moved (design 47 G §507, A-173).
|
|
func (h *catalogHolder) moveSource(rev int64, group string, heads map[string]string) {
|
|
h.t.Helper()
|
|
h.mu.Lock()
|
|
h.src = meshdelivery.SourceAnswer{Revision: rev, Group: group}
|
|
for repository, head := range heads {
|
|
h.src.Repositories = append(h.src.Repositories, meshdelivery.SourceHead{Repository: repository, Head: head, Trunk: catalogBranch})
|
|
}
|
|
h.mu.Unlock()
|
|
body, _ := json.Marshal(meshdelivery.SourceMovedEvent{Branch: catalogBranch, Revision: rev, Group: group})
|
|
if _, err := h.js.Publish(h.ctx, meshdelivery.SubjectSourceMoved(catalogBranch), body); err != nil {
|
|
h.t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// put is the holder's next trunk revision with the entry of one module changed, said as catalog-revision.
|
|
func (h *catalogHolder) put(e meshcatalog.CatalogEntry) {
|
|
h.t.Helper()
|
|
h.mu.Lock()
|
|
h.rev++
|
|
h.entries[e.Module] = e
|
|
rev := h.rev
|
|
h.mu.Unlock()
|
|
body, _ := json.Marshal(meshcatalog.CatalogRevisionEvent{Kind: meshcatalogclient.KindTrunk, Branch: catalogBranch, Revision: rev, Changed: []string{e.Module}})
|
|
if _, err := h.js.Publish(h.ctx, "mesh.seat.mesh-catalog.event."+meshcatalog.EventCatalogRevision+"."+catalogBranch, body); err != nil {
|
|
h.t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// TestProcessWholeSeesTheBuckets starts the program the way the node does (its entry, run, with its environment and a
|
|
// broker file) against an embedded bus holding the three buckets, and asks its socket.
|
|
func TestProcessWholeSeesTheBuckets(t *testing.T) { // 1134/AC8
|
|
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
go srv.Start()
|
|
if !srv.ReadyForConnections(10 * time.Second) {
|
|
t.Fatal("the bus did not come up")
|
|
}
|
|
t.Cleanup(srv.Shutdown)
|
|
|
|
nc, err := nats.Connect(srv.ClientURL())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(nc.Close)
|
|
js, err := jetstream.New(nc)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
defer cancel()
|
|
kvs := map[string]jetstream.KeyValue{}
|
|
for _, b := range []string{view.HaveBucket, view.AssignmentsBucket, view.PresenceBucket, view.StuckBucket} {
|
|
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
kvs[b] = kv
|
|
}
|
|
put := func(bucket, key string, value []byte) {
|
|
t.Helper()
|
|
if _, err := kvs[bucket].Put(ctx, key, value); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
haveValue := func(build string, phase string, health string) []byte {
|
|
b, err := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Build: build, Phase: phase, Health: health})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return b
|
|
}
|
|
// The seat's assignments (issue 774, ADR 0549 §3): x and y on a, nothing on b.
|
|
place := func(node string, modules ...string) {
|
|
p := meshmoduleassigner.Assignments{Node: node}
|
|
for _, m := range modules {
|
|
p.Modules = append(p.Modules, meshmoduleassigner.PlacedModule{Module: m})
|
|
}
|
|
values, err := meshmoduleassignerclient.AssignmentsValues(p)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, k := range slices.Sorted(maps.Keys(values)) { // a key per module, then the node key
|
|
put(view.AssignmentsBucket, k, values[k])
|
|
}
|
|
}
|
|
place("a", "x", "y")
|
|
place("b")
|
|
put(view.HaveBucket, "a.x", haveValue("b1", nodeengineclient.PhaseChecking, nodeengineclient.HealthWaits))
|
|
put(view.HaveBucket, "b.x", haveValue("b1", nodeengineclient.PhaseRunning, nodeengineclient.HealthOK))
|
|
|
|
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { os.RemoveAll(dir) })
|
|
broker := filepath.Join(dir, "broker")
|
|
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
|
|
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
socket := filepath.Join(dir, "run", "cli.sock")
|
|
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"}
|
|
runCtx, stop := context.WithCancel(ctx)
|
|
done := make(chan error, 1)
|
|
serveOwners(t, ctx, js, nc)
|
|
serveCatalog(t, ctx, js, nc)
|
|
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
|
t.Cleanup(func() { stop(); <-done })
|
|
|
|
rowsMatch := func(check func(view.Snapshot) bool) view.Snapshot {
|
|
t.Helper()
|
|
var last view.Snapshot
|
|
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
|
if s, err := view.Ask(socket); err == nil {
|
|
last = s
|
|
if check(s) {
|
|
return s
|
|
}
|
|
}
|
|
}
|
|
t.Fatalf("the socket never showed the expected rows; last: %+v", last)
|
|
return last
|
|
}
|
|
byKey := func(s view.Snapshot, node, module string) view.Row {
|
|
for _, r := range s.Rows {
|
|
if r.Node == node && r.Module == module {
|
|
return r
|
|
}
|
|
}
|
|
return view.Row{}
|
|
}
|
|
s := rowsMatch(func(s view.Snapshot) bool {
|
|
return byKey(s, "a", "x").Health == "waits" && byKey(s, "a", "y").Waits == "no have yet" && byKey(s, "b", "x").Status == view.StatusLeftBehind
|
|
})
|
|
if s.Counted() != 2 {
|
|
t.Fatalf("counted = %d, want 2 (a.x, a.y): %+v", s.Counted(), s.Rows)
|
|
}
|
|
|
|
put(view.HaveBucket, "a.x", haveValue("b1", nodeengineclient.PhaseRunning, nodeengineclient.HealthOK))
|
|
rowsMatch(func(s view.Snapshot) bool { r := byKey(s, "a", "x"); return r.Phase == "running" && r.Health == "ok" })
|
|
}
|
|
|
|
// TestProcessWholeShowsWhatTheWatchmanSays starts the program whole and writes an offline presence and a stuck key to the
|
|
// watchman's buckets, sees both shown, then writes them online and moved and sees both gone.
|
|
func TestProcessWholeShowsWhatTheWatchmanSays(t *testing.T) { // 1168/AC5
|
|
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
go srv.Start()
|
|
if !srv.ReadyForConnections(10 * time.Second) {
|
|
t.Fatal("the bus did not come up")
|
|
}
|
|
t.Cleanup(srv.Shutdown)
|
|
nc, err := nats.Connect(srv.ClientURL())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(nc.Close)
|
|
js, err := jetstream.New(nc)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
defer cancel()
|
|
kvs := map[string]jetstream.KeyValue{}
|
|
for _, b := range []string{view.HaveBucket, view.AssignmentsBucket, view.PresenceBucket, view.StuckBucket} {
|
|
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
kvs[b] = kv
|
|
}
|
|
presence := func(key string, online bool) {
|
|
t.Helper()
|
|
b, _ := meshwatchdogsclient.EncodeNodePresence(meshwatchdogsclient.NodePresence{Online: online})
|
|
if _, err := kvs[view.PresenceBucket].Put(ctx, key, b); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
stuck := func(moved bool) {
|
|
t.Helper()
|
|
b, _ := meshwatchdogsclient.EncodeStuck(meshwatchdogsclient.Stuck{Bucket: view.HaveBucket, Key: "a.x", Waited: "a report from a", Since: time.Now(), Moved: moved})
|
|
if _, err := kvs[view.StuckBucket].Put(ctx, view.HaveBucket+".a.x", b); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
presence("a", false)
|
|
presence("part.mesh-delivery-view.i1", false)
|
|
stuck(false)
|
|
|
|
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { os.RemoveAll(dir) })
|
|
broker := filepath.Join(dir, "broker")
|
|
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
|
|
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
socket := filepath.Join(dir, "run", "cli.sock")
|
|
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"}
|
|
runCtx, stop := context.WithCancel(ctx)
|
|
done := make(chan error, 1)
|
|
serveOwners(t, ctx, js, nc)
|
|
serveCatalog(t, ctx, js, nc)
|
|
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
|
t.Cleanup(func() { stop(); <-done })
|
|
|
|
wait := func(check func(view.Snapshot) bool) {
|
|
t.Helper()
|
|
var last view.Snapshot
|
|
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
|
if s, err := view.Ask(socket); err == nil {
|
|
last = s
|
|
if check(s) {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
t.Fatalf("the socket never showed the expected lists; last: %+v", last)
|
|
}
|
|
wait(func(s view.Snapshot) bool { return len(s.Offline) == 2 && len(s.Stuck) == 1 && s.Stuck[0].Key == "a.x" })
|
|
presence("a", true)
|
|
presence("part.mesh-delivery-view.i1", true)
|
|
stuck(true)
|
|
wait(func(s view.Snapshot) bool { return len(s.Offline) == 0 && len(s.Stuck) == 0 })
|
|
}
|
|
|
|
// TestProcessWholeShowsTheIndexersBucketsOnceTheyExist starts the program whole before the indexer's buckets exist, then
|
|
// creates them and puts keys: the view's absent-bucket retry (follow, unchanged) finds them and the snapshot holds them.
|
|
// The retry is the view's own, every 5 s, so the wait here is 15 s, not 5.
|
|
func TestProcessWholeShowsTheIndexersBucketsOnceTheyExist(t *testing.T) { // 1241/AC5
|
|
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
go srv.Start()
|
|
if !srv.ReadyForConnections(10 * time.Second) {
|
|
t.Fatal("the bus did not come up")
|
|
}
|
|
t.Cleanup(srv.Shutdown)
|
|
nc, err := nats.Connect(srv.ClientURL())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(nc.Close)
|
|
js, err := jetstream.New(nc)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
defer cancel()
|
|
for _, b := range []string{view.HaveBucket, view.AssignmentsBucket, view.PresenceBucket, view.StuckBucket} {
|
|
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { os.RemoveAll(dir) })
|
|
broker := filepath.Join(dir, "broker")
|
|
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
|
|
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
socket := filepath.Join(dir, "run", "cli.sock")
|
|
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"}
|
|
runCtx, stop := context.WithCancel(ctx)
|
|
done := make(chan error, 1)
|
|
serveOwners(t, ctx, js, nc)
|
|
catalog := serveCatalog(t, ctx, js, nc)
|
|
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
|
t.Cleanup(func() { stop(); <-done })
|
|
|
|
// The view is up and the indexer's buckets are not there.
|
|
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
|
if _, err := view.Ask(socket); err == nil {
|
|
break
|
|
}
|
|
}
|
|
cands, err := js.KeyValue(ctx, view.CandidatesBucket)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
const head = "0123456789abcdef0123456789abcdef01234567"
|
|
hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: head}})
|
|
cand, err := json.Marshal(meshcatalog.CandidatesByGroupValue{Group: "feat/x", Heads: hash, Pulls: []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-delivery", Number: 12, Head: head, Modules: []string{"a"}}}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := cands.Put(ctx, "feat/x", cand); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
catalog.put(meshcatalog.CatalogEntry{Module: "a", None: "no tree", Repository: "novox/mesh-delivery", Commit: head, Manifest: map[string]any{"module": "a"}})
|
|
var last view.Snapshot
|
|
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
|
if s, err := view.Ask(socket); err == nil {
|
|
last = s
|
|
if len(s.Groups) == 1 && len(s.Fingerprints) == 1 {
|
|
if s.Groups[0].Group != "feat/x" || s.Fingerprints[0].Module != "a" || s.Fingerprints[0].State != view.FPNone {
|
|
t.Fatalf("snapshot = %+v", s)
|
|
}
|
|
return
|
|
}
|
|
}
|
|
}
|
|
t.Fatalf("the snapshot never held the indexer's keys; last: %+v", last)
|
|
}
|
|
|
|
// TestProcessWholeListsEveryDifference starts the program whole before the assigner's differences bucket exists, then
|
|
// creates it and writes a difference explained by a rule and one unexplained: the socket's snapshot lists both with their
|
|
// rule, and a difference taken away leaves the list (1297/B8: the view lists every difference with its rule).
|
|
func TestProcessWholeListsEveryDifference(t *testing.T) { // 1297/B8 AC
|
|
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
go srv.Start()
|
|
if !srv.ReadyForConnections(10 * time.Second) {
|
|
t.Fatal("the bus did not come up")
|
|
}
|
|
t.Cleanup(srv.Shutdown)
|
|
nc, err := nats.Connect(srv.ClientURL())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(nc.Close)
|
|
js, err := jetstream.New(nc)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
defer cancel()
|
|
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { os.RemoveAll(dir) })
|
|
broker := filepath.Join(dir, "broker")
|
|
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
|
|
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
socket := filepath.Join(dir, "run", "cli.sock")
|
|
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"}
|
|
runCtx, stop := context.WithCancel(ctx)
|
|
done := make(chan error, 1)
|
|
serveOwners(t, ctx, js, nc)
|
|
serveCatalog(t, ctx, js, nc)
|
|
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
|
t.Cleanup(func() { stop(); <-done })
|
|
|
|
kv, err := js.KeyValue(ctx, view.AssignerWantsBucket)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
const fpOld = "src2:1111111111111111111111111111111111111111111111111111111111111111"
|
|
const fpNew = "src2:2222222222222222222222222222222222222222222222222222222222222222"
|
|
for key, d := range map[string]meshwantsclient.Want{
|
|
"want.ace.plex": {Fp: fpNew, Composed: "sha256:" + strings.Repeat("c", 64)},
|
|
"want.g14.plex": {Fp: fpOld, Composed: "sha256:" + strings.Repeat("c", 64), Newest: fpNew, Rule: meshwantsclient.RuleCanary, Words: "the canary ace does not run it healthy yet"},
|
|
} {
|
|
v, err := meshwantsclient.EncodeWant(d)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := kv.Put(ctx, key, v); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
wait := func(check func(view.Snapshot) bool, what string) {
|
|
t.Helper()
|
|
var last view.Snapshot
|
|
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
|
if s, err := view.Ask(socket); err == nil {
|
|
last = s
|
|
if check(s) {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
t.Fatalf("%s; last: %+v", what, last)
|
|
}
|
|
wait(func(s view.Snapshot) bool {
|
|
d := s.Wants
|
|
return len(d) == 2 && d[0].Node == "ace" && d[0].State == view.WantPending && d[1].Node == "g14" && d[1].State == view.WantHeldBack && d[1].Rule == meshwantsclient.RuleCanary
|
|
}, "the two wants never showed with where they stand")
|
|
if err := kv.Delete(ctx, "want.g14.plex"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wait(func(s view.Snapshot) bool { return len(s.Wants) == 1 }, "the want taken away never left the list")
|
|
}
|
|
|
|
// TestProcessWholeWritesTheAnswerToAnAskNeedingNoProof starts the program whole, with the assigner's asks, the view's
|
|
// answers and the teller's answers buckets, and a fixture hold ask (in release 5 the assigner writes none yet). The
|
|
// operator's answer over the socket is written under `<ask key>.view` in the view's bucket; the ask shows the choice, and
|
|
// leaves the list once the teller's answer is under its key. The merge's Approve is refused with its reason and writes
|
|
// nothing (1297/B9a AC).
|
|
func TestProcessWholeWritesTheAnswerToAnAskNeedingNoProof(t *testing.T) { // 1297/B9a AC
|
|
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
go srv.Start()
|
|
if !srv.ReadyForConnections(10 * time.Second) {
|
|
t.Fatal("the bus did not come up")
|
|
}
|
|
t.Cleanup(srv.Shutdown)
|
|
nc, err := nats.Connect(srv.ClientURL())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(nc.Close)
|
|
js, err := jetstream.New(nc)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
defer cancel()
|
|
bucket := func(name string, history uint8) jetstream.KeyValue {
|
|
t.Helper()
|
|
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: history})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return kv
|
|
}
|
|
askBucket := bucket(view.AssignerAsksBucket, view.AssignerAsksHistory)
|
|
viewBucket := bucket(viewstate.ViewAnswersBucket, viewstate.ViewAnswersHistory)
|
|
bucket(viewstate.ViewModule+"_"+views.MarksState, 1)
|
|
teller := bucket(view.AnswersBucket, view.AnswersHistory)
|
|
// The view says its answers on its views family (design 47 G A-178), which the teller follows.
|
|
said := make(chan *nats.Msg, 8)
|
|
sub, err := nc.ChanSubscribe("mesh.mod."+viewstate.ViewModule+".event."+views.EventPrefix+viewstate.ViewAnswersState, said)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { sub.Unsubscribe() })
|
|
|
|
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { os.RemoveAll(dir) })
|
|
broker := filepath.Join(dir, "broker")
|
|
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
|
|
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
socket := filepath.Join(dir, "run", "cli.sock")
|
|
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"}
|
|
runCtx, stop := context.WithCancel(ctx)
|
|
done := make(chan error, 1)
|
|
serveOwners(t, ctx, js, nc)
|
|
serveCatalog(t, ctx, js, nc)
|
|
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
|
t.Cleanup(func() { stop(); <-done })
|
|
|
|
const fp = "src2:2222222222222222222222222222222222222222222222222222222222222222"
|
|
key, err := asks.HoldAskKey("plex", fp)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ask := oc.Ask{ID: meshdeliveryclient.AskID(key), Headline: "plex is held after a failed health check", Explanation: "ace failed the check.", Who: occ.Operator,
|
|
Expires: occ.FormatTime(time.Now().Add(time.Hour)), OnExpiry: "the fingerprint stays held",
|
|
Options: []oc.Option{{ID: "release", Label: "Release", Does: "lets the move go on", Level: occ.Approve}, {ID: "keep", Label: "Keep held", Does: "keeps it held", Level: occ.Approve}}}
|
|
value, _ := asks.EncodeAsk(key, asks.Ask{Ask: rawAsk(t, ask)})
|
|
if _, err := askBucket.Put(ctx, key, value); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wait := func(check func(view.Snapshot) bool, what string) {
|
|
t.Helper()
|
|
var last view.Snapshot
|
|
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
|
if s, err := view.Ask(socket); err == nil {
|
|
last = s
|
|
if check(s) {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
t.Fatalf("%s; last: %+v", what, last)
|
|
}
|
|
wait(func(s view.Snapshot) bool { return len(s.Asks) == 1 && s.Asks[0].How == view.AnswerInTheView }, "the open ask never showed")
|
|
|
|
if err := view.AnswerAsk(socket, key, "release", "the check was a flake"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
e, err := viewBucket.Get(ctx, key+".view")
|
|
if err != nil {
|
|
t.Fatalf("the view wrote no answer under %s.view: %v", key, err)
|
|
}
|
|
if a, err := viewstate.DecodeViewAnswer(e.Value()); err != nil || a.Option != "release" {
|
|
t.Fatalf("answer = %+v, %v", a, err)
|
|
}
|
|
wait(func(s view.Snapshot) bool { return len(s.Asks) == 1 && s.Asks[0].Chosen == "release" }, "the ask never showed the view's choice")
|
|
select {
|
|
case msg := <-said:
|
|
if !strings.Contains(string(msg.Data), key+".view") {
|
|
t.Errorf("the view said %s, not its answer under %s.view", msg.Data, key)
|
|
}
|
|
case <-time.After(20 * time.Second):
|
|
t.Error("the view never said its answer on views.answers")
|
|
}
|
|
|
|
// The merge's Approve is no ask of the view's (design 47 G §431): refused by its key, never written.
|
|
mkey, err := meshdeliveryclient.MergeAskKey("feat/x", []meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: "0123456789abcdef0123456789abcdef01234567"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := view.AnswerAsk(socket, mkey, "approve", ""); err == nil || !strings.Contains(err.Error(), "answer it on the phone (issue 953)") {
|
|
t.Errorf("merge Approve = %v, want the refusal naming the phone", err)
|
|
}
|
|
if _, err := viewBucket.Get(ctx, mkey+".view"); err == nil {
|
|
t.Error("the view wrote an answer to the merge's ask")
|
|
}
|
|
|
|
// The teller fills the ask: it leaves the list.
|
|
answer, err := meshdeliveryclient.EncodeAnswer(meshdeliveryclient.Answer{Option: "release", Warrant: ask.ID + "@2026-10-23T12:00:00Z", Words: "answered in the view"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := teller.Put(ctx, key, answer); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wait(func(s view.Snapshot) bool { return len(s.Asks) == 0 }, "the answered ask never left the list")
|
|
}
|
|
|
|
// TestProcessWholeShowsTheBacklogOfBuildAsks starts the program whole against a bus holding the go seat's work-queue stream
|
|
// with its three shared workers, three check asks waiting and one held by a node: the socket's backlog row names three
|
|
// waiting, one held, and the oldest's age (1887/A8a AC; compiled in the unit run, run by the merge check).
|
|
func TestProcessWholeShowsTheBacklogOfBuildAsks(t *testing.T) { // 1887/A8a AC
|
|
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
go srv.Start()
|
|
if !srv.ReadyForConnections(10 * time.Second) {
|
|
t.Fatal("the bus did not come up")
|
|
}
|
|
t.Cleanup(srv.Shutdown)
|
|
nc, err := nats.Connect(srv.ClientURL())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(nc.Close)
|
|
js, err := jetstream.New(nc)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
defer cancel()
|
|
stream, _ := buildstep.AskStream(buildstep.AskGo)
|
|
st, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{"mesh.seat.node-build-go.accept.>"}, Retention: jetstream.WorkQueuePolicy})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var checkWorker jetstream.Consumer
|
|
for _, kind := range buildstep.AskKinds {
|
|
name, _ := buildstep.AskWorker(buildstep.AskGo, kind)
|
|
subject, _ := buildstep.AskSubject(buildstep.AskGo, kind)
|
|
c, err := st.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: name, FilterSubject: subject, AckPolicy: jetstream.AckExplicitPolicy})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if kind == buildstep.AskCheck {
|
|
checkWorker = c
|
|
}
|
|
}
|
|
subject, _ := buildstep.AskSubject(buildstep.AskGo, buildstep.AskCheck)
|
|
oldest := time.Now().UTC().Add(-2 * time.Minute)
|
|
for i, id := range []string{"m0000001aaaaaa", "m0000002bbbbbb", "m0000003cccccc", "m0000004dddddd"} {
|
|
body, err := buildstep.EncodeStepAsk(buildstep.StepAsk{ID: id, Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: "mesh-delivery-checker", At: oldest.Add(time.Duration(i) * time.Second),
|
|
Check: &buildstep.CheckJob{Fetch: buildstep.Fetch{Repository: "novox/x", Commit: "0123456789abcdef0123456789abcdef01234567"}, Levels: []string{buildstep.LevelUnit}}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := js.Publish(ctx, subject, body); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
// A node pulls one and holds it unacknowledged.
|
|
held, err := checkWorker.Fetch(1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for range held.Messages() {
|
|
}
|
|
|
|
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { os.RemoveAll(dir) })
|
|
broker := filepath.Join(dir, "broker")
|
|
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
|
|
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
socket := filepath.Join(dir, "run", "cli.sock")
|
|
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"}
|
|
runCtx, stop := context.WithCancel(ctx)
|
|
done := make(chan error, 1)
|
|
serveOwners(t, ctx, js, nc)
|
|
serveCatalog(t, ctx, js, nc)
|
|
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
|
t.Cleanup(func() { stop(); <-done })
|
|
|
|
var last view.Snapshot
|
|
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
|
s, err := view.Ask(socket)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
last = s
|
|
for _, b := range s.Backlog {
|
|
if b.Language == "go" && b.Kind == "check" && b.Waiting == 3 && b.Held == 1 && b.OldestSecs >= 118 {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
t.Fatalf("the backlog never showed three waiting asks with the oldest's age; last: %+v", last.Backlog)
|
|
}
|
|
|
|
// 1906/D9, whole on the bus: when the wants and the haves make the source revision delivered, the view says it once, as its event
|
|
// `delivered` with the revision, the group and the heads, which the issue tracker derives "fix delivered" from.
|
|
func TestProcessWholeSaysTheSourceRevisionIsDelivered(t *testing.T) {
|
|
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
go srv.Start()
|
|
if !srv.ReadyForConnections(10 * time.Second) {
|
|
t.Fatal("the bus did not come up")
|
|
}
|
|
t.Cleanup(srv.Shutdown)
|
|
nc, err := nats.Connect(srv.ClientURL())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(nc.Close)
|
|
js, err := jetstream.New(nc)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
defer cancel()
|
|
events, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
kvs := map[string]jetstream.KeyValue{}
|
|
for _, name := range []string{view.AssignerWantsBucket, view.HaveBucket, viewstate.ReachBucket} {
|
|
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 5})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
kvs[name] = kv
|
|
}
|
|
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { os.RemoveAll(dir) })
|
|
broker := filepath.Join(dir, "broker")
|
|
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
|
|
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": filepath.Join(dir, "run", "cli.sock"), "MESH_DELIVERY_WEB": "127.0.0.1:0"}
|
|
runCtx, stop := context.WithCancel(ctx)
|
|
done := make(chan error, 1)
|
|
serveOwners(t, ctx, js, nc)
|
|
catalog := serveCatalog(t, ctx, js, nc)
|
|
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
|
t.Cleanup(func() { stop(); <-done })
|
|
|
|
fp := "src2:" + strings.Repeat("2", 64)
|
|
want, _ := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fp, Composed: "sha256:" + strings.Repeat("c", 64)})
|
|
failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed})
|
|
good, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
|
|
catalog.put(meshcatalog.CatalogEntry{Module: "web", Fp: fp, Manifest: map[string]any{"module": "a"}, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
|
|
if _, err := kvs[view.AssignerWantsBucket].Put(ctx, "want.ace.web", want); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := kvs[view.HaveBucket].Put(ctx, "ace.web", failed); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The gate's source of the watched branch moves: the view asks the verb source for it and reads no gate state (A-173).
|
|
const rev = 41
|
|
catalog.moveSource(rev, "feat/x", map[string]string{"novox/web": strings.Repeat("1", 40)})
|
|
time.Sleep(time.Second)
|
|
if info, err := events.Info(ctx, jetstream.WithSubjectFilter("mesh.mod.mesh-delivery-view.event."+DeliveredEventType)); err != nil || len(info.State.Subjects) != 0 {
|
|
t.Fatalf("an event was said before the revision was delivered: %v %+v", err, info.State)
|
|
}
|
|
if _, err := kvs[view.HaveBucket].Put(ctx, "ace.web", good); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var msg *jetstream.RawStreamMsg
|
|
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && msg == nil; time.Sleep(100 * time.Millisecond) {
|
|
msg, _ = events.GetLastMsgForSubject(ctx, "mesh.mod.mesh-delivery-view.event."+DeliveredEventType)
|
|
}
|
|
if msg == nil {
|
|
t.Fatal("the delivery of the source revision was never said")
|
|
}
|
|
var e view.DeliveredEvent
|
|
if err := json.Unmarshal(msg.Data, &e); err != nil || e.Branch != catalogBranch || e.Revision != rev || e.Group != "feat/x" || e.Heads["novox/web"] != strings.Repeat("1", 40) {
|
|
t.Errorf("the event = %+v, %v; want the branch %s, revision %d, group feat/x and the heads", e, err, catalogBranch, rev)
|
|
}
|
|
if msg.Header.Get("x-source") != "mesh-delivery-view" || msg.Header.Get("x-event-id") == "" {
|
|
t.Errorf("the headers = %v", msg.Header)
|
|
}
|
|
// The reach it wrote is said as the event reached, with the reach as written (ADR 0596 §2).
|
|
var rm *jetstream.RawStreamMsg
|
|
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && rm == nil; time.Sleep(100 * time.Millisecond) {
|
|
rm, _ = events.GetLastMsgForSubject(ctx, "mesh.mod.mesh-delivery-view.event."+ReachedEventType)
|
|
}
|
|
if rm == nil {
|
|
t.Fatal("the reach of novox/web was written and never said as the event reached")
|
|
}
|
|
var re ReachedEvent
|
|
if err := json.Unmarshal(rm.Data, &re); err != nil || re.Key != "novox/web" {
|
|
t.Fatalf("the event reached = %s, %v", rm.Data, err)
|
|
}
|
|
if stored, gerr := kvs[viewstate.ReachBucket].Get(ctx, re.Key); gerr != nil || string(stored.Value()) != string(re.Reach) {
|
|
t.Errorf("the event carries %s, the state reach holds %v (%v)", re.Reach, stored, gerr)
|
|
}
|
|
}
|
|
|
|
// 1906/D10, whole on the bus (through run): a want that differs from its have is said `moving`, the have reaching it running and
|
|
// healthy `settled`, a have held or failed `stuck`, each once with the body the messenger reads and an event id per type, node, module
|
|
// and composition.
|
|
func TestProcessWholeSaysModulesInMotion(t *testing.T) {
|
|
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
go srv.Start()
|
|
if !srv.ReadyForConnections(10 * time.Second) {
|
|
t.Fatal("the bus did not come up")
|
|
}
|
|
t.Cleanup(srv.Shutdown)
|
|
nc, err := nats.Connect(srv.ClientURL())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(nc.Close)
|
|
js, err := jetstream.New(nc)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
defer cancel()
|
|
events, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
kvs := map[string]jetstream.KeyValue{}
|
|
for _, name := range []string{view.AssignerWantsBucket, view.HaveBucket, view.StuckBucket} {
|
|
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 5})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
kvs[name] = kv
|
|
}
|
|
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { os.RemoveAll(dir) })
|
|
broker := filepath.Join(dir, "broker")
|
|
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
|
|
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": filepath.Join(dir, "run", "cli.sock"), "MESH_DELIVERY_WEB": "127.0.0.1:0"}
|
|
runCtx, stop := context.WithCancel(ctx)
|
|
done := make(chan error, 1)
|
|
serveOwners(t, ctx, js, nc)
|
|
serveCatalog(t, ctx, js, nc)
|
|
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
|
t.Cleanup(func() { stop(); <-done })
|
|
time.Sleep(2 * time.Second) // the view has read what stood: nothing
|
|
|
|
fpOld, fpNew := "src2:"+strings.Repeat("1", 64), "src2:"+strings.Repeat("2", 64)
|
|
digest := "sha256:" + strings.Repeat("c", 64)
|
|
put := func(bucket, key string, v []byte, err error) {
|
|
t.Helper()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := kvs[bucket].Put(ctx, key, v); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
want := func(node, module string) {
|
|
v, err := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fpNew, Composed: digest})
|
|
put(view.AssignerWantsBucket, "want."+node+"."+module, v, err)
|
|
}
|
|
have := func(node, module, fp string, phase string, health string, detail string) {
|
|
v, err := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: phase, Health: health, Detail: detail})
|
|
put(view.HaveBucket, node+"."+module, v, err)
|
|
}
|
|
have("ace", "web", fpOld, nodeengineclient.PhaseRunning, nodeengineclient.HealthOK, "")
|
|
want("ace", "web")
|
|
have("g14", "web", fpOld, nodeengineclient.PhaseRunning, nodeengineclient.HealthOK, "")
|
|
want("g14", "web")
|
|
|
|
said := func() map[string]jetstream.RawStreamMsg {
|
|
out := map[string]jetstream.RawStreamMsg{}
|
|
for seq := uint64(1); ; seq++ {
|
|
m, err := events.GetMsg(ctx, seq)
|
|
if err != nil {
|
|
return out
|
|
}
|
|
out[m.Header.Get("x-event-id")] = *m
|
|
}
|
|
}
|
|
waitFor := func(id string) jetstream.RawStreamMsg {
|
|
t.Helper()
|
|
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
|
if m, ok := said()[id]; ok {
|
|
return m
|
|
}
|
|
}
|
|
t.Fatalf("never said %s; said %v", id, func() []string {
|
|
var ids []string
|
|
for k := range said() {
|
|
ids = append(ids, k)
|
|
}
|
|
return ids
|
|
}())
|
|
return jetstream.RawStreamMsg{}
|
|
}
|
|
hex := strings.Repeat("c", 64)
|
|
mv := waitFor("moving.ace.web." + hex)
|
|
var body view.MotionBody
|
|
if err := json.Unmarshal(mv.Data, &body); err != nil || body.Node != "ace" || body.Module != "web" || body.FromFP != fpOld || body.ToFP != fpNew || body.Composed != digest || body.BoundSeconds != view.MotionBoundSeconds || body.Since.IsZero() {
|
|
t.Errorf("moving = %+v, %v", body, err)
|
|
}
|
|
if mv.Subject != "mesh.mod.mesh-delivery-view.event.moving" {
|
|
t.Errorf("subject = %s", mv.Subject)
|
|
}
|
|
have("ace", "web", fpNew, nodeengineclient.PhaseRunning, nodeengineclient.HealthOK, "")
|
|
st := waitFor("settled.ace.web." + hex)
|
|
if err := json.Unmarshal(st.Data, &body); err != nil || body.Outcome != "running" {
|
|
t.Errorf("settled = %+v, %v", body, err)
|
|
}
|
|
have("g14", "web", fpOld, nodeengineclient.PhaseHeld, nodeengineclient.HealthWaits, "no space left")
|
|
stk := waitFor("stuck.g14.web." + hex)
|
|
if err := json.Unmarshal(stk.Data, &body); err != nil || body.Outcome != "stopped" || body.Words != "no space left" {
|
|
t.Errorf("stuck = %+v, %v", body, err)
|
|
}
|
|
time.Sleep(time.Second)
|
|
count := map[string]int{}
|
|
for id := range said() {
|
|
count[strings.SplitN(id, ".", 2)[0]]++
|
|
}
|
|
if count["moving"] != 2 || count["settled"] != 1 || count["stuck"] != 1 {
|
|
t.Errorf("events said by type = %v, want moving 2, settled 1, stuck 1", count)
|
|
}
|
|
}
|