The subscription is a wake-up; the backoff bounds the asks without it. The refusal is said with its subject and error (principle 3), the composer goes on, and the subscription is tried again on laterAgain's steps until it holds. Builds: design 47 G-22 A-487a, principle 3
817 lines
35 KiB
Go
817 lines
35 KiB
Go
package selfcompose
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"runtime"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
|
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
|
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
|
"github.com/novox/mesh-host/internal/agent"
|
|
"github.com/novox/mesh-host/internal/buckets"
|
|
"github.com/novox/mesh-host/internal/buckets/bucketstest"
|
|
"github.com/novox/mesh-host/internal/compositions"
|
|
)
|
|
|
|
// Design 47 G §421, §683, §698 (the lead's answers A-47, A-50): the node-engine reads the entry its want names by the
|
|
// fingerprint, through its own calls of the seat mesh-catalog's verb entry, and reads neither the indexer's states nor the
|
|
// builder's; a want whose entry the catalog half holds none of is held, its have saying entry-missing and naming the
|
|
// fingerprint; the entries and wants it composes from are kept in its composition state, written before the composition that
|
|
// uses them and read at its start before anything is asked; a want removed takes its kept want and entry away.
|
|
func composeFromEntries(t *testing.T) {
|
|
b := newFakeBus()
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
|
|
ghost := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64)
|
|
var manifest map[string]any
|
|
if err := json.Unmarshal([]byte(lemurs), &manifest); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
entry := meshcatalog.CatalogEntry{Module: "lemurs", Fp: fp, Manifest: manifest, Repository: "novox/x", Commit: "c",
|
|
ManifestDigest: meshcatalogclient.ManifestDigest([]byte(lemurs))}
|
|
var mu sync.Mutex
|
|
var asked []string
|
|
ask := func(_ context.Context, seat, verb string, args any) (json.RawMessage, error) {
|
|
in, isEntry := args.(meshcatalog.EntryInput)
|
|
if !isEntry {
|
|
t.Errorf("the trunk build is asked (%s %v) of a module nothing says runs here (A-482)", verb, args)
|
|
return nil, errors.New("the catalog half holds no trunk build here")
|
|
}
|
|
mu.Lock()
|
|
asked = append(asked, in.Fingerprint)
|
|
mu.Unlock()
|
|
if seat != "mesh-catalog" || verb != "entry" {
|
|
t.Errorf("asked %s.%s", seat, verb)
|
|
}
|
|
if in.Fingerprint != fp {
|
|
return nil, errors.New("no entry of fingerprint " + in.Fingerprint + " is kept")
|
|
}
|
|
return json.Marshal(meshcatalog.EntryAnswer{Entry: entry})
|
|
}
|
|
state := &fakeComposed{kv: map[string]ComposedEntry{}}
|
|
var held map[string]string
|
|
env := b.env()
|
|
env.Entry, env.ComposedState = EntryCall(ask), state
|
|
refused := make(chan struct{}, 1) // fed at each refusal the composer says, so a wait follows what it says, not a pause
|
|
env.Refused = func(_ string, h map[string]string) {
|
|
mu.Lock()
|
|
held = h
|
|
mu.Unlock()
|
|
select {
|
|
case refused <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
told := map[string]bool{}
|
|
env.EntryNotFound = func(fp string, notFound bool) { mu.Lock(); told[fp] = notFound; mu.Unlock() }
|
|
c := New(env)
|
|
out, err := c.Watch(ctx, "n1.>")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
place := assignedValues(t, meshmoduleassigner.Assignments{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"},
|
|
Modules: []meshmoduleassigner.PlacedModule{{Module: "lemurs"}, {Module: "ghost"}}})
|
|
have, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}})
|
|
nodeWant, _ := bucketstest.NodeWant(meshwants.WantsByWantNodeNodeValue{Self: true})
|
|
b.putAssigned(t, place)
|
|
b.put(t, buckets.Have, "n1.node", have)
|
|
b.put(t, buckets.Wants, "want.n1.node", nodeWant)
|
|
for module, w := range map[string]meshwants.WantsByWantNodeModuleValue{"lemurs": {Fp: fp, Self: true}, "ghost": {Fp: ghost, Self: true}} {
|
|
raw, _ := bucketstest.Want(w)
|
|
b.put(t, buckets.Wants, "want.n1."+module, raw)
|
|
}
|
|
for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, ResolvedBucket} {
|
|
b.send(bucket, agent.Update{Caught: true})
|
|
}
|
|
lemursKey, _ := compositions.Key("n1", "lemurs", fp)
|
|
handed := map[string]bool{}
|
|
for !handed[lemursKey] {
|
|
handed[next(t, out).Key] = true
|
|
}
|
|
// Neither the indexer's states nor the builder's are watched or read.
|
|
b.mu.Lock()
|
|
for _, r := range b.reads {
|
|
if r[0] == buckets.Modules || r[0] == buckets.Manifests || r[0] == buckets.Verdicts || strings.HasSuffix(r[0], "_builds") {
|
|
t.Errorf("the node-engine read %s of %s", r[1], r[0])
|
|
}
|
|
}
|
|
b.mu.Unlock()
|
|
// A want whose entry the half holds none of is held, saying entry-missing and the fingerprint; the rest is composed.
|
|
// The ghost's entry is asked first and answered not found after: wait until the refusal says entry-missing.
|
|
var why string
|
|
for deadline := time.After(10 * time.Second); !strings.HasPrefix(why, agent.EntryMissing); {
|
|
mu.Lock()
|
|
why = held["ghost"]
|
|
mu.Unlock()
|
|
if strings.HasPrefix(why, agent.EntryMissing) {
|
|
break
|
|
}
|
|
select {
|
|
case <-refused:
|
|
case <-deadline:
|
|
t.Fatalf("the ghost's reason is still %q after 10 s, want %s", why, agent.EntryMissing)
|
|
}
|
|
}
|
|
if !strings.HasPrefix(why, agent.EntryMissing) || !strings.Contains(why, ghost) || held["lemurs"] != "" {
|
|
t.Errorf("held %q, want ghost held with %s naming %s", held, agent.EntryMissing, ghost)
|
|
}
|
|
// And the finding entry-not-found is told for the fingerprint the half holds none of, and for no other (design 47 G §614, A-117).
|
|
mu.Lock()
|
|
nf, nfOther := told[ghost], told[fp]
|
|
mu.Unlock()
|
|
if !nf || nfOther {
|
|
t.Errorf("entry-not-found told %v for the ghost and %v for the entry held", nf, nfOther)
|
|
}
|
|
// An entry asked again by hand is asked again in the shadow's cache too (lead answer A-319): the shadow holds the ghost as not
|
|
// found, the half then holds it, and the node's own composer asked again takes it in on the shadow as well.
|
|
var late atomic.Bool
|
|
shadowEnv := b.env()
|
|
shadowEnv.Entry = EntryCall(func(ctx context.Context, seat, verb string, args any) (json.RawMessage, error) {
|
|
if in := args.(meshcatalog.EntryInput); in.Fingerprint == ghost && late.Load() {
|
|
return json.Marshal(meshcatalog.EntryAnswer{Entry: meshcatalog.CatalogEntry{Module: "ghost", Fp: ghost}})
|
|
}
|
|
return ask(ctx, seat, verb, args)
|
|
})
|
|
shadow := NewShadow(shadowEnv, nil)
|
|
shadow.c.entries.Need(ctx, ghost)
|
|
for deadline := time.Now().Add(10 * time.Second); shadow.c.entries.Rows() == nil || shadow.c.entries.Rows()[0].State != meshcatalogclient.EntryNotFound; {
|
|
if time.Now().After(deadline) {
|
|
t.Fatal("the shadow did not hold the ghost as not found")
|
|
}
|
|
runtime.Gosched()
|
|
}
|
|
late.Store(true)
|
|
c.AlsoEntryAgain = shadow.EntryAgain
|
|
if err := c.EntryAgain(ctx, ghost); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, state := shadow.c.entries.Need(ctx, ghost); state != meshcatalogclient.EntryAnswered {
|
|
t.Errorf("the shadow holds the ghost's entry as %s after entry-again, want answered", state)
|
|
}
|
|
// Each entry and want it composes from is kept in the composition state before the composition that uses them.
|
|
state.mu.Lock()
|
|
_, hasEntry := state.kv[entryKey(fp)]
|
|
_, hasWant := state.kv["want.lemurs"]
|
|
state.mu.Unlock()
|
|
if !hasEntry || !hasWant {
|
|
t.Errorf("the composition state holds entry %v and want %v of lemurs: %v", hasEntry, hasWant, state.written())
|
|
}
|
|
// The seat's first-views lists them.
|
|
_, entries, wants := c.FirstViews()
|
|
if len(entries) < 2 || len(wants) != 2 {
|
|
t.Errorf("first-views lists %v and %v", entries, wants)
|
|
}
|
|
// A restart reads what is kept before it asks anything: lemurs's entry is not asked again.
|
|
mu.Lock()
|
|
asked = nil
|
|
mu.Unlock()
|
|
env2 := b.env()
|
|
env2.Entry, env2.ComposedState = EntryCall(ask), state
|
|
// The node's own record, stated by the node-engine of itself in memory (A-120).
|
|
env2.NodeRecord = func() (nodeengine.NodeRecord, bool) {
|
|
return nodeengine.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}, true
|
|
}
|
|
c2 := New(env2)
|
|
out2, err := c2.Watch(ctx, "n1.>")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// And composes the module it kept a want and an entry of before any input is read, nothing of the others (design 47 G §683, A-117).
|
|
if u := next(t, out2); u.Key != lemursKey || u.Deleted {
|
|
t.Errorf("the early pass handed %q, want the composition %s of lemurs", u.Key, lemursKey)
|
|
}
|
|
c2.mu.Lock()
|
|
kept, keptWant := c2.keptEnt[fp], c2.wants["lemurs"].Fp
|
|
c2.mu.Unlock()
|
|
if kept.entry.Module != "lemurs" || keptWant != fp {
|
|
t.Errorf("a restart read entry %q and want %q from the state", kept.entry.Module, keptWant)
|
|
}
|
|
// A want removed takes its kept want and entry away.
|
|
raw, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fp, Removed: true})
|
|
b.put(t, buckets.Wants, "want.n1.lemurs", raw)
|
|
for range 20 {
|
|
state.mu.Lock()
|
|
_, hasWant = state.kv["want.lemurs"]
|
|
_, hasEntry = state.kv[entryKey(fp)]
|
|
state.mu.Unlock()
|
|
if !hasWant && !hasEntry {
|
|
break
|
|
}
|
|
next(t, out)
|
|
}
|
|
if hasWant || hasEntry {
|
|
t.Errorf("a removed want left its kept want %v and entry %v", hasWant, hasEntry)
|
|
}
|
|
}
|
|
|
|
// catalogHalf is the catalog half's entries a test composes from, by fingerprint, answered through the seat mesh-catalog's verbs
|
|
// entry and module (design 47 G §421, A-312): the node-engine reads no indexer.
|
|
type catalogHalf map[string]meshcatalog.CatalogEntry
|
|
|
|
// add holds the entry of module at fp, made of the manifest body.
|
|
func (h catalogHalf) add(t *testing.T, module, fp, body string) {
|
|
t.Helper()
|
|
var m map[string]any
|
|
if err := json.Unmarshal([]byte(body), &m); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h[fp] = meshcatalog.CatalogEntry{Module: module, Fp: fp, Manifest: m, Repository: "novox/" + module, Commit: "c", ManifestDigest: meshcatalogclient.ManifestDigest([]byte(body))}
|
|
}
|
|
|
|
func (h catalogHalf) call() meshcatalogclient.Call {
|
|
return EntryCall(func(_ context.Context, _, verb string, args any) (json.RawMessage, error) {
|
|
switch in := args.(type) {
|
|
case meshcatalog.ModuleInput:
|
|
for _, e := range h {
|
|
if e.Module == in.Module {
|
|
return json.Marshal(meshcatalog.ModuleAnswer{Module: meshcatalog.ModuleAnswerModule{Module: e.Module, Repository: e.Repository, Commit: e.Commit, Fp: e.Fp}})
|
|
}
|
|
}
|
|
return nil, fmt.Errorf("the catalog holds no module %s", in.Module)
|
|
case meshcatalog.EntryInput:
|
|
if e, ok := h[in.Fingerprint]; ok {
|
|
return json.Marshal(meshcatalog.EntryAnswer{Entry: e})
|
|
}
|
|
return nil, errors.New("no entry of fingerprint " + in.Fingerprint + " is kept")
|
|
}
|
|
return nil, fmt.Errorf("asked %s of %T", verb, args)
|
|
})
|
|
}
|
|
|
|
// holdEntry makes the composer hold the entry of module at fp, as its call of the verb entry would have answered it (§421), for the
|
|
// tests that set the rest of its picture directly.
|
|
func (c *Composer) holdEntry(t *testing.T, module, fp, body string) {
|
|
t.Helper()
|
|
half := catalogHalf{}
|
|
half.add(t, module, fp, body)
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
if c.keptEnt == nil {
|
|
c.keptEnt = map[string]keptEntry{}
|
|
}
|
|
c.keptEnt[fp] = keptEntry{entry: half[fp]}
|
|
}
|
|
|
|
// A-405, A-407: a module the node runs whose wanted build has no entry stands, for resolution alone, as the build its want says
|
|
// is being built, at the want's fingerprint, so its want stays held; one not running here, or whose want names no newer build the
|
|
// half holds, stands as its name alone.
|
|
func runningModuleWithoutEntryStandsAsTheNewestBuild(t *testing.T) {
|
|
old := meshcatalogclient.SourcePrefix2 + strings.Repeat("1", 64)
|
|
newest := meshcatalogclient.SourcePrefix2 + strings.Repeat("2", 64)
|
|
body := `{"module":"postgres","version":"2"}`
|
|
for _, tc := range []struct {
|
|
name string
|
|
runs bool
|
|
newest string
|
|
holds bool
|
|
laid bool
|
|
wantFp string
|
|
stopped bool // A-480a: heard stopped
|
|
trunk bool // A-480a: the catalog half's trunk build of the module has an entry (the newest build has none)
|
|
have string // A-482: the phase its have says, naming the build it runs ("" for none)
|
|
named string // A-480a: the manifest its running event names ("held": one this node holds, "unread": one it does not)
|
|
}{
|
|
{"runs, newest entry held", true, newest, true, true, old, false, false, "", ""},
|
|
{"not running here", false, newest, true, false, old, false, false, "", ""},
|
|
{"names no newer build", true, "", false, false, old, false, false, "", ""},
|
|
{"newest entry not held", true, newest, false, false, old, false, false, "", ""},
|
|
{"runs, no newest build, trunk entry held (A-480a)", true, "", false, true, old, false, true, "", ""},
|
|
{"heard no running event, its have says running (A-480a, A-482)", false, "", false, true, old, false, true, "running", ""},
|
|
{"heard no running event, its have says held (A-482)", false, "", false, true, old, false, true, "held", ""},
|
|
{"wanted here, heard no running event, no have (A-482)", false, "", false, false, old, false, true, "", ""},
|
|
{"its running event names a manifest this node holds (A-484a)", true, "", false, true, old, false, true, "", "held"},
|
|
{"its running event names a manifest this node cannot read (A-480a)", true, "", false, true, old, false, true, "", "unread"},
|
|
{"runs a composition at another build, no manifest read (A-480a)", true, "", false, true, old, false, true, "other", ""},
|
|
{"heard stopped, trunk entry held (A-480a)", false, "", false, false, old, true, true, "running", ""},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
env := newFakeBus().env()
|
|
env.Node = "n1"
|
|
trunkFp := meshcatalogclient.SourcePrefix2 + strings.Repeat("4", 64)
|
|
env.Entry = EntryCall(func(_ context.Context, _, verb string, _ any) (json.RawMessage, error) {
|
|
if verb == meshcatalog.VerbModule && tc.trunk {
|
|
var man map[string]any
|
|
_ = json.Unmarshal([]byte(body), &man)
|
|
return json.Marshal(meshcatalog.ModuleAnswer{Module: meshcatalog.ModuleAnswerModule{Fp: trunkFp, Manifest: man}})
|
|
}
|
|
return nil, errors.New("not held")
|
|
})
|
|
c := New(env)
|
|
c.mu.Lock()
|
|
c.wants["postgres"] = meshwants.WantsByWantNodeModuleValue{Fp: old, Newest: tc.newest}
|
|
c.assignments["n1"] = meshmoduleassigner.Assignments{Node: "n1", Modules: []meshmoduleassigner.PlacedModule{{Module: "postgres"}}}
|
|
if tc.runs {
|
|
c.heard = map[string]map[string]heardRun{"n1": {"postgres": {fingerprint: old}}}
|
|
}
|
|
if tc.named != "" {
|
|
digest := "sha256:" + strings.Repeat("9", 64)
|
|
c.heard = map[string]map[string]heardRun{"n1": {"postgres": {fingerprint: old, manifest: digest}}}
|
|
if tc.named == "held" {
|
|
c.manifests[digest] = []byte(body)
|
|
}
|
|
}
|
|
if tc.have != "" {
|
|
c.haveRuns["postgres"] = true
|
|
}
|
|
if tc.have == "other" {
|
|
c.env.Running = func() []compositions.Composed {
|
|
return []compositions.Composed{{Node: "n1", Module: "postgres", FP: meshcatalogclient.SourcePrefix2 + strings.Repeat("5", 64)}}
|
|
}
|
|
}
|
|
if tc.stopped {
|
|
c.heard = map[string]map[string]heardRun{"n1": {"postgres": {fingerprint: old, stopped: true}}}
|
|
}
|
|
c.mu.Unlock()
|
|
if tc.holds {
|
|
c.holdEntry(t, "postgres", newest, body)
|
|
}
|
|
if tc.trunk {
|
|
c.holdEntry(t, "postgres", trunkFp, body)
|
|
}
|
|
c.askTrunkBuilds()()
|
|
if got := c.wantedBuilds()[newest]; tc.newest != "" && got {
|
|
t.Errorf("the newest build is named by a want, want only asked")
|
|
}
|
|
c.layModules()
|
|
c.mu.Lock()
|
|
m, ok := c.modules["postgres"]
|
|
raw := c.manifests[m.Manifest]
|
|
c.mu.Unlock()
|
|
if !ok || m.Fp2 != tc.wantFp {
|
|
t.Fatalf("modules[postgres] = %+v (%v), want fingerprint %s", m, ok, tc.wantFp)
|
|
}
|
|
if got := strings.Contains(string(raw), `"version":"2"`); got != tc.laid {
|
|
t.Errorf("manifest of the newest build laid = %v, want %v (%s)", got, tc.laid, raw)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// The node judges a move's data forms from the entry it kept of the build it runs, though the catalog half dropped that entry and no
|
|
// running event names it; and an answered entry has the composition handed again, which the agent waits on (novox, mesh-hand-acts).
|
|
func TestRunningBuildEntryKeptAndHandedAgain(t *testing.T) {
|
|
old := meshcatalogclient.SourcePrefix2 + strings.Repeat("1", 64)
|
|
next := meshcatalogclient.SourcePrefix2 + strings.Repeat("2", 64)
|
|
env := newFakeBus().env()
|
|
env.Node = "n1"
|
|
env.Running = func() []compositions.Composed {
|
|
return []compositions.Composed{{Node: "n1", Module: "hand", FP: old}}
|
|
}
|
|
c := New(env)
|
|
c.mu.Lock()
|
|
c.wants["hand"] = meshwants.WantsByWantNodeModuleValue{Fp: next}
|
|
c.mu.Unlock()
|
|
if named, ok := c.wantedBuilds()[old]; !ok || named {
|
|
t.Errorf("the running build %s is asked of no want yet kept: %v, %v", old, named, ok)
|
|
}
|
|
key, _ := compositions.Key("n1", "hand", next)
|
|
c.mu.Lock()
|
|
c.handed[key] = "d"
|
|
c.mu.Unlock()
|
|
c.entryChanged(meshcatalogclient.EntryRow{Fingerprint: old, State: meshcatalogclient.EntryAnswered,
|
|
Entry: &meshcatalog.CatalogEntry{Module: "hand", Fp: old}})
|
|
c.mu.Lock()
|
|
_, still := c.handed[key]
|
|
c.mu.Unlock()
|
|
if still {
|
|
t.Error("an entry of the running build was answered, and the composition was not handed again")
|
|
}
|
|
|
|
// A-483: a running build with no entry is read from the composition it runs here: where neither it nor the wanted build's manifest
|
|
// keeps data on the bus no form can change and the move goes on; where either does, or no composition is run, it still waits.
|
|
for _, tc := range []struct {
|
|
name string
|
|
runs, data bool // a composition of the module runs here, keeping data
|
|
wantedData bool
|
|
wantedHeld bool
|
|
moves bool
|
|
}{
|
|
{"no entry of the running build, its composition and the wanted build keep no data", true, false, false, true, true},
|
|
{"no entry, its composition keeps data", true, true, false, true, false},
|
|
{"no entry, the wanted build keeps data", true, false, true, true, false},
|
|
{"no entry, no composition of it runs here", false, false, false, true, false},
|
|
{"no entry, the wanted build's entry is not held either", true, false, false, false, false},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
env := newFakeBus().env()
|
|
env.Node = "n1"
|
|
env.Running = func() []compositions.Composed {
|
|
if !tc.runs {
|
|
return nil
|
|
}
|
|
return []compositions.Composed{{Node: "n1", Module: "feh", FP: old, Data: tc.data}}
|
|
}
|
|
c := New(env)
|
|
if tc.wantedHeld {
|
|
body := `{"module":"feh","version":"2"}`
|
|
if tc.wantedData {
|
|
body = `{"module":"feh","version":"2","data":{"own":[{"id":"state","kind":"state"}]}}`
|
|
}
|
|
c.holdEntry(t, "feh", next, body)
|
|
}
|
|
changes, err := c.FormChanges(context.Background(), "feh", old, next)
|
|
if (err == nil) != tc.moves || len(changes) != 0 {
|
|
t.Errorf("FormChanges = %v, %v; the move goes on: %v", changes, err, tc.moves)
|
|
}
|
|
})
|
|
}
|
|
|
|
// A-484: where neither the entry nor a composition answers, the trunk's manifest is the running build's: a form change is judged
|
|
// against it, and its words name the three sources where the trunk answered none either.
|
|
for _, trunk := range []bool{true, false} {
|
|
env := newFakeBus().env()
|
|
env.Node = "n1"
|
|
c := New(env)
|
|
c.holdEntry(t, "feh", next, `{"module":"feh","version":"2","data":{"own":[{"id":"state","kind":"state"}]}}`)
|
|
if trunk {
|
|
c.mu.Lock()
|
|
c.trunkBuilds = map[string]trunkBuild{"feh": {running: old, fp: next, manifest: map[string]any{"module": "feh", "version": "1"}}}
|
|
c.mu.Unlock()
|
|
}
|
|
changes, err := c.FormChanges(context.Background(), "feh", old, next)
|
|
switch {
|
|
case trunk && err != nil:
|
|
t.Errorf("the trunk's manifest answers, FormChanges = %v, %v", changes, err)
|
|
case !trunk && (err == nil || !strings.Contains(err.Error(), SourceEntry) || !strings.Contains(err.Error(), SourceStatement) || !strings.Contains(err.Error(), SourceTrunk)):
|
|
t.Errorf("no source answers, FormChanges says %v, want the three named", err)
|
|
}
|
|
}
|
|
|
|
// A-484, A-484a, A-482: the manifest of the build a module runs here is answered in one place, from the entry of the running build,
|
|
// else the manifest its running statement names, else the module's trunk manifest; the composition says whether it keeps data. It
|
|
// says its source once per module and source, whoever asks, and where none answers its words name the three.
|
|
{
|
|
running := meshcatalogclient.SourcePrefix2 + strings.Repeat("1", 64)
|
|
other := meshcatalogclient.SourcePrefix2 + strings.Repeat("2", 64)
|
|
body := `{"module":"xorg","version":"1","provides":[{"name":"x11-display"}]}`
|
|
for _, tc := range []struct {
|
|
name string
|
|
entry string // the entry held, by fingerprint ("running", "other") or none
|
|
composition string // the composition this node runs: its build ("running", "other") or none
|
|
data bool
|
|
trunk bool
|
|
statement bool // the running event names a manifest this node holds (A-484a)
|
|
haveOnly bool // heard no running event: only its have names the build it runs (A-482)
|
|
wantSource string
|
|
wantKnown bool
|
|
}{
|
|
{"its entry answers first", "running", "running", false, true, false, false, SourceEntry, true},
|
|
{"the entry of the composition's build is the running build's", "other", "other", true, true, false, false, SourceEntry, true},
|
|
{"a composition with no manifest keeps the trunk's manifest and its data", "", "running", true, true, false, false, SourceTrunk, true},
|
|
{"a composition and no trunk answer: its build and its data alone", "", "running", true, false, false, false, SourceComposition, false},
|
|
{"the trunk alone", "", "", false, true, false, false, SourceTrunk, true},
|
|
{"its running statement's manifest comes before the trunk's", "", "running", true, true, true, false, SourceStatement, true},
|
|
{"only its have names the build, whose entry is held (A-482)", "running", "", false, false, false, true, SourceEntry, true},
|
|
{"none answers", "", "", false, false, false, false, "", false},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
env := newFakeBus().env()
|
|
env.Node = "n1"
|
|
var said []string
|
|
env.Say = func(s string) { said = append(said, s) }
|
|
if tc.composition != "" {
|
|
fp := map[string]string{"running": running, "other": other}[tc.composition]
|
|
env.Running = func() []compositions.Composed {
|
|
return []compositions.Composed{{Node: "n1", Module: "xorg", FP: fp, Data: tc.data}}
|
|
}
|
|
}
|
|
c := New(env)
|
|
c.mu.Lock()
|
|
c.assignments["n1"] = meshmoduleassigner.Assignments{Node: "n1", Modules: []meshmoduleassigner.PlacedModule{{Module: "xorg"}}}
|
|
c.heard = map[string]map[string]heardRun{"n1": {"xorg": {fingerprint: running}}}
|
|
if tc.haveOnly {
|
|
c.heard = map[string]map[string]heardRun{}
|
|
c.haveRuns["xorg"], c.haveFp["xorg"] = true, running
|
|
}
|
|
if tc.statement {
|
|
digest := "sha256:" + strings.Repeat("9", 64)
|
|
c.heard["n1"]["xorg"] = heardRun{fingerprint: running, manifest: digest}
|
|
c.manifests[digest] = []byte(body)
|
|
}
|
|
if tc.trunk {
|
|
var man map[string]any
|
|
_ = json.Unmarshal([]byte(body), &man)
|
|
c.trunkBuilds = map[string]trunkBuild{"xorg": {running: running, fp: other, manifest: man}}
|
|
}
|
|
c.mu.Unlock()
|
|
if tc.entry != "" {
|
|
c.holdEntry(t, "xorg", map[string]string{"running": running, "other": other}[tc.entry], body)
|
|
}
|
|
c.askTrunkBuilds()()
|
|
c.layModules()
|
|
a := c.runningManifest("n1", "xorg", "")
|
|
again := c.runningManifest("n1", "xorg", "")
|
|
if a.Source != tc.wantSource || a.Known != tc.wantKnown || (a.Known && len(a.Manifest.Provides) != 1) {
|
|
t.Fatalf("answered %q (known %v, provides %d), want %q (known %v)", a.Source, a.Known, len(a.Manifest.Provides), tc.wantSource, tc.wantKnown)
|
|
}
|
|
sources := 0
|
|
for _, line := range said {
|
|
if strings.Contains(line, "runs a build whose manifest is read from "+tc.wantSource) {
|
|
sources++
|
|
}
|
|
}
|
|
if tc.wantSource != "" && (sources != 1 || again.Said != a.Said) {
|
|
t.Errorf("said %q, want its source once", said)
|
|
}
|
|
if tc.composition != "" && a.Data != tc.data && a.Source != SourceEntry && a.Source != SourceStatement {
|
|
t.Errorf("the composition's data = %v, want %v", a.Data, tc.data)
|
|
}
|
|
if !tc.wantKnown && !strings.Contains(a.Why, SourceEntry) || !tc.wantKnown && !strings.Contains(a.Why, SourceStatement) || !tc.wantKnown && !strings.Contains(a.Why, SourceTrunk) {
|
|
t.Errorf("the words of an answer that reads no manifest are %q, want the three sources named", a.Why)
|
|
}
|
|
})
|
|
}
|
|
|
|
// A module counted as running here alone answers a held module's offer and what the module keeps (A-482): one wanted here that
|
|
// runs nothing and whose have names no build answers nothing, and HoldsData reads the manifest the one place read.
|
|
env := newFakeBus().env()
|
|
env.Node = "n1"
|
|
c := New(env)
|
|
c.holdEntry(t, "xorg", running, body)
|
|
c.mu.Lock()
|
|
c.wants["xorg"] = meshwants.WantsByWantNodeModuleValue{Fp: other}
|
|
c.haveFp["xorg"] = running
|
|
c.mu.Unlock()
|
|
if _, ok := c.runningFor("n1")("xorg"); ok {
|
|
t.Error("a module wanted here that runs nothing answered a manifest as running")
|
|
}
|
|
if data, known := c.HoldsData(context.Background(), "xorg"); data || !known {
|
|
t.Errorf("HoldsData = %v, %v, want a manifest read that declares no data", data, known)
|
|
}
|
|
c.mu.Lock()
|
|
c.haveRuns["xorg"] = true
|
|
c.mu.Unlock()
|
|
if man, ok := c.runningFor("n1")("xorg"); !ok || len(man.Provides) != 1 {
|
|
t.Error("a module its have says runs answered no manifest")
|
|
}
|
|
}
|
|
|
|
// A-461, A-461a, A-461b: another node's module whose running build has no entry is laid from the entry of the build the assignment
|
|
// places for it on that node, or, where the placement names none, of its trunk build; with neither entry it stays as its name
|
|
// alone, and this node's own module is never laid from a placement.
|
|
placedFp := meshcatalogclient.SourcePrefix2 + strings.Repeat("3", 64)
|
|
for _, tc := range []struct {
|
|
name string
|
|
node string
|
|
build, trunk string
|
|
holds, laid bool
|
|
heard string // A-461d: what is heard of it: "" its running build, "none", "stopped", "no-build"
|
|
}{
|
|
{"placed build has an entry", "n2", placedFp, "", true, true, ""},
|
|
{"neither build has an entry", "n2", placedFp, "", false, false, ""},
|
|
{"placement names no build, trunk build has an entry", "n2", "", placedFp, true, true, ""},
|
|
{"placement names no build, trunk names none", "n2", "", "", false, false, ""},
|
|
{"this node's own module is not laid from a placement", "n1", placedFp, "", true, false, ""},
|
|
{"placed, not heard, placement names no build, trunk build has an entry (A-461d)", "n2", "", placedFp, true, true, "none"},
|
|
{"placed, not heard, placed build has an entry (A-461d)", "n2", placedFp, "", true, true, "none"},
|
|
{"placed, stopped there, placement names no build, trunk build has an entry (A-461d)", "n2", "", placedFp, true, true, "stopped"},
|
|
{"placed, heard with no build, placement names no build, trunk build has an entry (A-461d)", "n2", "", placedFp, true, true, "no-build"},
|
|
{"placed, not heard, trunk names none (A-461d)", "n2", "", "", false, false, "none"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
env := newFakeBus().env()
|
|
env.Node = "n1"
|
|
env.Entry = EntryCall(func(_ context.Context, _, verb string, _ any) (json.RawMessage, error) {
|
|
if verb == meshcatalog.VerbModule {
|
|
var man map[string]any
|
|
if tc.trunk != "" {
|
|
man = map[string]any{"module": "distribution", "version": "2"}
|
|
}
|
|
return json.Marshal(meshcatalog.ModuleAnswer{Module: meshcatalog.ModuleAnswerModule{Fp: tc.trunk, Manifest: man}})
|
|
}
|
|
return nil, errors.New("not held")
|
|
})
|
|
c := New(env)
|
|
c.mu.Lock()
|
|
c.assignments[tc.node] = meshmoduleassigner.Assignments{Node: tc.node, Modules: []meshmoduleassigner.PlacedModule{{Module: "distribution", Build: tc.build}},
|
|
Holds: []meshmoduleassigner.Holding{{Claim: "container-image-registry", Module: "distribution", Scope: "mesh"}}}
|
|
c.heard = map[string]map[string]heardRun{tc.node: {"distribution": {fingerprint: old}}}
|
|
switch tc.heard {
|
|
case "none":
|
|
c.heard = map[string]map[string]heardRun{}
|
|
case "stopped":
|
|
c.heard = map[string]map[string]heardRun{tc.node: {"distribution": {fingerprint: old, stopped: true}}}
|
|
case "no-build":
|
|
c.heard = map[string]map[string]heardRun{tc.node: {"distribution": {}}}
|
|
}
|
|
c.mu.Unlock()
|
|
if tc.holds {
|
|
c.holdEntry(t, "distribution", placedFp, `{"module":"distribution","version":"2"}`)
|
|
}
|
|
c.askTrunkBuilds()()
|
|
c.mu.Lock()
|
|
_, kept := c.trunkBuilds["distribution"]
|
|
c.mu.Unlock()
|
|
if tc.node == "n2" && tc.build == "" && tc.trunk == "" && kept {
|
|
t.Error("a trunk build that could not be had was kept, so it is asked once per process (A-461e)")
|
|
}
|
|
if _, asked := c.wantedBuilds()[placedFp]; asked != (tc.node == "n2" && tc.build != "") {
|
|
t.Errorf("the placed build asked = %v", asked)
|
|
}
|
|
c.layModules()
|
|
c.mu.Lock()
|
|
m, ok := c.modules["distribution"]
|
|
raw := c.manifests[m.Manifest]
|
|
c.mu.Unlock()
|
|
if got := ok && strings.Contains(string(raw), `"version":"2"`); got != tc.laid {
|
|
t.Errorf("laid from the placed build = %v, want %v (%+v %s)", got, tc.laid, m, raw)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A-461, A-461e: a node asks the trunk build only of what it needs of another node's placement (the store's holder, the provider
|
|
// it is bound to), never of every module placed there, and beside the pass.
|
|
func TestOnlyWhatTheNodeNeedsOfAnotherNodesPlacementIsAsked(t *testing.T) {
|
|
env := newFakeBus().env()
|
|
env.Node = "n1"
|
|
var mu sync.Mutex
|
|
asked := map[string]int{}
|
|
env.Entry = EntryCall(func(_ context.Context, _, verb string, in any) (json.RawMessage, error) {
|
|
if verb == meshcatalog.VerbModule {
|
|
mu.Lock()
|
|
asked[in.(meshcatalog.ModuleInput).Module]++
|
|
mu.Unlock()
|
|
}
|
|
return nil, errors.New("not held")
|
|
})
|
|
c := New(env)
|
|
var unrelated []meshmoduleassigner.PlacedModule
|
|
for i := 0; i < 40; i++ {
|
|
unrelated = append(unrelated, meshmoduleassigner.PlacedModule{Module: fmt.Sprintf("unrelated%d", i)})
|
|
}
|
|
c.mu.Lock()
|
|
c.assignments["n2"] = meshmoduleassigner.Assignments{Node: "n2",
|
|
Modules: append(unrelated, meshmoduleassigner.PlacedModule{Module: "distribution"}, meshmoduleassigner.PlacedModule{Module: "provider"}),
|
|
Holds: []meshmoduleassigner.Holding{{Claim: "container-image-registry", Module: "distribution", Scope: "mesh"}}}
|
|
c.assignments["n1"] = meshmoduleassigner.Assignments{Node: "n1", Modules: []meshmoduleassigner.PlacedModule{
|
|
{Module: "app", Bindings: map[string]meshmoduleassigner.Provider{"postgres-database": {Module: "provider", Node: "n2"}}}}}
|
|
c.mu.Unlock()
|
|
c.askTrunkBuilds()()
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if len(asked) != 2 || asked["distribution"] != 1 || asked["provider"] != 1 {
|
|
t.Errorf("the trunk builds asked are %v, want the store's holder and the bound provider alone", asked)
|
|
}
|
|
}
|
|
|
|
// A-487a (A-461e): a trunk build the catalog gave none of is not asked again at every pass: it waits laterAgain's step for the module
|
|
// (5 s, then 15 s), and the seat's event catalog-revision ends the wait at once; an answer starts the steps again from 5 s. The
|
|
// event is heard through Env.Hear on the seat's subject, whatever its branch.
|
|
func TestTrunkAskBackoffAndCatalogRevision(t *testing.T) {
|
|
old := meshcatalogclient.SourcePrefix2 + strings.Repeat("1", 64)
|
|
now := time.Date(2026, 11, 19, 10, 0, 0, 0, time.UTC)
|
|
var asks int
|
|
answers := false
|
|
env := newFakeBus().env()
|
|
env.Node = "n1"
|
|
env.Now = func() time.Time { return now }
|
|
env.Entry = EntryCall(func(_ context.Context, _, verb string, _ any) (json.RawMessage, error) {
|
|
if verb != meshcatalog.VerbModule {
|
|
return nil, errors.New("not held")
|
|
}
|
|
asks++
|
|
if !answers {
|
|
return nil, errors.New("the catalog is busy")
|
|
}
|
|
return json.Marshal(meshcatalog.ModuleAnswer{Module: meshcatalog.ModuleAnswerModule{Fp: old, Manifest: map[string]any{"module": "postgres"}}})
|
|
})
|
|
c := New(env)
|
|
c.mu.Lock()
|
|
c.wants["postgres"] = meshwants.WantsByWantNodeModuleValue{Fp: old}
|
|
c.assignments["n1"] = meshmoduleassigner.Assignments{Node: "n1", Modules: []meshmoduleassigner.PlacedModule{{Module: "postgres"}}}
|
|
c.heard = map[string]map[string]heardRun{"n1": {"postgres": {fingerprint: old}}}
|
|
c.mu.Unlock()
|
|
pass := func(want int, what string) {
|
|
t.Helper()
|
|
c.askTrunkBuilds()()
|
|
if asks != want {
|
|
t.Fatalf("%s: the catalog was asked %d times, want %d", what, asks, want)
|
|
}
|
|
}
|
|
pass(1, "first pass")
|
|
pass(1, "a second pass at once")
|
|
now = now.Add(4 * time.Second)
|
|
pass(1, "within the 5 s step")
|
|
now = now.Add(2 * time.Second)
|
|
pass(2, "after the 5 s step")
|
|
now = now.Add(14 * time.Second)
|
|
pass(2, "within the 15 s step")
|
|
c.trunkRetryCleared() // what hearing catalog-revision does
|
|
pass(3, "after catalog-revision")
|
|
pass(3, "the revision's ask gave none again: a step waits again")
|
|
answers = true
|
|
now = now.Add(time.Minute)
|
|
pass(4, "after the longer step")
|
|
if _, backing := c.trunkRetry["postgres"]; backing {
|
|
t.Errorf("an answer came and the backoff of the module stands: %+v", c.trunkRetry)
|
|
}
|
|
|
|
// The wiring: the composer hears the seat's event on its subject and clears the waits.
|
|
heard := make(chan []byte, 1)
|
|
env.Hear = func(subject string) (<-chan []byte, func(), error) {
|
|
if subject == CatalogRevisionEvents {
|
|
return heard, func() {}, nil
|
|
}
|
|
return make(chan []byte), func() {}, nil
|
|
}
|
|
answers = false
|
|
c = New(env)
|
|
c.mu.Lock()
|
|
c.trunkRetry = map[string]trunkBackoff{"postgres": {tries: 1, next: now.Add(time.Hour)}}
|
|
c.mu.Unlock()
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
if _, err := c.Watch(ctx, "n1.>"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
heard <- []byte(`{}`)
|
|
deadline := time.After(5 * time.Second)
|
|
for {
|
|
c.mu.Lock()
|
|
next := c.trunkRetry["postgres"].next
|
|
c.mu.Unlock()
|
|
if next.IsZero() {
|
|
break
|
|
}
|
|
select {
|
|
case <-deadline:
|
|
t.Fatal("catalog-revision was heard and the wait of the module stands")
|
|
case <-time.After(5 * time.Millisecond):
|
|
}
|
|
}
|
|
}
|
|
|
|
// A-487a, principle 3: a refused subscription to catalog-revision does not refuse Watch; it is said once with its subject and error, and
|
|
// tried again on laterAgain's steps until it holds, after which an event clears the waits.
|
|
func TestCatalogRevisionSubscriptionRefusedIsSaidAndRetried(t *testing.T) {
|
|
setFor(t, &laterAgain, []time.Duration{5 * time.Millisecond})
|
|
b := newFakeBus()
|
|
env := b.env()
|
|
heard := make(chan []byte, 1)
|
|
var tries atomic.Int64
|
|
env.Hear = func(subject string) (<-chan []byte, func(), error) {
|
|
if subject == CatalogRevisionEvents {
|
|
if tries.Add(1) < 3 {
|
|
return nil, nil, errors.New("permissions violation")
|
|
}
|
|
return heard, func() {}, nil
|
|
}
|
|
return make(chan []byte), func() {}, nil
|
|
}
|
|
c := New(env)
|
|
c.mu.Lock()
|
|
c.trunkRetry = map[string]trunkBackoff{"postgres": {tries: 1, next: time.Now().Add(time.Hour)}}
|
|
c.mu.Unlock()
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
if _, err := c.Watch(ctx, "n1.>"); err != nil {
|
|
t.Fatalf("a refused catalog-revision subscription refused Watch: %v", err)
|
|
}
|
|
deadline := time.After(5 * time.Second)
|
|
for tries.Load() < 3 {
|
|
select {
|
|
case <-deadline:
|
|
t.Fatal("the subscription was not tried again")
|
|
case <-time.After(2 * time.Millisecond):
|
|
}
|
|
}
|
|
said := 0
|
|
b.mu.Lock()
|
|
for _, s := range b.said {
|
|
if strings.Contains(s, CatalogRevisionEvents) && strings.Contains(s, "permissions violation") {
|
|
said++
|
|
}
|
|
}
|
|
b.mu.Unlock()
|
|
if said != 1 {
|
|
t.Errorf("the refusal was said %d times, want once", said)
|
|
}
|
|
heard <- []byte(`{}`)
|
|
for {
|
|
c.mu.Lock()
|
|
z := c.trunkRetry["postgres"].next.IsZero()
|
|
c.mu.Unlock()
|
|
if z {
|
|
return
|
|
}
|
|
select {
|
|
case <-deadline:
|
|
t.Fatal("the event of the subscription that held did not clear the wait")
|
|
case <-time.After(2 * time.Millisecond):
|
|
}
|
|
}
|
|
}
|