Files
mesh-host/internal/agent/agent_test.go
T

559 lines
18 KiB
Go

package agent
import (
"context"
"crypto/ed25519"
"encoding/json"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
const node = "n1"
// The keys of the tests: the controller's and the operator's are trusted, a stranger's is not.
var controllerPub, controllerKey = newKey()
var operatorPub, operatorKey = newKey()
var _, strangerKey = newKey()
func newKey() (ed25519.PublicKey, ed25519.PrivateKey) {
pub, key, err := ed25519.GenerateKey(nil)
if err != nil {
panic(err)
}
return pub, key
}
func fpOf(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) }
// A watcher the test writes to; it says Caught once the keys written before the watch began are out.
type feed struct {
mu sync.Mutex
ch chan Update
rev uint64
first []Update
}
func newFeed() *feed { return &feed{ch: make(chan Update, 256)} }
func (f *feed) Watch(_ context.Context, _ string) (<-chan Update, error) {
f.mu.Lock()
defer f.mu.Unlock()
for _, u := range f.first {
f.ch <- u
}
f.ch <- Update{Caught: true}
return f.ch, nil
}
// before writes a key as if it was there when the watch began.
func (f *feed) before(key string, value []byte) {
f.mu.Lock()
defer f.mu.Unlock()
f.rev++
f.first = append(f.first, Update{Key: key, Value: value, Revision: f.rev})
}
func (f *feed) put(key string, value []byte) {
f.mu.Lock()
f.rev++
r := f.rev
f.mu.Unlock()
f.ch <- Update{Key: key, Value: value, Revision: r}
}
type haves struct {
mu sync.Mutex
stored map[string]Have
log []Have
removed []string
}
func (h *haves) Read(_ context.Context, module string) (Have, bool) {
h.mu.Lock()
defer h.mu.Unlock()
x, ok := h.stored[module]
return x, ok
}
func (h *haves) Write(_ context.Context, x Have) {
h.mu.Lock()
defer h.mu.Unlock()
if h.stored == nil {
h.stored = map[string]Have{}
}
h.stored[x.Module] = x
h.log = append(h.log, x)
}
func (h *haves) Removed(_ context.Context, m string) {
h.mu.Lock()
defer h.mu.Unlock()
h.removed = append(h.removed, m)
}
func (h *haves) last(module string) Have {
h.mu.Lock()
defer h.mu.Unlock()
return h.stored[module]
}
type bed struct {
t *testing.T
wants *feed
comps *feed
haves *haves
mu sync.Mutex
applied []string
removed []string
backups []string
failWith string
backup func(module string) Backup
kept map[string][]json.RawMessage
env Env
}
func newBed(t *testing.T) *bed {
b := &bed{t: t, wants: newFeed(), comps: newFeed(), haves: &haves{}}
b.env = Env{Node: node, Wants: b.wants, Composed: b.comps, Haves: b.haves,
Apply: func(_ context.Context, c delivery.Composed, _ []delivery.Composed) Result {
b.mu.Lock()
defer b.mu.Unlock()
name := c.Module
if name == "" {
name = NodeModule
}
b.applied = append(b.applied, name)
if b.failWith != "" {
return Result{Failed: true, Detail: b.failWith}
}
return Result{}
},
Remove: func(_ context.Context, m string, _ []delivery.Composed) Result {
b.mu.Lock()
defer b.mu.Unlock()
b.removed = append(b.removed, m)
return Result{}
},
Backup: func(_ context.Context, m string) Backup {
b.mu.Lock()
defer b.mu.Unlock()
b.backups = append(b.backups, m)
if b.backup != nil {
return b.backup(m)
}
return Backup{Done: true}
},
Kept: func(m string) ([]json.RawMessage, bool) {
r, ok := b.kept[m]
return r, ok
},
Trusted: func() []ed25519.PublicKey { return []ed25519.PublicKey{controllerPub, operatorPub} },
Say: func(s string) { t.Log(s) },
}
return b
}
func (b *bed) run() (stop func()) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() { defer close(done); _ = New(b.env).Run(ctx) }()
return func() { cancel(); <-done }
}
func file(module, id, content string) json.RawMessage {
raw, _ := json.Marshal(map[string]any{"id": module + "." + id, "type": "file", "path": "/tmp/r6/" + module + "/" + id,
"content": content, "module": module})
return raw
}
// composition is a module's composition at a fingerprint: its key, its bytes and its digest, from the writer's code.
func composition(t *testing.T, module, fp string, data bool, rs ...json.RawMessage) (key string, raw []byte, digest string) {
t.Helper()
c := delivery.Composed{Node: node, Module: module, FP: fp, Data: data, Resources: rs}
var err error
if module == NodeModule {
c.Module, c.FP = "", ""
key, err = delivery.ComposedNodeKey(node)
} else {
key, err = delivery.ComposedKey(node, module, fp)
}
if err != nil {
t.Fatal(err)
}
return signedBy(t, c, key, controllerKey)
}
// signedBy is c signed with signer, as the writer's code signs it: its key, its bytes and its digest.
func signedBy(t *testing.T, c delivery.Composed, key string, signer ed25519.PrivateKey) (string, []byte, string) {
t.Helper()
signed, err := delivery.SignComposed(c, signer)
if err != nil {
t.Fatal(err)
}
raw, err := delivery.EncodeComposed(signed)
if err != nil {
t.Fatal(err)
}
digest, err := delivery.ComposedDigestOf(signed)
if err != nil {
t.Fatal(err)
}
return key, raw, digest
}
func wantFor(t *testing.T, module, fp, digest, backup string) (string, []byte) {
t.Helper()
key, err := delivery.WantKey(node, module)
if err != nil {
t.Fatal(err)
}
raw, err := delivery.EncodeWant(delivery.Want{FP: fp, Composed: digest, Backup: backup})
if err != nil {
t.Fatal(err)
}
return key, raw
}
func eventually(t *testing.T, what string, ok func() bool) {
t.Helper()
deadline := time.Now().Add(3 * time.Second)
for !ok() {
if time.Now().After(deadline) {
t.Fatal(what)
}
time.Sleep(5 * time.Millisecond)
}
}
func (b *bed) appliedList() []string {
b.mu.Lock()
defer b.mu.Unlock()
return append([]string(nil), b.applied...)
}
// R6-A AC1 (the agent, with a bus that is a fake): given a want naming a digest and that composition, the module is
// applied and its have names the fingerprint and the digest; a composition whose digest no want names is not applied.
func TestAWantAndItsCompositionApplyTheModuleAndTheHaveNamesBoth(t *testing.T) {
b := newBed(t)
stop := b.run()
defer stop()
fp := fpOf("a")
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
// A composition no want names yet is not applied.
b.comps.put(ck, craw)
time.Sleep(100 * time.Millisecond)
if got := b.appliedList(); len(got) != 0 {
t.Fatalf("a composition no want names was applied: %v", got)
}
wk, wraw := wantFor(t, "web", fp, dig, "")
b.wants.put(wk, wraw)
eventually(t, "the module was not applied", func() bool { return len(b.appliedList()) == 1 })
eventually(t, "the have does not name both", func() bool {
h := b.haves.last("web")
return h.FP == fp && h.Composed == dig && h.Phase == string(delivery.PhaseRunning)
})
// A want naming a digest the composition at that fingerprint does not have: nothing is applied.
fp2 := fpOf("b")
ck2, craw2, _ := composition(t, "web", fp2, false, file("web", "x", "2"))
b.comps.put(ck2, craw2)
wk2, wraw2 := wantFor(t, "web", fp2, "sha256:"+strings.Repeat("0", 64), "")
b.wants.put(wk2, wraw2)
time.Sleep(100 * time.Millisecond)
if got := b.appliedList(); len(got) != 1 {
t.Fatalf("a want naming another digest applied the composition: %v", got)
}
}
// R6-A AC2: given a kept declaration and wants and compositions equal to it, when the agent starts, nothing is applied
// and every have names its fingerprint and digest.
func TestStartingOnWhatTheKeptDeclarationHoldsAppliesNothing(t *testing.T) {
b := newBed(t)
fp := fpOf("a")
rs := []json.RawMessage{file("web", "x", "1"), file("web", "y", "2")}
ck, craw, dig := composition(t, "web", fp, false, rs...)
wk, wraw := wantFor(t, "web", fp, dig, "")
b.comps.before(ck, craw)
b.wants.before(wk, wraw)
// The kept declaration holds the same resources, in another order and spacing.
b.kept = map[string][]json.RawMessage{"web": {json.RawMessage(strings.ReplaceAll(string(rs[1]), ",", " , ")), rs[0]}}
stop := b.run()
defer stop()
eventually(t, "the have was not written", func() bool { return b.haves.last("web").Composed == dig })
if h := b.haves.last("web"); h.FP != fp || h.Phase != string(delivery.PhaseRunning) {
t.Fatalf("the have: %+v", h)
}
time.Sleep(50 * time.Millisecond)
if got := b.appliedList(); len(got) != 0 {
t.Fatalf("a module that already runs what the want names was applied: %v", got)
}
}
// What the node-engine itself last applied is what runs after a restart, not the kept declaration.
func TestStartingOnItsOwnHaveAppliesNothing(t *testing.T) {
b := newBed(t)
fp := fpOf("a")
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
wk, wraw := wantFor(t, "web", fp, dig, "")
b.haves.stored = map[string]Have{"web": {Module: "web", FP: fp, Composed: dig, Phase: "running"}}
b.comps.before(ck, craw)
b.wants.before(wk, wraw)
stop := b.run()
defer stop()
time.Sleep(150 * time.Millisecond)
if got := b.appliedList(); len(got) != 0 {
t.Fatalf("a module that runs what the want names was applied after a restart: %v", got)
}
}
// A module whose kept resources differ from the composition is applied, and a failed apply is retried only by a change
// of either key.
func TestAFailedApplyIsRetriedOnlyByAChangeOfEitherKey(t *testing.T) {
b := newBed(t)
b.failWith = "web.x: no space"
stop := b.run()
defer stop()
fp := fpOf("a")
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
wk, wraw := wantFor(t, "web", fp, dig, "")
b.comps.put(ck, craw)
b.wants.put(wk, wraw)
eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
eventually(t, "the have does not say failed", func() bool {
h := b.haves.last("web")
return h.Phase == string(delivery.PhaseFailed) && strings.Contains(h.Detail, "no space") && h.FP == fp
})
// An unrelated key changes: not a retry.
ck9, craw9, _ := composition(t, "web", fpOf("9"), false, file("web", "x", "9"))
b.comps.put(ck9, craw9)
time.Sleep(150 * time.Millisecond)
if n := len(b.appliedList()); n != 1 {
t.Fatalf("a failed apply was retried by an unrelated key (%d applies)", n)
}
// The want is written again: the retry.
b.mu.Lock()
b.failWith = ""
b.mu.Unlock()
b.wants.put(wk, wraw)
eventually(t, "the change of the want did not retry", func() bool { return len(b.appliedList()) == 2 })
eventually(t, "not running", func() bool { return b.haves.last("web").Phase == string(delivery.PhaseRunning) })
}
// R6-A AC5: a data module that ran before asks its backup before any resource changes; a refusal holds it, nothing is
// applied and the have says held; when the want then says skip, the module is applied.
func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) {
b := newBed(t)
fp1, fp2 := fpOf("a"), fpOf("b")
ck1, craw1, dig1 := composition(t, "db", fp1, true, file("db", "x", "1"))
wk1, wraw1 := wantFor(t, "db", fp1, dig1, "")
b.comps.before(ck1, craw1)
b.wants.before(wk1, wraw1)
b.kept = map[string][]json.RawMessage{"db": {file("db", "x", "1")}}
refuse := true
b.backup = func(string) Backup {
if refuse {
return Backup{Words: "restic: the repository is locked"}
}
return Backup{Done: true}
}
stop := b.run()
defer stop()
eventually(t, "baseline", func() bool { return b.haves.last("db").Composed == dig1 })
ck2, craw2, dig2 := composition(t, "db", fp2, true, file("db", "x", "2"))
b.comps.put(ck2, craw2)
wk2, wraw2 := wantFor(t, "db", fp2, dig2, "")
b.wants.put(wk2, wraw2)
eventually(t, "the backup was not asked", func() bool {
b.mu.Lock()
defer b.mu.Unlock()
return len(b.backups) == 1
})
eventually(t, "the have does not say held", func() bool {
h := b.haves.last("db")
return h.Phase == string(delivery.PhaseHeld) && strings.Contains(h.Detail, "repository is locked") && h.FP == fp1
})
if got := b.appliedList(); len(got) != 0 {
t.Fatalf("something was applied behind a failed backup: %v", got)
}
// The answer arrives in the module's own want: back up again (it fails again), then skip.
wk, again := wantFor(t, "db", fp2, dig2, delivery.BackupAgain)
b.wants.put(wk, again)
eventually(t, "back up again did not ask the backup", func() bool {
b.mu.Lock()
defer b.mu.Unlock()
return len(b.backups) == 2
})
if got := b.appliedList(); len(got) != 0 {
t.Fatalf("applied behind a second failed backup: %v", got)
}
_, skip := wantFor(t, "db", fp2, dig2, delivery.BackupSkip)
b.wants.put(wk, skip)
eventually(t, "skip did not apply the module", func() bool { return len(b.appliedList()) == 1 })
b.mu.Lock()
n := len(b.backups)
b.mu.Unlock()
if n != 2 {
t.Fatalf("a backup was asked after the answer was to skip: %d asks", n)
}
eventually(t, "not running at the new fingerprint", func() bool {
h := b.haves.last("db")
return h.Phase == string(delivery.PhaseRunning) && h.FP == fp2 && h.Composed == dig2
})
}
// A data module with a good backup is applied after it; one that never ran has nothing to back up.
func TestABackupDoneIsFollowedByTheApplyAndANewModuleIsNotBackedUp(t *testing.T) {
b := newBed(t)
stop := b.run()
defer stop()
fp := fpOf("c")
ck, craw, dig := composition(t, "db", fp, true, file("db", "x", "1"))
wk, wraw := wantFor(t, "db", fp, dig, "")
b.comps.put(ck, craw)
b.wants.put(wk, wraw)
eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
b.mu.Lock()
n := len(b.backups)
b.mu.Unlock()
if n != 0 {
t.Fatalf("a module that never ran was backed up (%d)", n)
}
fp2 := fpOf("d")
ck2, craw2, dig2 := composition(t, "db", fp2, true, file("db", "x", "2"))
wk2, wraw2 := wantFor(t, "db", fp2, dig2, "")
b.comps.put(ck2, craw2)
b.wants.put(wk2, wraw2)
eventually(t, "second not applied", func() bool { return len(b.appliedList()) == 2 })
b.mu.Lock()
n = len(b.backups)
b.mu.Unlock()
if n != 1 {
t.Fatalf("a module that ran before was backed up %d times, want once", n)
}
}
// R6-A AC4 at the agent: a removed want removes that module only, and its have is written removed.
func TestARemovedWantRemovesThatModuleOnly(t *testing.T) {
b := newBed(t)
stop := b.run()
defer stop()
fp := fpOf("a")
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
b.comps.put(ck, craw)
wk, wraw := wantFor(t, "web", fp, dig, "")
b.wants.put(wk, wraw)
eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
gone, _ := delivery.EncodeWant(delivery.Want{Removed: true})
b.wants.put(wk, gone)
eventually(t, "not removed", func() bool {
b.mu.Lock()
defer b.mu.Unlock()
return len(b.removed) == 1 && b.removed[0] == "web"
})
eventually(t, "the have was not written removed", func() bool {
b.haves.mu.Lock()
defer b.haves.mu.Unlock()
return len(b.haves.removed) == 1
})
}
// The node's own composition is applied before any module's, whichever key came first.
func TestTheNodesOwnCompositionComesBeforeAnyModules(t *testing.T) {
b := newBed(t)
fp := fpOf("a")
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
wk, wraw := wantFor(t, "web", fp, dig, "")
nk, nraw, ndig := composition(t, NodeModule, "", false)
nwk, err := delivery.WantNodeKey(node)
if err != nil {
t.Fatal(err)
}
nwraw, err := delivery.EncodeNodeWant(delivery.NodeWant{Composed: ndig})
if err != nil {
t.Fatal(err)
}
b.comps.before(ck, craw)
b.comps.before(nk, nraw)
b.wants.before(wk, wraw)
b.wants.before(nwk, nwraw)
stop := b.run()
defer stop()
eventually(t, "both not applied", func() bool { return len(b.appliedList()) == 2 })
if got := b.appliedList(); got[0] != NodeModule || got[1] != "web" {
t.Fatalf("the order was %v, the node's own first", got)
}
}
// R6-A AC9: a composition signed by either trusted key is applied; one signed by another key, one changed after it was
// signed and one unsigned are refused and nothing of them is applied (ADR 0539 §3).
func TestOnlyACompositionSignedByATrustedKeyIsApplied(t *testing.T) {
b := newBed(t)
stop := b.run()
defer stop()
plain := func(module, fp, content string) delivery.Composed {
return delivery.Composed{Node: node, Module: module, FP: fp, Resources: []json.RawMessage{file(module, "x", content)}}
}
put := func(c delivery.Composed, signer ed25519.PrivateKey, tamper func([]byte) []byte) {
key, _ := delivery.ComposedKey(node, c.Module, c.FP)
var raw []byte
var dig string
if signer == nil {
raw, _ = delivery.EncodeComposed(c)
dig, _ = delivery.ComposedDigestOf(c)
} else {
_, raw, dig = signedBy(t, c, key, signer)
}
if tamper != nil {
raw = tamper(raw)
}
b.comps.put(key, raw)
wk, wraw := wantFor(t, c.Module, c.FP, dig, "")
b.wants.put(wk, wraw)
}
// Refused: signed by a stranger, changed after signing, unsigned.
put(plain("stranger", fpOf("1"), "x"), strangerKey, nil)
put(plain("changed", fpOf("2"), "before"), controllerKey, func(raw []byte) []byte {
return []byte(strings.Replace(string(raw), "before", "after!", 1))
})
put(plain("unsigned", fpOf("3"), "x"), nil, nil)
time.Sleep(200 * time.Millisecond)
if got := b.appliedList(); len(got) != 0 {
t.Fatalf("a composition that is not signed by a trusted key was applied: %v", got)
}
// Applied: signed by the controller, signed by the operator.
put(plain("bycontroller", fpOf("4"), "x"), controllerKey, nil)
put(plain("byoperator", fpOf("5"), "x"), operatorKey, nil)
eventually(t, "the trusted signers' compositions were not applied", func() bool { return len(b.appliedList()) == 2 })
got := b.appliedList()
if !(got[0] == "bycontroller" && got[1] == "byoperator") && !(got[0] == "byoperator" && got[1] == "bycontroller") {
t.Fatalf("applied %v", got)
}
if h := b.haves.last("byoperator"); h.Phase != string(delivery.PhaseRunning) || h.FP != fpOf("5") {
t.Fatalf("the operator's module has no running have: %+v", h)
}
}
// With no key trusted, nothing is applied: a node that does not know its signers fails closed.
func TestWithNoTrustedKeyNothingIsApplied(t *testing.T) {
b := newBed(t)
b.env.Trusted = nil
stop := b.run()
defer stop()
fp := fpOf("a")
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
b.comps.put(ck, craw)
wk, wraw := wantFor(t, "web", fp, dig, "")
b.wants.put(wk, wraw)
time.Sleep(200 * time.Millisecond)
if got := b.appliedList(); len(got) != 0 {
t.Fatalf("applied with no trusted key: %v", got)
}
}