The node agent: follows the wants and the node's compositions, applies one module at a time, backs a data module up first, holds it on a failed backup (R6-A A1, A2, A5, A6)
This commit is contained in:
@@ -0,0 +1,485 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const node = "n1"
|
||||
|
||||
func fpOf(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) }
|
||||
|
||||
// A watcher the test writes to; it says Caught once the keys written before the watch began are out.
|
||||
type feed struct {
|
||||
mu sync.Mutex
|
||||
ch chan Update
|
||||
rev uint64
|
||||
first []Update
|
||||
}
|
||||
|
||||
func newFeed() *feed { return &feed{ch: make(chan Update, 256)} }
|
||||
|
||||
func (f *feed) Watch(_ context.Context, _ string) (<-chan Update, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
for _, u := range f.first {
|
||||
f.ch <- u
|
||||
}
|
||||
f.ch <- Update{Caught: true}
|
||||
return f.ch, nil
|
||||
}
|
||||
|
||||
// before writes a key as if it was there when the watch began.
|
||||
func (f *feed) before(key string, value []byte) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.rev++
|
||||
f.first = append(f.first, Update{Key: key, Value: value, Revision: f.rev})
|
||||
}
|
||||
|
||||
func (f *feed) put(key string, value []byte) {
|
||||
f.mu.Lock()
|
||||
f.rev++
|
||||
r := f.rev
|
||||
f.mu.Unlock()
|
||||
f.ch <- Update{Key: key, Value: value, Revision: r}
|
||||
}
|
||||
|
||||
type haves struct {
|
||||
mu sync.Mutex
|
||||
stored map[string]Have
|
||||
log []Have
|
||||
removed []string
|
||||
}
|
||||
|
||||
func (h *haves) Read(_ context.Context, module string) (Have, bool) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
x, ok := h.stored[module]
|
||||
return x, ok
|
||||
}
|
||||
func (h *haves) Write(_ context.Context, x Have) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.stored == nil {
|
||||
h.stored = map[string]Have{}
|
||||
}
|
||||
h.stored[x.Module] = x
|
||||
h.log = append(h.log, x)
|
||||
}
|
||||
func (h *haves) Removed(_ context.Context, m string) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.removed = append(h.removed, m)
|
||||
}
|
||||
func (h *haves) last(module string) Have {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.stored[module]
|
||||
}
|
||||
|
||||
type bed struct {
|
||||
t *testing.T
|
||||
wants *feed
|
||||
comps *feed
|
||||
haves *haves
|
||||
mu sync.Mutex
|
||||
applied []string
|
||||
removed []string
|
||||
backups []string
|
||||
failWith string
|
||||
backup func(module string) Backup
|
||||
kept map[string][]json.RawMessage
|
||||
env Env
|
||||
}
|
||||
|
||||
func newBed(t *testing.T) *bed {
|
||||
b := &bed{t: t, wants: newFeed(), comps: newFeed(), haves: &haves{}}
|
||||
b.env = Env{Node: node, Wants: b.wants, Composed: b.comps, Haves: b.haves,
|
||||
Apply: func(_ context.Context, c delivery.Composed, _ []delivery.Composed) Result {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
name := c.Module
|
||||
if name == "" {
|
||||
name = NodeModule
|
||||
}
|
||||
b.applied = append(b.applied, name)
|
||||
if b.failWith != "" {
|
||||
return Result{Failed: true, Detail: b.failWith}
|
||||
}
|
||||
return Result{}
|
||||
},
|
||||
Remove: func(_ context.Context, m string, _ []delivery.Composed) Result {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.removed = append(b.removed, m)
|
||||
return Result{}
|
||||
},
|
||||
Backup: func(_ context.Context, m string) Backup {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.backups = append(b.backups, m)
|
||||
if b.backup != nil {
|
||||
return b.backup(m)
|
||||
}
|
||||
return Backup{Done: true}
|
||||
},
|
||||
Kept: func(m string) ([]json.RawMessage, bool) {
|
||||
r, ok := b.kept[m]
|
||||
return r, ok
|
||||
},
|
||||
Say: func(s string) { t.Log(s) },
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *bed) run() (stop func()) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); _ = New(b.env).Run(ctx) }()
|
||||
return func() { cancel(); <-done }
|
||||
}
|
||||
|
||||
func file(module, id, content string) json.RawMessage {
|
||||
raw, _ := json.Marshal(map[string]any{"id": module + "." + id, "type": "file", "path": "/tmp/r6/" + module + "/" + id,
|
||||
"content": content, "module": module})
|
||||
return raw
|
||||
}
|
||||
|
||||
// composition is a module's composition at a fingerprint: its key, its bytes and its digest, from the writer's code.
|
||||
func composition(t *testing.T, module, fp string, data bool, rs ...json.RawMessage) (key string, raw []byte, digest string) {
|
||||
t.Helper()
|
||||
c := delivery.Composed{Node: node, Module: module, FP: fp, Data: data, Resources: rs}
|
||||
var err error
|
||||
if module == NodeModule {
|
||||
c.Module, c.FP = "", ""
|
||||
key, err = delivery.ComposedNodeKey(node)
|
||||
} else {
|
||||
key, err = delivery.ComposedKey(node, module, fp)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if raw, err = delivery.EncodeComposed(c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if digest, err = delivery.ComposedDigestOf(c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, raw, digest
|
||||
}
|
||||
|
||||
func wantFor(t *testing.T, module, fp, digest, backup string) (string, []byte) {
|
||||
t.Helper()
|
||||
key, err := delivery.WantKey(node, module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, err := delivery.EncodeWant(delivery.Want{FP: fp, Composed: digest, Backup: backup})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, raw
|
||||
}
|
||||
|
||||
func eventually(t *testing.T, what string, ok func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for !ok() {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal(what)
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *bed) appliedList() []string {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return append([]string(nil), b.applied...)
|
||||
}
|
||||
|
||||
// R6-A AC1 (the agent, with a bus that is a fake): given a want naming a digest and that composition, the module is
|
||||
// applied and its have names the fingerprint and the digest; a composition whose digest no want names is not applied.
|
||||
func TestAWantAndItsCompositionApplyTheModuleAndTheHaveNamesBoth(t *testing.T) {
|
||||
b := newBed(t)
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
|
||||
fp := fpOf("a")
|
||||
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
|
||||
// A composition no want names yet is not applied.
|
||||
b.comps.put(ck, craw)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if got := b.appliedList(); len(got) != 0 {
|
||||
t.Fatalf("a composition no want names was applied: %v", got)
|
||||
}
|
||||
wk, wraw := wantFor(t, "web", fp, dig, "")
|
||||
b.wants.put(wk, wraw)
|
||||
eventually(t, "the module was not applied", func() bool { return len(b.appliedList()) == 1 })
|
||||
eventually(t, "the have does not name both", func() bool {
|
||||
h := b.haves.last("web")
|
||||
return h.FP == fp && h.Composed == dig && h.Phase == string(delivery.PhaseRunning)
|
||||
})
|
||||
|
||||
// A want naming a digest the composition at that fingerprint does not have: nothing is applied.
|
||||
fp2 := fpOf("b")
|
||||
ck2, craw2, _ := composition(t, "web", fp2, false, file("web", "x", "2"))
|
||||
b.comps.put(ck2, craw2)
|
||||
wk2, wraw2 := wantFor(t, "web", fp2, "sha256:"+strings.Repeat("0", 64), "")
|
||||
b.wants.put(wk2, wraw2)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if got := b.appliedList(); len(got) != 1 {
|
||||
t.Fatalf("a want naming another digest applied the composition: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// R6-A AC2: given a kept declaration and wants and compositions equal to it, when the agent starts, nothing is applied
|
||||
// and every have names its fingerprint and digest.
|
||||
func TestStartingOnWhatTheKeptDeclarationHoldsAppliesNothing(t *testing.T) {
|
||||
b := newBed(t)
|
||||
fp := fpOf("a")
|
||||
rs := []json.RawMessage{file("web", "x", "1"), file("web", "y", "2")}
|
||||
ck, craw, dig := composition(t, "web", fp, false, rs...)
|
||||
wk, wraw := wantFor(t, "web", fp, dig, "")
|
||||
b.comps.before(ck, craw)
|
||||
b.wants.before(wk, wraw)
|
||||
// The kept declaration holds the same resources, in another order and spacing.
|
||||
b.kept = map[string][]json.RawMessage{"web": {json.RawMessage(strings.ReplaceAll(string(rs[1]), ",", " , ")), rs[0]}}
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
eventually(t, "the have was not written", func() bool { return b.haves.last("web").Composed == dig })
|
||||
if h := b.haves.last("web"); h.FP != fp || h.Phase != string(delivery.PhaseRunning) {
|
||||
t.Fatalf("the have: %+v", h)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if got := b.appliedList(); len(got) != 0 {
|
||||
t.Fatalf("a module that already runs what the want names was applied: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// What the node-engine itself last applied is what runs after a restart, not the kept declaration.
|
||||
func TestStartingOnItsOwnHaveAppliesNothing(t *testing.T) {
|
||||
b := newBed(t)
|
||||
fp := fpOf("a")
|
||||
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
|
||||
wk, wraw := wantFor(t, "web", fp, dig, "")
|
||||
b.haves.stored = map[string]Have{"web": {Module: "web", FP: fp, Composed: dig, Phase: "running"}}
|
||||
b.comps.before(ck, craw)
|
||||
b.wants.before(wk, wraw)
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
if got := b.appliedList(); len(got) != 0 {
|
||||
t.Fatalf("a module that runs what the want names was applied after a restart: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A module whose kept resources differ from the composition is applied, and a failed apply is retried only by a change
|
||||
// of either key.
|
||||
func TestAFailedApplyIsRetriedOnlyByAChangeOfEitherKey(t *testing.T) {
|
||||
b := newBed(t)
|
||||
b.failWith = "web.x: no space"
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
fp := fpOf("a")
|
||||
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
|
||||
wk, wraw := wantFor(t, "web", fp, dig, "")
|
||||
b.comps.put(ck, craw)
|
||||
b.wants.put(wk, wraw)
|
||||
eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
|
||||
eventually(t, "the have does not say failed", func() bool {
|
||||
h := b.haves.last("web")
|
||||
return h.Phase == string(delivery.PhaseFailed) && strings.Contains(h.Detail, "no space") && h.FP == fp
|
||||
})
|
||||
// An unrelated key changes: not a retry.
|
||||
ck9, craw9, _ := composition(t, "web", fpOf("9"), false, file("web", "x", "9"))
|
||||
b.comps.put(ck9, craw9)
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
if n := len(b.appliedList()); n != 1 {
|
||||
t.Fatalf("a failed apply was retried by an unrelated key (%d applies)", n)
|
||||
}
|
||||
// The want is written again: the retry.
|
||||
b.mu.Lock()
|
||||
b.failWith = ""
|
||||
b.mu.Unlock()
|
||||
b.wants.put(wk, wraw)
|
||||
eventually(t, "the change of the want did not retry", func() bool { return len(b.appliedList()) == 2 })
|
||||
eventually(t, "not running", func() bool { return b.haves.last("web").Phase == string(delivery.PhaseRunning) })
|
||||
}
|
||||
|
||||
// R6-A AC5: a data module that ran before asks its backup before any resource changes; a refusal holds it, nothing is
|
||||
// applied and the have says held; when the want then says skip, the module is applied.
|
||||
func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) {
|
||||
b := newBed(t)
|
||||
fp1, fp2 := fpOf("a"), fpOf("b")
|
||||
ck1, craw1, dig1 := composition(t, "db", fp1, true, file("db", "x", "1"))
|
||||
wk1, wraw1 := wantFor(t, "db", fp1, dig1, "")
|
||||
b.comps.before(ck1, craw1)
|
||||
b.wants.before(wk1, wraw1)
|
||||
b.kept = map[string][]json.RawMessage{"db": {file("db", "x", "1")}}
|
||||
refuse := true
|
||||
b.backup = func(string) Backup {
|
||||
if refuse {
|
||||
return Backup{Words: "restic: the repository is locked"}
|
||||
}
|
||||
return Backup{Done: true}
|
||||
}
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
eventually(t, "baseline", func() bool { return b.haves.last("db").Composed == dig1 })
|
||||
|
||||
ck2, craw2, dig2 := composition(t, "db", fp2, true, file("db", "x", "2"))
|
||||
b.comps.put(ck2, craw2)
|
||||
wk2, wraw2 := wantFor(t, "db", fp2, dig2, "")
|
||||
b.wants.put(wk2, wraw2)
|
||||
eventually(t, "the backup was not asked", func() bool {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return len(b.backups) == 1
|
||||
})
|
||||
eventually(t, "the have does not say held", func() bool {
|
||||
h := b.haves.last("db")
|
||||
return h.Phase == string(delivery.PhaseHeld) && strings.Contains(h.Detail, "repository is locked") && h.FP == fp1
|
||||
})
|
||||
if got := b.appliedList(); len(got) != 0 {
|
||||
t.Fatalf("something was applied behind a failed backup: %v", got)
|
||||
}
|
||||
|
||||
// The answer arrives in the module's own want: back up again (it fails again), then skip.
|
||||
wk, again := wantFor(t, "db", fp2, dig2, delivery.BackupAgain)
|
||||
b.wants.put(wk, again)
|
||||
eventually(t, "back up again did not ask the backup", func() bool {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return len(b.backups) == 2
|
||||
})
|
||||
if got := b.appliedList(); len(got) != 0 {
|
||||
t.Fatalf("applied behind a second failed backup: %v", got)
|
||||
}
|
||||
_, skip := wantFor(t, "db", fp2, dig2, delivery.BackupSkip)
|
||||
b.wants.put(wk, skip)
|
||||
eventually(t, "skip did not apply the module", func() bool { return len(b.appliedList()) == 1 })
|
||||
b.mu.Lock()
|
||||
n := len(b.backups)
|
||||
b.mu.Unlock()
|
||||
if n != 2 {
|
||||
t.Fatalf("a backup was asked after the answer was to skip: %d asks", n)
|
||||
}
|
||||
eventually(t, "not running at the new fingerprint", func() bool {
|
||||
h := b.haves.last("db")
|
||||
return h.Phase == string(delivery.PhaseRunning) && h.FP == fp2 && h.Composed == dig2
|
||||
})
|
||||
}
|
||||
|
||||
// A data module with a good backup is applied after it; one that never ran has nothing to back up.
|
||||
func TestABackupDoneIsFollowedByTheApplyAndANewModuleIsNotBackedUp(t *testing.T) {
|
||||
b := newBed(t)
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
fp := fpOf("c")
|
||||
ck, craw, dig := composition(t, "db", fp, true, file("db", "x", "1"))
|
||||
wk, wraw := wantFor(t, "db", fp, dig, "")
|
||||
b.comps.put(ck, craw)
|
||||
b.wants.put(wk, wraw)
|
||||
eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
|
||||
b.mu.Lock()
|
||||
n := len(b.backups)
|
||||
b.mu.Unlock()
|
||||
if n != 0 {
|
||||
t.Fatalf("a module that never ran was backed up (%d)", n)
|
||||
}
|
||||
fp2 := fpOf("d")
|
||||
ck2, craw2, dig2 := composition(t, "db", fp2, true, file("db", "x", "2"))
|
||||
wk2, wraw2 := wantFor(t, "db", fp2, dig2, "")
|
||||
b.comps.put(ck2, craw2)
|
||||
b.wants.put(wk2, wraw2)
|
||||
eventually(t, "second not applied", func() bool { return len(b.appliedList()) == 2 })
|
||||
b.mu.Lock()
|
||||
n = len(b.backups)
|
||||
b.mu.Unlock()
|
||||
if n != 1 {
|
||||
t.Fatalf("a module that ran before was backed up %d times, want once", n)
|
||||
}
|
||||
}
|
||||
|
||||
// R6-A AC4 at the agent: a removed want removes that module only, and its have is written removed.
|
||||
func TestARemovedWantRemovesThatModuleOnly(t *testing.T) {
|
||||
b := newBed(t)
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
fp := fpOf("a")
|
||||
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
|
||||
b.comps.put(ck, craw)
|
||||
wk, wraw := wantFor(t, "web", fp, dig, "")
|
||||
b.wants.put(wk, wraw)
|
||||
eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
|
||||
gone, _ := delivery.EncodeWant(delivery.Want{Removed: true})
|
||||
b.wants.put(wk, gone)
|
||||
eventually(t, "not removed", func() bool {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return len(b.removed) == 1 && b.removed[0] == "web"
|
||||
})
|
||||
eventually(t, "the have was not written removed", func() bool {
|
||||
b.haves.mu.Lock()
|
||||
defer b.haves.mu.Unlock()
|
||||
return len(b.haves.removed) == 1
|
||||
})
|
||||
}
|
||||
|
||||
// The node's own composition is applied before any module's, whichever key came first.
|
||||
func TestTheNodesOwnCompositionComesBeforeAnyModules(t *testing.T) {
|
||||
b := newBed(t)
|
||||
fp := fpOf("a")
|
||||
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
|
||||
wk, wraw := wantFor(t, "web", fp, dig, "")
|
||||
nk, nraw, ndig := composition(t, NodeModule, "", false)
|
||||
nwk, err := delivery.WantNodeKey(node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
nwraw, err := delivery.EncodeNodeWant(delivery.NodeWant{Composed: ndig})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.comps.before(ck, craw)
|
||||
b.comps.before(nk, nraw)
|
||||
b.wants.before(wk, wraw)
|
||||
b.wants.before(nwk, nwraw)
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
eventually(t, "both not applied", func() bool { return len(b.appliedList()) == 2 })
|
||||
if got := b.appliedList(); got[0] != NodeModule || got[1] != "web" {
|
||||
t.Fatalf("the order was %v, the node's own first", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A composition the verifier refuses is never applied (the hook of ADR 0539 §3).
|
||||
func TestARefusedCompositionIsNeverApplied(t *testing.T) {
|
||||
b := newBed(t)
|
||||
b.env.Verify = func(string, []byte) error { return errRefused }
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
fp := fpOf("a")
|
||||
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
|
||||
b.comps.put(ck, craw)
|
||||
wk, wraw := wantFor(t, "web", fp, dig, "")
|
||||
b.wants.put(wk, wraw)
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
if got := b.appliedList(); len(got) != 0 {
|
||||
t.Fatalf("a refused composition was applied: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
type refusal string
|
||||
|
||||
func (r refusal) Error() string { return string(r) }
|
||||
|
||||
var errRefused error = refusal("signed by a key this node does not trust")
|
||||
Reference in New Issue
Block a user