package agent import ( "context" "crypto/ed25519" "encoding/json" "strings" "sync" "testing" "time" "git.novox.be/novox/mesh-sdk/go/delivery" ) const node = "n1" // The keys of the tests: the controller's and the operator's are trusted, a stranger's is not. var controllerPub, controllerKey = newKey() var operatorPub, operatorKey = newKey() var _, strangerKey = newKey() func newKey() (ed25519.PublicKey, ed25519.PrivateKey) { pub, key, err := ed25519.GenerateKey(nil) if err != nil { panic(err) } return pub, key } func fpOf(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) } // A watcher the test writes to; it says Caught once the keys written before the watch began are out. type feed struct { mu sync.Mutex ch chan Update rev uint64 first []Update } func newFeed() *feed { return &feed{ch: make(chan Update, 256)} } func (f *feed) Watch(_ context.Context, _ string) (<-chan Update, error) { f.mu.Lock() defer f.mu.Unlock() for _, u := range f.first { f.ch <- u } f.ch <- Update{Caught: true} return f.ch, nil } // before writes a key as if it was there when the watch began. func (f *feed) before(key string, value []byte) { f.mu.Lock() defer f.mu.Unlock() f.rev++ f.first = append(f.first, Update{Key: key, Value: value, Revision: f.rev}) } func (f *feed) put(key string, value []byte) { f.mu.Lock() f.rev++ r := f.rev f.mu.Unlock() f.ch <- Update{Key: key, Value: value, Revision: r} } type haves struct { mu sync.Mutex stored map[string]Have log []Have removed []string } func (h *haves) Read(_ context.Context, module string) (Have, bool) { h.mu.Lock() defer h.mu.Unlock() x, ok := h.stored[module] return x, ok } func (h *haves) Write(_ context.Context, x Have) { h.mu.Lock() defer h.mu.Unlock() if h.stored == nil { h.stored = map[string]Have{} } h.stored[x.Module] = x h.log = append(h.log, x) } func (h *haves) Removed(_ context.Context, m string) { h.mu.Lock() defer h.mu.Unlock() h.removed = append(h.removed, m) } func (h *haves) last(module string) Have { h.mu.Lock() defer h.mu.Unlock() return h.stored[module] } type bed struct { t *testing.T wants *feed comps *feed haves *haves mu sync.Mutex applied []string removed []string backups []string failWith string backup func(module string) Backup kept map[string][]json.RawMessage env Env } func newBed(t *testing.T) *bed { b := &bed{t: t, wants: newFeed(), comps: newFeed(), haves: &haves{}} b.env = Env{Node: node, Wants: b.wants, Composed: b.comps, Haves: b.haves, Apply: func(_ context.Context, c delivery.Composed, _ []delivery.Composed) Result { b.mu.Lock() defer b.mu.Unlock() name := c.Module if name == "" { name = NodeModule } b.applied = append(b.applied, name) if b.failWith != "" { return Result{Failed: true, Detail: b.failWith} } return Result{} }, Remove: func(_ context.Context, m string, _ []delivery.Composed) Result { b.mu.Lock() defer b.mu.Unlock() b.removed = append(b.removed, m) return Result{} }, Backup: func(_ context.Context, m string) Backup { b.mu.Lock() defer b.mu.Unlock() b.backups = append(b.backups, m) if b.backup != nil { return b.backup(m) } return Backup{Done: true} }, Kept: func(m string) ([]json.RawMessage, bool) { r, ok := b.kept[m] return r, ok }, Trusted: func() []ed25519.PublicKey { return []ed25519.PublicKey{controllerPub, operatorPub} }, Say: func(s string) { t.Log(s) }, } return b } func (b *bed) run() (stop func()) { ctx, cancel := context.WithCancel(context.Background()) done := make(chan struct{}) go func() { defer close(done); _ = New(b.env).Run(ctx) }() return func() { cancel(); <-done } } func file(module, id, content string) json.RawMessage { raw, _ := json.Marshal(map[string]any{"id": module + "." + id, "type": "file", "path": "/tmp/r6/" + module + "/" + id, "content": content, "module": module}) return raw } // composition is a module's composition at a fingerprint: its key, its bytes and its digest, from the writer's code. func composition(t *testing.T, module, fp string, data bool, rs ...json.RawMessage) (key string, raw []byte, digest string) { t.Helper() c := delivery.Composed{Node: node, Module: module, FP: fp, Data: data, Resources: rs} var err error if module == NodeModule { c.Module, c.FP = "", "" key, err = delivery.ComposedNodeKey(node) } else { key, err = delivery.ComposedKey(node, module, fp) } if err != nil { t.Fatal(err) } return signedBy(t, c, key, controllerKey) } // signedBy is c signed with signer, as the writer's code signs it: its key, its bytes and its digest. func signedBy(t *testing.T, c delivery.Composed, key string, signer ed25519.PrivateKey) (string, []byte, string) { t.Helper() signed, err := delivery.SignComposed(c, signer) if err != nil { t.Fatal(err) } raw, err := delivery.EncodeComposed(signed) if err != nil { t.Fatal(err) } digest, err := delivery.ComposedDigestOf(signed) if err != nil { t.Fatal(err) } return key, raw, digest } func wantFor(t *testing.T, module, fp, digest, backup string) (string, []byte) { t.Helper() key, err := delivery.WantKey(node, module) if err != nil { t.Fatal(err) } raw, err := delivery.EncodeWant(delivery.Want{FP: fp, Composed: digest, Backup: backup}) if err != nil { t.Fatal(err) } return key, raw } func eventually(t *testing.T, what string, ok func() bool) { t.Helper() deadline := time.Now().Add(3 * time.Second) for !ok() { if time.Now().After(deadline) { t.Fatal(what) } time.Sleep(5 * time.Millisecond) } } func (b *bed) appliedList() []string { b.mu.Lock() defer b.mu.Unlock() return append([]string(nil), b.applied...) } // R6-A AC1 (the agent, with a bus that is a fake): given a want naming a digest and that composition, the module is // applied and its have names the fingerprint and the digest; a composition whose digest no want names is not applied. func TestAWantAndItsCompositionApplyTheModuleAndTheHaveNamesBoth(t *testing.T) { b := newBed(t) stop := b.run() defer stop() fp := fpOf("a") ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1")) // A composition no want names yet is not applied. b.comps.put(ck, craw) time.Sleep(100 * time.Millisecond) if got := b.appliedList(); len(got) != 0 { t.Fatalf("a composition no want names was applied: %v", got) } wk, wraw := wantFor(t, "web", fp, dig, "") b.wants.put(wk, wraw) eventually(t, "the module was not applied", func() bool { return len(b.appliedList()) == 1 }) eventually(t, "the have does not name both", func() bool { h := b.haves.last("web") return h.FP == fp && h.Composed == dig && h.Phase == string(delivery.PhaseRunning) }) // A want naming a digest the composition at that fingerprint does not have: nothing is applied. fp2 := fpOf("b") ck2, craw2, _ := composition(t, "web", fp2, false, file("web", "x", "2")) b.comps.put(ck2, craw2) wk2, wraw2 := wantFor(t, "web", fp2, "sha256:"+strings.Repeat("0", 64), "") b.wants.put(wk2, wraw2) time.Sleep(100 * time.Millisecond) if got := b.appliedList(); len(got) != 1 { t.Fatalf("a want naming another digest applied the composition: %v", got) } } // R6-A AC2: given a kept declaration and wants and compositions equal to it, when the agent starts, nothing is applied // and every have names its fingerprint and digest. func TestStartingOnWhatTheKeptDeclarationHoldsAppliesNothing(t *testing.T) { b := newBed(t) fp := fpOf("a") rs := []json.RawMessage{file("web", "x", "1"), file("web", "y", "2")} ck, craw, dig := composition(t, "web", fp, false, rs...) wk, wraw := wantFor(t, "web", fp, dig, "") b.comps.before(ck, craw) b.wants.before(wk, wraw) // The kept declaration holds the same resources, in another order and spacing. b.kept = map[string][]json.RawMessage{"web": {json.RawMessage(strings.ReplaceAll(string(rs[1]), ",", " , ")), rs[0]}} stop := b.run() defer stop() eventually(t, "the have was not written", func() bool { return b.haves.last("web").Composed == dig }) if h := b.haves.last("web"); h.FP != fp || h.Phase != string(delivery.PhaseRunning) { t.Fatalf("the have: %+v", h) } time.Sleep(50 * time.Millisecond) if got := b.appliedList(); len(got) != 0 { t.Fatalf("a module that already runs what the want names was applied: %v", got) } } // What the node-engine itself last applied is what runs after a restart, not the kept declaration. func TestStartingOnItsOwnHaveAppliesNothing(t *testing.T) { b := newBed(t) fp := fpOf("a") ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1")) wk, wraw := wantFor(t, "web", fp, dig, "") b.haves.stored = map[string]Have{"web": {Module: "web", FP: fp, Composed: dig, Phase: "running"}} b.comps.before(ck, craw) b.wants.before(wk, wraw) stop := b.run() defer stop() time.Sleep(150 * time.Millisecond) if got := b.appliedList(); len(got) != 0 { t.Fatalf("a module that runs what the want names was applied after a restart: %v", got) } } // A module whose kept resources differ from the composition is applied, and a failed apply is retried only by a change // of either key. func TestAFailedApplyIsRetriedOnlyByAChangeOfEitherKey(t *testing.T) { b := newBed(t) b.failWith = "web.x: no space" stop := b.run() defer stop() fp := fpOf("a") ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1")) wk, wraw := wantFor(t, "web", fp, dig, "") b.comps.put(ck, craw) b.wants.put(wk, wraw) eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 }) eventually(t, "the have does not say failed", func() bool { h := b.haves.last("web") return h.Phase == string(delivery.PhaseFailed) && strings.Contains(h.Detail, "no space") && h.FP == fp }) // An unrelated key changes: not a retry. ck9, craw9, _ := composition(t, "web", fpOf("9"), false, file("web", "x", "9")) b.comps.put(ck9, craw9) time.Sleep(150 * time.Millisecond) if n := len(b.appliedList()); n != 1 { t.Fatalf("a failed apply was retried by an unrelated key (%d applies)", n) } // The want is written again: the retry. b.mu.Lock() b.failWith = "" b.mu.Unlock() b.wants.put(wk, wraw) eventually(t, "the change of the want did not retry", func() bool { return len(b.appliedList()) == 2 }) eventually(t, "not running", func() bool { return b.haves.last("web").Phase == string(delivery.PhaseRunning) }) } // R6-A AC5: a data module that ran before asks its backup before any resource changes; a refusal holds it, nothing is // applied and the have says held; when the want then says skip, the module is applied. func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) { b := newBed(t) fp1, fp2 := fpOf("a"), fpOf("b") ck1, craw1, dig1 := composition(t, "db", fp1, true, file("db", "x", "1")) wk1, wraw1 := wantFor(t, "db", fp1, dig1, "") b.comps.before(ck1, craw1) b.wants.before(wk1, wraw1) b.kept = map[string][]json.RawMessage{"db": {file("db", "x", "1")}} refuse := true b.backup = func(string) Backup { if refuse { return Backup{Words: "restic: the repository is locked"} } return Backup{Done: true} } stop := b.run() defer stop() eventually(t, "baseline", func() bool { return b.haves.last("db").Composed == dig1 }) ck2, craw2, dig2 := composition(t, "db", fp2, true, file("db", "x", "2")) b.comps.put(ck2, craw2) wk2, wraw2 := wantFor(t, "db", fp2, dig2, "") b.wants.put(wk2, wraw2) eventually(t, "the backup was not asked", func() bool { b.mu.Lock() defer b.mu.Unlock() return len(b.backups) == 1 }) eventually(t, "the have does not say held", func() bool { h := b.haves.last("db") return h.Phase == string(delivery.PhaseHeld) && strings.Contains(h.Detail, "repository is locked") && h.FP == fp1 }) if got := b.appliedList(); len(got) != 0 { t.Fatalf("something was applied behind a failed backup: %v", got) } // The answer arrives in the module's own want: back up again (it fails again), then skip. wk, again := wantFor(t, "db", fp2, dig2, delivery.BackupAgain) b.wants.put(wk, again) eventually(t, "back up again did not ask the backup", func() bool { b.mu.Lock() defer b.mu.Unlock() return len(b.backups) == 2 }) if got := b.appliedList(); len(got) != 0 { t.Fatalf("applied behind a second failed backup: %v", got) } _, skip := wantFor(t, "db", fp2, dig2, delivery.BackupSkip) b.wants.put(wk, skip) eventually(t, "skip did not apply the module", func() bool { return len(b.appliedList()) == 1 }) b.mu.Lock() n := len(b.backups) b.mu.Unlock() if n != 2 { t.Fatalf("a backup was asked after the answer was to skip: %d asks", n) } eventually(t, "not running at the new fingerprint", func() bool { h := b.haves.last("db") return h.Phase == string(delivery.PhaseRunning) && h.FP == fp2 && h.Composed == dig2 }) } // A data module with a good backup is applied after it; one that never ran has nothing to back up. func TestABackupDoneIsFollowedByTheApplyAndANewModuleIsNotBackedUp(t *testing.T) { b := newBed(t) stop := b.run() defer stop() fp := fpOf("c") ck, craw, dig := composition(t, "db", fp, true, file("db", "x", "1")) wk, wraw := wantFor(t, "db", fp, dig, "") b.comps.put(ck, craw) b.wants.put(wk, wraw) eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 }) b.mu.Lock() n := len(b.backups) b.mu.Unlock() if n != 0 { t.Fatalf("a module that never ran was backed up (%d)", n) } fp2 := fpOf("d") ck2, craw2, dig2 := composition(t, "db", fp2, true, file("db", "x", "2")) wk2, wraw2 := wantFor(t, "db", fp2, dig2, "") b.comps.put(ck2, craw2) b.wants.put(wk2, wraw2) eventually(t, "second not applied", func() bool { return len(b.appliedList()) == 2 }) b.mu.Lock() n = len(b.backups) b.mu.Unlock() if n != 1 { t.Fatalf("a module that ran before was backed up %d times, want once", n) } } // R6-A AC4 at the agent: a removed want removes that module only, and its have is written removed. func TestARemovedWantRemovesThatModuleOnly(t *testing.T) { b := newBed(t) stop := b.run() defer stop() fp := fpOf("a") ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1")) b.comps.put(ck, craw) wk, wraw := wantFor(t, "web", fp, dig, "") b.wants.put(wk, wraw) eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 }) gone, _ := delivery.EncodeWant(delivery.Want{Removed: true}) b.wants.put(wk, gone) eventually(t, "not removed", func() bool { b.mu.Lock() defer b.mu.Unlock() return len(b.removed) == 1 && b.removed[0] == "web" }) eventually(t, "the have was not written removed", func() bool { b.haves.mu.Lock() defer b.haves.mu.Unlock() return len(b.haves.removed) == 1 }) } // The node's own composition is applied before any module's, whichever key came first. func TestTheNodesOwnCompositionComesBeforeAnyModules(t *testing.T) { b := newBed(t) fp := fpOf("a") ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1")) wk, wraw := wantFor(t, "web", fp, dig, "") nk, nraw, ndig := composition(t, NodeModule, "", false) nwk, err := delivery.WantNodeKey(node) if err != nil { t.Fatal(err) } nwraw, err := delivery.EncodeNodeWant(delivery.NodeWant{Composed: ndig}) if err != nil { t.Fatal(err) } b.comps.before(ck, craw) b.comps.before(nk, nraw) b.wants.before(wk, wraw) b.wants.before(nwk, nwraw) stop := b.run() defer stop() eventually(t, "both not applied", func() bool { return len(b.appliedList()) == 2 }) if got := b.appliedList(); got[0] != NodeModule || got[1] != "web" { t.Fatalf("the order was %v, the node's own first", got) } } // R6-A AC9: a composition signed by either trusted key is applied; one signed by another key, one changed after it was // signed and one unsigned are refused and nothing of them is applied (ADR 0539 ยง3). func TestOnlyACompositionSignedByATrustedKeyIsApplied(t *testing.T) { b := newBed(t) stop := b.run() defer stop() plain := func(module, fp, content string) delivery.Composed { return delivery.Composed{Node: node, Module: module, FP: fp, Resources: []json.RawMessage{file(module, "x", content)}} } put := func(c delivery.Composed, signer ed25519.PrivateKey, tamper func([]byte) []byte) { key, _ := delivery.ComposedKey(node, c.Module, c.FP) var raw []byte var dig string if signer == nil { raw, _ = delivery.EncodeComposed(c) dig, _ = delivery.ComposedDigestOf(c) } else { _, raw, dig = signedBy(t, c, key, signer) } if tamper != nil { raw = tamper(raw) } b.comps.put(key, raw) wk, wraw := wantFor(t, c.Module, c.FP, dig, "") b.wants.put(wk, wraw) } // Refused: signed by a stranger, changed after signing, unsigned. put(plain("stranger", fpOf("1"), "x"), strangerKey, nil) put(plain("changed", fpOf("2"), "before"), controllerKey, func(raw []byte) []byte { return []byte(strings.Replace(string(raw), "before", "after!", 1)) }) put(plain("unsigned", fpOf("3"), "x"), nil, nil) time.Sleep(200 * time.Millisecond) if got := b.appliedList(); len(got) != 0 { t.Fatalf("a composition that is not signed by a trusted key was applied: %v", got) } // Applied: signed by the controller, signed by the operator. put(plain("bycontroller", fpOf("4"), "x"), controllerKey, nil) put(plain("byoperator", fpOf("5"), "x"), operatorKey, nil) eventually(t, "the trusted signers' compositions were not applied", func() bool { return len(b.appliedList()) == 2 }) got := b.appliedList() if !(got[0] == "bycontroller" && got[1] == "byoperator") && !(got[0] == "byoperator" && got[1] == "bycontroller") { t.Fatalf("applied %v", got) } if h := b.haves.last("byoperator"); h.Phase != string(delivery.PhaseRunning) || h.FP != fpOf("5") { t.Fatalf("the operator's module has no running have: %+v", h) } } // With no key trusted, nothing is applied: a node that does not know its signers fails closed. func TestWithNoTrustedKeyNothingIsApplied(t *testing.T) { b := newBed(t) b.env.Trusted = nil stop := b.run() defer stop() fp := fpOf("a") ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1")) b.comps.put(ck, craw) wk, wraw := wantFor(t, "web", fp, dig, "") b.wants.put(wk, wraw) time.Sleep(200 * time.Millisecond) if got := b.appliedList(); len(got) != 0 { t.Fatalf("applied with no trusted key: %v", got) } }