jschoubben
81672b183d
The reviewer showed how76e214astill lets the 07:54 outage happen at the re-landed engine's first start on novox. applied/ is empty there, so loadApplied loads nothing. The kept declaration retires, because every placed module has a have. Nothing then carries the node's own composition. The guard misses nats, which has no want there, so its nats.conf is guessed as nats's and removed. - At the start, when the store has no node's own composition, the node's own runs as the kept declaration has it, and the store keeps it now. The kept declaration retires only once the store holds `node`. - What this engine applied runs until loadHad reads the composition a differing have names. The bucket no longer keeps the one novox's have names, so the stored one stays. - In the node's own apply, a resource of another module goes only when that module's want says removed (Scope.RemovedByWant). A module the node's own composition carried has no want, so Placed never named it. Cost: the node's own apply cannot drop one of its own dotted resources without such a want. It is refused whole and said, never removed silently. Tests, with no new test functions: - A first-start case in the agent's Everything test (empty store, then a start with the declaration retired). It fails on76e214a. - Two rows in the guard's table: no want refuses; a want saying removed removes. - A case for the store in the authority test.
1597 lines
65 KiB
Go
1597 lines
65 KiB
Go
package agent
|
|
|
|
import (
|
|
"context"
|
|
"crypto/ed25519"
|
|
"encoding/json"
|
|
"fmt"
|
|
"maps"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
)
|
|
|
|
const node = "n1"
|
|
|
|
// The keys of the tests: the controller's and the operator's are trusted, a stranger's is not.
|
|
var controllerPub, controllerKey = newKey()
|
|
var operatorPub, operatorKey = newKey()
|
|
var _, strangerKey = newKey()
|
|
|
|
func newKey() (ed25519.PublicKey, ed25519.PrivateKey) {
|
|
pub, key, err := ed25519.GenerateKey(nil)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
return pub, key
|
|
}
|
|
|
|
func fpOf(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) }
|
|
|
|
// A watcher the test writes to; it says Caught once the keys written before the watch began are out.
|
|
type feed struct {
|
|
mu sync.Mutex
|
|
ch chan Update
|
|
rev uint64
|
|
first []Update
|
|
}
|
|
|
|
func newFeed() *feed { return &feed{ch: make(chan Update, 256)} }
|
|
|
|
func (f *feed) Watch(_ context.Context, _ string) (<-chan Update, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
for _, u := range f.first {
|
|
f.ch <- u
|
|
}
|
|
f.ch <- Update{Caught: true}
|
|
return f.ch, nil
|
|
}
|
|
|
|
// before writes a key as if it was there when the watch began.
|
|
func (f *feed) before(key string, value []byte) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.rev++
|
|
f.first = append(f.first, Update{Key: key, Value: value, Revision: f.rev})
|
|
}
|
|
|
|
func (f *feed) put(key string, value []byte) {
|
|
f.mu.Lock()
|
|
f.rev++
|
|
r := f.rev
|
|
f.mu.Unlock()
|
|
f.ch <- Update{Key: key, Value: value, Revision: r}
|
|
}
|
|
|
|
type haves struct {
|
|
mu sync.Mutex
|
|
stored map[string]Have
|
|
log []Have
|
|
removed []string
|
|
// unreadable is each module whose have the bus cannot read now (ReadErr).
|
|
unreadable map[string]bool
|
|
}
|
|
|
|
func (h *haves) ReadErr(ctx context.Context, module string) (Have, bool, error) {
|
|
h.mu.Lock()
|
|
bad := h.unreadable[module]
|
|
h.mu.Unlock()
|
|
if bad {
|
|
return Have{}, false, fmt.Errorf("nats: no responders available for request")
|
|
}
|
|
x, ok := h.Read(ctx, module)
|
|
return x, ok, nil
|
|
}
|
|
|
|
func (h *haves) Read(_ context.Context, module string) (Have, bool) {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
x, ok := h.stored[module]
|
|
return x, ok
|
|
}
|
|
func (h *haves) Write(_ context.Context, x Have) {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
if h.stored == nil {
|
|
h.stored = map[string]Have{}
|
|
}
|
|
h.stored[x.Module] = x
|
|
h.log = append(h.log, x)
|
|
}
|
|
func (h *haves) Removed(_ context.Context, m string) {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
h.removed = append(h.removed, m)
|
|
}
|
|
func (h *haves) last(module string) Have {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
return h.stored[module]
|
|
}
|
|
|
|
type bed struct {
|
|
t *testing.T
|
|
wants *feed
|
|
comps *feed
|
|
haves *haves
|
|
mu sync.Mutex
|
|
applied []string
|
|
removed []string
|
|
backups []string
|
|
failWith string
|
|
// transient marks failWith as the bus not answering; failTimes > 0 fails only the first that many applies.
|
|
transient bool
|
|
failTimes int
|
|
backup func(module string) Backup
|
|
kept map[string][]json.RawMessage
|
|
others [][]delivery.Composed
|
|
said []string
|
|
// settled is each module Settled was told, and settledRan whether what runs held it then.
|
|
settled []string
|
|
settledRan []bool
|
|
ag *Agent // the agent run() runs
|
|
env Env
|
|
}
|
|
|
|
func newBed(t *testing.T) *bed {
|
|
b := &bed{t: t, wants: newFeed(), comps: newFeed(), haves: &haves{}}
|
|
b.env = Env{Node: node, Wants: b.wants, Composed: b.comps, Haves: b.haves,
|
|
Apply: func(_ context.Context, c delivery.Composed, others []delivery.Composed) Result {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
b.others = append(b.others, others)
|
|
name := c.Module
|
|
if name == "" {
|
|
name = NodeModule
|
|
}
|
|
b.applied = append(b.applied, name)
|
|
if b.failWith != "" && (b.failTimes == 0 || len(b.applied) <= b.failTimes) {
|
|
return Result{Failed: true, Transient: b.transient, Detail: b.failWith}
|
|
}
|
|
return Result{}
|
|
},
|
|
Remove: func(_ context.Context, m string, _ []delivery.Composed) Result {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
b.removed = append(b.removed, m)
|
|
return Result{}
|
|
},
|
|
Backup: func(_ context.Context, m string) Backup {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
b.backups = append(b.backups, m)
|
|
if b.backup != nil {
|
|
return b.backup(m)
|
|
}
|
|
return Backup{Done: true}
|
|
},
|
|
Settled: func(m string) {
|
|
ran := false
|
|
if b.ag != nil {
|
|
for _, c := range b.ag.Everything() {
|
|
ran = ran || c.Module == m
|
|
}
|
|
}
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
b.settled = append(b.settled, m)
|
|
b.settledRan = append(b.settledRan, ran)
|
|
},
|
|
Kept: func() map[string][]json.RawMessage { return b.kept },
|
|
Trusted: func() []ed25519.PublicKey { return []ed25519.PublicKey{controllerPub, operatorPub} },
|
|
Say: func(s string) {
|
|
t.Log(s)
|
|
b.mu.Lock()
|
|
b.said = append(b.said, s)
|
|
b.mu.Unlock()
|
|
},
|
|
}
|
|
return b
|
|
}
|
|
|
|
func (b *bed) run() (stop func()) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan struct{})
|
|
b.ag = New(b.env)
|
|
go func() { defer close(done); _ = b.ag.Run(ctx) }()
|
|
return func() { cancel(); <-done }
|
|
}
|
|
|
|
func file(module, id, content string) json.RawMessage {
|
|
raw, _ := json.Marshal(map[string]any{"id": module + "." + id, "type": "file", "path": "/tmp/r6/" + module + "/" + id,
|
|
"content": content, "module": module})
|
|
return raw
|
|
}
|
|
|
|
// composition is a module's composition at a fingerprint: its key, its bytes and its digest, from the writer's code.
|
|
func composition(t *testing.T, module, fp string, data bool, rs ...json.RawMessage) (key string, raw []byte, digest string) {
|
|
t.Helper()
|
|
c := delivery.Composed{Node: node, Module: module, FP: fp, Data: data, Resources: rs}
|
|
var err error
|
|
if module == NodeModule {
|
|
c.Module, c.FP = "", ""
|
|
key, err = delivery.ComposedNodeKey(node)
|
|
} else {
|
|
key, err = delivery.ComposedKey(node, module, fp)
|
|
}
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return signedBy(t, c, key, controllerKey)
|
|
}
|
|
|
|
// signedBy is c signed with signer, as the writer's code signs it: its key, its bytes and its digest.
|
|
func signedBy(t *testing.T, c delivery.Composed, key string, signer ed25519.PrivateKey) (string, []byte, string) {
|
|
t.Helper()
|
|
signed, err := delivery.SignComposed(c, signer)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
raw, err := delivery.EncodeComposed(signed)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
digest, err := delivery.ComposedDigestOf(signed)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return key, raw, digest
|
|
}
|
|
|
|
func wantFor(t *testing.T, module, fp, digest, backup string) (string, []byte) {
|
|
t.Helper()
|
|
key, err := delivery.WantKey(node, module)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
raw, err := delivery.EncodeWant(delivery.Want{FP: fp, Composed: digest, Backup: backup})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return key, raw
|
|
}
|
|
|
|
func eventually(t *testing.T, what string, ok func() bool) {
|
|
t.Helper()
|
|
deadline := time.Now().Add(3 * time.Second)
|
|
for !ok() {
|
|
if time.Now().After(deadline) {
|
|
t.Fatal(what)
|
|
}
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
func (b *bed) appliedList() []string {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
return append([]string(nil), b.applied...)
|
|
}
|
|
|
|
// R6-A AC1 (the agent, with a bus that is a fake): given a want naming a digest and that composition, the module is
|
|
// applied and its have names the fingerprint and the digest; a composition whose digest no want names is not applied.
|
|
func TestAWantAndItsCompositionApplyTheModuleAndTheHaveNamesBoth(t *testing.T) {
|
|
b := newBed(t)
|
|
stop := b.run()
|
|
defer stop()
|
|
|
|
fp := fpOf("a")
|
|
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
|
|
// A composition no want names yet is not applied.
|
|
b.comps.put(ck, craw)
|
|
time.Sleep(100 * time.Millisecond)
|
|
if got := b.appliedList(); len(got) != 0 {
|
|
t.Fatalf("a composition no want names was applied: %v", got)
|
|
}
|
|
wk, wraw := wantFor(t, "web", fp, dig, "")
|
|
b.wants.put(wk, wraw)
|
|
eventually(t, "the module was not applied", func() bool { return len(b.appliedList()) == 1 })
|
|
eventually(t, "the have does not name both", func() bool {
|
|
h := b.haves.last("web")
|
|
return h.FP == fp && h.Composed == dig && h.Phase == string(delivery.PhaseRunning)
|
|
})
|
|
// Settled is told once what runs holds the module, so a declaration read then names its files (the filling's Knock;
|
|
// ace, 2026-10-25 21:17).
|
|
eventually(t, "Settled was not told web with web in what runs", func() bool {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
return slices.Equal(b.settled, []string{"web"}) && b.settledRan[0]
|
|
})
|
|
|
|
// A want naming a digest the composition at that fingerprint does not have: nothing is applied.
|
|
fp2 := fpOf("b")
|
|
ck2, craw2, _ := composition(t, "web", fp2, false, file("web", "x", "2"))
|
|
b.comps.put(ck2, craw2)
|
|
wk2, wraw2 := wantFor(t, "web", fp2, "sha256:"+strings.Repeat("0", 64), "")
|
|
b.wants.put(wk2, wraw2)
|
|
time.Sleep(100 * time.Millisecond)
|
|
if got := b.appliedList(); len(got) != 1 {
|
|
t.Fatalf("a want naming another digest applied the composition: %v", got)
|
|
}
|
|
}
|
|
|
|
// R6-A AC2: given a kept declaration and wants and compositions equal to it, when the agent starts, nothing is applied
|
|
// and every have names its fingerprint and digest.
|
|
func TestStartingOnWhatTheKeptDeclarationHoldsAppliesNothing(t *testing.T) {
|
|
b := newBed(t)
|
|
fp := fpOf("a")
|
|
rs := []json.RawMessage{file("web", "x", "1"), file("web", "y", "2")}
|
|
ck, craw, dig := composition(t, "web", fp, false, rs...)
|
|
wk, wraw := wantFor(t, "web", fp, dig, "")
|
|
b.comps.before(ck, craw)
|
|
b.wants.before(wk, wraw)
|
|
// The kept declaration holds the same resources, in another order and spacing.
|
|
b.kept = map[string][]json.RawMessage{"web": {json.RawMessage(strings.ReplaceAll(string(rs[1]), ",", " , ")), rs[0]}}
|
|
stop := b.run()
|
|
defer stop()
|
|
eventually(t, "the have was not written", func() bool { return b.haves.last("web").Composed == dig })
|
|
if h := b.haves.last("web"); h.FP != fp || h.Phase != string(delivery.PhaseRunning) {
|
|
t.Fatalf("the have: %+v", h)
|
|
}
|
|
time.Sleep(50 * time.Millisecond)
|
|
if got := b.appliedList(); len(got) != 0 {
|
|
t.Fatalf("a module that already runs what the want names was applied: %v", got)
|
|
}
|
|
}
|
|
|
|
// What the node-engine itself last applied is what runs after a restart, not the kept declaration.
|
|
func TestStartingOnItsOwnHaveAppliesNothing(t *testing.T) {
|
|
t.Run("1957: an engine start reads the machine's facts", testAnEngineStartReconcilesAModuleFromTheMachinesFacts)
|
|
t.Run("says the have again", testStartingOnItsOwnHaveSaysTheHaveAgainSoItsHealthIsThisEngines)
|
|
// An apply that failed is not what runs, though its have names the want's build and digest: the agent starting on
|
|
// it applies again (review of the switch, round 1).
|
|
t.Run("applies again after an apply failure", func(t *testing.T) {
|
|
b := newBed(t)
|
|
fp := fpOf("a")
|
|
ck, craw, dig := composition(t, "avahi", fp, false, file("avahi", "x", "1"))
|
|
wk, wraw := wantFor(t, "avahi", fp, dig, "")
|
|
b.haves.stored = map[string]Have{"avahi": {Module: "avahi", FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed),
|
|
Detail: "apply: unit failed to start"}}
|
|
b.comps.before(ck, craw)
|
|
b.wants.before(wk, wraw)
|
|
stop := b.run()
|
|
defer stop()
|
|
eventually(t, "a failed apply was not applied again", func() bool { return len(b.appliedList()) == 1 })
|
|
})
|
|
// Issue 1126, the stale marks after the seed's revert: a have a composer marked failed names what still runs; the
|
|
// agent starting on it applies nothing and says it running, so a mark it did not make clears.
|
|
t.Run("clears a composer's mark", func(t *testing.T) {
|
|
b := newBed(t)
|
|
fp := fpOf("a")
|
|
ck, craw, dig := composition(t, "avahi", fp, false, file("avahi", "x", "1"))
|
|
wk, wraw := wantFor(t, "avahi", fp, dig, "")
|
|
b.haves.stored = map[string]Have{"avahi": {Module: "avahi", FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed),
|
|
Detail: NotComposedDetail + "frombus: the credentials of ace: context canceled"}}
|
|
b.comps.before(ck, craw)
|
|
b.wants.before(wk, wraw)
|
|
stop := b.run()
|
|
defer stop()
|
|
eventually(t, "the mark was not cleared", func() bool {
|
|
h := b.haves.last("avahi")
|
|
return h.Phase == string(delivery.PhaseRunning) && h.FP == fp && h.Composed == dig && h.Detail == ""
|
|
})
|
|
if got := b.appliedList(); len(got) != 0 {
|
|
t.Fatalf("a module whose mark names what the want names was applied: %v", got)
|
|
}
|
|
})
|
|
b := newBed(t)
|
|
fp := fpOf("a")
|
|
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
|
|
wk, wraw := wantFor(t, "web", fp, dig, "")
|
|
b.haves.stored = map[string]Have{"web": {Module: "web", FP: fp, Composed: dig, Phase: "running"}}
|
|
b.comps.before(ck, craw)
|
|
b.wants.before(wk, wraw)
|
|
stop := b.run()
|
|
defer stop()
|
|
time.Sleep(150 * time.Millisecond)
|
|
if got := b.appliedList(); len(got) != 0 {
|
|
t.Fatalf("a module that runs what the want names was applied after a restart: %v", got)
|
|
}
|
|
}
|
|
|
|
// A module found running on its own have after a restart is said again, once, through the haves: the session then holds
|
|
// it and writes the health this engine judges. Not said, its key kept the health an older engine wrote: xorg's have on
|
|
// g14 and shanks said unknown from before the engine said a module with nothing to judge ok, and the assigner held every
|
|
// module requiring x11-display back for it (2026-10-25).
|
|
func testStartingOnItsOwnHaveSaysTheHaveAgainSoItsHealthIsThisEngines(t *testing.T) {
|
|
b := newBed(t)
|
|
fp := fpOf("a")
|
|
ck, craw, dig := composition(t, "xorg", fp, false, file("xorg", "x", "1"))
|
|
wk, wraw := wantFor(t, "xorg", fp, dig, "")
|
|
b.haves.stored = map[string]Have{"xorg": {Module: "xorg", FP: fp, Composed: dig, Phase: "running", Detail: "kept"}}
|
|
b.comps.before(ck, craw)
|
|
b.wants.before(wk, wraw)
|
|
stop := b.run()
|
|
defer stop()
|
|
eventually(t, "the have found running was not said again", func() bool {
|
|
b.haves.mu.Lock()
|
|
defer b.haves.mu.Unlock()
|
|
return len(b.haves.log) > 0
|
|
})
|
|
time.Sleep(50 * time.Millisecond)
|
|
b.haves.mu.Lock()
|
|
log := append([]Have(nil), b.haves.log...)
|
|
b.haves.mu.Unlock()
|
|
if len(log) != 1 {
|
|
t.Fatalf("the have was said %d times, want once: %+v", len(log), log)
|
|
}
|
|
if h := log[0]; h.Module != "xorg" || h.FP != fp || h.Composed != dig || h.Phase != string(delivery.PhaseRunning) || h.Detail != "kept" {
|
|
t.Fatalf("the have said again: %+v", h)
|
|
}
|
|
if got := b.appliedList(); len(got) != 0 {
|
|
t.Fatalf("a module that runs what the want names was applied: %v", got)
|
|
}
|
|
}
|
|
|
|
// A module whose kept resources differ from the composition is applied, and a failed apply is retried only by a change
|
|
// of either key.
|
|
func TestAFailedApplyIsRetriedOnlyByAChangeOfEitherKey(t *testing.T) {
|
|
t.Run("1957: the judge's edge reconciles, and a failed re-apply is failed", testReconcileAppliesARunningModuleAgainAndLeavesTheHeldAndTheUnwantedAlone)
|
|
// A real failure is not retried by the clock, however short the backoff: only by a change of either key.
|
|
t.Run("a real failure", func(t *testing.T) {
|
|
b := newBed(t)
|
|
b.failWith = "web.x: no space"
|
|
b.env.Backoff = func(int) time.Duration { return time.Millisecond }
|
|
stop := b.run()
|
|
defer stop()
|
|
fp := fpOf("a")
|
|
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
|
|
wk, wraw := wantFor(t, "web", fp, dig, "")
|
|
b.comps.put(ck, craw)
|
|
b.wants.put(wk, wraw)
|
|
eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
|
|
eventually(t, "the have does not say failed", func() bool {
|
|
h := b.haves.last("web")
|
|
return h.Phase == string(delivery.PhaseFailed) && strings.Contains(h.Detail, "no space") && h.FP == fp
|
|
})
|
|
// An unrelated key changes: not a retry.
|
|
ck9, craw9, _ := composition(t, "web", fpOf("9"), false, file("web", "x", "9"))
|
|
b.comps.put(ck9, craw9)
|
|
time.Sleep(150 * time.Millisecond)
|
|
if n := len(b.appliedList()); n != 1 {
|
|
t.Fatalf("a failed apply was retried by an unrelated key or the clock (%d applies)", n)
|
|
}
|
|
// The want is written again: the retry.
|
|
b.mu.Lock()
|
|
b.failWith = ""
|
|
b.mu.Unlock()
|
|
b.wants.put(wk, wraw)
|
|
eventually(t, "the change of the want did not retry", func() bool { return len(b.appliedList()) == 2 })
|
|
eventually(t, "not running", func() bool { return b.haves.last("web").Phase == string(delivery.PhaseRunning) })
|
|
})
|
|
// The bus did not answer for a moment (mesh-vault on the bus node, 2026-10-25 23:04:24): tried again with no new
|
|
// revision of either key, by the backoff or by RetryTransient once the engine knows the bus is back; the have says
|
|
// retrying meanwhile, never failed.
|
|
for name, backoff := range map[string]func(int) time.Duration{
|
|
"a transient failure, by the backoff": func(int) time.Duration { return time.Millisecond },
|
|
"a transient failure, by RetryTransient": nil,
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
b := newBed(t)
|
|
b.failWith, b.transient, b.failTimes = "fetching the object: nats: no responders available for request", true, 1
|
|
b.env.Backoff = backoff
|
|
stop := b.run()
|
|
defer stop()
|
|
fp := fpOf("a")
|
|
ck, craw, dig := composition(t, "vault", fp, false, file("vault", "x", "1"))
|
|
wk, wraw := wantFor(t, "vault", fp, dig, "")
|
|
b.comps.put(ck, craw)
|
|
b.wants.put(wk, wraw)
|
|
eventually(t, "not applied", func() bool { return len(b.appliedList()) >= 1 })
|
|
if backoff == nil {
|
|
eventually(t, "the have does not say retrying", func() bool {
|
|
return strings.HasPrefix(b.haves.last("vault").Detail, "retrying")
|
|
})
|
|
b.ag.RetryTransient()
|
|
}
|
|
eventually(t, "not applied again without a new revision", func() bool { return len(b.appliedList()) == 2 })
|
|
eventually(t, "not running", func() bool { return b.haves.last("vault").Phase == string(delivery.PhaseRunning) })
|
|
b.haves.mu.Lock()
|
|
defer b.haves.mu.Unlock()
|
|
for _, h := range b.haves.log {
|
|
if h.Phase == string(delivery.PhaseFailed) {
|
|
t.Errorf("a transient failure was written failed: %+v", h)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// R6-A AC5: a data module that ran before asks its backup before any resource changes; a refusal holds it, nothing is
|
|
// applied and the have says held; when the want then says skip, the module is applied.
|
|
func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) {
|
|
b := newBed(t)
|
|
fp1, fp2 := fpOf("a"), fpOf("b")
|
|
ck1, craw1, dig1 := composition(t, "db", fp1, true, file("db", "x", "1"))
|
|
wk1, wraw1 := wantFor(t, "db", fp1, dig1, "")
|
|
b.comps.before(ck1, craw1)
|
|
b.wants.before(wk1, wraw1)
|
|
b.kept = map[string][]json.RawMessage{"db": {file("db", "x", "1")}}
|
|
refuse := true
|
|
b.backup = func(string) Backup {
|
|
if refuse {
|
|
return Backup{Words: "restic: the repository is locked"}
|
|
}
|
|
return Backup{Done: true}
|
|
}
|
|
stop := b.run()
|
|
defer stop()
|
|
eventually(t, "baseline", func() bool { return b.haves.last("db").Composed == dig1 })
|
|
|
|
ck2, craw2, dig2 := composition(t, "db", fp2, true, file("db", "x", "2"))
|
|
b.comps.put(ck2, craw2)
|
|
wk2, wraw2 := wantFor(t, "db", fp2, dig2, "")
|
|
b.wants.put(wk2, wraw2)
|
|
eventually(t, "the backup was not asked", func() bool {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
return len(b.backups) == 1
|
|
})
|
|
eventually(t, "the have does not say held", func() bool {
|
|
h := b.haves.last("db")
|
|
return h.Phase == string(delivery.PhaseHeld) && strings.Contains(h.Detail, "repository is locked") && h.FP == fp1
|
|
})
|
|
if got := b.appliedList(); len(got) != 0 {
|
|
t.Fatalf("something was applied behind a failed backup: %v", got)
|
|
}
|
|
|
|
// The answer arrives in the module's own want: back up again (it fails again), then skip.
|
|
wk, again := wantFor(t, "db", fp2, dig2, delivery.BackupAgain)
|
|
b.wants.put(wk, again)
|
|
eventually(t, "back up again did not ask the backup", func() bool {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
return len(b.backups) == 2
|
|
})
|
|
if got := b.appliedList(); len(got) != 0 {
|
|
t.Fatalf("applied behind a second failed backup: %v", got)
|
|
}
|
|
_, skip := wantFor(t, "db", fp2, dig2, delivery.BackupSkip)
|
|
b.wants.put(wk, skip)
|
|
eventually(t, "skip did not apply the module", func() bool { return len(b.appliedList()) == 1 })
|
|
b.mu.Lock()
|
|
n := len(b.backups)
|
|
b.mu.Unlock()
|
|
if n != 2 {
|
|
t.Fatalf("a backup was asked after the answer was to skip: %d asks", n)
|
|
}
|
|
eventually(t, "not running at the new fingerprint", func() bool {
|
|
h := b.haves.last("db")
|
|
return h.Phase == string(delivery.PhaseRunning) && h.FP == fp2 && h.Composed == dig2
|
|
})
|
|
}
|
|
|
|
// A data module with a good backup is applied after it; one that never ran has nothing to back up.
|
|
func TestABackupDoneIsFollowedByTheApplyAndANewModuleIsNotBackedUp(t *testing.T) {
|
|
b := newBed(t)
|
|
stop := b.run()
|
|
defer stop()
|
|
fp := fpOf("c")
|
|
ck, craw, dig := composition(t, "db", fp, true, file("db", "x", "1"))
|
|
wk, wraw := wantFor(t, "db", fp, dig, "")
|
|
b.comps.put(ck, craw)
|
|
b.wants.put(wk, wraw)
|
|
eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
|
|
b.mu.Lock()
|
|
n := len(b.backups)
|
|
b.mu.Unlock()
|
|
if n != 0 {
|
|
t.Fatalf("a module that never ran was backed up (%d)", n)
|
|
}
|
|
fp2 := fpOf("d")
|
|
ck2, craw2, dig2 := composition(t, "db", fp2, true, file("db", "x", "2"))
|
|
wk2, wraw2 := wantFor(t, "db", fp2, dig2, "")
|
|
b.comps.put(ck2, craw2)
|
|
b.wants.put(wk2, wraw2)
|
|
eventually(t, "second not applied", func() bool { return len(b.appliedList()) == 2 })
|
|
b.mu.Lock()
|
|
n = len(b.backups)
|
|
b.mu.Unlock()
|
|
if n != 1 {
|
|
t.Fatalf("a module that ran before was backed up %d times, want once", n)
|
|
}
|
|
}
|
|
|
|
// R6-A AC4 at the agent: a removed want removes that module only, and its have is written removed.
|
|
func TestARemovedWantRemovesThatModuleOnly(t *testing.T) {
|
|
t.Run("its composition holds nothing back", testARemovedModulesCompositionHoldsNothingBack)
|
|
b := newBed(t)
|
|
stop := b.run()
|
|
defer stop()
|
|
fp := fpOf("a")
|
|
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
|
|
b.comps.put(ck, craw)
|
|
wk, wraw := wantFor(t, "web", fp, dig, "")
|
|
b.wants.put(wk, wraw)
|
|
eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
|
|
gone, _ := delivery.EncodeWant(delivery.Want{Removed: true})
|
|
b.wants.put(wk, gone)
|
|
eventually(t, "not removed", func() bool {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
return len(b.removed) == 1 && b.removed[0] == "web"
|
|
})
|
|
eventually(t, "the have was not written removed", func() bool {
|
|
b.haves.mu.Lock()
|
|
defer b.haves.mu.Unlock()
|
|
return len(b.haves.removed) == 1
|
|
})
|
|
}
|
|
|
|
// The node's own composition is applied before any module's, whichever key came first.
|
|
func TestTheNodesOwnCompositionComesBeforeAnyModules(t *testing.T) {
|
|
b := newBed(t)
|
|
fp := fpOf("a")
|
|
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
|
|
wk, wraw := wantFor(t, "web", fp, dig, "")
|
|
nk, nraw, ndig := composition(t, NodeModule, "", false)
|
|
nwk, err := delivery.WantNodeKey(node)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
nwraw, err := delivery.EncodeNodeWant(delivery.NodeWant{Composed: ndig})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
b.comps.before(ck, craw)
|
|
b.comps.before(nk, nraw)
|
|
b.wants.before(wk, wraw)
|
|
b.wants.before(nwk, nwraw)
|
|
stop := b.run()
|
|
defer stop()
|
|
eventually(t, "both not applied", func() bool { return len(b.appliedList()) == 2 })
|
|
if got := b.appliedList(); got[0] != NodeModule || got[1] != "web" {
|
|
t.Fatalf("the order was %v, the node's own first", got)
|
|
}
|
|
}
|
|
|
|
// R6-A AC9: a composition signed by either trusted key is applied; one signed by another key, one changed after it was
|
|
// signed and one unsigned are refused and nothing of them is applied (ADR 0539 §3).
|
|
func TestOnlyACompositionSignedByATrustedKeyIsApplied(t *testing.T) {
|
|
t.Run("with no trusted key", testWithNoTrustedKeyNothingIsApplied)
|
|
b := newBed(t)
|
|
stop := b.run()
|
|
defer stop()
|
|
|
|
plain := func(module, fp, content string) delivery.Composed {
|
|
return delivery.Composed{Node: node, Module: module, FP: fp, Resources: []json.RawMessage{file(module, "x", content)}}
|
|
}
|
|
put := func(c delivery.Composed, signer ed25519.PrivateKey, tamper func([]byte) []byte) {
|
|
key, _ := delivery.ComposedKey(node, c.Module, c.FP)
|
|
var raw []byte
|
|
var dig string
|
|
if signer == nil {
|
|
raw, _ = delivery.EncodeComposed(c)
|
|
dig, _ = delivery.ComposedDigestOf(c)
|
|
} else {
|
|
_, raw, dig = signedBy(t, c, key, signer)
|
|
}
|
|
if tamper != nil {
|
|
raw = tamper(raw)
|
|
}
|
|
b.comps.put(key, raw)
|
|
wk, wraw := wantFor(t, c.Module, c.FP, dig, "")
|
|
b.wants.put(wk, wraw)
|
|
}
|
|
|
|
// Refused: signed by a stranger, changed after signing, unsigned.
|
|
put(plain("stranger", fpOf("1"), "x"), strangerKey, nil)
|
|
put(plain("changed", fpOf("2"), "before"), controllerKey, func(raw []byte) []byte {
|
|
return []byte(strings.Replace(string(raw), "before", "after!", 1))
|
|
})
|
|
put(plain("unsigned", fpOf("3"), "x"), nil, nil)
|
|
time.Sleep(200 * time.Millisecond)
|
|
if got := b.appliedList(); len(got) != 0 {
|
|
t.Fatalf("a composition that is not signed by a trusted key was applied: %v", got)
|
|
}
|
|
|
|
// Applied: signed by the controller, signed by the operator.
|
|
put(plain("bycontroller", fpOf("4"), "x"), controllerKey, nil)
|
|
put(plain("byoperator", fpOf("5"), "x"), operatorKey, nil)
|
|
eventually(t, "the trusted signers' compositions were not applied", func() bool { return len(b.appliedList()) == 2 })
|
|
got := b.appliedList()
|
|
if !(got[0] == "bycontroller" && got[1] == "byoperator") && !(got[0] == "byoperator" && got[1] == "bycontroller") {
|
|
t.Fatalf("applied %v", got)
|
|
}
|
|
if h := b.haves.last("byoperator"); h.Phase != string(delivery.PhaseRunning) || h.FP != fpOf("5") {
|
|
t.Fatalf("the operator's module has no running have: %+v", h)
|
|
}
|
|
}
|
|
|
|
// With no key trusted, nothing is applied: a node that does not know its signers fails closed.
|
|
func testWithNoTrustedKeyNothingIsApplied(t *testing.T) {
|
|
b := newBed(t)
|
|
b.env.Trusted = nil
|
|
stop := b.run()
|
|
defer stop()
|
|
fp := fpOf("a")
|
|
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
|
|
b.comps.put(ck, craw)
|
|
wk, wraw := wantFor(t, "web", fp, dig, "")
|
|
b.wants.put(wk, wraw)
|
|
time.Sleep(200 * time.Millisecond)
|
|
if got := b.appliedList(); len(got) != 0 {
|
|
t.Fatalf("applied with no trusted key: %v", got)
|
|
}
|
|
}
|
|
|
|
// What a module's apply must not take is every other module that runs, those the agent holds and those still running from
|
|
// the kept declaration (ADR 0429 §4): the resources of the kept declaration reach Apply as the others.
|
|
func TestApplyingOneModuleHoldsOutTheModulesStillRunningFromTheKeptDeclaration(t *testing.T) {
|
|
b := newBed(t)
|
|
b.kept = map[string][]json.RawMessage{"db": {file("db", "x", "1")}, NodeModule: {file("node", "n", "n")}}
|
|
stop := b.run()
|
|
defer stop()
|
|
fp := fpOf("a")
|
|
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
|
|
b.comps.put(ck, craw)
|
|
wk, wraw := wantFor(t, "web", fp, dig, "")
|
|
b.wants.put(wk, wraw)
|
|
eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
var names []string
|
|
for _, c := range b.others[0] {
|
|
names = append(names, c.Module)
|
|
}
|
|
if len(names) != 2 || !strings.Contains(strings.Join(names, ","), "db") {
|
|
t.Fatalf("the others of web were %v: db and the node's own run from the kept declaration", names)
|
|
}
|
|
}
|
|
|
|
// Everything is what runs: the agent's compositions and the kept resources of each module it does not hold.
|
|
func TestEverythingIsWhatTheAgentRunsAndWhatIsKeptForTheRest(t *testing.T) {
|
|
b := newBed(t)
|
|
b.kept = map[string][]json.RawMessage{"db": {file("db", "x", "1")}, "web": {file("web", "x", "old")}}
|
|
a := New(b.env)
|
|
if got := a.Everything(); len(got) != 2 {
|
|
t.Fatalf("before the agent holds anything: %d compositions, want the 2 kept", len(got))
|
|
}
|
|
a.running["web"] = delivery.Have{FP: fpOf("a"), Composed: "sha256:" + strings.Repeat("1", 64), Phase: delivery.PhaseRunning}
|
|
ck, craw, dig := composition(t, "web", fpOf("a"), false, file("web", "x", "new"))
|
|
a.running["web"] = delivery.Have{FP: fpOf("a"), Composed: dig, Phase: delivery.PhaseRunning}
|
|
a.takeComposition(Update{Key: ck, Value: craw, Revision: 1})
|
|
got := a.Everything()
|
|
if len(got) != 2 {
|
|
t.Fatalf("%d compositions", len(got))
|
|
}
|
|
for _, c := range got {
|
|
if c.Module == "web" && !strings.Contains(string(c.Resources[0]), "new") {
|
|
t.Fatal("web is the kept declaration's, not the composition the agent runs")
|
|
}
|
|
}
|
|
if a.HoldsNode() {
|
|
t.Fatal("the agent holds no node composition")
|
|
}
|
|
|
|
// Issue 774 (novox, 2026-10-29 07:54 UTC): the kept declaration retired and the node's own composition in no bucket, a
|
|
// fresh engine's first own composition was composed from what runs with the node's own missing, so its folded modules
|
|
// (the bus server among them) were composed as nothing and removed. What this engine applied is kept on the machine
|
|
// and is what runs from before the first composition is watched; an apply keeps, a removal forgets.
|
|
t.Run("what this engine applied is what runs before the node composes its own", func(t *testing.T) {
|
|
b := newBed(t)
|
|
b.env.SelfComposed, b.env.Trusted, b.kept = true, nil, nil
|
|
store := &memApplied{kept: map[string][]byte{}}
|
|
b.env.Applied = store
|
|
// A folded module's resource in the node's own composition: its id names the module, as the controller folds it.
|
|
natsConf, _ := json.Marshal(map[string]any{"id": "nats.conf", "type": "file", "path": "/tmp/r6/nats/conf", "content": "listen 4222"})
|
|
own := delivery.Composed{Node: node, Resources: []json.RawMessage{natsConf}}
|
|
ownRaw, err := delivery.EncodeComposed(own)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ownDig, _ := delivery.ComposedDigestOf(own)
|
|
web := delivery.Composed{Node: node, Module: "web", FP: fpOf("a"), Resources: []json.RawMessage{file("web", "x", "1")}}
|
|
webRaw, _ := delivery.EncodeComposed(web)
|
|
webDig, _ := delivery.ComposedDigestOf(web)
|
|
store.kept[NodeModule], store.kept["web"] = ownRaw, webRaw
|
|
b.haves.Write(context.Background(), Have{Module: NodeModule, Composed: ownDig})
|
|
b.haves.Write(context.Background(), Have{Module: "web", FP: fpOf("a"), Composed: webDig, Phase: string(delivery.PhaseRunning)})
|
|
// The composer's first pass reads what runs when its watch begins, before the agent reads any have.
|
|
firstCh := make(chan []delivery.Composed, 1)
|
|
comps := b.comps
|
|
var ag *Agent
|
|
b.env.Composed = watchFunc(func(ctx context.Context, p string) (<-chan Update, error) {
|
|
firstCh <- ag.Everything()
|
|
return comps.Watch(ctx, p)
|
|
})
|
|
ag = New(b.env)
|
|
b.ag = ag
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan struct{})
|
|
go func() { defer close(done); _ = ag.Run(ctx) }()
|
|
defer func() { cancel(); <-done }()
|
|
var first []delivery.Composed
|
|
select {
|
|
case first = <-firstCh:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("the composer was not watched")
|
|
}
|
|
var ranOwn, ranWeb bool
|
|
for _, c := range first {
|
|
ranOwn = ranOwn || (c.Module == "" && len(c.Resources) == 1 && strings.Contains(string(c.Resources[0]), "4222"))
|
|
ranWeb = ranWeb || c.Module == "web"
|
|
}
|
|
if !ranOwn || !ranWeb {
|
|
t.Fatalf("what runs at the node's first own composition: own %v, web %v (%d compositions); want both as this engine applied them",
|
|
ranOwn, ranWeb, len(first))
|
|
}
|
|
// An apply keeps the composition it applied; a removal forgets it.
|
|
web2 := delivery.Composed{Node: node, Module: "web", FP: fpOf("b"), Resources: []json.RawMessage{file("web", "x", "2")}}
|
|
ck, _ := delivery.ComposedKey(node, "web", fpOf("b"))
|
|
raw2, _ := delivery.EncodeComposed(web2)
|
|
comps.put(ck, raw2)
|
|
wk, _ := delivery.WantKey(node, "web")
|
|
wraw, _ := delivery.EncodeWant(delivery.Want{FP: fpOf("b"), Self: true})
|
|
b.wants.put(wk, wraw)
|
|
eventually(t, "the applied composition was not kept", func() bool { return string(store.get("web")) == string(raw2) })
|
|
rraw, _ := delivery.EncodeWant(delivery.Want{Removed: true})
|
|
b.wants.put(wk, rraw)
|
|
eventually(t, "the removed module's composition was not forgotten", func() bool { return store.get("web") == nil })
|
|
})
|
|
|
|
// Issue 774, review of the re-land, round 1: at the re-landed engine's first start nothing is kept yet, and the node's
|
|
// have names a composition the node composed itself, in no bucket. The node's own runs as the kept declaration has it
|
|
// and is kept then, so the next start, the declaration retired, still runs nats.
|
|
t.Run("at the first start nothing is kept: the node's own runs as the kept declaration has it", func(t *testing.T) {
|
|
natsConf, _ := json.Marshal(map[string]any{"id": "nats.conf", "type": "file", "path": "/tmp/r6/nats/conf", "content": "listen 4222"})
|
|
store := &memApplied{kept: map[string][]byte{}}
|
|
h := &haves{}
|
|
h.Write(context.Background(), Have{Module: NodeModule, Composed: "sha256:" + strings.Repeat("c", 64)})
|
|
firstPass := func(kept map[string][]json.RawMessage) []delivery.Composed {
|
|
b := newBed(t)
|
|
b.env.SelfComposed, b.env.Trusted, b.kept, b.env.Applied, b.env.Haves = true, nil, kept, store, h
|
|
firstCh := make(chan []delivery.Composed, 1)
|
|
comps := b.comps
|
|
var ag *Agent
|
|
b.env.Composed = watchFunc(func(ctx context.Context, p string) (<-chan Update, error) {
|
|
firstCh <- ag.Everything()
|
|
return comps.Watch(ctx, p)
|
|
})
|
|
ag = New(b.env)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan struct{})
|
|
go func() { defer close(done); _ = ag.Run(ctx) }()
|
|
defer func() { cancel(); <-done }()
|
|
select {
|
|
case got := <-firstCh:
|
|
return got
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("the composer was not watched")
|
|
}
|
|
return nil
|
|
}
|
|
runsNats := func(cs []delivery.Composed) bool {
|
|
for _, c := range cs {
|
|
if c.Module == "" && len(c.Resources) == 1 && strings.Contains(string(c.Resources[0]), "4222") {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
if got := firstPass(map[string][]json.RawMessage{NodeModule: {natsConf}}); !runsNats(got) {
|
|
t.Fatalf("the first start does not run nats in the node's own (%d compositions)", len(got))
|
|
}
|
|
if store.get(NodeModule) == nil {
|
|
t.Fatal("the first start kept no node's own composition")
|
|
}
|
|
if got := firstPass(nil); !runsNats(got) {
|
|
t.Fatalf("the next start, the kept declaration retired, does not run nats in the node's own (%d compositions)", len(got))
|
|
}
|
|
})
|
|
}
|
|
|
|
// watchFunc is a Watcher from a function.
|
|
type watchFunc func(ctx context.Context, pattern string) (<-chan Update, error)
|
|
|
|
func (f watchFunc) Watch(ctx context.Context, pattern string) (<-chan Update, error) {
|
|
return f(ctx, pattern)
|
|
}
|
|
|
|
// memApplied is an AppliedStore in memory.
|
|
type memApplied struct {
|
|
mu sync.Mutex
|
|
kept map[string][]byte
|
|
}
|
|
|
|
func (m *memApplied) Load() (map[string][]byte, error) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
out := map[string][]byte{}
|
|
for k, v := range m.kept {
|
|
out[k] = v
|
|
}
|
|
return out, nil
|
|
}
|
|
func (m *memApplied) Keep(module string, raw []byte) error {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
m.kept[module] = raw
|
|
return nil
|
|
}
|
|
func (m *memApplied) Drop(module string) error {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
delete(m.kept, module)
|
|
return nil
|
|
}
|
|
func (m *memApplied) get(module string) []byte {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
return m.kept[module]
|
|
}
|
|
|
|
// A failed apply is applied again after a restart even when the kept declaration holds the composition: once this
|
|
// node-engine wrote its haves, they are what it knows (the kept declaration is only for the first start after the switch).
|
|
func TestAFailedApplyIsAppliedAgainAfterARestartWhateverIsKept(t *testing.T) {
|
|
b := newBed(t)
|
|
fp := fpOf("a")
|
|
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
|
|
wk, wraw := wantFor(t, "web", fp, dig, "")
|
|
b.comps.before(ck, craw)
|
|
b.wants.before(wk, wraw)
|
|
b.kept = map[string][]json.RawMessage{"web": {file("web", "x", "1")}}
|
|
b.haves.stored = map[string]Have{"web": {Module: "web", FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed)}}
|
|
stop := b.run()
|
|
defer stop()
|
|
eventually(t, "a failed apply was marked running from the kept declaration", func() bool { return len(b.appliedList()) == 1 })
|
|
}
|
|
|
|
// A have written before the switch carries no composition: then the kept declaration says what runs.
|
|
func TestAHaveWithNoCompositionDefersToTheKeptDeclaration(t *testing.T) {
|
|
b := newBed(t)
|
|
fp := fpOf("a")
|
|
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
|
|
wk, wraw := wantFor(t, "web", fp, dig, "")
|
|
b.comps.before(ck, craw)
|
|
b.wants.before(wk, wraw)
|
|
b.kept = map[string][]json.RawMessage{"web": {file("web", "x", "1")}}
|
|
b.haves.stored = map[string]Have{"web": {Module: "web", Phase: "running"}}
|
|
stop := b.run()
|
|
defer stop()
|
|
eventually(t, "the have was not written", func() bool { return b.haves.last("web").Composed == dig })
|
|
time.Sleep(50 * time.Millisecond)
|
|
if got := b.appliedList(); len(got) != 0 {
|
|
t.Fatalf("applied %v", got)
|
|
}
|
|
}
|
|
|
|
// A module that leaves the node's own composition for a composition of its own keeps its resources until that one is
|
|
// applied: the node's own apply holds out every module whose composition the node's bucket holds and the agent does not
|
|
// run yet (issue 1945, live on 2026-10-25 at 02:32: shanks' node-engine applied a node's own composition that no longer
|
|
// carried systemd-resolved, whose own composition waited for its credential, and removed its guard, so the machine lost
|
|
// its resolver).
|
|
func TestTheNodesOwnApplyHoldsOutAModuleThatMovedToItsOwnComposition(t *testing.T) {
|
|
b := newBed(t)
|
|
fp := fpOf("b")
|
|
// systemd-resolved's own composition is written; no want names it yet.
|
|
rk, rraw, _ := composition(t, "systemd-resolved", fp, false, file("systemd-resolved", "guard", "g"))
|
|
b.comps.before(rk, rraw)
|
|
nk, nraw, ndig := composition(t, NodeModule, "", false)
|
|
nwk, err := delivery.WantNodeKey(node)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
nwraw, err := delivery.EncodeNodeWant(delivery.NodeWant{Composed: ndig})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
b.comps.before(nk, nraw)
|
|
b.wants.before(nwk, nwraw)
|
|
stop := b.run()
|
|
defer stop()
|
|
eventually(t, "the node's own not applied", func() bool { return len(b.appliedList()) == 1 })
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
var names []string
|
|
for _, c := range b.others[0] {
|
|
names = append(names, c.Module)
|
|
}
|
|
if !strings.Contains(strings.Join(names, ","), "systemd-resolved") {
|
|
t.Fatalf("the others of the node's own apply were %v: systemd-resolved's resources would be removed", names)
|
|
}
|
|
|
|
// Issue 1126, review of the apply, round 1: a module the composer holds alone (de-spiegel leaving the fold with its
|
|
// names unset) has no composition of its own in the bucket: what runs of it in the node's own is held out of the
|
|
// node's own apply until one is handed.
|
|
b2 := newBed(t)
|
|
spiegel := file("de-spiegel", "server-env", "A=1")
|
|
var bare map[string]any
|
|
_ = json.Unmarshal(spiegel, &bare)
|
|
delete(bare, "module")
|
|
spiegel, _ = json.Marshal(bare)
|
|
b2.env.Kept = func() map[string][]json.RawMessage { return map[string][]json.RawMessage{NodeModule: {spiegel}} }
|
|
nk2, nraw2, ndig2 := composition(t, NodeModule, "", false)
|
|
nwraw2, _ := delivery.EncodeNodeWant(delivery.NodeWant{Composed: ndig2})
|
|
b2.comps.before(nk2, nraw2)
|
|
b2.wants.before(nwk, nwraw2)
|
|
ctx2, cancel2 := context.WithCancel(context.Background())
|
|
b2.ag = New(b2.env)
|
|
b2.ag.NotComposedEach(ctx2, "", map[string]string{"de-spiegel": "de-spiegel says ${config:praktijk-name}, and nothing sets it"})
|
|
done2 := make(chan struct{})
|
|
go func() { defer close(done2); _ = b2.ag.Run(ctx2) }()
|
|
defer func() { cancel2(); <-done2 }()
|
|
eventually(t, "the node's own not applied beside a held module", func() bool { return len(b2.appliedList()) == 1 })
|
|
b2.mu.Lock()
|
|
defer b2.mu.Unlock()
|
|
heldOut := false
|
|
for _, c := range b2.others[0] {
|
|
heldOut = heldOut || (c.Module == "de-spiegel" && len(c.Resources) == 1)
|
|
}
|
|
if !heldOut {
|
|
t.Errorf("the others of the node's own apply were %+v: held de-spiegel's resources would be removed", b2.others[0])
|
|
}
|
|
}
|
|
|
|
// A module the assigner removed holds nothing back, its compositions still in the bucket or not: its resources are its
|
|
// removal's to take (issue 1945).
|
|
func testARemovedModulesCompositionHoldsNothingBack(t *testing.T) {
|
|
b := newBed(t)
|
|
fp := fpOf("b")
|
|
rk, rraw, _ := composition(t, "gone", fp, false, file("gone", "x", "1"))
|
|
b.comps.before(rk, rraw)
|
|
gk, err := delivery.WantKey(node, "gone")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
graw, err := delivery.EncodeWant(delivery.Want{Removed: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
b.wants.before(gk, graw)
|
|
nk, nraw, ndig := composition(t, NodeModule, "", false)
|
|
nwk, _ := delivery.WantNodeKey(node)
|
|
nwraw, _ := delivery.EncodeNodeWant(delivery.NodeWant{Composed: ndig})
|
|
b.comps.before(nk, nraw)
|
|
b.wants.before(nwk, nwraw)
|
|
stop := b.run()
|
|
defer stop()
|
|
eventually(t, "the node's own not applied", func() bool {
|
|
for _, m := range b.appliedList() {
|
|
if m == NodeModule {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
})
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
for _, others := range b.others {
|
|
for _, c := range others {
|
|
if c.Module == "gone" {
|
|
t.Fatal("a removed module's composition was held out")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// The node's own composition is not applied while a module's composition in the node's bucket cannot be read or is not
|
|
// signed by a trusted key: what that one carries may be what left the node's own, and the node's own apply would remove
|
|
// it (issue 1945, the review's gap 2). It is said, naming the composition, and applied once that one reads.
|
|
func TestTheNodesOwnWaitsWhileAModulesCompositionCannotBeRead(t *testing.T) {
|
|
b := newBed(t)
|
|
bad, err := delivery.ComposedKey(node, "broken", fpOf("c"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
b.comps.before(bad, []byte(`{"v":1,"not":"a composition"}`))
|
|
nk, nraw, ndig := composition(t, NodeModule, "", false)
|
|
nwk, _ := delivery.WantNodeKey(node)
|
|
nwraw, _ := delivery.EncodeNodeWant(delivery.NodeWant{Composed: ndig})
|
|
b.comps.before(nk, nraw)
|
|
b.wants.before(nwk, nwraw)
|
|
stop := b.run()
|
|
defer stop()
|
|
eventually(t, "the wait not said", func() bool {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
for _, s := range b.said {
|
|
if strings.Contains(s, bad) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
})
|
|
if got := b.appliedList(); len(got) != 0 {
|
|
t.Fatalf("applied %v while a composition could not be read", got)
|
|
}
|
|
// Read again, the node's own is applied.
|
|
gk, graw, _ := composition(t, "broken", fpOf("c"), false, file("broken", "x", "1"))
|
|
b.comps.put(gk, graw)
|
|
eventually(t, "the node's own not applied", func() bool { return len(b.appliedList()) == 1 })
|
|
}
|
|
|
|
// A stale refused composition beside a newer readable one of the same module does not hold the node's own apply back:
|
|
// only the composition the module's want names, else its newest, counts (the review of issue 1945: the bucket keeps
|
|
// compositions written before the filled paths, which the SDK now refuses, and nobody applies them).
|
|
func TestAStaleRefusedCompositionDoesNotHoldTheNodesOwnBack(t *testing.T) {
|
|
b := newBed(t)
|
|
stale, err := delivery.ComposedKey(node, "power", fpOf("a"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
b.comps.before(stale, []byte(`{"v":1,"not":"a composition"}`))
|
|
gk, graw, _ := composition(t, "power", fpOf("b"), false, file("power", "x", "1"))
|
|
b.comps.before(gk, graw)
|
|
nk, nraw, ndig := composition(t, NodeModule, "", false)
|
|
nwk, _ := delivery.WantNodeKey(node)
|
|
nwraw, _ := delivery.EncodeNodeWant(delivery.NodeWant{Composed: ndig})
|
|
b.comps.before(nk, nraw)
|
|
b.wants.before(nwk, nwraw)
|
|
stop := b.run()
|
|
defer stop()
|
|
eventually(t, "the node's own not applied", func() bool { return len(b.appliedList()) == 1 })
|
|
}
|
|
|
|
// A composition the kept declaration or the node's own have says runs is applied all the same when the engine holds no
|
|
// record of its resources: they were removed since (issue 1945, live on 2026-10-25: ace's and g14's node-engines removed
|
|
// ca-trust's, power's and matrix's resources in a node's own apply, and the wants that named those modules' own
|
|
// compositions were judged already running from the kept declaration, so nothing made them again).
|
|
func TestACompositionWhoseResourcesWereRemovedIsAppliedAgain(t *testing.T) {
|
|
for _, from := range []string{"kept", "have"} {
|
|
t.Run(from, func(t *testing.T) {
|
|
b := newBed(t)
|
|
fp := fpOf("a")
|
|
rs := []json.RawMessage{file("ca-trust", "trust", "1")}
|
|
ck, craw, dig := composition(t, "ca-trust", fp, false, rs...)
|
|
wk, wraw := wantFor(t, "ca-trust", fp, dig, "")
|
|
b.comps.before(ck, craw)
|
|
b.wants.before(wk, wraw)
|
|
if from == "kept" {
|
|
b.kept = map[string][]json.RawMessage{"ca-trust": rs}
|
|
} else {
|
|
b.haves.stored = map[string]Have{"ca-trust": {Module: "ca-trust", FP: fp, Composed: dig, Phase: "running"}}
|
|
}
|
|
b.env.Recorded = func(context.Context, delivery.Composed) bool { return false }
|
|
stop := b.run()
|
|
defer stop()
|
|
eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
|
|
})
|
|
}
|
|
}
|
|
|
|
// The composition a want names is applied from the key's history when a newer one has replaced it under the same key:
|
|
// a want held at a fingerprint keeps naming the digest it was written with, and the controller writes every new
|
|
// composition at that fingerprint over the same key (ADR 0539 clause 1, the sweep's kept revisions; issue 1945, live on
|
|
// 2026-10-25: shanks' node-tools composition had digest 575a5c96, its held want named a36e63c8, and nothing was applied).
|
|
func TestTheRevisionAWantNamesIsAppliedFromHistory(t *testing.T) {
|
|
b := newBed(t)
|
|
fp := fpOf("a")
|
|
ck, oldRaw, oldDig := composition(t, "node-tools", fp, false, file("node-tools", "x", "old"))
|
|
_, newRaw, _ := composition(t, "node-tools", fp, false, file("node-tools", "x", "new"))
|
|
b.comps.before(ck, newRaw) // the key's latest value
|
|
wk, wraw := wantFor(t, "node-tools", fp, oldDig, "")
|
|
b.wants.before(wk, wraw)
|
|
b.env.Revisions = func(_ context.Context, key string) ([]Update, error) {
|
|
if key != ck {
|
|
return nil, nil
|
|
}
|
|
return []Update{{Key: ck, Value: oldRaw, Revision: 3}, {Key: ck, Value: newRaw, Revision: 7}}, nil
|
|
}
|
|
stop := b.run()
|
|
defer stop()
|
|
eventually(t, "the revision the want names was not applied", func() bool { return len(b.appliedList()) == 1 })
|
|
if h := b.haves.last("node-tools"); h.Composed != oldDig {
|
|
t.Fatalf("the have names %s, want %s", h.Composed, oldDig)
|
|
}
|
|
}
|
|
|
|
// 1945/AC18 and issue 924 (unit): what runs of a module the agent has not taken yet carries the filled paths the
|
|
// filling judges by. From the kept declaration while the module has no have naming a composition ("not named as filled"
|
|
// for claude-code and screen-lock, 2026-10-25); from the composition its have names once it has one, the kept
|
|
// declaration being older, read at the start before any module settles and told to the filling then, so a file the
|
|
// newer composition fills is accepted on the first fill (power's three drop-ins on ace, g14 and shanks, 2026-10-29).
|
|
// Only a have in a phase baseline takes as running names what runs; and what runs carries Data, from the composition
|
|
// or, from the kept declaration, from the module's manifest, so a data module is not judged to move its data against
|
|
// its first composition after a restart (mongodb and mssql on novox).
|
|
func TestWhatRunsBeforeTheAgentTakesAModuleNamesItsFilledPaths(t *testing.T) {
|
|
const old, added = "/tmp/r6/power/settings", "/tmp/r6/power/bound"
|
|
filledFile := func(id string) json.RawMessage {
|
|
return json.RawMessage(`{"id":"power.` + id + `","type":"file","module":"power","path":"/tmp/r6/power/` + id +
|
|
`","mode":"0644","owner-at-runtime":"owned"}`)
|
|
}
|
|
fp := fpOf("a")
|
|
ck, _ := delivery.ComposedKey(node, "power", fp)
|
|
applied := delivery.Composed{Node: node, Module: "power", FP: fp, Filled: []string{old, added}, Data: true,
|
|
Resources: []json.RawMessage{filledFile("settings"), filledFile("bound")}}
|
|
_, appliedRaw, appliedDig := signedBy(t, applied, ck, controllerKey)
|
|
newer := applied
|
|
newer.Resources = append([]json.RawMessage{}, applied.Resources...)
|
|
newer.Resources = append(newer.Resources, file("power", "later", "1"))
|
|
_, newerRaw, _ := signedBy(t, newer, ck, controllerKey)
|
|
for _, tc := range []struct {
|
|
name string
|
|
// have is the module's have; bucket the value its composition key holds now, history the revisions kept.
|
|
have *Have
|
|
bucket []byte
|
|
history []Update
|
|
filled, told string
|
|
// data is what runs' Data; holdsData the manifest's answer (nil: none read).
|
|
data bool
|
|
holdsData func(context.Context, string) (bool, bool)
|
|
}{
|
|
{name: "no have: the kept declaration's", filled: old},
|
|
{name: "a have naming the bucket's value", have: &Have{Module: "power", FP: fp, Composed: appliedDig},
|
|
bucket: appliedRaw, filled: old + "," + added, told: "the haves were read at the start", data: true},
|
|
{name: "a have applying names nothing that runs yet", have: &Have{Module: "power", FP: fp, Composed: appliedDig,
|
|
Phase: string(delivery.PhaseApplying)}, bucket: appliedRaw, filled: old, told: "the haves were read at the start"},
|
|
{name: "a data module from the kept declaration says Data as its manifest does", filled: old,
|
|
told: "the haves were read at the start", data: true,
|
|
holdsData: func(context.Context, string) (bool, bool) { return true, true }},
|
|
{name: "a have naming a revision replaced since", have: &Have{Module: "power", FP: fp, Composed: appliedDig},
|
|
bucket: newerRaw, history: []Update{{Key: ck, Value: appliedRaw, Revision: 3}, {Key: ck, Value: newerRaw, Revision: 7}},
|
|
filled: old + "," + added, told: "the haves were read at the start", data: true},
|
|
{name: "a have naming a composition the bucket no longer keeps", have: &Have{Module: "power", FP: fp, Composed: appliedDig},
|
|
bucket: newerRaw, filled: old, told: "the haves were read at the start"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
b := newBed(t)
|
|
b.kept = map[string][]json.RawMessage{"power": {filledFile("settings")}}
|
|
b.env.KeptFilled = func() map[string][]string { return map[string][]string{"power": {old}} }
|
|
if tc.have != nil {
|
|
b.haves.stored = map[string]Have{"power": *tc.have}
|
|
}
|
|
if tc.bucket != nil {
|
|
b.comps.before(ck, tc.bucket)
|
|
}
|
|
b.env.Revisions = func(_ context.Context, key string) ([]Update, error) { return tc.history, nil }
|
|
b.env.HoldsData = tc.holdsData
|
|
var told []string
|
|
var mu sync.Mutex
|
|
b.env.RunsChanged = func(why string) { mu.Lock(); told = append(told, why); mu.Unlock() }
|
|
filledNow := func() string {
|
|
for _, c := range b.ag.Everything() {
|
|
if c.Module == "power" {
|
|
if c.Data != tc.data {
|
|
t.Errorf("what runs says Data %v, want %v", c.Data, tc.data)
|
|
}
|
|
return strings.Join(c.Filled, ",")
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
if tc.told == "" { // no want, no have: nothing to start, and the kept declaration is what runs
|
|
a := New(b.env)
|
|
b.ag = a
|
|
if got := filledNow(); got != tc.filled {
|
|
t.Fatalf("filled %q, want %q", got, tc.filled)
|
|
}
|
|
return
|
|
}
|
|
stop := b.run()
|
|
defer stop()
|
|
eventually(t, "the filling was not told what runs at the start", func() bool {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
return len(told) > 0 && told[0] == tc.told
|
|
})
|
|
if got := filledNow(); got != tc.filled {
|
|
t.Fatalf("filled %q, want %q", got, tc.filled)
|
|
}
|
|
if got := b.appliedList(); len(got) != 0 {
|
|
t.Fatalf("applied %v: nothing has a want", got)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Issue 1126 (ADR 0547 §2, §3): with the node's own compositions, unsigned, and wants naming the build alone, the
|
|
// composition composed for the want's build is applied, and only when it differs from what runs; for a module of
|
|
// policy record a move that recreates a container waits in phase waiting-yes with its judgement until the want names
|
|
// the yes for that composition, and is then applied.
|
|
func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(t *testing.T) {
|
|
b := newBed(t)
|
|
b.env.SelfComposed, b.env.Trusted = true, nil
|
|
b.env.Policy = func(_ context.Context, m string) string {
|
|
if m == "db" || m == "rec" {
|
|
return catalogue.PolicyRecord
|
|
}
|
|
return catalogue.PolicyRoll
|
|
}
|
|
stop := b.run()
|
|
defer stop()
|
|
unsigned := func(module, fp string, rs ...json.RawMessage) (string, []byte, string) {
|
|
c := delivery.Composed{Node: node, Module: module, FP: fp, Resources: rs}
|
|
key, _ := delivery.ComposedKey(node, module, fp)
|
|
raw, err := delivery.EncodeComposed(c)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
dig, _ := delivery.ComposedDigestOf(c)
|
|
return key, raw, dig
|
|
}
|
|
want := func(module, fp, yes string) {
|
|
key, _ := delivery.WantKey(node, module)
|
|
raw, err := delivery.EncodeWant(delivery.Want{FP: fp, Yes: yes, Self: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
b.wants.put(key, raw)
|
|
}
|
|
|
|
fp := fpOf("a")
|
|
pw, _ := json.Marshal(map[string]any{"id": "web.pw", "type": "file", "path": "/tmp/r6/web/pw", "sealed": "1", "module": "web"})
|
|
ck, craw, dig := unsigned("web", fp, file("web", "x", "1"), pw)
|
|
b.comps.put(ck, craw)
|
|
want("web", fp, "")
|
|
eventually(t, "the node's own composition was not applied", func() bool { return len(b.appliedList()) == 1 })
|
|
eventually(t, "the have does not name the composed digest", func() bool { return b.haves.last("web").Composed == dig })
|
|
// The same composition handed again is no move.
|
|
b.comps.put(ck, craw)
|
|
time.Sleep(50 * time.Millisecond)
|
|
// The node's own composition carries its sealed values in its key, as a module's have does, so a restart reuses its
|
|
// core boxes and applies nothing again (review of L2, round 3, finding 1).
|
|
box, _ := json.Marshal(map[string]any{"id": "core-box", "type": "file", "path": "/tmp/r6/core-box", "sealed": "core"})
|
|
own := delivery.Composed{Node: node, Resources: []json.RawMessage{box}}
|
|
nk, _ := delivery.ComposedNodeKey(node)
|
|
nraw, _ := delivery.EncodeComposed(own)
|
|
ndig, _ := delivery.ComposedDigestOf(own)
|
|
b.comps.put(nk, nraw)
|
|
nwk, _ := delivery.WantNodeKey(node)
|
|
nwraw, _ := delivery.EncodeNodeWant(delivery.NodeWant{Self: true})
|
|
b.wants.put(nwk, nwraw)
|
|
eventually(t, "the node's own key does not carry its sealed values", func() bool {
|
|
h := b.haves.last(NodeModule)
|
|
return h.Composed == ndig && len(h.Sealed) == 1 && h.Sealed["core-box#sealed"] == "core"
|
|
})
|
|
// The node cannot compose (a stand-in refused): each wanted module's have says so, failed, naming why, and still
|
|
// names what runs; composed again, the have is what it runs. web was applied by this agent; idle was not since the
|
|
// agent's start (the switch and every delivery of the node-engine restart it), so what runs is what its have said.
|
|
idleFP, idleDig := fpOf("c"), "sha256:"+strings.Repeat("9", 64)
|
|
b.haves.Write(context.Background(), Have{Module: "idle", FP: idleFP, Composed: idleDig, Phase: string(delivery.PhaseRunning),
|
|
Sealed: delivery.Seals{"idle.key#sealed": `{"box":"idle"}`}})
|
|
want("idle", idleFP, "")
|
|
eventually(t, "idle's want was not taken", func() bool {
|
|
b.ag.mu.Lock()
|
|
defer b.ag.mu.Unlock()
|
|
return b.ag.wants["idle"].Present
|
|
})
|
|
if h := b.haves.last("web"); len(h.Sealed) != 1 || h.Sealed["web.pw#sealed"] != "1" {
|
|
t.Errorf("web's running have does not carry its sealed value: %+v", h)
|
|
}
|
|
b.ag.NotComposed(context.Background(), "a stand-in for web's password")
|
|
for _, row := range []struct {
|
|
module, fp, composed, slot, sealed string
|
|
}{{"web", fp, dig, "web.pw#sealed", "1"}, {"idle", idleFP, idleDig, "idle.key#sealed", `{"box":"idle"}`}} {
|
|
h := b.haves.last(row.module)
|
|
if h.Phase != string(delivery.PhaseFailed) || h.FP != row.fp || h.Composed != row.composed ||
|
|
len(h.Sealed) != 1 || h.Sealed[row.slot] != row.sealed || !strings.Contains(h.Detail, "web's password") {
|
|
t.Errorf("%s's have while the node cannot compose: %+v", row.module, h)
|
|
}
|
|
}
|
|
// A wanted module whose have cannot be read (the bus paused, issue 1126) is not marked: a mark would drop the build
|
|
// and digest its have names.
|
|
readAgain = time.Millisecond
|
|
b.haves.mu.Lock()
|
|
b.haves.unreadable = map[string]bool{"unread": true}
|
|
b.haves.mu.Unlock()
|
|
want("unread", fpOf("d"), "")
|
|
want("fresh", fpOf("e"), "") // never ran, its have absent: marked, naming nothing it runs
|
|
// A have that already says another cause keeps it (review of the switch, round 1): an apply that failed.
|
|
brokenFP, brokenDig := fpOf("f"), "sha256:"+strings.Repeat("8", 64)
|
|
b.haves.Write(context.Background(), Have{Module: "broken", FP: brokenFP, Composed: brokenDig, Phase: string(delivery.PhaseFailed),
|
|
Detail: "apply: unit failed to start"})
|
|
want("broken", brokenFP, "")
|
|
eventually(t, "unread's want was not taken", func() bool {
|
|
b.ag.mu.Lock()
|
|
defer b.ag.mu.Unlock()
|
|
return b.ag.wants["unread"].Present
|
|
})
|
|
b.ag.NotComposed(context.Background(), "a stand-in for web's password, again")
|
|
if h := b.haves.last("unread"); h.Module != "" {
|
|
t.Errorf("a module whose have could not be read was marked: %+v", h)
|
|
}
|
|
if h := b.haves.last("fresh"); h.Phase != string(delivery.PhaseFailed) || h.FP != "" {
|
|
t.Errorf("a module that never ran, its have absent, is not marked: %+v", h)
|
|
}
|
|
if h := b.haves.last("broken"); h.Detail != "apply: unit failed to start" || h.Composed != brokenDig {
|
|
t.Errorf("an apply failure was overwritten by a composer's mark: %+v", h)
|
|
}
|
|
// A run that ended marks nothing: what it would say is the next run's (review of the switch, round 1).
|
|
ended, end := context.WithCancel(context.Background())
|
|
end()
|
|
before := len(b.haves.log)
|
|
b.ag.NotComposed(ended, "a refusal from a run that ended")
|
|
if len(b.haves.log) != before {
|
|
t.Errorf("a run that ended wrote %d haves", len(b.haves.log)-before)
|
|
}
|
|
b.ag.NotComposed(context.Background(), "")
|
|
for _, row := range []struct{ module, composed string }{{"web", dig}, {"idle", idleDig}} {
|
|
if h := b.haves.last(row.module); h.Phase != string(delivery.PhaseRunning) || h.Composed != row.composed || len(h.Sealed) != 1 {
|
|
t.Errorf("%s's have once the node composes again: %+v", row.module, h)
|
|
}
|
|
}
|
|
// Issue 1126, the applying path: a module the node holds alone (its own refusal: de-spiegel's unset route names on
|
|
// novox) is marked alone, naming its own words; web, which composes, keeps running, unmarked. Composed again, idle's
|
|
// have is what it runs.
|
|
b.ag.NotComposedEach(context.Background(), "", map[string]string{"idle": "idle says ${config:name}, and nothing sets it"})
|
|
if h := b.haves.last("idle"); h.Phase != string(delivery.PhaseFailed) || h.Composed != idleDig || !strings.Contains(h.Detail, "${config:name}") {
|
|
t.Errorf("idle's have while it alone is held: %+v", h)
|
|
}
|
|
if h := b.haves.last("web"); h.Phase != string(delivery.PhaseRunning) || h.Composed != dig {
|
|
t.Errorf("web, which composes, is marked beside idle: %+v", h)
|
|
}
|
|
b.ag.NotComposedEach(context.Background(), "", nil)
|
|
if h := b.haves.last("idle"); h.Phase != string(delivery.PhaseRunning) || h.Composed != idleDig {
|
|
t.Errorf("idle's have once it composes again: %+v", h)
|
|
}
|
|
// A revision at the same build (a config value changed) is applied.
|
|
_, craw2, dig2 := unsigned("web", fp, file("web", "x", "2"))
|
|
b.comps.put(ck, craw2)
|
|
eventually(t, "the revision was not applied", func() bool { return len(b.appliedList()) == 3 && b.haves.last("web").Composed == dig2 })
|
|
|
|
// A record module: its first composition is no move; a new image recreates its container and waits for a yes.
|
|
container := func(image string) json.RawMessage {
|
|
module := strings.SplitN(image, "@", 2)[0]
|
|
raw, _ := json.Marshal(map[string]any{"id": module + ".server", "type": "container", "module": module, "name": module, "image": image})
|
|
return raw
|
|
}
|
|
dk, draw, _ := unsigned("db", fp, container("db@sha256:"+strings.Repeat("1", 64)))
|
|
b.comps.put(dk, draw)
|
|
want("db", fp, "")
|
|
eventually(t, "the record module's first composition was not applied", func() bool { return len(b.appliedList()) == 4 })
|
|
fpB := fpOf("b")
|
|
dk2, draw2, ddig2 := unsigned("db", fpB, container("db@sha256:"+strings.Repeat("2", 64)))
|
|
// rec is a record module this agent has run nothing of since its start (the switch and every delivery of the
|
|
// node-engine restart it): its have alone says what runs. Its move waits for a yes as db's does, and its have keeps
|
|
// what runs (fingerprint, digest, sealed values) rather than naming nothing (review of L2, round 3).
|
|
recFP, recDig := fpOf("d"), "sha256:"+strings.Repeat("8", 64)
|
|
b.haves.Write(context.Background(), Have{Module: "rec", FP: recFP, Composed: recDig, Phase: string(delivery.PhaseRunning),
|
|
Sealed: delivery.Seals{"rec.key#sealed": `{"box":"rec"}`}})
|
|
rk, rraw, rdig := unsigned("rec", fpB, container("rec@sha256:"+strings.Repeat("3", 64)))
|
|
b.comps.put(dk2, draw2)
|
|
b.comps.put(rk, rraw)
|
|
want("db", fpB, "")
|
|
want("rec", fpB, "")
|
|
applied := len(b.appliedList())
|
|
for _, row := range []struct {
|
|
module, fp, composed, waits string
|
|
sealed delivery.Seals
|
|
}{{"db", fp, "", ddig2, nil}, {"rec", recFP, recDig, rdig, delivery.Seals{"rec.key#sealed": `{"box":"rec"}`}}} {
|
|
eventually(t, row.module+"'s move did not wait for a yes", func() bool {
|
|
h := b.haves.last(row.module)
|
|
return h.Phase == string(delivery.PhaseWaitingYes) && h.Waits == row.waits
|
|
})
|
|
h := b.haves.last(row.module)
|
|
if h.FP != row.fp || (row.composed != "" && h.Composed != row.composed) || h.Composed == "" ||
|
|
(row.sealed != nil && !maps.Equal(h.Sealed, row.sealed)) {
|
|
t.Errorf("%s's have waiting for a yes does not keep what runs: %+v", row.module, h)
|
|
}
|
|
}
|
|
if got := b.appliedList(); len(got) != applied {
|
|
t.Fatalf("a move waiting for a yes was applied: %v", got)
|
|
}
|
|
want("db", fpB, ddig2)
|
|
want("rec", fpB, rdig)
|
|
eventually(t, "the yes did not apply the move", func() bool {
|
|
h, r := b.haves.last("db"), b.haves.last("rec")
|
|
return len(b.appliedList()) == applied+2 && h.Phase == string(delivery.PhaseRunning) && h.FP == fpB &&
|
|
r.Phase == string(delivery.PhaseRunning) && r.FP == fpB && r.Composed == rdig
|
|
})
|
|
}
|
|
|
|
// 1957/AC1 and AC4 (unit; ADR 0568 §1, §4): an engine start takes a module from what the machine shows, not only from the
|
|
// have or the kept declaration it kept. Every resource up applies nothing; one declared running and not active applies the
|
|
// module once, from a have or from the kept declaration; a module with no want is not looked at.
|
|
func testAnEngineStartReconcilesAModuleFromTheMachinesFacts(t *testing.T) {
|
|
for _, c := range []struct {
|
|
name string
|
|
have bool // the engine's own have says running at the want
|
|
held bool // else its have says held, from a failed backup
|
|
kept bool // else the kept declaration says it runs
|
|
inactive bool
|
|
unwanted bool
|
|
applied int
|
|
asked bool // the machine was looked at
|
|
}{
|
|
{"all up: nothing is applied", true, false, false, false, false, 0, true},
|
|
{"a unit not active: applied once", true, false, false, true, false, 1, true},
|
|
{"kept declaration, a unit not active: applied once", false, false, true, true, false, 1, true},
|
|
{"kept declaration, all up: nothing is applied", false, false, true, false, false, 0, true},
|
|
{"a held module applies nothing at start", false, true, false, true, false, 0, false},
|
|
{"no want: nothing is looked at or applied", true, false, false, true, true, 0, false},
|
|
} {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
b := newBed(t)
|
|
fp := fpOf("a")
|
|
rs := []json.RawMessage{file("web", "x", "1")}
|
|
ck, craw, dig := composition(t, "web", fp, false, rs...)
|
|
wk, wraw := wantFor(t, "web", fp, dig, "")
|
|
b.comps.before(ck, craw)
|
|
if !c.unwanted {
|
|
b.wants.before(wk, wraw)
|
|
}
|
|
if c.have {
|
|
b.haves.stored = map[string]Have{"web": {Module: "web", FP: fp, Composed: dig, Phase: "running"}}
|
|
}
|
|
if c.held {
|
|
b.haves.stored = map[string]Have{"web": {Module: "web", FP: fpOf("0"), Composed: "sha256:" + strings.Repeat("0", 64),
|
|
Phase: string(delivery.PhaseHeld), Detail: "restic: the repository is locked"}}
|
|
}
|
|
if c.kept {
|
|
b.kept = map[string][]json.RawMessage{"web": rs}
|
|
}
|
|
var mu sync.Mutex
|
|
asked := false
|
|
b.env.Inactive = func(context.Context, delivery.Composed) bool {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
asked = true
|
|
return c.inactive
|
|
}
|
|
stop := b.run()
|
|
defer stop()
|
|
if !c.unwanted && !c.held {
|
|
eventually(t, "the module was not settled", func() bool { return b.haves.last("web").Composed == dig })
|
|
}
|
|
time.Sleep(50 * time.Millisecond)
|
|
if got := b.appliedList(); len(got) != c.applied {
|
|
t.Fatalf("applied %v, want %d", got, c.applied)
|
|
}
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if asked != c.asked {
|
|
t.Fatalf("the machine was asked: %v, want %v", asked, c.asked)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// 1957/AC2, AC3 and AC4 (unit; ADR 0568 §2, §3, §4): the judge's edge, Reconcile, applies a running module again; one held
|
|
// for a failed backup and one with no want are left alone; and an apply that fails writes the have `failed` with the
|
|
// apply's own words.
|
|
func testReconcileAppliesARunningModuleAgainAndLeavesTheHeldAndTheUnwantedAlone(t *testing.T) {
|
|
b := newBed(t)
|
|
fp := fpOf("a")
|
|
ckW, crawW, digW := composition(t, "web", fp, false, file("web", "x", "1"))
|
|
wkW, wrawW := wantFor(t, "web", fp, digW, "")
|
|
ckD, crawD, digD := composition(t, "db", fp, true, file("db", "x", "1"))
|
|
wkD, wrawD := wantFor(t, "db", fp, digD, "")
|
|
b.comps.before(ckW, crawW)
|
|
b.wants.before(wkW, wrawW)
|
|
b.comps.before(ckD, crawD)
|
|
b.wants.before(wkD, wrawD)
|
|
b.haves.stored = map[string]Have{"web": {Module: "web", FP: fp, Composed: digW, Phase: "running"},
|
|
"db": {Module: "db", FP: fpOf("0"), Composed: "sha256:" + strings.Repeat("0", 64), Phase: "running"}}
|
|
b.backup = func(string) Backup { return Backup{Words: "restic: the repository is locked"} }
|
|
stop := b.run()
|
|
defer stop()
|
|
eventually(t, "web was not settled", func() bool { return b.haves.last("web").Composed == digW })
|
|
eventually(t, "db was not held", func() bool { return b.haves.last("db").Phase == string(delivery.PhaseHeld) })
|
|
if got := b.appliedList(); len(got) != 0 {
|
|
t.Fatalf("applied at the start: %v", got)
|
|
}
|
|
|
|
b.ag.Reconcile("db", "ghost")
|
|
time.Sleep(50 * time.Millisecond)
|
|
if got := b.appliedList(); len(got) != 0 {
|
|
t.Fatalf("a held module and an unwanted one were applied: %v", got)
|
|
}
|
|
if h := b.haves.last("db"); h.Phase != string(delivery.PhaseHeld) {
|
|
t.Fatalf("the held module's have changed: %+v", h)
|
|
}
|
|
|
|
b.ag.Reconcile("web")
|
|
eventually(t, "web was not applied again", func() bool { return len(b.appliedList()) == 1 })
|
|
eventually(t, "web is not running again", func() bool { return b.haves.last("web").Phase == string(delivery.PhaseRunning) })
|
|
|
|
b.mu.Lock()
|
|
b.failWith = "systemctl start web.service: exit status 1"
|
|
b.mu.Unlock()
|
|
b.ag.Reconcile("web")
|
|
eventually(t, "the failed re-apply did not write the have failed with its words", func() bool {
|
|
h := b.haves.last("web")
|
|
return h.Phase == string(delivery.PhaseFailed) && h.Detail == "systemctl start web.service: exit status 1"
|
|
})
|
|
time.Sleep(50 * time.Millisecond)
|
|
if got := b.appliedList(); len(got) != 2 {
|
|
t.Fatalf("a failed apply was tried again with no edge: %v", got)
|
|
}
|
|
}
|