mesh-tools
661b83249c
mesh/merge-gate head 661b83249c2a merged into trunk bb291b5ad8af — pass: builds gitea, ssh-client → ace, g14, novox, shanks; no bus step; every machine com…
mesh/repo-check head 661b83249c2a merged into trunk bb291b5ad8af — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
544 lines
16 KiB
Go
544 lines
16 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/provider"
|
|
|
|
"gitea/internal/forgehttp"
|
|
)
|
|
|
|
const (
|
|
pubA = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
|
|
pubB = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"
|
|
)
|
|
|
|
// fakeBus is the bus as the provider reads and writes it.
|
|
type fakeBus struct {
|
|
mu sync.Mutex
|
|
buckets map[string]map[string][]byte
|
|
}
|
|
|
|
func newBus() *fakeBus { return &fakeBus{buckets: map[string]map[string][]byte{}} }
|
|
|
|
func (b *fakeBus) set(bucket, key string, v []byte) {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
if b.buckets[bucket] == nil {
|
|
b.buckets[bucket] = map[string][]byte{}
|
|
}
|
|
b.buckets[bucket][key] = v
|
|
}
|
|
func (b *fakeBus) Watch(_ context.Context, bucket string, h func(string, []byte)) error {
|
|
b.mu.Lock()
|
|
cur := map[string][]byte{}
|
|
for k, v := range b.buckets[bucket] {
|
|
cur[k] = v
|
|
}
|
|
b.mu.Unlock()
|
|
for k, v := range cur {
|
|
h(k, v)
|
|
}
|
|
return nil
|
|
}
|
|
func (b *fakeBus) Put(_ context.Context, bucket, key string, v []byte) error {
|
|
b.set(bucket, key, v)
|
|
return nil
|
|
}
|
|
func (b *fakeBus) Delete(_ context.Context, bucket, key string) error {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
delete(b.buckets[bucket], key)
|
|
return nil
|
|
}
|
|
func (b *fakeBus) entry(t *testing.T, key string) provider.Entry {
|
|
t.Helper()
|
|
b.mu.Lock()
|
|
raw := b.buckets[provider.BucketOf("gitea")][key]
|
|
b.mu.Unlock()
|
|
if raw == nil {
|
|
t.Fatalf("no entry %s", key)
|
|
}
|
|
var e provider.Entry
|
|
if err := json.Unmarshal(raw, &e); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return e
|
|
}
|
|
func (b *fakeBus) has(key string) bool {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
_, ok := b.buckets[provider.BucketOf("gitea")][key]
|
|
return ok
|
|
}
|
|
|
|
// bindClient makes ssh-client on node bind gitea's forge-account in the assignments mirror.
|
|
func (b *fakeBus) bindClient(node string) {
|
|
b.set(provider.AssignmentsBucket, node+".ssh-client", []byte(fmt.Sprintf(
|
|
`{"node":%q,"module":"ssh-client","provisioning":{"provisions":[{"provision":"forge-account","provider":"novox","provider-module":"gitea","as":"mesh_%s_ssh_client"}]}}`,
|
|
node, node)))
|
|
}
|
|
|
|
// fakeForge is gitea's API as far as the provider uses it.
|
|
type fakeForge struct {
|
|
mu sync.Mutex
|
|
users map[string]bool
|
|
teams map[string]bool
|
|
members []string
|
|
held map[string][]map[string]any
|
|
next int64
|
|
}
|
|
|
|
func newForge() *fakeForge {
|
|
return &fakeForge{users: map[string]bool{}, held: map[string][]map[string]any{}}
|
|
}
|
|
|
|
var keyRe = regexp.MustCompile(`^/api/v1/(?:admin/)?users/([^/]+)/keys(?:/(\d+))?$`)
|
|
|
|
func (f *fakeForge) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
body, _ := io.ReadAll(r.Body)
|
|
var in map[string]any
|
|
_ = json.Unmarshal(body, &in)
|
|
reply := func(code int, v any) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(code)
|
|
if v != nil {
|
|
_ = json.NewEncoder(w).Encode(v)
|
|
}
|
|
}
|
|
p := r.URL.Path
|
|
if m := keyRe.FindStringSubmatch(p); m != nil {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
reply(200, f.held[m[1]])
|
|
case http.MethodPost:
|
|
f.next++
|
|
f.held[m[1]] = append(f.held[m[1]], map[string]any{"id": f.next, "title": in["title"], "key": in["key"]})
|
|
reply(201, map[string]any{"id": f.next})
|
|
case http.MethodDelete:
|
|
var kept []map[string]any
|
|
for _, k := range f.held[m[1]] {
|
|
if fmt.Sprint(k["id"]) != m[2] {
|
|
kept = append(kept, k)
|
|
}
|
|
}
|
|
f.held[m[1]] = kept
|
|
reply(204, nil)
|
|
}
|
|
return
|
|
}
|
|
switch {
|
|
case r.Method == http.MethodGet && strings.HasPrefix(p, "/api/v1/users/"):
|
|
if !f.users[strings.TrimPrefix(p, "/api/v1/users/")] {
|
|
reply(404, map[string]any{"message": "no"})
|
|
return
|
|
}
|
|
reply(200, map[string]any{"active": true})
|
|
case r.Method == http.MethodPost && p == "/api/v1/admin/users":
|
|
f.users[in["username"].(string)] = true
|
|
reply(201, map[string]any{})
|
|
case p == "/api/v1/orgs/novox/teams" && r.Method == http.MethodGet:
|
|
reply(200, []map[string]any{{"id": 5, "name": "agents"}})
|
|
case p == "/api/v1/teams/5" && r.Method == http.MethodPatch:
|
|
reply(200, map[string]any{})
|
|
case strings.HasPrefix(p, "/api/v1/teams/5/members/") && r.Method == http.MethodPut:
|
|
f.members = append(f.members, strings.TrimPrefix(p, "/api/v1/teams/5/members/"))
|
|
reply(204, nil)
|
|
default:
|
|
http.Error(w, "unexpected "+r.Method+" "+p, 400)
|
|
}
|
|
}
|
|
|
|
func (f *fakeForge) titles(user string) []string {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
var out []string
|
|
for _, k := range f.held[user] {
|
|
out = append(out, k["title"].(string))
|
|
}
|
|
return out
|
|
}
|
|
|
|
type fakeRunner struct {
|
|
out string
|
|
err error
|
|
args []string
|
|
}
|
|
|
|
func (r *fakeRunner) Run(_ context.Context, name string, args ...string) ([]byte, error) {
|
|
r.args = append([]string{name}, args...)
|
|
return []byte(r.out), r.err
|
|
}
|
|
|
|
const scanned = "# 127.0.0.1:222 SSH-2.0-Go\n[127.0.0.1]:222 ssh-ed25519 AAAAC3HOSTKEY\n"
|
|
|
|
type rig struct {
|
|
t *testing.T
|
|
bus *fakeBus
|
|
forge *fakeForge
|
|
svc *Service
|
|
runner *fakeRunner
|
|
}
|
|
|
|
func newRig(t *testing.T) *rig {
|
|
t.Helper()
|
|
forge := newForge()
|
|
srv := httptest.NewServer(forge)
|
|
t.Cleanup(srv.Close)
|
|
dir := t.TempDir()
|
|
pw := filepath.Join(dir, "pw")
|
|
host := filepath.Join(dir, "host")
|
|
origins := filepath.Join(dir, "origins")
|
|
for p, c := range map[string]string{pw: "x\n", host: "novox.internal\n", origins: "https://git.novox.be\n"} {
|
|
if err := os.WriteFile(p, []byte(c), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
runner := &fakeRunner{out: scanned}
|
|
svc := NewService(Config{Node: "novox", HostFile: host, OriginsFile: origins, Port: 222,
|
|
Forge: forgehttp.Client{URL: srv.URL, User: "mesh-admin", PasswordFile: pw}, Runner: runner,
|
|
Log: func(string, ...any) {}})
|
|
svc.loaded = true // the machines' keys are read; a test that means otherwise says so
|
|
return &rig{t: t, bus: newBus(), forge: forge, svc: svc, runner: runner}
|
|
}
|
|
|
|
func (r *rig) pass() []provider.Outcome {
|
|
r.t.Helper()
|
|
p := r.svc.Provider(r.bus)
|
|
out, err := p.Pass(context.Background())
|
|
if err != nil {
|
|
r.t.Fatal(err)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (r *rig) key(node, account, public string) {
|
|
b, _ := json.Marshal(map[string]string{"node": node, "account": account, "kind": "machine", "public": public,
|
|
"fingerprint": "SHA256:x", "at": "2026-10-16T10:00:00Z"})
|
|
r.svc.KeyChanged(node+"."+account, b)
|
|
}
|
|
|
|
func TestAConsumerWithAKeyGetsItsEntryAndItsKeyIsRegistered(t *testing.T) {
|
|
r := newRig(t)
|
|
r.bus.bindClient("g14")
|
|
r.key("g14", "nox", pubA+" nox@g14 (mesh)")
|
|
r.pass()
|
|
e := r.bus.entry(t, "g14.ssh-client.forge-account")
|
|
want := map[string]any{"user": "agent-g14", "ssh_host": "novox.internal", "ssh_port": float64(222),
|
|
"known_host": "[novox.internal]:222 ssh-ed25519 AAAAC3HOSTKEY", "ssh_alias": "mesh-forge",
|
|
"host_key": "ssh-ed25519 AAAAC3HOSTKEY", "ssh_base": "ssh://git@novox.internal:222/",
|
|
"https_base": "https://git.novox.be/"}
|
|
for k, v := range want {
|
|
if e.Values[k] != v {
|
|
t.Errorf("%s = %v, want %v", k, e.Values[k], v)
|
|
}
|
|
}
|
|
if e.VaultName != "" || e.User != "" {
|
|
t.Errorf("a public key is no secret; the entry names a credential: %+v", e)
|
|
}
|
|
if got := r.forge.titles("agent-g14"); len(got) != 1 || got[0] != "mesh:g14.nox" {
|
|
t.Errorf("registered keys = %v", got)
|
|
}
|
|
if !r.forge.users["agent-g14"] || len(r.forge.members) != 1 || r.forge.members[0] != "agent-g14" {
|
|
t.Errorf("account %v, team members %v", r.forge.users, r.forge.members)
|
|
}
|
|
if strings.Join(r.runner.args, " ") != "ssh-keyscan -t ed25519 -p 222 127.0.0.1" {
|
|
t.Errorf("scan = %v", r.runner.args)
|
|
}
|
|
}
|
|
|
|
func TestAConsumerWithoutAKeyGetsNoEntryAndNoAccount(t *testing.T) {
|
|
r := newRig(t)
|
|
r.bus.bindClient("g14")
|
|
r.key("ace", "nox", pubB) // another machine's key is not g14's
|
|
out := r.pass()
|
|
if r.bus.has("g14.ssh-client.forge-account") {
|
|
t.Fatal("an entry was written for a machine that reported no key")
|
|
}
|
|
if len(r.forge.users) != 0 {
|
|
t.Fatalf("an account was made: %v", r.forge.users)
|
|
}
|
|
if len(out) != 1 || !strings.Contains(out[0].Why, "no key") {
|
|
t.Fatalf("the wait should be said: %+v", out)
|
|
}
|
|
}
|
|
|
|
func TestAKeyOfAnotherKindIsNotTheMachinesKey(t *testing.T) {
|
|
r := newRig(t)
|
|
r.bus.bindClient("g14")
|
|
b, _ := json.Marshal(map[string]string{"node": "g14", "account": "x", "kind": "session", "public": pubA})
|
|
r.svc.KeyChanged("g14.x", b)
|
|
r.pass()
|
|
if r.bus.has("g14.ssh-client.forge-account") {
|
|
t.Fatal("answered with a key that is not a machine's")
|
|
}
|
|
}
|
|
|
|
func TestAChangedKeyReplacesTheOldMeshKey(t *testing.T) {
|
|
r := newRig(t)
|
|
r.bus.bindClient("g14")
|
|
r.key("g14", "nox", pubA+" nox@g14 (mesh)")
|
|
r.pass()
|
|
r.forge.mu.Lock()
|
|
r.forge.held["agent-g14"] = append(r.forge.held["agent-g14"], map[string]any{"id": int64(99), "title": "my own", "key": pubB})
|
|
r.forge.mu.Unlock()
|
|
r.key("g14", "nox", "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC nox@g14 (mesh)")
|
|
r.pass()
|
|
got := strings.Join(r.forge.titles("agent-g14"), ",")
|
|
if got != "my own,mesh:g14.nox" {
|
|
t.Fatalf("keys = %s", got)
|
|
}
|
|
for _, k := range r.forge.held["agent-g14"] {
|
|
if k["title"] == "mesh:g14.nox" && !strings.Contains(k["key"].(string), "CCCC") {
|
|
t.Fatalf("the old key stayed: %v", k)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAConsumerThatNoLongerBindsLosesItsMeshKeysAndKeepsItsAccount(t *testing.T) {
|
|
r := newRig(t)
|
|
r.bus.bindClient("g14")
|
|
r.key("g14", "nox", pubA+" nox@g14 (mesh)")
|
|
r.pass()
|
|
r.forge.mu.Lock()
|
|
r.forge.held["agent-g14"] = append(r.forge.held["agent-g14"], map[string]any{"id": int64(99), "title": "my own", "key": pubB})
|
|
r.forge.mu.Unlock()
|
|
r.bus.mu.Lock()
|
|
delete(r.bus.buckets[provider.AssignmentsBucket], "g14.ssh-client")
|
|
r.bus.mu.Unlock()
|
|
r.pass()
|
|
if r.bus.has("g14.ssh-client.forge-account") {
|
|
t.Fatal("the entry stayed")
|
|
}
|
|
if got := r.forge.titles("agent-g14"); len(got) != 1 || got[0] != "my own" {
|
|
t.Fatalf("keys = %v", got)
|
|
}
|
|
if !r.forge.users["agent-g14"] {
|
|
t.Fatal("the account was removed")
|
|
}
|
|
}
|
|
|
|
func TestAKeyChangeNamesTheConsumersToAnswerAgain(t *testing.T) {
|
|
r := newRig(t)
|
|
var got []string
|
|
r.svc.OnKeys = func(node string) { got = append(got, node) }
|
|
r.key("g14", "nox", pubA)
|
|
if len(got) != 1 || got[0] != "g14" {
|
|
t.Fatalf("answered for %v", got)
|
|
}
|
|
}
|
|
|
|
func TestAKeyThatIsNotReadableIsNotKept(t *testing.T) {
|
|
r := newRig(t)
|
|
r.svc.KeyChanged("g14.nox", []byte(`{"node":"g14","kind":"machine"`))
|
|
r.svc.KeyChanged("g14.nox", []byte(`{"node":"g14","account":"nox","kind":"machine","public":"PRIVATE KEY"}`))
|
|
r.bus.bindClient("g14")
|
|
r.pass()
|
|
if r.bus.has("g14.ssh-client.forge-account") {
|
|
t.Fatal("a value that is no public key was registered")
|
|
}
|
|
}
|
|
|
|
func TestTheForgesHostKeyUnreadableMeansNoEntry(t *testing.T) {
|
|
r := newRig(t)
|
|
r.runner.err = errors.New("connection refused")
|
|
r.bus.bindClient("g14")
|
|
r.key("g14", "nox", pubA)
|
|
r.pass()
|
|
if r.bus.has("g14.ssh-client.forge-account") {
|
|
t.Fatal("an entry without the forge's host key")
|
|
}
|
|
}
|
|
|
|
func TestKnownHostLineIsRewrittenToTheMeshName(t *testing.T) {
|
|
line, err := knownHostLine(scanned, "novox.internal", 222)
|
|
if err != nil || line != "[novox.internal]:222 ssh-ed25519 AAAAC3HOSTKEY" {
|
|
t.Fatalf("%q %v", line, err)
|
|
}
|
|
if _, err := knownHostLine("# nothing\n", "novox.internal", 222); err == nil {
|
|
t.Fatal("no key line was accepted")
|
|
}
|
|
}
|
|
|
|
func TestKeysWatchedRetriesUntilItAnswersAndPassesOnChanges(t *testing.T) {
|
|
r := newRig(t)
|
|
src := &fakeKeys{failFirst: 2}
|
|
r.svc.Pause = func(time.Duration) {}
|
|
r.svc.loaded = false
|
|
passes := 0
|
|
r.svc.OnKeys = func(string) { passes++ }
|
|
if err := r.svc.WatchKeys(context.Background(), src); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !r.svc.loaded {
|
|
t.Fatal("the keys are not loaded once the watch has answered")
|
|
}
|
|
if src.calls != 3 {
|
|
t.Fatalf("asked %d times", src.calls)
|
|
}
|
|
src.send("g14.nox", "put", []byte(`{"node":"g14","account":"nox","kind":"machine","public":"`+pubA+`"}`), false)
|
|
if passes != 1 {
|
|
t.Fatalf("passes = %d", passes)
|
|
}
|
|
// What was there when the watch began is taken in, and one pass answers it, not one per key.
|
|
src.send("ace.nox", "put", []byte(`{"node":"ace","account":"nox","kind":"machine","public":"`+pubB+`"}`), true)
|
|
if passes != 1 {
|
|
t.Fatalf("a current value passed on its own: %d", passes)
|
|
}
|
|
src.send("g14.nox", "delete", nil, false)
|
|
if passes != 2 || len(r.svc.keysOf("g14")) != 0 {
|
|
t.Fatalf("passes = %d, keys = %v", passes, r.svc.keysOf("g14"))
|
|
}
|
|
}
|
|
|
|
type fakeKeys struct {
|
|
failFirst int
|
|
calls int
|
|
handle func(key, op string, value []byte, current bool)
|
|
}
|
|
|
|
func (f *fakeKeys) Watch(handle func(key, op string, value []byte, current bool)) error {
|
|
f.calls++
|
|
if f.calls <= f.failFirst {
|
|
return errors.New("not yet")
|
|
}
|
|
f.handle = handle
|
|
return nil
|
|
}
|
|
func (f *fakeKeys) send(key, op string, v []byte, current bool) { f.handle(key, op, v, current) }
|
|
|
|
func TestADeletedKeyEntryRemovesTheMeshKeyAndTheBoundEntryButNotTheOperatorsKey(t *testing.T) {
|
|
r := newRig(t)
|
|
r.bus.bindClient("g14")
|
|
r.key("g14", "nox", pubA+" nox@g14 (mesh)")
|
|
r.pass()
|
|
r.forge.mu.Lock()
|
|
r.forge.held["agent-g14"] = append(r.forge.held["agent-g14"], map[string]any{"id": int64(99), "title": "my own", "key": pubB})
|
|
r.forge.mu.Unlock()
|
|
r.svc.KeyGone("g14.nox")
|
|
r.pass()
|
|
if got := r.forge.titles("agent-g14"); len(got) != 1 || got[0] != "my own" {
|
|
t.Fatalf("keys = %v", got)
|
|
}
|
|
if r.bus.has("g14.ssh-client.forge-account") {
|
|
t.Fatal("the entry still serves a machine whose key is gone")
|
|
}
|
|
}
|
|
|
|
func TestAPassBeforeTheKeysAreLoadedRemovesAndDeletesNothing(t *testing.T) {
|
|
r := newRig(t)
|
|
r.bus.bindClient("g14")
|
|
r.key("g14", "nox", pubA+" nox@g14 (mesh)")
|
|
r.pass()
|
|
r.forge.mu.Lock()
|
|
r.forge.held["agent-g14"] = append(r.forge.held["agent-g14"], map[string]any{"id": int64(99), "title": "my own", "key": pubB})
|
|
r.forge.mu.Unlock()
|
|
// A restart: the keys are not read yet, so a node with none is not a node that reported none.
|
|
r.svc.mu.Lock()
|
|
r.svc.keys = map[string]machineKey{}
|
|
r.svc.loaded = false
|
|
r.svc.mu.Unlock()
|
|
out := r.pass()
|
|
if got := strings.Join(r.forge.titles("agent-g14"), ","); got != "mesh:g14.nox,my own" {
|
|
t.Fatalf("keys = %s", got)
|
|
}
|
|
if !r.bus.has("g14.ssh-client.forge-account") {
|
|
t.Fatal("the entry was deleted before the keys were read")
|
|
}
|
|
if len(out) != 1 || !strings.Contains(out[0].Why, "not read yet") {
|
|
t.Fatalf("the wait should be said: %+v", out)
|
|
}
|
|
// Loaded, and the node reports none: now its keys go.
|
|
r.svc.mu.Lock()
|
|
r.svc.loaded = true
|
|
r.svc.mu.Unlock()
|
|
r.pass()
|
|
if got := strings.Join(r.forge.titles("agent-g14"), ","); got != "my own" || r.bus.has("g14.ssh-client.forge-account") {
|
|
t.Fatalf("keys = %s", got)
|
|
}
|
|
}
|
|
|
|
// Issue 955: a watch the runtime refuses (a grant that comes later) must not end the provider for good.
|
|
func TestKeepRunningStartsTheProviderAgainAfterARefusedWatch(t *testing.T) {
|
|
r := newRig(t)
|
|
var pauses []time.Duration
|
|
var said []string
|
|
r.svc.Pause = func(d time.Duration) { pauses = append(pauses, d) }
|
|
r.svc.cfg.Log = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }
|
|
ctx, stop := context.WithCancel(context.Background())
|
|
defer stop()
|
|
calls := 0
|
|
serving := make(chan struct{})
|
|
done := make(chan struct{})
|
|
go func() {
|
|
defer close(done)
|
|
r.svc.KeepRunning(ctx, func(ctx context.Context) error {
|
|
calls++
|
|
if calls <= 2 {
|
|
return errors.New(`watching mesh-controller_assignments: gitea reads no state called "mesh-controller.assignments"`)
|
|
}
|
|
close(serving)
|
|
<-ctx.Done()
|
|
return ctx.Err()
|
|
})
|
|
}()
|
|
select {
|
|
case <-serving:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("the provider was not started again after its watch was refused")
|
|
}
|
|
stop()
|
|
<-done
|
|
if calls != 3 {
|
|
t.Fatalf("run %d times", calls)
|
|
}
|
|
if len(pauses) != 2 || pauses[0] != 5*time.Second || pauses[1] != 10*time.Second {
|
|
t.Fatalf("the backoff is %v", pauses)
|
|
}
|
|
if len(said) == 0 || !strings.Contains(said[0], "mesh-controller.assignments") {
|
|
t.Fatalf("a refusal is said loudly: %v", said)
|
|
}
|
|
loud := 0
|
|
for _, s := range said {
|
|
if strings.Contains(s, "stopped") {
|
|
loud++
|
|
}
|
|
}
|
|
if loud != 1 {
|
|
t.Fatalf("the same refusal is said %d times in one spell: %v", loud, said)
|
|
}
|
|
}
|
|
|
|
func TestKeepRunningBacksOffToTwoMinutes(t *testing.T) {
|
|
r := newRig(t)
|
|
var pauses []time.Duration
|
|
r.svc.Pause = func(d time.Duration) { pauses = append(pauses, d) }
|
|
ctx, stop := context.WithCancel(context.Background())
|
|
calls := 0
|
|
r.svc.KeepRunning(ctx, func(context.Context) error {
|
|
if calls++; calls == 9 {
|
|
stop()
|
|
}
|
|
return errors.New("refused")
|
|
})
|
|
last := pauses[len(pauses)-1]
|
|
if pauses[0] != 5*time.Second || last != 2*time.Minute {
|
|
t.Fatalf("the backoff is %v", pauses)
|
|
}
|
|
}
|