diff --git a/modules/nats/dump_test.go b/modules/nats/dump_test.go new file mode 100644 index 00000000..7f8009c6 --- /dev/null +++ b/modules/nats/dump_test.go @@ -0,0 +1,72 @@ +package main_test + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// A failed snapshot dump keeps and says the real exit status (a container killed by a signal answers +// 137), names the container's state, and leaves no partial archive. +func TestTheDumpKeepsAndSaysTheStatusOfAFailedSnapshot(t *testing.T) { + raw, err := os.ReadFile("module.json") + if err != nil { + t.Fatal(err) + } + var m struct { + Data struct { + Own []struct { + ID string `json:"id"` + Backup struct { + Dump string `json:"dump"` + } `json:"backup"` + } `json:"own"` + } `json:"data"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + dump := "" + for _, d := range m.Data.Own { + if d.ID == "jetstream" { + dump = d.Backup.Dump + } + } + for _, c := range []struct { + name string + status string + want string + }{{"killed", "137", "exited 137 (killed by a signal)"}, {"failed", "3", "exited 3"}} { + dir := t.TempDir() + bin := filepath.Join(dir, "bin") + os.MkdirAll(bin, 0o755) + fake := "#!/bin/sh\nif [ \"$1\" = inspect ]; then echo 'container exited since yesterday'; exit 0; fi\necho partial-bytes\nexit " + c.status + "\n" + os.WriteFile(filepath.Join(bin, "docker"), []byte(fake), 0o755) + os.WriteFile(filepath.Join(dir, "broker"), nil, 0o644) + cmdline := strings.NewReplacer("${dir:snapshots}", dir, "${dir:mesh-state}", dir).Replace(dump) + cmd := exec.Command("sh", "-c", cmdline) + cmd.Env = []string{"PATH=" + bin + ":/usr/bin:/bin"} + out, err := cmd.CombinedOutput() + ee, ok := err.(*exec.ExitError) + if !ok || ee.ExitCode() != atoi(c.status) { + t.Errorf("%s: exit %v, want %s; output %s", c.name, err, c.status, out) + } + if !strings.Contains(string(out), c.want) || !strings.Contains(string(out), "container exited since") { + t.Errorf("%s: output %q", c.name, out) + } + if _, err := os.Stat(filepath.Join(dir, "bus.tar.partial")); err == nil { + t.Errorf("%s: partial left", c.name) + } + } +} + +func atoi(s string) int { + n := 0 + for _, r := range s { + n = n*10 + int(r-'0') + } + return n +} diff --git a/modules/nats/module.json b/modules/nats/module.json index 3e58475c..4f028cc2 100644 --- a/modules/nats/module.json +++ b/modules/nats/module.json @@ -82,7 +82,7 @@ "class": "valuable", "active": "1d", "backup": { - "dump": "docker exec -i mesh-broker-nats /opt/mesh/mesh-nats-snapshot snapshot < ${dir:mesh-state}/broker > ${dir:snapshots}/bus.tar.partial && mv ${dir:snapshots}/bus.tar.partial ${dir:snapshots}/bus.tar || { rm -f ${dir:snapshots}/bus.tar.partial; exit 1; }", + "dump": "p=${dir:snapshots}/bus.tar.partial; s=0; docker exec -i mesh-broker-nats /opt/mesh/mesh-nats-snapshot snapshot < ${dir:mesh-state}/broker > \"$p\" || s=$?; if [ $s -ne 0 ]; then echo \"mesh-nats-snapshot snapshot exited $s$([ $s -eq 137 -o $s -eq 143 ] && echo ' (killed by a signal)')\" >&2; docker inspect -f 'container {{.State.Status}} since {{.State.StartedAt}}' mesh-broker-nats >&2; rm -f \"$p\"; exit $s; fi; mv \"$p\" ${dir:snapshots}/bus.tar || exit 1", "into": "snapshots" }, "why": "the bus's streams and key-value buckets: the hand-act log, conditions, every module's state; written all the time, so copied by the server's own snapshot of each stream (novox/hq ADR 0235), never as live files" diff --git a/modules/restic/cmd/restic-backups/backups.go b/modules/restic/cmd/restic-backups/backups.go index bc8181e7..6c468dcc 100644 --- a/modules/restic/cmd/restic-backups/backups.go +++ b/modules/restic/cmd/restic-backups/backups.go @@ -36,6 +36,7 @@ import ( "sync" "syscall" "time" + "unicode/utf8" ) // Runner runs one command and answers what it printed, so the backups can be tested without restic @@ -69,7 +70,7 @@ func execRunner(ctx context.Context, name string, args ...string) (string, error // failureTail bounds what a failed command's output adds to the failure it reports: its last 2 KB. const failureTail = 2048 -var secretLooking = regexp.MustCompile(`(?i)((?:pass(?:word|wd)?|secret|token|api[_-]?key)\s*[=:]\s*)\S+`) +var secretLooking = regexp.MustCompile(`(?i)((?:pass(?:word|wd)?|secret|token|api[_-]?key)["']?\s*[=:]\s*)("[^"]*"|'[^']*'|\S+)`) // failure is the error of a command that exited badly: its last lines of stderr (stdout when stderr is // empty), bounded to failureTail bytes, with what looks like a secret blotted out, so the report of a @@ -86,7 +87,11 @@ func failure(program, name string, err error, stdout, stderr string) error { return fmt.Errorf("%s: %w (it printed nothing)", name, err) } if len(said) > failureTail { - said = "..." + said[len(said)-failureTail:] + cut := len(said) - failureTail + for cut < len(said) && !utf8.RuneStart(said[cut]) { + cut++ + } + said = "..." + said[cut:] } said = secretLooking.ReplaceAllString(said, "${1}***") return fmt.Errorf("%s: %v: %s", name, err, strings.Join(strings.Split(said, "\n"), " / ")) diff --git a/modules/restic/cmd/restic-backups/backups_test.go b/modules/restic/cmd/restic-backups/backups_test.go index 64695cff..c0560558 100644 --- a/modules/restic/cmd/restic-backups/backups_test.go +++ b/modules/restic/cmd/restic-backups/backups_test.go @@ -342,6 +342,8 @@ func TestAFailedCommandReportsTheTailOfWhatItPrintedWithSecretsBlotted(t *testin {"stdout when stderr is empty", "sh", "dump failed\n", "", "sh: exit status 1: dump failed", ""}, {"nothing printed", "sh", "", "", "sh: exit status 1 (it printed nothing)", ""}, {"bounded and blotted", "sh", "", long, "password=*** refused", "hunter2"}, + {"json credential", "sh", "", `{"password": "hunter2", "user": "a"}`, `"password": ***`, "hunter2"}, + {"cut on a character", "sh", "", strings.Repeat("é", 1500), "...é", ""}, {"sudo prompt", "sudo", "", "sudo: a password is required", "needs root", ""}, } { got := failure(c.program, "sh", exit, c.out, c.errOut).Error()