Merge pull request 'A failed backup names its cause: nats's dump keeps the snapshot's exit status and the container's state, restic reports the tail of what a command printed (issue 1585)' (#608) from fix/1585-dump-stderr into main
This commit was merged in pull request #608.
This commit is contained in:
@@ -0,0 +1,72 @@
|
||||
package main_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A failed snapshot dump keeps and says the real exit status (a container killed by a signal answers
|
||||
// 137), names the container's state, and leaves no partial archive.
|
||||
func TestTheDumpKeepsAndSaysTheStatusOfAFailedSnapshot(t *testing.T) {
|
||||
raw, err := os.ReadFile("module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Data struct {
|
||||
Own []struct {
|
||||
ID string `json:"id"`
|
||||
Backup struct {
|
||||
Dump string `json:"dump"`
|
||||
} `json:"backup"`
|
||||
} `json:"own"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dump := ""
|
||||
for _, d := range m.Data.Own {
|
||||
if d.ID == "jetstream" {
|
||||
dump = d.Backup.Dump
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
status string
|
||||
want string
|
||||
}{{"killed", "137", "exited 137 (killed by a signal)"}, {"failed", "3", "exited 3"}} {
|
||||
dir := t.TempDir()
|
||||
bin := filepath.Join(dir, "bin")
|
||||
os.MkdirAll(bin, 0o755)
|
||||
fake := "#!/bin/sh\nif [ \"$1\" = inspect ]; then echo 'container exited since yesterday'; exit 0; fi\necho partial-bytes\nexit " + c.status + "\n"
|
||||
os.WriteFile(filepath.Join(bin, "docker"), []byte(fake), 0o755)
|
||||
os.WriteFile(filepath.Join(dir, "broker"), nil, 0o644)
|
||||
cmdline := strings.NewReplacer("${dir:snapshots}", dir, "${dir:mesh-state}", dir).Replace(dump)
|
||||
cmd := exec.Command("sh", "-c", cmdline)
|
||||
cmd.Env = []string{"PATH=" + bin + ":/usr/bin:/bin"}
|
||||
out, err := cmd.CombinedOutput()
|
||||
ee, ok := err.(*exec.ExitError)
|
||||
if !ok || ee.ExitCode() != atoi(c.status) {
|
||||
t.Errorf("%s: exit %v, want %s; output %s", c.name, err, c.status, out)
|
||||
}
|
||||
if !strings.Contains(string(out), c.want) || !strings.Contains(string(out), "container exited since") {
|
||||
t.Errorf("%s: output %q", c.name, out)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "bus.tar.partial")); err == nil {
|
||||
t.Errorf("%s: partial left", c.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func atoi(s string) int {
|
||||
n := 0
|
||||
for _, r := range s {
|
||||
n = n*10 + int(r-'0')
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -82,7 +82,7 @@
|
||||
"class": "valuable",
|
||||
"active": "1d",
|
||||
"backup": {
|
||||
"dump": "docker exec -i mesh-broker-nats /opt/mesh/mesh-nats-snapshot snapshot < ${dir:mesh-state}/broker > ${dir:snapshots}/bus.tar.partial && mv ${dir:snapshots}/bus.tar.partial ${dir:snapshots}/bus.tar || { rm -f ${dir:snapshots}/bus.tar.partial; exit 1; }",
|
||||
"dump": "p=${dir:snapshots}/bus.tar.partial; s=0; docker exec -i mesh-broker-nats /opt/mesh/mesh-nats-snapshot snapshot < ${dir:mesh-state}/broker > \"$p\" || s=$?; if [ $s -ne 0 ]; then echo \"mesh-nats-snapshot snapshot exited $s$([ $s -eq 137 -o $s -eq 143 ] && echo ' (killed by a signal)')\" >&2; docker inspect -f 'container {{.State.Status}} since {{.State.StartedAt}}' mesh-broker-nats >&2; rm -f \"$p\"; exit $s; fi; mv \"$p\" ${dir:snapshots}/bus.tar || exit 1",
|
||||
"into": "snapshots"
|
||||
},
|
||||
"why": "the bus's streams and key-value buckets: the hand-act log, conditions, every module's state; written all the time, so copied by the server's own snapshot of each stream (novox/hq ADR 0235), never as live files"
|
||||
|
||||
@@ -36,6 +36,7 @@ import (
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// Runner runs one command and answers what it printed, so the backups can be tested without restic
|
||||
@@ -61,25 +62,41 @@ func execRunner(ctx context.Context, name string, args ...string) (string, error
|
||||
cmd := exec.CommandContext(ctx, program, argv...)
|
||||
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
said := strings.TrimSpace(stderr.String())
|
||||
if said == "" {
|
||||
said = strings.TrimSpace(stdout.String())
|
||||
}
|
||||
if program == "sudo" && strings.HasPrefix(said, "sudo:") {
|
||||
return "", fmt.Errorf("%s needs root and the runtime's account may not run it without a prompt: %s", name, said)
|
||||
}
|
||||
lines := strings.Split(said, "\n")
|
||||
if len(lines) > 3 {
|
||||
lines = lines[len(lines)-3:]
|
||||
}
|
||||
if said == "" {
|
||||
return "", fmt.Errorf("%s: %w", name, err)
|
||||
}
|
||||
return "", errors.New(strings.Join(lines, " / "))
|
||||
return "", failure(program, name, err, stdout.String(), stderr.String())
|
||||
}
|
||||
return stdout.String(), nil
|
||||
}
|
||||
|
||||
// failureTail bounds what a failed command's output adds to the failure it reports: its last 2 KB.
|
||||
const failureTail = 2048
|
||||
|
||||
var secretLooking = regexp.MustCompile(`(?i)((?:pass(?:word|wd)?|secret|token|api[_-]?key)["']?\s*[=:]\s*)("[^"]*"|'[^']*'|\S+)`)
|
||||
|
||||
// failure is the error of a command that exited badly: its last lines of stderr (stdout when stderr is
|
||||
// empty), bounded to failureTail bytes, with what looks like a secret blotted out, so the report of a
|
||||
// failed dump says why and not only that it exited. With nothing printed it keeps the exit status.
|
||||
func failure(program, name string, err error, stdout, stderr string) error {
|
||||
said := strings.TrimSpace(stderr)
|
||||
if said == "" {
|
||||
said = strings.TrimSpace(stdout)
|
||||
}
|
||||
if program == "sudo" && strings.HasPrefix(said, "sudo:") {
|
||||
return fmt.Errorf("%s needs root and the runtime's account may not run it without a prompt: %s", name, said)
|
||||
}
|
||||
if said == "" {
|
||||
return fmt.Errorf("%s: %w (it printed nothing)", name, err)
|
||||
}
|
||||
if len(said) > failureTail {
|
||||
cut := len(said) - failureTail
|
||||
for cut < len(said) && !utf8.RuneStart(said[cut]) {
|
||||
cut++
|
||||
}
|
||||
said = "..." + said[cut:]
|
||||
}
|
||||
said = secretLooking.ReplaceAllString(said, "${1}***")
|
||||
return fmt.Errorf("%s: %v: %s", name, err, strings.Join(strings.Split(said, "\n"), " / "))
|
||||
}
|
||||
|
||||
// Declared is what one module declared.
|
||||
type Declared struct {
|
||||
Module string `json:"module"`
|
||||
|
||||
@@ -333,3 +333,22 @@ func TestARestoreBesideAnotherAccountsDirectoryGoesToTheState(t *testing.T) {
|
||||
}
|
||||
_ = ran
|
||||
}
|
||||
|
||||
func TestAFailedCommandReportsTheTailOfWhatItPrintedWithSecretsBlotted(t *testing.T) {
|
||||
exit := errors.New("exit status 1")
|
||||
long := strings.Repeat("x", 5000) + "\npg_dump: error: password=hunter2 refused"
|
||||
for _, c := range []struct{ name, program, out, errOut, want, not string }{
|
||||
{"stderr", "sh", "", "pg_dump: connection refused\nsecond", "sh: exit status 1: pg_dump: connection refused / second", ""},
|
||||
{"stdout when stderr is empty", "sh", "dump failed\n", "", "sh: exit status 1: dump failed", ""},
|
||||
{"nothing printed", "sh", "", "", "sh: exit status 1 (it printed nothing)", ""},
|
||||
{"bounded and blotted", "sh", "", long, "password=*** refused", "hunter2"},
|
||||
{"json credential", "sh", "", `{"password": "hunter2", "user": "a"}`, `"password": ***`, "hunter2"},
|
||||
{"cut on a character", "sh", "", strings.Repeat("é", 1500), "...é", ""},
|
||||
{"sudo prompt", "sudo", "", "sudo: a password is required", "needs root", ""},
|
||||
} {
|
||||
got := failure(c.program, "sh", exit, c.out, c.errOut).Error()
|
||||
if !strings.Contains(got, c.want) || (c.not != "" && strings.Contains(got, c.not)) || len(got) > failureTail+100 {
|
||||
t.Errorf("%s: got %q", c.name, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user