package main import ( "context" "crypto/rand" "encoding/hex" "encoding/json" "errors" "fmt" "io" "net" "os" "os/exec" "path/filepath" "strings" "syscall" "time" "github.com/nats-io/nats.go" ) // restoreChunk is what is sent at a time; the server answers each before the next goes. const restoreChunk = 64 * 1024 // RestoreAll puts every stream of the archive into the server nc reaches, one at a time, and holds // each restored stream's state to the manifest. **Nothing is restored over anything**: a server that // already holds any of the streams is refused before the first is sent. func RestoreAll(ctx context.Context, nc *nats.Conn, a *Archive, log func(string, ...any)) error { var present []string for _, s := range a.Manifest.Streams { err := request(ctx, nc, apiStreamInfo+s.Name, nil, nil) var api *apiError switch { case err == nil: present = append(present, s.Name) case errors.As(err, &api) && api.Code == 404: default: return err } } if len(present) > 0 { return fmt.Errorf("the server already holds %s; nothing is restored over a live stream — restore into a new "+ "store (--into) and swap it in with the bus stopped", strings.Join(present, ", ")) } for _, s := range a.Manifest.Streams { started := time.Now() if err := restoreStream(ctx, nc, a, s); err != nil { return fmt.Errorf("stream %s: %w", s.Name, err) } log("%s: restored, at least %d message(s) to sequence %d, in %.2fs", s.Name, s.Messages, s.LastSeq, time.Since(started).Seconds()) } return nil } func restoreStream(ctx context.Context, nc *nats.Conn, a *Archive, s StreamEntry) error { meta, archive, err := a.open(s) if err != nil { return err } defer archive.Close() var req map[string]json.RawMessage if err := json.Unmarshal(meta, &req); err != nil || req["config"] == nil { return errors.New("its meta is not a stream's configuration and state") } var resp struct { DeliverSubject string `json:"deliver_subject"` } if err := request(ctx, nc, apiStreamRestore+s.Name, req, &resp); err != nil { return err } if resp.DeliverSubject == "" { return errors.New("the server gave nowhere to send the archive") } buf := make([]byte, restoreChunk) for { n, err := archive.Read(buf) if n > 0 { cctx, cancel := context.WithTimeout(ctx, 30*time.Second) reply, rerr := nc.RequestWithContext(cctx, resp.DeliverSubject, buf[:n]) cancel() if rerr != nil { return fmt.Errorf("sending the archive: %w", rerr) } if len(reply.Data) > 0 { return fmt.Errorf("the server refused the archive: %s", truncate(reply.Data)) } } if err == io.EOF { break } if err != nil { return err } } // The end: an empty message, answered once the server has rebuilt the stream from what it was // sent — which for a large stream takes a while. fctx, cancel := context.WithTimeout(ctx, 30*time.Minute) defer cancel() final, err := nc.RequestWithContext(fctx, resp.DeliverSubject, nil) if err != nil { return fmt.Errorf("finishing the restore: %w", err) } var created struct { Error *apiError `json:"error"` State streamState `json:"state"` } if err := json.Unmarshal(final.Data, &created); err != nil { return fmt.Errorf("the server's last answer is not JSON: %q", truncate(final.Data)) } if created.Error != nil { return created.Error } return heldTo(created.State, s) } // heldTo is whether a restored stream holds at least what its snapshot said it held. // // **At least, not exactly — found by the restore test, not assumed.** The state the server answers a // snapshot request with is taken when the snapshot starts; the stream goes on taking messages while // its blocks are read out, and a block read a moment later carries some of what was appended in that // moment. So a stream written during its snapshot restores with a later last sequence than the // manifest says (90075 where it said 90024, with a writer publishing every 2ms through it), and of // the messages in that tail some may be there and some not. Every message up to the manifest's last // sequence is there, exactly; that is the snapshot's promise, and what the test holds it to message // by message. What must never happen is a restored stream that ends before the manifest said it did. func heldTo(restored streamState, s StreamEntry) error { if restored.LastSeq < s.LastSeq || (s.Messages > 0 && restored.Messages == 0) { return fmt.Errorf("restored with %d message(s) up to sequence %d; the snapshot held %d up to %d", restored.Messages, restored.LastSeq, s.Messages, s.LastSeq) } return nil } // RestoreOffline fills a new store directory with the archive's streams, by a server of this // program's own: nats-server, the same binary the bus runs, on loopback, with the bus's one account // and a user that exists only for this run. It is stopped when the streams are in. What it leaves // is a store directory laid out exactly as the bus's — `jetstream//streams/` — which // a person swaps in for the bus's own with the bus stopped. It answers that jetstream directory. // // Why a server of its own rather than the live bus: a stream is restored only where it does not // exist, and on the live bus every stream exists — the controller asserts them on every start. So a // restore beside the live data, swapped in by a person, is the only one that never touches it. func RestoreOffline(ctx context.Context, a *Archive, into, account string, log func(string, ...any)) (string, error) { if !validName(account) { return "", fmt.Errorf("%q is not an account name", account) } if entries, err := os.ReadDir(into); err == nil && len(entries) > 0 { return "", fmt.Errorf("%s is not empty; a restore goes into a new directory, never over one", into) } if err := os.MkdirAll(into, 0o700); err != nil { return "", err } store, err := filepath.Abs(into) if err != nil { return "", err } binary, err := exec.LookPath("nats-server") if err != nil { return "", errors.New("no nats-server here to restore with; run this in the bus's own image") } work, err := os.MkdirTemp("", "mesh-nats-restore-server-*") if err != nil { return "", err } defer os.RemoveAll(work) port, err := freePort() if err != nil { return "", err } password := randomHex(24) conf := filepath.Join(work, "restore.conf") config := fmt.Sprintf(`listen: "127.0.0.1:%d" jetstream { store_dir: %q } accounts { %s { jetstream: enabled, users: [ { user: "restore", password: %q } ] } } `, port, store, account, password) if err := os.WriteFile(conf, []byte(config), 0o600); err != nil { return "", err } cmd := exec.Command(binary, "-c", conf) logFile, err := os.Create(filepath.Join(work, "server.log")) if err != nil { return "", err } defer logFile.Close() cmd.Stdout, cmd.Stderr = logFile, logFile if err := cmd.Start(); err != nil { return "", err } exited := make(chan error, 1) go func() { exited <- cmd.Wait() }() stop := func() error { _ = cmd.Process.Signal(syscall.SIGTERM) select { case err := <-exited: return err case <-time.After(30 * time.Second): _ = cmd.Process.Kill() <-exited return errors.New("the restore's server did not stop within 30s and was killed") } } serverLog := func() string { raw, _ := os.ReadFile(filepath.Join(work, "server.log")) return strings.TrimSpace(string(raw)) } url := fmt.Sprintf("nats://127.0.0.1:%d", port) var nc *nats.Conn for deadline := time.Now().Add(20 * time.Second); ; { nc, err = connect(url, Credential{User: "restore", Password: password}) if err == nil { break } if time.Now().After(deadline) { _ = stop() return "", fmt.Errorf("the restore's server did not come up: %v\n%s", err, serverLog()) } select { case err := <-exited: return "", fmt.Errorf("the restore's server stopped: %v\n%s", err, serverLog()) case <-time.After(200 * time.Millisecond): } } err = RestoreAll(ctx, nc, a, log) nc.Close() if stopErr := stop(); err == nil && stopErr != nil && !isTerminated(stopErr) { err = stopErr } if err != nil { return "", err } return filepath.Join(store, "jetstream"), nil } // isTerminated is a server that stopped because it was asked to. func isTerminated(err error) bool { var exit *exec.ExitError if errors.As(err, &exit) { if status, ok := exit.Sys().(syscall.WaitStatus); ok && status.Signaled() && status.Signal() == syscall.SIGTERM { return true } } return false } func freePort() (int, error) { l, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { return 0, err } defer l.Close() return l.Addr().(*net.TCPAddr).Port, nil } func randomHex(n int) string { b := make([]byte, n) if _, err := rand.Read(b); err != nil { panic(err) } return hex.EncodeToString(b) }