295 lines
10 KiB
Go
295 lines
10 KiB
Go
package keeper
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"encoding/json"
|
|
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
|
)
|
|
|
|
// The keeper's snapshots before a drop or recreate (design 47 G §681, ADR 0675): each is recorded in its request's record
|
|
// in the state requests (path, size, digest, protected) before the live data is touched, counts against the directory's
|
|
// cap, is removed by the keeper after its days, and is told as a finding when the directory is filling.
|
|
|
|
const (
|
|
// DefaultSnapshotsCap is the cap of the snapshots directory unless the setting `snapshots-cap` says otherwise: 20 GiB.
|
|
DefaultSnapshotsCap int64 = 20 << 30
|
|
// DefaultSnapshotKeepDays is how long a snapshot and its record are kept after their request (ADR 0667 §12).
|
|
DefaultSnapshotKeepDays = 90
|
|
// SnapshotsFillingMark is the fill of the cap, in percent, past which `snapshots-filling` is raised; it is cleared
|
|
// back under it.
|
|
SnapshotsFillingMark int64 = 80
|
|
// FindingSnapshotsFilling is the finding's kind; its scope is the bus module and its id SnapshotsID.
|
|
FindingSnapshotsFilling = "snapshots-filling"
|
|
// SnapshotsID is the id of the finding `snapshots-filling`.
|
|
SnapshotsID = "nats.snapshots"
|
|
)
|
|
|
|
// Findings is the bus module's finding writer, a held writer of mesh-sdk (views.HeldWriter): the keeper writes and clears
|
|
// `snapshots-filling` through it (design 47 G §681, A-29).
|
|
type Findings interface {
|
|
Write(ctx context.Context, finding meshconditions.Finding) error
|
|
Clear(ctx context.Context, scope, id, kind, why string) error
|
|
}
|
|
|
|
// Files is the keeper's snapshot files: read whole for their size and digest, and removed.
|
|
type Files interface {
|
|
// Digest reads the file's size and the `sha256:` digest of its bytes.
|
|
Digest(path string) (size int64, digest string, err error)
|
|
Remove(path string) error
|
|
}
|
|
|
|
// DiskFiles is Files on the keeper's own disk.
|
|
type DiskFiles struct{}
|
|
|
|
// Digest reads the file.
|
|
func (DiskFiles) Digest(path string) (int64, string, error) {
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return 0, "", err
|
|
}
|
|
defer f.Close()
|
|
h := sha256.New()
|
|
n, err := io.Copy(h, f)
|
|
if err != nil {
|
|
return 0, "", err
|
|
}
|
|
return n, "sha256:" + hex.EncodeToString(h.Sum(nil)), nil
|
|
}
|
|
|
|
// Remove removes the file; one already gone is no error.
|
|
func (DiskFiles) Remove(path string) error {
|
|
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (k *Keeper) files() Files {
|
|
if k.Files == nil {
|
|
return DiskFiles{}
|
|
}
|
|
return k.Files
|
|
}
|
|
|
|
func (k *Keeper) snapshotsCap() int64 {
|
|
if k.SnapshotsCap <= 0 {
|
|
return DefaultSnapshotsCap
|
|
}
|
|
return k.SnapshotsCap
|
|
}
|
|
|
|
func (k *Keeper) keepDays() int {
|
|
if k.SnapshotKeepDays <= 0 {
|
|
return DefaultSnapshotKeepDays
|
|
}
|
|
return k.SnapshotKeepDays
|
|
}
|
|
|
|
// Recorded says whether the request's snapshot was written whole by a keeper of milestone G: it holds a path and a
|
|
// digest. A snapshot of before G holds neither, and the keeper never sends nor deletes it by age (§681).
|
|
func (r Request) Recorded() bool { return r.Path != "" && r.Digest != "" }
|
|
|
|
// Expired says whether the recorded snapshot is past its days at now.
|
|
func (r Request) Expired(now time.Time, days int) bool {
|
|
return r.Recorded() && !r.At.IsZero() && now.After(r.At.AddDate(0, 0, days))
|
|
}
|
|
|
|
// snapshotRecords is every request in the state requests holding a recorded snapshot, by key.
|
|
func (k *Keeper) snapshotRecords() (map[string]Request, error) {
|
|
keys, err := k.Store.Keys(StateRequests)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := map[string]Request{}
|
|
for _, key := range keys {
|
|
if strings.HasPrefix(key, ActedPrefix) {
|
|
continue
|
|
}
|
|
raw, _, found, err := k.Store.Get(StateRequests, key)
|
|
if err != nil || !found {
|
|
continue
|
|
}
|
|
var r Request
|
|
if json.Unmarshal(raw, &r) == nil && r.Recorded() {
|
|
out[key] = r
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// SnapshotsUsed is the bytes the recorded snapshots hold, but for the request skip.
|
|
func (k *Keeper) SnapshotsUsed(skip string) (int64, error) {
|
|
recs, err := k.snapshotRecords()
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
var used int64
|
|
for key, r := range recs {
|
|
if key != skip {
|
|
used += r.Size
|
|
}
|
|
}
|
|
return used, nil
|
|
}
|
|
|
|
// recordSnapshot reads the snapshot's size and digest, refuses one that would pass the cap (removing the file it wrote,
|
|
// so nothing is left over), and writes path, size, digest and protected into the request's record before anything is
|
|
// removed from the live data.
|
|
func (k *Keeper) recordSnapshot(ctx context.Context, r *Request, path string, streams []string) error {
|
|
size, digest, err := k.files().Digest(path)
|
|
if err != nil {
|
|
return fmt.Errorf("the snapshot at %s cannot be read whole, so nothing was removed: %w", path, err)
|
|
}
|
|
protected, err := k.Bus.Restricted(ctx, streams)
|
|
if err != nil {
|
|
_ = k.files().Remove(path)
|
|
return fmt.Errorf("whether the snapshot holds a stream no other account hears cannot be read, so nothing was removed: %w", err)
|
|
}
|
|
used, err := k.SnapshotsUsed(r.ID)
|
|
if err != nil {
|
|
_ = k.files().Remove(path)
|
|
return fmt.Errorf("the snapshots' fill cannot be read, so nothing was removed: %w", err)
|
|
}
|
|
if used+size > k.snapshotsCap() {
|
|
_ = k.files().Remove(path)
|
|
return fmt.Errorf("the snapshot (%d bytes) would pass the cap %d of the snapshots directory (%d held), so nothing was removed: raise snapshots-cap or remove a snapshot", size, k.snapshotsCap(), used)
|
|
}
|
|
r.Path, r.Size, r.Digest, r.Protected = path, size, digest, protected
|
|
data, _ := json.Marshal(r)
|
|
if _, err := k.Store.Put(StateRequests, r.ID, data); err != nil {
|
|
_ = k.files().Remove(path)
|
|
r.Path, r.Size, r.Digest, r.Protected = "", 0, "", false
|
|
return fmt.Errorf("the snapshot cannot be recorded, so nothing was removed: %w", err)
|
|
}
|
|
k.snapshotsFill(used + size)
|
|
return nil
|
|
}
|
|
|
|
// ExpireSnapshots removes every recorded snapshot past its days, and its record, each removal said in the journal:
|
|
// at the keeper's start and at each new request, never on a timer. It judges the fill after.
|
|
func (k *Keeper) ExpireSnapshots() { k.expireSnapshots() }
|
|
|
|
// expireSnapshots is ExpireSnapshots, answering how many it removed.
|
|
func (k *Keeper) expireSnapshots() int {
|
|
recs, err := k.snapshotRecords()
|
|
if err != nil {
|
|
k.logf("the snapshots cannot be read: %v", err)
|
|
return 0
|
|
}
|
|
removed := 0
|
|
var used int64
|
|
for key, r := range recs {
|
|
if !r.Expired(k.now(), k.keepDays()) {
|
|
used += r.Size
|
|
continue
|
|
}
|
|
if err := k.files().Remove(r.Path); err != nil {
|
|
k.logf("the snapshot %s of %s, past its %d days, cannot be removed: %v", r.Path, key, k.keepDays(), err)
|
|
used += r.Size
|
|
continue
|
|
}
|
|
if err := k.Store.Delete(StateRequests, key); err != nil {
|
|
k.logf("the record of the removed snapshot %s cannot be removed: %v", key, err)
|
|
used += r.Size
|
|
continue
|
|
}
|
|
k.logf("removed the snapshot %s of the request %s, taken %s, past its %d days", r.Path, key, r.At.Format(time.RFC3339), k.keepDays())
|
|
removed++
|
|
}
|
|
k.snapshotsFill(used)
|
|
return removed
|
|
}
|
|
|
|
// removeSnapshots is the seat mesh-broker's snapshot-remove: the snapshot of one request, or every one past its days
|
|
// (all-expired). The hand act is written first, and a snapshot that cannot be recorded stays; a snapshot of before G goes only
|
|
// by its request (design 47 G §681). The answer is `accepted` with the request's own id as the act.
|
|
func (k *Keeper) removeSnapshots(r *Request) (string, error) {
|
|
if (r.Of == "") == !r.AllExpired {
|
|
return "error", fmt.Errorf("snapshot-remove names a request, or all-expired, not both and not neither")
|
|
}
|
|
what := "removed every snapshot past its days"
|
|
if r.Of != "" {
|
|
what = "removed the snapshot of the request " + r.Of
|
|
}
|
|
if k.Act != nil {
|
|
if err := k.Act(what+", asked by "+r.Caller, r.Why, "mesh-broker_snapshot_remove"); err != nil {
|
|
return "error", fmt.Errorf("the hand act cannot be recorded, so nothing is removed: %w", err)
|
|
}
|
|
}
|
|
if r.AllExpired {
|
|
r.Result = fmt.Sprintf("%d removed", k.expireSnapshots())
|
|
return "accepted", nil
|
|
}
|
|
raw, _, found, err := k.Store.Get(StateRequests, r.Of)
|
|
var rec Request
|
|
if err != nil || !found || json.Unmarshal(raw, &rec) != nil || (rec.Path == "" && rec.Snapshot == "") {
|
|
return "error", fmt.Errorf("no snapshot of the request %s is held", r.Of)
|
|
}
|
|
path := rec.Path
|
|
if path == "" {
|
|
path = rec.Snapshot
|
|
}
|
|
if err := k.files().Remove(path); err != nil {
|
|
return "error", fmt.Errorf("the snapshot %s cannot be removed: %w", path, err)
|
|
}
|
|
if err := k.Store.Delete(StateRequests, r.Of); err != nil {
|
|
return "error", fmt.Errorf("the record of the removed snapshot %s cannot be removed: %w", r.Of, err)
|
|
}
|
|
k.logf("removed the snapshot %s of the request %s by hand, asked by %s", path, r.Of, r.Caller)
|
|
if used, err := k.SnapshotsUsed(""); err == nil {
|
|
k.snapshotsFill(used)
|
|
}
|
|
return "accepted", nil
|
|
}
|
|
|
|
// SnapshotsAtStart is the keeper's first look at the snapshots at its start: what is past its days removed, and the fill
|
|
// judged afresh, so a finding raised before a restart that no longer holds is cleared.
|
|
func (k *Keeper) SnapshotsAtStart() {
|
|
k.mu.Lock()
|
|
k.fillUnjudged = true
|
|
k.mu.Unlock()
|
|
k.ExpireSnapshots()
|
|
}
|
|
|
|
// snapshotsFill raises the finding `snapshots-filling` past 80 % of the cap and clears it once back under, judged at
|
|
// every snapshot taken and removed and at the keeper's start.
|
|
func (k *Keeper) snapshotsFill(used int64) {
|
|
capBytes := k.snapshotsCap()
|
|
pct := used * 100 / capBytes
|
|
k.mu.Lock()
|
|
was, unjudged := k.filling, k.fillUnjudged
|
|
k.filling, k.fillUnjudged = pct >= SnapshotsFillingMark, false
|
|
now := k.filling
|
|
k.mu.Unlock()
|
|
if k.Finding == nil {
|
|
return
|
|
}
|
|
ctx := context.Background()
|
|
switch {
|
|
case now:
|
|
f := meshconditions.Finding{Scope: "module", ID: SnapshotsID, Kind: FindingSnapshotsFilling,
|
|
Subject: meshconditions.Subject{Scope: "module", ID: Module},
|
|
Severity: meshconditions.FindingSeverityWarning, Headline: "The bus module's snapshots are filling up",
|
|
Summary: fmt.Sprintf("the snapshots hold %d of %d bytes (%d%%)", used, capBytes, pct),
|
|
Needs: "snapshots removed, or a larger snapshots-cap",
|
|
Explanation: fmt.Sprintf("the snapshots hold %d of %d bytes (%d%%); at the cap a drop or recreate is refused; cleared under %d%%",
|
|
used, capBytes, pct, SnapshotsFillingMark),
|
|
Source: "snapshots", Observer: Module, At: k.now()}
|
|
if err := k.Finding.Write(ctx, f); err != nil {
|
|
k.logf("the finding %s cannot be written: %v", FindingSnapshotsFilling, err)
|
|
}
|
|
case was || unjudged:
|
|
if err := k.Finding.Clear(ctx, "module", SnapshotsID, FindingSnapshotsFilling, "the snapshots are under 80 % of their cap"); err != nil {
|
|
k.logf("the finding %s cannot be cleared: %v", FindingSnapshotsFilling, err)
|
|
}
|
|
}
|
|
}
|