package keeper import ( "context" "crypto/sha256" "encoding/hex" "fmt" "io" "os" "strings" "time" "encoding/json" "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" ) // The keeper's snapshots before a drop or recreate (design 47 G §681, ADR 0675): each is recorded in its request's record // in the state requests (path, size, digest, protected) before the live data is touched, counts against the directory's // cap, is removed by the keeper after its days, and is told as a finding when the directory is filling. const ( // DefaultSnapshotsCap is the cap of the snapshots directory unless the setting `snapshots-cap` says otherwise: 20 GiB. DefaultSnapshotsCap int64 = 20 << 30 // DefaultSnapshotKeepDays is how long a snapshot and its record are kept after their request (ADR 0667 §12). DefaultSnapshotKeepDays = 90 // SnapshotsFillingMark is the fill of the cap, in percent, past which `snapshots-filling` is raised; it is cleared // back under it. SnapshotsFillingMark int64 = 80 // FindingSnapshotsFilling is the finding's kind; its scope is the bus module and its id SnapshotsID. FindingSnapshotsFilling = "snapshots-filling" // SnapshotsID is the id of the finding `snapshots-filling`. SnapshotsID = "nats.snapshots" ) // Findings is the bus module's finding writer, a held writer of mesh-sdk (views.HeldWriter): the keeper writes and clears // `snapshots-filling` through it (design 47 G §681, A-29). type Findings interface { Write(ctx context.Context, finding meshconditions.Finding) error Clear(ctx context.Context, scope, id, kind, why string) error } // Files is the keeper's snapshot files: read whole for their size and digest, and removed. type Files interface { // Digest reads the file's size and the `sha256:` digest of its bytes. Digest(path string) (size int64, digest string, err error) Remove(path string) error } // DiskFiles is Files on the keeper's own disk. type DiskFiles struct{} // Digest reads the file. func (DiskFiles) Digest(path string) (int64, string, error) { f, err := os.Open(path) if err != nil { return 0, "", err } defer f.Close() h := sha256.New() n, err := io.Copy(h, f) if err != nil { return 0, "", err } return n, "sha256:" + hex.EncodeToString(h.Sum(nil)), nil } // Remove removes the file; one already gone is no error. func (DiskFiles) Remove(path string) error { if err := os.Remove(path); err != nil && !os.IsNotExist(err) { return err } return nil } func (k *Keeper) files() Files { if k.Files == nil { return DiskFiles{} } return k.Files } func (k *Keeper) snapshotsCap() int64 { if k.SnapshotsCap <= 0 { return DefaultSnapshotsCap } return k.SnapshotsCap } func (k *Keeper) keepDays() int { if k.SnapshotKeepDays <= 0 { return DefaultSnapshotKeepDays } return k.SnapshotKeepDays } // Recorded says whether the request's snapshot was written whole by a keeper of milestone G: it holds a path and a // digest. A snapshot of before G holds neither, and the keeper never sends nor deletes it by age (§681). func (r Request) Recorded() bool { return r.Path != "" && r.Digest != "" } // Expired says whether the recorded snapshot is past its days at now. func (r Request) Expired(now time.Time, days int) bool { return r.Recorded() && !r.At.IsZero() && now.After(r.At.AddDate(0, 0, days)) } // snapshotRecords is every request in the state requests holding a recorded snapshot, by key. func (k *Keeper) snapshotRecords() (map[string]Request, error) { keys, err := k.Store.Keys(StateRequests) if err != nil { return nil, err } out := map[string]Request{} for _, key := range keys { if strings.HasPrefix(key, ActedPrefix) { continue } raw, _, found, err := k.Store.Get(StateRequests, key) if err != nil || !found { continue } var r Request if json.Unmarshal(raw, &r) == nil && r.Recorded() { out[key] = r } } return out, nil } // SnapshotsUsed is the bytes the recorded snapshots hold, but for the request skip. func (k *Keeper) SnapshotsUsed(skip string) (int64, error) { recs, err := k.snapshotRecords() if err != nil { return 0, err } var used int64 for key, r := range recs { if key != skip { used += r.Size } } return used, nil } // recordSnapshot reads the snapshot's size and digest, refuses one that would pass the cap (removing the file it wrote, // so nothing is left over), and writes path, size, digest and protected into the request's record before anything is // removed from the live data. func (k *Keeper) recordSnapshot(ctx context.Context, r *Request, path string, streams []string) error { size, digest, err := k.files().Digest(path) if err != nil { return fmt.Errorf("the snapshot at %s cannot be read whole, so nothing was removed: %w", path, err) } protected, err := k.Bus.Restricted(ctx, streams) if err != nil { _ = k.files().Remove(path) return fmt.Errorf("whether the snapshot holds a stream no other account hears cannot be read, so nothing was removed: %w", err) } used, err := k.SnapshotsUsed(r.ID) if err != nil { _ = k.files().Remove(path) return fmt.Errorf("the snapshots' fill cannot be read, so nothing was removed: %w", err) } if used+size > k.snapshotsCap() { _ = k.files().Remove(path) return fmt.Errorf("the snapshot (%d bytes) would pass the cap %d of the snapshots directory (%d held), so nothing was removed: raise snapshots-cap or remove a snapshot", size, k.snapshotsCap(), used) } r.Path, r.Size, r.Digest, r.Protected = path, size, digest, protected data, _ := json.Marshal(r) if _, err := k.Store.Put(StateRequests, r.ID, data); err != nil { _ = k.files().Remove(path) r.Path, r.Size, r.Digest, r.Protected = "", 0, "", false return fmt.Errorf("the snapshot cannot be recorded, so nothing was removed: %w", err) } k.snapshotsFill(used + size) return nil } // ExpireSnapshots removes every recorded snapshot past its days, and its record, each removal said in the journal: // at the keeper's start and at each new request, never on a timer. It judges the fill after. func (k *Keeper) ExpireSnapshots() { k.expireSnapshots() } // expireSnapshots is ExpireSnapshots, answering how many it removed. func (k *Keeper) expireSnapshots() int { recs, err := k.snapshotRecords() if err != nil { k.logf("the snapshots cannot be read: %v", err) return 0 } removed := 0 var used int64 for key, r := range recs { if !r.Expired(k.now(), k.keepDays()) { used += r.Size continue } if err := k.files().Remove(r.Path); err != nil { k.logf("the snapshot %s of %s, past its %d days, cannot be removed: %v", r.Path, key, k.keepDays(), err) used += r.Size continue } if err := k.Store.Delete(StateRequests, key); err != nil { k.logf("the record of the removed snapshot %s cannot be removed: %v", key, err) used += r.Size continue } k.logf("removed the snapshot %s of the request %s, taken %s, past its %d days", r.Path, key, r.At.Format(time.RFC3339), k.keepDays()) removed++ } k.snapshotsFill(used) return removed } // removeSnapshots is the seat mesh-broker's snapshot-remove: the snapshot of one request, or every one past its days // (all-expired). The hand act is written first, and a snapshot that cannot be recorded stays; a snapshot of before G goes only // by its request (design 47 G §681). The answer is `accepted` with the request's own id as the act. func (k *Keeper) removeSnapshots(r *Request) (string, error) { if (r.Of == "") == !r.AllExpired { return "error", fmt.Errorf("snapshot-remove names a request, or all-expired, not both and not neither") } what := "removed every snapshot past its days" if r.Of != "" { what = "removed the snapshot of the request " + r.Of } if k.Act != nil { if err := k.Act(what+", asked by "+r.Caller, r.Why, "mesh-broker_snapshot_remove"); err != nil { return "error", fmt.Errorf("the hand act cannot be recorded, so nothing is removed: %w", err) } } if r.AllExpired { r.Result = fmt.Sprintf("%d removed", k.expireSnapshots()) return "accepted", nil } raw, _, found, err := k.Store.Get(StateRequests, r.Of) var rec Request if err != nil || !found || json.Unmarshal(raw, &rec) != nil || (rec.Path == "" && rec.Snapshot == "") { return "error", fmt.Errorf("no snapshot of the request %s is held", r.Of) } path := rec.Path if path == "" { path = rec.Snapshot } if err := k.files().Remove(path); err != nil { return "error", fmt.Errorf("the snapshot %s cannot be removed: %w", path, err) } if err := k.Store.Delete(StateRequests, r.Of); err != nil { return "error", fmt.Errorf("the record of the removed snapshot %s cannot be removed: %w", r.Of, err) } k.logf("removed the snapshot %s of the request %s by hand, asked by %s", path, r.Of, r.Caller) if used, err := k.SnapshotsUsed(""); err == nil { k.snapshotsFill(used) } return "accepted", nil } // SnapshotsAtStart is the keeper's first look at the snapshots at its start: what is past its days removed, and the fill // judged afresh, so a finding raised before a restart that no longer holds is cleared. func (k *Keeper) SnapshotsAtStart() { k.mu.Lock() k.fillUnjudged = true k.mu.Unlock() k.ExpireSnapshots() } // snapshotsFill raises the finding `snapshots-filling` past 80 % of the cap and clears it once back under, judged at // every snapshot taken and removed and at the keeper's start. func (k *Keeper) snapshotsFill(used int64) { capBytes := k.snapshotsCap() pct := used * 100 / capBytes k.mu.Lock() was, unjudged := k.filling, k.fillUnjudged k.filling, k.fillUnjudged = pct >= SnapshotsFillingMark, false now := k.filling k.mu.Unlock() if k.Finding == nil { return } ctx := context.Background() switch { case now: f := meshconditions.Finding{Scope: "module", ID: SnapshotsID, Kind: FindingSnapshotsFilling, Subject: meshconditions.Subject{Scope: "module", ID: Module}, Severity: meshconditions.FindingSeverityWarning, Headline: "The bus module's snapshots are filling up", Summary: fmt.Sprintf("the snapshots hold %d of %d bytes (%d%%)", used, capBytes, pct), Needs: "snapshots removed, or a larger snapshots-cap", Explanation: fmt.Sprintf("the snapshots hold %d of %d bytes (%d%%); at the cap a drop or recreate is refused; cleared under %d%%", used, capBytes, pct, SnapshotsFillingMark), Source: "snapshots", Observer: Module, At: k.now()} if err := k.Finding.Write(ctx, f); err != nil { k.logf("the finding %s cannot be written: %v", FindingSnapshotsFilling, err) } case was || unjudged: if err := k.Finding.Clear(ctx, "module", SnapshotsID, FindingSnapshotsFilling, "the snapshots are under 80 % of their cap"); err != nil { k.logf("the finding %s cannot be cleared: %v", FindingSnapshotsFilling, err) } } }