// The data the modules on this machine declare (novox/hq ADR 0233), and what the readings say of them. // // The mesh composes a file beside the backup lines: every data item every module on the machine declares, one line // each, which mesh-sdk reads (catalogue.ParseDataItems). disk-load measures the items and keeps each reading and its // seven-day peak as its own state (ADR 0571 clause 3); this holder reads them there and says each, with each module's // last good backup, in `backed-up`. // // And it deletes, when a person has decided: an item the mesh retired — its module gone from this // machine — is removed by `backup_delete_retired`, and only after a last restore point of it has been // taken, so the deletion can be undone until a person forgets that restore point. package main import ( "context" "encoding/json" "errors" "fmt" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/health" "os" "path/filepath" "regexp" "strings" "sync" "time" ) // Item is one data item a module declares, as placed on this machine (mesh-sdk's reading of the holder's data text). type Item = catalogue.PlacedDataItem // ItemReport is one item as `backed-up` says it: the item, its newest reading from disk-load (ADR 0571 clause 3) and // its newest good backup. type ItemReport struct { Item health.Measured // Peak is the largest size in the last seven days, on PeakDay. Peak *int64 `json:"peak_bytes,omitempty"` PeakDay string `json:"peak_day,omitempty"` LastBackup *time.Time `json:"last_backup,omitempty"` } var safePath = regexp.MustCompile(`^/[^\s'"\\$` + "`" + `]*$`) // Items is what the modules on this machine declare; none when the mesh composed no data file — an // older controller, which composes none. func (b *Backups) Items() (map[string][]Item, error) { if b.Where.Data == "" { return map[string][]Item{}, nil } raw, err := os.ReadFile(b.Where.Data) if errors.Is(err, os.ErrNotExist) { return map[string][]Item{}, nil } if err != nil { return nil, err } return catalogue.ParseDataItems(string(raw)) } // Measurements is the newest reading disk-load keeps of each item on this machine, by module and item (its state // `readings`, ADR 0571 clause 3); none where it cannot be read, which `backed-up` then says as no reading. func (b *Backups) Measurements() map[string]map[string]health.Reading { out := map[string]map[string]health.Reading{} if b.Readings == nil { return out } rs, err := b.Readings() if err != nil { b.Say("disk-load's readings could not be read: %v", err) return out } for _, r := range rs { if out[r.Module] == nil { out[r.Module] = map[string]health.Reading{} } out[r.Module][r.ID] = r } return out } func (b *Backups) goodFile() string { return filepath.Join(b.Where.State, "good.json") } // Good is each module's newest good night: what a night that failed since does not erase. func (b *Backups) Good() map[string]time.Time { out := map[string]time.Time{} if raw, err := os.ReadFile(b.goodFile()); err == nil { _ = json.Unmarshal(raw, &out) } // A night recorded good before this file existed counts. for module, n := range b.Nights() { if n.OK && n.At.After(out[module]) { out[module] = n.At } } return out } func (b *Backups) recordGood(module string, at time.Time) { good := b.Good() good[module] = at raw, _ := json.MarshalIndent(good, "", " ") if err := os.WriteFile(b.goodFile(), append(raw, '\n'), 0o600); err != nil { b.Say("recording %s's good night failed: %v", module, err) } b.keepGood() } // itemReports is every item of one module, with its newest reading and its newest good backup: the // module's newest good night, where that night keeps the path that covers the item. func itemReports(its []Item, measured map[string]health.Reading, paths []string, good time.Time) []ItemReport { out := []ItemReport{} for _, it := range its { r := ItemReport{Item: it} if m, ok := measured[it.ID]; ok { r.Measured, r.Peak, r.PeakDay = m.Measured, m.Peak, m.PeakAt } if it.CoveredBy != "" && !good.IsZero() { for _, p := range paths { if p == it.CoveredBy { g := good r.LastBackup = &g } } } out = append(out, r) } return out } // ---- deleting a retired item ------------------------------------------------------------------- // Deletion is how one deletion went, by path. type Deletion struct { Module string `json:"module"` Item string `json:"item"` Started time.Time `json:"started"` Running bool `json:"running"` Done bool `json:"done"` OK bool `json:"ok"` Snapshot string `json:"snapshot,omitempty"` Error string `json:"error,omitempty"` By string `json:"by,omitempty"` Why string `json:"why,omitempty"` } func (b *Backups) deletionsFile() string { return filepath.Join(b.Where.State, "deleted.json") } var deletionsMu sync.Mutex func (b *Backups) deletions() map[string]Deletion { out := map[string]Deletion{} if raw, err := os.ReadFile(b.deletionsFile()); err == nil { _ = json.Unmarshal(raw, &out) } return out } func (b *Backups) keepDeletion(path string, d Deletion) { deletionsMu.Lock() defer deletionsMu.Unlock() all := b.deletions() all[path] = d raw, _ := json.MarshalIndent(all, "", " ") if err := os.WriteFile(b.deletionsFile(), append(raw, '\n'), 0o600); err != nil { b.Say("recording the deletion of %s failed: %v", path, err) } } // refuseDeleting says why a path may not be deleted: not absolute, too near the root, or declared now // — by any module's item or backup line on this machine, itself, inside one, or holding one. func (b *Backups) refuseDeleting(path string) error { clean := filepath.Clean(path) if !safePath.MatchString(path) || clean != strings.TrimRight(path, "/") { return fmt.Errorf("%q is not a path this holder deletes", path) } if strings.Count(clean, "/") < 2 { return fmt.Errorf("%s is too near the root to be a module's data", clean) } near := func(declared string) bool { d := filepath.Clean(declared) return d == clean || strings.HasPrefix(d, clean+"/") || strings.HasPrefix(clean, d+"/") } items, err := b.Items() if err != nil { return fmt.Errorf("what is declared here cannot be read, so nothing is deleted: %v", err) } for module, its := range items { for _, it := range its { if near(it.Path) { return fmt.Errorf("%s is declared now — %s's %s at %s — so it is not retired; nothing is deleted", clean, module, it.ID, it.Path) } } } declared, err := b.Declared() if err != nil && !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("what is backed up here cannot be read, so nothing is deleted: %v", err) } for _, d := range declared { for _, p := range d.Paths { if near(p) { return fmt.Errorf("%s is backed up now as %s's %s, so it is not retired; nothing is deleted", clean, d.Module, p) } } } if clean == filepath.Clean(b.Where.Repository) || strings.HasPrefix(b.Where.Repository, clean+"/") || clean == filepath.Clean(b.Where.State) { return fmt.Errorf("%s holds this machine's backups; nothing is deleted", clean) } return nil } // DeleteRetired starts deleting one retired item: a last restore point of it, tagged as retired, then // its removal — never the removal without the restore point. Answers at once; DeletedOutcome follows // it. A path whose deletion already finished answers how it went. func (b *Backups) DeleteRetired(ctx context.Context, module, item, path, confirm, by, why string) (Deletion, error) { if module == "" || item == "" || path == "" { return Deletion{}, errors.New("module, item and path are required") } if confirm != item { return Deletion{}, fmt.Errorf("confirm is the item's name again (%s), to say this is meant", item) } if strings.TrimSpace(why) == "" { return Deletion{}, errors.New("why is required: a deletion is a person's decision, and says why") } if d, ok := b.deletions()[path]; ok && (d.Running || (d.Done && d.OK)) { return d, nil } if err := b.refuseDeleting(path); err != nil { return Deletion{}, err } if !b.exists(ctx, path) { d := Deletion{Module: module, Item: item, Started: b.Now().UTC(), Done: true, OK: true, Error: path + " was already gone", By: by, Why: why} b.keepDeletion(path, d) return d, nil } d := Deletion{Module: module, Item: item, Started: b.Now().UTC(), Running: true, By: by, Why: why} b.keepDeletion(path, d) go b.deleteNow(context.WithoutCancel(ctx), path, d) return d, nil } func (b *Backups) deleteNow(ctx context.Context, path string, d Deletion) { release, err := b.hold(nightLock) if err != nil { d.Running, d.Done, d.Error = false, true, err.Error() b.keepDeletion(path, d) return } defer release() finish := func(err error) { d.Running, d.Done = false, true if err != nil { d.Error = err.Error() b.Say("%s's retired %s at %s was NOT deleted: %v", d.Module, d.Item, path, err) } else { d.OK = true b.Say("%s's retired %s at %s DELETED by %s: %s; its last restore point is %s", d.Module, d.Item, path, orSomebody(d.By), d.Why, d.Snapshot) } b.keepDeletion(path, d) } if err := b.ensureRepository(ctx); err != nil { finish(fmt.Errorf("no restore point could be taken first: %w", err)) return } out, err := b.restic(ctx, "backup", "--json", "--tag", tagOf(d.Module), "--tag", "retired="+d.Item, path) if err != nil { finish(fmt.Errorf("the last restore point could not be taken, so nothing was deleted: %w", err)) return } d.Snapshot = snapshotOf(out) if d.Snapshot == "" { finish(errors.New("restic took the last restore point and named none, so nothing was deleted")) return } if _, err := b.Run(ctx, "rm", "-rf", "--one-file-system", "--", path); err != nil { finish(err) return } finish(nil) } // DeletedOutcome is how the deletion of a path went. func (b *Backups) DeletedOutcome(path string) (Deletion, error) { d, ok := b.deletions()[path] if !ok { return Deletion{}, fmt.Errorf("no deletion of %s was asked of this holder", path) } return d, nil } func orSomebody(by string) string { if by == "" { return "somebody" } return by }