diff --git a/modules/mesh-recovery/cmd/mesh-recovery-tools/main.go b/modules/mesh-recovery/cmd/mesh-recovery-tools/main.go index 6bd188dc5..efb4a2ddd 100644 --- a/modules/mesh-recovery/cmd/mesh-recovery-tools/main.go +++ b/modules/mesh-recovery/cmd/mesh-recovery-tools/main.go @@ -10,6 +10,7 @@ import ( "strings" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/health" ) func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } @@ -77,8 +78,7 @@ func Tools(ask Ask, callerOf func(map[string]any) string) []stdio.Tool { if err != nil { return nil, err } - var out any - return out, json.Unmarshal(raw, &out) + return keptPointsHealth(raw) }}, hand(ask, callerOf, "recovery_take", "take", "Take a restore point of one module's kept data now, at the form of the given fingerprint.", map[string]any{"module": module, "kept": kept, "fingerprint": str("the fingerprint whose form the point is taken at")}, "module", "kept", "fingerprint"), @@ -98,3 +98,32 @@ func Tools(ask Ask, callerOf func(map[string]any) string) []stdio.Tool { hand(ask, callerOf, "recovery_expire_points", "expire-points", "Remove the restore points past their keep days now, never the newest of a kept data.", map[string]any{}), } } + +// keptPointsHealth is what kept-points answers with the health answer beside it, which the module's declared health +// check reads (§554, §657, A-129): unhealthy while an aside only it holds the data of is listed as unrestored. +func keptPointsHealth(raw json.RawMessage) (map[string]any, error) { + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + return nil, err + } + var kp struct { + Unrestored []struct { + ID string `json:"id"` + Names []string `json:"names"` + } `json:"unrestored"` + } + if err := json.Unmarshal(raw, &kp); err != nil { + return nil, err + } + h := health.Healthy("No aside holds data the live bus lacks.") + if len(kp.Unrestored) > 0 { + var said []string + for _, a := range kp.Unrestored { + said = append(said, a.ID+" ("+strings.Join(a.Names, ", ")+")") + } + h = health.Unhealthy(fmt.Sprintf("%d aside(s) hold data whose live objects were removed and not restored: %s; recovery_restore_aside puts one back.", + len(kp.Unrestored), strings.Join(said, "; "))) + } + out["state"], out["healthy"], out["why"] = h.State, h.Healthy, h.Why + return out, nil +} diff --git a/modules/mesh-recovery/cmd/mesh-recovery-tools/main_test.go b/modules/mesh-recovery/cmd/mesh-recovery-tools/main_test.go index fe28938ea..ded8d73f7 100644 --- a/modules/mesh-recovery/cmd/mesh-recovery-tools/main_test.go +++ b/modules/mesh-recovery/cmd/mesh-recovery-tools/main_test.go @@ -6,6 +6,8 @@ import ( "slices" "strings" "testing" + + "git.novox.be/novox/mesh-sdk/go/health" ) // The bundle serves exactly the tools the manifest names, each named recovery_, and each acting tool calls its verb of the @@ -51,3 +53,26 @@ func TestEachToolCallsItsVerbWithTheProvedAsker(t *testing.T) { t.Errorf("tools %v, the manifest names %v", names, want) } } + +// recovery_kept_points answers what kept-points answers with the health answer beside it: unhealthy while an aside is +// unrestored, healthy otherwise, as the declared health check reads it (§657, A-129). +func TestKeptPointsIsTheHealthCheck(t *testing.T) { + for _, c := range []struct { + answer string + state string + }{ + {`{"points":[],"asides":[],"unrestored":[]}`, health.StateHealthy}, + {`{"points":[],"asides":[{"id":"x","names":["app_notes"],"size":1}],"unrestored":[{"id":"x","names":["app_notes"],"size":1}]}`, health.StateUnhealthy}, + } { + tools := Tools(func(string, any) (json.RawMessage, error) { return json.RawMessage(c.answer), nil }, func(map[string]any) string { return "" }) + out, err := tools[0].Run(map[string]any{}) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(out) + a, err := health.Read(raw) + if err != nil || a.State != c.state || !strings.Contains(string(raw), `"unrestored"`) { + t.Errorf("%s answered %s: %+v, %v", c.answer, raw, a, err) + } + } +} diff --git a/modules/mesh-recovery/internal/recovery/control.go b/modules/mesh-recovery/internal/recovery/control.go index 868fd9f9e..66e725d4e 100644 --- a/modules/mesh-recovery/internal/recovery/control.go +++ b/modules/mesh-recovery/internal/recovery/control.go @@ -89,30 +89,44 @@ func (e *Engine) KeptPoints(module, kept string) (seat.KeptPointsAnswer, error) out.Running = &seat.Running{Act: e.running.ID, Step: e.running.Step, Since: e.running.Since, WaitsOn: e.running.WaitsOn} } e.mu.Unlock() + // The asides only an aside holds, which the health check fails on, read from the engine (A-129). + out.Unrestored = []seat.Aside{} + for _, x := range e.Unrestored() { + if (module == "" || x.Module == module) && (kept == "" || x.Kept == kept) { + out.Unrestored = append(out.Unrestored, x.Aside) + } + } return out, nil } // Unrestored are the asides whose record ends at `removed` that no running act holds (§554 health, §561): the live data // of those names is gone and only the aside holds it. -func (e *Engine) Unrestored() []seat.Aside { +func (e *Engine) Unrestored() []UnrestoredAside { _, recs, _ := e.Asides() e.mu.Lock() defer e.mu.Unlock() - var out []seat.Aside + var out []UnrestoredAside for _, r := range recs { held := e.running != nil && e.running.Module == r.Module && e.running.Kept == r.Kept if r.Step == stepRemoved && !held { - out = append(out, r.aside()) + out = append(out, UnrestoredAside{Module: r.Module, Kept: r.Kept, Aside: r.aside()}) } } return out } +// UnrestoredAside is an aside only it holds the data of, with the module and kept data it was taken of: the scope of the +// finding aside-unrestored (§561). +type UnrestoredAside struct { + Module, Kept string + Aside seat.Aside +} + // Start repairs what a restart cut off (§516, §564, §560): points under a temporary name are removed, an aside whose // record ends at `set aside` (the live data never touched) is removed, an aside that ends at `removed` is put back under // the live names, and every accepted act that never ended ends failed with `stopped`, so its caller asks again. It // answers the asides it could not put back (the finding aside-unrestored, §561). -func (e *Engine) Start(ctx context.Context) []seat.Aside { +func (e *Engine) Start(ctx context.Context) []UnrestoredAside { _ = filepath.WalkDir(filepath.Join(e.Dir, "points"), func(path string, d os.DirEntry, err error) error { if err == nil && d.IsDir() && strings.HasPrefix(d.Name(), ".tmp-") { _ = os.RemoveAll(path) diff --git a/modules/mesh-recovery/internal/recovery/restore_test.go b/modules/mesh-recovery/internal/recovery/restore_test.go index 2242705b1..853a68645 100644 --- a/modules/mesh-recovery/internal/recovery/restore_test.go +++ b/modules/mesh-recovery/internal/recovery/restore_test.go @@ -249,6 +249,10 @@ func TestAsideMovesTheLiveDataIntoAnAsideAndRestoreAsidePutsItBack(t *testing.T) if got := r.e.Unrestored(); len(got) != 1 { t.Errorf("unrestored %+v", got) } + // kept-points says it, which the tool and the health check read (A-129). + if kp, _ := r.e.KeptPoints("app", ""); len(kp.Unrestored) != 1 || kp.Unrestored[0].ID != sa.ID { + t.Errorf("kept-points unrestored %+v", kp.Unrestored) + } r.s.events = nil r.do(recovery.Act{Kind: recovery.KindRestoreAside, Module: "app", Kept: "notes", AsideID: sa.ID, Why: "done looking", Caller: "tools"}) if lastRestored(t, r).Outcome != "done" || r.b.streams["app_notes"] != "hello" || len(r.e.Unrestored()) != 0 {