mesh-catalog/modules/mesh-mail/cmd/mesh-mail/health.go

package main

// How mesh-mail is (novox/hq ADR 0359 §6). Its keeper's liveness is its unit's; its watching is judged here, by the
// watch process, once a minute, and left in a status file the bundle's tool `mail_health` reads:
//
//   - waits, naming the own secret mail-credential-<account>, for an account whose password nobody gave yet (ADR 0283),
//     so the condition is needs-operator and the gate passes;
//   - unhealthy, naming the account and why, when its connection is down or its server has not answered for
//     SilentAfter (five minutes) — an IDLE that went silent;
//   - healthy otherwise.
//
// A fault beside a wait is unhealthy and says both.

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"strings"
	"time"

	"git.novox.be/novox/mesh-sdk/go/health"
)

// HealthTool is the module's tool the node-engine asks how it is.
const HealthTool = "mail_health"

// WatchEvery is how often the watch judges; StatusStale how old its verdict may be before the tool says it stopped.
const (
	WatchEvery  = time.Minute
	StatusStale = 3 * WatchEvery
	// SignInGrace is how long a password just given may wait for its keeper to sign in before that is a fault.
	SignInGrace = 3 * time.Minute
)

// Judge is the module's health from what the watch reads: the accounts setting (or why it cannot be read), whether the
// index serves, which passwords were given, and each account's state as its keeper said it.
func Judge(accounts []Account, accountsErr, readyErr error, given func(Account) bool, states map[string]AccountState, now time.Time) health.Answer {
	if accountsErr != nil {
		return health.Unhealthy(accountsErr.Error())
	}
	var problems, fine []string
	var waits []health.Wait
	if readyErr != nil {
		problems = append(problems, "the index cannot serve: "+readyErr.Error())
	}
	for _, a := range accounts {
		if !given(a) {
			waits = append(waits, health.Wait{
				Part:   "the mail account " + a.Name,
				Secret: a.Secret(),
				What:   "the password of the mail account " + a.Name,
			})
			continue
		}
		if readyErr != nil {
			continue
		}
		st, said := states[a.Name]
		switch {
		case !said:
			problems = append(problems, fmt.Sprintf("the account %s: its keeper has said nothing of it", a.Name))
		case st.State == AccountWaits:
			if now.Sub(st.Said) > SignInGrace {
				problems = append(problems, fmt.Sprintf("the account %s: its password was given, and its keeper has not signed in for %s",
					a.Name, now.Sub(st.Said).Round(time.Second)))
			} else {
				fine = append(fine, a.Name+" is signing in")
			}
		case st.State == AccountFailing:
			problems = append(problems, fmt.Sprintf("the account %s is not watched: %s", a.Name, st.Why))
		case st.Heard.IsZero():
			problems = append(problems, fmt.Sprintf("the account %s: its server has not answered since its keeper started", a.Name))
		case now.Sub(st.Heard) > SilentAfter:
			problems = append(problems, fmt.Sprintf("the account %s: its server has not answered for %s (its IDLE on the inbox is silent)",
				a.Name, now.Sub(st.Heard).Round(time.Second)))
		default:
			fine = append(fine, fmt.Sprintf("%s is %s (%s)", a.Name, st.State, st.Why))
		}
	}
	waitSaid := ""
	if len(waits) > 0 {
		var names []string
		for _, w := range waits {
			names = append(names, w.Secret)
		}
		waitSaid = "waits for its password: " + strings.Join(names, ", ")
	}
	switch {
	case len(problems) > 0:
		why := strings.Join(problems, "; ")
		if waitSaid != "" {
			why += "; and it " + waitSaid
		}
		return health.Unhealthy(why, waits...)
	case len(waits) > 0:
		why := "nothing is wrong, and it " + waitSaid
		if len(fine) > 0 {
			why += "; " + strings.Join(fine, "; ")
		}
		return health.Waiting(why, waits...)
	case len(accounts) == 0:
		return health.Healthy("no mail account is declared, so nothing is watched")
	}
	return health.Healthy(fmt.Sprintf("%d account(s) watched: %s", len(accounts), strings.Join(fine, "; ")))
}

// WithOverdue folds any overdue send (Overdue, send.go) into base: unhealthy, naming each one, whatever
// base itself said — an overdue send is never excused by an otherwise-healthy account (novox/hq ADR 0436
// clause 11, issue 1199).
func WithOverdue(base health.Answer, overdue []SendRequest) health.Answer {
	if len(overdue) == 0 {
		return base
	}
	names := make([]string, 0, len(overdue))
	for _, req := range overdue {
		names = append(names, fmt.Sprintf("%s (from %s)", req.ID, req.Account))
	}
	why := fmt.Sprintf("the send(s) %s have waited 10 minutes or more for the sender", strings.Join(names, ", "))
	if base.Why != "" {
		why = base.Why + "; and " + why
	}
	return health.Unhealthy(why, base.Waits...)
}

// Status is the watch's verdict as it leaves it: the answer, and when it looked.
type Status struct {
	health.Answer
	At time.Time `json:"at"`
}

func writeStatus(path string, s Status) error {
	raw, err := json.Marshal(s)
	if err != nil {
		return err
	}
	tmp := path + ".next"
	if err := os.WriteFile(tmp, raw, 0o600); err != nil {
		return err
	}
	return os.Rename(tmp, path)
}

// ReadStatus is the watch's last verdict, as the tool answers it: unhealthy when there is none, when it cannot be read,
// or when it is older than StatusStale.
func ReadStatus(path string, now time.Time) health.Answer {
	raw, err := os.ReadFile(path)
	if err != nil {
		return health.Unhealthy("the watch has left no verdict yet: " + err.Error())
	}
	var s Status
	if err := json.Unmarshal(raw, &s); err != nil {
		return health.Unhealthy("the watch's verdict cannot be read")
	}
	if age := now.Sub(s.At); age > StatusStale {
		return health.Unhealthy(fmt.Sprintf("the watch has not looked for %s (its last verdict: %s)", age.Round(time.Second), s.Why))
	}
	// Read through the SDK's reader, so a wait it cannot check is never excused.
	a, err := health.Read(mustJSON(s.Answer))
	if err != nil {
		return health.Unhealthy("the watch's verdict cannot be read: " + err.Error())
	}
	return a
}

func mustJSON(v any) []byte {
	raw, _ := json.Marshal(v)
	return raw
}

// watchOnce is one look: what the watch reads, judged.
func watchOnce(ctx context.Context, env Env, open func(context.Context) (Store, func(), error), now time.Time) health.Answer {
	accounts, accountsErr := ReadAccounts(env.Accounts)
	var readyErr error
	states := map[string]AccountState{}
	st, done, err := open(ctx)
	if err != nil {
		readyErr = err
	} else {
		defer done()
		if readyErr = st.Ready(ctx); readyErr == nil {
			if states, err = st.States(ctx); err != nil {
				readyErr = err
			}
		}
	}
	return Judge(accounts, accountsErr, readyErr, func(a Account) bool { return SecretGiven(env.Secrets, a) }, states, now)
}