mesh-catalog/modules/mesh-mail/cmd/mesh-mail/mirror.go

package main

// The account bucket (novox/hq ADR 0439 §2, issue 1255): a view (such as the mail web client) can watch it
// instead of asking mail-store's verbs for an account's state and folder counts. The bundle's own Mirror keeps
// it in step with the index (design 59's prepare step, settlement 6): it reads the index fresh each round and
// writes only what changed — a key put when its value differs, never otherwise (novox/hq ADR 0201 §12). keep
// (imap.go) is not changed: it writes the index; the Mirror alone writes this bucket.

import (
	"bytes"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"strings"
	"time"
	"unicode/utf8"

	stdio "git.novox.be/novox/mesh-sdk/go"
)

// AccountsState is the module's state the Mirror keeps one key per account in (novox/hq ADR 0439 §2).
const AccountsState = "accounts"

// MailsState is the module's state the Mirror keeps one key per mail in (novox/hq ADR 0439 §3, issue 1282).
const MailsState = "mails"

// The bounds ADR 0439 §2 gives the account bucket's value.
const (
	// MostAccountValue is the value's own bound, 64 KiB.
	MostAccountValue = 64 << 10
	// MostFolderName is one folder's name, cut at a character's boundary when it is longer.
	MostFolderName = 120
	// DefaultKeptMails is how many of an account's newest mails the mail bucket keeps, 200 unless ADR 0439 §4's
	// 81-account rule (issue 1282) gives every account the same smaller number instead.
	DefaultKeptMails = 200
	// MostMailValue is ADR 0439 §3's own bound on one mail's value in the mail bucket, 4 KiB.
	MostMailValue = 4 << 10
	// MostMailBytes is ADR 0439 §4's own bound across every account's newest mails together, 64 MiB.
	MostMailBytes = 64 << 20
	// MostMailSubject is one mail's subject in the mail bucket, cut at a character's boundary when longer.
	MostMailSubject = 300
	// MostMailShown is how many recipients, and how many attachment names, the mail bucket's value names one
	// by one before it counts the rest (ADR 0439 §3's "first 10").
	MostMailShown = 10
)

// AccountValue is state accounts' own value, one per account (ADR 0439 §2): never a time that moves on each
// sync, so two rounds with nothing changed write the byte-identical value.
type AccountValue struct {
	// State is the account's state as its keeper last said it (current, syncing, waits-for-sign-in or failing),
	// or "unknown" while its keeper has said nothing of it yet (as mail_accounts itself answers, tools.go).
	State string `json:"state"`
	// Kept is the number of the account's newest mails the mail bucket (issue 1282) keeps.
	Kept int `json:"kept"`
	// Folders are the account's folders, each by its name (cut to MostFolderName bytes), its count of mails and
	// its count of unread mails.
	Folders []AccountFolderValue `json:"folders"`
	// DroppedFolders is how many folders were left out, fewest-mail first, to keep the value at MostAccountValue
	// bytes or fewer; left out of the value (omitted, not zero) when nothing was dropped.
	DroppedFolders int `json:"dropped-folders,omitempty"`
}

// AccountFolderValue is one folder in an AccountValue.
type AccountFolderValue struct {
	Name   string `json:"name"`
	Mails  int    `json:"mails"`
	Unread int    `json:"unread"`
}

// MailValue is state mails' own value, one per kept mail (novox/hq ADR 0439 §3): never a body. The subject is
// cut to MostMailSubject bytes, and every address, name, folder and attachment name to MostFolderName bytes, at
// a character's boundary; recipients and attachment names are each named one by one up to MostMailShown, the
// rest counted, and when the value is still over MostMailValue bytes, recipients and then attachment names are
// dropped from the end (RecipientsDropped, AttachmentsDropped growing) until it fits.
type MailValue struct {
	Account            string       `json:"account"`
	Folder             string       `json:"folder"`
	Date               string       `json:"date"` // RFC3339; the mail's own Date header, never a time that moves
	Sender             MailPerson   `json:"sender"`
	Subject            string       `json:"subject"`
	Recipients         []MailPerson `json:"recipients,omitempty"`
	RecipientsDropped  int          `json:"recipients-dropped,omitempty"`
	Attachments        []string     `json:"attachments,omitempty"`
	AttachmentsDropped int          `json:"attachments-dropped,omitempty"`
	Size               int64        `json:"size"`
	Flags              []string     `json:"flags,omitempty"`
}

// MailPerson is a sender or a recipient in a MailValue, each field already cut to MostFolderName bytes.
type MailPerson struct {
	Name    string `json:"name,omitempty"`
	Address string `json:"address"`
}

// MirrorState is what the Mirror needs of one state bucket it writes: a key's current raw value (nil when it
// has none), a value put — the Mirror is this bucket's only writer, so never by compare-and-set — and every key
// it holds. An interface, so a unit test gives a fake instead of a runtime (novox/hq ADR 0439 §2, §3; issue
// 1255, 1282's "Prepare findings").
type MirrorState interface {
	Get(key string) (json.RawMessage, bool, error)
	Put(key string, value any) error
	Delete(key string) error
	Keys() ([]string, error)
}

// keptMirrorState is a MirrorState on the bus, over the sdk's own KeptState (state.go, seat.go).
type keptMirrorState struct{ s stdio.KeptState }

func (k keptMirrorState) Get(key string) (json.RawMessage, bool, error) {
	e, err := k.s.Get(key)
	if err != nil || e == nil {
		return nil, false, err
	}
	return e.Value, true, nil
}
func (k keptMirrorState) Put(key string, value any) error {
	_, err := k.s.Put(key, value)
	return err
}
func (k keptMirrorState) Delete(key string) error { return k.s.Delete(key) }
func (k keptMirrorState) Keys() ([]string, error) { return k.s.Keys() }

// Mirror keeps the module's state buckets in step with the index: one round reads the index fresh and writes
// only what changed (ADR 0201 §12). It runs beside the Publisher (serve(), main.go): after each Publisher
// round that said something, and at least every MirrorEvery either way (settlement 6).
type Mirror struct {
	Index    func(ctx context.Context) (Store, error)
	Accounts func() ([]Account, error)
	Accts    MirrorState // the `accounts` state
	Mails    MirrorState // the `mails` state (issue 1282); nil skips mirroring it (tests of accounts alone)
	Logf     func(format string, a ...any)
}

// Round mirrors every configured account's current state into the `accounts` bucket, and (when Mails is given)
// each one's newest mails into the `mails` bucket — both at the same, shared kept count (ADR 0439 §4's
// 81-account rule, issue 1282).
func (m *Mirror) Round(ctx context.Context) error {
	accounts, err := m.Accounts()
	if err != nil {
		return err
	}
	st, err := m.Index(ctx)
	if err != nil {
		return err
	}
	states, err := st.States(ctx)
	if err != nil {
		return err
	}
	kept := keptPerAccount(len(accounts))
	for _, a := range accounts {
		v, err := accountValue(ctx, st, a.Name, states[a.Name], kept)
		if err != nil {
			return err
		}
		if err := putIfChanged(m.Accts, a.Name, v); err != nil {
			return err
		}
	}
	if m.Mails == nil {
		return nil
	}
	for _, a := range accounts {
		if err := m.roundMails(ctx, st, a.Name, kept); err != nil {
			return err
		}
	}
	return nil
}

// roundMails mirrors one account's newest kept mails into the `mails` bucket (novox/hq ADR 0439 §3, issue
// 1282): a key put for one new or changed, a key deleted for one no longer among the newest kept — never for
// another account's own keys, which mailKey's account prefix tells apart.
func (m *Mirror) roundMails(ctx context.Context, st Store, account string, kept int) error {
	msgs, err := st.Newest(ctx, account, kept)
	if err != nil {
		return err
	}
	desired := map[string]MailValue{}
	for _, msg := range msgs {
		desired[mailKey(account, msg)] = mailValue(account, msg)
	}
	for key, v := range desired {
		if err := putIfChanged(m.Mails, key, v); err != nil {
			return err
		}
	}
	keys, err := m.Mails.Keys()
	if err != nil {
		return err
	}
	prefix := account + "."
	for _, k := range keys {
		if !strings.HasPrefix(k, prefix) {
			continue
		}
		if _, still := desired[k]; !still {
			if err := m.Mails.Delete(k); err != nil {
				return err
			}
		}
	}
	return nil
}

// accountValue is one account's own value: its state word, the number of mails kept for it, and its folders
// with their counts, trimmed to MostAccountValue.
func accountValue(ctx context.Context, st Store, account string, state AccountState, kept int) (AccountValue, error) {
	folders, err := st.Folders(ctx, account)
	if err != nil {
		return AccountValue{}, err
	}
	fv := make([]AccountFolderValue, 0, len(folders))
	for _, f := range folders {
		unread, err := unreadCount(ctx, st, account, f.Name)
		if err != nil {
			return AccountValue{}, err
		}
		fv = append(fv, AccountFolderValue{Name: cutUTF8(f.Name, MostFolderName), Mails: f.Messages, Unread: unread})
	}
	word := state.State
	if word == "" {
		word = "unknown"
	}
	return trimAccountValue(AccountValue{State: word, Kept: kept, Folders: fv}), nil
}

// unreadCount is a folder's count of messages without the \Seen flag.
func unreadCount(ctx context.Context, st Store, account, folder string) (int, error) {
	flags, err := st.Flags(ctx, account, folder)
	if err != nil {
		return 0, err
	}
	n := 0
	for _, fl := range flags {
		if !hasSeenFlag(fl) {
			n++
		}
	}
	return n, nil
}

// trimAccountValue is issue 1255/AC5: while the value marshals to more than MostAccountValue bytes, the folder
// with the fewest mails is dropped, and DroppedFolders counts it, until it fits (or no folder is left).
func trimAccountValue(v AccountValue) AccountValue {
	for accountValueSize(v) > MostAccountValue && len(v.Folders) > 0 {
		fewest := 0
		for i := 1; i < len(v.Folders); i++ {
			if v.Folders[i].Mails < v.Folders[fewest].Mails {
				fewest = i
			}
		}
		v.Folders = append(append([]AccountFolderValue{}, v.Folders[:fewest]...), v.Folders[fewest+1:]...)
		v.DroppedFolders++
	}
	return v
}

func accountValueSize(v AccountValue) int {
	raw, err := json.Marshal(v)
	if err != nil {
		return 0
	}
	return len(raw)
}

// keptPerAccount is ADR 0439 §4's 81-account rule (issue 1282): DefaultKeptMails, unless that many of every
// account's mails could together pass MostMailBytes, in which case every account keeps the same smaller number
// instead, so that all of them fit.
func keptPerAccount(numAccounts int) int {
	if numAccounts <= 0 {
		return DefaultKeptMails
	}
	if numAccounts*DefaultKeptMails*MostMailValue <= MostMailBytes {
		return DefaultKeptMails
	}
	n := MostMailBytes / (numAccounts * MostMailValue)
	if n < 1 {
		n = 1
	}
	if n > DefaultKeptMails {
		n = DefaultKeptMails
	}
	return n
}

// putIfChanged writes value at key only when it differs from what the bucket currently holds there (ADR 0201
// §12): the Mirror's own values never carry a time that moves on each sync, so the same JSON bytes (Go's own
// json.Marshal is stable for a fixed type) mean nothing changed.
func putIfChanged(bucket MirrorState, key string, value any) error {
	raw, err := json.Marshal(value)
	if err != nil {
		return err
	}
	cur, ok, err := bucket.Get(key)
	if err != nil {
		return err
	}
	if ok && bytes.Equal(cur, raw) {
		return nil
	}
	return bucket.Put(key, value)
}

// mailDigest is the digest named by a mail's own message-id, or by its folder and uid when it has none (novox/hq
// ADR 0439 §3, issue 1282, AC5): two mails without a message-id in different folders or uids digest apart; one
// message-id kept in two folders of the same account digests the same, so it is mirrored once.
func mailDigest(m Message) string {
	basis := m.MessageID
	if basis == "" {
		basis = m.Folder + "\x00" + itoa(int64(m.UID))
	}
	sum := sha256.Sum256([]byte(basis))
	return hex.EncodeToString(sum[:])
}

// mailKey is the mail bucket's own key: the account, so one account's keys are told apart from another's
// (roundMails's own deletion), then a dot (account names hold none, config.go's accountName), then the digest.
func mailKey(account string, m Message) string {
	return account + "." + mailDigest(m)
}

// mailValue is one mail's own value (ADR 0439 §3): account, folder, date, sender, subject, recipients and
// attachment names (each capped to MostMailShown, the rest counted), size and flags, never a body, trimmed to
// MostMailValue.
func mailValue(account string, m Message) MailValue {
	sender := MailPerson{}
	if len(m.From) > 0 {
		sender = mailPerson(m.From[0])
	}
	recipients, recipientsDropped := mailPeople(m.To)
	attachments, attachmentsDropped := mailAttachmentNames(m.Attachments)
	v := MailValue{
		Account:            account,
		Folder:             cutUTF8(m.Folder, MostFolderName),
		Date:               m.Date.UTC().Format(time.RFC3339),
		Sender:             sender,
		Subject:            cutUTF8(m.Subject, MostMailSubject),
		Recipients:         recipients,
		RecipientsDropped:  recipientsDropped,
		Attachments:        attachments,
		AttachmentsDropped: attachmentsDropped,
		Size:               m.Size,
		Flags:              append([]string(nil), m.Flags...),
	}
	return trimMailValue(v)
}

// mailPerson is one Person as a MailValue names it: name and address each cut to MostFolderName bytes.
func mailPerson(p Person) MailPerson {
	return MailPerson{Name: cutUTF8(p.Name, MostFolderName), Address: cutUTF8(p.Address, MostFolderName)}
}

// mailPeople is a list of recipients, capped to MostMailShown with the rest counted (ADR 0439 §3's "first 10
// recipients").
func mailPeople(ps []Person) ([]MailPerson, int) {
	dropped := 0
	if len(ps) > MostMailShown {
		dropped = len(ps) - MostMailShown
		ps = ps[:MostMailShown]
	}
	out := make([]MailPerson, 0, len(ps))
	for _, p := range ps {
		out = append(out, mailPerson(p))
	}
	return out, dropped
}

// mailAttachmentNames is a list of attachment names, capped to MostMailShown with the rest counted (ADR 0439
// §3's "first 10 attachment names"), each cut to MostFolderName bytes.
func mailAttachmentNames(refs []AttachmentRef) ([]string, int) {
	dropped := 0
	if len(refs) > MostMailShown {
		dropped = len(refs) - MostMailShown
		refs = refs[:MostMailShown]
	}
	out := make([]string, 0, len(refs))
	for _, r := range refs {
		out = append(out, cutUTF8(r.Name, MostFolderName))
	}
	return out, dropped
}

// trimMailValue is issue 1282/AC4: while the value marshals to more than MostMailValue bytes, a recipient is
// dropped from the end (RecipientsDropped counting it) until none is left, then an attachment name the same
// way (AttachmentsDropped counting it), until it fits or both are empty.
func trimMailValue(v MailValue) MailValue {
	for mailValueSize(v) > MostMailValue {
		switch {
		case len(v.Recipients) > 0:
			v.Recipients = v.Recipients[:len(v.Recipients)-1]
			v.RecipientsDropped++
		case len(v.Attachments) > 0:
			v.Attachments = v.Attachments[:len(v.Attachments)-1]
			v.AttachmentsDropped++
		default:
			return v
		}
	}
	return v
}

func mailValueSize(v MailValue) int {
	raw, err := json.Marshal(v)
	if err != nil {
		return 0
	}
	return len(raw)
}

// cutUTF8 cuts s to at most n bytes, never splitting a multi-byte character (novox/hq ADR 0439 §2, §3).
func cutUTF8(s string, n int) string {
	if len(s) <= n {
		return s
	}
	b := s[:n]
	for len(b) > 0 {
		r, size := utf8.DecodeLastRuneInString(b)
		if r == utf8.RuneError && size <= 1 {
			b = b[:len(b)-1]
			continue
		}
		break
	}
	return b
}