package main // The account bucket (novox/hq ADR 0439 §2, issue 1255): a view (such as the mail web client) can watch it // instead of asking mail-store's verbs for an account's state and folder counts. The bundle's own Mirror keeps // it in step with the index (design 59's prepare step, settlement 6): it reads the index fresh each round and // writes only what changed — a key put when its value differs, never otherwise (novox/hq ADR 0201 §12). keep // (imap.go) is not changed: it writes the index; the Mirror alone writes this bucket. import ( "bytes" "context" "crypto/sha256" "encoding/hex" "encoding/json" "strings" "time" "unicode/utf8" stdio "git.novox.be/novox/mesh-sdk/go" ) // AccountsState is the module's state the Mirror keeps one key per account in (novox/hq ADR 0439 §2). const AccountsState = "accounts" // MailsState is the module's state the Mirror keeps one key per mail in (novox/hq ADR 0439 §3, issue 1282). const MailsState = "mails" // The bounds ADR 0439 §2 gives the account bucket's value. const ( // MostAccountValue is the value's own bound, 64 KiB. MostAccountValue = 64 << 10 // MostFolderName is one folder's name, cut at a character's boundary when it is longer. MostFolderName = 120 // DefaultKeptMails is how many of an account's newest mails the mail bucket keeps, 200 unless ADR 0439 §4's // 81-account rule (issue 1282) gives every account the same smaller number instead. DefaultKeptMails = 200 // MostMailValue is ADR 0439 §3's own bound on one mail's value in the mail bucket, 4 KiB. MostMailValue = 4 << 10 // MostMailBytes is ADR 0439 §4's own bound across every account's newest mails together, 64 MiB. MostMailBytes = 64 << 20 // MostMailSubject is one mail's subject in the mail bucket, cut at a character's boundary when longer. MostMailSubject = 300 // MostMailShown is how many recipients, and how many attachment names, the mail bucket's value names one // by one before it counts the rest (ADR 0439 §3's "first 10"). MostMailShown = 10 ) // AccountValue is state accounts' own value, one per account (ADR 0439 §2): never a time that moves on each // sync, so two rounds with nothing changed write the byte-identical value. type AccountValue struct { // State is the account's state as its keeper last said it (current, syncing, waits-for-sign-in or failing), // or "unknown" while its keeper has said nothing of it yet (as mail_accounts itself answers, tools.go). State string `json:"state"` // Kept is the number of the account's newest mails the mail bucket (issue 1282) keeps. Kept int `json:"kept"` // Folders are the account's folders, each by its name (cut to MostFolderName bytes), its count of mails and // its count of unread mails. Folders []AccountFolderValue `json:"folders"` // DroppedFolders is how many folders were left out, fewest-mail first, to keep the value at MostAccountValue // bytes or fewer; left out of the value (omitted, not zero) when nothing was dropped. DroppedFolders int `json:"dropped-folders,omitempty"` } // AccountFolderValue is one folder in an AccountValue. type AccountFolderValue struct { Name string `json:"name"` Mails int `json:"mails"` Unread int `json:"unread"` } // MailValue is state mails' own value, one per kept mail (novox/hq ADR 0439 §3): never a body. The subject is // cut to MostMailSubject bytes, and every address, name, folder and attachment name to MostFolderName bytes, at // a character's boundary; recipients and attachment names are each named one by one up to MostMailShown, the // rest counted, and when the value is still over MostMailValue bytes, recipients and then attachment names are // dropped from the end (RecipientsDropped, AttachmentsDropped growing) until it fits. type MailValue struct { Account string `json:"account"` Folder string `json:"folder"` Date string `json:"date"` // RFC3339; the mail's own Date header, never a time that moves Sender MailPerson `json:"sender"` Subject string `json:"subject"` Recipients []MailPerson `json:"recipients,omitempty"` RecipientsDropped int `json:"recipients-dropped,omitempty"` Attachments []string `json:"attachments,omitempty"` AttachmentsDropped int `json:"attachments-dropped,omitempty"` Size int64 `json:"size"` Flags []string `json:"flags,omitempty"` } // MailPerson is a sender or a recipient in a MailValue, each field already cut to MostFolderName bytes. type MailPerson struct { Name string `json:"name,omitempty"` Address string `json:"address"` } // MirrorState is what the Mirror needs of one state bucket it writes: a key's current raw value (nil when it // has none), a value put — the Mirror is this bucket's only writer, so never by compare-and-set — and every key // it holds. An interface, so a unit test gives a fake instead of a runtime (novox/hq ADR 0439 §2, §3; issue // 1255, 1282's "Prepare findings"). type MirrorState interface { Get(key string) (json.RawMessage, bool, error) Put(key string, value any) error Delete(key string) error Keys() ([]string, error) } // keptMirrorState is a MirrorState on the bus, over the sdk's own KeptState (state.go, seat.go). type keptMirrorState struct{ s stdio.KeptState } func (k keptMirrorState) Get(key string) (json.RawMessage, bool, error) { e, err := k.s.Get(key) if err != nil || e == nil { return nil, false, err } return e.Value, true, nil } func (k keptMirrorState) Put(key string, value any) error { _, err := k.s.Put(key, value) return err } func (k keptMirrorState) Delete(key string) error { return k.s.Delete(key) } func (k keptMirrorState) Keys() ([]string, error) { return k.s.Keys() } // Mirror keeps the module's state buckets in step with the index: one round reads the index fresh and writes // only what changed (ADR 0201 §12). It runs beside the Publisher (serve(), main.go): after each Publisher // round that said something, and at least every MirrorEvery either way (settlement 6). type Mirror struct { Index func(ctx context.Context) (Store, error) Accounts func() ([]Account, error) Accts MirrorState // the `accounts` state Mails MirrorState // the `mails` state (issue 1282); nil skips mirroring it (tests of accounts alone) Logf func(format string, a ...any) } // Round mirrors every configured account's current state into the `accounts` bucket, and (when Mails is given) // each one's newest mails into the `mails` bucket — both at the same, shared kept count (ADR 0439 §4's // 81-account rule, issue 1282). func (m *Mirror) Round(ctx context.Context) error { accounts, err := m.Accounts() if err != nil { return err } st, err := m.Index(ctx) if err != nil { return err } states, err := st.States(ctx) if err != nil { return err } kept := keptPerAccount(len(accounts)) for _, a := range accounts { v, err := accountValue(ctx, st, a.Name, states[a.Name], kept) if err != nil { return err } if err := putIfChanged(m.Accts, a.Name, v); err != nil { return err } } if m.Mails == nil { return nil } for _, a := range accounts { if err := m.roundMails(ctx, st, a.Name, kept); err != nil { return err } } return nil } // roundMails mirrors one account's newest kept mails into the `mails` bucket (novox/hq ADR 0439 §3, issue // 1282): a key put for one new or changed, a key deleted for one no longer among the newest kept — never for // another account's own keys, which mailKey's account prefix tells apart. func (m *Mirror) roundMails(ctx context.Context, st Store, account string, kept int) error { msgs, err := st.Newest(ctx, account, kept) if err != nil { return err } desired := map[string]MailValue{} for _, msg := range msgs { desired[mailKey(account, msg)] = mailValue(account, msg) } for key, v := range desired { if err := putIfChanged(m.Mails, key, v); err != nil { return err } } keys, err := m.Mails.Keys() if err != nil { return err } prefix := account + "." for _, k := range keys { if !strings.HasPrefix(k, prefix) { continue } if _, still := desired[k]; !still { if err := m.Mails.Delete(k); err != nil { return err } } } return nil } // accountValue is one account's own value: its state word, the number of mails kept for it, and its folders // with their counts, trimmed to MostAccountValue. func accountValue(ctx context.Context, st Store, account string, state AccountState, kept int) (AccountValue, error) { folders, err := st.Folders(ctx, account) if err != nil { return AccountValue{}, err } fv := make([]AccountFolderValue, 0, len(folders)) for _, f := range folders { unread, err := unreadCount(ctx, st, account, f.Name) if err != nil { return AccountValue{}, err } fv = append(fv, AccountFolderValue{Name: cutUTF8(f.Name, MostFolderName), Mails: f.Messages, Unread: unread}) } word := state.State if word == "" { word = "unknown" } return trimAccountValue(AccountValue{State: word, Kept: kept, Folders: fv}), nil } // unreadCount is a folder's count of messages without the \Seen flag. func unreadCount(ctx context.Context, st Store, account, folder string) (int, error) { flags, err := st.Flags(ctx, account, folder) if err != nil { return 0, err } n := 0 for _, fl := range flags { if !hasSeenFlag(fl) { n++ } } return n, nil } // trimAccountValue is issue 1255/AC5: while the value marshals to more than MostAccountValue bytes, the folder // with the fewest mails is dropped, and DroppedFolders counts it, until it fits (or no folder is left). func trimAccountValue(v AccountValue) AccountValue { for accountValueSize(v) > MostAccountValue && len(v.Folders) > 0 { fewest := 0 for i := 1; i < len(v.Folders); i++ { if v.Folders[i].Mails < v.Folders[fewest].Mails { fewest = i } } v.Folders = append(append([]AccountFolderValue{}, v.Folders[:fewest]...), v.Folders[fewest+1:]...) v.DroppedFolders++ } return v } func accountValueSize(v AccountValue) int { raw, err := json.Marshal(v) if err != nil { return 0 } return len(raw) } // keptPerAccount is ADR 0439 §4's 81-account rule (issue 1282): DefaultKeptMails, unless that many of every // account's mails could together pass MostMailBytes, in which case every account keeps the same smaller number // instead, so that all of them fit. func keptPerAccount(numAccounts int) int { if numAccounts <= 0 { return DefaultKeptMails } if numAccounts*DefaultKeptMails*MostMailValue <= MostMailBytes { return DefaultKeptMails } n := MostMailBytes / (numAccounts * MostMailValue) if n < 1 { n = 1 } if n > DefaultKeptMails { n = DefaultKeptMails } return n } // putIfChanged writes value at key only when it differs from what the bucket currently holds there (ADR 0201 // §12): the Mirror's own values never carry a time that moves on each sync, so the same JSON bytes (Go's own // json.Marshal is stable for a fixed type) mean nothing changed. func putIfChanged(bucket MirrorState, key string, value any) error { raw, err := json.Marshal(value) if err != nil { return err } cur, ok, err := bucket.Get(key) if err != nil { return err } if ok && bytes.Equal(cur, raw) { return nil } return bucket.Put(key, value) } // mailDigest is the digest named by a mail's own message-id, or by its folder and uid when it has none (novox/hq // ADR 0439 §3, issue 1282, AC5): two mails without a message-id in different folders or uids digest apart; one // message-id kept in two folders of the same account digests the same, so it is mirrored once. func mailDigest(m Message) string { basis := m.MessageID if basis == "" { basis = m.Folder + "\x00" + itoa(int64(m.UID)) } sum := sha256.Sum256([]byte(basis)) return hex.EncodeToString(sum[:]) } // mailKey is the mail bucket's own key: the account, so one account's keys are told apart from another's // (roundMails's own deletion), then a dot (account names hold none, config.go's accountName), then the digest. func mailKey(account string, m Message) string { return account + "." + mailDigest(m) } // mailValue is one mail's own value (ADR 0439 §3): account, folder, date, sender, subject, recipients and // attachment names (each capped to MostMailShown, the rest counted), size and flags, never a body, trimmed to // MostMailValue. func mailValue(account string, m Message) MailValue { sender := MailPerson{} if len(m.From) > 0 { sender = mailPerson(m.From[0]) } recipients, recipientsDropped := mailPeople(m.To) attachments, attachmentsDropped := mailAttachmentNames(m.Attachments) v := MailValue{ Account: account, Folder: cutUTF8(m.Folder, MostFolderName), Date: m.Date.UTC().Format(time.RFC3339), Sender: sender, Subject: cutUTF8(m.Subject, MostMailSubject), Recipients: recipients, RecipientsDropped: recipientsDropped, Attachments: attachments, AttachmentsDropped: attachmentsDropped, Size: m.Size, Flags: append([]string(nil), m.Flags...), } return trimMailValue(v) } // mailPerson is one Person as a MailValue names it: name and address each cut to MostFolderName bytes. func mailPerson(p Person) MailPerson { return MailPerson{Name: cutUTF8(p.Name, MostFolderName), Address: cutUTF8(p.Address, MostFolderName)} } // mailPeople is a list of recipients, capped to MostMailShown with the rest counted (ADR 0439 §3's "first 10 // recipients"). func mailPeople(ps []Person) ([]MailPerson, int) { dropped := 0 if len(ps) > MostMailShown { dropped = len(ps) - MostMailShown ps = ps[:MostMailShown] } out := make([]MailPerson, 0, len(ps)) for _, p := range ps { out = append(out, mailPerson(p)) } return out, dropped } // mailAttachmentNames is a list of attachment names, capped to MostMailShown with the rest counted (ADR 0439 // §3's "first 10 attachment names"), each cut to MostFolderName bytes. func mailAttachmentNames(refs []AttachmentRef) ([]string, int) { dropped := 0 if len(refs) > MostMailShown { dropped = len(refs) - MostMailShown refs = refs[:MostMailShown] } out := make([]string, 0, len(refs)) for _, r := range refs { out = append(out, cutUTF8(r.Name, MostFolderName)) } return out, dropped } // trimMailValue is issue 1282/AC4: while the value marshals to more than MostMailValue bytes, a recipient is // dropped from the end (RecipientsDropped counting it) until none is left, then an attachment name the same // way (AttachmentsDropped counting it), until it fits or both are empty. func trimMailValue(v MailValue) MailValue { for mailValueSize(v) > MostMailValue { switch { case len(v.Recipients) > 0: v.Recipients = v.Recipients[:len(v.Recipients)-1] v.RecipientsDropped++ case len(v.Attachments) > 0: v.Attachments = v.Attachments[:len(v.Attachments)-1] v.AttachmentsDropped++ default: return v } } return v } func mailValueSize(v MailValue) int { raw, err := json.Marshal(v) if err != nil { return 0 } return len(raw) } // cutUTF8 cuts s to at most n bytes, never splitting a multi-byte character (novox/hq ADR 0439 §2, §3). func cutUTF8(s string, n int) string { if len(s) <= n { return s } b := s[:n] for len(b) > 0 { r, size := utf8.DecodeLastRuneInString(b) if r == utf8.RuneError && size <= 1 { b = b[:len(b)-1] continue } break } return b }