Keep each account's newest mails in a bucket, bounded past 81 accounts (issue 1282)
This commit is contained in:
@@ -191,6 +191,7 @@ func serve(env Env) {
|
||||
Index: b.Index,
|
||||
Accounts: func() ([]Account, error) { return ReadAccounts(env.Accounts) },
|
||||
Accts: keptMirrorState{stdio.State(AccountsState)},
|
||||
Mails: keptMirrorState{stdio.State(MailsState)},
|
||||
Logf: logf,
|
||||
}
|
||||
trigger := make(chan struct{}, 1)
|
||||
|
||||
@@ -88,9 +88,9 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
|
||||
if fam["issued-by"] != "outside" || fam["path"] != "${dir:state}/"+SecretFamily+"*.secret" || len(m.OwnSecrets) != 2 {
|
||||
t.Fatalf("own secrets %v", m.OwnSecrets)
|
||||
}
|
||||
// 1255/AC7: the module declares the account bucket's own state, beside `sends`.
|
||||
if !reflect.DeepEqual(m.State, []string{"sends", "accounts"}) {
|
||||
t.Fatalf("state %v, want [sends accounts]", m.State)
|
||||
// 1255/AC7, 1282: the module declares the account and mail buckets' own state, beside `sends`.
|
||||
if !reflect.DeepEqual(m.State, []string{"sends", "accounts", "mails"}) {
|
||||
t.Fatalf("state %v, want [sends accounts mails]", m.State)
|
||||
}
|
||||
var names []string
|
||||
for _, tool := range (&Bundle{}).Tools() {
|
||||
|
||||
@@ -9,7 +9,11 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
@@ -18,6 +22,9 @@ import (
|
||||
// AccountsState is the module's state the Mirror keeps one key per account in (novox/hq ADR 0439 §2).
|
||||
const AccountsState = "accounts"
|
||||
|
||||
// MailsState is the module's state the Mirror keeps one key per mail in (novox/hq ADR 0439 §3, issue 1282).
|
||||
const MailsState = "mails"
|
||||
|
||||
// The bounds ADR 0439 §2 gives the account bucket's value.
|
||||
const (
|
||||
// MostAccountValue is the value's own bound, 64 KiB.
|
||||
@@ -31,6 +38,11 @@ const (
|
||||
MostMailValue = 4 << 10
|
||||
// MostMailBytes is ADR 0439 §4's own bound across every account's newest mails together, 64 MiB.
|
||||
MostMailBytes = 64 << 20
|
||||
// MostMailSubject is one mail's subject in the mail bucket, cut at a character's boundary when longer.
|
||||
MostMailSubject = 300
|
||||
// MostMailShown is how many recipients, and how many attachment names, the mail bucket's value names one
|
||||
// by one before it counts the rest (ADR 0439 §3's "first 10").
|
||||
MostMailShown = 10
|
||||
)
|
||||
|
||||
// AccountValue is state accounts' own value, one per account (ADR 0439 §2): never a time that moves on each
|
||||
@@ -56,6 +68,31 @@ type AccountFolderValue struct {
|
||||
Unread int `json:"unread"`
|
||||
}
|
||||
|
||||
// MailValue is state mails' own value, one per kept mail (novox/hq ADR 0439 §3): never a body. The subject is
|
||||
// cut to MostMailSubject bytes, and every address, name, folder and attachment name to MostFolderName bytes, at
|
||||
// a character's boundary; recipients and attachment names are each named one by one up to MostMailShown, the
|
||||
// rest counted, and when the value is still over MostMailValue bytes, recipients and then attachment names are
|
||||
// dropped from the end (RecipientsDropped, AttachmentsDropped growing) until it fits.
|
||||
type MailValue struct {
|
||||
Account string `json:"account"`
|
||||
Folder string `json:"folder"`
|
||||
Date string `json:"date"` // RFC3339; the mail's own Date header, never a time that moves
|
||||
Sender MailPerson `json:"sender"`
|
||||
Subject string `json:"subject"`
|
||||
Recipients []MailPerson `json:"recipients,omitempty"`
|
||||
RecipientsDropped int `json:"recipients-dropped,omitempty"`
|
||||
Attachments []string `json:"attachments,omitempty"`
|
||||
AttachmentsDropped int `json:"attachments-dropped,omitempty"`
|
||||
Size int64 `json:"size"`
|
||||
Flags []string `json:"flags,omitempty"`
|
||||
}
|
||||
|
||||
// MailPerson is a sender or a recipient in a MailValue, each field already cut to MostFolderName bytes.
|
||||
type MailPerson struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Address string `json:"address"`
|
||||
}
|
||||
|
||||
// MirrorState is what the Mirror needs of one state bucket it writes: a key's current raw value (nil when it
|
||||
// has none), a value put — the Mirror is this bucket's only writer, so never by compare-and-set — and every key
|
||||
// it holds. An interface, so a unit test gives a fake instead of a runtime (novox/hq ADR 0439 §2, §3; issue
|
||||
@@ -91,10 +128,13 @@ type Mirror struct {
|
||||
Index func(ctx context.Context) (Store, error)
|
||||
Accounts func() ([]Account, error)
|
||||
Accts MirrorState // the `accounts` state
|
||||
Mails MirrorState // the `mails` state (issue 1282); nil skips mirroring it (tests of accounts alone)
|
||||
Logf func(format string, a ...any)
|
||||
}
|
||||
|
||||
// Round mirrors every configured account's current state into the `accounts` bucket (issue 1255's AC1-AC7).
|
||||
// Round mirrors every configured account's current state into the `accounts` bucket, and (when Mails is given)
|
||||
// each one's newest mails into the `mails` bucket — both at the same, shared kept count (ADR 0439 §4's
|
||||
// 81-account rule, issue 1282).
|
||||
func (m *Mirror) Round(ctx context.Context) error {
|
||||
accounts, err := m.Accounts()
|
||||
if err != nil {
|
||||
@@ -118,6 +158,49 @@ func (m *Mirror) Round(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if m.Mails == nil {
|
||||
return nil
|
||||
}
|
||||
for _, a := range accounts {
|
||||
if err := m.roundMails(ctx, st, a.Name, kept); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// roundMails mirrors one account's newest kept mails into the `mails` bucket (novox/hq ADR 0439 §3, issue
|
||||
// 1282): a key put for one new or changed, a key deleted for one no longer among the newest kept — never for
|
||||
// another account's own keys, which mailKey's account prefix tells apart.
|
||||
func (m *Mirror) roundMails(ctx context.Context, st Store, account string, kept int) error {
|
||||
msgs, err := st.Newest(ctx, account, kept)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
desired := map[string]MailValue{}
|
||||
for _, msg := range msgs {
|
||||
desired[mailKey(account, msg)] = mailValue(account, msg)
|
||||
}
|
||||
for key, v := range desired {
|
||||
if err := putIfChanged(m.Mails, key, v); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
keys, err := m.Mails.Keys()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
prefix := account + "."
|
||||
for _, k := range keys {
|
||||
if !strings.HasPrefix(k, prefix) {
|
||||
continue
|
||||
}
|
||||
if _, still := desired[k]; !still {
|
||||
if err := m.Mails.Delete(k); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -220,6 +303,112 @@ func putIfChanged(bucket MirrorState, key string, value any) error {
|
||||
return bucket.Put(key, value)
|
||||
}
|
||||
|
||||
// mailDigest is the digest named by a mail's own message-id, or by its folder and uid when it has none (novox/hq
|
||||
// ADR 0439 §3, issue 1282, AC5): two mails without a message-id in different folders or uids digest apart; one
|
||||
// message-id kept in two folders of the same account digests the same, so it is mirrored once.
|
||||
func mailDigest(m Message) string {
|
||||
basis := m.MessageID
|
||||
if basis == "" {
|
||||
basis = m.Folder + "\x00" + itoa(int64(m.UID))
|
||||
}
|
||||
sum := sha256.Sum256([]byte(basis))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// mailKey is the mail bucket's own key: the account, so one account's keys are told apart from another's
|
||||
// (roundMails's own deletion), then a dot (account names hold none, config.go's accountName), then the digest.
|
||||
func mailKey(account string, m Message) string {
|
||||
return account + "." + mailDigest(m)
|
||||
}
|
||||
|
||||
// mailValue is one mail's own value (ADR 0439 §3): account, folder, date, sender, subject, recipients and
|
||||
// attachment names (each capped to MostMailShown, the rest counted), size and flags, never a body, trimmed to
|
||||
// MostMailValue.
|
||||
func mailValue(account string, m Message) MailValue {
|
||||
sender := MailPerson{}
|
||||
if len(m.From) > 0 {
|
||||
sender = mailPerson(m.From[0])
|
||||
}
|
||||
recipients, recipientsDropped := mailPeople(m.To)
|
||||
attachments, attachmentsDropped := mailAttachmentNames(m.Attachments)
|
||||
v := MailValue{
|
||||
Account: account,
|
||||
Folder: cutUTF8(m.Folder, MostFolderName),
|
||||
Date: m.Date.UTC().Format(time.RFC3339),
|
||||
Sender: sender,
|
||||
Subject: cutUTF8(m.Subject, MostMailSubject),
|
||||
Recipients: recipients,
|
||||
RecipientsDropped: recipientsDropped,
|
||||
Attachments: attachments,
|
||||
AttachmentsDropped: attachmentsDropped,
|
||||
Size: m.Size,
|
||||
Flags: append([]string(nil), m.Flags...),
|
||||
}
|
||||
return trimMailValue(v)
|
||||
}
|
||||
|
||||
// mailPerson is one Person as a MailValue names it: name and address each cut to MostFolderName bytes.
|
||||
func mailPerson(p Person) MailPerson {
|
||||
return MailPerson{Name: cutUTF8(p.Name, MostFolderName), Address: cutUTF8(p.Address, MostFolderName)}
|
||||
}
|
||||
|
||||
// mailPeople is a list of recipients, capped to MostMailShown with the rest counted (ADR 0439 §3's "first 10
|
||||
// recipients").
|
||||
func mailPeople(ps []Person) ([]MailPerson, int) {
|
||||
dropped := 0
|
||||
if len(ps) > MostMailShown {
|
||||
dropped = len(ps) - MostMailShown
|
||||
ps = ps[:MostMailShown]
|
||||
}
|
||||
out := make([]MailPerson, 0, len(ps))
|
||||
for _, p := range ps {
|
||||
out = append(out, mailPerson(p))
|
||||
}
|
||||
return out, dropped
|
||||
}
|
||||
|
||||
// mailAttachmentNames is a list of attachment names, capped to MostMailShown with the rest counted (ADR 0439
|
||||
// §3's "first 10 attachment names"), each cut to MostFolderName bytes.
|
||||
func mailAttachmentNames(refs []AttachmentRef) ([]string, int) {
|
||||
dropped := 0
|
||||
if len(refs) > MostMailShown {
|
||||
dropped = len(refs) - MostMailShown
|
||||
refs = refs[:MostMailShown]
|
||||
}
|
||||
out := make([]string, 0, len(refs))
|
||||
for _, r := range refs {
|
||||
out = append(out, cutUTF8(r.Name, MostFolderName))
|
||||
}
|
||||
return out, dropped
|
||||
}
|
||||
|
||||
// trimMailValue is issue 1282/AC4: while the value marshals to more than MostMailValue bytes, a recipient is
|
||||
// dropped from the end (RecipientsDropped counting it) until none is left, then an attachment name the same
|
||||
// way (AttachmentsDropped counting it), until it fits or both are empty.
|
||||
func trimMailValue(v MailValue) MailValue {
|
||||
for mailValueSize(v) > MostMailValue {
|
||||
switch {
|
||||
case len(v.Recipients) > 0:
|
||||
v.Recipients = v.Recipients[:len(v.Recipients)-1]
|
||||
v.RecipientsDropped++
|
||||
case len(v.Attachments) > 0:
|
||||
v.Attachments = v.Attachments[:len(v.Attachments)-1]
|
||||
v.AttachmentsDropped++
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func mailValueSize(v MailValue) int {
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return len(raw)
|
||||
}
|
||||
|
||||
// cutUTF8 cuts s to at most n bytes, never splitting a multi-byte character (novox/hq ADR 0439 §2, §3).
|
||||
func cutUTF8(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
package main
|
||||
|
||||
// The account bucket (novox/hq ADR 0439 §2, issue 1255): a fake MirrorState, so these tests need no runtime,
|
||||
// and memStore for the index, as the rest of this package's unit tests already use it.
|
||||
// The account bucket (novox/hq ADR 0439 §2, issue 1255) and the mail bucket (ADR 0439 §3, issue 1282): a fake
|
||||
// MirrorState, so these tests need no runtime, and memStore for the index, as the rest of this package's unit
|
||||
// tests already use it.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
@@ -61,6 +63,13 @@ func mirrorOf(accounts []Account, st *memStore, bucket MirrorState) *Mirror {
|
||||
}
|
||||
}
|
||||
|
||||
// mirrorWithMails is mirrorOf, also mirroring the `mails` bucket (issue 1282).
|
||||
func mirrorWithMails(accounts []Account, st *memStore, accts, mails MirrorState) *Mirror {
|
||||
m := mirrorOf(accounts, st, accts)
|
||||
m.Mails = mails
|
||||
return m
|
||||
}
|
||||
|
||||
// 1255/AC1: the account's value holds its state word, the number of newest mails kept (200, no other account
|
||||
// configured, so none of ADR 0439 §4's 81-account rule applies), and its folders with their counts.
|
||||
func TestMirrorRoundWritesTheAccountsStateWord200AndItsFolders(t *testing.T) {
|
||||
@@ -288,3 +297,287 @@ func TestAccountsStateNameMatchesTheManifestsWord(t *testing.T) {
|
||||
t.Fatalf("AccountsState = %q", AccountsState)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- the mail bucket (novox/hq ADR 0439 §3, issue 1282) -----------------------------------------------------
|
||||
|
||||
// 1282/AC1: a new mail indexed gets one key in `mails`, named by its account and a digest of its message-id,
|
||||
// whose value carries its account, folder, date, sender, subject, recipients, attachment names, size and
|
||||
// flags, and no body.
|
||||
func TestMirrorRoundWritesANewMailsKeyWithItsFields(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
accounts, st := accountsAndStore(t)
|
||||
msg := Message{Account: "mesh", Folder: "INBOX", UID: 1, MessageID: "m1@example.org", Date: judgedAt,
|
||||
From: []Person{{Name: "Bob", Address: "bob@example.org"}}, To: []Person{{Address: "ops@example.org"}},
|
||||
Subject: "Hi", Size: 1234, Flags: []string{`\Seen`},
|
||||
Attachments: []AttachmentRef{{Key: "k1", Name: "invoice.pdf", Size: 10, Type: "application/pdf"}},
|
||||
Body: "should never appear in the mirrored value",
|
||||
}
|
||||
if err := st.Add(ctx, msg, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = st.SetFolder(ctx, Folder{Account: "mesh", Name: "INBOX", Messages: 1})
|
||||
|
||||
acctsBucket, mailsBucket := newMemMirrorState(), newMemMirrorState()
|
||||
m := mirrorWithMails(accounts, st, acctsBucket, mailsBucket)
|
||||
if err := m.Round(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
raw, ok, err := mailsBucket.Get(mailKey("mesh", msg))
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("no value kept for the new mail: %v %v", ok, err)
|
||||
}
|
||||
var v MailValue
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v.Account != "mesh" || v.Folder != "INBOX" || v.Subject != "Hi" || v.Size != 1234 {
|
||||
t.Fatalf("%+v", v)
|
||||
}
|
||||
if v.Sender.Name != "Bob" || v.Sender.Address != "bob@example.org" {
|
||||
t.Fatalf("sender: %+v", v.Sender)
|
||||
}
|
||||
if len(v.Recipients) != 1 || v.Recipients[0].Address != "ops@example.org" {
|
||||
t.Fatalf("recipients: %+v", v.Recipients)
|
||||
}
|
||||
if len(v.Attachments) != 1 || v.Attachments[0] != "invoice.pdf" {
|
||||
t.Fatalf("attachments: %+v", v.Attachments)
|
||||
}
|
||||
if len(v.Flags) != 1 || v.Flags[0] != `\Seen` {
|
||||
t.Fatalf("flags: %+v", v.Flags)
|
||||
}
|
||||
if strings.Contains(string(raw), "should never appear") {
|
||||
t.Fatalf("the value carries the body: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// 1282/AC2: a kept mail whose flags change has its key rewritten; a round with nothing changed writes nothing.
|
||||
func TestMirrorRoundRewritesOnlyAChangedMailsKey(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
accounts, st := accountsAndStore(t)
|
||||
a := Message{Account: "mesh", Folder: "INBOX", UID: 1, MessageID: "a@example.org", Date: judgedAt, Subject: "a"}
|
||||
b := Message{Account: "mesh", Folder: "INBOX", UID: 2, MessageID: "b@example.org", Date: judgedAt, Subject: "b"}
|
||||
if err := st.Add(ctx, a, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.Add(ctx, b, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = st.SetFolder(ctx, Folder{Account: "mesh", Name: "INBOX", Messages: 2})
|
||||
|
||||
acctsBucket, mailsBucket := newMemMirrorState(), newMemMirrorState()
|
||||
m := mirrorWithMails(accounts, st, acctsBucket, mailsBucket)
|
||||
if err := m.Round(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mailsBucket.calls = map[string]int{}
|
||||
|
||||
// Nothing changed: a second round writes neither key again.
|
||||
if err := m.Round(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if mailsBucket.calls[mailKey("mesh", a)] != 0 || mailsBucket.calls[mailKey("mesh", b)] != 0 {
|
||||
t.Fatalf("a round with nothing changed wrote a mail key: %v", mailsBucket.calls)
|
||||
}
|
||||
|
||||
// a's flags change: only a's key is rewritten.
|
||||
if err := st.SetFlags(ctx, "mesh", "INBOX", 1, []string{`\Seen`}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Round(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if mailsBucket.calls[mailKey("mesh", a)] != 1 {
|
||||
t.Fatalf("a's key was not rewritten: %v", mailsBucket.calls)
|
||||
}
|
||||
if mailsBucket.calls[mailKey("mesh", b)] != 0 {
|
||||
t.Fatalf("b's key was written though nothing of it changed: %v", mailsBucket.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// 1282/AC3 (removed): a mail removed from the index has its key deleted.
|
||||
func TestMirrorRoundDeletesAMailsKeyForARemovedMail(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
_, st := accountsAndStore(t)
|
||||
msg := Message{Account: "mesh", Folder: "INBOX", UID: 1, MessageID: "m@example.org", Date: judgedAt, Subject: "s"}
|
||||
if err := st.Add(context.Background(), msg, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bucket := newMemMirrorState()
|
||||
mir := &Mirror{Mails: bucket}
|
||||
if err := mir.roundMails(ctx, st, "mesh", 200); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok, _ := bucket.Get(mailKey("mesh", msg)); !ok {
|
||||
t.Fatalf("the mail's key was not written")
|
||||
}
|
||||
|
||||
if err := st.Remove(ctx, "mesh", "INBOX", 1, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := mir.roundMails(ctx, st, "mesh", 200); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok, _ := bucket.Get(mailKey("mesh", msg)); ok {
|
||||
t.Fatalf("the removed mail's key was not deleted")
|
||||
}
|
||||
}
|
||||
|
||||
// 1282/AC3 (pushed out): a mail pushed out of the newest kept by a newer one has its key deleted.
|
||||
func TestMirrorRoundDeletesAMailsKeyPushedOutOfTheNewestKept(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
_, st := accountsAndStore(t)
|
||||
old := Message{Account: "mesh", Folder: "INBOX", UID: 1, MessageID: "old@example.org", Date: judgedAt.Add(-time.Hour), Subject: "old"}
|
||||
if err := st.Add(ctx, old, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bucket := newMemMirrorState()
|
||||
mir := &Mirror{Mails: bucket}
|
||||
if err := mir.roundMails(ctx, st, "mesh", 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok, _ := bucket.Get(mailKey("mesh", old)); !ok {
|
||||
t.Fatalf("the old mail's key was not written")
|
||||
}
|
||||
|
||||
newer := Message{Account: "mesh", Folder: "INBOX", UID: 2, MessageID: "new@example.org", Date: judgedAt, Subject: "new"}
|
||||
if err := st.Add(ctx, newer, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := mir.roundMails(ctx, st, "mesh", 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok, _ := bucket.Get(mailKey("mesh", old)); ok {
|
||||
t.Fatalf("the old mail's key was not deleted once pushed out of the newest kept")
|
||||
}
|
||||
if _, ok, _ := bucket.Get(mailKey("mesh", newer)); !ok {
|
||||
t.Fatalf("the newer mail's key was not written")
|
||||
}
|
||||
}
|
||||
|
||||
// 1282/AC4: a mail with 500 recipients of 254-byte addresses, 50 attachments of 255-byte names and a
|
||||
// 5,000-character multibyte subject is mirrored at most MostMailValue bytes, its subject at most
|
||||
// MostMailSubject, each address and attachment name at most MostFolderName, and the dropped counts are in it.
|
||||
func TestMirrorRoundTrimsAnOversizedMailValueToAtMost4KiB(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
accounts, st := accountsAndStore(t)
|
||||
var to []Person
|
||||
for i := 0; i < 500; i++ {
|
||||
to = append(to, Person{Address: strings.Repeat("r", 240) + "@example.org"}) // 254 bytes
|
||||
}
|
||||
var atts []AttachmentRef
|
||||
for i := 0; i < 50; i++ {
|
||||
atts = append(atts, AttachmentRef{Key: "k" + itoa(int64(i)), Name: strings.Repeat("n", 255), Size: 10})
|
||||
}
|
||||
subject := strings.Repeat("é", 5000)
|
||||
msg := Message{Account: "mesh", Folder: "INBOX", UID: 1, MessageID: "big@example.org", Date: judgedAt,
|
||||
From: []Person{{Address: "sender@example.org"}}, To: to, Subject: subject, Size: 99, Attachments: atts}
|
||||
if err := st.Add(ctx, msg, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = st.SetFolder(ctx, Folder{Account: "mesh", Name: "INBOX", Messages: 1})
|
||||
|
||||
acctsBucket, mailsBucket := newMemMirrorState(), newMemMirrorState()
|
||||
m := mirrorWithMails(accounts, st, acctsBucket, mailsBucket)
|
||||
if err := m.Round(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
raw, ok, err := mailsBucket.Get(mailKey("mesh", msg))
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("no value kept: %v %v", ok, err)
|
||||
}
|
||||
if len(raw) > MostMailValue {
|
||||
t.Fatalf("the value is %d bytes, more than %d", len(raw), MostMailValue)
|
||||
}
|
||||
var v MailValue
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(v.Subject) > MostMailSubject {
|
||||
t.Fatalf("the subject is %d bytes, more than %d", len(v.Subject), MostMailSubject)
|
||||
}
|
||||
if len(v.Recipients) > MostMailShown {
|
||||
t.Fatalf("%d recipients shown, more than %d", len(v.Recipients), MostMailShown)
|
||||
}
|
||||
for _, r := range v.Recipients {
|
||||
if len(r.Address) > MostFolderName {
|
||||
t.Fatalf("a recipient's address is %d bytes, more than %d", len(r.Address), MostFolderName)
|
||||
}
|
||||
}
|
||||
for _, a := range v.Attachments {
|
||||
if len(a) > MostFolderName {
|
||||
t.Fatalf("an attachment name is %d bytes, more than %d", len(a), MostFolderName)
|
||||
}
|
||||
}
|
||||
if v.RecipientsDropped == 0 {
|
||||
t.Fatalf("recipients-dropped is 0, though 500 recipients were given")
|
||||
}
|
||||
if v.AttachmentsDropped == 0 {
|
||||
t.Fatalf("attachments-dropped is 0, though 50 attachments were given")
|
||||
}
|
||||
}
|
||||
|
||||
// 1282/AC5: two mails without a message-id, in different folders or uids, digest apart (two keys); one
|
||||
// message-id kept in two folders of one account digests the same (one key).
|
||||
func TestMailDigestDistinguishesByFolderAndUIDAndMergesByMessageID(t *testing.T) {
|
||||
a := Message{Folder: "INBOX", UID: 1}
|
||||
b := Message{Folder: "INBOX", UID: 2}
|
||||
if mailDigest(a) == mailDigest(b) {
|
||||
t.Fatalf("two mails without a message-id, different uids, digest the same")
|
||||
}
|
||||
c := Message{Folder: "Archive", UID: 1}
|
||||
if mailDigest(a) == mailDigest(c) {
|
||||
t.Fatalf("two mails without a message-id, different folders, digest the same")
|
||||
}
|
||||
withID1 := Message{Folder: "INBOX", UID: 9, MessageID: "x@example.org"}
|
||||
withID2 := Message{Folder: "Sent", UID: 10, MessageID: "x@example.org"}
|
||||
if mailDigest(withID1) != mailDigest(withID2) {
|
||||
t.Fatalf("one message-id kept in two folders of the account digests apart: %q vs %q",
|
||||
mailDigest(withID1), mailDigest(withID2))
|
||||
}
|
||||
}
|
||||
|
||||
// 1282/AC6: with 82 accounts, every account keeps the same smaller number n of newest mails, with
|
||||
// 82 x n x MostMailValue at most MostMailBytes, and n is written in each account's own `accounts` value
|
||||
// (issue 1255's Kept field).
|
||||
func TestMirrorRoundUses81AccountRuleForTheKeptNumber(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
var accounts []Account
|
||||
for i := 0; i < 82; i++ {
|
||||
accounts = append(accounts, Account{Name: "a" + itoa(int64(i)), Kind: KindIMAP, Address: "a@example.org"})
|
||||
}
|
||||
st := newMemStore()
|
||||
for _, a := range accounts {
|
||||
if err := st.SetState(ctx, AccountState{Account: a.Name, State: AccountCurrent}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
n := keptPerAccount(82)
|
||||
if n >= DefaultKeptMails {
|
||||
t.Fatalf("82 accounts did not reduce the kept number below %d: got %d", DefaultKeptMails, n)
|
||||
}
|
||||
if 82*n*MostMailValue > MostMailBytes {
|
||||
t.Fatalf("82 x %d x %d bytes is more than %d", n, MostMailValue, MostMailBytes)
|
||||
}
|
||||
|
||||
acctsBucket, mailsBucket := newMemMirrorState(), newMemMirrorState()
|
||||
m := mirrorWithMails(accounts, st, acctsBucket, mailsBucket)
|
||||
if err := m.Round(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, a := range accounts {
|
||||
raw, ok, err := acctsBucket.Get(a.Name)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("%s: no value: %v %v", a.Name, ok, err)
|
||||
}
|
||||
var v AccountValue
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v.Kept != n {
|
||||
t.Fatalf("%s: kept %d, want %d", a.Name, v.Kept, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -634,6 +634,13 @@ func (s *PgStore) Search(ctx context.Context, q Query) ([]Message, error) {
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// Newest is 1282: one account's newest n mails, via Search's own ordering (newest first) and deduplication (no
|
||||
// folder named, so dedupedMessagesSQL applies — the same copy Search itself would answer). Covered only by the
|
||||
// integration tests (issue 1244): this package's unit tests use memStore.
|
||||
func (s *PgStore) Newest(ctx context.Context, account string, n int) ([]Message, error) {
|
||||
return s.Search(ctx, Query{Accounts: []string{account}, Limit: n})
|
||||
}
|
||||
|
||||
// GroupBySender is 1230/AC10: one line per sender among the messages searchWhere's filters match, by count
|
||||
// descending (address ascending to break ties), Limit bounding the number of lines.
|
||||
func (s *PgStore) GroupBySender(ctx context.Context, q Query) ([]SenderGroup, error) {
|
||||
|
||||
@@ -183,6 +183,9 @@ type Store interface {
|
||||
Get(ctx context.Context, account, folder string, uid uint32) (Message, bool, error)
|
||||
// Search finds messages, newest first by default (OrderBy, OrderDesc choose otherwise), without their bodies.
|
||||
Search(ctx context.Context, q Query) ([]Message, error)
|
||||
// Newest is one account's newest n mails, without their bodies, each (account, message-id) counted once the
|
||||
// same way Search answers with no folder named (novox/hq ADR 0439 §3, issue 1282): for the mail bucket.
|
||||
Newest(ctx context.Context, account string, n int) ([]Message, error)
|
||||
// GroupBySender is 1230/AC10: one line per sender among the messages Query's other filters match, by count
|
||||
// descending, Limit bounding the number of lines.
|
||||
GroupBySender(ctx context.Context, q Query) ([]SenderGroup, error)
|
||||
@@ -379,6 +382,12 @@ func (s *memStore) Search(_ context.Context, q Query) ([]Message, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Newest is 1282: one account's newest n mails, via Search's own ordering (newest first) and deduplication (no
|
||||
// folder named, so dedupeArchive applies — the same copy Search itself would answer).
|
||||
func (s *memStore) Newest(ctx context.Context, account string, n int) ([]Message, error) {
|
||||
return s.Search(ctx, Query{Accounts: []string{account}, Limit: n})
|
||||
}
|
||||
|
||||
// orderLess is 1230 (AC7, AC11, AC17's schema): date (the default when orderBy is empty), sender (by its first
|
||||
// address), subject or size, ascending unless desc; ties break on (account, folder, uid), always ascending, so
|
||||
// the order is stable regardless of the map order Go would otherwise give equal messages.
|
||||
|
||||
@@ -88,7 +88,8 @@
|
||||
],
|
||||
"state": [
|
||||
"sends",
|
||||
"accounts"
|
||||
"accounts",
|
||||
"mails"
|
||||
],
|
||||
"emits": [
|
||||
"mail.send.sent",
|
||||
|
||||
Reference in New Issue
Block a user