package main // The account bucket (novox/hq ADR 0439 §2, issue 1255) and the mail bucket (ADR 0439 §3, issue 1282): a fake // MirrorState, so these tests need no runtime, and memStore for the index, as the rest of this package's unit // tests already use it. import ( "context" "encoding/json" "strings" "testing" "time" "unicode/utf8" ) // memMirrorState is a fake MirrorState (mirror.go), kept in memory: a map, no revision, since the Mirror is a // bucket's only writer and never writes it by compare-and-set. type memMirrorState struct { vals map[string]json.RawMessage calls map[string]int // Put calls per key, so a test can tell a skipped write from a repeated one } func newMemMirrorState() *memMirrorState { return &memMirrorState{vals: map[string]json.RawMessage{}, calls: map[string]int{}} } func (m *memMirrorState) Get(key string) (json.RawMessage, bool, error) { v, ok := m.vals[key] return v, ok, nil } func (m *memMirrorState) Put(key string, value any) error { raw, err := json.Marshal(value) if err != nil { return err } m.vals[key] = raw m.calls[key]++ return nil } func (m *memMirrorState) Delete(key string) error { delete(m.vals, key) return nil } func (m *memMirrorState) Keys() ([]string, error) { out := make([]string, 0, len(m.vals)) for k := range m.vals { out = append(out, k) } return out, nil } // accountsAndStore is one account "mesh", granted no caller (the Mirror reads no grant), with the index given. func accountsAndStore(t *testing.T) ([]Account, *memStore) { t.Helper() return []Account{{Name: "mesh", Kind: KindIMAP, Address: "jschoubben@novox.be"}}, newMemStore() } func mirrorOf(accounts []Account, st *memStore, bucket MirrorState) *Mirror { return &Mirror{ Index: func(context.Context) (Store, error) { return st, nil }, Accounts: func() ([]Account, error) { return accounts, nil }, Accts: bucket, } } // mirrorWithMails is mirrorOf, also mirroring the `mails` bucket (issue 1282). func mirrorWithMails(accounts []Account, st *memStore, accts, mails MirrorState) *Mirror { m := mirrorOf(accounts, st, accts) m.Mails = mails return m } // 1255/AC1: the account's value holds its state word, the number of newest mails kept (200, no other account // configured, so none of ADR 0439 §4's 81-account rule applies), and its folders with their counts. func TestMirrorRoundWritesTheAccountsStateWord200AndItsFolders(t *testing.T) { ctx := context.Background() accounts, st := accountsAndStore(t) _ = st.SetState(ctx, AccountState{Account: "mesh", State: AccountCurrent, Why: "watching"}) addMail(t, st, "mesh", "INBOX", 1, []string{}) // unread addMail(t, st, "mesh", "INBOX", 2, []string{`\Seen`}) // read addMail(t, st, "mesh", "Sent", 3, []string{`\Seen`}) // read _ = st.SetFolder(ctx, Folder{Account: "mesh", Name: "INBOX", Messages: 10}) _ = st.SetFolder(ctx, Folder{Account: "mesh", Name: "Sent", Messages: 5}) bucket := newMemMirrorState() m := mirrorOf(accounts, st, bucket) if err := m.Round(ctx); err != nil { t.Fatal(err) } raw, ok, err := bucket.Get("mesh") if err != nil || !ok { t.Fatalf("no value kept for mesh: %v %v", ok, err) } var v AccountValue if err := json.Unmarshal(raw, &v); err != nil { t.Fatal(err) } if v.State != AccountCurrent || v.Kept != DefaultKeptMails { t.Fatalf("state %q, kept %d; want %q, %d", v.State, v.Kept, AccountCurrent, DefaultKeptMails) } byName := map[string]AccountFolderValue{} for _, f := range v.Folders { byName[f.Name] = f } if byName["INBOX"].Mails != 10 || byName["INBOX"].Unread != 1 { t.Fatalf("INBOX: %+v", byName["INBOX"]) } if byName["Sent"].Mails != 5 || byName["Sent"].Unread != 0 { t.Fatalf("Sent: %+v", byName["Sent"]) } } // addMail indexes one bare message with the given flags, for folder-count tests that only need its UID and // flags (mirroring the shape imap.go's Keeper writes, store.go's Message). func addMail(t *testing.T, st *memStore, account, folder string, uid uint32, flags []string) { t.Helper() m := Message{Account: account, Folder: folder, UID: uid, Flags: flags} if err := st.Add(context.Background(), m, nil); err != nil { t.Fatal(err) } } // 1255/AC2: nothing is written when the index is unchanged since the last round. func TestMirrorRoundWritesNothingWhenUnchanged(t *testing.T) { ctx := context.Background() accounts, st := accountsAndStore(t) _ = st.SetState(ctx, AccountState{Account: "mesh", State: AccountCurrent}) _ = st.SetFolder(ctx, Folder{Account: "mesh", Name: "INBOX", Messages: 1}) bucket := newMemMirrorState() m := mirrorOf(accounts, st, bucket) if err := m.Round(ctx); err != nil { t.Fatal(err) } if bucket.calls["mesh"] != 1 { t.Fatalf("the first round wrote %d time(s); want 1", bucket.calls["mesh"]) } if err := m.Round(ctx); err != nil { t.Fatal(err) } if bucket.calls["mesh"] != 1 { t.Fatalf("a round with nothing changed wrote again: %d time(s)", bucket.calls["mesh"]) } } // 1255/AC3: one mail of INBOX marked read changes only that account's key, to 1 unread. func TestMirrorRoundWritesOnlyTheChangedAccountsKey(t *testing.T) { ctx := context.Background() accounts, st := accountsAndStore(t) accounts = append(accounts, Account{Name: "other", Kind: KindIMAP, Address: "other@novox.be"}) _ = st.SetState(ctx, AccountState{Account: "mesh", State: AccountCurrent}) _ = st.SetState(ctx, AccountState{Account: "other", State: AccountCurrent}) addMail(t, st, "mesh", "INBOX", 1, []string{}) _ = st.SetFolder(ctx, Folder{Account: "mesh", Name: "INBOX", Messages: 1}) _ = st.SetFolder(ctx, Folder{Account: "other", Name: "INBOX", Messages: 0}) bucket := newMemMirrorState() m := mirrorOf(accounts, st, bucket) if err := m.Round(ctx); err != nil { t.Fatal(err) } bucket.calls = map[string]int{} // reset the counters; only the next round's writes matter here // mesh's one message is now marked read. if err := st.SetFlags(ctx, "mesh", "INBOX", 1, []string{`\Seen`}, nil); err != nil { t.Fatal(err) } if err := m.Round(ctx); err != nil { t.Fatal(err) } if bucket.calls["mesh"] != 1 { t.Fatalf("mesh's key was not rewritten: %d call(s)", bucket.calls["mesh"]) } if bucket.calls["other"] != 0 { t.Fatalf("other's key was written though nothing of it changed: %d call(s)", bucket.calls["other"]) } raw, _, _ := bucket.Get("mesh") var v AccountValue _ = json.Unmarshal(raw, &v) for _, f := range v.Folders { if f.Name == "INBOX" && f.Unread != 0 { t.Fatalf("INBOX unread after marking read: %d", f.Unread) } } } // 1255/AC4: a folder name of 200 bytes of multibyte characters is cut to at most 120 bytes at a character's // boundary (never split mid-character). func TestMirrorRoundCutsALongMultibyteFolderNameAtACharacterBoundary(t *testing.T) { ctx := context.Background() accounts, st := accountsAndStore(t) _ = st.SetState(ctx, AccountState{Account: "mesh", State: AccountCurrent}) // 119 one-byte letters, then 2-byte "é"s: the 120th byte (MostFolderName) lands mid-character, inside the // first é's 2-byte encoding, so a naive byte cut would split it. long := strings.Repeat("a", 119) + strings.Repeat("é", 40) _ = st.SetFolder(ctx, Folder{Account: "mesh", Name: long, Messages: 1}) bucket := newMemMirrorState() m := mirrorOf(accounts, st, bucket) if err := m.Round(ctx); err != nil { t.Fatal(err) } raw, _, _ := bucket.Get("mesh") var v AccountValue _ = json.Unmarshal(raw, &v) if len(v.Folders) != 1 { t.Fatalf("folders: %+v", v.Folders) } got := v.Folders[0].Name if len(got) > MostFolderName { t.Fatalf("the folder name is %d bytes, more than %d", len(got), MostFolderName) } if !strings.HasPrefix(long, got) { t.Fatalf("the cut name is not a prefix of the original: %q", got) } if !utf8.ValidString(got) { t.Fatalf("the cut name is not valid UTF-8: a multibyte character was split: %q", got) } } // 1255/AC5: an account with enough folders that the value would pass 64 KiB has the folders with the fewest // mails dropped, their number counted, and the value fits. func TestMirrorRoundDropsTheFewestMailFoldersToFitUnder64KiB(t *testing.T) { ctx := context.Background() accounts, st := accountsAndStore(t) _ = st.SetState(ctx, AccountState{Account: "mesh", State: AccountCurrent}) // Enough folders, with long enough names, that the whole value passes MostAccountValue; each folder's own // mail count is distinct, so "fewest first" is unambiguous. const folders = 1200 for i := 0; i < folders; i++ { name := "folder-" + itoa(int64(i)) + "-abcdefghijklmnopqrstuvwxyz0123456789" _ = st.SetFolder(ctx, Folder{Account: "mesh", Name: name, Messages: i + 1}) } bucket := newMemMirrorState() m := mirrorOf(accounts, st, bucket) if err := m.Round(ctx); err != nil { t.Fatal(err) } raw, _, _ := bucket.Get("mesh") if len(raw) > MostAccountValue { t.Fatalf("the value is %d bytes, more than %d", len(raw), MostAccountValue) } var v AccountValue _ = json.Unmarshal(raw, &v) if v.DroppedFolders == 0 { t.Fatalf("no folder was dropped, though %d folders would overflow %d bytes", folders, MostAccountValue) } if v.DroppedFolders+len(v.Folders) != folders { t.Fatalf("dropped %d + kept %d != %d folders in all", v.DroppedFolders, len(v.Folders), folders) } // The kept folders are never among the ones with the fewest mails: folder i has i+1 mails, so every kept // folder's count is at least as large as every dropped folder's would have been. fewestKept := -1 for _, f := range v.Folders { if fewestKept == -1 || f.Mails < fewestKept { fewestKept = f.Mails } } if fewestKept < v.DroppedFolders { t.Fatalf("the fewest-mail folders were not the ones dropped: kept as few as %d mails, dropped %d folders", fewestKept, v.DroppedFolders) } } // 1255/AC6: two rounds an hour apart with nothing indexed between write the byte-identical value (no time that // moves on each sync). func TestMirrorRoundTwoRoundsWithNothingChangedWriteByteIdenticalValues(t *testing.T) { ctx := context.Background() accounts, st := accountsAndStore(t) _ = st.SetState(ctx, AccountState{Account: "mesh", State: AccountCurrent}) _ = st.SetFolder(ctx, Folder{Account: "mesh", Name: "INBOX", Messages: 3}) bucket := newMemMirrorState() m := mirrorOf(accounts, st, bucket) if err := m.Round(ctx); err != nil { t.Fatal(err) } first, _, _ := bucket.Get("mesh") firstCopy := append([]byte(nil), first...) // Force a second write by deleting the key directly and running again: the value computed must still be // byte-identical to the first, since nothing in the index changed. _ = bucket.Delete("mesh") if err := m.Round(ctx); err != nil { t.Fatal(err) } second, _, _ := bucket.Get("mesh") if string(firstCopy) != string(second) { t.Fatalf("the two rounds wrote different bytes:\n%s\n%s", firstCopy, second) } } // 1255/AC7: the manifest declares the state accounts (manifest_test.go asserts the exact list; this just // anchors the constant the code and the manifest both use). func TestAccountsStateNameMatchesTheManifestsWord(t *testing.T) { if AccountsState != "accounts" { t.Fatalf("AccountsState = %q", AccountsState) } } // ---- the mail bucket (novox/hq ADR 0439 §3, issue 1282) ----------------------------------------------------- // 1282/AC1: a new mail indexed gets one key in `mails`, named by its account and a digest of its message-id, // whose value carries its account, folder, date, sender, subject, recipients, attachment names, size and // flags, and no body. func TestMirrorRoundWritesANewMailsKeyWithItsFields(t *testing.T) { ctx := context.Background() accounts, st := accountsAndStore(t) msg := Message{Account: "mesh", Folder: "INBOX", UID: 1, MessageID: "m1@example.org", Date: judgedAt, From: []Person{{Name: "Bob", Address: "bob@example.org"}}, To: []Person{{Address: "ops@example.org"}}, Subject: "Hi", Size: 1234, Flags: []string{`\Seen`}, Attachments: []AttachmentRef{{Key: "k1", Name: "invoice.pdf", Size: 10, Type: "application/pdf"}}, Body: "should never appear in the mirrored value", } if err := st.Add(ctx, msg, nil); err != nil { t.Fatal(err) } _ = st.SetFolder(ctx, Folder{Account: "mesh", Name: "INBOX", Messages: 1}) acctsBucket, mailsBucket := newMemMirrorState(), newMemMirrorState() m := mirrorWithMails(accounts, st, acctsBucket, mailsBucket) if err := m.Round(ctx); err != nil { t.Fatal(err) } raw, ok, err := mailsBucket.Get(mailKey("mesh", msg)) if err != nil || !ok { t.Fatalf("no value kept for the new mail: %v %v", ok, err) } var v MailValue if err := json.Unmarshal(raw, &v); err != nil { t.Fatal(err) } if v.Account != "mesh" || v.Folder != "INBOX" || v.Subject != "Hi" || v.Size != 1234 { t.Fatalf("%+v", v) } if v.Sender.Name != "Bob" || v.Sender.Address != "bob@example.org" { t.Fatalf("sender: %+v", v.Sender) } if len(v.Recipients) != 1 || v.Recipients[0].Address != "ops@example.org" { t.Fatalf("recipients: %+v", v.Recipients) } if len(v.Attachments) != 1 || v.Attachments[0] != "invoice.pdf" { t.Fatalf("attachments: %+v", v.Attachments) } if len(v.Flags) != 1 || v.Flags[0] != `\Seen` { t.Fatalf("flags: %+v", v.Flags) } if strings.Contains(string(raw), "should never appear") { t.Fatalf("the value carries the body: %s", raw) } } // 1282/AC2: a kept mail whose flags change has its key rewritten; a round with nothing changed writes nothing. func TestMirrorRoundRewritesOnlyAChangedMailsKey(t *testing.T) { ctx := context.Background() accounts, st := accountsAndStore(t) a := Message{Account: "mesh", Folder: "INBOX", UID: 1, MessageID: "a@example.org", Date: judgedAt, Subject: "a"} b := Message{Account: "mesh", Folder: "INBOX", UID: 2, MessageID: "b@example.org", Date: judgedAt, Subject: "b"} if err := st.Add(ctx, a, nil); err != nil { t.Fatal(err) } if err := st.Add(ctx, b, nil); err != nil { t.Fatal(err) } _ = st.SetFolder(ctx, Folder{Account: "mesh", Name: "INBOX", Messages: 2}) acctsBucket, mailsBucket := newMemMirrorState(), newMemMirrorState() m := mirrorWithMails(accounts, st, acctsBucket, mailsBucket) if err := m.Round(ctx); err != nil { t.Fatal(err) } mailsBucket.calls = map[string]int{} // Nothing changed: a second round writes neither key again. if err := m.Round(ctx); err != nil { t.Fatal(err) } if mailsBucket.calls[mailKey("mesh", a)] != 0 || mailsBucket.calls[mailKey("mesh", b)] != 0 { t.Fatalf("a round with nothing changed wrote a mail key: %v", mailsBucket.calls) } // a's flags change: only a's key is rewritten. if err := st.SetFlags(ctx, "mesh", "INBOX", 1, []string{`\Seen`}, nil); err != nil { t.Fatal(err) } if err := m.Round(ctx); err != nil { t.Fatal(err) } if mailsBucket.calls[mailKey("mesh", a)] != 1 { t.Fatalf("a's key was not rewritten: %v", mailsBucket.calls) } if mailsBucket.calls[mailKey("mesh", b)] != 0 { t.Fatalf("b's key was written though nothing of it changed: %v", mailsBucket.calls) } } // 1282/AC3 (removed): a mail removed from the index has its key deleted. func TestMirrorRoundDeletesAMailsKeyForARemovedMail(t *testing.T) { ctx := context.Background() _, st := accountsAndStore(t) msg := Message{Account: "mesh", Folder: "INBOX", UID: 1, MessageID: "m@example.org", Date: judgedAt, Subject: "s"} if err := st.Add(context.Background(), msg, nil); err != nil { t.Fatal(err) } bucket := newMemMirrorState() mir := &Mirror{Mails: bucket} if err := mir.roundMails(ctx, st, "mesh", 200); err != nil { t.Fatal(err) } if _, ok, _ := bucket.Get(mailKey("mesh", msg)); !ok { t.Fatalf("the mail's key was not written") } if err := st.Remove(ctx, "mesh", "INBOX", 1, nil); err != nil { t.Fatal(err) } if err := mir.roundMails(ctx, st, "mesh", 200); err != nil { t.Fatal(err) } if _, ok, _ := bucket.Get(mailKey("mesh", msg)); ok { t.Fatalf("the removed mail's key was not deleted") } } // 1282/AC3 (pushed out): a mail pushed out of the newest kept by a newer one has its key deleted. func TestMirrorRoundDeletesAMailsKeyPushedOutOfTheNewestKept(t *testing.T) { ctx := context.Background() _, st := accountsAndStore(t) old := Message{Account: "mesh", Folder: "INBOX", UID: 1, MessageID: "old@example.org", Date: judgedAt.Add(-time.Hour), Subject: "old"} if err := st.Add(ctx, old, nil); err != nil { t.Fatal(err) } bucket := newMemMirrorState() mir := &Mirror{Mails: bucket} if err := mir.roundMails(ctx, st, "mesh", 1); err != nil { t.Fatal(err) } if _, ok, _ := bucket.Get(mailKey("mesh", old)); !ok { t.Fatalf("the old mail's key was not written") } newer := Message{Account: "mesh", Folder: "INBOX", UID: 2, MessageID: "new@example.org", Date: judgedAt, Subject: "new"} if err := st.Add(ctx, newer, nil); err != nil { t.Fatal(err) } if err := mir.roundMails(ctx, st, "mesh", 1); err != nil { t.Fatal(err) } if _, ok, _ := bucket.Get(mailKey("mesh", old)); ok { t.Fatalf("the old mail's key was not deleted once pushed out of the newest kept") } if _, ok, _ := bucket.Get(mailKey("mesh", newer)); !ok { t.Fatalf("the newer mail's key was not written") } } // 1282/AC4: a mail with 500 recipients of 254-byte addresses, 50 attachments of 255-byte names and a // 5,000-character multibyte subject is mirrored at most MostMailValue bytes, its subject at most // MostMailSubject, each address and attachment name at most MostFolderName, and the dropped counts are in it. func TestMirrorRoundTrimsAnOversizedMailValueToAtMost4KiB(t *testing.T) { ctx := context.Background() accounts, st := accountsAndStore(t) var to []Person for i := 0; i < 500; i++ { to = append(to, Person{Address: strings.Repeat("r", 240) + "@example.org"}) // 254 bytes } var atts []AttachmentRef for i := 0; i < 50; i++ { atts = append(atts, AttachmentRef{Key: "k" + itoa(int64(i)), Name: strings.Repeat("n", 255), Size: 10}) } subject := strings.Repeat("é", 5000) msg := Message{Account: "mesh", Folder: "INBOX", UID: 1, MessageID: "big@example.org", Date: judgedAt, From: []Person{{Address: "sender@example.org"}}, To: to, Subject: subject, Size: 99, Attachments: atts} if err := st.Add(ctx, msg, nil); err != nil { t.Fatal(err) } _ = st.SetFolder(ctx, Folder{Account: "mesh", Name: "INBOX", Messages: 1}) acctsBucket, mailsBucket := newMemMirrorState(), newMemMirrorState() m := mirrorWithMails(accounts, st, acctsBucket, mailsBucket) if err := m.Round(ctx); err != nil { t.Fatal(err) } raw, ok, err := mailsBucket.Get(mailKey("mesh", msg)) if err != nil || !ok { t.Fatalf("no value kept: %v %v", ok, err) } if len(raw) > MostMailValue { t.Fatalf("the value is %d bytes, more than %d", len(raw), MostMailValue) } var v MailValue if err := json.Unmarshal(raw, &v); err != nil { t.Fatal(err) } if len(v.Subject) > MostMailSubject { t.Fatalf("the subject is %d bytes, more than %d", len(v.Subject), MostMailSubject) } if len(v.Recipients) > MostMailShown { t.Fatalf("%d recipients shown, more than %d", len(v.Recipients), MostMailShown) } for _, r := range v.Recipients { if len(r.Address) > MostFolderName { t.Fatalf("a recipient's address is %d bytes, more than %d", len(r.Address), MostFolderName) } } for _, a := range v.Attachments { if len(a) > MostFolderName { t.Fatalf("an attachment name is %d bytes, more than %d", len(a), MostFolderName) } } if v.RecipientsDropped == 0 { t.Fatalf("recipients-dropped is 0, though 500 recipients were given") } if v.AttachmentsDropped == 0 { t.Fatalf("attachments-dropped is 0, though 50 attachments were given") } } // 1282/AC5: two mails without a message-id, in different folders or uids, digest apart (two keys); one // message-id kept in two folders of one account digests the same (one key). func TestMailDigestDistinguishesByFolderAndUIDAndMergesByMessageID(t *testing.T) { a := Message{Folder: "INBOX", UID: 1} b := Message{Folder: "INBOX", UID: 2} if mailDigest(a) == mailDigest(b) { t.Fatalf("two mails without a message-id, different uids, digest the same") } c := Message{Folder: "Archive", UID: 1} if mailDigest(a) == mailDigest(c) { t.Fatalf("two mails without a message-id, different folders, digest the same") } withID1 := Message{Folder: "INBOX", UID: 9, MessageID: "x@example.org"} withID2 := Message{Folder: "Sent", UID: 10, MessageID: "x@example.org"} if mailDigest(withID1) != mailDigest(withID2) { t.Fatalf("one message-id kept in two folders of the account digests apart: %q vs %q", mailDigest(withID1), mailDigest(withID2)) } } // 1282/AC6: with 82 accounts, every account keeps the same smaller number n of newest mails, with // 82 x n x MostMailValue at most MostMailBytes, and n is written in each account's own `accounts` value // (issue 1255's Kept field). func TestMirrorRoundUses81AccountRuleForTheKeptNumber(t *testing.T) { ctx := context.Background() var accounts []Account for i := 0; i < 82; i++ { accounts = append(accounts, Account{Name: "a" + itoa(int64(i)), Kind: KindIMAP, Address: "a@example.org"}) } st := newMemStore() for _, a := range accounts { if err := st.SetState(ctx, AccountState{Account: a.Name, State: AccountCurrent}); err != nil { t.Fatal(err) } } n := keptPerAccount(82) if n >= DefaultKeptMails { t.Fatalf("82 accounts did not reduce the kept number below %d: got %d", DefaultKeptMails, n) } if 82*n*MostMailValue > MostMailBytes { t.Fatalf("82 x %d x %d bytes is more than %d", n, MostMailValue, MostMailBytes) } acctsBucket, mailsBucket := newMemMirrorState(), newMemMirrorState() m := mirrorWithMails(accounts, st, acctsBucket, mailsBucket) if err := m.Round(ctx); err != nil { t.Fatal(err) } for _, a := range accounts { raw, ok, err := acctsBucket.Get(a.Name) if err != nil || !ok { t.Fatalf("%s: no value: %v %v", a.Name, ok, err) } var v AccountValue if err := json.Unmarshal(raw, &v); err != nil { t.Fatal(err) } if v.Kept != n { t.Fatalf("%s: kept %d, want %d", a.Name, v.Kept, n) } } }