mesh-catalog/modules/mesh-mail/cmd/mesh-mail/imap_test.go

package main

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/emersion/go-imap/v2"
)

// keeping starts a keeper on one account of the test server, its paces short, and stops it when the test ends.
func keeping(t *testing.T, s *mailServer, st Store, objects Objects, secrets string) {
	t.Helper()
	k := &Keeper{Store: st, Objects: objects, Dial: s.dialer(), SecretsDir: secrets, Logf: t.Logf,
		IdleCycle: 300 * time.Millisecond, PollEvery: 400 * time.Millisecond, RetryFirst: 100 * time.Millisecond}
	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan struct{})
	go func() { k.Watch(ctx, testAccount()); close(done) }()
	t.Cleanup(func() { cancel(); <-done })
}

func stateOf(st Store, account string) AccountState {
	states, _ := st.States(context.Background())
	return states[account]
}

// float64OrUint reads a uid from an event body whether it came from memory (uint32) or from postgres (float64).
type float64OrUint struct{ v any }

func (f float64OrUint) value() uint32 {
	switch n := f.v.(type) {
	case uint32:
		return n
	case float64:
		return uint32(n)
	}
	return 0
}

func uidOf(e Event) uint32 { return float64OrUint{e.Body["uid"]}.value() }

func eventsNamed(st Store, key string) []Event {
	evs, _ := st.Pending(context.Background(), 1000)
	var out []Event
	for _, e := range evs {
		if e.Key == key {
			out = append(out, e)
		}
	}
	return out
}

func describe(st Store) string {
	evs, _ := st.Pending(context.Background(), 1000)
	var keys []string
	for _, e := range evs {
		keys = append(keys, e.Key)
	}
	s := stateOf(st, "mesh")
	return "state " + s.State + " (" + s.Why + "); events " + strings.Join(keys, ", ")
}

// The whole watch, over a real IMAP conversation (ADR 0359 §2, §3, §5): the first look indexes what is there and says
// nothing; then a new mail in the inbox is said within moments of its delivery, through IDLE, with its metadata and
// its attachment by reference, never its body; its flags changed are said; its removal is said; a mail in another
// folder is said at the next poll; and watching never marks a message seen.
func TestTheKeeperSaysEachChangeOnTheBusAndOnlyReads(t *testing.T) {
	eachStore(t, func(t *testing.T, st Store) {
		s := newMailServer(t)
		op := s.operator(t)
		deliver(t, op, "INBOX", plainMail("before the watch", "an old letter"))
		deliver(t, op, "Archive", plainMail("archived long ago", "an older letter"))
		secrets := t.TempDir()
		giveSecret(t, secrets, testAccount(), testPassword)
		objects := newMemObjects()
		keeping(t, s, st, objects, secrets)

		eventually(t, 10*time.Second, "the first look", func() (bool, string) {
			return stateOf(st, "mesh").State == AccountCurrent, describe(st)
		})
		ctx := context.Background()
		if hits, _ := st.Search(ctx, Query{Text: "letter", Accounts: []string{"mesh"}}); len(hits) != 2 {
			t.Fatalf("the first look indexed %d messages, not 2", len(hits))
		}
		if evs, _ := st.Pending(ctx, 100); len(evs) != 0 {
			t.Fatalf("the first look said %d event(s): the history of a mailbox is not news", len(evs))
		}

		// A new mail with an attachment, delivered while the keeper idles on the inbox.
		deliver(t, op, "INBOX", attachedMail("the invoice"))
		var added Event
		eventually(t, 5*time.Second, "the new mail said", func() (bool, string) {
			evs := eventsNamed(st, "mail.mesh.message.added")
			if len(evs) == 1 {
				added = evs[0]
			}
			return len(evs) == 1, describe(st)
		})
		body := added.Body
		if body["subject"] != "the invoice" || body["folder"] != "INBOX" || body["account"] != "mesh" || body["message-id"] != "att@example.org" {
			t.Fatalf("the event says %v", body)
		}
		raw, _ := json.Marshal(body)
		for _, never := range []string{"Please find", "dGhlIGludm9pY2Ug", attachmentData} {
			if strings.Contains(string(raw), never) {
				t.Fatalf("the event carries the body or an attachment's bytes (%q): %s", never, raw)
			}
		}
		sum := sha256.Sum256([]byte(attachmentData))
		key := hex.EncodeToString(sum[:])
		if !strings.Contains(string(raw), `"key":"`+key+`"`) || !strings.Contains(string(raw), `"name":"invoice.pdf"`) {
			t.Fatalf("the event does not name the attachment by its SHA-256 %s: %s", key, raw)
		}
		if kept, _ := objects.Get(ctx, key); string(kept) != attachmentData {
			t.Fatalf("the attachment store keeps %q under %s", kept, key)
		}
		uid := uidOf(added)
		m, ok, _ := st.Get(ctx, "mesh", "INBOX", uid)
		if !ok || !strings.Contains(m.Body, "Please find the invoice attached.") {
			t.Fatalf("the index keeps the text for mail_read: %+v", m)
		}

		// Flagged by the operator elsewhere: said as changed, with the flags before and after.
		if _, err := op.Select("INBOX", nil).Wait(); err != nil {
			t.Fatal(err)
		}
		if err := op.Store(imap.UIDSetNum(imap.UID(uid)), &imap.StoreFlags{Op: imap.StoreFlagsAdd, Flags: []imap.Flag{imap.FlagFlagged}, Silent: true}, nil).Close(); err != nil {
			t.Fatal(err)
		}
		eventually(t, 5*time.Second, "the flag said", func() (bool, string) {
			evs := eventsNamed(st, "mail.mesh.message.changed")
			if len(evs) != 1 {
				return false, describe(st)
			}
			raw, _ := json.Marshal(evs[0].Body["changes"])
			return strings.Contains(string(raw), `"after":["\\Flagged"]`), string(raw)
		})

		// Never marked seen by the watching: the message is still unseen on the server.
		fetched, err := op.Fetch(imap.UIDSetNum(imap.UID(uid)), &imap.FetchOptions{Flags: true, UID: true}).Collect()
		if err != nil || len(fetched) != 1 {
			t.Fatalf("%v %v", fetched, err)
		}
		for _, f := range fetched[0].Flags {
			if f == imap.FlagSeen {
				t.Fatal("watching marked the message seen")
			}
		}

		// Deleted by the operator elsewhere: said as removed.
		if err := op.Store(imap.UIDSetNum(imap.UID(uid)), &imap.StoreFlags{Op: imap.StoreFlagsAdd, Flags: []imap.Flag{imap.FlagDeleted}, Silent: true}, nil).Close(); err != nil {
			t.Fatal(err)
		}
		if err := op.Expunge().Close(); err != nil {
			t.Fatal(err)
		}
		eventually(t, 5*time.Second, "the removal said", func() (bool, string) {
			evs := eventsNamed(st, "mail.mesh.message.removed")
			return len(evs) == 1 && uidOf(evs[0]) == uid && evs[0].Body["message-id"] == "att@example.org", describe(st)
		})
		if _, ok, _ := st.Get(ctx, "mesh", "INBOX", uid); ok {
			t.Fatal("the removed message is still indexed")
		}

		// A mail in another folder is said at the next poll.
		deliver(t, op, "Archive", plainMail("filed away", "a letter filed"))
		eventually(t, 5*time.Second, "the archived mail said", func() (bool, string) {
			for _, e := range eventsNamed(st, "mail.mesh.message.added") {
				if e.Body["folder"] == "Archive" && e.Body["subject"] == "filed away" {
					return true, ""
				}
			}
			return false, describe(st)
		})
		if s := stateOf(st, "mesh"); s.State != AccountCurrent || time.Since(s.Heard) > time.Minute {
			t.Fatalf("the account is %+v", s)
		}
	})
}

// No password given: the account waits, and nothing is dialled. A wrong one: the account fails, saying the server
// refused the sign-in. Given later: it is picked up without a restart.
func TestTheKeeperWaitsForThePasswordAndSaysARefusal(t *testing.T) {
	s := newMailServer(t)
	st := newMemStore()
	secrets := t.TempDir()
	keeping(t, s, st, newMemObjects(), secrets)
	eventually(t, 5*time.Second, "the wait", func() (bool, string) {
		got := stateOf(st, "mesh")
		return got.State == AccountWaits && strings.Contains(got.Why, "mail-credential-mesh"), describe(st)
	})
	giveSecret(t, secrets, testAccount(), "not-the-password")
	eventually(t, 5*time.Second, "the refusal", func() (bool, string) {
		got := stateOf(st, "mesh")
		return got.State == AccountFailing && strings.Contains(got.Why, "refused the sign-in"), describe(st)
	})
	giveSecret(t, secrets, testAccount(), testPassword)
	eventually(t, 10*time.Second, "the sign-in", func() (bool, string) {
		return stateOf(st, "mesh").State == AccountCurrent, describe(st)
	})
}

// It only reads a mailbox (ADR 0359 §5): no code of the module, but its tests, issues a command that flags, moves,
// copies, deletes, appends, creates, renames or otherwise writes to one over IMAP, and every folder is opened
// read-only and every body peeked. mail_send is the one exception the operator's own Approve carves out (ADR 0361
// §2): it never touches a mailbox either — it speaks SMTP, in smtp.go alone, to the account's outgoing server, and
// only once the operator approved it (send.go).
func TestNothingButTheTestsWritesToAMailbox(t *testing.T) {
	files, err := filepath.Glob("*.go")
	if err != nil {
		t.Fatal(err)
	}
	writes := regexp.MustCompile(`([A-Za-z_][A-Za-z0-9_]*)\.(Store|Move|Copy|Expunge|UIDExpunge|Append|Create|Delete|Rename|Subscribe|Unsubscribe|SetMetadata|SetQuota)\(`)
	for _, f := range files {
		if strings.HasSuffix(f, "_test.go") || f == "send.go" || f == "smtp.go" {
			// Neither touches a mailbox: send.go keeps the `sends` state on the bus, never IMAP; smtp.go speaks SMTP
			// to the account's outgoing server, never to a mailbox, and only on the operator's Approve (ADR 0361 §2).
			continue
		}
		raw, err := os.ReadFile(f)
		if err != nil {
			t.Fatal(err)
		}
		for _, m := range writes.FindAllSubmatch(raw, -1) {
			// os is a file of the module's own, never a mailbox; stdio is the bus SDK (main.go's hear subscribes to
			// the router's own decided subject, exactly as mesh-knowledge does, novox/hq issue 790 review round 1)
			// — its Subscribe has nothing to do with an IMAP mailbox's.
			if string(m[1]) != "os" && string(m[1]) != "stdio" {
				t.Errorf("%s calls %s, which writes to a mailbox", f, m[0])
			}
		}
		if strings.Contains(string(raw), "net/smtp") {
			t.Errorf("%s speaks SMTP: only smtp.go does, and only on the operator's Approve (ADR 0361 §2)", f)
		}
	}
	src, _ := os.ReadFile("imap.go")
	if !strings.Contains(string(src), "&imap.SelectOptions{ReadOnly: true}") || !strings.Contains(string(src), "{{Peek: true}}") {
		t.Fatal("folders are not opened read-only, or bodies not peeked")
	}
}

// A fetch of bodies is bounded by bytes (issue 733): batches cut by size and by count, and a message larger than a
// batch holds alone.
func TestBatchesAreBoundedByBytesAndALargerMessageGoesAlone(t *testing.T) {
	sizes := map[uint32]int64{1: 40, 2: 40, 3: 500, 4: 10, 5: 10, 6: 10}
	got := Batches([]uint32{1, 2, 3, 4, 5, 6}, sizes, 100, 2)
	want := [][]uint32{{1, 2}, {3}, {4, 5}, {6}}
	if fmt.Sprint(got) != fmt.Sprint(want) {
		t.Fatalf("%v, not %v", got, want)
	}
	if got := Batches(nil, nil, 100, 25); len(got) != 0 {
		t.Fatalf("%v", got)
	}
}

// Over a real conversation: with a cap of a few hundred bytes, a large mail is fetched alone and said so, and every
// mail is still indexed and said.
func TestTheKeeperFetchesUnderAByteCap(t *testing.T) {
	s := newMailServer(t)
	op := s.operator(t)
	st := newMemStore()
	secrets := t.TempDir()
	giveSecret(t, secrets, testAccount(), testPassword)
	var mu sync.Mutex
	var said []string
	k := &Keeper{Store: st, Objects: newMemObjects(), Dial: s.dialer(), SecretsDir: secrets,
		Logf:      func(f string, a ...any) { mu.Lock(); said = append(said, fmt.Sprintf(f, a...)); mu.Unlock() },
		IdleCycle: 300 * time.Millisecond, PollEvery: 400 * time.Millisecond, RetryFirst: 100 * time.Millisecond, BatchBytes: 600}
	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan struct{})
	go func() { k.Watch(ctx, testAccount()); close(done) }()
	t.Cleanup(func() { cancel(); <-done })
	eventually(t, 10*time.Second, "the first look", func() (bool, string) { return stateOf(st, "mesh").State == AccountCurrent, describe(st) })
	deliver(t, op, "INBOX", plainMail("small one", "short"))
	deliver(t, op, "INBOX", plainMail("large one", strings.Repeat("a long letter ", 200)))
	deliver(t, op, "INBOX", plainMail("small two", "short"))
	eventually(t, 5*time.Second, "three mails said", func() (bool, string) {
		return len(eventsNamed(st, "mail.mesh.message.added")) == 3, describe(st)
	})
	mu.Lock()
	defer mu.Unlock()
	alone := 0
	for _, line := range said {
		if strings.Contains(line, "fetched alone") {
			alone++
		}
	}
	if alone != 1 {
		t.Fatalf("said %d message(s) fetched alone: %v", alone, said)
	}
}