From 2985122856f8414a0fa655b9ba70e6c0163482e3 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 19 Oct 2026 14:29:15 +0200 Subject: [PATCH] mesh-mail: mail_search's Query gains subject/body words, attachment type/name and read/unread, so a word in one field, an attachment or a flag can filter mails (issue 1229) --- modules/mesh-mail/cmd/mesh-mail/pgstore.go | 70 +++++++++ .../mesh-mail/cmd/mesh-mail/search_test.go | 144 +++++++++++++++++- modules/mesh-mail/cmd/mesh-mail/store.go | 97 ++++++++++++ 3 files changed, 310 insertions(+), 1 deletion(-) diff --git a/modules/mesh-mail/cmd/mesh-mail/pgstore.go b/modules/mesh-mail/cmd/mesh-mail/pgstore.go index f91f257d..f81fda04 100644 --- a/modules/mesh-mail/cmd/mesh-mail/pgstore.go +++ b/modules/mesh-mail/cmd/mesh-mail/pgstore.go @@ -403,6 +403,54 @@ func addressFilterSQL(column, param string) string { "lower(el->>'address') like '%%@' || %s)", column, param, param) } +// wordFilterSQL is 1229/AC5, AC6(words), AC9, AC12, AC13, AC15: one condition per word of the list, each a +// whole-word, case-insensitive match (Postgres's \y, its own word boundary) against column, so a hyphenated word +// such as "4711-882" is never matched by only its digits (AC12); joined with "and" ("all", the default) or "or" +// ("any"). Empty answers "" (no filter). The word itself always reaches SQL as param's parameter, never +// concatenated; only its \y anchors are fixed SQL text. +func wordFilterSQL(column string, words []string, any bool, param func(any) string) string { + if len(words) == 0 { + return "" + } + parts := make([]string, 0, len(words)) + for _, w := range words { + parts = append(parts, fmt.Sprintf(`%s ~* ('\y' || %s || '\y')`, column, param(quoteRegex(w)))) + } + op := " and " + if any { + op = " or " + } + return "(" + strings.Join(parts, op) + ")" +} + +// quoteRegex escapes a word for Postgres's regex operators (ARE, a superset of POSIX ERE): the word is always +// matched literally, never as a pattern of its own, mirroring Go's regexp.QuoteMeta used by memStore's own match. +var quoteRegexEscaper = strings.NewReplacer( + `\`, `\\`, ".", `\.`, "+", `\+`, "*", `\*`, "?", `\?`, "(", `\(`, ")", `\)`, + "[", `\[`, "]", `\]`, "{", `\{`, "}", `\}`, "^", `\^`, "$", `\$`, "|", `\|`) + +func quoteRegex(word string) string { return quoteRegexEscaper.Replace(word) } + +// globToLike turns a shell glob ("*" and "?") into a SQL LIKE pattern, escaping LIKE's own metacharacters first +// (issue 1229, AC5, AC9: an attachment name pattern such as "*.pdf"). +func globToLike(pattern string) string { + var b strings.Builder + for _, r := range pattern { + switch r { + case '*': + b.WriteByte('%') + case '?': + b.WriteByte('_') + case '%', '_', '\\': + b.WriteByte('\\') + b.WriteRune(r) + default: + b.WriteRune(r) + } + } + return b.String() +} + func (s *PgStore) Search(ctx context.Context, q Query) ([]Message, error) { limit := q.Limit if limit <= 0 { @@ -437,6 +485,28 @@ func (s *PgStore) Search(ctx context.Context, q Query) ([]Message, error) { if !q.Before.IsZero() { where = append(where, "date < "+param(q.Before)) } + if c := wordFilterSQL("subject", q.SubjectWords, q.SubjectWordsAny, param); c != "" { + where = append(where, c) + } + if c := wordFilterSQL("body", q.BodyWords, q.BodyWordsAny, param); c != "" { + where = append(where, c) + } + if q.HasAttachment { + where = append(where, "jsonb_array_length(attachments) > 0") + } + if q.AttachmentType != "" || q.AttachmentNamePattern != "" { + var conds []string + if q.AttachmentType != "" { + conds = append(conds, "lower(el->>'type') = "+param(strings.ToLower(q.AttachmentType))) + } + if q.AttachmentNamePattern != "" { + conds = append(conds, "lower(el->>'name') like "+param(globToLike(strings.ToLower(q.AttachmentNamePattern)))) + } + where = append(where, "exists (select 1 from jsonb_array_elements(attachments) el where "+strings.Join(conds, " or ")+")") + } + if q.Unread != nil { + where = append(where, fmt.Sprintf("(%s = any(flags)) = %s", param(`\Seen`), param(!*q.Unread))) + } sqlText := "select " + messageColumns + " from message where " + strings.Join(where, " and ") + " order by date desc limit " + param(limit) rows, err := s.pool.Query(ctx, sqlText, args...) diff --git a/modules/mesh-mail/cmd/mesh-mail/search_test.go b/modules/mesh-mail/cmd/mesh-mail/search_test.go index 1fdbe4b5..331ff044 100644 --- a/modules/mesh-mail/cmd/mesh-mail/search_test.go +++ b/modules/mesh-mail/cmd/mesh-mail/search_test.go @@ -50,6 +50,24 @@ func searchFixtures() []Message { {Account: "mesh", Folder: "INBOX", UID: 6, MessageID: "", InReplyTo: "", Subject: "Re: quick note", Date: at(12, 30), From: []Person{{Name: "Bob", Address: "bob@example.org"}}, To: []Person{{Address: "jschoubben@novox.be"}}, Flags: []string{}, Attachments: []AttachmentRef{}, Body: "sounds good, thanks for the note", Size: 400}, + // 1229/AC5: from a domain (google.com), has-attachment, name *.pdf and the word "factuur". + {Account: "mesh", Folder: "INBOX", UID: 7, MessageID: "", Subject: "the factuur for Q3", + Date: at(13, 0), From: []Person{{Name: "Erik", Address: "erik@google.com"}}, To: []Person{{Address: "jschoubben@novox.be"}}, + Flags: []string{}, Attachments: []AttachmentRef{{Bucket: "mail", Key: "a4", Size: 200, Type: "application/pdf", Name: "factuur.pdf"}}, + Body: "see the attached factuur, due on receipt", Size: 850}, + // 1229/AC12: a word with digits and a dash ("4711-882"), matched whole. + {Account: "mesh", Folder: "INBOX", UID: 8, MessageID: "", Subject: "order 4711-882 confirmed", + Date: at(14, 0), From: []Person{{Address: "ops@example.org"}}, To: []Person{{Address: "jschoubben@novox.be"}}, + Flags: []string{}, Attachments: []AttachmentRef{}, Body: "your order 4711-882 has shipped", Size: 300}, + // Control for 1229/AC12: "4711" and "882" appear as separate tokens, never hyphenated together. + {Account: "mesh", Folder: "INBOX", UID: 9, MessageID: "", Subject: "order 4711 shipped, item 882 noted", + Date: at(14, 5), From: []Person{{Address: "ops@example.org"}}, To: []Person{{Address: "jschoubben@novox.be"}}, + Flags: []string{}, Attachments: []AttachmentRef{}, Body: "item 4711 and item 882 are unrelated", Size: 300}, + // 1229/AC15 (the "any" word list): only "verzekering" is present, not "schade", with an attachment. + {Account: "mesh", Folder: "INBOX", UID: 10, MessageID: "", Subject: "verzekering aanvraag", + Date: at(14, 10), From: []Person{{Address: "broker@example.org"}}, To: []Person{{Address: "jschoubben@novox.be"}}, + Flags: []string{}, Attachments: []AttachmentRef{{Bucket: "mail", Key: "a5", Size: 150, Type: "application/pdf", Name: "polis.pdf"}}, + Body: "nieuwe polis aanvraag for verzekering", Size: 450}, // mesh/Sent --------------------------------------------------------------------------------------------------- {Account: "mesh", Folder: "Sent", UID: 1, MessageID: "", InReplyTo: "", Subject: "Re: October invoice", @@ -127,7 +145,7 @@ func TestSearchFiltersByAccountFolderAndOnOrAfter(t *testing.T) { if err != nil { t.Fatal(err) } - sameSet(t, got, "", "", "") + sameSet(t, got, "", "", "", "", "", "", "") }) } @@ -189,3 +207,127 @@ func TestSearchFiltersByCcNotTo(t *testing.T) { sameSet(t, got, "") }) } + +// 1229/AC6 (words part): cc and the word "renovatie" in subject or body, combined with the cc filter +// from 1228: the mail with it only in to is still excluded. +func TestSearchFiltersByCcAndSubjectOrBodyWord(t *testing.T) { + eachStore(t, func(t *testing.T, st Store) { + ctx := context.Background() + addFixtures(t, ctx, st) + got, err := st.Search(ctx, Query{Accounts: []string{"mesh"}, Cc: "jschoubben@novox.be", BodyWords: []string{"renovatie"}, Limit: 10}) + if err != nil { + t.Fatal(err) + } + sameSet(t, got, "") + }) +} + +// 1229/AC5: from a domain, has-attachment, an attachment name pattern and a word in subject or body all combine. +func TestSearchFiltersByDomainAttachmentAndWord(t *testing.T) { + eachStore(t, func(t *testing.T, st Store) { + ctx := context.Background() + addFixtures(t, ctx, st) + got, err := st.Search(ctx, Query{Accounts: []string{"mesh"}, From: "google.com", HasAttachment: true, + AttachmentNamePattern: "*.pdf", BodyWords: []string{"factuur"}, Limit: 10}) + if err != nil { + t.Fatal(err) + } + // carol@google.com's security-alert mail (uid 4) is from the same domain but has neither an attachment nor + // the word, proving every one of the four conditions is enforced, not just one. + sameSet(t, got, "") + }) +} + +// 1229/AC9: on-or-after a date and an attachment type or name pattern (either suffices) answers a calendar invite. +func TestSearchFiltersByDateAndAttachmentTypeOrName(t *testing.T) { + eachStore(t, func(t *testing.T, st Store) { + ctx := context.Background() + addFixtures(t, ctx, st) + got, err := st.Search(ctx, Query{Accounts: []string{"mesh"}, OnOrAfter: time.Date(2026, 10, 19, 0, 0, 0, 0, time.UTC), + AttachmentType: "text/calendar", AttachmentNamePattern: "*.ics", Limit: 10}) + if err != nil { + t.Fatal(err) + } + sameSet(t, got, "") + }) +} + +// 1229/AC12: a word with digits and a dash is matched whole, not as its separate parts. +func TestSearchFiltersBySubjectOrBodyWordWhole(t *testing.T) { + eachStore(t, func(t *testing.T, st Store) { + ctx := context.Background() + addFixtures(t, ctx, st) + got, err := st.Search(ctx, Query{Accounts: []string{"mesh", "gmail"}, BodyWords: []string{"4711-882"}, Limit: 10}) + if err != nil { + t.Fatal(err) + } + sameSet(t, got, "") + }) +} + +// 1229/AC13: from a domain and the subject words (default "all": every word of the list), across every granted +// account, each answer naming its own account. +func TestSearchFiltersByDomainAndSubjectWordsAcrossAccounts(t *testing.T) { + eachStore(t, func(t *testing.T, st Store) { + ctx := context.Background() + addFixtures(t, ctx, st) + got, err := st.Search(ctx, Query{Accounts: []string{"mesh", "gmail"}, From: "google.com", + SubjectWords: []string{"security", "alert"}, Limit: 10}) + if err != nil { + t.Fatal(err) + } + sameSet(t, got, "", "") + accounts := map[string]bool{} + for _, m := range got { + accounts[m.Account] = true + } + if !accounts["mesh"] || !accounts["gmail"] { + t.Fatalf("each answer should name its own account: %+v", got) + } + }) +} + +// 1229/AC15: a word list marked "any" (schade or verzekering) combined with has-attachment; the word list's "any" +// mode matches a mail holding only one of the words, not both. +func TestSearchFiltersByAnyWordsAndHasAttachment(t *testing.T) { + eachStore(t, func(t *testing.T, st Store) { + ctx := context.Background() + addFixtures(t, ctx, st) + got, err := st.Search(ctx, Query{Accounts: []string{"mesh", "gmail"}, BodyWords: []string{"schade", "verzekering"}, + BodyWordsAny: true, HasAttachment: true, Limit: 10}) + if err != nil { + t.Fatal(err) + } + sameSet(t, got, "", "") + + // Without "any" (the default, "all"), only the mail holding both words together matches: polis1@mesh has + // "verzekering" alone, so it drops out. + all, err := st.Search(ctx, Query{Accounts: []string{"mesh", "gmail"}, BodyWords: []string{"schade", "verzekering"}, + HasAttachment: true, Limit: 10}) + if err != nil { + t.Fatal(err) + } + sameSet(t, all, "") + }) +} + +// 1229 (unread/read, from 1228/AC2): the flag filter, read or unread, independent of the other filters. +func TestSearchFiltersByUnreadFlag(t *testing.T) { + eachStore(t, func(t *testing.T, st Store) { + ctx := context.Background() + addFixtures(t, ctx, st) + unread := true + got, err := st.Search(ctx, Query{Accounts: []string{"mesh"}, Folder: "INBOX", Unread: &unread, Limit: 20}) + if err != nil { + t.Fatal(err) + } + sameSet(t, got, "", "", "", "", "", "") + + read := false + gotRead, err := st.Search(ctx, Query{Accounts: []string{"mesh"}, Folder: "INBOX", Unread: &read, Limit: 20}) + if err != nil { + t.Fatal(err) + } + sameSet(t, gotRead, "", "", "", "") + }) +} diff --git a/modules/mesh-mail/cmd/mesh-mail/store.go b/modules/mesh-mail/cmd/mesh-mail/store.go index a24a4eda..f4e53990 100644 --- a/modules/mesh-mail/cmd/mesh-mail/store.go +++ b/modules/mesh-mail/cmd/mesh-mail/store.go @@ -6,6 +6,8 @@ package main import ( "context" + "path" + "regexp" "sort" "strconv" "strings" @@ -101,6 +103,28 @@ type Query struct { // inclusive, before is exclusive. A zero time means no bound. OnOrAfter time.Time Before time.Time + + // SubjectWords and BodyWords (issue 1229, AC5, AC6's words part, AC12, AC13, AC15) are each a whole word, + // matched case-insensitively against Subject or Body respectively (a word with digits and a dash, such as + // "4711-882", is matched whole, never by its separate parts, AC12). By default every word of the list must be + // found ("all"); the *WordsAny flag switches that list to "any of them" (AC15). Empty means no filter. + SubjectWords []string + SubjectWordsAny bool + BodyWords []string + BodyWordsAny bool + + // HasAttachment, AttachmentType and AttachmentNamePattern (issue 1229, AC5, AC9) filter by attachment: + // HasAttachment requires at least one; AttachmentType (an exact media type, case-insensitive) and + // AttachmentNamePattern (a shell glob such as "*.pdf", case-insensitive) each require one attachment matching + // it, but when both are given, either suffices for the same attachment (AC9: "type text/calendar or name + // *.ics", since a calendar invite may carry either). Empty/false means no filter. + HasAttachment bool + AttachmentType string + AttachmentNamePattern string + + // Unread (issue 1229, carrying 1228/AC2's flag part) is nil for no filter; true answers only messages without + // the \Seen flag, false only those with it. + Unread *bool } // Store is the index. @@ -260,6 +284,17 @@ func (s *memStore) Search(_ context.Context, q Query) ([]Message, error) { if !q.Before.IsZero() && !m.Date.Before(q.Before) { continue } + // 1229/AC5, 1229/AC6(words), 1229/AC9, 1229/AC12, 1229/AC13, 1229/AC15: the word, attachment and + // read/unread filters, mirroring PgStore.Search. + if !wordMatches(m.Subject, q.SubjectWords, q.SubjectWordsAny) || !wordMatches(m.Body, q.BodyWords, q.BodyWordsAny) { + continue + } + if !attachmentMatches(m.Attachments, q.HasAttachment, q.AttachmentType, q.AttachmentNamePattern) { + continue + } + if q.Unread != nil && hasSeenFlag(m.Flags) == *q.Unread { + continue + } hay := strings.ToLower(m.Subject + " " + m.Body + " " + peopleText(m.From) + " " + peopleText(m.To) + " " + peopleText(m.Cc)) all := true for _, w := range words { @@ -358,6 +393,68 @@ func addressMatches(ps []Person, filter string) bool { return false } +// wordRegexp is a whole-word, case-insensitive match (issue 1229, AC12): \b, Go's RE2 word boundary, treats a +// dash as a non-word character, so a hyphenated word such as "4711-882" is bounded at both ends and never matched +// by only its digits. +func wordRegexp(word string) *regexp.Regexp { + return regexp.MustCompile(`(?i)\b` + regexp.QuoteMeta(word) + `\b`) +} + +// wordMatches is 1229/AC5, AC6(words), AC9, AC12, AC13, AC15: an empty list matches every message; otherwise +// every word of the list must be found in text ("all"), or at least one when any is true. +func wordMatches(text string, words []string, any bool) bool { + if len(words) == 0 { + return true + } + for _, w := range words { + found := wordRegexp(w).MatchString(text) + if any && found { + return true + } + if !any && !found { + return false + } + } + return !any +} + +// attachmentMatches is 1229/AC5, AC9: hasAttachment requires at least one attachment; typ and namePattern, when +// either is given, each require one attachment matching it, but when both are given either suffices for the same +// attachment (AC9's "type text/calendar or name *.ics"). +func attachmentMatches(atts []AttachmentRef, hasAttachment bool, typ, namePattern string) bool { + if hasAttachment && len(atts) == 0 { + return false + } + if typ == "" && namePattern == "" { + return true + } + for _, a := range atts { + if typ != "" && strings.EqualFold(a.Type, typ) { + return true + } + if namePattern != "" && globMatches(namePattern, a.Name) { + return true + } + } + return false +} + +// globMatches is 1229/AC5, AC9: a shell glob such as "*.pdf" or "*.ics", case-insensitive. +func globMatches(pattern, name string) bool { + ok, _ := path.Match(strings.ToLower(pattern), strings.ToLower(name)) + return ok +} + +// hasSeenFlag is 1229 (carrying 1228/AC2's flag part): whether the message is read, by IMAP's own \Seen flag. +func hasSeenFlag(flags []string) bool { + for _, f := range flags { + if strings.EqualFold(f, `\Seen`) { + return true + } + } + return false +} + func peopleText(ps []Person) string { var parts []string for _, p := range ps {