From a9ff48e200777d1aa2e32e8fb74bb2fc92244d30 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 19 Oct 2026 21:44:12 +0200 Subject: [PATCH 01/18] Answer a send's state only to its caller (issue 1261) --- .../mesh-mail/cmd/mesh-mail/manifest_test.go | 6 +- modules/mesh-mail/cmd/mesh-mail/send.go | 33 ++-- modules/mesh-mail/cmd/mesh-mail/send_test.go | 160 ++++++++++++++++++ modules/mesh-mail/cmd/mesh-mail/tools.go | 20 ++- modules/mesh-mail/module.json | 7 +- 5 files changed, 204 insertions(+), 22 deletions(-) diff --git a/modules/mesh-mail/cmd/mesh-mail/manifest_test.go b/modules/mesh-mail/cmd/mesh-mail/manifest_test.go index c4031ed9..5673e945 100644 --- a/modules/mesh-mail/cmd/mesh-mail/manifest_test.go +++ b/modules/mesh-mail/cmd/mesh-mail/manifest_test.go @@ -94,7 +94,7 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { } wantTools := []string{ "mail_accounts", "mail_search", "mail_read", "mail_health", "mail_send", "mail_send_status", - "mail-store.accounts", "mail-store.search", "mail-store.read", "mail-store.send", + "mail-store.accounts", "mail-store.search", "mail-store.read", "mail-store.send", "mail-store.send-status", } if !reflect.DeepEqual(m.Tools, names) || !reflect.DeepEqual(names, wantTools) { t.Fatalf("tools %v, the code serves %v", m.Tools, names) @@ -140,7 +140,7 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { // mail-webclient each held once for the mesh, mail-notifier held once per machine and needing node-notifier // there — and claims only mail-store, serving the same four verbs as its own tools (§2, §3). wantSeats := []seat{ - {Name: "mail-store", Scope: "mesh", Serves: []string{"accounts", "search", "read", "send"}}, + {Name: "mail-store", Scope: "mesh", Serves: []string{"accounts", "search", "read", "send", "send-status"}}, {Name: "mail-webclient", Scope: "mesh", Serves: []string{"open", "status"}}, {Name: "mail-notifier", Scope: "node", Serves: []string{"status"}, Needs: []string{"node-notifier"}}, } @@ -148,7 +148,7 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { t.Fatalf("seats %+v, want %+v", m.Seats, wantSeats) } wantClaims := []seat{ - {Name: "mail-store", Scope: "mesh", Serves: []string{"accounts", "search", "read", "send"}}, + {Name: "mail-store", Scope: "mesh", Serves: []string{"accounts", "search", "read", "send", "send-status"}}, } if !reflect.DeepEqual(m.Claims, wantClaims) { t.Fatalf("claims %+v, want %+v", m.Claims, wantClaims) diff --git a/modules/mesh-mail/cmd/mesh-mail/send.go b/modules/mesh-mail/cmd/mesh-mail/send.go index 66d130d3..6dbbb3fe 100644 --- a/modules/mesh-mail/cmd/mesh-mail/send.go +++ b/modules/mesh-mail/cmd/mesh-mail/send.go @@ -1312,19 +1312,30 @@ func tellWaits(store SendStore, tell TellOperator, id string, now time.Time) err // ---- reading the status ----------------------------------------------------------------------------------------- -// ReadSendStatus is where a send asked with mail_send stands, by its ask id: waiting for the operator, sent, -// declined, ended without a choice (expired, refused, cancelled), or not proven. Read from the router's own -// record; when the router's word is a close one that was not decided yet, this is also where a send is first -// decided (and so sent), exactly once (Decide). -func ReadSendStatus(store SendStore, bus AskBus, publish QueuePublish, id string, now time.Time) (map[string]any, error) { +// ReadSendStatus is where a send asked with mail_send stands, answered only to the caller that asked for it +// (novox/hq ADR 0440 §7, issue 1261): an id no send is kept under, or a send whose own caller differs from the +// one asking, answers status "unknown" and nothing else of it — never an error, and never the account, +// recipients or subject of a send that is not the caller's own. The sweep, which settles a send regardless of +// who asked for it, calls readSendStatus directly instead, never through this caller check. +func ReadSendStatus(store SendStore, bus AskBus, publish QueuePublish, id, caller string, now time.Time) (map[string]any, error) { trimmed := strings.TrimSpace(id) req, rev, ok, err := getSend(store, trimmed) if err != nil { return nil, fmt.Errorf("the send %q cannot be read now (%v); ask again", trimmed, err) } - if !ok { - return nil, fmt.Errorf("no send %q is kept: the id is the one mail_send gave", trimmed) + if !ok || req.Caller != caller { + return map[string]any{"status": "unknown"}, nil } + return readSendStatus(store, bus, publish, req, rev, now) +} + +// readSendStatus is ReadSendStatus's own reading of one send already fetched (req, rev): waiting for the +// operator, sent, declined, ended without a choice (expired, refused, cancelled), or not proven. Read from the +// router's own record; when the router's word is a close one that was not decided yet, this is also where a +// send is first decided (and so sent), exactly once (Decide). It never checks who is asking: ReadSendStatus +// does that before calling it, and the sweep (which settles a send no matter who asked for it) calls it +// directly. +func readSendStatus(store SendStore, bus AskBus, publish QueuePublish, req SendRequest, rev uint64, now time.Time) (map[string]any, error) { out := map[string]any{"ask": req.ID, "account": req.Account, "to": req.To, "subject": req.Subject, "asked": req.At.Format(time.RFC3339)} if req.Decided != nil { @@ -1342,7 +1353,7 @@ func ReadSendStatus(store SendStore, bus AskBus, publish QueuePublish, id string if now.Sub(req.At) > SendAskLasts { d, _, err := settleSend(store, req, rev, SendDecision{At: now, Outcome: "ended: never heard"}) if err != nil { - return nil, fmt.Errorf("the send %q could not be settled as ended (%v); ask again", trimmed, err) + return nil, fmt.Errorf("the send %q could not be settled as ended (%v); ask again", req.ID, err) } reportDecision(out, d, now) return out, nil @@ -1448,11 +1459,13 @@ func (b *Bundle) Sweep(now time.Time) { } } else { for _, id := range undecided { - req, _, ok, err := getSend(store, id) + req, rev, ok, err := getSend(store, id) if err != nil || !ok || req.Decided != nil { continue } - if _, err := ReadSendStatus(store, b.askBus(), b.queuePublish(), id, now); err != nil { + // 1261/AC6: the sweep settles a send regardless of who asked for it — readSendStatus directly, + // never ReadSendStatus's caller check. + if _, err := readSendStatus(store, b.askBus(), b.queuePublish(), req, rev, now); err != nil { if b.Logf != nil { b.Logf("[mesh-mail] sweeping the send %s: %v", id, err) } diff --git a/modules/mesh-mail/cmd/mesh-mail/send_test.go b/modules/mesh-mail/cmd/mesh-mail/send_test.go index 36939403..ea27bbc4 100644 --- a/modules/mesh-mail/cmd/mesh-mail/send_test.go +++ b/modules/mesh-mail/cmd/mesh-mail/send_test.go @@ -861,3 +861,163 @@ func TestADigestMismatchIsRefusedAndNothingIsQueued(t *testing.T) { t.Fatalf("the queue received %d message(s) despite the digest mismatch", n) } } + +// ---- mail_send_status answers only its caller (novox/hq ADR 0440 §7, issue 1261) --------------------------------- + +// 1261/AC1: the caller that made a send reads its own status in full, with its account, recipients and subject +// as before. +func TestSendStatusAnswersTheCallersOwnSendInFull(t *testing.T) { + b, _, _ := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"}) + rt := startBundle(t, b.Tools()) + + text, isErr := rt.call(SendTool, "novox.triage", sendArgs) + if isErr { + t.Fatalf("%s", text) + } + var out map[string]any + _ = json.Unmarshal([]byte(text), &out) + id := out["ask"].(string) + + text, isErr = rt.call(SendStatusTool, "novox.triage", map[string]any{"id": id}) + if isErr { + t.Fatalf("%s", text) + } + var status map[string]any + if err := json.Unmarshal([]byte(text), &status); err != nil { + t.Fatal(err) + } + if status["status"] != "waiting" || status["account"] != "mesh" || status["subject"] != sendArgs["subject"] { + t.Fatalf("the caller's own send: %v", status) + } + to, _ := status["to"].([]any) + if len(to) != 2 { + t.Fatalf("to: %v", status["to"]) + } +} + +// 1261/AC2: a caller other than the one that made the send reads status unknown and nothing else of it — no +// account, recipients or subject, and no error either. +func TestSendStatusAnswersUnknownAndNothingElseToAnotherCaller(t *testing.T) { + b, _, _ := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"}) + rt := startBundle(t, b.Tools()) + + text, isErr := rt.call(SendTool, "novox.triage", sendArgs) + if isErr { + t.Fatalf("%s", text) + } + var out map[string]any + _ = json.Unmarshal([]byte(text), &out) + id := out["ask"].(string) + + text, isErr = rt.call(SendStatusTool, "novox.someone-else", map[string]any{"id": id}) + if isErr { + t.Fatalf("another caller's read of the send was an error: %s", text) + } + var status map[string]any + if err := json.Unmarshal([]byte(text), &status); err != nil { + t.Fatal(err) + } + if len(status) != 1 || status["status"] != "unknown" { + t.Fatalf("another caller's read of the send: %v, want only status unknown", status) + } +} + +// 1261/AC3: an id no send is kept under answers status unknown and no error, for any caller. +func TestSendStatusAnswersUnknownForAnUnknownIDWithoutAnError(t *testing.T) { + b, _, _ := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"}) + rt := startBundle(t, b.Tools()) + + text, isErr := rt.call(SendStatusTool, "novox.triage", map[string]any{"id": "send-0000000000000000"}) + if isErr { + t.Fatalf("an unknown id was an error: %s", text) + } + var status map[string]any + if err := json.Unmarshal([]byte(text), &status); err != nil { + t.Fatal(err) + } + if len(status) != 1 || status["status"] != "unknown" { + t.Fatalf("%v", status) + } +} + +// 1261/AC4: every kept decision's status word is one of the fixed vocabulary (waiting, sent, failed, refused, +// not proven, declined, expired), answered to the send's own caller. +func TestSendStatusWordsAreTheFixedVocabularyForTheCallersOwnSend(t *testing.T) { + sends := newMemSends() + const caller = "novox.triage" + cases := []struct { + name, outcome, want string + }{ + {"queued", "queued", "waiting"}, + {"sending", "sending", "waiting"}, + {"retrying", "retrying", "waiting"}, + {"sent", "sent", "sent"}, + {"declined", "declined", "declined"}, + {"not-proven", "not proven", "not proven"}, + {"failed", "failed: boom", "failed"}, + {"refused", "refused: nope", "refused"}, + {"ended-never-heard", "ended: never heard", "expired"}, + {"ended-other", "ended: cancelled", "expired"}, + } + for _, c := range cases { + id := "send-" + c.name + req := SendRequest{ID: id, Account: "mesh", To: []string{"ops@example.org"}, Subject: "s", Caller: caller, + At: judgedAt, Decided: &SendDecision{At: judgedAt, Outcome: c.outcome}} + if _, err := sends.Create(id, req); err != nil { + t.Fatalf("%s: keeping: %v", c.name, err) + } + out, err := ReadSendStatus(sends, nil, nil, id, caller, judgedAt) + if err != nil { + t.Fatalf("%s: %v", c.name, err) + } + if out["status"] != c.want { + t.Fatalf("%s: status %v, want %v", c.name, out["status"], c.want) + } + } +} + +// 1261/AC5: the seat verb mail-store.send-status answers exactly as mail_send_status, for the same caller. +func TestSeatSendStatusAnswersExactlyAsMailSendStatus(t *testing.T) { + b, _, _ := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"}) + rt := startBundle(t, b.Tools()) + + text, isErr := rt.call(SendTool, "novox.triage", sendArgs) + if isErr { + t.Fatalf("%s", text) + } + var out map[string]any + _ = json.Unmarshal([]byte(text), &out) + id := out["ask"].(string) + + toolText, toolErr := rt.call(SendStatusTool, "novox.triage", map[string]any{"id": id}) + seatText, seatErr := rt.call(SeatSendStatusTool, "novox.triage", map[string]any{"id": id}) + if toolErr != seatErr || toolText != seatText { + t.Fatalf("mail_send_status: %v %s; mail-store.send-status: %v %s", toolErr, toolText, seatErr, seatText) + } +} + +// 1261/AC6: the sweep settles a send left undecided no matter who asked for it — never refused by the caller +// check mail_send_status itself applies. +func TestSweepSettlesASendRegardlessOfCaller(t *testing.T) { + b, router, _ := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"}) + queue := newMemQueue() + b.Queue = queue.publish + rt := startBundle(t, b.Tools()) + + text, isErr := rt.call(SendTool, "novox.triage", sendArgs) + if isErr { + t.Fatalf("%s", text) + } + var out map[string]any + _ = json.Unmarshal([]byte(text), &out) + id := out["ask"].(string) + + w := router.chose(id, OptionSend, judgedAt.Add(time.Minute)) + router.close(w) // only the router's record holds the answer + + // The sweep itself names no caller at all; it must still settle "novox.triage"'s send. + b.Sweep(judgedAt.Add(2 * time.Minute)) + if n := queue.count(); n != 1 { + t.Fatalf("the sweep did not settle the send regardless of caller: %d message(s)", n) + } +} diff --git a/modules/mesh-mail/cmd/mesh-mail/tools.go b/modules/mesh-mail/cmd/mesh-mail/tools.go index 14c9e9d3..c03e74c9 100644 --- a/modules/mesh-mail/cmd/mesh-mail/tools.go +++ b/modules/mesh-mail/cmd/mesh-mail/tools.go @@ -30,10 +30,11 @@ const ( // its verbs through the same handlers as the mail_* tools above — the seat's name is the one a caller grants // reach the same account's mail through, whichever name it calls. const ( - SeatAccountsTool = "mail-store.accounts" - SeatSearchTool = "mail-store.search" - SeatReadTool = "mail-store.read" - SeatSendTool = "mail-store.send" + SeatAccountsTool = "mail-store.accounts" + SeatSearchTool = "mail-store.search" + SeatReadTool = "mail-store.read" + SeatSendTool = "mail-store.send" + SeatSendStatusTool = "mail-store.send-status" ) // Bounds of what a tool answers. @@ -212,7 +213,9 @@ func (b *Bundle) Tools() []stdio.Tool { {Name: SendStatusTool, Description: "Where a send asked with mail_send stands, by its ask id: waiting for the operator, sent, declined, " + "ended without a choice (expired, refused, cancelled), or not proven. Read from the router's own record, held " + - "to the request mail_send asked (so a warrant for another request, or one changed since, sends nothing here).", + "to the request mail_send asked (so a warrant for another request, or one changed since, sends nothing here). " + + "Answered only to the caller that asked for the send: an id this caller did not ask for, known or not, " + + "answers status unknown and nothing else of it (novox/hq ADR 0440 §7, issue 1261).", Input: schema([]string{"id"}, map[string]any{"id": text("the ask id mail_send gave")}), Run: b.sendStatusTool}, {Name: SeatAccountsTool, @@ -242,6 +245,10 @@ func (b *Bundle) Tools() []stdio.Tool { "body": text("the message's plain-text body"), }), Run: b.sendTool}, + {Name: SeatSendStatusTool, + Description: "The mail-store seat's send-status verb (novox/hq ADR 0378 §2, issue 1261): the same as mail_send_status.", + Input: schema([]string{"id"}, map[string]any{"id": text("the ask id mail_send gave")}), + Run: b.sendStatusTool}, } } @@ -612,7 +619,8 @@ func (b *Bundle) sendStatusTool(args map[string]any) (any, error) { if id == "" { return nil, fmt.Errorf("%s needs id, the ask %s gave", SendStatusTool, SendTool) } - return ReadSendStatus(b.sends(), b.askBus(), b.queuePublish(), id, b.now()) + caller := b.caller(args) + return ReadSendStatus(b.sends(), b.askBus(), b.queuePublish(), id, caller, b.now()) } // ---- arguments ---------------------------------------------------------------------------------------------------- diff --git a/modules/mesh-mail/module.json b/modules/mesh-mail/module.json index b6aaa7ce..d1399727 100644 --- a/modules/mesh-mail/module.json +++ b/modules/mesh-mail/module.json @@ -6,7 +6,7 @@ { "name": "mail-store", "scope": "mesh", - "serves": ["accounts", "search", "read", "send"] + "serves": ["accounts", "search", "read", "send", "send-status"] }, { "name": "mail-webclient", @@ -24,7 +24,7 @@ { "name": "mail-store", "scope": "mesh", - "serves": ["accounts", "search", "read", "send"] + "serves": ["accounts", "search", "read", "send", "send-status"] } ], "requires": [ @@ -80,7 +80,8 @@ "mail-store.accounts", "mail-store.search", "mail-store.read", - "mail-store.send" + "mail-store.send", + "mail-store.send-status" ], "uses": [ "operator-channel" From 08200bc178ecd762a081faccd7747f95a279889e Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 19 Oct 2026 21:50:12 +0200 Subject: [PATCH 02/18] Keep each account's state in a bucket (issue 1255) --- modules/mesh-mail/cmd/mesh-mail/imap_test.go | 6 +- modules/mesh-mail/cmd/mesh-mail/main.go | 49 ++- .../mesh-mail/cmd/mesh-mail/manifest_test.go | 4 + modules/mesh-mail/cmd/mesh-mail/mirror.go | 238 ++++++++++++++ .../mesh-mail/cmd/mesh-mail/mirror_test.go | 290 ++++++++++++++++++ modules/mesh-mail/module.json | 3 +- 6 files changed, 586 insertions(+), 4 deletions(-) create mode 100644 modules/mesh-mail/cmd/mesh-mail/mirror.go create mode 100644 modules/mesh-mail/cmd/mesh-mail/mirror_test.go diff --git a/modules/mesh-mail/cmd/mesh-mail/imap_test.go b/modules/mesh-mail/cmd/mesh-mail/imap_test.go index 0159c710..5ffa1c86 100644 --- a/modules/mesh-mail/cmd/mesh-mail/imap_test.go +++ b/modules/mesh-mail/cmd/mesh-mail/imap_test.go @@ -221,12 +221,14 @@ func TestNothingButTheTestsWritesToAMailbox(t *testing.T) { } writes := regexp.MustCompile(`([A-Za-z_][A-Za-z0-9_]*)\.(Store|Move|Copy|Expunge|UIDExpunge|Append|Create|Delete|Rename|Subscribe|Unsubscribe|SetMetadata|SetQuota)\(`) for _, f := range files { - if strings.HasSuffix(f, "_test.go") || f == "send.go" || f == "smtp.go" || f == "sender.go" { + if strings.HasSuffix(f, "_test.go") || f == "send.go" || f == "smtp.go" || f == "sender.go" || f == "mirror.go" { // None touches a mailbox: send.go keeps the `sends` state on the bus, never IMAP; smtp.go speaks SMTP // to the account's outgoing server, never to a mailbox, and only on the operator's Approve (ADR 0361 // §2); sender.go is the sender process (novox/hq ADR 0436 §7) — its own bus account's `sends` state // (process.State.Create, matched here only because its receiver is named s) and router record, never - // IMAP either. + // IMAP either; mirror.go (novox/hq ADR 0439 §2, §3, issue 1255) keeps the `accounts` and `mails` + // state buckets on the bus, never IMAP — its Delete is a bucket key's, matched here only because its + // KeptState field is named s. continue } raw, err := os.ReadFile(f) diff --git a/modules/mesh-mail/cmd/mesh-mail/main.go b/modules/mesh-mail/cmd/mesh-mail/main.go index 7065834b..ebb444ac 100644 --- a/modules/mesh-mail/cmd/mesh-mail/main.go +++ b/modules/mesh-mail/cmd/mesh-mail/main.go @@ -187,8 +187,22 @@ func serve(env Env) { }, } b.Logf = logf - p := &Publisher{Index: b.Index, Emit: func(key string, body any) error { return stdio.Emit(key, body) }, Logf: logf} + mirror := &Mirror{ + Index: b.Index, + Accounts: func() ([]Account, error) { return ReadAccounts(env.Accounts) }, + Accts: keptMirrorState{stdio.State(AccountsState)}, + Logf: logf, + } + trigger := make(chan struct{}, 1) + p := &Publisher{Index: b.Index, Emit: func(key string, body any) error { return stdio.Emit(key, body) }, Logf: logf, + Said: func(int) { + select { + case trigger <- struct{}{}: + default: + } + }} go p.Run(context.Background()) + go mirrorTicker(mirror, trigger) go hear(b) go sweepSends(b) go waitsTellTicker(b) @@ -252,6 +266,32 @@ func hear(b *Bundle) { } } +// MirrorEvery is the Mirror's own round's slowest pace (settlement 6, issue 1255): at least this often, even +// with nothing published in between. +const MirrorEvery = time.Minute + +// mirrorTicker runs the Mirror's round at least every MirrorEvery, and at once again whenever trigger fires (a +// Publisher round that said something, settlement 6), until the process ends. +func mirrorTicker(m *Mirror, trigger <-chan struct{}) { + tick := time.NewTicker(MirrorEvery) + defer tick.Stop() + round := func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := m.Round(ctx); err != nil && m.Logf != nil { + m.Logf("[mesh-mail] the state buckets could not be mirrored: %v", err) + } + } + for { + select { + case <-trigger: + round() + case <-tick.C: + round() + } + } +} + // sweepSends reads every send still undecided once a minute and asks the router's own record for its answer, so // an Approve answered while this module could not hear it directly is still acted on (novox/hq issue 790, review // round 1). @@ -272,6 +312,10 @@ type Publisher struct { Emit func(key string, body any) error Logf func(format string, a ...any) Every time.Duration + // Said, when given, is called after a round that said one event or more (novox/hq issue 1255, settlement + // 6): the Mirror's own round runs beside it, so the state buckets catch up without waiting for the next + // tick. + Said func(n int) } // PublishBatch is how many events one round says at most. @@ -300,6 +344,9 @@ func (p *Publisher) Once(ctx context.Context) (int, error) { if err := st.Said(ctx, said); err != nil { return 0, err } + if p.Said != nil { + p.Said(len(said)) + } } return len(said), failed } diff --git a/modules/mesh-mail/cmd/mesh-mail/manifest_test.go b/modules/mesh-mail/cmd/mesh-mail/manifest_test.go index 5673e945..c379af5b 100644 --- a/modules/mesh-mail/cmd/mesh-mail/manifest_test.go +++ b/modules/mesh-mail/cmd/mesh-mail/manifest_test.go @@ -88,6 +88,10 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { if fam["issued-by"] != "outside" || fam["path"] != "${dir:state}/"+SecretFamily+"*.secret" || len(m.OwnSecrets) != 2 { t.Fatalf("own secrets %v", m.OwnSecrets) } + // 1255/AC7: the module declares the account bucket's own state, beside `sends`. + if !reflect.DeepEqual(m.State, []string{"sends", "accounts"}) { + t.Fatalf("state %v, want [sends accounts]", m.State) + } var names []string for _, tool := range (&Bundle{}).Tools() { names = append(names, tool.Name) diff --git a/modules/mesh-mail/cmd/mesh-mail/mirror.go b/modules/mesh-mail/cmd/mesh-mail/mirror.go new file mode 100644 index 00000000..bce48c25 --- /dev/null +++ b/modules/mesh-mail/cmd/mesh-mail/mirror.go @@ -0,0 +1,238 @@ +package main + +// The account bucket (novox/hq ADR 0439 §2, issue 1255): a view (such as the mail web client) can watch it +// instead of asking mail-store's verbs for an account's state and folder counts. The bundle's own Mirror keeps +// it in step with the index (design 59's prepare step, settlement 6): it reads the index fresh each round and +// writes only what changed — a key put when its value differs, never otherwise (novox/hq ADR 0201 §12). keep +// (imap.go) is not changed: it writes the index; the Mirror alone writes this bucket. + +import ( + "bytes" + "context" + "encoding/json" + "unicode/utf8" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// AccountsState is the module's state the Mirror keeps one key per account in (novox/hq ADR 0439 §2). +const AccountsState = "accounts" + +// The bounds ADR 0439 §2 gives the account bucket's value. +const ( + // MostAccountValue is the value's own bound, 64 KiB. + MostAccountValue = 64 << 10 + // MostFolderName is one folder's name, cut at a character's boundary when it is longer. + MostFolderName = 120 + // DefaultKeptMails is how many of an account's newest mails the mail bucket keeps, 200 unless ADR 0439 §4's + // 81-account rule (issue 1282) gives every account the same smaller number instead. + DefaultKeptMails = 200 + // MostMailValue is ADR 0439 §3's own bound on one mail's value in the mail bucket, 4 KiB. + MostMailValue = 4 << 10 + // MostMailBytes is ADR 0439 §4's own bound across every account's newest mails together, 64 MiB. + MostMailBytes = 64 << 20 +) + +// AccountValue is state accounts' own value, one per account (ADR 0439 §2): never a time that moves on each +// sync, so two rounds with nothing changed write the byte-identical value. +type AccountValue struct { + // State is the account's state as its keeper last said it (current, syncing, waits-for-sign-in or failing), + // or "unknown" while its keeper has said nothing of it yet (as mail_accounts itself answers, tools.go). + State string `json:"state"` + // Kept is the number of the account's newest mails the mail bucket (issue 1282) keeps. + Kept int `json:"kept"` + // Folders are the account's folders, each by its name (cut to MostFolderName bytes), its count of mails and + // its count of unread mails. + Folders []AccountFolderValue `json:"folders"` + // DroppedFolders is how many folders were left out, fewest-mail first, to keep the value at MostAccountValue + // bytes or fewer; left out of the value (omitted, not zero) when nothing was dropped. + DroppedFolders int `json:"dropped-folders,omitempty"` +} + +// AccountFolderValue is one folder in an AccountValue. +type AccountFolderValue struct { + Name string `json:"name"` + Mails int `json:"mails"` + Unread int `json:"unread"` +} + +// MirrorState is what the Mirror needs of one state bucket it writes: a key's current raw value (nil when it +// has none), a value put — the Mirror is this bucket's only writer, so never by compare-and-set — and every key +// it holds. An interface, so a unit test gives a fake instead of a runtime (novox/hq ADR 0439 §2, §3; issue +// 1255, 1282's "Prepare findings"). +type MirrorState interface { + Get(key string) (json.RawMessage, bool, error) + Put(key string, value any) error + Delete(key string) error + Keys() ([]string, error) +} + +// keptMirrorState is a MirrorState on the bus, over the sdk's own KeptState (state.go, seat.go). +type keptMirrorState struct{ s stdio.KeptState } + +func (k keptMirrorState) Get(key string) (json.RawMessage, bool, error) { + e, err := k.s.Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} +func (k keptMirrorState) Put(key string, value any) error { + _, err := k.s.Put(key, value) + return err +} +func (k keptMirrorState) Delete(key string) error { return k.s.Delete(key) } +func (k keptMirrorState) Keys() ([]string, error) { return k.s.Keys() } + +// Mirror keeps the module's state buckets in step with the index: one round reads the index fresh and writes +// only what changed (ADR 0201 §12). It runs beside the Publisher (serve(), main.go): after each Publisher +// round that said something, and at least every MirrorEvery either way (settlement 6). +type Mirror struct { + Index func(ctx context.Context) (Store, error) + Accounts func() ([]Account, error) + Accts MirrorState // the `accounts` state + Logf func(format string, a ...any) +} + +// Round mirrors every configured account's current state into the `accounts` bucket (issue 1255's AC1-AC7). +func (m *Mirror) Round(ctx context.Context) error { + accounts, err := m.Accounts() + if err != nil { + return err + } + st, err := m.Index(ctx) + if err != nil { + return err + } + states, err := st.States(ctx) + if err != nil { + return err + } + kept := keptPerAccount(len(accounts)) + for _, a := range accounts { + v, err := accountValue(ctx, st, a.Name, states[a.Name], kept) + if err != nil { + return err + } + if err := putIfChanged(m.Accts, a.Name, v); err != nil { + return err + } + } + return nil +} + +// accountValue is one account's own value: its state word, the number of mails kept for it, and its folders +// with their counts, trimmed to MostAccountValue. +func accountValue(ctx context.Context, st Store, account string, state AccountState, kept int) (AccountValue, error) { + folders, err := st.Folders(ctx, account) + if err != nil { + return AccountValue{}, err + } + fv := make([]AccountFolderValue, 0, len(folders)) + for _, f := range folders { + unread, err := unreadCount(ctx, st, account, f.Name) + if err != nil { + return AccountValue{}, err + } + fv = append(fv, AccountFolderValue{Name: cutUTF8(f.Name, MostFolderName), Mails: f.Messages, Unread: unread}) + } + word := state.State + if word == "" { + word = "unknown" + } + return trimAccountValue(AccountValue{State: word, Kept: kept, Folders: fv}), nil +} + +// unreadCount is a folder's count of messages without the \Seen flag. +func unreadCount(ctx context.Context, st Store, account, folder string) (int, error) { + flags, err := st.Flags(ctx, account, folder) + if err != nil { + return 0, err + } + n := 0 + for _, fl := range flags { + if !hasSeenFlag(fl) { + n++ + } + } + return n, nil +} + +// trimAccountValue is issue 1255/AC5: while the value marshals to more than MostAccountValue bytes, the folder +// with the fewest mails is dropped, and DroppedFolders counts it, until it fits (or no folder is left). +func trimAccountValue(v AccountValue) AccountValue { + for accountValueSize(v) > MostAccountValue && len(v.Folders) > 0 { + fewest := 0 + for i := 1; i < len(v.Folders); i++ { + if v.Folders[i].Mails < v.Folders[fewest].Mails { + fewest = i + } + } + v.Folders = append(append([]AccountFolderValue{}, v.Folders[:fewest]...), v.Folders[fewest+1:]...) + v.DroppedFolders++ + } + return v +} + +func accountValueSize(v AccountValue) int { + raw, err := json.Marshal(v) + if err != nil { + return 0 + } + return len(raw) +} + +// keptPerAccount is ADR 0439 §4's 81-account rule (issue 1282): DefaultKeptMails, unless that many of every +// account's mails could together pass MostMailBytes, in which case every account keeps the same smaller number +// instead, so that all of them fit. +func keptPerAccount(numAccounts int) int { + if numAccounts <= 0 { + return DefaultKeptMails + } + if numAccounts*DefaultKeptMails*MostMailValue <= MostMailBytes { + return DefaultKeptMails + } + n := MostMailBytes / (numAccounts * MostMailValue) + if n < 1 { + n = 1 + } + if n > DefaultKeptMails { + n = DefaultKeptMails + } + return n +} + +// putIfChanged writes value at key only when it differs from what the bucket currently holds there (ADR 0201 +// §12): the Mirror's own values never carry a time that moves on each sync, so the same JSON bytes (Go's own +// json.Marshal is stable for a fixed type) mean nothing changed. +func putIfChanged(bucket MirrorState, key string, value any) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + cur, ok, err := bucket.Get(key) + if err != nil { + return err + } + if ok && bytes.Equal(cur, raw) { + return nil + } + return bucket.Put(key, value) +} + +// cutUTF8 cuts s to at most n bytes, never splitting a multi-byte character (novox/hq ADR 0439 §2, §3). +func cutUTF8(s string, n int) string { + if len(s) <= n { + return s + } + b := s[:n] + for len(b) > 0 { + r, size := utf8.DecodeLastRuneInString(b) + if r == utf8.RuneError && size <= 1 { + b = b[:len(b)-1] + continue + } + break + } + return b +} diff --git a/modules/mesh-mail/cmd/mesh-mail/mirror_test.go b/modules/mesh-mail/cmd/mesh-mail/mirror_test.go new file mode 100644 index 00000000..fd193482 --- /dev/null +++ b/modules/mesh-mail/cmd/mesh-mail/mirror_test.go @@ -0,0 +1,290 @@ +package main + +// The account bucket (novox/hq ADR 0439 §2, issue 1255): a fake MirrorState, so these tests need no runtime, +// and memStore for the index, as the rest of this package's unit tests already use it. + +import ( + "context" + "encoding/json" + "strings" + "testing" + "unicode/utf8" +) + +// memMirrorState is a fake MirrorState (mirror.go), kept in memory: a map, no revision, since the Mirror is a +// bucket's only writer and never writes it by compare-and-set. +type memMirrorState struct { + vals map[string]json.RawMessage + calls map[string]int // Put calls per key, so a test can tell a skipped write from a repeated one +} + +func newMemMirrorState() *memMirrorState { + return &memMirrorState{vals: map[string]json.RawMessage{}, calls: map[string]int{}} +} + +func (m *memMirrorState) Get(key string) (json.RawMessage, bool, error) { + v, ok := m.vals[key] + return v, ok, nil +} +func (m *memMirrorState) Put(key string, value any) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + m.vals[key] = raw + m.calls[key]++ + return nil +} +func (m *memMirrorState) Delete(key string) error { + delete(m.vals, key) + return nil +} +func (m *memMirrorState) Keys() ([]string, error) { + out := make([]string, 0, len(m.vals)) + for k := range m.vals { + out = append(out, k) + } + return out, nil +} + +// accountsAndStore is one account "mesh", granted no caller (the Mirror reads no grant), with the index given. +func accountsAndStore(t *testing.T) ([]Account, *memStore) { + t.Helper() + return []Account{{Name: "mesh", Kind: KindIMAP, Address: "jschoubben@novox.be"}}, newMemStore() +} + +func mirrorOf(accounts []Account, st *memStore, bucket MirrorState) *Mirror { + return &Mirror{ + Index: func(context.Context) (Store, error) { return st, nil }, + Accounts: func() ([]Account, error) { return accounts, nil }, + Accts: bucket, + } +} + +// 1255/AC1: the account's value holds its state word, the number of newest mails kept (200, no other account +// configured, so none of ADR 0439 §4's 81-account rule applies), and its folders with their counts. +func TestMirrorRoundWritesTheAccountsStateWord200AndItsFolders(t *testing.T) { + ctx := context.Background() + accounts, st := accountsAndStore(t) + _ = st.SetState(ctx, AccountState{Account: "mesh", State: AccountCurrent, Why: "watching"}) + addMail(t, st, "mesh", "INBOX", 1, []string{}) // unread + addMail(t, st, "mesh", "INBOX", 2, []string{`\Seen`}) // read + addMail(t, st, "mesh", "Sent", 3, []string{`\Seen`}) // read + _ = st.SetFolder(ctx, Folder{Account: "mesh", Name: "INBOX", Messages: 10}) + _ = st.SetFolder(ctx, Folder{Account: "mesh", Name: "Sent", Messages: 5}) + + bucket := newMemMirrorState() + m := mirrorOf(accounts, st, bucket) + if err := m.Round(ctx); err != nil { + t.Fatal(err) + } + raw, ok, err := bucket.Get("mesh") + if err != nil || !ok { + t.Fatalf("no value kept for mesh: %v %v", ok, err) + } + var v AccountValue + if err := json.Unmarshal(raw, &v); err != nil { + t.Fatal(err) + } + if v.State != AccountCurrent || v.Kept != DefaultKeptMails { + t.Fatalf("state %q, kept %d; want %q, %d", v.State, v.Kept, AccountCurrent, DefaultKeptMails) + } + byName := map[string]AccountFolderValue{} + for _, f := range v.Folders { + byName[f.Name] = f + } + if byName["INBOX"].Mails != 10 || byName["INBOX"].Unread != 1 { + t.Fatalf("INBOX: %+v", byName["INBOX"]) + } + if byName["Sent"].Mails != 5 || byName["Sent"].Unread != 0 { + t.Fatalf("Sent: %+v", byName["Sent"]) + } +} + +// addMail indexes one bare message with the given flags, for folder-count tests that only need its UID and +// flags (mirroring the shape imap.go's Keeper writes, store.go's Message). +func addMail(t *testing.T, st *memStore, account, folder string, uid uint32, flags []string) { + t.Helper() + m := Message{Account: account, Folder: folder, UID: uid, Flags: flags} + if err := st.Add(context.Background(), m, nil); err != nil { + t.Fatal(err) + } +} + +// 1255/AC2: nothing is written when the index is unchanged since the last round. +func TestMirrorRoundWritesNothingWhenUnchanged(t *testing.T) { + ctx := context.Background() + accounts, st := accountsAndStore(t) + _ = st.SetState(ctx, AccountState{Account: "mesh", State: AccountCurrent}) + _ = st.SetFolder(ctx, Folder{Account: "mesh", Name: "INBOX", Messages: 1}) + + bucket := newMemMirrorState() + m := mirrorOf(accounts, st, bucket) + if err := m.Round(ctx); err != nil { + t.Fatal(err) + } + if bucket.calls["mesh"] != 1 { + t.Fatalf("the first round wrote %d time(s); want 1", bucket.calls["mesh"]) + } + if err := m.Round(ctx); err != nil { + t.Fatal(err) + } + if bucket.calls["mesh"] != 1 { + t.Fatalf("a round with nothing changed wrote again: %d time(s)", bucket.calls["mesh"]) + } +} + +// 1255/AC3: one mail of INBOX marked read changes only that account's key, to 1 unread. +func TestMirrorRoundWritesOnlyTheChangedAccountsKey(t *testing.T) { + ctx := context.Background() + accounts, st := accountsAndStore(t) + accounts = append(accounts, Account{Name: "other", Kind: KindIMAP, Address: "other@novox.be"}) + _ = st.SetState(ctx, AccountState{Account: "mesh", State: AccountCurrent}) + _ = st.SetState(ctx, AccountState{Account: "other", State: AccountCurrent}) + addMail(t, st, "mesh", "INBOX", 1, []string{}) + _ = st.SetFolder(ctx, Folder{Account: "mesh", Name: "INBOX", Messages: 1}) + _ = st.SetFolder(ctx, Folder{Account: "other", Name: "INBOX", Messages: 0}) + + bucket := newMemMirrorState() + m := mirrorOf(accounts, st, bucket) + if err := m.Round(ctx); err != nil { + t.Fatal(err) + } + bucket.calls = map[string]int{} // reset the counters; only the next round's writes matter here + + // mesh's one message is now marked read. + if err := st.SetFlags(ctx, "mesh", "INBOX", 1, []string{`\Seen`}, nil); err != nil { + t.Fatal(err) + } + if err := m.Round(ctx); err != nil { + t.Fatal(err) + } + if bucket.calls["mesh"] != 1 { + t.Fatalf("mesh's key was not rewritten: %d call(s)", bucket.calls["mesh"]) + } + if bucket.calls["other"] != 0 { + t.Fatalf("other's key was written though nothing of it changed: %d call(s)", bucket.calls["other"]) + } + raw, _, _ := bucket.Get("mesh") + var v AccountValue + _ = json.Unmarshal(raw, &v) + for _, f := range v.Folders { + if f.Name == "INBOX" && f.Unread != 0 { + t.Fatalf("INBOX unread after marking read: %d", f.Unread) + } + } +} + +// 1255/AC4: a folder name of 200 bytes of multibyte characters is cut to at most 120 bytes at a character's +// boundary (never split mid-character). +func TestMirrorRoundCutsALongMultibyteFolderNameAtACharacterBoundary(t *testing.T) { + ctx := context.Background() + accounts, st := accountsAndStore(t) + _ = st.SetState(ctx, AccountState{Account: "mesh", State: AccountCurrent}) + // 119 one-byte letters, then 2-byte "é"s: the 120th byte (MostFolderName) lands mid-character, inside the + // first é's 2-byte encoding, so a naive byte cut would split it. + long := strings.Repeat("a", 119) + strings.Repeat("é", 40) + _ = st.SetFolder(ctx, Folder{Account: "mesh", Name: long, Messages: 1}) + + bucket := newMemMirrorState() + m := mirrorOf(accounts, st, bucket) + if err := m.Round(ctx); err != nil { + t.Fatal(err) + } + raw, _, _ := bucket.Get("mesh") + var v AccountValue + _ = json.Unmarshal(raw, &v) + if len(v.Folders) != 1 { + t.Fatalf("folders: %+v", v.Folders) + } + got := v.Folders[0].Name + if len(got) > MostFolderName { + t.Fatalf("the folder name is %d bytes, more than %d", len(got), MostFolderName) + } + if !strings.HasPrefix(long, got) { + t.Fatalf("the cut name is not a prefix of the original: %q", got) + } + if !utf8.ValidString(got) { + t.Fatalf("the cut name is not valid UTF-8: a multibyte character was split: %q", got) + } +} + +// 1255/AC5: an account with enough folders that the value would pass 64 KiB has the folders with the fewest +// mails dropped, their number counted, and the value fits. +func TestMirrorRoundDropsTheFewestMailFoldersToFitUnder64KiB(t *testing.T) { + ctx := context.Background() + accounts, st := accountsAndStore(t) + _ = st.SetState(ctx, AccountState{Account: "mesh", State: AccountCurrent}) + // Enough folders, with long enough names, that the whole value passes MostAccountValue; each folder's own + // mail count is distinct, so "fewest first" is unambiguous. + const folders = 1200 + for i := 0; i < folders; i++ { + name := "folder-" + itoa(int64(i)) + "-abcdefghijklmnopqrstuvwxyz0123456789" + _ = st.SetFolder(ctx, Folder{Account: "mesh", Name: name, Messages: i + 1}) + } + + bucket := newMemMirrorState() + m := mirrorOf(accounts, st, bucket) + if err := m.Round(ctx); err != nil { + t.Fatal(err) + } + raw, _, _ := bucket.Get("mesh") + if len(raw) > MostAccountValue { + t.Fatalf("the value is %d bytes, more than %d", len(raw), MostAccountValue) + } + var v AccountValue + _ = json.Unmarshal(raw, &v) + if v.DroppedFolders == 0 { + t.Fatalf("no folder was dropped, though %d folders would overflow %d bytes", folders, MostAccountValue) + } + if v.DroppedFolders+len(v.Folders) != folders { + t.Fatalf("dropped %d + kept %d != %d folders in all", v.DroppedFolders, len(v.Folders), folders) + } + // The kept folders are never among the ones with the fewest mails: folder i has i+1 mails, so every kept + // folder's count is at least as large as every dropped folder's would have been. + fewestKept := -1 + for _, f := range v.Folders { + if fewestKept == -1 || f.Mails < fewestKept { + fewestKept = f.Mails + } + } + if fewestKept < v.DroppedFolders { + t.Fatalf("the fewest-mail folders were not the ones dropped: kept as few as %d mails, dropped %d folders", fewestKept, v.DroppedFolders) + } +} + +// 1255/AC6: two rounds an hour apart with nothing indexed between write the byte-identical value (no time that +// moves on each sync). +func TestMirrorRoundTwoRoundsWithNothingChangedWriteByteIdenticalValues(t *testing.T) { + ctx := context.Background() + accounts, st := accountsAndStore(t) + _ = st.SetState(ctx, AccountState{Account: "mesh", State: AccountCurrent}) + _ = st.SetFolder(ctx, Folder{Account: "mesh", Name: "INBOX", Messages: 3}) + + bucket := newMemMirrorState() + m := mirrorOf(accounts, st, bucket) + if err := m.Round(ctx); err != nil { + t.Fatal(err) + } + first, _, _ := bucket.Get("mesh") + firstCopy := append([]byte(nil), first...) + + // Force a second write by deleting the key directly and running again: the value computed must still be + // byte-identical to the first, since nothing in the index changed. + _ = bucket.Delete("mesh") + if err := m.Round(ctx); err != nil { + t.Fatal(err) + } + second, _, _ := bucket.Get("mesh") + if string(firstCopy) != string(second) { + t.Fatalf("the two rounds wrote different bytes:\n%s\n%s", firstCopy, second) + } +} + +// 1255/AC7: the manifest declares the state accounts (manifest_test.go asserts the exact list; this just +// anchors the constant the code and the manifest both use). +func TestAccountsStateNameMatchesTheManifestsWord(t *testing.T) { + if AccountsState != "accounts" { + t.Fatalf("AccountsState = %q", AccountsState) + } +} diff --git a/modules/mesh-mail/module.json b/modules/mesh-mail/module.json index d1399727..5aa126c7 100644 --- a/modules/mesh-mail/module.json +++ b/modules/mesh-mail/module.json @@ -87,7 +87,8 @@ "operator-channel" ], "state": [ - "sends" + "sends", + "accounts" ], "emits": [ "mail.send.sent", From a3e9985c15ed46cb72f07bede555a39486c609e8 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 19 Oct 2026 21:56:04 +0200 Subject: [PATCH 03/18] Keep each account's newest mails in a bucket, bounded past 81 accounts (issue 1282) --- modules/mesh-mail/cmd/mesh-mail/main.go | 1 + .../mesh-mail/cmd/mesh-mail/manifest_test.go | 6 +- modules/mesh-mail/cmd/mesh-mail/mirror.go | 191 ++++++++++- .../mesh-mail/cmd/mesh-mail/mirror_test.go | 297 +++++++++++++++++- modules/mesh-mail/cmd/mesh-mail/pgstore.go | 7 + modules/mesh-mail/cmd/mesh-mail/store.go | 9 + modules/mesh-mail/module.json | 3 +- 7 files changed, 507 insertions(+), 7 deletions(-) diff --git a/modules/mesh-mail/cmd/mesh-mail/main.go b/modules/mesh-mail/cmd/mesh-mail/main.go index ebb444ac..cb66ae89 100644 --- a/modules/mesh-mail/cmd/mesh-mail/main.go +++ b/modules/mesh-mail/cmd/mesh-mail/main.go @@ -191,6 +191,7 @@ func serve(env Env) { Index: b.Index, Accounts: func() ([]Account, error) { return ReadAccounts(env.Accounts) }, Accts: keptMirrorState{stdio.State(AccountsState)}, + Mails: keptMirrorState{stdio.State(MailsState)}, Logf: logf, } trigger := make(chan struct{}, 1) diff --git a/modules/mesh-mail/cmd/mesh-mail/manifest_test.go b/modules/mesh-mail/cmd/mesh-mail/manifest_test.go index c379af5b..2c791c9a 100644 --- a/modules/mesh-mail/cmd/mesh-mail/manifest_test.go +++ b/modules/mesh-mail/cmd/mesh-mail/manifest_test.go @@ -88,9 +88,9 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { if fam["issued-by"] != "outside" || fam["path"] != "${dir:state}/"+SecretFamily+"*.secret" || len(m.OwnSecrets) != 2 { t.Fatalf("own secrets %v", m.OwnSecrets) } - // 1255/AC7: the module declares the account bucket's own state, beside `sends`. - if !reflect.DeepEqual(m.State, []string{"sends", "accounts"}) { - t.Fatalf("state %v, want [sends accounts]", m.State) + // 1255/AC7, 1282: the module declares the account and mail buckets' own state, beside `sends`. + if !reflect.DeepEqual(m.State, []string{"sends", "accounts", "mails"}) { + t.Fatalf("state %v, want [sends accounts mails]", m.State) } var names []string for _, tool := range (&Bundle{}).Tools() { diff --git a/modules/mesh-mail/cmd/mesh-mail/mirror.go b/modules/mesh-mail/cmd/mesh-mail/mirror.go index bce48c25..4e83469c 100644 --- a/modules/mesh-mail/cmd/mesh-mail/mirror.go +++ b/modules/mesh-mail/cmd/mesh-mail/mirror.go @@ -9,7 +9,11 @@ package main import ( "bytes" "context" + "crypto/sha256" + "encoding/hex" "encoding/json" + "strings" + "time" "unicode/utf8" stdio "git.novox.be/novox/mesh-sdk/go" @@ -18,6 +22,9 @@ import ( // AccountsState is the module's state the Mirror keeps one key per account in (novox/hq ADR 0439 §2). const AccountsState = "accounts" +// MailsState is the module's state the Mirror keeps one key per mail in (novox/hq ADR 0439 §3, issue 1282). +const MailsState = "mails" + // The bounds ADR 0439 §2 gives the account bucket's value. const ( // MostAccountValue is the value's own bound, 64 KiB. @@ -31,6 +38,11 @@ const ( MostMailValue = 4 << 10 // MostMailBytes is ADR 0439 §4's own bound across every account's newest mails together, 64 MiB. MostMailBytes = 64 << 20 + // MostMailSubject is one mail's subject in the mail bucket, cut at a character's boundary when longer. + MostMailSubject = 300 + // MostMailShown is how many recipients, and how many attachment names, the mail bucket's value names one + // by one before it counts the rest (ADR 0439 §3's "first 10"). + MostMailShown = 10 ) // AccountValue is state accounts' own value, one per account (ADR 0439 §2): never a time that moves on each @@ -56,6 +68,31 @@ type AccountFolderValue struct { Unread int `json:"unread"` } +// MailValue is state mails' own value, one per kept mail (novox/hq ADR 0439 §3): never a body. The subject is +// cut to MostMailSubject bytes, and every address, name, folder and attachment name to MostFolderName bytes, at +// a character's boundary; recipients and attachment names are each named one by one up to MostMailShown, the +// rest counted, and when the value is still over MostMailValue bytes, recipients and then attachment names are +// dropped from the end (RecipientsDropped, AttachmentsDropped growing) until it fits. +type MailValue struct { + Account string `json:"account"` + Folder string `json:"folder"` + Date string `json:"date"` // RFC3339; the mail's own Date header, never a time that moves + Sender MailPerson `json:"sender"` + Subject string `json:"subject"` + Recipients []MailPerson `json:"recipients,omitempty"` + RecipientsDropped int `json:"recipients-dropped,omitempty"` + Attachments []string `json:"attachments,omitempty"` + AttachmentsDropped int `json:"attachments-dropped,omitempty"` + Size int64 `json:"size"` + Flags []string `json:"flags,omitempty"` +} + +// MailPerson is a sender or a recipient in a MailValue, each field already cut to MostFolderName bytes. +type MailPerson struct { + Name string `json:"name,omitempty"` + Address string `json:"address"` +} + // MirrorState is what the Mirror needs of one state bucket it writes: a key's current raw value (nil when it // has none), a value put — the Mirror is this bucket's only writer, so never by compare-and-set — and every key // it holds. An interface, so a unit test gives a fake instead of a runtime (novox/hq ADR 0439 §2, §3; issue @@ -91,10 +128,13 @@ type Mirror struct { Index func(ctx context.Context) (Store, error) Accounts func() ([]Account, error) Accts MirrorState // the `accounts` state + Mails MirrorState // the `mails` state (issue 1282); nil skips mirroring it (tests of accounts alone) Logf func(format string, a ...any) } -// Round mirrors every configured account's current state into the `accounts` bucket (issue 1255's AC1-AC7). +// Round mirrors every configured account's current state into the `accounts` bucket, and (when Mails is given) +// each one's newest mails into the `mails` bucket — both at the same, shared kept count (ADR 0439 §4's +// 81-account rule, issue 1282). func (m *Mirror) Round(ctx context.Context) error { accounts, err := m.Accounts() if err != nil { @@ -118,6 +158,49 @@ func (m *Mirror) Round(ctx context.Context) error { return err } } + if m.Mails == nil { + return nil + } + for _, a := range accounts { + if err := m.roundMails(ctx, st, a.Name, kept); err != nil { + return err + } + } + return nil +} + +// roundMails mirrors one account's newest kept mails into the `mails` bucket (novox/hq ADR 0439 §3, issue +// 1282): a key put for one new or changed, a key deleted for one no longer among the newest kept — never for +// another account's own keys, which mailKey's account prefix tells apart. +func (m *Mirror) roundMails(ctx context.Context, st Store, account string, kept int) error { + msgs, err := st.Newest(ctx, account, kept) + if err != nil { + return err + } + desired := map[string]MailValue{} + for _, msg := range msgs { + desired[mailKey(account, msg)] = mailValue(account, msg) + } + for key, v := range desired { + if err := putIfChanged(m.Mails, key, v); err != nil { + return err + } + } + keys, err := m.Mails.Keys() + if err != nil { + return err + } + prefix := account + "." + for _, k := range keys { + if !strings.HasPrefix(k, prefix) { + continue + } + if _, still := desired[k]; !still { + if err := m.Mails.Delete(k); err != nil { + return err + } + } + } return nil } @@ -220,6 +303,112 @@ func putIfChanged(bucket MirrorState, key string, value any) error { return bucket.Put(key, value) } +// mailDigest is the digest named by a mail's own message-id, or by its folder and uid when it has none (novox/hq +// ADR 0439 §3, issue 1282, AC5): two mails without a message-id in different folders or uids digest apart; one +// message-id kept in two folders of the same account digests the same, so it is mirrored once. +func mailDigest(m Message) string { + basis := m.MessageID + if basis == "" { + basis = m.Folder + "\x00" + itoa(int64(m.UID)) + } + sum := sha256.Sum256([]byte(basis)) + return hex.EncodeToString(sum[:]) +} + +// mailKey is the mail bucket's own key: the account, so one account's keys are told apart from another's +// (roundMails's own deletion), then a dot (account names hold none, config.go's accountName), then the digest. +func mailKey(account string, m Message) string { + return account + "." + mailDigest(m) +} + +// mailValue is one mail's own value (ADR 0439 §3): account, folder, date, sender, subject, recipients and +// attachment names (each capped to MostMailShown, the rest counted), size and flags, never a body, trimmed to +// MostMailValue. +func mailValue(account string, m Message) MailValue { + sender := MailPerson{} + if len(m.From) > 0 { + sender = mailPerson(m.From[0]) + } + recipients, recipientsDropped := mailPeople(m.To) + attachments, attachmentsDropped := mailAttachmentNames(m.Attachments) + v := MailValue{ + Account: account, + Folder: cutUTF8(m.Folder, MostFolderName), + Date: m.Date.UTC().Format(time.RFC3339), + Sender: sender, + Subject: cutUTF8(m.Subject, MostMailSubject), + Recipients: recipients, + RecipientsDropped: recipientsDropped, + Attachments: attachments, + AttachmentsDropped: attachmentsDropped, + Size: m.Size, + Flags: append([]string(nil), m.Flags...), + } + return trimMailValue(v) +} + +// mailPerson is one Person as a MailValue names it: name and address each cut to MostFolderName bytes. +func mailPerson(p Person) MailPerson { + return MailPerson{Name: cutUTF8(p.Name, MostFolderName), Address: cutUTF8(p.Address, MostFolderName)} +} + +// mailPeople is a list of recipients, capped to MostMailShown with the rest counted (ADR 0439 §3's "first 10 +// recipients"). +func mailPeople(ps []Person) ([]MailPerson, int) { + dropped := 0 + if len(ps) > MostMailShown { + dropped = len(ps) - MostMailShown + ps = ps[:MostMailShown] + } + out := make([]MailPerson, 0, len(ps)) + for _, p := range ps { + out = append(out, mailPerson(p)) + } + return out, dropped +} + +// mailAttachmentNames is a list of attachment names, capped to MostMailShown with the rest counted (ADR 0439 +// §3's "first 10 attachment names"), each cut to MostFolderName bytes. +func mailAttachmentNames(refs []AttachmentRef) ([]string, int) { + dropped := 0 + if len(refs) > MostMailShown { + dropped = len(refs) - MostMailShown + refs = refs[:MostMailShown] + } + out := make([]string, 0, len(refs)) + for _, r := range refs { + out = append(out, cutUTF8(r.Name, MostFolderName)) + } + return out, dropped +} + +// trimMailValue is issue 1282/AC4: while the value marshals to more than MostMailValue bytes, a recipient is +// dropped from the end (RecipientsDropped counting it) until none is left, then an attachment name the same +// way (AttachmentsDropped counting it), until it fits or both are empty. +func trimMailValue(v MailValue) MailValue { + for mailValueSize(v) > MostMailValue { + switch { + case len(v.Recipients) > 0: + v.Recipients = v.Recipients[:len(v.Recipients)-1] + v.RecipientsDropped++ + case len(v.Attachments) > 0: + v.Attachments = v.Attachments[:len(v.Attachments)-1] + v.AttachmentsDropped++ + default: + return v + } + } + return v +} + +func mailValueSize(v MailValue) int { + raw, err := json.Marshal(v) + if err != nil { + return 0 + } + return len(raw) +} + // cutUTF8 cuts s to at most n bytes, never splitting a multi-byte character (novox/hq ADR 0439 §2, §3). func cutUTF8(s string, n int) string { if len(s) <= n { diff --git a/modules/mesh-mail/cmd/mesh-mail/mirror_test.go b/modules/mesh-mail/cmd/mesh-mail/mirror_test.go index fd193482..a6ad8837 100644 --- a/modules/mesh-mail/cmd/mesh-mail/mirror_test.go +++ b/modules/mesh-mail/cmd/mesh-mail/mirror_test.go @@ -1,13 +1,15 @@ package main -// The account bucket (novox/hq ADR 0439 §2, issue 1255): a fake MirrorState, so these tests need no runtime, -// and memStore for the index, as the rest of this package's unit tests already use it. +// The account bucket (novox/hq ADR 0439 §2, issue 1255) and the mail bucket (ADR 0439 §3, issue 1282): a fake +// MirrorState, so these tests need no runtime, and memStore for the index, as the rest of this package's unit +// tests already use it. import ( "context" "encoding/json" "strings" "testing" + "time" "unicode/utf8" ) @@ -61,6 +63,13 @@ func mirrorOf(accounts []Account, st *memStore, bucket MirrorState) *Mirror { } } +// mirrorWithMails is mirrorOf, also mirroring the `mails` bucket (issue 1282). +func mirrorWithMails(accounts []Account, st *memStore, accts, mails MirrorState) *Mirror { + m := mirrorOf(accounts, st, accts) + m.Mails = mails + return m +} + // 1255/AC1: the account's value holds its state word, the number of newest mails kept (200, no other account // configured, so none of ADR 0439 §4's 81-account rule applies), and its folders with their counts. func TestMirrorRoundWritesTheAccountsStateWord200AndItsFolders(t *testing.T) { @@ -288,3 +297,287 @@ func TestAccountsStateNameMatchesTheManifestsWord(t *testing.T) { t.Fatalf("AccountsState = %q", AccountsState) } } + +// ---- the mail bucket (novox/hq ADR 0439 §3, issue 1282) ----------------------------------------------------- + +// 1282/AC1: a new mail indexed gets one key in `mails`, named by its account and a digest of its message-id, +// whose value carries its account, folder, date, sender, subject, recipients, attachment names, size and +// flags, and no body. +func TestMirrorRoundWritesANewMailsKeyWithItsFields(t *testing.T) { + ctx := context.Background() + accounts, st := accountsAndStore(t) + msg := Message{Account: "mesh", Folder: "INBOX", UID: 1, MessageID: "m1@example.org", Date: judgedAt, + From: []Person{{Name: "Bob", Address: "bob@example.org"}}, To: []Person{{Address: "ops@example.org"}}, + Subject: "Hi", Size: 1234, Flags: []string{`\Seen`}, + Attachments: []AttachmentRef{{Key: "k1", Name: "invoice.pdf", Size: 10, Type: "application/pdf"}}, + Body: "should never appear in the mirrored value", + } + if err := st.Add(ctx, msg, nil); err != nil { + t.Fatal(err) + } + _ = st.SetFolder(ctx, Folder{Account: "mesh", Name: "INBOX", Messages: 1}) + + acctsBucket, mailsBucket := newMemMirrorState(), newMemMirrorState() + m := mirrorWithMails(accounts, st, acctsBucket, mailsBucket) + if err := m.Round(ctx); err != nil { + t.Fatal(err) + } + + raw, ok, err := mailsBucket.Get(mailKey("mesh", msg)) + if err != nil || !ok { + t.Fatalf("no value kept for the new mail: %v %v", ok, err) + } + var v MailValue + if err := json.Unmarshal(raw, &v); err != nil { + t.Fatal(err) + } + if v.Account != "mesh" || v.Folder != "INBOX" || v.Subject != "Hi" || v.Size != 1234 { + t.Fatalf("%+v", v) + } + if v.Sender.Name != "Bob" || v.Sender.Address != "bob@example.org" { + t.Fatalf("sender: %+v", v.Sender) + } + if len(v.Recipients) != 1 || v.Recipients[0].Address != "ops@example.org" { + t.Fatalf("recipients: %+v", v.Recipients) + } + if len(v.Attachments) != 1 || v.Attachments[0] != "invoice.pdf" { + t.Fatalf("attachments: %+v", v.Attachments) + } + if len(v.Flags) != 1 || v.Flags[0] != `\Seen` { + t.Fatalf("flags: %+v", v.Flags) + } + if strings.Contains(string(raw), "should never appear") { + t.Fatalf("the value carries the body: %s", raw) + } +} + +// 1282/AC2: a kept mail whose flags change has its key rewritten; a round with nothing changed writes nothing. +func TestMirrorRoundRewritesOnlyAChangedMailsKey(t *testing.T) { + ctx := context.Background() + accounts, st := accountsAndStore(t) + a := Message{Account: "mesh", Folder: "INBOX", UID: 1, MessageID: "a@example.org", Date: judgedAt, Subject: "a"} + b := Message{Account: "mesh", Folder: "INBOX", UID: 2, MessageID: "b@example.org", Date: judgedAt, Subject: "b"} + if err := st.Add(ctx, a, nil); err != nil { + t.Fatal(err) + } + if err := st.Add(ctx, b, nil); err != nil { + t.Fatal(err) + } + _ = st.SetFolder(ctx, Folder{Account: "mesh", Name: "INBOX", Messages: 2}) + + acctsBucket, mailsBucket := newMemMirrorState(), newMemMirrorState() + m := mirrorWithMails(accounts, st, acctsBucket, mailsBucket) + if err := m.Round(ctx); err != nil { + t.Fatal(err) + } + mailsBucket.calls = map[string]int{} + + // Nothing changed: a second round writes neither key again. + if err := m.Round(ctx); err != nil { + t.Fatal(err) + } + if mailsBucket.calls[mailKey("mesh", a)] != 0 || mailsBucket.calls[mailKey("mesh", b)] != 0 { + t.Fatalf("a round with nothing changed wrote a mail key: %v", mailsBucket.calls) + } + + // a's flags change: only a's key is rewritten. + if err := st.SetFlags(ctx, "mesh", "INBOX", 1, []string{`\Seen`}, nil); err != nil { + t.Fatal(err) + } + if err := m.Round(ctx); err != nil { + t.Fatal(err) + } + if mailsBucket.calls[mailKey("mesh", a)] != 1 { + t.Fatalf("a's key was not rewritten: %v", mailsBucket.calls) + } + if mailsBucket.calls[mailKey("mesh", b)] != 0 { + t.Fatalf("b's key was written though nothing of it changed: %v", mailsBucket.calls) + } +} + +// 1282/AC3 (removed): a mail removed from the index has its key deleted. +func TestMirrorRoundDeletesAMailsKeyForARemovedMail(t *testing.T) { + ctx := context.Background() + _, st := accountsAndStore(t) + msg := Message{Account: "mesh", Folder: "INBOX", UID: 1, MessageID: "m@example.org", Date: judgedAt, Subject: "s"} + if err := st.Add(context.Background(), msg, nil); err != nil { + t.Fatal(err) + } + bucket := newMemMirrorState() + mir := &Mirror{Mails: bucket} + if err := mir.roundMails(ctx, st, "mesh", 200); err != nil { + t.Fatal(err) + } + if _, ok, _ := bucket.Get(mailKey("mesh", msg)); !ok { + t.Fatalf("the mail's key was not written") + } + + if err := st.Remove(ctx, "mesh", "INBOX", 1, nil); err != nil { + t.Fatal(err) + } + if err := mir.roundMails(ctx, st, "mesh", 200); err != nil { + t.Fatal(err) + } + if _, ok, _ := bucket.Get(mailKey("mesh", msg)); ok { + t.Fatalf("the removed mail's key was not deleted") + } +} + +// 1282/AC3 (pushed out): a mail pushed out of the newest kept by a newer one has its key deleted. +func TestMirrorRoundDeletesAMailsKeyPushedOutOfTheNewestKept(t *testing.T) { + ctx := context.Background() + _, st := accountsAndStore(t) + old := Message{Account: "mesh", Folder: "INBOX", UID: 1, MessageID: "old@example.org", Date: judgedAt.Add(-time.Hour), Subject: "old"} + if err := st.Add(ctx, old, nil); err != nil { + t.Fatal(err) + } + bucket := newMemMirrorState() + mir := &Mirror{Mails: bucket} + if err := mir.roundMails(ctx, st, "mesh", 1); err != nil { + t.Fatal(err) + } + if _, ok, _ := bucket.Get(mailKey("mesh", old)); !ok { + t.Fatalf("the old mail's key was not written") + } + + newer := Message{Account: "mesh", Folder: "INBOX", UID: 2, MessageID: "new@example.org", Date: judgedAt, Subject: "new"} + if err := st.Add(ctx, newer, nil); err != nil { + t.Fatal(err) + } + if err := mir.roundMails(ctx, st, "mesh", 1); err != nil { + t.Fatal(err) + } + if _, ok, _ := bucket.Get(mailKey("mesh", old)); ok { + t.Fatalf("the old mail's key was not deleted once pushed out of the newest kept") + } + if _, ok, _ := bucket.Get(mailKey("mesh", newer)); !ok { + t.Fatalf("the newer mail's key was not written") + } +} + +// 1282/AC4: a mail with 500 recipients of 254-byte addresses, 50 attachments of 255-byte names and a +// 5,000-character multibyte subject is mirrored at most MostMailValue bytes, its subject at most +// MostMailSubject, each address and attachment name at most MostFolderName, and the dropped counts are in it. +func TestMirrorRoundTrimsAnOversizedMailValueToAtMost4KiB(t *testing.T) { + ctx := context.Background() + accounts, st := accountsAndStore(t) + var to []Person + for i := 0; i < 500; i++ { + to = append(to, Person{Address: strings.Repeat("r", 240) + "@example.org"}) // 254 bytes + } + var atts []AttachmentRef + for i := 0; i < 50; i++ { + atts = append(atts, AttachmentRef{Key: "k" + itoa(int64(i)), Name: strings.Repeat("n", 255), Size: 10}) + } + subject := strings.Repeat("é", 5000) + msg := Message{Account: "mesh", Folder: "INBOX", UID: 1, MessageID: "big@example.org", Date: judgedAt, + From: []Person{{Address: "sender@example.org"}}, To: to, Subject: subject, Size: 99, Attachments: atts} + if err := st.Add(ctx, msg, nil); err != nil { + t.Fatal(err) + } + _ = st.SetFolder(ctx, Folder{Account: "mesh", Name: "INBOX", Messages: 1}) + + acctsBucket, mailsBucket := newMemMirrorState(), newMemMirrorState() + m := mirrorWithMails(accounts, st, acctsBucket, mailsBucket) + if err := m.Round(ctx); err != nil { + t.Fatal(err) + } + + raw, ok, err := mailsBucket.Get(mailKey("mesh", msg)) + if err != nil || !ok { + t.Fatalf("no value kept: %v %v", ok, err) + } + if len(raw) > MostMailValue { + t.Fatalf("the value is %d bytes, more than %d", len(raw), MostMailValue) + } + var v MailValue + if err := json.Unmarshal(raw, &v); err != nil { + t.Fatal(err) + } + if len(v.Subject) > MostMailSubject { + t.Fatalf("the subject is %d bytes, more than %d", len(v.Subject), MostMailSubject) + } + if len(v.Recipients) > MostMailShown { + t.Fatalf("%d recipients shown, more than %d", len(v.Recipients), MostMailShown) + } + for _, r := range v.Recipients { + if len(r.Address) > MostFolderName { + t.Fatalf("a recipient's address is %d bytes, more than %d", len(r.Address), MostFolderName) + } + } + for _, a := range v.Attachments { + if len(a) > MostFolderName { + t.Fatalf("an attachment name is %d bytes, more than %d", len(a), MostFolderName) + } + } + if v.RecipientsDropped == 0 { + t.Fatalf("recipients-dropped is 0, though 500 recipients were given") + } + if v.AttachmentsDropped == 0 { + t.Fatalf("attachments-dropped is 0, though 50 attachments were given") + } +} + +// 1282/AC5: two mails without a message-id, in different folders or uids, digest apart (two keys); one +// message-id kept in two folders of one account digests the same (one key). +func TestMailDigestDistinguishesByFolderAndUIDAndMergesByMessageID(t *testing.T) { + a := Message{Folder: "INBOX", UID: 1} + b := Message{Folder: "INBOX", UID: 2} + if mailDigest(a) == mailDigest(b) { + t.Fatalf("two mails without a message-id, different uids, digest the same") + } + c := Message{Folder: "Archive", UID: 1} + if mailDigest(a) == mailDigest(c) { + t.Fatalf("two mails without a message-id, different folders, digest the same") + } + withID1 := Message{Folder: "INBOX", UID: 9, MessageID: "x@example.org"} + withID2 := Message{Folder: "Sent", UID: 10, MessageID: "x@example.org"} + if mailDigest(withID1) != mailDigest(withID2) { + t.Fatalf("one message-id kept in two folders of the account digests apart: %q vs %q", + mailDigest(withID1), mailDigest(withID2)) + } +} + +// 1282/AC6: with 82 accounts, every account keeps the same smaller number n of newest mails, with +// 82 x n x MostMailValue at most MostMailBytes, and n is written in each account's own `accounts` value +// (issue 1255's Kept field). +func TestMirrorRoundUses81AccountRuleForTheKeptNumber(t *testing.T) { + ctx := context.Background() + var accounts []Account + for i := 0; i < 82; i++ { + accounts = append(accounts, Account{Name: "a" + itoa(int64(i)), Kind: KindIMAP, Address: "a@example.org"}) + } + st := newMemStore() + for _, a := range accounts { + if err := st.SetState(ctx, AccountState{Account: a.Name, State: AccountCurrent}); err != nil { + t.Fatal(err) + } + } + + n := keptPerAccount(82) + if n >= DefaultKeptMails { + t.Fatalf("82 accounts did not reduce the kept number below %d: got %d", DefaultKeptMails, n) + } + if 82*n*MostMailValue > MostMailBytes { + t.Fatalf("82 x %d x %d bytes is more than %d", n, MostMailValue, MostMailBytes) + } + + acctsBucket, mailsBucket := newMemMirrorState(), newMemMirrorState() + m := mirrorWithMails(accounts, st, acctsBucket, mailsBucket) + if err := m.Round(ctx); err != nil { + t.Fatal(err) + } + for _, a := range accounts { + raw, ok, err := acctsBucket.Get(a.Name) + if err != nil || !ok { + t.Fatalf("%s: no value: %v %v", a.Name, ok, err) + } + var v AccountValue + if err := json.Unmarshal(raw, &v); err != nil { + t.Fatal(err) + } + if v.Kept != n { + t.Fatalf("%s: kept %d, want %d", a.Name, v.Kept, n) + } + } +} diff --git a/modules/mesh-mail/cmd/mesh-mail/pgstore.go b/modules/mesh-mail/cmd/mesh-mail/pgstore.go index 19832e74..bcb519a0 100644 --- a/modules/mesh-mail/cmd/mesh-mail/pgstore.go +++ b/modules/mesh-mail/cmd/mesh-mail/pgstore.go @@ -634,6 +634,13 @@ func (s *PgStore) Search(ctx context.Context, q Query) ([]Message, error) { return out, rows.Err() } +// Newest is 1282: one account's newest n mails, via Search's own ordering (newest first) and deduplication (no +// folder named, so dedupedMessagesSQL applies — the same copy Search itself would answer). Covered only by the +// integration tests (issue 1244): this package's unit tests use memStore. +func (s *PgStore) Newest(ctx context.Context, account string, n int) ([]Message, error) { + return s.Search(ctx, Query{Accounts: []string{account}, Limit: n}) +} + // GroupBySender is 1230/AC10: one line per sender among the messages searchWhere's filters match, by count // descending (address ascending to break ties), Limit bounding the number of lines. func (s *PgStore) GroupBySender(ctx context.Context, q Query) ([]SenderGroup, error) { diff --git a/modules/mesh-mail/cmd/mesh-mail/store.go b/modules/mesh-mail/cmd/mesh-mail/store.go index 2a1d6784..c221fe00 100644 --- a/modules/mesh-mail/cmd/mesh-mail/store.go +++ b/modules/mesh-mail/cmd/mesh-mail/store.go @@ -183,6 +183,9 @@ type Store interface { Get(ctx context.Context, account, folder string, uid uint32) (Message, bool, error) // Search finds messages, newest first by default (OrderBy, OrderDesc choose otherwise), without their bodies. Search(ctx context.Context, q Query) ([]Message, error) + // Newest is one account's newest n mails, without their bodies, each (account, message-id) counted once the + // same way Search answers with no folder named (novox/hq ADR 0439 §3, issue 1282): for the mail bucket. + Newest(ctx context.Context, account string, n int) ([]Message, error) // GroupBySender is 1230/AC10: one line per sender among the messages Query's other filters match, by count // descending, Limit bounding the number of lines. GroupBySender(ctx context.Context, q Query) ([]SenderGroup, error) @@ -379,6 +382,12 @@ func (s *memStore) Search(_ context.Context, q Query) ([]Message, error) { return out, nil } +// Newest is 1282: one account's newest n mails, via Search's own ordering (newest first) and deduplication (no +// folder named, so dedupeArchive applies — the same copy Search itself would answer). +func (s *memStore) Newest(ctx context.Context, account string, n int) ([]Message, error) { + return s.Search(ctx, Query{Accounts: []string{account}, Limit: n}) +} + // orderLess is 1230 (AC7, AC11, AC17's schema): date (the default when orderBy is empty), sender (by its first // address), subject or size, ascending unless desc; ties break on (account, folder, uid), always ascending, so // the order is stable regardless of the map order Go would otherwise give equal messages. diff --git a/modules/mesh-mail/module.json b/modules/mesh-mail/module.json index 5aa126c7..1eba0cf3 100644 --- a/modules/mesh-mail/module.json +++ b/modules/mesh-mail/module.json @@ -88,7 +88,8 @@ ], "state": [ "sends", - "accounts" + "accounts", + "mails" ], "emits": [ "mail.send.sent", From a9ad306693b404e5f472e4b410f37ce2598fa8d6 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 19 Oct 2026 22:07:36 +0200 Subject: [PATCH 04/18] Add the module mesh-mail-web, with its status verb (issue 1262) --- .../mesh-mail-web/cmd/mesh-mail-web/config.go | 77 ++++++++ .../cmd/mesh-mail-web/config_test.go | 56 ++++++ .../mesh-mail-web/cmd/mesh-mail-web/main.go | 31 +++ .../cmd/mesh-mail-web/manifest_test.go | 176 ++++++++++++++++++ .../mesh-mail-web/cmd/mesh-mail-web/tools.go | 106 +++++++++++ .../cmd/mesh-mail-web/tools_test.go | 64 +++++++ modules/mesh-mail-web/files/oidc.json | 5 + modules/mesh-mail-web/files/operators.json | 3 + modules/mesh-mail-web/go.mod | 5 + modules/mesh-mail-web/go.sum | 2 + modules/mesh-mail-web/module.json | 124 ++++++++++++ 11 files changed, 649 insertions(+) create mode 100644 modules/mesh-mail-web/cmd/mesh-mail-web/config.go create mode 100644 modules/mesh-mail-web/cmd/mesh-mail-web/config_test.go create mode 100644 modules/mesh-mail-web/cmd/mesh-mail-web/main.go create mode 100644 modules/mesh-mail-web/cmd/mesh-mail-web/manifest_test.go create mode 100644 modules/mesh-mail-web/cmd/mesh-mail-web/tools.go create mode 100644 modules/mesh-mail-web/cmd/mesh-mail-web/tools_test.go create mode 100644 modules/mesh-mail-web/files/oidc.json create mode 100644 modules/mesh-mail-web/files/operators.json create mode 100644 modules/mesh-mail-web/go.mod create mode 100644 modules/mesh-mail-web/go.sum create mode 100644 modules/mesh-mail-web/module.json diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/config.go b/modules/mesh-mail-web/cmd/mesh-mail-web/config.go new file mode 100644 index 00000000..8d419e81 --- /dev/null +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/config.go @@ -0,0 +1,77 @@ +package main + +// The trusted setting `operators` (novox/hq ADR 0440 §3, settlement 5 of design 59 milestone 1's prepare step): +// the Keycloak identities (the ID token's preferred_username) let in. Nobody by default; a setting that cannot +// be read, is empty, or holds something not its own shape lets nobody in either, and says why (issue 1262/AC4). + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "strings" +) + +// Env is where the mesh puts what the module reads, as the manifest's build env names it. +type Env struct { + Operators string // the trusted setting operators + OIDC string // the oidc-client binding, rendered (issue 1262's oidc.json) + Listen string // the port the mesh gave endpoint `web` (MESH_MAIL_WEB_LISTEN) + Node string // this machine's own name (MESH_NODE), for the internal host mail..internal +} + +func envFromOS() Env { + return Env{ + Operators: os.Getenv("MESH_MAIL_WEB_OPERATORS"), + OIDC: os.Getenv("MESH_MAIL_WEB_OIDC"), + Listen: os.Getenv("MESH_MAIL_WEB_LISTEN"), + Node: os.Getenv("MESH_NODE"), + } +} + +// OperatorsFile is the shape of the setting as the mesh writes it. +type OperatorsFile struct { + Operators []string `json:"operators"` +} + +// ReadOperators reads the operators setting (issue 1262/AC4): a setting missing, empty, or holding an unknown +// field lets nobody in, and the reason is kept (never silently "everyone", never a panic). +func ReadOperators(path string) ([]string, error) { + if path == "" { + return nil, errors.New("the module was started without its operators setting (MESH_MAIL_WEB_OPERATORS)") + } + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("the operators setting cannot be read: %w", err) + } + dec := json.NewDecoder(strings.NewReader(string(raw))) + dec.DisallowUnknownFields() + var f OperatorsFile + if err := dec.Decode(&f); err != nil { + return nil, fmt.Errorf("the operators setting is not the module's: %w", err) + } + return f.Operators, nil +} + +// Operators is the setting read fresh each time it is asked (never cached): it who may sign in, so a setting +// that cannot be read grants nobody rather than whoever it last allowed. +type Operators struct { + Path string +} + +// Allows says whether identity (the ID token's preferred_username) is let in, and why not when it is refused. +func (o Operators) Allows(identity string) (bool, string) { + if identity == "" { + return false, "no identity was given" + } + names, err := ReadOperators(o.Path) + if err != nil { + return false, err.Error() + } + for _, n := range names { + if n == identity { + return true, "" + } + } + return false, fmt.Sprintf("%q is not in the setting operators", identity) +} diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/config_test.go b/modules/mesh-mail-web/cmd/mesh-mail-web/config_test.go new file mode 100644 index 00000000..415a7df2 --- /dev/null +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/config_test.go @@ -0,0 +1,56 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } +} + +// 1262/AC4: the operators file missing, empty, or holding an unknown field lets nobody in, and says why. +func TestOperatorsLetsNobodyInWhenTheSettingIsMissingEmptyOrUnknown(t *testing.T) { + dir := t.TempDir() + + missing := Operators{Path: filepath.Join(dir, "does-not-exist.json")} + if ok, why := missing.Allows("alice"); ok || why == "" { + t.Fatalf("missing: allowed=%v why=%q", ok, why) + } + + empty := filepath.Join(dir, "empty.json") + writeFile(t, empty, ``) + if ok, why := (Operators{Path: empty}).Allows("alice"); ok || why == "" { + t.Fatalf("empty: allowed=%v why=%q", ok, why) + } + + unknown := filepath.Join(dir, "unknown.json") + writeFile(t, unknown, `{"operators": ["alice"], "surprise": true}`) + if ok, why := (Operators{Path: unknown}).Allows("alice"); ok || why == "" { + t.Fatalf("unknown field: allowed=%v why=%q", ok, why) + } + + // A well-formed setting still lets nobody in who is not named. + good := filepath.Join(dir, "good.json") + writeFile(t, good, `{"operators": ["alice"]}`) + if ok, _ := (Operators{Path: good}).Allows("bob"); ok { + t.Fatalf("bob was let in by a setting naming only alice") + } + if ok, why := (Operators{Path: good}).Allows("alice"); !ok || why != "" { + t.Fatalf("alice: allowed=%v why=%q", ok, why) + } +} + +// Also 1262/AC4: an empty operators list (the default) lets nobody in. +func TestOperatorsDefaultEmptyListLetsNobodyIn(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "operators.json") + writeFile(t, path, `{"operators": []}`) + if ok, why := (Operators{Path: path}).Allows("anyone"); ok || why == "" { + t.Fatalf("allowed=%v why=%q", ok, why) + } +} diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/main.go b/modules/mesh-mail-web/cmd/mesh-mail-web/main.go new file mode 100644 index 00000000..328a6c6b --- /dev/null +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/main.go @@ -0,0 +1,31 @@ +// mesh-mail-web: the mail web client (novox/hq ADR 0440), built from mesh-mail's source as mesh-mail-notifier is +// (novox/hq ADR 0198 §1, ADR 0175 §1). A Go bundle the node's runtime launches, its own account (runs-as): it +// claims mail-webclient (open, status), reaches mail-store's seat verbs (accounts, search, read, send, +// send-status), and — once issue 1283 lands — answers its own HTTP endpoint `web`. No database, no state, no +// writable directory beyond the one the runtime account needs for its own settings. What this module says, it +// says on stderr; stdout is the MCP channel. +package main + +import ( + "fmt" + "io" + "os" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +func main() { + b := &Bundle{Env: envFromOS()} + if err := run(b, os.Stdin, os.Stdout); err != nil { + logf("%v", err) + os.Exit(1) + } +} + +// run is main's startup path, factored so a test can run it over a pipe instead of the real stdin/stdout +// (mirroring mesh-mail-notifier's own run). +func run(b *Bundle, in io.Reader, out io.Writer) error { + return stdio.ServeOn("", b.Tools(), in, out) +} diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/manifest_test.go b/modules/mesh-mail-web/cmd/mesh-mail-web/manifest_test.go new file mode 100644 index 00000000..6f670ec3 --- /dev/null +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/manifest_test.go @@ -0,0 +1,176 @@ +package main + +// The manifest says what the code does (novox/hq ADR 0440 §1, §2, §3, §8; issue 1262). +// +// 1262/AC5: the repository's own way to run "the controller's manifest rules" is merge-check.sh, which shells +// out to the real controller binary ("$MESH_GATE" module check modules/*/module.json, merge-check.sh:127) — not +// something this package can call as a unit test, and this batch's brief refuses running merge-check.sh or any +// integration test. mesh-mail's and mesh-mail-notifier's own manifest_test.go files do not run the controller +// either: they assert the manifest's shape by hand, field by field, in Go. There is no unit-level way in this +// repository to run the controller's actual module-check rules, so TestTheManifestSaysWhatTheCodeDoes below is +// as far as the repo allows for AC5: it is the same hand-written shape assertion the sibling modules use, now +// covering every field AC1 names. The real module check still runs once, for real, through the lead's +// mesh-controller.try. + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "testing" +) + +type seat struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` +} + +type listen struct { + Name string `json:"name"` + Port int `json:"port"` + Protocol string `json:"protocol"` + From string `json:"from"` +} + +type configEntry struct { + Type string `json:"type"` + Trusted bool `json:"trusted"` + Default []string `json:"default"` +} + +type manifest struct { + Module string `json:"module"` + RunsAs string `json:"runs-as"` + Claims []seat `json:"claims"` + Invokes []string `json:"invokes"` + Reads []string `json:"reads"` + Requires []string `json:"requires"` + Contributes map[string]map[string]any `json:"contributes"` + Binds map[string]string `json:"binds"` + Secrets map[string]string `json:"secrets"` + Listens []listen `json:"listens"` + Config map[string]configEntry `json:"config"` + Tools []string `json:"tools"` + State []any `json:"state"` + OwnSecrets map[string]string `json:"own-secrets"` + SecretsOwner string `json:"secrets-owner"` + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []struct { + From string `json:"from"` + Binary string `json:"binary"` + Loads []string `json:"loads"` + Env map[string]string `json:"env"` + } `json:"artifacts"` + } `json:"build"` +} + +func readManifest(t *testing.T) manifest { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m manifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + return m +} + +// 1262/AC1 (and, as far as the repo allows, AC5 — see the file's own comment above). +func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { + m := readManifest(t) + + if m.Module != "mesh-mail-web" { + t.Fatalf("module %q", m.Module) + } + if m.RunsAs != "mesh-mail-web" { + t.Fatalf("runs-as %q, want its own account", m.RunsAs) + } + wantClaims := []seat{{Name: "mail-webclient", Scope: "mesh", Serves: []string{"open", "status"}}} + if !reflect.DeepEqual(m.Claims, wantClaims) { + t.Fatalf("claims %+v, want %+v", m.Claims, wantClaims) + } + wantInvokes := []string{ + "seat:mail-store.accounts", "seat:mail-store.search", "seat:mail-store.read", + "seat:mail-store.send", "seat:mail-store.send-status", + } + if !reflect.DeepEqual(m.Invokes, wantInvokes) { + t.Fatalf("invokes %v, want exactly the five mail-store verbs %v", m.Invokes, wantInvokes) + } + if !reflect.DeepEqual(m.Reads, []string{"mesh-mail.accounts"}) { + t.Fatalf("reads %v, want [mesh-mail.accounts]", m.Reads) + } + if !contains(m.Requires, "route") || !contains(m.Requires, "oidc-client") { + t.Fatalf("requires %v, want route and oidc-client", m.Requires) + } + if len(m.Listens) != 1 || m.Listens[0].Name != "web" || m.Listens[0].From != "mesh" || m.Listens[0].Protocol != "tcp" { + t.Fatalf("listens %+v, want exactly one, named web, from the mesh only", m.Listens) + } + op, ok := m.Config["operators"] + if !ok || op.Type != "array" || !op.Trusted || len(op.Default) != 0 { + t.Fatalf("config.operators %+v, want a trusted array defaulting to nobody", op) + } + // No database, no state bucket, no writable directory beyond the account's own state dir: only the + // account (a user resource, not writable), the state directory, and the two files rendered into it. + if len(m.State) != 0 { + t.Fatalf("state %v, want none", m.State) + } + for _, dbish := range []string{"postgres-database", "s3-bucket"} { + if contains(m.Requires, dbish) { + t.Fatalf("requires %v, want no database or object store", m.Requires) + } + } + wantResourceIDs := map[string]bool{"account": true, "state": true, "operators": true, "oidc-config": true} + if len(m.Resources) != len(wantResourceIDs) { + t.Fatalf("resources %v, want exactly %v", m.Resources, wantResourceIDs) + } + for _, r := range m.Resources { + id := r["id"].(string) + if !wantResourceIDs[id] { + t.Fatalf("an unexpected resource %q: no writable directory beyond the state dir is wanted", id) + } + if id != "account" && r["type"] == "directory" && id != "state" { + t.Fatalf("an extra writable directory %q", id) + } + } + + // The manifest's own tools match the code's (mirroring mesh-mail's and mesh-mail-notifier's own check). + var names []string + for _, tool := range (&Bundle{}).Tools() { + names = append(names, tool.Name) + } + if !reflect.DeepEqual(m.Tools, names) { + t.Fatalf("tools %v, the code serves %v", m.Tools, names) + } + + a := m.Build.Artifacts + if len(a) != 2 || a[0].From != "cmd/mesh-mail-web" || a[0].Binary != "mesh-mail-web" || !reflect.DeepEqual(a[0].Loads, []string{"mesh-mail-web"}) { + t.Fatalf("build %+v", m.Build) + } + for k, v := range map[string]string{"MESH_MAIL_WEB_OPERATORS": "operators", "MESH_MAIL_WEB_OIDC": "oidc-config"} { + found := false + for _, r := range m.Resources { + if r["id"] == v { + found = a[0].Env[k] == r["path"] + } + } + if !found { + t.Fatalf("the bundle reads %s at %s, which does not match resource %s's path", k, a[0].Env[k], v) + } + } + if a[0].Env["MESH_MAIL_WEB_LISTEN"] != "${port:web}" { + t.Fatalf("MESH_MAIL_WEB_LISTEN is %q, want the web endpoint's port", a[0].Env["MESH_MAIL_WEB_LISTEN"]) + } +} + +func contains(list []string, s string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/tools.go b/modules/mesh-mail-web/cmd/mesh-mail-web/tools.go new file mode 100644 index 00000000..53638f44 --- /dev/null +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/tools.go @@ -0,0 +1,106 @@ +package main + +// The module's seat verbs (novox/hq ADR 0440 §8): mail-webclient.status (whether it serves, and which accounts +// it is granted) and mail-webclient.open (issue 1283: the page's address). mail_web_health is its own health +// tool, read-only, naming nothing of a page or an account. + +import ( + "context" + "encoding/json" + "fmt" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Tool names, as the manifest declares them. +const ( + StatusTool = "mail-webclient.status" + HealthTool = "mail_web_health" +) + +// AccountsAsk is how the Bundle reaches seat:mail-store.accounts: the bus's own stdio.Ask, unless a test gives +// its own fake (so a unit test needs no runtime). +type AccountsAsk func(ctx context.Context) (json.RawMessage, error) + +func sdkAccountsAsk(ctx context.Context) (json.RawMessage, error) { + return stdio.Ask("seat:mail-store.accounts", map[string]any{}) +} + +// Bundle is what the tools reach: the module's own settings, and the seat it calls. +type Bundle struct { + Env Env + Accounts AccountsAsk + Now func() time.Time +} + +func (b *Bundle) accountsAsk(ctx context.Context) (json.RawMessage, error) { + if b.Accounts != nil { + return b.Accounts(ctx) + } + return sdkAccountsAsk(ctx) +} + +func (b *Bundle) now() time.Time { + if b.Now != nil { + return b.Now() + } + return time.Now() +} + +// Tools are the module's tools. +func (b *Bundle) Tools() []stdio.Tool { + return []stdio.Tool{ + {Name: StatusTool, + Description: "Whether mesh-mail-web serves, and the mail accounts it is granted through " + + "seat:mail-store.accounts (novox/hq ADR 0440 §8). Read-only.", + Input: schema(nil, map[string]any{}), + Run: b.statusTool}, + {Name: HealthTool, + Description: "How mesh-mail-web is: healthy once its own settings can be read, unhealthy naming why " + + "otherwise. Read-only.", + Input: schema(nil, map[string]any{}), + Run: b.healthTool}, + } +} + +// statusTool is issue 1262/AC2, AC3: mail-store's own accounts answer, read as the granted accounts; an ask +// mail-store does not answer, or refuses, answers "it does not serve" and why, never a crash. +func (b *Bundle) statusTool(map[string]any) (any, error) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + raw, err := b.accountsAsk(ctx) + if err != nil { + return map[string]any{"serves": false, "why": fmt.Sprintf("mail-store does not answer: %v", err)}, nil + } + var answer struct { + Accounts []struct { + Account string `json:"account"` + } `json:"accounts"` + } + if err := json.Unmarshal(raw, &answer); err != nil { + return map[string]any{"serves": false, "why": fmt.Sprintf("mail-store's answer could not be read: %v", err)}, nil + } + names := make([]string, 0, len(answer.Accounts)) + for _, a := range answer.Accounts { + names = append(names, a.Account) + } + return map[string]any{"serves": true, "accounts": names}, nil +} + +func (b *Bundle) healthTool(map[string]any) (any, error) { + if _, err := ReadOperators(b.Env.Operators); err != nil { + return map[string]any{"healthy": false, "why": err.Error()}, nil + } + return map[string]any{"healthy": true}, nil +} + +// ---- schema helpers (mirroring mesh-mail's tools.go) ----------------------------------------------------------- + +func schema(required []string, props map[string]any) map[string]any { + s := map[string]any{"type": "object", "properties": props, "additionalProperties": false} + if len(required) > 0 { + s["required"] = required + } + return s +} diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/tools_test.go b/modules/mesh-mail-web/cmd/mesh-mail-web/tools_test.go new file mode 100644 index 00000000..d4986149 --- /dev/null +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/tools_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "testing" +) + +// 1262/AC2: mail-store answers accounts a and b; status answers that it serves, and is granted a and b. +func TestStatusAnswersItServesAndTheGrantedAccounts(t *testing.T) { + b := &Bundle{Accounts: func(context.Context) (json.RawMessage, error) { + return json.Marshal(map[string]any{"accounts": []map[string]any{ + {"account": "a", "kind": "imap"}, {"account": "b", "kind": "imap"}, + }}) + }} + out, err := b.statusTool(nil) + if err != nil { + t.Fatal(err) + } + m := out.(map[string]any) + if m["serves"] != true { + t.Fatalf("serves: %v", m["serves"]) + } + got, _ := m["accounts"].([]string) + if len(got) != 2 || got[0] != "a" || got[1] != "b" { + t.Fatalf("accounts: %v", m["accounts"]) + } +} + +// 1262/AC3: mail-store does not answer (or refuses); status answers that it does not serve, and why, and does +// not panic. +func TestStatusAnswersItDoesNotServeWhenMailStoreRefuses(t *testing.T) { + b := &Bundle{Accounts: func(context.Context) (json.RawMessage, error) { + return nil, errors.New("refused: mesh-mail-web is not granted any account") + }} + out, err := b.statusTool(nil) + if err != nil { + t.Fatal(err) + } + m := out.(map[string]any) + if m["serves"] != false { + t.Fatalf("serves: %v", m["serves"]) + } + why, _ := m["why"].(string) + if why == "" { + t.Fatalf("no reason given: %v", m) + } +} + +// Same AC3, the other half: mail-store answers something this module cannot read. +func TestStatusAnswersItDoesNotServeWhenMailStoresAnswerCannotBeRead(t *testing.T) { + b := &Bundle{Accounts: func(context.Context) (json.RawMessage, error) { + return json.RawMessage(`{"accounts": "not a list"}`), nil + }} + out, err := b.statusTool(nil) + if err != nil { + t.Fatal(err) + } + m := out.(map[string]any) + if m["serves"] != false || m["why"] == "" { + t.Fatalf("%v", m) + } +} diff --git a/modules/mesh-mail-web/files/oidc.json b/modules/mesh-mail-web/files/oidc.json new file mode 100644 index 00000000..f8646087 --- /dev/null +++ b/modules/mesh-mail-web/files/oidc.json @@ -0,0 +1,5 @@ +{ + "client-id": "${bound:oidc-client:as}", + "client-secret": "${secret:oidc-client}", + "issuer": "${bound:oidc-client:issuer}" +} diff --git a/modules/mesh-mail-web/files/operators.json b/modules/mesh-mail-web/files/operators.json new file mode 100644 index 00000000..dea781c4 --- /dev/null +++ b/modules/mesh-mail-web/files/operators.json @@ -0,0 +1,3 @@ +{ + "operators": [] +} diff --git a/modules/mesh-mail-web/go.mod b/modules/mesh-mail-web/go.mod new file mode 100644 index 00000000..baf38bd6 --- /dev/null +++ b/modules/mesh-mail-web/go.mod @@ -0,0 +1,5 @@ +module mesh-mail-web + +go 1.26.0 + +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019062050-34392e74916c diff --git a/modules/mesh-mail-web/go.sum b/modules/mesh-mail-web/go.sum new file mode 100644 index 00000000..25aaa963 --- /dev/null +++ b/modules/mesh-mail-web/go.sum @@ -0,0 +1,2 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019062050-34392e74916c h1:/f6f2ugss172hDJSL3EFZqb9PdfO5DKzYVfPiX2LdAM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019062050-34392e74916c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/mesh-mail-web/module.json b/modules/mesh-mail-web/module.json new file mode 100644 index 00000000..544ed697 --- /dev/null +++ b/modules/mesh-mail-web/module.json @@ -0,0 +1,124 @@ +{ + "module": "mesh-mail-web", + "version": "1", + "slug": "mail-web", + "runs-as": "mesh-mail-web", + "claims": [ + { + "name": "mail-webclient", + "scope": "mesh", + "serves": ["open", "status"] + } + ], + "invokes": [ + "seat:mail-store.accounts", + "seat:mail-store.search", + "seat:mail-store.read", + "seat:mail-store.send", + "seat:mail-store.send-status" + ], + "reads": ["mesh-mail.accounts"], + "requires": ["route", "oidc-client"], + "contributes": { + "route": { + "label": "mail", + "endpoint": "web" + }, + "oidc-client": { + "label": "mail", + "endpoint": "web", + "callback": "/signed-in" + } + }, + "binds": { + "route": "${dir:state}/route.json", + "oidc-client": "${dir:state}/oidc-bound.json" + }, + "secrets": { + "oidc-client": "${dir:state}/oidc-bound.secret" + }, + "listens": [ + { + "name": "web", + "port": 8099, + "protocol": "tcp", + "from": "mesh", + "why": "the mail web client's pages, over http; only the route proxy reaches it, and the operator's own browser through the internal name (novox/hq ADR 0440 §2) — its public name, if any, is a route grant" + } + ], + "config": { + "operators": { + "type": "array", + "description": "The Keycloak identities (the ID token's preferred_username) let to sign in; nobody by default (novox/hq ADR 0440 §3).", + "trusted": true, + "default": [] + } + }, + "tools": [ + "mail-webclient.status", + "mail_web_health" + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-mail-web", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-mail-web", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-mail-web" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-mail-web", + "place": ".", + "unknown": "program" + }, + { + "id": "operators", + "trusted": true, + "type": "file", + "path": "${dir:state}/operators.json", + "mode": "0600", + "owner": "mesh-mail-web", + "merge": "json", + "template": "files/operators.json", + "takes-config": ["operators"] + }, + { + "id": "oidc-config", + "type": "file", + "path": "${dir:state}/oidc.json", + "mode": "0600", + "owner": "mesh-mail-web", + "template": "files/oidc.json" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-mail-web", + "binary": "mesh-mail-web", + "loads": ["mesh-mail-web"], + "env": { + "MESH_MAIL_WEB_OPERATORS": "${dir:state}/operators.json", + "MESH_MAIL_WEB_OIDC": "${dir:state}/oidc.json", + "MESH_MAIL_WEB_LISTEN": "${port:web}" + } + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } +} From 68c141849febdc725a330524e9c4dc885b1902d9 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 19 Oct 2026 22:09:49 +0200 Subject: [PATCH 05/18] Serve mesh-mail-web's HTTP endpoint: internal host check, CSP, open (issue 1283) --- .../mesh-mail-web/cmd/mesh-mail-web/main.go | 15 ++ .../mesh-mail-web/cmd/mesh-mail-web/server.go | 107 ++++++++++++++ .../cmd/mesh-mail-web/server_test.go | 136 ++++++++++++++++++ .../mesh-mail-web/cmd/mesh-mail-web/tools.go | 13 ++ modules/mesh-mail-web/module.json | 1 + 5 files changed, 272 insertions(+) create mode 100644 modules/mesh-mail-web/cmd/mesh-mail-web/server.go create mode 100644 modules/mesh-mail-web/cmd/mesh-mail-web/server_test.go diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/main.go b/modules/mesh-mail-web/cmd/mesh-mail-web/main.go index 328a6c6b..46009ecb 100644 --- a/modules/mesh-mail-web/cmd/mesh-mail-web/main.go +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/main.go @@ -9,6 +9,7 @@ package main import ( "fmt" "io" + "net/http" "os" stdio "git.novox.be/novox/mesh-sdk/go" @@ -18,12 +19,26 @@ func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } func main() { b := &Bundle{Env: envFromOS()} + go serveHTTP(b) if err := run(b, os.Stdin, os.Stdout); err != nil { logf("%v", err) os.Exit(1) } } +// serveHTTP starts the HTTP endpoint `web` (issue 1283), on the port the mesh gave it; a listen it cannot +// start is logged, never fatal to the seat verbs stdio.Serve answers on the MCP channel. +func serveHTTP(b *Bundle) { + if b.Env.Listen == "" { + logf("[mesh-mail-web] started without MESH_MAIL_WEB_LISTEN; its HTTP endpoint is not served") + return + } + s := NewServer(InternalHost(b.Env.Node)) + if err := http.ListenAndServe(":"+b.Env.Listen, s); err != nil { + logf("[mesh-mail-web] its HTTP endpoint ended: %v", err) + } +} + // run is main's startup path, factored so a test can run it over a pipe instead of the real stdin/stdout // (mirroring mesh-mail-notifier's own run). func run(b *Bundle, in io.Reader, out io.Writer) error { diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/server.go b/modules/mesh-mail-web/cmd/mesh-mail-web/server.go new file mode 100644 index 00000000..087a9963 --- /dev/null +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/server.go @@ -0,0 +1,107 @@ +package main + +// The web server (novox/hq ADR 0440 §2, §5; issue 1283): answers only its internal name mail..internal +// (404 for any other Host — a public name, an IP, or none), sets a content security policy allowing only its +// own origin on every answer, and writes nothing to disk. Pages are rendered through html/template only (its +// auto-escaping), never written to a file. The seat verb open (issue 1262's claim) answers the page's address: +// the root, or one mail's, by account, folder and uid (mail_read's own addressing, tools.go of mesh-mail). + +import ( + "fmt" + "html/template" + "net" + "net/http" + "net/url" + "strconv" +) + +// OpenTool is the mail-webclient seat's open verb. +const OpenTool = "mail-webclient.open" + +// CSP is ADR 0440 §5's own policy, set on every answer: nothing loaded from elsewhere, and this page never +// framed by another site. +const CSP = "default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self'; " + + "frame-ancestors 'none'; base-uri 'none'; form-action 'self'" + +// InternalHost is this machine's own internal name for the module (ADR 0440 §2): the operator's own setting +// (set on the machine, outside the manifest) assigns it the label `mail`, so it is always mail..internal. +func InternalHost(node string) string { return "mail." + node + ".internal" } + +// Server answers mesh-mail-web's HTTP endpoint: a Host check and a CSP on every answer (issue 1283), wrapping +// a plain mux that later issues (1284's form check, 1264's pages, ...) add routes to. +type Server struct { + Host string // InternalHost(node) + Mux *http.ServeMux +} + +// NewServer is a Server for host, with the root page issue 1283 itself answers (a stand-in: the real accounts +// and folder pages are issue 1264's). +func NewServer(host string) *Server { + s := &Server{Host: host, Mux: http.NewServeMux()} + s.Mux.HandleFunc("/", s.rootPage) + return s +} + +// ServeHTTP is issue 1283/AC1, AC2: every answer carries the CSP; a request for any Host but this server's own +// internal name is 404, naming nothing of the mail. +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Security-Policy", CSP) + host := r.Host + if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + if host != s.Host { + http.NotFound(w, r) + return + } + s.Mux.ServeHTTP(w, r) +} + +// rootTemplate is 1283/AC4's own stand-in page: no account or mail content yet (issue 1264), only enough to +// prove the server answers at all, through html/template (auto-escaping), never a file written to disk. +var rootTemplate = template.Must(template.New("root").Parse( + `Mail

mesh-mail-web is up.

`)) + +func (s *Server) rootPage(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _ = rootTemplate.Execute(w, nil) +} + +// openTool is issue 1262's claimed verb open, built here (1283) now that the server's own host exists: with +// nothing given, the root; with account, folder and uid together, that mail's own address. +func (b *Bundle) openTool(args map[string]any) (any, error) { + if err := only(args, "account", "folder", "uid"); err != nil { + return nil, err + } + account, _ := args["account"].(string) + folder, _ := args["folder"].(string) + uidArg, hasUID := args["uid"] + + base := "https://" + InternalHost(b.Env.Node) + "/" + if account == "" && folder == "" && !hasUID { + return map[string]any{"url": base}, nil + } + uid, isNum := uidArg.(float64) + if account == "" || folder == "" || !isNum || uid <= 0 { + return nil, fmt.Errorf("%s needs account, folder and uid together, or none of them", OpenTool) + } + q := url.Values{"account": {account}, "folder": {folder}, "uid": {strconv.FormatFloat(uid, 'f', 0, 64)}} + return map[string]any{"url": "https://" + InternalHost(b.Env.Node) + "/mail?" + q.Encode()}, nil +} + +// only refuses an argument the tool does not take (mirroring mesh-mail's own tools.go). +func only(args map[string]any, takes ...string) error { + for k := range args { + ok := false + for _, t := range takes { + if k == t { + ok = true + break + } + } + if !ok { + return fmt.Errorf("the argument %q is not taken", k) + } + } + return nil +} diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/server_test.go b/modules/mesh-mail-web/cmd/mesh-mail-web/server_test.go new file mode 100644 index 00000000..b2382043 --- /dev/null +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/server_test.go @@ -0,0 +1,136 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +// 1283/AC1: a request whose Host is not mail..internal — a public name, an IP, or none — is answered +// 404, naming nothing of the mail. +func TestServerAnswers404ForAnyHostButItsOwnInternalName(t *testing.T) { + s := NewServer(InternalHost("g14")) + for _, host := range []string{"example.com", "203.0.113.5", "", "mail.other-node.internal"} { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/", nil) + r.Host = host + s.ServeHTTP(w, r) + if w.Code != http.StatusNotFound { + t.Fatalf("host %q: status %d, want 404", host, w.Code) + } + if strings.Contains(strings.ToLower(w.Body.String()), "mail") { + t.Fatalf("host %q: the 404 body names mail: %s", host, w.Body.String()) + } + } +} + +// Its own internal name is answered (not 404). +func TestServerAnswersItsOwnInternalName(t *testing.T) { + s := NewServer(InternalHost("g14")) + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/", nil) + r.Host = InternalHost("g14") + s.ServeHTTP(w, r) + if w.Code != http.StatusOK { + t.Fatalf("status %d, want 200", w.Code) + } +} + +// A Host with a port still matches, port and all (the mesh's own listen carries one). +func TestServerAnswersItsOwnInternalNameWithAPort(t *testing.T) { + s := NewServer(InternalHost("g14")) + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/", nil) + r.Host = InternalHost("g14") + ":8099" + s.ServeHTTP(w, r) + if w.Code != http.StatusOK { + t.Fatalf("status %d, want 200", w.Code) + } +} + +// 1283/AC2: every answer (200 and 404 alike) carries the content security policy. +func TestServerSetsTheContentSecurityPolicyOnEveryAnswer(t *testing.T) { + s := NewServer(InternalHost("g14")) + cases := []string{"example.com", InternalHost("g14")} + for _, host := range cases { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/", nil) + r.Host = host + s.ServeHTTP(w, r) + if got := w.Header().Get("Content-Security-Policy"); got != CSP { + t.Fatalf("host %q: CSP %q, want %q", host, got, CSP) + } + } +} + +// 1283/AC3: open with nothing given answers the root; with account, folder and uid together, that mail's own +// address. +func TestOpenAnswersTheRootOrOneMailsAddress(t *testing.T) { + b := &Bundle{Env: Env{Node: "g14"}} + out, err := b.openTool(map[string]any{}) + if err != nil { + t.Fatal(err) + } + if got := out.(map[string]any)["url"]; got != "https://mail.g14.internal/" { + t.Fatalf("root url: %v", got) + } + + out, err = b.openTool(map[string]any{"account": "mesh", "folder": "INBOX", "uid": float64(7)}) + if err != nil { + t.Fatal(err) + } + url := out.(map[string]any)["url"].(string) + if !strings.HasPrefix(url, "https://mail.g14.internal/mail?") || + !strings.Contains(url, "account=mesh") || !strings.Contains(url, "folder=INBOX") || !strings.Contains(url, "uid=7") { + t.Fatalf("mail url: %s", url) + } + + if _, err := b.openTool(map[string]any{"account": "mesh"}); err == nil { + t.Fatalf("account alone, with no folder or uid, was not refused") + } +} + +// 1283/AC4: with the working directory and home read-only, the server still starts and answers a page (it +// writes nothing to disk). +func TestServerStartsAndAnswersAPageWithACwdAndHomeReadOnly(t *testing.T) { + ro := t.TempDir() + if err := os.Chmod(ro, 0o500); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(ro, 0o700) }) + + oldWd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + oldHome := os.Getenv("HOME") + t.Cleanup(func() { + _ = os.Chdir(oldWd) + _ = os.Setenv("HOME", oldHome) + }) + if err := os.Chdir(ro); err != nil { + t.Fatal(err) + } + _ = os.Setenv("HOME", filepath.Join(ro, "home-does-not-exist")) + + s := NewServer(InternalHost("g14")) + srv := httptest.NewServer(s) + defer srv.Close() + + req, err := http.NewRequest(http.MethodGet, srv.URL+"/", nil) + if err != nil { + t.Fatal(err) + } + req.Host = InternalHost("g14") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status %d, want 200", resp.StatusCode) + } +} diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/tools.go b/modules/mesh-mail-web/cmd/mesh-mail-web/tools.go index 53638f44..495e79a2 100644 --- a/modules/mesh-mail-web/cmd/mesh-mail-web/tools.go +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/tools.go @@ -51,6 +51,16 @@ func (b *Bundle) now() time.Time { // Tools are the module's tools. func (b *Bundle) Tools() []stdio.Tool { return []stdio.Tool{ + {Name: OpenTool, + Description: "The mail web client's own address (novox/hq ADR 0440 §8): with nothing given, the " + + "page's root; with account, folder and uid together (mail_read's own addressing), that mail's " + + "own page. Read-only; answers a URL, never a page's content.", + Input: schema(nil, map[string]any{ + "account": text("the account, by its name in mail_accounts"), + "folder": text("the folder, such as INBOX"), + "uid": integer("the message's uid in that folder"), + }), + Run: b.openTool}, {Name: StatusTool, Description: "Whether mesh-mail-web serves, and the mail accounts it is granted through " + "seat:mail-store.accounts (novox/hq ADR 0440 §8). Read-only.", @@ -104,3 +114,6 @@ func schema(required []string, props map[string]any) map[string]any { } return s } + +func text(d string) map[string]any { return map[string]any{"type": "string", "description": d} } +func integer(d string) map[string]any { return map[string]any{"type": "integer", "description": d} } diff --git a/modules/mesh-mail-web/module.json b/modules/mesh-mail-web/module.json index 544ed697..a0dd2d7f 100644 --- a/modules/mesh-mail-web/module.json +++ b/modules/mesh-mail-web/module.json @@ -55,6 +55,7 @@ } }, "tools": [ + "mail-webclient.open", "mail-webclient.status", "mail_web_health" ], From 8e0ef84060fd0481cdc62d7c5ea4d5d2a35241c4 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 19 Oct 2026 22:16:15 +0200 Subject: [PATCH 06/18] Sign in through Keycloak: code flow with PKCE, verified ID token, 12-hour session (issue 1263) Library choice: golang.org/x/oauth2 for the code exchange, github.com/coreos/go-oidc/v3 for the ID token's verification (issuer, audience, expiry, signature from the issuer's JWKS). --- .../mesh-mail-web/cmd/mesh-mail-web/auth.go | 207 ++++++++++ .../cmd/mesh-mail-web/auth_test.go | 364 ++++++++++++++++++ .../mesh-mail-web/cmd/mesh-mail-web/main.go | 38 +- .../mesh-mail-web/cmd/mesh-mail-web/server.go | 15 +- .../cmd/mesh-mail-web/server_test.go | 10 +- .../cmd/mesh-mail-web/sessions.go | 143 +++++++ modules/mesh-mail-web/go.mod | 8 +- modules/mesh-mail-web/go.sum | 6 + 8 files changed, 779 insertions(+), 12 deletions(-) create mode 100644 modules/mesh-mail-web/cmd/mesh-mail-web/auth.go create mode 100644 modules/mesh-mail-web/cmd/mesh-mail-web/auth_test.go create mode 100644 modules/mesh-mail-web/cmd/mesh-mail-web/sessions.go diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/auth.go b/modules/mesh-mail-web/cmd/mesh-mail-web/auth.go new file mode 100644 index 00000000..92ee1a97 --- /dev/null +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/auth.go @@ -0,0 +1,207 @@ +package main + +// Keycloak sign-in (novox/hq ADR 0440 §3, issue 1263): the authorisation-code flow with state and PKCE against +// the bound issuer, the callback at /signed-in, the ID token verified (issuer, audience, expiry, signature from +// the issuer's JWKS), the identity (its preferred_username) checked against the trusted setting operators, and +// a session kept in memory only, its cookie Secure, HttpOnly, SameSite=Strict, ending SessionLifetime after +// sign-in. Library choice, said as the brief asks: golang.org/x/oauth2 for the code exchange and +// github.com/coreos/go-oidc/v3 for the ID token's verification (issuer, audience, expiry and signature against +// the issuer's own JWKS) — chosen over hand-rolling JWT checks here, since go-oidc's verifier is the +// already-reviewed, standard implementation of exactly what ADR 0440 §3 asks for. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "strings" + "time" + + "github.com/coreos/go-oidc/v3/oidc" + "golang.org/x/oauth2" +) + +// SignedInPath is the callback the issuer redirects back to (the oidc-client contribution's own callback, +// module.json). +const SignedInPath = "/signed-in" + +// OIDCConfig is the oidc-client binding's own facts this module reads (issue 1262's oidc.json). +type OIDCConfig struct { + ClientID string `json:"client-id"` + ClientSecret string `json:"client-secret"` + Issuer string `json:"issuer"` +} + +// ReadOIDCConfig reads the rendered oidc-client binding. +func ReadOIDCConfig(path string) (OIDCConfig, error) { + if path == "" { + return OIDCConfig{}, errors.New("the module was started without its oidc setting (MESH_MAIL_WEB_OIDC)") + } + raw, err := os.ReadFile(path) + if err != nil { + return OIDCConfig{}, fmt.Errorf("the oidc setting cannot be read: %w", err) + } + dec := json.NewDecoder(strings.NewReader(string(raw))) + dec.DisallowUnknownFields() + var c OIDCConfig + if err := dec.Decode(&c); err != nil { + return OIDCConfig{}, fmt.Errorf("the oidc setting is not the module's: %w", err) + } + if c.ClientID == "" || c.Issuer == "" { + return OIDCConfig{}, errors.New("the oidc setting names no client id or issuer") + } + return c, nil +} + +// Authenticator is how this module reaches the issuer: build the redirect to it, exchange a code for tokens, +// and verify the ID token it names. +type Authenticator struct { + OAuth2 oauth2.Config + Verifier *oidc.IDTokenVerifier +} + +// NewAuthenticator discovers the issuer (its authorisation, token and JWKS endpoints, novox/hq issue 1263) and +// builds an Authenticator for it. redirectURL is this module's own callback, https:///signed-in. +func NewAuthenticator(ctx context.Context, cfg OIDCConfig, redirectURL string) (*Authenticator, error) { + provider, err := oidc.NewProvider(ctx, cfg.Issuer) + if err != nil { + return nil, fmt.Errorf("the issuer %s could not be reached: %w", cfg.Issuer, err) + } + return &Authenticator{ + OAuth2: oauth2.Config{ + ClientID: cfg.ClientID, + ClientSecret: cfg.ClientSecret, + Endpoint: provider.Endpoint(), + RedirectURL: redirectURL, + Scopes: []string{oidc.ScopeOpenID, "profile"}, + }, + Verifier: provider.Verifier(&oidc.Config{ClientID: cfg.ClientID}), + }, nil +} + +// Auth wires sign-in into a Server: RequireSession protects a page, an attachment or a send (issue 1263/AC1), +// and Callback is /signed-in. +type Auth struct { + Auth *Authenticator + Sessions *Sessions + Pending *PendingAuths + Operators Operators + Logf func(format string, a ...any) +} + +func (a *Auth) logf(format string, args ...any) { + if a.Logf != nil { + a.Logf(format, args...) + } +} + +// sessionKey is the context key RequireSession hands the signed-in Session on to the handler it wraps. +type sessionKey struct{} + +// SessionFrom is the session a handler wrapped by RequireSession was asked with. +func SessionFrom(r *http.Request) (Session, bool) { + s, ok := r.Context().Value(sessionKey{}).(Session) + return s, ok +} + +// RequireSession is issue 1263/AC1, AC5: no session, or one that ended 12 hours ago, is answered a redirect to +// the issuer's authorisation endpoint (state, PKCE challenge, this module's own callback) rather than the page, +// attachment or send asked for; a live session is handed on to next. +func (a *Auth) RequireSession(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if cookie, err := r.Cookie(SessionCookie); err == nil { + if sess, ok := a.Sessions.Get(cookie.Value); ok { + ctx := context.WithValue(r.Context(), sessionKey{}, sess) + next(w, r.WithContext(ctx)) + return + } + } + a.redirectToSignIn(w, r) + } +} + +func (a *Auth) redirectToSignIn(w http.ResponseWriter, r *http.Request) { + verifier := oauth2.GenerateVerifier() + state, err := a.Pending.Create(verifier, r.URL.RequestURI()) + if err != nil { + http.Error(w, "sign-in could not be started", http.StatusInternalServerError) + return + } + url := a.Auth.OAuth2.AuthCodeURL(state, oauth2.S256ChallengeOption(verifier)) + http.Redirect(w, r, url, http.StatusFound) +} + +// Callback is /signed-in (issue 1263/AC2, AC3, AC4): a state this module did not issue, an exchange that +// fails, an ID token of another audience, signed by another key, or expired, or an identity operators does not +// name, is answered 403 and makes no session. Only once the ID token verifies and names an operator is a +// session made, its cookie set, and the browser sent back to the page it first asked for. +func (a *Auth) Callback(w http.ResponseWriter, r *http.Request) { + refuse := func(why string) { + a.logf("[mesh-mail-web] sign-in refused: %s", why) + http.Error(w, "forbidden", http.StatusForbidden) + } + query := r.URL.Query() + if errDesc := query.Get("error"); errDesc != "" { + refuse("the issuer answered " + errDesc) + return + } + pending, ok := a.Pending.Take(query.Get("state")) + if !ok { + refuse("the state is not one this module issued, or it expired") + return + } + code := query.Get("code") + if code == "" { + refuse("the callback carries no code") + return + } + tok, err := a.Auth.OAuth2.Exchange(r.Context(), code, oauth2.VerifierOption(pending.Verifier)) + if err != nil { + refuse(fmt.Sprintf("the code could not be exchanged: %v", err)) + return + } + rawIDToken, ok := tok.Extra("id_token").(string) + if !ok || rawIDToken == "" { + refuse("the issuer's answer carries no ID token") + return + } + idToken, err := a.Auth.Verifier.Verify(r.Context(), rawIDToken) + if err != nil { + // Issuer, audience, expiry and signature are all checked here, by the verifier (1263/AC4). + refuse(fmt.Sprintf("the ID token does not verify: %v", err)) + return + } + var claims struct { + PreferredUsername string `json:"preferred_username"` + } + if err := idToken.Claims(&claims); err != nil { + refuse(fmt.Sprintf("the ID token's claims could not be read: %v", err)) + return + } + allowed, why := a.Operators.Allows(claims.PreferredUsername) + if !allowed { + refuse(why) + return + } + id, err := a.Sessions.Create(claims.PreferredUsername) + if err != nil { + refuse(fmt.Sprintf("the session could not be made: %v", err)) + return + } + http.SetCookie(w, &http.Cookie{ + Name: SessionCookie, + Value: id, + Path: "/", + Secure: true, + HttpOnly: true, + SameSite: http.SameSiteStrictMode, + MaxAge: int(SessionLifetime / time.Second), + }) + returnTo := pending.Return + if returnTo == "" { + returnTo = "/" + } + http.Redirect(w, r, returnTo, http.StatusFound) +} diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/auth_test.go b/modules/mesh-mail-web/cmd/mesh-mail-web/auth_test.go new file mode 100644 index 00000000..b16d269a --- /dev/null +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/auth_test.go @@ -0,0 +1,364 @@ +package main + +// Keycloak sign-in (novox/hq ADR 0440 §3, issue 1263): an httptest stand-in issuer, as the lead's prepare +// findings said to check this against, minting its own ID tokens (RS256) so a test controls exactly what +// go-oidc's verifier is asked to check: the right issuer, audience, expiry and signature, and the wrong ones. + +import ( + "context" + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "math/big" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" +) + +// fakeIssuer is a minimal OIDC issuer: discovery, JWKS and a token endpoint that answers whatever ID token the +// test set up, regardless of the code it is asked to exchange (the code's own validity is not what issue +// 1263's acceptance criteria are about; the ID token's is). +type fakeIssuer struct { + *httptest.Server + key *rsa.PrivateKey + kid string + nextIDToken string +} + +func newFakeIssuer(t *testing.T) *fakeIssuer { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + fi := &fakeIssuer{key: key, kid: "test-key"} + mux := http.NewServeMux() + mux.HandleFunc("/.well-known/openid-configuration", fi.discovery) + mux.HandleFunc("/jwks", fi.jwks) + mux.HandleFunc("/token", fi.token) + fi.Server = httptest.NewServer(mux) + t.Cleanup(fi.Server.Close) + return fi +} + +func (fi *fakeIssuer) discovery(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "issuer": fi.Server.URL, + "authorization_endpoint": fi.Server.URL + "/authorize", + "token_endpoint": fi.Server.URL + "/token", + "jwks_uri": fi.Server.URL + "/jwks", + "userinfo_endpoint": fi.Server.URL + "/userinfo", + "id_token_signing_alg_values_supported": []string{"RS256"}, + "response_types_supported": []string{"code"}, + "subject_types_supported": []string{"public"}, + }) +} + +func (fi *fakeIssuer) jwks(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + n := base64.RawURLEncoding.EncodeToString(fi.key.PublicKey.N.Bytes()) + e := base64.RawURLEncoding.EncodeToString(big.NewInt(int64(fi.key.PublicKey.E)).Bytes()) + _ = json.NewEncoder(w).Encode(map[string]any{"keys": []map[string]any{ + {"kty": "RSA", "alg": "RS256", "use": "sig", "kid": fi.kid, "n": n, "e": e}, + }}) +} + +func (fi *fakeIssuer) token(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "access_token": "fake-access-token", + "token_type": "Bearer", + "expires_in": 3600, + "id_token": fi.nextIDToken, + }) +} + +func b64url(b []byte) string { return base64.RawURLEncoding.EncodeToString(b) } + +// sign mints an RS256 JWT of claims, signed with key under kid — the real issuer's own key unless a test asks +// for "signed by another key" (1263/AC4). +func sign(claims map[string]any, kid string, key *rsa.PrivateKey) string { + header, _ := json.Marshal(map[string]any{"alg": "RS256", "typ": "JWT", "kid": kid}) + payload, _ := json.Marshal(claims) + signingInput := b64url(header) + "." + b64url(payload) + sum := sha256.Sum256([]byte(signingInput)) + sig, err := rsa.SignPKCS1v15(rand.Reader, key, crypto.SHA256, sum[:]) + if err != nil { + panic(err) + } + return signingInput + "." + b64url(sig) +} + +// idToken is a valid ID token's claims, as claimsPatch overrides them (so a test changes exactly one thing — +// the audience, the expiry, the signing key — and nothing else). +func (fi *fakeIssuer) idToken(clientID, username string, now time.Time, claimsPatch map[string]any, key *rsa.PrivateKey, kid string) string { + claims := map[string]any{ + "iss": fi.Server.URL, + "aud": clientID, + "exp": now.Add(time.Hour).Unix(), + "iat": now.Unix(), + "sub": "user-" + username, + "preferred_username": username, + } + for k, v := range claimsPatch { + claims[k] = v + } + return sign(claims, kid, key) +} + +// testAuth is an Auth over fi, with its own callback redirect https://mail.g14.internal/signed-in, and +// deterministic time so a test controls what "12 hours ago" or "expired" means. +func testAuth(t *testing.T, fi *fakeIssuer, operatorsPath string, now func() time.Time) *Auth { + t.Helper() + cfg := OIDCConfig{ClientID: "mail-web", ClientSecret: "s3cr3t", Issuer: fi.Server.URL} + authenticator, err := NewAuthenticator(context.Background(), cfg, "https://"+InternalHost("g14")+SignedInPath) + if err != nil { + t.Fatal(err) + } + return &Auth{ + Auth: authenticator, + Sessions: NewSessions(now), + Pending: NewPendingAuths(now), + Operators: Operators{Path: operatorsPath}, + Logf: t.Logf, + } +} + +func operatorsFile(t *testing.T, names ...string) string { + t.Helper() + dir := t.TempDir() + path := dir + "/operators.json" + raw, _ := json.Marshal(map[string]any{"operators": names}) + writeFile(t, path, string(raw)) + return path +} + +// 1263/AC1: no session: a redirect to the issuer's authorisation endpoint with the client id, a state, a PKCE +// challenge, and the redirect https://mail..internal/signed-in. +func TestNoSessionRedirectsToTheIssuersAuthorisationEndpoint(t *testing.T) { + fi := newFakeIssuer(t) + auth := testAuth(t, fi, operatorsFile(t, "alice"), nil) + s := NewServer(InternalHost("g14"), auth) + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/", nil) + r.Host = InternalHost("g14") + s.ServeHTTP(w, r) + + if w.Code != http.StatusFound { + t.Fatalf("status %d, want 302", w.Code) + } + loc, err := url.Parse(w.Header().Get("Location")) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(loc.String(), fi.Server.URL+"/authorize") { + t.Fatalf("redirected to %s, not the issuer's authorisation endpoint", loc) + } + q := loc.Query() + if q.Get("client_id") != "mail-web" { + t.Fatalf("client_id: %q", q.Get("client_id")) + } + if q.Get("state") == "" { + t.Fatalf("no state") + } + if q.Get("code_challenge") == "" || q.Get("code_challenge_method") != "S256" { + t.Fatalf("no PKCE challenge: %q %q", q.Get("code_challenge"), q.Get("code_challenge_method")) + } + if q.Get("redirect_uri") != "https://"+InternalHost("g14")+SignedInPath { + t.Fatalf("redirect_uri: %q", q.Get("redirect_uri")) + } +} + +// startSignIn drives the redirect step for a test, answering the state and PKCE verifier it issued (read back +// from the pending store by the state the redirect carried), as the next call (the callback) needs them. +func startSignIn(t *testing.T, s *Server, auth *Auth) (state string) { + t.Helper() + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/", nil) + r.Host = InternalHost("g14") + s.ServeHTTP(w, r) + loc, err := url.Parse(w.Header().Get("Location")) + if err != nil { + t.Fatal(err) + } + return loc.Query().Get("state") +} + +func callback(s *Server, state, code string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, SignedInPath+"?state="+url.QueryEscape(state)+"&code="+url.QueryEscape(code), nil) + r.Host = InternalHost("g14") + s.ServeHTTP(w, r) + return w +} + +// 1263/AC2: a valid callback whose ID token names an operator sets a session cookie Secure, HttpOnly, +// SameSite=Strict, lasting 12 hours, and sends the browser back to the page it first asked for. +func TestACallbackWithAValidIDTokenNamingAnOperatorSetsTheSessionCookie(t *testing.T) { + fi := newFakeIssuer(t) + now := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + auth := testAuth(t, fi, operatorsFile(t, "alice"), func() time.Time { return now }) + s := NewServer(InternalHost("g14"), auth) + + state := startSignIn(t, s, auth) + // The session clock (now) is mocked to test the 12-hour bound (TestASessionTwelveHoursOldIsTreatedAsNoSession + // below); the ID token's own iat/exp are judged by go-oidc against the real wall clock, so they are minted + // from it, not from the mocked now. + fi.nextIDToken = fi.idToken("mail-web", "alice", time.Now(), nil, fi.key, fi.kid) + w := callback(s, state, "some-code") + + if w.Code != http.StatusFound { + t.Fatalf("status %d, want 302: %s", w.Code, w.Body.String()) + } + if loc := w.Header().Get("Location"); loc != "/" { + t.Fatalf("returned to %q, want the page first asked for (/)", loc) + } + var found *http.Cookie + for _, c := range w.Result().Cookies() { + if c.Name == SessionCookie { + found = c + } + } + if found == nil { + t.Fatal("no session cookie set") + } + if !found.Secure || !found.HttpOnly || found.SameSite != http.SameSiteStrictMode { + t.Fatalf("cookie attributes: Secure=%v HttpOnly=%v SameSite=%v", found.Secure, found.HttpOnly, found.SameSite) + } + if found.MaxAge != int(SessionLifetime/time.Second) { + t.Fatalf("MaxAge %d, want %d (12h)", found.MaxAge, int(SessionLifetime/time.Second)) + } +} + +// 1263/AC3: an ID token naming an identity not in operators is answered 403, and no session cookie is set. +func TestACallbackForAnIdentityNotInOperatorsIsRefused(t *testing.T) { + fi := newFakeIssuer(t) + now := time.Now() + auth := testAuth(t, fi, operatorsFile(t, "alice"), func() time.Time { return now }) + s := NewServer(InternalHost("g14"), auth) + + state := startSignIn(t, s, auth) + fi.nextIDToken = fi.idToken("mail-web", "mallory", now, nil, fi.key, fi.kid) + w := callback(s, state, "some-code") + + if w.Code != http.StatusForbidden { + t.Fatalf("status %d, want 403", w.Code) + } + if len(w.Result().Cookies()) != 0 { + t.Fatalf("a cookie was set for a refused identity: %v", w.Result().Cookies()) + } +} + +// 1263/AC4: a state this module did not issue, an ID token of another audience, one signed by another key, or +// one expired — each answered 403, no session made. +func TestACallbackIsRefusedOnAnUnknownStateOrAnInvalidIDToken(t *testing.T) { + fi := newFakeIssuer(t) + now := time.Now() + otherKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + + cases := []struct { + name string + setup func(auth *Auth, s *Server) (state string) + token func() string + }{ + {"unknown state", func(auth *Auth, s *Server) string { return "a-state-nobody-issued" }, + func() string { return fi.idToken("mail-web", "alice", now, nil, fi.key, fi.kid) }}, + {"wrong audience", func(auth *Auth, s *Server) string { return startSignIn(t, s, auth) }, + func() string { return fi.idToken("someone-else", "alice", now, nil, fi.key, fi.kid) }}, + {"wrong signing key", func(auth *Auth, s *Server) string { return startSignIn(t, s, auth) }, + func() string { return fi.idToken("mail-web", "alice", now, nil, otherKey, fi.kid) }}, + {"expired", func(auth *Auth, s *Server) string { return startSignIn(t, s, auth) }, + func() string { + return fi.idToken("mail-web", "alice", now, map[string]any{"exp": now.Add(-time.Hour).Unix()}, fi.key, fi.kid) + }}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + auth := testAuth(t, fi, operatorsFile(t, "alice"), func() time.Time { return now }) + s := NewServer(InternalHost("g14"), auth) + state := c.setup(auth, s) + fi.nextIDToken = c.token() + w := callback(s, state, "some-code") + if w.Code != http.StatusForbidden { + t.Fatalf("%s: status %d, want 403: %s", c.name, w.Code, w.Body.String()) + } + if len(w.Result().Cookies()) != 0 { + t.Fatalf("%s: a cookie was set", c.name) + } + }) + } +} + +// 1263/AC5: a session made 12 hours ago is treated as no session: a page asked then is redirected to sign in +// again. +func TestASessionTwelveHoursOldIsTreatedAsNoSession(t *testing.T) { + fi := newFakeIssuer(t) + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + now := start + auth := testAuth(t, fi, operatorsFile(t, "alice"), func() time.Time { return now }) + s := NewServer(InternalHost("g14"), auth) + + state := startSignIn(t, s, auth) + // The ID token's own iat/exp are judged against the real wall clock (see the comment in + // TestACallbackWithAValidIDTokenNamingAnOperatorSetsTheSessionCookie above); the session's own age is what + // this test mocks, through the session clock (now). + fi.nextIDToken = fi.idToken("mail-web", "alice", time.Now(), nil, fi.key, fi.kid) + w := callback(s, state, "some-code") + var cookie *http.Cookie + for _, c := range w.Result().Cookies() { + if c.Name == SessionCookie { + cookie = c + } + } + if cookie == nil { + t.Fatal("no session cookie set") + } + + // Just under 12 hours: still a session. + now = start.Add(12*time.Hour - time.Minute) + r := httptest.NewRequest(http.MethodGet, "/", nil) + r.Host = InternalHost("g14") + r.AddCookie(cookie) + w2 := httptest.NewRecorder() + s.ServeHTTP(w2, r) + if w2.Code != http.StatusOK { + t.Fatalf("just under 12h: status %d, want 200", w2.Code) + } + + // Past 12 hours: no session any more — redirected to sign in again. + now = start.Add(12*time.Hour + time.Minute) + r2 := httptest.NewRequest(http.MethodGet, "/", nil) + r2.Host = InternalHost("g14") + r2.AddCookie(cookie) + w3 := httptest.NewRecorder() + s.ServeHTTP(w3, r2) + if w3.Code != http.StatusFound { + t.Fatalf("past 12h: status %d, want 302 (redirect to sign in)", w3.Code) + } +} + +// 1263/AC6: operators empty lets no identity in. +func TestCallbackWithEmptyOperatorsLetsNobodyIn(t *testing.T) { + fi := newFakeIssuer(t) + now := time.Now() + auth := testAuth(t, fi, operatorsFile(t), func() time.Time { return now }) + s := NewServer(InternalHost("g14"), auth) + + state := startSignIn(t, s, auth) + fi.nextIDToken = fi.idToken("mail-web", "alice", now, nil, fi.key, fi.kid) + w := callback(s, state, "some-code") + if w.Code != http.StatusForbidden { + t.Fatalf("status %d, want 403", w.Code) + } +} diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/main.go b/modules/mesh-mail-web/cmd/mesh-mail-web/main.go index 46009ecb..35af11ce 100644 --- a/modules/mesh-mail-web/cmd/mesh-mail-web/main.go +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/main.go @@ -7,10 +7,12 @@ package main import ( + "context" "fmt" "io" "net/http" "os" + "time" stdio "git.novox.be/novox/mesh-sdk/go" ) @@ -27,18 +29,50 @@ func main() { } // serveHTTP starts the HTTP endpoint `web` (issue 1283), on the port the mesh gave it; a listen it cannot -// start is logged, never fatal to the seat verbs stdio.Serve answers on the MCP channel. +// start is logged, never fatal to the seat verbs stdio.Serve answers on the MCP channel. Sign-in (issue 1263) +// is set up first, retried on a failure (the issuer may not answer yet): never served with no session guard +// at all, which a nil Auth would be (server.go's own fallback, used by its unit tests alone). func serveHTTP(b *Bundle) { if b.Env.Listen == "" { logf("[mesh-mail-web] started without MESH_MAIL_WEB_LISTEN; its HTTP endpoint is not served") return } - s := NewServer(InternalHost(b.Env.Node)) + host := InternalHost(b.Env.Node) + var auth *Auth + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + a, err := buildAuth(host, b.Env) + if err == nil { + auth = a + break + } + logf("[mesh-mail-web] sign-in cannot be set up yet (%v); trying again in %s", err, wait) + time.Sleep(wait) + } + s := NewServer(host, auth) if err := http.ListenAndServe(":"+b.Env.Listen, s); err != nil { logf("[mesh-mail-web] its HTTP endpoint ended: %v", err) } } +// buildAuth reads the oidc-client binding (issue 1262's oidc.json) and discovers the issuer (issue 1263). +func buildAuth(host string, env Env) (*Auth, error) { + cfg, err := ReadOIDCConfig(env.OIDC) + if err != nil { + return nil, err + } + authenticator, err := NewAuthenticator(context.Background(), cfg, "https://"+host+SignedInPath) + if err != nil { + return nil, err + } + return &Auth{ + Auth: authenticator, + Sessions: NewSessions(nil), + Pending: NewPendingAuths(nil), + Operators: Operators{Path: env.Operators}, + Logf: logf, + }, nil +} + // run is main's startup path, factored so a test can run it over a pipe instead of the real stdin/stdout // (mirroring mesh-mail-notifier's own run). func run(b *Bundle, in io.Reader, out io.Writer) error { diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/server.go b/modules/mesh-mail-web/cmd/mesh-mail-web/server.go index 087a9963..6fbd5a35 100644 --- a/modules/mesh-mail-web/cmd/mesh-mail-web/server.go +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/server.go @@ -32,13 +32,20 @@ func InternalHost(node string) string { return "mail." + node + ".internal" } type Server struct { Host string // InternalHost(node) Mux *http.ServeMux + Auth *Auth // nil refuses no page by session (1283 alone); issue 1263 gives one } // NewServer is a Server for host, with the root page issue 1283 itself answers (a stand-in: the real accounts -// and folder pages are issue 1264's). -func NewServer(host string) *Server { - s := &Server{Host: host, Mux: http.NewServeMux()} - s.Mux.HandleFunc("/", s.rootPage) +// and folder pages are issue 1264's) and, when auth is given, the sign-in callback (issue 1263) and every page +// behind a session (RequireSession). +func NewServer(host string, auth *Auth) *Server { + s := &Server{Host: host, Mux: http.NewServeMux(), Auth: auth} + root := s.rootPage + if auth != nil { + s.Mux.HandleFunc(SignedInPath, auth.Callback) + root = auth.RequireSession(root) + } + s.Mux.HandleFunc("/", root) return s } diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/server_test.go b/modules/mesh-mail-web/cmd/mesh-mail-web/server_test.go index b2382043..2c25e35a 100644 --- a/modules/mesh-mail-web/cmd/mesh-mail-web/server_test.go +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/server_test.go @@ -12,7 +12,7 @@ import ( // 1283/AC1: a request whose Host is not mail..internal — a public name, an IP, or none — is answered // 404, naming nothing of the mail. func TestServerAnswers404ForAnyHostButItsOwnInternalName(t *testing.T) { - s := NewServer(InternalHost("g14")) + s := NewServer(InternalHost("g14"), nil) for _, host := range []string{"example.com", "203.0.113.5", "", "mail.other-node.internal"} { w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/", nil) @@ -29,7 +29,7 @@ func TestServerAnswers404ForAnyHostButItsOwnInternalName(t *testing.T) { // Its own internal name is answered (not 404). func TestServerAnswersItsOwnInternalName(t *testing.T) { - s := NewServer(InternalHost("g14")) + s := NewServer(InternalHost("g14"), nil) w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/", nil) r.Host = InternalHost("g14") @@ -41,7 +41,7 @@ func TestServerAnswersItsOwnInternalName(t *testing.T) { // A Host with a port still matches, port and all (the mesh's own listen carries one). func TestServerAnswersItsOwnInternalNameWithAPort(t *testing.T) { - s := NewServer(InternalHost("g14")) + s := NewServer(InternalHost("g14"), nil) w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/", nil) r.Host = InternalHost("g14") + ":8099" @@ -53,7 +53,7 @@ func TestServerAnswersItsOwnInternalNameWithAPort(t *testing.T) { // 1283/AC2: every answer (200 and 404 alike) carries the content security policy. func TestServerSetsTheContentSecurityPolicyOnEveryAnswer(t *testing.T) { - s := NewServer(InternalHost("g14")) + s := NewServer(InternalHost("g14"), nil) cases := []string{"example.com", InternalHost("g14")} for _, host := range cases { w := httptest.NewRecorder() @@ -116,7 +116,7 @@ func TestServerStartsAndAnswersAPageWithACwdAndHomeReadOnly(t *testing.T) { } _ = os.Setenv("HOME", filepath.Join(ro, "home-does-not-exist")) - s := NewServer(InternalHost("g14")) + s := NewServer(InternalHost("g14"), nil) srv := httptest.NewServer(s) defer srv.Close() diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/sessions.go b/modules/mesh-mail-web/cmd/mesh-mail-web/sessions.go new file mode 100644 index 00000000..26579612 --- /dev/null +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/sessions.go @@ -0,0 +1,143 @@ +package main + +// Sessions and the pending authorisations in flight (novox/hq ADR 0440 §3, issue 1263): both kept in memory +// only, never on disk — a restart signs everyone out, accepted (design 59 milestone 1's prepare step). + +import ( + "crypto/rand" + "encoding/base64" + "sync" + "time" +) + +// SessionCookie is the cookie's own name. +const SessionCookie = "mesh_mail_web_session" + +// SessionLifetime is how long a session lasts after it is made (novox/hq ADR 0440 §3, issue 1263/AC2, AC5). +const SessionLifetime = 12 * time.Hour + +// PendingLifetime is how long a sign-in may take between the redirect to the issuer and its callback, before +// the state is no longer honoured. +const PendingLifetime = 10 * time.Minute + +func randomID() (string, error) { + var b [32]byte + if _, err := rand.Read(b[:]); err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(b[:]), nil +} + +// Session is one signed-in operator's own session: its identity, when it was made (SessionLifetime decides +// when it ends), and the per-session token issue 1284 binds every changing form to. +type Session struct { + Operator string + Made time.Time + Token string +} + +// Sessions keeps every session in memory, by its cookie's own id. +type Sessions struct { + mu sync.Mutex + now func() time.Time + byID map[string]Session +} + +// NewSessions is an empty Sessions; now is time.Now unless a test gives its own. +func NewSessions(now func() time.Time) *Sessions { + if now == nil { + now = time.Now + } + return &Sessions{now: now, byID: map[string]Session{}} +} + +// Create starts a new session for operator, answering its id (the cookie's own value) and its own per-session +// token (issue 1284). +func (s *Sessions) Create(operator string) (id string, err error) { + id, err = randomID() + if err != nil { + return "", err + } + token, err := randomID() + if err != nil { + return "", err + } + s.mu.Lock() + defer s.mu.Unlock() + s.byID[id] = Session{Operator: operator, Made: s.now(), Token: token} + return id, nil +} + +// Get is the session at id, or false when there is none or it ended SessionLifetime ago (1263/AC5): an expired +// session reads as no session at all, dropped on the same look (no background sweep is needed: every read +// already checks the time). +func (s *Sessions) Get(id string) (Session, bool) { + if id == "" { + return Session{}, false + } + s.mu.Lock() + defer s.mu.Unlock() + sess, ok := s.byID[id] + if !ok { + return Session{}, false + } + if s.now().Sub(sess.Made) > SessionLifetime { + delete(s.byID, id) + return Session{}, false + } + return sess, true +} + +// PendingAuth is one sign-in asked of the issuer, remembered between the redirect and its callback: the PKCE +// verifier the challenge was made from, and the page the browser first asked for. +type PendingAuth struct { + Verifier string + Return string + Created time.Time +} + +// PendingAuths keeps every sign-in in flight, by its own state, in memory only. +type PendingAuths struct { + mu sync.Mutex + now func() time.Time + byState map[string]PendingAuth +} + +// NewPendingAuths is an empty PendingAuths; now is time.Now unless a test gives its own. +func NewPendingAuths(now func() time.Time) *PendingAuths { + if now == nil { + now = time.Now + } + return &PendingAuths{now: now, byState: map[string]PendingAuth{}} +} + +// Create remembers one sign-in in flight, answering the state it is kept under. +func (p *PendingAuths) Create(verifier, returnTo string) (state string, err error) { + state, err = randomID() + if err != nil { + return "", err + } + p.mu.Lock() + defer p.mu.Unlock() + p.byState[state] = PendingAuth{Verifier: verifier, Return: returnTo, Created: p.now()} + return state, nil +} + +// Take reads back and removes the sign-in kept under state — once; a state never issued, already taken, or +// older than PendingLifetime answers false (1263/AC4's "a state it did not issue"). +func (p *PendingAuths) Take(state string) (PendingAuth, bool) { + if state == "" { + return PendingAuth{}, false + } + p.mu.Lock() + defer p.mu.Unlock() + pa, ok := p.byState[state] + if !ok { + return PendingAuth{}, false + } + delete(p.byState, state) + if p.now().Sub(pa.Created) > PendingLifetime { + return PendingAuth{}, false + } + return pa, true +} diff --git a/modules/mesh-mail-web/go.mod b/modules/mesh-mail-web/go.mod index baf38bd6..9664b173 100644 --- a/modules/mesh-mail-web/go.mod +++ b/modules/mesh-mail-web/go.mod @@ -2,4 +2,10 @@ module mesh-mail-web go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019062050-34392e74916c +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019062050-34392e74916c + github.com/coreos/go-oidc/v3 v3.21.0 + golang.org/x/oauth2 v0.37.0 +) + +require github.com/go-jose/go-jose/v4 v4.1.4 // indirect diff --git a/modules/mesh-mail-web/go.sum b/modules/mesh-mail-web/go.sum index 25aaa963..f405e4dd 100644 --- a/modules/mesh-mail-web/go.sum +++ b/modules/mesh-mail-web/go.sum @@ -1,2 +1,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019062050-34392e74916c h1:/f6f2ugss172hDJSL3EFZqb9PdfO5DKzYVfPiX2LdAM= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019062050-34392e74916c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/coreos/go-oidc/v3 v3.21.0 h1:wZo4Q9Pum8dYEj0eMUPrqR+kvuGkeUplbLpNCkBqoWM= +github.com/coreos/go-oidc/v3 v3.21.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= +golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= From 3ee80c98abf3a7cc7e51d17175f138f7291ced37 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 19 Oct 2026 22:18:03 +0200 Subject: [PATCH 07/18] Bind every changing form's token to its session (issue 1284) --- .../mesh-mail-web/cmd/mesh-mail-web/csrf.go | 49 +++++++ .../cmd/mesh-mail-web/csrf_test.go | 132 ++++++++++++++++++ 2 files changed, 181 insertions(+) create mode 100644 modules/mesh-mail-web/cmd/mesh-mail-web/csrf.go create mode 100644 modules/mesh-mail-web/cmd/mesh-mail-web/csrf_test.go diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/csrf.go b/modules/mesh-mail-web/cmd/mesh-mail-web/csrf.go new file mode 100644 index 00000000..661541f2 --- /dev/null +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/csrf.go @@ -0,0 +1,49 @@ +package main + +// The form token bound to the session (novox/hq ADR 0440 §3's last clause, issue 1284): every form that +// changes something — today mail_send, once issue 1267 builds the write page — carries a hidden field with +// the session's own token (a random 32 bytes, made once per session, Sessions.Create); a POST missing it, or +// carrying another session's, is refused 403 before the change it would have made ever runs, compared in +// constant time so the check is never a timing side-channel. + +import ( + "crypto/subtle" + "html/template" + "net/http" +) + +// FormTokenField is the hidden field's own name every changing form carries. +const FormTokenField = "csrf_token" + +// FormTokenInput is the hidden field a changing form's own template includes (issue 1284/AC1): bound to the +// session handed to the page that renders it (RequireSession, SessionFrom). +func FormTokenInput(sess Session) template.HTML { + return template.HTML(``) +} + +// RequireFormToken wraps a POST handler that changes something (novox/hq issue 1284): it calls next only once +// the session RequireSession handed on (SessionFrom) and the form's own FormTokenField compare equal, in +// constant time; a POST with no token, or another session's, is answered 403 and next is never called, so the +// change it would have made — today mail_send — never runs (AC2, AC3). RequireFormToken is placed inside +// RequireSession (RequireSession(RequireFormToken(next))): a POST with no session at all is the sign-in +// redirect RequireSession itself gives, never reaching here. +func RequireFormToken(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + sess, ok := SessionFrom(r) + if !ok { + http.Error(w, "forbidden", http.StatusForbidden) + return + } + if err := r.ParseForm(); err != nil { + http.Error(w, "forbidden", http.StatusForbidden) + return + } + given := r.PostFormValue(FormTokenField) + if given == "" || subtle.ConstantTimeCompare([]byte(given), []byte(sess.Token)) != 1 { + http.Error(w, "forbidden", http.StatusForbidden) + return + } + next(w, r) + } +} diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/csrf_test.go b/modules/mesh-mail-web/cmd/mesh-mail-web/csrf_test.go new file mode 100644 index 00000000..b2f62166 --- /dev/null +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/csrf_test.go @@ -0,0 +1,132 @@ +package main + +// The form token bound to the session (novox/hq ADR 0440 §3, issue 1284). There is no write page to test +// against yet (issue 1267 builds it); these tests stand in for it with a minimal changing handler ("send"), +// proving the mechanism issue 1267 will wire mail_send behind: a session, and that session's own token. + +import ( + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" +) + +// sessionAndCookie starts a session directly (bypassing sign-in, which auth_test.go already covers on its +// own) and answers it with the cookie a request would carry. +func sessionAndCookie(t *testing.T, sessions *Sessions, operator string) (Session, *http.Cookie) { + t.Helper() + id, err := sessions.Create(operator) + if err != nil { + t.Fatal(err) + } + sess, ok := sessions.Get(id) + if !ok { + t.Fatal("the session just created is not there") + } + return sess, &http.Cookie{Name: SessionCookie, Value: id} +} + +// 1284/AC1: a signed-in session's own form carries that session's token. +func TestFormTokenInputCarriesTheSessionsOwnToken(t *testing.T) { + sess, _ := sessionAndCookie(t, NewSessions(nil), "alice") + html := string(FormTokenInput(sess)) + if !strings.Contains(html, `name="csrf_token"`) || !strings.Contains(html, sess.Token) { + t.Fatalf("the form's hidden field does not carry the session's token: %s", html) + } +} + +// changingHandler is the stand-in for mail_send (issue 1267 is the real one): it counts how many times it was +// actually reached. +func changingHandler(calls *int) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + *calls++ + w.WriteHeader(http.StatusOK) + } +} + +func postSend(t *testing.T, handler http.HandlerFunc, cookie *http.Cookie, token string) *httptest.ResponseRecorder { + t.Helper() + form := url.Values{} + if token != "" { + form.Set(FormTokenField, token) + } + r := httptest.NewRequest(http.MethodPost, "/send", strings.NewReader(form.Encode())) + r.Header.Set("Content-Type", "application/x-www-form-urlencoded") + if cookie != nil { + r.AddCookie(cookie) + } + w := httptest.NewRecorder() + handler(w, r) + return w +} + +// 1284/AC2: a POST of a send without the token is answered 403, and the change (here, the stand-in) is never +// called. +func TestPOSTWithoutTheTokenIsRefusedAndTheChangeIsNotCalled(t *testing.T) { + sessions := NewSessions(nil) + _, cookie := sessionAndCookie(t, sessions, "alice") + auth := &Auth{Sessions: sessions} + var calls int + handler := auth.RequireSession(RequireFormToken(changingHandler(&calls))) + + w := postSend(t, handler, cookie, "") + if w.Code != http.StatusForbidden { + t.Fatalf("status %d, want 403", w.Code) + } + if calls != 0 { + t.Fatalf("the change was called %d time(s) with no token", calls) + } +} + +// 1284/AC3: a POST carrying another session's token is answered 403, and the change is never called. +func TestPOSTWithAnotherSessionsTokenIsRefusedAndTheChangeIsNotCalled(t *testing.T) { + sessions := NewSessions(nil) + _, cookie := sessionAndCookie(t, sessions, "alice") + otherSess, _ := sessionAndCookie(t, sessions, "bob") + auth := &Auth{Sessions: sessions} + var calls int + handler := auth.RequireSession(RequireFormToken(changingHandler(&calls))) + + w := postSend(t, handler, cookie, otherSess.Token) + if w.Code != http.StatusForbidden { + t.Fatalf("status %d, want 403", w.Code) + } + if calls != 0 { + t.Fatalf("the change was called %d time(s) with another session's token", calls) + } +} + +// 1284/AC4: a POST carrying its own session's token proceeds. +func TestPOSTWithItsOwnSessionsTokenProceeds(t *testing.T) { + sessions := NewSessions(nil) + sess, cookie := sessionAndCookie(t, sessions, "alice") + auth := &Auth{Sessions: sessions} + var calls int + handler := auth.RequireSession(RequireFormToken(changingHandler(&calls))) + + w := postSend(t, handler, cookie, sess.Token) + if w.Code != http.StatusOK { + t.Fatalf("status %d, want 200: %s", w.Code, w.Body.String()) + } + if calls != 1 { + t.Fatalf("the change was called %d time(s), want 1", calls) + } +} + +// No session at all: RequireSession's own redirect runs, RequireFormToken is never reached, and the change is +// never called either (a POST attempted with no session is treated the same as a page with no session). +func TestPOSTWithNoSessionDoesNotReachTheFormTokenCheckOrTheChange(t *testing.T) { + fi := newFakeIssuer(t) + auth := testAuth(t, fi, operatorsFile(t, "alice"), nil) + var calls int + handler := auth.RequireSession(RequireFormToken(changingHandler(&calls))) + + w := postSend(t, handler, nil, "whatever") + if w.Code != http.StatusFound { + t.Fatalf("status %d, want 302 (redirect to sign in)", w.Code) + } + if calls != 0 { + t.Fatalf("the change was called %d time(s) with no session", calls) + } +} From 28fdbbd346011fcf919c11454b64e45d9fd82760 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 19 Oct 2026 22:22:52 +0200 Subject: [PATCH 08/18] Show every granted account, its state and its folders' counts (issue 1264) Accounts bucket fixture (accountBucketValue/accountBucketFolder) copied from mesh-mail's own modules/mesh-mail/cmd/mesh-mail/mirror.go AccountValue/AccountFolderValue (issue 1255); mesh-mail-web is its own Go module, so this is a copy, not an import. --- .../cmd/mesh-mail-web/accounts.go | 203 ++++++++++++++++++ .../cmd/mesh-mail-web/accounts_test.go | 180 ++++++++++++++++ .../cmd/mesh-mail-web/auth_test.go | 12 +- .../mesh-mail-web/cmd/mesh-mail-web/main.go | 2 +- .../mesh-mail-web/cmd/mesh-mail-web/server.go | 31 ++- .../cmd/mesh-mail-web/server_test.go | 10 +- .../mesh-mail-web/cmd/mesh-mail-web/tools.go | 66 +++++- 7 files changed, 478 insertions(+), 26 deletions(-) create mode 100644 modules/mesh-mail-web/cmd/mesh-mail-web/accounts.go create mode 100644 modules/mesh-mail-web/cmd/mesh-mail-web/accounts_test.go diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/accounts.go b/modules/mesh-mail-web/cmd/mesh-mail-web/accounts.go new file mode 100644 index 00000000..0ab2cd79 --- /dev/null +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/accounts.go @@ -0,0 +1,203 @@ +package main + +// The accounts and folder pages (novox/hq ADR 0440 §4, issue 1264): the accounts page lists every account +// mail-store.accounts answers, each with its state and its folders' mail and unread counts; the folder page +// lists one folder's mails, newest first, 50 a page, from mail-store.search, paged by the date of the last one +// shown. Every page behind the sign-in (NewServer's own RequireSession wrapping); rendered only through +// html/template, so nothing a mail carries is ever interpreted as markup. + +import ( + "context" + "encoding/json" + "fmt" + "html/template" + "net/http" + "time" +) + +// accountBucketValue and accountBucketFolder mirror mesh-mail's own AccountValue and AccountFolderValue +// (modules/mesh-mail/cmd/mesh-mail/mirror.go, issue 1255) — copied, not imported: mesh-mail-web is its own Go +// module, and the shape is mesh-mail's own to change. Only the fields this page reads are kept. +type accountBucketValue struct { + State string `json:"state"` + Folders []accountBucketFolder `json:"folders"` +} + +type accountBucketFolder struct { + Name string `json:"name"` + Mails int `json:"mails"` + Unread int `json:"unread"` +} + +// accountsAnswer is mail-store.accounts' own answer (mesh-mail tools.go accountsTool), the fields this page +// reads. +type accountsAnswer struct { + Accounts []struct { + Account string `json:"account"` + Kind string `json:"kind"` + Address string `json:"address"` + State string `json:"state"` + Why string `json:"why"` + } `json:"accounts"` +} + +// accountRow is one account's own line on the accounts page. +type accountRow struct { + Name, Kind, Address, State, Why string + FoldersKnown bool + Folders []accountBucketFolder +} + +var accountsTemplate = template.Must(template.New("accounts").Parse(` +Mail accounts +

Accounts

+{{if .Error}}

{{.Error}}

{{else}} +
    +{{range .Accounts}} +{{$row := .}} +
  • {{.Name}} ({{.Kind}}, {{.Address}}) — {{.State}}{{if .Why}}: {{.Why}}{{end}} +{{if .FoldersKnown}} +
      +{{range .Folders}}
    • {{.Name}}: {{.Mails}} mail(s), {{.Unread}} unread
      • {{end}} +
    +{{else}}

    its folders are not known yet

    {{end}} +
    • +{{end}} +
+{{end}} +`)) + +// accountsTemplateData avoids the html/template range variable losing access to the account's own name inside +// its nested range over folders (Go templates have no closures): each row carries its own name twice. +type accountsTemplateData struct { + Accounts []accountRowWithName + Error string +} + +type accountRowWithName struct { + accountRow + CurrentAccount string +} + +// accountsPage is issue 1264/AC1-AC5: lists every account mail-store.accounts answers (never one it did not, +// AC2), each with its folders' mail and unread counts read from mesh-mail's own account bucket — one Get per +// account mail-store answered, never Keys() (AC2), and never failing the whole page when an account has no +// key there yet (AC4). The only seats this page calls are mail-store.accounts and (the folder page, below) +// mail-store.search (AC5). +func (b *Bundle) accountsPage(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second) + defer cancel() + raw, err := b.accountsAsk(ctx) + if err != nil { + renderPage(w, accountsTemplate, accountsTemplateData{Error: "mail-store does not answer; try again shortly"}) + return + } + var answer accountsAnswer + if err := json.Unmarshal(raw, &answer); err != nil { + renderPage(w, accountsTemplate, accountsTemplateData{Error: "mail-store's answer could not be read"}) + return + } + rows := make([]accountRowWithName, 0, len(answer.Accounts)) + for _, a := range answer.Accounts { + row := accountRow{Name: a.Account, Kind: a.Kind, Address: a.Address, State: a.State, Why: a.Why} + // 1264/AC2, AC4: read only this account's own key, never the bucket's whole list; a missing key (the + // account's keeper has not mirrored it yet) is not an error, just "not known yet". + entry, err := b.accountBucketGet(a.Account) + if err == nil && entry != nil { + var v accountBucketValue + if json.Unmarshal(entry.Value, &v) == nil { + row.Folders = v.Folders + row.FoldersKnown = true + } + } + rows = append(rows, accountRowWithName{accountRow: row, CurrentAccount: a.Account}) + } + renderPage(w, accountsTemplate, accountsTemplateData{Accounts: rows}) +} + +// MostFolderResults is the folder and search pages' own page size (novox/hq ADR 0440 §4, issue 1264/AC3). +const MostFolderResults = 50 + +// searchResult is one mail as mail-store.search answers it (mesh-mail imap.go's metadata), the fields the +// folder and search pages read. +type searchResult struct { + Account string `json:"account"` + Folder string `json:"folder"` + UID int `json:"uid"` + Date string `json:"date"` + From []personResult `json:"from"` + To []personResult `json:"to"` + Subject string `json:"subject"` + Size int64 `json:"size"` + Flags []string `json:"flags"` +} + +type personResult struct { + Name string `json:"name,omitempty"` + Address string `json:"address"` +} + +// searchAnswer is mail-store.search's own answer shape for a plain (non group-by-sender) search. +type searchAnswer struct { + Found []searchResult `json:"found"` +} + +var folderTemplate = template.Must(template.New("folder").Parse(` +{{.Folder}} +

{{.Account}} — {{.Folder}}

+{{if .Error}}

{{.Error}}

{{else}} +
    +{{range .Mails}}
  • {{.Date}} — {{range .From}}{{.Name}} <{{.Address}}> {{end}}— {{.Subject}}
    • {{end}} +
+{{if .NextBefore}}

Older

{{end}} +{{end}} +`)) + +type folderTemplateData struct { + Account, Folder, NextBefore, Error string + Mails []searchResult +} + +// folderPage is issue 1264/AC3: the 50 newest mails of one folder, newest first, from mail-store.search; the +// next link asks for mails before the date of the 50th shown. A page with fewer than 50 has no next link: a +// mail sharing that exact second with the last one shown could be skipped by the next page (a known limit of +// paging by date alone, accepted rather than fixed here). +func (b *Bundle) folderPage(w http.ResponseWriter, r *http.Request) { + account := r.URL.Query().Get("account") + folder := r.URL.Query().Get("folder") + before := r.URL.Query().Get("before") + if account == "" || folder == "" { + http.Error(w, "account and folder are both needed", http.StatusBadRequest) + return + } + args := map[string]any{"account": account, "folder": folder, "limit": float64(MostFolderResults)} + if before != "" { + args["before"] = before + } + ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second) + defer cancel() + raw, err := b.searchAsk(ctx, args) + if err != nil { + renderPage(w, folderTemplate, folderTemplateData{Account: account, Folder: folder, Error: "the folder could not be answered"}) + return + } + var answer searchAnswer + if err := json.Unmarshal(raw, &answer); err != nil { + renderPage(w, folderTemplate, folderTemplateData{Account: account, Folder: folder, Error: "the answer could not be read"}) + return + } + data := folderTemplateData{Account: account, Folder: folder, Mails: answer.Found} + if len(answer.Found) >= MostFolderResults { + data.NextBefore = answer.Found[len(answer.Found)-1].Date + } + renderPage(w, folderTemplate, data) +} + +// renderPage executes tmpl, setting the page's own content type first; a template that fails to execute (it +// never should, given these fixed templates) says so in words, never a stack trace. +func renderPage(w http.ResponseWriter, tmpl *template.Template, data any) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + if err := tmpl.Execute(w, data); err != nil { + fmt.Fprintf(w, "\n", err) + } +} diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/accounts_test.go b/modules/mesh-mail-web/cmd/mesh-mail-web/accounts_test.go new file mode 100644 index 00000000..5814333a --- /dev/null +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/accounts_test.go @@ -0,0 +1,180 @@ +package main + +// The accounts and folder pages (novox/hq ADR 0440 §4, issue 1264). + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// fakeAccountsBucket is a fake AccountsBucketGet (tools.go): a map, and a counter of every Get call so a test +// can prove which keys were read. +type fakeAccountsBucket struct { + values map[string]accountBucketValue + gets []string +} + +func (f *fakeAccountsBucket) get(account string) (*stdio.StateEntry, error) { + f.gets = append(f.gets, account) + v, ok := f.values[account] + if !ok { + return nil, nil + } + raw, _ := json.Marshal(v) + return &stdio.StateEntry{Key: account, Value: raw}, nil +} + +func accountsAnswerJSON(accounts ...map[string]any) ([]byte, error) { + return json.Marshal(map[string]any{"accounts": accounts}) +} + +// 1264/AC1: mail-store answers accounts a and b, both held in the bucket; the page lists both, each with its +// state and its folders' mail and unread counts. +func TestAccountsPageListsEachAccountWithItsStateAndFolderCounts(t *testing.T) { + raw, err := accountsAnswerJSON( + map[string]any{"account": "a", "kind": "imap", "address": "a@example.org", "state": "current"}, + map[string]any{"account": "b", "kind": "imap", "address": "b@example.org", "state": "syncing"}, + ) + if err != nil { + t.Fatal(err) + } + bucket := &fakeAccountsBucket{values: map[string]accountBucketValue{ + "a": {State: "current", Folders: []accountBucketFolder{{Name: "INBOX", Mails: 10, Unread: 2}}}, + "b": {State: "syncing", Folders: []accountBucketFolder{{Name: "INBOX", Mails: 3, Unread: 0}}}, + }} + b := &Bundle{ + Accounts: func(context.Context) (json.RawMessage, error) { return raw, nil }, + AccountBucket: bucket.get, + } + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/", nil) + b.accountsPage(w, r) + + body := w.Body.String() + for _, want := range []string{"a", "current", "INBOX", "10 mail(s)", "2 unread", "b", "syncing", "3 mail(s)", "0 unread"} { + if !strings.Contains(body, want) { + t.Fatalf("the page is missing %q:\n%s", want, body) + } + } +} + +// 1264/AC2: the bucket also holds account c, which mail-store did not answer; nothing of c appears, and its +// key is never read. +func TestAccountsPageNeverShowsOrReadsAnAccountMailStoreDidNotAnswer(t *testing.T) { + raw, err := accountsAnswerJSON(map[string]any{"account": "a", "kind": "imap", "address": "a@example.org", "state": "current"}) + if err != nil { + t.Fatal(err) + } + bucket := &fakeAccountsBucket{values: map[string]accountBucketValue{ + "a": {State: "current", Folders: []accountBucketFolder{{Name: "INBOX", Mails: 1, Unread: 0}}}, + "c": {State: "current", Folders: []accountBucketFolder{{Name: "Secret", Mails: 99, Unread: 99}}}, + }} + b := &Bundle{ + Accounts: func(context.Context) (json.RawMessage, error) { return raw, nil }, + AccountBucket: bucket.get, + } + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/", nil) + b.accountsPage(w, r) + + if strings.Contains(w.Body.String(), "c") && strings.Contains(w.Body.String(), "Secret") { + t.Fatalf("the page names account c, which mail-store did not answer:\n%s", w.Body.String()) + } + for _, g := range bucket.gets { + if g == "c" { + t.Fatalf("account c's key was read, though mail-store did not answer it: %v", bucket.gets) + } + } +} + +// 1264/AC3: a folder with 120 mails shows the 50 newest, newest first, and its next link asks mail-store.search +// for that folder with before the date of the 50th shown. +func TestFolderPageShows50NewestAndLinksToTheNextPageByDate(t *testing.T) { + var found []map[string]any + for i := 0; i < 50; i++ { + found = append(found, map[string]any{ + "account": "mesh", "folder": "INBOX", "uid": 120 - i, "subject": "s", + "date": "2026-01-01T12:00:00Z", "from": []map[string]any{{"address": "x@example.org"}}, + }) + } + raw, _ := json.Marshal(map[string]any{"found": found}) + var gotArgs map[string]any + b := &Bundle{Search: func(ctx context.Context, args map[string]any) (json.RawMessage, error) { + gotArgs = args + return raw, nil + }} + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/folder?account=mesh&folder=INBOX", nil) + b.folderPage(w, r) + + if gotArgs["account"] != "mesh" || gotArgs["folder"] != "INBOX" || gotArgs["limit"] != float64(50) { + t.Fatalf("mail-store.search called with %v", gotArgs) + } + body := strings.ToLower(w.Body.String()) + if !strings.Contains(body, "before=2026-01-01t12") { + t.Fatalf("no next link with the 50th mail's date:\n%s", w.Body.String()) + } +} + +// 1264/AC4: an account mail-store answered with no key in the bucket yet is listed, its folders said not known +// yet, and the page does not fail. +func TestAccountsPageSaysFoldersNotKnownYetWithoutFailing(t *testing.T) { + raw, err := accountsAnswerJSON(map[string]any{"account": "a", "kind": "imap", "address": "a@example.org", "state": "unknown"}) + if err != nil { + t.Fatal(err) + } + bucket := &fakeAccountsBucket{values: map[string]accountBucketValue{}} + b := &Bundle{ + Accounts: func(context.Context) (json.RawMessage, error) { return raw, nil }, + AccountBucket: bucket.get, + } + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/", nil) + b.accountsPage(w, r) + + if w.Code != http.StatusOK { + t.Fatalf("status %d, want 200", w.Code) + } + if !strings.Contains(w.Body.String(), "not known yet") { + t.Fatalf("the page does not say folders are not known yet:\n%s", w.Body.String()) + } +} + +// 1264/AC5: across the accounts and folder pages, the only verbs called are mail-store.accounts and +// mail-store.search. +func TestAccountsAndFolderPagesCallOnlyAccountsAndSearch(t *testing.T) { + var accountsCalled, searchCalled int + raw, _ := accountsAnswerJSON(map[string]any{"account": "a", "kind": "imap", "address": "a@example.org", "state": "current"}) + b := &Bundle{ + Accounts: func(context.Context) (json.RawMessage, error) { accountsCalled++; return raw, nil }, + Search: func(context.Context, map[string]any) (json.RawMessage, error) { + searchCalled++ + return json.Marshal(map[string]any{"found": []any{}}) + }, + AccountBucket: (&fakeAccountsBucket{values: map[string]accountBucketValue{}}).get, + Read: func(context.Context, map[string]any) (json.RawMessage, error) { + t.Fatal("mail-store.read called") + return nil, nil + }, + Send: func(context.Context, map[string]any) (json.RawMessage, error) { + t.Fatal("mail-store.send called") + return nil, nil + }, + SendStatus: func(context.Context, map[string]any) (json.RawMessage, error) { + t.Fatal("mail-store.send-status called") + return nil, nil + }, + } + b.accountsPage(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/", nil)) + b.folderPage(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/folder?account=a&folder=INBOX", nil)) + + if accountsCalled != 1 || searchCalled != 1 { + t.Fatalf("accounts called %d time(s), search %d time(s)", accountsCalled, searchCalled) + } +} diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/auth_test.go b/modules/mesh-mail-web/cmd/mesh-mail-web/auth_test.go index b16d269a..bec8539f 100644 --- a/modules/mesh-mail-web/cmd/mesh-mail-web/auth_test.go +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/auth_test.go @@ -145,7 +145,7 @@ func operatorsFile(t *testing.T, names ...string) string { func TestNoSessionRedirectsToTheIssuersAuthorisationEndpoint(t *testing.T) { fi := newFakeIssuer(t) auth := testAuth(t, fi, operatorsFile(t, "alice"), nil) - s := NewServer(InternalHost("g14"), auth) + s := NewServer(InternalHost("g14"), auth, nil) w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/", nil) @@ -206,7 +206,7 @@ func TestACallbackWithAValidIDTokenNamingAnOperatorSetsTheSessionCookie(t *testi fi := newFakeIssuer(t) now := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) auth := testAuth(t, fi, operatorsFile(t, "alice"), func() time.Time { return now }) - s := NewServer(InternalHost("g14"), auth) + s := NewServer(InternalHost("g14"), auth, nil) state := startSignIn(t, s, auth) // The session clock (now) is mocked to test the 12-hour bound (TestASessionTwelveHoursOldIsTreatedAsNoSession @@ -243,7 +243,7 @@ func TestACallbackForAnIdentityNotInOperatorsIsRefused(t *testing.T) { fi := newFakeIssuer(t) now := time.Now() auth := testAuth(t, fi, operatorsFile(t, "alice"), func() time.Time { return now }) - s := NewServer(InternalHost("g14"), auth) + s := NewServer(InternalHost("g14"), auth, nil) state := startSignIn(t, s, auth) fi.nextIDToken = fi.idToken("mail-web", "mallory", now, nil, fi.key, fi.kid) @@ -286,7 +286,7 @@ func TestACallbackIsRefusedOnAnUnknownStateOrAnInvalidIDToken(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { auth := testAuth(t, fi, operatorsFile(t, "alice"), func() time.Time { return now }) - s := NewServer(InternalHost("g14"), auth) + s := NewServer(InternalHost("g14"), auth, nil) state := c.setup(auth, s) fi.nextIDToken = c.token() w := callback(s, state, "some-code") @@ -307,7 +307,7 @@ func TestASessionTwelveHoursOldIsTreatedAsNoSession(t *testing.T) { start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) now := start auth := testAuth(t, fi, operatorsFile(t, "alice"), func() time.Time { return now }) - s := NewServer(InternalHost("g14"), auth) + s := NewServer(InternalHost("g14"), auth, nil) state := startSignIn(t, s, auth) // The ID token's own iat/exp are judged against the real wall clock (see the comment in @@ -353,7 +353,7 @@ func TestCallbackWithEmptyOperatorsLetsNobodyIn(t *testing.T) { fi := newFakeIssuer(t) now := time.Now() auth := testAuth(t, fi, operatorsFile(t), func() time.Time { return now }) - s := NewServer(InternalHost("g14"), auth) + s := NewServer(InternalHost("g14"), auth, nil) state := startSignIn(t, s, auth) fi.nextIDToken = fi.idToken("mail-web", "alice", now, nil, fi.key, fi.kid) diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/main.go b/modules/mesh-mail-web/cmd/mesh-mail-web/main.go index 35af11ce..94daefeb 100644 --- a/modules/mesh-mail-web/cmd/mesh-mail-web/main.go +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/main.go @@ -48,7 +48,7 @@ func serveHTTP(b *Bundle) { logf("[mesh-mail-web] sign-in cannot be set up yet (%v); trying again in %s", err, wait) time.Sleep(wait) } - s := NewServer(host, auth) + s := NewServer(host, auth, b) if err := http.ListenAndServe(":"+b.Env.Listen, s); err != nil { logf("[mesh-mail-web] its HTTP endpoint ended: %v", err) } diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/server.go b/modules/mesh-mail-web/cmd/mesh-mail-web/server.go index 6fbd5a35..4c198dc1 100644 --- a/modules/mesh-mail-web/cmd/mesh-mail-web/server.go +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/server.go @@ -28,24 +28,35 @@ const CSP = "default-src 'self'; script-src 'self'; style-src 'self'; img-src 's func InternalHost(node string) string { return "mail." + node + ".internal" } // Server answers mesh-mail-web's HTTP endpoint: a Host check and a CSP on every answer (issue 1283), wrapping -// a plain mux that later issues (1284's form check, 1264's pages, ...) add routes to. +// a plain mux every page is added to, each behind the sign-in (RequireSession). type Server struct { Host string // InternalHost(node) Mux *http.ServeMux - Auth *Auth // nil refuses no page by session (1283 alone); issue 1263 gives one + Auth *Auth // nil refuses no page by session (1283 alone); issue 1263 gives one + Web *Bundle // nil answers only the stand-in root page (1283 alone); issue 1264 gives one } -// NewServer is a Server for host, with the root page issue 1283 itself answers (a stand-in: the real accounts -// and folder pages are issue 1264's) and, when auth is given, the sign-in callback (issue 1263) and every page -// behind a session (RequireSession). -func NewServer(host string, auth *Auth) *Server { - s := &Server{Host: host, Mux: http.NewServeMux(), Auth: auth} - root := s.rootPage +// NewServer is a Server for host: with auth given, the sign-in callback (issue 1263) and every page behind a +// session (RequireSession); with b also given, the real pages (accounts and folders, issue 1264; search, +// issue 1265; a mail and its attachments, issue 1266; the write page and a send's state, issue 1267) in place +// of 1283's own stand-in root. +func NewServer(host string, auth *Auth, b *Bundle) *Server { + s := &Server{Host: host, Mux: http.NewServeMux(), Auth: auth, Web: b} + protect := func(h http.HandlerFunc) http.HandlerFunc { + if auth != nil { + return auth.RequireSession(h) + } + return h + } if auth != nil { s.Mux.HandleFunc(SignedInPath, auth.Callback) - root = auth.RequireSession(root) } - s.Mux.HandleFunc("/", root) + if b != nil { + s.Mux.HandleFunc("/", protect(b.accountsPage)) + s.Mux.HandleFunc("/folder", protect(b.folderPage)) + } else { + s.Mux.HandleFunc("/", protect(s.rootPage)) + } return s } diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/server_test.go b/modules/mesh-mail-web/cmd/mesh-mail-web/server_test.go index 2c25e35a..c7804eaf 100644 --- a/modules/mesh-mail-web/cmd/mesh-mail-web/server_test.go +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/server_test.go @@ -12,7 +12,7 @@ import ( // 1283/AC1: a request whose Host is not mail..internal — a public name, an IP, or none — is answered // 404, naming nothing of the mail. func TestServerAnswers404ForAnyHostButItsOwnInternalName(t *testing.T) { - s := NewServer(InternalHost("g14"), nil) + s := NewServer(InternalHost("g14"), nil, nil) for _, host := range []string{"example.com", "203.0.113.5", "", "mail.other-node.internal"} { w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/", nil) @@ -29,7 +29,7 @@ func TestServerAnswers404ForAnyHostButItsOwnInternalName(t *testing.T) { // Its own internal name is answered (not 404). func TestServerAnswersItsOwnInternalName(t *testing.T) { - s := NewServer(InternalHost("g14"), nil) + s := NewServer(InternalHost("g14"), nil, nil) w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/", nil) r.Host = InternalHost("g14") @@ -41,7 +41,7 @@ func TestServerAnswersItsOwnInternalName(t *testing.T) { // A Host with a port still matches, port and all (the mesh's own listen carries one). func TestServerAnswersItsOwnInternalNameWithAPort(t *testing.T) { - s := NewServer(InternalHost("g14"), nil) + s := NewServer(InternalHost("g14"), nil, nil) w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/", nil) r.Host = InternalHost("g14") + ":8099" @@ -53,7 +53,7 @@ func TestServerAnswersItsOwnInternalNameWithAPort(t *testing.T) { // 1283/AC2: every answer (200 and 404 alike) carries the content security policy. func TestServerSetsTheContentSecurityPolicyOnEveryAnswer(t *testing.T) { - s := NewServer(InternalHost("g14"), nil) + s := NewServer(InternalHost("g14"), nil, nil) cases := []string{"example.com", InternalHost("g14")} for _, host := range cases { w := httptest.NewRecorder() @@ -116,7 +116,7 @@ func TestServerStartsAndAnswersAPageWithACwdAndHomeReadOnly(t *testing.T) { } _ = os.Setenv("HOME", filepath.Join(ro, "home-does-not-exist")) - s := NewServer(InternalHost("g14"), nil) + s := NewServer(InternalHost("g14"), nil, nil) srv := httptest.NewServer(s) defer srv.Close() diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/tools.go b/modules/mesh-mail-web/cmd/mesh-mail-web/tools.go index 495e79a2..bff8f550 100644 --- a/modules/mesh-mail-web/cmd/mesh-mail-web/tools.go +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/tools.go @@ -27,11 +27,34 @@ func sdkAccountsAsk(ctx context.Context) (json.RawMessage, error) { return stdio.Ask("seat:mail-store.accounts", map[string]any{}) } -// Bundle is what the tools reach: the module's own settings, and the seat it calls. +// SeatAsk is how the Bundle reaches one of mail-store's other verbs (search, read, send, send-status): the +// bus's own stdio.Ask, unless a test gives its own fake. +type SeatAsk func(ctx context.Context, args map[string]any) (json.RawMessage, error) + +func sdkSeatAsk(verb string) SeatAsk { + return func(ctx context.Context, args map[string]any) (json.RawMessage, error) { + return stdio.Ask("seat:"+verb, args) + } +} + +// AccountsBucketGet is how the Bundle reads mesh-mail's own account bucket (novox/hq ADR 0439 §2, issue 1255), +// one account at a time (issue 1264/AC2: never Keys(), never an account mail-store did not itself answer). +type AccountsBucketGet func(account string) (*stdio.StateEntry, error) + +func sdkAccountsBucketGet(account string) (*stdio.StateEntry, error) { + return stdio.State("mesh-mail.accounts").Get(account) +} + +// Bundle is what the tools reach: the module's own settings, and the seats it calls. type Bundle struct { - Env Env - Accounts AccountsAsk - Now func() time.Time + Env Env + Accounts AccountsAsk + Search SeatAsk + Read SeatAsk + Send SeatAsk + SendStatus SeatAsk + AccountBucket AccountsBucketGet + Now func() time.Time } func (b *Bundle) accountsAsk(ctx context.Context) (json.RawMessage, error) { @@ -41,6 +64,41 @@ func (b *Bundle) accountsAsk(ctx context.Context) (json.RawMessage, error) { return sdkAccountsAsk(ctx) } +func (b *Bundle) searchAsk(ctx context.Context, args map[string]any) (json.RawMessage, error) { + if b.Search != nil { + return b.Search(ctx, args) + } + return sdkSeatAsk("mail-store.search")(ctx, args) +} + +func (b *Bundle) readAsk(ctx context.Context, args map[string]any) (json.RawMessage, error) { + if b.Read != nil { + return b.Read(ctx, args) + } + return sdkSeatAsk("mail-store.read")(ctx, args) +} + +func (b *Bundle) sendAsk(ctx context.Context, args map[string]any) (json.RawMessage, error) { + if b.Send != nil { + return b.Send(ctx, args) + } + return sdkSeatAsk("mail-store.send")(ctx, args) +} + +func (b *Bundle) sendStatusAsk(ctx context.Context, args map[string]any) (json.RawMessage, error) { + if b.SendStatus != nil { + return b.SendStatus(ctx, args) + } + return sdkSeatAsk("mail-store.send-status")(ctx, args) +} + +func (b *Bundle) accountBucketGet(account string) (*stdio.StateEntry, error) { + if b.AccountBucket != nil { + return b.AccountBucket(account) + } + return sdkAccountsBucketGet(account) +} + func (b *Bundle) now() time.Time { if b.Now != nil { return b.Now() From ff1886ac625f6088dac7c1ca07ffa924c29205f3 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 19 Oct 2026 22:24:52 +0200 Subject: [PATCH 09/18] Add the search page, by mail-store.search's own argument names (issue 1265) --- .../mesh-mail-web/cmd/mesh-mail-web/search.go | 116 +++++++++++++++++ .../cmd/mesh-mail-web/search_test.go | 118 ++++++++++++++++++ .../mesh-mail-web/cmd/mesh-mail-web/server.go | 1 + 3 files changed, 235 insertions(+) create mode 100644 modules/mesh-mail-web/cmd/mesh-mail-web/search.go create mode 100644 modules/mesh-mail-web/cmd/mesh-mail-web/search_test.go diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/search.go b/modules/mesh-mail-web/cmd/mesh-mail-web/search.go new file mode 100644 index 00000000..f800faf8 --- /dev/null +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/search.go @@ -0,0 +1,116 @@ +package main + +// The search page (novox/hq ADR 0440 §4, issue 1265): the form's own field names are exactly mail-store.search's +// own argument names (mesh-mail tools.go searchArgs, issue 1232), so a filled field maps straight onto the +// seat's own argument of the same name; an empty field is left out of the call entirely, never sent as "". +// Paged by date exactly as the folder page (issue 1264): 50 a page, the next link asking again with before the +// last shown mail's own date. + +import ( + "context" + "encoding/json" + "html/template" + "net/http" + "strings" + "time" +) + +var searchTemplate = template.Must(template.New("search").Parse(` +Search mail +

Search

+
+ + + + + + + +
+{{if .Error}}

{{.Error}}

{{else}} +
    +{{range .Mails}}
  • {{.Date}} — {{range .From}}{{.Name}} <{{.Address}}> {{end}}— {{.Subject}}
    • {{end}} +
+{{if .NextBefore}}

Older

{{end}} +{{end}} +`)) + +type searchTemplateData struct { + Query, From, To, OnOrAfter, Before, Error, NextBefore, NextLink string + HasAttachment bool + Mails []searchResult +} + +// searchArgsFromForm builds mail-store.search's own arguments from the request's query parameters: a field +// left empty is left out of the call entirely (issue 1265/AC1) — never sent empty. "from" is split into a list +// on whitespace or commas, mail-store.search's own array argument. limit is always MostFolderResults (AC2), +// mechanical paging the form itself never carries. +func searchArgsFromForm(q map[string][]string) map[string]any { + get := func(name string) string { + if v, ok := q[name]; ok && len(v) > 0 { + return strings.TrimSpace(v[0]) + } + return "" + } + args := map[string]any{"limit": float64(MostFolderResults)} + if v := get("query"); v != "" { + args["query"] = v + } + if v := get("from"); v != "" { + var list []string + for _, part := range strings.FieldsFunc(v, func(r rune) bool { return r == ',' || r == ' ' }) { + if part != "" { + list = append(list, part) + } + } + if len(list) > 0 { + args["from"] = list + } + } + if v := get("to"); v != "" { + args["to"] = v + } + if v := get("on-or-after"); v != "" { + args["on-or-after"] = v + } + if v := get("before"); v != "" { + args["before"] = v + } + if v := get("has-attachment"); v != "" { + args["has-attachment"] = true + } + return args +} + +// searchPage is issue 1265/AC1-AC4. +func (b *Bundle) searchPage(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + data := searchTemplateData{ + Query: q.Get("query"), From: q.Get("from"), To: q.Get("to"), + OnOrAfter: q.Get("on-or-after"), Before: q.Get("before"), HasAttachment: q.Get("has-attachment") != "", + } + args := searchArgsFromForm(q) + ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second) + defer cancel() + raw, err := b.searchAsk(ctx, args) + if err != nil { + data.Error = "the search could not be answered" + renderPage(w, searchTemplate, data) + return + } + var answer searchAnswer + if err := json.Unmarshal(raw, &answer); err != nil { + data.Error = "the search could not be answered" + renderPage(w, searchTemplate, data) + return + } + data.Mails = answer.Found + if len(answer.Found) >= MostFolderResults { + last := answer.Found[len(answer.Found)-1].Date + data.NextBefore = last + next := r.URL.Query() + next.Set("before", last) + data.NextLink = "/search?" + next.Encode() + } + renderPage(w, searchTemplate, data) +} diff --git a/modules/mesh-mail-web/cmd/mesh-mail-web/search_test.go b/modules/mesh-mail-web/cmd/mesh-mail-web/search_test.go new file mode 100644 index 00000000..33aa1a36 --- /dev/null +++ b/modules/mesh-mail-web/cmd/mesh-mail-web/search_test.go @@ -0,0 +1,118 @@ +package main + +// The search page (novox/hq ADR 0440 §4, issue 1265). + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "strings" + "testing" +) + +// 1265/AC1: the search form filled with words, sender, recipient, a date range and has-attachment calls +// mail-store.search with exactly query, from, to, on-or-after, before and has-attachment carrying those +// values (limit is the page's own mechanical paging, not a form field). +func TestSearchPageCallsSearchWithExactlyTheFilledFormFields(t *testing.T) { + var gotArgs map[string]any + b := &Bundle{Search: func(ctx context.Context, args map[string]any) (json.RawMessage, error) { + gotArgs = args + return json.Marshal(map[string]any{"found": []any{}}) + }} + q := url.Values{ + "query": {"invoice"}, "from": {"ops@example.org lead@example.org"}, "to": {"me@example.org"}, + "on-or-after": {"2026-01-01"}, "before": {"2026-02-01"}, "has-attachment": {"on"}, + } + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/search?"+q.Encode(), nil) + b.searchPage(w, r) + + want := map[string]any{ + "query": "invoice", "from": []string{"ops@example.org", "lead@example.org"}, "to": "me@example.org", + "on-or-after": "2026-01-01", "before": "2026-02-01", "has-attachment": true, "limit": float64(50), + } + if !reflect.DeepEqual(gotArgs, want) { + t.Fatalf("args %#v, want %#v", gotArgs, want) + } +} + +// Unfilled fields are left out of the call entirely, never sent as "". +func TestSearchPageLeavesEmptyFieldsOutOfTheCall(t *testing.T) { + var gotArgs map[string]any + b := &Bundle{Search: func(ctx context.Context, args map[string]any) (json.RawMessage, error) { + gotArgs = args + return json.Marshal(map[string]any{"found": []any{}}) + }} + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/search?query=hi", nil) + b.searchPage(w, r) + + want := map[string]any{"query": "hi", "limit": float64(50)} + if !reflect.DeepEqual(gotArgs, want) { + t.Fatalf("args %#v, want %#v", gotArgs, want) + } +} + +// 1265/AC2: more than 50 results shows 50 newest first, and the next link asks again with before the date of +// the 50th shown. +func TestSearchPageShows50AndLinksToTheNextPageByDate(t *testing.T) { + var found []map[string]any + for i := 0; i < 50; i++ { + found = append(found, map[string]any{ + "account": "mesh", "folder": "INBOX", "uid": 200 - i, "subject": "s", + "date": "2026-03-01T08:00:00Z", "from": []map[string]any{{"address": "x@example.org"}}, + }) + } + raw, _ := json.Marshal(map[string]any{"found": found}) + b := &Bundle{Search: func(context.Context, map[string]any) (json.RawMessage, error) { return raw, nil }} + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/search?query=x", nil) + b.searchPage(w, r) + + body := strings.ToLower(w.Body.String()) + if !strings.Contains(body, "before=2026-03-01t08") { + t.Fatalf("no next link with the 50th mail's date:\n%s", w.Body.String()) + } +} + +// 1265/AC3: words holding ", + "date": "2026-01-01T00:00:00Z", "from": []map[string]any{{"address": "x@example.org"}}}, + }}) + b := &Bundle{Search: func(context.Context, map[string]any) (json.RawMessage, error) { return raw, nil }} + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/search?query=x", nil) + b.searchPage(w, r) + + body := w.Body.String() + if strings.Contains(body, " and shows that +// text escaped, carries no script or img element, and nothing on the page names another origin. +func TestMailPageEscapesScriptAndRemoteImageInTheBody(t *testing.T) { + raw, _ := json.Marshal(map[string]any{ + "account": "mesh", "folder": "INBOX", "uid": 7, "subject": "hi", "date": "2026-01-01T00:00:00Z", + "from": []map[string]any{{"address": "bob@example.org"}}, "to": []map[string]any{{"address": "me@example.org"}}, + "text": ` and `, + }) + b := &Bundle{Read: func(context.Context, map[string]any) (json.RawMessage, error) { return raw, nil }} + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/mail?account=mesh&folder=INBOX&uid=7", nil) + b.mailPage(w, r) + + body := w.Body.String() + if strings.Contains(body, " and `, + "attachments": []map[string]any{ + {"key": "att1", "name": "invoice.pdf", "size": 11, "type": "application/pdf"}, + }, + }, nil +} + +// sendStandIn answers mail-store.send once with a fixed ask id, and mail-store.send-status "waiting" the +// first time it is asked, "sent" from then on — so the write page's own flow (issue 1267) is followed from +// its POST to a final state exactly as a browser's own meta-refresh polling would (design test 3). +type sendStandIn struct{ statusCalls int } + +func (s *sendStandIn) send(map[string]any) (any, error) { + return map[string]any{"ask": "send-1"}, nil +} + +func (s *sendStandIn) status(map[string]any) (any, error) { + s.statusCalls++ + if s.statusCalls == 1 { + return map[string]any{"status": "waiting", "words": "the operator has not answered yet"}, nil + } + return map[string]any{"status": "sent", "words": "sent on the operator's Approve"}, nil +} + +// 1268/design test 1 (AC1): no session redirects a page, an attachment and a send to sign-in; an identity +// not in operators is 403; a send without its token is 403; the session cookie is Secure, HttpOnly, +// SameSite=Strict, 12 hours. +func TestAcceptanceSignInRedirectsAndRefusals(t *testing.T) { + mw := startMailWeb(t, acceptanceAccounts(), []string{"alice"}) + mw.store.Accounts = func(map[string]any) (any, error) { + return map[string]any{"accounts": []any{map[string]any{"account": "a", "kind": "imap", "address": "a@example.org", "state": "current"}}}, nil + } + + for _, path := range []string{"/", "/attachment?account=a&folder=INBOX&uid=1&key=att1", "/write"} { + resp := doWithCookie(t, mw, http.MethodGet, path, nil, nil) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusFound { + t.Fatalf("%s with no session: status %d, want 302 (redirect to sign in)", path, resp.StatusCode) + } + } + sendResp := doForm(t, mw, "/send", url.Values{}, nil) + _ = sendResp.Body.Close() + if sendResp.StatusCode != http.StatusFound { + t.Fatalf("POST /send with no session: status %d, want 302", sendResp.StatusCode) + } + + // An identity not in operators: 403, no session cookie. + req, err := http.NewRequest(http.MethodGet, "http://"+mw.internalHost+"/", nil) + if err != nil { + t.Fatal(err) + } + req.Host = mw.internalHost + resp, err := mw.client.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + loc, err := url.Parse(resp.Header.Get("Location")) + if err != nil { + t.Fatal(err) + } + mw.fakeIssuer.nextIDToken = mw.fakeIssuer.idToken(testClientID, "mallory", time.Now(), nil, mw.fakeIssuer.key, mw.fakeIssuer.kid) + cbReq, err := http.NewRequest(http.MethodGet, + "http://"+mw.internalHost+SignedInPath+"?state="+url.QueryEscape(loc.Query().Get("state"))+"&code=c", nil) + if err != nil { + t.Fatal(err) + } + cbReq.Host = mw.internalHost + cbResp, err := mw.client.Do(cbReq) + if err != nil { + t.Fatal(err) + } + defer cbResp.Body.Close() + if cbResp.StatusCode != http.StatusForbidden { + t.Fatalf("an identity not in operators: status %d, want 403", cbResp.StatusCode) + } + for _, c := range cbResp.Cookies() { + if c.Name == SessionCookie { + t.Fatal("a session cookie was set for an identity not in operators") + } + } + + // A signed-in operator's send without the token: 403. + cookie := signIn(t, mw, "alice", "/") + if !cookie.Secure || !cookie.HttpOnly || cookie.SameSite != http.SameSiteStrictMode { + t.Fatalf("cookie attributes: Secure=%v HttpOnly=%v SameSite=%v", cookie.Secure, cookie.HttpOnly, cookie.SameSite) + } + if cookie.MaxAge != int(SessionLifetime/time.Second) { + t.Fatalf("cookie MaxAge %d, want %d (12h)", cookie.MaxAge, int(SessionLifetime/time.Second)) + } + + noTokenResp := doForm(t, mw, "/send", url.Values{"account": {"a"}, "to": {"x@example.org"}, "subject": {"s"}, "body": {"b"}}, cookie) + defer noTokenResp.Body.Close() + if noTokenResp.StatusCode != http.StatusForbidden { + t.Fatalf("a send with no form token: status %d, want 403", noTokenResp.StatusCode) + } +} + +// 1268/design test 2 (AC2): a request with any other Host is answered 404. +func TestAcceptanceOtherHostIs404(t *testing.T) { + mw := startMailWeb(t, nil, nil) + req, err := http.NewRequest(http.MethodGet, "http://"+mw.internalHost+"/", nil) + if err != nil { + t.Fatal(err) + } + req.Host = "example.com" + resp, err := mw.client.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("Host example.com: status %d, want 404", resp.StatusCode) + } +} + +// 1268/design test 3 (AC3): a signed-in operator browsing the accounts page, a folder's two pages, a search, +// a mail and a send followed to sent — the stand-in recorded only the five mail verbs; folders appear only +// for account a; the second folder page asked before the last date shown; the send's own page reached sent; +// the manifest declares no database and no writable directory beyond the state directory. +func TestAcceptanceBrowsingAndSendingEndToEnd(t *testing.T) { + mw := startMailWeb(t, acceptanceAccounts(), []string{"alice"}) + mw.store.Accounts = func(map[string]any) (any, error) { + return map[string]any{"accounts": []any{map[string]any{"account": "a", "kind": "imap", "address": "a@example.org", "state": "current"}}}, nil + } + mails := acceptanceMails(time.Now()) + mw.store.Search = searchStandIn(mails) + mw.store.Read = readStandIn + sends := &sendStandIn{} + mw.store.Send = sends.send + mw.store.SendStatus = sends.status + + cookie := signIn(t, mw, "alice", "/") + + // The accounts page: a appears, c (ungranted) never does. + accResp := doWithCookie(t, mw, http.MethodGet, "/", nil, cookie) + accBody, _ := io.ReadAll(accResp.Body) + _ = accResp.Body.Close() + if accResp.StatusCode != http.StatusOK { + t.Fatalf("accounts page: status %d: %s", accResp.StatusCode, accBody) + } + if !strings.Contains(string(accBody), "INBOX") { + t.Fatalf("the accounts page does not show account a's folder:\n%s", accBody) + } + if strings.Contains(string(accBody), "Secret") { + t.Fatalf("the accounts page shows account c's folder, though it was never granted:\n%s", accBody) + } + + // The folder's first page: 50 mails, a next link to the 51st-55th. + page1 := doWithCookie(t, mw, http.MethodGet, "/folder?account=a&folder=INBOX", nil, cookie) + page1Body, _ := io.ReadAll(page1.Body) + _ = page1.Body.Close() + fiftieth := mails[49]["date"].(string) + // html/template percent-encodes an href's own query value (":" becomes "%3a" etc — issue 1264's own + // unit test, TestFolderPageShows50NewestAndLinksToTheNextPageByDate, hit the same shape): compared only + // up to the date's own colon-free prefix, never byte-for-byte against the raw RFC3339 string. + datePrefix := strings.SplitN(fiftieth, ":", 2)[0] // "2026-01-02T12" + if !strings.Contains(strings.ToLower(string(page1Body)), strings.ToLower("before="+datePrefix)) { + t.Fatalf("the folder's first page does not link to the next page before the 50th mail's date (%s):\n%s", fiftieth, page1Body) + } + + // The folder's second page: asked with that same before. + page2 := doWithCookie(t, mw, http.MethodGet, "/folder?account=a&folder=INBOX&before="+url.QueryEscape(fiftieth), nil, cookie) + page2Body, _ := io.ReadAll(page2.Body) + _ = page2.Body.Close() + if page2.StatusCode != http.StatusOK { + t.Fatalf("the folder's second page: status %d: %s", page2.StatusCode, page2Body) + } + if !strings.Contains(string(page2Body), "mail 5") { // the 55th-50th mails' own subjects + t.Fatalf("the folder's second page does not show the older mails:\n%s", page2Body) + } + + // A search. + searchResp := doWithCookie(t, mw, http.MethodGet, "/search?query=mail", nil, cookie) + _ = searchResp.Body.Close() + if searchResp.StatusCode != http.StatusOK { + t.Fatalf("the search page: status %d", searchResp.StatusCode) + } + + // A mail, and its attachment. + mailResp := doWithCookie(t, mw, http.MethodGet, "/mail?account=a&folder=INBOX&uid=1", nil, cookie) + mailBody, _ := io.ReadAll(mailResp.Body) + _ = mailResp.Body.Close() + if mailResp.StatusCode != http.StatusOK { + t.Fatalf("the mail page: status %d: %s", mailResp.StatusCode, mailBody) + } + attResp := doWithCookie(t, mw, http.MethodGet, "/attachment?account=a&folder=INBOX&uid=1&key=att1", nil, cookie) + attBody, _ := io.ReadAll(attResp.Body) + _ = attResp.Body.Close() + if attResp.StatusCode != http.StatusOK || string(attBody) != "hello world" { + t.Fatalf("the attachment: status %d, body %q", attResp.StatusCode, attBody) + } + + // The write page's own form token, then a send followed to sent. + writeResp := doWithCookie(t, mw, http.MethodGet, "/write", nil, cookie) + writeBody, _ := io.ReadAll(writeResp.Body) + _ = writeResp.Body.Close() + token := extractHiddenValue(string(writeBody), FormTokenField) + if token == "" { + t.Fatalf("the write page carries no form token:\n%s", writeBody) + } + sendResp := doForm(t, mw, "/send", url.Values{ + "account": {"a"}, "to": {"ops@example.org"}, "subject": {"hi"}, "body": {"the body"}, FormTokenField: {token}, + }, cookie) + sendBody, _ := io.ReadAll(sendResp.Body) + _ = sendResp.Body.Close() + if sendResp.StatusCode != http.StatusOK || !strings.Contains(string(sendBody), "waiting") { + t.Fatalf("the send's first answer: status %d: %s", sendResp.StatusCode, sendBody) + } + statusResp := doWithCookie(t, mw, http.MethodGet, "/send-status?id=send-1", nil, cookie) + statusBody, _ := io.ReadAll(statusResp.Body) + _ = statusResp.Body.Close() + if !strings.Contains(string(statusBody), "sent") { + t.Fatalf("the send's state did not reach sent:\n%s", statusBody) + } + if strings.Contains(string(statusBody), `http-equiv="refresh"`) { + t.Fatalf("the send's final page still refreshes itself:\n%s", statusBody) + } + + // Only the five mail verbs were ever asked. + wantKeys := map[string]bool{ + "seat:mail-store.accounts": true, "seat:mail-store.search": true, "seat:mail-store.read": true, + "seat:mail-store.send": true, "seat:mail-store.send-status": true, + } + for _, k := range mw.store.Keys() { + if !wantKeys[k] { + t.Fatalf("an unexpected verb was asked: %s", k) + } + } + + // The manifest declares no database and no writable directory beyond the state directory (reusing + // manifest_test.go's own reader, this package's always-compiled parser). + m := readManifest(t) + for _, dbish := range []string{"postgres-database", "s3-bucket"} { + if contains(m.Requires, dbish) { + t.Fatalf("the manifest requires %s, a database or object store", dbish) + } + } + wantResourceIDs := map[string]bool{"account": true, "state": true, "operators": true, "oidc-config": true} + for _, r := range m.Resources { + id, _ := r["id"].(string) + if !wantResourceIDs[id] { + t.Fatalf("an unexpected resource %q: no writable directory beyond the state dir is wanted", id) + } + } +} + +// 1268/design test 4 (AC4): a mail with a script element and a remote image is shown as text; an attachment +// comes as a download with nosniff; every page answered carries the content security policy. +func TestAcceptanceMailEscapingAttachmentDownloadAndCSP(t *testing.T) { + mw := startMailWeb(t, acceptanceAccounts(), []string{"alice"}) + mw.store.Accounts = func(map[string]any) (any, error) { + return map[string]any{"accounts": []any{map[string]any{"account": "a", "kind": "imap", "address": "a@example.org", "state": "current"}}}, nil + } + mw.store.Read = readStandIn + cookie := signIn(t, mw, "alice", "/") + + mailResp := doWithCookie(t, mw, http.MethodGet, "/mail?account=a&folder=INBOX&uid=1", nil, cookie) + mailBody, _ := io.ReadAll(mailResp.Body) + _ = mailResp.Body.Close() + body := string(mailBody) + if strings.Contains(body, "