diff --git a/modules/mesh-mail/cmd/mesh-mail/acceptance_integration_test.go b/modules/mesh-mail/cmd/mesh-mail/acceptance_integration_test.go new file mode 100644 index 00000000..33ae896c --- /dev/null +++ b/modules/mesh-mail/cmd/mesh-mail/acceptance_integration_test.go @@ -0,0 +1,328 @@ +//go:build integration + +package main + +// Release 3's acceptance test (novox/hq ADR 0436, issue 1198 AC4): the whole module, over a JetStream bus +// of the test's own (nats-server v2.11.17, copied from modules/mesh-delivery/cmd/mesh-delivery/bus_test.go +// as newTestBus already does in sender_integration_test.go), a fake SMTP server, the router's records +// bucket with a closed Approve record, the account file and a password file written by the test. Compiled +// by the merge gate's `go vet -tags integration`; never run on this machine (the context file's hard rule). +// +// "The tools queue an approved send" is played here directly against the real bus (Decide, send.go) rather +// than through the tool runner's own RPC — the tool runner's part (stdio.Serve, grants, the ask's wording) +// is proved elsewhere (send_test.go, ask_test.go); what this test proves is the handover from an approved, +// queued send through to the sender's own claim, send and outcome, end to end over a real bus. + +import ( + "context" + "encoding/json" + "errors" + "path/filepath" + "strings" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +// kvSends adapts a raw jetstream.KeyValue to SendStore (send.go): the test's own stand-in for both the +// tools' and the sender's `sends` state, which in production are the same bucket reached under two +// different accounts. +type kvSends struct{ kv jetstream.KeyValue } + +func (k kvSends) Get(key string) (json.RawMessage, uint64, bool, error) { + e, err := k.kv.Get(context.Background(), key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, 0, false, nil + } + if err != nil { + return nil, 0, false, err + } + return json.RawMessage(e.Value()), e.Revision(), true, nil +} +func (k kvSends) Create(key string, value any) (uint64, error) { + data, err := json.Marshal(value) + if err != nil { + return 0, err + } + rev, err := k.kv.Create(context.Background(), key, data) + if errors.Is(err, jetstream.ErrKeyExists) { + return 0, stdio.ErrChanged + } + return rev, err +} +func (k kvSends) Update(key string, value any, revision uint64) (uint64, error) { + data, err := json.Marshal(value) + if err != nil { + return 0, err + } + rev, err := k.kv.Update(context.Background(), key, data, revision) + if errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + return 0, stdio.ErrChanged + } + return rev, err +} +func (k kvSends) Keys() ([]string, error) { return nil, errors.New("not used by this test") } + +// acceptanceBus is everything the test built on a fresh bus: ready-made buckets and streams, and the raw +// clients to act as the tools with. +type acceptanceBus struct { + nc *nats.Conn + js jetstream.JetStream + sends jetstream.KeyValue + asks jetstream.KeyValue +} + +func newAcceptanceBus(t *testing.T) (*acceptanceBus, string) { + t.Helper() + url := newTestBus(t) + ctx := context.Background() + nc, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + sends, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: Module + "_" + SendsState}) + if err != nil { + t.Fatal(err) + } + asksKV, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: MessengerAsksBucket}) + if err != nil { + t.Fatal(err) + } + if _, err := js.CreateStream(ctx, mailSendsStreamConfig()); err != nil { + t.Fatal(err) + } + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod." + Module + ".event.>"}}); err != nil { + t.Fatal(err) + } + return &acceptanceBus{nc: nc, js: js, sends: sends, asks: asksKV}, url +} + +// queueAnApprovedSend plays the tools' own part (send.go's Decide, the queue publish) directly against the +// real bus: a send asked, approved by a warrant whose digest really is the one req.Ask() binds, recorded +// approved and queued — exactly what issue 1194 made the tools do, without needing the tool runner's RPC. +func queueAnApprovedSend(t *testing.T, bus *acceptanceBus, account string, approvedAt time.Time) (SendRequest, string) { + t.Helper() + ctx := context.Background() + req := SendRequest{ID: newSendID(), Account: account, To: []string{"ops@example.org"}, Subject: "The nightly gate is red", + Body: "See the job log.", Caller: "person.op", At: approvedAt, AskTo: "o… at example org"} + store := kvSends{kv: bus.sends} + if _, err := store.Create(req.ID, req); err != nil { + t.Fatalf("keeping the send: %v", err) + } + w := warrantFor(req, approvedAt) + rec := routerRecord{State: string(w.Outcome), Warrant: &w} + raw, _ := json.Marshal(rec) + if _, err := bus.asks.Put(ctx, Module+"."+req.ID, raw); err != nil { + t.Fatalf("writing the router's record: %v", err) + } + // The tools' own publish path (send.go's QueuePublish, wired for real to stdio.Emit/sdkQueuePublish): + // the real wire subject the controller composes for mail-sends' one event (mailSendApprovedSubject), + // the send id as the message id, so the stream's own duplicate window is the one actually exercised + // (novox/hq review round 1, finding 7). + publish := func(id string, body any) error { + data, err := json.Marshal(body) + if err != nil { + return err + } + _, err = bus.js.Publish(ctx, mailSendApprovedSubject, data, jetstream.WithMsgID(id)) + return err + } + if _, err := Decide(store, publish, w, approvedAt); err != nil { + t.Fatalf("the tools' own Decide: %v", err) + } + kept, _, ok, err := getSend(store, req.ID) + if err != nil || !ok || kept.Decided == nil || kept.Decided.Outcome != "queued" { + t.Fatalf("not left queued: %v %v %+v", err, ok, kept.Decided) + } + // Published a second time under the same message id, exactly as the sweep would for a record it finds + // still queued past its grace (Sweep, send.go): the stream's 24-hour duplicate window keeps it one + // message (novox/hq review round 1, finding 7) — the sender's own claim (clause 9) is what then keeps + // it one send, proved by this test's own "sent once" assertion below. + if err := publish(req.ID, QueuedSend{Send: req.ID, Account: req.Account, Ask: req.ID}); err != nil { + t.Fatalf("the duplicate publish: %v", err) + } + return req, req.ID +} + +// 1198/AC4: when the tools queue an approved send and the send process runs, the fake server receives +// exactly one mail carrying the send id's Message-ID, sends says sent, and mail.send.sent is on the stream. +func TestMailSenderAcceptanceAnApprovedSendIsSentOnceAndSaidOnce(t *testing.T) { + bus, url := newAcceptanceBus(t) + fake := newFakeSMTPServer(t) + + dir := t.TempDir() + accountsPath := filepath.Join(dir, "accounts.json") + writeFile(t, accountsPath, `{"accounts": {"mesh": {"kind": "imap", "address": "jschoubben@novox.be", + "incoming": {"host": "mail.novox.be", "port": 993, "security": "tls"}, + "outgoing": {"host": "127.0.0.1", "port": 1, "security": "tls"}}}}`) + writeFile(t, filepath.Join(dir, "mail-credential-mesh.secret"), "the-password\n") + + approvedAt := judgedAt + req, id := queueAnApprovedSend(t, bus, "mesh", approvedAt) + + account := process.Account{URL: url, Node: "g14", Module: Module, User: "g14." + Module + "." + SendProcessID, Password: "pw"} + if _, err := bus.js.CreateConsumer(context.Background(), MailSendsStream, jetstream.ConsumerConfig{ + Durable: account.QueueConsumerName(), AckPolicy: jetstream.AckExplicitPolicy, AckWait: 5 * time.Second, MaxDeliver: MostAttempts}); err != nil { + t.Fatal(err) + } + conn, err := process.Connect(account) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + h := &senderHandler{ + Store: kvSends{kv: bus.sends}, + Record: routerRecordOf(conn), + Accounts: func() ([]Account, error) { return ReadAccounts(accountsPath) }, + SecretsDir: dir, + Dial: fake.dialer(), + Emit: emitOf(conn), + Now: func() time.Time { return approvedAt.Add(time.Minute) }, + } + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { done <- conn.Consume(ctx, MailSendsStream, h.handle) }() + + eventually(t, 10*time.Second, "the mail sent", func() (bool, string) { + return fake.count() == 1, "received so far: " + describeReceived(fake) + }) + cancel() + <-done + + store := kvSends{kv: bus.sends} + kept, _, ok, err := getSend(store, id) + if err != nil || !ok || kept.Decided == nil || kept.Decided.Outcome != "sent" { + t.Fatalf("the kept send: %v %v %+v", err, ok, kept.Decided) + } + got := fake.last() + if got.to[0] != "ops@example.org" { + t.Fatalf("received: %+v", got) + } + if !strings.Contains(got.data, "Message-Id: <"+req.ID+"@novox.be>") { + t.Fatalf("the mail's Message-ID does not name the send: %s", got.data) + } + + // mail.send.sent is on the EVENTS stream, naming the send id, never the recipient, subject or body. + eventually(t, 10*time.Second, "mail.send.sent on the stream", func() (bool, string) { + msg, err := fetchOneRaw(bus, "mesh.mod."+Module+".event."+MailSendSent) + if err != nil || msg == nil { + return false, "not yet" + } + var e OutcomeEvent + _ = json.Unmarshal(msg, &e) + return e.Send == req.ID && e.Account == "mesh", "got " + string(msg) + }) +} + +// 1198/AC4: when the fake server refuses sign-in, then after the attempts (shortened here) the record says +// failed and nothing was received. +func TestMailSenderAcceptanceARefusedSignInFailsAfterItsAttemptsAndNothingIsReceived(t *testing.T) { + orig := RetryDelays + RetryDelays = []time.Duration{10 * time.Millisecond, 10 * time.Millisecond, 10 * time.Millisecond, 10 * time.Millisecond} + t.Cleanup(func() { RetryDelays = orig }) + + bus, url := newAcceptanceBus(t) + s := newScriptedSMTPServer(t) + s.auth = reply(535, "authentication failed") + + dir := t.TempDir() + accountsPath := filepath.Join(dir, "accounts.json") + writeFile(t, accountsPath, `{"accounts": {"mesh": {"kind": "imap", "address": "jschoubben@novox.be", + "incoming": {"host": "mail.novox.be", "port": 993, "security": "tls"}, + "outgoing": {"host": "127.0.0.1", "port": 1, "security": "tls"}}}}`) + writeFile(t, filepath.Join(dir, "mail-credential-mesh.secret"), "the-wrong-password\n") + + approvedAt := judgedAt + _, id := queueAnApprovedSend(t, bus, "mesh", approvedAt) + + account := process.Account{URL: url, Node: "g14", Module: Module, User: "g14." + Module + "." + SendProcessID, Password: "pw"} + if _, err := bus.js.CreateConsumer(context.Background(), MailSendsStream, jetstream.ConsumerConfig{ + Durable: account.QueueConsumerName(), AckPolicy: jetstream.AckExplicitPolicy, AckWait: 2 * time.Second, MaxDeliver: MostAttempts}); err != nil { + t.Fatal(err) + } + conn, err := process.Connect(account) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + h := &senderHandler{ + Store: kvSends{kv: bus.sends}, + Record: routerRecordOf(conn), + Accounts: func() ([]Account, error) { return ReadAccounts(accountsPath) }, + SecretsDir: dir, + Dial: s.dialer(), + Emit: emitOf(conn), + Now: func() time.Time { return approvedAt.Add(time.Minute) }, + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { done <- conn.Consume(ctx, MailSendsStream, h.handle) }() + + store := kvSends{kv: bus.sends} + eventually(t, 25*time.Second, "the send to fail after its attempts", func() (bool, string) { + kept, _, ok, err := getSend(store, id) + if err != nil || !ok || kept.Decided == nil { + return false, "not yet decided" + } + // "queued" is the send's own starting state (queueAnApprovedSend's own write, before the sender + // ever claims it): without excluding it here too, this check reads done on the very first look, + // long before the sender connects or claims anything at all (novox/hq review: build-1792400882114092431, + // 1198/AC4). + o := kept.Decided.Outcome + return o != "queued" && o != "retrying" && o != "sending", o + }) + cancel() + <-done + + kept, _, ok, err := getSend(store, id) + if err != nil || !ok || kept.Decided == nil || !strings.HasPrefix(kept.Decided.Outcome, "failed: ") { + t.Fatalf("the kept send did not end failed: %v %v %+v", err, ok, kept.Decided) + } + if kept.Decided.Attempt != MostAttempts { + t.Fatalf("failed after %d attempt(s), want all %d", kept.Decided.Attempt, MostAttempts) + } + if n := s.receivedCount(); n != 0 { + t.Fatalf("the scripted server received %d message(s), want none: a refused sign-in never reaches DATA", n) + } +} + +// fetchOneRaw pulls at most one message off a stream's subject, or nil within a short wait, for a test to +// check an emitted event without binding a durable consumer of its own. +func fetchOneRaw(bus *acceptanceBus, subject string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + cons, err := bus.js.OrderedConsumer(ctx, "EVENTS", jetstream.OrderedConsumerConfig{FilterSubjects: []string{subject}}) + if err != nil { + return nil, err + } + batch, err := cons.Fetch(1, jetstream.FetchMaxWait(time.Second)) + if err != nil { + return nil, err + } + msg, ok := <-batch.Messages() + if !ok { + return nil, nil + } + return msg.Data(), nil +} + +func describeReceived(f *fakeSMTPServer) string { + if f.count() == 0 { + return "none" + } + got := f.last() + return got.from +} diff --git a/modules/mesh-mail/cmd/mesh-mail/claim_test.go b/modules/mesh-mail/cmd/mesh-mail/claim_test.go new file mode 100644 index 00000000..8b2e9535 --- /dev/null +++ b/modules/mesh-mail/cmd/mesh-mail/claim_test.go @@ -0,0 +1,451 @@ +package main + +// The sender's own claim (novox/hq ADR 0436 clauses 9, 10, issue 1196): it never trusts the queue's message, +// verifying the operator's warrant again from the router's own record before it claims the send by +// compare-and-set — exactly mirroring the tools' own check, so the boundary clause 9 describes (the tool +// runner's account may publish on the queue and write `sends`, but only a warrant that really closed the +// ask moves a send to "sending"). + +import ( + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" +) + +// claimTestSend asks and approves one send (through the tools, as issue 1194 built it), leaving it queued, +// and returns its id and the router that holds its warrant. +func claimTestSend(t *testing.T) (id string, router *sendRouter, sends *memSends) { + t.Helper() + b, router, sends := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"}) + queue := newMemQueue() + b.Queue = queue.publish + rt := startBundle(t, b.Tools()) + + text, isErr := rt.call(SendTool, "novox.triage", sendArgs) + if isErr { + t.Fatalf("%s", text) + } + var out map[string]any + _ = json.Unmarshal([]byte(text), &out) + id = out["ask"].(string) + + w := router.chose(id, OptionSend, judgedAt.Add(time.Minute)) + router.close(w) + if _, err := Decide(sends, b.queuePublish(), w, judgedAt.Add(2*time.Minute)); err != nil { + t.Fatalf("approving: %v", err) + } + req, _, ok, err := getSend(sends, id) + if err != nil || !ok || req.Decided == nil || req.Decided.Outcome != "queued" { + t.Fatalf("not left queued: %v %v %+v", err, ok, req.Decided) + } + return id, router, sends +} + +// rawQueuedSend creates a send directly in the queued state, without going through the tools or the router +// at all, so an AC2 test controls the router's record independently of how the send reached queued: the +// defence in depth clause 9 describes, since the tool runner's account could write `sends` and the queue +// without ever having closed a real warrant. +func rawQueuedSend(t *testing.T, sends *memSends) SendRequest { + t.Helper() + req := SendRequest{ID: newSendID(), Account: "mesh", To: []string{"ops@example.org"}, Subject: "Status", + Body: "the body", Caller: "person.op", At: judgedAt, AskTo: "o… at example org"} + req.Decided = &SendDecision{At: judgedAt.Add(time.Minute), Outcome: "queued"} + if _, err := sends.Create(req.ID, req); err != nil { + t.Fatal(err) + } + return req +} + +// warrantFor is a well-formed warrant answering req's ask with Send, at, and the ask's own digest — the +// same shape sendRouter.chose builds, for a test that never published the ask to a router at all. +func warrantFor(req SendRequest, at time.Time) asks.Warrant { + ask := req.Ask() + opt, _ := ask.Option(OptionSend) + return asks.Warrant{Ask: req.ID, Asker: Module, Outcome: asks.OutcomeChosen, Option: opt.ID, Label: opt.Label, + Level: opt.Level, + By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}, + Channel: "telegram", Proofs: []string{"P1"}, At: at, AskDigest: ask.Digest()} +} + +// 1196/AC1: given a queued send whose router record is closed by the operator's Approve at level approve +// within its expiry and whose recomputed act digest is the option's binds, when the sender takes the +// message, then it claims the send by compare-and-set from queued to sending with the attempt and time +// before any connection (nothing here connects to anything: claimQueuedSend never dials). +func TestClaimQueuedSendClaimsSendingWithTheAttemptBeforeAnyConnection(t *testing.T) { + id, router, sends := claimTestSend(t) + + emit := &memEmit{} + req, d, _, outcome, err := claimQueuedSend(sends, router.Record, emit.emit, id, judgedAt.Add(3*time.Minute)) + if err != nil { + t.Fatalf("claiming: %v", err) + } + if outcome != claimToSend { + t.Fatalf("outcome: %v, want claimToSend", outcome) + } + if d.Outcome != "sending" || d.Attempt != 1 || !d.At.Equal(judgedAt.Add(3*time.Minute)) { + t.Fatalf("claimed: %+v", d) + } + if req.ID != id { + t.Fatalf("req: %+v", req) + } + kept, _, ok, err := getSend(sends, id) + if err != nil || !ok || kept.Decided == nil || kept.Decided.Outcome != "sending" { + t.Fatalf("not kept as sending: %v %v %+v", err, ok, kept.Decided) + } + if len(emit.calls) != 0 { + t.Fatalf("claiming to send emitted something: %+v", emit.calls) + } +} + +// 1196/AC2: given a router record that is not mesh-mail's, not closed by Approve, approved after its +// expiry, or whose digest differs from the request in sends, when taken, then the sender records refused +// with the reason, terminates the message and connects to no server (claimQueuedSend never dials, so +// "connects to no server" holds for every path by construction; this proves each is refused and the claim +// never advances to sending). +func TestClaimQueuedSendRefusesAMismatchedOrUnclosedWarrant(t *testing.T) { + t.Run("not mesh-mail's ask", func(t *testing.T) { + sends, router := newMemSends(), newSendRouter() + req := rawQueuedSend(t, sends) + w := warrantFor(req, judgedAt.Add(time.Minute)) + w.Asker = "another-module" + router.close(w) + emit := &memEmit{} + _, d, _, outcome, err := claimQueuedSend(sends, router.Record, emit.emit, req.ID, judgedAt.Add(3*time.Minute)) + if err != nil || outcome != claimRefused || !strings.HasPrefix(d.Outcome, "refused: ") { + t.Fatalf("outcome %v, decision %+v, err %v", outcome, d, err) + } + assertEmittedFailed(t, emit, req.ID, d.Outcome) + }) + t.Run("not closed by approve", func(t *testing.T) { + sends, router := newMemSends(), newSendRouter() + req := rawQueuedSend(t, sends) + // The router keeps no record of this ask at all: never published, or never answered. + emit := &memEmit{} + _, d, _, outcome, err := claimQueuedSend(sends, router.Record, emit.emit, req.ID, judgedAt.Add(3*time.Minute)) + if err != nil || outcome != claimRefused || !strings.HasPrefix(d.Outcome, "refused: ") { + t.Fatalf("outcome %v, decision %+v, err %v", outcome, d, err) + } + assertEmittedFailed(t, emit, req.ID, d.Outcome) + }) + t.Run("approved after the ask's expiry", func(t *testing.T) { + sends, router := newMemSends(), newSendRouter() + req := rawQueuedSend(t, sends) + w := warrantFor(req, req.Ask().Expires.Add(time.Hour)) + router.close(w) + emit := &memEmit{} + _, d, _, outcome, err := claimQueuedSend(sends, router.Record, emit.emit, req.ID, judgedAt.Add(3*time.Minute)) + if err != nil || outcome != claimRefused || !strings.HasPrefix(d.Outcome, "refused: ") { + t.Fatalf("outcome %v, decision %+v, err %v", outcome, d, err) + } + assertEmittedFailed(t, emit, req.ID, d.Outcome) + }) + t.Run("a digest the published ask never bound", func(t *testing.T) { + sends, router := newMemSends(), newSendRouter() + req := rawQueuedSend(t, sends) + w := warrantFor(req, judgedAt.Add(time.Minute)) + w.AskDigest = "sha256:" + strings.Repeat("0", 64) + router.close(w) + emit := &memEmit{} + _, d, _, outcome, err := claimQueuedSend(sends, router.Record, emit.emit, req.ID, judgedAt.Add(3*time.Minute)) + if err != nil || outcome != claimRefused || !strings.HasPrefix(d.Outcome, "refused: ") { + t.Fatalf("outcome %v, decision %+v, err %v", outcome, d, err) + } + assertEmittedFailed(t, emit, req.ID, d.Outcome) + }) +} + +// assertEmittedFailed checks that exactly one mail.send.failed was emitted, naming the send id and the +// decision's own outcome in words (novox/hq review round 1 finding 4). +func assertEmittedFailed(t *testing.T, emit *memEmit, id, outcome string) { + t.Helper() + if len(emit.calls) != 1 { + t.Fatalf("emitted %d time(s), want exactly one mail.send.failed: %+v", len(emit.calls), emit.calls) + } + c := emit.calls[0] + if c.eventType != MailSendFailed || c.id != id || c.body.Send != id || c.body.Outcome != outcome { + t.Fatalf("emitted %+v, want mail.send.failed naming %s and outcome %q", c, id, outcome) + } +} + +// 1196/AC3: given a send already sending (the sender stopped mid-send), when its message is delivered +// again, then the sender sends nothing, records not proven and terminates. +func TestClaimQueuedSendOnARedeliveryOfAnAlreadySendingClaimRecordsNotProven(t *testing.T) { + id, router, sends := claimTestSend(t) + emit := &memEmit{} + req, _, _, outcome, err := claimQueuedSend(sends, router.Record, emit.emit, id, judgedAt.Add(3*time.Minute)) + if err != nil || outcome != claimToSend { + t.Fatalf("first claim: %v %v", outcome, err) + } + _ = req + + // Redelivered: the first claim's handler may have stopped mid-send. + _, d, _, outcome, err := claimQueuedSend(sends, router.Record, emit.emit, id, judgedAt.Add(4*time.Minute)) + if err != nil { + t.Fatalf("redelivered claim: %v", err) + } + if outcome != claimNotProven || d.Outcome != "not proven" { + t.Fatalf("outcome %v, decision %+v", outcome, d) + } + assertEmittedFailed(t, emit, id, d.Outcome) +} + +// 1196/AC4: given a lost compare-and-set (another claim), when taken, then it sends nothing and leaves the +// message to the bus (no decision of this call's own making is kept; it reads back whatever the other +// writer decided). +func TestClaimQueuedSendOnALostRaceSendsNothingAndLeavesTheMessageToTheBus(t *testing.T) { + id, router, sends := claimTestSend(t) + racy := &raceOnFirstUpdate{SendStore: sends, race: func() { + r, rv, ok, err := getSend(sends, id) + if err != nil || !ok { + t.Fatalf("racing claim could not read the send: %v %v", err, ok) + } + if _, _, err := settleSend(sends, r, rv, SendDecision{At: judgedAt.Add(3 * time.Minute), Outcome: "sending", Attempt: 1}); err != nil { + t.Fatalf("racing claim: %v", err) + } + }} + + emit := &memEmit{} + _, d, _, outcome, err := claimQueuedSend(racy, router.Record, emit.emit, id, judgedAt.Add(3*time.Minute)) + if err != nil { + t.Fatalf("claiming: %v", err) + } + if outcome != claimLostRace { + t.Fatalf("outcome: %v, want claimLostRace", outcome) + } + if d.Outcome != "sending" || d.Attempt != 1 { + t.Fatalf("the decision read back is not the racing claim's own: %+v", d) + } + if len(emit.calls) != 0 { + t.Fatalf("a lost race emitted something: %+v", emit.calls) + } +} + +// review round 1 finding 3: a failed write of the tells-pending index does not stop the refusal itself +// from being recorded — the record write and the message's own fate are never undone by an index that +// could not be kept — but the index failure is still reported, never silently dropped. +func TestClaimQueuedSendStillRefusesWhenTheTellsPendingIndexCannotBeWritten(t *testing.T) { + sends, router := newMemSends(), newSendRouter() + req := rawQueuedSend(t, sends) + sends.failKey, sends.failErr = tellsPendingKey, errors.New("the store is down") + + emit := &memEmit{} + _, d, _, outcome, err := claimQueuedSend(sends, router.Record, emit.emit, req.ID, judgedAt.Add(3*time.Minute)) + if outcome != claimRefused || !strings.HasPrefix(d.Outcome, "refused: ") { + t.Fatalf("outcome %v, decision %+v", outcome, d) + } + if err == nil { + t.Fatalf("the index failure was not reported") + } + kept, _, ok, kerr := getSend(sends, req.ID) + if kerr != nil || !ok || kept.Decided == nil || kept.Decided.Outcome != d.Outcome { + t.Fatalf("the refusal itself was not kept despite the index failing: %v %v %+v", kerr, ok, kept.Decided) + } + assertEmittedFailed(t, emit, req.ID, d.Outcome) +} + +// Same finding, for the redelivery-finds-sending path (not proven). +func TestClaimQueuedSendStillRecordsNotProvenWhenTheTellsPendingIndexCannotBeWritten(t *testing.T) { + sends, router := newMemSends(), newSendRouter() + req := rawQueuedSend(t, sends) + current, rev, ok, err := getSend(sends, req.ID) + if err != nil || !ok { + t.Fatalf("reading it back: %v %v", err, ok) + } + current.Decided.Outcome = "sending" + if _, err := sends.Update(req.ID, current, rev); err != nil { + t.Fatalf("leaving it sending: %v", err) + } + sends.failKey, sends.failErr = tellsPendingKey, errors.New("the store is down") + + emit := &memEmit{} + _, d, _, outcome, err := claimQueuedSend(sends, router.Record, emit.emit, req.ID, judgedAt.Add(4*time.Minute)) + if outcome != claimNotProven || d.Outcome != "not proven" { + t.Fatalf("outcome %v, decision %+v", outcome, d) + } + if err == nil { + t.Fatalf("the index failure was not reported") + } + kept, _, ok2, kerr := getSend(sends, req.ID) + if kerr != nil || !ok2 || kept.Decided == nil || kept.Decided.Outcome != "not proven" { + t.Fatalf("not proven was not kept despite the index failing: %v %v %+v", kerr, ok2, kept.Decided) + } + assertEmittedFailed(t, emit, req.ID, d.Outcome) +} + +// Same finding, for decideSend: a failed sender-pending index write does not stop "queued" from being +// recorded, nor the publish from being attempted — there is no separate "approved" state any more (novox/hq +// ADR 0436 clause 8, as decided to replace the two-state design, review round 1). +func TestDecideSendStillRecordsQueuedAndPublishesWhenTheSenderPendingIndexCannotBeWritten(t *testing.T) { + sends := newMemSends() + req := SendRequest{ID: newSendID(), Account: "mesh", To: []string{"ops@example.org"}, Subject: "s", Body: "body", At: judgedAt} + rev, err := sends.Create(req.ID, req) + if err != nil { + t.Fatal(err) + } + sends.failKey, sends.failErr = senderPendingKey, errors.New("the store is down") + queue := newMemQueue() + w := warrantFor(req, judgedAt.Add(time.Minute)) + + d, err := decideSend(sends, queue.publish, req, rev, w, judgedAt.Add(time.Minute)) + if d.Outcome != "queued" { + t.Fatalf("decision: %+v, want queued despite the index failing", d) + } + if err == nil { + t.Fatalf("the index failure was not reported") + } + if n := queue.count(); n != 1 { + t.Fatalf("published %d time(s)", n) + } + kept, _, ok, kerr := getSend(sends, req.ID) + if kerr != nil || !ok || kept.Decided == nil || kept.Decided.Outcome != "queued" { + t.Fatalf("not kept as queued despite the index failing: %v %v %+v", kerr, ok, kept.Decided) + } +} + +// 1194/AC1 (as decided to replace the two-state design, review round 1): on a verified warrant the record +// moves straight from undecided to queued by one compare-and-set, with its time, and mail.send.approved is +// published with the send id as its message id — no "approved" state in between. +func TestDecideSendRecordsQueuedDirectlyAndPublishes(t *testing.T) { + sends := newMemSends() + req := SendRequest{ID: newSendID(), Account: "mesh", To: []string{"ops@example.org"}, Subject: "s", Body: "body", At: judgedAt} + rev, err := sends.Create(req.ID, req) + if err != nil { + t.Fatal(err) + } + queue := newMemQueue() + w := warrantFor(req, judgedAt.Add(time.Minute)) + now := judgedAt.Add(time.Minute) + + d, err := decideSend(sends, queue.publish, req, rev, w, now) + if err != nil { + t.Fatalf("deciding: %v", err) + } + if d.Outcome != "queued" || !d.At.Equal(now) { + t.Fatalf("decision: %+v, want queued at %v", d, now) + } + if n := queue.count(); n != 1 { + t.Fatalf("published %d time(s), want exactly 1", n) + } + got := queue.last() + if got.id != req.ID || got.body.Send != req.ID || got.body.Account != "mesh" || got.body.Ask != req.ID { + t.Fatalf("published %+v, want the send id as the message id and body", got) + } + kept, _, ok, kerr := getSend(sends, req.ID) + if kerr != nil || !ok || kept.Decided == nil || kept.Decided.Outcome != "queued" { + t.Fatalf("not kept as queued: %v %v %+v", kerr, ok, kept.Decided) + } + ids, err := senderPendingIDs(sends) + if err != nil || len(ids) != 1 || ids[0] != req.ID { + t.Fatalf("not indexed as sender-pending: %v %v", err, ids) + } +} + +// 1196/AC2 (review round 2): a store that changes the revision between the claim and the refusal's own +// write — another writer's decision landing in between — never emits mail.send.failed for a write that did +// not actually land (refuse() does not retry: that is recordOutcome's own job, send.go, review round 2). +func TestClaimQueuedSendRefuseEmitsNothingWhenTheWriteDidNotLand(t *testing.T) { + sends, router := newMemSends(), newSendRouter() + req := rawQueuedSend(t, sends) + w := warrantFor(req, judgedAt.Add(time.Minute)) + w.AskDigest = "sha256:" + strings.Repeat("0", 64) // mismatched: claimQueuedSend refuses it + router.close(w) + racy := &raceOnFirstUpdate{SendStore: sends, race: func() { + r, rv, ok, err := getSend(sends, req.ID) + if err != nil || !ok { + t.Fatalf("racing write could not read the send: %v %v", err, ok) + } + if _, _, err := settleSend(sends, r, rv, SendDecision{At: judgedAt, Outcome: "not proven", Attempt: 1}); err != nil { + t.Fatalf("racing write: %v", err) + } + }} + emit := &memEmit{} + + _, d, _, outcome, _ := claimQueuedSend(racy, router.Record, emit.emit, req.ID, judgedAt.Add(3*time.Minute)) + if outcome != claimRefused { + t.Fatalf("outcome: %v, want claimRefused", outcome) + } + if len(emit.calls) != 0 { + t.Fatalf("emitted mail.send.failed despite the refusal's own write not landing: %+v", emit.calls) + } + // The kept record is the racing write's own decision, not this call's: refuse() never retries. + kept, _, ok, kerr := getSend(sends, req.ID) + if kerr != nil || !ok || kept.Decided == nil || kept.Decided.Outcome != "not proven" { + t.Fatalf("kept: %v %v %+v (returned %+v)", kerr, ok, kept.Decided, d) + } +} + +// 1196/AC5 (operator-approved change to clause 10): given a send already sent (and one already failed, +// refused, or not proven), when its message is delivered again, then the record is unchanged, nothing is +// emitted or told, and the message is terminated (claimAlreadyFinished; sender.go's handler maps it to +// Term alone, tested here the same way every other claimOutcome is in this file — by the outcome value). +func TestClaimQueuedSendOnARedeliveryOfAnAlreadyFinishedSendTerminatesWithoutTouchingAnything(t *testing.T) { + for name, outcome := range map[string]string{ + "sent": "sent", + "failed": "failed: 550 no such mailbox", + "refused": "refused: the warrant does not bind the request", + "not proven": "not proven", + } { + t.Run(name, func(t *testing.T) { + sends, router := newMemSends(), newSendRouter() + req := rawQueuedSend(t, sends) + current, rev, ok, err := getSend(sends, req.ID) + if err != nil || !ok { + t.Fatalf("reading it back: %v %v", err, ok) + } + current.Decided.Outcome = outcome + current.Decided.Warrant = "the operator, via telegram (user id verified), chose Send" + current.Decided.Attempt = 1 + if _, err := sends.Update(req.ID, current, rev); err != nil { + t.Fatalf("leaving it %s: %v", outcome, err) + } + before := *current.Decided + + emit := &memEmit{} + _, d, _, got, err := claimQueuedSend(sends, router.Record, emit.emit, req.ID, judgedAt.Add(3*time.Minute)) + if err != nil { + t.Fatalf("claiming: %v", err) + } + if got != claimAlreadyFinished { + t.Fatalf("outcome: %v, want claimAlreadyFinished", got) + } + if d.Outcome != before.Outcome || d.Warrant != before.Warrant || d.Attempt != before.Attempt || !d.At.Equal(before.At) { + t.Fatalf("decision returned: %+v, want unchanged %+v", d, before) + } + if len(emit.calls) != 0 { + t.Fatalf("emitted something for an already-finished send: %+v", emit.calls) + } + kept, _, ok, kerr := getSend(sends, req.ID) + if kerr != nil || !ok || kept.Decided == nil { + t.Fatalf("the send could not be read back: %v %v", kerr, ok) + } + if kept.Decided.Outcome != before.Outcome || kept.Decided.Warrant != before.Warrant || kept.Decided.Attempt != before.Attempt { + t.Fatalf("the record changed: %+v, want unchanged %+v", kept.Decided, before) + } + ids, terr := tellPendingIDs(sends) + if terr != nil || len(ids) != 0 { + t.Fatalf("a tell was queued for an already-finished send: %v %v", terr, ids) + } + }) + } +} + +// raceOnFirstUpdate performs one competing write the first time Update is called — simulating another +// delivery's claim landing first — so the caller's own compare-and-set loses with stdio.ErrChanged +// (novox/hq issue 1196, AC4). +type raceOnFirstUpdate struct { + SendStore + raced bool + race func() +} + +func (r *raceOnFirstUpdate) Update(key string, value any, revision uint64) (uint64, error) { + if !r.raced { + r.raced = true + r.race() + } + return r.SendStore.Update(key, value, revision) +} diff --git a/modules/mesh-mail/cmd/mesh-mail/config.go b/modules/mesh-mail/cmd/mesh-mail/config.go index 4591f9f9..03c167d9 100644 --- a/modules/mesh-mail/cmd/mesh-mail/config.go +++ b/modules/mesh-mail/cmd/mesh-mail/config.go @@ -159,33 +159,17 @@ func (s Server) check(which string) error { return nil } -// SecretPath is where the account's own secret is placed in the secrets directory, as the manifest declares the -// family: /mail-credential-.secret. -func SecretPath(dir string, a Account) string { return filepath.Join(dir, a.Secret()+".secret") } - -// ReadSecret is the account's password, or given false when nobody gave it yet: the mesh never makes a member of the -// family, so a member not given is no file (ADR 0283 §6). -func ReadSecret(dir string, a Account) (string, bool, error) { - raw, err := os.ReadFile(SecretPath(dir, a)) - if errors.Is(err, os.ErrNotExist) { - return "", false, nil - } - if err != nil { - return "", false, err - } - pw := strings.TrimRight(string(raw), "\r\n") - if pw == "" { - return "", false, nil - } - return pw, true, nil -} +// ReadSecret and SecretPath moved into cmd/mesh-mail/internal/secret (novox/hq ADR 0436 §12, issue 1193): only +// keep (imap.go) and the sender's own actor (sendactor.go) read a password, never the tools, which the operator +// account's tool runner serves and which must never open a password file. // SecretGiven says whether the account's own secret was given. // // It looks at the file and never reads it: whoever judges whether a password was given has no need of the password. -// A file that is there and not empty was given; whether it signs in is the keeper's to say. +// A file that is there and not empty was given; whether it signs in is the keeper's to say. It computes the +// secret's path itself (not through the moved package) so the tools, which call this, never import it. func SecretGiven(dir string, a Account) bool { - info, err := os.Stat(SecretPath(dir, a)) + info, err := os.Stat(filepath.Join(dir, a.Secret()+".secret")) return err == nil && info.Mode().IsRegular() && info.Size() > 0 } diff --git a/modules/mesh-mail/cmd/mesh-mail/config_test.go b/modules/mesh-mail/cmd/mesh-mail/config_test.go index 21acbb3a..3f3560a5 100644 --- a/modules/mesh-mail/cmd/mesh-mail/config_test.go +++ b/modules/mesh-mail/cmd/mesh-mail/config_test.go @@ -6,6 +6,8 @@ import ( "reflect" "strings" "testing" + + "mesh-mail/cmd/mesh-mail/internal/secret" ) // The first stage's account, as the operator sets the setting `accounts` (ADR 0359 §5): its address and its server, @@ -22,7 +24,7 @@ func TestTheFirstStagesAccountIsReadAndItsSecretNamed(t *testing.T) { got[0].Incoming.Address() != "mail.novox.be:993" || got[0].Secret() != "mail-credential-mesh" || got[0].Outgoing != nil { t.Fatalf("%+v", got) } - if p := SecretPath("/state", got[0]); p != "/state/mail-credential-mesh.secret" { + if p := secret.Path("/state", got[0].Secret()); p != "/state/mail-credential-mesh.secret" { t.Fatalf("the secret is placed at %s", p) } } @@ -99,16 +101,16 @@ func TestASecretIsGivenByItsFileNotByReadingIt(t *testing.T) { if SecretGiven(dir, a) { t.Fatal("given before anything was written") } - _ = os.WriteFile(SecretPath(dir, a), nil, 0o600) + _ = os.WriteFile(secret.Path(dir, a.Secret()), nil, 0o600) if SecretGiven(dir, a) { t.Fatal("an empty file is a password") } - _ = os.WriteFile(SecretPath(dir, a), []byte("pw\n"), 0o600) - if pw, given, _ := ReadSecret(dir, a); !given || pw != "pw" || strings.Contains(pw, "\n") { + _ = os.WriteFile(secret.Path(dir, a.Secret()), []byte("pw\n"), 0o600) + if pw, given, _ := secret.Read(dir, a.Secret()); !given || pw != "pw" || strings.Contains(pw, "\n") { t.Fatalf("%q %v", pw, given) } // Unreadable to the judge, as a secret owned by root is to the watch: still given. - if err := os.Chmod(SecretPath(dir, a), 0); err != nil { + if err := os.Chmod(secret.Path(dir, a.Secret()), 0); err != nil { t.Fatal(err) } if !SecretGiven(dir, a) { diff --git a/modules/mesh-mail/cmd/mesh-mail/health.go b/modules/mesh-mail/cmd/mesh-mail/health.go index 9eee668c..f670851b 100644 --- a/modules/mesh-mail/cmd/mesh-mail/health.go +++ b/modules/mesh-mail/cmd/mesh-mail/health.go @@ -105,6 +105,24 @@ func Judge(accounts []Account, accountsErr, readyErr error, given func(Account) return health.Healthy(fmt.Sprintf("%d account(s) watched: %s", len(accounts), strings.Join(fine, "; "))) } +// WithOverdue folds any overdue send (Overdue, send.go) into base: unhealthy, naming each one, whatever +// base itself said — an overdue send is never excused by an otherwise-healthy account (novox/hq ADR 0436 +// clause 11, issue 1199). +func WithOverdue(base health.Answer, overdue []SendRequest) health.Answer { + if len(overdue) == 0 { + return base + } + names := make([]string, 0, len(overdue)) + for _, req := range overdue { + names = append(names, fmt.Sprintf("%s (from %s)", req.ID, req.Account)) + } + why := fmt.Sprintf("the send(s) %s have waited 10 minutes or more for the sender", strings.Join(names, ", ")) + if base.Why != "" { + why = base.Why + "; and " + why + } + return health.Unhealthy(why, base.Waits...) +} + // Status is the watch's verdict as it leaves it: the answer, and when it looked. type Status struct { health.Answer diff --git a/modules/mesh-mail/cmd/mesh-mail/health_test.go b/modules/mesh-mail/cmd/mesh-mail/health_test.go index 44252013..67a642ee 100644 --- a/modules/mesh-mail/cmd/mesh-mail/health_test.go +++ b/modules/mesh-mail/cmd/mesh-mail/health_test.go @@ -114,3 +114,39 @@ func TestTheWatchJudgesFromTheFilesAndTheIndex(t *testing.T) { t.Fatalf("the status file's shape: %s", raw) } } + +// 1199/AC2: mail_health (the tool, not only Overdue or WithOverdue in isolation) turns unhealthy naming an +// overdue send, even while the watch's own verdict is healthy (novox/hq review round 1 finding 6). +func TestMailHealthToolTurnsUnhealthyNamingAnOverdueSend(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "health.json") + if err := writeStatus(path, Status{Answer: health.Healthy("1 account(s) watched"), At: judgedAt}); err != nil { + t.Fatal(err) + } + sends := newMemSends() + overdueID := "send-overdue1" + req := SendRequest{ID: overdueID, Account: "mesh", At: judgedAt} + req.Decided = &SendDecision{At: judgedAt.Add(-20 * time.Minute), Outcome: "queued"} + if _, err := sends.Create(req.ID, req); err != nil { + t.Fatal(err) + } + if err := addSenderPending(sends, req.ID); err != nil { + t.Fatal(err) + } + b := &Bundle{Env: Env{Status: path}, Sends: sends, Now: func() time.Time { return judgedAt }} + + out, err := b.healthTool(nil) + if err != nil { + t.Fatalf("mail_health: %v", err) + } + a, ok := out.(health.Answer) + if !ok { + t.Fatalf("mail_health answered %T, want health.Answer", out) + } + if a.State != health.StateUnhealthy { + t.Fatalf("mail_health: %+v, want unhealthy", a) + } + if !strings.Contains(a.Why, overdueID) { + t.Fatalf("mail_health's why does not name the overdue send: %q", a.Why) + } +} diff --git a/modules/mesh-mail/cmd/mesh-mail/imap.go b/modules/mesh-mail/cmd/mesh-mail/imap.go index 5488e535..caf66144 100644 --- a/modules/mesh-mail/cmd/mesh-mail/imap.go +++ b/modules/mesh-mail/cmd/mesh-mail/imap.go @@ -22,6 +22,8 @@ import ( "github.com/emersion/go-imap/v2" "github.com/emersion/go-imap/v2/imapclient" + + "mesh-mail/cmd/mesh-mail/internal/secret" ) // How the keeper paces itself. IdleCycle is how long one IDLE lasts before it is ended and begun again, which proves @@ -170,7 +172,7 @@ func (k *Keeper) Watch(ctx context.Context, a Account) { retry := orDefault(k.RetryFirst, RetryFirst) var heard, synced time.Time for ctx.Err() == nil { - pw, given, err := ReadSecret(k.SecretsDir, a) + pw, given, err := secret.Read(k.SecretsDir, a.Secret()) switch { case err != nil: k.say(ctx, a, AccountFailing, "its password was given and cannot be read: "+err.Error(), heard, synced) diff --git a/modules/mesh-mail/cmd/mesh-mail/imap_test.go b/modules/mesh-mail/cmd/mesh-mail/imap_test.go index 47d9d53d..0159c710 100644 --- a/modules/mesh-mail/cmd/mesh-mail/imap_test.go +++ b/modules/mesh-mail/cmd/mesh-mail/imap_test.go @@ -221,9 +221,12 @@ func TestNothingButTheTestsWritesToAMailbox(t *testing.T) { } writes := regexp.MustCompile(`([A-Za-z_][A-Za-z0-9_]*)\.(Store|Move|Copy|Expunge|UIDExpunge|Append|Create|Delete|Rename|Subscribe|Unsubscribe|SetMetadata|SetQuota)\(`) for _, f := range files { - if strings.HasSuffix(f, "_test.go") || f == "send.go" || f == "smtp.go" { - // Neither touches a mailbox: send.go keeps the `sends` state on the bus, never IMAP; smtp.go speaks SMTP - // to the account's outgoing server, never to a mailbox, and only on the operator's Approve (ADR 0361 §2). + if strings.HasSuffix(f, "_test.go") || f == "send.go" || f == "smtp.go" || f == "sender.go" { + // None touches a mailbox: send.go keeps the `sends` state on the bus, never IMAP; smtp.go speaks SMTP + // to the account's outgoing server, never to a mailbox, and only on the operator's Approve (ADR 0361 + // §2); sender.go is the sender process (novox/hq ADR 0436 §7) — its own bus account's `sends` state + // (process.State.Create, matched here only because its receiver is named s) and router record, never + // IMAP either. continue } raw, err := os.ReadFile(f) diff --git a/modules/mesh-mail/cmd/mesh-mail/internal/secret/secret.go b/modules/mesh-mail/cmd/mesh-mail/internal/secret/secret.go new file mode 100644 index 00000000..cfb2ef83 --- /dev/null +++ b/modules/mesh-mail/cmd/mesh-mail/internal/secret/secret.go @@ -0,0 +1,36 @@ +// Package secret reads an account's own password from the secrets directory (novox/hq ADR 0283 §6). It is +// imported only by mesh-mail's keeper (which signs in to IMAP as root) and its sender (which signs in to SMTP +// as its own account, mesh-mail-send): never by the tools, which the operator account's tool runner serves and +// which must never open a password file (novox/hq ADR 0436 §12, issue 1193). Moved out of package main's own +// files so a test (secret_boundary_test.go) can prove the tools' bundle (main.go's serve path, tools.go, +// send.go, health.go) never imports it. +package secret + +import ( + "errors" + "os" + "path/filepath" + "strings" +) + +// Path is where an account's own secret is placed in the secrets directory, given the family member name +// (mail-credential-): /.secret. +func Path(dir, member string) string { return filepath.Join(dir, member+".secret") } + +// Read is the member's password, or given false when nobody gave it yet: the mesh never makes a member of the +// family, so a member not given is no file (ADR 0283 §6). A directory that does not exist at all answers the +// same as a file that does not exist: given false, no error (novox/hq issue 1193, AC3). +func Read(dir, member string) (string, bool, error) { + raw, err := os.ReadFile(Path(dir, member)) + if errors.Is(err, os.ErrNotExist) { + return "", false, nil + } + if err != nil { + return "", false, err + } + pw := strings.TrimRight(string(raw), "\r\n") + if pw == "" { + return "", false, nil + } + return pw, true, nil +} diff --git a/modules/mesh-mail/cmd/mesh-mail/kit_test.go b/modules/mesh-mail/cmd/mesh-mail/kit_test.go index e7308661..e018e2b6 100644 --- a/modules/mesh-mail/cmd/mesh-mail/kit_test.go +++ b/modules/mesh-mail/cmd/mesh-mail/kit_test.go @@ -22,6 +22,8 @@ import ( "github.com/emersion/go-imap/v2/imapclient" "github.com/emersion/go-imap/v2/imapserver" "github.com/emersion/go-imap/v2/imapserver/imapmemserver" + + "mesh-mail/cmd/mesh-mail/internal/secret" ) func TestMain(m *testing.M) { @@ -134,7 +136,7 @@ func testAccount() Account { func giveSecret(t *testing.T, dir string, a Account, pw string) { t.Helper() - if err := os.WriteFile(SecretPath(dir, a), []byte(pw+"\n"), 0o600); err != nil { + if err := os.WriteFile(secret.Path(dir, a.Secret()), []byte(pw+"\n"), 0o600); err != nil { t.Fatal(err) } } diff --git a/modules/mesh-mail/cmd/mesh-mail/main.go b/modules/mesh-mail/cmd/mesh-mail/main.go index 02d4fffa..7065834b 100644 --- a/modules/mesh-mail/cmd/mesh-mail/main.go +++ b/modules/mesh-mail/cmd/mesh-mail/main.go @@ -9,12 +9,16 @@ // mesh-mail keep acts: watches every account (IDLE on the inbox, a look at the other folders every five // minutes), keeps the index and the attachment store, and writes each change's event beside it // mesh-mail watch judges: reads every minute how each account is, and leaves its verdict for mail_health +// mesh-mail send the sender (novox/hq ADR 0436 §7): its own bus account, consuming the work queue +// mail-sends; it alone reads a password, and performs the one send an operator's +// Approve authorises, once the tools have recorded it approved and queued it // // The two long-running processes are the act and the judge, as mesh-knowledge's are: keep is judged by its unit, watch -// by the tool mail_health (ADR 0359 §6, ADR 0240 §3 and §7). Only the bundle reaches the bus, through the runtime that -// launched it, so the keeper hands its events over in the index, written in the same step as the change they say, -// and the bundle says them and removes each once the bus took it. stdout is the bundle's MCP channel; what this module -// says, it says on stderr. +// by the tool mail_health (ADR 0359 §6, ADR 0240 §3 and §7); send is judged by its unit too (it holds a bus account +// of its own, never a seat, so no tool answers for it). Only the bundle reaches the bus through the runtime that +// launched it; send reaches it on its own account instead (mesh-sdk go/process), so the keeper hands its events over +// in the index, written in the same step as the change they say, and the bundle says them and removes each once the +// bus took it. stdout is the bundle's MCP channel; what this module says, it says on stderr. package main import ( @@ -64,6 +68,8 @@ func main() { os.Exit(keep(envFromOS())) case "watch": os.Exit(watch(envFromOS())) + case "send": + os.Exit(send(envFromOS())) } } serve(envFromOS()) @@ -185,12 +191,45 @@ func serve(env Env) { go p.Run(context.Background()) go hear(b) go sweepSends(b) + go waitsTellTicker(b) if err := stdio.Serve("", b.Tools()); err != nil { logf("%v", err) os.Exit(1) } } +// sdkTellOperator is the tools' own TellOperator (send.go), wrapping stdio.Ask to the operator channel's +// tell verb, exactly as mesh-files' own pattern (modules/mesh-files/cmd/mesh-files/main.go tellOperator): +// no lines, no subject naming anything the content rule would refuse. +func sdkTellOperator(headline string) error { + _, err := stdio.Ask("seat:operator-channel.tell", map[string]any{"headline": headline, "lines": []string{}, "subject": "mesh-mail"}) + return err +} + +// WaitsTellEvery is how often the tools look for an overdue send still owing their own "waits" tell +// (novox/hq ADR 0436 clause 11, issue 1199): a second ticker, apart from sweepSends, since the two tells — +// the sender's own failure, the tools' own overdue wait — are never the same look. +const WaitsTellEvery = time.Minute + +// waitsTellTicker runs the tools' own "waits" tell for every overdue send, at least every WaitsTellEvery, +// until the process ends (novox/hq ADR 0436 clause 11, issue 1199). +func waitsTellTicker(b *Bundle) { + tick := time.NewTicker(WaitsTellEvery) + defer tick.Stop() + for { + overdue, err := Overdue(b.sends(), time.Now()) + if err != nil { + logf("[mesh-mail] the overdue check cannot be read: %v", err) + } + for _, req := range overdue { + if err := tellWaits(b.sends(), sdkTellOperator, req.ID, time.Now()); err != nil { + logf("[mesh-mail] telling the operator that %s waits: %v", req.ID, err) + } + } + <-tick.C + } +} + // hear subscribes to the router's word on this module's own asks, asked again until the runtime takes it, exactly // as mesh-knowledge does. Each warrant heard goes to Heard, the same Decide that mail_send_status uses: an // approved send goes out as soon as the operator answers it, never only when something later calls diff --git a/modules/mesh-mail/cmd/mesh-mail/manifest_test.go b/modules/mesh-mail/cmd/mesh-mail/manifest_test.go index 87b01d30..c4031ed9 100644 --- a/modules/mesh-mail/cmd/mesh-mail/manifest_test.go +++ b/modules/mesh-mail/cmd/mesh-mail/manifest_test.go @@ -30,11 +30,16 @@ type manifest struct { OwnSecrets map[string]map[string]string `json:"own-secrets"` Tools []string `json:"tools"` Emits []string `json:"emits"` + State []string `json:"state"` Streams []struct { - Name string `json:"name"` - Events []string `json:"events"` - MaxAgeDays int `json:"max-age-days"` - Grant struct { + Name string `json:"name"` + Events []string `json:"events"` + MaxAgeDays int `json:"max-age-days"` + MaxMiB int `json:"max-mib"` + Retention string `json:"retention"` + MaxDeliveries int `json:"max-deliveries"` + Duplicates int `json:"duplicates"` + Grant struct { Config string `json:"config"` Events string `json:"events"` } `json:"grant"` @@ -80,7 +85,7 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { t.Fatalf("the slug %q overflows an S3 access key", m.Slug) } fam := m.OwnSecrets[SecretFamily+"*"] - if fam["issued-by"] != "outside" || fam["path"] != "${dir:state}/"+SecretFamily+"*.secret" || len(m.OwnSecrets) != 1 { + if fam["issued-by"] != "outside" || fam["path"] != "${dir:state}/"+SecretFamily+"*.secret" || len(m.OwnSecrets) != 2 { t.Fatalf("own secrets %v", m.OwnSecrets) } var names []string @@ -95,13 +100,35 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { t.Fatalf("tools %v, the code serves %v", m.Tools, names) } // Its events go to its own stream MAIL, kept a week, each account's part heard only by the consumers the trusted - // config value grants names for it (ADR 0359 §2, §4; ADR 0363): none on the shared EVENTS stream. + // config value grants names for it (ADR 0359 §2, §4; ADR 0363): none on the shared EVENTS stream. The module's + // top-level emits is the sender's two outcomes (novox/hq ADR 0436 §7, §11, issue 1172): the module as a whole may + // publish them, though only the process send actually does (its own bus.emits, checked in + // TestItsManifestDeclaresTheSenderProcess). var events []string for _, c := range Changes { events = append(events, EventKey("*", c)) } - if len(m.Emits) != 0 || len(m.Streams) != 1 || m.Streams[0].Name != "mail" || !reflect.DeepEqual(m.Streams[0].Events, events) || - m.Streams[0].MaxAgeDays != 7 || m.Streams[0].Grant.Config != "grants" || m.Streams[0].Grant.Events != "mail.{key}.**" { + var mailStream *struct { + Name string `json:"name"` + Events []string `json:"events"` + MaxAgeDays int `json:"max-age-days"` + MaxMiB int `json:"max-mib"` + Retention string `json:"retention"` + MaxDeliveries int `json:"max-deliveries"` + Duplicates int `json:"duplicates"` + Grant struct { + Config string `json:"config"` + Events string `json:"events"` + } `json:"grant"` + } + for i := range m.Streams { + if m.Streams[i].Name == "mail" { + mailStream = &m.Streams[i] + } + } + if !reflect.DeepEqual(m.Emits, []string{"mail.send.sent", "mail.send.failed"}) || len(m.Streams) != 2 || mailStream == nil || + !reflect.DeepEqual(mailStream.Events, events) || mailStream.MaxAgeDays != 7 || mailStream.Grant.Config != "grants" || + mailStream.Grant.Events != "mail.{key}.**" { t.Fatalf("emits %v, streams %+v; the code says %v", m.Emits, m.Streams, events) } // It calls nothing and consumes nothing: mesh-mail-notifier (issue 800 part B) is the one that invokes and @@ -187,3 +214,112 @@ func TestItsProcessesAreThePreparationTheActAndTheJudge(t *testing.T) { t.Fatalf("keep restarts on %v", keep["restart-on"]) } } + +// 1172/AC2: the manifest declares the sender's account, own-secrets owner, work-queue stream and process bus, +// and widens the state directory and accounts.json so the sender (not the operator account) can read them, +// mirroring mesh-controller's own fixtures (internal/catalogue/process_bus_test.go:14-31, +// own_secret_owner_test.go:16-29, stream_work_queue_test.go:13-14). +func TestItsManifestDeclaresTheSenderProcess(t *testing.T) { + m, byID := readManifest(t) + + account := byID["account"] + if account == nil || account["type"] != "user" || account["name"] != "mesh-mail-send" { + t.Fatalf("the sender's account resource: %v", account) + } + + fam := m.OwnSecrets[SecretFamily+"*"] + if fam["owner"] != "mesh-mail-send" { + t.Fatalf("mail-credential-*'s owner: %v; want mesh-mail-send", fam["owner"]) + } + + // The sender's own bus account is delivered as the own secret broker-, never the module's + // own `broker` (novox/hq ADR 0436 §2): without it, the controller's mayIssue refuses to issue the + // module at all, naming the process and the secret (review round 1, finding 1). + broker := m.OwnSecrets["broker-send"] + if broker["path"] != "${dir:state}/broker-send.secret" || broker["owner"] != "mesh-mail-send" { + t.Fatalf("broker-send: %v; want path ${dir:state}/broker-send.secret, owner mesh-mail-send", broker) + } + + var sendsStream *struct { + Name string `json:"name"` + Events []string `json:"events"` + MaxAgeDays int `json:"max-age-days"` + MaxMiB int `json:"max-mib"` + Retention string `json:"retention"` + MaxDeliveries int `json:"max-deliveries"` + Duplicates int `json:"duplicates"` + Grant struct { + Config string `json:"config"` + Events string `json:"events"` + } `json:"grant"` + } + for i := range m.Streams { + if m.Streams[i].Name == "mail-sends" { + sendsStream = &m.Streams[i] + } + } + if sendsStream == nil || !reflect.DeepEqual(sendsStream.Events, []string{"mail.send.approved"}) || + sendsStream.MaxAgeDays != 1 || sendsStream.MaxMiB != 64 || sendsStream.Retention != "work-queue" || + sendsStream.MaxDeliveries != 5 || sendsStream.Duplicates != 24 { + t.Fatalf("the stream mail-sends: %+v", sendsStream) + } + + if !reflect.DeepEqual(m.Emits, []string{"mail.send.sent", "mail.send.failed"}) { + t.Fatalf("top-level emits: %v", m.Emits) + } + + send := byID["send"] + if send == nil || send["type"] != "process" || send["user"] != "mesh-mail-send" { + t.Fatalf("the process send: %v", send) + } + run, _ := send["run"].([]any) + if len(run) != 2 || run[0] != "./mesh-mail" || run[1] != "send" { + t.Fatalf("send's run: %v", send["run"]) + } + env, _ := send["env"].(map[string]any) + keepEnv, _ := byID["keep"]["env"].(map[string]any) + for _, k := range []string{"MESH_MAIL_ACCOUNTS", "MESH_MAIL_SECRETS"} { + if env[k] != keepEnv[k] { + t.Fatalf("send's %s is %v, keep's is %v", k, env[k], keepEnv[k]) + } + } + h, _ := send["health"].(map[string]any) + if h["kind"] != "unit" { + t.Fatalf("send's health: %v", send["health"]) + } + bus, _ := send["bus"].(map[string]any) + if bus == nil { + t.Fatalf("send declares no bus") + } + if !reflect.DeepEqual(toStrings(bus["emits"]), []string{"mail.send.sent", "mail.send.failed"}) || + !reflect.DeepEqual(toStrings(bus["queues"]), []string{"mail-sends"}) || + !reflect.DeepEqual(toStrings(bus["state"]), []string{"sends"}) || bus["asks"] != true || + !reflect.DeepEqual(toStrings(bus["invokes"]), []string{"seat:operator-channel.tell"}) { + t.Fatalf("send's bus: %+v", bus) + } + + // On novox the operator account's state directory is 0700, so mesh-mail-send cannot reach its own 0600 + // files inside it: the state directory becomes 0711 (passable, not listable) and accounts.json (server + // names and addresses, no password) becomes 0644, so the sender can read the outgoing server. No password + // file's mode widens (the lead's decision, issue 1172). + if byID["state"]["mode"] != "0711" { + t.Fatalf("the state directory's mode: %v; want 0711", byID["state"]["mode"]) + } + if byID["accounts"]["mode"] != "0644" { + t.Fatalf("accounts.json's mode: %v; want 0644", byID["accounts"]["mode"]) + } + if byID["grants"]["mode"] != "0600" { + t.Fatalf("grants.json's mode widened to %v; it holds no password, but was not asked to widen", byID["grants"]["mode"]) + } +} + +func toStrings(v any) []string { + list, _ := v.([]any) + out := make([]string, 0, len(list)) + for _, item := range list { + if s, ok := item.(string); ok { + out = append(out, s) + } + } + return out +} diff --git a/modules/mesh-mail/cmd/mesh-mail/outcome_test.go b/modules/mesh-mail/cmd/mesh-mail/outcome_test.go new file mode 100644 index 00000000..f4358bec --- /dev/null +++ b/modules/mesh-mail/cmd/mesh-mail/outcome_test.go @@ -0,0 +1,308 @@ +package main + +// recordOutcome writes the final decision for one claimed send and says what the caller does with the +// queue message next (novox/hq ADR 0436 clauses 10, 11, issue 1198). + +import ( + "errors" + "strings" + "testing" + "time" +) + +// emitted is one call recordOutcome's Emit made. +type emitted struct { + eventType, id string + body OutcomeEvent +} + +// memEmit collects every emit call, for a test to check the event and its body. +type memEmit struct{ calls []emitted } + +func (m *memEmit) emit(eventType, id string, body any) error { + b, _ := body.(OutcomeEvent) + m.calls = append(m.calls, emitted{eventType: eventType, id: id, body: b}) + return nil +} + +// outcomeTestClaim keeps one send claimed "sending" at the given attempt, returning its request, the +// claimed decision and the revision recordOutcome needs. +func outcomeTestClaim(t *testing.T, attempt int) (SendRequest, SendDecision, uint64, *memSends) { + t.Helper() + sends := newMemSends() + req := SendRequest{ID: newSendID(), Account: "mesh", To: []string{"ops@example.org"}, Subject: "s", Body: "body", At: judgedAt} + claim := SendDecision{At: judgedAt.Add(time.Minute), Outcome: "sending", Warrant: "the operator, via telegram, chose Send", Attempt: attempt} + req.Decided = &claim + rev, err := sends.Create(req.ID, req) + if err != nil { + t.Fatal(err) + } + return req, claim, rev, sends +} + +// 1198/AC1: given a claimed send the server accepts, when sent, then sent is recorded, mail.send.sent is +// emitted (send id, account, outcome in words; no recipient, subject or body), and only then — by +// construction, since the caller acknowledges only once this call returns — the message is acknowledged. +func TestRecordOutcomeOnSuccessRecordsSentAndEmitsIt(t *testing.T) { + req, claimed, rev, sends := outcomeTestClaim(t, 1) + emit := &memEmit{} + d, action, _, err := recordOutcome(sends, emit.emit, req, claimed, rev, nil, judgedAt.Add(2*time.Minute)) + if err != nil { + t.Fatalf("recordOutcome: %v", err) + } + if d.Outcome != "sent" { + t.Fatalf("outcome: %+v", d) + } + if action != actionAck { + t.Fatalf("action: %v, want actionAck", action) + } + kept, _, ok, err := getSend(sends, req.ID) + if err != nil || !ok || kept.Decided == nil || kept.Decided.Outcome != "sent" { + t.Fatalf("not kept as sent: %v %v %+v", err, ok, kept.Decided) + } + if len(emit.calls) != 1 || emit.calls[0].eventType != MailSendSent || emit.calls[0].id != req.ID { + t.Fatalf("emitted: %+v", emit.calls) + } + body := emit.calls[0].body + if body.Send != req.ID || body.Account != "mesh" || body.Outcome == "" { + t.Fatalf("the event's body: %+v", body) + } + whole := body.Send + body.Account + body.Outcome + for _, forbidden := range []string{"ops@example.org", "s", "body"} { + if forbidden == "s" { + continue // too short to search for meaningfully without false positives in the id/account + } + if strings.Contains(whole, forbidden) { + t.Fatalf("the emitted body carries %q: %+v", forbidden, body) + } + } +} + +// 1198/AC2: given a not-taken failure on attempt n (n < 5), then retrying is recorded with attempt, reason +// and next attempt, and the caller is told to ask the bus to deliver it again after 1, 5, 15, 30 minutes +// for n = 1..4; on the fifth, failed is recorded and mail.send.failed emitted. +func TestRecordOutcomeOnNotTakenRetriesUpToTheFourthAttempt(t *testing.T) { + for n, want := range map[int]time.Duration{1: time.Minute, 2: 5 * time.Minute, 3: 15 * time.Minute, 4: 30 * time.Minute} { + req, claimed, rev, sends := outcomeTestClaim(t, n) + emit := &memEmit{} + now := judgedAt.Add(2 * time.Minute) + d, action, delay, err := recordOutcome(sends, emit.emit, req, claimed, rev, &SendError{Outcome: NotTaken, Err: errors.New("no connection")}, now) + if err != nil { + t.Fatalf("attempt %d: recordOutcome: %v", n, err) + } + if d.Outcome != "retrying" || d.Attempt != n { + t.Fatalf("attempt %d: outcome %+v", n, d) + } + if !d.NextAttempt.Equal(now.Add(want)) { + t.Fatalf("attempt %d: next attempt %v, want %v", n, d.NextAttempt, now.Add(want)) + } + if action != actionRetry || delay != want { + t.Fatalf("attempt %d: action %v, delay %v, want actionRetry, %v", n, action, delay, want) + } + if len(emit.calls) != 0 { + t.Fatalf("attempt %d: retrying emitted something: %+v", n, emit.calls) + } + } +} + +func TestRecordOutcomeOnNotTakenFailsOnTheFifthAttempt(t *testing.T) { + req, claimed, rev, sends := outcomeTestClaim(t, MostAttempts) + emit := &memEmit{} + d, action, _, err := recordOutcome(sends, emit.emit, req, claimed, rev, &SendError{Outcome: NotTaken, Err: errors.New("no connection")}, + judgedAt.Add(2*time.Minute)) + if err != nil { + t.Fatalf("recordOutcome: %v", err) + } + if !strings.HasPrefix(d.Outcome, "failed: ") || d.Attempt != MostAttempts { + t.Fatalf("outcome: %+v", d) + } + if action != actionTerm { + t.Fatalf("action: %v, want actionTerm", action) + } + if len(emit.calls) != 1 || emit.calls[0].eventType != MailSendFailed { + t.Fatalf("emitted: %+v", emit.calls) + } +} + +// 1198/AC3: given a refusal, then failed is recorded with the reason, mail.send.failed emitted, and the +// message terminated; given not proven, the same with not proven. +func TestRecordOutcomeOnRefusedRecordsFailedAndTerminates(t *testing.T) { + req, claimed, rev, sends := outcomeTestClaim(t, 1) + emit := &memEmit{} + d, action, _, err := recordOutcome(sends, emit.emit, req, claimed, rev, &SendError{Outcome: Refused, Err: errors.New("550 no such mailbox")}, + judgedAt.Add(2*time.Minute)) + if err != nil { + t.Fatalf("recordOutcome: %v", err) + } + if !strings.HasPrefix(d.Outcome, "failed: ") || !strings.Contains(d.Outcome, "550 no such mailbox") { + t.Fatalf("outcome: %+v", d) + } + if action != actionTerm { + t.Fatalf("action: %v, want actionTerm", action) + } + if len(emit.calls) != 1 || emit.calls[0].eventType != MailSendFailed || emit.calls[0].body.Outcome != d.Outcome { + t.Fatalf("emitted: %+v", emit.calls) + } +} + +func TestRecordOutcomeOnNotProvenRecordsNotProvenAndTerminates(t *testing.T) { + req, claimed, rev, sends := outcomeTestClaim(t, 1) + emit := &memEmit{} + d, action, _, err := recordOutcome(sends, emit.emit, req, claimed, rev, + &SendError{Outcome: NotProven, Err: errors.New("connection lost after the data was written")}, judgedAt.Add(2*time.Minute)) + if err != nil { + t.Fatalf("recordOutcome: %v", err) + } + if d.Outcome != "not proven" { + t.Fatalf("outcome: %+v", d) + } + if action != actionTerm { + t.Fatalf("action: %v, want actionTerm", action) + } + if len(emit.calls) != 1 || emit.calls[0].eventType != MailSendFailed || emit.calls[0].body.Outcome != "not proven" { + t.Fatalf("emitted: %+v", emit.calls) + } +} + +// review round 1 finding 3: a failed write of the tells-pending index does not stop the failed decision +// itself from being recorded and emitted — only the index write is reported as failing. +func TestRecordOutcomeStillRecordsFailedWhenTheTellsPendingIndexCannotBeWritten(t *testing.T) { + req, claimed, rev, sends := outcomeTestClaim(t, 1) + sends.failKey, sends.failErr = tellsPendingKey, errors.New("the store is down") + emit := &memEmit{} + d, action, _, err := recordOutcome(sends, emit.emit, req, claimed, rev, &SendError{Outcome: Refused, Err: errors.New("550 no such mailbox")}, + judgedAt.Add(2*time.Minute)) + if err == nil { + t.Fatalf("the index failure was not reported") + } + if !strings.HasPrefix(d.Outcome, "failed: ") || action != actionTerm { + t.Fatalf("outcome %+v, action %v", d, action) + } + kept, _, ok, kerr := getSend(sends, req.ID) + if kerr != nil || !ok || kept.Decided == nil || kept.Decided.Outcome != d.Outcome { + t.Fatalf("the decision itself was not kept despite the index failing: %v %v %+v", kerr, ok, kept.Decided) + } + if len(emit.calls) != 1 { + t.Fatalf("emitted %d time(s) despite the index failing", len(emit.calls)) + } +} + +// review round 1 finding 5: if writing "sent" cannot be kept (even retried on a changed revision), the +// message is never acknowledged and mail.send.sent is never emitted — a mail actually sent is never +// silently announced successful when the record never actually carried it. +func TestRecordOutcomeOnSentDoesNotAckOrEmitWhenTheWriteCannotBeKept(t *testing.T) { + req, claimed, rev, sends := outcomeTestClaim(t, 1) + sends.failKey, sends.failErr = req.ID, errors.New("the store is down") + emit := &memEmit{} + + d, action, _, err := recordOutcome(sends, emit.emit, req, claimed, rev, nil, judgedAt.Add(2*time.Minute)) + if err == nil { + t.Fatalf("the write failure was not reported") + } + if action != actionLeave { + t.Fatalf("action: %v, want actionLeave (left to the bus, never acked)", action) + } + if len(emit.calls) != 0 { + t.Fatalf("emitted mail.send.sent despite the write never being kept: %+v", emit.calls) + } + // The kept record is unaffected: still "sending", as claimQueuedSend originally left it. + kept, _, ok, kerr := getSend(sends, req.ID) + if kerr != nil || !ok || kept.Decided == nil || kept.Decided.Outcome != "sending" { + t.Fatalf("the record changed despite the write failing: %v %v %+v (returned %+v)", kerr, ok, kept.Decided, d) + } +} + +// Same finding, for a failed outcome (refused): the message is left to the bus, never terminated, and +// mail.send.failed is never emitted, when the write itself cannot be kept. +func TestRecordOutcomeOnRefusedDoesNotTerminateOrEmitWhenTheWriteCannotBeKept(t *testing.T) { + req, claimed, rev, sends := outcomeTestClaim(t, 1) + sends.failKey, sends.failErr = req.ID, errors.New("the store is down") + emit := &memEmit{} + + _, action, _, err := recordOutcome(sends, emit.emit, req, claimed, rev, &SendError{Outcome: Refused, Err: errors.New("550 no such mailbox")}, + judgedAt.Add(2*time.Minute)) + if err == nil { + t.Fatalf("the write failure was not reported") + } + if action != actionLeave { + t.Fatalf("action: %v, want actionLeave", action) + } + if len(emit.calls) != 0 { + t.Fatalf("emitted mail.send.failed despite the write never being kept: %+v", emit.calls) + } +} + +// 1198/AC1 (review round 2): a store that changes the revision between the claim and the outcome — another +// writer's decision landing in between — is retried once with the fresh revision; recordOutcome's own +// decision, once it does land, is what is kept, acked and emitted, never the other writer's state read +// back with a nil error mistaken for success. +func TestRecordOutcomeRetriesOnceWhenTheRevisionChangedBetweenClaimAndOutcome(t *testing.T) { + req, claimed, rev, sends := outcomeTestClaim(t, 1) + racy := &raceOnFirstUpdate{SendStore: sends, race: func() { + // A different delivery's own claim lands first, bumping the revision without ever finishing this + // call's own write — settleSend (via updateDecision) answers that with revision 0 and a nil error. + r, rv, ok, err := getSend(sends, req.ID) + if err != nil || !ok { + t.Fatalf("racing write could not read the send: %v %v", err, ok) + } + if _, _, err := settleSend(sends, r, rv, SendDecision{At: judgedAt, Outcome: "sending", Attempt: 2}); err != nil { + t.Fatalf("racing write: %v", err) + } + }} + emit := &memEmit{} + + d, action, _, err := recordOutcome(racy, emit.emit, req, claimed, rev, nil, judgedAt.Add(2*time.Minute)) + if err != nil { + t.Fatalf("recordOutcome: %v", err) + } + if d.Outcome != "sent" || action != actionAck { + t.Fatalf("outcome %+v, action %v, want sent/actionAck once retried with the fresh revision", d, action) + } + if len(emit.calls) != 1 || emit.calls[0].eventType != MailSendSent || emit.calls[0].body.Outcome != "sent" { + t.Fatalf("emitted: %+v", emit.calls) + } + kept, _, ok, kerr := getSend(sends, req.ID) + if kerr != nil || !ok || kept.Decided == nil || kept.Decided.Outcome != "sent" { + t.Fatalf("the kept record is not this call's own decision: %v %v %+v", kerr, ok, kept.Decided) + } +} + +// Same finding: when the race repeats on the retry too (a second writer keeps landing first), recordOutcome +// gives up rather than looping or mistaking the other writer's state for its own — actionLeave, nothing +// emitted, the error reported. +func TestRecordOutcomeGivesUpWhenTheRevisionKeepsChanging(t *testing.T) { + req, claimed, rev, sends := outcomeTestClaim(t, 1) + always := &raceEveryUpdate{SendStore: sends, race: func() { + r, rv, ok, err := getSend(sends, req.ID) + if err != nil || !ok { + t.Fatalf("racing write could not read the send: %v %v", err, ok) + } + if _, _, err := settleSend(sends, r, rv, SendDecision{At: judgedAt, Outcome: "sending", Attempt: r.Decided.Attempt + 1}); err != nil { + t.Fatalf("racing write: %v", err) + } + }} + emit := &memEmit{} + + _, action, _, err := recordOutcome(always, emit.emit, req, claimed, rev, nil, judgedAt.Add(2*time.Minute)) + if err == nil { + t.Fatalf("a persistently changing revision was not reported") + } + if action != actionLeave { + t.Fatalf("action: %v, want actionLeave", action) + } + if len(emit.calls) != 0 { + t.Fatalf("emitted despite never keeping its own write: %+v", emit.calls) + } +} + +// raceEveryUpdate performs the competing write before every Update call, not only the first — simulating a +// revision that keeps changing out from under every retry (novox/hq review round 2, issue 1198). +type raceEveryUpdate struct { + SendStore + race func() +} + +func (r *raceEveryUpdate) Update(key string, value any, revision uint64) (uint64, error) { + r.race() + return r.SendStore.Update(key, value, revision) +} diff --git a/modules/mesh-mail/cmd/mesh-mail/sdk_pin_test.go b/modules/mesh-mail/cmd/mesh-mail/sdk_pin_test.go new file mode 100644 index 00000000..44f0a371 --- /dev/null +++ b/modules/mesh-mail/cmd/mesh-mail/sdk_pin_test.go @@ -0,0 +1,31 @@ +package main + +// The SDK pin a process's own bus connection needs (novox/hq ADR 0436 §4, issue 1172): mesh-mail's go.mod +// requires mesh-sdk go at the release-2 commit (main 34392e7) and go 1.26.0, and go/process is importable. +// The blank import below is what proves importability at compile time; go vet (run before every commit, per +// the context file) already fails the build if it is not. + +import ( + "os" + "strings" + "testing" + + _ "git.novox.be/novox/mesh-sdk/go/process" +) + +// 1172/AC1: given mesh-mail's go.mod, when built, then it requires mesh-sdk go at +// v0.1.14-0.20261019062050-34392e74916c and go 1.26.0, and go/process is importable (the import above). +func TestGoModPinsTheSDKsProcessRelease(t *testing.T) { + raw, err := os.ReadFile("../../go.mod") + if err != nil { + t.Fatal(err) + } + mod := string(raw) + const wantSDK = "git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019062050-34392e74916c" + if !strings.Contains(mod, wantSDK) { + t.Fatalf("go.mod does not require %q:\n%s", wantSDK, mod) + } + if !strings.Contains(mod, "go 1.26.0") { + t.Fatalf("go.mod does not declare go 1.26.0:\n%s", mod) + } +} diff --git a/modules/mesh-mail/cmd/mesh-mail/secret_boundary_test.go b/modules/mesh-mail/cmd/mesh-mail/secret_boundary_test.go new file mode 100644 index 00000000..6dd6d1b5 --- /dev/null +++ b/modules/mesh-mail/cmd/mesh-mail/secret_boundary_test.go @@ -0,0 +1,124 @@ +package main + +// The tools never open a password file (novox/hq ADR 0436 §12, issue 1193): ReadSecret and SecretPath moved +// into cmd/mesh-mail/internal/secret, imported only by keep (imap.go) and — once issue 1195 adds it — the +// sender's own process (sender.go). Since issue 1194, Decide no longer reaches any actor that reads a secret +// at all (it records approved and queues the send for the sender); the import list this test checks still +// holds, now even more plainly. The build seat runs every test under one account, so no permission test can +// show the split (context file, "Clause 12's proof"); this is the file-level proxy the lead chose instead: an +// AST check over the tools' own files. + +import ( + "encoding/json" + "go/parser" + "go/token" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// secretPackage is the import path the tools' bundle must never carry. +const secretPackage = "mesh-mail/cmd/mesh-mail/internal/secret" + +// 1193/AC1: none of the tools' bundle files (main.go's serve path, tools.go, send.go, health.go) imports the +// password-reading package; the failure names the offending file. +func TestTheToolsBundleNeverImportsThePasswordPackage(t *testing.T) { + for _, file := range []string{"main.go", "tools.go", "send.go", "health.go"} { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, file, nil, parser.ImportsOnly) + if err != nil { + t.Fatalf("%s: %v", file, err) + } + for _, imp := range f.Imports { + path := strings.Trim(imp.Path.Value, `"`) + if path == secretPackage { + t.Errorf("%s imports %s, which only keep and the sender's actor may (novox/hq ADR 0436 §12)", file, secretPackage) + } + } + } +} + +// 1193/AC2: keep (imap.go) and the sender (sender.go, issue 1195) read the password through the moved +// package; SecretGiven (stat only) stays in config.go, available to the tools, and never reads the password +// itself. (sender.go's own account file is read through secret.Path too, though its own account is not a +// password; the real per-message password read is issue 1196's.) +func TestKeepAndTheSenderReadThePasswordThroughTheMovedPackage(t *testing.T) { + for _, file := range []string{"imap.go", "sender.go"} { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, file, nil, parser.ImportsOnly) + if err != nil { + t.Fatalf("%s: %v", file, err) + } + found := false + for _, imp := range f.Imports { + if strings.Trim(imp.Path.Value, `"`) == secretPackage { + found = true + } + } + if !found { + t.Errorf("%s does not import %s", file, secretPackage) + } + } + // SecretGiven itself (config.go) never calls ReadFile, os.Open or the moved package: whether a password was + // given is read from the file's presence and size alone (issue 733, kept by config_test.go's own check too). + src, err := os.ReadFile("config.go") + if err != nil { + t.Fatal(err) + } + body := string(src) + body = body[strings.Index(body, "func SecretGiven("):] + body = body[:strings.Index(body, "\n}\n")] + for _, bad := range []string{"ReadSecret", "ReadFile", "os.Open", "secret.Read"} { + if strings.Contains(body, bad) { + t.Fatalf("SecretGiven reads the password (%s):\n%s", bad, body) + } + } +} + +// 1193/AC3: with the secrets directory absent, mail_send raises its ask, the warrant of an Approve is heard, +// and mail_send_status reads it back — each succeeds, and no error mentions a secret file. Since issue 1194 +// the tools never read a secret on this path at all (they queue the send for the sender), so this also proves +// the stronger property the tools' bundle never touches b.Env.Secrets. +func TestTheToolPathSucceedsWithTheSecretsDirectoryAbsent(t *testing.T) { + dir := t.TempDir() + missing := filepath.Join(dir, "never-created") + env := Env{Accounts: filepath.Join(dir, "accounts.json"), Grants: filepath.Join(dir, "grants.json"), Secrets: missing} + writeFile(t, env.Accounts, `{"accounts": {"mesh": {"kind": "imap", "address": "jschoubben@novox.be", + "incoming": {"host": "mail.novox.be", "port": 993, "security": "tls"}, + "outgoing": {"host": "h", "port": 465, "security": "tls"}}}}`) + writeFile(t, env.Grants, `{"grants": {"mesh": ["triage"]}}`) + router, sends, queue := newSendRouter(), newMemSends(), newMemQueue() + b := &Bundle{Env: env, Now: func() time.Time { return judgedAt }, Asks: router, Sends: sends, Queue: queue.publish} + rt := startBundle(t, b.Tools()) + + text, isErr := rt.call(SendTool, "novox.triage", sendArgs) + if isErr { + t.Fatalf("mail_send: %s", text) + } + var out map[string]any + _ = json.Unmarshal([]byte(text), &out) + id := out["ask"].(string) + + w := router.chose(id, OptionSend, judgedAt.Add(time.Minute)) + router.close(w) + d, err := Decide(sends, b.queuePublish(), w, judgedAt.Add(2*time.Minute)) + if err != nil { + t.Fatalf("the warrant was not heard cleanly: %v", err) + } + if d.Outcome != "queued" { + t.Fatalf("the decision: %+v; want queued (the tools never read the secrets directory)", d) + } + if strings.Contains(d.Outcome, missing) || strings.Contains(strings.ToLower(d.Outcome), "secret") { + t.Fatalf("the decision names a secret file: %q", d.Outcome) + } + + text, isErr = rt.call(SendStatusTool, "novox.triage", map[string]any{"id": id}) + if isErr { + t.Fatalf("mail_send_status: %s", text) + } + if strings.Contains(text, missing) || strings.Contains(strings.ToLower(text), "secret") { + t.Fatalf("mail_send_status names a secret file: %s", text) + } +} diff --git a/modules/mesh-mail/cmd/mesh-mail/send.go b/modules/mesh-mail/cmd/mesh-mail/send.go index bd8f31c6..66d130d3 100644 --- a/modules/mesh-mail/cmd/mesh-mail/send.go +++ b/modules/mesh-mail/cmd/mesh-mail/send.go @@ -186,6 +186,20 @@ type SendDecision struct { Outcome string `json:"outcome"` Warrant string `json:"warrant,omitempty"` Error string `json:"error,omitempty"` + // Attempt is which handover to the server this is, 1 for the first: the sender's own claim from queued + // or retrying to sending counts it (novox/hq ADR 0436 clauses 9, 10, issue 1196). + Attempt int `json:"attempt,omitempty"` + // NextAttempt is when a "retrying" send is next claimed, the bus's own redelivery time (novox/hq ADR + // 0436 clause 10, issue 1198): read by mail_health to say a send overdue (issue 1199). + NextAttempt time.Time `json:"next-attempt,omitempty"` + // Tell is the sender's own tell to the operator for a failed, refused or not-proven send, kept to once: + // "" (not owed, or not yet attempted), "telling" (claimed, the channel not yet confirmed it took the + // message) or "told" (novox/hq ADR 0436 clause 11, issue 1199). + Tell string `json:"tell,omitempty"` + // WaitsTell is the tools' own tell that an overdue send's sender has not taken it, kept to once the + // same way as Tell, on its own field: the two are never the same tell (novox/hq ADR 0436 clause 11, + // issue 1199). + WaitsTell string `json:"waits-tell,omitempty"` } // bodyDigest is the digest an act binds in place of the body itself: never the body, which the act's digest is @@ -554,18 +568,31 @@ func looksLikeHostName(w string) bool { // ---- deciding --------------------------------------------------------------------------------------------------- -// AccountLookup resolves an account by name, as the settings currently hold it (not as it stood when it was asked: -// the account's server may have changed since, and its password is read fresh), or found false. -type AccountLookup func(name string) (Account, bool) +// MailSendApproved is the event mesh-mail's tools publish on the work queue mail-sends once the operator's +// Approve is verified and the send recorded approved (novox/hq ADR 0436 clause 8). +const MailSendApproved = "mail.send.approved" -// SendActor performs the one send a chosen option authorises, resolving the account's own secret itself. It is -// called at most once per request (Decide claims the request first). -type SendActor func(a Account, to []string, subject, body string, now time.Time) error +// QueuedSend is the body mail.send.approved carries: never the recipients, the subject or the body, which stay +// in the send's own record in `sends` (novox/hq ADR 0436 clause 8). +type QueuedSend struct { + Send string `json:"send"` + Account string `json:"account"` + Ask string `json:"ask"` +} -// settleSend writes a decision onto a kept send (claiming it, or ending it outright) and, once the write lands, -// drops it from the undecided index: an index entry is never left behind once the record it points to carries a -// decision (novox/hq issue 811). On a race lost to another writer, it reads back whatever that writer decided. -func settleSend(store SendStore, req SendRequest, rev uint64, d SendDecision) (SendDecision, uint64, error) { +// QueuePublish publishes one message on the module's own work-queue stream, the message id chosen (here, the +// send id) so a repeated publish inside the stream's duplicate window is kept to one (novox/hq ADR 0436 +// clauses 6, 8); nothing here reads or holds a password. Real is sdkQueuePublish; a test gives its own. +type QueuePublish func(id string, body any) error + +func sdkQueuePublish(id string, body any) error { + return stdio.Emit(MailSendApproved, body, stdio.EmitOptions{MessageID: id}) +} + +// updateDecision writes a decision onto a kept send by compare-and-set, without touching the undecided index: +// the index is settleSend's to drop, once the decision is one the sweep no longer needs to look at again +// (novox/hq issue 811). On a race lost to another writer, it reads back whatever that writer decided. +func updateDecision(store SendStore, req SendRequest, rev uint64, d SendDecision) (SendDecision, uint64, error) { next := req next.Decided = &d newRev, err := store.Update(req.ID, next, rev) @@ -577,16 +604,30 @@ func settleSend(store SendStore, req SendRequest, rev uint64, d SendDecision) (S } return d, 0, err } - _ = removeUndecided(store, req.ID) // best-effort (see removeUndecided) return d, newRev, nil } +// settleSend writes a decision onto a kept send (claiming it, or ending it outright) and, once the write lands, +// drops it from the undecided index: an index entry is never left behind once the record it points to carries a +// decision the sweep no longer needs to act on (novox/hq issue 811). On a race lost to another writer, it reads +// back whatever that writer decided. +func settleSend(store SendStore, req SendRequest, rev uint64, d SendDecision) (SendDecision, uint64, error) { + final, newRev, err := updateDecision(store, req, rev, d) + if err != nil { + return final, 0, err + } + if newRev != 0 { + _ = removeUndecided(store, req.ID) // best-effort (see removeUndecided) + } + return final, newRev, nil +} + // Decide holds one closed ask's warrant to the request it answers (asks.Warrant.For, asks.Option.Performs), and — -// on Approve, exactly once — performs the send. A request already decided is never decided again: decide answers -// its kept decision at once, and two warrants for the same request (a redelivery, or two readers racing) have only -// the first claim the request by a compare-and-set write; the second finds it already claimed or decided and sends -// nothing. -func Decide(store SendStore, lookup AccountLookup, send SendActor, w asks.Warrant, now time.Time) (SendDecision, error) { +// on Approve, exactly once — records the send approved and queues it for the sender (novox/hq ADR 0436 clause 8); +// it never sends. A request already decided is never decided again: decide answers its kept decision at once, and +// two warrants for the same request (a redelivery, or two readers racing) have only the first claim the request +// by a compare-and-set write; the second finds it already claimed or decided and queues nothing twice. +func Decide(store SendStore, publish QueuePublish, w asks.Warrant, now time.Time) (SendDecision, error) { req, rev, ok, err := getSend(store, w.Ask) if err != nil { return SendDecision{}, err @@ -623,45 +664,650 @@ func Decide(store SendStore, lookup AccountLookup, send SendActor, w asks.Warran case OptionDecline: return settle(SendDecision{At: now, Outcome: "declined", Warrant: w.Says()}) case OptionSend: - return decideSend(store, lookup, send, req, rev, w, now) + return decideSend(store, publish, req, rev, w, now) default: return settle(SendDecision{At: now, Outcome: "refused: the option " + option.ID + " is not one mail_send offers"}) } } -// decideSend claims the request by a compare-and-set write before it sends, so a redelivered warrant finds it -// already claimed (or decided) and acts on nothing: a person's Approve is performed once, never retried on a -// transient failure (the failure is kept in the decision, not hidden). -func decideSend(store SendStore, lookup AccountLookup, send SendActor, req SendRequest, rev uint64, w asks.Warrant, now time.Time) (SendDecision, error) { - claimed := req - claim := SendDecision{At: now, Outcome: "sending", Warrant: w.Says()} - claimedDecision, newRev, err := settleSend(store, req, rev, claim) +// decideSend records the request queued by a compare-and-set write before it publishes, so a redelivered +// warrant finds it already claimed (or decided) and queues nothing twice: a person's Approve is queued once +// (novox/hq ADR 0436 clause 8, as decided to replace the two-state approved/queued design — review round 1, +// replacing finding 2's work-around with a decision changing the clause itself). There is no "approved" +// state in a send's life at all: the record moves straight from undecided to queued; the publish that +// follows is best-effort, and a publish that fails (or the tools stopping before it ever runs) leaves the +// record "queued" with its own time, which the sweep finds and republishes once a short grace has passed +// (Sweep, below) — the send id as the message id, and the stream's duplicate window, keep it one message +// either way, and the sender's own claim (clause 9) keeps it one send. It never sends: the sender (clause 7) +// takes the queued message, verifies the warrant again itself (clause 9) and performs the one handover to +// the server (clause 10). +func decideSend(store SendStore, publish QueuePublish, req SendRequest, rev uint64, w asks.Warrant, now time.Time) (SendDecision, error) { + // Indexed before the record carries "queued" (novox/hq review round 1 finding 3): a crash between this + // and the write below must still leave it findable by mail_health's overdue check and the sweep's own + // stale-queued retry once "queued" does land. + indexErr := addSenderPending(store, req.ID) + queued := SendDecision{At: now, Outcome: "queued", Warrant: w.Says()} + claimed, newRev, err := settleSend(store, req, rev, queued) if err != nil { - return SendDecision{}, err + return SendDecision{}, errOr(indexErr, err) } if newRev == 0 { // Lost the claim race: another writer already decided this send. - return claimedDecision, nil + return claimed, indexErr + } + // Best-effort: its own failure is never returned to the caller as this call's own error — the record is + // already durably "queued" either way, and the sweep retries a publish that did not land. + _ = publish(req.ID, QueuedSend{Send: req.ID, Account: req.Account, Ask: req.ID}) + return claimed, indexErr +} + +// ---- the sender's own claim (novox/hq ADR 0436 clauses 9, 10, issue 1196) -------------------------------------- +// +// The sender never trusts the queue's message: it reads the router's record of mesh-mail's own ask under its +// own account, and mirrors the tools' own check (Decide, above) exactly — w.For, then option.Performs on the +// request's own Act — before it claims the send by compare-and-set and, only then, connects to anything. + +// RouterRecord is how the sender reads the router's record of one ask: a function so a test gives a fake, and +// the real one (sender.go) wraps a process.Conn's Ask(bucket, id) over the operator channel's records. +type RouterRecord func(id string) (*stdio.StateEntry, error) + +// claimOutcome is what became of one claim, so the caller (the message handler) knows the message's fate. +type claimOutcome int + +const ( + // claimToSend: claimed "sending" by this call; the caller may now connect and send. + claimToSend claimOutcome = iota + // claimRefused: recorded refused for a reason this function names; the caller terminates the message. + claimRefused + // claimNotProven: the send was already "sending" (a previous delivery may have handed it to the server); + // recorded not proven; the caller terminates the message, never retrying it automatically. + claimNotProven + // claimLostRace: the compare-and-set to "sending" lost to another writer (another delivery of the same + // message, racing this one); nothing is recorded here and nothing sent; the caller leaves the message to + // the bus entirely — no Ack, no Nak, no Term. + claimLostRace + // claimAlreadyFinished: a redelivery of a message whose send already finished (sent, failed, refused + // or not proven) — the sender's own ack was lost, or two deliveries raced and the other one already + // finished it. Nothing is recorded, emitted or told again; the caller only terminates the message + // (novox/hq ADR 0436 clause 10, operator-approved change). + claimAlreadyFinished +) + +// isFinishedOutcome says whether a decided send's outcome is already over, in the sense clause 10 means it: +// sent, failed (any reason), refused (any reason, including the sender's own clause-9 refusal), or not +// proven. A redelivery that finds one of these is a repeat, never a new refusal (novox/hq review, the +// operator-approved change to clause 10). +func isFinishedOutcome(outcome string) bool { + return outcome == "sent" || outcome == "not proven" || + strings.HasPrefix(outcome, "failed: ") || strings.HasPrefix(outcome, "refused: ") +} + +// claimQueuedSend verifies the operator's warrant for id (mesh-mail's own ask) and claims the send by +// compare-and-set from queued or retrying to sending, with the attempt and time, before any connection +// (novox/hq ADR 0436 clauses 9, 10). It is never trusted from the queue's message alone: the ask is read +// fresh from the router under the sender's own account, closed by the operator's Approve at level approve +// within its expiry (Warrant.For), and its act's digest (the account, recipients, subject and body's digest) +// is the option's binds (Option.Performs) — exactly the tools' own check (Decide, above) mirrored here, +// because the tool runner's account, which every agent session on this machine also reaches the bus as, may +// publish on the queue and write `sends` (this is the boundary clause 9 describes). +func claimQueuedSend(store SendStore, record RouterRecord, emit Emit, id string, now time.Time) (SendRequest, SendDecision, uint64, claimOutcome, error) { + req, rev, ok, err := getSend(store, id) + if err != nil { + return SendRequest{}, SendDecision{}, 0, claimLostRace, err + } + if !ok { + return SendRequest{}, SendDecision{}, 0, claimLostRace, fmt.Errorf("no send %q is kept", id) + } + if req.Decided == nil { + // Nothing was ever decided at all (never queued): no tell owed (clause 11 ties the tell to a + // *failed approved* send), and nothing emitted either — clause 11's two events are for a send the + // operator did approve. + d, _, err := settleSend(store, req, rev, SendDecision{At: now, Outcome: "refused: the send is not yet approved"}) + return req, d, 0, claimRefused, err + } + // Every other refusal below follows a real Approve (the send reached at least queued): clause 9's own + // refusal owes the operator a tell and the same mail.send.failed event as clause 10's refusal does + // (novox/hq ADR 0436 clause 11, issue 1199; review round 1 finding 4). + refuse := func(reason string) (SendRequest, SendDecision, uint64, claimOutcome, error) { + // Indexed for a tell BEFORE the record carries "refused" (novox/hq review round 1 finding 3): a + // crash between this and the write below must still leave it findable once refused does land; + // tellSend itself tolerates finding it still sending, queued or retrying in the meantime. + indexErr := addTellPending(store, req.ID) + d, newRev, err := settleSend(store, req, rev, SendDecision{At: now, Outcome: "refused: " + reason}) + if newRev == 0 { + // The write did not land (another writer's decision already stands, or it failed outright): + // nothing of this call's own making to emit (novox/hq review round 2, issue 1198). + return req, d, 0, claimRefused, errOr(indexErr, err) + } + _ = emit(MailSendFailed, req.ID, OutcomeEvent{Send: req.ID, Account: req.Account, Outcome: d.Outcome}) + // Only once refused is durably recorded is the sender-pending index's entry dropped. + if rmErr := removeSenderPending(store, req.ID); rmErr != nil { + indexErr = errOr(indexErr, fmt.Errorf("dropping %s from the sender-pending index: %w", req.ID, rmErr)) + } + return req, d, 0, claimRefused, indexErr } - claimed.Decided = &claim - acc, found := lookup(req.Account) - var final SendDecision switch { - case !found: - final = SendDecision{At: now, Outcome: "refused: the account no longer exists", Warrant: w.Says()} - default: - if sendErr := send(acc, req.To, req.Subject, req.Body, now); sendErr != nil { - final = SendDecision{At: now, Outcome: "failed: " + sendErr.Error(), Warrant: w.Says(), Error: sendErr.Error()} - } else { - final = SendDecision{At: now, Outcome: "sent", Warrant: w.Says()} + case req.Decided.Outcome == "sending": + // A previous delivery may have handed it to the server already; never sent twice, and never + // retried automatically either way (ADR 0436 clause 10). Same ordering as refuse() above. + indexErr := addTellPending(store, req.ID) + d, newRev, err := settleSend(store, req, rev, SendDecision{At: now, Outcome: "not proven", Attempt: req.Decided.Attempt}) + if newRev == 0 { + // The write did not land: nothing of this call's own making to emit (novox/hq review round 2, + // issue 1198). + return req, d, 0, claimNotProven, errOr(indexErr, err) + } + _ = emit(MailSendFailed, req.ID, OutcomeEvent{Send: req.ID, Account: req.Account, Outcome: d.Outcome}) + if rmErr := removeSenderPending(store, req.ID); rmErr != nil { + indexErr = errOr(indexErr, fmt.Errorf("dropping %s from the sender-pending index: %w", req.ID, rmErr)) + } + return req, d, 0, claimNotProven, indexErr + case isFinishedOutcome(req.Decided.Outcome): + // A redelivery of a message whose send already finished — sent, failed, refused by the sender's + // own claim, or not proven: the sender's own ack was lost (a bus reconnect) and the work-queue gave + // it up after AckWait, or two deliveries raced and the other one already finished it. The operator + // was already told (or sent) once; telling, emitting or recording anything again here would be a + // second word about a send that is already over. Just a repeat: terminated, nothing else touched + // (novox/hq ADR 0436 clause 10, operator-approved change). + return req, *req.Decided, 0, claimAlreadyFinished, nil + case req.Decided.Outcome != "queued" && req.Decided.Outcome != "retrying": + return refuse(fmt.Sprintf("the send is %q, neither queued nor retrying", req.Decided.Outcome)) + } + entry, err := record(id) + if err != nil { + return SendRequest{}, SendDecision{}, 0, claimLostRace, err + } + if entry == nil { + return refuse("the router keeps no record of this ask") + } + var rec routerRecord + if json.Unmarshal(entry.Value, &rec) != nil { + return refuse("the router's record of it cannot be read") + } + if rec.State == "" || rec.State == "open" || rec.Warrant == nil { + return refuse("the ask is not closed by the operator's Approve") + } + w := *rec.Warrant + if w.Asker != Module { + return refuse(fmt.Sprintf("the warrant is for %s's ask, not %s's", w.Asker, Module)) + } + ask := req.Ask() + option, err := w.For(Module, ask) + if err == nil { + err = option.Performs(req.Act(option.ID)) + } + if err != nil { + return refuse(err.Error()) + } + if option.ID != OptionSend { + return refuse("the option " + option.ID + " does not authorise a send") + } + claim := SendDecision{At: now, Outcome: "sending", Warrant: w.Says(), Attempt: req.Decided.Attempt + 1} + claimed, newRev, err := updateDecision(store, req, rev, claim) + if err != nil { + return SendRequest{}, SendDecision{}, 0, claimLostRace, err + } + if newRev == 0 { + // Lost the claim race: another delivery of the same message is already handling it. + return req, claimed, 0, claimLostRace, nil + } + return req, claimed, newRev, claimToSend, nil +} + +// ---- the send's outcome, retry and emit (novox/hq ADR 0436 clauses 10, 11, issue 1198) -------------------------- + +// MailSendSent and MailSendFailed are the two events the sender emits, never anything else (novox/hq ADR +// 0436 clause 11): the send id, the account and the outcome in words — never the recipients, the subject or +// the body. +const ( + MailSendSent = "mail.send.sent" + MailSendFailed = "mail.send.failed" +) + +// OutcomeEvent is the body mail.send.sent and mail.send.failed carry. +type OutcomeEvent struct { + Send string `json:"send"` + Account string `json:"account"` + Outcome string `json:"outcome"` +} + +// Emit publishes one of this module's own events with a chosen message id (novox/hq ADR 0436 clauses 4, +// 11): the real one (sender.go) wraps a process.Conn's own Emit; a test gives its own. +type Emit func(eventType, id string, body any) error + +// RetryDelays is how long after each not-taken attempt (1-indexed) the bus is asked to redeliver the +// message, before the next attempt: 1, 5, 15 and 30 minutes for attempts 1 to 4 (novox/hq ADR 0436 clause +// 10). +var RetryDelays = []time.Duration{time.Minute, 5 * time.Minute, 15 * time.Minute, 30 * time.Minute} + +// MostAttempts is how many times the sender hands a claimed send to the server before giving up: the fifth +// failure records failed, and the bus's own max-deliveries (module.json's mail-sends) dead-letters the +// message once it is terminated rather than acknowledged (novox/hq ADR 0436 clause 10). +const MostAttempts = 5 + +// msgAction is what the caller (the message handler, sender.go) does with the queue message once +// recordOutcome has written the decision: Ack (sent), NakWithDelay (retrying, not the last attempt yet) or +// Term (failed, or not proven — the bus is never asked to redeliver those; a dead letter past the fifth +// attempt is still reached by Term, since clause 10 records failed itself rather than waiting for the bus's +// own max-deliveries to notice). actionLeave is none of these: the record's own write could not be kept +// (even after a retry on a changed revision), so nothing is emitted and the message is left entirely to +// the bus — a later redelivery finds the record still at its previous outcome and reconciles from there, +// never a send silently announced sent, failed or retrying when it was not actually kept that way (novox/hq +// review round 1 finding 5). +type msgAction int + +const ( + actionAck msgAction = iota + actionRetry + actionTerm + actionLeave +) + +// recordOutcome writes the final decision for one claimed send once SendMail has run (or could not even be +// attempted — the account vanished, or its password cannot be read, both passed in as a NotTaken +// *SendError, exactly as a real SendMail failure would be: safe to retry), and says what the caller does +// with the message next. Attempt (req's own, from the claim) decides retrying from failed on a not-taken +// failure: the fifth attempt always fails rather than retries again (novox/hq ADR 0436 clauses 10, 11). +func recordOutcome(store SendStore, emit Emit, req SendRequest, claimed SendDecision, rev uint64, sendErr error, now time.Time) (SendDecision, msgAction, time.Duration, error) { + // write keeps d, retrying once on a changed revision (re-reading the send and trying again with its + // fresh one) before giving up: kept is false only when the record's own write could not be kept even + // after that retry, in which case the caller emits nothing and leaves the message to the bus entirely — + // never announcing an outcome (sent, failed, retrying) that was not actually recorded (novox/hq review + // round 1 finding 5). "Kept" is decided from settleSend's own returned revision, never from its error + // alone: on a changed revision whose record already carries another writer's decision, settleSend + // (through updateDecision) returns that other decision with revision 0 and a nil error — a nil error + // is not success here, and without checking the revision the retry never ran at all, so the caller + // could emit an outcome (and ack) while the record actually held someone else's state (novox/hq review + // round 2, issue 1198). + write := func(d SendDecision, final, tellWorthy bool) (result SendDecision, kept bool, outErr error) { + var indexErr error + if tellWorthy { + // Indexed for a tell BEFORE the record carries the outcome that owes it (novox/hq review round + // 1 finding 3): a crash between this and the write below must still leave it findable once the + // outcome does land; tellSend itself tolerates finding it still sending, queued or retrying. + indexErr = addTellPending(store, req.ID) + } + curReq, curRev := req, rev + var newRev uint64 + var writeErr error + for attempt := 0; attempt < 2; attempt++ { + result, newRev, writeErr = settleSend(store, curReq, curRev, d) + if newRev != 0 { + break // this write landed + } + if attempt == 1 { + break // retried once already; give up + } + if writeErr != nil && !errors.Is(writeErr, stdio.ErrChanged) { + break // a hard failure, not a revision race: retrying blindly would not help + } + fresh, freshRev, ok, rerr := getSend(store, req.ID) + if rerr != nil || !ok { + break + } + curReq, curRev = fresh, freshRev + } + if newRev == 0 { + reportErr := writeErr + if reportErr == nil { + reportErr = fmt.Errorf("the send %s's outcome could not be kept: another writer's decision already stands, even retried on its fresh revision", req.ID) + } + return d, false, errOr(indexErr, reportErr) + } + if final { + // Only once the outcome is durably recorded is the sender-pending index's entry dropped: from + // here the sender is done with it, one way or another (novox/hq issue 1199). + if rmErr := removeSenderPending(store, req.ID); rmErr != nil { + indexErr = errOr(indexErr, fmt.Errorf("dropping %s from the sender-pending index: %w", req.ID, rmErr)) + } + } + return result, true, indexErr + } + if sendErr == nil { + d, kept, err := write(SendDecision{At: now, Outcome: "sent", Warrant: claimed.Warrant, Attempt: claimed.Attempt}, true, false) + if !kept { + return d, actionLeave, 0, err + } + _ = emit(MailSendSent, req.ID, OutcomeEvent{Send: req.ID, Account: req.Account, Outcome: d.Outcome}) + return d, actionAck, 0, err + } + var se *SendError + if !errors.As(sendErr, &se) { + // SendMail itself never returns an error that is not a *SendError; a caller passing one of its own + // (the account vanished, or its password cannot be read) is read as not taken, never as a refusal or + // a proven send, which would be worse guesses than "safe to retry" (novox/hq ADR 0436 clause 10). + se = &SendError{Outcome: NotTaken, Err: sendErr} + } + switch se.Outcome { + case Refused: + d, kept, err := write(SendDecision{At: now, Outcome: "failed: " + se.Err.Error(), Warrant: claimed.Warrant, + Attempt: claimed.Attempt, Error: se.Err.Error()}, true, true) + if !kept { + return d, actionLeave, 0, err + } + _ = emit(MailSendFailed, req.ID, OutcomeEvent{Send: req.ID, Account: req.Account, Outcome: d.Outcome}) + return d, actionTerm, 0, err + case NotProven: + d, kept, err := write(SendDecision{At: now, Outcome: "not proven", Warrant: claimed.Warrant, Attempt: claimed.Attempt, + Error: se.Err.Error()}, true, true) + if !kept { + return d, actionLeave, 0, err + } + _ = emit(MailSendFailed, req.ID, OutcomeEvent{Send: req.ID, Account: req.Account, Outcome: d.Outcome}) + return d, actionTerm, 0, err + default: // NotTaken + if claimed.Attempt >= MostAttempts { + d, kept, err := write(SendDecision{At: now, Outcome: "failed: " + se.Err.Error(), Warrant: claimed.Warrant, + Attempt: claimed.Attempt, Error: se.Err.Error()}, true, true) + if !kept { + return d, actionLeave, 0, err + } + _ = emit(MailSendFailed, req.ID, OutcomeEvent{Send: req.ID, Account: req.Account, Outcome: d.Outcome}) + return d, actionTerm, 0, err + } + delay := RetryDelays[claimed.Attempt-1] + d, kept, err := write(SendDecision{At: now, Outcome: "retrying", Warrant: claimed.Warrant, Attempt: claimed.Attempt, + Error: se.Err.Error(), NextAttempt: now.Add(delay)}, false, false) + if !kept { + return d, actionLeave, 0, err + } + return d, actionRetry, delay, err // not a final outcome: no emit (clause 11 names only sent and failed) + } +} + +// ---- two more indices (novox/hq issue 1199), the same shape as the undecided index (issue 811) ------------------ +// +// Neither the tools' nor the sender's own `sends` state lists every key it holds (mesh-sdk go/process's +// State offers no such call, and listing it all on every health check or every look would grow with every +// send ever sent, not with the few still live): a reserved key stands in for a list, exactly as the +// undecided index already does. + +// errOr is a, or b when a is nil: the first of two errors a caller that can only report one keeps (novox/hq +// review round 1 finding 3), so an index-bookkeeping failure is never silently dropped once there is also a +// record-write failure to report, and vice versa. +func errOr(a, b error) error { + if a != nil { + return a + } + return b +} + +// idIndex is the shape a send-id index keeps: generic across all three (undecided, sender-pending, +// tells-pending). +type idIndex struct { + IDs []string `json:"ids"` +} + +// addToIndex adds id to the index at key, retrying its compare-and-set write on a race. A stored index +// that cannot be parsed is never overwritten (which would silently drop every id it already named): it is +// reported as an error naming the key instead (novox/hq issue 811, review round 3). +func addToIndex(store SendStore, key, id string) error { + for { + raw, rev, ok, err := store.Get(key) + if err != nil { + return err + } + var idx idIndex + if ok { + if err := json.Unmarshal(raw, &idx); err != nil { + return fmt.Errorf("the index %q cannot be parsed: %w", key, err) + } + for _, existing := range idx.IDs { + if existing == id { + return nil + } + } + } + idx.IDs = append(idx.IDs, id) + if !ok { + if _, err := store.Create(key, idx); err != nil { + if errors.Is(err, stdio.ErrChanged) { + continue + } + return err + } + return nil + } + if _, err := store.Update(key, idx, rev); err != nil { + if errors.Is(err, stdio.ErrChanged) { + continue + } + return err + } + return nil + } +} + +// removeFromIndex drops id from the index at key, retrying its compare-and-set write on a race. An id +// never in the index (already absent) is left as it is. +func removeFromIndex(store SendStore, key, id string) error { + for { + raw, rev, ok, err := store.Get(key) + if err != nil || !ok { + return err + } + var idx idIndex + if err := json.Unmarshal(raw, &idx); err != nil { + return fmt.Errorf("the index %q cannot be parsed: %w", key, err) + } + at := -1 + for i, existing := range idx.IDs { + if existing == id { + at = i + break + } + } + if at == -1 { + return nil + } + idx.IDs = append(idx.IDs[:at], idx.IDs[at+1:]...) + if _, err := store.Update(key, idx, rev); err != nil { + if errors.Is(err, stdio.ErrChanged) { + continue + } + return err + } + return nil + } +} + +// indexIDs is every id the index at key currently names. +func indexIDs(store SendStore, key string) ([]string, error) { + raw, _, ok, err := store.Get(key) + if err != nil || !ok { + return nil, err + } + var idx idIndex + if err := json.Unmarshal(raw, &idx); err != nil { + return nil, fmt.Errorf("the index %q cannot be parsed: %w", key, err) + } + return idx.IDs, nil +} + +// senderPendingKey indexes every send the sender currently owns — queued, retrying or sending — so +// mail_health's overdue check (Overdue, below) finds them without listing `sends` whole. +const senderPendingKey = "sender-pending" + +func addSenderPending(store SendStore, id string) error { + return addToIndex(store, senderPendingKey, id) +} +func removeSenderPending(store SendStore, id string) error { + return removeFromIndex(store, senderPendingKey, id) +} +func senderPendingIDs(store SendStore) ([]string, error) { return indexIDs(store, senderPendingKey) } + +// tellsPendingKey indexes every send still owing the sender's own tell (needsTell, below), so its own +// look (sender.go) finds them without listing `sends` whole. +const tellsPendingKey = "tells-pending" + +func addTellPending(store SendStore, id string) error { return addToIndex(store, tellsPendingKey, id) } +func removeTellPending(store SendStore, id string) error { + return removeFromIndex(store, tellsPendingKey, id) +} +func tellPendingIDs(store SendStore) ([]string, error) { return indexIDs(store, tellsPendingKey) } + +// ---- overdue (novox/hq ADR 0436 clause 11, issue 1199) ------------------------------------------------------ + +// OverdueAfter is how long a send may sit queued, or wait past its own next attempt, before mail_health +// calls it unhealthy (novox/hq ADR 0436 clause 11). +const OverdueAfter = 10 * time.Minute + +// Overdue is every send the sender currently owns (senderPendingIDs) that is overdue: queued more than +// OverdueAfter ago, or retrying more than OverdueAfter past its own next attempt. A send waiting for its +// next attempt within that window, or one actually being sent, is never overdue (novox/hq ADR 0436 clause +// 11). +func Overdue(store SendStore, now time.Time) ([]SendRequest, error) { + ids, err := senderPendingIDs(store) + if err != nil { + return nil, err + } + var out []SendRequest + for _, id := range ids { + req, _, ok, err := getSend(store, id) + if err != nil || !ok || req.Decided == nil { + continue + } + switch { + case req.Decided.Outcome == "queued" && now.Sub(req.Decided.At) > OverdueAfter: + out = append(out, req) + case req.Decided.Outcome == "retrying" && now.Sub(req.Decided.NextAttempt) > OverdueAfter: + out = append(out, req) } } - claimed.Decided = &final - if _, err := store.Update(req.ID, claimed, newRev); err != nil { - // The claim stands either way: a person's Approve is acted on once. The caller logs this; it is not retried. - return final, fmt.Errorf("the send %s was %s, and that could not be kept (%w)", req.ID, final.Outcome, err) + return out, nil +} + +// ---- telling the operator (novox/hq ADR 0436 clause 11, issue 1199) ---------------------------------------- +// +// Two separate tells, each kept to once on its own field: the sender's own, for a failed, refused or +// not-proven send (Tell); the tools' own, for an overdue one — still queued or retrying, its sender not +// having taken it yet (WaitsTell). Neither ever names an address, held to the operator channel's content +// rule by using only the account's own name. + +// TellOperator calls seat:operator-channel.tell once with the one line to say (novox/hq ADR 0380 §8). The +// real ones wrap a process.Conn's own Call (sender.go) or stdio.Ask (main.go); a test gives its own. +type TellOperator func(headline string) error + +// needsTell says whether a decided send's outcome owes the operator the sender's own tell (novox/hq ADR +// 0436 clause 11): every outcome after the Approve but sent — failed, refused or not proven. +func needsTell(outcome string) bool { + return outcome == "not proven" || strings.HasPrefix(outcome, "failed: ") || strings.HasPrefix(outcome, "refused: ") +} + +// tellReason is the sender's own one line for a failed, refused or not-proven send (novox/hq ADR 0436 +// clause 11). +func tellReason(account, outcome string) string { + if outcome == "not proven" { + return fmt.Sprintf("The mail you approved from %s may not have been sent; its sent folder says", account) } - return final, nil + reason := outcome + for _, prefix := range []string{"failed: ", "refused: "} { + if r, ok := strings.CutPrefix(outcome, prefix); ok { + reason = r + break + } + } + return fmt.Sprintf("The mail you approved from %s was not sent: %s", account, reason) +} + +// tellSend attempts the sender's own tell for one failed, refused or not-proven send, kept to once by +// Tell: telling then told, by compare-and-set, on the record itself (novox/hq ADR 0436 clause 11). A tell +// the channel refuses, or that nothing answers, leaves telling, so the next look (the sender's own ticker, +// at least every 10 minutes) tries it again; a record already told, or whose outcome no longer needs one, +// is dropped from the index without being told again. +func tellSend(store SendStore, tell TellOperator, id string, now time.Time) error { + req, rev, ok, err := getSend(store, id) + if err != nil { + return err + } + if !ok || req.Decided == nil { + _ = removeTellPending(store, id) + return nil + } + switch req.Decided.Outcome { + case "sending", "queued", "retrying": + // Indexed for a tell before the outcome that owes it had actually landed (addTellPending now runs + // before the write, novox/hq review round 1 finding 3): not decided yet whether a tell is owed. + // Left in the index; the next look finds it once the write lands. + return nil + } + if !needsTell(req.Decided.Outcome) || req.Decided.Tell == "told" { + _ = removeTellPending(store, id) + return nil + } + d := *req.Decided + if d.Tell != "telling" { + d.Tell = "telling" + next := req + next.Decided = &d + newRev, err := store.Update(id, next, rev) + if err != nil { + if errors.Is(err, stdio.ErrChanged) { + return nil // raced with another writer; the next look sorts it out + } + return err + } + rev = newRev + } + if err := tell(tellReason(req.Account, d.Outcome)); err != nil { + return nil // left telling; tried again on the next look + } + d.Tell = "told" + next := req + next.Decided = &d + if _, err := store.Update(id, next, rev); err != nil { + if errors.Is(err, stdio.ErrChanged) { + return nil + } + return err + } + _ = removeTellPending(store, id) + return nil +} + +// waitsReason is the tools' own one line for an overdue send (novox/hq ADR 0436 clause 11). +func waitsReason(account string) string { + return fmt.Sprintf("The mail you approved from %s waits: its sender has not taken it", account) +} + +// tellWaits attempts the tools' own "waits" tell for one overdue send, kept to once by WaitsTell — the +// same telling/told sequence as tellSend, on the record's own separate field, since the two tells are +// never the same one (novox/hq ADR 0436 clause 11). +func tellWaits(store SendStore, tell TellOperator, id string, now time.Time) error { + req, rev, ok, err := getSend(store, id) + if err != nil { + return err + } + if !ok || req.Decided == nil || (req.Decided.Outcome != "queued" && req.Decided.Outcome != "retrying") || req.Decided.WaitsTell == "told" { + return nil + } + d := *req.Decided + if d.WaitsTell != "telling" { + d.WaitsTell = "telling" + next := req + next.Decided = &d + newRev, err := store.Update(id, next, rev) + if err != nil { + if errors.Is(err, stdio.ErrChanged) { + return nil + } + return err + } + rev = newRev + } + if err := tell(waitsReason(req.Account)); err != nil { + return nil + } + d.WaitsTell = "told" + next := req + next.Decided = &d + if _, err := store.Update(id, next, rev); err != nil { + if errors.Is(err, stdio.ErrChanged) { + return nil + } + return err + } + return nil } // ---- reading the status ----------------------------------------------------------------------------------------- @@ -670,7 +1316,7 @@ func decideSend(store SendStore, lookup AccountLookup, send SendActor, req SendR // declined, ended without a choice (expired, refused, cancelled), or not proven. Read from the router's own // record; when the router's word is a close one that was not decided yet, this is also where a send is first // decided (and so sent), exactly once (Decide). -func ReadSendStatus(store SendStore, bus AskBus, lookup AccountLookup, send SendActor, id string, now time.Time) (map[string]any, error) { +func ReadSendStatus(store SendStore, bus AskBus, publish QueuePublish, id string, now time.Time) (map[string]any, error) { trimmed := strings.TrimSpace(id) req, rev, ok, err := getSend(store, trimmed) if err != nil { @@ -713,7 +1359,7 @@ func ReadSendStatus(store SendStore, bus AskBus, lookup AccountLookup, send Send out["status"], out["words"] = "waiting", "the operator has not answered yet" return out, nil } - d, err := Decide(store, lookup, send, *rec.Warrant, now) + d, err := Decide(store, publish, *rec.Warrant, now) if err != nil { out["status"], out["words"] = "not proven", err.Error() return out, nil @@ -726,9 +1372,17 @@ func ReadSendStatus(store SendStore, bus AskBus, lookup AccountLookup, send Send // and the words that explain it. now is when it is asked, not when the send was decided: a claim still "sending" // longer than StaleSendAfter with no final write is "not proven" (the send may or may not have gone out), never // "waiting" forever — and it is never retried automatically either way (novox/hq issue 790, review round 1). +// reportDecision's status word is one of: waiting (for the operator, or for the sender — approved, queued, +// being sent or retrying, each said apart in words), sent, failed, refused, not proven, declined or expired +// (novox/hq ADR 0436 clause 12, issue 1199) — the whole vocabulary mail_send_status answers. func reportDecision(out map[string]any, d SendDecision, now time.Time) { out["at"] = d.At.Format(time.RFC3339) switch { + case d.Outcome == "queued": + // Recorded queued by the one compare-and-set the Approve makes (novox/hq ADR 0436 clause 8): there + // is no separate "approved" state in a send's life. Published to mail-sends already, or the sweep + // will publish it again once a short grace has passed. + out["status"], out["words"] = "waiting", "approved and waiting for the sender; ask again shortly" case d.Outcome == "sent": out["status"], out["words"] = "sent", "sent on the operator's Approve ("+d.Warrant+")" case d.Outcome == "declined": @@ -738,17 +1392,23 @@ func reportDecision(out map[string]any, d SendDecision, now time.Time) { out["words"] = "not proven: the send may or may not have gone out; the operator's Approve was being acted " + "on at " + d.At.Format(time.RFC3339) + ", with no final word since" case d.Outcome == "sending": - out["status"], out["words"] = "waiting", "the operator's Approve is being acted on; ask again shortly" + out["status"], out["words"] = "waiting", "being sent; ask again shortly" + case d.Outcome == "retrying": + out["status"] = "waiting" + out["words"] = "waiting for its next attempt, at " + d.NextAttempt.Format(time.RFC3339) + ": " + d.Error + case d.Outcome == "not proven": + out["status"] = "not proven" + out["words"] = "not proven: the send may or may not have gone out: " + d.Error case d.Outcome == "ended: never heard": - out["status"], out["words"] = "ended", "its ask to the operator was never heard, so it was not sent" + out["status"], out["words"] = "expired", "its ask to the operator was never heard, so it was not sent" case strings.HasPrefix(d.Outcome, "failed: "): out["status"], out["words"] = "failed", "nothing reached the server: "+strings.TrimPrefix(d.Outcome, "failed: ") case strings.HasPrefix(d.Outcome, "ended: "): - out["status"], out["words"] = "ended", "nothing was sent: "+strings.TrimPrefix(d.Outcome, "ended: ") + out["status"], out["words"] = "expired", "nothing was sent: "+strings.TrimPrefix(d.Outcome, "ended: ") case strings.HasPrefix(d.Outcome, "refused: "): - out["status"], out["words"] = "not proven", "nothing was sent: "+strings.TrimPrefix(d.Outcome, "refused: ") + out["status"], out["words"] = "refused", "nothing was sent: "+strings.TrimPrefix(d.Outcome, "refused: ") default: - out["status"], out["words"] = "ended", d.Outcome + out["status"], out["words"] = "expired", d.Outcome } } @@ -758,35 +1418,62 @@ func reportDecision(out map[string]any, d SendDecision, now time.Time) { // the same Decide that mail_send_status uses: an approved send goes out as soon as the operator answers, never // only when something later happens to call mail_send_status (novox/hq ADR 0361 §2, review round 1 of issue 790). func (b *Bundle) Heard(w asks.Warrant) { - if _, err := Decide(b.sends(), b.lookupAccount, b.sendActor(), w, b.now()); err != nil { + if _, err := Decide(b.sends(), b.queuePublish(), w, b.now()); err != nil { if b.Logf != nil { b.Logf("[mesh-mail] the warrant for the send %s could not be acted on: %v", w.Ask, err) } } } -// Sweep reads only the sends the undecided index still names, and asks the router's own record for each one's -// answer, the same path mail_send_status takes (ReadSendStatus): an Approve recorded while this module did not -// hear it directly (down, or not yet subscribed) is still acted on, once. Unlike reading every send ever kept, its -// work grows only with the sends still undecided, never with every one a module has ever sent (novox/hq ADR 0361 -// §2, review round 1 of issue 790; the index, issue 811). +// QueuedGrace is how long a newly queued send is given to be published before the sweep considers it +// possibly stuck and publishes it again: long enough that the tools' own publish, right after the same +// compare-and-set write, is never raced by the sweep under ordinary load (novox/hq ADR 0436 clause 8, as +// decided to replace the two-state approved/queued design, review round 1). +const QueuedGrace = time.Minute + +// Sweep does two things, neither reading every send ever kept (its work grows only with the few still +// live, never with history): a send the undecided index still names and that is not yet decided at all is +// asked about the same way mail_send_status does (ReadSendStatus) — an Approve recorded while this module +// did not hear it directly (down, or not yet subscribed) is still acted on, once (novox/hq ADR 0361 §2, +// review round 1 of issue 790; the index, issue 811); and a send the sender-pending index names that is +// still "queued" past QueuedGrace — its publish to mail-sends may have failed, or the tools stopped before +// it ever ran — is published again, the send id still its message id, so the duplicate window and the +// sender's own claim keep it one send either way (novox/hq ADR 0436 clause 8, review round 1). func (b *Bundle) Sweep(now time.Time) { store := b.sends() - ids, err := undecidedIDs(store) + undecided, err := undecidedIDs(store) if err != nil { if b.Logf != nil { b.Logf("[mesh-mail] the undecided index cannot be read: %v", err) } + } else { + for _, id := range undecided { + req, _, ok, err := getSend(store, id) + if err != nil || !ok || req.Decided != nil { + continue + } + if _, err := ReadSendStatus(store, b.askBus(), b.queuePublish(), id, now); err != nil { + if b.Logf != nil { + b.Logf("[mesh-mail] sweeping the send %s: %v", id, err) + } + } + } + } + pending, err := senderPendingIDs(store) + if err != nil { + if b.Logf != nil { + b.Logf("[mesh-mail] the sender-pending index cannot be read: %v", err) + } return } - for _, id := range ids { + for _, id := range pending { req, _, ok, err := getSend(store, id) - if err != nil || !ok || req.Decided != nil { + if err != nil || !ok || req.Decided == nil || req.Decided.Outcome != "queued" || now.Sub(req.Decided.At) < QueuedGrace { continue } - if _, err := ReadSendStatus(store, b.askBus(), b.lookupAccount, b.sendActor(), id, now); err != nil { + if err := b.queuePublish()(req.ID, QueuedSend{Send: req.ID, Account: req.Account, Ask: req.ID}); err != nil { if b.Logf != nil { - b.Logf("[mesh-mail] sweeping the send %s: %v", id, err) + b.Logf("[mesh-mail] republishing the queued send %s: %v", id, err) } } } diff --git a/modules/mesh-mail/cmd/mesh-mail/send_test.go b/modules/mesh-mail/cmd/mesh-mail/send_test.go index 176a7287..36939403 100644 --- a/modules/mesh-mail/cmd/mesh-mail/send_test.go +++ b/modules/mesh-mail/cmd/mesh-mail/send_test.go @@ -166,6 +166,47 @@ func (r *sendRouter) published1() int { return len(r.published) } +// ---- a fake work-queue publish, as the tools reach mail-sends (novox/hq ADR 0436 clause 8) ---------------------- + +type queuedMsg struct { + id string + body QueuedSend +} + +type memQueue struct { + mu sync.Mutex + published []queuedMsg + failErr error +} + +func newMemQueue() *memQueue { return &memQueue{} } + +func (q *memQueue) publish(id string, body any) error { + q.mu.Lock() + defer q.mu.Unlock() + if q.failErr != nil { + return q.failErr + } + b, _ := body.(QueuedSend) + q.published = append(q.published, queuedMsg{id: id, body: b}) + return nil +} + +func (q *memQueue) count() int { + q.mu.Lock() + defer q.mu.Unlock() + return len(q.published) +} + +func (q *memQueue) last() queuedMsg { + q.mu.Lock() + defer q.mu.Unlock() + if len(q.published) == 0 { + return queuedMsg{} + } + return q.published[len(q.published)-1] +} + // ---- an in-process fake SMTP server, speaking the protocol over a real socket ------------------------------------- type receivedMail struct { @@ -375,11 +416,13 @@ func TestMailSendAsksWithTheAccountRecipientsSubjectAndFirstLines(t *testing.T) } } -// (3) On Approve, the fake SMTP server receives exactly one message, even if the warrant is delivered twice. -func TestOnApproveTheFakeSMTPServerReceivesExactlyOneMessageEvenDeliveredTwice(t *testing.T) { - fake := newFakeSMTPServer(t) +// 1194/AC1: On Approve, the send is queued for the sender exactly once, even if the warrant is delivered twice +// (novox/hq ADR 0436 clause 8). Since issue 1194 the tools never dial an outgoing server: they record approved +// and queue the send for the sender process, which alone performs it (issue 1196, 1198). +func TestOnApproveTheSendIsQueuedExactlyOnceEvenDeliveredTwice(t *testing.T) { b, router, sends := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"}) - b.Dial = fake.dialer() + queue := newMemQueue() + b.Queue = queue.publish rt := startBundle(t, b.Tools()) text, isErr := rt.call(SendTool, "novox.triage", sendArgs) @@ -393,44 +436,41 @@ func TestOnApproveTheFakeSMTPServerReceivesExactlyOneMessageEvenDeliveredTwice(t w := router.chose(id, OptionSend, judgedAt.Add(time.Minute)) router.close(w) - lookup := b.lookupAccount - actor := b.sendActor() - if _, err := Decide(sends, lookup, actor, w, judgedAt.Add(2*time.Minute)); err != nil { + publish := b.queuePublish() + if _, err := Decide(sends, publish, w, judgedAt.Add(2*time.Minute)); err != nil { t.Fatalf("deciding: %v", err) } - if n := fake.count(); n != 1 { - t.Fatalf("the fake server received %d message(s) after one Approve", n) + if n := queue.count(); n != 1 { + t.Fatalf("the queue received %d message(s) after one Approve", n) } - // The warrant delivered a second time: still exactly one message. - if _, err := Decide(sends, lookup, actor, w, judgedAt.Add(3*time.Minute)); err != nil { + // The warrant delivered a second time: still queued exactly once. + if _, err := Decide(sends, publish, w, judgedAt.Add(3*time.Minute)); err != nil { t.Fatalf("deciding again: %v", err) } - if n := fake.count(); n != 1 { - t.Fatalf("the fake server received %d message(s) after the warrant was delivered twice", n) + if n := queue.count(); n != 1 { + t.Fatalf("the queue received %d message(s) after the warrant was delivered twice", n) } - got := fake.last() - if got.from != "jschoubben@novox.be" || len(got.to) != 2 || got.to[0] != "ops@example.org" || got.to[1] != "lead@example.org" { - t.Fatalf("received %+v", got) - } - if !strings.Contains(got.data, "The nightly gate is red") || !strings.Contains(got.data, "The merge gate failed on the resolver change") { - t.Fatalf("the message sent: %q", got.data) + got := queue.last() + if got.id != id || got.body.Send != id || got.body.Account != "mesh" || got.body.Ask != id { + t.Fatalf("queued %+v, want the send id %s as the message id, account and ask", got, id) } - // mail_send_status reports it sent, and reports the same thing again without sending a third time. + // mail_send_status reports it waiting on the sender, and reports the same thing again without queueing a + // third time. text, isErr = rt.call(SendStatusTool, "novox.triage", map[string]any{"id": id}) - if isErr || !strings.Contains(text, `"status":"sent"`) { + if isErr || !strings.Contains(text, `"status":"waiting"`) { t.Fatalf("%v %s", isErr, text) } - if n := fake.count(); n != 1 { - t.Fatalf("mail_send_status sent again: %d", n) + if n := queue.count(); n != 1 { + t.Fatalf("mail_send_status queued again: %d", n) } } -// (4) On Decline, nothing is sent, and the status says declined. -func TestOnDeclineNothingIsSentAndTheStatusSaysDeclined(t *testing.T) { - fake := newFakeSMTPServer(t) +// 1194/AC3: On Decline, nothing is queued, and the status says declined. +func TestOnDeclineNothingIsQueuedAndTheStatusSaysDeclined(t *testing.T) { b, router, sends := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"}) - b.Dial = fake.dialer() + queue := newMemQueue() + b.Queue = queue.publish rt := startBundle(t, b.Tools()) text, isErr := rt.call(SendTool, "novox.triage", sendArgs) @@ -443,11 +483,11 @@ func TestOnDeclineNothingIsSentAndTheStatusSaysDeclined(t *testing.T) { w := router.chose(id, OptionDecline, judgedAt.Add(time.Minute)) router.close(w) - if _, err := Decide(sends, b.lookupAccount, b.sendActor(), w, judgedAt.Add(2*time.Minute)); err != nil { + if _, err := Decide(sends, b.queuePublish(), w, judgedAt.Add(2*time.Minute)); err != nil { t.Fatalf("deciding: %v", err) } - if n := fake.count(); n != 0 { - t.Fatalf("the fake server received %d message(s) on Decline", n) + if n := queue.count(); n != 0 { + t.Fatalf("the queue received %d message(s) on Decline", n) } text, isErr = rt.call(SendStatusTool, "novox.triage", map[string]any{"id": id}) @@ -474,12 +514,12 @@ func TestMailSendRefusesAnAccountWithNoOutgoingServer(t *testing.T) { } } -// (5) An approved send goes out once the warrant reaches the module through the hear handler alone — never -// through mail_send_status (novox/hq ADR 0361 §2, review round 1 of issue 790). -func TestAnApprovedSendGoesOutOnceWithNoStatusCall(t *testing.T) { - fake := newFakeSMTPServer(t) +// 1194/AC1: An approved send is queued once the warrant reaches the module through the hear handler alone — +// never through mail_send_status (novox/hq ADR 0361 §2, review round 1 of issue 790). +func TestAnApprovedSendIsQueuedOnceWithNoStatusCall(t *testing.T) { b, router, _ := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"}) - b.Dial = fake.dialer() + queue := newMemQueue() + b.Queue = queue.publish rt := startBundle(t, b.Tools()) text, isErr := rt.call(SendTool, "novox.triage", sendArgs) @@ -494,21 +534,21 @@ func TestAnApprovedSendGoesOutOnceWithNoStatusCall(t *testing.T) { router.close(w) b.Heard(w) // the warrant delivered through the hear handler only - if n := fake.count(); n != 1 { - t.Fatalf("the fake server received %d message(s) after the warrant was heard once", n) + if n := queue.count(); n != 1 { + t.Fatalf("the queue received %d message(s) after the warrant was heard once", n) } b.Heard(w) // the same warrant delivered a second time - if n := fake.count(); n != 1 { - t.Fatalf("the fake server received %d message(s) after the warrant was heard twice", n) + if n := queue.count(); n != 1 { + t.Fatalf("the queue received %d message(s) after the warrant was heard twice", n) } } -// (6) The sweep acts on a send the operator approved while this module was never told directly: it reads the -// router's own record, the same path mail_send_status takes (novox/hq issue 790, review round 1). +// 1194/AC1: The sweep acts on a send the operator approved while this module was never told directly: it reads +// the router's own record, the same path mail_send_status takes (novox/hq issue 790, review round 1). func TestSweepActsOnAnApprovedSendItWasNotToldAboutDirectly(t *testing.T) { - fake := newFakeSMTPServer(t) b, router, _ := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"}) - b.Dial = fake.dialer() + queue := newMemQueue() + b.Queue = queue.publish rt := startBundle(t, b.Tools()) text, isErr := rt.call(SendTool, "novox.triage", sendArgs) @@ -523,12 +563,66 @@ func TestSweepActsOnAnApprovedSendItWasNotToldAboutDirectly(t *testing.T) { router.close(w) // only the router's record holds the answer; mesh-mail is never told directly b.Sweep(judgedAt.Add(2 * time.Minute)) - if n := fake.count(); n != 1 { + if n := queue.count(); n != 1 { t.Fatalf("the sweep did not act on the approved send: %d message(s)", n) } - b.Sweep(judgedAt.Add(3 * time.Minute)) - if n := fake.count(); n != 1 { - t.Fatalf("a second sweep sent again: %d message(s)", n) + // A second sweep, still within the just-queued record's own grace, does not publish it again (past the + // grace, it would: TestSweepPublishesAgainAQueuedRecordPastItsGrace, novox/hq review round 1). + b.Sweep(judgedAt.Add(2*time.Minute + QueuedGrace/2)) + if n := queue.count(); n != 1 { + t.Fatalf("a second sweep within the grace queued again: %d message(s)", n) + } +} + +// 1194/AC2 (as decided to replace the two-state approved/queued design, review round 1): the sweep +// republishes a queued, unclaimed record whose queued time is older than QueuedGrace, the send id still +// its message id — the stream's duplicate window and the sender's own claim keep it one send either way +// (proved end to end over a real bus by the module-whole acceptance test, issue 1198 AC4). +func TestSweepPublishesAgainAQueuedRecordPastItsGrace(t *testing.T) { + b, router, sends := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"}) + queue := newMemQueue() + queue.failErr = errors.New("the bus is down") + b.Queue = queue.publish + rt := startBundle(t, b.Tools()) + + text, isErr := rt.call(SendTool, "novox.triage", sendArgs) + if isErr { + t.Fatalf("%s", text) + } + var out map[string]any + _ = json.Unmarshal([]byte(text), &out) + id := out["ask"].(string) + + w := router.chose(id, OptionSend, judgedAt.Add(time.Minute)) + router.close(w) + decidedAt := judgedAt.Add(2 * time.Minute) + if _, err := Decide(sends, b.queuePublish(), w, decidedAt); err != nil { + t.Fatalf("deciding: %v", err) + } + if n := queue.count(); n != 0 { + t.Fatalf("the queue received %d message(s) despite failing", n) + } + // There is no "approved" state: the record is already "queued", its own publish just failed. + req, _, ok, err := getSend(sends, id) + if err != nil || !ok || req.Decided == nil || req.Decided.Outcome != "queued" { + t.Fatalf("left queued despite the publish failing: %v %v %+v", err, ok, req.Decided) + } + + queue.failErr = nil + // Within its grace, the sweep leaves a freshly queued record alone. + b.Sweep(decidedAt.Add(QueuedGrace / 2)) + if n := queue.count(); n != 0 { + t.Fatalf("the sweep republished within the grace: %d message(s)", n) + } + + // Past its grace, the sweep publishes the still-queued send again. + b.Sweep(decidedAt.Add(QueuedGrace + time.Second)) + if n := queue.count(); n != 1 { + t.Fatalf("the sweep did not republish the queued send past its grace: %d message(s)", n) + } + got := queue.last() + if got.id != id || got.body.Send != id { + t.Fatalf("republished %+v, want the send id as the message id", got) } } @@ -590,26 +684,15 @@ func TestMailSendRefusesAControlCharacterInTheSubjectOrAnAddress(t *testing.T) { } // (9) Rcpt takes the parsed address alone; the To header takes the formatted address (novox/hq issue 790, review -// round 1). +// round 1). Tested directly against SendMail (smtp.go): since issue 1194 the tools never dial an outgoing +// server themselves — Decide only records approved and queues the send for the sender (issue 1196, 1198), which +// is where SendMail is actually called from. func TestSendUsesTheParsedAddressForRcptAndTheFormattedAddressForTheToHeader(t *testing.T) { fake := newFakeSMTPServer(t) - b, router, _ := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"}) - b.Dial = fake.dialer() - rt := startBundle(t, b.Tools()) - - args := map[string]any{"account": "mesh", "to": []any{"Ops Team "}, - "subject": "Status", "body": "body"} - text, isErr := rt.call(SendTool, "novox.triage", args) - if isErr { - t.Fatalf("%s", text) + acc := Account{Name: "mesh", Address: "jschoubben@novox.be", Outgoing: &Server{Host: "127.0.0.1", Port: 25, Security: "tls"}} + if err := SendMail(fake.dialer(), acc, "the-password", []string{"Ops Team "}, "Status", "body", judgedAt, "send-abc123"); err != nil { + t.Fatalf("SendMail: %v", err) } - var out map[string]any - _ = json.Unmarshal([]byte(text), &out) - id := out["ask"].(string) - - w := router.chose(id, OptionSend, judgedAt.Add(time.Minute)) - router.close(w) - b.Heard(w) got := fake.last() if len(got.to) != 1 || got.to[0] != "ops@example.org" { @@ -660,8 +743,8 @@ func TestASendWhoseAskNeverPublishedEndsAfterTheAskWouldHaveLasted(t *testing.T) if isErr { t.Fatalf("%s", text) } - if !strings.Contains(text, `"status":"ended"`) { - t.Fatalf("does not read ended once the ask would have lasted: %s", text) + if !strings.Contains(text, `"status":"expired"`) { + t.Fatalf("does not read expired once the ask would have lasted: %s", text) } if !strings.Contains(text, "its ask to the operator was never heard, so it was not sent") { t.Fatalf("does not say why: %s", text) @@ -671,9 +754,7 @@ func TestASendWhoseAskNeverPublishedEndsAfterTheAskWouldHaveLasted(t *testing.T) // (11) The sweep reads only the sends still undecided, by the undecided index, not every send ever kept: a send // already decided is never asked about again (novox/hq issue 811). func TestSweepReadsOnlyUndecidedSends(t *testing.T) { - fake := newFakeSMTPServer(t) b, router, sends := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"}) - b.Dial = fake.dialer() rt := startBundle(t, b.Tools()) // One send, declined right away: decided, so it leaves the undecided index. @@ -686,7 +767,7 @@ func TestSweepReadsOnlyUndecidedSends(t *testing.T) { declinedID := out["ask"].(string) w := router.chose(declinedID, OptionDecline, judgedAt.Add(time.Minute)) router.close(w) - if _, err := Decide(sends, b.lookupAccount, b.sendActor(), w, judgedAt.Add(2*time.Minute)); err != nil { + if _, err := Decide(sends, b.queuePublish(), w, judgedAt.Add(2*time.Minute)); err != nil { t.Fatalf("deciding: %v", err) } @@ -748,3 +829,35 @@ func TestAddAndRemoveUndecidedFailOnACorruptIndexRatherThanResettingIt(t *testin t.Fatalf("removeUndecided overwrote the corrupt index instead of leaving it: %s", sends.vals[undecidedIndexKey]) } } + +// 1194/AC3: a warrant that does not bind the request (its digest is not the ask's) is refused, as today, and +// nothing is published on mail-sends. +func TestADigestMismatchIsRefusedAndNothingIsQueued(t *testing.T) { + b, router, sends := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"}) + queue := newMemQueue() + b.Queue = queue.publish + rt := startBundle(t, b.Tools()) + + text, isErr := rt.call(SendTool, "novox.triage", sendArgs) + if isErr { + t.Fatalf("%s", text) + } + var out map[string]any + _ = json.Unmarshal([]byte(text), &out) + id := out["ask"].(string) + + w := router.chose(id, OptionSend, judgedAt.Add(time.Minute)) + w.AskDigest = "sha256:" + strings.Repeat("0", 64) // not the digest of the ask actually published + router.close(w) + + d, err := Decide(sends, b.queuePublish(), w, judgedAt.Add(2*time.Minute)) + if err != nil { + t.Fatalf("deciding: %v", err) + } + if !strings.HasPrefix(d.Outcome, "refused: ") { + t.Fatalf("outcome: %q, want refused: ...", d.Outcome) + } + if n := queue.count(); n != 0 { + t.Fatalf("the queue received %d message(s) despite the digest mismatch", n) + } +} diff --git a/modules/mesh-mail/cmd/mesh-mail/sender.go b/modules/mesh-mail/cmd/mesh-mail/sender.go new file mode 100644 index 00000000..4217270e --- /dev/null +++ b/modules/mesh-mail/cmd/mesh-mail/sender.go @@ -0,0 +1,295 @@ +package main + +// mesh-mail send (novox/hq ADR 0436 §7): the sender, a process with a bus account of its own +// (mesh-mail-send), never a seat. It alone of the module's parts reads an account's password (issue 1193); +// the tools never send — they record the operator's Approve and queue it (issue 1194) on the work queue +// mail-sends, which this process consumes through its one durable consumer (issue 1195). For each message +// it verifies the warrant again and claims the send once (issue 1196), performs the one handover to the +// server (issue 1197's classification), and records the outcome, retries a not-taken failure and emits the +// result (issue 1198). The tell to the operator and the overdue health check are issue 1199's. + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-mail/cmd/mesh-mail/internal/secret" +) + +// SendProcessID is this process's id, as module.json's resource "send" names it: the controller delivers +// its own bus account as the own secret broker- (novox/hq ADR 0436 §2). +const SendProcessID = "send" + +// MailSendsStream is the work-queue stream mail-sends carries on the bus (novox/hq ADR 0436 §8): the +// manifest's local name "mail-sends", capitalised with its hyphen an underscore, exactly as the controller +// composes a stream's own bus name (mesh-controller internal/catalogue/streams.go StreamBusName, +// strings.ToUpper(strings.ReplaceAll(local, "-", "_")); confirmed against its own fixture, mesh-controller +// cmd/mesh-controller/process_live_test.go's mailSends, which names it "MAIL_SENDS" — review round 1, +// finding 7). +const MailSendsStream = "MAIL_SENDS" + +// MessengerAsksBucket is the operator channel's own records bucket, as the controller's own fixtures name +// it (mesh-controller cmd/mesh-controller/process_live_test.go, internal/broker/process_grants_test.go): +// never hard-coded in mesh-sdk go/process itself, since a process may read more than one seat's records. +const MessengerAsksBucket = "messenger_asks" + +// routerRecordOf is the real RouterRecord (send.go), reading the router's record of one of mesh-mail's own +// asks under conn's own account (novox/hq ADR 0436 clause 9) — the same direct-get the tools use +// (stdio.SeatRecord), reached here as the sender's own account instead of the tool runner's. Wired into the +// handler once the claim, the send and its outcome are all in place (issues 1197, 1198). +func routerRecordOf(conn *process.Conn) RouterRecord { + return func(id string) (*stdio.StateEntry, error) { return conn.Ask(MessengerAsksBucket, id) } +} + +// processSends adapts conn's own `sends` state (novox/hq ADR 0436 clause 7's bus.state) to SendStore +// (send.go), the same CAS the tools use, now reached under the sender's own account. Keys is never called +// by the claim or outcome recording (issues 1196, 1198) — only the tools' sweep calls it, over its own +// account — so it is refused by name if ever called by mistake, rather than silently answering empty. +type processSends struct{ state process.State } + +func (p processSends) Get(key string) (json.RawMessage, uint64, bool, error) { + e, err := p.state.Get(key) + if err != nil || e == nil { + return nil, 0, false, err + } + return e.Value, e.Revision, true, nil +} +func (p processSends) Create(key string, value any) (uint64, error) { + return p.state.Create(key, value) +} +func (p processSends) Update(key string, value any, revision uint64) (uint64, error) { + return p.state.Update(key, value, revision) +} +func (p processSends) Keys() ([]string, error) { + return nil, fmt.Errorf("the sender's own `sends` state is never listed whole; only the tools' sweep does that") +} + +// send is `mesh-mail send`: it reads its own bus account and the accounts setting, connects, and consumes +// mail-sends until stopped. It refuses to start naming what is missing when the account or the accounts +// setting cannot be read — never a password, which no error here or anywhere in this process logs +// (novox/hq ADR 0436 §7, §12). +func send(env Env) int { + if env.Secrets == "" { + logf("[mesh-mail] send was started without its secrets directory (MESH_MAIL_SECRETS)") + return 2 + } + if _, err := ReadAccounts(env.Accounts); err != nil { + logf("[mesh-mail] send: %v", err) + return 1 + } + account, err := process.ReadAccount(secret.Path(env.Secrets, "broker-"+SendProcessID)) + if err != nil { + logf("[mesh-mail] send: %v", err) + return 1 + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + conn, err := process.Connect(account) + if err != nil { + logf("[mesh-mail] send: the bus cannot be reached: %v", err) + return 1 + } + defer conn.Close() + h := &senderHandler{ + Store: processSends{state: conn.State(SendsState)}, + Record: routerRecordOf(conn), + Accounts: func() ([]Account, error) { return ReadAccounts(env.Accounts) }, + SecretsDir: env.Secrets, + Emit: emitOf(conn), + Tell: tellOperatorOf(conn), + Logf: logf, + } + go h.tellTicker(ctx) + logf("[mesh-mail] send is consuming %s as %s", MailSendsStream, account.User) + if err := conn.Consume(ctx, MailSendsStream, h.handle); err != nil && ctx.Err() == nil { + logf("[mesh-mail] send stopped: %v", err) + return 1 + } + return 0 +} + +// tellOperatorOf is the real TellOperator (send.go), wrapping conn's own Call to the operator channel's +// tell verb (novox/hq ADR 0380 §8, ADR 0436 clause 11): no lines, no subject naming anything the content +// rule would refuse — the headline alone carries the one line. +func tellOperatorOf(conn *process.Conn) TellOperator { + return func(headline string) error { + _, err := conn.Call("seat:operator-channel.tell", map[string]any{"headline": headline, "lines": []string{}, "subject": "mesh-mail"}) + return err + } +} + +// TellEvery is how often the sender looks for a send still owing its own tell (novox/hq ADR 0436 clause +// 11: "at least every 10 minutes"). +const TellEvery = 10 * time.Minute + +// tellTicker runs every send still owing the sender's own tell at least every TellEvery, until ctx ends +// (novox/hq ADR 0436 clause 11, issue 1199): once right away, so a tell left over a restart is not made to +// wait a full interval. +func (h *senderHandler) tellTicker(ctx context.Context) { + t := time.NewTicker(TellEvery) + defer t.Stop() + for { + h.tellPending() + select { + case <-ctx.Done(): + return + case <-t.C: + } + } +} + +func (h *senderHandler) tellPending() { + ids, err := tellPendingIDs(h.Store) + if err != nil { + h.logf("[mesh-mail] send: the tells-pending index cannot be read: %v", err) + return + } + for _, id := range ids { + if err := tellSend(h.Store, h.Tell, id, h.now()); err != nil { + h.logf("[mesh-mail] send: telling the operator about %s: %v", id, err) + } + } +} + +// runSend connects as the sender's own account and consumes mail-sends until ctx ends, handing each message +// to handle. Apart from send so a test can drive it over a bus of its own, with a handler of its own +// (novox/hq issue 1195, AC2): on ctx ending (a signal), Consume stops pulling and returns without +// acknowledging a message still in handle's hands — only handle decides a message's fate (process/queue.go). +func runSend(ctx context.Context, account process.Account, handle func(*process.Message)) error { + conn, err := process.Connect(account) + if err != nil { + return fmt.Errorf("the bus cannot be reached: %w", err) + } + defer conn.Close() + return conn.Consume(ctx, MailSendsStream, handle) +} + +// emitOf is the real Emit (send.go), wrapping conn's own Emit with the given id as both x-event-id and the +// bus's own dedup id (novox/hq ADR 0436 clauses 4, 11). +func emitOf(conn *process.Conn) Emit { + return func(eventType, id string, body any) error { + return conn.Emit(eventType, body, stdio.EmitOptions{MessageID: id}) + } +} + +// senderHandler is everything handleQueuedSend needs: the `sends` state and the router's record, both +// reached under the sender's own account, the accounts setting (to find the server to connect to) and the +// secrets directory (to read the one password this process holds), and how it sends and emits — real +// unless a test gives its own (novox/hq ADR 0436 §7, issues 1196 to 1198). +type senderHandler struct { + Store SendStore + Record RouterRecord + Accounts func() ([]Account, error) + SecretsDir string + Dial SMTPDialer + Emit Emit + Tell TellOperator + Now func() time.Time + Logf func(format string, a ...any) +} + +func (h *senderHandler) now() time.Time { + if h.Now != nil { + return h.Now() + } + return time.Now() +} + +func (h *senderHandler) logf(format string, a ...any) { + if h.Logf != nil { + h.Logf(format, a...) + } +} + +// handle is the handler mesh-mail send's Consume hands every message to mail-sends (novox/hq ADR 0436 +// clauses 9, 10, 11): the warrant verified again and the send claimed (claimQueuedSend, send.go), the one +// handover to the server attempted, and the outcome recorded, retried or emitted (recordOutcome, send.go) — +// then, and only then, the message is acknowledged, asked redelivered, or terminated, exactly as that +// outcome says. +func (h *senderHandler) handle(msg *process.Message) { + var q QueuedSend + if err := json.Unmarshal(msg.Body, &q); err != nil { + h.logf("[mesh-mail] send: a queued message on %s could not be read: %v", msg.Subject, err) + if err := msg.Term(); err != nil { + h.logf("[mesh-mail] send: terminating %s: %v", msg.Subject, err) + } + return + } + req, claimed, rev, outcome, err := claimQueuedSend(h.Store, h.Record, h.Emit, q.Send, h.now()) + if err != nil { + // Logged, never returned on alone: an index-bookkeeping error (novox/hq review round 1 finding 3) + // can stand beside a real decision (claimRefused, claimNotProven) that the switch below still + // must act on; a fatal read or claim failure, with no decision made, always reads claimLostRace, + // which the switch already leaves to the bus. + h.logf("[mesh-mail] send: claiming %s: %v", q.Send, err) + } + switch outcome { + case claimLostRace: + return // another delivery of the same message already owns it. + case claimRefused, claimNotProven, claimAlreadyFinished: + // claimAlreadyFinished: a repeat of a message whose send already finished (the operator was + // already told, or not, exactly once) — terminated, and nothing else touched (novox/hq review, + // operator-approved change to clause 10). + if err := msg.Term(); err != nil { + h.logf("[mesh-mail] send: terminating %s: %v", q.Send, err) + } + return + } + sendErr := h.attempt(req) + _, action, delay, err := recordOutcome(h.Store, h.Emit, req, claimed, rev, sendErr, h.now()) + if err != nil { + h.logf("[mesh-mail] send: recording the outcome of %s: %v", q.Send, err) + } + switch action { + case actionAck: + if err := msg.Ack(); err != nil { + h.logf("[mesh-mail] send: acknowledging %s: %v", q.Send, err) + } + case actionRetry: + if err := msg.NakWithDelay(delay); err != nil { + h.logf("[mesh-mail] send: asking %s redelivered: %v", q.Send, err) + } + case actionTerm: + if err := msg.Term(); err != nil { + h.logf("[mesh-mail] send: terminating %s: %v", q.Send, err) + } + case actionLeave: + // The outcome's own write could not be kept, even retried on a changed revision (novox/hq review + // round 1 finding 5): nothing was emitted above, and nothing is done to the message either — left + // entirely to the bus, so a later redelivery finds the record still at its previous outcome. + } +} + +// attempt is the one handover to req.Account's outgoing server (novox/hq ADR 0436 clause 10): it reads the +// accounts setting and the account's own password fresh, never from the queue's message, and every way it +// cannot even be attempted (the account vanished, its password not given, the setting unreadable) is a +// *SendError classified NotTaken, exactly as SendMail's own would be — safe to retry. +func (h *senderHandler) attempt(req SendRequest) error { + accounts, err := h.Accounts() + if err != nil { + return &SendError{Outcome: NotTaken, Err: fmt.Errorf("the accounts setting cannot be read: %w", err)} + } + acc, found := accountByName(accounts, req.Account) + if !found { + return &SendError{Outcome: NotTaken, Err: fmt.Errorf("the account %s no longer exists", req.Account)} + } + pw, given, err := secret.Read(h.SecretsDir, acc.Secret()) + if err != nil { + return &SendError{Outcome: NotTaken, Err: fmt.Errorf("the account %s's password cannot be read: %w", acc.Name, err)} + } + if !given { + return &SendError{Outcome: NotTaken, Err: fmt.Errorf("the account %s has no password yet; the operator gives it at the desk", acc.Name)} + } + dial := h.Dial + if dial == nil { + dial = DialSMTP + } + return SendMail(dial, acc, pw, req.To, req.Subject, req.Body, h.now(), req.ID) +} diff --git a/modules/mesh-mail/cmd/mesh-mail/sender_integration_test.go b/modules/mesh-mail/cmd/mesh-mail/sender_integration_test.go new file mode 100644 index 00000000..e4f9a3ea --- /dev/null +++ b/modules/mesh-mail/cmd/mesh-mail/sender_integration_test.go @@ -0,0 +1,163 @@ +//go:build integration + +package main + +// mesh-mail send over a real bus (novox/hq ADR 0436 §7, issue 1195 AC2): a JetStream server of the test's +// own, nats-server v2.11.17, copied from modules/mesh-delivery/cmd/mesh-delivery/bus_test.go:52-58 +// (newTestBus) — the SDK's own testbus is internal, so this module keeps its own copy. Compiled by the +// merge gate's `go vet -tags integration`; never run on this machine (the context file's hard rule). + +import ( + "context" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +// mailSendApprovedSubject is the real wire subject the controller composes for mesh-mail's own stream +// mail-sends' one event, mail.send.approved: "mesh.stream." + module + ".event." + event (mesh-controller +// internal/broker/modstreams.go ModuleStreamSubject, moduleStreamPrefix). The sender's own Consume binds +// by stream and consumer name alone (mesh-sdk go/process queue.go), so this subject matters only to how a +// test builds the stream and publishes to it — it must match what the controller actually composes, or an +// integration test proves nothing about the real path (novox/hq review round 1, finding 7). +const mailSendApprovedSubject = "mesh.stream." + Module + ".event." + MailSendApproved + +// mailSendsStreamConfig is the work-queue stream mail-sends, composed exactly as the controller would +// (mesh-controller internal/catalogue/streams.go StreamDeclaration.Stream and +// internal/broker/modstreams.go ModuleStream.Stream): retention work-queue, discard new, module.json's +// max-age-days 1 and max-mib 64, and its "duplicates": 24 — in hours (novox/hq ADR 0436 clause 6) — which +// the controller turns into DuplicatesSeconds = Duplicates * 60 * 60, 24 hours here. +func mailSendsStreamConfig() jetstream.StreamConfig { + return jetstream.StreamConfig{ + Name: MailSendsStream, + Subjects: []string{mailSendApprovedSubject}, + Retention: jetstream.WorkQueuePolicy, + Discard: jetstream.DiscardNew, + MaxAge: 24 * time.Hour, + MaxBytes: 64 << 20, + Duplicates: 24 * time.Hour, + } +} + +// mailSendsConsumerConfig is the one durable consumer the controller composes for a process's bus.queues +// (mesh-controller internal/broker/modstreams.go QueueConsumer): AckWaitSeconds 30, MaxDeliver from the +// stream's own max-deliveries (module.json: 5, MostAttempts here). +func mailSendsConsumerConfig(durable string) jetstream.ConsumerConfig { + return jetstream.ConsumerConfig{Durable: durable, AckPolicy: jetstream.AckExplicitPolicy, AckWait: 30 * time.Second, MaxDeliver: MostAttempts} +} + +// newTestBus starts a JetStream-enabled nats-server for the test alone, on a random port with its own temp +// store (modules/mesh-delivery/cmd/mesh-delivery/bus_test.go:52-58). +func newTestBus(t *testing.T) string { + t.Helper() + opts := &server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, JetStream: true, StoreDir: t.TempDir(), + NoLog: true, NoSigs: true} + s, err := server.NewServer(opts) + if err != nil { + t.Fatalf("a bus for this test could not be made: %v", err) + } + go s.Start() + if !s.ReadyForConnections(30 * time.Second) { + s.Shutdown() + t.Fatal("a bus for this test did not come up within 30s") + } + t.Cleanup(func() { s.Shutdown(); s.WaitForShutdown() }) + return s.ClientURL() +} + +// 1195/AC2: given the mail-sends work-queue stream and its consumer _mesh-mail_send, when the +// process runs with a handler and a message is published, then the handler receives it; when the process +// gets SIGTERM (here, its context cancelled, the same signal main.go's send() reacts to) it stops without +// acknowledging a message in hand. +func TestSendConsumesMailSendsAndStopsCleanlyOnSignal(t *testing.T) { + url := newTestBus(t) + ctx := context.Background() + + nc, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + if _, err := js.CreateStream(ctx, mailSendsStreamConfig()); err != nil { + t.Fatal(err) + } + + account := process.Account{URL: url, Node: "g14", Module: Module, User: "g14." + Module + "." + SendProcessID, Password: "pw"} + if _, err := js.CreateConsumer(ctx, MailSendsStream, mailSendsConsumerConfig(account.QueueConsumerName())); err != nil { + t.Fatal(err) + } + + received := make(chan *process.Message, 1) + runCtx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- runSend(runCtx, account, func(m *process.Message) { received <- m }) }() + + body := []byte(`{"send":"send-1","account":"mesh","ask":"send-1"}`) + // Published twice under the same message id (the send id, as the tools' own publish path always uses, + // novox/hq ADR 0436 clause 8): the stream's own 24-hour duplicate window keeps it one message, proving + // the dedup this test's stream config actually carries (novox/hq review round 1, finding 7). + if _, err := js.Publish(ctx, mailSendApprovedSubject, body, jetstream.WithMsgID("send-1")); err != nil { + t.Fatal(err) + } + if _, err := js.Publish(ctx, mailSendApprovedSubject, body, jetstream.WithMsgID("send-1")); err != nil { + t.Fatal(err) + } + + select { + case msg := <-received: + if msg.Subject != mailSendApprovedSubject { + t.Fatalf("received on %q, want %q", msg.Subject, mailSendApprovedSubject) + } + // Left exactly here, never acknowledged: the handler's own fate-deciding (Ack, NakWithDelay, Term) is + // issues 1196-1198's, not this skeleton's. + case <-time.After(10 * time.Second): + t.Fatal("the handler never received the published message") + } + select { + case <-received: + t.Fatal("the duplicate publish under the same message id was delivered a second time") + case <-time.After(500 * time.Millisecond): + // The duplicate was dropped by the stream's own window, as it should be. + } + + cancel() // the signal send() itself reacts to (SIGTERM, or ^C) + select { + case err := <-done: + if err == nil { + t.Fatal("runSend returned nil, want ctx.Err() once its context was cancelled") + } + case <-time.After(10 * time.Second): + t.Fatal("runSend did not stop once its context was cancelled") + } + + // The message was never acknowledged: it is still pending (or ack-pending) on the same durable consumer, + // read fresh over a new connection. + nc2, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer nc2.Close() + js2, err := jetstream.New(nc2) + if err != nil { + t.Fatal(err) + } + cons, err := js2.Consumer(ctx, MailSendsStream, account.QueueConsumerName()) + if err != nil { + t.Fatal(err) + } + info, err := cons.Info(ctx) + if err != nil { + t.Fatal(err) + } + if info.NumPending == 0 && info.NumAckPending == 0 { + t.Fatal("the message was acknowledged despite the process stopping mid-handle") + } +} diff --git a/modules/mesh-mail/cmd/mesh-mail/sender_test.go b/modules/mesh-mail/cmd/mesh-mail/sender_test.go new file mode 100644 index 00000000..5df2f9cb --- /dev/null +++ b/modules/mesh-mail/cmd/mesh-mail/sender_test.go @@ -0,0 +1,80 @@ +package main + +// mesh-mail send (novox/hq ADR 0436 §7, issue 1195): refuses to start naming what is missing, never +// attempting to reach a bus when it cannot even read its own account or the accounts setting. + +import ( + "path/filepath" + "testing" +) + +// 1195/AC1: given `mesh-mail send` with MESH_MAIL_SECRETS holding broker-send.secret in the controller's +// form, when it starts, then it reads the account and the accounts file, and refuses to start naming what +// is missing when either is absent. +func TestSendRefusesToStartNamingWhatIsMissing(t *testing.T) { + validAccount := `{"url": "nats://127.0.0.1:4222", "node": "g14", "module": "mesh-mail", ` + + `"user": "g14.mesh-mail.send", "password": "pw"}` + + t.Run("no secrets directory", func(t *testing.T) { + dir := t.TempDir() + env := Env{Accounts: filepath.Join(dir, "accounts.json")} + writeFile(t, env.Accounts, stageOne) + if code := send(env); code == 0 { + t.Fatal("send started with no secrets directory at all") + } + }) + + t.Run("no accounts file", func(t *testing.T) { + dir := t.TempDir() + env := Env{Accounts: filepath.Join(dir, "accounts.json"), Secrets: dir} + writeFile(t, filepath.Join(dir, "broker-send.secret"), validAccount) + if code := send(env); code == 0 { + t.Fatal("send started with no accounts setting") + } + }) + + t.Run("accounts file is not the module's", func(t *testing.T) { + dir := t.TempDir() + env := Env{Accounts: filepath.Join(dir, "accounts.json"), Secrets: dir} + writeFile(t, env.Accounts, `not json`) + writeFile(t, filepath.Join(dir, "broker-send.secret"), validAccount) + if code := send(env); code == 0 { + t.Fatal("send started with an accounts setting it cannot read") + } + }) + + t.Run("no own account delivered yet", func(t *testing.T) { + dir := t.TempDir() + env := Env{Accounts: filepath.Join(dir, "accounts.json"), Secrets: dir} + writeFile(t, env.Accounts, stageOne) + // broker-send.secret is not written at all. + if code := send(env); code == 0 { + t.Fatal("send started with no own account delivered") + } + }) + + t.Run("own account missing a field", func(t *testing.T) { + dir := t.TempDir() + env := Env{Accounts: filepath.Join(dir, "accounts.json"), Secrets: dir} + writeFile(t, env.Accounts, stageOne) + writeFile(t, filepath.Join(dir, "broker-send.secret"), `{"url": "nats://127.0.0.1:4222", "node": "g14", "module": "mesh-mail"}`) + code := send(env) + if code == 0 { + t.Fatal("send started with an own account missing its user and password") + } + }) +} + +// The account file's path is exactly what secret.Path gives for the member "broker-", inside +// the directory MESH_MAIL_SECRETS already names — no secret of its own needs declaring in the manifest for +// it (novox/hq ADR 0436 §2: the controller delivers it automatically for every process with a bus). +func TestSendProcessIDNamesItsOwnAccountFile(t *testing.T) { + if SendProcessID != "send" { + t.Fatalf("SendProcessID: %q; module.json's resource id is \"send\"", SendProcessID) + } + // mesh-controller's own StreamBusName: "mail-sends" capitalised, its hyphen an underscore (review round + // 1, finding 7 — confirmed against cmd/mesh-controller/process_live_test.go's own fixture). + if MailSendsStream != "MAIL_SENDS" { + t.Fatalf("MailSendsStream: %q, want MAIL_SENDS", MailSendsStream) + } +} diff --git a/modules/mesh-mail/cmd/mesh-mail/smtp.go b/modules/mesh-mail/cmd/mesh-mail/smtp.go index bc12d406..0d542ba6 100644 --- a/modules/mesh-mail/cmd/mesh-mail/smtp.go +++ b/modules/mesh-mail/cmd/mesh-mail/smtp.go @@ -8,10 +8,12 @@ package main import ( "crypto/tls" + "errors" "fmt" "mime" "net/mail" "net/smtp" + "net/textproto" "strings" "time" "unicode" @@ -55,10 +57,63 @@ func DialSMTP(server Server) (*smtp.Client, error) { } } -// SendMail sends one message through the account's outgoing server (novox/hq ADR 0361 §1, §2): signs in with the -// password given, states every recipient, and writes the message once. Never logs the password or the body; an -// error names the server and what it said, never either. -func SendMail(dial SMTPDialer, a Account, password string, to []string, subject, body string, now time.Time) error { +// SendOutcome classifies how a send attempt ended (novox/hq ADR 0436 clause 10), so the sender knows +// whether a retry is safe, certainly wrong, or neither: +// +// - NotTaken: no connection, the sign-in refused, or a 4xx reply to MAIL, RCPT or DATA, or to the end of +// data — the server provably never took the mail; retrying is safe. +// - Refused: a 5xx reply before or at the end of data — the server provably refused the mail; never +// retried. +// - NotProven: the connection was lost after the data was written, with no reply at all — the mail may or +// may not have gone out; never retried automatically either way. +type SendOutcome int + +const ( + NotTaken SendOutcome = iota + Refused + NotProven +) + +func (o SendOutcome) String() string { + switch o { + case Refused: + return "refused" + case NotProven: + return "not proven" + default: + return "not taken" + } +} + +// SendError classifies a SendMail failure (novox/hq ADR 0436 clause 10): Outcome says which, and the error +// itself (Unwrap) carries the server's own words where there were any, never the password or the body. +type SendError struct { + Outcome SendOutcome + Err error +} + +func (e *SendError) Error() string { return e.Err.Error() } +func (e *SendError) Unwrap() error { return e.Err } + +// classifyReply is NotTaken for a 4xx reply, Refused for a 5xx, and NotTaken for anything else (no +// connection, or the connection lost before a reply was read at all) — the safe default everywhere but the +// one case clause 10 names its own outcome for: the connection lost after the data was written, which the +// caller classifies as NotProven itself, never through this. +func classifyReply(err error) SendOutcome { + var proto *textproto.Error + if errors.As(err, &proto) && proto.Code >= 500 { + return Refused + } + return NotTaken +} + +// SendMail sends one message through the account's outgoing server (novox/hq ADR 0361 §1, §2; ADR 0436 +// clause 10): signs in with the password given, states every recipient, and writes the message once, its +// Message-ID naming sendID so the mail is never sent twice within the day (clause 10's own guarantee, kept +// by the sender's compare-and-set claim and the bus's duplicate window on the id alone — this is the +// mail's own proof of it). Never logs the password or the body; an error names the server and what it +// said, never either, and is always a *SendError (classifyReply, or NotProven below). +func SendMail(dial SMTPDialer, a Account, password string, to []string, subject, body string, now time.Time, sendID string) error { if a.Outgoing == nil { return fmt.Errorf("the account %s has no outgoing server", a.Name) } @@ -76,44 +131,63 @@ func SendMail(dial SMTPDialer, a Account, password string, to []string, subject, } client, err := dial(*a.Outgoing) if err != nil { - return err + return &SendError{Outcome: NotTaken, Err: err} } defer client.Close() from := a.SignInAs() auth := smtp.PlainAuth("", from, password, a.Outgoing.Host) if err := client.Auth(auth); err != nil { - return fmt.Errorf("signing in to %s failed: %w", a.Outgoing.Address(), err) + // A refused sign-in is its own case, never classified by a reply code: the server never took the + // mail either way (novox/hq ADR 0436 clause 10). + return &SendError{Outcome: NotTaken, Err: fmt.Errorf("signing in to %s failed: %w", a.Outgoing.Address(), err)} } if err := client.Mail(from); err != nil { - return fmt.Errorf("the outgoing server refused the sender: %w", err) + return &SendError{Outcome: classifyReply(err), Err: fmt.Errorf("the outgoing server refused the sender: %w", err)} } for _, rcpt := range rcpts { if err := client.Rcpt(rcpt); err != nil { - return fmt.Errorf("the outgoing server refused the recipient %s: %w", rcpt, err) + return &SendError{Outcome: classifyReply(err), Err: fmt.Errorf("the outgoing server refused the recipient %s: %w", rcpt, err)} } } w, err := client.Data() if err != nil { - return fmt.Errorf("the outgoing server refused the message: %w", err) + return &SendError{Outcome: classifyReply(err), Err: fmt.Errorf("the outgoing server refused the message: %w", err)} } - if _, err := w.Write(composeMessage(from, headers, subject, body, now)); err != nil { - return fmt.Errorf("the message could not be written: %w", err) + if _, err := w.Write(composeMessage(from, headers, subject, body, now, sendID)); err != nil { + // The data was not written whole: never "after the data was written" (clause 10's not-proven case), + // so this is not taken, not not-proven — nothing coherent reached the server. + return &SendError{Outcome: NotTaken, Err: fmt.Errorf("the message could not be written: %w", err)} } if err := w.Close(); err != nil { - return fmt.Errorf("the outgoing server did not accept the message: %w", err) + // Close is what sends the terminating "." and reads the server's one reply to the whole message + // (SMTP's "end of data", clause 10): a real reply here classifies exactly as MAIL or RCPT's does; no + // reply at all (the connection lost after the data was written) is not proven — never retried + // automatically either way. + outcome := NotProven + var proto *textproto.Error + if errors.As(err, &proto) { + outcome = classifyReply(err) + } + return &SendError{Outcome: outcome, Err: fmt.Errorf("the outgoing server did not accept the message: %w", err)} } - return client.Quit() + // The message is already accepted (Close's 250 reply); Quit is a courtesy goodbye, and its own failure + // here never turns an accepted send into a failed one. + _ = client.Quit() + return nil } // composeMessage is a plain-text message, its lines CRLF-terminated as SMTP takes them. to is the recipients // already formatted (mail.Address.String()), the same ones Rcpt was given, never the raw strings a caller gave. -func composeMessage(from string, to []string, subject, body string, now time.Time) []byte { +// Its Message-ID names sendID (novox/hq ADR 0436 clause 10), so the mail carries the proof of which send it +// is, at @. +func composeMessage(from string, to []string, subject, body string, now time.Time, sendID string) []byte { var b strings.Builder header := func(k, v string) { b.WriteString(k + ": " + v + "\r\n") } header("From", from) header("To", strings.Join(to, ", ")) header("Subject", encodeHeader(subject)) header("Date", now.Format(time.RFC1123Z)) + header("Message-Id", "<"+sendID+"@"+messageIDDomain(from)+">") header("MIME-Version", "1.0") header("Content-Type", "text/plain; charset=utf-8") header("Content-Transfer-Encoding", "8bit") @@ -122,6 +196,20 @@ func composeMessage(from string, to []string, subject, body string, now time.Tim return []byte(b.String()) } +// messageIDDomain is the domain a Message-ID is minted under: the part of from after its @, as a bare +// address or parsed out of "Name "; "localhost" when from carries none (never empty, which is not a +// valid Message-ID). +func messageIDDomain(from string) string { + addr := from + if parsed, err := mail.ParseAddress(from); err == nil { + addr = parsed.Address + } + if _, domain, found := strings.Cut(addr, "@"); found && domain != "" { + return domain + } + return "localhost" +} + // encodeHeader word-encodes a header's value when it holds more than ASCII; a header a phone never needs to show. func encodeHeader(s string) string { for _, r := range s { diff --git a/modules/mesh-mail/cmd/mesh-mail/smtp_classify_test.go b/modules/mesh-mail/cmd/mesh-mail/smtp_classify_test.go new file mode 100644 index 00000000..97bc90f5 --- /dev/null +++ b/modules/mesh-mail/cmd/mesh-mail/smtp_classify_test.go @@ -0,0 +1,262 @@ +package main + +// SendMail classifies every failure (novox/hq ADR 0436 clause 10, issue 1197): not taken (no connection, a +// refused sign-in, or a 4xx reply to MAIL, RCPT or DATA or to the end of data), refused (a 5xx before or at +// the end of data), or not proven (the connection lost after the data was written, before any reply). Its +// Message-ID names the send id. + +import ( + "bufio" + "errors" + "fmt" + "net" + "net/smtp" + "strings" + "sync/atomic" + "testing" +) + +// step is one scripted SMTP reply, or a drop (the connection closed instead of a reply). +type step struct { + drop bool + code int + msg string +} + +func reply(code int, msg string) step { return step{code: code, msg: msg} } +func drop() step { return step{drop: true} } + +// scriptedSMTPServer answers EHLO, AUTH, MAIL, RCPT, DATA's own 354 and the end-of-data reply each with a +// step a test sets, defaulting to the ordinary success sequence — so a test overrides exactly the one stage +// its case is about. +type scriptedSMTPServer struct { + ln net.Listener + ehlo, auth, mail, rcpt, dataOpen, dataEnd step + received int32 // atomic: how many full DATA bodies this server has read +} + +func newScriptedSMTPServer(t *testing.T) *scriptedSMTPServer { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + s := &scriptedSMTPServer{ln: ln, ehlo: reply(250, "fake.local"), auth: reply(235, "authenticated"), + mail: reply(250, "OK"), rcpt: reply(250, "OK"), dataOpen: reply(354, "go ahead"), dataEnd: reply(250, "queued")} + go s.serve() + t.Cleanup(func() { _ = ln.Close() }) + return s +} + +func (s *scriptedSMTPServer) addr() string { return s.ln.Addr().String() } + +// receivedCount is how many full DATA bodies this server has read, for a test to prove nothing was +// received at all (novox/hq review round 1 finding 6). +func (s *scriptedSMTPServer) receivedCount() int { return int(atomic.LoadInt32(&s.received)) } + +func (s *scriptedSMTPServer) dialer() SMTPDialer { + addr := s.addr() + return func(Server) (*smtp.Client, error) { return smtp.Dial(addr) } +} + +func (s *scriptedSMTPServer) send(conn net.Conn, st step) bool { + if st.drop { + _ = conn.Close() + return false + } + fmt.Fprintf(conn, "%d %s\r\n", st.code, st.msg) + return true +} + +func (s *scriptedSMTPServer) serve() { + for { + conn, err := s.ln.Accept() + if err != nil { + return + } + go s.handle(conn) + } +} + +func (s *scriptedSMTPServer) handle(conn net.Conn) { + defer conn.Close() + r := bufio.NewReader(conn) + fmt.Fprintf(conn, "220 fake.local ESMTP\r\n") + for { + line, err := r.ReadString('\n') + if err != nil { + return + } + upper := strings.ToUpper(strings.TrimRight(line, "\r\n")) + switch { + case strings.HasPrefix(upper, "EHLO"), strings.HasPrefix(upper, "HELO"): + if !s.send(conn, s.ehlo) { + return + } + case strings.HasPrefix(upper, "AUTH"): + if !s.send(conn, s.auth) { + return + } + case strings.HasPrefix(upper, "MAIL"): + if !s.send(conn, s.mail) { + return + } + case strings.HasPrefix(upper, "RCPT"): + if !s.send(conn, s.rcpt) { + return + } + case upper == "DATA": + if !s.send(conn, s.dataOpen) { + return + } + if s.dataOpen.code >= 400 { + continue // not entering data mode; the client ends the transaction itself + } + for { + dl, err := r.ReadString('\n') + if err != nil { + return + } + if dl == ".\r\n" || dl == ".\n" { + break + } + } + atomic.AddInt32(&s.received, 1) // a full message body was read, whatever dataEnd then says + s.send(conn, s.dataEnd) // a drop here is "the connection lost after the data was written" + return + case upper == "QUIT": + fmt.Fprintf(conn, "221 bye\r\n") + return + default: + fmt.Fprintf(conn, "500 unrecognized\r\n") + } + } +} + +// classifyTestAccount is an account whose outgoing server is the given address: Host is its loopback +// literal, which PlainAuth itself accepts without TLS (its own isLocalhost check). +func classifyTestAccount(addr string) Account { + host, portStr, _ := net.SplitHostPort(addr) + var port int + _, _ = fmt.Sscanf(portStr, "%d", &port) + return Account{Name: "mesh", Address: "jschoubben@novox.be", Outgoing: &Server{Host: host, Port: port, Security: "tls"}} +} + +func outcomeOf(t *testing.T, err error) SendOutcome { + t.Helper() + var se *SendError + if !errors.As(err, &se) { + t.Fatalf("SendMail's error is not a *SendError: %v (%T)", err, err) + } + return se.Outcome +} + +// 1197/AC1: given no connection, a refused sign-in, or a 4xx reply to MAIL, RCPT or DATA or to the end of +// data, when SendMail runs, then it says not taken with the server's words. +func TestSendMailSaysNotTaken(t *testing.T) { + t.Run("no connection", func(t *testing.T) { + s := newScriptedSMTPServer(t) + addr := s.addr() + s.ln.Close() // nothing listens by the time SendMail dials + acc := classifyTestAccount(addr) + dial := func(Server) (*smtp.Client, error) { return smtp.Dial(addr) } + err := SendMail(dial, acc, "pw", []string{"a@b.c"}, "s", "body", judgedAt, "send-1") + if err == nil || outcomeOf(t, err) != NotTaken { + t.Fatalf("err %v, want NotTaken", err) + } + }) + t.Run("a refused sign-in", func(t *testing.T) { + s := newScriptedSMTPServer(t) + s.auth = reply(535, "authentication failed") + acc := classifyTestAccount(s.addr()) + err := SendMail(s.dialer(), acc, "pw", []string{"a@b.c"}, "s", "body", judgedAt, "send-1") + if err == nil || outcomeOf(t, err) != NotTaken || !strings.Contains(err.Error(), "signing in") { + t.Fatalf("err %v, want NotTaken naming the sign-in", err) + } + }) + for _, c := range []struct { + name string + set func(*scriptedSMTPServer) + }{ + {"a 4xx to MAIL", func(s *scriptedSMTPServer) { s.mail = reply(451, "try again later") }}, + {"a 4xx to RCPT", func(s *scriptedSMTPServer) { s.rcpt = reply(450, "mailbox busy") }}, + {"a 4xx to DATA", func(s *scriptedSMTPServer) { s.dataOpen = reply(452, "insufficient storage") }}, + {"a 4xx at the end of data", func(s *scriptedSMTPServer) { s.dataEnd = reply(451, "local error") }}, + } { + t.Run(c.name, func(t *testing.T) { + s := newScriptedSMTPServer(t) + c.set(s) + acc := classifyTestAccount(s.addr()) + err := SendMail(s.dialer(), acc, "pw", []string{"a@b.c"}, "s", "body", judgedAt, "send-1") + if err == nil || outcomeOf(t, err) != NotTaken { + t.Fatalf("err %v, want NotTaken", err) + } + }) + } +} + +// 1197/AC2: given a 5xx reply before or at the end of data, then it says refused with the server's words. +func TestSendMailSaysRefused(t *testing.T) { + for _, c := range []struct { + name string + set func(*scriptedSMTPServer) + }{ + {"a 5xx to MAIL", func(s *scriptedSMTPServer) { s.mail = reply(550, "no such sender") }}, + {"a 5xx to RCPT", func(s *scriptedSMTPServer) { s.rcpt = reply(550, "no such recipient") }}, + {"a 5xx to DATA", func(s *scriptedSMTPServer) { s.dataOpen = reply(554, "transaction failed") }}, + {"a 5xx at the end of data", func(s *scriptedSMTPServer) { s.dataEnd = reply(552, "quota exceeded") }}, + } { + t.Run(c.name, func(t *testing.T) { + s := newScriptedSMTPServer(t) + c.set(s) + acc := classifyTestAccount(s.addr()) + err := SendMail(s.dialer(), acc, "pw", []string{"a@b.c"}, "s", "body", judgedAt, "send-1") + if err == nil || outcomeOf(t, err) != Refused { + t.Fatalf("err %v, want Refused", err) + } + }) + } +} + +// 1197/AC3: given the connection lost after the data was written and before any reply, then it says not +// proven. +func TestSendMailSaysNotProvenWhenTheConnectionIsLostAfterTheDataWasWritten(t *testing.T) { + s := newScriptedSMTPServer(t) + s.dataEnd = drop() + acc := classifyTestAccount(s.addr()) + err := SendMail(s.dialer(), acc, "pw", []string{"a@b.c"}, "s", "body", judgedAt, "send-1") + if err == nil || outcomeOf(t, err) != NotProven { + t.Fatalf("err %v, want NotProven", err) + } +} + +// 1197/AC4: given a send id, when the message is composed, then it carries a Message-ID naming the send id +// (@). Proved both in isolation (composeMessage) and over the wire +// (SendMail, with the existing fakeSMTPServer, which keeps the body it received). +func TestComposeMessageCarriesTheSendIDInItsMessageID(t *testing.T) { + raw := composeMessage("jschoubben@novox.be", []string{"a@b.c"}, "s", "body", judgedAt, "send-abc123") + if !strings.Contains(string(raw), "Message-Id: ") { + t.Fatalf("the message does not carry the send id's Message-ID:\n%s", raw) + } +} + +// composeMessage falls back to "localhost" when From carries no domain at all (never an empty or +// malformed Message-ID). +func TestComposeMessageFallsBackToLocalhostWithNoDomain(t *testing.T) { + raw := composeMessage("postmaster", nil, "s", "body", judgedAt, "send-1") + if !strings.Contains(string(raw), "Message-Id: ") { + t.Fatalf("message: %s", raw) + } +} + +func TestSendMailCarriesTheMessageIDOverTheWire(t *testing.T) { + fake := newFakeSMTPServer(t) + acc := classifyTestAccount(fake.addr()) + if err := SendMail(fake.dialer(), acc, "pw", []string{"a@b.c"}, "s", "body", judgedAt, "send-xyz"); err != nil { + t.Fatalf("SendMail: %v", err) + } + got := fake.last() + if !strings.Contains(got.data, "Message-Id: ") { + t.Fatalf("the message sent does not carry the send id's Message-ID:\n%s", got.data) + } +} diff --git a/modules/mesh-mail/cmd/mesh-mail/tell_test.go b/modules/mesh-mail/cmd/mesh-mail/tell_test.go new file mode 100644 index 00000000..2ecc9f36 --- /dev/null +++ b/modules/mesh-mail/cmd/mesh-mail/tell_test.go @@ -0,0 +1,264 @@ +package main + +// The tells to the operator, overdue health and mail_send_status's words (novox/hq ADR 0436 clauses 11, +// 12, issue 1199). + +import ( + "errors" + "strings" + "testing" + "time" +) + +// memTell collects every headline a TellOperator was called with, optionally refusing the first n calls +// (novox/hq issue 1199). +type memTell struct { + headlines []string + refuseN int +} + +func (m *memTell) tell(headline string) error { + if m.refuseN > 0 { + m.refuseN-- + return errors.New("the channel is down") + } + m.headlines = append(m.headlines, headline) + return nil +} + +func tellTestSend(t *testing.T, sends *memSends, outcome string) SendRequest { + t.Helper() + req := SendRequest{ID: newSendID(), Account: "mesh", To: []string{"ops@example.org"}, At: judgedAt} + req.Decided = &SendDecision{At: judgedAt, Outcome: outcome} + if _, err := sends.Create(req.ID, req); err != nil { + t.Fatal(err) + } + if err := addTellPending(sends, req.ID); err != nil { + t.Fatal(err) + } + return req +} + +// 1199/AC1: given a send recorded failed, refused or not proven after the operator's Approve, when the +// sender looks, then it sets tell to telling by compare-and-set, calls seat:operator-channel.tell once +// (the headline naming the account and the reason, or for not proven that it may not have been sent; no +// address in it), and on the channel taking it sets told. A refused or unanswered tell leaves telling, and +// the next look tells only records left telling. A sent send is never told. +func TestTellSendTellsOnceAndSettlesToldOnlyOnceTheChannelTakesIt(t *testing.T) { + sends := newMemSends() + req := tellTestSend(t, sends, "failed: 550 no such mailbox") + tell := &memTell{} + + if err := tellSend(sends, tell.tell, req.ID, judgedAt.Add(time.Minute)); err != nil { + t.Fatalf("telling: %v", err) + } + if len(tell.headlines) != 1 { + t.Fatalf("told %d time(s): %v", len(tell.headlines), tell.headlines) + } + h := tell.headlines[0] + if !strings.Contains(h, "mesh") || !strings.Contains(h, "550 no such mailbox") { + t.Fatalf("headline: %q", h) + } + if strings.Contains(h, "@") { + t.Fatalf("the headline carries an address: %q", h) + } + kept, _, ok, err := getSend(sends, req.ID) + if err != nil || !ok || kept.Decided == nil || kept.Decided.Tell != "told" { + t.Fatalf("tell not settled told: %v %v %+v", err, ok, kept.Decided) + } + ids, err := tellPendingIDs(sends) + if err != nil || len(ids) != 0 { + t.Fatalf("still indexed after being told: %v %v", err, ids) + } + + // A second look at an already-told send tells nothing more. + if err := tellSend(sends, tell.tell, req.ID, judgedAt.Add(2*time.Minute)); err != nil { + t.Fatalf("telling again: %v", err) + } + if len(tell.headlines) != 1 { + t.Fatalf("told again: %v", tell.headlines) + } +} + +func TestTellSendLeavesTellingOnARefusedChannelAndTriesAgain(t *testing.T) { + sends := newMemSends() + req := tellTestSend(t, sends, "not proven") + tell := &memTell{refuseN: 1} + + if err := tellSend(sends, tell.tell, req.ID, judgedAt.Add(time.Minute)); err != nil { + t.Fatalf("telling: %v", err) + } + kept, _, ok, err := getSend(sends, req.ID) + if err != nil || !ok || kept.Decided == nil || kept.Decided.Tell != "telling" { + t.Fatalf("not left telling: %v %v %+v", err, ok, kept.Decided) + } + ids, err := tellPendingIDs(sends) + if err != nil || len(ids) != 1 { + t.Fatalf("no longer indexed despite the refusal: %v %v", err, ids) + } + + // The next look succeeds. + if err := tellSend(sends, tell.tell, req.ID, judgedAt.Add(2*time.Minute)); err != nil { + t.Fatalf("telling again: %v", err) + } + if len(tell.headlines) != 1 { + t.Fatalf("told: %v", tell.headlines) + } + if !strings.Contains(tell.headlines[0], "may not have been sent") { + t.Fatalf("not-proven headline: %q", tell.headlines[0]) + } +} + +func TestTellSendNeverTellsForASentSend(t *testing.T) { + sends := newMemSends() + req := SendRequest{ID: newSendID(), Account: "mesh", At: judgedAt} + req.Decided = &SendDecision{At: judgedAt, Outcome: "sent"} + if _, err := sends.Create(req.ID, req); err != nil { + t.Fatal(err) + } + tell := &memTell{} + if err := tellSend(sends, tell.tell, req.ID, judgedAt.Add(time.Minute)); err != nil { + t.Fatalf("telling: %v", err) + } + if len(tell.headlines) != 0 { + t.Fatalf("a sent send was told: %v", tell.headlines) + } +} + +// review round 1 finding 3: a send indexed for a tell before its own outcome write has landed (recordOutcome +// and claimQueuedSend now index first) must not be dropped by a look that finds it still sending, queued or +// retrying — it is left in the index for the next look, once the real outcome lands. +func TestTellSendKeepsAnEntryWhoseRecordIsStillInFlight(t *testing.T) { + for _, outcome := range []string{"sending", "queued", "retrying"} { + sends := newMemSends() + req := SendRequest{ID: newSendID(), Account: "mesh", At: judgedAt} + req.Decided = &SendDecision{At: judgedAt, Outcome: outcome} + if _, err := sends.Create(req.ID, req); err != nil { + t.Fatal(err) + } + if err := addTellPending(sends, req.ID); err != nil { + t.Fatal(err) + } + tell := &memTell{} + if err := tellSend(sends, tell.tell, req.ID, judgedAt.Add(time.Minute)); err != nil { + t.Fatalf("%s: telling: %v", outcome, err) + } + if len(tell.headlines) != 0 { + t.Fatalf("%s: told before its outcome landed: %v", outcome, tell.headlines) + } + ids, err := tellPendingIDs(sends) + if err != nil || len(ids) != 1 { + t.Fatalf("%s: dropped from the index while still in flight: %v %v", outcome, err, ids) + } + } +} + +// 1199/AC2: given a send queued more than 10 minutes ago, or retrying 10 minutes past its next attempt, +// when mail_health runs (Overdue), then it names the send; a send waiting for its next attempt is never +// overdue. +func TestOverdueFindsAQueuedOrRetryingSendPastTenMinutes(t *testing.T) { + sends := newMemSends() + queueTestSend := func(outcome string, at, next time.Time) string { + req := SendRequest{ID: newSendID(), Account: "mesh", At: judgedAt} + req.Decided = &SendDecision{At: at, Outcome: outcome, NextAttempt: next} + if _, err := sends.Create(req.ID, req); err != nil { + t.Fatal(err) + } + if err := addSenderPending(sends, req.ID); err != nil { + t.Fatal(err) + } + return req.ID + } + now := judgedAt + overdueQueued := queueTestSend("queued", now.Add(-11*time.Minute), time.Time{}) + freshQueued := queueTestSend("queued", now.Add(-5*time.Minute), time.Time{}) + overdueRetrying := queueTestSend("retrying", now.Add(-20*time.Minute), now.Add(-11*time.Minute)) + notYetDueRetrying := queueTestSend("retrying", now.Add(-5*time.Minute), now.Add(3*time.Minute)) + sendingNotOverdue := queueTestSend("sending", now.Add(-20*time.Minute), time.Time{}) + + overdue, err := Overdue(sends, now) + if err != nil { + t.Fatalf("Overdue: %v", err) + } + var ids []string + for _, r := range overdue { + ids = append(ids, r.ID) + } + want := map[string]bool{overdueQueued: true, overdueRetrying: true} + if len(ids) != len(want) { + t.Fatalf("overdue: %v, want exactly %v", ids, want) + } + for _, id := range ids { + if !want[id] { + t.Fatalf("unexpectedly overdue: %s (fresh %s, not-yet-due %s, sending %s)", id, freshQueued, notYetDueRetrying, sendingNotOverdue) + } + } +} + +// 1199/AC3: given an overdue send, when the tools' second ticker looks, then they tell once through +// waits-tell (telling then told), separate from the sender's own tell. +func TestTellWaitsTellsOnceThroughItsOwnField(t *testing.T) { + sends := newMemSends() + req := SendRequest{ID: newSendID(), Account: "mesh", At: judgedAt} + req.Decided = &SendDecision{At: judgedAt.Add(-20 * time.Minute), Outcome: "queued"} + if _, err := sends.Create(req.ID, req); err != nil { + t.Fatal(err) + } + tell := &memTell{} + if err := tellWaits(sends, tell.tell, req.ID, judgedAt); err != nil { + t.Fatalf("telling waits: %v", err) + } + if len(tell.headlines) != 1 || !strings.Contains(tell.headlines[0], "waits") { + t.Fatalf("waits-tell: %v", tell.headlines) + } + kept, _, ok, err := getSend(sends, req.ID) + if err != nil || !ok || kept.Decided == nil || kept.Decided.WaitsTell != "told" || kept.Decided.Tell != "" { + t.Fatalf("waits-tell not settled apart from the sender's own tell: %v %v %+v", err, ok, kept.Decided) + } + // Told once: a second look tells nothing more. + if err := tellWaits(sends, tell.tell, req.ID, judgedAt.Add(time.Minute)); err != nil { + t.Fatalf("telling waits again: %v", err) + } + if len(tell.headlines) != 1 { + t.Fatalf("told again: %v", tell.headlines) + } +} + +// 1199/AC4: given each state, when mail_send_status reads it (reportDecision), then it says: waiting for +// the operator (covered by send_test.go's own asking tests, which read ReadSendStatus before any decision +// exists — reportDecision itself only ever sees an already-decided SendDecision), approved and waiting for +// the sender, being sent, waiting for its next attempt, sent, failed with its reason, refused, not proven, +// declined, or expired — checking both the status word and the words that actually say which of these it +// is (novox/hq review round 1 finding 6: checking status alone proves too little). +func TestReportDecisionCoversTheWholeStatusVocabulary(t *testing.T) { + for name, c := range map[string]struct { + d SendDecision + wantStatus string + wantInWords string + }{ + // There is no "approved" state in a send's life: Decide's one compare-and-set write moves it + // straight from undecided to "queued" (novox/hq ADR 0436 clause 8, as decided to replace the + // two-state design, review round 1). + "queued": {SendDecision{At: judgedAt, Outcome: "queued"}, "waiting", "approved and waiting for the sender"}, + "being sent": {SendDecision{At: judgedAt, Outcome: "sending"}, "waiting", "being sent"}, + "retrying": {SendDecision{At: judgedAt, Outcome: "retrying", NextAttempt: judgedAt.Add(time.Minute)}, "waiting", "waiting for its next attempt"}, + "sent": {SendDecision{At: judgedAt, Outcome: "sent"}, "sent", "sent on the operator's Approve"}, + "failed with its reason": {SendDecision{At: judgedAt, Outcome: "failed: 550 no such mailbox"}, "failed", "550 no such mailbox"}, + "refused": {SendDecision{At: judgedAt, Outcome: "refused: the warrant does not bind the request"}, "refused", "the warrant does not bind the request"}, + "not proven": {SendDecision{At: judgedAt, Outcome: "not proven"}, "not proven", "not proven"}, + "declined": {SendDecision{At: judgedAt, Outcome: "declined"}, "declined", "the operator declined"}, + "expired (never heard)": {SendDecision{At: judgedAt, Outcome: "ended: never heard"}, "expired", "never heard"}, + } { + t.Run(name, func(t *testing.T) { + out := map[string]any{} + reportDecision(out, c.d, judgedAt.Add(time.Minute)) + if out["status"] != c.wantStatus { + t.Fatalf("status: %v, want %q (words: %v)", out["status"], c.wantStatus, out["words"]) + } + words, _ := out["words"].(string) + if !strings.Contains(words, c.wantInWords) { + t.Fatalf("words: %q, want it to contain %q", words, c.wantInWords) + } + }) + } +} diff --git a/modules/mesh-mail/cmd/mesh-mail/tools.go b/modules/mesh-mail/cmd/mesh-mail/tools.go index 095ee3d4..51c434df 100644 --- a/modules/mesh-mail/cmd/mesh-mail/tools.go +++ b/modules/mesh-mail/cmd/mesh-mail/tools.go @@ -45,9 +45,10 @@ const ( ) // Bundle is what the tools reach: the settings, the index, the attachment store, and — for mail_send — the -// operator channel, the `sends` state and the outgoing dialer. The last three are nil outside a test: the real -// bus and dialer (sdkAskBus, keptSends, DialSMTP) are used lazily, so building a Bundle for the read-only tools -// never needs them. +// operator channel, the `sends` state and the work-queue publisher. The last three are nil outside a test: the +// real bus (sdkAskBus, keptSends, sdkQueuePublish) is used lazily, so building a Bundle for the read-only tools +// never needs them. Nothing here reads a password: the tools record the operator's Approve and queue the send +// for the sender, which alone performs it (novox/hq ADR 0436 §12, issue 1193, 1194). type Bundle struct { Env Env Index func(ctx context.Context) (Store, error) @@ -57,7 +58,7 @@ type Bundle struct { Asks AskBus Sends SendStore - Dial SMTPDialer + Queue QueuePublish Logf func(format string, a ...any) } @@ -138,10 +139,11 @@ func (b *Bundle) Tools() []stdio.Tool { Run: b.readTool}, {Name: HealthTool, Description: "How mesh-mail is, as its watch last saw it: healthy when every account's server answered within five " + - "minutes; unhealthy, naming the account and why, when a connection is down or its IDLE on the inbox went silent; " + - "waits, naming the account's password, while the operator has not given it at the desk.", + "minutes; unhealthy, naming the account and why, when a connection is down or its IDLE on the inbox went silent, " + + "or an approved send has waited 10 minutes or more for its sender; waits, naming the account's password, while " + + "the operator has not given it at the desk.", Input: schema(nil, map[string]any{}), - Run: func(map[string]any) (any, error) { return ReadStatus(b.Env.Status, b.now()), nil }}, + Run: b.healthTool}, {Name: SendTool, Description: "Send mail from an account you are granted (the same check as mail_read, novox/hq ADR 0361 §2). It " + "sends nothing itself: it raises an ask on the operator channel — the account, the recipients, the subject and " + @@ -205,6 +207,22 @@ func (b *Bundle) caller(args map[string]any) string { return stdio.CallerOf(args) } +// healthTool folds an overdue send (novox/hq ADR 0436 clause 11, issue 1199) into the watch's own verdict: +// unhealthy, naming the send, no matter what the watch itself said. Overdue reads `sends` directly — the +// watch process has no access to it — so a send stuck past its own 10 minutes is caught even while every +// account's own connection is fine. +func (b *Bundle) healthTool(map[string]any) (any, error) { + base := ReadStatus(b.Env.Status, b.now()) + overdue, err := Overdue(b.sends(), b.now()) + if err != nil { + if b.Logf != nil { + b.Logf("[mesh-mail] mail_health: the overdue check failed: %v", err) + } + return base, nil + } + return WithOverdue(base, overdue), nil +} + func (b *Bundle) accountsTool(args map[string]any) (any, error) { if err := only(args); err != nil { return nil, err @@ -397,14 +415,13 @@ func (b *Bundle) sends() SendStore { return keptSends{stdio.State(SendsState)} } -// lookupAccount resolves an account by name as the settings currently hold it — read fresh at decide time, never -// the account as it stood when it was asked. -func (b *Bundle) lookupAccount(name string) (Account, bool) { - accounts, err := ReadAccounts(b.Env.Accounts) - if err != nil { - return Account{}, false +// queuePublish is how this bundle publishes onto the module's own work-queue stream: the bus's own, unless a +// test gave one. Nothing here reads a password (novox/hq ADR 0436 §12). +func (b *Bundle) queuePublish() QueuePublish { + if b.Queue != nil { + return b.Queue } - return accountByName(accounts, name) + return sdkQueuePublish } func accountByName(accounts []Account, name string) (Account, bool) { @@ -416,25 +433,6 @@ func accountByName(accounts []Account, name string) (Account, bool) { return Account{}, false } -// sendActor performs one send on the operator's Approve: it reads the account's own secret itself, so Decide and -// the `sends` state never hold a password. -func (b *Bundle) sendActor() SendActor { - dial := b.Dial - if dial == nil { - dial = DialSMTP - } - return func(a Account, to []string, subject, body string, now time.Time) error { - pw, given, err := ReadSecret(b.Env.Secrets, a) - if err != nil { - return fmt.Errorf("the account %s's password cannot be read: %w", a.Name, err) - } - if !given { - return fmt.Errorf("the account %s has no password yet; the operator gives it at the desk", a.Name) - } - return SendMail(dial, a, pw, to, subject, body, now) - } -} - func stringList(v any) []string { list, _ := v.([]any) out := make([]string, 0, len(list)) @@ -483,7 +481,7 @@ func (b *Bundle) sendStatusTool(args map[string]any) (any, error) { if id == "" { return nil, fmt.Errorf("%s needs id, the ask %s gave", SendStatusTool, SendTool) } - return ReadSendStatus(b.sends(), b.askBus(), b.lookupAccount, b.sendActor(), id, b.now()) + return ReadSendStatus(b.sends(), b.askBus(), b.queuePublish(), id, b.now()) } // ---- arguments ---------------------------------------------------------------------------------------------------- diff --git a/modules/mesh-mail/go.mod b/modules/mesh-mail/go.mod index 16e2aaf6..b5826dcc 100644 --- a/modules/mesh-mail/go.mod +++ b/modules/mesh-mail/go.mod @@ -1,19 +1,31 @@ module mesh-mail -go 1.25.0 +go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019062050-34392e74916c github.com/emersion/go-imap/v2 v2.0.0-beta.8 github.com/emersion/go-message v0.18.2 github.com/jackc/pgx/v5 v5.11.0 + github.com/nats-io/nats-server/v2 v2.11.17 + github.com/nats-io/nats.go v1.54.0 ) require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect + github.com/google/go-tpm v0.9.8 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect - golang.org/x/sync v0.17.0 // indirect - golang.org/x/text v0.29.0 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect + golang.org/x/time v0.15.0 // indirect ) diff --git a/modules/mesh-mail/go.sum b/modules/mesh-mail/go.sum index da60653c..8aa47987 100644 --- a/modules/mesh-mail/go.sum +++ b/modules/mesh-mail/go.sum @@ -1,5 +1,7 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44 h1:r5xIo5qWOXQdCSMxvqJy8kV4U+P/QWXFK6971Ha6wU8= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019062050-34392e74916c h1:/f6f2ugss172hDJSL3EFZqb9PdfO5DKzYVfPiX2LdAM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019062050-34392e74916c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -9,6 +11,8 @@ github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7 github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA= github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 h1:oP4q0fw+fOSWn3DfFi4EXdT+B+gTtzx8GC9xsc26Znk= github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= @@ -17,6 +21,20 @@ github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -27,6 +45,8 @@ github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -36,14 +56,17 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -52,8 +75,10 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= -golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/modules/mesh-mail/module.json b/modules/mesh-mail/module.json index 38a5ccea..b6aaa7ce 100644 --- a/modules/mesh-mail/module.json +++ b/modules/mesh-mail/module.json @@ -48,7 +48,12 @@ "mail-credential-*": { "path": "${dir:state}/mail-credential-*.secret", "issued-by": "outside", - "prompt": "the password of the mail account {member}" + "prompt": "the password of the mail account {member}", + "owner": "mesh-mail-send" + }, + "broker-send": { + "path": "${dir:state}/broker-send.secret", + "owner": "mesh-mail-send" } }, "config": { @@ -83,6 +88,10 @@ "state": [ "sends" ], + "emits": [ + "mail.send.sent", + "mail.send.failed" + ], "streams": [ { "name": "mail", @@ -96,13 +105,29 @@ "config": "grants", "events": "mail.{key}.**" } + }, + { + "name": "mail-sends", + "events": [ + "mail.send.approved" + ], + "max-age-days": 1, + "max-mib": 64, + "retention": "work-queue", + "max-deliveries": 5, + "duplicates": 24 } ], "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-mail-send" + }, { "id": "state", "type": "directory", - "mode": "0700", + "mode": "0711", "place": ".", "unknown": "program" }, @@ -125,7 +150,7 @@ "trusted": true, "type": "file", "path": "${dir:state}/accounts.json", - "mode": "0600", + "mode": "0644", "merge": "json", "template": "files/accounts.json", "takes-config": [ @@ -214,6 +239,43 @@ "looks": 2, "grace": "180s" } + }, + { + "id": "send", + "type": "process", + "name": "mesh-mail-send", + "artifact": "tools", + "run": [ + "./mesh-mail", + "send" + ], + "user": "mesh-mail-send", + "env": { + "MESH_MAIL_ACCOUNTS": "${dir:state}/accounts.json", + "MESH_MAIL_SECRETS": "${dir:state}" + }, + "restart-on": [ + "accounts" + ], + "bus": { + "emits": [ + "mail.send.sent", + "mail.send.failed" + ], + "queues": [ + "mail-sends" + ], + "state": [ + "sends" + ], + "asks": true, + "invokes": [ + "seat:operator-channel.tell" + ] + }, + "health": { + "kind": "unit" + } } ], "build": {