novox/mesh-catalog
13
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases Wiki Activity

Merge pull request 'mesh-mail sends an approved mail from its own sender process, and tells a failed send once (ADR 0436, ADR 0438; issues 1172, 1193-1199)' (#371) from feat/mail-sender into main

Browse Source
This commit was merged in pull request #371.
This commit is contained in:
mesh-tools
2026-10-19 11:36:51 +00:00
parent c441d83ba7 a2c0c58677
commit 7bb588ca1b
27 changed files with 3782 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-mail/cmd/mesh-mail/acceptance_integration_test.go
+328
View File
@@ -0,0 +1,328 @@
//go:build integration
package main
// Release 3's acceptance test (novox/hq ADR 0436, issue 1198 AC4): the whole module, over a JetStream bus
// of the test's own (nats-server v2.11.17, copied from modules/mesh-delivery/cmd/mesh-delivery/bus_test.go
// as newTestBus already does in sender_integration_test.go), a fake SMTP server, the router's records
// bucket with a closed Approve record, the account file and a password file written by the test. Compiled
// by the merge gate's `go vet -tags integration`; never run on this machine (the context file's hard rule).
//
// "The tools queue an approved send" is played here directly against the real bus (Decide, send.go) rather
// than through the tool runner's own RPC — the tool runner's part (stdio.Serve, grants, the ask's wording)
// is proved elsewhere (send_test.go, ask_test.go); what this test proves is the handover from an approved,
// queued send through to the sender's own claim, send and outcome, end to end over a real bus.
import (
"context"
"encoding/json"
"errors"
"path/filepath"
"strings"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
)
// kvSends adapts a raw jetstream.KeyValue to SendStore (send.go): the test's own stand-in for both the
// tools' and the sender's `sends` state, which in production are the same bucket reached under two
// different accounts.
type kvSends struct{ kv jetstream.KeyValue }
func (k kvSends) Get(key string) (json.RawMessage, uint64, bool, error) {
e, err := k.kv.Get(context.Background(), key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, 0, false, nil
}
if err != nil {
return nil, 0, false, err
}
return json.RawMessage(e.Value()), e.Revision(), true, nil
}
func (k kvSends) Create(key string, value any) (uint64, error) {
data, err := json.Marshal(value)
if err != nil {
return 0, err
}
rev, err := k.kv.Create(context.Background(), key, data)
if errors.Is(err, jetstream.ErrKeyExists) {
return 0, stdio.ErrChanged
}
return rev, err
}
func (k kvSends) Update(key string, value any, revision uint64) (uint64, error) {
data, err := json.Marshal(value)
if err != nil {
return 0, err
}
rev, err := k.kv.Update(context.Background(), key, data, revision)
if errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
return 0, stdio.ErrChanged
}
return rev, err
}
func (k kvSends) Keys() ([]string, error) { return nil, errors.New("not used by this test") }
// acceptanceBus is everything the test built on a fresh bus: ready-made buckets and streams, and the raw
// clients to act as the tools with.
type acceptanceBus struct {
nc *nats.Conn
js jetstream.JetStream
sends jetstream.KeyValue
asks jetstream.KeyValue
}
func newAcceptanceBus(t *testing.T) (*acceptanceBus, string) {
t.Helper()
url := newTestBus(t)
ctx := context.Background()
nc, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
sends, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: Module + "_" + SendsState})
if err != nil {
t.Fatal(err)
}
asksKV, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: MessengerAsksBucket})
if err != nil {
t.Fatal(err)
}
if _, err := js.CreateStream(ctx, mailSendsStreamConfig()); err != nil {
t.Fatal(err)
}
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod." + Module + ".event.>"}}); err != nil {
t.Fatal(err)
}
return &acceptanceBus{nc: nc, js: js, sends: sends, asks: asksKV}, url
}
// queueAnApprovedSend plays the tools' own part (send.go's Decide, the queue publish) directly against the
// real bus: a send asked, approved by a warrant whose digest really is the one req.Ask() binds, recorded
// approved and queued — exactly what issue 1194 made the tools do, without needing the tool runner's RPC.
func queueAnApprovedSend(t *testing.T, bus *acceptanceBus, account string, approvedAt time.Time) (SendRequest, string) {
t.Helper()
ctx := context.Background()
req := SendRequest{ID: newSendID(), Account: account, To: []string{"ops@example.org"}, Subject: "The nightly gate is red",
Body: "See the job log.", Caller: "person.op", At: approvedAt, AskTo: "o… at example org"}
store := kvSends{kv: bus.sends}
if _, err := store.Create(req.ID, req); err != nil {
t.Fatalf("keeping the send: %v", err)
}
w := warrantFor(req, approvedAt)
rec := routerRecord{State: string(w.Outcome), Warrant: &w}
raw, _ := json.Marshal(rec)
if _, err := bus.asks.Put(ctx, Module+"."+req.ID, raw); err != nil {
t.Fatalf("writing the router's record: %v", err)
}
// The tools' own publish path (send.go's QueuePublish, wired for real to stdio.Emit/sdkQueuePublish):
// the real wire subject the controller composes for mail-sends' one event (mailSendApprovedSubject),
// the send id as the message id, so the stream's own duplicate window is the one actually exercised
// (novox/hq review round 1, finding 7).
publish := func(id string, body any) error {
data, err := json.Marshal(body)
if err != nil {
return err
}
_, err = bus.js.Publish(ctx, mailSendApprovedSubject, data, jetstream.WithMsgID(id))
return err
}
if _, err := Decide(store, publish, w, approvedAt); err != nil {
t.Fatalf("the tools' own Decide: %v", err)
}
kept, _, ok, err := getSend(store, req.ID)
if err != nil || !ok || kept.Decided == nil || kept.Decided.Outcome != "queued" {
t.Fatalf("not left queued: %v %v %+v", err, ok, kept.Decided)
}
// Published a second time under the same message id, exactly as the sweep would for a record it finds
// still queued past its grace (Sweep, send.go): the stream's 24-hour duplicate window keeps it one
// message (novox/hq review round 1, finding 7) — the sender's own claim (clause 9) is what then keeps
// it one send, proved by this test's own "sent once" assertion below.
if err := publish(req.ID, QueuedSend{Send: req.ID, Account: req.Account, Ask: req.ID}); err != nil {
t.Fatalf("the duplicate publish: %v", err)
}
return req, req.ID
}
// 1198/AC4: when the tools queue an approved send and the send process runs, the fake server receives
// exactly one mail carrying the send id's Message-ID, sends says sent, and mail.send.sent is on the stream.
func TestMailSenderAcceptanceAnApprovedSendIsSentOnceAndSaidOnce(t *testing.T) {
bus, url := newAcceptanceBus(t)
fake := newFakeSMTPServer(t)
dir := t.TempDir()
accountsPath := filepath.Join(dir, "accounts.json")
writeFile(t, accountsPath, `{"accounts": {"mesh": {"kind": "imap", "address": "jschoubben@novox.be",
"incoming": {"host": "mail.novox.be", "port": 993, "security": "tls"},
"outgoing": {"host": "127.0.0.1", "port": 1, "security": "tls"}}}}`)
writeFile(t, filepath.Join(dir, "mail-credential-mesh.secret"), "the-password\n")
approvedAt := judgedAt
req, id := queueAnApprovedSend(t, bus, "mesh", approvedAt)
account := process.Account{URL: url, Node: "g14", Module: Module, User: "g14." + Module + "." + SendProcessID, Password: "pw"}
if _, err := bus.js.CreateConsumer(context.Background(), MailSendsStream, jetstream.ConsumerConfig{
Durable: account.QueueConsumerName(), AckPolicy: jetstream.AckExplicitPolicy, AckWait: 5 * time.Second, MaxDeliver: MostAttempts}); err != nil {
t.Fatal(err)
}
conn, err := process.Connect(account)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
h := &senderHandler{
Store: kvSends{kv: bus.sends},
Record: routerRecordOf(conn),
Accounts: func() ([]Account, error) { return ReadAccounts(accountsPath) },
SecretsDir: dir,
Dial: fake.dialer(),
Emit: emitOf(conn),
Now: func() time.Time { return approvedAt.Add(time.Minute) },
}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
done := make(chan error, 1)
go func() { done <- conn.Consume(ctx, MailSendsStream, h.handle) }()
eventually(t, 10*time.Second, "the mail sent", func() (bool, string) {
return fake.count() == 1, "received so far: " + describeReceived(fake)
})
cancel()
<-done
store := kvSends{kv: bus.sends}
kept, _, ok, err := getSend(store, id)
if err != nil || !ok || kept.Decided == nil || kept.Decided.Outcome != "sent" {
t.Fatalf("the kept send: %v %v %+v", err, ok, kept.Decided)
}
got := fake.last()
if got.to[0] != "ops@example.org" {
t.Fatalf("received: %+v", got)
}
if !strings.Contains(got.data, "Message-Id: <"+req.ID+"@novox.be>") {
t.Fatalf("the mail's Message-ID does not name the send: %s", got.data)
}
// mail.send.sent is on the EVENTS stream, naming the send id, never the recipient, subject or body.
eventually(t, 10*time.Second, "mail.send.sent on the stream", func() (bool, string) {
msg, err := fetchOneRaw(bus, "mesh.mod."+Module+".event."+MailSendSent)
if err != nil || msg == nil {
return false, "not yet"
}
var e OutcomeEvent
_ = json.Unmarshal(msg, &e)
return e.Send == req.ID && e.Account == "mesh", "got " + string(msg)
})
}
// 1198/AC4: when the fake server refuses sign-in, then after the attempts (shortened here) the record says
// failed and nothing was received.
func TestMailSenderAcceptanceARefusedSignInFailsAfterItsAttemptsAndNothingIsReceived(t *testing.T) {
orig := RetryDelays
RetryDelays = []time.Duration{10 * time.Millisecond, 10 * time.Millisecond, 10 * time.Millisecond, 10 * time.Millisecond}
t.Cleanup(func() { RetryDelays = orig })
bus, url := newAcceptanceBus(t)
s := newScriptedSMTPServer(t)
s.auth = reply(535, "authentication failed")
dir := t.TempDir()
accountsPath := filepath.Join(dir, "accounts.json")
writeFile(t, accountsPath, `{"accounts": {"mesh": {"kind": "imap", "address": "jschoubben@novox.be",
"incoming": {"host": "mail.novox.be", "port": 993, "security": "tls"},
"outgoing": {"host": "127.0.0.1", "port": 1, "security": "tls"}}}}`)
writeFile(t, filepath.Join(dir, "mail-credential-mesh.secret"), "the-wrong-password\n")
approvedAt := judgedAt
_, id := queueAnApprovedSend(t, bus, "mesh", approvedAt)
account := process.Account{URL: url, Node: "g14", Module: Module, User: "g14." + Module + "." + SendProcessID, Password: "pw"}
if _, err := bus.js.CreateConsumer(context.Background(), MailSendsStream, jetstream.ConsumerConfig{
Durable: account.QueueConsumerName(), AckPolicy: jetstream.AckExplicitPolicy, AckWait: 2 * time.Second, MaxDeliver: MostAttempts}); err != nil {
t.Fatal(err)
}
conn, err := process.Connect(account)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
h := &senderHandler{
Store: kvSends{kv: bus.sends},
Record: routerRecordOf(conn),
Accounts: func() ([]Account, error) { return ReadAccounts(accountsPath) },
SecretsDir: dir,
Dial: s.dialer(),
Emit: emitOf(conn),
Now: func() time.Time { return approvedAt.Add(time.Minute) },
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
done := make(chan error, 1)
go func() { done <- conn.Consume(ctx, MailSendsStream, h.handle) }()
store := kvSends{kv: bus.sends}
eventually(t, 25*time.Second, "the send to fail after its attempts", func() (bool, string) {
kept, _, ok, err := getSend(store, id)
if err != nil || !ok || kept.Decided == nil {
return false, "not yet decided"
}
// "queued" is the send's own starting state (queueAnApprovedSend's own write, before the sender
// ever claims it): without excluding it here too, this check reads done on the very first look,
// long before the sender connects or claims anything at all (novox/hq review: build-1792400882114092431,
// 1198/AC4).
o := kept.Decided.Outcome
return o != "queued" && o != "retrying" && o != "sending", o
})
cancel()
<-done
kept, _, ok, err := getSend(store, id)
if err != nil || !ok || kept.Decided == nil || !strings.HasPrefix(kept.Decided.Outcome, "failed: ") {
t.Fatalf("the kept send did not end failed: %v %v %+v", err, ok, kept.Decided)
}
if kept.Decided.Attempt != MostAttempts {
t.Fatalf("failed after %d attempt(s), want all %d", kept.Decided.Attempt, MostAttempts)
}
if n := s.receivedCount(); n != 0 {
t.Fatalf("the scripted server received %d message(s), want none: a refused sign-in never reaches DATA", n)
}
}
// fetchOneRaw pulls at most one message off a stream's subject, or nil within a short wait, for a test to
// check an emitted event without binding a durable consumer of its own.
func fetchOneRaw(bus *acceptanceBus, subject string) ([]byte, error) {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
cons, err := bus.js.OrderedConsumer(ctx, "EVENTS", jetstream.OrderedConsumerConfig{FilterSubjects: []string{subject}})
if err != nil {
return nil, err
}
batch, err := cons.Fetch(1, jetstream.FetchMaxWait(time.Second))
if err != nil {
return nil, err
}
msg, ok := <-batch.Messages()
if !ok {
return nil, nil
}
return msg.Data(), nil
}
func describeReceived(f *fakeSMTPServer) string {
if f.count() == 0 {
return "none"
}
got := f.last()
return got.from
}
modules/mesh-mail/cmd/mesh-mail/claim_test.go
+451
View File
@@ -0,0 +1,451 @@
package main
// The sender's own claim (novox/hq ADR 0436 clauses 9, 10, issue 1196): it never trusts the queue's message,
// verifying the operator's warrant again from the router's own record before it claims the send by
// compare-and-set — exactly mirroring the tools' own check, so the boundary clause 9 describes (the tool
// runner's account may publish on the queue and write `sends`, but only a warrant that really closed the
// ask moves a send to "sending").
import (
"encoding/json"
"errors"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
)
// claimTestSend asks and approves one send (through the tools, as issue 1194 built it), leaving it queued,
// and returns its id and the router that holds its warrant.
func claimTestSend(t *testing.T) (id string, router *sendRouter, sends *memSends) {
t.Helper()
b, router, sends := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"})
queue := newMemQueue()
b.Queue = queue.publish
rt := startBundle(t, b.Tools())
text, isErr := rt.call(SendTool, "novox.triage", sendArgs)
if isErr {
t.Fatalf("%s", text)
}
var out map[string]any
_ = json.Unmarshal([]byte(text), &out)
id = out["ask"].(string)
w := router.chose(id, OptionSend, judgedAt.Add(time.Minute))
router.close(w)
if _, err := Decide(sends, b.queuePublish(), w, judgedAt.Add(2*time.Minute)); err != nil {
t.Fatalf("approving: %v", err)
}
req, _, ok, err := getSend(sends, id)
if err != nil || !ok || req.Decided == nil || req.Decided.Outcome != "queued" {
t.Fatalf("not left queued: %v %v %+v", err, ok, req.Decided)
}
return id, router, sends
}
// rawQueuedSend creates a send directly in the queued state, without going through the tools or the router
// at all, so an AC2 test controls the router's record independently of how the send reached queued: the
// defence in depth clause 9 describes, since the tool runner's account could write `sends` and the queue
// without ever having closed a real warrant.
func rawQueuedSend(t *testing.T, sends *memSends) SendRequest {
t.Helper()
req := SendRequest{ID: newSendID(), Account: "mesh", To: []string{"ops@example.org"}, Subject: "Status",
Body: "the body", Caller: "person.op", At: judgedAt, AskTo: "o… at example org"}
req.Decided = &SendDecision{At: judgedAt.Add(time.Minute), Outcome: "queued"}
if _, err := sends.Create(req.ID, req); err != nil {
t.Fatal(err)
}
return req
}
// warrantFor is a well-formed warrant answering req's ask with Send, at, and the ask's own digest — the
// same shape sendRouter.chose builds, for a test that never published the ask to a router at all.
func warrantFor(req SendRequest, at time.Time) asks.Warrant {
ask := req.Ask()
opt, _ := ask.Option(OptionSend)
return asks.Warrant{Ask: req.ID, Asker: Module, Outcome: asks.OutcomeChosen, Option: opt.ID, Label: opt.Label,
Level: opt.Level,
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"},
Channel: "telegram", Proofs: []string{"P1"}, At: at, AskDigest: ask.Digest()}
}
// 1196/AC1: given a queued send whose router record is closed by the operator's Approve at level approve
// within its expiry and whose recomputed act digest is the option's binds, when the sender takes the
// message, then it claims the send by compare-and-set from queued to sending with the attempt and time
// before any connection (nothing here connects to anything: claimQueuedSend never dials).
func TestClaimQueuedSendClaimsSendingWithTheAttemptBeforeAnyConnection(t *testing.T) {
id, router, sends := claimTestSend(t)
emit := &memEmit{}
req, d, _, outcome, err := claimQueuedSend(sends, router.Record, emit.emit, id, judgedAt.Add(3*time.Minute))
if err != nil {
t.Fatalf("claiming: %v", err)
}
if outcome != claimToSend {
t.Fatalf("outcome: %v, want claimToSend", outcome)
}
if d.Outcome != "sending" || d.Attempt != 1 || !d.At.Equal(judgedAt.Add(3*time.Minute)) {
t.Fatalf("claimed: %+v", d)
}
if req.ID != id {
t.Fatalf("req: %+v", req)
}
kept, _, ok, err := getSend(sends, id)
if err != nil || !ok || kept.Decided == nil || kept.Decided.Outcome != "sending" {
t.Fatalf("not kept as sending: %v %v %+v", err, ok, kept.Decided)
}
if len(emit.calls) != 0 {
t.Fatalf("claiming to send emitted something: %+v", emit.calls)
}
}
// 1196/AC2: given a router record that is not mesh-mail's, not closed by Approve, approved after its
// expiry, or whose digest differs from the request in sends, when taken, then the sender records refused
// with the reason, terminates the message and connects to no server (claimQueuedSend never dials, so
// "connects to no server" holds for every path by construction; this proves each is refused and the claim
// never advances to sending).
func TestClaimQueuedSendRefusesAMismatchedOrUnclosedWarrant(t *testing.T) {
t.Run("not mesh-mail's ask", func(t *testing.T) {
sends, router := newMemSends(), newSendRouter()
req := rawQueuedSend(t, sends)
w := warrantFor(req, judgedAt.Add(time.Minute))
w.Asker = "another-module"
router.close(w)
emit := &memEmit{}
_, d, _, outcome, err := claimQueuedSend(sends, router.Record, emit.emit, req.ID, judgedAt.Add(3*time.Minute))
if err != nil || outcome != claimRefused || !strings.HasPrefix(d.Outcome, "refused: ") {
t.Fatalf("outcome %v, decision %+v, err %v", outcome, d, err)
}
assertEmittedFailed(t, emit, req.ID, d.Outcome)
})
t.Run("not closed by approve", func(t *testing.T) {
sends, router := newMemSends(), newSendRouter()
req := rawQueuedSend(t, sends)
// The router keeps no record of this ask at all: never published, or never answered.
emit := &memEmit{}
_, d, _, outcome, err := claimQueuedSend(sends, router.Record, emit.emit, req.ID, judgedAt.Add(3*time.Minute))
if err != nil || outcome != claimRefused || !strings.HasPrefix(d.Outcome, "refused: ") {
t.Fatalf("outcome %v, decision %+v, err %v", outcome, d, err)
}
assertEmittedFailed(t, emit, req.ID, d.Outcome)
})
t.Run("approved after the ask's expiry", func(t *testing.T) {
sends, router := newMemSends(), newSendRouter()
req := rawQueuedSend(t, sends)
w := warrantFor(req, req.Ask().Expires.Add(time.Hour))
router.close(w)
emit := &memEmit{}
_, d, _, outcome, err := claimQueuedSend(sends, router.Record, emit.emit, req.ID, judgedAt.Add(3*time.Minute))
if err != nil || outcome != claimRefused || !strings.HasPrefix(d.Outcome, "refused: ") {
t.Fatalf("outcome %v, decision %+v, err %v", outcome, d, err)
}
assertEmittedFailed(t, emit, req.ID, d.Outcome)
})
t.Run("a digest the published ask never bound", func(t *testing.T) {
sends, router := newMemSends(), newSendRouter()
req := rawQueuedSend(t, sends)
w := warrantFor(req, judgedAt.Add(time.Minute))
w.AskDigest = "sha256:" + strings.Repeat("0", 64)
router.close(w)
emit := &memEmit{}
_, d, _, outcome, err := claimQueuedSend(sends, router.Record, emit.emit, req.ID, judgedAt.Add(3*time.Minute))
if err != nil || outcome != claimRefused || !strings.HasPrefix(d.Outcome, "refused: ") {
t.Fatalf("outcome %v, decision %+v, err %v", outcome, d, err)
}
assertEmittedFailed(t, emit, req.ID, d.Outcome)
})
}
// assertEmittedFailed checks that exactly one mail.send.failed was emitted, naming the send id and the
// decision's own outcome in words (novox/hq review round 1 finding 4).
func assertEmittedFailed(t *testing.T, emit *memEmit, id, outcome string) {
t.Helper()
if len(emit.calls) != 1 {
t.Fatalf("emitted %d time(s), want exactly one mail.send.failed: %+v", len(emit.calls), emit.calls)
}
c := emit.calls[0]
if c.eventType != MailSendFailed || c.id != id || c.body.Send != id || c.body.Outcome != outcome {
t.Fatalf("emitted %+v, want mail.send.failed naming %s and outcome %q", c, id, outcome)
}
}
// 1196/AC3: given a send already sending (the sender stopped mid-send), when its message is delivered
// again, then the sender sends nothing, records not proven and terminates.
func TestClaimQueuedSendOnARedeliveryOfAnAlreadySendingClaimRecordsNotProven(t *testing.T) {
id, router, sends := claimTestSend(t)
emit := &memEmit{}
req, _, _, outcome, err := claimQueuedSend(sends, router.Record, emit.emit, id, judgedAt.Add(3*time.Minute))
if err != nil || outcome != claimToSend {
t.Fatalf("first claim: %v %v", outcome, err)
}
_ = req
// Redelivered: the first claim's handler may have stopped mid-send.
_, d, _, outcome, err := claimQueuedSend(sends, router.Record, emit.emit, id, judgedAt.Add(4*time.Minute))
if err != nil {
t.Fatalf("redelivered claim: %v", err)
}
if outcome != claimNotProven || d.Outcome != "not proven" {
t.Fatalf("outcome %v, decision %+v", outcome, d)
}
assertEmittedFailed(t, emit, id, d.Outcome)
}
// 1196/AC4: given a lost compare-and-set (another claim), when taken, then it sends nothing and leaves the
// message to the bus (no decision of this call's own making is kept; it reads back whatever the other
// writer decided).
func TestClaimQueuedSendOnALostRaceSendsNothingAndLeavesTheMessageToTheBus(t *testing.T) {
id, router, sends := claimTestSend(t)
racy := &raceOnFirstUpdate{SendStore: sends, race: func() {
r, rv, ok, err := getSend(sends, id)
if err != nil || !ok {
t.Fatalf("racing claim could not read the send: %v %v", err, ok)
}
if _, _, err := settleSend(sends, r, rv, SendDecision{At: judgedAt.Add(3 * time.Minute), Outcome: "sending", Attempt: 1}); err != nil {
t.Fatalf("racing claim: %v", err)
}
}}
emit := &memEmit{}
_, d, _, outcome, err := claimQueuedSend(racy, router.Record, emit.emit, id, judgedAt.Add(3*time.Minute))
if err != nil {
t.Fatalf("claiming: %v", err)
}
if outcome != claimLostRace {
t.Fatalf("outcome: %v, want claimLostRace", outcome)
}
if d.Outcome != "sending" || d.Attempt != 1 {
t.Fatalf("the decision read back is not the racing claim's own: %+v", d)
}
if len(emit.calls) != 0 {
t.Fatalf("a lost race emitted something: %+v", emit.calls)
}
}
// review round 1 finding 3: a failed write of the tells-pending index does not stop the refusal itself
// from being recorded — the record write and the message's own fate are never undone by an index that
// could not be kept — but the index failure is still reported, never silently dropped.
func TestClaimQueuedSendStillRefusesWhenTheTellsPendingIndexCannotBeWritten(t *testing.T) {
sends, router := newMemSends(), newSendRouter()
req := rawQueuedSend(t, sends)
sends.failKey, sends.failErr = tellsPendingKey, errors.New("the store is down")
emit := &memEmit{}
_, d, _, outcome, err := claimQueuedSend(sends, router.Record, emit.emit, req.ID, judgedAt.Add(3*time.Minute))
if outcome != claimRefused || !strings.HasPrefix(d.Outcome, "refused: ") {
t.Fatalf("outcome %v, decision %+v", outcome, d)
}
if err == nil {
t.Fatalf("the index failure was not reported")
}
kept, _, ok, kerr := getSend(sends, req.ID)
if kerr != nil || !ok || kept.Decided == nil || kept.Decided.Outcome != d.Outcome {
t.Fatalf("the refusal itself was not kept despite the index failing: %v %v %+v", kerr, ok, kept.Decided)
}
assertEmittedFailed(t, emit, req.ID, d.Outcome)
}
// Same finding, for the redelivery-finds-sending path (not proven).
func TestClaimQueuedSendStillRecordsNotProvenWhenTheTellsPendingIndexCannotBeWritten(t *testing.T) {
sends, router := newMemSends(), newSendRouter()
req := rawQueuedSend(t, sends)
current, rev, ok, err := getSend(sends, req.ID)
if err != nil || !ok {
t.Fatalf("reading it back: %v %v", err, ok)
}
current.Decided.Outcome = "sending"
if _, err := sends.Update(req.ID, current, rev); err != nil {
t.Fatalf("leaving it sending: %v", err)
}
sends.failKey, sends.failErr = tellsPendingKey, errors.New("the store is down")
emit := &memEmit{}
_, d, _, outcome, err := claimQueuedSend(sends, router.Record, emit.emit, req.ID, judgedAt.Add(4*time.Minute))
if outcome != claimNotProven || d.Outcome != "not proven" {
t.Fatalf("outcome %v, decision %+v", outcome, d)
}
if err == nil {
t.Fatalf("the index failure was not reported")
}
kept, _, ok2, kerr := getSend(sends, req.ID)
if kerr != nil || !ok2 || kept.Decided == nil || kept.Decided.Outcome != "not proven" {
t.Fatalf("not proven was not kept despite the index failing: %v %v %+v", kerr, ok2, kept.Decided)
}
assertEmittedFailed(t, emit, req.ID, d.Outcome)
}
// Same finding, for decideSend: a failed sender-pending index write does not stop "queued" from being
// recorded, nor the publish from being attempted — there is no separate "approved" state any more (novox/hq
// ADR 0436 clause 8, as decided to replace the two-state design, review round 1).
func TestDecideSendStillRecordsQueuedAndPublishesWhenTheSenderPendingIndexCannotBeWritten(t *testing.T) {
sends := newMemSends()
req := SendRequest{ID: newSendID(), Account: "mesh", To: []string{"ops@example.org"}, Subject: "s", Body: "body", At: judgedAt}
rev, err := sends.Create(req.ID, req)
if err != nil {
t.Fatal(err)
}
sends.failKey, sends.failErr = senderPendingKey, errors.New("the store is down")
queue := newMemQueue()
w := warrantFor(req, judgedAt.Add(time.Minute))
d, err := decideSend(sends, queue.publish, req, rev, w, judgedAt.Add(time.Minute))
if d.Outcome != "queued" {
t.Fatalf("decision: %+v, want queued despite the index failing", d)
}
if err == nil {
t.Fatalf("the index failure was not reported")
}
if n := queue.count(); n != 1 {
t.Fatalf("published %d time(s)", n)
}
kept, _, ok, kerr := getSend(sends, req.ID)
if kerr != nil || !ok || kept.Decided == nil || kept.Decided.Outcome != "queued" {
t.Fatalf("not kept as queued despite the index failing: %v %v %+v", kerr, ok, kept.Decided)
}
}
// 1194/AC1 (as decided to replace the two-state design, review round 1): on a verified warrant the record
// moves straight from undecided to queued by one compare-and-set, with its time, and mail.send.approved is
// published with the send id as its message id — no "approved" state in between.
func TestDecideSendRecordsQueuedDirectlyAndPublishes(t *testing.T) {
sends := newMemSends()
req := SendRequest{ID: newSendID(), Account: "mesh", To: []string{"ops@example.org"}, Subject: "s", Body: "body", At: judgedAt}
rev, err := sends.Create(req.ID, req)
if err != nil {
t.Fatal(err)
}
queue := newMemQueue()
w := warrantFor(req, judgedAt.Add(time.Minute))
now := judgedAt.Add(time.Minute)
d, err := decideSend(sends, queue.publish, req, rev, w, now)
if err != nil {
t.Fatalf("deciding: %v", err)
}
if d.Outcome != "queued" || !d.At.Equal(now) {
t.Fatalf("decision: %+v, want queued at %v", d, now)
}
if n := queue.count(); n != 1 {
t.Fatalf("published %d time(s), want exactly 1", n)
}
got := queue.last()
if got.id != req.ID || got.body.Send != req.ID || got.body.Account != "mesh" || got.body.Ask != req.ID {
t.Fatalf("published %+v, want the send id as the message id and body", got)
}
kept, _, ok, kerr := getSend(sends, req.ID)
if kerr != nil || !ok || kept.Decided == nil || kept.Decided.Outcome != "queued" {
t.Fatalf("not kept as queued: %v %v %+v", kerr, ok, kept.Decided)
}
ids, err := senderPendingIDs(sends)
if err != nil || len(ids) != 1 || ids[0] != req.ID {
t.Fatalf("not indexed as sender-pending: %v %v", err, ids)
}
}
// 1196/AC2 (review round 2): a store that changes the revision between the claim and the refusal's own
// write — another writer's decision landing in between — never emits mail.send.failed for a write that did
// not actually land (refuse() does not retry: that is recordOutcome's own job, send.go, review round 2).
func TestClaimQueuedSendRefuseEmitsNothingWhenTheWriteDidNotLand(t *testing.T) {
sends, router := newMemSends(), newSendRouter()
req := rawQueuedSend(t, sends)
w := warrantFor(req, judgedAt.Add(time.Minute))
w.AskDigest = "sha256:" + strings.Repeat("0", 64) // mismatched: claimQueuedSend refuses it
router.close(w)
racy := &raceOnFirstUpdate{SendStore: sends, race: func() {
r, rv, ok, err := getSend(sends, req.ID)
if err != nil || !ok {
t.Fatalf("racing write could not read the send: %v %v", err, ok)
}
if _, _, err := settleSend(sends, r, rv, SendDecision{At: judgedAt, Outcome: "not proven", Attempt: 1}); err != nil {
t.Fatalf("racing write: %v", err)
}
}}
emit := &memEmit{}
_, d, _, outcome, _ := claimQueuedSend(racy, router.Record, emit.emit, req.ID, judgedAt.Add(3*time.Minute))
if outcome != claimRefused {
t.Fatalf("outcome: %v, want claimRefused", outcome)
}
if len(emit.calls) != 0 {
t.Fatalf("emitted mail.send.failed despite the refusal's own write not landing: %+v", emit.calls)
}
// The kept record is the racing write's own decision, not this call's: refuse() never retries.
kept, _, ok, kerr := getSend(sends, req.ID)
if kerr != nil || !ok || kept.Decided == nil || kept.Decided.Outcome != "not proven" {
t.Fatalf("kept: %v %v %+v (returned %+v)", kerr, ok, kept.Decided, d)
}
}
// 1196/AC5 (operator-approved change to clause 10): given a send already sent (and one already failed,
// refused, or not proven), when its message is delivered again, then the record is unchanged, nothing is
// emitted or told, and the message is terminated (claimAlreadyFinished; sender.go's handler maps it to
// Term alone, tested here the same way every other claimOutcome is in this file — by the outcome value).
func TestClaimQueuedSendOnARedeliveryOfAnAlreadyFinishedSendTerminatesWithoutTouchingAnything(t *testing.T) {
for name, outcome := range map[string]string{
"sent": "sent",
"failed": "failed: 550 no such mailbox",
"refused": "refused: the warrant does not bind the request",
"not proven": "not proven",
} {
t.Run(name, func(t *testing.T) {
sends, router := newMemSends(), newSendRouter()
req := rawQueuedSend(t, sends)
current, rev, ok, err := getSend(sends, req.ID)
if err != nil || !ok {
t.Fatalf("reading it back: %v %v", err, ok)
}
current.Decided.Outcome = outcome
current.Decided.Warrant = "the operator, via telegram (user id verified), chose Send"
current.Decided.Attempt = 1
if _, err := sends.Update(req.ID, current, rev); err != nil {
t.Fatalf("leaving it %s: %v", outcome, err)
}
before := *current.Decided
emit := &memEmit{}
_, d, _, got, err := claimQueuedSend(sends, router.Record, emit.emit, req.ID, judgedAt.Add(3*time.Minute))
if err != nil {
t.Fatalf("claiming: %v", err)
}
if got != claimAlreadyFinished {
t.Fatalf("outcome: %v, want claimAlreadyFinished", got)
}
if d.Outcome != before.Outcome || d.Warrant != before.Warrant || d.Attempt != before.Attempt || !d.At.Equal(before.At) {
t.Fatalf("decision returned: %+v, want unchanged %+v", d, before)
}
if len(emit.calls) != 0 {
t.Fatalf("emitted something for an already-finished send: %+v", emit.calls)
}
kept, _, ok, kerr := getSend(sends, req.ID)
if kerr != nil || !ok || kept.Decided == nil {
t.Fatalf("the send could not be read back: %v %v", kerr, ok)
}
if kept.Decided.Outcome != before.Outcome || kept.Decided.Warrant != before.Warrant || kept.Decided.Attempt != before.Attempt {
t.Fatalf("the record changed: %+v, want unchanged %+v", kept.Decided, before)
}
ids, terr := tellPendingIDs(sends)
if terr != nil || len(ids) != 0 {
t.Fatalf("a tell was queued for an already-finished send: %v %v", terr, ids)
}
})
}
}
// raceOnFirstUpdate performs one competing write the first time Update is called — simulating another
// delivery's claim landing first — so the caller's own compare-and-set loses with stdio.ErrChanged
// (novox/hq issue 1196, AC4).
type raceOnFirstUpdate struct {
SendStore
raced bool
race func()
}
func (r *raceOnFirstUpdate) Update(key string, value any, revision uint64) (uint64, error) {
if !r.raced {
r.raced = true
r.race()
}
return r.SendStore.Update(key, value, revision)
}
modules/mesh-mail/cmd/mesh-mail/config.go
+6 -22
View File
@@ -159,33 +159,17 @@ func (s Server) check(which string) error {
return nil
}
// SecretPath is where the account's own secret is placed in the secrets directory, as the manifest declares the
// family: <dir>/mail-credential-<account>.secret.
func SecretPath(dir string, a Account) string { return filepath.Join(dir, a.Secret()+".secret") }
// ReadSecret is the account's password, or given false when nobody gave it yet: the mesh never makes a member of the
// family, so a member not given is no file (ADR 0283 §6).
func ReadSecret(dir string, a Account) (string, bool, error) {
raw, err := os.ReadFile(SecretPath(dir, a))
if errors.Is(err, os.ErrNotExist) {
return "", false, nil
}
if err != nil {
return "", false, err
}
pw := strings.TrimRight(string(raw), "\r\n")
if pw == "" {
return "", false, nil
}
return pw, true, nil
}
// ReadSecret and SecretPath moved into cmd/mesh-mail/internal/secret (novox/hq ADR 0436 §12, issue 1193): only
// keep (imap.go) and the sender's own actor (sendactor.go) read a password, never the tools, which the operator
// account's tool runner serves and which must never open a password file.
// SecretGiven says whether the account's own secret was given.
//
// It looks at the file and never reads it: whoever judges whether a password was given has no need of the password.
// A file that is there and not empty was given; whether it signs in is the keeper's to say.
// A file that is there and not empty was given; whether it signs in is the keeper's to say. It computes the
// secret's path itself (not through the moved package) so the tools, which call this, never import it.
func SecretGiven(dir string, a Account) bool {
info, err := os.Stat(SecretPath(dir, a))
info, err := os.Stat(filepath.Join(dir, a.Secret()+".secret"))
return err == nil && info.Mode().IsRegular() && info.Size() > 0
}
modules/mesh-mail/cmd/mesh-mail/config_test.go
+7 -5
View File
@@ -6,6 +6,8 @@ import (
"reflect"
"strings"
"testing"
"mesh-mail/cmd/mesh-mail/internal/secret"
)
// The first stage's account, as the operator sets the setting `accounts` (ADR 0359 §5): its address and its server,
@@ -22,7 +24,7 @@ func TestTheFirstStagesAccountIsReadAndItsSecretNamed(t *testing.T) {
got[0].Incoming.Address() != "mail.novox.be:993" || got[0].Secret() != "mail-credential-mesh" || got[0].Outgoing != nil {
t.Fatalf("%+v", got)
}
if p := SecretPath("/state", got[0]); p != "/state/mail-credential-mesh.secret" {
if p := secret.Path("/state", got[0].Secret()); p != "/state/mail-credential-mesh.secret" {
t.Fatalf("the secret is placed at %s", p)
}
}
@@ -99,16 +101,16 @@ func TestASecretIsGivenByItsFileNotByReadingIt(t *testing.T) {
if SecretGiven(dir, a) {
t.Fatal("given before anything was written")
}
_ = os.WriteFile(SecretPath(dir, a), nil, 0o600)
_ = os.WriteFile(secret.Path(dir, a.Secret()), nil, 0o600)
if SecretGiven(dir, a) {
t.Fatal("an empty file is a password")
}
_ = os.WriteFile(SecretPath(dir, a), []byte("pw\n"), 0o600)
if pw, given, _ := ReadSecret(dir, a); !given || pw != "pw" || strings.Contains(pw, "\n") {
_ = os.WriteFile(secret.Path(dir, a.Secret()), []byte("pw\n"), 0o600)
if pw, given, _ := secret.Read(dir, a.Secret()); !given || pw != "pw" || strings.Contains(pw, "\n") {
t.Fatalf("%q %v", pw, given)
}
// Unreadable to the judge, as a secret owned by root is to the watch: still given.
if err := os.Chmod(SecretPath(dir, a), 0); err != nil {
if err := os.Chmod(secret.Path(dir, a.Secret()), 0); err != nil {
t.Fatal(err)
}
if !SecretGiven(dir, a) {
modules/mesh-mail/cmd/mesh-mail/health.go
+18
View File
@@ -105,6 +105,24 @@ func Judge(accounts []Account, accountsErr, readyErr error, given func(Account)
return health.Healthy(fmt.Sprintf("%d account(s) watched: %s", len(accounts), strings.Join(fine, "; ")))
}
// WithOverdue folds any overdue send (Overdue, send.go) into base: unhealthy, naming each one, whatever
// base itself said — an overdue send is never excused by an otherwise-healthy account (novox/hq ADR 0436
// clause 11, issue 1199).
func WithOverdue(base health.Answer, overdue []SendRequest) health.Answer {
if len(overdue) == 0 {
return base
}
names := make([]string, 0, len(overdue))
for _, req := range overdue {
names = append(names, fmt.Sprintf("%s (from %s)", req.ID, req.Account))
}
why := fmt.Sprintf("the send(s) %s have waited 10 minutes or more for the sender", strings.Join(names, ", "))
if base.Why != "" {
why = base.Why + "; and " + why
}
return health.Unhealthy(why, base.Waits...)
}
// Status is the watch's verdict as it leaves it: the answer, and when it looked.
type Status struct {
health.Answer
modules/mesh-mail/cmd/mesh-mail/health_test.go
+36
View File
@@ -114,3 +114,39 @@ func TestTheWatchJudgesFromTheFilesAndTheIndex(t *testing.T) {
t.Fatalf("the status file's shape: %s", raw)
}
}
// 1199/AC2: mail_health (the tool, not only Overdue or WithOverdue in isolation) turns unhealthy naming an
// overdue send, even while the watch's own verdict is healthy (novox/hq review round 1 finding 6).
func TestMailHealthToolTurnsUnhealthyNamingAnOverdueSend(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "health.json")
if err := writeStatus(path, Status{Answer: health.Healthy("1 account(s) watched"), At: judgedAt}); err != nil {
t.Fatal(err)
}
sends := newMemSends()
overdueID := "send-overdue1"
req := SendRequest{ID: overdueID, Account: "mesh", At: judgedAt}
req.Decided = &SendDecision{At: judgedAt.Add(-20 * time.Minute), Outcome: "queued"}
if _, err := sends.Create(req.ID, req); err != nil {
t.Fatal(err)
}
if err := addSenderPending(sends, req.ID); err != nil {
t.Fatal(err)
}
b := &Bundle{Env: Env{Status: path}, Sends: sends, Now: func() time.Time { return judgedAt }}
out, err := b.healthTool(nil)
if err != nil {
t.Fatalf("mail_health: %v", err)
}
a, ok := out.(health.Answer)
if !ok {
t.Fatalf("mail_health answered %T, want health.Answer", out)
}
if a.State != health.StateUnhealthy {
t.Fatalf("mail_health: %+v, want unhealthy", a)
}
if !strings.Contains(a.Why, overdueID) {
t.Fatalf("mail_health's why does not name the overdue send: %q", a.Why)
}
}
modules/mesh-mail/cmd/mesh-mail/imap.go
+3 -1
View File
@@ -22,6 +22,8 @@ import (
"github.com/emersion/go-imap/v2"
"github.com/emersion/go-imap/v2/imapclient"
"mesh-mail/cmd/mesh-mail/internal/secret"
)
// How the keeper paces itself. IdleCycle is how long one IDLE lasts before it is ended and begun again, which proves
@@ -170,7 +172,7 @@ func (k *Keeper) Watch(ctx context.Context, a Account) {
retry := orDefault(k.RetryFirst, RetryFirst)
var heard, synced time.Time
for ctx.Err() == nil {
pw, given, err := ReadSecret(k.SecretsDir, a)
pw, given, err := secret.Read(k.SecretsDir, a.Secret())
switch {
case err != nil:
k.say(ctx, a, AccountFailing, "its password was given and cannot be read: "+err.Error(), heard, synced)
modules/mesh-mail/cmd/mesh-mail/imap_test.go
+6 -3
View File
@@ -221,9 +221,12 @@ func TestNothingButTheTestsWritesToAMailbox(t *testing.T) {
}
writes := regexp.MustCompile(`([A-Za-z_][A-Za-z0-9_]*)\.(Store|Move|Copy|Expunge|UIDExpunge|Append|Create|Delete|Rename|Subscribe|Unsubscribe|SetMetadata|SetQuota)\(`)
for _, f := range files {
if strings.HasSuffix(f, "_test.go") || f == "send.go" || f == "smtp.go" {
// Neither touches a mailbox: send.go keeps the `sends` state on the bus, never IMAP; smtp.go speaks SMTP
// to the account's outgoing server, never to a mailbox, and only on the operator's Approve (ADR 0361 §2).
if strings.HasSuffix(f, "_test.go") || f == "send.go" || f == "smtp.go" || f == "sender.go" {
// None touches a mailbox: send.go keeps the `sends` state on the bus, never IMAP; smtp.go speaks SMTP
// to the account's outgoing server, never to a mailbox, and only on the operator's Approve (ADR 0361
// §2); sender.go is the sender process (novox/hq ADR 0436 §7) — its own bus account's `sends` state
// (process.State.Create, matched here only because its receiver is named s) and router record, never
// IMAP either.
continue
}
raw, err := os.ReadFile(f)
modules/mesh-mail/cmd/mesh-mail/internal/secret/secret.go
+36
View File
@@ -0,0 +1,36 @@
// Package secret reads an account's own password from the secrets directory (novox/hq ADR 0283 §6). It is
// imported only by mesh-mail's keeper (which signs in to IMAP as root) and its sender (which signs in to SMTP
// as its own account, mesh-mail-send): never by the tools, which the operator account's tool runner serves and
// which must never open a password file (novox/hq ADR 0436 §12, issue 1193). Moved out of package main's own
// files so a test (secret_boundary_test.go) can prove the tools' bundle (main.go's serve path, tools.go,
// send.go, health.go) never imports it.
package secret
import (
"errors"
"os"
"path/filepath"
"strings"
)
// Path is where an account's own secret is placed in the secrets directory, given the family member name
// (mail-credential-<account>): <dir>/<member>.secret.
func Path(dir, member string) string { return filepath.Join(dir, member+".secret") }
// Read is the member's password, or given false when nobody gave it yet: the mesh never makes a member of the
// family, so a member not given is no file (ADR 0283 §6). A directory that does not exist at all answers the
// same as a file that does not exist: given false, no error (novox/hq issue 1193, AC3).
func Read(dir, member string) (string, bool, error) {
raw, err := os.ReadFile(Path(dir, member))
if errors.Is(err, os.ErrNotExist) {
return "", false, nil
}
if err != nil {
return "", false, err
}
pw := strings.TrimRight(string(raw), "\r\n")
if pw == "" {
return "", false, nil
}
return pw, true, nil
}
modules/mesh-mail/cmd/mesh-mail/kit_test.go
+3 -1
View File
@@ -22,6 +22,8 @@ import (
"github.com/emersion/go-imap/v2/imapclient"
"github.com/emersion/go-imap/v2/imapserver"
"github.com/emersion/go-imap/v2/imapserver/imapmemserver"
"mesh-mail/cmd/mesh-mail/internal/secret"
)
func TestMain(m *testing.M) {
@@ -134,7 +136,7 @@ func testAccount() Account {
func giveSecret(t *testing.T, dir string, a Account, pw string) {
t.Helper()
if err := os.WriteFile(SecretPath(dir, a), []byte(pw+"\n"), 0o600); err != nil {
if err := os.WriteFile(secret.Path(dir, a.Secret()), []byte(pw+"\n"), 0o600); err != nil {
t.Fatal(err)
}
}
modules/mesh-mail/cmd/mesh-mail/main.go
+43 -4
View File
@@ -9,12 +9,16 @@
// mesh-mail keep acts: watches every account (IDLE on the inbox, a look at the other folders every five
// minutes), keeps the index and the attachment store, and writes each change's event beside it
// mesh-mail watch judges: reads every minute how each account is, and leaves its verdict for mail_health
// mesh-mail send the sender (novox/hq ADR 0436 §7): its own bus account, consuming the work queue
// mail-sends; it alone reads a password, and performs the one send an operator's
// Approve authorises, once the tools have recorded it approved and queued it
//
// The two long-running processes are the act and the judge, as mesh-knowledge's are: keep is judged by its unit, watch
// by the tool mail_health (ADR 0359 §6, ADR 0240 §3 and §7). Only the bundle reaches the bus, through the runtime that
// launched it, so the keeper hands its events over in the index, written in the same step as the change they say,
// and the bundle says them and removes each once the bus took it. stdout is the bundle's MCP channel; what this module
// says, it says on stderr.
// by the tool mail_health (ADR 0359 §6, ADR 0240 §3 and §7); send is judged by its unit too (it holds a bus account
// of its own, never a seat, so no tool answers for it). Only the bundle reaches the bus through the runtime that
// launched it; send reaches it on its own account instead (mesh-sdk go/process), so the keeper hands its events over
// in the index, written in the same step as the change they say, and the bundle says them and removes each once the
// bus took it. stdout is the bundle's MCP channel; what this module says, it says on stderr.
package main
import (
@@ -64,6 +68,8 @@ func main() {
os.Exit(keep(envFromOS()))
case "watch":
os.Exit(watch(envFromOS()))
case "send":
os.Exit(send(envFromOS()))
}
}
serve(envFromOS())
@@ -185,12 +191,45 @@ func serve(env Env) {
go p.Run(context.Background())
go hear(b)
go sweepSends(b)
go waitsTellTicker(b)
if err := stdio.Serve("", b.Tools()); err != nil {
logf("%v", err)
os.Exit(1)
}
}
// sdkTellOperator is the tools' own TellOperator (send.go), wrapping stdio.Ask to the operator channel's
// tell verb, exactly as mesh-files' own pattern (modules/mesh-files/cmd/mesh-files/main.go tellOperator):
// no lines, no subject naming anything the content rule would refuse.
func sdkTellOperator(headline string) error {
_, err := stdio.Ask("seat:operator-channel.tell", map[string]any{"headline": headline, "lines": []string{}, "subject": "mesh-mail"})
return err
}
// WaitsTellEvery is how often the tools look for an overdue send still owing their own "waits" tell
// (novox/hq ADR 0436 clause 11, issue 1199): a second ticker, apart from sweepSends, since the two tells —
// the sender's own failure, the tools' own overdue wait — are never the same look.
const WaitsTellEvery = time.Minute
// waitsTellTicker runs the tools' own "waits" tell for every overdue send, at least every WaitsTellEvery,
// until the process ends (novox/hq ADR 0436 clause 11, issue 1199).
func waitsTellTicker(b *Bundle) {
tick := time.NewTicker(WaitsTellEvery)
defer tick.Stop()
for {
overdue, err := Overdue(b.sends(), time.Now())
if err != nil {
logf("[mesh-mail] the overdue check cannot be read: %v", err)
}
for _, req := range overdue {
if err := tellWaits(b.sends(), sdkTellOperator, req.ID, time.Now()); err != nil {
logf("[mesh-mail] telling the operator that %s waits: %v", req.ID, err)
}
}
<-tick.C
}
}
// hear subscribes to the router's word on this module's own asks, asked again until the runtime takes it, exactly
// as mesh-knowledge does. Each warrant heard goes to Heard, the same Decide that mail_send_status uses: an
// approved send goes out as soon as the operator answers it, never only when something later calls
modules/mesh-mail/cmd/mesh-mail/manifest_test.go
+144 -8
View File
@@ -30,11 +30,16 @@ type manifest struct {
OwnSecrets map[string]map[string]string `json:"own-secrets"`
Tools []string `json:"tools"`
Emits []string `json:"emits"`
State []string `json:"state"`
Streams []struct {
Name string `json:"name"`
Events []string `json:"events"`
MaxAgeDays int `json:"max-age-days"`
Grant struct {
Name string `json:"name"`
Events []string `json:"events"`
MaxAgeDays int `json:"max-age-days"`
MaxMiB int `json:"max-mib"`
Retention string `json:"retention"`
MaxDeliveries int `json:"max-deliveries"`
Duplicates int `json:"duplicates"`
Grant struct {
Config string `json:"config"`
Events string `json:"events"`
} `json:"grant"`
@@ -80,7 +85,7 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
t.Fatalf("the slug %q overflows an S3 access key", m.Slug)
}
fam := m.OwnSecrets[SecretFamily+"*"]
if fam["issued-by"] != "outside" || fam["path"] != "${dir:state}/"+SecretFamily+"*.secret" || len(m.OwnSecrets) != 1 {
if fam["issued-by"] != "outside" || fam["path"] != "${dir:state}/"+SecretFamily+"*.secret" || len(m.OwnSecrets) != 2 {
t.Fatalf("own secrets %v", m.OwnSecrets)
}
var names []string
@@ -95,13 +100,35 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
t.Fatalf("tools %v, the code serves %v", m.Tools, names)
}
// Its events go to its own stream MAIL, kept a week, each account's part heard only by the consumers the trusted
// config value grants names for it (ADR 0359 §2, §4; ADR 0363): none on the shared EVENTS stream.
// config value grants names for it (ADR 0359 §2, §4; ADR 0363): none on the shared EVENTS stream. The module's
// top-level emits is the sender's two outcomes (novox/hq ADR 0436 §7, §11, issue 1172): the module as a whole may
// publish them, though only the process send actually does (its own bus.emits, checked in
// TestItsManifestDeclaresTheSenderProcess).
var events []string
for _, c := range Changes {
events = append(events, EventKey("*", c))
}
if len(m.Emits) != 0 || len(m.Streams) != 1 || m.Streams[0].Name != "mail" || !reflect.DeepEqual(m.Streams[0].Events, events) ||
m.Streams[0].MaxAgeDays != 7 || m.Streams[0].Grant.Config != "grants" || m.Streams[0].Grant.Events != "mail.{key}.**" {
var mailStream *struct {
Name string `json:"name"`
Events []string `json:"events"`
MaxAgeDays int `json:"max-age-days"`
MaxMiB int `json:"max-mib"`
Retention string `json:"retention"`
MaxDeliveries int `json:"max-deliveries"`
Duplicates int `json:"duplicates"`
Grant struct {
Config string `json:"config"`
Events string `json:"events"`
} `json:"grant"`
}
for i := range m.Streams {
if m.Streams[i].Name == "mail" {
mailStream = &m.Streams[i]
}
}
if !reflect.DeepEqual(m.Emits, []string{"mail.send.sent", "mail.send.failed"}) || len(m.Streams) != 2 || mailStream == nil ||
!reflect.DeepEqual(mailStream.Events, events) || mailStream.MaxAgeDays != 7 || mailStream.Grant.Config != "grants" ||
mailStream.Grant.Events != "mail.{key}.**" {
t.Fatalf("emits %v, streams %+v; the code says %v", m.Emits, m.Streams, events)
}
// It calls nothing and consumes nothing: mesh-mail-notifier (issue 800 part B) is the one that invokes and
@@ -187,3 +214,112 @@ func TestItsProcessesAreThePreparationTheActAndTheJudge(t *testing.T) {
t.Fatalf("keep restarts on %v", keep["restart-on"])
}
}
// 1172/AC2: the manifest declares the sender's account, own-secrets owner, work-queue stream and process bus,
// and widens the state directory and accounts.json so the sender (not the operator account) can read them,
// mirroring mesh-controller's own fixtures (internal/catalogue/process_bus_test.go:14-31,
// own_secret_owner_test.go:16-29, stream_work_queue_test.go:13-14).
func TestItsManifestDeclaresTheSenderProcess(t *testing.T) {
m, byID := readManifest(t)
account := byID["account"]
if account == nil || account["type"] != "user" || account["name"] != "mesh-mail-send" {
t.Fatalf("the sender's account resource: %v", account)
}
fam := m.OwnSecrets[SecretFamily+"*"]
if fam["owner"] != "mesh-mail-send" {
t.Fatalf("mail-credential-*'s owner: %v; want mesh-mail-send", fam["owner"])
}
// The sender's own bus account is delivered as the own secret broker-<process id>, never the module's
// own `broker` (novox/hq ADR 0436 §2): without it, the controller's mayIssue refuses to issue the
// module at all, naming the process and the secret (review round 1, finding 1).
broker := m.OwnSecrets["broker-send"]
if broker["path"] != "${dir:state}/broker-send.secret" || broker["owner"] != "mesh-mail-send" {
t.Fatalf("broker-send: %v; want path ${dir:state}/broker-send.secret, owner mesh-mail-send", broker)
}
var sendsStream *struct {
Name string `json:"name"`
Events []string `json:"events"`
MaxAgeDays int `json:"max-age-days"`
MaxMiB int `json:"max-mib"`
Retention string `json:"retention"`
MaxDeliveries int `json:"max-deliveries"`
Duplicates int `json:"duplicates"`
Grant struct {
Config string `json:"config"`
Events string `json:"events"`
} `json:"grant"`
}
for i := range m.Streams {
if m.Streams[i].Name == "mail-sends" {
sendsStream = &m.Streams[i]
}
}
if sendsStream == nil || !reflect.DeepEqual(sendsStream.Events, []string{"mail.send.approved"}) ||
sendsStream.MaxAgeDays != 1 || sendsStream.MaxMiB != 64 || sendsStream.Retention != "work-queue" ||
sendsStream.MaxDeliveries != 5 || sendsStream.Duplicates != 24 {
t.Fatalf("the stream mail-sends: %+v", sendsStream)
}
if !reflect.DeepEqual(m.Emits, []string{"mail.send.sent", "mail.send.failed"}) {
t.Fatalf("top-level emits: %v", m.Emits)
}
send := byID["send"]
if send == nil || send["type"] != "process" || send["user"] != "mesh-mail-send" {
t.Fatalf("the process send: %v", send)
}
run, _ := send["run"].([]any)
if len(run) != 2 || run[0] != "./mesh-mail" || run[1] != "send" {
t.Fatalf("send's run: %v", send["run"])
}
env, _ := send["env"].(map[string]any)
keepEnv, _ := byID["keep"]["env"].(map[string]any)
for _, k := range []string{"MESH_MAIL_ACCOUNTS", "MESH_MAIL_SECRETS"} {
if env[k] != keepEnv[k] {
t.Fatalf("send's %s is %v, keep's is %v", k, env[k], keepEnv[k])
}
}
h, _ := send["health"].(map[string]any)
if h["kind"] != "unit" {
t.Fatalf("send's health: %v", send["health"])
}
bus, _ := send["bus"].(map[string]any)
if bus == nil {
t.Fatalf("send declares no bus")
}
if !reflect.DeepEqual(toStrings(bus["emits"]), []string{"mail.send.sent", "mail.send.failed"}) ||
!reflect.DeepEqual(toStrings(bus["queues"]), []string{"mail-sends"}) ||
!reflect.DeepEqual(toStrings(bus["state"]), []string{"sends"}) || bus["asks"] != true ||
!reflect.DeepEqual(toStrings(bus["invokes"]), []string{"seat:operator-channel.tell"}) {
t.Fatalf("send's bus: %+v", bus)
}
// On novox the operator account's state directory is 0700, so mesh-mail-send cannot reach its own 0600
// files inside it: the state directory becomes 0711 (passable, not listable) and accounts.json (server
// names and addresses, no password) becomes 0644, so the sender can read the outgoing server. No password
// file's mode widens (the lead's decision, issue 1172).
if byID["state"]["mode"] != "0711" {
t.Fatalf("the state directory's mode: %v; want 0711", byID["state"]["mode"])
}
if byID["accounts"]["mode"] != "0644" {
t.Fatalf("accounts.json's mode: %v; want 0644", byID["accounts"]["mode"])
}
if byID["grants"]["mode"] != "0600" {
t.Fatalf("grants.json's mode widened to %v; it holds no password, but was not asked to widen", byID["grants"]["mode"])
}
}
func toStrings(v any) []string {
list, _ := v.([]any)
out := make([]string, 0, len(list))
for _, item := range list {
if s, ok := item.(string); ok {
out = append(out, s)
}
}
return out
}
modules/mesh-mail/cmd/mesh-mail/outcome_test.go
+308
View File
@@ -0,0 +1,308 @@
package main
// recordOutcome writes the final decision for one claimed send and says what the caller does with the
// queue message next (novox/hq ADR 0436 clauses 10, 11, issue 1198).
import (
"errors"
"strings"
"testing"
"time"
)
// emitted is one call recordOutcome's Emit made.
type emitted struct {
eventType, id string
body OutcomeEvent
}
// memEmit collects every emit call, for a test to check the event and its body.
type memEmit struct{ calls []emitted }
func (m *memEmit) emit(eventType, id string, body any) error {
b, _ := body.(OutcomeEvent)
m.calls = append(m.calls, emitted{eventType: eventType, id: id, body: b})
return nil
}
// outcomeTestClaim keeps one send claimed "sending" at the given attempt, returning its request, the
// claimed decision and the revision recordOutcome needs.
func outcomeTestClaim(t *testing.T, attempt int) (SendRequest, SendDecision, uint64, *memSends) {
t.Helper()
sends := newMemSends()
req := SendRequest{ID: newSendID(), Account: "mesh", To: []string{"ops@example.org"}, Subject: "s", Body: "body", At: judgedAt}
claim := SendDecision{At: judgedAt.Add(time.Minute), Outcome: "sending", Warrant: "the operator, via telegram, chose Send", Attempt: attempt}
req.Decided = &claim
rev, err := sends.Create(req.ID, req)
if err != nil {
t.Fatal(err)
}
return req, claim, rev, sends
}
// 1198/AC1: given a claimed send the server accepts, when sent, then sent is recorded, mail.send.sent is
// emitted (send id, account, outcome in words; no recipient, subject or body), and only then — by
// construction, since the caller acknowledges only once this call returns — the message is acknowledged.
func TestRecordOutcomeOnSuccessRecordsSentAndEmitsIt(t *testing.T) {
req, claimed, rev, sends := outcomeTestClaim(t, 1)
emit := &memEmit{}
d, action, _, err := recordOutcome(sends, emit.emit, req, claimed, rev, nil, judgedAt.Add(2*time.Minute))
if err != nil {
t.Fatalf("recordOutcome: %v", err)
}
if d.Outcome != "sent" {
t.Fatalf("outcome: %+v", d)
}
if action != actionAck {
t.Fatalf("action: %v, want actionAck", action)
}
kept, _, ok, err := getSend(sends, req.ID)
if err != nil || !ok || kept.Decided == nil || kept.Decided.Outcome != "sent" {
t.Fatalf("not kept as sent: %v %v %+v", err, ok, kept.Decided)
}
if len(emit.calls) != 1 || emit.calls[0].eventType != MailSendSent || emit.calls[0].id != req.ID {
t.Fatalf("emitted: %+v", emit.calls)
}
body := emit.calls[0].body
if body.Send != req.ID || body.Account != "mesh" || body.Outcome == "" {
t.Fatalf("the event's body: %+v", body)
}
whole := body.Send + body.Account + body.Outcome
for _, forbidden := range []string{"ops@example.org", "s", "body"} {
if forbidden == "s" {
continue // too short to search for meaningfully without false positives in the id/account
}
if strings.Contains(whole, forbidden) {
t.Fatalf("the emitted body carries %q: %+v", forbidden, body)
}
}
}
// 1198/AC2: given a not-taken failure on attempt n (n < 5), then retrying is recorded with attempt, reason
// and next attempt, and the caller is told to ask the bus to deliver it again after 1, 5, 15, 30 minutes
// for n = 1..4; on the fifth, failed is recorded and mail.send.failed emitted.
func TestRecordOutcomeOnNotTakenRetriesUpToTheFourthAttempt(t *testing.T) {
for n, want := range map[int]time.Duration{1: time.Minute, 2: 5 * time.Minute, 3: 15 * time.Minute, 4: 30 * time.Minute} {
req, claimed, rev, sends := outcomeTestClaim(t, n)
emit := &memEmit{}
now := judgedAt.Add(2 * time.Minute)
d, action, delay, err := recordOutcome(sends, emit.emit, req, claimed, rev, &SendError{Outcome: NotTaken, Err: errors.New("no connection")}, now)
if err != nil {
t.Fatalf("attempt %d: recordOutcome: %v", n, err)
}
if d.Outcome != "retrying" || d.Attempt != n {
t.Fatalf("attempt %d: outcome %+v", n, d)
}
if !d.NextAttempt.Equal(now.Add(want)) {
t.Fatalf("attempt %d: next attempt %v, want %v", n, d.NextAttempt, now.Add(want))
}
if action != actionRetry || delay != want {
t.Fatalf("attempt %d: action %v, delay %v, want actionRetry, %v", n, action, delay, want)
}
if len(emit.calls) != 0 {
t.Fatalf("attempt %d: retrying emitted something: %+v", n, emit.calls)
}
}
}
func TestRecordOutcomeOnNotTakenFailsOnTheFifthAttempt(t *testing.T) {
req, claimed, rev, sends := outcomeTestClaim(t, MostAttempts)
emit := &memEmit{}
d, action, _, err := recordOutcome(sends, emit.emit, req, claimed, rev, &SendError{Outcome: NotTaken, Err: errors.New("no connection")},
judgedAt.Add(2*time.Minute))
if err != nil {
t.Fatalf("recordOutcome: %v", err)
}
if !strings.HasPrefix(d.Outcome, "failed: ") || d.Attempt != MostAttempts {
t.Fatalf("outcome: %+v", d)
}
if action != actionTerm {
t.Fatalf("action: %v, want actionTerm", action)
}
if len(emit.calls) != 1 || emit.calls[0].eventType != MailSendFailed {
t.Fatalf("emitted: %+v", emit.calls)
}
}
// 1198/AC3: given a refusal, then failed is recorded with the reason, mail.send.failed emitted, and the
// message terminated; given not proven, the same with not proven.
func TestRecordOutcomeOnRefusedRecordsFailedAndTerminates(t *testing.T) {
req, claimed, rev, sends := outcomeTestClaim(t, 1)
emit := &memEmit{}
d, action, _, err := recordOutcome(sends, emit.emit, req, claimed, rev, &SendError{Outcome: Refused, Err: errors.New("550 no such mailbox")},
judgedAt.Add(2*time.Minute))
if err != nil {
t.Fatalf("recordOutcome: %v", err)
}
if !strings.HasPrefix(d.Outcome, "failed: ") || !strings.Contains(d.Outcome, "550 no such mailbox") {
t.Fatalf("outcome: %+v", d)
}
if action != actionTerm {
t.Fatalf("action: %v, want actionTerm", action)
}
if len(emit.calls) != 1 || emit.calls[0].eventType != MailSendFailed || emit.calls[0].body.Outcome != d.Outcome {
t.Fatalf("emitted: %+v", emit.calls)
}
}
func TestRecordOutcomeOnNotProvenRecordsNotProvenAndTerminates(t *testing.T) {
req, claimed, rev, sends := outcomeTestClaim(t, 1)
emit := &memEmit{}
d, action, _, err := recordOutcome(sends, emit.emit, req, claimed, rev,
&SendError{Outcome: NotProven, Err: errors.New("connection lost after the data was written")}, judgedAt.Add(2*time.Minute))
if err != nil {
t.Fatalf("recordOutcome: %v", err)
}
if d.Outcome != "not proven" {
t.Fatalf("outcome: %+v", d)
}
if action != actionTerm {
t.Fatalf("action: %v, want actionTerm", action)
}
if len(emit.calls) != 1 || emit.calls[0].eventType != MailSendFailed || emit.calls[0].body.Outcome != "not proven" {
t.Fatalf("emitted: %+v", emit.calls)
}
}
// review round 1 finding 3: a failed write of the tells-pending index does not stop the failed decision
// itself from being recorded and emitted — only the index write is reported as failing.
func TestRecordOutcomeStillRecordsFailedWhenTheTellsPendingIndexCannotBeWritten(t *testing.T) {
req, claimed, rev, sends := outcomeTestClaim(t, 1)
sends.failKey, sends.failErr = tellsPendingKey, errors.New("the store is down")
emit := &memEmit{}
d, action, _, err := recordOutcome(sends, emit.emit, req, claimed, rev, &SendError{Outcome: Refused, Err: errors.New("550 no such mailbox")},
judgedAt.Add(2*time.Minute))
if err == nil {
t.Fatalf("the index failure was not reported")
}
if !strings.HasPrefix(d.Outcome, "failed: ") || action != actionTerm {
t.Fatalf("outcome %+v, action %v", d, action)
}
kept, _, ok, kerr := getSend(sends, req.ID)
if kerr != nil || !ok || kept.Decided == nil || kept.Decided.Outcome != d.Outcome {
t.Fatalf("the decision itself was not kept despite the index failing: %v %v %+v", kerr, ok, kept.Decided)
}
if len(emit.calls) != 1 {
t.Fatalf("emitted %d time(s) despite the index failing", len(emit.calls))
}
}
// review round 1 finding 5: if writing "sent" cannot be kept (even retried on a changed revision), the
// message is never acknowledged and mail.send.sent is never emitted — a mail actually sent is never
// silently announced successful when the record never actually carried it.
func TestRecordOutcomeOnSentDoesNotAckOrEmitWhenTheWriteCannotBeKept(t *testing.T) {
req, claimed, rev, sends := outcomeTestClaim(t, 1)
sends.failKey, sends.failErr = req.ID, errors.New("the store is down")
emit := &memEmit{}
d, action, _, err := recordOutcome(sends, emit.emit, req, claimed, rev, nil, judgedAt.Add(2*time.Minute))
if err == nil {
t.Fatalf("the write failure was not reported")
}
if action != actionLeave {
t.Fatalf("action: %v, want actionLeave (left to the bus, never acked)", action)
}
if len(emit.calls) != 0 {
t.Fatalf("emitted mail.send.sent despite the write never being kept: %+v", emit.calls)
}
// The kept record is unaffected: still "sending", as claimQueuedSend originally left it.
kept, _, ok, kerr := getSend(sends, req.ID)
if kerr != nil || !ok || kept.Decided == nil || kept.Decided.Outcome != "sending" {
t.Fatalf("the record changed despite the write failing: %v %v %+v (returned %+v)", kerr, ok, kept.Decided, d)
}
}
// Same finding, for a failed outcome (refused): the message is left to the bus, never terminated, and
// mail.send.failed is never emitted, when the write itself cannot be kept.
func TestRecordOutcomeOnRefusedDoesNotTerminateOrEmitWhenTheWriteCannotBeKept(t *testing.T) {
req, claimed, rev, sends := outcomeTestClaim(t, 1)
sends.failKey, sends.failErr = req.ID, errors.New("the store is down")
emit := &memEmit{}
_, action, _, err := recordOutcome(sends, emit.emit, req, claimed, rev, &SendError{Outcome: Refused, Err: errors.New("550 no such mailbox")},
judgedAt.Add(2*time.Minute))
if err == nil {
t.Fatalf("the write failure was not reported")
}
if action != actionLeave {
t.Fatalf("action: %v, want actionLeave", action)
}
if len(emit.calls) != 0 {
t.Fatalf("emitted mail.send.failed despite the write never being kept: %+v", emit.calls)
}
}
// 1198/AC1 (review round 2): a store that changes the revision between the claim and the outcome — another
// writer's decision landing in between — is retried once with the fresh revision; recordOutcome's own
// decision, once it does land, is what is kept, acked and emitted, never the other writer's state read
// back with a nil error mistaken for success.
func TestRecordOutcomeRetriesOnceWhenTheRevisionChangedBetweenClaimAndOutcome(t *testing.T) {
req, claimed, rev, sends := outcomeTestClaim(t, 1)
racy := &raceOnFirstUpdate{SendStore: sends, race: func() {
// A different delivery's own claim lands first, bumping the revision without ever finishing this
// call's own write — settleSend (via updateDecision) answers that with revision 0 and a nil error.
r, rv, ok, err := getSend(sends, req.ID)
if err != nil || !ok {
t.Fatalf("racing write could not read the send: %v %v", err, ok)
}
if _, _, err := settleSend(sends, r, rv, SendDecision{At: judgedAt, Outcome: "sending", Attempt: 2}); err != nil {
t.Fatalf("racing write: %v", err)
}
}}
emit := &memEmit{}
d, action, _, err := recordOutcome(racy, emit.emit, req, claimed, rev, nil, judgedAt.Add(2*time.Minute))
if err != nil {
t.Fatalf("recordOutcome: %v", err)
}
if d.Outcome != "sent" || action != actionAck {
t.Fatalf("outcome %+v, action %v, want sent/actionAck once retried with the fresh revision", d, action)
}
if len(emit.calls) != 1 || emit.calls[0].eventType != MailSendSent || emit.calls[0].body.Outcome != "sent" {
t.Fatalf("emitted: %+v", emit.calls)
}
kept, _, ok, kerr := getSend(sends, req.ID)
if kerr != nil || !ok || kept.Decided == nil || kept.Decided.Outcome != "sent" {
t.Fatalf("the kept record is not this call's own decision: %v %v %+v", kerr, ok, kept.Decided)
}
}
// Same finding: when the race repeats on the retry too (a second writer keeps landing first), recordOutcome
// gives up rather than looping or mistaking the other writer's state for its own — actionLeave, nothing
// emitted, the error reported.
func TestRecordOutcomeGivesUpWhenTheRevisionKeepsChanging(t *testing.T) {
req, claimed, rev, sends := outcomeTestClaim(t, 1)
always := &raceEveryUpdate{SendStore: sends, race: func() {
r, rv, ok, err := getSend(sends, req.ID)
if err != nil || !ok {
t.Fatalf("racing write could not read the send: %v %v", err, ok)
}
if _, _, err := settleSend(sends, r, rv, SendDecision{At: judgedAt, Outcome: "sending", Attempt: r.Decided.Attempt + 1}); err != nil {
t.Fatalf("racing write: %v", err)
}
}}
emit := &memEmit{}
_, action, _, err := recordOutcome(always, emit.emit, req, claimed, rev, nil, judgedAt.Add(2*time.Minute))
if err == nil {
t.Fatalf("a persistently changing revision was not reported")
}
if action != actionLeave {
t.Fatalf("action: %v, want actionLeave", action)
}
if len(emit.calls) != 0 {
t.Fatalf("emitted despite never keeping its own write: %+v", emit.calls)
}
}
// raceEveryUpdate performs the competing write before every Update call, not only the first — simulating a
// revision that keeps changing out from under every retry (novox/hq review round 2, issue 1198).
type raceEveryUpdate struct {
SendStore
race func()
}
func (r *raceEveryUpdate) Update(key string, value any, revision uint64) (uint64, error) {
r.race()
return r.SendStore.Update(key, value, revision)
}
modules/mesh-mail/cmd/mesh-mail/sdk_pin_test.go
+31
View File
@@ -0,0 +1,31 @@
package main
// The SDK pin a process's own bus connection needs (novox/hq ADR 0436 §4, issue 1172): mesh-mail's go.mod
// requires mesh-sdk go at the release-2 commit (main 34392e7) and go 1.26.0, and go/process is importable.
// The blank import below is what proves importability at compile time; go vet (run before every commit, per
// the context file) already fails the build if it is not.
import (
"os"
"strings"
"testing"
_ "git.novox.be/novox/mesh-sdk/go/process"
)
// 1172/AC1: given mesh-mail's go.mod, when built, then it requires mesh-sdk go at
// v0.1.14-0.20261019062050-34392e74916c and go 1.26.0, and go/process is importable (the import above).
func TestGoModPinsTheSDKsProcessRelease(t *testing.T) {
raw, err := os.ReadFile("../../go.mod")
if err != nil {
t.Fatal(err)
}
mod := string(raw)
const wantSDK = "git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019062050-34392e74916c"
if !strings.Contains(mod, wantSDK) {
t.Fatalf("go.mod does not require %q:\n%s", wantSDK, mod)
}
if !strings.Contains(mod, "go 1.26.0") {
t.Fatalf("go.mod does not declare go 1.26.0:\n%s", mod)
}
}
modules/mesh-mail/cmd/mesh-mail/secret_boundary_test.go
+124
View File
@@ -0,0 +1,124 @@
package main
// The tools never open a password file (novox/hq ADR 0436 §12, issue 1193): ReadSecret and SecretPath moved
// into cmd/mesh-mail/internal/secret, imported only by keep (imap.go) and — once issue 1195 adds it — the
// sender's own process (sender.go). Since issue 1194, Decide no longer reaches any actor that reads a secret
// at all (it records approved and queues the send for the sender); the import list this test checks still
// holds, now even more plainly. The build seat runs every test under one account, so no permission test can
// show the split (context file, "Clause 12's proof"); this is the file-level proxy the lead chose instead: an
// AST check over the tools' own files.
import (
"encoding/json"
"go/parser"
"go/token"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
// secretPackage is the import path the tools' bundle must never carry.
const secretPackage = "mesh-mail/cmd/mesh-mail/internal/secret"
// 1193/AC1: none of the tools' bundle files (main.go's serve path, tools.go, send.go, health.go) imports the
// password-reading package; the failure names the offending file.
func TestTheToolsBundleNeverImportsThePasswordPackage(t *testing.T) {
for _, file := range []string{"main.go", "tools.go", "send.go", "health.go"} {
fset := token.NewFileSet()
f, err := parser.ParseFile(fset, file, nil, parser.ImportsOnly)
if err != nil {
t.Fatalf("%s: %v", file, err)
}
for _, imp := range f.Imports {
path := strings.Trim(imp.Path.Value, `"`)
if path == secretPackage {
t.Errorf("%s imports %s, which only keep and the sender's actor may (novox/hq ADR 0436 §12)", file, secretPackage)
}
}
}
}
// 1193/AC2: keep (imap.go) and the sender (sender.go, issue 1195) read the password through the moved
// package; SecretGiven (stat only) stays in config.go, available to the tools, and never reads the password
// itself. (sender.go's own account file is read through secret.Path too, though its own account is not a
// password; the real per-message password read is issue 1196's.)
func TestKeepAndTheSenderReadThePasswordThroughTheMovedPackage(t *testing.T) {
for _, file := range []string{"imap.go", "sender.go"} {
fset := token.NewFileSet()
f, err := parser.ParseFile(fset, file, nil, parser.ImportsOnly)
if err != nil {
t.Fatalf("%s: %v", file, err)
}
found := false
for _, imp := range f.Imports {
if strings.Trim(imp.Path.Value, `"`) == secretPackage {
found = true
}
}
if !found {
t.Errorf("%s does not import %s", file, secretPackage)
}
}
// SecretGiven itself (config.go) never calls ReadFile, os.Open or the moved package: whether a password was
// given is read from the file's presence and size alone (issue 733, kept by config_test.go's own check too).
src, err := os.ReadFile("config.go")
if err != nil {
t.Fatal(err)
}
body := string(src)
body = body[strings.Index(body, "func SecretGiven("):]
body = body[:strings.Index(body, "\n}\n")]
for _, bad := range []string{"ReadSecret", "ReadFile", "os.Open", "secret.Read"} {
if strings.Contains(body, bad) {
t.Fatalf("SecretGiven reads the password (%s):\n%s", bad, body)
}
}
}
// 1193/AC3: with the secrets directory absent, mail_send raises its ask, the warrant of an Approve is heard,
// and mail_send_status reads it back — each succeeds, and no error mentions a secret file. Since issue 1194
// the tools never read a secret on this path at all (they queue the send for the sender), so this also proves
// the stronger property the tools' bundle never touches b.Env.Secrets.
func TestTheToolPathSucceedsWithTheSecretsDirectoryAbsent(t *testing.T) {
dir := t.TempDir()
missing := filepath.Join(dir, "never-created")
env := Env{Accounts: filepath.Join(dir, "accounts.json"), Grants: filepath.Join(dir, "grants.json"), Secrets: missing}
writeFile(t, env.Accounts, `{"accounts": {"mesh": {"kind": "imap", "address": "jschoubben@novox.be",
"incoming": {"host": "mail.novox.be", "port": 993, "security": "tls"},
"outgoing": {"host": "h", "port": 465, "security": "tls"}}}}`)
writeFile(t, env.Grants, `{"grants": {"mesh": ["triage"]}}`)
router, sends, queue := newSendRouter(), newMemSends(), newMemQueue()
b := &Bundle{Env: env, Now: func() time.Time { return judgedAt }, Asks: router, Sends: sends, Queue: queue.publish}
rt := startBundle(t, b.Tools())
text, isErr := rt.call(SendTool, "novox.triage", sendArgs)
if isErr {
t.Fatalf("mail_send: %s", text)
}
var out map[string]any
_ = json.Unmarshal([]byte(text), &out)
id := out["ask"].(string)
w := router.chose(id, OptionSend, judgedAt.Add(time.Minute))
router.close(w)
d, err := Decide(sends, b.queuePublish(), w, judgedAt.Add(2*time.Minute))
if err != nil {
t.Fatalf("the warrant was not heard cleanly: %v", err)
}
if d.Outcome != "queued" {
t.Fatalf("the decision: %+v; want queued (the tools never read the secrets directory)", d)
}
if strings.Contains(d.Outcome, missing) || strings.Contains(strings.ToLower(d.Outcome), "secret") {
t.Fatalf("the decision names a secret file: %q", d.Outcome)
}
text, isErr = rt.call(SendStatusTool, "novox.triage", map[string]any{"id": id})
if isErr {
t.Fatalf("mail_send_status: %s", text)
}
if strings.Contains(text, missing) || strings.Contains(strings.ToLower(text), "secret") {
t.Fatalf("mail_send_status names a secret file: %s", text)
}
}
modules/mesh-mail/cmd/mesh-mail/send.go
+746 -59
View File
@@ -186,6 +186,20 @@ type SendDecision struct {
Outcome string `json:"outcome"`
Warrant string `json:"warrant,omitempty"`
Error string `json:"error,omitempty"`
// Attempt is which handover to the server this is, 1 for the first: the sender's own claim from queued
// or retrying to sending counts it (novox/hq ADR 0436 clauses 9, 10, issue 1196).
Attempt int `json:"attempt,omitempty"`
// NextAttempt is when a "retrying" send is next claimed, the bus's own redelivery time (novox/hq ADR
// 0436 clause 10, issue 1198): read by mail_health to say a send overdue (issue 1199).
NextAttempt time.Time `json:"next-attempt,omitempty"`
// Tell is the sender's own tell to the operator for a failed, refused or not-proven send, kept to once:
// "" (not owed, or not yet attempted), "telling" (claimed, the channel not yet confirmed it took the
// message) or "told" (novox/hq ADR 0436 clause 11, issue 1199).
Tell string `json:"tell,omitempty"`
// WaitsTell is the tools' own tell that an overdue send's sender has not taken it, kept to once the
// same way as Tell, on its own field: the two are never the same tell (novox/hq ADR 0436 clause 11,
// issue 1199).
WaitsTell string `json:"waits-tell,omitempty"`
}
// bodyDigest is the digest an act binds in place of the body itself: never the body, which the act's digest is
@@ -554,18 +568,31 @@ func looksLikeHostName(w string) bool {
// ---- deciding ---------------------------------------------------------------------------------------------------
// AccountLookup resolves an account by name, as the settings currently hold it (not as it stood when it was asked:
// the account's server may have changed since, and its password is read fresh), or found false.
type AccountLookup func(name string) (Account, bool)
// MailSendApproved is the event mesh-mail's tools publish on the work queue mail-sends once the operator's
// Approve is verified and the send recorded approved (novox/hq ADR 0436 clause 8).
const MailSendApproved = "mail.send.approved"
// SendActor performs the one send a chosen option authorises, resolving the account's own secret itself. It is
// called at most once per request (Decide claims the request first).
type SendActor func(a Account, to []string, subject, body string, now time.Time) error
// QueuedSend is the body mail.send.approved carries: never the recipients, the subject or the body, which stay
// in the send's own record in `sends` (novox/hq ADR 0436 clause 8).
type QueuedSend struct {
Send string `json:"send"`
Account string `json:"account"`
Ask string `json:"ask"`
}
// settleSend writes a decision onto a kept send (claiming it, or ending it outright) and, once the write lands,
// drops it from the undecided index: an index entry is never left behind once the record it points to carries a
// decision (novox/hq issue 811). On a race lost to another writer, it reads back whatever that writer decided.
func settleSend(store SendStore, req SendRequest, rev uint64, d SendDecision) (SendDecision, uint64, error) {
// QueuePublish publishes one message on the module's own work-queue stream, the message id chosen (here, the
// send id) so a repeated publish inside the stream's duplicate window is kept to one (novox/hq ADR 0436
// clauses 6, 8); nothing here reads or holds a password. Real is sdkQueuePublish; a test gives its own.
type QueuePublish func(id string, body any) error
func sdkQueuePublish(id string, body any) error {
return stdio.Emit(MailSendApproved, body, stdio.EmitOptions{MessageID: id})
}
// updateDecision writes a decision onto a kept send by compare-and-set, without touching the undecided index:
// the index is settleSend's to drop, once the decision is one the sweep no longer needs to look at again
// (novox/hq issue 811). On a race lost to another writer, it reads back whatever that writer decided.
func updateDecision(store SendStore, req SendRequest, rev uint64, d SendDecision) (SendDecision, uint64, error) {
next := req
next.Decided = &d
newRev, err := store.Update(req.ID, next, rev)
@@ -577,16 +604,30 @@ func settleSend(store SendStore, req SendRequest, rev uint64, d SendDecision) (S
}
return d, 0, err
}
_ = removeUndecided(store, req.ID) // best-effort (see removeUndecided)
return d, newRev, nil
}
// settleSend writes a decision onto a kept send (claiming it, or ending it outright) and, once the write lands,
// drops it from the undecided index: an index entry is never left behind once the record it points to carries a
// decision the sweep no longer needs to act on (novox/hq issue 811). On a race lost to another writer, it reads
// back whatever that writer decided.
func settleSend(store SendStore, req SendRequest, rev uint64, d SendDecision) (SendDecision, uint64, error) {
final, newRev, err := updateDecision(store, req, rev, d)
if err != nil {
return final, 0, err
}
if newRev != 0 {
_ = removeUndecided(store, req.ID) // best-effort (see removeUndecided)
}
return final, newRev, nil
}
// Decide holds one closed ask's warrant to the request it answers (asks.Warrant.For, asks.Option.Performs), and —
// on Approve, exactly once — performs the send. A request already decided is never decided again: decide answers
// its kept decision at once, and two warrants for the same request (a redelivery, or two readers racing) have only
// the first claim the request by a compare-and-set write; the second finds it already claimed or decided and sends
// nothing.
func Decide(store SendStore, lookup AccountLookup, send SendActor, w asks.Warrant, now time.Time) (SendDecision, error) {
// on Approve, exactly once — records the send approved and queues it for the sender (novox/hq ADR 0436 clause 8);
// it never sends. A request already decided is never decided again: decide answers its kept decision at once, and
// two warrants for the same request (a redelivery, or two readers racing) have only the first claim the request
// by a compare-and-set write; the second finds it already claimed or decided and queues nothing twice.
func Decide(store SendStore, publish QueuePublish, w asks.Warrant, now time.Time) (SendDecision, error) {
req, rev, ok, err := getSend(store, w.Ask)
if err != nil {
return SendDecision{}, err
@@ -623,45 +664,650 @@ func Decide(store SendStore, lookup AccountLookup, send SendActor, w asks.Warran
case OptionDecline:
return settle(SendDecision{At: now, Outcome: "declined", Warrant: w.Says()})
case OptionSend:
return decideSend(store, lookup, send, req, rev, w, now)
return decideSend(store, publish, req, rev, w, now)
default:
return settle(SendDecision{At: now, Outcome: "refused: the option " + option.ID + " is not one mail_send offers"})
}
}
// decideSend claims the request by a compare-and-set write before it sends, so a redelivered warrant finds it
// already claimed (or decided) and acts on nothing: a person's Approve is performed once, never retried on a
// transient failure (the failure is kept in the decision, not hidden).
func decideSend(store SendStore, lookup AccountLookup, send SendActor, req SendRequest, rev uint64, w asks.Warrant, now time.Time) (SendDecision, error) {
claimed := req
claim := SendDecision{At: now, Outcome: "sending", Warrant: w.Says()}
claimedDecision, newRev, err := settleSend(store, req, rev, claim)
// decideSend records the request queued by a compare-and-set write before it publishes, so a redelivered
// warrant finds it already claimed (or decided) and queues nothing twice: a person's Approve is queued once
// (novox/hq ADR 0436 clause 8, as decided to replace the two-state approved/queued design — review round 1,
// replacing finding 2's work-around with a decision changing the clause itself). There is no "approved"
// state in a send's life at all: the record moves straight from undecided to queued; the publish that
// follows is best-effort, and a publish that fails (or the tools stopping before it ever runs) leaves the
// record "queued" with its own time, which the sweep finds and republishes once a short grace has passed
// (Sweep, below) — the send id as the message id, and the stream's duplicate window, keep it one message
// either way, and the sender's own claim (clause 9) keeps it one send. It never sends: the sender (clause 7)
// takes the queued message, verifies the warrant again itself (clause 9) and performs the one handover to
// the server (clause 10).
func decideSend(store SendStore, publish QueuePublish, req SendRequest, rev uint64, w asks.Warrant, now time.Time) (SendDecision, error) {
// Indexed before the record carries "queued" (novox/hq review round 1 finding 3): a crash between this
// and the write below must still leave it findable by mail_health's overdue check and the sweep's own
// stale-queued retry once "queued" does land.
indexErr := addSenderPending(store, req.ID)
queued := SendDecision{At: now, Outcome: "queued", Warrant: w.Says()}
claimed, newRev, err := settleSend(store, req, rev, queued)
if err != nil {
return SendDecision{}, err
return SendDecision{}, errOr(indexErr, err)
}
if newRev == 0 {
// Lost the claim race: another writer already decided this send.
return claimedDecision, nil
return claimed, indexErr
}
// Best-effort: its own failure is never returned to the caller as this call's own error — the record is
// already durably "queued" either way, and the sweep retries a publish that did not land.
_ = publish(req.ID, QueuedSend{Send: req.ID, Account: req.Account, Ask: req.ID})
return claimed, indexErr
}
// ---- the sender's own claim (novox/hq ADR 0436 clauses 9, 10, issue 1196) --------------------------------------
//
// The sender never trusts the queue's message: it reads the router's record of mesh-mail's own ask under its
// own account, and mirrors the tools' own check (Decide, above) exactly — w.For, then option.Performs on the
// request's own Act — before it claims the send by compare-and-set and, only then, connects to anything.
// RouterRecord is how the sender reads the router's record of one ask: a function so a test gives a fake, and
// the real one (sender.go) wraps a process.Conn's Ask(bucket, id) over the operator channel's records.
type RouterRecord func(id string) (*stdio.StateEntry, error)
// claimOutcome is what became of one claim, so the caller (the message handler) knows the message's fate.
type claimOutcome int
const (
// claimToSend: claimed "sending" by this call; the caller may now connect and send.
claimToSend claimOutcome = iota
// claimRefused: recorded refused for a reason this function names; the caller terminates the message.
claimRefused
// claimNotProven: the send was already "sending" (a previous delivery may have handed it to the server);
// recorded not proven; the caller terminates the message, never retrying it automatically.
claimNotProven
// claimLostRace: the compare-and-set to "sending" lost to another writer (another delivery of the same
// message, racing this one); nothing is recorded here and nothing sent; the caller leaves the message to
// the bus entirely — no Ack, no Nak, no Term.
claimLostRace
// claimAlreadyFinished: a redelivery of a message whose send already finished (sent, failed, refused
// or not proven) — the sender's own ack was lost, or two deliveries raced and the other one already
// finished it. Nothing is recorded, emitted or told again; the caller only terminates the message
// (novox/hq ADR 0436 clause 10, operator-approved change).
claimAlreadyFinished
)
// isFinishedOutcome says whether a decided send's outcome is already over, in the sense clause 10 means it:
// sent, failed (any reason), refused (any reason, including the sender's own clause-9 refusal), or not
// proven. A redelivery that finds one of these is a repeat, never a new refusal (novox/hq review, the
// operator-approved change to clause 10).
func isFinishedOutcome(outcome string) bool {
return outcome == "sent" || outcome == "not proven" ||
strings.HasPrefix(outcome, "failed: ") || strings.HasPrefix(outcome, "refused: ")
}
// claimQueuedSend verifies the operator's warrant for id (mesh-mail's own ask) and claims the send by
// compare-and-set from queued or retrying to sending, with the attempt and time, before any connection
// (novox/hq ADR 0436 clauses 9, 10). It is never trusted from the queue's message alone: the ask is read
// fresh from the router under the sender's own account, closed by the operator's Approve at level approve
// within its expiry (Warrant.For), and its act's digest (the account, recipients, subject and body's digest)
// is the option's binds (Option.Performs) — exactly the tools' own check (Decide, above) mirrored here,
// because the tool runner's account, which every agent session on this machine also reaches the bus as, may
// publish on the queue and write `sends` (this is the boundary clause 9 describes).
func claimQueuedSend(store SendStore, record RouterRecord, emit Emit, id string, now time.Time) (SendRequest, SendDecision, uint64, claimOutcome, error) {
req, rev, ok, err := getSend(store, id)
if err != nil {
return SendRequest{}, SendDecision{}, 0, claimLostRace, err
}
if !ok {
return SendRequest{}, SendDecision{}, 0, claimLostRace, fmt.Errorf("no send %q is kept", id)
}
if req.Decided == nil {
// Nothing was ever decided at all (never queued): no tell owed (clause 11 ties the tell to a
// *failed approved* send), and nothing emitted either — clause 11's two events are for a send the
// operator did approve.
d, _, err := settleSend(store, req, rev, SendDecision{At: now, Outcome: "refused: the send is not yet approved"})
return req, d, 0, claimRefused, err
}
// Every other refusal below follows a real Approve (the send reached at least queued): clause 9's own
// refusal owes the operator a tell and the same mail.send.failed event as clause 10's refusal does
// (novox/hq ADR 0436 clause 11, issue 1199; review round 1 finding 4).
refuse := func(reason string) (SendRequest, SendDecision, uint64, claimOutcome, error) {
// Indexed for a tell BEFORE the record carries "refused" (novox/hq review round 1 finding 3): a
// crash between this and the write below must still leave it findable once refused does land;
// tellSend itself tolerates finding it still sending, queued or retrying in the meantime.
indexErr := addTellPending(store, req.ID)
d, newRev, err := settleSend(store, req, rev, SendDecision{At: now, Outcome: "refused: " + reason})
if newRev == 0 {
// The write did not land (another writer's decision already stands, or it failed outright):
// nothing of this call's own making to emit (novox/hq review round 2, issue 1198).
return req, d, 0, claimRefused, errOr(indexErr, err)
}
_ = emit(MailSendFailed, req.ID, OutcomeEvent{Send: req.ID, Account: req.Account, Outcome: d.Outcome})
// Only once refused is durably recorded is the sender-pending index's entry dropped.
if rmErr := removeSenderPending(store, req.ID); rmErr != nil {
indexErr = errOr(indexErr, fmt.Errorf("dropping %s from the sender-pending index: %w", req.ID, rmErr))
}
return req, d, 0, claimRefused, indexErr
}
claimed.Decided = &claim
acc, found := lookup(req.Account)
var final SendDecision
switch {
case !found:
final = SendDecision{At: now, Outcome: "refused: the account no longer exists", Warrant: w.Says()}
default:
if sendErr := send(acc, req.To, req.Subject, req.Body, now); sendErr != nil {
final = SendDecision{At: now, Outcome: "failed: " + sendErr.Error(), Warrant: w.Says(), Error: sendErr.Error()}
} else {
final = SendDecision{At: now, Outcome: "sent", Warrant: w.Says()}
case req.Decided.Outcome == "sending":
// A previous delivery may have handed it to the server already; never sent twice, and never
// retried automatically either way (ADR 0436 clause 10). Same ordering as refuse() above.
indexErr := addTellPending(store, req.ID)
d, newRev, err := settleSend(store, req, rev, SendDecision{At: now, Outcome: "not proven", Attempt: req.Decided.Attempt})
if newRev == 0 {
// The write did not land: nothing of this call's own making to emit (novox/hq review round 2,
// issue 1198).
return req, d, 0, claimNotProven, errOr(indexErr, err)
}
_ = emit(MailSendFailed, req.ID, OutcomeEvent{Send: req.ID, Account: req.Account, Outcome: d.Outcome})
if rmErr := removeSenderPending(store, req.ID); rmErr != nil {
indexErr = errOr(indexErr, fmt.Errorf("dropping %s from the sender-pending index: %w", req.ID, rmErr))
}
return req, d, 0, claimNotProven, indexErr
case isFinishedOutcome(req.Decided.Outcome):
// A redelivery of a message whose send already finished — sent, failed, refused by the sender's
// own claim, or not proven: the sender's own ack was lost (a bus reconnect) and the work-queue gave
// it up after AckWait, or two deliveries raced and the other one already finished it. The operator
// was already told (or sent) once; telling, emitting or recording anything again here would be a
// second word about a send that is already over. Just a repeat: terminated, nothing else touched
// (novox/hq ADR 0436 clause 10, operator-approved change).
return req, *req.Decided, 0, claimAlreadyFinished, nil
case req.Decided.Outcome != "queued" && req.Decided.Outcome != "retrying":
return refuse(fmt.Sprintf("the send is %q, neither queued nor retrying", req.Decided.Outcome))
}
entry, err := record(id)
if err != nil {
return SendRequest{}, SendDecision{}, 0, claimLostRace, err
}
if entry == nil {
return refuse("the router keeps no record of this ask")
}
var rec routerRecord
if json.Unmarshal(entry.Value, &rec) != nil {
return refuse("the router's record of it cannot be read")
}
if rec.State == "" || rec.State == "open" || rec.Warrant == nil {
return refuse("the ask is not closed by the operator's Approve")
}
w := *rec.Warrant
if w.Asker != Module {
return refuse(fmt.Sprintf("the warrant is for %s's ask, not %s's", w.Asker, Module))
}
ask := req.Ask()
option, err := w.For(Module, ask)
if err == nil {
err = option.Performs(req.Act(option.ID))
}
if err != nil {
return refuse(err.Error())
}
if option.ID != OptionSend {
return refuse("the option " + option.ID + " does not authorise a send")
}
claim := SendDecision{At: now, Outcome: "sending", Warrant: w.Says(), Attempt: req.Decided.Attempt + 1}
claimed, newRev, err := updateDecision(store, req, rev, claim)
if err != nil {
return SendRequest{}, SendDecision{}, 0, claimLostRace, err
}
if newRev == 0 {
// Lost the claim race: another delivery of the same message is already handling it.
return req, claimed, 0, claimLostRace, nil
}
return req, claimed, newRev, claimToSend, nil
}
// ---- the send's outcome, retry and emit (novox/hq ADR 0436 clauses 10, 11, issue 1198) --------------------------
// MailSendSent and MailSendFailed are the two events the sender emits, never anything else (novox/hq ADR
// 0436 clause 11): the send id, the account and the outcome in words — never the recipients, the subject or
// the body.
const (
MailSendSent = "mail.send.sent"
MailSendFailed = "mail.send.failed"
)
// OutcomeEvent is the body mail.send.sent and mail.send.failed carry.
type OutcomeEvent struct {
Send string `json:"send"`
Account string `json:"account"`
Outcome string `json:"outcome"`
}
// Emit publishes one of this module's own events with a chosen message id (novox/hq ADR 0436 clauses 4,
// 11): the real one (sender.go) wraps a process.Conn's own Emit; a test gives its own.
type Emit func(eventType, id string, body any) error
// RetryDelays is how long after each not-taken attempt (1-indexed) the bus is asked to redeliver the
// message, before the next attempt: 1, 5, 15 and 30 minutes for attempts 1 to 4 (novox/hq ADR 0436 clause
// 10).
var RetryDelays = []time.Duration{time.Minute, 5 * time.Minute, 15 * time.Minute, 30 * time.Minute}
// MostAttempts is how many times the sender hands a claimed send to the server before giving up: the fifth
// failure records failed, and the bus's own max-deliveries (module.json's mail-sends) dead-letters the
// message once it is terminated rather than acknowledged (novox/hq ADR 0436 clause 10).
const MostAttempts = 5
// msgAction is what the caller (the message handler, sender.go) does with the queue message once
// recordOutcome has written the decision: Ack (sent), NakWithDelay (retrying, not the last attempt yet) or
// Term (failed, or not proven — the bus is never asked to redeliver those; a dead letter past the fifth
// attempt is still reached by Term, since clause 10 records failed itself rather than waiting for the bus's
// own max-deliveries to notice). actionLeave is none of these: the record's own write could not be kept
// (even after a retry on a changed revision), so nothing is emitted and the message is left entirely to
// the bus — a later redelivery finds the record still at its previous outcome and reconciles from there,
// never a send silently announced sent, failed or retrying when it was not actually kept that way (novox/hq
// review round 1 finding 5).
type msgAction int
const (
actionAck msgAction = iota
actionRetry
actionTerm
actionLeave
)
// recordOutcome writes the final decision for one claimed send once SendMail has run (or could not even be
// attempted — the account vanished, or its password cannot be read, both passed in as a NotTaken
// *SendError, exactly as a real SendMail failure would be: safe to retry), and says what the caller does
// with the message next. Attempt (req's own, from the claim) decides retrying from failed on a not-taken
// failure: the fifth attempt always fails rather than retries again (novox/hq ADR 0436 clauses 10, 11).
func recordOutcome(store SendStore, emit Emit, req SendRequest, claimed SendDecision, rev uint64, sendErr error, now time.Time) (SendDecision, msgAction, time.Duration, error) {
// write keeps d, retrying once on a changed revision (re-reading the send and trying again with its
// fresh one) before giving up: kept is false only when the record's own write could not be kept even
// after that retry, in which case the caller emits nothing and leaves the message to the bus entirely —
// never announcing an outcome (sent, failed, retrying) that was not actually recorded (novox/hq review
// round 1 finding 5). "Kept" is decided from settleSend's own returned revision, never from its error
// alone: on a changed revision whose record already carries another writer's decision, settleSend
// (through updateDecision) returns that other decision with revision 0 and a nil error — a nil error
// is not success here, and without checking the revision the retry never ran at all, so the caller
// could emit an outcome (and ack) while the record actually held someone else's state (novox/hq review
// round 2, issue 1198).
write := func(d SendDecision, final, tellWorthy bool) (result SendDecision, kept bool, outErr error) {
var indexErr error
if tellWorthy {
// Indexed for a tell BEFORE the record carries the outcome that owes it (novox/hq review round
// 1 finding 3): a crash between this and the write below must still leave it findable once the
// outcome does land; tellSend itself tolerates finding it still sending, queued or retrying.
indexErr = addTellPending(store, req.ID)
}
curReq, curRev := req, rev
var newRev uint64
var writeErr error
for attempt := 0; attempt < 2; attempt++ {
result, newRev, writeErr = settleSend(store, curReq, curRev, d)
if newRev != 0 {
break // this write landed
}
if attempt == 1 {
break // retried once already; give up
}
if writeErr != nil && !errors.Is(writeErr, stdio.ErrChanged) {
break // a hard failure, not a revision race: retrying blindly would not help
}
fresh, freshRev, ok, rerr := getSend(store, req.ID)
if rerr != nil || !ok {
break
}
curReq, curRev = fresh, freshRev
}
if newRev == 0 {
reportErr := writeErr
if reportErr == nil {
reportErr = fmt.Errorf("the send %s's outcome could not be kept: another writer's decision already stands, even retried on its fresh revision", req.ID)
}
return d, false, errOr(indexErr, reportErr)
}
if final {
// Only once the outcome is durably recorded is the sender-pending index's entry dropped: from
// here the sender is done with it, one way or another (novox/hq issue 1199).
if rmErr := removeSenderPending(store, req.ID); rmErr != nil {
indexErr = errOr(indexErr, fmt.Errorf("dropping %s from the sender-pending index: %w", req.ID, rmErr))
}
}
return result, true, indexErr
}
if sendErr == nil {
d, kept, err := write(SendDecision{At: now, Outcome: "sent", Warrant: claimed.Warrant, Attempt: claimed.Attempt}, true, false)
if !kept {
return d, actionLeave, 0, err
}
_ = emit(MailSendSent, req.ID, OutcomeEvent{Send: req.ID, Account: req.Account, Outcome: d.Outcome})
return d, actionAck, 0, err
}
var se *SendError
if !errors.As(sendErr, &se) {
// SendMail itself never returns an error that is not a *SendError; a caller passing one of its own
// (the account vanished, or its password cannot be read) is read as not taken, never as a refusal or
// a proven send, which would be worse guesses than "safe to retry" (novox/hq ADR 0436 clause 10).
se = &SendError{Outcome: NotTaken, Err: sendErr}
}
switch se.Outcome {
case Refused:
d, kept, err := write(SendDecision{At: now, Outcome: "failed: " + se.Err.Error(), Warrant: claimed.Warrant,
Attempt: claimed.Attempt, Error: se.Err.Error()}, true, true)
if !kept {
return d, actionLeave, 0, err
}
_ = emit(MailSendFailed, req.ID, OutcomeEvent{Send: req.ID, Account: req.Account, Outcome: d.Outcome})
return d, actionTerm, 0, err
case NotProven:
d, kept, err := write(SendDecision{At: now, Outcome: "not proven", Warrant: claimed.Warrant, Attempt: claimed.Attempt,
Error: se.Err.Error()}, true, true)
if !kept {
return d, actionLeave, 0, err
}
_ = emit(MailSendFailed, req.ID, OutcomeEvent{Send: req.ID, Account: req.Account, Outcome: d.Outcome})
return d, actionTerm, 0, err
default: // NotTaken
if claimed.Attempt >= MostAttempts {
d, kept, err := write(SendDecision{At: now, Outcome: "failed: " + se.Err.Error(), Warrant: claimed.Warrant,
Attempt: claimed.Attempt, Error: se.Err.Error()}, true, true)
if !kept {
return d, actionLeave, 0, err
}
_ = emit(MailSendFailed, req.ID, OutcomeEvent{Send: req.ID, Account: req.Account, Outcome: d.Outcome})
return d, actionTerm, 0, err
}
delay := RetryDelays[claimed.Attempt-1]
d, kept, err := write(SendDecision{At: now, Outcome: "retrying", Warrant: claimed.Warrant, Attempt: claimed.Attempt,
Error: se.Err.Error(), NextAttempt: now.Add(delay)}, false, false)
if !kept {
return d, actionLeave, 0, err
}
return d, actionRetry, delay, err // not a final outcome: no emit (clause 11 names only sent and failed)
}
}
// ---- two more indices (novox/hq issue 1199), the same shape as the undecided index (issue 811) ------------------
//
// Neither the tools' nor the sender's own `sends` state lists every key it holds (mesh-sdk go/process's
// State offers no such call, and listing it all on every health check or every look would grow with every
// send ever sent, not with the few still live): a reserved key stands in for a list, exactly as the
// undecided index already does.
// errOr is a, or b when a is nil: the first of two errors a caller that can only report one keeps (novox/hq
// review round 1 finding 3), so an index-bookkeeping failure is never silently dropped once there is also a
// record-write failure to report, and vice versa.
func errOr(a, b error) error {
if a != nil {
return a
}
return b
}
// idIndex is the shape a send-id index keeps: generic across all three (undecided, sender-pending,
// tells-pending).
type idIndex struct {
IDs []string `json:"ids"`
}
// addToIndex adds id to the index at key, retrying its compare-and-set write on a race. A stored index
// that cannot be parsed is never overwritten (which would silently drop every id it already named): it is
// reported as an error naming the key instead (novox/hq issue 811, review round 3).
func addToIndex(store SendStore, key, id string) error {
for {
raw, rev, ok, err := store.Get(key)
if err != nil {
return err
}
var idx idIndex
if ok {
if err := json.Unmarshal(raw, &idx); err != nil {
return fmt.Errorf("the index %q cannot be parsed: %w", key, err)
}
for _, existing := range idx.IDs {
if existing == id {
return nil
}
}
}
idx.IDs = append(idx.IDs, id)
if !ok {
if _, err := store.Create(key, idx); err != nil {
if errors.Is(err, stdio.ErrChanged) {
continue
}
return err
}
return nil
}
if _, err := store.Update(key, idx, rev); err != nil {
if errors.Is(err, stdio.ErrChanged) {
continue
}
return err
}
return nil
}
}
// removeFromIndex drops id from the index at key, retrying its compare-and-set write on a race. An id
// never in the index (already absent) is left as it is.
func removeFromIndex(store SendStore, key, id string) error {
for {
raw, rev, ok, err := store.Get(key)
if err != nil || !ok {
return err
}
var idx idIndex
if err := json.Unmarshal(raw, &idx); err != nil {
return fmt.Errorf("the index %q cannot be parsed: %w", key, err)
}
at := -1
for i, existing := range idx.IDs {
if existing == id {
at = i
break
}
}
if at == -1 {
return nil
}
idx.IDs = append(idx.IDs[:at], idx.IDs[at+1:]...)
if _, err := store.Update(key, idx, rev); err != nil {
if errors.Is(err, stdio.ErrChanged) {
continue
}
return err
}
return nil
}
}
// indexIDs is every id the index at key currently names.
func indexIDs(store SendStore, key string) ([]string, error) {
raw, _, ok, err := store.Get(key)
if err != nil || !ok {
return nil, err
}
var idx idIndex
if err := json.Unmarshal(raw, &idx); err != nil {
return nil, fmt.Errorf("the index %q cannot be parsed: %w", key, err)
}
return idx.IDs, nil
}
// senderPendingKey indexes every send the sender currently owns — queued, retrying or sending — so
// mail_health's overdue check (Overdue, below) finds them without listing `sends` whole.
const senderPendingKey = "sender-pending"
func addSenderPending(store SendStore, id string) error {
return addToIndex(store, senderPendingKey, id)
}
func removeSenderPending(store SendStore, id string) error {
return removeFromIndex(store, senderPendingKey, id)
}
func senderPendingIDs(store SendStore) ([]string, error) { return indexIDs(store, senderPendingKey) }
// tellsPendingKey indexes every send still owing the sender's own tell (needsTell, below), so its own
// look (sender.go) finds them without listing `sends` whole.
const tellsPendingKey = "tells-pending"
func addTellPending(store SendStore, id string) error { return addToIndex(store, tellsPendingKey, id) }
func removeTellPending(store SendStore, id string) error {
return removeFromIndex(store, tellsPendingKey, id)
}
func tellPendingIDs(store SendStore) ([]string, error) { return indexIDs(store, tellsPendingKey) }
// ---- overdue (novox/hq ADR 0436 clause 11, issue 1199) ------------------------------------------------------
// OverdueAfter is how long a send may sit queued, or wait past its own next attempt, before mail_health
// calls it unhealthy (novox/hq ADR 0436 clause 11).
const OverdueAfter = 10 * time.Minute
// Overdue is every send the sender currently owns (senderPendingIDs) that is overdue: queued more than
// OverdueAfter ago, or retrying more than OverdueAfter past its own next attempt. A send waiting for its
// next attempt within that window, or one actually being sent, is never overdue (novox/hq ADR 0436 clause
// 11).
func Overdue(store SendStore, now time.Time) ([]SendRequest, error) {
ids, err := senderPendingIDs(store)
if err != nil {
return nil, err
}
var out []SendRequest
for _, id := range ids {
req, _, ok, err := getSend(store, id)
if err != nil || !ok || req.Decided == nil {
continue
}
switch {
case req.Decided.Outcome == "queued" && now.Sub(req.Decided.At) > OverdueAfter:
out = append(out, req)
case req.Decided.Outcome == "retrying" && now.Sub(req.Decided.NextAttempt) > OverdueAfter:
out = append(out, req)
}
}
claimed.Decided = &final
if _, err := store.Update(req.ID, claimed, newRev); err != nil {
// The claim stands either way: a person's Approve is acted on once. The caller logs this; it is not retried.
return final, fmt.Errorf("the send %s was %s, and that could not be kept (%w)", req.ID, final.Outcome, err)
return out, nil
}
// ---- telling the operator (novox/hq ADR 0436 clause 11, issue 1199) ----------------------------------------
//
// Two separate tells, each kept to once on its own field: the sender's own, for a failed, refused or
// not-proven send (Tell); the tools' own, for an overdue one — still queued or retrying, its sender not
// having taken it yet (WaitsTell). Neither ever names an address, held to the operator channel's content
// rule by using only the account's own name.
// TellOperator calls seat:operator-channel.tell once with the one line to say (novox/hq ADR 0380 §8). The
// real ones wrap a process.Conn's own Call (sender.go) or stdio.Ask (main.go); a test gives its own.
type TellOperator func(headline string) error
// needsTell says whether a decided send's outcome owes the operator the sender's own tell (novox/hq ADR
// 0436 clause 11): every outcome after the Approve but sent — failed, refused or not proven.
func needsTell(outcome string) bool {
return outcome == "not proven" || strings.HasPrefix(outcome, "failed: ") || strings.HasPrefix(outcome, "refused: ")
}
// tellReason is the sender's own one line for a failed, refused or not-proven send (novox/hq ADR 0436
// clause 11).
func tellReason(account, outcome string) string {
if outcome == "not proven" {
return fmt.Sprintf("The mail you approved from %s may not have been sent; its sent folder says", account)
}
return final, nil
reason := outcome
for _, prefix := range []string{"failed: ", "refused: "} {
if r, ok := strings.CutPrefix(outcome, prefix); ok {
reason = r
break
}
}
return fmt.Sprintf("The mail you approved from %s was not sent: %s", account, reason)
}
// tellSend attempts the sender's own tell for one failed, refused or not-proven send, kept to once by
// Tell: telling then told, by compare-and-set, on the record itself (novox/hq ADR 0436 clause 11). A tell
// the channel refuses, or that nothing answers, leaves telling, so the next look (the sender's own ticker,
// at least every 10 minutes) tries it again; a record already told, or whose outcome no longer needs one,
// is dropped from the index without being told again.
func tellSend(store SendStore, tell TellOperator, id string, now time.Time) error {
req, rev, ok, err := getSend(store, id)
if err != nil {
return err
}
if !ok || req.Decided == nil {
_ = removeTellPending(store, id)
return nil
}
switch req.Decided.Outcome {
case "sending", "queued", "retrying":
// Indexed for a tell before the outcome that owes it had actually landed (addTellPending now runs
// before the write, novox/hq review round 1 finding 3): not decided yet whether a tell is owed.
// Left in the index; the next look finds it once the write lands.
return nil
}
if !needsTell(req.Decided.Outcome) || req.Decided.Tell == "told" {
_ = removeTellPending(store, id)
return nil
}
d := *req.Decided
if d.Tell != "telling" {
d.Tell = "telling"
next := req
next.Decided = &d
newRev, err := store.Update(id, next, rev)
if err != nil {
if errors.Is(err, stdio.ErrChanged) {
return nil // raced with another writer; the next look sorts it out
}
return err
}
rev = newRev
}
if err := tell(tellReason(req.Account, d.Outcome)); err != nil {
return nil // left telling; tried again on the next look
}
d.Tell = "told"
next := req
next.Decided = &d
if _, err := store.Update(id, next, rev); err != nil {
if errors.Is(err, stdio.ErrChanged) {
return nil
}
return err
}
_ = removeTellPending(store, id)
return nil
}
// waitsReason is the tools' own one line for an overdue send (novox/hq ADR 0436 clause 11).
func waitsReason(account string) string {
return fmt.Sprintf("The mail you approved from %s waits: its sender has not taken it", account)
}
// tellWaits attempts the tools' own "waits" tell for one overdue send, kept to once by WaitsTell — the
// same telling/told sequence as tellSend, on the record's own separate field, since the two tells are
// never the same one (novox/hq ADR 0436 clause 11).
func tellWaits(store SendStore, tell TellOperator, id string, now time.Time) error {
req, rev, ok, err := getSend(store, id)
if err != nil {
return err
}
if !ok || req.Decided == nil || (req.Decided.Outcome != "queued" && req.Decided.Outcome != "retrying") || req.Decided.WaitsTell == "told" {
return nil
}
d := *req.Decided
if d.WaitsTell != "telling" {
d.WaitsTell = "telling"
next := req
next.Decided = &d
newRev, err := store.Update(id, next, rev)
if err != nil {
if errors.Is(err, stdio.ErrChanged) {
return nil
}
return err
}
rev = newRev
}
if err := tell(waitsReason(req.Account)); err != nil {
return nil
}
d.WaitsTell = "told"
next := req
next.Decided = &d
if _, err := store.Update(id, next, rev); err != nil {
if errors.Is(err, stdio.ErrChanged) {
return nil
}
return err
}
return nil
}
// ---- reading the status -----------------------------------------------------------------------------------------
@@ -670,7 +1316,7 @@ func decideSend(store SendStore, lookup AccountLookup, send SendActor, req SendR
// declined, ended without a choice (expired, refused, cancelled), or not proven. Read from the router's own
// record; when the router's word is a close one that was not decided yet, this is also where a send is first
// decided (and so sent), exactly once (Decide).
func ReadSendStatus(store SendStore, bus AskBus, lookup AccountLookup, send SendActor, id string, now time.Time) (map[string]any, error) {
func ReadSendStatus(store SendStore, bus AskBus, publish QueuePublish, id string, now time.Time) (map[string]any, error) {
trimmed := strings.TrimSpace(id)
req, rev, ok, err := getSend(store, trimmed)
if err != nil {
@@ -713,7 +1359,7 @@ func ReadSendStatus(store SendStore, bus AskBus, lookup AccountLookup, send Send
out["status"], out["words"] = "waiting", "the operator has not answered yet"
return out, nil
}
d, err := Decide(store, lookup, send, *rec.Warrant, now)
d, err := Decide(store, publish, *rec.Warrant, now)
if err != nil {
out["status"], out["words"] = "not proven", err.Error()
return out, nil
@@ -726,9 +1372,17 @@ func ReadSendStatus(store SendStore, bus AskBus, lookup AccountLookup, send Send
// and the words that explain it. now is when it is asked, not when the send was decided: a claim still "sending"
// longer than StaleSendAfter with no final write is "not proven" (the send may or may not have gone out), never
// "waiting" forever — and it is never retried automatically either way (novox/hq issue 790, review round 1).
// reportDecision's status word is one of: waiting (for the operator, or for the sender — approved, queued,
// being sent or retrying, each said apart in words), sent, failed, refused, not proven, declined or expired
// (novox/hq ADR 0436 clause 12, issue 1199) — the whole vocabulary mail_send_status answers.
func reportDecision(out map[string]any, d SendDecision, now time.Time) {
out["at"] = d.At.Format(time.RFC3339)
switch {
case d.Outcome == "queued":
// Recorded queued by the one compare-and-set the Approve makes (novox/hq ADR 0436 clause 8): there
// is no separate "approved" state in a send's life. Published to mail-sends already, or the sweep
// will publish it again once a short grace has passed.
out["status"], out["words"] = "waiting", "approved and waiting for the sender; ask again shortly"
case d.Outcome == "sent":
out["status"], out["words"] = "sent", "sent on the operator's Approve ("+d.Warrant+")"
case d.Outcome == "declined":
@@ -738,17 +1392,23 @@ func reportDecision(out map[string]any, d SendDecision, now time.Time) {
out["words"] = "not proven: the send may or may not have gone out; the operator's Approve was being acted " +
"on at " + d.At.Format(time.RFC3339) + ", with no final word since"
case d.Outcome == "sending":
out["status"], out["words"] = "waiting", "the operator's Approve is being acted on; ask again shortly"
out["status"], out["words"] = "waiting", "being sent; ask again shortly"
case d.Outcome == "retrying":
out["status"] = "waiting"
out["words"] = "waiting for its next attempt, at " + d.NextAttempt.Format(time.RFC3339) + ": " + d.Error
case d.Outcome == "not proven":
out["status"] = "not proven"
out["words"] = "not proven: the send may or may not have gone out: " + d.Error
case d.Outcome == "ended: never heard":
out["status"], out["words"] = "ended", "its ask to the operator was never heard, so it was not sent"
out["status"], out["words"] = "expired", "its ask to the operator was never heard, so it was not sent"
case strings.HasPrefix(d.Outcome, "failed: "):
out["status"], out["words"] = "failed", "nothing reached the server: "+strings.TrimPrefix(d.Outcome, "failed: ")
case strings.HasPrefix(d.Outcome, "ended: "):
out["status"], out["words"] = "ended", "nothing was sent: "+strings.TrimPrefix(d.Outcome, "ended: ")
out["status"], out["words"] = "expired", "nothing was sent: "+strings.TrimPrefix(d.Outcome, "ended: ")
case strings.HasPrefix(d.Outcome, "refused: "):
out["status"], out["words"] = "not proven", "nothing was sent: "+strings.TrimPrefix(d.Outcome, "refused: ")
out["status"], out["words"] = "refused", "nothing was sent: "+strings.TrimPrefix(d.Outcome, "refused: ")
default:
out["status"], out["words"] = "ended", d.Outcome
out["status"], out["words"] = "expired", d.Outcome
}
}
@@ -758,35 +1418,62 @@ func reportDecision(out map[string]any, d SendDecision, now time.Time) {
// the same Decide that mail_send_status uses: an approved send goes out as soon as the operator answers, never
// only when something later happens to call mail_send_status (novox/hq ADR 0361 §2, review round 1 of issue 790).
func (b *Bundle) Heard(w asks.Warrant) {
if _, err := Decide(b.sends(), b.lookupAccount, b.sendActor(), w, b.now()); err != nil {
if _, err := Decide(b.sends(), b.queuePublish(), w, b.now()); err != nil {
if b.Logf != nil {
b.Logf("[mesh-mail] the warrant for the send %s could not be acted on: %v", w.Ask, err)
}
}
}
// Sweep reads only the sends the undecided index still names, and asks the router's own record for each one's
// answer, the same path mail_send_status takes (ReadSendStatus): an Approve recorded while this module did not
// hear it directly (down, or not yet subscribed) is still acted on, once. Unlike reading every send ever kept, its
// work grows only with the sends still undecided, never with every one a module has ever sent (novox/hq ADR 0361
// §2, review round 1 of issue 790; the index, issue 811).
// QueuedGrace is how long a newly queued send is given to be published before the sweep considers it
// possibly stuck and publishes it again: long enough that the tools' own publish, right after the same
// compare-and-set write, is never raced by the sweep under ordinary load (novox/hq ADR 0436 clause 8, as
// decided to replace the two-state approved/queued design, review round 1).
const QueuedGrace = time.Minute
// Sweep does two things, neither reading every send ever kept (its work grows only with the few still
// live, never with history): a send the undecided index still names and that is not yet decided at all is
// asked about the same way mail_send_status does (ReadSendStatus) — an Approve recorded while this module
// did not hear it directly (down, or not yet subscribed) is still acted on, once (novox/hq ADR 0361 §2,
// review round 1 of issue 790; the index, issue 811); and a send the sender-pending index names that is
// still "queued" past QueuedGrace — its publish to mail-sends may have failed, or the tools stopped before
// it ever ran — is published again, the send id still its message id, so the duplicate window and the
// sender's own claim keep it one send either way (novox/hq ADR 0436 clause 8, review round 1).
func (b *Bundle) Sweep(now time.Time) {
store := b.sends()
ids, err := undecidedIDs(store)
undecided, err := undecidedIDs(store)
if err != nil {
if b.Logf != nil {
b.Logf("[mesh-mail] the undecided index cannot be read: %v", err)
}
} else {
for _, id := range undecided {
req, _, ok, err := getSend(store, id)
if err != nil || !ok || req.Decided != nil {
continue
}
if _, err := ReadSendStatus(store, b.askBus(), b.queuePublish(), id, now); err != nil {
if b.Logf != nil {
b.Logf("[mesh-mail] sweeping the send %s: %v", id, err)
}
}
}
}
pending, err := senderPendingIDs(store)
if err != nil {
if b.Logf != nil {
b.Logf("[mesh-mail] the sender-pending index cannot be read: %v", err)
}
return
}
for _, id := range ids {
for _, id := range pending {
req, _, ok, err := getSend(store, id)
if err != nil || !ok || req.Decided != nil {
if err != nil || !ok || req.Decided == nil || req.Decided.Outcome != "queued" || now.Sub(req.Decided.At) < QueuedGrace {
continue
}
if _, err := ReadSendStatus(store, b.askBus(), b.lookupAccount, b.sendActor(), id, now); err != nil {
if err := b.queuePublish()(req.ID, QueuedSend{Send: req.ID, Account: req.Account, Ask: req.ID}); err != nil {
if b.Logf != nil {
b.Logf("[mesh-mail] sweeping the send %s: %v", id, err)
b.Logf("[mesh-mail] republishing the queued send %s: %v", id, err)
}
}
}
modules/mesh-mail/cmd/mesh-mail/send_test.go
+182 -69
View File
@@ -166,6 +166,47 @@ func (r *sendRouter) published1() int {
return len(r.published)
}
// ---- a fake work-queue publish, as the tools reach mail-sends (novox/hq ADR 0436 clause 8) ----------------------
type queuedMsg struct {
id string
body QueuedSend
}
type memQueue struct {
mu sync.Mutex
published []queuedMsg
failErr error
}
func newMemQueue() *memQueue { return &memQueue{} }
func (q *memQueue) publish(id string, body any) error {
q.mu.Lock()
defer q.mu.Unlock()
if q.failErr != nil {
return q.failErr
}
b, _ := body.(QueuedSend)
q.published = append(q.published, queuedMsg{id: id, body: b})
return nil
}
func (q *memQueue) count() int {
q.mu.Lock()
defer q.mu.Unlock()
return len(q.published)
}
func (q *memQueue) last() queuedMsg {
q.mu.Lock()
defer q.mu.Unlock()
if len(q.published) == 0 {
return queuedMsg{}
}
return q.published[len(q.published)-1]
}
// ---- an in-process fake SMTP server, speaking the protocol over a real socket -------------------------------------
type receivedMail struct {
@@ -375,11 +416,13 @@ func TestMailSendAsksWithTheAccountRecipientsSubjectAndFirstLines(t *testing.T)
}
}
// (3) On Approve, the fake SMTP server receives exactly one message, even if the warrant is delivered twice.
func TestOnApproveTheFakeSMTPServerReceivesExactlyOneMessageEvenDeliveredTwice(t *testing.T) {
fake := newFakeSMTPServer(t)
// 1194/AC1: On Approve, the send is queued for the sender exactly once, even if the warrant is delivered twice
// (novox/hq ADR 0436 clause 8). Since issue 1194 the tools never dial an outgoing server: they record approved
// and queue the send for the sender process, which alone performs it (issue 1196, 1198).
func TestOnApproveTheSendIsQueuedExactlyOnceEvenDeliveredTwice(t *testing.T) {
b, router, sends := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"})
b.Dial = fake.dialer()
queue := newMemQueue()
b.Queue = queue.publish
rt := startBundle(t, b.Tools())
text, isErr := rt.call(SendTool, "novox.triage", sendArgs)
@@ -393,44 +436,41 @@ func TestOnApproveTheFakeSMTPServerReceivesExactlyOneMessageEvenDeliveredTwice(t
w := router.chose(id, OptionSend, judgedAt.Add(time.Minute))
router.close(w)
lookup := b.lookupAccount
actor := b.sendActor()
if _, err := Decide(sends, lookup, actor, w, judgedAt.Add(2*time.Minute)); err != nil {
publish := b.queuePublish()
if _, err := Decide(sends, publish, w, judgedAt.Add(2*time.Minute)); err != nil {
t.Fatalf("deciding: %v", err)
}
if n := fake.count(); n != 1 {
t.Fatalf("the fake server received %d message(s) after one Approve", n)
if n := queue.count(); n != 1 {
t.Fatalf("the queue received %d message(s) after one Approve", n)
}
// The warrant delivered a second time: still exactly one message.
if _, err := Decide(sends, lookup, actor, w, judgedAt.Add(3*time.Minute)); err != nil {
// The warrant delivered a second time: still queued exactly once.
if _, err := Decide(sends, publish, w, judgedAt.Add(3*time.Minute)); err != nil {
t.Fatalf("deciding again: %v", err)
}
if n := fake.count(); n != 1 {
t.Fatalf("the fake server received %d message(s) after the warrant was delivered twice", n)
if n := queue.count(); n != 1 {
t.Fatalf("the queue received %d message(s) after the warrant was delivered twice", n)
}
got := fake.last()
if got.from != "jschoubben@novox.be" || len(got.to) != 2 || got.to[0] != "ops@example.org" || got.to[1] != "lead@example.org" {
t.Fatalf("received %+v", got)
}
if !strings.Contains(got.data, "The nightly gate is red") || !strings.Contains(got.data, "The merge gate failed on the resolver change") {
t.Fatalf("the message sent: %q", got.data)
got := queue.last()
if got.id != id || got.body.Send != id || got.body.Account != "mesh" || got.body.Ask != id {
t.Fatalf("queued %+v, want the send id %s as the message id, account and ask", got, id)
}
// mail_send_status reports it sent, and reports the same thing again without sending a third time.
// mail_send_status reports it waiting on the sender, and reports the same thing again without queueing a
// third time.
text, isErr = rt.call(SendStatusTool, "novox.triage", map[string]any{"id": id})
if isErr || !strings.Contains(text, `"status":"sent"`) {
if isErr || !strings.Contains(text, `"status":"waiting"`) {
t.Fatalf("%v %s", isErr, text)
}
if n := fake.count(); n != 1 {
t.Fatalf("mail_send_status sent again: %d", n)
if n := queue.count(); n != 1 {
t.Fatalf("mail_send_status queued again: %d", n)
}
}
// (4) On Decline, nothing is sent, and the status says declined.
func TestOnDeclineNothingIsSentAndTheStatusSaysDeclined(t *testing.T) {
fake := newFakeSMTPServer(t)
// 1194/AC3: On Decline, nothing is queued, and the status says declined.
func TestOnDeclineNothingIsQueuedAndTheStatusSaysDeclined(t *testing.T) {
b, router, sends := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"})
b.Dial = fake.dialer()
queue := newMemQueue()
b.Queue = queue.publish
rt := startBundle(t, b.Tools())
text, isErr := rt.call(SendTool, "novox.triage", sendArgs)
@@ -443,11 +483,11 @@ func TestOnDeclineNothingIsSentAndTheStatusSaysDeclined(t *testing.T) {
w := router.chose(id, OptionDecline, judgedAt.Add(time.Minute))
router.close(w)
if _, err := Decide(sends, b.lookupAccount, b.sendActor(), w, judgedAt.Add(2*time.Minute)); err != nil {
if _, err := Decide(sends, b.queuePublish(), w, judgedAt.Add(2*time.Minute)); err != nil {
t.Fatalf("deciding: %v", err)
}
if n := fake.count(); n != 0 {
t.Fatalf("the fake server received %d message(s) on Decline", n)
if n := queue.count(); n != 0 {
t.Fatalf("the queue received %d message(s) on Decline", n)
}
text, isErr = rt.call(SendStatusTool, "novox.triage", map[string]any{"id": id})
@@ -474,12 +514,12 @@ func TestMailSendRefusesAnAccountWithNoOutgoingServer(t *testing.T) {
}
}
// (5) An approved send goes out once the warrant reaches the module through the hear handler alone — never
// through mail_send_status (novox/hq ADR 0361 §2, review round 1 of issue 790).
func TestAnApprovedSendGoesOutOnceWithNoStatusCall(t *testing.T) {
fake := newFakeSMTPServer(t)
// 1194/AC1: An approved send is queued once the warrant reaches the module through the hear handler alone —
// never through mail_send_status (novox/hq ADR 0361 §2, review round 1 of issue 790).
func TestAnApprovedSendIsQueuedOnceWithNoStatusCall(t *testing.T) {
b, router, _ := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"})
b.Dial = fake.dialer()
queue := newMemQueue()
b.Queue = queue.publish
rt := startBundle(t, b.Tools())
text, isErr := rt.call(SendTool, "novox.triage", sendArgs)
@@ -494,21 +534,21 @@ func TestAnApprovedSendGoesOutOnceWithNoStatusCall(t *testing.T) {
router.close(w)
b.Heard(w) // the warrant delivered through the hear handler only
if n := fake.count(); n != 1 {
t.Fatalf("the fake server received %d message(s) after the warrant was heard once", n)
if n := queue.count(); n != 1 {
t.Fatalf("the queue received %d message(s) after the warrant was heard once", n)
}
b.Heard(w) // the same warrant delivered a second time
if n := fake.count(); n != 1 {
t.Fatalf("the fake server received %d message(s) after the warrant was heard twice", n)
if n := queue.count(); n != 1 {
t.Fatalf("the queue received %d message(s) after the warrant was heard twice", n)
}
}
// (6) The sweep acts on a send the operator approved while this module was never told directly: it reads the
// router's own record, the same path mail_send_status takes (novox/hq issue 790, review round 1).
// 1194/AC1: The sweep acts on a send the operator approved while this module was never told directly: it reads
// the router's own record, the same path mail_send_status takes (novox/hq issue 790, review round 1).
func TestSweepActsOnAnApprovedSendItWasNotToldAboutDirectly(t *testing.T) {
fake := newFakeSMTPServer(t)
b, router, _ := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"})
b.Dial = fake.dialer()
queue := newMemQueue()
b.Queue = queue.publish
rt := startBundle(t, b.Tools())
text, isErr := rt.call(SendTool, "novox.triage", sendArgs)
@@ -523,12 +563,66 @@ func TestSweepActsOnAnApprovedSendItWasNotToldAboutDirectly(t *testing.T) {
router.close(w) // only the router's record holds the answer; mesh-mail is never told directly
b.Sweep(judgedAt.Add(2 * time.Minute))
if n := fake.count(); n != 1 {
if n := queue.count(); n != 1 {
t.Fatalf("the sweep did not act on the approved send: %d message(s)", n)
}
b.Sweep(judgedAt.Add(3 * time.Minute))
if n := fake.count(); n != 1 {
t.Fatalf("a second sweep sent again: %d message(s)", n)
// A second sweep, still within the just-queued record's own grace, does not publish it again (past the
// grace, it would: TestSweepPublishesAgainAQueuedRecordPastItsGrace, novox/hq review round 1).
b.Sweep(judgedAt.Add(2*time.Minute + QueuedGrace/2))
if n := queue.count(); n != 1 {
t.Fatalf("a second sweep within the grace queued again: %d message(s)", n)
}
}
// 1194/AC2 (as decided to replace the two-state approved/queued design, review round 1): the sweep
// republishes a queued, unclaimed record whose queued time is older than QueuedGrace, the send id still
// its message id — the stream's duplicate window and the sender's own claim keep it one send either way
// (proved end to end over a real bus by the module-whole acceptance test, issue 1198 AC4).
func TestSweepPublishesAgainAQueuedRecordPastItsGrace(t *testing.T) {
b, router, sends := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"})
queue := newMemQueue()
queue.failErr = errors.New("the bus is down")
b.Queue = queue.publish
rt := startBundle(t, b.Tools())
text, isErr := rt.call(SendTool, "novox.triage", sendArgs)
if isErr {
t.Fatalf("%s", text)
}
var out map[string]any
_ = json.Unmarshal([]byte(text), &out)
id := out["ask"].(string)
w := router.chose(id, OptionSend, judgedAt.Add(time.Minute))
router.close(w)
decidedAt := judgedAt.Add(2 * time.Minute)
if _, err := Decide(sends, b.queuePublish(), w, decidedAt); err != nil {
t.Fatalf("deciding: %v", err)
}
if n := queue.count(); n != 0 {
t.Fatalf("the queue received %d message(s) despite failing", n)
}
// There is no "approved" state: the record is already "queued", its own publish just failed.
req, _, ok, err := getSend(sends, id)
if err != nil || !ok || req.Decided == nil || req.Decided.Outcome != "queued" {
t.Fatalf("left queued despite the publish failing: %v %v %+v", err, ok, req.Decided)
}
queue.failErr = nil
// Within its grace, the sweep leaves a freshly queued record alone.
b.Sweep(decidedAt.Add(QueuedGrace / 2))
if n := queue.count(); n != 0 {
t.Fatalf("the sweep republished within the grace: %d message(s)", n)
}
// Past its grace, the sweep publishes the still-queued send again.
b.Sweep(decidedAt.Add(QueuedGrace + time.Second))
if n := queue.count(); n != 1 {
t.Fatalf("the sweep did not republish the queued send past its grace: %d message(s)", n)
}
got := queue.last()
if got.id != id || got.body.Send != id {
t.Fatalf("republished %+v, want the send id as the message id", got)
}
}
@@ -590,26 +684,15 @@ func TestMailSendRefusesAControlCharacterInTheSubjectOrAnAddress(t *testing.T) {
}
// (9) Rcpt takes the parsed address alone; the To header takes the formatted address (novox/hq issue 790, review
// round 1).
// round 1). Tested directly against SendMail (smtp.go): since issue 1194 the tools never dial an outgoing
// server themselves — Decide only records approved and queues the send for the sender (issue 1196, 1198), which
// is where SendMail is actually called from.
func TestSendUsesTheParsedAddressForRcptAndTheFormattedAddressForTheToHeader(t *testing.T) {
fake := newFakeSMTPServer(t)
b, router, _ := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"})
b.Dial = fake.dialer()
rt := startBundle(t, b.Tools())
args := map[string]any{"account": "mesh", "to": []any{"Ops Team <ops@example.org>"},
"subject": "Status", "body": "body"}
text, isErr := rt.call(SendTool, "novox.triage", args)
if isErr {
t.Fatalf("%s", text)
acc := Account{Name: "mesh", Address: "jschoubben@novox.be", Outgoing: &Server{Host: "127.0.0.1", Port: 25, Security: "tls"}}
if err := SendMail(fake.dialer(), acc, "the-password", []string{"Ops Team <ops@example.org>"}, "Status", "body", judgedAt, "send-abc123"); err != nil {
t.Fatalf("SendMail: %v", err)
}
var out map[string]any
_ = json.Unmarshal([]byte(text), &out)
id := out["ask"].(string)
w := router.chose(id, OptionSend, judgedAt.Add(time.Minute))
router.close(w)
b.Heard(w)
got := fake.last()
if len(got.to) != 1 || got.to[0] != "ops@example.org" {
@@ -660,8 +743,8 @@ func TestASendWhoseAskNeverPublishedEndsAfterTheAskWouldHaveLasted(t *testing.T)
if isErr {
t.Fatalf("%s", text)
}
if !strings.Contains(text, `"status":"ended"`) {
t.Fatalf("does not read ended once the ask would have lasted: %s", text)
if !strings.Contains(text, `"status":"expired"`) {
t.Fatalf("does not read expired once the ask would have lasted: %s", text)
}
if !strings.Contains(text, "its ask to the operator was never heard, so it was not sent") {
t.Fatalf("does not say why: %s", text)
@@ -671,9 +754,7 @@ func TestASendWhoseAskNeverPublishedEndsAfterTheAskWouldHaveLasted(t *testing.T)
// (11) The sweep reads only the sends still undecided, by the undecided index, not every send ever kept: a send
// already decided is never asked about again (novox/hq issue 811).
func TestSweepReadsOnlyUndecidedSends(t *testing.T) {
fake := newFakeSMTPServer(t)
b, router, sends := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"})
b.Dial = fake.dialer()
rt := startBundle(t, b.Tools())
// One send, declined right away: decided, so it leaves the undecided index.
@@ -686,7 +767,7 @@ func TestSweepReadsOnlyUndecidedSends(t *testing.T) {
declinedID := out["ask"].(string)
w := router.chose(declinedID, OptionDecline, judgedAt.Add(time.Minute))
router.close(w)
if _, err := Decide(sends, b.lookupAccount, b.sendActor(), w, judgedAt.Add(2*time.Minute)); err != nil {
if _, err := Decide(sends, b.queuePublish(), w, judgedAt.Add(2*time.Minute)); err != nil {
t.Fatalf("deciding: %v", err)
}
@@ -748,3 +829,35 @@ func TestAddAndRemoveUndecidedFailOnACorruptIndexRatherThanResettingIt(t *testin
t.Fatalf("removeUndecided overwrote the corrupt index instead of leaving it: %s", sends.vals[undecidedIndexKey])
}
}
// 1194/AC3: a warrant that does not bind the request (its digest is not the ask's) is refused, as today, and
// nothing is published on mail-sends.
func TestADigestMismatchIsRefusedAndNothingIsQueued(t *testing.T) {
b, router, sends := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"})
queue := newMemQueue()
b.Queue = queue.publish
rt := startBundle(t, b.Tools())
text, isErr := rt.call(SendTool, "novox.triage", sendArgs)
if isErr {
t.Fatalf("%s", text)
}
var out map[string]any
_ = json.Unmarshal([]byte(text), &out)
id := out["ask"].(string)
w := router.chose(id, OptionSend, judgedAt.Add(time.Minute))
w.AskDigest = "sha256:" + strings.Repeat("0", 64) // not the digest of the ask actually published
router.close(w)
d, err := Decide(sends, b.queuePublish(), w, judgedAt.Add(2*time.Minute))
if err != nil {
t.Fatalf("deciding: %v", err)
}
if !strings.HasPrefix(d.Outcome, "refused: ") {
t.Fatalf("outcome: %q, want refused: ...", d.Outcome)
}
if n := queue.count(); n != 0 {
t.Fatalf("the queue received %d message(s) despite the digest mismatch", n)
}
}
modules/mesh-mail/cmd/mesh-mail/sender.go
+295
View File
@@ -0,0 +1,295 @@
package main
// mesh-mail send (novox/hq ADR 0436 §7): the sender, a process with a bus account of its own
// (mesh-mail-send), never a seat. It alone of the module's parts reads an account's password (issue 1193);
// the tools never send — they record the operator's Approve and queue it (issue 1194) on the work queue
// mail-sends, which this process consumes through its one durable consumer (issue 1195). For each message
// it verifies the warrant again and claims the send once (issue 1196), performs the one handover to the
// server (issue 1197's classification), and records the outcome, retries a not-taken failure and emits the
// result (issue 1198). The tell to the operator and the overdue health check are issue 1199's.
import (
"context"
"encoding/json"
"fmt"
"os"
"os/signal"
"syscall"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-mail/cmd/mesh-mail/internal/secret"
)
// SendProcessID is this process's id, as module.json's resource "send" names it: the controller delivers
// its own bus account as the own secret broker-<SendProcessID> (novox/hq ADR 0436 §2).
const SendProcessID = "send"
// MailSendsStream is the work-queue stream mail-sends carries on the bus (novox/hq ADR 0436 §8): the
// manifest's local name "mail-sends", capitalised with its hyphen an underscore, exactly as the controller
// composes a stream's own bus name (mesh-controller internal/catalogue/streams.go StreamBusName,
// strings.ToUpper(strings.ReplaceAll(local, "-", "_")); confirmed against its own fixture, mesh-controller
// cmd/mesh-controller/process_live_test.go's mailSends, which names it "MAIL_SENDS" — review round 1,
// finding 7).
const MailSendsStream = "MAIL_SENDS"
// MessengerAsksBucket is the operator channel's own records bucket, as the controller's own fixtures name
// it (mesh-controller cmd/mesh-controller/process_live_test.go, internal/broker/process_grants_test.go):
// never hard-coded in mesh-sdk go/process itself, since a process may read more than one seat's records.
const MessengerAsksBucket = "messenger_asks"
// routerRecordOf is the real RouterRecord (send.go), reading the router's record of one of mesh-mail's own
// asks under conn's own account (novox/hq ADR 0436 clause 9) — the same direct-get the tools use
// (stdio.SeatRecord), reached here as the sender's own account instead of the tool runner's. Wired into the
// handler once the claim, the send and its outcome are all in place (issues 1197, 1198).
func routerRecordOf(conn *process.Conn) RouterRecord {
return func(id string) (*stdio.StateEntry, error) { return conn.Ask(MessengerAsksBucket, id) }
}
// processSends adapts conn's own `sends` state (novox/hq ADR 0436 clause 7's bus.state) to SendStore
// (send.go), the same CAS the tools use, now reached under the sender's own account. Keys is never called
// by the claim or outcome recording (issues 1196, 1198) — only the tools' sweep calls it, over its own
// account — so it is refused by name if ever called by mistake, rather than silently answering empty.
type processSends struct{ state process.State }
func (p processSends) Get(key string) (json.RawMessage, uint64, bool, error) {
e, err := p.state.Get(key)
if err != nil || e == nil {
return nil, 0, false, err
}
return e.Value, e.Revision, true, nil
}
func (p processSends) Create(key string, value any) (uint64, error) {
return p.state.Create(key, value)
}
func (p processSends) Update(key string, value any, revision uint64) (uint64, error) {
return p.state.Update(key, value, revision)
}
func (p processSends) Keys() ([]string, error) {
return nil, fmt.Errorf("the sender's own `sends` state is never listed whole; only the tools' sweep does that")
}
// send is `mesh-mail send`: it reads its own bus account and the accounts setting, connects, and consumes
// mail-sends until stopped. It refuses to start naming what is missing when the account or the accounts
// setting cannot be read — never a password, which no error here or anywhere in this process logs
// (novox/hq ADR 0436 §7, §12).
func send(env Env) int {
if env.Secrets == "" {
logf("[mesh-mail] send was started without its secrets directory (MESH_MAIL_SECRETS)")
return 2
}
if _, err := ReadAccounts(env.Accounts); err != nil {
logf("[mesh-mail] send: %v", err)
return 1
}
account, err := process.ReadAccount(secret.Path(env.Secrets, "broker-"+SendProcessID))
if err != nil {
logf("[mesh-mail] send: %v", err)
return 1
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
conn, err := process.Connect(account)
if err != nil {
logf("[mesh-mail] send: the bus cannot be reached: %v", err)
return 1
}
defer conn.Close()
h := &senderHandler{
Store: processSends{state: conn.State(SendsState)},
Record: routerRecordOf(conn),
Accounts: func() ([]Account, error) { return ReadAccounts(env.Accounts) },
SecretsDir: env.Secrets,
Emit: emitOf(conn),
Tell: tellOperatorOf(conn),
Logf: logf,
}
go h.tellTicker(ctx)
logf("[mesh-mail] send is consuming %s as %s", MailSendsStream, account.User)
if err := conn.Consume(ctx, MailSendsStream, h.handle); err != nil && ctx.Err() == nil {
logf("[mesh-mail] send stopped: %v", err)
return 1
}
return 0
}
// tellOperatorOf is the real TellOperator (send.go), wrapping conn's own Call to the operator channel's
// tell verb (novox/hq ADR 0380 §8, ADR 0436 clause 11): no lines, no subject naming anything the content
// rule would refuse — the headline alone carries the one line.
func tellOperatorOf(conn *process.Conn) TellOperator {
return func(headline string) error {
_, err := conn.Call("seat:operator-channel.tell", map[string]any{"headline": headline, "lines": []string{}, "subject": "mesh-mail"})
return err
}
}
// TellEvery is how often the sender looks for a send still owing its own tell (novox/hq ADR 0436 clause
// 11: "at least every 10 minutes").
const TellEvery = 10 * time.Minute
// tellTicker runs every send still owing the sender's own tell at least every TellEvery, until ctx ends
// (novox/hq ADR 0436 clause 11, issue 1199): once right away, so a tell left over a restart is not made to
// wait a full interval.
func (h *senderHandler) tellTicker(ctx context.Context) {
t := time.NewTicker(TellEvery)
defer t.Stop()
for {
h.tellPending()
select {
case <-ctx.Done():
return
case <-t.C:
}
}
}
func (h *senderHandler) tellPending() {
ids, err := tellPendingIDs(h.Store)
if err != nil {
h.logf("[mesh-mail] send: the tells-pending index cannot be read: %v", err)
return
}
for _, id := range ids {
if err := tellSend(h.Store, h.Tell, id, h.now()); err != nil {
h.logf("[mesh-mail] send: telling the operator about %s: %v", id, err)
}
}
}
// runSend connects as the sender's own account and consumes mail-sends until ctx ends, handing each message
// to handle. Apart from send so a test can drive it over a bus of its own, with a handler of its own
// (novox/hq issue 1195, AC2): on ctx ending (a signal), Consume stops pulling and returns without
// acknowledging a message still in handle's hands — only handle decides a message's fate (process/queue.go).
func runSend(ctx context.Context, account process.Account, handle func(*process.Message)) error {
conn, err := process.Connect(account)
if err != nil {
return fmt.Errorf("the bus cannot be reached: %w", err)
}
defer conn.Close()
return conn.Consume(ctx, MailSendsStream, handle)
}
// emitOf is the real Emit (send.go), wrapping conn's own Emit with the given id as both x-event-id and the
// bus's own dedup id (novox/hq ADR 0436 clauses 4, 11).
func emitOf(conn *process.Conn) Emit {
return func(eventType, id string, body any) error {
return conn.Emit(eventType, body, stdio.EmitOptions{MessageID: id})
}
}
// senderHandler is everything handleQueuedSend needs: the `sends` state and the router's record, both
// reached under the sender's own account, the accounts setting (to find the server to connect to) and the
// secrets directory (to read the one password this process holds), and how it sends and emits — real
// unless a test gives its own (novox/hq ADR 0436 §7, issues 1196 to 1198).
type senderHandler struct {
Store SendStore
Record RouterRecord
Accounts func() ([]Account, error)
SecretsDir string
Dial SMTPDialer
Emit Emit
Tell TellOperator
Now func() time.Time
Logf func(format string, a ...any)
}
func (h *senderHandler) now() time.Time {
if h.Now != nil {
return h.Now()
}
return time.Now()
}
func (h *senderHandler) logf(format string, a ...any) {
if h.Logf != nil {
h.Logf(format, a...)
}
}
// handle is the handler mesh-mail send's Consume hands every message to mail-sends (novox/hq ADR 0436
// clauses 9, 10, 11): the warrant verified again and the send claimed (claimQueuedSend, send.go), the one
// handover to the server attempted, and the outcome recorded, retried or emitted (recordOutcome, send.go) —
// then, and only then, the message is acknowledged, asked redelivered, or terminated, exactly as that
// outcome says.
func (h *senderHandler) handle(msg *process.Message) {
var q QueuedSend
if err := json.Unmarshal(msg.Body, &q); err != nil {
h.logf("[mesh-mail] send: a queued message on %s could not be read: %v", msg.Subject, err)
if err := msg.Term(); err != nil {
h.logf("[mesh-mail] send: terminating %s: %v", msg.Subject, err)
}
return
}
req, claimed, rev, outcome, err := claimQueuedSend(h.Store, h.Record, h.Emit, q.Send, h.now())
if err != nil {
// Logged, never returned on alone: an index-bookkeeping error (novox/hq review round 1 finding 3)
// can stand beside a real decision (claimRefused, claimNotProven) that the switch below still
// must act on; a fatal read or claim failure, with no decision made, always reads claimLostRace,
// which the switch already leaves to the bus.
h.logf("[mesh-mail] send: claiming %s: %v", q.Send, err)
}
switch outcome {
case claimLostRace:
return // another delivery of the same message already owns it.
case claimRefused, claimNotProven, claimAlreadyFinished:
// claimAlreadyFinished: a repeat of a message whose send already finished (the operator was
// already told, or not, exactly once) — terminated, and nothing else touched (novox/hq review,
// operator-approved change to clause 10).
if err := msg.Term(); err != nil {
h.logf("[mesh-mail] send: terminating %s: %v", q.Send, err)
}
return
}
sendErr := h.attempt(req)
_, action, delay, err := recordOutcome(h.Store, h.Emit, req, claimed, rev, sendErr, h.now())
if err != nil {
h.logf("[mesh-mail] send: recording the outcome of %s: %v", q.Send, err)
}
switch action {
case actionAck:
if err := msg.Ack(); err != nil {
h.logf("[mesh-mail] send: acknowledging %s: %v", q.Send, err)
}
case actionRetry:
if err := msg.NakWithDelay(delay); err != nil {
h.logf("[mesh-mail] send: asking %s redelivered: %v", q.Send, err)
}
case actionTerm:
if err := msg.Term(); err != nil {
h.logf("[mesh-mail] send: terminating %s: %v", q.Send, err)
}
case actionLeave:
// The outcome's own write could not be kept, even retried on a changed revision (novox/hq review
// round 1 finding 5): nothing was emitted above, and nothing is done to the message either — left
// entirely to the bus, so a later redelivery finds the record still at its previous outcome.
}
}
// attempt is the one handover to req.Account's outgoing server (novox/hq ADR 0436 clause 10): it reads the
// accounts setting and the account's own password fresh, never from the queue's message, and every way it
// cannot even be attempted (the account vanished, its password not given, the setting unreadable) is a
// *SendError classified NotTaken, exactly as SendMail's own would be — safe to retry.
func (h *senderHandler) attempt(req SendRequest) error {
accounts, err := h.Accounts()
if err != nil {
return &SendError{Outcome: NotTaken, Err: fmt.Errorf("the accounts setting cannot be read: %w", err)}
}
acc, found := accountByName(accounts, req.Account)
if !found {
return &SendError{Outcome: NotTaken, Err: fmt.Errorf("the account %s no longer exists", req.Account)}
}
pw, given, err := secret.Read(h.SecretsDir, acc.Secret())
if err != nil {
return &SendError{Outcome: NotTaken, Err: fmt.Errorf("the account %s's password cannot be read: %w", acc.Name, err)}
}
if !given {
return &SendError{Outcome: NotTaken, Err: fmt.Errorf("the account %s has no password yet; the operator gives it at the desk", acc.Name)}
}
dial := h.Dial
if dial == nil {
dial = DialSMTP
}
return SendMail(dial, acc, pw, req.To, req.Subject, req.Body, h.now(), req.ID)
}
modules/mesh-mail/cmd/mesh-mail/sender_integration_test.go
+163
View File
@@ -0,0 +1,163 @@
//go:build integration
package main
// mesh-mail send over a real bus (novox/hq ADR 0436 §7, issue 1195 AC2): a JetStream server of the test's
// own, nats-server v2.11.17, copied from modules/mesh-delivery/cmd/mesh-delivery/bus_test.go:52-58
// (newTestBus) — the SDK's own testbus is internal, so this module keeps its own copy. Compiled by the
// merge gate's `go vet -tags integration`; never run on this machine (the context file's hard rule).
import (
"context"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
)
// mailSendApprovedSubject is the real wire subject the controller composes for mesh-mail's own stream
// mail-sends' one event, mail.send.approved: "mesh.stream." + module + ".event." + event (mesh-controller
// internal/broker/modstreams.go ModuleStreamSubject, moduleStreamPrefix). The sender's own Consume binds
// by stream and consumer name alone (mesh-sdk go/process queue.go), so this subject matters only to how a
// test builds the stream and publishes to it — it must match what the controller actually composes, or an
// integration test proves nothing about the real path (novox/hq review round 1, finding 7).
const mailSendApprovedSubject = "mesh.stream." + Module + ".event." + MailSendApproved
// mailSendsStreamConfig is the work-queue stream mail-sends, composed exactly as the controller would
// (mesh-controller internal/catalogue/streams.go StreamDeclaration.Stream and
// internal/broker/modstreams.go ModuleStream.Stream): retention work-queue, discard new, module.json's
// max-age-days 1 and max-mib 64, and its "duplicates": 24 — in hours (novox/hq ADR 0436 clause 6) — which
// the controller turns into DuplicatesSeconds = Duplicates * 60 * 60, 24 hours here.
func mailSendsStreamConfig() jetstream.StreamConfig {
return jetstream.StreamConfig{
Name: MailSendsStream,
Subjects: []string{mailSendApprovedSubject},
Retention: jetstream.WorkQueuePolicy,
Discard: jetstream.DiscardNew,
MaxAge: 24 * time.Hour,
MaxBytes: 64 << 20,
Duplicates: 24 * time.Hour,
}
}
// mailSendsConsumerConfig is the one durable consumer the controller composes for a process's bus.queues
// (mesh-controller internal/broker/modstreams.go QueueConsumer): AckWaitSeconds 30, MaxDeliver from the
// stream's own max-deliveries (module.json: 5, MostAttempts here).
func mailSendsConsumerConfig(durable string) jetstream.ConsumerConfig {
return jetstream.ConsumerConfig{Durable: durable, AckPolicy: jetstream.AckExplicitPolicy, AckWait: 30 * time.Second, MaxDeliver: MostAttempts}
}
// newTestBus starts a JetStream-enabled nats-server for the test alone, on a random port with its own temp
// store (modules/mesh-delivery/cmd/mesh-delivery/bus_test.go:52-58).
func newTestBus(t *testing.T) string {
t.Helper()
opts := &server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, JetStream: true, StoreDir: t.TempDir(),
NoLog: true, NoSigs: true}
s, err := server.NewServer(opts)
if err != nil {
t.Fatalf("a bus for this test could not be made: %v", err)
}
go s.Start()
if !s.ReadyForConnections(30 * time.Second) {
s.Shutdown()
t.Fatal("a bus for this test did not come up within 30s")
}
t.Cleanup(func() { s.Shutdown(); s.WaitForShutdown() })
return s.ClientURL()
}
// 1195/AC2: given the mail-sends work-queue stream and its consumer <machine>_mesh-mail_send, when the
// process runs with a handler and a message is published, then the handler receives it; when the process
// gets SIGTERM (here, its context cancelled, the same signal main.go's send() reacts to) it stops without
// acknowledging a message in hand.
func TestSendConsumesMailSendsAndStopsCleanlyOnSignal(t *testing.T) {
url := newTestBus(t)
ctx := context.Background()
nc, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer nc.Close()
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
if _, err := js.CreateStream(ctx, mailSendsStreamConfig()); err != nil {
t.Fatal(err)
}
account := process.Account{URL: url, Node: "g14", Module: Module, User: "g14." + Module + "." + SendProcessID, Password: "pw"}
if _, err := js.CreateConsumer(ctx, MailSendsStream, mailSendsConsumerConfig(account.QueueConsumerName())); err != nil {
t.Fatal(err)
}
received := make(chan *process.Message, 1)
runCtx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- runSend(runCtx, account, func(m *process.Message) { received <- m }) }()
body := []byte(`{"send":"send-1","account":"mesh","ask":"send-1"}`)
// Published twice under the same message id (the send id, as the tools' own publish path always uses,
// novox/hq ADR 0436 clause 8): the stream's own 24-hour duplicate window keeps it one message, proving
// the dedup this test's stream config actually carries (novox/hq review round 1, finding 7).
if _, err := js.Publish(ctx, mailSendApprovedSubject, body, jetstream.WithMsgID("send-1")); err != nil {
t.Fatal(err)
}
if _, err := js.Publish(ctx, mailSendApprovedSubject, body, jetstream.WithMsgID("send-1")); err != nil {
t.Fatal(err)
}
select {
case msg := <-received:
if msg.Subject != mailSendApprovedSubject {
t.Fatalf("received on %q, want %q", msg.Subject, mailSendApprovedSubject)
}
// Left exactly here, never acknowledged: the handler's own fate-deciding (Ack, NakWithDelay, Term) is
// issues 1196-1198's, not this skeleton's.
case <-time.After(10 * time.Second):
t.Fatal("the handler never received the published message")
}
select {
case <-received:
t.Fatal("the duplicate publish under the same message id was delivered a second time")
case <-time.After(500 * time.Millisecond):
// The duplicate was dropped by the stream's own window, as it should be.
}
cancel() // the signal send() itself reacts to (SIGTERM, or ^C)
select {
case err := <-done:
if err == nil {
t.Fatal("runSend returned nil, want ctx.Err() once its context was cancelled")
}
case <-time.After(10 * time.Second):
t.Fatal("runSend did not stop once its context was cancelled")
}
// The message was never acknowledged: it is still pending (or ack-pending) on the same durable consumer,
// read fresh over a new connection.
nc2, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer nc2.Close()
js2, err := jetstream.New(nc2)
if err != nil {
t.Fatal(err)
}
cons, err := js2.Consumer(ctx, MailSendsStream, account.QueueConsumerName())
if err != nil {
t.Fatal(err)
}
info, err := cons.Info(ctx)
if err != nil {
t.Fatal(err)
}
if info.NumPending == 0 && info.NumAckPending == 0 {
t.Fatal("the message was acknowledged despite the process stopping mid-handle")
}
}
modules/mesh-mail/cmd/mesh-mail/sender_test.go
+80
View File
@@ -0,0 +1,80 @@
package main
// mesh-mail send (novox/hq ADR 0436 §7, issue 1195): refuses to start naming what is missing, never
// attempting to reach a bus when it cannot even read its own account or the accounts setting.
import (
"path/filepath"
"testing"
)
// 1195/AC1: given `mesh-mail send` with MESH_MAIL_SECRETS holding broker-send.secret in the controller's
// form, when it starts, then it reads the account and the accounts file, and refuses to start naming what
// is missing when either is absent.
func TestSendRefusesToStartNamingWhatIsMissing(t *testing.T) {
validAccount := `{"url": "nats://127.0.0.1:4222", "node": "g14", "module": "mesh-mail", ` +
`"user": "g14.mesh-mail.send", "password": "pw"}`
t.Run("no secrets directory", func(t *testing.T) {
dir := t.TempDir()
env := Env{Accounts: filepath.Join(dir, "accounts.json")}
writeFile(t, env.Accounts, stageOne)
if code := send(env); code == 0 {
t.Fatal("send started with no secrets directory at all")
}
})
t.Run("no accounts file", func(t *testing.T) {
dir := t.TempDir()
env := Env{Accounts: filepath.Join(dir, "accounts.json"), Secrets: dir}
writeFile(t, filepath.Join(dir, "broker-send.secret"), validAccount)
if code := send(env); code == 0 {
t.Fatal("send started with no accounts setting")
}
})
t.Run("accounts file is not the module's", func(t *testing.T) {
dir := t.TempDir()
env := Env{Accounts: filepath.Join(dir, "accounts.json"), Secrets: dir}
writeFile(t, env.Accounts, `not json`)
writeFile(t, filepath.Join(dir, "broker-send.secret"), validAccount)
if code := send(env); code == 0 {
t.Fatal("send started with an accounts setting it cannot read")
}
})
t.Run("no own account delivered yet", func(t *testing.T) {
dir := t.TempDir()
env := Env{Accounts: filepath.Join(dir, "accounts.json"), Secrets: dir}
writeFile(t, env.Accounts, stageOne)
// broker-send.secret is not written at all.
if code := send(env); code == 0 {
t.Fatal("send started with no own account delivered")
}
})
t.Run("own account missing a field", func(t *testing.T) {
dir := t.TempDir()
env := Env{Accounts: filepath.Join(dir, "accounts.json"), Secrets: dir}
writeFile(t, env.Accounts, stageOne)
writeFile(t, filepath.Join(dir, "broker-send.secret"), `{"url": "nats://127.0.0.1:4222", "node": "g14", "module": "mesh-mail"}`)
code := send(env)
if code == 0 {
t.Fatal("send started with an own account missing its user and password")
}
})
}
// The account file's path is exactly what secret.Path gives for the member "broker-<process id>", inside
// the directory MESH_MAIL_SECRETS already names — no secret of its own needs declaring in the manifest for
// it (novox/hq ADR 0436 §2: the controller delivers it automatically for every process with a bus).
func TestSendProcessIDNamesItsOwnAccountFile(t *testing.T) {
if SendProcessID != "send" {
t.Fatalf("SendProcessID: %q; module.json's resource id is \"send\"", SendProcessID)
}
// mesh-controller's own StreamBusName: "mail-sends" capitalised, its hyphen an underscore (review round
// 1, finding 7 — confirmed against cmd/mesh-controller/process_live_test.go's own fixture).
if MailSendsStream != "MAIL_SENDS" {
t.Fatalf("MailSendsStream: %q, want MAIL_SENDS", MailSendsStream)
}
}
modules/mesh-mail/cmd/mesh-mail/smtp.go
+102 -14
View File
@@ -8,10 +8,12 @@ package main
import (
"crypto/tls"
"errors"
"fmt"
"mime"
"net/mail"
"net/smtp"
"net/textproto"
"strings"
"time"
"unicode"
@@ -55,10 +57,63 @@ func DialSMTP(server Server) (*smtp.Client, error) {
}
}
// SendMail sends one message through the account's outgoing server (novox/hq ADR 0361 §1, §2): signs in with the
// password given, states every recipient, and writes the message once. Never logs the password or the body; an
// error names the server and what it said, never either.
func SendMail(dial SMTPDialer, a Account, password string, to []string, subject, body string, now time.Time) error {
// SendOutcome classifies how a send attempt ended (novox/hq ADR 0436 clause 10), so the sender knows
// whether a retry is safe, certainly wrong, or neither:
//
// - NotTaken: no connection, the sign-in refused, or a 4xx reply to MAIL, RCPT or DATA, or to the end of
// data — the server provably never took the mail; retrying is safe.
// - Refused: a 5xx reply before or at the end of data — the server provably refused the mail; never
// retried.
// - NotProven: the connection was lost after the data was written, with no reply at all — the mail may or
// may not have gone out; never retried automatically either way.
type SendOutcome int
const (
NotTaken SendOutcome = iota
Refused
NotProven
)
func (o SendOutcome) String() string {
switch o {
case Refused:
return "refused"
case NotProven:
return "not proven"
default:
return "not taken"
}
}
// SendError classifies a SendMail failure (novox/hq ADR 0436 clause 10): Outcome says which, and the error
// itself (Unwrap) carries the server's own words where there were any, never the password or the body.
type SendError struct {
Outcome SendOutcome
Err error
}
func (e *SendError) Error() string { return e.Err.Error() }
func (e *SendError) Unwrap() error { return e.Err }
// classifyReply is NotTaken for a 4xx reply, Refused for a 5xx, and NotTaken for anything else (no
// connection, or the connection lost before a reply was read at all) — the safe default everywhere but the
// one case clause 10 names its own outcome for: the connection lost after the data was written, which the
// caller classifies as NotProven itself, never through this.
func classifyReply(err error) SendOutcome {
var proto *textproto.Error
if errors.As(err, &proto) && proto.Code >= 500 {
return Refused
}
return NotTaken
}
// SendMail sends one message through the account's outgoing server (novox/hq ADR 0361 §1, §2; ADR 0436
// clause 10): signs in with the password given, states every recipient, and writes the message once, its
// Message-ID naming sendID so the mail is never sent twice within the day (clause 10's own guarantee, kept
// by the sender's compare-and-set claim and the bus's duplicate window on the id alone — this is the
// mail's own proof of it). Never logs the password or the body; an error names the server and what it
// said, never either, and is always a *SendError (classifyReply, or NotProven below).
func SendMail(dial SMTPDialer, a Account, password string, to []string, subject, body string, now time.Time, sendID string) error {
if a.Outgoing == nil {
return fmt.Errorf("the account %s has no outgoing server", a.Name)
}
@@ -76,44 +131,63 @@ func SendMail(dial SMTPDialer, a Account, password string, to []string, subject,
}
client, err := dial(*a.Outgoing)
if err != nil {
return err
return &SendError{Outcome: NotTaken, Err: err}
}
defer client.Close()
from := a.SignInAs()
auth := smtp.PlainAuth("", from, password, a.Outgoing.Host)
if err := client.Auth(auth); err != nil {
return fmt.Errorf("signing in to %s failed: %w", a.Outgoing.Address(), err)
// A refused sign-in is its own case, never classified by a reply code: the server never took the
// mail either way (novox/hq ADR 0436 clause 10).
return &SendError{Outcome: NotTaken, Err: fmt.Errorf("signing in to %s failed: %w", a.Outgoing.Address(), err)}
}
if err := client.Mail(from); err != nil {
return fmt.Errorf("the outgoing server refused the sender: %w", err)
return &SendError{Outcome: classifyReply(err), Err: fmt.Errorf("the outgoing server refused the sender: %w", err)}
}
for _, rcpt := range rcpts {
if err := client.Rcpt(rcpt); err != nil {
return fmt.Errorf("the outgoing server refused the recipient %s: %w", rcpt, err)
return &SendError{Outcome: classifyReply(err), Err: fmt.Errorf("the outgoing server refused the recipient %s: %w", rcpt, err)}
}
}
w, err := client.Data()
if err != nil {
return fmt.Errorf("the outgoing server refused the message: %w", err)
return &SendError{Outcome: classifyReply(err), Err: fmt.Errorf("the outgoing server refused the message: %w", err)}
}
if _, err := w.Write(composeMessage(from, headers, subject, body, now)); err != nil {
return fmt.Errorf("the message could not be written: %w", err)
if _, err := w.Write(composeMessage(from, headers, subject, body, now, sendID)); err != nil {
// The data was not written whole: never "after the data was written" (clause 10's not-proven case),
// so this is not taken, not not-proven — nothing coherent reached the server.
return &SendError{Outcome: NotTaken, Err: fmt.Errorf("the message could not be written: %w", err)}
}
if err := w.Close(); err != nil {
return fmt.Errorf("the outgoing server did not accept the message: %w", err)
// Close is what sends the terminating "." and reads the server's one reply to the whole message
// (SMTP's "end of data", clause 10): a real reply here classifies exactly as MAIL or RCPT's does; no
// reply at all (the connection lost after the data was written) is not proven — never retried
// automatically either way.
outcome := NotProven
var proto *textproto.Error
if errors.As(err, &proto) {
outcome = classifyReply(err)
}
return &SendError{Outcome: outcome, Err: fmt.Errorf("the outgoing server did not accept the message: %w", err)}
}
return client.Quit()
// The message is already accepted (Close's 250 reply); Quit is a courtesy goodbye, and its own failure
// here never turns an accepted send into a failed one.
_ = client.Quit()
return nil
}
// composeMessage is a plain-text message, its lines CRLF-terminated as SMTP takes them. to is the recipients
// already formatted (mail.Address.String()), the same ones Rcpt was given, never the raw strings a caller gave.
func composeMessage(from string, to []string, subject, body string, now time.Time) []byte {
// Its Message-ID names sendID (novox/hq ADR 0436 clause 10), so the mail carries the proof of which send it
// is, at <sendID>@<the account's own domain, from the From address>.
func composeMessage(from string, to []string, subject, body string, now time.Time, sendID string) []byte {
var b strings.Builder
header := func(k, v string) { b.WriteString(k + ": " + v + "\r\n") }
header("From", from)
header("To", strings.Join(to, ", "))
header("Subject", encodeHeader(subject))
header("Date", now.Format(time.RFC1123Z))
header("Message-Id", "<"+sendID+"@"+messageIDDomain(from)+">")
header("MIME-Version", "1.0")
header("Content-Type", "text/plain; charset=utf-8")
header("Content-Transfer-Encoding", "8bit")
@@ -122,6 +196,20 @@ func composeMessage(from string, to []string, subject, body string, now time.Tim
return []byte(b.String())
}
// messageIDDomain is the domain a Message-ID is minted under: the part of from after its @, as a bare
// address or parsed out of "Name <addr>"; "localhost" when from carries none (never empty, which is not a
// valid Message-ID).
func messageIDDomain(from string) string {
addr := from
if parsed, err := mail.ParseAddress(from); err == nil {
addr = parsed.Address
}
if _, domain, found := strings.Cut(addr, "@"); found && domain != "" {
return domain
}
return "localhost"
}
// encodeHeader word-encodes a header's value when it holds more than ASCII; a header a phone never needs to show.
func encodeHeader(s string) string {
for _, r := range s {
modules/mesh-mail/cmd/mesh-mail/smtp_classify_test.go
+262
View File
@@ -0,0 +1,262 @@
package main
// SendMail classifies every failure (novox/hq ADR 0436 clause 10, issue 1197): not taken (no connection, a
// refused sign-in, or a 4xx reply to MAIL, RCPT or DATA or to the end of data), refused (a 5xx before or at
// the end of data), or not proven (the connection lost after the data was written, before any reply). Its
// Message-ID names the send id.
import (
"bufio"
"errors"
"fmt"
"net"
"net/smtp"
"strings"
"sync/atomic"
"testing"
)
// step is one scripted SMTP reply, or a drop (the connection closed instead of a reply).
type step struct {
drop bool
code int
msg string
}
func reply(code int, msg string) step { return step{code: code, msg: msg} }
func drop() step { return step{drop: true} }
// scriptedSMTPServer answers EHLO, AUTH, MAIL, RCPT, DATA's own 354 and the end-of-data reply each with a
// step a test sets, defaulting to the ordinary success sequence — so a test overrides exactly the one stage
// its case is about.
type scriptedSMTPServer struct {
ln net.Listener
ehlo, auth, mail, rcpt, dataOpen, dataEnd step
received int32 // atomic: how many full DATA bodies this server has read
}
func newScriptedSMTPServer(t *testing.T) *scriptedSMTPServer {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
s := &scriptedSMTPServer{ln: ln, ehlo: reply(250, "fake.local"), auth: reply(235, "authenticated"),
mail: reply(250, "OK"), rcpt: reply(250, "OK"), dataOpen: reply(354, "go ahead"), dataEnd: reply(250, "queued")}
go s.serve()
t.Cleanup(func() { _ = ln.Close() })
return s
}
func (s *scriptedSMTPServer) addr() string { return s.ln.Addr().String() }
// receivedCount is how many full DATA bodies this server has read, for a test to prove nothing was
// received at all (novox/hq review round 1 finding 6).
func (s *scriptedSMTPServer) receivedCount() int { return int(atomic.LoadInt32(&s.received)) }
func (s *scriptedSMTPServer) dialer() SMTPDialer {
addr := s.addr()
return func(Server) (*smtp.Client, error) { return smtp.Dial(addr) }
}
func (s *scriptedSMTPServer) send(conn net.Conn, st step) bool {
if st.drop {
_ = conn.Close()
return false
}
fmt.Fprintf(conn, "%d %s\r\n", st.code, st.msg)
return true
}
func (s *scriptedSMTPServer) serve() {
for {
conn, err := s.ln.Accept()
if err != nil {
return
}
go s.handle(conn)
}
}
func (s *scriptedSMTPServer) handle(conn net.Conn) {
defer conn.Close()
r := bufio.NewReader(conn)
fmt.Fprintf(conn, "220 fake.local ESMTP\r\n")
for {
line, err := r.ReadString('\n')
if err != nil {
return
}
upper := strings.ToUpper(strings.TrimRight(line, "\r\n"))
switch {
case strings.HasPrefix(upper, "EHLO"), strings.HasPrefix(upper, "HELO"):
if !s.send(conn, s.ehlo) {
return
}
case strings.HasPrefix(upper, "AUTH"):
if !s.send(conn, s.auth) {
return
}
case strings.HasPrefix(upper, "MAIL"):
if !s.send(conn, s.mail) {
return
}
case strings.HasPrefix(upper, "RCPT"):
if !s.send(conn, s.rcpt) {
return
}
case upper == "DATA":
if !s.send(conn, s.dataOpen) {
return
}
if s.dataOpen.code >= 400 {
continue // not entering data mode; the client ends the transaction itself
}
for {
dl, err := r.ReadString('\n')
if err != nil {
return
}
if dl == ".\r\n" || dl == ".\n" {
break
}
}
atomic.AddInt32(&s.received, 1) // a full message body was read, whatever dataEnd then says
s.send(conn, s.dataEnd) // a drop here is "the connection lost after the data was written"
return
case upper == "QUIT":
fmt.Fprintf(conn, "221 bye\r\n")
return
default:
fmt.Fprintf(conn, "500 unrecognized\r\n")
}
}
}
// classifyTestAccount is an account whose outgoing server is the given address: Host is its loopback
// literal, which PlainAuth itself accepts without TLS (its own isLocalhost check).
func classifyTestAccount(addr string) Account {
host, portStr, _ := net.SplitHostPort(addr)
var port int
_, _ = fmt.Sscanf(portStr, "%d", &port)
return Account{Name: "mesh", Address: "jschoubben@novox.be", Outgoing: &Server{Host: host, Port: port, Security: "tls"}}
}
func outcomeOf(t *testing.T, err error) SendOutcome {
t.Helper()
var se *SendError
if !errors.As(err, &se) {
t.Fatalf("SendMail's error is not a *SendError: %v (%T)", err, err)
}
return se.Outcome
}
// 1197/AC1: given no connection, a refused sign-in, or a 4xx reply to MAIL, RCPT or DATA or to the end of
// data, when SendMail runs, then it says not taken with the server's words.
func TestSendMailSaysNotTaken(t *testing.T) {
t.Run("no connection", func(t *testing.T) {
s := newScriptedSMTPServer(t)
addr := s.addr()
s.ln.Close() // nothing listens by the time SendMail dials
acc := classifyTestAccount(addr)
dial := func(Server) (*smtp.Client, error) { return smtp.Dial(addr) }
err := SendMail(dial, acc, "pw", []string{"a@b.c"}, "s", "body", judgedAt, "send-1")
if err == nil || outcomeOf(t, err) != NotTaken {
t.Fatalf("err %v, want NotTaken", err)
}
})
t.Run("a refused sign-in", func(t *testing.T) {
s := newScriptedSMTPServer(t)
s.auth = reply(535, "authentication failed")
acc := classifyTestAccount(s.addr())
err := SendMail(s.dialer(), acc, "pw", []string{"a@b.c"}, "s", "body", judgedAt, "send-1")
if err == nil || outcomeOf(t, err) != NotTaken || !strings.Contains(err.Error(), "signing in") {
t.Fatalf("err %v, want NotTaken naming the sign-in", err)
}
})
for _, c := range []struct {
name string
set func(*scriptedSMTPServer)
}{
{"a 4xx to MAIL", func(s *scriptedSMTPServer) { s.mail = reply(451, "try again later") }},
{"a 4xx to RCPT", func(s *scriptedSMTPServer) { s.rcpt = reply(450, "mailbox busy") }},
{"a 4xx to DATA", func(s *scriptedSMTPServer) { s.dataOpen = reply(452, "insufficient storage") }},
{"a 4xx at the end of data", func(s *scriptedSMTPServer) { s.dataEnd = reply(451, "local error") }},
} {
t.Run(c.name, func(t *testing.T) {
s := newScriptedSMTPServer(t)
c.set(s)
acc := classifyTestAccount(s.addr())
err := SendMail(s.dialer(), acc, "pw", []string{"a@b.c"}, "s", "body", judgedAt, "send-1")
if err == nil || outcomeOf(t, err) != NotTaken {
t.Fatalf("err %v, want NotTaken", err)
}
})
}
}
// 1197/AC2: given a 5xx reply before or at the end of data, then it says refused with the server's words.
func TestSendMailSaysRefused(t *testing.T) {
for _, c := range []struct {
name string
set func(*scriptedSMTPServer)
}{
{"a 5xx to MAIL", func(s *scriptedSMTPServer) { s.mail = reply(550, "no such sender") }},
{"a 5xx to RCPT", func(s *scriptedSMTPServer) { s.rcpt = reply(550, "no such recipient") }},
{"a 5xx to DATA", func(s *scriptedSMTPServer) { s.dataOpen = reply(554, "transaction failed") }},
{"a 5xx at the end of data", func(s *scriptedSMTPServer) { s.dataEnd = reply(552, "quota exceeded") }},
} {
t.Run(c.name, func(t *testing.T) {
s := newScriptedSMTPServer(t)
c.set(s)
acc := classifyTestAccount(s.addr())
err := SendMail(s.dialer(), acc, "pw", []string{"a@b.c"}, "s", "body", judgedAt, "send-1")
if err == nil || outcomeOf(t, err) != Refused {
t.Fatalf("err %v, want Refused", err)
}
})
}
}
// 1197/AC3: given the connection lost after the data was written and before any reply, then it says not
// proven.
func TestSendMailSaysNotProvenWhenTheConnectionIsLostAfterTheDataWasWritten(t *testing.T) {
s := newScriptedSMTPServer(t)
s.dataEnd = drop()
acc := classifyTestAccount(s.addr())
err := SendMail(s.dialer(), acc, "pw", []string{"a@b.c"}, "s", "body", judgedAt, "send-1")
if err == nil || outcomeOf(t, err) != NotProven {
t.Fatalf("err %v, want NotProven", err)
}
}
// 1197/AC4: given a send id, when the message is composed, then it carries a Message-ID naming the send id
// (<send id>@<the account's domain>). Proved both in isolation (composeMessage) and over the wire
// (SendMail, with the existing fakeSMTPServer, which keeps the body it received).
func TestComposeMessageCarriesTheSendIDInItsMessageID(t *testing.T) {
raw := composeMessage("jschoubben@novox.be", []string{"a@b.c"}, "s", "body", judgedAt, "send-abc123")
if !strings.Contains(string(raw), "Message-Id: <send-abc123@novox.be>") {
t.Fatalf("the message does not carry the send id's Message-ID:\n%s", raw)
}
}
// composeMessage falls back to "localhost" when From carries no domain at all (never an empty or
// malformed Message-ID).
func TestComposeMessageFallsBackToLocalhostWithNoDomain(t *testing.T) {
raw := composeMessage("postmaster", nil, "s", "body", judgedAt, "send-1")
if !strings.Contains(string(raw), "Message-Id: <send-1@localhost>") {
t.Fatalf("message: %s", raw)
}
}
func TestSendMailCarriesTheMessageIDOverTheWire(t *testing.T) {
fake := newFakeSMTPServer(t)
acc := classifyTestAccount(fake.addr())
if err := SendMail(fake.dialer(), acc, "pw", []string{"a@b.c"}, "s", "body", judgedAt, "send-xyz"); err != nil {
t.Fatalf("SendMail: %v", err)
}
got := fake.last()
if !strings.Contains(got.data, "Message-Id: <send-xyz@novox.be>") {
t.Fatalf("the message sent does not carry the send id's Message-ID:\n%s", got.data)
}
}
modules/mesh-mail/cmd/mesh-mail/tell_test.go
+264
View File
@@ -0,0 +1,264 @@
package main
// The tells to the operator, overdue health and mail_send_status's words (novox/hq ADR 0436 clauses 11,
// 12, issue 1199).
import (
"errors"
"strings"
"testing"
"time"
)
// memTell collects every headline a TellOperator was called with, optionally refusing the first n calls
// (novox/hq issue 1199).
type memTell struct {
headlines []string
refuseN int
}
func (m *memTell) tell(headline string) error {
if m.refuseN > 0 {
m.refuseN--
return errors.New("the channel is down")
}
m.headlines = append(m.headlines, headline)
return nil
}
func tellTestSend(t *testing.T, sends *memSends, outcome string) SendRequest {
t.Helper()
req := SendRequest{ID: newSendID(), Account: "mesh", To: []string{"ops@example.org"}, At: judgedAt}
req.Decided = &SendDecision{At: judgedAt, Outcome: outcome}
if _, err := sends.Create(req.ID, req); err != nil {
t.Fatal(err)
}
if err := addTellPending(sends, req.ID); err != nil {
t.Fatal(err)
}
return req
}
// 1199/AC1: given a send recorded failed, refused or not proven after the operator's Approve, when the
// sender looks, then it sets tell to telling by compare-and-set, calls seat:operator-channel.tell once
// (the headline naming the account and the reason, or for not proven that it may not have been sent; no
// address in it), and on the channel taking it sets told. A refused or unanswered tell leaves telling, and
// the next look tells only records left telling. A sent send is never told.
func TestTellSendTellsOnceAndSettlesToldOnlyOnceTheChannelTakesIt(t *testing.T) {
sends := newMemSends()
req := tellTestSend(t, sends, "failed: 550 no such mailbox")
tell := &memTell{}
if err := tellSend(sends, tell.tell, req.ID, judgedAt.Add(time.Minute)); err != nil {
t.Fatalf("telling: %v", err)
}
if len(tell.headlines) != 1 {
t.Fatalf("told %d time(s): %v", len(tell.headlines), tell.headlines)
}
h := tell.headlines[0]
if !strings.Contains(h, "mesh") || !strings.Contains(h, "550 no such mailbox") {
t.Fatalf("headline: %q", h)
}
if strings.Contains(h, "@") {
t.Fatalf("the headline carries an address: %q", h)
}
kept, _, ok, err := getSend(sends, req.ID)
if err != nil || !ok || kept.Decided == nil || kept.Decided.Tell != "told" {
t.Fatalf("tell not settled told: %v %v %+v", err, ok, kept.Decided)
}
ids, err := tellPendingIDs(sends)
if err != nil || len(ids) != 0 {
t.Fatalf("still indexed after being told: %v %v", err, ids)
}
// A second look at an already-told send tells nothing more.
if err := tellSend(sends, tell.tell, req.ID, judgedAt.Add(2*time.Minute)); err != nil {
t.Fatalf("telling again: %v", err)
}
if len(tell.headlines) != 1 {
t.Fatalf("told again: %v", tell.headlines)
}
}
func TestTellSendLeavesTellingOnARefusedChannelAndTriesAgain(t *testing.T) {
sends := newMemSends()
req := tellTestSend(t, sends, "not proven")
tell := &memTell{refuseN: 1}
if err := tellSend(sends, tell.tell, req.ID, judgedAt.Add(time.Minute)); err != nil {
t.Fatalf("telling: %v", err)
}
kept, _, ok, err := getSend(sends, req.ID)
if err != nil || !ok || kept.Decided == nil || kept.Decided.Tell != "telling" {
t.Fatalf("not left telling: %v %v %+v", err, ok, kept.Decided)
}
ids, err := tellPendingIDs(sends)
if err != nil || len(ids) != 1 {
t.Fatalf("no longer indexed despite the refusal: %v %v", err, ids)
}
// The next look succeeds.
if err := tellSend(sends, tell.tell, req.ID, judgedAt.Add(2*time.Minute)); err != nil {
t.Fatalf("telling again: %v", err)
}
if len(tell.headlines) != 1 {
t.Fatalf("told: %v", tell.headlines)
}
if !strings.Contains(tell.headlines[0], "may not have been sent") {
t.Fatalf("not-proven headline: %q", tell.headlines[0])
}
}
func TestTellSendNeverTellsForASentSend(t *testing.T) {
sends := newMemSends()
req := SendRequest{ID: newSendID(), Account: "mesh", At: judgedAt}
req.Decided = &SendDecision{At: judgedAt, Outcome: "sent"}
if _, err := sends.Create(req.ID, req); err != nil {
t.Fatal(err)
}
tell := &memTell{}
if err := tellSend(sends, tell.tell, req.ID, judgedAt.Add(time.Minute)); err != nil {
t.Fatalf("telling: %v", err)
}
if len(tell.headlines) != 0 {
t.Fatalf("a sent send was told: %v", tell.headlines)
}
}
// review round 1 finding 3: a send indexed for a tell before its own outcome write has landed (recordOutcome
// and claimQueuedSend now index first) must not be dropped by a look that finds it still sending, queued or
// retrying — it is left in the index for the next look, once the real outcome lands.
func TestTellSendKeepsAnEntryWhoseRecordIsStillInFlight(t *testing.T) {
for _, outcome := range []string{"sending", "queued", "retrying"} {
sends := newMemSends()
req := SendRequest{ID: newSendID(), Account: "mesh", At: judgedAt}
req.Decided = &SendDecision{At: judgedAt, Outcome: outcome}
if _, err := sends.Create(req.ID, req); err != nil {
t.Fatal(err)
}
if err := addTellPending(sends, req.ID); err != nil {
t.Fatal(err)
}
tell := &memTell{}
if err := tellSend(sends, tell.tell, req.ID, judgedAt.Add(time.Minute)); err != nil {
t.Fatalf("%s: telling: %v", outcome, err)
}
if len(tell.headlines) != 0 {
t.Fatalf("%s: told before its outcome landed: %v", outcome, tell.headlines)
}
ids, err := tellPendingIDs(sends)
if err != nil || len(ids) != 1 {
t.Fatalf("%s: dropped from the index while still in flight: %v %v", outcome, err, ids)
}
}
}
// 1199/AC2: given a send queued more than 10 minutes ago, or retrying 10 minutes past its next attempt,
// when mail_health runs (Overdue), then it names the send; a send waiting for its next attempt is never
// overdue.
func TestOverdueFindsAQueuedOrRetryingSendPastTenMinutes(t *testing.T) {
sends := newMemSends()
queueTestSend := func(outcome string, at, next time.Time) string {
req := SendRequest{ID: newSendID(), Account: "mesh", At: judgedAt}
req.Decided = &SendDecision{At: at, Outcome: outcome, NextAttempt: next}
if _, err := sends.Create(req.ID, req); err != nil {
t.Fatal(err)
}
if err := addSenderPending(sends, req.ID); err != nil {
t.Fatal(err)
}
return req.ID
}
now := judgedAt
overdueQueued := queueTestSend("queued", now.Add(-11*time.Minute), time.Time{})
freshQueued := queueTestSend("queued", now.Add(-5*time.Minute), time.Time{})
overdueRetrying := queueTestSend("retrying", now.Add(-20*time.Minute), now.Add(-11*time.Minute))
notYetDueRetrying := queueTestSend("retrying", now.Add(-5*time.Minute), now.Add(3*time.Minute))
sendingNotOverdue := queueTestSend("sending", now.Add(-20*time.Minute), time.Time{})
overdue, err := Overdue(sends, now)
if err != nil {
t.Fatalf("Overdue: %v", err)
}
var ids []string
for _, r := range overdue {
ids = append(ids, r.ID)
}
want := map[string]bool{overdueQueued: true, overdueRetrying: true}
if len(ids) != len(want) {
t.Fatalf("overdue: %v, want exactly %v", ids, want)
}
for _, id := range ids {
if !want[id] {
t.Fatalf("unexpectedly overdue: %s (fresh %s, not-yet-due %s, sending %s)", id, freshQueued, notYetDueRetrying, sendingNotOverdue)
}
}
}
// 1199/AC3: given an overdue send, when the tools' second ticker looks, then they tell once through
// waits-tell (telling then told), separate from the sender's own tell.
func TestTellWaitsTellsOnceThroughItsOwnField(t *testing.T) {
sends := newMemSends()
req := SendRequest{ID: newSendID(), Account: "mesh", At: judgedAt}
req.Decided = &SendDecision{At: judgedAt.Add(-20 * time.Minute), Outcome: "queued"}
if _, err := sends.Create(req.ID, req); err != nil {
t.Fatal(err)
}
tell := &memTell{}
if err := tellWaits(sends, tell.tell, req.ID, judgedAt); err != nil {
t.Fatalf("telling waits: %v", err)
}
if len(tell.headlines) != 1 || !strings.Contains(tell.headlines[0], "waits") {
t.Fatalf("waits-tell: %v", tell.headlines)
}
kept, _, ok, err := getSend(sends, req.ID)
if err != nil || !ok || kept.Decided == nil || kept.Decided.WaitsTell != "told" || kept.Decided.Tell != "" {
t.Fatalf("waits-tell not settled apart from the sender's own tell: %v %v %+v", err, ok, kept.Decided)
}
// Told once: a second look tells nothing more.
if err := tellWaits(sends, tell.tell, req.ID, judgedAt.Add(time.Minute)); err != nil {
t.Fatalf("telling waits again: %v", err)
}
if len(tell.headlines) != 1 {
t.Fatalf("told again: %v", tell.headlines)
}
}
// 1199/AC4: given each state, when mail_send_status reads it (reportDecision), then it says: waiting for
// the operator (covered by send_test.go's own asking tests, which read ReadSendStatus before any decision
// exists — reportDecision itself only ever sees an already-decided SendDecision), approved and waiting for
// the sender, being sent, waiting for its next attempt, sent, failed with its reason, refused, not proven,
// declined, or expired — checking both the status word and the words that actually say which of these it
// is (novox/hq review round 1 finding 6: checking status alone proves too little).
func TestReportDecisionCoversTheWholeStatusVocabulary(t *testing.T) {
for name, c := range map[string]struct {
d SendDecision
wantStatus string
wantInWords string
}{
// There is no "approved" state in a send's life: Decide's one compare-and-set write moves it
// straight from undecided to "queued" (novox/hq ADR 0436 clause 8, as decided to replace the
// two-state design, review round 1).
"queued": {SendDecision{At: judgedAt, Outcome: "queued"}, "waiting", "approved and waiting for the sender"},
"being sent": {SendDecision{At: judgedAt, Outcome: "sending"}, "waiting", "being sent"},
"retrying": {SendDecision{At: judgedAt, Outcome: "retrying", NextAttempt: judgedAt.Add(time.Minute)}, "waiting", "waiting for its next attempt"},
"sent": {SendDecision{At: judgedAt, Outcome: "sent"}, "sent", "sent on the operator's Approve"},
"failed with its reason": {SendDecision{At: judgedAt, Outcome: "failed: 550 no such mailbox"}, "failed", "550 no such mailbox"},
"refused": {SendDecision{At: judgedAt, Outcome: "refused: the warrant does not bind the request"}, "refused", "the warrant does not bind the request"},
"not proven": {SendDecision{At: judgedAt, Outcome: "not proven"}, "not proven", "not proven"},
"declined": {SendDecision{At: judgedAt, Outcome: "declined"}, "declined", "the operator declined"},
"expired (never heard)": {SendDecision{At: judgedAt, Outcome: "ended: never heard"}, "expired", "never heard"},
} {
t.Run(name, func(t *testing.T) {
out := map[string]any{}
reportDecision(out, c.d, judgedAt.Add(time.Minute))
if out["status"] != c.wantStatus {
t.Fatalf("status: %v, want %q (words: %v)", out["status"], c.wantStatus, out["words"])
}
words, _ := out["words"].(string)
if !strings.Contains(words, c.wantInWords) {
t.Fatalf("words: %q, want it to contain %q", words, c.wantInWords)
}
})
}
}
modules/mesh-mail/cmd/mesh-mail/tools.go
+32 -34
View File
@@ -45,9 +45,10 @@ const (
)
// Bundle is what the tools reach: the settings, the index, the attachment store, and — for mail_send — the
// operator channel, the `sends` state and the outgoing dialer. The last three are nil outside a test: the real
// bus and dialer (sdkAskBus, keptSends, DialSMTP) are used lazily, so building a Bundle for the read-only tools
// never needs them.
// operator channel, the `sends` state and the work-queue publisher. The last three are nil outside a test: the
// real bus (sdkAskBus, keptSends, sdkQueuePublish) is used lazily, so building a Bundle for the read-only tools
// never needs them. Nothing here reads a password: the tools record the operator's Approve and queue the send
// for the sender, which alone performs it (novox/hq ADR 0436 §12, issue 1193, 1194).
type Bundle struct {
Env Env
Index func(ctx context.Context) (Store, error)
@@ -57,7 +58,7 @@ type Bundle struct {
Asks AskBus
Sends SendStore
Dial SMTPDialer
Queue QueuePublish
Logf func(format string, a ...any)
}
@@ -138,10 +139,11 @@ func (b *Bundle) Tools() []stdio.Tool {
Run: b.readTool},
{Name: HealthTool,
Description: "How mesh-mail is, as its watch last saw it: healthy when every account's server answered within five " +
"minutes; unhealthy, naming the account and why, when a connection is down or its IDLE on the inbox went silent; " +
"waits, naming the account's password, while the operator has not given it at the desk.",
"minutes; unhealthy, naming the account and why, when a connection is down or its IDLE on the inbox went silent, " +
"or an approved send has waited 10 minutes or more for its sender; waits, naming the account's password, while " +
"the operator has not given it at the desk.",
Input: schema(nil, map[string]any{}),
Run: func(map[string]any) (any, error) { return ReadStatus(b.Env.Status, b.now()), nil }},
Run: b.healthTool},
{Name: SendTool,
Description: "Send mail from an account you are granted (the same check as mail_read, novox/hq ADR 0361 §2). It " +
"sends nothing itself: it raises an ask on the operator channel — the account, the recipients, the subject and " +
@@ -205,6 +207,22 @@ func (b *Bundle) caller(args map[string]any) string {
return stdio.CallerOf(args)
}
// healthTool folds an overdue send (novox/hq ADR 0436 clause 11, issue 1199) into the watch's own verdict:
// unhealthy, naming the send, no matter what the watch itself said. Overdue reads `sends` directly — the
// watch process has no access to it — so a send stuck past its own 10 minutes is caught even while every
// account's own connection is fine.
func (b *Bundle) healthTool(map[string]any) (any, error) {
base := ReadStatus(b.Env.Status, b.now())
overdue, err := Overdue(b.sends(), b.now())
if err != nil {
if b.Logf != nil {
b.Logf("[mesh-mail] mail_health: the overdue check failed: %v", err)
}
return base, nil
}
return WithOverdue(base, overdue), nil
}
func (b *Bundle) accountsTool(args map[string]any) (any, error) {
if err := only(args); err != nil {
return nil, err
@@ -397,14 +415,13 @@ func (b *Bundle) sends() SendStore {
return keptSends{stdio.State(SendsState)}
}
// lookupAccount resolves an account by name as the settings currently hold it — read fresh at decide time, never
// the account as it stood when it was asked.
func (b *Bundle) lookupAccount(name string) (Account, bool) {
accounts, err := ReadAccounts(b.Env.Accounts)
if err != nil {
return Account{}, false
// queuePublish is how this bundle publishes onto the module's own work-queue stream: the bus's own, unless a
// test gave one. Nothing here reads a password (novox/hq ADR 0436 §12).
func (b *Bundle) queuePublish() QueuePublish {
if b.Queue != nil {
return b.Queue
}
return accountByName(accounts, name)
return sdkQueuePublish
}
func accountByName(accounts []Account, name string) (Account, bool) {
@@ -416,25 +433,6 @@ func accountByName(accounts []Account, name string) (Account, bool) {
return Account{}, false
}
// sendActor performs one send on the operator's Approve: it reads the account's own secret itself, so Decide and
// the `sends` state never hold a password.
func (b *Bundle) sendActor() SendActor {
dial := b.Dial
if dial == nil {
dial = DialSMTP
}
return func(a Account, to []string, subject, body string, now time.Time) error {
pw, given, err := ReadSecret(b.Env.Secrets, a)
if err != nil {
return fmt.Errorf("the account %s's password cannot be read: %w", a.Name, err)
}
if !given {
return fmt.Errorf("the account %s has no password yet; the operator gives it at the desk", a.Name)
}
return SendMail(dial, a, pw, to, subject, body, now)
}
}
func stringList(v any) []string {
list, _ := v.([]any)
out := make([]string, 0, len(list))
@@ -483,7 +481,7 @@ func (b *Bundle) sendStatusTool(args map[string]any) (any, error) {
if id == "" {
return nil, fmt.Errorf("%s needs id, the ask %s gave", SendStatusTool, SendTool)
}
return ReadSendStatus(b.sends(), b.askBus(), b.lookupAccount, b.sendActor(), id, b.now())
return ReadSendStatus(b.sends(), b.askBus(), b.queuePublish(), id, b.now())
}
// ---- arguments ----------------------------------------------------------------------------------------------------
modules/mesh-mail/go.mod
+16 -4
View File
@@ -1,19 +1,31 @@
module mesh-mail
go 1.25.0
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019062050-34392e74916c
github.com/emersion/go-imap/v2 v2.0.0-beta.8
github.com/emersion/go-message v0.18.2
github.com/jackc/pgx/v5 v5.11.0
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
require (
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
golang.org/x/sync v0.17.0 // indirect
golang.org/x/text v0.29.0 // indirect
github.com/klauspost/compress v1.20.0 // indirect
github.com/minio/highwayhash v1.0.4 // indirect
github.com/nats-io/jwt/v2 v2.8.1 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sync v0.23.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/text v0.42.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
modules/mesh-mail/go.sum
+31 -6
View File
@@ -1,5 +1,7 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44 h1:r5xIo5qWOXQdCSMxvqJy8kV4U+P/QWXFK6971Ha6wU8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019062050-34392e74916c h1:/f6f2ugss172hDJSL3EFZqb9PdfO5DKzYVfPiX2LdAM=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019062050-34392e74916c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -9,6 +11,8 @@ github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7
github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA=
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 h1:oP4q0fw+fOSWn3DfFi4EXdT+B+gTtzx8GC9xsc26Znk=
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
@@ -17,6 +21,20 @@ github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -27,6 +45,8 @@ github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -36,14 +56,17 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -52,8 +75,10 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
modules/mesh-mail/module.json
+65 -3
View File
@@ -48,7 +48,12 @@
"mail-credential-*": {
"path": "${dir:state}/mail-credential-*.secret",
"issued-by": "outside",
"prompt": "the password of the mail account {member}"
"prompt": "the password of the mail account {member}",
"owner": "mesh-mail-send"
},
"broker-send": {
"path": "${dir:state}/broker-send.secret",
"owner": "mesh-mail-send"
}
},
"config": {
@@ -83,6 +88,10 @@
"state": [
"sends"
],
"emits": [
"mail.send.sent",
"mail.send.failed"
],
"streams": [
{
"name": "mail",
@@ -96,13 +105,29 @@
"config": "grants",
"events": "mail.{key}.**"
}
},
{
"name": "mail-sends",
"events": [
"mail.send.approved"
],
"max-age-days": 1,
"max-mib": 64,
"retention": "work-queue",
"max-deliveries": 5,
"duplicates": 24
}
],
"resources": [
{
"id": "account",
"type": "user",
"name": "mesh-mail-send"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"mode": "0711",
"place": ".",
"unknown": "program"
},
@@ -125,7 +150,7 @@
"trusted": true,
"type": "file",
"path": "${dir:state}/accounts.json",
"mode": "0600",
"mode": "0644",
"merge": "json",
"template": "files/accounts.json",
"takes-config": [
@@ -214,6 +239,43 @@
"looks": 2,
"grace": "180s"
}
},
{
"id": "send",
"type": "process",
"name": "mesh-mail-send",
"artifact": "tools",
"run": [
"./mesh-mail",
"send"
],
"user": "mesh-mail-send",
"env": {
"MESH_MAIL_ACCOUNTS": "${dir:state}/accounts.json",
"MESH_MAIL_SECRETS": "${dir:state}"
},
"restart-on": [
"accounts"
],
"bus": {
"emits": [
"mail.send.sent",
"mail.send.failed"
],
"queues": [
"mail-sends"
],
"state": [
"sends"
],
"asks": true,
"invokes": [
"seat:operator-channel.tell"
]
},
"health": {
"kind": "unit"
}
}
],
"build": {