diff --git a/modules/brother/README.md b/modules/brother/README.md index 5d924552..47287774 100644 --- a/modules/brother/README.md +++ b/modules/brother/README.md @@ -122,8 +122,8 @@ the login page again. The logout follows a logout link or form on the page; a pa ## The event -`scan.added`, said by this module, on the wire `mesh.mod.brother.event.scan.added`; a consumer declares `brother.scan.added`. Its body: -`path`, `node`, `bytes`, `sha256`, `width-mm`, `height-mm`, `resolution` (dpi, as the scanner gave it), +`scan.added`, said by the device process once the PDF is stored, before the file is written (the tool side files it afterwards, from the store, by a link the device signed; the verb's answer adds the file's `path`), on the wire `mesh.mod.brother.event.scan.added`; a consumer declares `brother.scan.added`. Its body: +`object` (the key of the stored PDF), `node`, `bytes`, `sha256`, `width-mm`, `height-mm`, `resolution` (dpi, as the scanner gave it), `color-mode`, `cropped`, `pages` (1 for the glass), `scanned-at` (RFC 3339, UTC). ## Files it does not know diff --git a/modules/brother/cmd/brother/convert.go b/modules/brother/cmd/brother/convert.go new file mode 100644 index 00000000..0cf0d4e5 --- /dev/null +++ b/modules/brother/cmd/brother/convert.go @@ -0,0 +1,207 @@ +package main + +// Conversion before printing (novox/hq ADR 0468 clause 7, issue 1600). The printer takes PDF and JPEG. A stored +// document of a type the module only-office converts (seat document-converter) is turned into a PDF first: the device +// process signs a GET link on the document and a PUT link on .pdf with the binding's key, calls +// document-converter.convert, checks that the PDF the converter says it put is the PDF the bucket holds, and prints +// that. Any other type is refused, naming the types taken. + +import ( + "archive/zip" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + "unicode/utf8" + + "git.novox.be/novox/mesh-sdk/go/objectlink" +) + +// ConvertFollow is how long the device process follows one conversion, in calls of under 25 s each. +const ConvertFollow = 3 * time.Minute + +// ConvertTypes are the types document-converter takes (clause 7). +var ConvertTypes = []string{"docx", "doc", "xlsx", "xls", "pptx", "ppt", "odt", "ods", "odp", "rtf", "txt", "html"} + +// contentTypes are the media types of a fetched document, by the type the converter is asked for. +var contentTypes = map[string]string{ + "text/html": "html", "application/xhtml+xml": "html", "text/plain": "txt", "application/rtf": "rtf", "text/rtf": "rtf", + "application/msword": "doc", "application/vnd.ms-excel": "xls", "application/vnd.ms-powerpoint": "ppt", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx", + "application/vnd.openxmlformats-officedocument.presentationml.presentation": "pptx", + "application/vnd.oasis.opendocument.text": "odt", + "application/vnd.oasis.opendocument.spreadsheet": "ods", + "application/vnd.oasis.opendocument.presentation": "odp", +} + +// the first bytes of an OLE compound file (the legacy Word, Excel and PowerPoint formats), and the stream names that +// tell them apart, as UTF-16 (MS-CFB: directory entry names are UTF-16LE). +var oleMagic = []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1} + +func utf16le(s string) []byte { + var b []byte + for _, r := range s { + b = append(b, byte(r), byte(r>>8)) + } + return b +} + +// ConvertibleType is the type a document is, among the converter's twelve, told by its bytes and, for a document that +// came from a web server, by its Content-Type; "" when it is none of them. A document that is a PDF or a JPEG is +// not asked about: the printer takes it as it is. +func ConvertibleType(doc []byte, contentType string) string { + if i := strings.IndexByte(contentType, ';'); i >= 0 { + contentType = contentType[:i] + } + byHeader := contentTypes[strings.ToLower(strings.TrimSpace(contentType))] + switch { + case bytes.HasPrefix(doc, []byte("PK\x03\x04")): + if t := zipType(doc); t != "" { + return t + } + return "" + case bytes.HasPrefix(doc, oleMagic): + for name, t := range map[string]string{"WordDocument": "doc", "Workbook": "xls", "PowerPoint Document": "ppt"} { + if bytes.Contains(doc, utf16le(name)) { + return t + } + } + return "" + case bytes.HasPrefix(doc, []byte(`{\rtf`)): + return "rtf" + } + if byHeader == "html" || looksLikeHTML(doc) { + if utf8.Valid(doc) && !bytes.Contains(doc, []byte{0}) { + return "html" + } + return "" + } + if utf8.Valid(doc) && !bytes.Contains(doc, []byte{0}) && len(doc) > 0 { + return "txt" + } + return "" +} + +func looksLikeHTML(doc []byte) bool { + head := doc + if len(head) > 512 { + head = head[:512] + } + s := strings.ToLower(strings.TrimLeft(strings.TrimPrefix(string(head), "\xef\xbb\xbf"), " \t\r\n")) + return strings.HasPrefix(s, "`, + "_rels/.rels": ``, + "word/document.xml": ``, + } { + w, err := zw.Create(name) + if err != nil { + t.Fatal(err) + } + w.Write([]byte(body)) + } + zw.Close() + return buf.Bytes() +} + +// converterFake stands for document-converter.convert: it records the call and, as the converter process would, puts a +// PDF in the bucket under the key the target link names and answers its size and SHA-256. +type converterFake struct { + r *printerRig + calls []map[string]any + pdf []byte + answer func(body map[string]any, stored []byte) (json.RawMessage, error) +} + +func withConverter(r *printerRig, pdf []byte) *converterFake { + c := &converterFake{r: r, pdf: pdf} + r.worker.Convert = func(body map[string]any) (json.RawMessage, error) { + c.calls = append(c.calls, body) + if body["target"] != nil { // a start puts the PDF; a read of an ask (convert {ask}) puts nothing + target, _ := url.Parse(body["target"].(string)) + r.objects.Put(context.Background(), strings.TrimPrefix(target.Path, "/"), "application/pdf", c.pdf) + } + if c.answer != nil { + return c.answer(body, c.pdf) + } + return json.Marshal(map[string]any{"size": len(c.pdf), "sha256": sha256Hex(c.pdf)}) + } + return c +} + +// 1600/AC1: a stored .docx is converted: document-converter.convert is called with a GET link on the source and a PUT +// link on .pdf, both signed in the bucket and expiring within ten minutes, and the PDF is printed. +func TestAStoredDocxIsConvertedThenPrinted(t *testing.T) { + fake := newIPPFake(t, 51, 5) + r := newRig(t, `{"printer":"192.0.2.10"}`, "pw") + doc := docx(t) + key := sha256Hex(doc) + r.objects.Put(t.Context(), key, "application/octet-stream", doc) + pdf := pdfOf(2500) + conv := withConverter(r, pdf) + got, err := r.call(t, "printer.print", map[string]any{"object": key}) + if err != nil { + t.Fatal(err) + } + if len(conv.calls) != 1 { + t.Fatalf("%d converter calls", len(conv.calls)) + } + call := conv.calls[0] + src, _ := url.Parse(call["source"].(string)) + dst, _ := url.Parse(call["target"].(string)) + if src.Path != "/"+key || dst.Path != "/"+key+".pdf" || call["type"] != "docx" || call["sha256"] != key { + t.Errorf("call = %v", call) + } + for _, u := range []*url.URL{src, dst} { + if secs, _ := strconv.Atoi(u.Query().Get("X-Amz-Expires")); secs < 1 || time.Duration(secs)*time.Second > 10*time.Minute { + t.Errorf("%s lives %d s", u, secs) + } + } + if strings.Join(r.objects.links, ",") != "GET "+key+" 600,PUT "+key+".pdf 600" { + t.Errorf("links signed: %v", r.objects.links) + } + m := got.(map[string]any) + if m["job-id"] != float64(51) || m["format"] != "application/pdf" || m["object"] != key+".pdf" || m["converted-from"] != key { + t.Errorf("answer = %v", m) + } + if _, sent := fake.request(t); string(sent) != string(pdf) { + t.Error("the printer was not sent the converted PDF") + } +} + +// 1600/AC2: a converter answer whose SHA-256 differs from the stored PDF's is refused and nothing is printed. +func TestAConverterAnswerOfAnotherHashPrintsNothing(t *testing.T) { + fake := newIPPFake(t, 1, 3) + r := newRig(t, `{"printer":"192.0.2.10"}`, "pw") + doc := docx(t) + r.objects.Put(t.Context(), sha256Hex(doc), "application/octet-stream", doc) + conv := withConverter(r, pdfOf(900)) + conv.answer = func(map[string]any, []byte) (json.RawMessage, error) { + return json.Marshal(map[string]any{"size": 900, "sha256": sha256Hex([]byte("another pdf"))}) + } + _, err := r.call(t, "printer.print", map[string]any{"object": sha256Hex(doc)}) + if err == nil || !strings.Contains(err.Error(), "nothing is printed") { + t.Fatalf("error = %v", err) + } + if fake.body != nil { + t.Error("something was printed") + } +} + +// 1600/AC3: a converter error is answered with its code and meaning. +func TestAConverterErrorIsAnsweredWithItsCodeAndMeaning(t *testing.T) { + fake := newIPPFake(t, 1, 3) + r := newRig(t, `{"printer":"192.0.2.10"}`, "pw") + doc := docx(t) + r.objects.Put(t.Context(), sha256Hex(doc), "application/octet-stream", doc) + conv := withConverter(r, pdfOf(10)) + conv.answer = func(map[string]any, []byte) (json.RawMessage, error) { + return nil, errors.New("the document server refused the conversion, error -4: error while downloading the document file to be converted") + } + _, err := r.call(t, "printer.print", map[string]any{"object": sha256Hex(doc)}) + if err == nil || !strings.Contains(err.Error(), "-4: error while downloading the document file to be converted") { + t.Fatalf("error = %v", err) + } + // An error the converter wrote in its answer is the same. + conv.answer = func(map[string]any, []byte) (json.RawMessage, error) { + return json.RawMessage(`{"error": "-3: conversion error"}`), nil + } + if _, err := r.call(t, "printer.print", map[string]any{"object": sha256Hex(doc)}); err == nil || !strings.Contains(err.Error(), "-3: conversion error") { + t.Fatalf("error = %v", err) + } + if fake.body != nil { + t.Error("something was printed") + } +} + +// 1600/AC4: a type that is neither PDF, JPEG nor one of the converter's twelve is refused naming the types taken, and +// convert is not called. +func TestAnotherTypeIsRefusedNamingTheTypesTaken(t *testing.T) { + fake := newIPPFake(t, 1, 3) + r := newRig(t, `{"printer":"192.0.2.10"}`, "pw") + exe := []byte("MZ\x90\x00\x03\x00\x00\x00\x04\x00\x00\x00\xff\xff\x00\x00") + r.objects.Put(t.Context(), sha256Hex(exe), "application/octet-stream", exe) + conv := withConverter(r, pdfOf(10)) + _, err := r.call(t, "printer.print", map[string]any{"object": sha256Hex(exe)}) + if err == nil || !strings.Contains(err.Error(), "docx, doc, xlsx, xls, pptx, ppt, odt, ods, odp, rtf, txt, html") { + t.Fatalf("error = %v", err) + } + if len(conv.calls) != 0 || fake.body != nil { + t.Errorf("%d converter calls, printed %v", len(conv.calls), fake.body != nil) + } +} + +// 1600: each of the twelve types is told by its bytes (and a web page also by its Content-Type). +func TestTheTypesAreToldByTheirBytes(t *testing.T) { + ole := func(stream string) []byte { + return append(append([]byte{}, oleMagic...), append(make([]byte, 64), utf16le(stream)...)...) + } + odf := func(mime string) []byte { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, _ := zw.Create("mimetype") + w.Write([]byte(mime)) + zw.Close() + return buf.Bytes() + } + ooxml := func(part string) []byte { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, _ := zw.Create(part) + w.Write([]byte("")) + zw.Close() + return buf.Bytes() + } + for want, c := range map[string]struct { + doc []byte + ct string + }{ + "docx": {docx(t), ""}, "xlsx": {ooxml("xl/workbook.xml"), ""}, "pptx": {ooxml("ppt/presentation.xml"), ""}, + "doc": {ole("WordDocument"), ""}, "xls": {ole("Workbook"), ""}, "ppt": {ole("PowerPoint Document"), ""}, + "odt": {odf("application/vnd.oasis.opendocument.text"), ""}, "ods": {odf("application/vnd.oasis.opendocument.spreadsheet"), ""}, + "odp": {odf("application/vnd.oasis.opendocument.presentation"), ""}, + "rtf": {[]byte(`{\rtf1\ansi hello}`), ""}, "txt": {[]byte("just some words\n"), ""}, + "html": {[]byte("hi"), ""}, "html ": {[]byte("

a fragment

"), "text/html; charset=utf-8"}, + } { + if got := ConvertibleType(c.doc, c.ct); got != strings.TrimSpace(want) { + t.Errorf("%s: told as %q", want, got) + } + } + for name, doc := range map[string][]byte{"empty": {}, "binary": {0, 1, 2, 3}, "a zip of nothing known": ooxml("other.xml"), "ole of nothing known": ole("Other")} { + if got := ConvertibleType(doc, ""); got != "" { + t.Errorf("%s: told as %q", name, got) + } + } +} + +// 1600/AC5: the device process invokes seat:document-converter.convert. +func TestTheDeviceProcessInvokesTheConverter(t *testing.T) { + for _, r := range readManifest(t).Resources { + if r["id"] == "device" { + inv, _ := r["bus"].(map[string]any)["invokes"].([]any) + if len(inv) != 1 || inv[0] != "seat:document-converter.convert" { + t.Errorf("invokes = %v", inv) + } + return + } + } + t.Fatal("no device process") +} + +// 1600 (review round 1): a conversion answered as converting is followed with convert {ask}, each call its own, until +// it is done; one still converting after the device process's bound is not printed, and the process.CallTimeout of the +// SDK is not touched. +func TestAConversionAnsweredAsConvertingIsFollowedWithItsAsk(t *testing.T) { + fake := newIPPFake(t, 61, 5) + r := newRig(t, `{"printer":"192.0.2.10"}`, "pw") + doc := docx(t) + key := sha256Hex(doc) + r.objects.Put(t.Context(), key, "application/octet-stream", doc) + pdf := pdfOf(700) + conv := withConverter(r, pdf) + rounds := 0 + conv.answer = func(body map[string]any, stored []byte) (json.RawMessage, error) { + if body["ask"] != nil { + rounds++ + if rounds < 3 { + return json.RawMessage(`{"state": "converting", "ask": "` + body["ask"].(string) + `"}`), nil + } + return json.Marshal(map[string]any{"size": len(stored), "sha256": sha256Hex(stored)}) + } + return json.RawMessage(`{"state": "converting", "ask": "a1b2c3a1b2c3a1b2c3a1b2c3"}`), nil + } + if _, err := r.call(t, "printer.print", map[string]any{"object": key}); err != nil { + t.Fatal(err) + } + if len(conv.calls) != 4 || conv.calls[1]["ask"] != "a1b2c3a1b2c3a1b2c3a1b2c3" || conv.calls[1]["source"] != nil { + t.Errorf("calls = %v", conv.calls) + } + if _, sent := fake.request(t); string(sent) != string(pdf) { + t.Error("the converted PDF was not printed") + } + + // Always converting: given up at the bound, nothing printed. + fake.body = nil + r.worker.ConvertFollow = 30 * time.Millisecond + conv.answer = func(body map[string]any, _ []byte) (json.RawMessage, error) { + time.Sleep(5 * time.Millisecond) + return json.RawMessage(`{"state": "converting", "ask": "a1b2c3a1b2c3a1b2c3a1b2c3"}`), nil + } + if _, err := r.call(t, "printer.print", map[string]any{"object": key}); err == nil || !strings.Contains(err.Error(), "still converting") || fake.body != nil { + t.Errorf("error = %v, printed %v", err, fake.body != nil) + } + src, _ := os.ReadFile("deviceproc.go") + if strings.Contains(string(src), "CallTimeout") { + t.Error("deviceproc.go sets the SDK's call timeout") + } +} diff --git a/modules/brother/cmd/brother/deviceproc.go b/modules/brother/cmd/brother/deviceproc.go index 134846af..6411cfae 100644 --- a/modules/brother/cmd/brother/deviceproc.go +++ b/modules/brother/cmd/brother/deviceproc.go @@ -11,14 +11,17 @@ import ( "encoding/json" "errors" "fmt" + "net/http" "os" "os/signal" + "slices" "strings" "sync" "syscall" "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/objectlink" "git.novox.be/novox/mesh-sdk/go/process" "github.com/nats-io/nats.go/jetstream" @@ -66,6 +69,12 @@ type deviceWorker struct { Node string // Capture scans; nil is the device's own (a test puts a stand-in). Capture func(ctx context.Context, req ScanRequest) ([]Capture, error) + // Convert calls the seat document-converter's convert verb as this process's own account (clause 7); nil: none. + Convert func(body map[string]any) (json.RawMessage, error) + // ConvertFollow is how long a conversion is followed; zero is ConvertFollow (a test makes it shorter). + ConvertFollow time.Duration + // Fetcher is the guarded fetch of a web link (fetch.go); it reads its facts when it is first needed. + Fetcher func() (*Fetcher, error) // answerPart is the raw size of a document part in an answer; zero is AnswerPartSize (a test makes it larger). answerPart int Now func() time.Time @@ -247,10 +256,14 @@ func (w *deviceWorker) keep(ctx context.Context, objects Objects, set Settings, // keepOne keeps one captured PDF and announces it. func (w *deviceWorker) keepOne(ctx context.Context, objects Objects, set Settings, c Capture) (map[string]any, error) { - // A scan the tool could not take is not kept nor announced: an announced scan is always filed (issue 1539). - if most := MostAnswerParts * AnswerPartSize; len(c.PDF) > most { - return nil, fmt.Errorf("the scan's PDF is %d bytes, over the %d the tool can fetch (%d parts of %d): it is not kept; scan with a lower resolution or fewer sheets", - len(c.PDF), most, MostAnswerParts, AnswerPartSize) + // A scan over a document's limit is not kept nor announced: an announced scan is always filed (issue 1539). + if len(c.PDF) > int(objectlink.MaxDocument) { + return nil, fmt.Errorf("the scan's PDF is %d bytes, over the %d a document may be (novox/hq ADR 0468 §3): it is not kept; scan with a lower resolution or fewer sheets", + len(c.PDF), objectlink.MaxDocument) + } + signer, ok := objects.(Presigner) + if !ok { + return nil, errors.New("the bucket cannot sign a link: the scan is not kept") } body := c.Body(w.Node, set.ColorMode) key := body["object"].(string) @@ -260,7 +273,13 @@ func (w *deviceWorker) keepOne(ctx context.Context, objects Objects, set Setting if err := w.Emit(body, c.SHA256); err != nil { return nil, fmt.Errorf("the scan is kept as %s, but %s could not be said on the bus: %w", key, EventAdded, err) } - answer := map[string]any{"key": key, "size": len(c.PDF), "parts": answerPartCount(len(c.PDF))} + // The answer carries a link to read the stored PDF, signed with the binding's key (novox/hq ADR 0468 §4): the tool + // side reads the scan by it and files it; no byte of the document is in the answer. + link, err := signer.Link(http.MethodGet, key, objectlink.MaxExpires) + if err != nil { + return nil, fmt.Errorf("the scan is kept as %s, but a link to it could not be signed: %w", key, err) + } + answer := map[string]any{"key": key, "size": len(c.PDF), "link": link, "link-seconds": int(objectlink.MaxExpires / time.Second)} for k, v := range body { answer[k] = v } @@ -370,7 +389,20 @@ func (w *deviceWorker) print(ctx context.Context, ask Ask) (map[string]any, erro return nil, err } key := ask.Object - if key == "" { + var doc []byte + contentType := "" + switch { + case ask.URL != "": + fetched, err := w.fetch(ctx, ask.URL) + if err != nil { + return nil, err + } + // The fetched document is kept once under its SHA-256, as a scan is, and printed as that object. + doc, key, contentType = fetched.Body, sha256Hex(fetched.Body), fetched.ContentType + if err := Store(ctx, objects, key, fetched.ContentType, doc); err != nil { + return nil, err + } + case key == "": for _, ext := range []string{"pdf", "jpg"} { k := "uploads/" + ask.Upload + "." + ext if has, err := objects.Has(ctx, k); err != nil { @@ -383,21 +415,55 @@ func (w *deviceWorker) print(ctx context.Context, ask Ask) (map[string]any, erro if key == "" { return nil, fmt.Errorf("upload %s is not complete: no document is kept under it", ask.Upload) } - } - doc, err := objects.Get(ctx, key) - if err != nil { - return nil, err + fallthrough + default: + var err error + if doc, err = objects.Get(ctx, key); err != nil { + return nil, err + } } format, _, err := SniffFormat(doc) + var convertedFrom string if err != nil { - return nil, err + // Not a PDF or a JPEG: a type the converter takes becomes a PDF first (clause 7), any other is refused. + pdf, pdfKey, cerr := w.toPDF(ctx, objects, key, doc, contentType) + if cerr != nil { + return nil, cerr + } + doc, convertedFrom, key = pdf, key, pdfKey + format = "application/pdf" + } + // The printer is asked what it takes before the job, and a format it does not list is refused with the list + // (novox/hq ADR 0468 clause 5): a printer that cannot be asked prints nothing. + listed, err := FormatsOf(ctx, set.Printer) + if err != nil { + return nil, fmt.Errorf("the printer's document formats could not be read, so nothing is sent: %w", err) + } + if !slices.Contains(listed, format) { + return nil, fmt.Errorf("the printer does not take %s: its document-format-supported lists %s", format, strings.Join(listed, ", ")) } res, err := PrintJob(ctx, set.Printer, PrintOptions{Format: format, Copies: ask.Copies, Sides: ask.Sides, Media: ask.Media, Name: "brother print"}, doc) if err != nil { return nil, err } - return map[string]any{"job-id": res.JobID, "job-state": res.State, "job-state-code": res.StateCode, - "job-state-reasons": res.Reasons, "format": format, "copies": ask.Copies, "sides": ask.Sides, "media": ask.Media, "object": key}, nil + out := map[string]any{"job-id": res.JobID, "job-state": res.State, "job-state-code": res.StateCode, + "job-state-reasons": res.Reasons, "format": format, "copies": ask.Copies, "sides": ask.Sides, "media": ask.Media, "object": key} + if convertedFrom != "" { + out["converted-from"] = convertedFrom + } + return out, nil +} + +// fetch gets a web link under the rules of fetch.go. +func (w *deviceWorker) fetch(ctx context.Context, link string) (*Fetched, error) { + if w.Fetcher == nil { + return nil, errors.New("this process has no way to fetch a web link") + } + f, err := w.Fetcher() + if err != nil { + return nil, err + } + return f.Fetch(ctx, link) } // read logs in, reads one page of the admin site and logs out. It is read-only: the login (and the logout) are @@ -497,7 +563,7 @@ func nonNilForms(f []Form) []Form { // It refuses to start naming what is missing when the account cannot be read, and never logs a password. func device() int { settings, passwordFile, brokerFile := os.Getenv("BROTHER_SETTINGS"), os.Getenv("BROTHER_ADMIN_PASSWORD"), os.Getenv("BROTHER_BROKER") - store := os.Getenv("BROTHER_STORE") + store, fetchFile := os.Getenv("BROTHER_STORE"), os.Getenv("BROTHER_FETCH") if brokerFile == "" { fmt.Fprintln(os.Stderr, "[brother] device was started without BROTHER_BROKER, its bus account's file") return 2 @@ -519,6 +585,13 @@ func device() int { } return NewS3(facts), nil }, + Fetcher: func() (*Fetcher, error) { + r, err := ReadFetchFacts(fetchFile) + if err != nil { + return nil, err + } + return NewFetcher(r), nil + }, Now: time.Now, Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, f+"\n", a...) }, } @@ -541,6 +614,9 @@ func runDevice(ctx context.Context, account process.Account, w *deviceWorker) er } defer conn.Close() w.Answers = conn.State(AnswersState) + w.Convert = func(body map[string]any) (json.RawMessage, error) { + return conn.Call("seat:document-converter.convert", body) + } if w.Node == "" { w.Node = account.Node } diff --git a/modules/brother/cmd/brother/documents_test.go b/modules/brother/cmd/brother/documents_test.go index a876c540..e645a02d 100644 --- a/modules/brother/cmd/brother/documents_test.go +++ b/modules/brother/cmd/brother/documents_test.go @@ -115,7 +115,7 @@ func TestScannerDocumentGivesEachPartOfAKeptScan(t *testing.T) { } } -// 1453/AC10: a scan of more than one part is fetched whole by scanner.scan, joined, checked and filed. +// 1453/AC10, 1601: a scan the size of several answer parts is read whole from the store by scanner.scan, checked and filed. func TestScanNowFetchesAScanOfSeveralParts(t *testing.T) { f := newFakeScanner(t, fixture(t, "scan.jpg")) r, _ := scanRig(t, f, Settings{Crop: false}) @@ -266,15 +266,41 @@ func TestAnIncompleteUploadIsDroppedAfterAWhile(t *testing.T) { // ippFake is a printer's IPP port: it records the request and answers a Print-Job answer with a job id and state. type ippFake struct { url string - body []byte + body []byte // the Print-Job request status int + // formats is document-format-supported, as the printer lists it; asked counts the Get-Printer-Attributes requests. + formats []string + asked int + // delay is how long a Print-Job takes to be answered. + delay time.Duration } func newIPPFake(t *testing.T, jobID, jobState int) *ippFake { t.Helper() - f := &ippFake{} + f := &ippFake{formats: []string{"application/octet-stream", "application/pdf", "image/jpeg"}} srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - f.body, _ = io.ReadAll(r.Body) + body, _ := io.ReadAll(r.Body) + if binary.BigEndian.Uint16(body[2:4]) == uint16(ippGetPrinterAttributes) { + // Get-Printer-Attributes (RFC 8011 §4.2.5): the formats are one attribute of several values, a value + // after the first carrying an empty name (§4.1.4), each of the mimeMediaType syntax. + f.asked++ + ans := []byte{0x02, 0x00, 0x00, 0x00, 0, 0, 0, 1, ippOperation} + ans = appendAttr(ans, ippCharset, "attributes-charset", "utf-8") + ans = append(ans, 0x04) // printer-attributes-tag, RFC 8010 §3.5.1 + for i, format := range f.formats { + name := "document-format-supported" + if i > 0 { + name = "" + } + ans = appendAttr(ans, ippMimeType, name, format) + } + ans = append(ans, ippEnd) + w.Header().Set("Content-Type", "application/ipp") + w.Write(ans) + return + } + f.body = body + time.Sleep(f.delay) ans := []byte{0x02, 0x00, byte(f.status >> 8), byte(f.status), 0, 0, 0, 1, ippOperation} ans = appendAttr(ans, ippCharset, "attributes-charset", "utf-8") ans = append(ans, ippJob) @@ -337,15 +363,14 @@ func TestPrinterPrintSendsAPrintJobAndAnswersTheJob(t *testing.T) { } } -// 1453/AC12: the defaults are one copy, one-sided, A4; a JPEG is sent as image/jpeg; an upload is printed by its id. -func TestPrinterPrintDefaultsAndFormatsAndUploads(t *testing.T) { +// 1453/AC12: the defaults are one copy, one-sided, A4; a JPEG is sent as image/jpeg; a document stored under its +// SHA-256 (as node-files.upload keeps one) is printed by its key. +func TestPrinterPrintDefaultsAndFormatsAndStoredDocuments(t *testing.T) { fake := newIPPFake(t, 8, 3) r := newRig(t, `{"printer":"192.0.2.10"}`, "pw") jpg := append([]byte{0xFF, 0xD8, 0xFF, 0xE0}, bytes.Repeat([]byte{9}, 200)...) - if _, err := upload(t, r, "photo", jpg, sha256Hex(jpg), 1, 1, jpg); err != nil { - t.Fatal(err) - } - got, err := r.call(t, "printer.print", map[string]any{"upload": "photo"}) + r.objects.Put(t.Context(), sha256Hex(jpg), "image/jpeg", jpg) + got, err := r.call(t, "printer.print", map[string]any{"object": sha256Hex(jpg)}) if err != nil { t.Fatal(err) } @@ -357,8 +382,8 @@ func TestPrinterPrintDefaultsAndFormatsAndUploads(t *testing.T) { strings.Join(attrs.texts("sides"), "") != "one-sided" || strings.Join(attrs.texts("media"), "") != DefaultMedia || attrs.Attributes["copies"][0].Integer != 1 { t.Errorf("attributes = %v, %d bytes sent", attrs.Attributes, len(sent)) } - if _, err := r.call(t, "printer.print", map[string]any{"upload": "never-uploaded"}); err == nil || !strings.Contains(err.Error(), "not complete") { - t.Errorf("an upload never made: error = %v", err) + if _, err := r.call(t, "printer.print", map[string]any{"object": sha256Hex([]byte("never stored"))}); err == nil { + t.Errorf("an object never stored was printed") } } @@ -392,9 +417,9 @@ func TestPrinterPrintRefusesWhatIsNotWellFormed(t *testing.T) { if _, err := r.call(t, "printer.print", map[string]any{"object": dk}); err == nil || !strings.Contains(err.Error(), "0x0400") { t.Errorf("a refused job: error = %v", err) } - // A kept object that is not a document the printer takes is not sent. + // A kept object that is no document the printer takes, nor one the converter takes (1600), is not sent. fake.status, fake.body = 0, nil - r.objects.Put(t.Context(), key, "text/plain", []byte("hello")) + r.objects.Put(t.Context(), key, "application/octet-stream", []byte{0x00, 0x01, 0xFE, 0xFF, 0x02}) if _, err := r.call(t, "printer.print", map[string]any{"object": key}); err == nil || !strings.Contains(err.Error(), "neither a PDF nor a JPEG") || fake.body != nil { t.Errorf("a text object: error = %v, sent %d bytes", err, len(fake.body)) } @@ -547,83 +572,3 @@ func home(t *testing.T) string { t.Setenv("HOME", h) return h } - -// 1551: printer.print with a path reads the file, uploads it in parts and prints it in one call; a document of -// several parts arrives whole, and the answer has the job's id and state. -func TestPrinterPrintTakesAPathAndUploadsAndPrintsInOneCall(t *testing.T) { - fake := newIPPFake(t, 31, 5) - r := newRig(t, `{"printer":"192.0.2.10"}`, "pw") - doc := pdfOf(2*PartSize + 1234) - file := filepath.Join(home(t), "letter.bin") // the name says nothing: the bytes are read - if err := os.WriteFile(file, doc, 0o600); err != nil { - t.Fatal(err) - } - got, err := r.call(t, "printer.print", map[string]any{"path": file, "copies": float64(2)}) - if err != nil { - t.Fatal(err) - } - m := got.(map[string]any) - if m["job-id"] != float64(31) || m["job-state"] != "processing" || m["format"] != "application/pdf" || m["bytes"] != len(doc) { - t.Errorf("answer = %v", m) - } - attrs, sent := fake.request(t) - if !bytes.Equal(sent, doc) || attrs.Attributes["copies"][0].Integer != 2 { - t.Errorf("sent %d bytes (want %d), attributes %v", len(sent), len(doc), attrs.Attributes) - } - ops := r.bus.ops() - if len(ops) != 4 || ops[0] != OpUpload || ops[2] != OpUpload || ops[3] != OpPrint { - t.Errorf("asks = %v, want three upload parts and a print", ops) - } -} - -// 1551: only a regular file under the tool account's home folder is read, after symbolic links are resolved; every -// other path is refused in one fixed way that says nothing of whether it exists or how large it is; a file in the -// home folder that is no PDF or JPEG by its bytes is refused by name of the type. -func TestPrinterPrintReadsOnlyAFileUnderTheHomeFolder(t *testing.T) { - fake := newIPPFake(t, 1, 5) - r := newRig(t, `{"printer":"192.0.2.10"}`, "pw") - h := home(t) - outside := t.TempDir() - secret := filepath.Join(outside, "secret.pdf") - os.WriteFile(secret, pdfOf(3000), 0o600) - link := filepath.Join(h, "link.pdf") - os.Symlink(secret, link) - dirLink := filepath.Join(h, "dirlink") - os.Symlink(outside, dirLink) - os.Mkdir(filepath.Join(h, "folder"), 0o700) - for name, path := range map[string]string{ - "outside home": secret, - "link out of home": link, - "folder link out": filepath.Join(dirLink, "secret.pdf"), - "dot-dot out of home": filepath.Join(h, "..", filepath.Base(outside), "secret.pdf"), - "missing in home": filepath.Join(h, "none.pdf"), - "missing outside home": filepath.Join(outside, "none.pdf"), - "folder": filepath.Join(h, "folder"), - "the home folder itself": h, - "relative": "secret.pdf", - "proc": "/proc/self/environ", - } { - _, err := r.call(t, "printer.print", map[string]any{"path": path}) - if err == nil || err.Error() != PathRefused { - t.Errorf("%s: error = %v, want exactly %q", name, err, PathRefused) - } - } - text := filepath.Join(h, "fake.pdf") - os.WriteFile(text, []byte("just text, named .pdf"), 0o600) - if _, err := r.call(t, "printer.print", map[string]any{"path": text}); err == nil || !strings.Contains(err.Error(), "neither a PDF nor a JPEG") { - t.Errorf("text in home: error = %v", err) - } - if _, err := r.call(t, "printer.print", map[string]any{"path": text, "object": sha256Hex([]byte("x")) + ".pdf"}); err == nil { - t.Error("path with object accepted") - } - if r.bus.asks() != 0 || fake.body != nil { - t.Errorf("%d asks queued, printer sent %d bytes", r.bus.asks(), len(fake.body)) - } - // A symbolic link inside the home folder to a file inside it is fine. - inside := filepath.Join(h, "real.pdf") - os.WriteFile(inside, pdfOf(3000), 0o600) - os.Symlink(inside, filepath.Join(h, "ok.pdf")) - if _, err := r.call(t, "printer.print", map[string]any{"path": filepath.Join(h, "ok.pdf")}); err != nil { - t.Errorf("a link within home: %v", err) - } -} diff --git a/modules/brother/cmd/brother/fetch.go b/modules/brother/cmd/brother/fetch.go new file mode 100644 index 00000000..17b6c108 --- /dev/null +++ b/modules/brother/cmd/brother/fetch.go @@ -0,0 +1,227 @@ +package main + +// A web link the device process fetches (novox/hq ADR 0468 clause 6, issue 1599). The process holds the bucket's key and +// the printer's password and sits inside the mesh, so a URL from any caller would otherwise reach the store, the +// printer's admin pages or the controller. Only http and https are taken. The fetch takes at most 60 s, 100 MiB and 5 +// redirects. Each host is resolved once, every address it gives is checked, and the connection is made to an address +// that was checked, never to a second lookup's. Loopback, private (RFC 1918, RFC 4193), link-local, shared (RFC 6598), +// unspecified, multicast and the mesh's own tunnel range are refused, after every redirect too. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/netip" + "net/url" + "os" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/objectlink" +) + +const ( + // FetchTime, FetchMost and FetchRedirects are the fetch's bounds (ADR 0468 clause 6). + FetchTime = 60 * time.Second + FetchMost = objectlink.MaxDocument + FetchRedirects = 5 + // MostURL is the longest address printer.print takes. + MostURL = 2048 +) + +// FetchFacts is the file the mesh writes for the fetch (files/fetch.json): the tunnel's range, which a module cannot +// know (mesh-sdk go/catalogue/machine_into_files.go, the fact `mesh-range`). +type FetchFacts struct { + MeshRange string `json:"mesh-range"` +} + +// ReadFetchFacts reads the fetch's facts; a file that names no range is refused, since a fetch that does not know the +// mesh's range could reach it. +func ReadFetchFacts(path string) (netip.Prefix, error) { + if path == "" { + return netip.Prefix{}, errors.New("the module was started without the fetch's facts (BROTHER_FETCH)") + } + raw, err := os.ReadFile(path) + if err != nil { + return netip.Prefix{}, fmt.Errorf("the fetch's facts cannot be read: %w", err) + } + var f FetchFacts + if err := json.Unmarshal(raw, &f); err != nil { + return netip.Prefix{}, fmt.Errorf("the fetch's facts are not the module's: %v", err) + } + p, err := netip.ParsePrefix(strings.TrimSpace(f.MeshRange)) + if err != nil { + return netip.Prefix{}, fmt.Errorf("the fetch's facts name no mesh range (%q): %v", f.MeshRange, err) + } + return p, nil +} + +// CheckURLScheme is what the tool side can say of an address without looking at the network: it is http or https. +func CheckURLScheme(raw string) error { + if len(raw) > MostURL { + return fmt.Errorf("url is at most %d characters", MostURL) + } + u, err := url.Parse(raw) + if err != nil || u.Host == "" { + return fmt.Errorf("url %q is not an address", raw) + } + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("url %q has the scheme %q: only http and https are fetched (novox/hq ADR 0468 clause 6)", raw, u.Scheme) + } + return nil +} + +var refusedRanges = []struct { + prefix netip.Prefix + class string +}{ + {netip.MustParsePrefix("0.0.0.0/8"), "an unspecified address"}, + {netip.MustParsePrefix("10.0.0.0/8"), "a private address (RFC 1918)"}, + {netip.MustParsePrefix("100.64.0.0/10"), "a shared address (RFC 6598)"}, + {netip.MustParsePrefix("127.0.0.0/8"), "a loopback address"}, + {netip.MustParsePrefix("169.254.0.0/16"), "a link-local address"}, + {netip.MustParsePrefix("172.16.0.0/12"), "a private address (RFC 1918)"}, + {netip.MustParsePrefix("192.168.0.0/16"), "a private address (RFC 1918)"}, + {netip.MustParsePrefix("224.0.0.0/4"), "a multicast address"}, + {netip.MustParsePrefix("::/128"), "an unspecified address"}, + {netip.MustParsePrefix("::1/128"), "a loopback address"}, + {netip.MustParsePrefix("fc00::/7"), "a private address (RFC 4193)"}, + {netip.MustParsePrefix("fe80::/10"), "a link-local address"}, + {netip.MustParsePrefix("ff00::/8"), "a multicast address"}, +} + +// Refusal says why an address may not be fetched from, or "" when it may. +func (f *Fetcher) Refusal(a netip.Addr) string { + a = a.Unmap() + for _, r := range refusedRanges { + if r.prefix.Contains(a) { + return r.class + } + } + if f.MeshRange.IsValid() && f.MeshRange.Contains(a) { + return "an address of the mesh's own network" + } + return "" +} + +// Fetcher fetches a web link under the rules above. Resolve and Dial are what a test replaces. +type Fetcher struct { + MeshRange netip.Prefix + Resolve func(ctx context.Context, host string) ([]netip.Addr, error) + Dial func(ctx context.Context, network, address string) (net.Conn, error) + Timeout time.Duration + Most int64 + Redirects int +} + +// NewFetcher is a fetcher with the real resolver and dialer and the bounds of the ADR. +func NewFetcher(meshRange netip.Prefix) *Fetcher { + d := &net.Dialer{Timeout: 15 * time.Second} + return &Fetcher{MeshRange: meshRange, Dial: d.DialContext, Timeout: FetchTime, Most: FetchMost, Redirects: FetchRedirects, + Resolve: func(ctx context.Context, host string) ([]netip.Addr, error) { + return net.DefaultResolver.LookupNetIP(ctx, "ip", host) + }} +} + +// Fetched is a document fetched: its bytes and the media type the server gave. +type Fetched struct { + Body []byte + ContentType string +} + +// Fetch gets the document at the address. Every refusal names why and happens before a connection to the refused +// address is made; a body over the limit or a fetch over the time stores nothing, since nothing is returned. +func (f *Fetcher) Fetch(ctx context.Context, raw string) (*Fetched, error) { + ctx, cancel := context.WithTimeout(ctx, f.Timeout) + defer cancel() + current := raw + for hop := 0; ; hop++ { + if err := CheckURLScheme(current); err != nil { + return nil, err + } + u, _ := url.Parse(current) + addr, err := f.checked(ctx, u.Hostname()) + if err != nil { + return nil, fmt.Errorf("%s: %w", current, err) + } + port := u.Port() + if port == "" { + port = map[string]string{"http": "80", "https": "443"}[u.Scheme] + } + target := net.JoinHostPort(addr.String(), port) + tr := &http.Transport{ + Proxy: nil, // never the environment's proxy: the address checked is the address dialled + DisableKeepAlives: true, + DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) { + return f.Dial(ctx, network, target) + }, + } + client := &http.Client{Transport: tr, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + req, err := http.NewRequestWithContext(ctx, http.MethodGet, current, nil) + if err != nil { + tr.CloseIdleConnections() + return nil, err + } + resp, err := client.Do(req) + if err != nil { + tr.CloseIdleConnections() + return nil, fmt.Errorf("%s could not be fetched: %w", current, err) + } + if resp.StatusCode >= 300 && resp.StatusCode < 400 && resp.Header.Get("Location") != "" { + loc, lerr := u.Parse(resp.Header.Get("Location")) + resp.Body.Close() + tr.CloseIdleConnections() + if lerr != nil { + return nil, fmt.Errorf("%s redirects to an address that is not one: %v", current, lerr) + } + if hop >= f.Redirects { + return nil, fmt.Errorf("%s is redirected more than %d times", raw, f.Redirects) + } + current = loc.String() + continue + } + defer tr.CloseIdleConnections() + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%s answered %s", current, resp.Status) + } + if resp.ContentLength > f.Most { + return nil, fmt.Errorf("%s is %d bytes, over the %d a document may be (novox/hq ADR 0468 clause 6)", current, resp.ContentLength, f.Most) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, f.Most+1)) + if err != nil { + return nil, fmt.Errorf("%s could not be read within %s: %w", current, f.Timeout, err) + } + if int64(len(body)) > f.Most { + return nil, fmt.Errorf("%s is over the %d bytes a document may be (novox/hq ADR 0468 clause 6)", current, f.Most) + } + return &Fetched{Body: body, ContentType: resp.Header.Get("Content-Type")}, nil + } +} + +// checked resolves the host once, refuses it when any address it gives is refused, and answers the address to dial. +func (f *Fetcher) checked(ctx context.Context, host string) (netip.Addr, error) { + var addrs []netip.Addr + if ip, err := netip.ParseAddr(host); err == nil { + addrs = []netip.Addr{ip} + } else { + found, err := f.Resolve(ctx, host) + if err != nil { + return netip.Addr{}, fmt.Errorf("the host %s cannot be resolved: %w", host, err) + } + addrs = found + } + if len(addrs) == 0 { + return netip.Addr{}, fmt.Errorf("the host %s resolves to no address", host) + } + for _, a := range addrs { + if class := f.Refusal(a); class != "" { + return netip.Addr{}, fmt.Errorf("refused: %s is %s, and the device process fetches nothing inside the mesh or the machine (novox/hq ADR 0468 clause 6)", a, class) + } + } + return addrs[0].Unmap(), nil +} diff --git a/modules/brother/cmd/brother/fetch_test.go b/modules/brother/cmd/brother/fetch_test.go new file mode 100644 index 00000000..434b5041 --- /dev/null +++ b/modules/brother/cmd/brother/fetch_test.go @@ -0,0 +1,262 @@ +package main + +import ( + "context" + "errors" + "net" + "net/http" + "net/http/httptest" + "net/netip" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +// A fetch rig: names resolve as the test says, and every dial goes to the test's server whatever address was asked, +// after the fetcher has checked the address it was about to dial and told the dial what that was. +type fetchRig struct { + f *Fetcher + mu sync.Mutex + dialed []string + lookups map[string]int + names map[string][]netip.Addr + // after is what a host answers from its second lookup on, when it answers differently (a rebinding). + after map[string][]netip.Addr + srv *httptest.Server +} + +func newFetchRig(t *testing.T, h http.HandlerFunc) *fetchRig { + t.Helper() + r := &fetchRig{lookups: map[string]int{}, names: map[string][]netip.Addr{}, after: map[string][]netip.Addr{}} + r.srv = httptest.NewServer(h) + t.Cleanup(r.srv.Close) + r.f = &Fetcher{MeshRange: netip.MustParsePrefix("10.10.0.0/24"), Timeout: 5 * time.Second, Most: 1 << 20, Redirects: FetchRedirects, + Resolve: func(_ context.Context, host string) ([]netip.Addr, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.lookups[host]++ + a, ok := r.names[host] + if !ok { + return nil, errors.New("no such host") + } + if again, rebinds := r.after[host]; rebinds && r.lookups[host] > 1 { + return again, nil + } + return a, nil + }, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + r.mu.Lock() + r.dialed = append(r.dialed, address) + r.mu.Unlock() + return (&net.Dialer{}).DialContext(ctx, network, r.srv.Listener.Addr().String()) + }} + return r +} + +func addrs(s ...string) []netip.Addr { + var out []netip.Addr + for _, a := range s { + out = append(out, netip.MustParseAddr(a)) + } + return out +} + +// 1599/AC1: a url with a scheme other than http or https is refused by name. +func TestAURLOfAnotherSchemeIsRefused(t *testing.T) { + r := newFetchRig(t, func(http.ResponseWriter, *http.Request) {}) + for _, u := range []string{"file:///etc/passwd", "ftp://example.com/a.pdf", "gopher://example.com/", "javascript:alert(1)", "//example.com/a.pdf"} { + _, err := r.f.Fetch(t.Context(), u) + if err == nil || (!strings.Contains(err.Error(), "only http and https") && !strings.Contains(err.Error(), "is not an address")) { + t.Errorf("%s: error = %v", u, err) + } + if err := CheckURLScheme(u); err == nil { + t.Errorf("%s: the tool side took it", u) + } + } + if len(r.dialed) != 0 { + t.Errorf("dialled %v", r.dialed) + } +} + +// 1599/AC2: a host that is, or resolves to, a loopback, private, link-local, shared, unspecified or multicast address, +// or one in the mesh's range, is refused naming the class, and no connection is made. +func TestAddressesInsideTheMeshAndTheMachineAreRefused(t *testing.T) { + r := newFetchRig(t, func(http.ResponseWriter, *http.Request) { t.Error("a request reached the server") }) + cases := map[string]string{ + "127.0.0.1": "loopback", "::1": "loopback", "10.0.0.1": "private address (RFC 1918)", "192.168.1.1": "private address (RFC 1918)", + "172.16.0.1": "private address (RFC 1918)", "fd00::1": "private address (RFC 4193)", "169.254.1.1": "link-local", "fe80::1": "link-local", + "100.64.0.1": "shared address (RFC 6598)", "0.0.0.0": "unspecified", "::": "unspecified", "224.0.0.1": "multicast", "ff02::1": "multicast", + "10.10.0.7": "private address (RFC 1918)", "::ffff:127.0.0.1": "loopback", + } + for ip, class := range cases { + host := ip + if strings.Contains(ip, ":") { + host = "[" + ip + "]" + } + _, err := r.f.Fetch(t.Context(), "http://"+host+"/a.pdf") + if err == nil || !strings.Contains(err.Error(), class) { + t.Errorf("%s: error = %v, want the class %q", ip, err, class) + } + // The same address behind a name. + r.names["inside.test"] = addrs("203.0.113.9", ip) + if _, err := r.f.Fetch(t.Context(), "http://inside.test/a.pdf"); err == nil || !strings.Contains(err.Error(), class) { + t.Errorf("inside.test -> %s: error = %v", ip, err) + } + } + // The mesh's range from fetch.json, when it is not a private range. + r.f.MeshRange = netip.MustParsePrefix("203.0.113.0/24") + if _, err := r.f.Fetch(t.Context(), "http://203.0.113.5/a.pdf"); err == nil || !strings.Contains(err.Error(), "the mesh's own network") { + t.Errorf("the mesh range: error = %v", err) + } + if len(r.dialed) != 0 { + t.Errorf("a connection was made to %v", r.dialed) + } +} + +// 1599/AC3: a redirect to such an address is refused; a sixth redirect is refused; a redirect to a public address is followed. +func TestRedirectsAreCheckedAndBounded(t *testing.T) { + hops := 0 + r := newFetchRig(t, func(w http.ResponseWriter, req *http.Request) { + switch req.URL.Path { + case "/to-loopback": + http.Redirect(w, req, "http://127.0.0.1/secret", http.StatusFound) + case "/to-name": + http.Redirect(w, req, "http://inside.test/x", http.StatusFound) + case "/loop": + hops++ + http.Redirect(w, req, "/loop", http.StatusFound) + case "/ok": + http.Redirect(w, req, "http://public.test/final.pdf", http.StatusMovedPermanently) + default: + w.Write([]byte("%PDF-1.7 final")) + } + }) + r.names["public.test"] = addrs("203.0.113.9") + r.names["inside.test"] = addrs("192.168.0.9") + if _, err := r.f.Fetch(t.Context(), "http://public.test/to-loopback"); err == nil || !strings.Contains(err.Error(), "loopback") { + t.Errorf("redirect to loopback: error = %v", err) + } + if _, err := r.f.Fetch(t.Context(), "http://public.test/to-name"); err == nil || !strings.Contains(err.Error(), "private address (RFC 1918)") { + t.Errorf("redirect to a name: error = %v", err) + } + if _, err := r.f.Fetch(t.Context(), "http://public.test/loop"); err == nil || !strings.Contains(err.Error(), "redirected more than 5 times") { + t.Errorf("a loop: error = %v", err) + } + if hops != 6 { + t.Errorf("the loop was followed %d times, want the first request and 5 redirects (6 requests)", hops) + } + got, err := r.f.Fetch(t.Context(), "http://public.test/ok") + if err != nil || string(got.Body) != "%PDF-1.7 final" { + t.Errorf("a redirect to a public address: %v, %v", got, err) + } +} + +// 1599/AC4: a body over the limit and a fetch over the time are refused, and nothing is returned to be stored. +func TestABodyOverTheLimitAndAFetchOverTheTimeAreRefused(t *testing.T) { + r := newFetchRig(t, func(w http.ResponseWriter, req *http.Request) { + switch req.URL.Path { + case "/big": + w.Write([]byte(strings.Repeat("x", 2000))) + case "/chunked": + w.(http.Flusher).Flush() // no length is declared + w.Write([]byte(strings.Repeat("x", 2000))) + case "/slow": + time.Sleep(500 * time.Millisecond) + w.Write([]byte("late")) + } + }) + r.names["public.test"] = addrs("203.0.113.9") + r.f.Most = 1000 + for _, p := range []string{"/big", "/chunked"} { + if got, err := r.f.Fetch(t.Context(), "http://public.test"+p); err == nil || got != nil || !strings.Contains(err.Error(), "over the") { + t.Errorf("%s: %v, %v", p, got, err) + } + } + r.f.Timeout = 100 * time.Millisecond + if got, err := r.f.Fetch(t.Context(), "http://public.test/slow"); err == nil || got != nil { + t.Errorf("a slow fetch: %v, %v", got, err) + } +} + +// 1599/AC5: the connection goes to the address that was checked: a name that answers differently on a second lookup +// is not dialled again, and each host is looked up once. +func TestTheCheckedAddressIsTheOneDialled(t *testing.T) { + r := newFetchRig(t, func(w http.ResponseWriter, req *http.Request) { w.Write([]byte("%PDF-1.7 x")) }) + r.names["rebind.test"] = addrs("203.0.113.9") + r.after["rebind.test"] = addrs("127.0.0.1") // a second lookup would say loopback + if _, err := r.f.Fetch(t.Context(), "http://rebind.test:8080/a.pdf"); err != nil { + t.Fatal(err) + } + if len(r.dialed) != 1 || r.dialed[0] != "203.0.113.9:8080" { + t.Errorf("dialled %v, want the checked address and the url's port", r.dialed) + } + if r.lookups["rebind.test"] != 1 { + t.Errorf("the host was looked up %d times", r.lookups["rebind.test"]) + } +} + +// 1599/AC6: a fetched PDF is stored under its SHA-256 and printed as that object. +func TestAFetchedPDFIsStoredUnderItsHashAndPrinted(t *testing.T) { + fake := newIPPFake(t, 41, 5) + r := newRig(t, `{"printer":"192.0.2.10"}`, "pw") + doc := pdfOf(4000) + web := newFetchRig(t, func(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Type", "application/pdf") + w.Write(doc) + }) + web.names["public.test"] = addrs("203.0.113.9") + r.worker.Fetcher = func() (*Fetcher, error) { return web.f, nil } + got, err := r.call(t, "printer.print", map[string]any{"url": "http://public.test/letter.pdf"}) + if err != nil { + t.Fatal(err) + } + m := got.(map[string]any) + key := sha256Hex(doc) + if m["object"] != key || m["job-id"] != float64(41) || m["format"] != "application/pdf" { + t.Errorf("answer = %v", m) + } + if stored, err := r.objects.Get(t.Context(), key); err != nil || string(stored) != string(doc) { + t.Errorf("stored under the hash: %v", err) + } + if _, sent := fake.request(t); string(sent) != string(doc) { + t.Error("the printer was not sent the fetched document") + } + queued := r.bus.published[0] + if queued.URL != "http://public.test/letter.pdf" || queued.Object != "" { + t.Errorf("queued %+v", queued) + } + if err := CheckURLScheme("https://public.test/letter.pdf"); err != nil { + t.Errorf("https refused: %v", err) + } + // A refusal inside the mesh is answered, and nothing is stored nor printed. + fake.body = nil + before := len(r.objects.keys()) + if _, err := r.call(t, "printer.print", map[string]any{"url": "http://127.0.0.1:9000/mesh-shanks-brother/x"}); err == nil || !strings.Contains(err.Error(), "loopback") { + t.Errorf("a loopback address: error = %v", err) + } + if len(r.objects.keys()) != before || fake.body != nil { + t.Error("something was stored or printed for a refused address") + } +} + +// The fetch's facts are the file the mesh writes (files/fetch.json) from the `mesh-range` machine fact, and the device +// process reads them. +func TestTheFetchFactsAreTheMeshRange(t *testing.T) { + raw, err := os.ReadFile("../../files/fetch.json") + if err != nil || !strings.Contains(string(raw), `"mesh-range": "${machine:mesh-range}"`) { + t.Fatalf("files/fetch.json = %s, %v", raw, err) + } + file := filepath.Join(t.TempDir(), "fetch.json") + os.WriteFile(file, []byte(`{"mesh-range": "10.10.0.0/24"}`), 0o600) + p, err := ReadFetchFacts(file) + if err != nil || p.String() != "10.10.0.0/24" { + t.Errorf("prefix %v, %v", p, err) + } + os.WriteFile(file, []byte(`{}`), 0o600) + if _, err := ReadFetchFacts(file); err == nil { + t.Error("facts with no range were taken") + } +} diff --git a/modules/brother/cmd/brother/glass_test.go b/modules/brother/cmd/brother/glass_test.go index ef89f713..6c5efcbc 100644 --- a/modules/brother/cmd/brother/glass_test.go +++ b/modules/brother/cmd/brother/glass_test.go @@ -264,7 +264,7 @@ func TestAnOpenDocumentIsFoundAgainAfterARestartAndItsPagesAreDeletedWhenClosed( docExpiry = 200 * time.Millisecond t.Cleanup(func() { docExpiry = old }) s := &Service{Settings: Settings{Scanner: f.base(), Folder: folder, Resolution: 200, ColorMode: "color"}, Now: func() time.Time { return fixedNow }, - Asks: askBus, AnswerWait: 5 * time.Second, ScanWait: 5 * time.Second, AnswerPoll: 5 * time.Millisecond, Jobs: NewJobs()} + Asks: askBus, AnswerWait: 5 * time.Second, ScanWait: 5 * time.Second, AnswerPoll: 5 * time.Millisecond, Jobs: NewJobs(), Fetch: fetchClient, DocExpiry: docExpiry} if err := s.Recover(context.Background()); err != nil { t.Fatal(err) } diff --git a/modules/brother/cmd/brother/ipp.go b/modules/brother/cmd/brother/ipp.go index 5296650c..449b8b33 100644 --- a/modules/brother/cmd/brother/ipp.go +++ b/modules/brother/cmd/brother/ipp.go @@ -245,6 +245,39 @@ func PrinterStateOf(ctx context.Context, host string) (PrinterState, error) { return ans.State() } +// formatAttributes is what the format check asks for: the document formats the printer takes (RFC 8011 §5.4.22). +var formatAttributes = []string{"document-format-supported"} + +// FormatsOf asks the printer, over IPP, which document formats it lists in document-format-supported. +func FormatsOf(ctx context.Context, host string) ([]string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, ippURL(host), + bytes.NewReader(EncodeGetPrinterAttributes(ippPrinterURI(host), formatAttributes))) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/ipp") + resp, err := (&http.Client{Timeout: 15 * time.Second}).Do(req) + if err != nil { + return nil, fmt.Errorf("the printer's IPP service does not answer: %w", err) + } + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return nil, fmt.Errorf("the printer's IPP answer cannot be read: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("the printer's IPP service answered %s", resp.Status) + } + ans, err := DecodeIPP(body) + if err != nil { + return nil, err + } + if ans.Status >= 0x0100 { + return nil, fmt.Errorf("the printer refused the IPP request with status 0x%04x", ans.Status) + } + return ans.texts("document-format-supported"), nil +} + // PrintOptions are what a Print-Job asks of the printer. type PrintOptions struct { Format string // document-format, application/pdf or image/jpeg diff --git a/modules/brother/cmd/brother/main.go b/modules/brother/cmd/brother/main.go index fe676e4f..34f0d577 100644 --- a/modules/brother/cmd/brother/main.go +++ b/modules/brother/cmd/brother/main.go @@ -11,6 +11,9 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" ) +// uploadVia is how the tool side asks a machine's node-files for an upload; a test puts its own in its place. +var uploadVia = sdkUpload + func main() { if len(os.Args) > 1 && os.Args[1] == "device" { os.Exit(device()) @@ -84,7 +87,7 @@ func askFrom(args map[string]any) (Ask, error) { var a Ask var err error for name, dst := range map[string]*string{"page": &a.Page, "object": &a.Object, "upload": &a.Upload, "sha256": &a.SHA256, - "data": &a.Data, "sides": &a.Sides, "media": &a.Media} { + "data": &a.Data, "sides": &a.Sides, "media": &a.Media, "path": &a.Path, "node": &a.Node, "url": &a.URL, "ask": &a.Of} { if *dst, err = strArg(args, name); err != nil { return a, err } @@ -109,7 +112,8 @@ func tools(settingsFile string, now func() time.Time) []stdio.Tool { } return &Service{Settings: s, Node: os.Getenv("MESH_NODE"), Now: now, Asks: askBus, AnswerWait: answerWait, ScanWait: scanWait, AnswerPoll: answerPoll, - QuickWait: quickWait, FeederWait: feederWait, Jobs: jobs}, nil + QuickWait: quickWait, FeederWait: feederWait, Jobs: jobs, Upload: uploadVia, Fetch: fetchClient, DocExpiry: docExpiry, DocRetry: docRetry, + Bucket: func() (string, error) { return ReadBucket(os.Getenv("BROTHER_BUCKET")) }}, nil } // ask is a tool that reads its arguments, and answers what the service gives. ask := func(run func(s *Service, ctx context.Context, a Ask) (map[string]any, error), bound time.Duration) func(map[string]any) (any, error) { @@ -147,39 +151,37 @@ func tools(settingsFile string, now func() time.Time) []stdio.Tool { }, { Name: "printer.print", - Description: "Print a document on the printer: a file under the home folder of the account the tool runs as, on the machine it runs on (path: reads it, uploads it in parts " + - "and prints it, in one call), a scan kept in the bucket (object), or a completed upload (upload; for a module that sends " + - "bytes over the bus). Only PDF and JPEG, told by the file's bytes and not its name. The device " + - "process sends it as an IPP Print-Job in its own format and answers the job's id and state (with path, the upload id and the size).", + Description: "Print a document on the printer, in one call: a key in the bucket (object: a scan, or a document stored there), a file on any " + + "machine of the mesh (path with node: that machine's node-files sends it into the bucket, a place it keeps a secret is " + + "refused there by name and nothing is printed), or a web link (url). Name exactly one. A Word, spreadsheet or slide " + + "document is turned into a PDF first; otherwise only PDF and JPEG, told by the bytes and not the name, and only a format " + + "the printer lists in document-format-supported. The device process sends it as an IPP Print-Job and answers the job's " + + "id and state. No document crosses the bus. A call answers within 25 s (the tool runner cuts a call at 30 s): a print not done by then goes on and is answered as started with its ask id, and printer.print with only that ask reads it.", Input: map[string]any{"type": "object", "properties": map[string]any{ - "path": str("an absolute path of a PDF or JPEG file under the tool account's home folder (symbolic links are resolved and the real path must lie there too; any other path is refused in one fixed way)"), - "object": str("a scan's key, as scan.added names it"), - "upload": str("a completed upload's id"), + "object": str("a stored document's key: a scan's key, as scan.added names it, or the key node-files.upload answered"), + "path": str("an absolute path of a file on the machine node names"), + "node": str("the machine path is on, such as g14"), + "url": str("an http or https address of a document or a page"), + "ask": str("the ask id of an earlier print that was answered as started: alone, it reads that print's outcome"), "copies": num("1 to 99 (default 1)"), "sides": str("one-sided (default), two-sided-long-edge or two-sided-short-edge"), "media": str("the paper's size name (default " + DefaultMedia + ")"), }}, Run: func(args map[string]any) (any, error) { - path, err := strArg(args, "path") - if err != nil { - return nil, err - } a, err := askFrom(args) if err != nil { return nil, err } + if a.Of != "" && len(args) > 1 { + return nil, fmt.Errorf("ask is read on its own: give the ask id alone to read an earlier print") + } s, err := service() if err != nil { return nil, err } - if path == "" { - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - return s.PrinterPrint(ctx, a) - } ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) defer cancel() - return s.PrintFile(ctx, path, a) + return s.PrinterPrint(ctx, a) }, }, { diff --git a/modules/brother/cmd/brother/manifest_test.go b/modules/brother/cmd/brother/manifest_test.go index 6782b773..044686dd 100644 --- a/modules/brother/cmd/brother/manifest_test.go +++ b/modules/brother/cmd/brother/manifest_test.go @@ -13,7 +13,8 @@ import ( ) type manifest struct { - Requires []string `json:"requires"` + Requires []requirement `json:"requires"` + Invokes []string `json:"invokes"` Binds map[string]string `json:"binds"` Secrets map[string]string `json:"secrets"` Emits []string `json:"emits"` @@ -28,6 +29,31 @@ type manifest struct { } `json:"build"` } +// requirement is an entry of `requires`: a name, or an object with the name and the shares of the bucket. +type requirement struct { + Name string `json:"name"` + Shares []struct { + To string `json:"to"` + Methods []string `json:"methods"` + } `json:"shares"` +} + +func (r *requirement) UnmarshalJSON(b []byte) error { + if err := json.Unmarshal(b, &r.Name); err == nil { + return nil + } + type plain requirement + return json.Unmarshal(b, (*plain)(r)) +} + +func (m manifest) requireNames() []string { + var out []string + for _, r := range m.Requires { + out = append(out, r.Name) + } + return out +} + type seat struct { Name string `json:"name"` Scope string `json:"scope"` @@ -265,6 +291,7 @@ func TestTheAdminProcessIsDeclaredAsMeshMailsSenderIs(t *testing.T) { "BROTHER_ADMIN_PASSWORD": m.OwnSecrets["admin-password"]["path"], "BROTHER_BROKER": m.OwnSecrets["broker-device"]["path"], "BROTHER_STORE": "${dir:state}/store.json", + "BROTHER_FETCH": "${dir:state}/fetch.json", // 1599: the tunnel's range, which the fetch never reaches } if !reflect.DeepEqual(env, want) { t.Errorf("env = %v, want %v", env, want) @@ -330,8 +357,8 @@ func TestThePrinterAsksStreamIsAWorkQueue(t *testing.T) { // written into store.json, a file only the device account can read; the tools' bundle is not told where it is. func TestTheBucketIsRequiredAndItsKeyIsOnlyTheDeviceAccounts(t *testing.T) { m := readManifest(t) - if !reflect.DeepEqual(m.Requires, []string{"s3-bucket"}) { - t.Errorf("requires = %v", m.Requires) + if !reflect.DeepEqual(m.requireNames(), []string{"s3-bucket"}) { + t.Errorf("requires = %v", m.requireNames()) } if m.Binds["s3-bucket"] != "${dir:state}/store-binding.json" || m.Secrets["s3-bucket"] != "${dir:state}/store.secret" { t.Errorf("binds %v, secrets %v", m.Binds, m.Secrets) diff --git a/modules/brother/cmd/brother/objects.go b/modules/brother/cmd/brother/objects.go index ffad97dc..9cd6ca9b 100644 --- a/modules/brother/cmd/brother/objects.go +++ b/modules/brother/cmd/brother/objects.go @@ -24,6 +24,8 @@ import ( "sort" "strings" "time" + + "git.novox.be/novox/mesh-sdk/go/objectlink" ) // Objects is the module's bucket. @@ -55,6 +57,13 @@ func Store(ctx context.Context, o Objects, key, mediaType string, data []byte) e return o.Put(ctx, key, mediaType, data) } +// Presigner is a store that signs a link to one of its objects (novox/hq ADR 0468 §3): the device process's, with the +// binding's key, which reaches this module's bucket only. +type Presigner interface { + // Link is a link for GET or PUT of one key that lives for the given time (at most ten minutes). + Link(method, key string, expires time.Duration) (string, error) +} + // StoreFacts is the file the mesh writes from the s3-bucket binding and its secret. type StoreFacts struct { Scheme string `json:"scheme"` @@ -114,6 +123,16 @@ func NewS3(f StoreFacts) *S3 { func (s *S3) Bucket() string { return s.Name } +// Link signs a link for one object of the bucket with the binding's key, for the host the binding gives. +func (s *S3) Link(method, key string, expires time.Duration) (string, error) { + now := time.Now + if s.Now != nil { + now = s.Now + } + return objectlink.Presign(objectlink.Request{Method: method, Endpoint: s.Endpoint, Bucket: s.Name, Key: key, + AccessKey: s.Access, SecretKey: s.Secret, Region: s.Region, Expires: expires, Now: now()}) +} + func (s *S3) do(ctx context.Context, method, key, mediaType string, body []byte) (*http.Response, error) { u, err := url.Parse(s.Endpoint + "/" + s.Name + "/" + key) if err != nil { diff --git a/modules/brother/cmd/brother/objects_test.go b/modules/brother/cmd/brother/objects_test.go index 1bcfbdc1..ab520ef9 100644 --- a/modules/brother/cmd/brother/objects_test.go +++ b/modules/brother/cmd/brother/objects_test.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "fmt" "io" @@ -9,9 +10,11 @@ import ( "os" "path/filepath" "sort" + "strconv" "strings" "sync" "testing" + "time" ) // memObjects is the bucket in memory, for the tests of what the device process keeps. @@ -19,6 +22,10 @@ type memObjects struct { mu sync.Mutex kept map[string][]byte puts int + // corrupt, when set, makes a GET of a link answer bytes that are not the kept ones. + corrupt bool + // links records each link signed: " ". + links []string } func newMemObjects() *memObjects { return &memObjects{kept: map[string][]byte{}} } @@ -68,6 +75,32 @@ func (m *memObjects) Delete(_ context.Context, key string) error { return nil } +// Link is a link the bucket in memory serves itself (RoundTrip): the key in the path and the life in the query, as a +// presigned link says them (X-Amz-Expires). +func (m *memObjects) Link(method, key string, expires time.Duration) (string, error) { + m.mu.Lock() + m.links = append(m.links, method+" "+key+" "+strconv.Itoa(int(expires/time.Second))) + m.mu.Unlock() + return "http://" + m.Bucket() + ".objects.test/" + key + "?X-Amz-Expires=" + strconv.Itoa(int(expires/time.Second)), nil +} + +// RoundTrip answers a GET of a link made by Link with the object kept under its key. +func (m *memObjects) RoundTrip(r *http.Request) (*http.Response, error) { + m.mu.Lock() + b, ok := m.kept[strings.TrimPrefix(r.URL.Path, "/")] + corrupt := m.corrupt + m.mu.Unlock() + if corrupt && ok { + b = append(append([]byte(nil), b...), 0) + } + status := 200 + if !ok { + status = 404 + } + return &http.Response{StatusCode: status, Body: io.NopCloser(bytes.NewReader(b)), ContentLength: int64(len(b)), + Header: http.Header{}, Request: r}, nil +} + func (m *memObjects) keys() []string { m.mu.Lock() defer m.mu.Unlock() diff --git a/modules/brother/cmd/brother/printer.go b/modules/brother/cmd/brother/printer.go index a7acc06e..75cba5ee 100644 --- a/modules/brother/cmd/brother/printer.go +++ b/modules/brother/cmd/brother/printer.go @@ -36,6 +36,11 @@ const ( // DeviceProcessID is the device process's id as module.json's resource "device" names it: the controller // delivers its bus account as the own secret broker-device. DeviceProcessID = "device" + // PrintTakeUp is how long a print ask may wait on the queue before the device process takes it up, and PrintFollow + // how long the tool side follows it to its end: the device process may fetch and convert before the job. A call + // answers within QuickWait whatever these are, and the print goes on (the tool runner cuts every call at 30 s). + PrintTakeUp = 3 * time.Minute + PrintFollow = 6 * time.Minute // AnswerWait is how long a verb waits for the device process; ScanWait how long a scan does. AnswerWait = 60 * time.Second ScanWait = 180 * time.Second @@ -88,6 +93,14 @@ type Ask struct { Document string `json:"document,omitempty"` // scan: start, or an open document's id: a glass page into it Finish bool `json:"finish,omitempty"` // scan: close the document named in Document into one PDF + + // Path with Node are what printer.print is asked for a file of a machine: the tool side turns them into an Object + // before anything is queued, so the queued ask carries a key and the options and never these (ADR 0468 clause 5). + // A URL is queued as given: the device process alone fetches it (clause 6). + Path string `json:"-"` // print: a file on the machine Node + Of string `json:"-"` // print: the ask id of an earlier print, whose outcome is read + Node string `json:"-"` // print: the machine Path is on + URL string `json:"url,omitempty"` // print: a web document the device process fetches (issue 1599) } // AskBus is how the tools reach the device process: they queue an ask and read an answer back. @@ -145,7 +158,10 @@ func (s *Service) call(ctx context.Context, ask Ask, wait time.Duration) (map[st // queue puts the ask on the work queue and answers its id, under which the answer is written. The asker waits // startBy for the device process to take the ask up (0 is for as long as it is waited for). func (s *Service) queue(ask Ask, startBy time.Duration) (string, error) { - ask.ID, ask.At = newAskID(), s.Now().UTC() + if ask.ID == "" { + ask.ID = newAskID() + } + ask.At = s.Now().UTC() if startBy > 0 { ask.Deadline = ask.At.Add(startBy) } @@ -211,9 +227,12 @@ func validPage(page string) bool { var ( scanKeyPattern = regexp.MustCompile(`^[0-9a-f]{64}\.pdf$`) - uploadPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`) - mediaPattern = regexp.MustCompile(`^[a-z0-9_.-]{1,64}$`) - sha256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`) + // printKeyPattern is a document the bucket keeps under its SHA-256: a scan's key, or the key node-files.upload + // answers for a file sent from a machine (the bare SHA-256). + printKeyPattern = regexp.MustCompile(`^[0-9a-f]{64}(\.pdf)?$`) + uploadPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`) + mediaPattern = regexp.MustCompile(`^[a-z0-9_.-]{1,64}$`) + sha256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`) ) // Sides are the values the printer's sides take. @@ -259,14 +278,23 @@ func checkUpload(a Ask) error { // checkPrint is what is wrong with an ask to print, or nil, after the defaults are laid in. func checkPrint(a *Ask) error { - if (a.Object == "") == (a.Upload == "") { - return fmt.Errorf("name either object (a scan's key) or upload (a completed upload's id), not both and not neither") + if a.Upload != "" { + return fmt.Errorf("upload is no longer a source of printer.print: send the file with path and node, or name its key in object") } - if a.Object != "" && !scanKeyPattern.MatchString(a.Object) { - return fmt.Errorf("object %q is not a scan's key: 64 hex digits and .pdf", a.Object) + if (a.Path == "") != (a.Node == "") { + return fmt.Errorf("path and node go together: path is a file on the machine node names") } - if a.Upload != "" && !uploadPattern.MatchString(a.Upload) { - return fmt.Errorf("upload %q is not an upload id: letters, digits, - and _, at most 64", a.Upload) + sources := 0 + for _, given := range []bool{a.Object != "", a.Path != "", a.URL != ""} { + if given { + sources++ + } + } + if sources != 1 { + return fmt.Errorf("name exactly one of object (a key in the bucket), path with node (a file on that machine) or url, not %s", map[bool]string{true: "none", false: "two or more"}[sources == 0]) + } + if a.Object != "" && !printKeyPattern.MatchString(a.Object) { + return fmt.Errorf("object %q is not a stored document's key: 64 hex digits, with .pdf for a scan", a.Object) } if a.Copies == 0 { a.Copies = 1 @@ -307,50 +335,42 @@ func (s *Service) PrinterUpload(ctx context.Context, a Ask) (map[string]any, err return s.call(ctx, a, s.AnswerWait) } -// PrintFile prints a PDF or a JPEG that lies on the machine the tool runs on: it reads the file, uploads it in -// parts under an upload id of its own and prints that, in one call (issue 1551). The type is read from the -// file's first bytes, never its name; any other file is refused before anything is queued. The answer is the -// print's, with the upload id and the file's size. -func (s *Service) PrintFile(ctx context.Context, path string, a Ask) (map[string]any, error) { - if a.Object != "" || a.Upload != "" { - return nil, fmt.Errorf("name path, or object, or upload: one of the three") - } - // The path is checked first, and refused in one fixed way that tells nothing about the file. - doc, err := ReadDocument(path, MostParts*PartSize) +// sdkUpload asks node's node-files to send a file of that machine into the bucket and answers the key it was +// stored under (issue 1597). The form is the seat's verb on one machine, `seat:.@`. +func sdkUpload(node, path, bucket string) (string, error) { + raw, err := stdio.Ask("seat:node-files.upload@"+node, map[string]any{"path": path, "bucket": bucket}) if err != nil { - return nil, err + return "", err } - a.Upload = "p-" + newAskID() - if err := checkPrint(&a); err != nil { - return nil, err + var ans struct { + Key string `json:"key"` + Answer json.RawMessage `json:"answer"` } - if s.Settings.Printer == "" { - return nil, unsetError("printer") + if err := json.Unmarshal(raw, &ans); err != nil { + return "", fmt.Errorf("node-files' answer is not JSON: %w", err) } - if _, _, err := SniffFormat(doc); err != nil { - return nil, fmt.Errorf("file %s: %w", path, err) + if ans.Key == "" && len(ans.Answer) > 0 { + _ = json.Unmarshal(ans.Answer, &ans) } - sum, n := sha256Hex(doc), partCount(len(doc)) - for i := 1; i <= n; i++ { - part, _, _ := cutPartOf(doc, i, PartSize) - up, err := s.PrinterUpload(ctx, Ask{Upload: a.Upload, Part: i, Parts: n, SHA256: sum, Data: base64.StdEncoding.EncodeToString(part)}) - if err != nil { - return nil, fmt.Errorf("part %d of %d of %s could not be uploaded: %w", i, n, path, err) - } - if i == n && up["complete"] != true { - return nil, fmt.Errorf("the upload of %s is not complete after its last part: %v", path, up) - } + if ans.Key == "" { + return "", fmt.Errorf("node-files' answer names no key") } - out, err := s.PrinterPrint(ctx, a) - if err != nil { - return nil, fmt.Errorf("%s is uploaded as %s but could not be printed: %w", path, a.Upload, err) - } - out["path"], out["upload"], out["bytes"] = path, a.Upload, len(doc) - return out, nil + return ans.Key, nil } -// PrinterPrint asks the device process to print a kept scan or a completed upload. +// PrinterPrint prints a stored document, a file of a machine, or a web link, in one call. A file of a machine is first +// sent into the bucket by that machine's node-files (refused there by name when it is a secret, and then nothing is +// printed); the print ask the device process takes carries the key and the options and no bytes. +// +// The tool runner cuts every call at 30 s, so the call answers within QuickWait, upload included. A print not done by +// then goes on in the background and is answered as started, with its ask id; printer.print {ask} reads its outcome. func (s *Service) PrinterPrint(ctx context.Context, a Ask) (map[string]any, error) { + if a.Of != "" { + if a.Object != "" || a.Path != "" || a.Node != "" || a.URL != "" || a.Upload != "" { + return nil, fmt.Errorf("ask is read on its own: name exactly one of object, path with node, url, or the ask id of an earlier print") + } + return s.PrintResult(ctx, a.Of) + } a.Op = OpPrint if err := checkPrint(&a); err != nil { return nil, err @@ -358,9 +378,103 @@ func (s *Service) PrinterPrint(ctx context.Context, a Ask) (map[string]any, erro if s.Settings.Printer == "" { return nil, unsetError("printer") } - return s.call(ctx, a, s.AnswerWait) + bucket := "" + switch { + case a.URL != "": + if err := CheckURLScheme(a.URL); err != nil { + return nil, err + } + case a.Path != "": + if s.Upload == nil { + return nil, fmt.Errorf("this module has no way to ask node-files for an upload") + } + var err error + if bucket, err = s.Bucket(); err != nil { + return nil, err + } + } + id := newAskID() + j := s.Jobs.put(id) + go func() { + j.body, j.err = s.printRun(id, a, bucket) + close(j.done) + }() + return s.printWait(ctx, id, j) } +// printRun is a print followed to its end, however long the caller waits: the upload, the ask, the device's answer. +func (s *Service) printRun(id string, a Ask, bucket string) (map[string]any, error) { + from := map[string]any{} + if a.Path != "" { + key, err := s.Upload(a.Node, a.Path, bucket) + if err != nil { + return nil, fmt.Errorf("%s on %s could not be sent to the bucket, and nothing is printed: %w", a.Path, a.Node, err) + } + from["path"], from["node"] = a.Path, a.Node + a.Object, a.Path, a.Node = key, "", "" + } + a.ID = id + if _, err := s.queue(a, PrintTakeUp); err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), PrintFollow) + defer cancel() + out, err := s.await(ctx, id, PrintFollow) + if err != nil { + return nil, err + } + for k, v := range from { + out[k] = v + } + return out, nil +} + +// printWait gives the print's outcome if it comes within QuickWait, and started with the ask id otherwise. +func (s *Service) printWait(ctx context.Context, id string, j *job) (map[string]any, error) { + quick := s.QuickWait + if quick <= 0 { + quick = QuickWait + } + select { + case <-j.done: + return j.body, j.err + case <-time.After(quick): + return map[string]any{"state": "started", "ask": id, + "note": "the print goes on; read its outcome with printer.print {ask}"}, nil + case <-ctx.Done(): + return nil, fmt.Errorf("the caller went away; the print %s goes on: %w", id, ctx.Err()) + } +} + +// PrintResult is the outcome of the print with this ask id: the device's answer when it is done (a failure is the +// error), and started, still printing, while it is not. A print this process followed is read from it, one it did not +// (it was restarted) from the device's answers, which are kept for 10 minutes. +func (s *Service) PrintResult(ctx context.Context, id string) (map[string]any, error) { + if !askIDPattern.MatchString(id) { + return nil, fmt.Errorf("ask %q is not an ask id: the 24 hex digits printer.print answered", id) + } + if j := s.Jobs.get(id); j != nil { + return s.printWait(ctx, id, j) + } + raw, err := s.Asks.Answer(id) + if err != nil { + return nil, fmt.Errorf("the device process's answers cannot be read: %w", err) + } + if raw == nil { + return map[string]any{"state": "started", "ask": id, "note": "still printing, or this ask is not known"}, nil + } + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("the device process's answer is not JSON: %w", err) + } + if msg, bad := out["error"].(string); bad { + return nil, fmt.Errorf("%s", msg) + } + return out, nil +} + +var askIDPattern = regexp.MustCompile(`^[0-9a-f]{24}$`) + // PrinterStatus is the printer's state over IPP and the scanner's over eSCL, none of it behind a login // (ADR 0467 clause 5). A part that does not answer is an answer, with why, not an error. Which of printer and // scanner is not set is said by name, and nothing is contacted. diff --git a/modules/brother/cmd/brother/printlink_test.go b/modules/brother/cmd/brother/printlink_test.go new file mode 100644 index 00000000..56670d1f --- /dev/null +++ b/modules/brother/cmd/brother/printlink_test.go @@ -0,0 +1,318 @@ +package main + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +// uploadFake stands for node-files.upload on another machine: it records the ask and answers a key, or refuses. +type uploadFake struct { + mu sync.Mutex + calls []string + key string + err error +} + +// withUpload puts a fake node-files in the tool side's place, and the bucket file the mesh writes from the binding +// (files/bucket.json) where the tool reads it. +// seen is the uploads asked so far, read under the fake's lock (the print runs in the background). +func (f *uploadFake) seen() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.calls...) +} + +func withUpload(t *testing.T, key string, err error) *uploadFake { + t.Helper() + f := &uploadFake{key: key, err: err} + old := uploadVia + uploadVia = func(node, path, bucket string) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, node+"|"+path+"|"+bucket) + return f.key, f.err + } + t.Cleanup(func() { uploadVia = old }) + file := filepath.Join(t.TempDir(), "bucket.json") + if err := os.WriteFile(file, []byte(`{"bucket": "brother-test"}`), 0o644); err != nil { + t.Fatal(err) + } + t.Setenv("BROTHER_BUCKET", file) + return f +} + +// 1598/AC1: none, or two, of object, path+node and url is refused naming the rule; path without node is refused. +func TestPrintTakesExactlyOneSource(t *testing.T) { + fake := newIPPFake(t, 1, 3) + up := withUpload(t, "k", nil) + r := newRig(t, `{"printer":"192.0.2.10"}`, "pw") + key := sha256Hex([]byte("x")) + ".pdf" + for name, args := range map[string]map[string]any{ + "none": {}, + "only options": {"copies": float64(2)}, + "object and path": {"object": key, "path": "/home/ops/a.pdf", "node": "g14"}, + "object and url": {"object": key, "url": "https://example.com/a.pdf"}, + "path+node and url": {"path": "/home/ops/a.pdf", "node": "g14", "url": "https://example.com/a.pdf"}, + "all three": {"object": key, "path": "/home/ops/a.pdf", "node": "g14", "url": "https://example.com/a.pdf"}, + "path without node": {"path": "/home/ops/a.pdf"}, + "node without path": {"node": "g14", "object": key}, + "the retired upload": {"upload": "u1"}, + "object and upload": {"object": key, "upload": "u1"}, + "path and its object": {"path": "/home/ops/a.pdf", "node": "g14", "object": key}, + } { + _, err := r.call(t, "printer.print", args) + if err == nil { + t.Errorf("%s: accepted", name) + continue + } + rule := strings.Contains(err.Error(), "exactly one of object") || strings.Contains(err.Error(), "path and node go together") || + strings.Contains(err.Error(), "upload is no longer") + if !rule { + t.Errorf("%s: the refusal does not name the rule: %v", name, err) + } + } + if r.bus.asks() != 0 || len(up.seen()) != 0 || fake.body != nil { + t.Errorf("%d asks queued, %d uploads asked, printer sent %d bytes", r.bus.asks(), len(up.seen()), len(fake.body)) + } +} + +// 1598/AC2: with path and node g14, printer.print asks g14's node-files to upload the path into brother's bucket, and +// queues a print ask that carries the returned key and the options: under 4 KiB, and no path, node or bytes. +func TestPrintOfAFileOfAnotherMachineUploadsThenPrintsTheKey(t *testing.T) { + fake := newIPPFake(t, 31, 5) + doc := pdfOf(6000) + key := sha256Hex(doc) + up := withUpload(t, key, nil) + r := newRig(t, `{"printer":"192.0.2.10"}`, "pw") + r.objects.Put(t.Context(), key, "application/pdf", doc) // what g14's upload stored + got, err := r.call(t, "printer.print", map[string]any{"path": "/home/ops/letter.pdf", "node": "g14", "copies": float64(2)}) + if err != nil { + t.Fatal(err) + } + if len(up.seen()) != 1 || up.seen()[0] != "g14|/home/ops/letter.pdf|brother-test" { + t.Fatalf("uploads asked: %v", up.seen()) + } + if len(r.bus.published) != 1 { + t.Fatalf("%d asks queued", len(r.bus.published)) + } + queued := r.bus.published[0] + raw, _ := json.Marshal(queued) + var fields map[string]any + json.Unmarshal(raw, &fields) + if queued.Op != OpPrint || queued.Object != key || queued.Copies != 2 || len(raw) >= 4096 { + t.Errorf("queued %s", raw) + } + for _, banned := range []string{"path", "node", "url", "data"} { + if _, has := fields[banned]; has { + t.Errorf("the queued ask carries %q: %s", banned, raw) + } + } + m := got.(map[string]any) + if m["job-id"] != float64(31) || m["node"] != "g14" || m["path"] != "/home/ops/letter.pdf" { + t.Errorf("answer = %v", m) + } + if _, sent := fake.request(t); string(sent) != string(doc) { + t.Errorf("the printer was not sent the stored document") + } +} + +// 1598/AC3: an upload refused by node-files is answered with node-files' reason, and nothing is printed. +func TestARefusedUploadIsAnsweredWithItsReasonAndNothingIsPrinted(t *testing.T) { + fake := newIPPFake(t, 1, 3) + up := withUpload(t, "", errors.New("/home/ops/.ssh/id_ed25519 is refused by name: it is in .ssh, which holds an account's credentials")) + r := newRig(t, `{"printer":"192.0.2.10"}`, "pw") + _, err := r.call(t, "printer.print", map[string]any{"path": "/home/ops/.ssh/id_ed25519", "node": "g14"}) + if err == nil || !strings.Contains(err.Error(), "is refused by name: it is in .ssh") { + t.Fatalf("error = %v", err) + } + if len(up.seen()) != 1 || r.bus.asks() != 0 || fake.body != nil { + t.Errorf("%d uploads, %d asks queued, printer sent %d bytes", len(up.seen()), r.bus.asks(), len(fake.body)) + } +} + +// 1598/AC4: the device reads document-format-supported before the job and refuses a format the printer does not +// list, naming the list; a printer that cannot be asked is sent nothing. +func TestTheDeviceChecksTheFormatsThePrinterListsBeforeTheJob(t *testing.T) { + fake := newIPPFake(t, 5, 3) + r := newRig(t, `{"printer":"192.0.2.10"}`, "pw") + doc := pdfOf(300) + key := sha256Hex(doc) + ".pdf" + r.objects.Put(t.Context(), key, "application/pdf", doc) + if _, err := r.call(t, "printer.print", map[string]any{"object": key}); err != nil { + t.Fatal(err) + } + if fake.asked != 1 || fake.body == nil { + t.Fatalf("the formats were asked %d times, the job was sent: %v", fake.asked, fake.body != nil) + } + + fake.body, fake.formats = nil, []string{"image/urf", "image/jpeg"} + _, err := r.call(t, "printer.print", map[string]any{"object": key}) + if err == nil || !strings.Contains(err.Error(), "application/pdf") || !strings.Contains(err.Error(), "image/urf, image/jpeg") { + t.Fatalf("a format not listed: error = %v", err) + } + if fake.body != nil { + t.Error("a job was sent in a format the printer does not list") + } + + fake.formats = nil + if _, err := r.call(t, "printer.print", map[string]any{"object": key}); err == nil || fake.body != nil { + t.Errorf("a printer that lists nothing: error = %v, sent %v", err, fake.body != nil) + } +} + +// 1598/AC5: brother's s3-bucket requirement shares put with seat:node-files; brother invokes seat:node-files.upload; +// the bucket's name reaches the tool side through a file the mesh writes from the binding. +func TestManifestSharesPutWithNodeFilesAndInvokesItsUpload(t *testing.T) { + m := readManifest(t) + if len(m.Requires) != 1 || m.Requires[0].Name != "s3-bucket" || len(m.Requires[0].Shares) != 1 || + m.Requires[0].Shares[0].To != "seat:node-files" || strings.Join(m.Requires[0].Shares[0].Methods, ",") != "put" { + t.Errorf("requires = %+v", m.Requires) + } + if len(m.Invokes) != 1 || m.Invokes[0] != "seat:node-files.upload" { + t.Errorf("invokes = %v", m.Invokes) + } + var bucket map[string]any + for _, r := range m.Resources { + if r["id"] == "bucket" { + bucket = r + } + } + if bucket == nil || bucket["path"] != "${dir:state}/bucket.json" || bucket["template"] != "files/bucket.json" { + t.Fatalf("bucket resource = %v", bucket) + } + raw, _ := os.ReadFile("../../files/bucket.json") + if !strings.Contains(string(raw), "${bound:s3-bucket:bucket}") || strings.Contains(string(raw), "secret") { + t.Errorf("files/bucket.json = %s", raw) + } + for _, a := range m.Build.Artifacts { + if a["name"] == "tools" && a["env"].(map[string]any)["BROTHER_BUCKET"] != "${dir:state}/bucket.json" { + t.Errorf("the tools bundle is not given BROTHER_BUCKET: %v", a["env"]) + } + } +} + +// quickly makes a call wait this long before it answers started, for the test. +func quickly(t *testing.T, d time.Duration) { + t.Helper() + old := quickWait + quickWait = d + t.Cleanup(func() { quickWait = old }) +} + +// 1598/AC7: a device answer slower than the call's bound is answered as started with the ask id within the bound, and +// printer.print {ask} then returns the device's answer. +func TestASlowPrintIsAnsweredAsStartedAndReadWithItsAsk(t *testing.T) { + fake := newIPPFake(t, 71, 5) + fake.delay = 400 * time.Millisecond + quickly(t, 60*time.Millisecond) + r := newRig(t, `{"printer":"192.0.2.10"}`, "pw") + doc := pdfOf(300) + key := sha256Hex(doc) + ".pdf" + r.objects.Put(t.Context(), key, "application/pdf", doc) + began := time.Now() + got, err := r.call(t, "printer.print", map[string]any{"object": key}) + if err != nil { + t.Fatal(err) + } + m := got.(map[string]any) + id, _ := m["ask"].(string) + if m["state"] != "started" || id == "" || time.Since(began) > 300*time.Millisecond { + t.Fatalf("answer %v after %s", m, time.Since(began)) + } + if QuickWait != 25*time.Second { + t.Errorf("a call waits %s, want 25s", QuickWait) + } + // Read while it goes on, and after: the same id. + deadline := time.Now().Add(5 * time.Second) + for { + read, err := r.call(t, "printer.print", map[string]any{"ask": id}) + if err != nil { + t.Fatal(err) + } + rm := read.(map[string]any) + if rm["state"] == nil { + if rm["job-id"] != float64(71) || rm["object"] != key { + t.Errorf("the outcome = %v", rm) + } + break + } + if time.Now().After(deadline) { + t.Fatalf("never done: %v", rm) + } + } + // A restarted tool process has no job: the device's answers are read. + r2 := newRig(t, `{"printer":"192.0.2.10"}`, "pw") + r2.answers.m = map[string]json.RawMessage{"0123456789abcdef01234567": json.RawMessage(`{"job-id": 5}`)} + if read, err := r2.call(t, "printer.print", map[string]any{"ask": "0123456789abcdef01234567"}); err != nil || read.(map[string]any)["job-id"] != float64(5) { + t.Errorf("read from the answers: %v, %v", read, err) + } + if _, err := r.call(t, "printer.print", map[string]any{"ask": id, "copies": float64(2)}); err == nil { + t.Error("ask with another argument was accepted") + } + if _, err := r.call(t, "printer.print", map[string]any{"ask": "../x"}); err == nil { + t.Error("a made-up ask id was accepted") + } +} + +// 1598/AC8: printer.print {path, node} whose upload and print together pass the bound answers started with the ask id, +// never a timeout, and the print follows. +func TestASlowUploadAndPrintAnswersStartedNeverATimeout(t *testing.T) { + fake := newIPPFake(t, 72, 5) + fake.delay = 100 * time.Millisecond + quickly(t, 60*time.Millisecond) + doc := pdfOf(300) + key := sha256Hex(doc) + up := withUpload(t, key, nil) + old := uploadVia + uploadVia = func(node, path, bucket string) (string, error) { + time.Sleep(150 * time.Millisecond) // node-files is slow + return old(node, path, bucket) + } + t.Cleanup(func() { uploadVia = old }) + r := newRig(t, `{"printer":"192.0.2.10"}`, "pw") + r.objects.Put(t.Context(), key, "application/pdf", doc) + got, err := r.call(t, "printer.print", map[string]any{"path": "/home/ops/letter.pdf", "node": "g14"}) + if err != nil { + t.Fatalf("a slow upload failed the call: %v", err) + } + m := got.(map[string]any) + id, _ := m["ask"].(string) + if m["state"] != "started" || id == "" || len(up.seen()) > 1 { + t.Fatalf("answer %v", m) + } + for deadline := time.Now().Add(5 * time.Second); ; { + read, err := r.call(t, "printer.print", map[string]any{"ask": id}) + if err != nil { + t.Fatal(err) + } + if rm := read.(map[string]any); rm["state"] == nil { + if rm["job-id"] != float64(72) || rm["path"] != "/home/ops/letter.pdf" || rm["node"] != "g14" { + t.Errorf("the outcome = %v", rm) + } + break + } + if time.Now().After(deadline) { + t.Fatal("never done") + } + } + if len(up.seen()) != 1 || len(r.bus.published) != 1 || r.bus.published[0].Object != key { + t.Errorf("uploads %v, asks %v", up.seen(), r.bus.published) + } + // An upload that fails after the call answered is the outcome, read with the ask. + uploadVia = func(node, path, bucket string) (string, error) { + time.Sleep(150 * time.Millisecond) + return "", errors.New("/home/ops/.ssh/id_ed25519 is refused by name") + } + got, _ = r.call(t, "printer.print", map[string]any{"path": "/home/ops/.ssh/id_ed25519", "node": "g14"}) + id2, _ := got.(map[string]any)["ask"].(string) + time.Sleep(300 * time.Millisecond) + if _, err := r.call(t, "printer.print", map[string]any{"ask": id2}); err == nil || !strings.Contains(err.Error(), "refused by name") { + t.Errorf("the failed upload's outcome: %v", err) + } +} diff --git a/modules/brother/cmd/brother/rig_test.go b/modules/brother/cmd/brother/rig_test.go index 75c6b7d9..22ae831f 100644 --- a/modules/brother/cmd/brother/rig_test.go +++ b/modules/brother/cmd/brother/rig_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "github.com/nats-io/nats.go/jetstream" + "net/http" "os" "path/filepath" "sync" @@ -144,7 +145,12 @@ func newRig(t *testing.T, settings, password string) *printerRig { bus := &fakeAskBus{worker: w, answers: answers} old, oldWait, oldScan, oldPoll := askBus, answerWait, scanWait, answerPoll askBus, answerWait, scanWait, answerPoll = bus, 5*time.Second, 5*time.Second, 5*time.Millisecond - t.Cleanup(func() { askBus, answerWait, scanWait, answerPoll = old, oldWait, oldScan, oldPoll }) + oldFetch := fetchClient + fetchClient = &http.Client{Transport: objects} + t.Cleanup(func() { + askBus, answerWait, scanWait, answerPoll = old, oldWait, oldScan, oldPoll + fetchClient = oldFetch + }) tl := tools(file, func() time.Time { return fixedNow }) return &printerRig{tl: tl, run: tool(t, tl, "printer.settings").Run, bus: bus, worker: w, answers: answers, objects: objects, said: &out, mu: &mu} } diff --git a/modules/brother/cmd/brother/scan.go b/modules/brother/cmd/brother/scan.go index 69cd3b36..a73faa9c 100644 --- a/modules/brother/cmd/brother/scan.go +++ b/modules/brother/cmd/brother/scan.go @@ -1,12 +1,15 @@ package main import ( + "bytes" "context" - "encoding/base64" "fmt" + "net/http" "strings" "sync" "time" + + "git.novox.be/novox/mesh-sdk/go/objectlink" ) // EventAdded is the module's event: a scan was kept in the bucket (ADR 0464, ADR 0468 clause 2), said by the device @@ -29,6 +32,15 @@ type Service struct { QuickWait, FeederWait time.Duration // Jobs are the scans this process follows. Jobs *Jobs + // Upload asks a machine's node-files to send one of its files into the bucket and answers the key; Bucket is + // brother's bucket. Both are the tool side's (printer.print with path and node). + // Fetch reads the links the device process signs; DocExpiry and DocRetry are the open documents' waits. They are + // copied into the Service when it is made, because background work reads them while a test (or a reload) changes the + // package's own, which would race. + Fetch *http.Client + DocExpiry, DocRetry time.Duration + Upload func(node, path, bucket string) (string, error) + Bucket func() (string, error) } // Jobs are the scans this tool process has started and not yet been asked the outcome of. A scan runs in the @@ -229,7 +241,7 @@ func (s *Service) finishScan(ctx context.Context, id string, wait time.Duration, } if doc, _ := ans["document"].(string); doc != "" && ans["key"] == nil { // A page was added to an open document: nothing to file yet, and the document is closed if it is left open. - s.armDocument(doc, docExpiry, 0) + s.armDocument(doc, s.docExpiry(), 0) return ans, nil } if ask.Document != "" && ask.Finish { @@ -280,26 +292,19 @@ func (s *Service) fileAnswer(ctx context.Context, ans map[string]any) (map[strin func (s *Service) fileOne(ctx context.Context, ans map[string]any) (map[string]any, error) { key, _ := ans["key"].(string) sum, _ := ans["sha256"].(string) - parts, _ := ans["parts"].(float64) - if key == "" || sum == "" || parts < 1 || parts > MostAnswerParts { - return nil, fmt.Errorf("the device process's answer to the scan names no stored scan: %v", ans) + link, _ := ans["link"].(string) + if key == "" || sum == "" || link == "" { + return nil, fmt.Errorf("the device process's answer to the scan names no stored scan or no link to it: %v", withoutLink(ans)) } - got := make([][]byte, 0, int(parts)) - for i := 1; i <= int(parts); i++ { - p, err := s.fetchPart(ctx, key, i) - if err != nil && ctx.Err() == nil { - // A part fetch is a read, safe to repeat: one retry rides out a short bus outage (issue 1536). - p, err = s.fetchPart(ctx, key, i) - } - if err != nil { - return nil, fmt.Errorf("the scan is kept as %s, but its part %d could not be fetched: %w", key, i, err) - } - got = append(got, p) - } - pdf, err := joinParts(got, sum) - if err != nil { - return nil, fmt.Errorf("the scan is kept as %s: %w", key, err) + // The scan is read from the store by the link the device process signed, never through the bus (ADR 0468 §4), and + // must hash to the scan's SHA-256 before anything is written. + var buf bytes.Buffer + gctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + if _, err := objectlink.Get(gctx, s.fetchClient(), link, &buf, objectlink.MaxDocument, sum); err != nil { + return nil, fmt.Errorf("the scan is kept as %s, but it could not be read from the store: %w", key, err) } + pdf := buf.Bytes() at, err := time.Parse(time.RFC3339, fmt.Sprint(ans["scanned-at"])) if err != nil { at = s.Now() @@ -310,7 +315,7 @@ func (s *Service) fileOne(ctx context.Context, ans map[string]any) (map[string]a } body := map[string]any{} for k, v := range ans { - if k != "key" && k != "size" && k != "parts" { + if k != "key" && k != "size" && k != "link" && k != "link-seconds" { body[k] = v } } @@ -318,20 +323,41 @@ func (s *Service) fileOne(ctx context.Context, ans map[string]any) (map[string]a return body, nil } -// fetchPart is one part of a stored object, asked of the device process and decoded. -func (s *Service) fetchPart(ctx context.Context, object string, part int) ([]byte, error) { - ans, err := s.call(ctx, Ask{Op: OpDocument, Object: object, Part: part}, s.AnswerWait) - if err != nil { - return nil, err +// withoutLink is an answer said in an error without its link, which carries a signature. +func withoutLink(ans map[string]any) map[string]any { + out := map[string]any{} + for k, v := range ans { + if k != "link" { + out[k] = v + } } - data, _ := ans["data"].(string) - raw, err := base64.StdEncoding.DecodeString(data) - if err != nil { - return nil, fmt.Errorf("the part is not base64: %w", err) - } - return raw, nil + return out } +func (s *Service) fetchClient() *http.Client { + if s.Fetch != nil { + return s.Fetch + } + return &http.Client{} +} + +func (s *Service) docExpiry() time.Duration { + if s.DocExpiry > 0 { + return s.DocExpiry + } + return 30 * time.Minute +} + +func (s *Service) docRetry() time.Duration { + if s.DocRetry > 0 { + return s.DocRetry + } + return 5 * time.Minute +} + +// fetchClientDefault is what a Service made by the tools reads the links with; a test puts a client of its own in its place before it makes the tools. +var fetchClient = &http.Client{} + // ScanStatus says whether the scanner answers, and what it is doing. A scanner that does not answer is an // answer, not an error: that is how the module's health is read. func (s *Service) ScanStatus(ctx context.Context) (map[string]any, error) { @@ -370,7 +396,7 @@ func (s *Service) armDocument(doc string, d time.Duration, attempts int) { if err == nil || attempts >= 10 || strings.Contains(err.Error(), "is closed already") || strings.Contains(err.Error(), "its id is wrong") { return } - s.armDocument(doc, docRetry, attempts+1) + s.armDocument(doc, s.docRetry(), attempts+1) }) } @@ -388,9 +414,9 @@ func (s *Service) Recover(ctx context.Context) error { if id == "" { continue } - left := docExpiry + left := s.docExpiry() if at, err := time.Parse(time.RFC3339, fmt.Sprint(m["updated"])); err == nil { - left = max(docExpiry-s.Now().Sub(at), time.Second) + left = max(s.docExpiry()-s.Now().Sub(at), time.Second) } s.armDocument(id, left, 0) } diff --git a/modules/brother/cmd/brother/scanlink_test.go b/modules/brother/cmd/brother/scanlink_test.go new file mode 100644 index 00000000..7f480eec --- /dev/null +++ b/modules/brother/cmd/brother/scanlink_test.go @@ -0,0 +1,157 @@ +package main + +import ( + "encoding/json" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" +) + +// 1601/AC1: scanner.scan reads the stored PDF by the GET link the device's answer carries (never by parts) and writes it +// into the scan folder; the answer is the event's body plus the path. +func TestTheScanIsReadFromTheStoreByTheLinkAndFiled(t *testing.T) { + f := newFakeScanner(t, fixture(t, "scan.jpg")) + r, folder := scanRig(t, f, Settings{Crop: false}) + body, err := r.scan(t) + if err != nil { + t.Fatal(err) + } + if len(*r.said) != 1 { + t.Fatalf("%d events", len(*r.said)) + } + stored, err := r.objects.Get(t.Context(), (*r.said)[0].body["object"].(string)) + if err != nil { + t.Fatal(err) + } + filed, err := os.ReadFile(body["path"].(string)) + if err != nil || string(filed) != string(stored) { + t.Fatalf("the filed scan is not the stored one: %v", err) + } + if filepath.Dir(body["path"].(string)) != folder { + t.Errorf("filed in %s, want %s", filepath.Dir(body["path"].(string)), folder) + } + for _, a := range r.bus.published { + if a.Op == OpDocument { + t.Errorf("a part of the scan was asked of the device process over the bus") + } + } + for k := range (*r.said)[0].body { + if body[k] == nil && k != "path" { + t.Errorf("the event's %s is not in the answer", k) + } + } + if body["link"] != nil || body["parts"] != nil || body["key"] != nil { + t.Errorf("the answer carries the transport's own fields: %v", body) + } +} + +// 1601/AC2: the device's answer carries a link that expires within ten minutes and no bytes of the document; the real +// store signs it with the binding's key for the host the binding gives. +func TestTheDeviceAnswerCarriesALinkAndNoBytes(t *testing.T) { + f := newFakeScanner(t, fixture(t, "scan.jpg")) + r, _ := scanRig(t, f, Settings{Crop: false}) + if _, err := r.scan(t); err != nil { + t.Fatal(err) + } + var ans map[string]any + for _, raw := range r.answers.m { + json.Unmarshal(raw, &ans) + } + link, _ := ans["link"].(string) + u, err := url.Parse(link) + if err != nil || link == "" { + t.Fatalf("answer %v", ans) + } + secs, _ := strconv.Atoi(u.Query().Get("X-Amz-Expires")) + if secs < 1 || time.Duration(secs)*time.Second > 10*time.Minute { + t.Errorf("the link lives %d s", secs) + } + raw, _ := json.Marshal(ans) + if ans["data"] != nil || len(raw) > 4096 { + t.Errorf("the answer is %d bytes and carries data %v", len(raw), ans["data"] != nil) + } + + // The real store: the binding's key signs, for the binding's host and port, in the module's bucket. + s3 := NewS3(StoreFacts{Scheme: "http", At: "novox.internal", Port: "20001", Bucket: "mesh-shanks-brother", Region: "eu-west", + Key: "binding-key", Secret: "binding-secret"}) + s3.Now = func() time.Time { return fixedNow } + got, err := s3.Link("GET", "abc.pdf", 10*time.Minute) + if err != nil { + t.Fatal(err) + } + gu, _ := url.Parse(got) + if gu.Host != "novox.internal:20001" || gu.Path != "/mesh-shanks-brother/abc.pdf" || + !strings.HasPrefix(gu.Query().Get("X-Amz-Credential"), "binding-key/") || gu.Query().Get("X-Amz-Expires") != "600" { + t.Errorf("link %s", got) + } + if _, err := s3.Link("GET", "abc.pdf", 11*time.Minute); err == nil { + t.Error("a link over ten minutes was signed") + } +} + +// 1601/AC3: a body whose SHA-256 differs from the scan's writes no file and answers the mismatch. +func TestABodyOfAnotherHashWritesNoFile(t *testing.T) { + f := newFakeScanner(t, fixture(t, "scan.jpg")) + r, folder := scanRig(t, f, Settings{Crop: false}) + r.objects.corrupt = true + _, err := r.scan(t) + if err == nil || !strings.Contains(err.Error(), "hashes to") { + t.Fatalf("error = %v", err) + } + if e, _ := os.ReadDir(folder); len(e) != 0 { + t.Errorf("a scan of the wrong hash was filed: %v", e) + } +} + +// 1601/AC4: brother.scan.added is said before the file is written, once the scan is stored, and is under 4 KiB. +func TestTheEventIsSaidBeforeTheFileIsWritten(t *testing.T) { + f := newFakeScanner(t, fixture(t, "scan.jpg")) + r, folder := scanRig(t, f, Settings{Crop: false}) + var filedWhenSaid = -1 + emit := r.worker.Emit + r.worker.Emit = func(body map[string]any, id string) error { + e, _ := os.ReadDir(folder) + filedWhenSaid = len(e) + return emit(body, id) + } + if _, err := r.scan(t); err != nil { + t.Fatal(err) + } + if filedWhenSaid != 0 { + t.Errorf("%d files in the folder when the event was said, want 0", filedWhenSaid) + } + said := (*r.said)[0] + if !said.seen { + t.Error("the event was said before the scan was stored") + } + raw, _ := json.Marshal(said.body) + if len(raw) >= 4096 || said.body["path"] != nil { + t.Errorf("the event is %d bytes, path %v", len(raw), said.body["path"]) + } +} + +// 1601/AC5: no code of the device process writes a file outside its state directory: the only writes to the disk in the +// module's code are the filing of a scan (file.go), made by the tool side (scan.go). +func TestOnlyTheFilingWritesFiles(t *testing.T) { + writes := []string{"os.WriteFile", "os.Create(", "os.OpenFile", "os.MkdirAll", "os.Mkdir(", "os.Rename", "os.CreateTemp", "File(s.Settings.Folder"} + files, _ := filepath.Glob("*.go") + for _, name := range files { + if strings.HasSuffix(name, "_test.go") || name == "file.go" { + continue + } + raw, _ := os.ReadFile(name) + for _, w := range writes { + if strings.Contains(string(raw), w) && !(name == "scan.go" && w == "File(s.Settings.Folder") { + t.Errorf("%s writes with %s: only file.go does, and scan.go (the tool side) calls it", name, w) + } + } + } + raw, _ := os.ReadFile("deviceproc.go") + if strings.Contains(string(raw), "File(") { + t.Error("deviceproc.go files a scan") + } +} diff --git a/modules/brother/cmd/brother/service_test.go b/modules/brother/cmd/brother/service_test.go index 11eea84c..99f83dc8 100644 --- a/modules/brother/cmd/brother/service_test.go +++ b/modules/brother/cmd/brother/service_test.go @@ -362,38 +362,3 @@ func (b *flakyBus) Publish(id string, body any) error { } return b.fakeAskBus.Publish(id, body) } - -// 1536: a part fetch that fails once is asked again, and the scan is filed; one that fails twice is a failure -// naming the stored object, so it can be filed later. -func TestAFailedPartFetchIsAskedOnceMoreBeforeTheScanFails(t *testing.T) { - f := newFakeScanner(t, fixture(t, "scan.jpg")) - r, _ := scanRig(t, f, Settings{Crop: false}) - r.bus = nil - flaky := &flakyBus{fakeAskBus: &fakeAskBus{worker: r.worker, answers: r.answers}, fails: 1} - askBus = flaky - body, err := r.scan(t) - if err != nil { - t.Fatalf("one failed part fetch ended the scan: %v", err) - } - if _, err := os.Stat(body["path"].(string)); err != nil { - t.Errorf("the scan is not filed: %v", err) - } - - f2 := newFakeScanner(t, fixture(t, "scan.jpg")) - r2, folder := scanRig(t, f2, Settings{Crop: false}) - askBus = &flakyBus{fakeAskBus: &fakeAskBus{worker: r2.worker, answers: r2.answers}, fails: 2} - _, err = r2.scan(t) - if err == nil { - t.Fatal("two failed part fetches did not fail the scan") - } - key := "" - if got := *r2.said; len(got) == 1 { - key, _ = got[0].body["object"].(string) - } - if key == "" || !strings.Contains(err.Error(), key) { - t.Errorf("the failure %q does not name the stored object %q", err, key) - } - if entries, _ := os.ReadDir(folder); len(entries) != 0 { - t.Errorf("a scan whose parts failed was filed: %v", entries) - } -} diff --git a/modules/brother/cmd/brother/settings.go b/modules/brother/cmd/brother/settings.go index b8f9c616..8bd31e19 100644 --- a/modules/brother/cmd/brother/settings.go +++ b/modules/brother/cmd/brother/settings.go @@ -108,3 +108,26 @@ func (s Settings) MissingPrinter() error { return fmt.Errorf("config value %s is not set: the operator gives it with the config verb for the brother module on this node", strings.Join(unset, " and ")) } + +// bucketFile is the file the mesh writes from the s3-bucket binding (files/bucket.json): the bucket's name and no secret. +// BucketFile is where the tool side reads brother's bucket from: a file the mesh writes from the s3-bucket binding +// (files/bucket.json), which holds the bucket's name and no secret. +type bucketFile struct { + Bucket string `json:"bucket"` +} + +// ReadBucket is brother's bucket, as the mesh named it. +func ReadBucket(path string) (string, error) { + if path == "" { + return "", fmt.Errorf("the module was started without the bucket's name (BROTHER_BUCKET)") + } + raw, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("the bucket's name cannot be read: %w", err) + } + var f bucketFile + if err := json.Unmarshal(raw, &f); err != nil || f.Bucket == "" { + return "", fmt.Errorf("the bucket file %s names no bucket", path) + } + return f.Bucket, nil +} diff --git a/modules/brother/files/bucket.json b/modules/brother/files/bucket.json new file mode 100644 index 00000000..b72b2331 --- /dev/null +++ b/modules/brother/files/bucket.json @@ -0,0 +1,3 @@ +{ + "bucket": "${bound:s3-bucket:bucket}" +} diff --git a/modules/brother/files/fetch.json b/modules/brother/files/fetch.json new file mode 100644 index 00000000..abef425f --- /dev/null +++ b/modules/brother/files/fetch.json @@ -0,0 +1,3 @@ +{ + "mesh-range": "${machine:mesh-range}" +} diff --git a/modules/brother/go.mod b/modules/brother/go.mod index 4c4a0e28..fd383c67 100644 --- a/modules/brother/go.mod +++ b/modules/brother/go.mod @@ -3,7 +3,7 @@ module brother go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/brother/go.sum b/modules/brother/go.sum index 628b153b..eb649969 100644 --- a/modules/brother/go.sum +++ b/modules/brother/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef h1:xy3wuGWJpqMAXYdXolocYG6w9xnWqBltVUffoaziIKo= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f h1:7J/9jOfSfjfYpf8MOj96HwQRViQUkBNMb71RHP2mjn8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/brother/module.json b/modules/brother/module.json index 57559d56..df2bed16 100644 --- a/modules/brother/module.json +++ b/modules/brother/module.json @@ -2,7 +2,20 @@ "module": "brother", "version": "1", "requires": [ - "s3-bucket" + { + "name": "s3-bucket", + "shares": [ + { + "to": "seat:node-files", + "methods": [ + "put" + ] + } + ] + } + ], + "invokes": [ + "seat:node-files.upload" ], "binds": { "s3-bucket": "${dir:state}/store-binding.json" @@ -199,6 +212,21 @@ "mode": "0600", "template": "files/store.json" }, + { + "id": "bucket", + "type": "file", + "path": "${dir:state}/bucket.json", + "mode": "0644", + "template": "files/bucket.json" + }, + { + "id": "fetch", + "type": "file", + "path": "${dir:state}/fetch.json", + "owner": "brother-device", + "mode": "0600", + "template": "files/fetch.json" + }, { "id": "device", "type": "process", @@ -213,11 +241,13 @@ "BROTHER_SETTINGS": "${dir:state}/settings.json", "BROTHER_ADMIN_PASSWORD": "${dir:state}/admin-password.secret", "BROTHER_BROKER": "${dir:state}/broker-device.secret", - "BROTHER_STORE": "${dir:state}/store.json" + "BROTHER_STORE": "${dir:state}/store.json", + "BROTHER_FETCH": "${dir:state}/fetch.json" }, "restart-on": [ "settings", - "store" + "store", + "fetch" ], "bus": { "queues": [ @@ -228,6 +258,9 @@ ], "emits": [ "scan.added" + ], + "invokes": [ + "seat:document-converter.convert" ] }, "health": { @@ -267,7 +300,8 @@ "brother" ], "env": { - "BROTHER_SETTINGS": "${dir:state}/settings.json" + "BROTHER_SETTINGS": "${dir:state}/settings.json", + "BROTHER_BUCKET": "${dir:state}/bucket.json" } }, { diff --git a/modules/gitea/go.mod b/modules/gitea/go.mod index 9629492d..35087244 100644 --- a/modules/gitea/go.mod +++ b/modules/gitea/go.mod @@ -1,22 +1,22 @@ module gitea -go 1.25.0 +go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 github.com/nats-io/nats-server/v2 v2.11.17 - github.com/nats-io/nats.go v1.51.0 - golang.org/x/crypto v0.50.0 + github.com/nats-io/nats.go v1.54.0 + golang.org/x/crypto v0.57.0 ) require ( github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect github.com/google/go-tpm v0.9.8 // indirect - github.com/klauspost/compress v1.18.5 // indirect + github.com/klauspost/compress v1.20.0 // indirect github.com/minio/highwayhash v1.0.4 // indirect github.com/nats-io/jwt/v2 v2.8.1 // indirect - github.com/nats-io/nkeys v0.4.15 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect - golang.org/x/sys v0.43.0 // indirect + golang.org/x/sys v0.48.0 // indirect golang.org/x/time v0.15.0 // indirect ) diff --git a/modules/gitea/go.sum b/modules/gitea/go.sum index 85f38bd2..226d88cc 100644 --- a/modules/gitea/go.sum +++ b/modules/gitea/go.sum @@ -1,29 +1,27 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0 h1:EK84so3Y8JdFT3BHrzj9cPQQrDUPa/d+IdOGnerlmEI= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6 h1:Sv1Qzix+U1RpaUkw5IbB0IJEyZ+FqgzHmbZrCZkcWf0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 h1:Cdl5N56QPfmFBoLHrDfvvyc5Q+PUnTjf3fgYD7imcco= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= -github.com/nats-io/nats.go v1.51.0 h1:ByW84XTz6W03GSSsygsZcA+xgKK8vPGaa/FCAAEHnAI= -github.com/nats-io/nats.go v1.51.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= -github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= -github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= -golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/keycloak/go.mod b/modules/keycloak/go.mod index d8fa2ddb..28481201 100644 --- a/modules/keycloak/go.mod +++ b/modules/keycloak/go.mod @@ -1,5 +1,5 @@ module keycloak -go 1.25.0 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 diff --git a/modules/keycloak/go.sum b/modules/keycloak/go.sum index 1a769534..6162079d 100644 --- a/modules/keycloak/go.sum +++ b/modules/keycloak/go.sum @@ -1,4 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0 h1:EK84so3Y8JdFT3BHrzj9cPQQrDUPa/d+IdOGnerlmEI= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6 h1:Sv1Qzix+U1RpaUkw5IbB0IJEyZ+FqgzHmbZrCZkcWf0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 h1:Cdl5N56QPfmFBoLHrDfvvyc5Q+PUnTjf3fgYD7imcco= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/mailu/automx.ts b/modules/mailu/automx.ts new file mode 100644 index 00000000..1512f0b9 --- /dev/null +++ b/modules/mailu/automx.ts @@ -0,0 +1,87 @@ +// What the autoconfiguration module (automx2) holds, as stored: its provider, server, domain and server_domain +// rows, read from its SQLite file inside its container with the sqlite3 program the image carries, opened +// read-only. Nothing is written and nothing is interpreted: the rows are the evidence for what automx answers +// to mail clients (novox/hq issue 1630, ADR 0480), and the reason the answer may be empty. + +import type { Exec } from "./health.js"; + +export const AUTOMX_DATABASE = "/data/db.sqlite"; + +/** The tables read, each with its columns and its ordering. Fixed text: the tool takes no query from its caller. */ +export const AUTOMX_TABLES: Record = { + provider: { columns: ["id", "name", "short_name"], order: "id" }, + server: { columns: ["id", "prio", "name", "port", "type", "socket_type", "user_name", "authentication"], order: "id" }, + domain: { columns: ["id", "name", "provider_id", "ldapserver_id"], order: "id" }, + server_domain: { columns: ["server_id", "domain_id"], order: "domain_id, server_id" }, +}; + +/** + * The one statement that reads a table. The sqlite3 in the mailu-automx image does not know `-json` (it answers + * "sqlite3: Error: unknown option: -json", issue 1630), so each column is printed as `:`, which + * the oldest sqlite3 prints in its default list mode: no quoting, no separator or newline inside a cell, and a NULL + * (`null:`) apart from an empty text (`text:`). + */ +export function selectOf(table: string): string { + const t = AUTOMX_TABLES[table]; + return `SELECT ${t.columns.map((c) => `typeof(${c}) || ':' || hex(${c})`).join(", ")} FROM ${table} ORDER BY ${t.order}`; +} + +export type Row = Record; + +export type AutomxRows = { + database: string; + tables: Record; + /** servers whose name is empty: the client is told an empty host name for them */ + empty_names: { id: number; type: string }[]; + /** tables that could not be read, each with what sqlite3 or docker said on one line */ + unreadable: Record; +}; + +/** One cell as selectOf prints it: `:`; text is the bytes read as UTF-8, a number is read from its text. */ +function cellOf(cell: string): string | number | null { + const at = cell.indexOf(":"); + if (at < 0 || cell.length - at - 1 > 0 && !/^([0-9A-Fa-f]{2})+$/.test(cell.slice(at + 1))) throw new Error(`sqlite3 printed ${JSON.stringify(cell.slice(0, 40))}, not a cell`); + const type = cell.slice(0, at); + const bytes = Buffer.from(cell.slice(at + 1), "hex").toString("utf8"); + switch (type) { + case "null": return null; + case "integer": case "real": return Number(bytes); + default: return bytes; + } +} + +/** sqlite3 prints one line per row, cells apart by `|`, and nothing at all for a table without rows. */ +export function parseRows(table: string, output: string): Row[] { + const columns = AUTOMX_TABLES[table].columns; + const rows: Row[] = []; + for (const line of output.split("\n")) { + if (line.trim() === "") continue; + const cells = line.replace(/\r$/, "").split("|"); + if (cells.length !== columns.length) throw new Error(`sqlite3 printed ${cells.length} cells for ${columns.length} columns`); + rows.push(Object.fromEntries(columns.map((c, i) => [c, cellOf(cells[i])]))); + } + return rows; +} + +/** The command that reads one table: read-only, one fixed statement. */ +export function readCommand(table: string): string[] { + return ["sqlite3", "-readonly", AUTOMX_DATABASE, selectOf(table)]; +} + +/** Read every table; one that fails is said in `unreadable` and does not hide the others. */ +export async function readAutomx(exec: Exec, container: string): Promise { + const answer: AutomxRows = { database: AUTOMX_DATABASE, tables: {}, empty_names: [], unreadable: {} }; + for (const table of Object.keys(AUTOMX_TABLES)) { + try { + answer.tables[table] = parseRows(table, await exec(container, readCommand(table))); + } catch (e) { + const err = e as { stderr?: string; message?: string }; + const text = (err?.stderr ?? "").trim() || (err?.message ?? String(e)).trim(); + answer.unreadable[table] = text.split("\n").map((l) => l.trim()).filter(Boolean)[0] ?? "it failed and said nothing"; + } + } + for (const s of answer.tables.server ?? []) { + if (String(s.name ?? "").trim() === "") answer.empty_names.push({ id: Number(s.id), type: String(s.type) }); + } + return answer; +} diff --git a/modules/mailu/cmd/automx/testdata/captured/01-thunderbird-autoconfig-ok.body b/modules/mailu/cmd/automx/testdata/captured/01-thunderbird-autoconfig-ok.body new file mode 100644 index 00000000..72d5c376 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/01-thunderbird-autoconfig-ok.body @@ -0,0 +1 @@ +novox.benovox.benovox.be143STARTTLS%EMAILADDRESS%password-cleartextmail.novox.be587STARTTLS%EMAILADDRESS%password-cleartext \ No newline at end of file diff --git a/modules/mailu/cmd/automx/testdata/captured/01-thunderbird-autoconfig-ok.headers b/modules/mailu/cmd/automx/testdata/captured/01-thunderbird-autoconfig-ok.headers new file mode 100644 index 00000000..01eb3d79 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/01-thunderbird-autoconfig-ok.headers @@ -0,0 +1,3 @@ +Content-Length: 612 +Content-Type: application/xml +Server: Werkzeug/3.1.9 Python/3.9.5 diff --git a/modules/mailu/cmd/automx/testdata/captured/01-thunderbird-autoconfig-ok.request b/modules/mailu/cmd/automx/testdata/captured/01-thunderbird-autoconfig-ok.request new file mode 100644 index 00000000..98ae52c5 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/01-thunderbird-autoconfig-ok.request @@ -0,0 +1,3 @@ +GET https://autoconfig.novox.be/mail/config-v1.1.xml?emailaddress=jochen@novox.be +User-Agent: Thunderbird +Accept: */* diff --git a/modules/mailu/cmd/automx/testdata/captured/01-thunderbird-autoconfig-ok.status b/modules/mailu/cmd/automx/testdata/captured/01-thunderbird-autoconfig-ok.status new file mode 100644 index 00000000..992a7e84 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/01-thunderbird-autoconfig-ok.status @@ -0,0 +1 @@ +200 OK diff --git a/modules/mailu/cmd/automx/testdata/captured/02-thunderbird-on-automx-name.body b/modules/mailu/cmd/automx/testdata/captured/02-thunderbird-on-automx-name.body new file mode 100644 index 00000000..72d5c376 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/02-thunderbird-on-automx-name.body @@ -0,0 +1 @@ +novox.benovox.benovox.be143STARTTLS%EMAILADDRESS%password-cleartextmail.novox.be587STARTTLS%EMAILADDRESS%password-cleartext \ No newline at end of file diff --git a/modules/mailu/cmd/automx/testdata/captured/02-thunderbird-on-automx-name.headers b/modules/mailu/cmd/automx/testdata/captured/02-thunderbird-on-automx-name.headers new file mode 100644 index 00000000..01eb3d79 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/02-thunderbird-on-automx-name.headers @@ -0,0 +1,3 @@ +Content-Length: 612 +Content-Type: application/xml +Server: Werkzeug/3.1.9 Python/3.9.5 diff --git a/modules/mailu/cmd/automx/testdata/captured/02-thunderbird-on-automx-name.request b/modules/mailu/cmd/automx/testdata/captured/02-thunderbird-on-automx-name.request new file mode 100644 index 00000000..2a6405cf --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/02-thunderbird-on-automx-name.request @@ -0,0 +1,2 @@ +GET https://automx.novox.be/mail/config-v1.1.xml?emailaddress=jochen@novox.be +User-Agent: Thunderbird diff --git a/modules/mailu/cmd/automx/testdata/captured/02-thunderbird-on-automx-name.status b/modules/mailu/cmd/automx/testdata/captured/02-thunderbird-on-automx-name.status new file mode 100644 index 00000000..992a7e84 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/02-thunderbird-on-automx-name.status @@ -0,0 +1 @@ +200 OK diff --git a/modules/mailu/cmd/automx/testdata/captured/03-thunderbird-on-autodiscover-name.body b/modules/mailu/cmd/automx/testdata/captured/03-thunderbird-on-autodiscover-name.body new file mode 100644 index 00000000..72d5c376 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/03-thunderbird-on-autodiscover-name.body @@ -0,0 +1 @@ +novox.benovox.benovox.be143STARTTLS%EMAILADDRESS%password-cleartextmail.novox.be587STARTTLS%EMAILADDRESS%password-cleartext \ No newline at end of file diff --git a/modules/mailu/cmd/automx/testdata/captured/03-thunderbird-on-autodiscover-name.headers b/modules/mailu/cmd/automx/testdata/captured/03-thunderbird-on-autodiscover-name.headers new file mode 100644 index 00000000..01eb3d79 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/03-thunderbird-on-autodiscover-name.headers @@ -0,0 +1,3 @@ +Content-Length: 612 +Content-Type: application/xml +Server: Werkzeug/3.1.9 Python/3.9.5 diff --git a/modules/mailu/cmd/automx/testdata/captured/03-thunderbird-on-autodiscover-name.request b/modules/mailu/cmd/automx/testdata/captured/03-thunderbird-on-autodiscover-name.request new file mode 100644 index 00000000..5b33fcce --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/03-thunderbird-on-autodiscover-name.request @@ -0,0 +1,2 @@ +GET https://autodiscover.novox.be/mail/config-v1.1.xml?emailaddress=jochen@novox.be +User-Agent: Thunderbird diff --git a/modules/mailu/cmd/automx/testdata/captured/03-thunderbird-on-autodiscover-name.status b/modules/mailu/cmd/automx/testdata/captured/03-thunderbird-on-autodiscover-name.status new file mode 100644 index 00000000..992a7e84 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/03-thunderbird-on-autodiscover-name.status @@ -0,0 +1 @@ +200 OK diff --git a/modules/mailu/cmd/automx/testdata/captured/04-thunderbird-unknown-domain.body b/modules/mailu/cmd/automx/testdata/captured/04-thunderbird-unknown-domain.body new file mode 100644 index 00000000..e69de29b diff --git a/modules/mailu/cmd/automx/testdata/captured/04-thunderbird-unknown-domain.headers b/modules/mailu/cmd/automx/testdata/captured/04-thunderbird-unknown-domain.headers new file mode 100644 index 00000000..85abd033 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/04-thunderbird-unknown-domain.headers @@ -0,0 +1,2 @@ +Content-Type: text/html; charset=utf-8 +Server: Werkzeug/3.1.9 Python/3.9.5 diff --git a/modules/mailu/cmd/automx/testdata/captured/04-thunderbird-unknown-domain.request b/modules/mailu/cmd/automx/testdata/captured/04-thunderbird-unknown-domain.request new file mode 100644 index 00000000..d2459a88 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/04-thunderbird-unknown-domain.request @@ -0,0 +1,2 @@ +GET https://autoconfig.novox.be/mail/config-v1.1.xml?emailaddress=nobody@unknown.example +User-Agent: Thunderbird diff --git a/modules/mailu/cmd/automx/testdata/captured/04-thunderbird-unknown-domain.status b/modules/mailu/cmd/automx/testdata/captured/04-thunderbird-unknown-domain.status new file mode 100644 index 00000000..bd3152a3 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/04-thunderbird-unknown-domain.status @@ -0,0 +1 @@ +204 No Content diff --git a/modules/mailu/cmd/automx/testdata/captured/05-thunderbird-missing-emailaddress.body b/modules/mailu/cmd/automx/testdata/captured/05-thunderbird-missing-emailaddress.body new file mode 100644 index 00000000..7f0ad25f --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/05-thunderbird-missing-emailaddress.body @@ -0,0 +1 @@ +Missing request argument "emailaddress" \ No newline at end of file diff --git a/modules/mailu/cmd/automx/testdata/captured/05-thunderbird-missing-emailaddress.headers b/modules/mailu/cmd/automx/testdata/captured/05-thunderbird-missing-emailaddress.headers new file mode 100644 index 00000000..64102801 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/05-thunderbird-missing-emailaddress.headers @@ -0,0 +1,3 @@ +Content-Length: 39 +Content-Type: text/html; charset=utf-8 +Server: Werkzeug/3.1.9 Python/3.9.5 diff --git a/modules/mailu/cmd/automx/testdata/captured/05-thunderbird-missing-emailaddress.request b/modules/mailu/cmd/automx/testdata/captured/05-thunderbird-missing-emailaddress.request new file mode 100644 index 00000000..5d521f35 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/05-thunderbird-missing-emailaddress.request @@ -0,0 +1,2 @@ +GET https://autoconfig.novox.be/mail/config-v1.1.xml +User-Agent: Thunderbird diff --git a/modules/mailu/cmd/automx/testdata/captured/05-thunderbird-missing-emailaddress.status b/modules/mailu/cmd/automx/testdata/captured/05-thunderbird-missing-emailaddress.status new file mode 100644 index 00000000..59db10c4 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/05-thunderbird-missing-emailaddress.status @@ -0,0 +1 @@ +400 Bad Request diff --git a/modules/mailu/cmd/automx/testdata/captured/06-autoconfig-root.body b/modules/mailu/cmd/automx/testdata/captured/06-autoconfig-root.body new file mode 100644 index 00000000..8c25486a --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/06-autoconfig-root.body @@ -0,0 +1 @@ +Show Thunderbird-style XML configuration for abc@example.com. \ No newline at end of file diff --git a/modules/mailu/cmd/automx/testdata/captured/06-autoconfig-root.headers b/modules/mailu/cmd/automx/testdata/captured/06-autoconfig-root.headers new file mode 100644 index 00000000..5cd620c0 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/06-autoconfig-root.headers @@ -0,0 +1,3 @@ +Content-Length: 152 +Content-Type: text/html; charset=utf-8 +Server: Werkzeug/3.1.9 Python/3.9.5 diff --git a/modules/mailu/cmd/automx/testdata/captured/06-autoconfig-root.request b/modules/mailu/cmd/automx/testdata/captured/06-autoconfig-root.request new file mode 100644 index 00000000..4dc45b54 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/06-autoconfig-root.request @@ -0,0 +1,2 @@ +GET https://autoconfig.novox.be/ +User-Agent: Mozilla/5.0 diff --git a/modules/mailu/cmd/automx/testdata/captured/06-autoconfig-root.status b/modules/mailu/cmd/automx/testdata/captured/06-autoconfig-root.status new file mode 100644 index 00000000..992a7e84 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/06-autoconfig-root.status @@ -0,0 +1 @@ +200 OK diff --git a/modules/mailu/cmd/automx/testdata/captured/07-wellknown-autoconfig-path.body b/modules/mailu/cmd/automx/testdata/captured/07-wellknown-autoconfig-path.body new file mode 100644 index 00000000..affacdf5 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/07-wellknown-autoconfig-path.body @@ -0,0 +1,5 @@ + + +404 Not Found +

Not Found

+

The requested URL was not found on the server. If you entered the URL manually please check your spelling and try again.

diff --git a/modules/mailu/cmd/automx/testdata/captured/07-wellknown-autoconfig-path.headers b/modules/mailu/cmd/automx/testdata/captured/07-wellknown-autoconfig-path.headers new file mode 100644 index 00000000..11f1fdb6 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/07-wellknown-autoconfig-path.headers @@ -0,0 +1,3 @@ +Content-Length: 207 +Content-Type: text/html; charset=utf-8 +Server: Werkzeug/3.1.9 Python/3.9.5 diff --git a/modules/mailu/cmd/automx/testdata/captured/07-wellknown-autoconfig-path.request b/modules/mailu/cmd/automx/testdata/captured/07-wellknown-autoconfig-path.request new file mode 100644 index 00000000..d45d9935 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/07-wellknown-autoconfig-path.request @@ -0,0 +1 @@ +GET https://autoconfig.novox.be/.well-known/autoconfig/mail/config-v1.1.xml?emailaddress=jochen@novox.be diff --git a/modules/mailu/cmd/automx/testdata/captured/07-wellknown-autoconfig-path.status b/modules/mailu/cmd/automx/testdata/captured/07-wellknown-autoconfig-path.status new file mode 100644 index 00000000..44d986c4 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/07-wellknown-autoconfig-path.status @@ -0,0 +1 @@ +404 Not Found diff --git a/modules/mailu/cmd/automx/testdata/captured/08-outlook-post-known-domain.body b/modules/mailu/cmd/automx/testdata/captured/08-outlook-post-known-domain.body new file mode 100644 index 00000000..a38132d8 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/08-outlook-post-known-domain.body @@ -0,0 +1,5 @@ + + +500 Internal Server Error +

Internal Server Error

+

The server encountered an internal error and was unable to complete your request. Either the server is overloaded or there is an error in the application.

diff --git a/modules/mailu/cmd/automx/testdata/captured/08-outlook-post-known-domain.headers b/modules/mailu/cmd/automx/testdata/captured/08-outlook-post-known-domain.headers new file mode 100644 index 00000000..463e2827 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/08-outlook-post-known-domain.headers @@ -0,0 +1,3 @@ +Content-Length: 265 +Content-Type: text/html; charset=utf-8 +Server: Werkzeug/3.1.9 Python/3.9.5 diff --git a/modules/mailu/cmd/automx/testdata/captured/08-outlook-post-known-domain.request b/modules/mailu/cmd/automx/testdata/captured/08-outlook-post-known-domain.request new file mode 100644 index 00000000..258c92a6 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/08-outlook-post-known-domain.request @@ -0,0 +1,12 @@ +POST https://autodiscover.novox.be/autodiscover/autodiscover.xml +User-Agent: Microsoft Office/16.0 (Windows NT 10.0; Microsoft Outlook 16.0.14326; Pro) +Content-Type: text/xml +Accept: */* + + + + + jochen@novox.be + http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a + + diff --git a/modules/mailu/cmd/automx/testdata/captured/08-outlook-post-known-domain.status b/modules/mailu/cmd/automx/testdata/captured/08-outlook-post-known-domain.status new file mode 100644 index 00000000..e534a491 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/08-outlook-post-known-domain.status @@ -0,0 +1 @@ +500 Internal Server Error diff --git a/modules/mailu/cmd/automx/testdata/captured/09-outlook-post-unknown-domain.body b/modules/mailu/cmd/automx/testdata/captured/09-outlook-post-unknown-domain.body new file mode 100644 index 00000000..e69de29b diff --git a/modules/mailu/cmd/automx/testdata/captured/09-outlook-post-unknown-domain.headers b/modules/mailu/cmd/automx/testdata/captured/09-outlook-post-unknown-domain.headers new file mode 100644 index 00000000..85abd033 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/09-outlook-post-unknown-domain.headers @@ -0,0 +1,2 @@ +Content-Type: text/html; charset=utf-8 +Server: Werkzeug/3.1.9 Python/3.9.5 diff --git a/modules/mailu/cmd/automx/testdata/captured/09-outlook-post-unknown-domain.request b/modules/mailu/cmd/automx/testdata/captured/09-outlook-post-unknown-domain.request new file mode 100644 index 00000000..990a009c --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/09-outlook-post-unknown-domain.request @@ -0,0 +1,6 @@ +POST https://autodiscover.novox.be/autodiscover/autodiscover.xml +User-Agent: Microsoft Office/16.0 +Content-Type: text/xml + + +nobody@unknown.examplehttp://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a diff --git a/modules/mailu/cmd/automx/testdata/captured/09-outlook-post-unknown-domain.status b/modules/mailu/cmd/automx/testdata/captured/09-outlook-post-unknown-domain.status new file mode 100644 index 00000000..bd3152a3 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/09-outlook-post-unknown-domain.status @@ -0,0 +1 @@ +204 No Content diff --git a/modules/mailu/cmd/automx/testdata/captured/10-outlook-post-malformed-body.body b/modules/mailu/cmd/automx/testdata/captured/10-outlook-post-malformed-body.body new file mode 100644 index 00000000..a38132d8 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/10-outlook-post-malformed-body.body @@ -0,0 +1,5 @@ + + +500 Internal Server Error +

Internal Server Error

+

The server encountered an internal error and was unable to complete your request. Either the server is overloaded or there is an error in the application.

diff --git a/modules/mailu/cmd/automx/testdata/captured/10-outlook-post-malformed-body.headers b/modules/mailu/cmd/automx/testdata/captured/10-outlook-post-malformed-body.headers new file mode 100644 index 00000000..463e2827 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/10-outlook-post-malformed-body.headers @@ -0,0 +1,3 @@ +Content-Length: 265 +Content-Type: text/html; charset=utf-8 +Server: Werkzeug/3.1.9 Python/3.9.5 diff --git a/modules/mailu/cmd/automx/testdata/captured/10-outlook-post-malformed-body.request b/modules/mailu/cmd/automx/testdata/captured/10-outlook-post-malformed-body.request new file mode 100644 index 00000000..7467eeee --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/10-outlook-post-malformed-body.request @@ -0,0 +1,5 @@ +POST https://autodiscover.novox.be/autodiscover/autodiscover.xml +User-Agent: Microsoft Office/16.0 +Content-Type: text/xml + +this is not xml \ No newline at end of file diff --git a/modules/mailu/cmd/automx/testdata/captured/10-outlook-post-malformed-body.status b/modules/mailu/cmd/automx/testdata/captured/10-outlook-post-malformed-body.status new file mode 100644 index 00000000..e534a491 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/10-outlook-post-malformed-body.status @@ -0,0 +1 @@ +500 Internal Server Error diff --git a/modules/mailu/cmd/automx/testdata/captured/11-outlook-post-missing-emailaddress.body b/modules/mailu/cmd/automx/testdata/captured/11-outlook-post-missing-emailaddress.body new file mode 100644 index 00000000..c52afca0 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/11-outlook-post-missing-emailaddress.body @@ -0,0 +1 @@ +Missing request argument "EMailAddress" \ No newline at end of file diff --git a/modules/mailu/cmd/automx/testdata/captured/11-outlook-post-missing-emailaddress.headers b/modules/mailu/cmd/automx/testdata/captured/11-outlook-post-missing-emailaddress.headers new file mode 100644 index 00000000..64102801 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/11-outlook-post-missing-emailaddress.headers @@ -0,0 +1,3 @@ +Content-Length: 39 +Content-Type: text/html; charset=utf-8 +Server: Werkzeug/3.1.9 Python/3.9.5 diff --git a/modules/mailu/cmd/automx/testdata/captured/11-outlook-post-missing-emailaddress.request b/modules/mailu/cmd/automx/testdata/captured/11-outlook-post-missing-emailaddress.request new file mode 100644 index 00000000..abea016e --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/11-outlook-post-missing-emailaddress.request @@ -0,0 +1,6 @@ +POST https://autodiscover.novox.be/autodiscover/autodiscover.xml +User-Agent: Microsoft Office/16.0 +Content-Type: text/xml + + +http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a diff --git a/modules/mailu/cmd/automx/testdata/captured/11-outlook-post-missing-emailaddress.status b/modules/mailu/cmd/automx/testdata/captured/11-outlook-post-missing-emailaddress.status new file mode 100644 index 00000000..59db10c4 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/11-outlook-post-missing-emailaddress.status @@ -0,0 +1 @@ +400 Bad Request diff --git a/modules/mailu/cmd/automx/testdata/captured/12-outlook-get-not-allowed.body b/modules/mailu/cmd/automx/testdata/captured/12-outlook-get-not-allowed.body new file mode 100644 index 00000000..08dacbdc --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/12-outlook-get-not-allowed.body @@ -0,0 +1,5 @@ + + +405 Method Not Allowed +

Method Not Allowed

+

The method is not allowed for the requested URL.

diff --git a/modules/mailu/cmd/automx/testdata/captured/12-outlook-get-not-allowed.headers b/modules/mailu/cmd/automx/testdata/captured/12-outlook-get-not-allowed.headers new file mode 100644 index 00000000..84e74dc7 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/12-outlook-get-not-allowed.headers @@ -0,0 +1,4 @@ +Allow: POST, OPTIONS +Content-Length: 153 +Content-Type: text/html; charset=utf-8 +Server: Werkzeug/3.1.9 Python/3.9.5 diff --git a/modules/mailu/cmd/automx/testdata/captured/12-outlook-get-not-allowed.request b/modules/mailu/cmd/automx/testdata/captured/12-outlook-get-not-allowed.request new file mode 100644 index 00000000..11530419 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/12-outlook-get-not-allowed.request @@ -0,0 +1,2 @@ +GET https://autodiscover.novox.be/autodiscover/autodiscover.xml +User-Agent: Microsoft Office/16.0 diff --git a/modules/mailu/cmd/automx/testdata/captured/12-outlook-get-not-allowed.status b/modules/mailu/cmd/automx/testdata/captured/12-outlook-get-not-allowed.status new file mode 100644 index 00000000..ddf33a47 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/12-outlook-get-not-allowed.status @@ -0,0 +1 @@ +405 Method Not Allowed diff --git a/modules/mailu/cmd/automx/testdata/captured/13-outlook-post-capitalised-path.body b/modules/mailu/cmd/automx/testdata/captured/13-outlook-post-capitalised-path.body new file mode 100644 index 00000000..affacdf5 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/13-outlook-post-capitalised-path.body @@ -0,0 +1,5 @@ + + +404 Not Found +

Not Found

+

The requested URL was not found on the server. If you entered the URL manually please check your spelling and try again.

diff --git a/modules/mailu/cmd/automx/testdata/captured/13-outlook-post-capitalised-path.headers b/modules/mailu/cmd/automx/testdata/captured/13-outlook-post-capitalised-path.headers new file mode 100644 index 00000000..11f1fdb6 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/13-outlook-post-capitalised-path.headers @@ -0,0 +1,3 @@ +Content-Length: 207 +Content-Type: text/html; charset=utf-8 +Server: Werkzeug/3.1.9 Python/3.9.5 diff --git a/modules/mailu/cmd/automx/testdata/captured/13-outlook-post-capitalised-path.request b/modules/mailu/cmd/automx/testdata/captured/13-outlook-post-capitalised-path.request new file mode 100644 index 00000000..ff129abe --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/13-outlook-post-capitalised-path.request @@ -0,0 +1,6 @@ +POST https://autodiscover.novox.be/Autodiscover/Autodiscover.xml +User-Agent: Microsoft Office/16.0 +Content-Type: text/xml + + +jochen@novox.behttp://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a diff --git a/modules/mailu/cmd/automx/testdata/captured/13-outlook-post-capitalised-path.status b/modules/mailu/cmd/automx/testdata/captured/13-outlook-post-capitalised-path.status new file mode 100644 index 00000000..44d986c4 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/13-outlook-post-capitalised-path.status @@ -0,0 +1 @@ +404 Not Found diff --git a/modules/mailu/cmd/automx/testdata/captured/14-mobileconfig-no-slash.body b/modules/mailu/cmd/automx/testdata/captured/14-mobileconfig-no-slash.body new file mode 100644 index 00000000..bc384fc9 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/14-mobileconfig-no-slash.body @@ -0,0 +1,5 @@ + + +Redirecting... +

Redirecting...

+

You should be redirected automatically to the target URL: https://automx.novox.be/mobileconfig/?emailaddress=jochen@novox.be. If not, click the link. diff --git a/modules/mailu/cmd/automx/testdata/captured/14-mobileconfig-no-slash.headers b/modules/mailu/cmd/automx/testdata/captured/14-mobileconfig-no-slash.headers new file mode 100644 index 00000000..75822817 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/14-mobileconfig-no-slash.headers @@ -0,0 +1,4 @@ +Content-Length: 319 +Content-Type: text/html; charset=utf-8 +Location: https://automx.novox.be/mobileconfig/?emailaddress=jochen@novox.be +Server: Werkzeug/3.1.9 Python/3.9.5 diff --git a/modules/mailu/cmd/automx/testdata/captured/14-mobileconfig-no-slash.request b/modules/mailu/cmd/automx/testdata/captured/14-mobileconfig-no-slash.request new file mode 100644 index 00000000..ee86c5ba --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/14-mobileconfig-no-slash.request @@ -0,0 +1,3 @@ +GET https://automx.novox.be/mobileconfig?emailaddress=jochen@novox.be +User-Agent: Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) Safari +Accept: */* diff --git a/modules/mailu/cmd/automx/testdata/captured/14-mobileconfig-no-slash.status b/modules/mailu/cmd/automx/testdata/captured/14-mobileconfig-no-slash.status new file mode 100644 index 00000000..8e8ce5e4 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/14-mobileconfig-no-slash.status @@ -0,0 +1 @@ +308 Permanent Redirect diff --git a/modules/mailu/cmd/automx/testdata/captured/15-mobileconfig-known-domain.body b/modules/mailu/cmd/automx/testdata/captured/15-mobileconfig-known-domain.body new file mode 100644 index 00000000..875543c0 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/15-mobileconfig-known-domain.body @@ -0,0 +1,5 @@ + + +400 Bad Request +

Bad Request

+

The browser (or proxy) sent a request that this server could not understand.

diff --git a/modules/mailu/cmd/automx/testdata/captured/15-mobileconfig-known-domain.headers b/modules/mailu/cmd/automx/testdata/captured/15-mobileconfig-known-domain.headers new file mode 100644 index 00000000..184e49a4 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/15-mobileconfig-known-domain.headers @@ -0,0 +1,3 @@ +Content-Length: 167 +Content-Type: text/html; charset=utf-8 +Server: Werkzeug/3.1.9 Python/3.9.5 diff --git a/modules/mailu/cmd/automx/testdata/captured/15-mobileconfig-known-domain.request b/modules/mailu/cmd/automx/testdata/captured/15-mobileconfig-known-domain.request new file mode 100644 index 00000000..47197a20 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/15-mobileconfig-known-domain.request @@ -0,0 +1,3 @@ +GET https://automx.novox.be/mobileconfig/?emailaddress=jochen@novox.be +User-Agent: Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) Safari +Accept: */* diff --git a/modules/mailu/cmd/automx/testdata/captured/15-mobileconfig-known-domain.status b/modules/mailu/cmd/automx/testdata/captured/15-mobileconfig-known-domain.status new file mode 100644 index 00000000..59db10c4 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/15-mobileconfig-known-domain.status @@ -0,0 +1 @@ +400 Bad Request diff --git a/modules/mailu/cmd/automx/testdata/captured/16-mobileconfig-unknown-domain.body b/modules/mailu/cmd/automx/testdata/captured/16-mobileconfig-unknown-domain.body new file mode 100644 index 00000000..e69de29b diff --git a/modules/mailu/cmd/automx/testdata/captured/16-mobileconfig-unknown-domain.headers b/modules/mailu/cmd/automx/testdata/captured/16-mobileconfig-unknown-domain.headers new file mode 100644 index 00000000..85abd033 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/16-mobileconfig-unknown-domain.headers @@ -0,0 +1,2 @@ +Content-Type: text/html; charset=utf-8 +Server: Werkzeug/3.1.9 Python/3.9.5 diff --git a/modules/mailu/cmd/automx/testdata/captured/16-mobileconfig-unknown-domain.request b/modules/mailu/cmd/automx/testdata/captured/16-mobileconfig-unknown-domain.request new file mode 100644 index 00000000..b38acb38 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/16-mobileconfig-unknown-domain.request @@ -0,0 +1,2 @@ +GET https://automx.novox.be/mobileconfig/?emailaddress=nobody@unknown.example +User-Agent: Mozilla/5.0 (iPhone) diff --git a/modules/mailu/cmd/automx/testdata/captured/16-mobileconfig-unknown-domain.status b/modules/mailu/cmd/automx/testdata/captured/16-mobileconfig-unknown-domain.status new file mode 100644 index 00000000..bd3152a3 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/16-mobileconfig-unknown-domain.status @@ -0,0 +1 @@ +204 No Content diff --git a/modules/mailu/cmd/automx/testdata/captured/17-mobileconfig-missing-emailaddress.body b/modules/mailu/cmd/automx/testdata/captured/17-mobileconfig-missing-emailaddress.body new file mode 100644 index 00000000..7f0ad25f --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/17-mobileconfig-missing-emailaddress.body @@ -0,0 +1 @@ +Missing request argument "emailaddress" \ No newline at end of file diff --git a/modules/mailu/cmd/automx/testdata/captured/17-mobileconfig-missing-emailaddress.headers b/modules/mailu/cmd/automx/testdata/captured/17-mobileconfig-missing-emailaddress.headers new file mode 100644 index 00000000..64102801 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/17-mobileconfig-missing-emailaddress.headers @@ -0,0 +1,3 @@ +Content-Length: 39 +Content-Type: text/html; charset=utf-8 +Server: Werkzeug/3.1.9 Python/3.9.5 diff --git a/modules/mailu/cmd/automx/testdata/captured/17-mobileconfig-missing-emailaddress.request b/modules/mailu/cmd/automx/testdata/captured/17-mobileconfig-missing-emailaddress.request new file mode 100644 index 00000000..a7f84362 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/17-mobileconfig-missing-emailaddress.request @@ -0,0 +1,2 @@ +GET https://automx.novox.be/mobileconfig/ +User-Agent: Mozilla/5.0 (iPhone) diff --git a/modules/mailu/cmd/automx/testdata/captured/17-mobileconfig-missing-emailaddress.status b/modules/mailu/cmd/automx/testdata/captured/17-mobileconfig-missing-emailaddress.status new file mode 100644 index 00000000..59db10c4 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/17-mobileconfig-missing-emailaddress.status @@ -0,0 +1 @@ +400 Bad Request diff --git a/modules/mailu/cmd/automx/testdata/captured/CAPTURED b/modules/mailu/cmd/automx/testdata/captured/CAPTURED new file mode 100644 index 00000000..88967524 --- /dev/null +++ b/modules/mailu/cmd/automx/testdata/captured/CAPTURED @@ -0,0 +1,30 @@ +What mailu's autoconfiguration module (automx2 2021.6, container mailu-automx on novox, Werkzeug/3.1.9 Python/3.9.5) +answers to the requests mail clients make (R4d step AMX-1, issue 1630, ADR 0480). + +Captured 2026-10-21 between 13:58:25 and 13:58:52 (CEST; the Date headers said 11:58 GMT) with the mesh tool +shanks/netcheck.netcheck_request, called from shanks, through the names clients use (autoconfig.novox.be, +autodiscover.novox.be, automx.novox.be: HTTPS, the mesh's route), no ssh. A redirect is reported, never followed. +Mail domain served: novox.be. Mailbox used: jochen@novox.be; unknown domain: nobody@unknown.example. + +Each capture is four files with the same stem: + .request method and URL, then the request headers sent, then (after a blank line) the body sent + .status the status line + .headers the response headers, sorted by name, without Date (it changes on every request) + .body the response body, byte for byte (Content-Length in .headers equals its size) +The files were written from the tool's answers by a script (not by retyping each by hand); the tool returns JSON, so +a later capture that wants exact bytes needs a tool that writes the answer to a file. + +What the answers show (read, not guessed): + - GET /mail/config-v1.1.xml?emailaddress=... answers 200 application/xml on all three names; same body on each. + - The incoming (IMAP) server has an EMPTY hostname (), port 143, STARTTLS. The outgoing server is + mail.novox.be:587 STARTTLS. Cause (from reading the module, not from the database): automx/files/add-domains:35 + reads AMX_IMAP_SERVER, files/mailu.env:58 sets AMX_IMAP_ADDRESS, so the IMAP name was never written. + A client given this answer cannot find its IMAP server. + - POST /autodiscover/autodiscover.xml with a valid request for the served domain answers 500 (the Outlook + answer for a real mailbox does not work today). Unknown domain: 204 no body. Malformed body: 500. + No EMailAddress element: 400 "Missing request argument "EMailAddress"". GET: 405 with Allow: POST, OPTIONS. + The path is case sensitive (/Autodiscover/Autodiscover.xml is 404). + - GET /mobileconfig?emailaddress=... redirects (308) to /mobileconfig/?emailaddress=...; for the served domain that + answers 400 with no profile; unknown domain 204; no emailaddress 400. + - GET / on autoconfig answers 200 with a small HTML page linking to an example address. + - Not captured: the database rows (no mesh tool reads /data/db.sqlite; issue 1630 AC4). diff --git a/modules/mailu/module.json b/modules/mailu/module.json index 568be145..98c3cb23 100644 --- a/modules/mailu/module.json +++ b/modules/mailu/module.json @@ -71,7 +71,8 @@ "mailu_list_domains", "mailu_read_mail", "mailu_search_mail", - "mailu_health" + "mailu_health", + "automx_rows" ], "emits": [ "user.created", diff --git a/modules/mailu/test/automx-rows.test.ts b/modules/mailu/test/automx-rows.test.ts new file mode 100644 index 00000000..8ef35222 --- /dev/null +++ b/modules/mailu/test/automx-rows.test.ts @@ -0,0 +1,103 @@ +// What automx_rows reads (novox/hq issue 1630): the rows of automx2's database as stored, through a read-only +// sqlite3 inside its container. The database here is a real SQLite file made by the test with the tables +// automx/files/setup-db creates and the rows add-domains writes — with the IMAP server's name empty, the case +// the live instance is in. The fake container runs each command's statement against that database and prints +// it the way the image's sqlite3 does in list mode (one line per row, cells apart by "|", nothing for no rows); that sqlite3 +// refuses `-json` ("sqlite3: Error: unknown option: -json", captured from novox/mailu.automx_rows, issue 1630). Run against the compiled module. + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { DatabaseSync } from "node:sqlite"; + +import { readAutomx, readCommand, selectOf, parseRows, AUTOMX_DATABASE } from "../dist/automx.js"; + +function database(imapName: string): DatabaseSync { + const db = new DatabaseSync(":memory:"); + db.exec(` + CREATE TABLE provider(id INT PRIMARY KEY NOT NULL, name TEXT NOT NULL, short_name TEXT NOT NULL); + CREATE TABLE server(id INT PRIMARY KEY NOT NULL, prio INT NOT NULL DEFAULT 10, name TEXT NOT NULL, port INT NOT NULL, + type TEXT NOT NULL, socket_type TEXT NOT NULL, user_name TEXT NOT NULL, authentication TEXT NOT NULL); + CREATE TABLE domain(id INT PRIMARY KEY NOT NULL, name TEXT NOT NULL, provider_id INT NOT NULL, ldapserver_id INT NULL); + CREATE TABLE server_domain(server_id INT NOT NULL, domain_id INT NOT NULL); + INSERT INTO provider VALUES (1, 'example.org', 'example.org'); + INSERT INTO server(id, port, type, name, socket_type, user_name, authentication) + VALUES (2, 143, 'imap', '${imapName}', 'STARTTLS', '%EMAILADDRESS%', 'password-cleartext'); + INSERT INTO server(id, port, type, name, socket_type, user_name, authentication) + VALUES (3, 587, 'smtp', 'mail.example.org', 'STARTTLS', '%EMAILADDRESS%', 'password-cleartext'); + INSERT INTO domain VALUES (5, 'example.org', 1, NULL); + INSERT INTO server_domain VALUES (2, 5), (3, 5); + `); + return db; +} + +function container(db: DatabaseSync, asked: string[][] = []) { + return async (_name: string, command: string[]) => { + asked.push(command); + if (command.includes("-json")) throw Object.assign(new Error("Command failed: docker exec"), { stderr: "sqlite3: Error: unknown option: -json\n" }); + const rows = db.prepare(command[command.length - 1]).all() as unknown as Record[]; + return rows.length === 0 ? "" : rows.map((r) => Object.values(r).join("|")).join("\n") + "\n"; + }; +} + +test("the rows come back as stored, and an empty server name is said", async () => { + const answer = await readAutomx(container(database("")), "mailu-automx"); + assert.equal(answer.database, AUTOMX_DATABASE); + assert.deepEqual(answer.unreadable, {}); + assert.deepEqual(answer.tables.provider, [{ id: 1, name: "example.org", short_name: "example.org" }]); + assert.deepEqual(answer.tables.domain, [{ id: 5, name: "example.org", provider_id: 1, ldapserver_id: null }]); + assert.deepEqual(answer.tables.server_domain, [{ server_id: 2, domain_id: 5 }, { server_id: 3, domain_id: 5 }]); + assert.deepEqual( + answer.tables.server.map((s) => [s.id, s.type, s.name, s.port]), + [[2, "imap", "", 143], [3, "smtp", "mail.example.org", 587]], + ); + assert.deepEqual(answer.empty_names, [{ id: 2, type: "imap" }]); +}); + +test("a named IMAP server is not reported empty", async () => { + const answer = await readAutomx(container(database("mail.example.org")), "mailu-automx"); + assert.deepEqual(answer.empty_names, []); +}); + +test("every command opens the file read-only and runs one fixed SELECT", async () => { + const asked: string[][] = []; + await readAutomx(container(database("x"), asked), "mailu-automx"); + assert.equal(asked.length, 4); + for (const c of asked) { + assert.deepEqual(c.slice(0, 3), ["sqlite3", "-readonly", AUTOMX_DATABASE]); + assert.match(c[3], /^SELECT /); + assert.equal(c.length, 4); + assert.ok(!c.includes("-json")); + } + assert.deepEqual(readCommand("provider"), ["sqlite3", "-readonly", AUTOMX_DATABASE, selectOf("provider")]); +}); + +test("a table that cannot be read is said and does not hide the others", async () => { + const db = database("x"); + const exec = async (name: string, command: string[]) => { + if (command[3].includes("FROM domain ")) throw Object.assign(new Error("Command failed: docker exec"), { stderr: "Error: unable to open database file\nmore" }); + return container(db)(name, command); + }; + const answer = await readAutomx(exec, "mailu-automx"); + assert.deepEqual(answer.unreadable, { domain: "Error: unable to open database file" }); + assert.equal(answer.tables.provider.length, 1); + assert.equal(answer.tables.domain, undefined); +}); + +test("a NULL, an empty text, a number and a text holding a bar or a newline are read apart", () => { + const hex = (t: string) => Buffer.from(t).toString("hex").toUpperCase(); + const line = [`integer:${hex("5")}`, `text:${hex("a|b\nc")}`, "text:", "null:"].join("|"); + const rows = parseRows("domain", line + "\n"); + assert.deepEqual(rows, [{ id: 5, name: "a|b\nc", provider_id: "", ldapserver_id: null }]); +}); + +test("output that is not cells is an unreadable table, not a guess", async () => { + const exec = async () => "Error: something else\n"; + const answer = await readAutomx(exec, "mailu-automx"); + assert.equal(Object.keys(answer.unreadable).length, 4); +}); + +test("the image's refusal of -json is said as the table's reason", async () => { + const exec = async (_n: string, command: string[]) => container(database("x"))("", ["sqlite3", "-json", ...command.slice(1)]); + const answer = await readAutomx(exec, "mailu-automx"); + assert.deepEqual(answer.unreadable.domain, "sqlite3: Error: unknown option: -json"); +}); diff --git a/modules/mailu/test/mail-flows.test.ts b/modules/mailu/test/mail-flows.test.ts index ddea9791..3950148c 100644 --- a/modules/mailu/test/mail-flows.test.ts +++ b/modules/mailu/test/mail-flows.test.ts @@ -135,6 +135,7 @@ test("the health tool is served without the admin API, on the smtp container it await import("../dist/tools/index.js"); const mailu = collectTools({ MESH_MAILU_SMTP_CONTAINER: "mailu-smtp" }).find((m) => m.module === "mailu"); const names = (mailu?.tools ?? []).map((t) => t.name); - assert.deepEqual(names, ["mailu_health"]); + assert.deepEqual(names, ["mailu_health", "automx_rows"]); assert.match(mailu!.tools[0].description, /inside mailu-smtp/); + assert.match(mailu!.tools[1].description, /read-only, inside mailu-automx/); }); diff --git a/modules/mailu/tools/index.ts b/modules/mailu/tools/index.ts index 6b8442e9..c6624f7e 100644 --- a/modules/mailu/tools/index.ts +++ b/modules/mailu/tools/index.ts @@ -4,6 +4,7 @@ import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools"; import { MailuClient } from "../client.js"; +import { readAutomx } from "../automx.js"; import { dockerExec, mailFlows, DEFERRED_BOUND_SECONDS, PUBLIC_NAME, SPREAD_DOMAINS } from "../health.js"; /** @@ -26,6 +27,23 @@ export function getHealthTool(env: NodeJS.ProcessEnv): ToolDefinition { }; } +/** + * What automx holds, as stored (novox/hq issue 1630): the rows behind the answers it gives mail clients. It needs + * no admin API, only the automx container, so like the health tool it is served without the API. + */ +export function getAutomxTool(env: NodeJS.ProcessEnv): ToolDefinition { + const container = env.MESH_MAILU_AUTOMX_CONTAINER || "mailu-automx"; + return { + name: "automx_rows", + description: + `Read the autoconfiguration module's database, as stored, read-only, inside ${container}: its provider, ` + + "server, domain and server_domain rows, and the servers whose name is empty (a mail client is then told " + + "an empty host name). Writes nothing.", + input: {}, + run: async () => readAutomx(dockerExec(), container), + }; +} + export function getMailuTools(mailu: MailuClient): ToolDefinition[] { return [ { @@ -173,5 +191,5 @@ registerModuleTools("mailu", (env) => { } catch { tools = []; } - return [...tools, getHealthTool(env)]; + return [...tools, getHealthTool(env), getAutomxTool(env)]; }); diff --git a/modules/mailu/tsconfig.json b/modules/mailu/tsconfig.json index a55a92ee..ad97ec61 100644 --- a/modules/mailu/tsconfig.json +++ b/modules/mailu/tsconfig.json @@ -8,5 +8,5 @@ "skipLibCheck": true, "noEmit": true }, - "include": ["client.ts", "health.ts", "index.ts", "tools/index.ts"] + "include": ["client.ts", "health.ts", "automx.ts", "index.ts", "tools/index.ts"] } diff --git a/modules/mesh-delivery-gate/.gitignore b/modules/mesh-delivery-gate/.gitignore new file mode 100644 index 00000000..03570e7b --- /dev/null +++ b/modules/mesh-delivery-gate/.gitignore @@ -0,0 +1 @@ +/cmd/mesh-delivery-gate/mesh-delivery-gate diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go index 95550c73..627ffabe 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go @@ -366,6 +366,9 @@ func (h *Holder) pullUpdated(p PullEvent) (isNew bool) { d.BaseSHA = p.BaseSHA h.keep(d) } + // A delivery made from its verdict before its head was heard (issue 1726) holds none of what an announcement + // carries: filled from this one, so its base is known and the operator is asked. + h.fillFromAnnouncement(d, p) // The review record, corrected in the description without a push, is read again (ADR 0421 §5). if by, at := ReviewIn(p.Body); p.Body != "" && (by != d.ReviewedBy || at != d.ReviewedAt) { d.ReviewedBy, d.ReviewedAt = by, at @@ -398,18 +401,7 @@ func (h *Holder) pullUpdated(p PullEvent) (isNew bool) { h.leaveGroup(old) } } - replaced := "" - for _, old := range h.deliveries { - if old.Repository == p.repository() && old.Number == p.Number && !sameCommit(old.Commit, p.HeadSHA) && - !old.State.Final() && old.MergedAs == "" { - old.NewerHead = p.HeadSHA - h.settleAndKeep(old) - if old.Group != "" { - h.leaveGroup(old) - replaced = old.ID - } - } - } + replaced := h.supersedeOlder(p) d, t, err := h.freshAnnouncement(p, now, startedOver) if err != nil { h.refuse(err) @@ -429,6 +421,49 @@ func (h *Holder) pullUpdated(p PullEvent) (isNew bool) { return isNew } +// supersedeOlder ends the older heads of p's pull request: each is told its newer head, and one in a group leaves it. It +// returns the id of the one that left a group, whose place the new head takes (takePlaceOf). Under the lock. +func (h *Holder) supersedeOlder(p PullEvent) (replaced string) { + for _, old := range h.deliveries { + if old.Repository == p.repository() && old.Number == p.Number && !sameCommit(old.Commit, p.HeadSHA) && + !old.State.Final() && old.MergedAs == "" { + old.NewerHead = p.HeadSHA + h.settleAndKeep(old) + if old.Group != "" { + h.leaveGroup(old) + replaced = old.ID + } + } + } + return replaced +} + +// fillFromAnnouncement fills a delivery made from its check's verdict before its pull request's head was heard (issue +// 1726: no base, branch, title, URLs or review record) from an announcement or an edit of that pull request, and then +// does what a first announcement does: the older heads of the pull request are superseded, and the delivery takes +// their place in a group or joins the group its branch makes. Nothing for a delivery that holds its base. Under the lock. +func (h *Holder) fillFromAnnouncement(d *Delivery, p PullEvent) { + if d.Base != "" || p.Base == "" || d.State.Final() || d.MergedAs != "" { + return + } + d.Title, d.Base, d.Branch, d.HTMLURL, d.CloneURL = p.Title, p.Base, p.Head, p.HTMLURL, p.CloneURL + d.Paths, d.PathsTruncated, d.Removed = p.Paths, p.PathsTruncated, p.Removed + d.ModuleDirs, d.ModuleDirsSaid = p.ModuleDirs, p.ModuleDirsSaid + d.Issues, d.After = IssuesIn(p.Title+"\n"+p.Body), AfterIn(p.Body) + d.ReviewedBy, d.ReviewedAt = ReviewIn(p.Body) + if p.BaseSHA != "" { + d.BaseSHA = p.BaseSHA + } + d.Number = p.Number + h.Logf("[mesh-delivery] %s: its pull request's announcement filled its base (%s), branch and title", d.ID, p.Base) + h.keep(d) + replaced := h.supersedeOlder(p) + if d.Group == "" && !h.takePlaceOf(replaced, d) { + h.joinGroup(d) + } + h.Poke() +} + // PullEdited takes a pull request's edit (novox/hq issue 1079): its description is read again for the review record, the // `after:` lines and the issues it names, exactly as an announcement reads them. An Approve ask still unanswered whose words // the new record changes is withdrawn and asked again with it read, without a word on the pull request: nothing was @@ -458,6 +493,9 @@ func (h *Holder) PullEdited(p PullEvent) { h.pullUpdated(p) return } + if d != nil && d.Number == p.Number { + h.fillFromAnnouncement(d, p) // a delivery made from its verdict, its head never heard (issue 1726) + } if p.Body == "" { return } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1726_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1726_test.go new file mode 100644 index 00000000..9d4ce895 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1726_test.go @@ -0,0 +1,126 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// novox/hq issue 1726: a delivery made from a check's verdict that arrived before its pull request's head was heard +// (transition "its verdict arrived before its head was heard", as mesh-controller#419 at a47476252772 was) holds no base, +// branch, title, URLs or review record. The ready pass skipped it in silence, so no Approve was ever asked, and the +// announcement of the same head, heard later, filled none of it. + +const verdictFirstBody = "Fixes a bug.\n\nReviewed-by: reviewer@shanks at " + readyHead + "\n" + +// verdictFirst is the live shape: the verdict heard, the head not, and the delivery ready. +func verdictFirst(t *testing.T) (*world, string) { + w := newWorld(t) + w.op.auto = false + id := IDOf("novox/app", readyHead) + w.h.Checked(judgedAt("novox/app", 419, readyHead, "pass", "t1t1t1t1t1t1")) + w.settleAll() + d := w.delivery(id) + if d.State != Ready || d.Base != "" || d.Branch != "" || d.Title != "" || d.HTMLURL != "" { + t.Fatalf("not the live shape: %s base %q branch %q title %q", d.State, d.Base, d.Branch, d.Title) + } + return w, id +} + +func TestADeliveryWithoutItsHeadSaysWhatItWaitsForAndAsksNoOne(t *testing.T) { + w, id := verdictFirst(t) + if n := len(w.op.open()); n != 0 { + t.Fatalf("an ask was made for a delivery with no base: %d", n) + } + d := w.delivery(id) + if d.Ready == nil || !strings.Contains(d.Ready.Waits, "announcement") { + t.Fatalf("it does not say what it waits for: %+v", d.Ready) + } + shown, err := w.h.Show(id) + if err != nil { + t.Fatal(err) + } + if waits, _ := shown.(map[string]any)["waits"].(string); !strings.Contains(waits, "announcement") { + t.Fatalf("show waits %q", waits) + } +} + +func TestAnAnnouncementFillsADeliveryMadeFromItsVerdictAndTheOperatorIsAsked(t *testing.T) { + w, id := verdictFirst(t) + p := pr("novox/app", 419, readyHead, "fix/x", "src/a.go") + p.Title, p.Body, p.CloneURL = "Fix x", verdictFirstBody, "https://forge.invalid/novox/app.git" + w.h.PullUpdated(p) + w.settleAll() + d := w.delivery(id) + if d.Base != "main" || d.Branch != "fix/x" || d.Title != "Fix x" || d.HTMLURL == "" || d.CloneURL == "" || + d.ReviewedBy != "reviewer@shanks" || len(d.Paths) != 1 { + t.Fatalf("the announcement did not fill the delivery: %+v", d) + } + if n := len(w.op.open()); n != 1 { + t.Fatalf("the operator was asked %d times; wanted once", n) + } + if d.Ready.Waits != "" && !strings.Contains(d.Ready.Waits, "Approve") { + t.Fatalf("still waits: %q", d.Ready.Waits) + } +} + +func TestADeliveryWithoutItsHeadPastItsBoundIsStalled(t *testing.T) { + w, id := verdictFirst(t) + if s := w.h.Stalled(); len(s) != 0 { + t.Fatalf("stalled before its bound: %+v", s) + } + w.later(headHeardBound + time.Minute) + w.settleAll() + var found bool + for _, l := range w.h.Stalled() { + if l.ID == id && strings.Contains(l.Says, "announcement") { + found = true + } + } + if !found { + t.Fatalf("not stalled past its bound: %+v", w.h.Stalled()) + } +} + +// A head asked, then a push whose verdict is heard before its head: the announcement of the new head supersedes the old +// one exactly as a first announcement does, so one Approve stands, for the new head. +func TestAnAnnouncementFillingAVerdictFirstHeadSupersedesTheOlderHeadAndItsAsk(t *testing.T) { + w := newWorld(t) + w.op.auto = false + a, b := IDOf("novox/app", readyHead), IDOf("novox/app", pushed) + w.h.PullUpdated(pr("novox/app", 419, readyHead, "fix/x", "src/a.go")) + w.h.Checked(judgedAt("novox/app", 419, readyHead, "pass", "t1t1t1t1t1t1")) + w.settleAll() + if n := len(w.op.open()); n != 1 { + t.Fatalf("head A: %d asks open; wanted 1", n) + } + w.h.Checked(judgedAt("novox/app", 419, pushed, "pass", "t1t1t1t1t1t1")) + w.settleAll() + w.h.PullUpdated(pr("novox/app", 419, pushed, "fix/x", "src/a.go")) + w.settleAll() + if got := w.delivery(a); got.NewerHead != pushed || got.State != Superseded { + t.Fatalf("head A is %s with newer head %q; wanted superseded by %s", got.State, got.NewerHead, pushed) + } + if got := w.delivery(b); got.Base != "main" || got.Branch != "fix/x" { + t.Fatalf("head B not filled: %+v", got) + } + if n := len(w.op.open()); n != 1 { + t.Fatalf("%d asks open; wanted one, for the new head", n) + } +} + +// A description edit is a natural trigger too: it fills a delivery made from its verdict. +func TestAnEditOfThePullRequestFillsADeliveryMadeFromItsVerdict(t *testing.T) { + w, id := verdictFirst(t) + p := pr("novox/app", 419, readyHead, "fix/x") + p.Title, p.Body = "Fix x", verdictFirstBody + w.h.PullEdited(p) + w.settleAll() + d := w.delivery(id) + if d.Base != "main" || d.Branch != "fix/x" || d.Title != "Fix x" || d.ReviewedBy != "reviewer@shanks" { + t.Fatalf("the edit did not fill the delivery: %+v", d) + } + if n := len(w.op.open()); n != 1 { + t.Fatalf("the operator was asked %d times; wanted once", n) + } +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go index 0a60efc2..ffccdb18 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go @@ -41,6 +41,12 @@ const readyRefusedFor time.Duration = 0 // groupRefusedFor is how long the merge tool may refuse a closed group's member before it is said as stalled: none (issue 1115). const groupRefusedFor time.Duration = 0 +// waitsForHead is what a delivery waits for when its verdict was heard and its pull request's head was not (issue 1726). +const waitsForHead = "its pull request's announcement: its base, branch and title are not heard, so what it merges into is not known" + +// headHeardBound is how long a delivery may wait for its pull request's announcement before it is said as stalled. +const headHeardBound = 10 * time.Minute + // groupClosed is whether a group is closed: a member reached the trunk. Under the lock. func (h *Holder) groupClosed(id string) bool { g := h.groups[id] @@ -181,6 +187,18 @@ func (h *Holder) readyAsks(now time.Time) []func() { } continue } + if d.Base == "" { + // A delivery made from its verdict before its head was heard (issue 1726): its base, branch and title are + // unknown, so what it merges into is too. It is neither asked nor passed over in silence: it waits for its + // pull request's announcement, which fills it (pullUpdated), and past headHeardBound it is stalled. + if r.Waits != waitsForHead { + r.Waits = waitsForHead + h.Logf("[mesh-delivery] %s waits: %s", d.ID, waitsForHead) + h.keep(d) + } + h.lookAt(r.Seen.Add(headHeardBound), "a delivery whose pull request was never announced") + continue + } if d.Base != MergeBranch { if r.OtherBase != d.Base { r.OtherBase = d.Base @@ -369,6 +387,10 @@ func (h *Holder) readyLines(now time.Time) []StalledLine { continue } switch { + case d.Base == "" && d.Number != 0 && now.Sub(r.Seen) >= headHeardBound: + out = append(out, StalledLine{ID: d.ID, Number: d.Number, State: d.State, For: now.Sub(r.Seen).Round(time.Second).String(), + Bound: headHeardBound.String(), H2: "none: a later announcement of the pull request's head (the merge tool announces it again, the forge's announcer restarts, or a new push) or an edit of its description; a recheck does not fill it — the operator's", + Says: fmt.Sprintf("%s#%d passed its checks at %s and waits for %s", d.Repository, d.Number, shortOf(r.Head, 12), waitsForHead)}) case r.Failed != "" && d.State == Rejected: out = append(out, StalledLine{ID: d.ID, Number: d.Number, State: d.State, For: now.Sub(r.Seen).Round(time.Second).String(), Bound: "0s", H2: "none: a new head, or a recheck that passes — the operator's", diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go index 7179b032..8e6c747c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go @@ -42,6 +42,9 @@ func (h *Holder) waitsFor(d *Delivery) string { if d.MergedAs != "" { return "a walk that answers its merge" } + if d.Base == "" && d.Number != 0 { + return waitsForHead + } return "its merge" case Rejected: return "a new head, or a recheck" diff --git a/modules/mesh-files/cmd/mesh-files/files.go b/modules/mesh-files/cmd/mesh-files/files.go index c8eea476..e001e3e0 100644 --- a/modules/mesh-files/cmd/mesh-files/files.go +++ b/modules/mesh-files/cmd/mesh-files/files.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "io/fs" + "net/http" "os" "path/filepath" "regexp" @@ -69,6 +70,12 @@ type Files struct { OwnerOf func(path string) (Ident, error) Lchown func(path string, uid, gid int) error NewID func() string + // Link asks the object store for a link (seat:object-store.link) and HTTP moves the bytes over it; upload and + // download use them (transfer.go). Tests replace both. + Link func(bucket, key, method string, seconds int) (string, error) + HTTP *http.Client + // Bound is how long a transfer may take; zero is TransferTime (a test makes it shorter). + Bound time.Duration mu sync.Mutex } diff --git a/modules/mesh-files/cmd/mesh-files/main.go b/modules/mesh-files/cmd/mesh-files/main.go index 8dc5ad20..c6b7af43 100644 --- a/modules/mesh-files/cmd/mesh-files/main.go +++ b/modules/mesh-files/cmd/mesh-files/main.go @@ -65,7 +65,7 @@ func newFiles(places []string) (*Files, error) { r.Root = sudoRoot{} node := os.Getenv("MESH_NODE") return &Files{Reader: r, State: state, Owned: owned, Node: node, Tell: tellOperator, - OwnerOf: ownerOnDisk, Lchown: os.Lchown, NewID: newID}, nil + OwnerOf: ownerOnDisk, Lchown: os.Lchown, NewID: newID, Link: askLink}, nil } // tellOperator tells the operator a plain notice, through the operator channel's tell verb: no condition is @@ -235,6 +235,22 @@ func tools(f *Files) []stdio.Tool { func(act Act, a map[string]any) (any, error) { return f.Chown(act, arg(a, "path"), arg(a, "owner"), arg(a, "group"), boolArg(a, "recursive")) }), + verb("upload", "Send one file of this node into a bucket of the object store, over a short signed link and never over "+ + "the bus; answers the key (the file's SHA-256 when none is given), the size and the SHA-256. At most 100 MiB. "+ + "A place a secret is kept, a known credential file, or a link to one is refused by name, before anything is sent. "+ + "Read "+asAccount+".", + object(map[string]any{"path": str("an absolute path to a file"), "bucket": str("the bucket the file goes into"), + "key": str("the object's key (optional; the file's SHA-256 when not given)")}, "path", "bucket"), + func(a map[string]any) (any, error) { return f.Upload(arg(a, "path"), arg(a, "bucket"), arg(a, "key")) }), + writeVerb("download", "Write one object of a bucket to a file of this node, over a short signed link and never over "+ + "the bus; the old file goes to the node's trash. At most 100 MiB. The bytes must hash to sha256, or to the key "+ + "when the key is a SHA-256; otherwise no file is written.", + map[string]any{"bucket": str("the bucket"), "key": str("the object's key"), "path": str("an absolute path"), + "sha256": str("the SHA-256 the bytes must have (optional; the key when that is a SHA-256)")}, + []string{"bucket", "key", "path"}, + func(act Act, a map[string]any) (any, error) { + return f.Download(act, arg(a, "bucket"), arg(a, "key"), arg(a, "path"), arg(a, "sha256")) + }), verb("trash", "What the node's trash holds: each deleted or replaced item with its id, original path and when "+ "it went.", object(map[string]any{}), func(map[string]any) (any, error) { return f.Trash() }), diff --git a/modules/mesh-files/cmd/mesh-files/manifest_test.go b/modules/mesh-files/cmd/mesh-files/manifest_test.go index 5d7def24..4b0ae531 100644 --- a/modules/mesh-files/cmd/mesh-files/manifest_test.go +++ b/modules/mesh-files/cmd/mesh-files/manifest_test.go @@ -58,7 +58,17 @@ func TestTheHolderRunsAsTheToolRunnersAccount(t *testing.T) { if !state { t.Error("no state directory for the trash and the act log") } - if len(m.Invokes) != 1 || m.Invokes[0] != "seat:operator-channel.tell" { + // 1597/AC7: the tell verb, and the object store's link verb that upload and download ask for. + if len(m.Invokes) != 2 || m.Invokes[0] != "seat:operator-channel.tell" || m.Invokes[1] != "seat:object-store.link" { t.Errorf("invokes %v", m.Invokes) } + for _, v := range []string{"upload", "download"} { + found := false + for _, s := range m.Claims[0].Serves { + found = found || s == v + } + if !found { + t.Errorf("1597/AC7: the manifest does not serve %s", v) + } + } } diff --git a/modules/mesh-files/cmd/mesh-files/transfer.go b/modules/mesh-files/cmd/mesh-files/transfer.go new file mode 100644 index 00000000..c91e1512 --- /dev/null +++ b/modules/mesh-files/cmd/mesh-files/transfer.go @@ -0,0 +1,259 @@ +package main + +// upload and download (novox/hq ADR 0468 §9, issue 1597): a file of this node goes into a bucket of the object store, +// and an object comes out of one into a file of this node, over a short signed link and never over the bus. +// +// upload sends a file's content off the node, so it is judged as a read (ADR 0394 §7): before anything else it +// refuses by name what a read refuses, a link's target included. download writes a file, so it is judged as a write +// (ADR 0394 §3 to §5): a path a module owns is refused without an override, the old file goes to the trash, and the +// act is one line in the act log. + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/objectlink" +) + +// LinkSeconds is how long the link asked for one transfer lives, and TransferTime how long a transfer may take: the +// tool runner cuts every tool call at 30 s, so a transfer that is not done in 25 s is refused rather than left to run +// on after the caller was told it failed. Nothing is kept of a transfer that did not finish. +const ( + LinkSeconds = 300 + TransferTime = 25 * time.Second +) + +func (f *Files) bound() time.Duration { + if f.Bound > 0 { + return f.Bound + } + return TransferTime +} + +// tooSlow says a transfer that ran out of time in words, naming the bound. +func (f *Files) tooSlow(err error, what string) error { + if errors.Is(err, context.DeadlineExceeded) || strings.Contains(err.Error(), "context deadline exceeded") || + strings.Contains(err.Error(), "Client.Timeout") { + return fmt.Errorf("%s did not finish within %s (a tool call is cut at 30 s), so it is refused and nothing is kept", what, f.bound()) + } + return err +} + +var sha256Key = regexp.MustCompile(`^[0-9a-f]{64}$`) + +// askLink asks the object store's seat for one link (ADR 0468 §8). The answer is {link, method, seconds, expires-at}. +func askLink(bucket, key, method string, seconds int) (string, error) { + raw, err := stdio.Ask("seat:object-store.link", map[string]any{"bucket": bucket, "key": key, "method": method, "seconds": seconds}) + if err != nil { + return "", fmt.Errorf("the object store gave no %s link: %w", method, err) + } + var a struct { + Link string `json:"link"` + Answer json.RawMessage `json:"answer"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return "", fmt.Errorf("the object store's answer is not JSON: %w", err) + } + if a.Link == "" && len(a.Answer) > 0 { + _ = json.Unmarshal(a.Answer, &a) + } + if a.Link == "" { + return "", errors.New("the object store's answer holds no link") + } + return a.Link, nil +} + +func (f *Files) link(bucket, key, method string) (string, error) { + if f.Link == nil { + return "", errors.New("this holder has no way to ask the object store for a link") + } + return f.Link(bucket, key, method, LinkSeconds) +} + +func (f *Files) client() *http.Client { + if f.HTTP != nil { + return f.HTTP + } + return &http.Client{Timeout: f.bound()} +} + +// UploadAnswer is what upload answers. +type UploadAnswer struct { + Bucket string `json:"bucket"` + Key string `json:"key"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` +} + +// Upload sends one file of this node into a bucket. The key is the file's SHA-256 when none is given; the hash is +// then made first (one read of the file), so the link is asked for the key it will be used for. +func (f *Files) Upload(raw, bucket, key string) (UploadAnswer, error) { + ctx, cancel := context.WithTimeout(context.Background(), f.bound()) + defer cancel() + if strings.TrimSpace(bucket) == "" { + return UploadAnswer{}, errors.New("bucket is needed: name the bucket the file goes into") + } + path, err := absolute(raw) + if err != nil { + return UploadAnswer{}, err + } + // Refused by name before anything else: no link is asked and nothing is read for a place a secret is kept. + if err := f.refusedRead(path); err != nil { + return UploadAnswer{}, err + } + in, err := os.Open(path) + if err != nil { + return UploadAnswer{}, fmt.Errorf("%s cannot be read by this account: %s", path, why(err)) + } + defer in.Close() + info, err := in.Stat() + if err != nil { + return UploadAnswer{}, err + } + if !info.Mode().IsRegular() { + return UploadAnswer{}, fmt.Errorf("%s is a %s, not a file: upload sends one file", path, typeOf(info.Mode())) + } + if info.Size() > objectlink.MaxDocument { + return UploadAnswer{}, fmt.Errorf("%s is %d bytes, over the %d a document may be (novox/hq ADR 0468 §3)", path, info.Size(), objectlink.MaxDocument) + } + want := "" + if strings.TrimSpace(key) == "" { + sum, err := hashOf(in) + if err != nil { + return UploadAnswer{}, fmt.Errorf("%s could not be read: %v", path, err) + } + if _, err := in.Seek(0, io.SeekStart); err != nil { + return UploadAnswer{}, err + } + key, want = sum, sum + } + link, err := f.link(bucket, key, "put") + if err != nil { + return UploadAnswer{}, err + } + res, err := objectlink.Put(ctx, f.client(), link, in, info.Size(), objectlink.MaxDocument) + if err != nil { + return UploadAnswer{}, f.tooSlow(fmt.Errorf("%s could not be sent: %w", path, err), "the upload of "+path) + } + if want != "" && res.SHA256 != want { + return UploadAnswer{}, fmt.Errorf("%s changed while it was sent: it was stored under %s but hashes to %s", path, want, res.SHA256) + } + return UploadAnswer{Bucket: bucket, Key: key, Size: res.Size, SHA256: res.SHA256}, nil +} + +func hashOf(r io.Reader) (string, error) { + h := sha256.New() + if _, err := io.Copy(h, r); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +// DownloadAnswer is what download answers. +type DownloadAnswer struct { + ActAnswer + Size int64 `json:"size"` + SHA256 string `json:"sha256"` +} + +// Download writes one object of a bucket to a file of this node. The bytes must hash to sha256 when it is given, or +// to the key when the key is itself a SHA-256 (the store keeps a document under its SHA-256, ADR 0468 §2); a +// mismatch leaves no file, and an old file is left as it was. +func (f *Files) Download(a Act, bucket, key, raw, wantSHA string) (DownloadAnswer, error) { + f.mu.Lock() + defer f.mu.Unlock() + if err := f.begin(a); err != nil { + return DownloadAnswer{}, err + } + if strings.TrimSpace(bucket) == "" || strings.TrimSpace(key) == "" { + return DownloadAnswer{}, errors.New("bucket and key are needed: name the object to fetch") + } + path, err := absolute(raw) + if err != nil { + return DownloadAnswer{}, err + } + hits, err := f.guard(a, false, path) + if err != nil { + return DownloadAnswer{}, err + } + dir := filepath.Dir(path) + if _, err := os.Stat(dir); err != nil { + return DownloadAnswer{}, fmt.Errorf("the folder of %s cannot be reached: %s", path, why(err)) + } + expect := strings.ToLower(strings.TrimSpace(wantSHA)) + if expect == "" && sha256Key.MatchString(key) { + expect = key + } + var perm uint32 = 0o644 + want, exists := Ident{}, false + if old, err := os.Lstat(path); err == nil { + if !old.Mode().IsRegular() { + return DownloadAnswer{}, fmt.Errorf("%s is a %s, not a file: download replaces a file whole", path, typeOf(old.Mode())) + } + exists, perm = true, permOf(old.Mode()) + if want, err = f.ownerOf(path); err != nil { + return DownloadAnswer{}, err + } + } else if want, err = f.ownerOf(dir); err != nil { + return DownloadAnswer{}, err + } + link, err := f.link(bucket, key, "get") + if err != nil { + return DownloadAnswer{}, err + } + tmp, err := os.CreateTemp(dir, ".download-*") + if err != nil { + return DownloadAnswer{}, fmt.Errorf("a file cannot be made in the folder of %s by this account: %s", path, why(err)) + } + done := false + defer func() { + tmp.Close() + if !done { + os.Remove(tmp.Name()) + } + }() + ctx, cancel := context.WithTimeout(context.Background(), f.bound()) + defer cancel() + res, err := objectlink.Get(ctx, f.client(), link, tmp, objectlink.MaxDocument, expect) + if err != nil { + return DownloadAnswer{}, f.tooSlow(fmt.Errorf("%s/%s could not be fetched: %w", bucket, key, err), "the download of "+bucket+"/"+key) + } + if err := tmp.Close(); err != nil { + return DownloadAnswer{}, err + } + if err := f.setMode(tmp.Name(), perm); err != nil { + return DownloadAnswer{}, err + } + if err := f.chownTo(tmp.Name(), want); err != nil { + return DownloadAnswer{}, fmt.Errorf("the owner of %s could not be set: %v", path, err) + } + trashed := "" + if exists { + if trashed, err = f.toTrash(a, path); err != nil { + return DownloadAnswer{}, err + } + } + if err := os.Rename(tmp.Name(), path); err != nil { + if trashed != "" { + if _, rerr := f.restore(trashed); rerr != nil { + err = fmt.Errorf("%v (and the old file could not be put back from the trash, id %s: %v)", err, trashed, rerr) + } + } + return DownloadAnswer{}, err + } + done = true + ans, err := f.finish(a, "download", hits, ActAnswer{Done: "wrote " + path, Paths: []string{path}, Trashed: trashed}) + return DownloadAnswer{ActAnswer: ans, Size: res.Size, SHA256: res.SHA256}, err +} diff --git a/modules/mesh-files/cmd/mesh-files/transfer_test.go b/modules/mesh-files/cmd/mesh-files/transfer_test.go new file mode 100644 index 00000000..ffc71864 --- /dev/null +++ b/modules/mesh-files/cmd/mesh-files/transfer_test.go @@ -0,0 +1,301 @@ +package main + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +// fakeStore stands for the object store behind signed links: a PUT keeps the body under its path, a GET answers it. +// objectlink (mesh-sdk go/objectlink) is the real client of both. +type fakeStore struct { + mu sync.Mutex + objects map[string][]byte + puts []string + srv *httptest.Server +} + +func newFakeStore(t *testing.T) *fakeStore { + t.Helper() + s := &fakeStore{objects: map[string][]byte{}} + s.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + switch r.Method { + case http.MethodPut: + b, _ := io.ReadAll(r.Body) + s.objects[r.URL.Path] = b + s.puts = append(s.puts, r.URL.Path) + case http.MethodGet: + b, ok := s.objects[r.URL.Path] + if !ok { + w.WriteHeader(404) + return + } + w.Write(b) + } + })) + t.Cleanup(s.srv.Close) + return s +} + +// linkVerb is the fake object-store.link: it records each ask and answers a link to the fake store. +type linkVerb struct { + store *fakeStore + asks []string + err error +} + +func (l *linkVerb) ask(bucket, key, method string, seconds int) (string, error) { + l.asks = append(l.asks, method+" "+bucket+"/"+key) + if l.err != nil { + return "", l.err + } + return l.store.srv.URL + "/" + bucket + "/" + key + "?sig=1", nil +} + +func transferRig(t *testing.T, places ...string) (*rig, *fakeStore, *linkVerb) { + t.Helper() + g := testFiles(t) + g.Refuser.Places = places + st := newFakeStore(t) + lv := &linkVerb{store: st} + g.Link = lv.ask + return g, st, lv +} + +func sum(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) } + +// 1597/AC1: the store receives exactly the file's bytes and the answer is {key: SHA-256, size, sha256}. +func TestUploadSendsTheFileUnderItsHash(t *testing.T) { + g, st, lv := transferRig(t) + body := bytes.Repeat([]byte("a document\n"), 1000) + write(t, g.path("home/ops/doc.pdf"), string(body)) + ans, err := g.Upload(g.path("home/ops/doc.pdf"), "mesh-shanks-brother", "") + if err != nil { + t.Fatal(err) + } + want := sum(body) + if ans.Key != want || ans.SHA256 != want || ans.Size != int64(len(body)) { + t.Fatalf("answer %+v", ans) + } + if !bytes.Equal(st.objects["/mesh-shanks-brother/"+want], body) { + t.Fatal("the store did not receive exactly the file's bytes") + } + if len(lv.asks) != 1 || lv.asks[0] != "put mesh-shanks-brother/"+want { + t.Fatalf("link asks %v", lv.asks) + } +} + +// 1597/AC2: a key given is the key it is stored under. +func TestUploadStoresUnderTheKeyGiven(t *testing.T) { + g, st, _ := transferRig(t) + write(t, g.path("a.txt"), "hello") + ans, err := g.Upload(g.path("a.txt"), "b", "scans/a.txt") + if err != nil { + t.Fatal(err) + } + if ans.Key != "scans/a.txt" || string(st.objects["/b/scans/a.txt"]) != "hello" || ans.SHA256 != sum([]byte("hello")) { + t.Fatalf("answer %+v, store %v", ans, st.puts) + } +} + +// 1597/AC3: places a read refuses are refused by name, and no link is asked and nothing reaches the store. +func TestUploadRefusesWhatAReadRefuses(t *testing.T) { + dir := t.TempDir() + // A vault directory and a module's own-secret directory are the controller's secret places (MESH_SECRET_PLACES). + places := []string{filepath.Join(dir, "var/lib/vault"), filepath.Join(dir, "var/lib/brother")} + g, st, lv := transferRig(t, places...) + secret := map[string]string{ + "vault": filepath.Join(dir, "var/lib/vault/box"), + "own-secret": filepath.Join(dir, "var/lib/brother/admin-password.secret"), + "key file": filepath.Join(dir, "srv/tls/server.key"), + "ssh key": filepath.Join(dir, "home/ops/.ssh/id_ed25519"), + "known profile": filepath.Join(dir, "home/ops/.mozilla/firefox/logins.json"), + } + for name, p := range secret { + write(t, p, "SECRET") + if _, err := g.Upload(p, "b", ""); err == nil || !strings.Contains(err.Error(), "refused by name") { + t.Errorf("%s: not refused by name: %v", name, err) + } + } + link := filepath.Join(dir, "home/ops/innocent.txt") + if err := os.Symlink(secret["ssh key"], link); err != nil { + t.Fatal(err) + } + if _, err := g.Upload(link, "b", ""); err == nil || !strings.Contains(err.Error(), "leads to") { + t.Errorf("a link to a secret: %v", err) + } + if len(lv.asks) != 0 || len(st.puts) != 0 { + t.Fatalf("a link was asked %v or the store was reached %v", lv.asks, st.puts) + } +} + +// 1597/AC4: a file over 100 MiB is refused before a link is asked. +func TestUploadRefusesOverOneHundredMiB(t *testing.T) { + g, st, lv := transferRig(t) + p := g.path("big.bin") + f, err := os.Create(p) + if err != nil { + t.Fatal(err) + } + if err := f.Truncate(100<<20 + 1); err != nil { // a sparse file: no bytes are written + t.Fatal(err) + } + f.Close() + if _, err := g.Upload(p, "b", ""); err == nil || !strings.Contains(err.Error(), "over the") { + t.Fatalf("not refused: %v", err) + } + if len(lv.asks) != 0 || len(st.puts) != 0 { + t.Fatal("a link was asked for a file over the limit") + } +} + +// 1597/AC5: download writes the object to the path; a module-owned path is refused without an override; a hash that +// does not match leaves no file. +func TestDownloadWritesTheObject(t *testing.T) { + g, st, lv := transferRig(t) + st.objects["/b/k.txt"] = []byte("from the store") + ans, err := g.Download(by, "b", "k.txt", g.path("out/k.txt"), "") + if err == nil { + t.Fatal("the folder does not exist, yet download succeeded") + } + if err := os.MkdirAll(g.path("out"), 0o755); err != nil { + t.Fatal(err) + } + ans, err = g.Download(by, "b", "k.txt", g.path("out/k.txt"), "") + if err != nil { + t.Fatal(err) + } + if mustRead(t, g.path("out/k.txt")) != "from the store" || ans.SHA256 != sum([]byte("from the store")) || ans.Size != 14 { + t.Fatalf("answer %+v", ans) + } + if len(lv.asks) != 1 || lv.asks[0] != "get b/k.txt" { + t.Fatalf("asks %v", lv.asks) + } + if lines := actLines(t, g); len(lines) != 1 || lines[0]["verb"] != "download" { + t.Fatalf("act log %v", lines) + } +} + +func TestDownloadRefusesAModuleOwnedPathWithoutOverride(t *testing.T) { + g, st, lv := transferRig(t) + st.objects["/b/k"] = []byte("x") + owned := g.path("etc/brother.conf") + write(t, owned, "old") + g.Owned[owned] = "brother (settings)" + _, err := g.Download(by, "b", "k", owned, "") + if err == nil || !strings.Contains(err.Error(), "is owned by brother") { + t.Fatalf("not refused: %v", err) + } + if mustRead(t, owned) != "old" || len(lv.asks) != 0 { + t.Fatal("the owned file changed or a link was asked") + } + if _, err := g.Download(Act{Caller: by.Caller, Reason: "a test", Override: "because"}, "b", "k", owned, ""); err != nil { + t.Fatalf("with an override: %v", err) + } +} + +func TestDownloadWithAWrongHashLeavesNoFile(t *testing.T) { + g, st, _ := transferRig(t) + st.objects["/b/"+sum([]byte("expected"))] = []byte("something else") // the key says a hash the bytes do not have + dir := g.path("out") + os.MkdirAll(dir, 0o755) + write(t, filepath.Join(dir, "doc"), "old content") + _, err := g.Download(by, "b", sum([]byte("expected")), filepath.Join(dir, "doc"), "") + if err == nil || !strings.Contains(err.Error(), "hashes to") { + t.Fatalf("not refused: %v", err) + } + if mustRead(t, filepath.Join(dir, "doc")) != "old content" { + t.Fatal("the old file was replaced") + } + entries, _ := os.ReadDir(dir) + if len(entries) != 1 { + t.Fatalf("a temporary file is left: %v", entries) + } + // An explicit sha256 is judged too, for a key that is not a hash. + st.objects["/b/plain"] = []byte("abc") + if _, err := g.Download(by, "b", "plain", filepath.Join(dir, "plain"), sum([]byte("other"))); err == nil { + t.Fatal("a wrong sha256 was accepted") + } + if _, err := os.Stat(filepath.Join(dir, "plain")); err == nil { + t.Fatal("a file was left for a wrong sha256") + } +} + +// 1597/AC6: upload then download keeps the SHA-256 across the round trip. +func TestUploadThenDownloadKeepsTheHash(t *testing.T) { + g, _, _ := transferRig(t) + body := bytes.Repeat([]byte{0, 1, 2, 3, 250}, 50000) + write(t, g.path("in.bin"), string(body)) + up, err := g.Upload(g.path("in.bin"), "b", "") + if err != nil { + t.Fatal(err) + } + down, err := g.Download(by, "b", up.Key, g.path("out.bin"), "") + if err != nil { + t.Fatal(err) + } + if up.SHA256 != down.SHA256 || sum([]byte(mustRead(t, g.path("out.bin")))) != up.SHA256 { + t.Fatalf("up %+v down %+v", up, down) + } +} + +// 1597/AC1: the verbs are listed on the seat with their inputs. +func TestTheSeatListsUploadAndDownload(t *testing.T) { + g, _, _ := transferRig(t) + names := map[string]bool{} + for _, tool := range tools(g.Files) { + names[tool.Name] = true + } + if !names[Seat+".upload"] || !names[Seat+".download"] { + t.Fatalf("tools %v", names) + } +} + +// 1597/AC9: a transfer slower than its bound is refused naming it, and no object or file is left. +func TestASlowTransferIsRefusedNamingTheBoundAndKeepsNothing(t *testing.T) { + g := testFiles(t) + g.Bound = 150 * time.Millisecond + release := make(chan struct{}) + slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + w.Write([]byte("half of the document")) + w.(http.Flusher).Flush() + } + select { // a store that does not finish: nothing is kept of a PUT whose body it never took whole + case <-release: + case <-r.Context().Done(): + } + })) + t.Cleanup(func() { close(release); slow.Close() }) + g.Link = func(bucket, key, method string, seconds int) (string, error) { + return slow.URL + "/" + bucket + "/" + key, nil + } + write(t, g.path("in.bin"), "a document") + if _, err := g.Upload(g.path("in.bin"), "b", "k"); err == nil || !strings.Contains(err.Error(), "did not finish within 150ms") || + !strings.Contains(err.Error(), "nothing is kept") { + t.Fatalf("upload error = %v", err) + } + if err := os.MkdirAll(g.path("out"), 0o755); err != nil { + t.Fatal(err) + } + if _, err := g.Download(by, "b", "k", g.path("out/doc"), ""); err == nil || !strings.Contains(err.Error(), "did not finish within 150ms") { + t.Fatalf("download error = %v", err) + } + if e, _ := os.ReadDir(g.path("out")); len(e) != 0 { + t.Errorf("a file is left: %v", e) + } + if TransferTime != 25*time.Second { + t.Errorf("the bound is %s, want 25s", TransferTime) + } +} diff --git a/modules/mesh-files/go.mod b/modules/mesh-files/go.mod index 89a969c6..46f7fe53 100644 --- a/modules/mesh-files/go.mod +++ b/modules/mesh-files/go.mod @@ -1,5 +1,5 @@ module mesh-files -go 1.25.0 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f diff --git a/modules/mesh-files/go.sum b/modules/mesh-files/go.sum index 480a5760..35a7408b 100644 --- a/modules/mesh-files/go.sum +++ b/modules/mesh-files/go.sum @@ -2,3 +2,5 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261012122749-67bfa22132f2 h1:1E5iapAO git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261012122749-67bfa22132f2/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44 h1:r5xIo5qWOXQdCSMxvqJy8kV4U+P/QWXFK6971Ha6wU8= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f h1:7J/9jOfSfjfYpf8MOj96HwQRViQUkBNMb71RHP2mjn8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/mesh-files/module.json b/modules/mesh-files/module.json index cc1df162..915d2aef 100644 --- a/modules/mesh-files/module.json +++ b/modules/mesh-files/module.json @@ -20,12 +20,15 @@ "chmod", "chown", "trash", - "restore" + "restore", + "upload", + "download" ] } ], "invokes": [ - "seat:operator-channel.tell" + "seat:operator-channel.tell", + "seat:object-store.link" ], "resources": [ { diff --git a/modules/mesh-provisioner-client/go.mod b/modules/mesh-provisioner-client/go.mod index 535479a9..1d3e1806 100644 --- a/modules/mesh-provisioner-client/go.mod +++ b/modules/mesh-provisioner-client/go.mod @@ -1,22 +1,22 @@ module mesh-provisioner-client -go 1.25.0 +go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 github.com/nats-io/nats-server/v2 v2.11.17 - github.com/nats-io/nats.go v1.51.0 + github.com/nats-io/nats.go v1.54.0 ) require ( github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect github.com/google/go-tpm v0.9.8 // indirect - github.com/klauspost/compress v1.18.5 // indirect + github.com/klauspost/compress v1.20.0 // indirect github.com/minio/highwayhash v1.0.4 // indirect github.com/nats-io/jwt/v2 v2.8.1 // indirect - github.com/nats-io/nkeys v0.4.15 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect - golang.org/x/crypto v0.50.0 // indirect - golang.org/x/sys v0.43.0 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect golang.org/x/time v0.15.0 // indirect ) diff --git a/modules/mesh-provisioner-client/go.sum b/modules/mesh-provisioner-client/go.sum index 85f38bd2..226d88cc 100644 --- a/modules/mesh-provisioner-client/go.sum +++ b/modules/mesh-provisioner-client/go.sum @@ -1,29 +1,27 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0 h1:EK84so3Y8JdFT3BHrzj9cPQQrDUPa/d+IdOGnerlmEI= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6 h1:Sv1Qzix+U1RpaUkw5IbB0IJEyZ+FqgzHmbZrCZkcWf0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 h1:Cdl5N56QPfmFBoLHrDfvvyc5Q+PUnTjf3fgYD7imcco= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= -github.com/nats-io/nats.go v1.51.0 h1:ByW84XTz6W03GSSsygsZcA+xgKK8vPGaa/FCAAEHnAI= -github.com/nats-io/nats.go v1.51.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= -github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= -github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= -golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/minio/client.ts b/modules/minio/client.ts index 9ffb5b23..af29f17a 100644 --- a/modules/minio/client.ts +++ b/modules/minio/client.ts @@ -5,7 +5,7 @@ // // It speaks two planes with node built-ins only (never the `minio` npm package): // - the S3 data plane over `fetch`, signed with AWS Signature V4 (node:crypto) — bucket and object -// operations, and presigned URLs; +// operations (links are signed by the object-store seat's Go tool with a link key, never here); // - the admin plane through the `mc` CLI (node:child_process) — scoped service accounts, whose // creation the MinIO admin REST API guards behind an encrypted payload `fetch` cannot form. // This mirrors hal's MinIOClient/MinIOAdmin split, folded into one client the module builds from env. @@ -193,26 +193,6 @@ export class MinioClient { }; } - /** A time-limited URL for GET (download) or PUT (upload) of one object — query-string SigV4. */ - presignedUrl(method: "GET" | "PUT", bucket: string, object: string, expires = 86400): string { - const { amzDate, dateStamp } = this.stamp(); - const scope = `${dateStamp}/${this.region}/s3/aws4_request`; - const host = new URL(this.baseUrl).host; - const params: Record = { - "X-Amz-Algorithm": "AWS4-HMAC-SHA256", - "X-Amz-Credential": `${this.rootUser}/${scope}`, - "X-Amz-Date": amzDate, - "X-Amz-Expires": String(expires), - "X-Amz-SignedHeaders": "host", - }; - const path = uriEncode(`/${bucket}/${object}`, false); - const canonicalQuery = encodeQuery(params); - const canonicalRequest = [method, path, canonicalQuery, `host:${host}\n`, "host", "UNSIGNED-PAYLOAD"].join("\n"); - const stringToSign = ["AWS4-HMAC-SHA256", amzDate, scope, sha256hex(canonicalRequest)].join("\n"); - const signature = hmac(this.signingKey(dateStamp), stringToSign).toString("hex"); - return `${this.baseUrl}${path}?${canonicalQuery}&X-Amz-Signature=${signature}`; - } - // --- admin plane (mc CLI) ------------------------------------------------ /** @@ -228,9 +208,14 @@ export class MinioClient { * but the machine's process table shows it to every local user for the second `mc` runs (novox/hq * issue 282, open): replacing it means the admin API's encrypted request made here. */ - async createAccessKey(bucket: string, accessKey: string, secretKey: string): Promise { + async createAccessKey( + bucket: string, + accessKey: string, + secretKey: string, + policy: string = bucketPolicy(bucket), + ): Promise { const policyPath = join(this.mcConfigDir, `policy-${accessKey}.json`); - writeFileSync(policyPath, bucketPolicy(bucket), { mode: 0o600 }); + writeFileSync(policyPath, policy, { mode: 0o600 }); try { await this.mc( "admin", "user", "svcacct", "add", "mesh", this.rootUser, @@ -248,6 +233,16 @@ export class MinioClient { await this.mc("admin", "user", "svcacct", "rm", "mesh", accessKey); } + /** Whether a service account of this access key exists (`mc admin user svcacct info` fails when it does not). */ + async accessKeyExists(accessKey: string): Promise { + try { + await this.mc("admin", "user", "svcacct", "info", "mesh", accessKey); + return true; + } catch { + return false; + } + } + /** * Run `mc` against the alias `mesh`, which it is given as MC_HOST_mesh in its environment — the * root user and password inside the URL — rather than by an `mc alias set` that carried the root @@ -320,7 +315,7 @@ export class MinioClient { // is a line of this module's manifest, filled per consumer and delivered to both ends). Both // survived with no callers, which is the state a rule comes back from; they are gone. -function bucketPolicy(bucket: string): string { +export function bucketPolicy(bucket: string): string { return JSON.stringify({ Version: "2012-10-17", Statement: [{ diff --git a/modules/minio/cmd/object-store/main.go b/modules/minio/cmd/object-store/main.go new file mode 100644 index 00000000..c789f03c --- /dev/null +++ b/modules/minio/cmd/object-store/main.go @@ -0,0 +1,52 @@ +// minio's object-store seat (novox/hq ADR 0468 §3, §8): the holder of `object-store`, serving `link`. A party that +// needs to put a document into a bucket, or get one out, asks for one short link and moves the bytes over HTTP +// itself; no document ever travels on the bus. +// +// The link is signed with the bucket's link key (issue 1594), which only reads and writes that bucket's objects, +// never with the root account. Who may have a link is judged here, at each call, from the grant file the mesh +// delivered to minio: the bucket's consumer, and whoever its `shares` name for that method. +// +// stdout is the MCP channel; what this module says, it says on stderr. +package main + +import ( + "fmt" + "os" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Words the manifest gives this bundle (modules/minio/module.json). +const ( + ConfigWord = "OBJECT_STORE_CONFIG" // the store as the machine reaches it: files/object-store.json, rendered + LinksWord = "MESH_MINIO_LINKS" // the directory of the link keys' secrets (issue 1594) + ReceivesWord = "MESH_RECEIVES" // the grant file +) + +func main() { + s, err := newServer(os.Getenv(ConfigWord), os.Getenv(LinksWord), os.Getenv(ReceivesWord)) + if err != nil { + fmt.Fprintf(os.Stderr, "[object-store] %v\n", err) + os.Exit(1) + } + if err := stdio.Serve("", tools(s)); err != nil { + fmt.Fprintf(os.Stderr, "[object-store] %v\n", err) + os.Exit(1) + } +} + +func tools(s *server) []stdio.Tool { + return []stdio.Tool{{ + Name: "object-store.link", + Description: "A signed link to one object of a bucket, for one method, that lives at most ten minutes: the caller " + + "moves the bytes over HTTP to it and never over the bus. Allowed to the bucket's own consumer and to whoever the " + + "consumer's shares name for the method; the link is signed with a key that reaches that bucket's objects only.", + Input: map[string]any{ + "bucket": map[string]any{"type": "string", "description": "the bucket"}, + "key": map[string]any{"type": "string", "description": "the object's key"}, + "method": map[string]any{"type": "string", "description": "get to read the object, put to write it"}, + "seconds": map[string]any{"type": "integer", "description": "how long the link lives, 1 to 600 (default 300)"}, + }, + Run: func(args map[string]any) (any, error) { return s.link(stdio.CallerOf(args), args) }, + }} +} diff --git a/modules/minio/cmd/object-store/manifest_test.go b/modules/minio/cmd/object-store/manifest_test.go new file mode 100644 index 00000000..34e5ebb9 --- /dev/null +++ b/modules/minio/cmd/object-store/manifest_test.go @@ -0,0 +1,107 @@ +package main + +import ( + "encoding/json" + "os" + "strings" + "testing" +) + +func manifest(t *testing.T) map[string]any { + t.Helper() + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + return m +} + +// 1516/AC6: minio claims object-store (scope mesh) serving link, invokes seat:mesh-controller.seats, builds the Go +// bundle, and no tool named minio_presigned_url remains. +func TestManifestClaimsTheSeat(t *testing.T) { + m := manifest(t) + seats, _ := m["seats"].([]any) + if len(seats) != 1 || seats[0].(map[string]any)["name"] != "object-store" || seats[0].(map[string]any)["scope"] != "mesh" || + len(seats[0].(map[string]any)["serves"].([]any)) != 1 || seats[0].(map[string]any)["serves"].([]any)[0] != "link" { + t.Fatalf("the seat object-store is not declared in seats (the mesh defines no seat a module does not declare): %v", seats) + } + claims := m["claims"].([]any) + if len(claims) != 1 { + t.Fatalf("claims: %v", claims) + } + c := claims[0].(map[string]any) + serves := c["serves"].([]any) + if c["name"] != "object-store" || c["scope"] != "mesh" || len(serves) != 1 || serves[0] != "link" { + t.Fatalf("claim: %v", c) + } + inv := m["invokes"].([]any) + if len(inv) != 1 || inv[0] != "seat:mesh-controller.seats" { + t.Fatalf("invokes: %v", inv) + } + var goBundle map[string]any + for _, a := range m["build"].(map[string]any)["artifacts"].([]any) { + if a.(map[string]any)["from"] == "cmd/object-store" { + goBundle = a.(map[string]any) + } + } + if goBundle == nil || goBundle["language"] != "go" || goBundle["kind"] != "bundle" { + t.Fatalf("no Go bundle for cmd/object-store: %v", goBundle) + } + env := goBundle["env"].(map[string]any) + for _, word := range []string{ConfigWord, LinksWord, ReceivesWord} { + if env[word] == nil { + t.Errorf("the Go bundle's env lacks %s", word) + } + } + for _, f := range []string{"../../tools/index.ts", "../../client.ts"} { + raw, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), "minio_presigned_url") || strings.Contains(string(raw), "presignedUrl") { + t.Errorf("%s still has the root-signed link", f) + } + } +} + +// 1516/AC1: the file the Go tool reads is rendered from files/object-store.json with the facts the s3-bucket binding +// renders for a consumer: at is the providing machine's `at`, port the S3 endpoint's port (brother's files/store.json +// is the writer of that shape: scheme, at, port, region). +func TestStoreFileSaysWhatTheBindingSays(t *testing.T) { + raw, err := os.ReadFile("../../files/object-store.json") + if err != nil { + t.Fatal(err) + } + var f map[string]string + if err := json.Unmarshal(raw, &f); err != nil { + t.Fatal(err) + } + if f["at"] != "${machine:at}" || f["port"] != "${port:9000}" || f["scheme"] != "http" || f["region"] != "eu-west" { + t.Fatalf("object-store.json: %v", f) + } + brother, err := os.ReadFile("../../../brother/files/store.json") + if err != nil { + t.Fatal(err) + } + var b map[string]string + if err := json.Unmarshal(brother, &b); err != nil { + t.Fatal(err) + } + for _, k := range []string{"scheme", "at", "port", "region"} { + if _, ok := b[k]; !ok { + t.Errorf("brother's store.json has no %s: the shape moved", k) + } + if _, ok := f[k]; !ok { + t.Errorf("object-store.json has no %s", k) + } + } + // The same values: minio's serves.s3-bucket.port is the port its S3 listen declares. + m := manifest(t) + if port := m["serves"].(map[string]any)["s3-bucket"].(map[string]any)["port"]; port != float64(9000) { + t.Errorf("serves port %v", port) + } +} diff --git a/modules/minio/cmd/object-store/server.go b/modules/minio/cmd/object-store/server.go new file mode 100644 index 00000000..9b4b59e5 --- /dev/null +++ b/modules/minio/cmd/object-store/server.go @@ -0,0 +1,306 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/objectlink" +) + +// DefaultSeconds is the life of a link when none is asked; MaxSeconds is the longest (ADR 0468 §3). +const ( + DefaultSeconds = 300 + MaxSeconds = 600 + holdersFor = 60 * time.Second +) + +// storeConfig is the store as this machine reaches it. The file is rendered by the mesh from files/object-store.json +// with the same facts the s3-bucket binding renders for a consumer (`at` is the providing machine's, `port` the one +// it gave the S3 endpoint), so a link names the host and port every consumer's own file names. +type storeConfig struct { + Scheme string `json:"scheme"` + At string `json:"at"` + Port string `json:"port"` + Region string `json:"region"` +} + +func (c storeConfig) endpoint() string { return c.Scheme + "://" + c.At + ":" + c.Port } + +func readConfig(path string) (storeConfig, error) { + var c storeConfig + raw, err := os.ReadFile(path) + if err != nil { + return c, fmt.Errorf("the store's address is unreadable (%s): %w", path, err) + } + if err := json.Unmarshal(raw, &c); err != nil { + return c, fmt.Errorf("the store's address file %s is not JSON: %w", path, err) + } + if c.Scheme == "" || c.At == "" || c.Port == "" { + return c, fmt.Errorf("the store's address file %s lacks a scheme, an at or a port", path) + } + return c, nil +} + +// linkKeyID is the access key id of a bucket's link key, as the provisioner makes it (modules/minio/links.ts). +func linkKeyID(bucket string) string { + sum := sha256.Sum256([]byte(bucket)) + return "link" + hex.EncodeToString(sum[:])[:16] +} + +type server struct { + configPath, linksDir, receives string + now func() time.Time + // seats asks the controller for its seats; a test replaces it. + seats func() (json.RawMessage, error) + + mu sync.Mutex + holders map[string]bool // "/." + readAt time.Time + haveSet bool +} + +func newServer(config, links, receives string) (*server, error) { + for word, v := range map[string]string{ConfigWord: config, LinksWord: links, ReceivesWord: receives} { + if strings.TrimSpace(v) == "" { + return nil, fmt.Errorf("%s is not given: the manifest names it", word) + } + } + return &server{configPath: config, linksDir: links, receives: receives, now: time.Now, seats: askSeats}, nil +} + +func askSeats() (json.RawMessage, error) { + return stdio.Ask("seat:mesh-controller.seats", map[string]any{}) +} + +// link answers one ask. +func (s *server) link(caller string, args map[string]any) (any, error) { + bucket, _ := args["bucket"].(string) + key, _ := args["key"].(string) + method, _ := args["method"].(string) + if bucket == "" { + return nil, errors.New("link: bucket is needed") + } + if key == "" { + return nil, errors.New("link: key is needed") + } + if method != "get" && method != "put" { + return nil, fmt.Errorf("link: method %q is not get or put", method) + } + seconds := DefaultSeconds + if v, given := args["seconds"]; given { + f, ok := v.(float64) + if !ok || f != float64(int(f)) { + return nil, fmt.Errorf("link: seconds %v is not a whole number", v) + } + seconds = int(f) + } + if seconds < 1 || seconds > MaxSeconds { + return nil, fmt.Errorf("link: seconds %d is outside 1 to %d: a link lives at most ten minutes (ADR 0468 §3)", seconds, MaxSeconds) + } + node, module, ok := splitCaller(caller) + if !ok { + return nil, fmt.Errorf("link: the caller is %q: the runtime did not prove who asks, and a link is not given to an unproven caller", caller) + } + grant, err := s.grantOf(bucket) + if err != nil { + return nil, err + } + if err := s.allowed(grant, node, module, method); err != nil { + return nil, fmt.Errorf("link: %s.%s may not %s in bucket %s: %w", node, module, method, bucket, err) + } + cfg, err := readConfig(s.configPath) + if err != nil { + return nil, err + } + secret, err := os.ReadFile(filepath.Join(s.linksDir, bucket+".secret")) + if err != nil || strings.TrimSpace(string(secret)) == "" { + return nil, fmt.Errorf("link: bucket %s has no link key secret yet (%s); minio's provisioner makes it", bucket, filepath.Join(s.linksDir, bucket+".secret")) + } + now := s.now() + url, err := objectlink.Presign(objectlink.Request{ + Method: strings.ToUpper(method), Endpoint: cfg.endpoint(), Bucket: bucket, Key: key, + AccessKey: linkKeyID(bucket), SecretKey: strings.TrimSpace(string(secret)), Region: cfg.Region, + Expires: time.Duration(seconds) * time.Second, Now: now, + }) + if err != nil { + return nil, err + } + return map[string]any{"link": url, "method": method, "seconds": seconds, "expires-at": now.Add(time.Duration(seconds) * time.Second).UTC().Format(time.RFC3339)}, nil +} + +// splitCaller splits the bus user `.` the runtime proved. +func splitCaller(caller string) (node, module string, ok bool) { + if caller == "" || caller == stdio.Unproven { + return "", "", false + } + node, module, ok = strings.Cut(caller, ".") + if !ok || node == "" || module == "" || strings.Contains(module, ".") { + return "", "", false + } + return node, module, true +} + +// grant is what the mesh delivered for one bucket: its consumer and the consumer's shares. +type grant struct { + consumerModule, consumerNode string + shares []share +} + +type share struct { + To string `json:"to"` + Methods []string `json:"methods"` +} + +// grantOf reads the grant file now (never cached, so a change of shares applies to the next ask) and finds the entry +// whose derived bucket is this one. The entry's shape is the mesh's Contribution (mesh-sdk go/catalogue/declaration.go +// `Contribution`: from, node, as, values, derived) inside `given` (receivedFile in the same file). +func (s *server) grantOf(bucket string) (grant, error) { + raw, err := os.ReadFile(s.receives) + if err != nil { + return grant{}, fmt.Errorf("link: the grant file is unreadable (%s): %w", s.receives, err) + } + var doc struct { + Given []struct { + From string `json:"from"` + Node string `json:"node"` + Values struct { + Shares []share `json:"shares"` + } `json:"values"` + Derived struct { + Bucket string `json:"bucket"` + } `json:"derived"` + } `json:"given"` + } + if err := json.Unmarshal(raw, &doc); err != nil { + return grant{}, fmt.Errorf("link: the grant file %s is not JSON: %w", s.receives, err) + } + for _, g := range doc.Given { + if g.Derived.Bucket == bucket { + return grant{consumerModule: g.From, consumerNode: g.Node, shares: g.Values.Shares}, nil + } + } + return grant{}, fmt.Errorf("link: no grant names bucket %s: nobody is given it", bucket) +} + +// allowed judges the caller: the bucket's consumer, or a share for this method naming the caller's module, or a seat +// the caller's module holds. +func (s *server) allowed(g grant, node, module, method string) error { + if g.consumerModule == module && (g.consumerNode == "" || g.consumerNode == node) { + return nil + } + // A share names the caller when its `to` is the caller's module, or a seat the caller's module holds. The + // controller is asked only for a seat share, and only when nothing earlier decided. + names := func(sh share) (bool, error) { + if sh.To == module { + return true, nil + } + if seat, isSeat := strings.CutPrefix(sh.To, "seat:"); isSeat { + held, err := s.holds(seat, node, module) + if err != nil { + return false, fmt.Errorf("a share names seat %s and the controller's seats cannot be read to tell whether this caller holds it: %w", seat, err) + } + return held, nil + } + return false, nil + } + var unread error + for _, sh := range g.shares { + if !has(sh.Methods, method) { + continue + } + named, err := names(sh) + if err != nil { + unread = err + continue + } + if named { + return nil + } + } + if unread != nil { + return unread + } + for _, sh := range g.shares { + if has(sh.Methods, method) { + continue + } + if named, err := names(sh); err == nil && named { + return fmt.Errorf("the bucket's shares give this caller %s, not %s", strings.Join(sh.Methods, " and "), method) + } + } + return errors.New("it is not the bucket's consumer and no share of the bucket names it for that method") +} + +func has(list []string, v string) bool { + for _, x := range list { + if x == v { + return true + } + } + return false +} + +// holds says whether node.module holds the seat, from the controller's seats, read at most once a minute. A read that +// failed is not cached: the next ask reads again. +func (s *server) holds(seat, node, module string) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + if !s.haveSet || s.now().Sub(s.readAt) >= holdersFor { + raw, err := s.seats() + if err != nil { + return false, err + } + set, err := holdersIn(raw) + if err != nil { + return false, err + } + s.holders, s.readAt, s.haveSet = set, s.now(), true + } + return s.holders[seat+"/"+node+"."+module], nil +} + +// holdersIn reads the controller's `seats` answer (answer.seats[].holders[] of {node, module}, and answer.outside[] of +// {Claim, Node, Module}) into "/.". The answer may arrive wrapped in {"answer": ...}. +func holdersIn(raw json.RawMessage) (map[string]bool, error) { + var doc struct { + Answer json.RawMessage `json:"answer"` + Seats []struct { + Seat string `json:"seat"` + Holders []struct { + Node string `json:"node"` + Module string `json:"module"` + } `json:"holders"` + } `json:"seats"` + Outside []struct { + Claim, Node, Module string + } `json:"outside"` + } + if err := json.Unmarshal(raw, &doc); err != nil { + return nil, fmt.Errorf("the controller's seats answer is not JSON: %w", err) + } + if len(doc.Answer) > 0 && string(doc.Answer) != "null" { + return holdersIn(doc.Answer) + } + if doc.Seats == nil && doc.Outside == nil { + return nil, errors.New("the controller's seats answer holds neither seats nor outside claims") + } + out := map[string]bool{} + for _, s := range doc.Seats { + for _, h := range s.Holders { + out[s.Seat+"/"+h.Node+"."+h.Module] = true + } + } + for _, o := range doc.Outside { + out[o.Claim+"/"+o.Node+"."+o.Module] = true + } + return out, nil +} diff --git a/modules/minio/cmd/object-store/server_test.go b/modules/minio/cmd/object-store/server_test.go new file mode 100644 index 00000000..2f944a27 --- /dev/null +++ b/modules/minio/cmd/object-store/server_test.go @@ -0,0 +1,217 @@ +package main + +import ( + "encoding/json" + "errors" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// The grant entry's shape is the mesh's Contribution (mesh-sdk go/catalogue/declaration.go: from, node, as, values, +// derived), written in a `given` list by receivedFile in the same file; `shares` is written into values by +// go/catalogue/bucket_shares.go withShares: [{"to": ..., "methods": [...]}]. +const grantFile = `{ + "contributions": 1, + "requirement": "s3-bucket", + "given": [ + {"from": "brother", "node": "shanks", "as": "brother-key", + "values": {"shares": [{"to": "seat:node-files", "methods": ["put"]}, {"to": "print-and-scan", "methods": ["get"]}]}, + "derived": {"bucket": "mesh-shanks-brother"}}, + {"from": "nextcloud", "node": "novox", "as": "nextcloud-key", "derived": {"bucket": "mesh-novox-nextcloud"}} + ] +}` + +// The seats answer is the controller's (mesh-controller.seats read on 2026-10-21): seats[].holders[] of {node, module}, +// and outside[] of {Claim, Node, Module}. +const seatsAnswer = `{"answer": {"seats": [ + {"seat": "node-files", "scope": "node", "holders": [{"node": "g14", "module": "mesh-files"}, {"node": "shanks", "module": "mesh-files"}]}, + {"seat": "node-backup", "scope": "node", "holders": [{"node": "g14", "module": "restic"}]} + ], "outside": [{"Claim": "printer", "Scope": "mesh", "Node": "shanks", "Module": "brother", "Site": "home", "Kind": ""}]}}` + +type rig struct { + s *server + dir string + now time.Time + seatsCalls int + seatsErr error +} + +func newRig(t *testing.T) *rig { + t.Helper() + dir := t.TempDir() + r := &rig{dir: dir, now: time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)} + write := func(name, content string) string { + p := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return p + } + // What the mesh renders from files/object-store.json for a store on novox.internal, port 20001. + cfg := write("object-store.json", `{"scheme":"http","at":"novox.internal","port":"20001","region":"eu-west"}`) + write("links/mesh-shanks-brother.secret", "brother-link-secret\n") + write("links/mesh-novox-nextcloud.secret", "nextcloud-link-secret\n") + grants := write("grants/mesh.json", grantFile) + s, err := newServer(cfg, filepath.Join(dir, "links"), grants) + if err != nil { + t.Fatal(err) + } + s.now = func() time.Time { return r.now } + s.seats = func() (json.RawMessage, error) { + r.seatsCalls++ + if r.seatsErr != nil { + return nil, r.seatsErr + } + return json.RawMessage(seatsAnswer), nil + } + r.s = s + return r +} + +func ask(bucket, method string, seconds any) map[string]any { + a := map[string]any{"bucket": bucket, "key": "doc.pdf", "method": method} + if seconds != nil { + a["seconds"] = seconds + } + return a +} + +func linkOf(t *testing.T, got any) *url.URL { + t.Helper() + m := got.(map[string]any) + u, err := url.Parse(m["link"].(string)) + if err != nil { + t.Fatal(err) + } + return u +} + +// 1516/AC1: brother's own ask gets a link signed with the bucket's link key, for the store's host and port, 300 s. +func TestLinkIsSignedWithTheBucketsLinkKey(t *testing.T) { + r := newRig(t) + got, err := r.s.link("shanks.brother", ask("mesh-shanks-brother", "get", float64(300))) + if err != nil { + t.Fatal(err) + } + u := linkOf(t, got) + if u.Host != "novox.internal:20001" || u.Scheme != "http" || u.Path != "/mesh-shanks-brother/doc.pdf" { + t.Fatalf("link goes to %s", u) + } + q := u.Query() + if cred := q.Get("X-Amz-Credential"); !strings.HasPrefix(cred, linkKeyID("mesh-shanks-brother")+"/20261021/eu-west/s3/") { + t.Fatalf("credential %q is not the bucket's link key", cred) + } + if strings.Contains(u.String(), "meshroot") { + t.Fatal("the root account signed a link") + } + if q.Get("X-Amz-Expires") != "300" || q.Get("X-Amz-Date") != "20261021T120000Z" { + t.Fatalf("expiry %q at %q", q.Get("X-Amz-Expires"), q.Get("X-Amz-Date")) + } +} + +// 1516/AC1: the link key id is the one the provisioner makes (modules/minio/links.ts linkKeyId, tested there with the +// same literal). +func TestLinkKeyIDMatchesTheProvisioners(t *testing.T) { + if got := linkKeyID("brother"); got != "link0b4bd77cec705cf5" { + t.Fatalf("linkKeyID = %s", got) + } +} + +// 1516/AC2: a share to seat:node-files for put, mesh-files holding node-files on g14: put given, get refused naming the method. +func TestSeatShareGivesTheHoldersTheSharedMethodOnly(t *testing.T) { + r := newRig(t) + if _, err := r.s.link("g14.mesh-files", ask("mesh-shanks-brother", "put", nil)); err != nil { + t.Fatalf("put: %v", err) + } + _, err := r.s.link("g14.mesh-files", ask("mesh-shanks-brother", "get", nil)) + if err == nil || !strings.Contains(err.Error(), "put, not get") { + t.Fatalf("get was not refused naming the method: %v", err) + } + // A module share is judged by name alone. + if _, err := r.s.link("novox.print-and-scan", ask("mesh-shanks-brother", "get", nil)); err != nil { + t.Fatalf("print-and-scan get: %v", err) + } +} + +// 1516/AC3: each of these is refused with its reason. +func TestRefusals(t *testing.T) { + r := newRig(t) + cases := []struct { + name, caller string + args map[string]any + want string + }{ + {"a stranger", "g14.restic", ask("mesh-shanks-brother", "get", nil), "not the bucket's consumer"}, + {"a seat it does not hold", "g14.restic", ask("mesh-shanks-brother", "put", nil), "not the bucket's consumer"}, + {"unproven", "unproven", ask("mesh-shanks-brother", "get", nil), "did not prove"}, + {"no grant", "shanks.brother", ask("no-such-bucket", "get", nil), "no grant names bucket"}, + {"seconds over 600", "shanks.brother", ask("mesh-shanks-brother", "get", float64(601)), "at most ten minutes"}, + {"seconds zero", "shanks.brother", ask("mesh-shanks-brother", "get", float64(0)), "outside 1 to 600"}, + {"method delete", "shanks.brother", ask("mesh-shanks-brother", "delete", nil), "not get or put"}, + {"another consumer's bucket", "shanks.brother", ask("mesh-novox-nextcloud", "get", nil), "not the bucket's consumer"}, + } + for _, c := range cases { + _, err := r.s.link(c.caller, c.args) + if err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("%s: want a refusal containing %q, got %v", c.name, c.want, err) + } + } +} + +// 1516/AC4: with the controller unreadable a seat share is refused naming why; a good read serves a second ask within +// 60 s from the cache and is read again after. +func TestSeatsUnreadableAndCached(t *testing.T) { + r := newRig(t) + r.seatsErr = errors.New("the controller did not answer") + _, err := r.s.link("g14.mesh-files", ask("mesh-shanks-brother", "put", nil)) + if err == nil || !strings.Contains(err.Error(), "the controller did not answer") || !strings.Contains(err.Error(), "node-files") { + t.Fatalf("not refused naming why: %v", err) + } + r.seatsErr = nil + put := func() { + t.Helper() + if _, err := r.s.link("g14.mesh-files", ask("mesh-shanks-brother", "put", nil)); err != nil { + t.Fatal(err) + } + } + put() // a failed read is not cached: this reads again + put() // served from the cache + if r.seatsCalls != 2 { + t.Fatalf("the controller was asked %d times, want 2 (one failure, one good read)", r.seatsCalls) + } + r.now = r.now.Add(59 * time.Second) + put() + if r.seatsCalls != 2 { + t.Fatalf("an ask within 60 s read the seats again (%d calls)", r.seatsCalls) + } + r.now = r.now.Add(2 * time.Second) + put() + if r.seatsCalls != 3 { + t.Fatalf("the seats were not read again after 60 s (%d calls)", r.seatsCalls) + } +} + +// 1516/AC5: the grant file is read at each call. +func TestGrantFileIsReadAtEachCall(t *testing.T) { + r := newRig(t) + if _, err := r.s.link("novox.print-and-scan", ask("mesh-shanks-brother", "get", nil)); err != nil { + t.Fatal(err) + } + changed := strings.Replace(grantFile, `{"to": "print-and-scan", "methods": ["get"]}`, `{"to": "someone-else", "methods": ["get"]}`, 1) + if err := os.WriteFile(r.s.receives, []byte(changed), 0o600); err != nil { + t.Fatal(err) + } + if _, err := r.s.link("novox.print-and-scan", ask("mesh-shanks-brother", "get", nil)); err == nil { + t.Fatal("the second call judged by the old shares") + } + if _, err := r.s.link("novox.someone-else", ask("mesh-shanks-brother", "get", nil)); err != nil { + t.Fatal(err) + } +} diff --git a/modules/minio/files/object-store.json b/modules/minio/files/object-store.json new file mode 100644 index 00000000..d2970bcc --- /dev/null +++ b/modules/minio/files/object-store.json @@ -0,0 +1,6 @@ +{ + "scheme": "http", + "at": "${machine:at}", + "port": "${port:9000}", + "region": "eu-west" +} diff --git a/modules/minio/go.mod b/modules/minio/go.mod new file mode 100644 index 00000000..c09ccfda --- /dev/null +++ b/modules/minio/go.mod @@ -0,0 +1,5 @@ +module minio + +go 1.26.0 + +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f diff --git a/modules/minio/go.sum b/modules/minio/go.sum new file mode 100644 index 00000000..9728ce93 --- /dev/null +++ b/modules/minio/go.sum @@ -0,0 +1,2 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f h1:7J/9jOfSfjfYpf8MOj96HwQRViQUkBNMb71RHP2mjn8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/minio/links.ts b/modules/minio/links.ts new file mode 100644 index 00000000..b870f0df --- /dev/null +++ b/modules/minio/links.ts @@ -0,0 +1,64 @@ +// The link key — a second service account per bucket, beside the consumer's own key (novox/hq ADR 0468 §8, +// issue 1594). The object-store seat signs the links (issue 1516) with this key and never with the root +// account: its policy allows only s3:GetObject and s3:PutObject on the bucket's objects, so a link can read or +// write an object of that bucket and do nothing else. Its secret is minio's own: a file under the state +// directory (links/.secret, 0600), never in a grant, never on the bus, never in an emitted event. +// +// This file imports nothing of the SDK, so its tests run without the toolchain's packages. + +import { createHash, randomBytes } from "node:crypto"; +import { chmodSync, existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +/** What the link key needs of the admin client (MinioClient has all three). */ +export interface LinkAdmin { + createAccessKey(bucket: string, accessKey: string, secretKey: string, policy?: string): Promise; + removeAccessKey(accessKey: string): Promise; + accessKeyExists(accessKey: string): Promise; +} + +/** The access key id of a bucket's link key: "link" and the first 16 hex digits of the SHA-256 of the bucket + * name (20 characters, the longest an access key may be; a bucket name may be 63). The signer of issue 1516 + * derives the same id. */ +export function linkKeyId(bucket: string): string { + return "link" + createHash("sha256").update(bucket, "utf8").digest("hex").slice(0, 16); +} + +/** Only get and put of the bucket's objects: no list, no delete, no bucket-level action. */ +export function linkPolicy(bucket: string): string { + return JSON.stringify({ + Version: "2012-10-17", + Statement: [{ + Effect: "Allow", + Action: ["s3:GetObject", "s3:PutObject"], + Resource: [`arn:aws:s3:::${bucket}/*`], + }], + }); +} + +export function linkSecretPath(dir: string, bucket: string): string { + return join(dir, `${bucket}.secret`); +} + +/** Make the bucket's link key if it is not there: the secret is kept if its file exists (so a key lost in + * minio is made again under the same secret), made fresh otherwise. Idempotent: a key that exists is left. */ +export async function ensureLinkKey(admin: LinkAdmin, dir: string, bucket: string): Promise { + const id = linkKeyId(bucket); + const path = linkSecretPath(dir, bucket); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + let secret = existsSync(path) ? readFileSync(path, "utf8").trim() : ""; + if (secret !== "" && (await admin.accessKeyExists(id))) return; + if (secret === "") { + secret = randomBytes(32).toString("hex"); + writeFileSync(path, secret + "\n", { mode: 0o600 }); + chmodSync(path, 0o600); + } + try { await admin.removeAccessKey(id); } catch { /* none yet */ } + await admin.createAccessKey(bucket, id, secret, linkPolicy(bucket)); +} + +/** Revoke the bucket's link key; the bucket and the secret file's absence are left as they are. */ +export async function revokeLinkKey(admin: LinkAdmin, dir: string, bucket: string): Promise { + try { await admin.removeAccessKey(linkKeyId(bucket)); } catch { /* already gone */ } + try { unlinkSync(linkSecretPath(dir, bucket)); } catch { /* none */ } +} diff --git a/modules/minio/module.json b/modules/minio/module.json index 85b82530..01a966af 100644 --- a/modules/minio/module.json +++ b/modules/minio/module.json @@ -18,6 +18,27 @@ "requires": [ "route" ], + "seats": [ + { + "name": "object-store", + "scope": "mesh", + "serves": [ + "link" + ] + } + ], + "claims": [ + { + "name": "object-store", + "scope": "mesh", + "serves": [ + "link" + ] + } + ], + "invokes": [ + "seat:mesh-controller.seats" + ], "contributes": { "route": { "api": { @@ -108,6 +129,13 @@ "mode": "0600", "template": "files/root.env" }, + { + "id": "object-store", + "type": "file", + "path": "${dir:state}/object-store.json", + "mode": "0600", + "template": "files/object-store.json" + }, { "id": "data", "type": "directory", @@ -173,11 +201,28 @@ "MESH_MINIO_ROOT_USER": "meshroot", "MESH_MINIO_ROOT_PASSWORD_FILE": "${dir:state}/root.secret", "MESH_MINIO_REGION": "eu-west", + "MESH_MINIO_LINKS": "${dir:state}/links", "MESH_MINIO_MC_BIN": "mcli", "MESH_MINIO_MC_CONFIG": "${dir:state}/mc", "MESH_RECEIVES": "${dir:grants}/mesh.json" } }, + { + "name": "tools-go", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/object-store", + "binary": "object-store", + "loads": [ + "object-store" + ], + "env": { + "OBJECT_STORE_CONFIG": "${dir:state}/object-store.json", + "MESH_MINIO_LINKS": "${dir:state}/links", + "MESH_RECEIVES": "${dir:grants}/mesh.json" + } + }, { "name": "files", "kind": "archive", diff --git a/modules/minio/package.json b/modules/minio/package.json index 892b0977..017e2af8 100644 --- a/modules/minio/package.json +++ b/modules/minio/package.json @@ -4,6 +4,9 @@ "description": "minio — S3-compatible object store. Its admin client, tools, provisioner and events live here (novox/hq ADR 0039).", "type": "module", "private": true, + "scripts": { + "test": "node --test --experimental-strip-types 'test/*.test.ts'" + }, "dependencies": { "@novox/mesh-sdk": "^0.1.7" }, diff --git a/modules/minio/provisioner/handlers.ts b/modules/minio/provisioner/handlers.ts new file mode 100644 index 00000000..70052810 --- /dev/null +++ b/modules/minio/provisioner/handlers.ts @@ -0,0 +1,78 @@ +// The s3-bucket provisioner's create, remove and holds, with what they touch handed in, so a unit test drives +// them with a fake admin client (the provisioner's index.ts runs them against minio). + +import type { Provision } from "@novox/mesh-sdk/provisioner"; +import { ensureLinkKey, revokeLinkKey, type LinkAdmin } from "../links.js"; + +export interface Admin extends LinkAdmin { + bucketExists(bucket: string): Promise; + createBucket(bucket: string): Promise; + canReachAs(bucket: string, accessKey: string, secretKey: string): Promise; + readonly baseUrl: string; +} + +export interface Deps { + minio: Admin; + linksDir: string; + announce(type: string, body: unknown): Promise; +} + +export function handlers({ minio, linksDir, announce }: Deps) { + return { + + async create(p: Provision): Promise { + const bucket = bucketNamed(p.derived); + const accessKeyId = p.as; + + if (!(await minio.bucketExists(bucket))) await minio.createBucket(bucket); + // Re-mint the scoped key idempotently: drop any prior one under this id, then add it back with + // the mesh's secret. + try { await minio.removeAccessKey(accessKeyId); } catch { /* none yet — first provision */ } + await minio.createAccessKey(bucket, accessKeyId, p.password); + await ensureLinkKey(minio, linksDir, bucket); + + await announce("bucket.created", { + bucket, + consumer: p.consumer ?? "", + accessKey: accessKeyId, + endpoint: minio.baseUrl, + }); + }, + + async remove(p: { as: string; derived: Readonly> }): Promise { + const bucket = bucketNamed(p.derived); + + // Revoking the key is what cuts the consumer's access, and the bucket is kept, empty or not + // (novox/hq issue 241: a withdrawal never destroys a consumer's data — on 2026-10-04 a misread grants file withdrew every consumer at once). A bucket is removed by a person, never by this loop. + try { await minio.removeAccessKey(p.as); } catch { /* already gone */ } + await revokeLinkKey(minio, linksDir, bucket); + console.error(`[minio] ${p.as} withdrawn: access key and link key revoked, bucket ${bucket} kept`); + + await announce("bucket.removed", { bucket, accessKey: p.as }); + }, + + // Asked every minute by the harness: whether the backend still holds this consumer exactly as + // the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120). + async holds(p: Provision): Promise { + return minio.canReachAs(bucketNamed(p.derived), p.as, p.password); + }, + }; +} + +/** The bucket the mesh derived for this consumer. + * + * Absent means this module is running against a control plane that does not fill `${consumer:…}` + * yet, or a manifest whose `serves` block lost the line. Both are the same mistake from here — + * nobody said which bucket — and both are said rather than guessed: a provisioner that fell back + * to deriving one would restore the second rule and hide the fault behind a bucket that happens + * to be right. */ +function bucketNamed(derived: Readonly>): string { + const bucket = derived.bucket; + if (typeof bucket !== "string" || bucket === "") { + throw new Error( + "the mesh did not say which bucket this consumer gets: minio's manifest must serve " + + "`bucket` under s3-bucket (novox/hq ADR 0202)", + ); + } + return bucket; +} diff --git a/modules/minio/provisioner/index.ts b/modules/minio/provisioner/index.ts index 15308b3d..345bee46 100644 --- a/modules/minio/provisioner/index.ts +++ b/modules/minio/provisioner/index.ts @@ -22,63 +22,28 @@ import { runProvisioner, type Provision } from "@novox/mesh-sdk/provisioner"; import { emit } from "@novox/mesh-sdk/events"; import { MinioClient } from "../client.js"; +import { ensureLinkKey } from "../links.js"; +import { handlers } from "./handlers.js"; const minio = MinioClient.fromEnv(); +// minio's own link keys (issue 1594): the secrets live under the state directory, nowhere else. +const linksDir = process.env.MESH_MINIO_LINKS ?? ""; -runProvisioner("s3-bucket", { - async create(p: Provision): Promise { - const bucket = bucketNamed(p.derived); - const accessKeyId = p.as; - - if (!(await minio.bucketExists(bucket))) await minio.createBucket(bucket); - // Re-mint the scoped key idempotently: drop any prior one under this id, then add it back with - // the mesh's secret. - try { await minio.removeAccessKey(accessKeyId); } catch { /* none yet — first provision */ } - await minio.createAccessKey(bucket, accessKeyId, p.password); - - await announce("bucket.created", { - bucket, - consumer: p.consumer ?? "", - accessKey: accessKeyId, - endpoint: minio.baseUrl, - }); - }, - - async remove(p: { as: string; derived: Readonly> }): Promise { - const bucket = bucketNamed(p.derived); - - // Revoking the key is what cuts the consumer's access, and the bucket is kept, empty or not - // (novox/hq issue 241: a withdrawal never destroys a consumer's data — on 2026-10-04 a misread grants file withdrew every consumer at once). A bucket is removed by a person, never by this loop. - try { await minio.removeAccessKey(p.as); } catch { /* already gone */ } - console.error(`[minio] ${p.as} withdrawn: access key revoked, bucket ${bucket} kept`); - - await announce("bucket.removed", { bucket, accessKey: p.as }); - }, - - // Asked every minute by the harness: whether the backend still holds this consumer exactly as - // the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120). - async holds(p: Provision): Promise { - return minio.canReachAs(bucketNamed(p.derived), p.as, p.password); - }, -}); - -/** The bucket the mesh derived for this consumer. - * - * Absent means this module is running against a control plane that does not fill `${consumer:…}` - * yet, or a manifest whose `serves` block lost the line. Both are the same mistake from here — - * nobody said which bucket — and both are said rather than guessed: a provisioner that fell back - * to deriving one would restore the second rule and hide the fault behind a bucket that happens - * to be right. */ -function bucketNamed(derived: Readonly>): string { - const bucket = derived.bucket; - if (typeof bucket !== "string" || bucket === "") { - throw new Error( - "the mesh did not say which bucket this consumer gets: minio's manifest must serve " + - "`bucket` under s3-bucket (novox/hq ADR 0202)", - ); +// At start every bucket that exists gets its link key too, those made before link keys were (best effort: +// a bucket that fails is said, the provisioner still serves). +void (async () => { + if (linksDir === "") { console.error("[minio] MESH_MINIO_LINKS is not set: no link keys made"); return; } + try { + for (const b of await minio.listBuckets()) { + try { await ensureLinkKey(minio, linksDir, b.name); } + catch (err) { console.error(`[minio] link key for ${b.name}: ${err}`); } + } + } catch (err) { + console.error(`[minio] could not list buckets for link keys: ${err}`); } - return bucket; -} +})(); + +runProvisioner("s3-bucket", handlers({ minio, linksDir, announce })); /** Emit best-effort: a broker hiccup is logged and dropped, never allowed to throw back and fail a * bucket that was made. */ diff --git a/modules/minio/test/links.test.ts b/modules/minio/test/links.test.ts new file mode 100644 index 00000000..62de8e75 --- /dev/null +++ b/modules/minio/test/links.test.ts @@ -0,0 +1,117 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { mkdtempSync, readFileSync, statSync, existsSync, readdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +// Issue 1594: a link key per bucket (novox/hq ADR 0468 §8). The fake admin below stands for minio's service +// accounts; the real client's policies come from client.ts's bucketPolicy, which the tests import. + +import { bucketPolicy } from "../client.ts"; +import { ensureLinkKey, linkKeyId, linkSecretPath } from "../links.ts"; +import { handlers } from "../provisioner/handlers.ts"; + +class FakeAdmin { + baseUrl = "http://127.0.0.1:9000"; + buckets = new Set(); + keys = new Map(); + created: string[] = []; + async bucketExists(b: string) { return this.buckets.has(b); } + async createBucket(b: string) { this.buckets.add(b); } + async canReachAs() { return true; } + async accessKeyExists(id: string) { return this.keys.has(id); } + async removeAccessKey(id: string) { + if (!this.keys.delete(id)) throw new Error("no such key"); + } + async createAccessKey(bucket: string, id: string, secret: string, policy: string = bucketPolicy(bucket)) { + this.keys.set(id, { bucket, secret, policy }); + this.created.push(id); + return { accessKey: id, secretKey: secret }; + } +} + +function setup() { + const minio = new FakeAdmin(); + const dir = join(mkdtempSync(join(process.env.TMPDIR ?? tmpdir(), "minio-links-")), "links"); + const events: Array<{ type: string; body: unknown }> = []; + const h = handlers({ minio, linksDir: dir, announce: async (type, body) => { events.push({ type, body }); } }); + return { minio, dir, events, h }; +} + +const provision = { as: "brother-key", password: "consumer-secret", consumer: "brother", derived: { bucket: "brother" } } as any; + +test("creating a consumer's bucket makes a link key allowed only get and put of the bucket's objects, its secret in links/.secret at 0600", async () => { + // 1594/AC1 + const { minio, dir, h } = setup(); + await h.create(provision); + const link = minio.keys.get(linkKeyId("brother")); + assert.ok(link, "a link key exists for the bucket"); + const policy = JSON.parse(link.policy); + assert.equal(policy.Statement.length, 1); + assert.deepEqual(policy.Statement[0].Action, ["s3:GetObject", "s3:PutObject"]); + assert.deepEqual(policy.Statement[0].Resource, ["arn:aws:s3:::brother/*"]); + const path = linkSecretPath(dir, "brother"); + assert.equal(statSync(path).mode & 0o777, 0o600); + assert.equal(readFileSync(path, "utf8").trim(), link.secret); + assert.ok(link.secret.length >= 32); +}); + +test("buckets that existed before get a link key at start, and a second start makes none twice", async () => { + // 1594/AC2 + const { minio, dir } = setup(); + for (const b of ["brother", "mesh-mail", "nextcloud", "invoicing"]) minio.buckets.add(b); + for (const b of minio.buckets) await ensureLinkKey(minio, dir, b); + assert.equal(minio.keys.size, 4); + const first = [...minio.created]; + const secret = readFileSync(linkSecretPath(dir, "nextcloud"), "utf8"); + for (const b of minio.buckets) await ensureLinkKey(minio, dir, b); + assert.deepEqual(minio.created, first, "no key made a second time"); + assert.equal(readFileSync(linkSecretPath(dir, "nextcloud"), "utf8"), secret, "the secret is kept"); +}); + +test("a link key lost in minio is made again under the kept secret", async () => { + // 1594/AC2 + const { minio, dir } = setup(); + await ensureLinkKey(minio, dir, "brother"); + const secret = minio.keys.get(linkKeyId("brother"))!.secret; + minio.keys.clear(); + await ensureLinkKey(minio, dir, "brother"); + assert.equal(minio.keys.get(linkKeyId("brother"))!.secret, secret); +}); + +test("removing a consumer revokes its bucket's link key and keeps the bucket", async () => { + // 1594/AC3 + const { minio, dir, h } = setup(); + await h.create(provision); + assert.ok(minio.keys.has(linkKeyId("brother"))); + await h.remove({ as: "brother-key", derived: { bucket: "brother" } }); + assert.ok(!minio.keys.has(linkKeyId("brother")), "link key revoked"); + assert.ok(!minio.keys.has("brother-key"), "consumer key revoked"); + assert.ok(minio.buckets.has("brother"), "bucket kept"); + assert.ok(!existsSync(linkSecretPath(dir, "brother")), "secret file removed"); +}); + +test("the consumer's own key keeps its s3:* policy, and the link key's secret is in no event", async () => { + // 1594/AC4 + const { minio, dir, events, h } = setup(); + await h.create(provision); + const own = minio.keys.get("brother-key")!; + assert.equal(own.secret, "consumer-secret"); + assert.equal(own.policy, bucketPolicy("brother")); + assert.deepEqual(JSON.parse(own.policy).Statement[0].Action, ["s3:*"]); + const linkSecret = minio.keys.get(linkKeyId("brother"))!.secret; + await h.remove({ as: "brother-key", derived: { bucket: "brother" } }); + assert.ok(events.length >= 2); + for (const e of events) assert.ok(!JSON.stringify(e).includes(linkSecret), `${e.type} carries no link secret`); + assert.deepEqual(readdirSync(dir), [], "nothing else is kept in the links directory"); +}); + +test("a link key id fits an access key (20 characters) whatever the bucket name", () => { + assert.equal(linkKeyId("a".repeat(63)).length, 20); + assert.notEqual(linkKeyId("brother"), linkKeyId("mesh-mail")); +}); + +test("the link key id is the one the object-store tool derives (cmd/object-store server_test.go holds the same literal)", () => { + // 1516/AC1 + assert.equal(linkKeyId("brother"), "link0b4bd77cec705cf5"); +}); diff --git a/modules/minio/tools/index.ts b/modules/minio/tools/index.ts index cd192825..35833579 100644 --- a/modules/minio/tools/index.ts +++ b/modules/minio/tools/index.ts @@ -50,22 +50,6 @@ export function getMinioTools(minio: MinioClient): ToolDefinition[] { input: { bucket: { type: "string", description: "the bucket name" } }, run: async (args) => minio.bucketInfo(String(args.bucket)), }, - { - name: "minio_presigned_url", - description: "A time-limited URL to download (GET) or upload (PUT) one object without credentials.", - input: { - bucket: { type: "string", description: "the bucket name" }, - object: { type: "string", description: "the object key" }, - method: { type: "string", description: "'GET' to download (default) or 'PUT' to upload" }, - expires: { type: "number", description: "seconds until the URL expires (default 86400 = 24h)" }, - }, - run: async (args) => { - const method = String(args.method ?? "GET").toUpperCase() === "PUT" ? "PUT" : "GET"; - const expires = args.expires ? Number(args.expires) : 86400; - const url = minio.presignedUrl(method, String(args.bucket), String(args.object), expires); - return { method, bucket: String(args.bucket), object: String(args.object), expiresInSeconds: expires, url }; - }, - }, ]; } diff --git a/modules/minio/tsconfig.json b/modules/minio/tsconfig.json index c2a8df00..f1e72155 100644 --- a/modules/minio/tsconfig.json +++ b/modules/minio/tsconfig.json @@ -10,7 +10,9 @@ }, "include": [ "client.ts", + "links.ts", "tools/index.ts", - "provisioner/index.ts" + "provisioner/index.ts", + "provisioner/handlers.ts" ] } \ No newline at end of file diff --git a/modules/only-office/cmd/only-office/convert.go b/modules/only-office/cmd/only-office/convert.go new file mode 100644 index 00000000..fc148278 --- /dev/null +++ b/modules/only-office/cmd/only-office/convert.go @@ -0,0 +1,204 @@ +package main + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "net/url" + "regexp" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// The ask the verb queues for the converter process (ADR 0468 §7; the shape of brother's device asks, ADR 0436 §6): +// on the work queue convert-asks, whose one consumer is the module's process `converter`, which writes its answer +// under the ask's id in the state convert-answers. +const ( + // EventAsked is the event on the work queue; the stream convert-asks carries it. + EventAsked = "convert.asked" + // AnswersState is the state the converter writes its answers in. + AnswersState = "convert-answers" + // ConvertWait is how long one call waits for the converter. The tool runner cuts every tool call at 30 s, so a + // call answers within 25 s: a conversion that is not done by then goes on, and is read with convert {ask}. + ConvertWait = 25 * time.Second + // TakeUpWithin is how long an ask may wait on the queue before the converter takes it up: brother's device process + // follows a conversion for 3 minutes. + TakeUpWithin = 3 * time.Minute +) + +// Types are the document types the converter takes (ADR 0468 §7). Any other is refused by name. +var Types = []string{"docx", "doc", "xlsx", "xls", "pptx", "ppt", "odt", "ods", "odp", "rtf", "txt", "html"} + +// Ask is what the verb queues: the operation, when it was asked, and the operation's arguments. +type Ask struct { + ID string `json:"id"` + Op string `json:"op"` + At time.Time `json:"asked-at"` + Deadline time.Time `json:"deadline"` + // Source is the signed GET link of the document, Type its type, Target the signed PUT link of the PDF. + Source string `json:"source"` + Type string `json:"type"` + Target string `json:"target"` + // SHA256 is the source's SHA-256 when the asker knows it; the converter then names the conversion by it. + SHA256 string `json:"sha256,omitempty"` +} + +var sha256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`) + +// OpConvert is the one operation. +const OpConvert = "convert" + +// AskBus is how the tool reaches the converter: it queues an ask and reads an answer back. +type AskBus interface { + Publish(id string, body any) error + Answer(id string) (json.RawMessage, error) +} + +type sdkAskBus struct{} + +func (sdkAskBus) Publish(id string, body any) error { + return stdio.Emit(EventAsked, body, stdio.EmitOptions{MessageID: id}) +} + +func (sdkAskBus) Answer(id string) (json.RawMessage, error) { + e, err := stdio.State(AnswersState).Get(id) + if err != nil || e == nil { + return nil, err + } + return e.Value, nil +} + +// Service is the tool side. +type Service struct { + Asks AskBus + Now func() time.Time + // Wait is how long an answer is waited for, Poll how often it is looked for. + Wait, Poll time.Duration +} + +func newService() *Service { + return &Service{Asks: sdkAskBus{}, Now: time.Now, Wait: ConvertWait, Poll: 500 * time.Millisecond} +} + +func newAskID() string { + b := make([]byte, 12) + rand.Read(b) + return hex.EncodeToString(b) +} + +// cleanType is a type as asked, lower-case and without a leading dot, when it is one the converter takes. +func cleanType(raw string) (string, error) { + t := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(raw), ".")) + for _, ok := range Types { + if t == ok { + return t, nil + } + } + taken := append([]string(nil), Types...) + sort.Strings(taken) + return "", fmt.Errorf("type %q is not one the converter takes: %s", raw, strings.Join(taken, ", ")) +} + +func checkLink(name, raw string) error { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") { + return fmt.Errorf("%s is a signed http(s) link, such as one object-store.link gives", name) + } + return nil +} + +// Convert queues one ask and waits for the converter's answer {size, sha256} (the PDF it put to target). Nothing is +// queued for a type the converter does not take or a link that is not one. +func (s *Service) Convert(ctx context.Context, source, typ, target, sha256 string) (map[string]any, error) { + t, err := cleanType(typ) + if err != nil { + return nil, err + } + if err := checkLink("source", source); err != nil { + return nil, err + } + if err := checkLink("target", target); err != nil { + return nil, err + } + sha256 = strings.ToLower(strings.TrimSpace(sha256)) + if sha256 != "" && !sha256Pattern.MatchString(sha256) { + return nil, fmt.Errorf("sha256 is the source's SHA-256, 64 hex digits, not %q", sha256) + } + at := s.Now().UTC() + ask := Ask{ID: newAskID(), Op: OpConvert, At: at, Deadline: at.Add(TakeUpWithin), Source: strings.TrimSpace(source), Type: t, Target: strings.TrimSpace(target), SHA256: sha256} + if err := s.Asks.Publish(ask.ID, ask); err != nil { + return nil, fmt.Errorf("the ask for the converter could not be queued: %w", err) + } + return s.await(ctx, ask.ID) +} + +// Read answers the ask of an earlier call: the converter's answer, or that it is still converting. +func (s *Service) Read(ctx context.Context, id string) (map[string]any, error) { + if !askPattern.MatchString(id) { + return nil, fmt.Errorf("ask %q is not an ask id: the 24 hex digits convert answered", id) + } + return s.await(ctx, id) +} + +var askPattern = regexp.MustCompile(`^[0-9a-f]{24}$`) + +// await waits at most Wait for the converter's answer under the id. A conversion still running is answered as +// converting with its ask id, never as a failure: it goes on, and convert {ask} reads its answer. +func (s *Service) await(ctx context.Context, id string) (map[string]any, error) { + ctx, cancel := context.WithTimeout(ctx, s.Wait) + defer cancel() + for { + raw, err := s.Asks.Answer(id) + if err != nil { + return nil, fmt.Errorf("the converter's answers cannot be read (ask %s): %w", id, err) + } + if raw != nil { + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("the converter's answer is not JSON (ask %s): %w", id, err) + } + if msg, bad := out["error"].(string); bad { + return nil, fmt.Errorf("%s (ask %s)", msg, id) + } + out["ask"] = id + return out, nil + } + select { + case <-ctx.Done(): + return map[string]any{"state": "converting", "ask": id, + "note": "the conversion goes on; call convert again with only ask to read its answer (kept 10 minutes)"}, nil + case <-time.After(s.Poll): + } + } +} + +func tools(s *Service) []stdio.Tool { + return []stdio.Tool{{ + Name: "document-converter.convert", + Description: "Turn a Word, spreadsheet, slide, text or web document into a PDF: the document is read from a signed GET " + + "link and the PDF is put to a signed PUT link, so no document crosses the bus. Takes " + strings.Join(Types, ", ") + + "; any other type is refused by name. Answers the PDF's size and SHA-256 or the document server's error; a conversion not done within 25 s is answered as converting with its ask id (a tool call is cut at 30 s), and convert with only ask reads it.", + Input: map[string]any{ + "source": map[string]any{"type": "string", "description": "a signed GET link of the document"}, + "type": map[string]any{"type": "string", "description": "the document's type: " + strings.Join(Types, ", ")}, + "target": map[string]any{"type": "string", "description": "a signed PUT link for the PDF"}, + "sha256": map[string]any{"type": "string", "description": "the document's SHA-256, if known (optional)"}, + "ask": map[string]any{"type": "string", "description": "the ask id of a conversion that was answered as converting: alone, it reads that conversion's answer"}, + }, + Run: func(a map[string]any) (any, error) { + str := func(k string) string { v, _ := a[k].(string); return v } + if ask := str("ask"); ask != "" { + if str("source") != "" || str("type") != "" || str("target") != "" || str("sha256") != "" { + return nil, fmt.Errorf("ask is read on its own: give ask alone to read an earlier conversion, or source, type and target to start one") + } + return s.Read(context.Background(), ask) + } + return s.Convert(context.Background(), str("source"), str("type"), str("target"), str("sha256")) + }, + }} +} diff --git a/modules/only-office/cmd/only-office/convert_test.go b/modules/only-office/cmd/only-office/convert_test.go new file mode 100644 index 00000000..27f4a787 --- /dev/null +++ b/modules/only-office/cmd/only-office/convert_test.go @@ -0,0 +1,221 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "strings" + "sync" + "testing" + "time" +) + +// fakeBus is the work queue and the state: it records each ask and lets a test write the answer, as the converter +// process would under the ask's id. +type fakeBus struct { + mu sync.Mutex + asks []Ask + answers map[string]json.RawMessage + answerOn func(Ask) json.RawMessage // answers an ask as it is published +} + +func (b *fakeBus) Publish(id string, body any) error { + b.mu.Lock() + defer b.mu.Unlock() + raw, _ := json.Marshal(body) + var a Ask + _ = json.Unmarshal(raw, &a) + b.asks = append(b.asks, a) + if b.answerOn != nil { + if ans := b.answerOn(a); ans != nil { + b.answers[id] = ans + } + } + return nil +} + +func (b *fakeBus) Answer(id string) (json.RawMessage, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.answers[id], nil +} + +func rig(wait time.Duration) (*Service, *fakeBus) { + b := &fakeBus{answers: map[string]json.RawMessage{}} + return &Service{Asks: b, Now: time.Now, Wait: wait, Poll: time.Millisecond}, b +} + +const get, put = "http://novox.internal:20001/b/doc.docx?X-Amz-Signature=a", "http://novox.internal:20001/b/doc.pdf?X-Amz-Signature=b" + +// 1595/AC1: one ask with those fields is published on convert-asks, and the verb answers what the converter wrote under +// the ask's id. +func TestConvertQueuesOneAskAndAnswersWhatTheConverterWrote(t *testing.T) { + s, b := rig(time.Second) + b.answerOn = func(a Ask) json.RawMessage { return json.RawMessage(`{"size": 4096, "sha256": "ab12"}`) } + got, err := s.Convert(context.Background(), get, "DOCX", put, "") + if err != nil { + t.Fatal(err) + } + if len(b.asks) != 1 { + t.Fatalf("%d asks queued", len(b.asks)) + } + a := b.asks[0] + if a.Op != OpConvert || a.Source != get || a.Type != "docx" || a.Target != put || a.ID == "" { + t.Fatalf("ask %+v", a) + } + if got["size"] != float64(4096) || got["sha256"] != "ab12" || got["ask"] != a.ID { + t.Fatalf("answer %v", got) + } +} + +// 1595/AC1: an error the converter wrote is the verb's error. +func TestConvertAnswersTheConvertersError(t *testing.T) { + s, b := rig(time.Second) + b.answerOn = func(a Ask) json.RawMessage { + return json.RawMessage(`{"error": "-4: the document server could not download the file"}`) + } + if _, err := s.Convert(context.Background(), get, "doc", put, ""); err == nil || !strings.Contains(err.Error(), "-4: the document server") { + t.Fatalf("error %v", err) + } +} + +// 1595/AC2: a type outside the twelve is refused naming the types taken, and nothing is queued. +func TestConvertRefusesOtherTypes(t *testing.T) { + s, b := rig(time.Second) + for _, typ := range []string{"exe", "pdf", "", "docm", "jpg"} { + _, err := s.Convert(context.Background(), get, typ, put, "") + if err == nil || !strings.Contains(err.Error(), "docx") || !strings.Contains(err.Error(), "txt") { + t.Errorf("type %q: %v", typ, err) + } + } + if _, err := s.Convert(context.Background(), "file:///etc/passwd", "docx", put, ""); err == nil { + t.Error("a file link was taken") + } + if len(b.asks) != 0 { + t.Fatalf("%d asks queued", len(b.asks)) + } + if len(Types) != 12 { + t.Fatalf("%d types", len(Types)) + } +} + +// 1595/AC3, AC6: with no answer within the call's bound the verb answers that it is converting, with the ask id, +// and does not fail; convert {ask} then returns the converter's answer once it is written, or converting again. +func TestAConversionNotDoneInTimeIsAnsweredAsConvertingAndRead(t *testing.T) { + s, b := rig(30 * time.Millisecond) + got, err := s.Convert(context.Background(), get, "docx", put, "") + if err != nil { + t.Fatalf("a slow converter failed the call: %v", err) + } + id := b.asks[0].ID + if got["state"] != "converting" || got["ask"] != id { + t.Fatalf("answer %v", got) + } + again, err := s.Read(context.Background(), id) + if err != nil || again["state"] != "converting" || again["ask"] != id { + t.Fatalf("read while converting: %v, %v", again, err) + } + b.mu.Lock() + b.answers[id] = json.RawMessage(`{"size": 77, "sha256": "cd34"}`) + b.mu.Unlock() + done, err := s.Read(context.Background(), id) + if err != nil || done["size"] != float64(77) || done["sha256"] != "cd34" || done["state"] != nil { + t.Fatalf("read after: %v, %v", done, err) + } + if len(b.asks) != 1 { + t.Errorf("%d asks queued, want the one", len(b.asks)) + } + if ConvertWait != 25*time.Second { + t.Errorf("a call waits %s, want 25s", ConvertWait) + } + // Through the tool: ask alone reads; ask with anything else is refused; a made-up ask id is refused. + tl := tools(s)[0] + if r, err := tl.Run(map[string]any{"ask": id}); err != nil || r.(map[string]any)["size"] != float64(77) { + t.Errorf("tool read: %v, %v", r, err) + } + if _, err := tl.Run(map[string]any{"ask": id, "source": get}); err == nil { + t.Error("ask with a source was accepted") + } + if _, err := tl.Run(map[string]any{"ask": "../x"}); err == nil { + t.Error("a made-up ask id was accepted") + } +} + +// 1595/AC4: the tool side opens no credential: its code never names the JWT or a secret file, and the manifest's +// tools bundle is given no env naming one. +func TestTheToolSideHoldsNoCredential(t *testing.T) { + // The tool side is these two files; the converter process (issue 1596) is code of its own, which alone reads the token. + for _, f := range []string{"main.go", "convert.go"} { + raw, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + for _, word := range []string{"jwt", "JWT", ".secret", "os.ReadFile", "os.Open"} { + if strings.Contains(string(raw), word) { + t.Errorf("%s mentions %q", f, word) + } + } + } + m := manifest(t) + for _, a := range m["build"].(map[string]any)["artifacts"].([]any) { + if env, ok := a.(map[string]any)["env"]; ok { + t.Errorf("an artifact is given env %v", env) + } + } +} + +func manifest(t *testing.T) map[string]any { + t.Helper() + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + return m +} + +// 1595/AC5: only-office claims document-converter (scope mesh) serving convert, declares the stream convert-asks as a +// work queue and the state convert-answers (the shape of brother's device-asks and device-answers). +func TestManifestClaimsTheSeat(t *testing.T) { + m := manifest(t) + seats, _ := m["seats"].([]any) + if len(seats) != 1 || seats[0].(map[string]any)["name"] != "document-converter" || seats[0].(map[string]any)["scope"] != "mesh" || + len(seats[0].(map[string]any)["serves"].([]any)) != 1 || seats[0].(map[string]any)["serves"].([]any)[0] != "convert" { + t.Fatalf("the seat document-converter is not declared in seats (the mesh defines no seat a module does not declare): %v", seats) + } + claims := m["claims"].([]any) + c := claims[0].(map[string]any) + if len(claims) != 1 || c["name"] != "document-converter" || c["scope"] != "mesh" || + len(c["serves"].([]any)) != 1 || c["serves"].([]any)[0] != "convert" { + t.Fatalf("claims %v", claims) + } + streams := m["streams"].([]any) + s := streams[0].(map[string]any) + if len(streams) != 1 || s["name"] != "convert-asks" || s["retention"] != "work-queue" || s["max-deliveries"] != float64(3) || + s["events"].([]any)[0] != EventAsked { + t.Fatalf("streams %v", streams) + } + state := m["state"].([]any) + st := state[0].(map[string]any) + if len(state) != 1 || st["name"] != AnswersState || st["ttl-seconds"] != float64(600) { + t.Fatalf("state %v", state) + } + found := false + for _, a := range m["build"].(map[string]any)["artifacts"].([]any) { + found = found || a.(map[string]any)["from"] == "cmd/only-office" + } + if !found { + t.Error("no Go bundle") + } +} + +func TestTheVerbIsListedUnderTheSeat(t *testing.T) { + s, _ := rig(time.Second) + ts := tools(s) + if len(ts) != 1 || ts[0].Name != "document-converter.convert" { + t.Fatalf("tools %v", ts) + } +} diff --git a/modules/only-office/cmd/only-office/converter.go b/modules/only-office/cmd/only-office/converter.go new file mode 100644 index 00000000..15f2119c --- /dev/null +++ b/modules/only-office/cmd/only-office/converter.go @@ -0,0 +1,311 @@ +package main + +// The converter process (novox/hq ADR 0468 §7, ADR 0436 §1, §3, §6; issue 1596). It runs under an account of its own +// and a bus account of its own, is the one consumer of the work queue convert-asks, serves no verb, and alone reads +// the document server's token (the JWT). For each ask it has the document server turn the source into a PDF, fetches +// the PDF from the document server and puts it to the ask's target link, and writes {size, sha256} -- or the error -- +// under the ask's id in the state convert-answers, where the tool side reads it. + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "os/signal" + "strings" + "syscall" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/objectlink" + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go/jetstream" +) + +const ( + // AsksStream is the work-queue stream's bus name: the manifest's convert-asks, capitalised with its hyphen an underscore. + AsksStream = "CONVERT_ASKS" + // ConvertPath is the document server's conversion service, reached on the machine's own published port. The path is + // the one the conversion API's request page names, https://api.onlyoffice.com/docs/docs-api/additional-api/conversion-api/request/ + // (read 2026-10-21): POST to https://documentserver/converter. + ConvertPath = "/converter" + // StaleAfter is how old an ask may be when the converter takes it: older than twice the longest wait, nobody waits. + StaleAfter = 6 * time.Minute + // ConvertRun is how long one conversion, its fetch and its put may take. + ConvertRun = 4 * time.Minute +) + +// errorMeanings are the conversion service's error codes. Read from the conversion API's documentation, page +// https://api.onlyoffice.com/docs/docs-api/additional-api/conversion-api/error-codes/ on 2026-10-21. +var errorMeanings = map[int]string{ + -1: "unknown error", + -2: "conversion timeout error", + -3: "conversion error", + -4: "error while downloading the document file to be converted", + -5: "incorrect password", + -6: "error while accessing the conversion result database", + -7: "input error", + -8: "invalid token", + -9: "the converter cannot determine the output file format", + -10: "size limit exceeded", +} + +// meaningOf is an error code with its meaning, "-4: error while ...". +func meaningOf(code int) string { + if m, ok := errorMeanings[code]; ok { + return fmt.Sprintf("%d: %s", code, m) + } + return fmt.Sprintf("%d: a code the conversion API does not document", code) +} + +// jwtHS256 signs a payload as a JSON Web Token with HMAC-SHA256. +func jwtHS256(secret string, payload []byte) string { + enc := base64.RawURLEncoding + signing := enc.EncodeToString([]byte(`{"alg":"HS256","typ":"JWT"}`)) + "." + enc.EncodeToString(payload) + mac := hmac.New(sha256.New, []byte(secret)) + mac.Write([]byte(signing)) + return signing + "." + enc.EncodeToString(mac.Sum(nil)) +} + +type answerStore interface { + Create(key string, value any) (uint64, error) +} + +type action int + +const ( + actionAck action = iota + actionTerm + actionRetry +) + +// converterWorker answers asks. Everything it touches is handed in, so a test drives it with fakes. +type converterWorker struct { + // JWT reads the document server's secret; only this process calls it. + JWT func() (string, error) + // Server is the document server's address, "http://127.0.0.1:9070". + Server string + // Docs talks to the document server, Store to the object store. + Docs, Store *http.Client + Answers answerStore + Now func() time.Time + Logf func(string, ...any) + NewKey func() string +} + +func (w *converterWorker) logf(f string, a ...any) { + if w.Logf != nil { + w.Logf(f, a...) + } +} + +// Handle takes one queued ask: it answers it under its id and says how the message is settled. +func (w *converterWorker) Handle(body []byte) (action, time.Duration) { + var ask Ask + if err := json.Unmarshal(body, &ask); err != nil || ask.ID == "" { + w.logf("[only-office] converter: a queued ask could not be read: %v", err) + return actionTerm, 0 + } + if age := w.Now().Sub(ask.At); age > StaleAfter { + w.logf("[only-office] converter: ask %s is %s old; nobody waits for it", ask.ID, age.Round(time.Second)) + return actionAck, 0 + } + if !ask.Deadline.IsZero() && w.Now().After(ask.Deadline) { + w.answer(ask.ID, map[string]any{"error": "the ask was taken up after its deadline: nothing was converted for it; ask again"}) + return actionTerm, 0 + } + ctx, cancel := context.WithTimeout(context.Background(), ConvertRun) + defer cancel() + out, err := w.convert(ctx, ask) + if err != nil { + out = map[string]any{"error": err.Error()} + } + if err := w.answer(ask.ID, out); err != nil { + return actionRetry, 5 * time.Second + } + return actionAck, 0 +} + +func (w *converterWorker) answer(id string, v map[string]any) error { + if _, err := w.Answers.Create(id, v); err != nil && !errors.Is(err, stdio.ErrChanged) { + w.logf("[only-office] converter: the answer to %s could not be written: %v", id, err) + var api *jetstream.APIError + if errors.As(err, &api) { + return nil // a refused value would be refused again + } + return err + } + return nil +} + +// convert does one ask: the document server converts, then the PDF is put to the target. +func (w *converterWorker) convert(ctx context.Context, ask Ask) (map[string]any, error) { + if ask.Op != OpConvert { + return nil, fmt.Errorf("the converter does not know the operation %q", ask.Op) + } + if _, err := cleanType(ask.Type); err != nil { + return nil, err + } + secret, err := w.JWT() + if err != nil { + return nil, fmt.Errorf("the document server's token cannot be read: %w", err) + } + key := ask.SHA256 + if key == "" { + key = w.NewKey() + } + req := map[string]any{"async": false, "filetype": ask.Type, "key": key, "outputtype": "pdf", "url": ask.Source} + payload, _ := json.Marshal(req) + req["token"] = jwtHS256(secret, payload) // JWT_IN_BODY: the document server reads the token from the body + reqBody, _ := json.Marshal(req) + hreq, err := http.NewRequestWithContext(ctx, http.MethodPost, w.Server+ConvertPath, bytes.NewReader(reqBody)) + if err != nil { + return nil, err + } + hreq.Header.Set("Content-Type", "application/json") + hreq.Header.Set("Accept", "application/json") + resp, err := w.Docs.Do(hreq) + if err != nil { + return nil, fmt.Errorf("the document server cannot be reached: %w", err) + } + defer resp.Body.Close() + raw, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return nil, fmt.Errorf("the document server's answer cannot be read: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("the document server answered %d", resp.StatusCode) + } + var out struct { + EndConvert bool `json:"endConvert"` + FileURL string `json:"fileUrl"` + Error int `json:"error"` + } + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("the document server's answer is not JSON: %w", err) + } + if out.Error != 0 { + return nil, fmt.Errorf("the document server refused the conversion, error %s", meaningOf(out.Error)) + } + if !out.EndConvert || out.FileURL == "" { + return nil, errors.New("the document server did not finish the conversion: its answer has no file") + } + return w.deliver(ctx, out.FileURL, ask.Target) +} + +// deliver fetches the PDF from the document server into a temporary file and puts it to the target. +func (w *converterWorker) deliver(ctx context.Context, fileURL, target string) (map[string]any, error) { + greq, err := http.NewRequestWithContext(ctx, http.MethodGet, fileURL, nil) + if err != nil { + return nil, err + } + gresp, err := w.Docs.Do(greq) + if err != nil { + return nil, fmt.Errorf("the converted file cannot be fetched from the document server: %w", err) + } + defer gresp.Body.Close() + if gresp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("the document server answered %d for the converted file", gresp.StatusCode) + } + tmp, err := os.CreateTemp("", "converted-*.pdf") + if err != nil { + return nil, fmt.Errorf("a temporary file cannot be made: %w", err) + } + defer os.Remove(tmp.Name()) + defer tmp.Close() + n, err := io.Copy(tmp, io.LimitReader(gresp.Body, objectlink.MaxDocument+1)) + if err != nil { + return nil, fmt.Errorf("the converted file cannot be read: %w", err) + } + if n > objectlink.MaxDocument { + return nil, fmt.Errorf("the converted file is over %d bytes (novox/hq ADR 0468 §3)", objectlink.MaxDocument) + } + if _, err := tmp.Seek(0, io.SeekStart); err != nil { + return nil, err + } + res, err := objectlink.Put(ctx, w.Store, target, tmp, n, objectlink.MaxDocument) + if err != nil { + return nil, fmt.Errorf("the store did not take the PDF: %w", err) + } + return map[string]any{"size": res.Size, "sha256": res.SHA256}, nil +} + +func newKey() string { + b := make([]byte, 16) + rand.Read(b) + return hex.EncodeToString(b) +} + +func readSecret(path string) func() (string, error) { + return func() (string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return "", err + } + s := strings.TrimSpace(string(raw)) + if s == "" { + return "", fmt.Errorf("%s is empty", path) + } + return s, nil + } +} + +// converter is `only-office converter`: it reads its own bus account, connects, and consumes convert-asks until stopped. +func converter() int { + brokerFile, jwtFile, server := os.Getenv("ONLY_OFFICE_BROKER"), os.Getenv("ONLY_OFFICE_JWT"), os.Getenv("ONLY_OFFICE_SERVER") + if brokerFile == "" || jwtFile == "" || server == "" { + fmt.Fprintln(os.Stderr, "[only-office] converter was started without ONLY_OFFICE_BROKER, ONLY_OFFICE_JWT or ONLY_OFFICE_SERVER") + return 2 + } + account, err := process.ReadAccount(brokerFile) + if err != nil { + fmt.Fprintf(os.Stderr, "[only-office] converter: %v\n", err) + return 1 + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + w := &converterWorker{JWT: readSecret(jwtFile), Server: strings.TrimRight(server, "/"), + Docs: &http.Client{Timeout: ConvertRun}, Store: &http.Client{Timeout: ConvertRun}, + Now: time.Now, NewKey: newKey, Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, f+"\n", a...) }} + fmt.Fprintf(os.Stderr, "[only-office] converter is consuming %s as %s\n", AsksStream, account.User) + if err := runConverter(ctx, account, w); err != nil && ctx.Err() == nil { + fmt.Fprintf(os.Stderr, "[only-office] converter stopped: %v\n", err) + return 1 + } + return 0 +} + +// runConverter connects as the converter's own account and consumes convert-asks until ctx ends, answering each ask in +// the state convert-answers. Apart from converter so a test can drive it over a bus of its own. +func runConverter(ctx context.Context, account process.Account, w *converterWorker) error { + conn, err := process.Connect(account) + if err != nil { + return fmt.Errorf("the bus cannot be reached: %w", err) + } + defer conn.Close() + w.Answers = conn.State(AnswersState) + return conn.Consume(ctx, AsksStream, func(msg *process.Message) { + act, delay := w.Handle(msg.Body) + var err error + switch act { + case actionAck: + err = msg.Ack() + case actionRetry: + err = msg.NakWithDelay(delay) + case actionTerm: + err = msg.Term() + } + if err != nil { + w.logf("[only-office] converter: settling %s: %v", msg.Subject, err) + } + }) +} diff --git a/modules/only-office/cmd/only-office/converter_integration_test.go b/modules/only-office/cmd/only-office/converter_integration_test.go new file mode 100644 index 00000000..fd9378f1 --- /dev/null +++ b/modules/only-office/cmd/only-office/converter_integration_test.go @@ -0,0 +1,98 @@ +//go:build integration + +package main + +// 1596/AC6: the tool side and the converter over a real bus (the shape of brother's device_integration_test.go, +// ADR 0436 §7): a JetStream server of the test's own, the work-queue stream convert-asks and its consumer, the state +// convert-answers, and a fake document server. The tool side's Convert queues an ask; the converter takes it and +// answers; Convert returns the answer. Compiled by the merge gate's `go vet -tags integration`; never run on this machine. + +import ( + "context" + "encoding/json" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +// convertAskedSubject is the wire subject the controller composes for the stream's one event: +// "mesh.stream." + module + ".event." + event (mesh-controller internal/broker/modstreams.go). +const convertAskedSubject = "mesh.stream.only-office.event." + EventAsked + +// natsBus is the tool side's bus: the ask published on the stream, the answer read from the state's bucket. +type natsBus struct { + js jetstream.JetStream + kv jetstream.KeyValue +} + +func (b natsBus) Publish(id string, body any) error { + raw, _ := json.Marshal(body) + _, err := b.js.Publish(context.Background(), convertAskedSubject, raw, jetstream.WithMsgID(id)) + return err +} + +func (b natsBus) Answer(id string) (json.RawMessage, error) { + e, err := b.kv.Get(context.Background(), id) + if err != nil { + return nil, nil + } + return e.Value(), nil +} + +func TestTheToolSideAndTheConverterConvertOverARealBus(t *testing.T) { + opts := &server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true} + s, err := server.NewServer(opts) + if err != nil { + t.Fatal(err) + } + go s.Start() + if !s.ReadyForConnections(30 * time.Second) { + t.Fatal("a bus for this test did not come up within 30s") + } + t.Cleanup(func() { s.Shutdown(); s.WaitForShutdown() }) + ctx := context.Background() + nc, err := nats.Connect(s.ClientURL()) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: AsksStream, Subjects: []string{convertAskedSubject}, + Retention: jetstream.WorkQueuePolicy, Discard: jetstream.DiscardNew, MaxAge: 24 * time.Hour, MaxBytes: 8 << 20}); err != nil { + t.Fatal(err) + } + account := process.Account{URL: s.ClientURL(), Node: "novox", Module: "only-office", User: "novox.only-office.converter", Password: "pw"} + if _, err := js.CreateConsumer(ctx, AsksStream, jetstream.ConsumerConfig{Durable: account.QueueConsumerName(), + AckPolicy: jetstream.AckExplicitPolicy, AckWait: 30 * time.Second, MaxDeliver: 3}); err != nil { + t.Fatal(err) + } + kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: account.Module + "_" + AnswersState, TTL: 10 * time.Minute}) + if err != nil { + t.Fatal(err) + } + + d, tgt := newFakeDocs(t), newFakeTarget(t) + w, _ := worker(d) + runCtx, cancel := context.WithCancel(ctx) + defer cancel() + go runConverter(runCtx, account, w) + + svc := &Service{Asks: natsBus{js: js, kv: kv}, Now: time.Now, Wait: 20 * time.Second, Poll: 50 * time.Millisecond} + got, err := svc.Convert(ctx, "http://novox.internal:20001/b/in.docx?X-Amz-Signature=a", "docx", tgt.srv.URL+"/b/in.pdf?sig=1", "") + if err != nil { + t.Fatal(err) + } + if got["size"] != float64(len(d.pdf)) || string(tgt.got) != string(d.pdf) { + t.Fatalf("answer %v, target got %q", got, tgt.got) + } + if len(d.requests) != 1 { + t.Fatalf("%d conversion requests", len(d.requests)) + } +} diff --git a/modules/only-office/cmd/only-office/converter_test.go b/modules/only-office/cmd/only-office/converter_test.go new file mode 100644 index 00000000..fe40c820 --- /dev/null +++ b/modules/only-office/cmd/only-office/converter_test.go @@ -0,0 +1,273 @@ +package main + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "testing" + "time" +) + +const jwtSecret = "module-jwt-secret" + +// fakeDocs stands for the document server's conversion service (the request and answer shapes are the conversion API's: +// https://api.onlyoffice.com/docs/docs-api/additional-api/conversion-api/request/ and .../error-codes/, read +// 2026-10-21) and the file it serves back. +type fakeDocs struct { + mu sync.Mutex + requests []map[string]any + accept string + answer string // the JSON the service answers; {fileUrl} is replaced by the file's address + pdf []byte + srv *httptest.Server +} + +func newFakeDocs(t *testing.T) *fakeDocs { + d := &fakeDocs{pdf: []byte("%PDF-1.7 converted"), answer: `{"endConvert": true, "fileUrl": "{fileUrl}", "percent": 100}`} + d.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + d.mu.Lock() + defer d.mu.Unlock() + switch r.URL.Path { + case ConvertPath: + var m map[string]any + json.NewDecoder(r.Body).Decode(&m) + d.requests = append(d.requests, m) + d.accept = r.Header.Get("Accept") + io.WriteString(w, strings.ReplaceAll(d.answer, "{fileUrl}", "http://"+r.Host+"/cache/out.pdf")) + case "/cache/out.pdf": + w.Write(d.pdf) + default: + w.WriteHeader(404) + } + })) + t.Cleanup(d.srv.Close) + return d +} + +type fakeTarget struct { + mu sync.Mutex + got []byte + puts int + status int + srv *httptest.Server +} + +func newFakeTarget(t *testing.T) *fakeTarget { + f := &fakeTarget{status: 200} + f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + if f.status != 200 { + w.WriteHeader(f.status) + io.WriteString(w, `AccessDeniedAccess Denied.`) + return + } + f.got, _ = io.ReadAll(r.Body) + f.puts++ + })) + t.Cleanup(f.srv.Close) + return f +} + +type answers struct { + mu sync.Mutex + m map[string]map[string]any +} + +func (a *answers) Create(key string, value any) (uint64, error) { + a.mu.Lock() + defer a.mu.Unlock() + raw, _ := json.Marshal(value) + var v map[string]any + json.Unmarshal(raw, &v) + a.m[key] = v + return 1, nil +} + +func worker(d *fakeDocs) (*converterWorker, *answers) { + a := &answers{m: map[string]map[string]any{}} + return &converterWorker{JWT: func() (string, error) { return jwtSecret, nil }, Server: d.srv.URL, + Docs: http.DefaultClient, Store: http.DefaultClient, Answers: a, Now: time.Now, + NewKey: func() string { return "fresh-key" }}, a +} + +func askBody(t *testing.T, target string, sha string) []byte { + raw, err := json.Marshal(Ask{ID: "ask1", Op: OpConvert, At: time.Now(), Deadline: time.Now().Add(time.Minute), + Source: "http://novox.internal:20001/b/in.docx?X-Amz-Signature=a", Type: "docx", Target: target, SHA256: sha}) + if err != nil { + t.Fatal(err) + } + return raw +} + +// verifies a JWT the way a document server does: three parts, HS256 over the first two with the secret. +func verifiesJWT(t *testing.T, token string) map[string]any { + t.Helper() + parts := strings.Split(token, ".") + if len(parts) != 3 { + t.Fatalf("token %q", token) + } + mac := hmac.New(sha256.New, []byte(jwtSecret)) + mac.Write([]byte(parts[0] + "." + parts[1])) + if base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) != parts[2] { + t.Fatal("the token does not verify with the module's secret") + } + payload, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + t.Fatal(err) + } + var m map[string]any + json.Unmarshal(payload, &m) + return m +} + +// 1596/AC1: the server receives one conversion request with the source link, the type, outputtype pdf and a JWT that +// verifies with the module's secret, and the PDF it returns is PUT to target. +func TestTheConverterPostsTheRequestAndPutsThePDF(t *testing.T) { + d, tgt := newFakeDocs(t), newFakeTarget(t) + w, _ := worker(d) + act, _ := w.Handle(askBody(t, tgt.srv.URL+"/b/in.pdf?sig=1", strings.Repeat("ab", 32))) + if act != actionAck { + t.Fatalf("action %v", act) + } + if len(d.requests) != 1 { + t.Fatalf("%d conversion requests", len(d.requests)) + } + r := d.requests[0] + if r["url"] != "http://novox.internal:20001/b/in.docx?X-Amz-Signature=a" || r["filetype"] != "docx" || r["outputtype"] != "pdf" || + r["async"] != false || r["key"] != strings.Repeat("ab", 32) || d.accept != "application/json" { + t.Fatalf("request %v accept %q", r, d.accept) + } + claims := verifiesJWT(t, r["token"].(string)) + for _, k := range []string{"url", "filetype", "outputtype", "key"} { + if claims[k] != r[k] { + t.Errorf("the token's %s is %v, the body's %v", k, claims[k], r[k]) + } + } + if string(tgt.got) != string(d.pdf) || tgt.puts != 1 { + t.Fatalf("the target got %q in %d puts", tgt.got, tgt.puts) + } +} + +// 1596/AC1: without a known SHA-256 the key is a fresh one. +func TestTheKeyIsFreshWhenTheSourceHashIsNotKnown(t *testing.T) { + d, tgt := newFakeDocs(t), newFakeTarget(t) + w, _ := worker(d) + w.Handle(askBody(t, tgt.srv.URL+"/x", "")) + if d.requests[0]["key"] != "fresh-key" { + t.Fatalf("key %v", d.requests[0]["key"]) + } +} + +// 1596/AC2: the answer under the ask id carries the PUT body's size and SHA-256. +func TestTheAnswerCarriesTheSizeAndHashOfWhatWasPut(t *testing.T) { + d, tgt := newFakeDocs(t), newFakeTarget(t) + w, a := worker(d) + w.Handle(askBody(t, tgt.srv.URL+"/x", "")) + sum := sha256.Sum256(d.pdf) + got := a.m["ask1"] + if got["size"] != float64(len(d.pdf)) || got["sha256"] != hex.EncodeToString(sum[:]) || got["error"] != nil { + t.Fatalf("answer %v", got) + } +} + +// 1596/AC3: a server answer with error -4 is answered with the code and its meaning, and nothing is PUT. +func TestAServerErrorIsAnsweredWithItsMeaningAndNothingIsPut(t *testing.T) { + d, tgt := newFakeDocs(t), newFakeTarget(t) + d.answer = `{"error": -4}` + w, a := worker(d) + w.Handle(askBody(t, tgt.srv.URL+"/x", "")) + msg, _ := a.m["ask1"]["error"].(string) + if !strings.Contains(msg, "-4: error while downloading the document file to be converted") { + t.Fatalf("answer %v", a.m["ask1"]) + } + if tgt.puts != 0 { + t.Fatal("something was put") + } + if got := meaningOf(-8); got != "-8: invalid token" { + t.Fatalf("-8 reads %q", got) + } +} + +// 1596/AC4: a failed PUT is answered as a failure naming the store's refusal; the ask is not answered as done. +func TestAFailedPutIsAFailureNamingTheStoresRefusal(t *testing.T) { + d, tgt := newFakeDocs(t), newFakeTarget(t) + tgt.status = 403 + w, a := worker(d) + w.Handle(askBody(t, tgt.srv.URL+"/x", "")) + got := a.m["ask1"] + msg, _ := got["error"].(string) + if !strings.Contains(msg, "403") || !strings.Contains(msg, "AccessDenied") || got["size"] != nil || got["sha256"] != nil { + t.Fatalf("answer %v", got) + } +} + +// An ask whose deadline passed is not converted, and an unreadable ask is ended. +func TestALateAskIsNotConverted(t *testing.T) { + d, tgt := newFakeDocs(t), newFakeTarget(t) + w, a := worker(d) + w.Now = func() time.Time { return time.Now().Add(2 * time.Minute) } + if act, _ := w.Handle(askBody(t, tgt.srv.URL+"/x", "")); act != actionTerm { + t.Fatalf("action %v", act) + } + if len(d.requests) != 0 || a.m["ask1"]["error"] == nil { + t.Fatalf("requests %d, answer %v", len(d.requests), a.m["ask1"]) + } + if act, _ := w.Handle([]byte("nonsense")); act != actionTerm { + t.Fatal("an unreadable ask was kept") + } +} + +// 1596/AC5: the process converter has no serves, is the one consumer of convert-asks and writes convert-answers, and +// its account owns the JWT file. +func TestManifestDeclaresTheConverterProcess(t *testing.T) { + m := manifest(t) + var proc, account map[string]any + for _, r := range m["resources"].([]any) { + res := r.(map[string]any) + if res["type"] == "process" && res["name"] == "only-office-converter" { + proc = res + } + if res["type"] == "user" && res["name"] == "only-office-converter" { + account = res + } + } + if proc == nil || account == nil { + t.Fatalf("process %v account %v", proc, account) + } + if _, serves := proc["serves"]; serves { + t.Error("the converter serves") + } + bus := proc["bus"].(map[string]any) + if q := bus["queues"].([]any); len(q) != 1 || q[0] != "convert-asks" { + t.Errorf("queues %v", q) + } + if s := bus["state"].([]any); len(s) != 1 || s[0] != AnswersState { + t.Errorf("state %v", s) + } + if proc["user"] != "only-office-converter" || account["shell"] != "/usr/bin/nologin" { + t.Errorf("user %v shell %v", proc["user"], account["shell"]) + } + jwt := m["own-secrets"].(map[string]any)["jwt"].(map[string]any) + if jwt["owner"] != "only-office-converter" || jwt["path"] != "${dir:state}/jwt.secret" { + t.Errorf("jwt %v", jwt) + } + broker := m["own-secrets"].(map[string]any)["broker-converter"].(map[string]any) + env := proc["env"].(map[string]any) + if broker["owner"] != "only-office-converter" || env["ONLY_OFFICE_BROKER"] != broker["path"] || env["ONLY_OFFICE_JWT"] != jwt["path"] { + t.Errorf("broker %v env %v", broker, env) + } + // The container still gets the token: server.env names it. + raw, _ := os.ReadFile("../../files/server.env") + if !strings.Contains(string(raw), "JWT_SECRET=${secret:jwt}") { + t.Error("the document server is no longer given the token") + } +} diff --git a/modules/only-office/cmd/only-office/main.go b/modules/only-office/cmd/only-office/main.go new file mode 100644 index 00000000..63f31170 --- /dev/null +++ b/modules/only-office/cmd/only-office/main.go @@ -0,0 +1,24 @@ +// only-office's Go part (novox/hq ADR 0468 §7): the tool side of the seat `document-converter`. A caller asks to +// have a document turned into a PDF, by a signed link to read it from and a signed link to put the PDF to; the +// tool side queues the ask for the module's `converter` process (issue 1596) and reads its answer. It holds no +// credential: the document server's token is the process's alone (ADR 0436 §12). +// +// stdout is the MCP channel; what this module says, it says on stderr. +package main + +import ( + "fmt" + "os" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func main() { + if len(os.Args) > 1 && os.Args[1] == "converter" { + os.Exit(converter()) + } + if err := stdio.Serve("", tools(newService())); err != nil { + fmt.Fprintf(os.Stderr, "[only-office] %v\n", err) + os.Exit(1) + } +} diff --git a/modules/only-office/go.mod b/modules/only-office/go.mod new file mode 100644 index 00000000..b06230a9 --- /dev/null +++ b/modules/only-office/go.mod @@ -0,0 +1,22 @@ +module only-office + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f + github.com/nats-io/nats-server/v2 v2.11.17 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/only-office/go.sum b/modules/only-office/go.sum new file mode 100644 index 00000000..cd9cb802 --- /dev/null +++ b/modules/only-office/go.sum @@ -0,0 +1,27 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f h1:7J/9jOfSfjfYpf8MOj96HwQRViQUkBNMb71RHP2mjn8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/only-office/module.json b/modules/only-office/module.json index 4af651f4..03ed3cc0 100644 --- a/modules/only-office/module.json +++ b/modules/only-office/module.json @@ -14,11 +14,55 @@ "endpoint": "web" } }, + "seats": [ + { + "name": "document-converter", + "scope": "mesh", + "serves": [ + "convert" + ] + } + ], + "claims": [ + { + "name": "document-converter", + "scope": "mesh", + "serves": [ + "convert" + ] + } + ], + "streams": [ + { + "name": "convert-asks", + "events": [ + "convert.asked" + ], + "max-age-days": 1, + "max-mib": 8, + "retention": "work-queue", + "max-deliveries": 3, + "duplicates": 24 + } + ], + "state": [ + { + "name": "convert-answers", + "ttl-seconds": 600 + } + ], "binds": { "route": "${dir:state}/route.json" }, "own-secrets": { - "jwt": "${dir:state}/jwt.secret" + "jwt": { + "path": "${dir:state}/jwt.secret", + "owner": "only-office-converter" + }, + "broker-converter": { + "path": "${dir:state}/broker-converter.secret", + "owner": "only-office-converter" + } }, "listens": [ { @@ -79,10 +123,16 @@ { "id": "state", "type": "directory", - "mode": "0700", + "mode": "0711", "place": ".", "unknown": "program" }, + { + "id": "converter-account", + "type": "user", + "name": "only-office-converter", + "shell": "/usr/bin/nologin" + }, { "id": "server-env", "type": "file", @@ -159,10 +209,48 @@ "${dir:fonts}:/usr/share/fonts/truetype/custom" ], "secrets-in-environment": "run-document-server.sh regenerates JWT_SECRET from the environment on every start and overwrites local.json; not convertible" + }, + { + "id": "converter", + "type": "process", + "name": "only-office-converter", + "artifact": "tools", + "run": [ + "./only-office", + "converter" + ], + "user": "only-office-converter", + "env": { + "ONLY_OFFICE_BROKER": "${dir:state}/broker-converter.secret", + "ONLY_OFFICE_JWT": "${dir:state}/jwt.secret", + "ONLY_OFFICE_SERVER": "http://127.0.0.1:9070" + }, + "bus": { + "queues": [ + "convert-asks" + ], + "state": [ + "convert-answers" + ] + }, + "health": { + "kind": "unit" + } } ], "build": { "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/only-office", + "binary": "only-office", + "loads": [ + "only-office" + ] + }, { "name": "files", "kind": "archive", diff --git a/modules/postgres/go.mod b/modules/postgres/go.mod index 9073f456..b1598d4e 100644 --- a/modules/postgres/go.mod +++ b/modules/postgres/go.mod @@ -1,9 +1,9 @@ module postgres -go 1.25.0 +go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 github.com/jackc/pgx/v5 v5.11.0 ) diff --git a/modules/postgres/go.sum b/modules/postgres/go.sum index a97bf94f..923b4ba3 100644 --- a/modules/postgres/go.sum +++ b/modules/postgres/go.sum @@ -1,7 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0 h1:EK84so3Y8JdFT3BHrzj9cPQQrDUPa/d+IdOGnerlmEI= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6 h1:Sv1Qzix+U1RpaUkw5IbB0IJEyZ+FqgzHmbZrCZkcWf0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 h1:Cdl5N56QPfmFBoLHrDfvvyc5Q+PUnTjf3fgYD7imcco= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/modules/route-adapter/go.mod b/modules/route-adapter/go.mod index 3fcef8ed..a4ec98b8 100644 --- a/modules/route-adapter/go.mod +++ b/modules/route-adapter/go.mod @@ -1,5 +1,5 @@ module route-adapter -go 1.25.0 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 diff --git a/modules/route-adapter/go.sum b/modules/route-adapter/go.sum index 1a769534..6162079d 100644 --- a/modules/route-adapter/go.sum +++ b/modules/route-adapter/go.sum @@ -1,4 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0 h1:EK84so3Y8JdFT3BHrzj9cPQQrDUPa/d+IdOGnerlmEI= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6 h1:Sv1Qzix+U1RpaUkw5IbB0IJEyZ+FqgzHmbZrCZkcWf0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 h1:Cdl5N56QPfmFBoLHrDfvvyc5Q+PUnTjf3fgYD7imcco= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=