diff --git a/modules/brother/cmd/brother/convert.go b/modules/brother/cmd/brother/convert.go new file mode 100644 index 00000000..ba7e1f21 --- /dev/null +++ b/modules/brother/cmd/brother/convert.go @@ -0,0 +1,181 @@ +package main + +// Conversion before printing (novox/hq ADR 0468 clause 7, issue 1600). The printer takes PDF and JPEG. A stored +// document of a type the module only-office converts (seat document-converter) is turned into a PDF first: the device +// process signs a GET link on the document and a PUT link on .pdf with the binding's key, calls +// document-converter.convert, checks that the PDF the converter says it put is the PDF the bucket holds, and prints +// that. Any other type is refused, naming the types taken. + +import ( + "archive/zip" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "unicode/utf8" + + "git.novox.be/novox/mesh-sdk/go/objectlink" +) + +// ConvertTypes are the types document-converter takes (clause 7). +var ConvertTypes = []string{"docx", "doc", "xlsx", "xls", "pptx", "ppt", "odt", "ods", "odp", "rtf", "txt", "html"} + +// contentTypes are the media types of a fetched document, by the type the converter is asked for. +var contentTypes = map[string]string{ + "text/html": "html", "application/xhtml+xml": "html", "text/plain": "txt", "application/rtf": "rtf", "text/rtf": "rtf", + "application/msword": "doc", "application/vnd.ms-excel": "xls", "application/vnd.ms-powerpoint": "ppt", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx", + "application/vnd.openxmlformats-officedocument.presentationml.presentation": "pptx", + "application/vnd.oasis.opendocument.text": "odt", + "application/vnd.oasis.opendocument.spreadsheet": "ods", + "application/vnd.oasis.opendocument.presentation": "odp", +} + +// the first bytes of an OLE compound file (the legacy Word, Excel and PowerPoint formats), and the stream names that +// tell them apart, as UTF-16 (MS-CFB: directory entry names are UTF-16LE). +var oleMagic = []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1} + +func utf16le(s string) []byte { + var b []byte + for _, r := range s { + b = append(b, byte(r), byte(r>>8)) + } + return b +} + +// ConvertibleType is the type a document is, among the converter's twelve, told by its bytes and, for a document that +// came from a web server, by its Content-Type; "" when it is none of them. A document that is a PDF or a JPEG is +// not asked about: the printer takes it as it is. +func ConvertibleType(doc []byte, contentType string) string { + if i := strings.IndexByte(contentType, ';'); i >= 0 { + contentType = contentType[:i] + } + byHeader := contentTypes[strings.ToLower(strings.TrimSpace(contentType))] + switch { + case bytes.HasPrefix(doc, []byte("PK\x03\x04")): + if t := zipType(doc); t != "" { + return t + } + return "" + case bytes.HasPrefix(doc, oleMagic): + for name, t := range map[string]string{"WordDocument": "doc", "Workbook": "xls", "PowerPoint Document": "ppt"} { + if bytes.Contains(doc, utf16le(name)) { + return t + } + } + return "" + case bytes.HasPrefix(doc, []byte(`{\rtf`)): + return "rtf" + } + if byHeader == "html" || looksLikeHTML(doc) { + if utf8.Valid(doc) && !bytes.Contains(doc, []byte{0}) { + return "html" + } + return "" + } + if utf8.Valid(doc) && !bytes.Contains(doc, []byte{0}) && len(doc) > 0 { + return "txt" + } + return "" +} + +func looksLikeHTML(doc []byte) bool { + head := doc + if len(head) > 512 { + head = head[:512] + } + s := strings.ToLower(strings.TrimLeft(strings.TrimPrefix(string(head), "\xef\xbb\xbf"), " \t\r\n")) + return strings.HasPrefix(s, "`, + "_rels/.rels": ``, + "word/document.xml": ``, + } { + w, err := zw.Create(name) + if err != nil { + t.Fatal(err) + } + w.Write([]byte(body)) + } + zw.Close() + return buf.Bytes() +} + +// converterFake stands for document-converter.convert: it records the call and, as the converter process would, puts a +// PDF in the bucket under the key the target link names and answers its size and SHA-256. +type converterFake struct { + r *printerRig + calls []map[string]any + pdf []byte + answer func(body map[string]any, stored []byte) (json.RawMessage, error) +} + +func withConverter(r *printerRig, pdf []byte) *converterFake { + c := &converterFake{r: r, pdf: pdf} + r.worker.Convert = func(body map[string]any) (json.RawMessage, error) { + c.calls = append(c.calls, body) + target, _ := url.Parse(body["target"].(string)) + r.objects.Put(context.Background(), strings.TrimPrefix(target.Path, "/"), "application/pdf", c.pdf) + if c.answer != nil { + return c.answer(body, c.pdf) + } + return json.Marshal(map[string]any{"size": len(c.pdf), "sha256": sha256Hex(c.pdf)}) + } + return c +} + +// 1600/AC1: a stored .docx is converted: document-converter.convert is called with a GET link on the source and a PUT +// link on .pdf, both signed in the bucket and expiring within ten minutes, and the PDF is printed. +func TestAStoredDocxIsConvertedThenPrinted(t *testing.T) { + fake := newIPPFake(t, 51, 5) + r := newRig(t, `{"printer":"192.0.2.10"}`, "pw") + doc := docx(t) + key := sha256Hex(doc) + r.objects.Put(t.Context(), key, "application/octet-stream", doc) + pdf := pdfOf(2500) + conv := withConverter(r, pdf) + got, err := r.call(t, "printer.print", map[string]any{"object": key}) + if err != nil { + t.Fatal(err) + } + if len(conv.calls) != 1 { + t.Fatalf("%d converter calls", len(conv.calls)) + } + call := conv.calls[0] + src, _ := url.Parse(call["source"].(string)) + dst, _ := url.Parse(call["target"].(string)) + if src.Path != "/"+key || dst.Path != "/"+key+".pdf" || call["type"] != "docx" || call["sha256"] != key { + t.Errorf("call = %v", call) + } + for _, u := range []*url.URL{src, dst} { + if secs, _ := strconv.Atoi(u.Query().Get("X-Amz-Expires")); secs < 1 || time.Duration(secs)*time.Second > 10*time.Minute { + t.Errorf("%s lives %d s", u, secs) + } + } + if strings.Join(r.objects.links, ",") != "GET "+key+" 600,PUT "+key+".pdf 600" { + t.Errorf("links signed: %v", r.objects.links) + } + m := got.(map[string]any) + if m["job-id"] != float64(51) || m["format"] != "application/pdf" || m["object"] != key+".pdf" || m["converted-from"] != key { + t.Errorf("answer = %v", m) + } + if _, sent := fake.request(t); string(sent) != string(pdf) { + t.Error("the printer was not sent the converted PDF") + } +} + +// 1600/AC2: a converter answer whose SHA-256 differs from the stored PDF's is refused and nothing is printed. +func TestAConverterAnswerOfAnotherHashPrintsNothing(t *testing.T) { + fake := newIPPFake(t, 1, 3) + r := newRig(t, `{"printer":"192.0.2.10"}`, "pw") + doc := docx(t) + r.objects.Put(t.Context(), sha256Hex(doc), "application/octet-stream", doc) + conv := withConverter(r, pdfOf(900)) + conv.answer = func(map[string]any, []byte) (json.RawMessage, error) { + return json.Marshal(map[string]any{"size": 900, "sha256": sha256Hex([]byte("another pdf"))}) + } + _, err := r.call(t, "printer.print", map[string]any{"object": sha256Hex(doc)}) + if err == nil || !strings.Contains(err.Error(), "nothing is printed") { + t.Fatalf("error = %v", err) + } + if fake.body != nil { + t.Error("something was printed") + } +} + +// 1600/AC3: a converter error is answered with its code and meaning. +func TestAConverterErrorIsAnsweredWithItsCodeAndMeaning(t *testing.T) { + fake := newIPPFake(t, 1, 3) + r := newRig(t, `{"printer":"192.0.2.10"}`, "pw") + doc := docx(t) + r.objects.Put(t.Context(), sha256Hex(doc), "application/octet-stream", doc) + conv := withConverter(r, pdfOf(10)) + conv.answer = func(map[string]any, []byte) (json.RawMessage, error) { + return nil, errors.New("the document server refused the conversion, error -4: error while downloading the document file to be converted") + } + _, err := r.call(t, "printer.print", map[string]any{"object": sha256Hex(doc)}) + if err == nil || !strings.Contains(err.Error(), "-4: error while downloading the document file to be converted") { + t.Fatalf("error = %v", err) + } + // An error the converter wrote in its answer is the same. + conv.answer = func(map[string]any, []byte) (json.RawMessage, error) { + return json.RawMessage(`{"error": "-3: conversion error"}`), nil + } + if _, err := r.call(t, "printer.print", map[string]any{"object": sha256Hex(doc)}); err == nil || !strings.Contains(err.Error(), "-3: conversion error") { + t.Fatalf("error = %v", err) + } + if fake.body != nil { + t.Error("something was printed") + } +} + +// 1600/AC4: a type that is neither PDF, JPEG nor one of the converter's twelve is refused naming the types taken, and +// convert is not called. +func TestAnotherTypeIsRefusedNamingTheTypesTaken(t *testing.T) { + fake := newIPPFake(t, 1, 3) + r := newRig(t, `{"printer":"192.0.2.10"}`, "pw") + exe := []byte("MZ\x90\x00\x03\x00\x00\x00\x04\x00\x00\x00\xff\xff\x00\x00") + r.objects.Put(t.Context(), sha256Hex(exe), "application/octet-stream", exe) + conv := withConverter(r, pdfOf(10)) + _, err := r.call(t, "printer.print", map[string]any{"object": sha256Hex(exe)}) + if err == nil || !strings.Contains(err.Error(), "docx, doc, xlsx, xls, pptx, ppt, odt, ods, odp, rtf, txt, html") { + t.Fatalf("error = %v", err) + } + if len(conv.calls) != 0 || fake.body != nil { + t.Errorf("%d converter calls, printed %v", len(conv.calls), fake.body != nil) + } +} + +// 1600: each of the twelve types is told by its bytes (and a web page also by its Content-Type). +func TestTheTypesAreToldByTheirBytes(t *testing.T) { + ole := func(stream string) []byte { + return append(append([]byte{}, oleMagic...), append(make([]byte, 64), utf16le(stream)...)...) + } + odf := func(mime string) []byte { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, _ := zw.Create("mimetype") + w.Write([]byte(mime)) + zw.Close() + return buf.Bytes() + } + ooxml := func(part string) []byte { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, _ := zw.Create(part) + w.Write([]byte("")) + zw.Close() + return buf.Bytes() + } + for want, c := range map[string]struct { + doc []byte + ct string + }{ + "docx": {docx(t), ""}, "xlsx": {ooxml("xl/workbook.xml"), ""}, "pptx": {ooxml("ppt/presentation.xml"), ""}, + "doc": {ole("WordDocument"), ""}, "xls": {ole("Workbook"), ""}, "ppt": {ole("PowerPoint Document"), ""}, + "odt": {odf("application/vnd.oasis.opendocument.text"), ""}, "ods": {odf("application/vnd.oasis.opendocument.spreadsheet"), ""}, + "odp": {odf("application/vnd.oasis.opendocument.presentation"), ""}, + "rtf": {[]byte(`{\rtf1\ansi hello}`), ""}, "txt": {[]byte("just some words\n"), ""}, + "html": {[]byte("hi"), ""}, "html ": {[]byte("

a fragment

"), "text/html; charset=utf-8"}, + } { + if got := ConvertibleType(c.doc, c.ct); got != strings.TrimSpace(want) { + t.Errorf("%s: told as %q", want, got) + } + } + for name, doc := range map[string][]byte{"empty": {}, "binary": {0, 1, 2, 3}, "a zip of nothing known": ooxml("other.xml"), "ole of nothing known": ole("Other")} { + if got := ConvertibleType(doc, ""); got != "" { + t.Errorf("%s: told as %q", name, got) + } + } +} + +// 1600/AC5: the device process invokes seat:document-converter.convert. +func TestTheDeviceProcessInvokesTheConverter(t *testing.T) { + for _, r := range readManifest(t).Resources { + if r["id"] == "device" { + inv, _ := r["bus"].(map[string]any)["invokes"].([]any) + if len(inv) != 1 || inv[0] != "seat:document-converter.convert" { + t.Errorf("invokes = %v", inv) + } + return + } + } + t.Fatal("no device process") +} diff --git a/modules/brother/cmd/brother/deviceproc.go b/modules/brother/cmd/brother/deviceproc.go index cc2e412d..dc6f8e67 100644 --- a/modules/brother/cmd/brother/deviceproc.go +++ b/modules/brother/cmd/brother/deviceproc.go @@ -69,6 +69,8 @@ type deviceWorker struct { Node string // Capture scans; nil is the device's own (a test puts a stand-in). Capture func(ctx context.Context, req ScanRequest) ([]Capture, error) + // Convert calls the seat document-converter's convert verb as this process's own account (clause 7); nil: none. + Convert func(body map[string]any) (json.RawMessage, error) // Fetcher is the guarded fetch of a web link (fetch.go); it reads its facts when it is first needed. Fetcher func() (*Fetcher, error) // answerPart is the raw size of a document part in an answer; zero is AnswerPartSize (a test makes it larger). @@ -386,6 +388,7 @@ func (w *deviceWorker) print(ctx context.Context, ask Ask) (map[string]any, erro } key := ask.Object var doc []byte + contentType := "" switch { case ask.URL != "": fetched, err := w.fetch(ctx, ask.URL) @@ -393,7 +396,7 @@ func (w *deviceWorker) print(ctx context.Context, ask Ask) (map[string]any, erro return nil, err } // The fetched document is kept once under its SHA-256, as a scan is, and printed as that object. - doc, key = fetched.Body, sha256Hex(fetched.Body) + doc, key, contentType = fetched.Body, sha256Hex(fetched.Body), fetched.ContentType if err := Store(ctx, objects, key, fetched.ContentType, doc); err != nil { return nil, err } @@ -418,8 +421,15 @@ func (w *deviceWorker) print(ctx context.Context, ask Ask) (map[string]any, erro } } format, _, err := SniffFormat(doc) + var convertedFrom string if err != nil { - return nil, err + // Not a PDF or a JPEG: a type the converter takes becomes a PDF first (clause 7), any other is refused. + pdf, pdfKey, cerr := w.toPDF(ctx, objects, key, doc, contentType) + if cerr != nil { + return nil, cerr + } + doc, convertedFrom, key = pdf, key, pdfKey + format = "application/pdf" } // The printer is asked what it takes before the job, and a format it does not list is refused with the list // (novox/hq ADR 0468 clause 5): a printer that cannot be asked prints nothing. @@ -434,8 +444,12 @@ func (w *deviceWorker) print(ctx context.Context, ask Ask) (map[string]any, erro if err != nil { return nil, err } - return map[string]any{"job-id": res.JobID, "job-state": res.State, "job-state-code": res.StateCode, - "job-state-reasons": res.Reasons, "format": format, "copies": ask.Copies, "sides": ask.Sides, "media": ask.Media, "object": key}, nil + out := map[string]any{"job-id": res.JobID, "job-state": res.State, "job-state-code": res.StateCode, + "job-state-reasons": res.Reasons, "format": format, "copies": ask.Copies, "sides": ask.Sides, "media": ask.Media, "object": key} + if convertedFrom != "" { + out["converted-from"] = convertedFrom + } + return out, nil } // fetch gets a web link under the rules of fetch.go. @@ -598,6 +612,11 @@ func runDevice(ctx context.Context, account process.Account, w *deviceWorker) er } defer conn.Close() w.Answers = conn.State(AnswersState) + // A conversion takes longer than the default call bound (30 s); the tool side waits ConvertWait for the converter. + process.CallTimeout = ConvertCallWait + w.Convert = func(body map[string]any) (json.RawMessage, error) { + return conn.Call("seat:document-converter.convert", body) + } if w.Node == "" { w.Node = account.Node } diff --git a/modules/brother/cmd/brother/documents_test.go b/modules/brother/cmd/brother/documents_test.go index f58d318b..87e0d014 100644 --- a/modules/brother/cmd/brother/documents_test.go +++ b/modules/brother/cmd/brother/documents_test.go @@ -414,9 +414,9 @@ func TestPrinterPrintRefusesWhatIsNotWellFormed(t *testing.T) { if _, err := r.call(t, "printer.print", map[string]any{"object": dk}); err == nil || !strings.Contains(err.Error(), "0x0400") { t.Errorf("a refused job: error = %v", err) } - // A kept object that is not a document the printer takes is not sent. + // A kept object that is no document the printer takes, nor one the converter takes (1600), is not sent. fake.status, fake.body = 0, nil - r.objects.Put(t.Context(), key, "text/plain", []byte("hello")) + r.objects.Put(t.Context(), key, "application/octet-stream", []byte{0x00, 0x01, 0xFE, 0xFF, 0x02}) if _, err := r.call(t, "printer.print", map[string]any{"object": key}); err == nil || !strings.Contains(err.Error(), "neither a PDF nor a JPEG") || fake.body != nil { t.Errorf("a text object: error = %v, sent %d bytes", err, len(fake.body)) } diff --git a/modules/brother/cmd/brother/main.go b/modules/brother/cmd/brother/main.go index a1a7709a..0b0f1bf2 100644 --- a/modules/brother/cmd/brother/main.go +++ b/modules/brother/cmd/brother/main.go @@ -111,7 +111,7 @@ func tools(settingsFile string, now func() time.Time) []stdio.Tool { return nil, err } return &Service{Settings: s, Node: os.Getenv("MESH_NODE"), Now: now, Asks: askBus, - AnswerWait: answerWait, ScanWait: scanWait, AnswerPoll: answerPoll, + AnswerWait: answerWait, ScanWait: scanWait, AnswerPoll: answerPoll, PrintWait: PrintWait, QuickWait: quickWait, FeederWait: feederWait, Jobs: jobs, Upload: uploadVia, Bucket: func() (string, error) { return ReadBucket(os.Getenv("BROTHER_BUCKET")) }}, nil } diff --git a/modules/brother/cmd/brother/objects_test.go b/modules/brother/cmd/brother/objects_test.go index b06708f8..ab520ef9 100644 --- a/modules/brother/cmd/brother/objects_test.go +++ b/modules/brother/cmd/brother/objects_test.go @@ -24,6 +24,8 @@ type memObjects struct { puts int // corrupt, when set, makes a GET of a link answer bytes that are not the kept ones. corrupt bool + // links records each link signed: " ". + links []string } func newMemObjects() *memObjects { return &memObjects{kept: map[string][]byte{}} } @@ -76,6 +78,9 @@ func (m *memObjects) Delete(_ context.Context, key string) error { // Link is a link the bucket in memory serves itself (RoundTrip): the key in the path and the life in the query, as a // presigned link says them (X-Amz-Expires). func (m *memObjects) Link(method, key string, expires time.Duration) (string, error) { + m.mu.Lock() + m.links = append(m.links, method+" "+key+" "+strconv.Itoa(int(expires/time.Second))) + m.mu.Unlock() return "http://" + m.Bucket() + ".objects.test/" + key + "?X-Amz-Expires=" + strconv.Itoa(int(expires/time.Second)), nil } diff --git a/modules/brother/cmd/brother/printer.go b/modules/brother/cmd/brother/printer.go index 9d67399d..7192aab8 100644 --- a/modules/brother/cmd/brother/printer.go +++ b/modules/brother/cmd/brother/printer.go @@ -36,6 +36,11 @@ const ( // DeviceProcessID is the device process's id as module.json's resource "device" names it: the controller // delivers its bus account as the own secret broker-device. DeviceProcessID = "device" + // ConvertCallWait is how long the device process waits for document-converter.convert (the tool side of + // only-office waits two minutes for the converter). + ConvertCallWait = 130 * time.Second + // PrintWait is how long printer.print waits for the device process: a fetch and a conversion come before the job. + PrintWait = 4 * time.Minute // AnswerWait is how long a verb waits for the device process; ScanWait how long a scan does. AnswerWait = 60 * time.Second ScanWait = 180 * time.Second @@ -381,7 +386,8 @@ func (s *Service) PrinterPrint(ctx context.Context, a Ask) (map[string]any, erro from["path"], from["node"] = a.Path, a.Node a.Object, a.Path, a.Node = key, "", "" } - out, err := s.call(ctx, a, s.AnswerWait) + wait := max(s.AnswerWait, s.PrintWait) + out, err := s.call(ctx, a, wait) if err != nil { return nil, err } diff --git a/modules/brother/cmd/brother/scan.go b/modules/brother/cmd/brother/scan.go index e8ddf14b..0475c433 100644 --- a/modules/brother/cmd/brother/scan.go +++ b/modules/brother/cmd/brother/scan.go @@ -28,6 +28,8 @@ type Service struct { // AnswerWait is how long a verb waits for the device process's answer (ScanWait for a scan); AnswerPoll how // often it looks. AnswerWait, ScanWait, AnswerPoll time.Duration + // PrintWait is how long printer.print waits: a fetch and a conversion come before the job. + PrintWait time.Duration // QuickWait is how long a feeder scan is waited for in one call, FeederWait how long it is followed. QuickWait, FeederWait time.Duration // Jobs are the scans this process follows. diff --git a/modules/brother/module.json b/modules/brother/module.json index 3a637020..df2bed16 100644 --- a/modules/brother/module.json +++ b/modules/brother/module.json @@ -258,6 +258,9 @@ ], "emits": [ "scan.added" + ], + "invokes": [ + "seat:document-converter.convert" ] }, "health": {