Scan several glass pages into one PDF with scanner.scan document start, an id and finish (issue 1538)

This commit is contained in:
2026-10-21 01:45:53 +02:00
parent 17f3c45182
commit a75e11e814
7 changed files with 403 additions and 21 deletions
+1 -1
View File
@@ -63,7 +63,7 @@ makes it (mode 0755) when it files the first scan.
| verb (tool) | does |
|---|---|
| `scanner.scan` | `source` (`glass`, default, or `feeder`) and `duplex` (feeder only): asks the device process to scan, crop (glass only), keep the PDF in the bucket under its sha256 and say `scan.added`; then fetches it in parts, checks the sha256 and files it in `scan-folder`, in the background, so a scan is filed whether or not the caller still waits. A glass scan is waited for up to 180 s. A feeder scan is waited for 25 s (the mesh console gives up after 30 s); if it is not done, the answer is `started: true` with its `id`, and `scanner.scan {id}` reads the outcome (`state` running, done with the event's body and `path`, or failed with `why`). The device process ends a feeder job at 20 s a page (at least 3 min, at most 30 min). A scan whose PDF the tool could not fetch (over 100 MiB) is refused before it is kept or announced. Answers the event's body and `path` |
| `scanner.scan` | `source` (`glass`, default, or `feeder`) and `duplex` (feeder only): asks the device process to scan, crop (glass only), keep the PDF in the bucket under its sha256 and say `scan.added`; then fetches it in parts, checks the sha256 and files it in `scan-folder`, in the background, so a scan is filed whether or not the caller still waits. A glass scan is waited for up to 180 s. Several glass pages in one PDF (a page turned over, a stapled document): `document` `start` scans page 1 into a new document and answers its id (nothing is filed yet), `document` with that id scans the next page, and `document` with the id and `finish` true closes it into one multi-page PDF, filed once with one `scan.added` (nothing is scanned). The pages are kept in the bucket under `pending/<id>/` (not in memory, so a restart of the device process loses none); a document that gets no page for 30 minutes is closed and filed so by the tool process (a restart of the tool process forgets that timer; `finish` still works). The pages in the bucket are not deleted, the bucket having no delete. A feeder scan is waited for 25 s (the mesh console gives up after 30 s); if it is not done, the answer is `started: true` with its `id`, and `scanner.scan {id}` reads the outcome (`state` running, done with the event's body and `path`, or failed with `why`). The device process ends a feeder job at 20 s a page (at least 3 min, at most 30 min). A scan whose PDF the tool could not fetch (over 100 MiB) is refused before it is kept or announced. Answers the event's body and `path` |
| `scanner.document` | one part of a kept scan (`object`, `part` from 1): `part`, `parts`, `size`, `sha256` (of the whole), `data` (base64). Join the parts and check the sha256 |
| `scanner.status` | whether the scanner answers, its state (Idle or Processing) and its document feeder's state; a scanner that does not answer is `answers: false` with why. This is how its health is read: the module has no long-running resource to declare a health check on |
| `scanner.list` | the newest filed scans, newest first (`limit`, default 20). Read-only |
+26
View File
@@ -132,3 +132,29 @@ func (c Capture) Body(node, colorMode string) map[string]any {
"scanned-at": c.At.UTC().Format(time.RFC3339),
}
}
// GlassImage is a page scanned from the glass.
type GlassImage struct {
JPEG []byte
DPI, Width, Height int
Cropped bool
}
// GlassPage scans one page from the glass for a document of several pages: cropped to its document when the settings
// say so (the first document, never split in several), with the resolution the scanner gave.
func (s *Service) GlassPage(ctx context.Context) (page GlassImage, err error) {
scans, err := NewClient(s.Settings.Scanner).ScanPages(ctx, ScanOptions{Resolution: s.Settings.Resolution, ColorMode: s.Settings.ColorMode})
if err != nil {
return page, err
}
dpi, err := Density(scans[0])
if err != nil {
return page, err
}
cut, err := Crop(scans[0], s.Settings.Crop)
if err != nil {
return page, err
}
page.JPEG, page.DPI, page.Width, page.Height, page.Cropped = cut.JPEG, dpi, cut.Width, cut.Height, cut.Cropped
return page, nil
}
+136
View File
@@ -73,6 +73,8 @@ type deviceWorker struct {
mu sync.Mutex
pending map[string]*pendingUpload
// docs serialises the work on open documents.
docs sync.Mutex
}
func (w *deviceWorker) logf(format string, a ...any) {
@@ -184,6 +186,9 @@ func (w *deviceWorker) scan(ctx context.Context, ask Ask) (map[string]any, error
return nil, err
}
svc := &Service{Settings: set, Node: w.Node, Now: w.Now}
if ask.Document != "" {
return w.scanDocument(ctx, ask, svc, objects)
}
capture := svc.Capture
if w.Capture != nil {
capture = w.Capture
@@ -551,3 +556,134 @@ func runDevice(ctx context.Context, account process.Account, w *deviceWorker) er
}
})
}
// An open document (issue 1538) is kept in the bucket, not in memory, so that a restart of the device process
// loses none of its pages: each page's JPEG under pending/<id>/<n>.jpg and a manifest, pending/<id>/manifest.json,
// that says how many pages there are and, once it is closed, the key of its PDF. The pages are not deleted when it
// is closed (the bucket has no delete); they are the bucket's to age out.
const mostDocumentPages = 100
type docPage struct {
Key string `json:"key"`
DPI int `json:"dpi"`
Width int `json:"width"`
Height int `json:"height"`
Cropped bool `json:"cropped"`
}
type docManifest struct {
Started time.Time `json:"started"`
Pages []docPage `json:"pages"`
Finished string `json:"finished,omitempty"` // the closed document's object key
}
func manifestKey(id string) string { return "pending/" + id + "/manifest.json" }
func loadManifest(ctx context.Context, o Objects, id string) (*docManifest, error) {
has, err := o.Has(ctx, manifestKey(id))
if err != nil {
return nil, err
}
if !has {
return nil, fmt.Errorf("no document %s is open: its id is wrong, or it was closed", id)
}
raw, err := o.Get(ctx, manifestKey(id))
if err != nil {
return nil, err
}
var m docManifest
if err := json.Unmarshal(raw, &m); err != nil {
return nil, fmt.Errorf("document %s: its manifest cannot be read: %w", id, err)
}
if m.Finished != "" {
return nil, fmt.Errorf("document %s is closed already, as %s", id, m.Finished)
}
return &m, nil
}
func saveManifest(ctx context.Context, o Objects, id string, m *docManifest) error {
raw, _ := json.Marshal(m)
return o.Put(ctx, manifestKey(id), "application/json", raw)
}
// scanDocument adds a glass page to an open document (document start opens one, with the ask's id as its id), or
// with finish closes the document into one multi-page PDF, kept and announced once like any scan.
func (w *deviceWorker) scanDocument(ctx context.Context, ask Ask, svc *Service, objects Objects) (map[string]any, error) {
if ask.Source == "feeder" || ask.Duplex {
return nil, fmt.Errorf("a document of several pages is scanned from the glass, page by page; the feeder takes the whole stack in one scan")
}
id := ask.Document
if id == "start" {
if ask.Finish {
return nil, fmt.Errorf("finish closes an open document: name its id, not start")
}
id = ask.ID
}
if !uploadPattern.MatchString(id) {
return nil, fmt.Errorf("document %q is not an id: letters, digits, - and _, at most 64", id)
}
w.docs.Lock()
defer w.docs.Unlock()
var m *docManifest
if ask.Document == "start" {
m = &docManifest{Started: w.Now().UTC()}
} else {
var err error
if m, err = loadManifest(ctx, objects, id); err != nil {
return nil, err
}
}
if ask.Finish {
return w.finishDocument(ctx, id, m, svc, objects)
}
if len(m.Pages) >= mostDocumentPages {
return nil, fmt.Errorf("document %s has %d pages, the most one takes: close it", id, len(m.Pages))
}
page, err := svc.GlassPage(ctx)
if err != nil {
return nil, err
}
key := fmt.Sprintf("pending/%s/%03d.jpg", id, len(m.Pages)+1)
if err := objects.Put(ctx, key, "image/jpeg", page.JPEG); err != nil {
return nil, err
}
m.Pages = append(m.Pages, docPage{Key: key, DPI: page.DPI, Width: page.Width, Height: page.Height, Cropped: page.Cropped})
if err := saveManifest(ctx, objects, id, m); err != nil {
return nil, err
}
return map[string]any{"document": id, "pages": len(m.Pages), "state": "open",
"next": "scanner.scan {document: \"" + id + "\"} adds a page; scanner.scan {document: \"" + id + "\", finish: true} closes it into one PDF"}, nil
}
// finishDocument builds the open document's pages into one PDF and keeps and announces it.
func (w *deviceWorker) finishDocument(ctx context.Context, id string, m *docManifest, svc *Service, objects Objects) (map[string]any, error) {
if len(m.Pages) == 0 {
return nil, fmt.Errorf("document %s has no page", id)
}
pages := make([]PDFPage, len(m.Pages))
for i, p := range m.Pages {
jpg, err := objects.Get(ctx, p.Key)
if err != nil {
return nil, fmt.Errorf("document %s: page %d is not kept: %w", id, i+1, err)
}
pages[i] = PDFPage{JPEG: jpg, DPI: p.DPI}
}
first := m.Pages[0]
c, err := captureOf(pages, Cropped{Width: first.Width, Height: first.Height, Cropped: first.Cropped}, first.DPI, w.Now())
if err != nil {
return nil, err
}
set, err := w.Settings()
if err != nil {
return nil, err
}
answer, err := w.keepOne(ctx, objects, set, c)
if err != nil {
return nil, err
}
m.Finished = answer["key"].(string)
if err := saveManifest(ctx, objects, id, m); err != nil {
return nil, fmt.Errorf("document %s is kept as %s, but could not be marked closed: %w", id, m.Finished, err)
}
return answer, nil
}
+115
View File
@@ -1,9 +1,13 @@
package main
import (
"bytes"
"image"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
// withDensity puts a JFIF header of dpi into a JPEG that has none (Go's encoder writes none).
@@ -67,3 +71,114 @@ func TestOneDocumentOnTheGlassIsAnsweredAsBefore(t *testing.T) {
t.Errorf("answer = %v, %d events", body, len(*r.said))
}
}
func docArgs(document string, finish bool) map[string]any {
a := map[string]any{"document": document}
if finish {
a["finish"] = true
}
return a
}
// 1538: start scans page 1 into a new document and files nothing; the id adds the next page; finish closes the
// pages into one PDF, one object, one scan.added, one file, and scans nothing.
func TestAGlassDocumentOfTwoPagesIsOnePDFFiledOnce(t *testing.T) {
f := newFakeScanner(t, fixture(t, "scan.jpg"))
r, folder := scanRig(t, f, Settings{Crop: false, Resolution: 200})
got, err := r.call(t, "scanner.scan", docArgs("start", false))
if err != nil {
t.Fatal(err)
}
id, _ := got.(map[string]any)["document"].(string)
if id == "" || got.(map[string]any)["pages"] != float64(1) {
t.Fatalf("start answered %v", got)
}
if got, err = r.call(t, "scanner.scan", docArgs(id, false)); err != nil || got.(map[string]any)["pages"] != float64(2) {
t.Fatalf("second page: %v, %v", got, err)
}
if e, _ := os.ReadDir(folder); len(e) != 0 || len(*r.said) != 0 {
t.Fatalf("an open document filed %d files and said %d events", len(e), len(*r.said))
}
posts := f.posts
got, err = r.call(t, "scanner.scan", docArgs(id, true))
if err != nil {
t.Fatal(err)
}
body := got.(map[string]any)
if body["pages"] != float64(2) || body["path"] == nil || f.posts != posts {
t.Fatalf("finish answered %v (%d scans after, %d before)", body, f.posts, posts)
}
pdf, _ := os.ReadFile(body["path"].(string))
if !bytes.Contains(pdf, []byte("/Count 2")) || bytes.Count(pdf, []byte("/Type /Page /Parent")) != 2 {
t.Error("the filed PDF does not have two pages")
}
if e, _ := os.ReadDir(folder); len(e) != 1 || len(*r.said) != 1 || (*r.said)[0].body["pages"] != 2 {
t.Errorf("%d files, %d events", len(e), len(*r.said))
}
// A closed document takes no more.
if _, err := r.call(t, "scanner.scan", docArgs(id, false)); err == nil || !strings.Contains(err.Error(), "closed already") {
t.Errorf("a page into a closed document: %v", err)
}
}
// 1538: what is refused: finish with start, an unknown id, a feeder document, an id that is no id.
func TestGlassDocumentsRefuseWhatIsNotWellFormed(t *testing.T) {
f := newFakeScanner(t, fixture(t, "scan.jpg"))
r, _ := scanRig(t, f, Settings{})
for name, args := range map[string]map[string]any{
"finish with start": docArgs("start", true),
"finish alone": {"finish": true},
"unknown id": docArgs("nosuchdocument", false),
"unknown finish": docArgs("nosuchdocument", true),
"bad id": docArgs("../../etc", false),
"on the feeder": {"document": "start", "source": "feeder"},
} {
if _, err := r.call(t, "scanner.scan", args); err == nil {
t.Errorf("%s: accepted", name)
}
}
if f.posts != 0 {
t.Errorf("%d scans were put for refused asks", f.posts)
}
}
// 1538: an open document that gets no page is closed and filed as one PDF by the tool: its pages are not lost.
func TestAnUnfinishedGlassDocumentIsFiledAfterItsExpiry(t *testing.T) {
f := newFakeScanner(t, fixture(t, "scan.jpg"))
r, folder := scanRig(t, f, Settings{Crop: false, Resolution: 200})
old := docExpiry
docExpiry = 150 * time.Millisecond
t.Cleanup(func() { docExpiry = old })
got, err := r.call(t, "scanner.scan", docArgs("start", false))
if err != nil {
t.Fatal(err)
}
id := got.(map[string]any)["document"].(string)
if _, err := r.call(t, "scanner.scan", docArgs(id, false)); err != nil {
t.Fatal(err)
}
waitFor(t, "the expired document to be filed", func() bool { e, _ := os.ReadDir(folder); return len(e) == 1 })
e, _ := os.ReadDir(folder)
pdf, _ := os.ReadFile(filepath.Join(folder, e[0].Name()))
if !bytes.Contains(pdf, []byte("/Count 2")) || len(*r.said) != 1 {
t.Errorf("the expired document: %d events, two pages %v", len(*r.said), bytes.Contains(pdf, []byte("/Count 2")))
}
}
// 1538: finishing a document does not wait for the expiry: its timer is gone, so nothing is filed twice.
func TestAFinishedGlassDocumentIsNotFiledAgainByItsExpiry(t *testing.T) {
f := newFakeScanner(t, fixture(t, "scan.jpg"))
r, folder := scanRig(t, f, Settings{Crop: false, Resolution: 200})
old := docExpiry
docExpiry = 150 * time.Millisecond
t.Cleanup(func() { docExpiry = old })
got, _ := r.call(t, "scanner.scan", docArgs("start", false))
id := got.(map[string]any)["document"].(string)
if _, err := r.call(t, "scanner.scan", docArgs(id, true)); err != nil {
t.Fatal(err)
}
time.Sleep(400 * time.Millisecond)
if e, _ := os.ReadDir(folder); len(e) != 1 {
t.Errorf("%d files, want 1", len(e))
}
}
+30 -4
View File
@@ -248,14 +248,19 @@ func tools(settingsFile string, now func() time.Time) []stdio.Tool {
"call after 30): when it is not done by then the answer is {started: true, id, state: running}; the scan goes on (the " +
"device process lets a feeder run last 20 seconds a page, 3 to 30 minutes) and is filed when done, and " +
"scanner.scan {id} reads the outcome: state running, state done with the answer below, or state failed with why. " +
"The id is known to the tool process that started the scan; scan.added also announces it. Answers the event's body: " +
"The id is known to the tool process that started the scan; scan.added also announces it. " +
"Several pages of the glass in one PDF (a page turned over, a stapled document): document start scans page 1 and answers {document: <id>, pages: 1}; " +
"document <id> scans the next page; document <id> with finish true closes it into one PDF, filed once, answered like a scan. " +
"An open document that gets no page for 30 minutes is closed and filed so by the tool, and its pages are kept meanwhile in the bucket. Answers the event's body: " +
"the object's key, its size and sha256, the first page's real size in millimetres, the resolution the scanner really gave, " +
"whether it was cropped, the number of pages and the file's path; for several documents on the glass {documents: [that, for each], count} (a failure of one is in failed). The image itself is never sent. " +
"Duplex on the glass is refused.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"source": map[string]any{"type": "string", "enum": []string{"glass", "feeder"}, "description": "glass (default) or feeder"},
"id": map[string]any{"type": "string", "description": "read the outcome of the scan with this id (from an answer that said started) instead of scanning"},
"duplex": map[string]any{"type": "boolean", "description": "feeder only: scan both sides of each sheet (default false)"},
"source": map[string]any{"type": "string", "enum": []string{"glass", "feeder"}, "description": "glass (default) or feeder"},
"id": map[string]any{"type": "string", "description": "read the outcome of the scan with this id (from an answer that said started) instead of scanning"},
"document": map[string]any{"type": "string", "description": "glass only, a document of several pages in one PDF: start opens one and scans its first page; the id of an open document scans the next page into it"},
"finish": map[string]any{"type": "boolean", "description": "with document: the id of an open document: close it into one PDF, filed once (nothing is scanned)"},
"duplex": map[string]any{"type": "boolean", "description": "feeder only: scan both sides of each sheet (default false)"},
}},
Run: func(args map[string]any) (any, error) {
s, err := service()
@@ -269,6 +274,27 @@ func tools(settingsFile string, now func() time.Time) []stdio.Tool {
if id != "" {
return s.ScanResult(id)
}
document, err := strArg(args, "document")
if err != nil {
return nil, err
}
finish := false
if v, ok := args["finish"]; ok {
if finish, ok = v.(bool); !ok {
return nil, fmt.Errorf("finish is true or false")
}
}
if document != "" || finish {
if document == "" {
return nil, fmt.Errorf("finish closes a document: name its id in document")
}
if v, _ := args["source"].(string); v == "feeder" {
return nil, fmt.Errorf("a document of several pages is scanned from the glass; the feeder takes the whole stack in one scan")
}
ctx, cancel := context.WithTimeout(context.Background(), scanWait+10*time.Minute)
defer cancel()
return s.ScanPage(ctx, document, finish)
}
source, err := strArg(args, "source")
if err != nil {
return nil, err
+3
View File
@@ -80,6 +80,9 @@ type Ask struct {
Media string `json:"media,omitempty"` // print
Source string `json:"source,omitempty"` // scan: glass (default) or feeder
Duplex bool `json:"duplex,omitempty"` // scan: both sides of each sheet, feeder only
Document string `json:"document,omitempty"` // scan: start, or an open document's id: a glass page into it
Finish bool `json:"finish,omitempty"` // scan: close the document named in Document into one PDF
}
// AskBus is how the tools reach the device process: they queue an ask and read an answer back.
+92 -16
View File
@@ -35,8 +35,35 @@ type Service struct {
// background from the moment it is asked for, so that it is filed whether or not the caller still waits (issue
// 1539); the caller waits for it, and reads its outcome later with its id when it did not finish in time.
type Jobs struct {
mu sync.Mutex
m map[string]*job
mu sync.Mutex
m map[string]*job
open map[string]*time.Timer // open documents: the timer that closes each
}
// arm (re)starts the timer that closes the open document after d.
func (j *Jobs) arm(doc string, d time.Duration, closeIt func()) {
j.mu.Lock()
defer j.mu.Unlock()
if j.open == nil {
j.open = map[string]*time.Timer{}
}
if t := j.open[doc]; t != nil {
t.Stop()
}
j.open[doc] = time.AfterFunc(d, func() {
j.disarm(doc)
closeIt()
})
}
// disarm forgets the open document's timer.
func (j *Jobs) disarm(doc string) {
j.mu.Lock()
defer j.mu.Unlock()
if t := j.open[doc]; t != nil {
t.Stop()
delete(j.open, doc)
}
}
type job struct {
@@ -84,23 +111,20 @@ func (s *Service) ScanNow(ctx context.Context, source string, duplex bool) (map[
if err != nil {
return nil, err
}
id, err := s.queue(Ask{Op: OpScan, Source: source, Duplex: duplex})
if err != nil {
return nil, err
}
bound, quick := s.ScanWait, s.ScanWait
if req.Feeder {
bound, quick = s.FeederWait, s.QuickWait
}
j := s.Jobs.put(id)
go func() {
// The goroutine outlives the caller: a context of its own, bounded by the scan's own bound and the time
// to fetch the parts.
ctx, cancel := context.WithTimeout(context.Background(), bound+10*time.Minute)
defer cancel()
j.body, j.err = s.finishScan(ctx, id, bound)
close(j.done)
}()
return s.runScan(ctx, Ask{Op: OpScan, Source: source, Duplex: duplex}, bound, quick)
}
// runScan queues the ask and follows it in the background to its end (filing included); the caller is given the
// outcome if it comes within quick, and started: true with the id otherwise.
func (s *Service) runScan(ctx context.Context, ask Ask, bound, quick time.Duration) (map[string]any, error) {
id, j, err := s.startScan(ask, bound)
if err != nil {
return nil, err
}
select {
case <-j.done:
return j.body, j.err
@@ -112,6 +136,44 @@ func (s *Service) ScanNow(ctx context.Context, source string, duplex bool) (map[
}
}
// startScan queues the ask and starts the goroutine that follows it, however long the caller waits.
func (s *Service) startScan(ask Ask, bound time.Duration) (string, *job, error) {
id, err := s.queue(ask)
if err != nil {
return "", nil, err
}
j := s.Jobs.put(id)
go func() {
// The goroutine outlives the caller: a context of its own, bounded by the scan's own bound and the time
// to fetch the parts.
ctx, cancel := context.WithTimeout(context.Background(), bound+10*time.Minute)
defer cancel()
j.body, j.err = s.finishScan(ctx, id, bound, ask)
close(j.done)
}()
return id, j, nil
}
// docExpiry is how long an open document waits for its next page before its pages are filed as one PDF anyway.
var docExpiry = 30 * time.Minute
// ScanPage scans the glass into a document of several pages (issue 1538). document is start, which opens a
// document and scans its first page into it, or the id of an open one, which scans the next page into it; with
// finish the document is closed into one PDF, filed once (nothing is scanned). An open document that gets no page
// for docExpiry is closed and filed so, by this process: its pages are never lost.
func (s *Service) ScanPage(ctx context.Context, document string, finish bool) (map[string]any, error) {
if err := s.Settings.Missing(); err != nil {
return nil, err
}
if document != "start" && !uploadPattern.MatchString(document) {
return nil, fmt.Errorf("document is start, or the id of an open document: letters, digits, - and _, at most 64")
}
if finish && document == "start" {
return nil, fmt.Errorf("finish closes an open document: name its id in document, not start")
}
return s.runScan(ctx, Ask{Op: OpScan, Document: document, Finish: finish}, s.ScanWait, s.ScanWait)
}
// ScanResult is the outcome of the scan with this id: its body and path when it is done, state running when it is
// not, and state failed with why when it failed. A scan this process does not know (it was restarted, or the id is
// wrong) is an error.
@@ -136,11 +198,25 @@ func (s *Service) ScanResult(id string) (map[string]any, error) {
}
// finishScan waits for the device process's answer to the scan ask, then fetches the stored scan and files it.
func (s *Service) finishScan(ctx context.Context, id string, wait time.Duration) (map[string]any, error) {
func (s *Service) finishScan(ctx context.Context, id string, wait time.Duration, ask Ask) (map[string]any, error) {
ans, err := s.await(ctx, id, wait)
if err != nil {
return nil, err
}
if doc, _ := ans["document"].(string); doc != "" && ans["key"] == nil {
// A page was added to an open document: nothing to file yet, and the document is closed if it is left open.
s.Jobs.arm(doc, docExpiry, func() {
_, _, _ = s.startScan(Ask{Op: OpScan, Document: doc, Finish: true}, s.ScanWait)
})
return ans, nil
}
if ask.Document != "" && ask.Finish {
out, err := s.fileAnswer(ctx, ans)
if err == nil {
s.Jobs.disarm(ask.Document)
}
return out, err
}
return s.fileAnswer(ctx, ans)
}