novox/mesh-catalog
18
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases Wiki Activity

Merge pull request 'Files go through the store: object-store links, node-files upload/download, a document converter, and brother printing a key, a path on any machine or a web link (design 60 F3, ADR 0468)' (#417) from feat/files-through-the-store into main

Browse Source
This commit was merged in pull request #417.
This commit is contained in:
mesh-tools
2026-10-21 16:31:22 +00:00
parent b2f5ebb27d 6cf51173d5
commit 7a7a403485
58 changed files with 4921 additions and 357 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/brother/README.md
+2 -2
View File
@@ -122,8 +122,8 @@ the login page again. The logout follows a logout link or form on the page; a pa
## The event
`scan.added`, said by this module, on the wire `mesh.mod.brother.event.scan.added`; a consumer declares `brother.scan.added`. Its body:
`path`, `node`, `bytes`, `sha256`, `width-mm`, `height-mm`, `resolution` (dpi, as the scanner gave it),
`scan.added`, said by the device process once the PDF is stored, before the file is written (the tool side files it afterwards, from the store, by a link the device signed; the verb's answer adds the file's `path`), on the wire `mesh.mod.brother.event.scan.added`; a consumer declares `brother.scan.added`. Its body:
`object` (the key of the stored PDF), `node`, `bytes`, `sha256`, `width-mm`, `height-mm`, `resolution` (dpi, as the scanner gave it),
`color-mode`, `cropped`, `pages` (1 for the glass), `scanned-at` (RFC 3339, UTC).
## Files it does not know
modules/brother/cmd/brother/convert.go
+207
View File
@@ -0,0 +1,207 @@
package main
// Conversion before printing (novox/hq ADR 0468 clause 7, issue 1600). The printer takes PDF and JPEG. A stored
// document of a type the module only-office converts (seat document-converter) is turned into a PDF first: the device
// process signs a GET link on the document and a PUT link on <sha>.pdf with the binding's key, calls
// document-converter.convert, checks that the PDF the converter says it put is the PDF the bucket holds, and prints
// that. Any other type is refused, naming the types taken.
import (
"archive/zip"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"time"
"unicode/utf8"
"git.novox.be/novox/mesh-sdk/go/objectlink"
)
// ConvertFollow is how long the device process follows one conversion, in calls of under 25 s each.
const ConvertFollow = 3 * time.Minute
// ConvertTypes are the types document-converter takes (clause 7).
var ConvertTypes = []string{"docx", "doc", "xlsx", "xls", "pptx", "ppt", "odt", "ods", "odp", "rtf", "txt", "html"}
// contentTypes are the media types of a fetched document, by the type the converter is asked for.
var contentTypes = map[string]string{
"text/html": "html", "application/xhtml+xml": "html", "text/plain": "txt", "application/rtf": "rtf", "text/rtf": "rtf",
"application/msword": "doc", "application/vnd.ms-excel": "xls", "application/vnd.ms-powerpoint": "ppt",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx",
"application/vnd.openxmlformats-officedocument.presentationml.presentation": "pptx",
"application/vnd.oasis.opendocument.text": "odt",
"application/vnd.oasis.opendocument.spreadsheet": "ods",
"application/vnd.oasis.opendocument.presentation": "odp",
}
// the first bytes of an OLE compound file (the legacy Word, Excel and PowerPoint formats), and the stream names that
// tell them apart, as UTF-16 (MS-CFB: directory entry names are UTF-16LE).
var oleMagic = []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1}
func utf16le(s string) []byte {
var b []byte
for _, r := range s {
b = append(b, byte(r), byte(r>>8))
}
return b
}
// ConvertibleType is the type a document is, among the converter's twelve, told by its bytes and, for a document that
// came from a web server, by its Content-Type; "" when it is none of them. A document that is a PDF or a JPEG is
// not asked about: the printer takes it as it is.
func ConvertibleType(doc []byte, contentType string) string {
if i := strings.IndexByte(contentType, ';'); i >= 0 {
contentType = contentType[:i]
}
byHeader := contentTypes[strings.ToLower(strings.TrimSpace(contentType))]
switch {
case bytes.HasPrefix(doc, []byte("PK\x03\x04")):
if t := zipType(doc); t != "" {
return t
}
return ""
case bytes.HasPrefix(doc, oleMagic):
for name, t := range map[string]string{"WordDocument": "doc", "Workbook": "xls", "PowerPoint Document": "ppt"} {
if bytes.Contains(doc, utf16le(name)) {
return t
}
}
return ""
case bytes.HasPrefix(doc, []byte(`{\rtf`)):
return "rtf"
}
if byHeader == "html" || looksLikeHTML(doc) {
if utf8.Valid(doc) && !bytes.Contains(doc, []byte{0}) {
return "html"
}
return ""
}
if utf8.Valid(doc) && !bytes.Contains(doc, []byte{0}) && len(doc) > 0 {
return "txt"
}
return ""
}
func looksLikeHTML(doc []byte) bool {
head := doc
if len(head) > 512 {
head = head[:512]
}
s := strings.ToLower(strings.TrimLeft(strings.TrimPrefix(string(head), "\xef\xbb\xbf"), " \t\r\n"))
return strings.HasPrefix(s, "<!doctype html") || strings.HasPrefix(s, "<html")
}
// zipType tells the packages apart by their parts: OOXML by its main part (ECMA-376 Part 2), OpenDocument by its
// first file `mimetype` (ODF 1.2 Part 3 §3.3).
func zipType(doc []byte) string {
zr, err := zip.NewReader(bytes.NewReader(doc), int64(len(doc)))
if err != nil {
return ""
}
for _, f := range zr.File {
switch f.Name {
case "word/document.xml":
return "docx"
case "xl/workbook.xml":
return "xlsx"
case "ppt/presentation.xml":
return "pptx"
case "mimetype":
rc, err := f.Open()
if err != nil {
return ""
}
mt, _ := io.ReadAll(io.LimitReader(rc, 256))
rc.Close()
switch strings.TrimSpace(string(mt)) {
case "application/vnd.oasis.opendocument.text":
return "odt"
case "application/vnd.oasis.opendocument.spreadsheet":
return "ods"
case "application/vnd.oasis.opendocument.presentation":
return "odp"
}
}
}
return ""
}
// toPDF turns a stored document of a converter type into a stored PDF and answers the PDF and its key. contentType is
// the media type a web server gave, or "".
func (w *deviceWorker) toPDF(ctx context.Context, objects Objects, key string, doc []byte, contentType string) ([]byte, string, error) {
typ := ConvertibleType(doc, contentType)
if typ == "" {
return nil, "", fmt.Errorf("the document is neither a PDF nor a JPEG, nor a type the converter takes (%s): nothing is converted or printed",
strings.Join(ConvertTypes, ", "))
}
signer, ok := objects.(Presigner)
if !ok {
return nil, "", errors.New("the bucket cannot sign a link, so the document cannot be converted")
}
if w.Convert == nil {
return nil, "", errors.New("this process has no way to call the document converter")
}
sum := sha256Hex(doc)
pdfKey := sum + ".pdf"
source, err := signer.Link(http.MethodGet, key, objectlink.MaxExpires)
if err != nil {
return nil, "", err
}
target, err := signer.Link(http.MethodPut, pdfKey, objectlink.MaxExpires)
if err != nil {
return nil, "", err
}
raw, err := w.Convert(map[string]any{"source": source, "type": typ, "target": target, "sha256": sum})
if err != nil {
return nil, "", fmt.Errorf("the document (%s) could not be converted: %w", typ, err)
}
// The converter answers within 25 s, the tool runner cutting every call at 30 s; a conversion not done by then is
// answered as converting with its ask id, and is read again with convert {ask}, each call under 25 s, for as long
// as the device process follows a conversion.
follow := w.ConvertFollow
if follow <= 0 {
follow = ConvertFollow
}
until := time.Now().Add(follow)
var ans struct {
SHA256 string `json:"sha256"`
Error string `json:"error"`
State string `json:"state"`
Ask string `json:"ask"`
}
for {
ans.State, ans.Ask, ans.SHA256, ans.Error = "", "", "", ""
if err := json.Unmarshal(raw, &ans); err != nil {
return nil, "", fmt.Errorf("the converter's answer is not JSON: %w", err)
}
if ans.State != "converting" {
break
}
if ans.Ask == "" || !time.Now().Before(until) {
return nil, "", fmt.Errorf("the document (%s) was still converting after %s (ask %s): nothing is printed", typ, follow, ans.Ask)
}
if raw, err = w.Convert(map[string]any{"ask": ans.Ask}); err != nil {
return nil, "", fmt.Errorf("the document (%s) could not be converted: %w", typ, err)
}
}
if ans.Error != "" {
return nil, "", fmt.Errorf("the document (%s) could not be converted: %s", typ, ans.Error)
}
pdf, err := objects.Get(ctx, pdfKey)
if err != nil {
return nil, "", fmt.Errorf("the converter answered, but no PDF is kept as %s: %w", pdfKey, err)
}
if got := sha256Hex(pdf); !strings.EqualFold(got, ans.SHA256) {
return nil, "", fmt.Errorf("the converter says it put a PDF of SHA-256 %s, but the bucket holds one of %s as %s: nothing is printed", ans.SHA256, got, pdfKey)
}
if _, _, err := SniffFormat(pdf); err != nil {
return nil, "", fmt.Errorf("the converter's result %s: %w", pdfKey, err)
}
return pdf, pdfKey, nil
}
modules/brother/cmd/brother/convert_test.go
+268
View File
@@ -0,0 +1,268 @@
package main
import (
"archive/zip"
"bytes"
"context"
"encoding/json"
"errors"
"net/url"
"os"
"strconv"
"strings"
"testing"
"time"
)
// docx builds a minimal Word package: the parts ECMA-376 Part 2 requires of one ([Content_Types].xml, the package
// relationships and the main part word/document.xml). No real Word file is captured; the layout is the standard's.
func docx(t *testing.T) []byte {
t.Helper()
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
for name, body := range map[string]string{
"[Content_Types].xml": `<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"/>`,
"_rels/.rels": `<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"/>`,
"word/document.xml": `<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body/></w:document>`,
} {
w, err := zw.Create(name)
if err != nil {
t.Fatal(err)
}
w.Write([]byte(body))
}
zw.Close()
return buf.Bytes()
}
// converterFake stands for document-converter.convert: it records the call and, as the converter process would, puts a
// PDF in the bucket under the key the target link names and answers its size and SHA-256.
type converterFake struct {
r *printerRig
calls []map[string]any
pdf []byte
answer func(body map[string]any, stored []byte) (json.RawMessage, error)
}
func withConverter(r *printerRig, pdf []byte) *converterFake {
c := &converterFake{r: r, pdf: pdf}
r.worker.Convert = func(body map[string]any) (json.RawMessage, error) {
c.calls = append(c.calls, body)
if body["target"] != nil { // a start puts the PDF; a read of an ask (convert {ask}) puts nothing
target, _ := url.Parse(body["target"].(string))
r.objects.Put(context.Background(), strings.TrimPrefix(target.Path, "/"), "application/pdf", c.pdf)
}
if c.answer != nil {
return c.answer(body, c.pdf)
}
return json.Marshal(map[string]any{"size": len(c.pdf), "sha256": sha256Hex(c.pdf)})
}
return c
}
// 1600/AC1: a stored .docx is converted: document-converter.convert is called with a GET link on the source and a PUT
// link on <sha>.pdf, both signed in the bucket and expiring within ten minutes, and the PDF is printed.
func TestAStoredDocxIsConvertedThenPrinted(t *testing.T) {
fake := newIPPFake(t, 51, 5)
r := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
doc := docx(t)
key := sha256Hex(doc)
r.objects.Put(t.Context(), key, "application/octet-stream", doc)
pdf := pdfOf(2500)
conv := withConverter(r, pdf)
got, err := r.call(t, "printer.print", map[string]any{"object": key})
if err != nil {
t.Fatal(err)
}
if len(conv.calls) != 1 {
t.Fatalf("%d converter calls", len(conv.calls))
}
call := conv.calls[0]
src, _ := url.Parse(call["source"].(string))
dst, _ := url.Parse(call["target"].(string))
if src.Path != "/"+key || dst.Path != "/"+key+".pdf" || call["type"] != "docx" || call["sha256"] != key {
t.Errorf("call = %v", call)
}
for _, u := range []*url.URL{src, dst} {
if secs, _ := strconv.Atoi(u.Query().Get("X-Amz-Expires")); secs < 1 || time.Duration(secs)*time.Second > 10*time.Minute {
t.Errorf("%s lives %d s", u, secs)
}
}
if strings.Join(r.objects.links, ",") != "GET "+key+" 600,PUT "+key+".pdf 600" {
t.Errorf("links signed: %v", r.objects.links)
}
m := got.(map[string]any)
if m["job-id"] != float64(51) || m["format"] != "application/pdf" || m["object"] != key+".pdf" || m["converted-from"] != key {
t.Errorf("answer = %v", m)
}
if _, sent := fake.request(t); string(sent) != string(pdf) {
t.Error("the printer was not sent the converted PDF")
}
}
// 1600/AC2: a converter answer whose SHA-256 differs from the stored PDF's is refused and nothing is printed.
func TestAConverterAnswerOfAnotherHashPrintsNothing(t *testing.T) {
fake := newIPPFake(t, 1, 3)
r := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
doc := docx(t)
r.objects.Put(t.Context(), sha256Hex(doc), "application/octet-stream", doc)
conv := withConverter(r, pdfOf(900))
conv.answer = func(map[string]any, []byte) (json.RawMessage, error) {
return json.Marshal(map[string]any{"size": 900, "sha256": sha256Hex([]byte("another pdf"))})
}
_, err := r.call(t, "printer.print", map[string]any{"object": sha256Hex(doc)})
if err == nil || !strings.Contains(err.Error(), "nothing is printed") {
t.Fatalf("error = %v", err)
}
if fake.body != nil {
t.Error("something was printed")
}
}
// 1600/AC3: a converter error is answered with its code and meaning.
func TestAConverterErrorIsAnsweredWithItsCodeAndMeaning(t *testing.T) {
fake := newIPPFake(t, 1, 3)
r := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
doc := docx(t)
r.objects.Put(t.Context(), sha256Hex(doc), "application/octet-stream", doc)
conv := withConverter(r, pdfOf(10))
conv.answer = func(map[string]any, []byte) (json.RawMessage, error) {
return nil, errors.New("the document server refused the conversion, error -4: error while downloading the document file to be converted")
}
_, err := r.call(t, "printer.print", map[string]any{"object": sha256Hex(doc)})
if err == nil || !strings.Contains(err.Error(), "-4: error while downloading the document file to be converted") {
t.Fatalf("error = %v", err)
}
// An error the converter wrote in its answer is the same.
conv.answer = func(map[string]any, []byte) (json.RawMessage, error) {
return json.RawMessage(`{"error": "-3: conversion error"}`), nil
}
if _, err := r.call(t, "printer.print", map[string]any{"object": sha256Hex(doc)}); err == nil || !strings.Contains(err.Error(), "-3: conversion error") {
t.Fatalf("error = %v", err)
}
if fake.body != nil {
t.Error("something was printed")
}
}
// 1600/AC4: a type that is neither PDF, JPEG nor one of the converter's twelve is refused naming the types taken, and
// convert is not called.
func TestAnotherTypeIsRefusedNamingTheTypesTaken(t *testing.T) {
fake := newIPPFake(t, 1, 3)
r := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
exe := []byte("MZ\x90\x00\x03\x00\x00\x00\x04\x00\x00\x00\xff\xff\x00\x00")
r.objects.Put(t.Context(), sha256Hex(exe), "application/octet-stream", exe)
conv := withConverter(r, pdfOf(10))
_, err := r.call(t, "printer.print", map[string]any{"object": sha256Hex(exe)})
if err == nil || !strings.Contains(err.Error(), "docx, doc, xlsx, xls, pptx, ppt, odt, ods, odp, rtf, txt, html") {
t.Fatalf("error = %v", err)
}
if len(conv.calls) != 0 || fake.body != nil {
t.Errorf("%d converter calls, printed %v", len(conv.calls), fake.body != nil)
}
}
// 1600: each of the twelve types is told by its bytes (and a web page also by its Content-Type).
func TestTheTypesAreToldByTheirBytes(t *testing.T) {
ole := func(stream string) []byte {
return append(append([]byte{}, oleMagic...), append(make([]byte, 64), utf16le(stream)...)...)
}
odf := func(mime string) []byte {
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
w, _ := zw.Create("mimetype")
w.Write([]byte(mime))
zw.Close()
return buf.Bytes()
}
ooxml := func(part string) []byte {
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
w, _ := zw.Create(part)
w.Write([]byte("<x/>"))
zw.Close()
return buf.Bytes()
}
for want, c := range map[string]struct {
doc []byte
ct string
}{
"docx": {docx(t), ""}, "xlsx": {ooxml("xl/workbook.xml"), ""}, "pptx": {ooxml("ppt/presentation.xml"), ""},
"doc": {ole("WordDocument"), ""}, "xls": {ole("Workbook"), ""}, "ppt": {ole("PowerPoint Document"), ""},
"odt": {odf("application/vnd.oasis.opendocument.text"), ""}, "ods": {odf("application/vnd.oasis.opendocument.spreadsheet"), ""},
"odp": {odf("application/vnd.oasis.opendocument.presentation"), ""},
"rtf": {[]byte(`{\rtf1\ansi hello}`), ""}, "txt": {[]byte("just some words\n"), ""},
"html": {[]byte("<!DOCTYPE html><html><body>hi</body></html>"), ""}, "html ": {[]byte("<p>a fragment</p>"), "text/html; charset=utf-8"},
} {
if got := ConvertibleType(c.doc, c.ct); got != strings.TrimSpace(want) {
t.Errorf("%s: told as %q", want, got)
}
}
for name, doc := range map[string][]byte{"empty": {}, "binary": {0, 1, 2, 3}, "a zip of nothing known": ooxml("other.xml"), "ole of nothing known": ole("Other")} {
if got := ConvertibleType(doc, ""); got != "" {
t.Errorf("%s: told as %q", name, got)
}
}
}
// 1600/AC5: the device process invokes seat:document-converter.convert.
func TestTheDeviceProcessInvokesTheConverter(t *testing.T) {
for _, r := range readManifest(t).Resources {
if r["id"] == "device" {
inv, _ := r["bus"].(map[string]any)["invokes"].([]any)
if len(inv) != 1 || inv[0] != "seat:document-converter.convert" {
t.Errorf("invokes = %v", inv)
}
return
}
}
t.Fatal("no device process")
}
// 1600 (review round 1): a conversion answered as converting is followed with convert {ask}, each call its own, until
// it is done; one still converting after the device process's bound is not printed, and the process.CallTimeout of the
// SDK is not touched.
func TestAConversionAnsweredAsConvertingIsFollowedWithItsAsk(t *testing.T) {
fake := newIPPFake(t, 61, 5)
r := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
doc := docx(t)
key := sha256Hex(doc)
r.objects.Put(t.Context(), key, "application/octet-stream", doc)
pdf := pdfOf(700)
conv := withConverter(r, pdf)
rounds := 0
conv.answer = func(body map[string]any, stored []byte) (json.RawMessage, error) {
if body["ask"] != nil {
rounds++
if rounds < 3 {
return json.RawMessage(`{"state": "converting", "ask": "` + body["ask"].(string) + `"}`), nil
}
return json.Marshal(map[string]any{"size": len(stored), "sha256": sha256Hex(stored)})
}
return json.RawMessage(`{"state": "converting", "ask": "a1b2c3a1b2c3a1b2c3a1b2c3"}`), nil
}
if _, err := r.call(t, "printer.print", map[string]any{"object": key}); err != nil {
t.Fatal(err)
}
if len(conv.calls) != 4 || conv.calls[1]["ask"] != "a1b2c3a1b2c3a1b2c3a1b2c3" || conv.calls[1]["source"] != nil {
t.Errorf("calls = %v", conv.calls)
}
if _, sent := fake.request(t); string(sent) != string(pdf) {
t.Error("the converted PDF was not printed")
}
// Always converting: given up at the bound, nothing printed.
fake.body = nil
r.worker.ConvertFollow = 30 * time.Millisecond
conv.answer = func(body map[string]any, _ []byte) (json.RawMessage, error) {
time.Sleep(5 * time.Millisecond)
return json.RawMessage(`{"state": "converting", "ask": "a1b2c3a1b2c3a1b2c3a1b2c3"}`), nil
}
if _, err := r.call(t, "printer.print", map[string]any{"object": key}); err == nil || !strings.Contains(err.Error(), "still converting") || fake.body != nil {
t.Errorf("error = %v, printed %v", err, fake.body != nil)
}
src, _ := os.ReadFile("deviceproc.go")
if strings.Contains(string(src), "CallTimeout") {
t.Error("deviceproc.go sets the SDK's call timeout")
}
}
modules/brother/cmd/brother/deviceproc.go
+90 -14
View File
@@ -11,14 +11,17 @@ import (
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
"os/signal"
"slices"
"strings"
"sync"
"syscall"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/objectlink"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
@@ -66,6 +69,12 @@ type deviceWorker struct {
Node string
// Capture scans; nil is the device's own (a test puts a stand-in).
Capture func(ctx context.Context, req ScanRequest) ([]Capture, error)
// Convert calls the seat document-converter's convert verb as this process's own account (clause 7); nil: none.
Convert func(body map[string]any) (json.RawMessage, error)
// ConvertFollow is how long a conversion is followed; zero is ConvertFollow (a test makes it shorter).
ConvertFollow time.Duration
// Fetcher is the guarded fetch of a web link (fetch.go); it reads its facts when it is first needed.
Fetcher func() (*Fetcher, error)
// answerPart is the raw size of a document part in an answer; zero is AnswerPartSize (a test makes it larger).
answerPart int
Now func() time.Time
@@ -247,10 +256,14 @@ func (w *deviceWorker) keep(ctx context.Context, objects Objects, set Settings,
// keepOne keeps one captured PDF and announces it.
func (w *deviceWorker) keepOne(ctx context.Context, objects Objects, set Settings, c Capture) (map[string]any, error) {
// A scan the tool could not take is not kept nor announced: an announced scan is always filed (issue 1539).
if most := MostAnswerParts * AnswerPartSize; len(c.PDF) > most {
return nil, fmt.Errorf("the scan's PDF is %d bytes, over the %d the tool can fetch (%d parts of %d): it is not kept; scan with a lower resolution or fewer sheets",
len(c.PDF), most, MostAnswerParts, AnswerPartSize)
// A scan over a document's limit is not kept nor announced: an announced scan is always filed (issue 1539).
if len(c.PDF) > int(objectlink.MaxDocument) {
return nil, fmt.Errorf("the scan's PDF is %d bytes, over the %d a document may be (novox/hq ADR 0468 §3): it is not kept; scan with a lower resolution or fewer sheets",
len(c.PDF), objectlink.MaxDocument)
}
signer, ok := objects.(Presigner)
if !ok {
return nil, errors.New("the bucket cannot sign a link: the scan is not kept")
}
body := c.Body(w.Node, set.ColorMode)
key := body["object"].(string)
@@ -260,7 +273,13 @@ func (w *deviceWorker) keepOne(ctx context.Context, objects Objects, set Setting
if err := w.Emit(body, c.SHA256); err != nil {
return nil, fmt.Errorf("the scan is kept as %s, but %s could not be said on the bus: %w", key, EventAdded, err)
}
answer := map[string]any{"key": key, "size": len(c.PDF), "parts": answerPartCount(len(c.PDF))}
// The answer carries a link to read the stored PDF, signed with the binding's key (novox/hq ADR 0468 §4): the tool
// side reads the scan by it and files it; no byte of the document is in the answer.
link, err := signer.Link(http.MethodGet, key, objectlink.MaxExpires)
if err != nil {
return nil, fmt.Errorf("the scan is kept as %s, but a link to it could not be signed: %w", key, err)
}
answer := map[string]any{"key": key, "size": len(c.PDF), "link": link, "link-seconds": int(objectlink.MaxExpires / time.Second)}
for k, v := range body {
answer[k] = v
}
@@ -370,7 +389,20 @@ func (w *deviceWorker) print(ctx context.Context, ask Ask) (map[string]any, erro
return nil, err
}
key := ask.Object
if key == "" {
var doc []byte
contentType := ""
switch {
case ask.URL != "":
fetched, err := w.fetch(ctx, ask.URL)
if err != nil {
return nil, err
}
// The fetched document is kept once under its SHA-256, as a scan is, and printed as that object.
doc, key, contentType = fetched.Body, sha256Hex(fetched.Body), fetched.ContentType
if err := Store(ctx, objects, key, fetched.ContentType, doc); err != nil {
return nil, err
}
case key == "":
for _, ext := range []string{"pdf", "jpg"} {
k := "uploads/" + ask.Upload + "." + ext
if has, err := objects.Has(ctx, k); err != nil {
@@ -383,21 +415,55 @@ func (w *deviceWorker) print(ctx context.Context, ask Ask) (map[string]any, erro
if key == "" {
return nil, fmt.Errorf("upload %s is not complete: no document is kept under it", ask.Upload)
}
}
doc, err := objects.Get(ctx, key)
if err != nil {
return nil, err
fallthrough
default:
var err error
if doc, err = objects.Get(ctx, key); err != nil {
return nil, err
}
}
format, _, err := SniffFormat(doc)
var convertedFrom string
if err != nil {
return nil, err
// Not a PDF or a JPEG: a type the converter takes becomes a PDF first (clause 7), any other is refused.
pdf, pdfKey, cerr := w.toPDF(ctx, objects, key, doc, contentType)
if cerr != nil {
return nil, cerr
}
doc, convertedFrom, key = pdf, key, pdfKey
format = "application/pdf"
}
// The printer is asked what it takes before the job, and a format it does not list is refused with the list
// (novox/hq ADR 0468 clause 5): a printer that cannot be asked prints nothing.
listed, err := FormatsOf(ctx, set.Printer)
if err != nil {
return nil, fmt.Errorf("the printer's document formats could not be read, so nothing is sent: %w", err)
}
if !slices.Contains(listed, format) {
return nil, fmt.Errorf("the printer does not take %s: its document-format-supported lists %s", format, strings.Join(listed, ", "))
}
res, err := PrintJob(ctx, set.Printer, PrintOptions{Format: format, Copies: ask.Copies, Sides: ask.Sides, Media: ask.Media, Name: "brother print"}, doc)
if err != nil {
return nil, err
}
return map[string]any{"job-id": res.JobID, "job-state": res.State, "job-state-code": res.StateCode,
"job-state-reasons": res.Reasons, "format": format, "copies": ask.Copies, "sides": ask.Sides, "media": ask.Media, "object": key}, nil
out := map[string]any{"job-id": res.JobID, "job-state": res.State, "job-state-code": res.StateCode,
"job-state-reasons": res.Reasons, "format": format, "copies": ask.Copies, "sides": ask.Sides, "media": ask.Media, "object": key}
if convertedFrom != "" {
out["converted-from"] = convertedFrom
}
return out, nil
}
// fetch gets a web link under the rules of fetch.go.
func (w *deviceWorker) fetch(ctx context.Context, link string) (*Fetched, error) {
if w.Fetcher == nil {
return nil, errors.New("this process has no way to fetch a web link")
}
f, err := w.Fetcher()
if err != nil {
return nil, err
}
return f.Fetch(ctx, link)
}
// read logs in, reads one page of the admin site and logs out. It is read-only: the login (and the logout) are
@@ -497,7 +563,7 @@ func nonNilForms(f []Form) []Form {
// It refuses to start naming what is missing when the account cannot be read, and never logs a password.
func device() int {
settings, passwordFile, brokerFile := os.Getenv("BROTHER_SETTINGS"), os.Getenv("BROTHER_ADMIN_PASSWORD"), os.Getenv("BROTHER_BROKER")
store := os.Getenv("BROTHER_STORE")
store, fetchFile := os.Getenv("BROTHER_STORE"), os.Getenv("BROTHER_FETCH")
if brokerFile == "" {
fmt.Fprintln(os.Stderr, "[brother] device was started without BROTHER_BROKER, its bus account's file")
return 2
@@ -519,6 +585,13 @@ func device() int {
}
return NewS3(facts), nil
},
Fetcher: func() (*Fetcher, error) {
r, err := ReadFetchFacts(fetchFile)
if err != nil {
return nil, err
}
return NewFetcher(r), nil
},
Now: time.Now,
Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, f+"\n", a...) },
}
@@ -541,6 +614,9 @@ func runDevice(ctx context.Context, account process.Account, w *deviceWorker) er
}
defer conn.Close()
w.Answers = conn.State(AnswersState)
w.Convert = func(body map[string]any) (json.RawMessage, error) {
return conn.Call("seat:document-converter.convert", body)
}
if w.Node == "" {
w.Node = account.Node
}
modules/brother/cmd/brother/documents_test.go
+39 -94
View File
@@ -115,7 +115,7 @@ func TestScannerDocumentGivesEachPartOfAKeptScan(t *testing.T) {
}
}
// 1453/AC10: a scan of more than one part is fetched whole by scanner.scan, joined, checked and filed.
// 1453/AC10, 1601: a scan the size of several answer parts is read whole from the store by scanner.scan, checked and filed.
func TestScanNowFetchesAScanOfSeveralParts(t *testing.T) {
f := newFakeScanner(t, fixture(t, "scan.jpg"))
r, _ := scanRig(t, f, Settings{Crop: false})
@@ -266,15 +266,41 @@ func TestAnIncompleteUploadIsDroppedAfterAWhile(t *testing.T) {
// ippFake is a printer's IPP port: it records the request and answers a Print-Job answer with a job id and state.
type ippFake struct {
url string
body []byte
body []byte // the Print-Job request
status int
// formats is document-format-supported, as the printer lists it; asked counts the Get-Printer-Attributes requests.
formats []string
asked int
// delay is how long a Print-Job takes to be answered.
delay time.Duration
}
func newIPPFake(t *testing.T, jobID, jobState int) *ippFake {
t.Helper()
f := &ippFake{}
f := &ippFake{formats: []string{"application/octet-stream", "application/pdf", "image/jpeg"}}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
f.body, _ = io.ReadAll(r.Body)
body, _ := io.ReadAll(r.Body)
if binary.BigEndian.Uint16(body[2:4]) == uint16(ippGetPrinterAttributes) {
// Get-Printer-Attributes (RFC 8011 §4.2.5): the formats are one attribute of several values, a value
// after the first carrying an empty name (§4.1.4), each of the mimeMediaType syntax.
f.asked++
ans := []byte{0x02, 0x00, 0x00, 0x00, 0, 0, 0, 1, ippOperation}
ans = appendAttr(ans, ippCharset, "attributes-charset", "utf-8")
ans = append(ans, 0x04) // printer-attributes-tag, RFC 8010 §3.5.1
for i, format := range f.formats {
name := "document-format-supported"
if i > 0 {
name = ""
}
ans = appendAttr(ans, ippMimeType, name, format)
}
ans = append(ans, ippEnd)
w.Header().Set("Content-Type", "application/ipp")
w.Write(ans)
return
}
f.body = body
time.Sleep(f.delay)
ans := []byte{0x02, 0x00, byte(f.status >> 8), byte(f.status), 0, 0, 0, 1, ippOperation}
ans = appendAttr(ans, ippCharset, "attributes-charset", "utf-8")
ans = append(ans, ippJob)
@@ -337,15 +363,14 @@ func TestPrinterPrintSendsAPrintJobAndAnswersTheJob(t *testing.T) {
}
}
// 1453/AC12: the defaults are one copy, one-sided, A4; a JPEG is sent as image/jpeg; an upload is printed by its id.
func TestPrinterPrintDefaultsAndFormatsAndUploads(t *testing.T) {
// 1453/AC12: the defaults are one copy, one-sided, A4; a JPEG is sent as image/jpeg; a document stored under its
// SHA-256 (as node-files.upload keeps one) is printed by its key.
func TestPrinterPrintDefaultsAndFormatsAndStoredDocuments(t *testing.T) {
fake := newIPPFake(t, 8, 3)
r := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
jpg := append([]byte{0xFF, 0xD8, 0xFF, 0xE0}, bytes.Repeat([]byte{9}, 200)...)
if _, err := upload(t, r, "photo", jpg, sha256Hex(jpg), 1, 1, jpg); err != nil {
t.Fatal(err)
}
got, err := r.call(t, "printer.print", map[string]any{"upload": "photo"})
r.objects.Put(t.Context(), sha256Hex(jpg), "image/jpeg", jpg)
got, err := r.call(t, "printer.print", map[string]any{"object": sha256Hex(jpg)})
if err != nil {
t.Fatal(err)
}
@@ -357,8 +382,8 @@ func TestPrinterPrintDefaultsAndFormatsAndUploads(t *testing.T) {
strings.Join(attrs.texts("sides"), "") != "one-sided" || strings.Join(attrs.texts("media"), "") != DefaultMedia || attrs.Attributes["copies"][0].Integer != 1 {
t.Errorf("attributes = %v, %d bytes sent", attrs.Attributes, len(sent))
}
if _, err := r.call(t, "printer.print", map[string]any{"upload": "never-uploaded"}); err == nil || !strings.Contains(err.Error(), "not complete") {
t.Errorf("an upload never made: error = %v", err)
if _, err := r.call(t, "printer.print", map[string]any{"object": sha256Hex([]byte("never stored"))}); err == nil {
t.Errorf("an object never stored was printed")
}
}
@@ -392,9 +417,9 @@ func TestPrinterPrintRefusesWhatIsNotWellFormed(t *testing.T) {
if _, err := r.call(t, "printer.print", map[string]any{"object": dk}); err == nil || !strings.Contains(err.Error(), "0x0400") {
t.Errorf("a refused job: error = %v", err)
}
// A kept object that is not a document the printer takes is not sent.
// A kept object that is no document the printer takes, nor one the converter takes (1600), is not sent.
fake.status, fake.body = 0, nil
r.objects.Put(t.Context(), key, "text/plain", []byte("hello"))
r.objects.Put(t.Context(), key, "application/octet-stream", []byte{0x00, 0x01, 0xFE, 0xFF, 0x02})
if _, err := r.call(t, "printer.print", map[string]any{"object": key}); err == nil || !strings.Contains(err.Error(), "neither a PDF nor a JPEG") || fake.body != nil {
t.Errorf("a text object: error = %v, sent %d bytes", err, len(fake.body))
}
@@ -547,83 +572,3 @@ func home(t *testing.T) string {
t.Setenv("HOME", h)
return h
}
// 1551: printer.print with a path reads the file, uploads it in parts and prints it in one call; a document of
// several parts arrives whole, and the answer has the job's id and state.
func TestPrinterPrintTakesAPathAndUploadsAndPrintsInOneCall(t *testing.T) {
fake := newIPPFake(t, 31, 5)
r := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
doc := pdfOf(2*PartSize + 1234)
file := filepath.Join(home(t), "letter.bin") // the name says nothing: the bytes are read
if err := os.WriteFile(file, doc, 0o600); err != nil {
t.Fatal(err)
}
got, err := r.call(t, "printer.print", map[string]any{"path": file, "copies": float64(2)})
if err != nil {
t.Fatal(err)
}
m := got.(map[string]any)
if m["job-id"] != float64(31) || m["job-state"] != "processing" || m["format"] != "application/pdf" || m["bytes"] != len(doc) {
t.Errorf("answer = %v", m)
}
attrs, sent := fake.request(t)
if !bytes.Equal(sent, doc) || attrs.Attributes["copies"][0].Integer != 2 {
t.Errorf("sent %d bytes (want %d), attributes %v", len(sent), len(doc), attrs.Attributes)
}
ops := r.bus.ops()
if len(ops) != 4 || ops[0] != OpUpload || ops[2] != OpUpload || ops[3] != OpPrint {
t.Errorf("asks = %v, want three upload parts and a print", ops)
}
}
// 1551: only a regular file under the tool account's home folder is read, after symbolic links are resolved; every
// other path is refused in one fixed way that says nothing of whether it exists or how large it is; a file in the
// home folder that is no PDF or JPEG by its bytes is refused by name of the type.
func TestPrinterPrintReadsOnlyAFileUnderTheHomeFolder(t *testing.T) {
fake := newIPPFake(t, 1, 5)
r := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
h := home(t)
outside := t.TempDir()
secret := filepath.Join(outside, "secret.pdf")
os.WriteFile(secret, pdfOf(3000), 0o600)
link := filepath.Join(h, "link.pdf")
os.Symlink(secret, link)
dirLink := filepath.Join(h, "dirlink")
os.Symlink(outside, dirLink)
os.Mkdir(filepath.Join(h, "folder"), 0o700)
for name, path := range map[string]string{
"outside home": secret,
"link out of home": link,
"folder link out": filepath.Join(dirLink, "secret.pdf"),
"dot-dot out of home": filepath.Join(h, "..", filepath.Base(outside), "secret.pdf"),
"missing in home": filepath.Join(h, "none.pdf"),
"missing outside home": filepath.Join(outside, "none.pdf"),
"folder": filepath.Join(h, "folder"),
"the home folder itself": h,
"relative": "secret.pdf",
"proc": "/proc/self/environ",
} {
_, err := r.call(t, "printer.print", map[string]any{"path": path})
if err == nil || err.Error() != PathRefused {
t.Errorf("%s: error = %v, want exactly %q", name, err, PathRefused)
}
}
text := filepath.Join(h, "fake.pdf")
os.WriteFile(text, []byte("just text, named .pdf"), 0o600)
if _, err := r.call(t, "printer.print", map[string]any{"path": text}); err == nil || !strings.Contains(err.Error(), "neither a PDF nor a JPEG") {
t.Errorf("text in home: error = %v", err)
}
if _, err := r.call(t, "printer.print", map[string]any{"path": text, "object": sha256Hex([]byte("x")) + ".pdf"}); err == nil {
t.Error("path with object accepted")
}
if r.bus.asks() != 0 || fake.body != nil {
t.Errorf("%d asks queued, printer sent %d bytes", r.bus.asks(), len(fake.body))
}
// A symbolic link inside the home folder to a file inside it is fine.
inside := filepath.Join(h, "real.pdf")
os.WriteFile(inside, pdfOf(3000), 0o600)
os.Symlink(inside, filepath.Join(h, "ok.pdf"))
if _, err := r.call(t, "printer.print", map[string]any{"path": filepath.Join(h, "ok.pdf")}); err != nil {
t.Errorf("a link within home: %v", err)
}
}
modules/brother/cmd/brother/fetch.go
+227
View File
@@ -0,0 +1,227 @@
package main
// A web link the device process fetches (novox/hq ADR 0468 clause 6, issue 1599). The process holds the bucket's key and
// the printer's password and sits inside the mesh, so a URL from any caller would otherwise reach the store, the
// printer's admin pages or the controller. Only http and https are taken. The fetch takes at most 60 s, 100 MiB and 5
// redirects. Each host is resolved once, every address it gives is checked, and the connection is made to an address
// that was checked, never to a second lookup's. Loopback, private (RFC 1918, RFC 4193), link-local, shared (RFC 6598),
// unspecified, multicast and the mesh's own tunnel range are refused, after every redirect too.
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/netip"
"net/url"
"os"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/objectlink"
)
const (
// FetchTime, FetchMost and FetchRedirects are the fetch's bounds (ADR 0468 clause 6).
FetchTime = 60 * time.Second
FetchMost = objectlink.MaxDocument
FetchRedirects = 5
// MostURL is the longest address printer.print takes.
MostURL = 2048
)
// FetchFacts is the file the mesh writes for the fetch (files/fetch.json): the tunnel's range, which a module cannot
// know (mesh-sdk go/catalogue/machine_into_files.go, the fact `mesh-range`).
type FetchFacts struct {
MeshRange string `json:"mesh-range"`
}
// ReadFetchFacts reads the fetch's facts; a file that names no range is refused, since a fetch that does not know the
// mesh's range could reach it.
func ReadFetchFacts(path string) (netip.Prefix, error) {
if path == "" {
return netip.Prefix{}, errors.New("the module was started without the fetch's facts (BROTHER_FETCH)")
}
raw, err := os.ReadFile(path)
if err != nil {
return netip.Prefix{}, fmt.Errorf("the fetch's facts cannot be read: %w", err)
}
var f FetchFacts
if err := json.Unmarshal(raw, &f); err != nil {
return netip.Prefix{}, fmt.Errorf("the fetch's facts are not the module's: %v", err)
}
p, err := netip.ParsePrefix(strings.TrimSpace(f.MeshRange))
if err != nil {
return netip.Prefix{}, fmt.Errorf("the fetch's facts name no mesh range (%q): %v", f.MeshRange, err)
}
return p, nil
}
// CheckURLScheme is what the tool side can say of an address without looking at the network: it is http or https.
func CheckURLScheme(raw string) error {
if len(raw) > MostURL {
return fmt.Errorf("url is at most %d characters", MostURL)
}
u, err := url.Parse(raw)
if err != nil || u.Host == "" {
return fmt.Errorf("url %q is not an address", raw)
}
if u.Scheme != "http" && u.Scheme != "https" {
return fmt.Errorf("url %q has the scheme %q: only http and https are fetched (novox/hq ADR 0468 clause 6)", raw, u.Scheme)
}
return nil
}
var refusedRanges = []struct {
prefix netip.Prefix
class string
}{
{netip.MustParsePrefix("0.0.0.0/8"), "an unspecified address"},
{netip.MustParsePrefix("10.0.0.0/8"), "a private address (RFC 1918)"},
{netip.MustParsePrefix("100.64.0.0/10"), "a shared address (RFC 6598)"},
{netip.MustParsePrefix("127.0.0.0/8"), "a loopback address"},
{netip.MustParsePrefix("169.254.0.0/16"), "a link-local address"},
{netip.MustParsePrefix("172.16.0.0/12"), "a private address (RFC 1918)"},
{netip.MustParsePrefix("192.168.0.0/16"), "a private address (RFC 1918)"},
{netip.MustParsePrefix("224.0.0.0/4"), "a multicast address"},
{netip.MustParsePrefix("::/128"), "an unspecified address"},
{netip.MustParsePrefix("::1/128"), "a loopback address"},
{netip.MustParsePrefix("fc00::/7"), "a private address (RFC 4193)"},
{netip.MustParsePrefix("fe80::/10"), "a link-local address"},
{netip.MustParsePrefix("ff00::/8"), "a multicast address"},
}
// Refusal says why an address may not be fetched from, or "" when it may.
func (f *Fetcher) Refusal(a netip.Addr) string {
a = a.Unmap()
for _, r := range refusedRanges {
if r.prefix.Contains(a) {
return r.class
}
}
if f.MeshRange.IsValid() && f.MeshRange.Contains(a) {
return "an address of the mesh's own network"
}
return ""
}
// Fetcher fetches a web link under the rules above. Resolve and Dial are what a test replaces.
type Fetcher struct {
MeshRange netip.Prefix
Resolve func(ctx context.Context, host string) ([]netip.Addr, error)
Dial func(ctx context.Context, network, address string) (net.Conn, error)
Timeout time.Duration
Most int64
Redirects int
}
// NewFetcher is a fetcher with the real resolver and dialer and the bounds of the ADR.
func NewFetcher(meshRange netip.Prefix) *Fetcher {
d := &net.Dialer{Timeout: 15 * time.Second}
return &Fetcher{MeshRange: meshRange, Dial: d.DialContext, Timeout: FetchTime, Most: FetchMost, Redirects: FetchRedirects,
Resolve: func(ctx context.Context, host string) ([]netip.Addr, error) {
return net.DefaultResolver.LookupNetIP(ctx, "ip", host)
}}
}
// Fetched is a document fetched: its bytes and the media type the server gave.
type Fetched struct {
Body []byte
ContentType string
}
// Fetch gets the document at the address. Every refusal names why and happens before a connection to the refused
// address is made; a body over the limit or a fetch over the time stores nothing, since nothing is returned.
func (f *Fetcher) Fetch(ctx context.Context, raw string) (*Fetched, error) {
ctx, cancel := context.WithTimeout(ctx, f.Timeout)
defer cancel()
current := raw
for hop := 0; ; hop++ {
if err := CheckURLScheme(current); err != nil {
return nil, err
}
u, _ := url.Parse(current)
addr, err := f.checked(ctx, u.Hostname())
if err != nil {
return nil, fmt.Errorf("%s: %w", current, err)
}
port := u.Port()
if port == "" {
port = map[string]string{"http": "80", "https": "443"}[u.Scheme]
}
target := net.JoinHostPort(addr.String(), port)
tr := &http.Transport{
Proxy: nil, // never the environment's proxy: the address checked is the address dialled
DisableKeepAlives: true,
DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) {
return f.Dial(ctx, network, target)
},
}
client := &http.Client{Transport: tr, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, current, nil)
if err != nil {
tr.CloseIdleConnections()
return nil, err
}
resp, err := client.Do(req)
if err != nil {
tr.CloseIdleConnections()
return nil, fmt.Errorf("%s could not be fetched: %w", current, err)
}
if resp.StatusCode >= 300 && resp.StatusCode < 400 && resp.Header.Get("Location") != "" {
loc, lerr := u.Parse(resp.Header.Get("Location"))
resp.Body.Close()
tr.CloseIdleConnections()
if lerr != nil {
return nil, fmt.Errorf("%s redirects to an address that is not one: %v", current, lerr)
}
if hop >= f.Redirects {
return nil, fmt.Errorf("%s is redirected more than %d times", raw, f.Redirects)
}
current = loc.String()
continue
}
defer tr.CloseIdleConnections()
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("%s answered %s", current, resp.Status)
}
if resp.ContentLength > f.Most {
return nil, fmt.Errorf("%s is %d bytes, over the %d a document may be (novox/hq ADR 0468 clause 6)", current, resp.ContentLength, f.Most)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, f.Most+1))
if err != nil {
return nil, fmt.Errorf("%s could not be read within %s: %w", current, f.Timeout, err)
}
if int64(len(body)) > f.Most {
return nil, fmt.Errorf("%s is over the %d bytes a document may be (novox/hq ADR 0468 clause 6)", current, f.Most)
}
return &Fetched{Body: body, ContentType: resp.Header.Get("Content-Type")}, nil
}
}
// checked resolves the host once, refuses it when any address it gives is refused, and answers the address to dial.
func (f *Fetcher) checked(ctx context.Context, host string) (netip.Addr, error) {
var addrs []netip.Addr
if ip, err := netip.ParseAddr(host); err == nil {
addrs = []netip.Addr{ip}
} else {
found, err := f.Resolve(ctx, host)
if err != nil {
return netip.Addr{}, fmt.Errorf("the host %s cannot be resolved: %w", host, err)
}
addrs = found
}
if len(addrs) == 0 {
return netip.Addr{}, fmt.Errorf("the host %s resolves to no address", host)
}
for _, a := range addrs {
if class := f.Refusal(a); class != "" {
return netip.Addr{}, fmt.Errorf("refused: %s is %s, and the device process fetches nothing inside the mesh or the machine (novox/hq ADR 0468 clause 6)", a, class)
}
}
return addrs[0].Unmap(), nil
}
modules/brother/cmd/brother/fetch_test.go
+262
View File
@@ -0,0 +1,262 @@
package main
import (
"context"
"errors"
"net"
"net/http"
"net/http/httptest"
"net/netip"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
)
// A fetch rig: names resolve as the test says, and every dial goes to the test's server whatever address was asked,
// after the fetcher has checked the address it was about to dial and told the dial what that was.
type fetchRig struct {
f *Fetcher
mu sync.Mutex
dialed []string
lookups map[string]int
names map[string][]netip.Addr
// after is what a host answers from its second lookup on, when it answers differently (a rebinding).
after map[string][]netip.Addr
srv *httptest.Server
}
func newFetchRig(t *testing.T, h http.HandlerFunc) *fetchRig {
t.Helper()
r := &fetchRig{lookups: map[string]int{}, names: map[string][]netip.Addr{}, after: map[string][]netip.Addr{}}
r.srv = httptest.NewServer(h)
t.Cleanup(r.srv.Close)
r.f = &Fetcher{MeshRange: netip.MustParsePrefix("10.10.0.0/24"), Timeout: 5 * time.Second, Most: 1 << 20, Redirects: FetchRedirects,
Resolve: func(_ context.Context, host string) ([]netip.Addr, error) {
r.mu.Lock()
defer r.mu.Unlock()
r.lookups[host]++
a, ok := r.names[host]
if !ok {
return nil, errors.New("no such host")
}
if again, rebinds := r.after[host]; rebinds && r.lookups[host] > 1 {
return again, nil
}
return a, nil
},
Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
r.mu.Lock()
r.dialed = append(r.dialed, address)
r.mu.Unlock()
return (&net.Dialer{}).DialContext(ctx, network, r.srv.Listener.Addr().String())
}}
return r
}
func addrs(s ...string) []netip.Addr {
var out []netip.Addr
for _, a := range s {
out = append(out, netip.MustParseAddr(a))
}
return out
}
// 1599/AC1: a url with a scheme other than http or https is refused by name.
func TestAURLOfAnotherSchemeIsRefused(t *testing.T) {
r := newFetchRig(t, func(http.ResponseWriter, *http.Request) {})
for _, u := range []string{"file:///etc/passwd", "ftp://example.com/a.pdf", "gopher://example.com/", "javascript:alert(1)", "//example.com/a.pdf"} {
_, err := r.f.Fetch(t.Context(), u)
if err == nil || (!strings.Contains(err.Error(), "only http and https") && !strings.Contains(err.Error(), "is not an address")) {
t.Errorf("%s: error = %v", u, err)
}
if err := CheckURLScheme(u); err == nil {
t.Errorf("%s: the tool side took it", u)
}
}
if len(r.dialed) != 0 {
t.Errorf("dialled %v", r.dialed)
}
}
// 1599/AC2: a host that is, or resolves to, a loopback, private, link-local, shared, unspecified or multicast address,
// or one in the mesh's range, is refused naming the class, and no connection is made.
func TestAddressesInsideTheMeshAndTheMachineAreRefused(t *testing.T) {
r := newFetchRig(t, func(http.ResponseWriter, *http.Request) { t.Error("a request reached the server") })
cases := map[string]string{
"127.0.0.1": "loopback", "::1": "loopback", "10.0.0.1": "private address (RFC 1918)", "192.168.1.1": "private address (RFC 1918)",
"172.16.0.1": "private address (RFC 1918)", "fd00::1": "private address (RFC 4193)", "169.254.1.1": "link-local", "fe80::1": "link-local",
"100.64.0.1": "shared address (RFC 6598)", "0.0.0.0": "unspecified", "::": "unspecified", "224.0.0.1": "multicast", "ff02::1": "multicast",
"10.10.0.7": "private address (RFC 1918)", "::ffff:127.0.0.1": "loopback",
}
for ip, class := range cases {
host := ip
if strings.Contains(ip, ":") {
host = "[" + ip + "]"
}
_, err := r.f.Fetch(t.Context(), "http://"+host+"/a.pdf")
if err == nil || !strings.Contains(err.Error(), class) {
t.Errorf("%s: error = %v, want the class %q", ip, err, class)
}
// The same address behind a name.
r.names["inside.test"] = addrs("203.0.113.9", ip)
if _, err := r.f.Fetch(t.Context(), "http://inside.test/a.pdf"); err == nil || !strings.Contains(err.Error(), class) {
t.Errorf("inside.test -> %s: error = %v", ip, err)
}
}
// The mesh's range from fetch.json, when it is not a private range.
r.f.MeshRange = netip.MustParsePrefix("203.0.113.0/24")
if _, err := r.f.Fetch(t.Context(), "http://203.0.113.5/a.pdf"); err == nil || !strings.Contains(err.Error(), "the mesh's own network") {
t.Errorf("the mesh range: error = %v", err)
}
if len(r.dialed) != 0 {
t.Errorf("a connection was made to %v", r.dialed)
}
}
// 1599/AC3: a redirect to such an address is refused; a sixth redirect is refused; a redirect to a public address is followed.
func TestRedirectsAreCheckedAndBounded(t *testing.T) {
hops := 0
r := newFetchRig(t, func(w http.ResponseWriter, req *http.Request) {
switch req.URL.Path {
case "/to-loopback":
http.Redirect(w, req, "http://127.0.0.1/secret", http.StatusFound)
case "/to-name":
http.Redirect(w, req, "http://inside.test/x", http.StatusFound)
case "/loop":
hops++
http.Redirect(w, req, "/loop", http.StatusFound)
case "/ok":
http.Redirect(w, req, "http://public.test/final.pdf", http.StatusMovedPermanently)
default:
w.Write([]byte("%PDF-1.7 final"))
}
})
r.names["public.test"] = addrs("203.0.113.9")
r.names["inside.test"] = addrs("192.168.0.9")
if _, err := r.f.Fetch(t.Context(), "http://public.test/to-loopback"); err == nil || !strings.Contains(err.Error(), "loopback") {
t.Errorf("redirect to loopback: error = %v", err)
}
if _, err := r.f.Fetch(t.Context(), "http://public.test/to-name"); err == nil || !strings.Contains(err.Error(), "private address (RFC 1918)") {
t.Errorf("redirect to a name: error = %v", err)
}
if _, err := r.f.Fetch(t.Context(), "http://public.test/loop"); err == nil || !strings.Contains(err.Error(), "redirected more than 5 times") {
t.Errorf("a loop: error = %v", err)
}
if hops != 6 {
t.Errorf("the loop was followed %d times, want the first request and 5 redirects (6 requests)", hops)
}
got, err := r.f.Fetch(t.Context(), "http://public.test/ok")
if err != nil || string(got.Body) != "%PDF-1.7 final" {
t.Errorf("a redirect to a public address: %v, %v", got, err)
}
}
// 1599/AC4: a body over the limit and a fetch over the time are refused, and nothing is returned to be stored.
func TestABodyOverTheLimitAndAFetchOverTheTimeAreRefused(t *testing.T) {
r := newFetchRig(t, func(w http.ResponseWriter, req *http.Request) {
switch req.URL.Path {
case "/big":
w.Write([]byte(strings.Repeat("x", 2000)))
case "/chunked":
w.(http.Flusher).Flush() // no length is declared
w.Write([]byte(strings.Repeat("x", 2000)))
case "/slow":
time.Sleep(500 * time.Millisecond)
w.Write([]byte("late"))
}
})
r.names["public.test"] = addrs("203.0.113.9")
r.f.Most = 1000
for _, p := range []string{"/big", "/chunked"} {
if got, err := r.f.Fetch(t.Context(), "http://public.test"+p); err == nil || got != nil || !strings.Contains(err.Error(), "over the") {
t.Errorf("%s: %v, %v", p, got, err)
}
}
r.f.Timeout = 100 * time.Millisecond
if got, err := r.f.Fetch(t.Context(), "http://public.test/slow"); err == nil || got != nil {
t.Errorf("a slow fetch: %v, %v", got, err)
}
}
// 1599/AC5: the connection goes to the address that was checked: a name that answers differently on a second lookup
// is not dialled again, and each host is looked up once.
func TestTheCheckedAddressIsTheOneDialled(t *testing.T) {
r := newFetchRig(t, func(w http.ResponseWriter, req *http.Request) { w.Write([]byte("%PDF-1.7 x")) })
r.names["rebind.test"] = addrs("203.0.113.9")
r.after["rebind.test"] = addrs("127.0.0.1") // a second lookup would say loopback
if _, err := r.f.Fetch(t.Context(), "http://rebind.test:8080/a.pdf"); err != nil {
t.Fatal(err)
}
if len(r.dialed) != 1 || r.dialed[0] != "203.0.113.9:8080" {
t.Errorf("dialled %v, want the checked address and the url's port", r.dialed)
}
if r.lookups["rebind.test"] != 1 {
t.Errorf("the host was looked up %d times", r.lookups["rebind.test"])
}
}
// 1599/AC6: a fetched PDF is stored under its SHA-256 and printed as that object.
func TestAFetchedPDFIsStoredUnderItsHashAndPrinted(t *testing.T) {
fake := newIPPFake(t, 41, 5)
r := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
doc := pdfOf(4000)
web := newFetchRig(t, func(w http.ResponseWriter, req *http.Request) {
w.Header().Set("Content-Type", "application/pdf")
w.Write(doc)
})
web.names["public.test"] = addrs("203.0.113.9")
r.worker.Fetcher = func() (*Fetcher, error) { return web.f, nil }
got, err := r.call(t, "printer.print", map[string]any{"url": "http://public.test/letter.pdf"})
if err != nil {
t.Fatal(err)
}
m := got.(map[string]any)
key := sha256Hex(doc)
if m["object"] != key || m["job-id"] != float64(41) || m["format"] != "application/pdf" {
t.Errorf("answer = %v", m)
}
if stored, err := r.objects.Get(t.Context(), key); err != nil || string(stored) != string(doc) {
t.Errorf("stored under the hash: %v", err)
}
if _, sent := fake.request(t); string(sent) != string(doc) {
t.Error("the printer was not sent the fetched document")
}
queued := r.bus.published[0]
if queued.URL != "http://public.test/letter.pdf" || queued.Object != "" {
t.Errorf("queued %+v", queued)
}
if err := CheckURLScheme("https://public.test/letter.pdf"); err != nil {
t.Errorf("https refused: %v", err)
}
// A refusal inside the mesh is answered, and nothing is stored nor printed.
fake.body = nil
before := len(r.objects.keys())
if _, err := r.call(t, "printer.print", map[string]any{"url": "http://127.0.0.1:9000/mesh-shanks-brother/x"}); err == nil || !strings.Contains(err.Error(), "loopback") {
t.Errorf("a loopback address: error = %v", err)
}
if len(r.objects.keys()) != before || fake.body != nil {
t.Error("something was stored or printed for a refused address")
}
}
// The fetch's facts are the file the mesh writes (files/fetch.json) from the `mesh-range` machine fact, and the device
// process reads them.
func TestTheFetchFactsAreTheMeshRange(t *testing.T) {
raw, err := os.ReadFile("../../files/fetch.json")
if err != nil || !strings.Contains(string(raw), `"mesh-range": "${machine:mesh-range}"`) {
t.Fatalf("files/fetch.json = %s, %v", raw, err)
}
file := filepath.Join(t.TempDir(), "fetch.json")
os.WriteFile(file, []byte(`{"mesh-range": "10.10.0.0/24"}`), 0o600)
p, err := ReadFetchFacts(file)
if err != nil || p.String() != "10.10.0.0/24" {
t.Errorf("prefix %v, %v", p, err)
}
os.WriteFile(file, []byte(`{}`), 0o600)
if _, err := ReadFetchFacts(file); err == nil {
t.Error("facts with no range were taken")
}
}
modules/brother/cmd/brother/glass_test.go
+1 -1
View File
@@ -264,7 +264,7 @@ func TestAnOpenDocumentIsFoundAgainAfterARestartAndItsPagesAreDeletedWhenClosed(
docExpiry = 200 * time.Millisecond
t.Cleanup(func() { docExpiry = old })
s := &Service{Settings: Settings{Scanner: f.base(), Folder: folder, Resolution: 200, ColorMode: "color"}, Now: func() time.Time { return fixedNow },
Asks: askBus, AnswerWait: 5 * time.Second, ScanWait: 5 * time.Second, AnswerPoll: 5 * time.Millisecond, Jobs: NewJobs()}
Asks: askBus, AnswerWait: 5 * time.Second, ScanWait: 5 * time.Second, AnswerPoll: 5 * time.Millisecond, Jobs: NewJobs(), Fetch: fetchClient, DocExpiry: docExpiry}
if err := s.Recover(context.Background()); err != nil {
t.Fatal(err)
}
modules/brother/cmd/brother/ipp.go
+33
View File
@@ -245,6 +245,39 @@ func PrinterStateOf(ctx context.Context, host string) (PrinterState, error) {
return ans.State()
}
// formatAttributes is what the format check asks for: the document formats the printer takes (RFC 8011 §5.4.22).
var formatAttributes = []string{"document-format-supported"}
// FormatsOf asks the printer, over IPP, which document formats it lists in document-format-supported.
func FormatsOf(ctx context.Context, host string) ([]string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, ippURL(host),
bytes.NewReader(EncodeGetPrinterAttributes(ippPrinterURI(host), formatAttributes)))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/ipp")
resp, err := (&http.Client{Timeout: 15 * time.Second}).Do(req)
if err != nil {
return nil, fmt.Errorf("the printer's IPP service does not answer: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if err != nil {
return nil, fmt.Errorf("the printer's IPP answer cannot be read: %w", err)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("the printer's IPP service answered %s", resp.Status)
}
ans, err := DecodeIPP(body)
if err != nil {
return nil, err
}
if ans.Status >= 0x0100 {
return nil, fmt.Errorf("the printer refused the IPP request with status 0x%04x", ans.Status)
}
return ans.texts("document-format-supported"), nil
}
// PrintOptions are what a Print-Job asks of the printer.
type PrintOptions struct {
Format string // document-format, application/pdf or image/jpeg
modules/brother/cmd/brother/main.go
+21 -19
View File
@@ -11,6 +11,9 @@ import (
stdio "git.novox.be/novox/mesh-sdk/go"
)
// uploadVia is how the tool side asks a machine's node-files for an upload; a test puts its own in its place.
var uploadVia = sdkUpload
func main() {
if len(os.Args) > 1 && os.Args[1] == "device" {
os.Exit(device())
@@ -84,7 +87,7 @@ func askFrom(args map[string]any) (Ask, error) {
var a Ask
var err error
for name, dst := range map[string]*string{"page": &a.Page, "object": &a.Object, "upload": &a.Upload, "sha256": &a.SHA256,
"data": &a.Data, "sides": &a.Sides, "media": &a.Media} {
"data": &a.Data, "sides": &a.Sides, "media": &a.Media, "path": &a.Path, "node": &a.Node, "url": &a.URL, "ask": &a.Of} {
if *dst, err = strArg(args, name); err != nil {
return a, err
}
@@ -109,7 +112,8 @@ func tools(settingsFile string, now func() time.Time) []stdio.Tool {
}
return &Service{Settings: s, Node: os.Getenv("MESH_NODE"), Now: now, Asks: askBus,
AnswerWait: answerWait, ScanWait: scanWait, AnswerPoll: answerPoll,
QuickWait: quickWait, FeederWait: feederWait, Jobs: jobs}, nil
QuickWait: quickWait, FeederWait: feederWait, Jobs: jobs, Upload: uploadVia, Fetch: fetchClient, DocExpiry: docExpiry, DocRetry: docRetry,
Bucket: func() (string, error) { return ReadBucket(os.Getenv("BROTHER_BUCKET")) }}, nil
}
// ask is a tool that reads its arguments, and answers what the service gives.
ask := func(run func(s *Service, ctx context.Context, a Ask) (map[string]any, error), bound time.Duration) func(map[string]any) (any, error) {
@@ -147,39 +151,37 @@ func tools(settingsFile string, now func() time.Time) []stdio.Tool {
},
{
Name: "printer.print",
Description: "Print a document on the printer: a file under the home folder of the account the tool runs as, on the machine it runs on (path: reads it, uploads it in parts " +
"and prints it, in one call), a scan kept in the bucket (object), or a completed upload (upload; for a module that sends " +
"bytes over the bus). Only PDF and JPEG, told by the file's bytes and not its name. The device " +
"process sends it as an IPP Print-Job in its own format and answers the job's id and state (with path, the upload id and the size).",
Description: "Print a document on the printer, in one call: a key in the bucket (object: a scan, or a document stored there), a file on any " +
"machine of the mesh (path with node: that machine's node-files sends it into the bucket, a place it keeps a secret is " +
"refused there by name and nothing is printed), or a web link (url). Name exactly one. A Word, spreadsheet or slide " +
"document is turned into a PDF first; otherwise only PDF and JPEG, told by the bytes and not the name, and only a format " +
"the printer lists in document-format-supported. The device process sends it as an IPP Print-Job and answers the job's " +
"id and state. No document crosses the bus. A call answers within 25 s (the tool runner cuts a call at 30 s): a print not done by then goes on and is answered as started with its ask id, and printer.print with only that ask reads it.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"path": str("an absolute path of a PDF or JPEG file under the tool account's home folder (symbolic links are resolved and the real path must lie there too; any other path is refused in one fixed way)"),
"object": str("a scan's key, as scan.added names it"),
"upload": str("a completed upload's id"),
"object": str("a stored document's key: a scan's key, as scan.added names it, or the key node-files.upload answered"),
"path": str("an absolute path of a file on the machine node names"),
"node": str("the machine path is on, such as g14"),
"url": str("an http or https address of a document or a page"),
"ask": str("the ask id of an earlier print that was answered as started: alone, it reads that print's outcome"),
"copies": num("1 to 99 (default 1)"),
"sides": str("one-sided (default), two-sided-long-edge or two-sided-short-edge"),
"media": str("the paper's size name (default " + DefaultMedia + ")"),
}},
Run: func(args map[string]any) (any, error) {
path, err := strArg(args, "path")
if err != nil {
return nil, err
}
a, err := askFrom(args)
if err != nil {
return nil, err
}
if a.Of != "" && len(args) > 1 {
return nil, fmt.Errorf("ask is read on its own: give the ask id alone to read an earlier print")
}
s, err := service()
if err != nil {
return nil, err
}
if path == "" {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
return s.PrinterPrint(ctx, a)
}
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
defer cancel()
return s.PrintFile(ctx, path, a)
return s.PrinterPrint(ctx, a)
},
},
{
modules/brother/cmd/brother/manifest_test.go
+30 -3
View File
@@ -13,7 +13,8 @@ import (
)
type manifest struct {
Requires []string `json:"requires"`
Requires []requirement `json:"requires"`
Invokes []string `json:"invokes"`
Binds map[string]string `json:"binds"`
Secrets map[string]string `json:"secrets"`
Emits []string `json:"emits"`
@@ -28,6 +29,31 @@ type manifest struct {
} `json:"build"`
}
// requirement is an entry of `requires`: a name, or an object with the name and the shares of the bucket.
type requirement struct {
Name string `json:"name"`
Shares []struct {
To string `json:"to"`
Methods []string `json:"methods"`
} `json:"shares"`
}
func (r *requirement) UnmarshalJSON(b []byte) error {
if err := json.Unmarshal(b, &r.Name); err == nil {
return nil
}
type plain requirement
return json.Unmarshal(b, (*plain)(r))
}
func (m manifest) requireNames() []string {
var out []string
for _, r := range m.Requires {
out = append(out, r.Name)
}
return out
}
type seat struct {
Name string `json:"name"`
Scope string `json:"scope"`
@@ -265,6 +291,7 @@ func TestTheAdminProcessIsDeclaredAsMeshMailsSenderIs(t *testing.T) {
"BROTHER_ADMIN_PASSWORD": m.OwnSecrets["admin-password"]["path"],
"BROTHER_BROKER": m.OwnSecrets["broker-device"]["path"],
"BROTHER_STORE": "${dir:state}/store.json",
"BROTHER_FETCH": "${dir:state}/fetch.json", // 1599: the tunnel's range, which the fetch never reaches
}
if !reflect.DeepEqual(env, want) {
t.Errorf("env = %v, want %v", env, want)
@@ -330,8 +357,8 @@ func TestThePrinterAsksStreamIsAWorkQueue(t *testing.T) {
// written into store.json, a file only the device account can read; the tools' bundle is not told where it is.
func TestTheBucketIsRequiredAndItsKeyIsOnlyTheDeviceAccounts(t *testing.T) {
m := readManifest(t)
if !reflect.DeepEqual(m.Requires, []string{"s3-bucket"}) {
t.Errorf("requires = %v", m.Requires)
if !reflect.DeepEqual(m.requireNames(), []string{"s3-bucket"}) {
t.Errorf("requires = %v", m.requireNames())
}
if m.Binds["s3-bucket"] != "${dir:state}/store-binding.json" || m.Secrets["s3-bucket"] != "${dir:state}/store.secret" {
t.Errorf("binds %v, secrets %v", m.Binds, m.Secrets)
modules/brother/cmd/brother/objects.go
+19
View File
@@ -24,6 +24,8 @@ import (
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/objectlink"
)
// Objects is the module's bucket.
@@ -55,6 +57,13 @@ func Store(ctx context.Context, o Objects, key, mediaType string, data []byte) e
return o.Put(ctx, key, mediaType, data)
}
// Presigner is a store that signs a link to one of its objects (novox/hq ADR 0468 §3): the device process's, with the
// binding's key, which reaches this module's bucket only.
type Presigner interface {
// Link is a link for GET or PUT of one key that lives for the given time (at most ten minutes).
Link(method, key string, expires time.Duration) (string, error)
}
// StoreFacts is the file the mesh writes from the s3-bucket binding and its secret.
type StoreFacts struct {
Scheme string `json:"scheme"`
@@ -114,6 +123,16 @@ func NewS3(f StoreFacts) *S3 {
func (s *S3) Bucket() string { return s.Name }
// Link signs a link for one object of the bucket with the binding's key, for the host the binding gives.
func (s *S3) Link(method, key string, expires time.Duration) (string, error) {
now := time.Now
if s.Now != nil {
now = s.Now
}
return objectlink.Presign(objectlink.Request{Method: method, Endpoint: s.Endpoint, Bucket: s.Name, Key: key,
AccessKey: s.Access, SecretKey: s.Secret, Region: s.Region, Expires: expires, Now: now()})
}
func (s *S3) do(ctx context.Context, method, key, mediaType string, body []byte) (*http.Response, error) {
u, err := url.Parse(s.Endpoint + "/" + s.Name + "/" + key)
if err != nil {
modules/brother/cmd/brother/objects_test.go
+33
View File
@@ -1,6 +1,7 @@
package main
import (
"bytes"
"context"
"fmt"
"io"
@@ -9,9 +10,11 @@ import (
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"testing"
"time"
)
// memObjects is the bucket in memory, for the tests of what the device process keeps.
@@ -19,6 +22,10 @@ type memObjects struct {
mu sync.Mutex
kept map[string][]byte
puts int
// corrupt, when set, makes a GET of a link answer bytes that are not the kept ones.
corrupt bool
// links records each link signed: "<method> <key> <seconds>".
links []string
}
func newMemObjects() *memObjects { return &memObjects{kept: map[string][]byte{}} }
@@ -68,6 +75,32 @@ func (m *memObjects) Delete(_ context.Context, key string) error {
return nil
}
// Link is a link the bucket in memory serves itself (RoundTrip): the key in the path and the life in the query, as a
// presigned link says them (X-Amz-Expires).
func (m *memObjects) Link(method, key string, expires time.Duration) (string, error) {
m.mu.Lock()
m.links = append(m.links, method+" "+key+" "+strconv.Itoa(int(expires/time.Second)))
m.mu.Unlock()
return "http://" + m.Bucket() + ".objects.test/" + key + "?X-Amz-Expires=" + strconv.Itoa(int(expires/time.Second)), nil
}
// RoundTrip answers a GET of a link made by Link with the object kept under its key.
func (m *memObjects) RoundTrip(r *http.Request) (*http.Response, error) {
m.mu.Lock()
b, ok := m.kept[strings.TrimPrefix(r.URL.Path, "/")]
corrupt := m.corrupt
m.mu.Unlock()
if corrupt && ok {
b = append(append([]byte(nil), b...), 0)
}
status := 200
if !ok {
status = 404
}
return &http.Response{StatusCode: status, Body: io.NopCloser(bytes.NewReader(b)), ContentLength: int64(len(b)),
Header: http.Header{}, Request: r}, nil
}
func (m *memObjects) keys() []string {
m.mu.Lock()
defer m.mu.Unlock()
modules/brother/cmd/brother/printer.go
+160 -46
View File
@@ -36,6 +36,11 @@ const (
// DeviceProcessID is the device process's id as module.json's resource "device" names it: the controller
// delivers its bus account as the own secret broker-device.
DeviceProcessID = "device"
// PrintTakeUp is how long a print ask may wait on the queue before the device process takes it up, and PrintFollow
// how long the tool side follows it to its end: the device process may fetch and convert before the job. A call
// answers within QuickWait whatever these are, and the print goes on (the tool runner cuts every call at 30 s).
PrintTakeUp = 3 * time.Minute
PrintFollow = 6 * time.Minute
// AnswerWait is how long a verb waits for the device process; ScanWait how long a scan does.
AnswerWait = 60 * time.Second
ScanWait = 180 * time.Second
@@ -88,6 +93,14 @@ type Ask struct {
Document string `json:"document,omitempty"` // scan: start, or an open document's id: a glass page into it
Finish bool `json:"finish,omitempty"` // scan: close the document named in Document into one PDF
// Path with Node are what printer.print is asked for a file of a machine: the tool side turns them into an Object
// before anything is queued, so the queued ask carries a key and the options and never these (ADR 0468 clause 5).
// A URL is queued as given: the device process alone fetches it (clause 6).
Path string `json:"-"` // print: a file on the machine Node
Of string `json:"-"` // print: the ask id of an earlier print, whose outcome is read
Node string `json:"-"` // print: the machine Path is on
URL string `json:"url,omitempty"` // print: a web document the device process fetches (issue 1599)
}
// AskBus is how the tools reach the device process: they queue an ask and read an answer back.
@@ -145,7 +158,10 @@ func (s *Service) call(ctx context.Context, ask Ask, wait time.Duration) (map[st
// queue puts the ask on the work queue and answers its id, under which the answer is written. The asker waits
// startBy for the device process to take the ask up (0 is for as long as it is waited for).
func (s *Service) queue(ask Ask, startBy time.Duration) (string, error) {
ask.ID, ask.At = newAskID(), s.Now().UTC()
if ask.ID == "" {
ask.ID = newAskID()
}
ask.At = s.Now().UTC()
if startBy > 0 {
ask.Deadline = ask.At.Add(startBy)
}
@@ -211,9 +227,12 @@ func validPage(page string) bool {
var (
scanKeyPattern = regexp.MustCompile(`^[0-9a-f]{64}\.pdf$`)
uploadPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`)
mediaPattern = regexp.MustCompile(`^[a-z0-9_.-]{1,64}$`)
sha256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
// printKeyPattern is a document the bucket keeps under its SHA-256: a scan's key, or the key node-files.upload
// answers for a file sent from a machine (the bare SHA-256).
printKeyPattern = regexp.MustCompile(`^[0-9a-f]{64}(\.pdf)?$`)
uploadPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`)
mediaPattern = regexp.MustCompile(`^[a-z0-9_.-]{1,64}$`)
sha256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
)
// Sides are the values the printer's sides take.
@@ -259,14 +278,23 @@ func checkUpload(a Ask) error {
// checkPrint is what is wrong with an ask to print, or nil, after the defaults are laid in.
func checkPrint(a *Ask) error {
if (a.Object == "") == (a.Upload == "") {
return fmt.Errorf("name either object (a scan's key) or upload (a completed upload's id), not both and not neither")
if a.Upload != "" {
return fmt.Errorf("upload is no longer a source of printer.print: send the file with path and node, or name its key in object")
}
if a.Object != "" && !scanKeyPattern.MatchString(a.Object) {
return fmt.Errorf("object %q is not a scan's key: 64 hex digits and .pdf", a.Object)
if (a.Path == "") != (a.Node == "") {
return fmt.Errorf("path and node go together: path is a file on the machine node names")
}
if a.Upload != "" && !uploadPattern.MatchString(a.Upload) {
return fmt.Errorf("upload %q is not an upload id: letters, digits, - and _, at most 64", a.Upload)
sources := 0
for _, given := range []bool{a.Object != "", a.Path != "", a.URL != ""} {
if given {
sources++
}
}
if sources != 1 {
return fmt.Errorf("name exactly one of object (a key in the bucket), path with node (a file on that machine) or url, not %s", map[bool]string{true: "none", false: "two or more"}[sources == 0])
}
if a.Object != "" && !printKeyPattern.MatchString(a.Object) {
return fmt.Errorf("object %q is not a stored document's key: 64 hex digits, with .pdf for a scan", a.Object)
}
if a.Copies == 0 {
a.Copies = 1
@@ -307,50 +335,42 @@ func (s *Service) PrinterUpload(ctx context.Context, a Ask) (map[string]any, err
return s.call(ctx, a, s.AnswerWait)
}
// PrintFile prints a PDF or a JPEG that lies on the machine the tool runs on: it reads the file, uploads it in
// parts under an upload id of its own and prints that, in one call (issue 1551). The type is read from the
// file's first bytes, never its name; any other file is refused before anything is queued. The answer is the
// print's, with the upload id and the file's size.
func (s *Service) PrintFile(ctx context.Context, path string, a Ask) (map[string]any, error) {
if a.Object != "" || a.Upload != "" {
return nil, fmt.Errorf("name path, or object, or upload: one of the three")
}
// The path is checked first, and refused in one fixed way that tells nothing about the file.
doc, err := ReadDocument(path, MostParts*PartSize)
// sdkUpload asks node's node-files to send a file of that machine into the bucket and answers the key it was
// stored under (issue 1597). The form is the seat's verb on one machine, `seat:<seat>.<verb>@<node>`.
func sdkUpload(node, path, bucket string) (string, error) {
raw, err := stdio.Ask("seat:node-files.upload@"+node, map[string]any{"path": path, "bucket": bucket})
if err != nil {
return nil, err
return "", err
}
a.Upload = "p-" + newAskID()
if err := checkPrint(&a); err != nil {
return nil, err
var ans struct {
Key string `json:"key"`
Answer json.RawMessage `json:"answer"`
}
if s.Settings.Printer == "" {
return nil, unsetError("printer")
if err := json.Unmarshal(raw, &ans); err != nil {
return "", fmt.Errorf("node-files' answer is not JSON: %w", err)
}
if _, _, err := SniffFormat(doc); err != nil {
return nil, fmt.Errorf("file %s: %w", path, err)
if ans.Key == "" && len(ans.Answer) > 0 {
_ = json.Unmarshal(ans.Answer, &ans)
}
sum, n := sha256Hex(doc), partCount(len(doc))
for i := 1; i <= n; i++ {
part, _, _ := cutPartOf(doc, i, PartSize)
up, err := s.PrinterUpload(ctx, Ask{Upload: a.Upload, Part: i, Parts: n, SHA256: sum, Data: base64.StdEncoding.EncodeToString(part)})
if err != nil {
return nil, fmt.Errorf("part %d of %d of %s could not be uploaded: %w", i, n, path, err)
}
if i == n && up["complete"] != true {
return nil, fmt.Errorf("the upload of %s is not complete after its last part: %v", path, up)
}
if ans.Key == "" {
return "", fmt.Errorf("node-files' answer names no key")
}
out, err := s.PrinterPrint(ctx, a)
if err != nil {
return nil, fmt.Errorf("%s is uploaded as %s but could not be printed: %w", path, a.Upload, err)
}
out["path"], out["upload"], out["bytes"] = path, a.Upload, len(doc)
return out, nil
return ans.Key, nil
}
// PrinterPrint asks the device process to print a kept scan or a completed upload.
// PrinterPrint prints a stored document, a file of a machine, or a web link, in one call. A file of a machine is first
// sent into the bucket by that machine's node-files (refused there by name when it is a secret, and then nothing is
// printed); the print ask the device process takes carries the key and the options and no bytes.
//
// The tool runner cuts every call at 30 s, so the call answers within QuickWait, upload included. A print not done by
// then goes on in the background and is answered as started, with its ask id; printer.print {ask} reads its outcome.
func (s *Service) PrinterPrint(ctx context.Context, a Ask) (map[string]any, error) {
if a.Of != "" {
if a.Object != "" || a.Path != "" || a.Node != "" || a.URL != "" || a.Upload != "" {
return nil, fmt.Errorf("ask is read on its own: name exactly one of object, path with node, url, or the ask id of an earlier print")
}
return s.PrintResult(ctx, a.Of)
}
a.Op = OpPrint
if err := checkPrint(&a); err != nil {
return nil, err
@@ -358,9 +378,103 @@ func (s *Service) PrinterPrint(ctx context.Context, a Ask) (map[string]any, erro
if s.Settings.Printer == "" {
return nil, unsetError("printer")
}
return s.call(ctx, a, s.AnswerWait)
bucket := ""
switch {
case a.URL != "":
if err := CheckURLScheme(a.URL); err != nil {
return nil, err
}
case a.Path != "":
if s.Upload == nil {
return nil, fmt.Errorf("this module has no way to ask node-files for an upload")
}
var err error
if bucket, err = s.Bucket(); err != nil {
return nil, err
}
}
id := newAskID()
j := s.Jobs.put(id)
go func() {
j.body, j.err = s.printRun(id, a, bucket)
close(j.done)
}()
return s.printWait(ctx, id, j)
}
// printRun is a print followed to its end, however long the caller waits: the upload, the ask, the device's answer.
func (s *Service) printRun(id string, a Ask, bucket string) (map[string]any, error) {
from := map[string]any{}
if a.Path != "" {
key, err := s.Upload(a.Node, a.Path, bucket)
if err != nil {
return nil, fmt.Errorf("%s on %s could not be sent to the bucket, and nothing is printed: %w", a.Path, a.Node, err)
}
from["path"], from["node"] = a.Path, a.Node
a.Object, a.Path, a.Node = key, "", ""
}
a.ID = id
if _, err := s.queue(a, PrintTakeUp); err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), PrintFollow)
defer cancel()
out, err := s.await(ctx, id, PrintFollow)
if err != nil {
return nil, err
}
for k, v := range from {
out[k] = v
}
return out, nil
}
// printWait gives the print's outcome if it comes within QuickWait, and started with the ask id otherwise.
func (s *Service) printWait(ctx context.Context, id string, j *job) (map[string]any, error) {
quick := s.QuickWait
if quick <= 0 {
quick = QuickWait
}
select {
case <-j.done:
return j.body, j.err
case <-time.After(quick):
return map[string]any{"state": "started", "ask": id,
"note": "the print goes on; read its outcome with printer.print {ask}"}, nil
case <-ctx.Done():
return nil, fmt.Errorf("the caller went away; the print %s goes on: %w", id, ctx.Err())
}
}
// PrintResult is the outcome of the print with this ask id: the device's answer when it is done (a failure is the
// error), and started, still printing, while it is not. A print this process followed is read from it, one it did not
// (it was restarted) from the device's answers, which are kept for 10 minutes.
func (s *Service) PrintResult(ctx context.Context, id string) (map[string]any, error) {
if !askIDPattern.MatchString(id) {
return nil, fmt.Errorf("ask %q is not an ask id: the 24 hex digits printer.print answered", id)
}
if j := s.Jobs.get(id); j != nil {
return s.printWait(ctx, id, j)
}
raw, err := s.Asks.Answer(id)
if err != nil {
return nil, fmt.Errorf("the device process's answers cannot be read: %w", err)
}
if raw == nil {
return map[string]any{"state": "started", "ask": id, "note": "still printing, or this ask is not known"}, nil
}
var out map[string]any
if err := json.Unmarshal(raw, &out); err != nil {
return nil, fmt.Errorf("the device process's answer is not JSON: %w", err)
}
if msg, bad := out["error"].(string); bad {
return nil, fmt.Errorf("%s", msg)
}
return out, nil
}
var askIDPattern = regexp.MustCompile(`^[0-9a-f]{24}$`)
// PrinterStatus is the printer's state over IPP and the scanner's over eSCL, none of it behind a login
// (ADR 0467 clause 5). A part that does not answer is an answer, with why, not an error. Which of printer and
// scanner is not set is said by name, and nothing is contacted.
modules/brother/cmd/brother/printlink_test.go
+318
View File
@@ -0,0 +1,318 @@
package main
import (
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
)
// uploadFake stands for node-files.upload on another machine: it records the ask and answers a key, or refuses.
type uploadFake struct {
mu sync.Mutex
calls []string
key string
err error
}
// withUpload puts a fake node-files in the tool side's place, and the bucket file the mesh writes from the binding
// (files/bucket.json) where the tool reads it.
// seen is the uploads asked so far, read under the fake's lock (the print runs in the background).
func (f *uploadFake) seen() []string {
f.mu.Lock()
defer f.mu.Unlock()
return append([]string(nil), f.calls...)
}
func withUpload(t *testing.T, key string, err error) *uploadFake {
t.Helper()
f := &uploadFake{key: key, err: err}
old := uploadVia
uploadVia = func(node, path, bucket string) (string, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.calls = append(f.calls, node+"|"+path+"|"+bucket)
return f.key, f.err
}
t.Cleanup(func() { uploadVia = old })
file := filepath.Join(t.TempDir(), "bucket.json")
if err := os.WriteFile(file, []byte(`{"bucket": "brother-test"}`), 0o644); err != nil {
t.Fatal(err)
}
t.Setenv("BROTHER_BUCKET", file)
return f
}
// 1598/AC1: none, or two, of object, path+node and url is refused naming the rule; path without node is refused.
func TestPrintTakesExactlyOneSource(t *testing.T) {
fake := newIPPFake(t, 1, 3)
up := withUpload(t, "k", nil)
r := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
key := sha256Hex([]byte("x")) + ".pdf"
for name, args := range map[string]map[string]any{
"none": {},
"only options": {"copies": float64(2)},
"object and path": {"object": key, "path": "/home/ops/a.pdf", "node": "g14"},
"object and url": {"object": key, "url": "https://example.com/a.pdf"},
"path+node and url": {"path": "/home/ops/a.pdf", "node": "g14", "url": "https://example.com/a.pdf"},
"all three": {"object": key, "path": "/home/ops/a.pdf", "node": "g14", "url": "https://example.com/a.pdf"},
"path without node": {"path": "/home/ops/a.pdf"},
"node without path": {"node": "g14", "object": key},
"the retired upload": {"upload": "u1"},
"object and upload": {"object": key, "upload": "u1"},
"path and its object": {"path": "/home/ops/a.pdf", "node": "g14", "object": key},
} {
_, err := r.call(t, "printer.print", args)
if err == nil {
t.Errorf("%s: accepted", name)
continue
}
rule := strings.Contains(err.Error(), "exactly one of object") || strings.Contains(err.Error(), "path and node go together") ||
strings.Contains(err.Error(), "upload is no longer")
if !rule {
t.Errorf("%s: the refusal does not name the rule: %v", name, err)
}
}
if r.bus.asks() != 0 || len(up.seen()) != 0 || fake.body != nil {
t.Errorf("%d asks queued, %d uploads asked, printer sent %d bytes", r.bus.asks(), len(up.seen()), len(fake.body))
}
}
// 1598/AC2: with path and node g14, printer.print asks g14's node-files to upload the path into brother's bucket, and
// queues a print ask that carries the returned key and the options: under 4 KiB, and no path, node or bytes.
func TestPrintOfAFileOfAnotherMachineUploadsThenPrintsTheKey(t *testing.T) {
fake := newIPPFake(t, 31, 5)
doc := pdfOf(6000)
key := sha256Hex(doc)
up := withUpload(t, key, nil)
r := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
r.objects.Put(t.Context(), key, "application/pdf", doc) // what g14's upload stored
got, err := r.call(t, "printer.print", map[string]any{"path": "/home/ops/letter.pdf", "node": "g14", "copies": float64(2)})
if err != nil {
t.Fatal(err)
}
if len(up.seen()) != 1 || up.seen()[0] != "g14|/home/ops/letter.pdf|brother-test" {
t.Fatalf("uploads asked: %v", up.seen())
}
if len(r.bus.published) != 1 {
t.Fatalf("%d asks queued", len(r.bus.published))
}
queued := r.bus.published[0]
raw, _ := json.Marshal(queued)
var fields map[string]any
json.Unmarshal(raw, &fields)
if queued.Op != OpPrint || queued.Object != key || queued.Copies != 2 || len(raw) >= 4096 {
t.Errorf("queued %s", raw)
}
for _, banned := range []string{"path", "node", "url", "data"} {
if _, has := fields[banned]; has {
t.Errorf("the queued ask carries %q: %s", banned, raw)
}
}
m := got.(map[string]any)
if m["job-id"] != float64(31) || m["node"] != "g14" || m["path"] != "/home/ops/letter.pdf" {
t.Errorf("answer = %v", m)
}
if _, sent := fake.request(t); string(sent) != string(doc) {
t.Errorf("the printer was not sent the stored document")
}
}
// 1598/AC3: an upload refused by node-files is answered with node-files' reason, and nothing is printed.
func TestARefusedUploadIsAnsweredWithItsReasonAndNothingIsPrinted(t *testing.T) {
fake := newIPPFake(t, 1, 3)
up := withUpload(t, "", errors.New("/home/ops/.ssh/id_ed25519 is refused by name: it is in .ssh, which holds an account's credentials"))
r := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
_, err := r.call(t, "printer.print", map[string]any{"path": "/home/ops/.ssh/id_ed25519", "node": "g14"})
if err == nil || !strings.Contains(err.Error(), "is refused by name: it is in .ssh") {
t.Fatalf("error = %v", err)
}
if len(up.seen()) != 1 || r.bus.asks() != 0 || fake.body != nil {
t.Errorf("%d uploads, %d asks queued, printer sent %d bytes", len(up.seen()), r.bus.asks(), len(fake.body))
}
}
// 1598/AC4: the device reads document-format-supported before the job and refuses a format the printer does not
// list, naming the list; a printer that cannot be asked is sent nothing.
func TestTheDeviceChecksTheFormatsThePrinterListsBeforeTheJob(t *testing.T) {
fake := newIPPFake(t, 5, 3)
r := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
doc := pdfOf(300)
key := sha256Hex(doc) + ".pdf"
r.objects.Put(t.Context(), key, "application/pdf", doc)
if _, err := r.call(t, "printer.print", map[string]any{"object": key}); err != nil {
t.Fatal(err)
}
if fake.asked != 1 || fake.body == nil {
t.Fatalf("the formats were asked %d times, the job was sent: %v", fake.asked, fake.body != nil)
}
fake.body, fake.formats = nil, []string{"image/urf", "image/jpeg"}
_, err := r.call(t, "printer.print", map[string]any{"object": key})
if err == nil || !strings.Contains(err.Error(), "application/pdf") || !strings.Contains(err.Error(), "image/urf, image/jpeg") {
t.Fatalf("a format not listed: error = %v", err)
}
if fake.body != nil {
t.Error("a job was sent in a format the printer does not list")
}
fake.formats = nil
if _, err := r.call(t, "printer.print", map[string]any{"object": key}); err == nil || fake.body != nil {
t.Errorf("a printer that lists nothing: error = %v, sent %v", err, fake.body != nil)
}
}
// 1598/AC5: brother's s3-bucket requirement shares put with seat:node-files; brother invokes seat:node-files.upload;
// the bucket's name reaches the tool side through a file the mesh writes from the binding.
func TestManifestSharesPutWithNodeFilesAndInvokesItsUpload(t *testing.T) {
m := readManifest(t)
if len(m.Requires) != 1 || m.Requires[0].Name != "s3-bucket" || len(m.Requires[0].Shares) != 1 ||
m.Requires[0].Shares[0].To != "seat:node-files" || strings.Join(m.Requires[0].Shares[0].Methods, ",") != "put" {
t.Errorf("requires = %+v", m.Requires)
}
if len(m.Invokes) != 1 || m.Invokes[0] != "seat:node-files.upload" {
t.Errorf("invokes = %v", m.Invokes)
}
var bucket map[string]any
for _, r := range m.Resources {
if r["id"] == "bucket" {
bucket = r
}
}
if bucket == nil || bucket["path"] != "${dir:state}/bucket.json" || bucket["template"] != "files/bucket.json" {
t.Fatalf("bucket resource = %v", bucket)
}
raw, _ := os.ReadFile("../../files/bucket.json")
if !strings.Contains(string(raw), "${bound:s3-bucket:bucket}") || strings.Contains(string(raw), "secret") {
t.Errorf("files/bucket.json = %s", raw)
}
for _, a := range m.Build.Artifacts {
if a["name"] == "tools" && a["env"].(map[string]any)["BROTHER_BUCKET"] != "${dir:state}/bucket.json" {
t.Errorf("the tools bundle is not given BROTHER_BUCKET: %v", a["env"])
}
}
}
// quickly makes a call wait this long before it answers started, for the test.
func quickly(t *testing.T, d time.Duration) {
t.Helper()
old := quickWait
quickWait = d
t.Cleanup(func() { quickWait = old })
}
// 1598/AC7: a device answer slower than the call's bound is answered as started with the ask id within the bound, and
// printer.print {ask} then returns the device's answer.
func TestASlowPrintIsAnsweredAsStartedAndReadWithItsAsk(t *testing.T) {
fake := newIPPFake(t, 71, 5)
fake.delay = 400 * time.Millisecond
quickly(t, 60*time.Millisecond)
r := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
doc := pdfOf(300)
key := sha256Hex(doc) + ".pdf"
r.objects.Put(t.Context(), key, "application/pdf", doc)
began := time.Now()
got, err := r.call(t, "printer.print", map[string]any{"object": key})
if err != nil {
t.Fatal(err)
}
m := got.(map[string]any)
id, _ := m["ask"].(string)
if m["state"] != "started" || id == "" || time.Since(began) > 300*time.Millisecond {
t.Fatalf("answer %v after %s", m, time.Since(began))
}
if QuickWait != 25*time.Second {
t.Errorf("a call waits %s, want 25s", QuickWait)
}
// Read while it goes on, and after: the same id.
deadline := time.Now().Add(5 * time.Second)
for {
read, err := r.call(t, "printer.print", map[string]any{"ask": id})
if err != nil {
t.Fatal(err)
}
rm := read.(map[string]any)
if rm["state"] == nil {
if rm["job-id"] != float64(71) || rm["object"] != key {
t.Errorf("the outcome = %v", rm)
}
break
}
if time.Now().After(deadline) {
t.Fatalf("never done: %v", rm)
}
}
// A restarted tool process has no job: the device's answers are read.
r2 := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
r2.answers.m = map[string]json.RawMessage{"0123456789abcdef01234567": json.RawMessage(`{"job-id": 5}`)}
if read, err := r2.call(t, "printer.print", map[string]any{"ask": "0123456789abcdef01234567"}); err != nil || read.(map[string]any)["job-id"] != float64(5) {
t.Errorf("read from the answers: %v, %v", read, err)
}
if _, err := r.call(t, "printer.print", map[string]any{"ask": id, "copies": float64(2)}); err == nil {
t.Error("ask with another argument was accepted")
}
if _, err := r.call(t, "printer.print", map[string]any{"ask": "../x"}); err == nil {
t.Error("a made-up ask id was accepted")
}
}
// 1598/AC8: printer.print {path, node} whose upload and print together pass the bound answers started with the ask id,
// never a timeout, and the print follows.
func TestASlowUploadAndPrintAnswersStartedNeverATimeout(t *testing.T) {
fake := newIPPFake(t, 72, 5)
fake.delay = 100 * time.Millisecond
quickly(t, 60*time.Millisecond)
doc := pdfOf(300)
key := sha256Hex(doc)
up := withUpload(t, key, nil)
old := uploadVia
uploadVia = func(node, path, bucket string) (string, error) {
time.Sleep(150 * time.Millisecond) // node-files is slow
return old(node, path, bucket)
}
t.Cleanup(func() { uploadVia = old })
r := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
r.objects.Put(t.Context(), key, "application/pdf", doc)
got, err := r.call(t, "printer.print", map[string]any{"path": "/home/ops/letter.pdf", "node": "g14"})
if err != nil {
t.Fatalf("a slow upload failed the call: %v", err)
}
m := got.(map[string]any)
id, _ := m["ask"].(string)
if m["state"] != "started" || id == "" || len(up.seen()) > 1 {
t.Fatalf("answer %v", m)
}
for deadline := time.Now().Add(5 * time.Second); ; {
read, err := r.call(t, "printer.print", map[string]any{"ask": id})
if err != nil {
t.Fatal(err)
}
if rm := read.(map[string]any); rm["state"] == nil {
if rm["job-id"] != float64(72) || rm["path"] != "/home/ops/letter.pdf" || rm["node"] != "g14" {
t.Errorf("the outcome = %v", rm)
}
break
}
if time.Now().After(deadline) {
t.Fatal("never done")
}
}
if len(up.seen()) != 1 || len(r.bus.published) != 1 || r.bus.published[0].Object != key {
t.Errorf("uploads %v, asks %v", up.seen(), r.bus.published)
}
// An upload that fails after the call answered is the outcome, read with the ask.
uploadVia = func(node, path, bucket string) (string, error) {
time.Sleep(150 * time.Millisecond)
return "", errors.New("/home/ops/.ssh/id_ed25519 is refused by name")
}
got, _ = r.call(t, "printer.print", map[string]any{"path": "/home/ops/.ssh/id_ed25519", "node": "g14"})
id2, _ := got.(map[string]any)["ask"].(string)
time.Sleep(300 * time.Millisecond)
if _, err := r.call(t, "printer.print", map[string]any{"ask": id2}); err == nil || !strings.Contains(err.Error(), "refused by name") {
t.Errorf("the failed upload's outcome: %v", err)
}
}
modules/brother/cmd/brother/rig_test.go
+7 -1
View File
@@ -4,6 +4,7 @@ import (
"encoding/json"
"fmt"
"github.com/nats-io/nats.go/jetstream"
"net/http"
"os"
"path/filepath"
"sync"
@@ -144,7 +145,12 @@ func newRig(t *testing.T, settings, password string) *printerRig {
bus := &fakeAskBus{worker: w, answers: answers}
old, oldWait, oldScan, oldPoll := askBus, answerWait, scanWait, answerPoll
askBus, answerWait, scanWait, answerPoll = bus, 5*time.Second, 5*time.Second, 5*time.Millisecond
t.Cleanup(func() { askBus, answerWait, scanWait, answerPoll = old, oldWait, oldScan, oldPoll })
oldFetch := fetchClient
fetchClient = &http.Client{Transport: objects}
t.Cleanup(func() {
askBus, answerWait, scanWait, answerPoll = old, oldWait, oldScan, oldPoll
fetchClient = oldFetch
})
tl := tools(file, func() time.Time { return fixedNow })
return &printerRig{tl: tl, run: tool(t, tl, "printer.settings").Run, bus: bus, worker: w, answers: answers, objects: objects, said: &out, mu: &mu}
}
modules/brother/cmd/brother/scan.go
+61 -35
View File
@@ -1,12 +1,15 @@
package main
import (
"bytes"
"context"
"encoding/base64"
"fmt"
"net/http"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/objectlink"
)
// EventAdded is the module's event: a scan was kept in the bucket (ADR 0464, ADR 0468 clause 2), said by the device
@@ -29,6 +32,15 @@ type Service struct {
QuickWait, FeederWait time.Duration
// Jobs are the scans this process follows.
Jobs *Jobs
// Upload asks a machine's node-files to send one of its files into the bucket and answers the key; Bucket is
// brother's bucket. Both are the tool side's (printer.print with path and node).
// Fetch reads the links the device process signs; DocExpiry and DocRetry are the open documents' waits. They are
// copied into the Service when it is made, because background work reads them while a test (or a reload) changes the
// package's own, which would race.
Fetch *http.Client
DocExpiry, DocRetry time.Duration
Upload func(node, path, bucket string) (string, error)
Bucket func() (string, error)
}
// Jobs are the scans this tool process has started and not yet been asked the outcome of. A scan runs in the
@@ -229,7 +241,7 @@ func (s *Service) finishScan(ctx context.Context, id string, wait time.Duration,
}
if doc, _ := ans["document"].(string); doc != "" && ans["key"] == nil {
// A page was added to an open document: nothing to file yet, and the document is closed if it is left open.
s.armDocument(doc, docExpiry, 0)
s.armDocument(doc, s.docExpiry(), 0)
return ans, nil
}
if ask.Document != "" && ask.Finish {
@@ -280,26 +292,19 @@ func (s *Service) fileAnswer(ctx context.Context, ans map[string]any) (map[strin
func (s *Service) fileOne(ctx context.Context, ans map[string]any) (map[string]any, error) {
key, _ := ans["key"].(string)
sum, _ := ans["sha256"].(string)
parts, _ := ans["parts"].(float64)
if key == "" || sum == "" || parts < 1 || parts > MostAnswerParts {
return nil, fmt.Errorf("the device process's answer to the scan names no stored scan: %v", ans)
link, _ := ans["link"].(string)
if key == "" || sum == "" || link == "" {
return nil, fmt.Errorf("the device process's answer to the scan names no stored scan or no link to it: %v", withoutLink(ans))
}
got := make([][]byte, 0, int(parts))
for i := 1; i <= int(parts); i++ {
p, err := s.fetchPart(ctx, key, i)
if err != nil && ctx.Err() == nil {
// A part fetch is a read, safe to repeat: one retry rides out a short bus outage (issue 1536).
p, err = s.fetchPart(ctx, key, i)
}
if err != nil {
return nil, fmt.Errorf("the scan is kept as %s, but its part %d could not be fetched: %w", key, i, err)
}
got = append(got, p)
}
pdf, err := joinParts(got, sum)
if err != nil {
return nil, fmt.Errorf("the scan is kept as %s: %w", key, err)
// The scan is read from the store by the link the device process signed, never through the bus (ADR 0468 §4), and
// must hash to the scan's SHA-256 before anything is written.
var buf bytes.Buffer
gctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()
if _, err := objectlink.Get(gctx, s.fetchClient(), link, &buf, objectlink.MaxDocument, sum); err != nil {
return nil, fmt.Errorf("the scan is kept as %s, but it could not be read from the store: %w", key, err)
}
pdf := buf.Bytes()
at, err := time.Parse(time.RFC3339, fmt.Sprint(ans["scanned-at"]))
if err != nil {
at = s.Now()
@@ -310,7 +315,7 @@ func (s *Service) fileOne(ctx context.Context, ans map[string]any) (map[string]a
}
body := map[string]any{}
for k, v := range ans {
if k != "key" && k != "size" && k != "parts" {
if k != "key" && k != "size" && k != "link" && k != "link-seconds" {
body[k] = v
}
}
@@ -318,20 +323,41 @@ func (s *Service) fileOne(ctx context.Context, ans map[string]any) (map[string]a
return body, nil
}
// fetchPart is one part of a stored object, asked of the device process and decoded.
func (s *Service) fetchPart(ctx context.Context, object string, part int) ([]byte, error) {
ans, err := s.call(ctx, Ask{Op: OpDocument, Object: object, Part: part}, s.AnswerWait)
if err != nil {
return nil, err
// withoutLink is an answer said in an error without its link, which carries a signature.
func withoutLink(ans map[string]any) map[string]any {
out := map[string]any{}
for k, v := range ans {
if k != "link" {
out[k] = v
}
}
data, _ := ans["data"].(string)
raw, err := base64.StdEncoding.DecodeString(data)
if err != nil {
return nil, fmt.Errorf("the part is not base64: %w", err)
}
return raw, nil
return out
}
func (s *Service) fetchClient() *http.Client {
if s.Fetch != nil {
return s.Fetch
}
return &http.Client{}
}
func (s *Service) docExpiry() time.Duration {
if s.DocExpiry > 0 {
return s.DocExpiry
}
return 30 * time.Minute
}
func (s *Service) docRetry() time.Duration {
if s.DocRetry > 0 {
return s.DocRetry
}
return 5 * time.Minute
}
// fetchClientDefault is what a Service made by the tools reads the links with; a test puts a client of its own in its place before it makes the tools.
var fetchClient = &http.Client{}
// ScanStatus says whether the scanner answers, and what it is doing. A scanner that does not answer is an
// answer, not an error: that is how the module's health is read.
func (s *Service) ScanStatus(ctx context.Context) (map[string]any, error) {
@@ -370,7 +396,7 @@ func (s *Service) armDocument(doc string, d time.Duration, attempts int) {
if err == nil || attempts >= 10 || strings.Contains(err.Error(), "is closed already") || strings.Contains(err.Error(), "its id is wrong") {
return
}
s.armDocument(doc, docRetry, attempts+1)
s.armDocument(doc, s.docRetry(), attempts+1)
})
}
@@ -388,9 +414,9 @@ func (s *Service) Recover(ctx context.Context) error {
if id == "" {
continue
}
left := docExpiry
left := s.docExpiry()
if at, err := time.Parse(time.RFC3339, fmt.Sprint(m["updated"])); err == nil {
left = max(docExpiry-s.Now().Sub(at), time.Second)
left = max(s.docExpiry()-s.Now().Sub(at), time.Second)
}
s.armDocument(id, left, 0)
}
modules/brother/cmd/brother/scanlink_test.go
+157
View File
@@ -0,0 +1,157 @@
package main
import (
"encoding/json"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
)
// 1601/AC1: scanner.scan reads the stored PDF by the GET link the device's answer carries (never by parts) and writes it
// into the scan folder; the answer is the event's body plus the path.
func TestTheScanIsReadFromTheStoreByTheLinkAndFiled(t *testing.T) {
f := newFakeScanner(t, fixture(t, "scan.jpg"))
r, folder := scanRig(t, f, Settings{Crop: false})
body, err := r.scan(t)
if err != nil {
t.Fatal(err)
}
if len(*r.said) != 1 {
t.Fatalf("%d events", len(*r.said))
}
stored, err := r.objects.Get(t.Context(), (*r.said)[0].body["object"].(string))
if err != nil {
t.Fatal(err)
}
filed, err := os.ReadFile(body["path"].(string))
if err != nil || string(filed) != string(stored) {
t.Fatalf("the filed scan is not the stored one: %v", err)
}
if filepath.Dir(body["path"].(string)) != folder {
t.Errorf("filed in %s, want %s", filepath.Dir(body["path"].(string)), folder)
}
for _, a := range r.bus.published {
if a.Op == OpDocument {
t.Errorf("a part of the scan was asked of the device process over the bus")
}
}
for k := range (*r.said)[0].body {
if body[k] == nil && k != "path" {
t.Errorf("the event's %s is not in the answer", k)
}
}
if body["link"] != nil || body["parts"] != nil || body["key"] != nil {
t.Errorf("the answer carries the transport's own fields: %v", body)
}
}
// 1601/AC2: the device's answer carries a link that expires within ten minutes and no bytes of the document; the real
// store signs it with the binding's key for the host the binding gives.
func TestTheDeviceAnswerCarriesALinkAndNoBytes(t *testing.T) {
f := newFakeScanner(t, fixture(t, "scan.jpg"))
r, _ := scanRig(t, f, Settings{Crop: false})
if _, err := r.scan(t); err != nil {
t.Fatal(err)
}
var ans map[string]any
for _, raw := range r.answers.m {
json.Unmarshal(raw, &ans)
}
link, _ := ans["link"].(string)
u, err := url.Parse(link)
if err != nil || link == "" {
t.Fatalf("answer %v", ans)
}
secs, _ := strconv.Atoi(u.Query().Get("X-Amz-Expires"))
if secs < 1 || time.Duration(secs)*time.Second > 10*time.Minute {
t.Errorf("the link lives %d s", secs)
}
raw, _ := json.Marshal(ans)
if ans["data"] != nil || len(raw) > 4096 {
t.Errorf("the answer is %d bytes and carries data %v", len(raw), ans["data"] != nil)
}
// The real store: the binding's key signs, for the binding's host and port, in the module's bucket.
s3 := NewS3(StoreFacts{Scheme: "http", At: "novox.internal", Port: "20001", Bucket: "mesh-shanks-brother", Region: "eu-west",
Key: "binding-key", Secret: "binding-secret"})
s3.Now = func() time.Time { return fixedNow }
got, err := s3.Link("GET", "abc.pdf", 10*time.Minute)
if err != nil {
t.Fatal(err)
}
gu, _ := url.Parse(got)
if gu.Host != "novox.internal:20001" || gu.Path != "/mesh-shanks-brother/abc.pdf" ||
!strings.HasPrefix(gu.Query().Get("X-Amz-Credential"), "binding-key/") || gu.Query().Get("X-Amz-Expires") != "600" {
t.Errorf("link %s", got)
}
if _, err := s3.Link("GET", "abc.pdf", 11*time.Minute); err == nil {
t.Error("a link over ten minutes was signed")
}
}
// 1601/AC3: a body whose SHA-256 differs from the scan's writes no file and answers the mismatch.
func TestABodyOfAnotherHashWritesNoFile(t *testing.T) {
f := newFakeScanner(t, fixture(t, "scan.jpg"))
r, folder := scanRig(t, f, Settings{Crop: false})
r.objects.corrupt = true
_, err := r.scan(t)
if err == nil || !strings.Contains(err.Error(), "hashes to") {
t.Fatalf("error = %v", err)
}
if e, _ := os.ReadDir(folder); len(e) != 0 {
t.Errorf("a scan of the wrong hash was filed: %v", e)
}
}
// 1601/AC4: brother.scan.added is said before the file is written, once the scan is stored, and is under 4 KiB.
func TestTheEventIsSaidBeforeTheFileIsWritten(t *testing.T) {
f := newFakeScanner(t, fixture(t, "scan.jpg"))
r, folder := scanRig(t, f, Settings{Crop: false})
var filedWhenSaid = -1
emit := r.worker.Emit
r.worker.Emit = func(body map[string]any, id string) error {
e, _ := os.ReadDir(folder)
filedWhenSaid = len(e)
return emit(body, id)
}
if _, err := r.scan(t); err != nil {
t.Fatal(err)
}
if filedWhenSaid != 0 {
t.Errorf("%d files in the folder when the event was said, want 0", filedWhenSaid)
}
said := (*r.said)[0]
if !said.seen {
t.Error("the event was said before the scan was stored")
}
raw, _ := json.Marshal(said.body)
if len(raw) >= 4096 || said.body["path"] != nil {
t.Errorf("the event is %d bytes, path %v", len(raw), said.body["path"])
}
}
// 1601/AC5: no code of the device process writes a file outside its state directory: the only writes to the disk in the
// module's code are the filing of a scan (file.go), made by the tool side (scan.go).
func TestOnlyTheFilingWritesFiles(t *testing.T) {
writes := []string{"os.WriteFile", "os.Create(", "os.OpenFile", "os.MkdirAll", "os.Mkdir(", "os.Rename", "os.CreateTemp", "File(s.Settings.Folder"}
files, _ := filepath.Glob("*.go")
for _, name := range files {
if strings.HasSuffix(name, "_test.go") || name == "file.go" {
continue
}
raw, _ := os.ReadFile(name)
for _, w := range writes {
if strings.Contains(string(raw), w) && !(name == "scan.go" && w == "File(s.Settings.Folder") {
t.Errorf("%s writes with %s: only file.go does, and scan.go (the tool side) calls it", name, w)
}
}
}
raw, _ := os.ReadFile("deviceproc.go")
if strings.Contains(string(raw), "File(") {
t.Error("deviceproc.go files a scan")
}
}
modules/brother/cmd/brother/service_test.go
-35
View File
@@ -362,38 +362,3 @@ func (b *flakyBus) Publish(id string, body any) error {
}
return b.fakeAskBus.Publish(id, body)
}
// 1536: a part fetch that fails once is asked again, and the scan is filed; one that fails twice is a failure
// naming the stored object, so it can be filed later.
func TestAFailedPartFetchIsAskedOnceMoreBeforeTheScanFails(t *testing.T) {
f := newFakeScanner(t, fixture(t, "scan.jpg"))
r, _ := scanRig(t, f, Settings{Crop: false})
r.bus = nil
flaky := &flakyBus{fakeAskBus: &fakeAskBus{worker: r.worker, answers: r.answers}, fails: 1}
askBus = flaky
body, err := r.scan(t)
if err != nil {
t.Fatalf("one failed part fetch ended the scan: %v", err)
}
if _, err := os.Stat(body["path"].(string)); err != nil {
t.Errorf("the scan is not filed: %v", err)
}
f2 := newFakeScanner(t, fixture(t, "scan.jpg"))
r2, folder := scanRig(t, f2, Settings{Crop: false})
askBus = &flakyBus{fakeAskBus: &fakeAskBus{worker: r2.worker, answers: r2.answers}, fails: 2}
_, err = r2.scan(t)
if err == nil {
t.Fatal("two failed part fetches did not fail the scan")
}
key := ""
if got := *r2.said; len(got) == 1 {
key, _ = got[0].body["object"].(string)
}
if key == "" || !strings.Contains(err.Error(), key) {
t.Errorf("the failure %q does not name the stored object %q", err, key)
}
if entries, _ := os.ReadDir(folder); len(entries) != 0 {
t.Errorf("a scan whose parts failed was filed: %v", entries)
}
}
modules/brother/cmd/brother/settings.go
+23
View File
@@ -108,3 +108,26 @@ func (s Settings) MissingPrinter() error {
return fmt.Errorf("config value %s is not set: the operator gives it with the config verb for the brother module on this node",
strings.Join(unset, " and "))
}
// bucketFile is the file the mesh writes from the s3-bucket binding (files/bucket.json): the bucket's name and no secret.
// BucketFile is where the tool side reads brother's bucket from: a file the mesh writes from the s3-bucket binding
// (files/bucket.json), which holds the bucket's name and no secret.
type bucketFile struct {
Bucket string `json:"bucket"`
}
// ReadBucket is brother's bucket, as the mesh named it.
func ReadBucket(path string) (string, error) {
if path == "" {
return "", fmt.Errorf("the module was started without the bucket's name (BROTHER_BUCKET)")
}
raw, err := os.ReadFile(path)
if err != nil {
return "", fmt.Errorf("the bucket's name cannot be read: %w", err)
}
var f bucketFile
if err := json.Unmarshal(raw, &f); err != nil || f.Bucket == "" {
return "", fmt.Errorf("the bucket file %s names no bucket", path)
}
return f.Bucket, nil
}
modules/brother/files/bucket.json
+3
View File
@@ -0,0 +1,3 @@
{
"bucket": "${bound:s3-bucket:bucket}"
}
modules/brother/files/fetch.json
+3
View File
@@ -0,0 +1,3 @@
{
"mesh-range": "${machine:mesh-range}"
}
modules/brother/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module brother
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/brother/go.sum
+2
View File
@@ -1,5 +1,7 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef h1:xy3wuGWJpqMAXYdXolocYG6w9xnWqBltVUffoaziIKo=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f h1:7J/9jOfSfjfYpf8MOj96HwQRViQUkBNMb71RHP2mjn8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/brother/module.json
+38 -4
View File
@@ -2,7 +2,20 @@
"module": "brother",
"version": "1",
"requires": [
"s3-bucket"
{
"name": "s3-bucket",
"shares": [
{
"to": "seat:node-files",
"methods": [
"put"
]
}
]
}
],
"invokes": [
"seat:node-files.upload"
],
"binds": {
"s3-bucket": "${dir:state}/store-binding.json"
@@ -199,6 +212,21 @@
"mode": "0600",
"template": "files/store.json"
},
{
"id": "bucket",
"type": "file",
"path": "${dir:state}/bucket.json",
"mode": "0644",
"template": "files/bucket.json"
},
{
"id": "fetch",
"type": "file",
"path": "${dir:state}/fetch.json",
"owner": "brother-device",
"mode": "0600",
"template": "files/fetch.json"
},
{
"id": "device",
"type": "process",
@@ -213,11 +241,13 @@
"BROTHER_SETTINGS": "${dir:state}/settings.json",
"BROTHER_ADMIN_PASSWORD": "${dir:state}/admin-password.secret",
"BROTHER_BROKER": "${dir:state}/broker-device.secret",
"BROTHER_STORE": "${dir:state}/store.json"
"BROTHER_STORE": "${dir:state}/store.json",
"BROTHER_FETCH": "${dir:state}/fetch.json"
},
"restart-on": [
"settings",
"store"
"store",
"fetch"
],
"bus": {
"queues": [
@@ -228,6 +258,9 @@
],
"emits": [
"scan.added"
],
"invokes": [
"seat:document-converter.convert"
]
},
"health": {
@@ -267,7 +300,8 @@
"brother"
],
"env": {
"BROTHER_SETTINGS": "${dir:state}/settings.json"
"BROTHER_SETTINGS": "${dir:state}/settings.json",
"BROTHER_BUCKET": "${dir:state}/bucket.json"
}
},
{
modules/mesh-files/cmd/mesh-files/files.go
+7
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"io"
"io/fs"
"net/http"
"os"
"path/filepath"
"regexp"
@@ -69,6 +70,12 @@ type Files struct {
OwnerOf func(path string) (Ident, error)
Lchown func(path string, uid, gid int) error
NewID func() string
// Link asks the object store for a link (seat:object-store.link) and HTTP moves the bytes over it; upload and
// download use them (transfer.go). Tests replace both.
Link func(bucket, key, method string, seconds int) (string, error)
HTTP *http.Client
// Bound is how long a transfer may take; zero is TransferTime (a test makes it shorter).
Bound time.Duration
mu sync.Mutex
}
modules/mesh-files/cmd/mesh-files/main.go
+17 -1
View File
@@ -65,7 +65,7 @@ func newFiles(places []string) (*Files, error) {
r.Root = sudoRoot{}
node := os.Getenv("MESH_NODE")
return &Files{Reader: r, State: state, Owned: owned, Node: node, Tell: tellOperator,
OwnerOf: ownerOnDisk, Lchown: os.Lchown, NewID: newID}, nil
OwnerOf: ownerOnDisk, Lchown: os.Lchown, NewID: newID, Link: askLink}, nil
}
// tellOperator tells the operator a plain notice, through the operator channel's tell verb: no condition is
@@ -235,6 +235,22 @@ func tools(f *Files) []stdio.Tool {
func(act Act, a map[string]any) (any, error) {
return f.Chown(act, arg(a, "path"), arg(a, "owner"), arg(a, "group"), boolArg(a, "recursive"))
}),
verb("upload", "Send one file of this node into a bucket of the object store, over a short signed link and never over "+
"the bus; answers the key (the file's SHA-256 when none is given), the size and the SHA-256. At most 100 MiB. "+
"A place a secret is kept, a known credential file, or a link to one is refused by name, before anything is sent. "+
"Read "+asAccount+".",
object(map[string]any{"path": str("an absolute path to a file"), "bucket": str("the bucket the file goes into"),
"key": str("the object's key (optional; the file's SHA-256 when not given)")}, "path", "bucket"),
func(a map[string]any) (any, error) { return f.Upload(arg(a, "path"), arg(a, "bucket"), arg(a, "key")) }),
writeVerb("download", "Write one object of a bucket to a file of this node, over a short signed link and never over "+
"the bus; the old file goes to the node's trash. At most 100 MiB. The bytes must hash to sha256, or to the key "+
"when the key is a SHA-256; otherwise no file is written.",
map[string]any{"bucket": str("the bucket"), "key": str("the object's key"), "path": str("an absolute path"),
"sha256": str("the SHA-256 the bytes must have (optional; the key when that is a SHA-256)")},
[]string{"bucket", "key", "path"},
func(act Act, a map[string]any) (any, error) {
return f.Download(act, arg(a, "bucket"), arg(a, "key"), arg(a, "path"), arg(a, "sha256"))
}),
verb("trash", "What the node's trash holds: each deleted or replaced item with its id, original path and when "+
"it went.", object(map[string]any{}),
func(map[string]any) (any, error) { return f.Trash() }),
modules/mesh-files/cmd/mesh-files/manifest_test.go
+11 -1
View File
@@ -58,7 +58,17 @@ func TestTheHolderRunsAsTheToolRunnersAccount(t *testing.T) {
if !state {
t.Error("no state directory for the trash and the act log")
}
if len(m.Invokes) != 1 || m.Invokes[0] != "seat:operator-channel.tell" {
// 1597/AC7: the tell verb, and the object store's link verb that upload and download ask for.
if len(m.Invokes) != 2 || m.Invokes[0] != "seat:operator-channel.tell" || m.Invokes[1] != "seat:object-store.link" {
t.Errorf("invokes %v", m.Invokes)
}
for _, v := range []string{"upload", "download"} {
found := false
for _, s := range m.Claims[0].Serves {
found = found || s == v
}
if !found {
t.Errorf("1597/AC7: the manifest does not serve %s", v)
}
}
}
modules/mesh-files/cmd/mesh-files/transfer.go
+259
View File
@@ -0,0 +1,259 @@
package main
// upload and download (novox/hq ADR 0468 §9, issue 1597): a file of this node goes into a bucket of the object store,
// and an object comes out of one into a file of this node, over a short signed link and never over the bus.
//
// upload sends a file's content off the node, so it is judged as a read (ADR 0394 §7): before anything else it
// refuses by name what a read refuses, a link's target included. download writes a file, so it is judged as a write
// (ADR 0394 §3 to §5): a path a module owns is refused without an override, the old file goes to the trash, and the
// act is one line in the act log.
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"regexp"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/objectlink"
)
// LinkSeconds is how long the link asked for one transfer lives, and TransferTime how long a transfer may take: the
// tool runner cuts every tool call at 30 s, so a transfer that is not done in 25 s is refused rather than left to run
// on after the caller was told it failed. Nothing is kept of a transfer that did not finish.
const (
LinkSeconds = 300
TransferTime = 25 * time.Second
)
func (f *Files) bound() time.Duration {
if f.Bound > 0 {
return f.Bound
}
return TransferTime
}
// tooSlow says a transfer that ran out of time in words, naming the bound.
func (f *Files) tooSlow(err error, what string) error {
if errors.Is(err, context.DeadlineExceeded) || strings.Contains(err.Error(), "context deadline exceeded") ||
strings.Contains(err.Error(), "Client.Timeout") {
return fmt.Errorf("%s did not finish within %s (a tool call is cut at 30 s), so it is refused and nothing is kept", what, f.bound())
}
return err
}
var sha256Key = regexp.MustCompile(`^[0-9a-f]{64}$`)
// askLink asks the object store's seat for one link (ADR 0468 §8). The answer is {link, method, seconds, expires-at}.
func askLink(bucket, key, method string, seconds int) (string, error) {
raw, err := stdio.Ask("seat:object-store.link", map[string]any{"bucket": bucket, "key": key, "method": method, "seconds": seconds})
if err != nil {
return "", fmt.Errorf("the object store gave no %s link: %w", method, err)
}
var a struct {
Link string `json:"link"`
Answer json.RawMessage `json:"answer"`
}
if err := json.Unmarshal(raw, &a); err != nil {
return "", fmt.Errorf("the object store's answer is not JSON: %w", err)
}
if a.Link == "" && len(a.Answer) > 0 {
_ = json.Unmarshal(a.Answer, &a)
}
if a.Link == "" {
return "", errors.New("the object store's answer holds no link")
}
return a.Link, nil
}
func (f *Files) link(bucket, key, method string) (string, error) {
if f.Link == nil {
return "", errors.New("this holder has no way to ask the object store for a link")
}
return f.Link(bucket, key, method, LinkSeconds)
}
func (f *Files) client() *http.Client {
if f.HTTP != nil {
return f.HTTP
}
return &http.Client{Timeout: f.bound()}
}
// UploadAnswer is what upload answers.
type UploadAnswer struct {
Bucket string `json:"bucket"`
Key string `json:"key"`
Size int64 `json:"size"`
SHA256 string `json:"sha256"`
}
// Upload sends one file of this node into a bucket. The key is the file's SHA-256 when none is given; the hash is
// then made first (one read of the file), so the link is asked for the key it will be used for.
func (f *Files) Upload(raw, bucket, key string) (UploadAnswer, error) {
ctx, cancel := context.WithTimeout(context.Background(), f.bound())
defer cancel()
if strings.TrimSpace(bucket) == "" {
return UploadAnswer{}, errors.New("bucket is needed: name the bucket the file goes into")
}
path, err := absolute(raw)
if err != nil {
return UploadAnswer{}, err
}
// Refused by name before anything else: no link is asked and nothing is read for a place a secret is kept.
if err := f.refusedRead(path); err != nil {
return UploadAnswer{}, err
}
in, err := os.Open(path)
if err != nil {
return UploadAnswer{}, fmt.Errorf("%s cannot be read by this account: %s", path, why(err))
}
defer in.Close()
info, err := in.Stat()
if err != nil {
return UploadAnswer{}, err
}
if !info.Mode().IsRegular() {
return UploadAnswer{}, fmt.Errorf("%s is a %s, not a file: upload sends one file", path, typeOf(info.Mode()))
}
if info.Size() > objectlink.MaxDocument {
return UploadAnswer{}, fmt.Errorf("%s is %d bytes, over the %d a document may be (novox/hq ADR 0468 §3)", path, info.Size(), objectlink.MaxDocument)
}
want := ""
if strings.TrimSpace(key) == "" {
sum, err := hashOf(in)
if err != nil {
return UploadAnswer{}, fmt.Errorf("%s could not be read: %v", path, err)
}
if _, err := in.Seek(0, io.SeekStart); err != nil {
return UploadAnswer{}, err
}
key, want = sum, sum
}
link, err := f.link(bucket, key, "put")
if err != nil {
return UploadAnswer{}, err
}
res, err := objectlink.Put(ctx, f.client(), link, in, info.Size(), objectlink.MaxDocument)
if err != nil {
return UploadAnswer{}, f.tooSlow(fmt.Errorf("%s could not be sent: %w", path, err), "the upload of "+path)
}
if want != "" && res.SHA256 != want {
return UploadAnswer{}, fmt.Errorf("%s changed while it was sent: it was stored under %s but hashes to %s", path, want, res.SHA256)
}
return UploadAnswer{Bucket: bucket, Key: key, Size: res.Size, SHA256: res.SHA256}, nil
}
func hashOf(r io.Reader) (string, error) {
h := sha256.New()
if _, err := io.Copy(h, r); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
// DownloadAnswer is what download answers.
type DownloadAnswer struct {
ActAnswer
Size int64 `json:"size"`
SHA256 string `json:"sha256"`
}
// Download writes one object of a bucket to a file of this node. The bytes must hash to sha256 when it is given, or
// to the key when the key is itself a SHA-256 (the store keeps a document under its SHA-256, ADR 0468 §2); a
// mismatch leaves no file, and an old file is left as it was.
func (f *Files) Download(a Act, bucket, key, raw, wantSHA string) (DownloadAnswer, error) {
f.mu.Lock()
defer f.mu.Unlock()
if err := f.begin(a); err != nil {
return DownloadAnswer{}, err
}
if strings.TrimSpace(bucket) == "" || strings.TrimSpace(key) == "" {
return DownloadAnswer{}, errors.New("bucket and key are needed: name the object to fetch")
}
path, err := absolute(raw)
if err != nil {
return DownloadAnswer{}, err
}
hits, err := f.guard(a, false, path)
if err != nil {
return DownloadAnswer{}, err
}
dir := filepath.Dir(path)
if _, err := os.Stat(dir); err != nil {
return DownloadAnswer{}, fmt.Errorf("the folder of %s cannot be reached: %s", path, why(err))
}
expect := strings.ToLower(strings.TrimSpace(wantSHA))
if expect == "" && sha256Key.MatchString(key) {
expect = key
}
var perm uint32 = 0o644
want, exists := Ident{}, false
if old, err := os.Lstat(path); err == nil {
if !old.Mode().IsRegular() {
return DownloadAnswer{}, fmt.Errorf("%s is a %s, not a file: download replaces a file whole", path, typeOf(old.Mode()))
}
exists, perm = true, permOf(old.Mode())
if want, err = f.ownerOf(path); err != nil {
return DownloadAnswer{}, err
}
} else if want, err = f.ownerOf(dir); err != nil {
return DownloadAnswer{}, err
}
link, err := f.link(bucket, key, "get")
if err != nil {
return DownloadAnswer{}, err
}
tmp, err := os.CreateTemp(dir, ".download-*")
if err != nil {
return DownloadAnswer{}, fmt.Errorf("a file cannot be made in the folder of %s by this account: %s", path, why(err))
}
done := false
defer func() {
tmp.Close()
if !done {
os.Remove(tmp.Name())
}
}()
ctx, cancel := context.WithTimeout(context.Background(), f.bound())
defer cancel()
res, err := objectlink.Get(ctx, f.client(), link, tmp, objectlink.MaxDocument, expect)
if err != nil {
return DownloadAnswer{}, f.tooSlow(fmt.Errorf("%s/%s could not be fetched: %w", bucket, key, err), "the download of "+bucket+"/"+key)
}
if err := tmp.Close(); err != nil {
return DownloadAnswer{}, err
}
if err := f.setMode(tmp.Name(), perm); err != nil {
return DownloadAnswer{}, err
}
if err := f.chownTo(tmp.Name(), want); err != nil {
return DownloadAnswer{}, fmt.Errorf("the owner of %s could not be set: %v", path, err)
}
trashed := ""
if exists {
if trashed, err = f.toTrash(a, path); err != nil {
return DownloadAnswer{}, err
}
}
if err := os.Rename(tmp.Name(), path); err != nil {
if trashed != "" {
if _, rerr := f.restore(trashed); rerr != nil {
err = fmt.Errorf("%v (and the old file could not be put back from the trash, id %s: %v)", err, trashed, rerr)
}
}
return DownloadAnswer{}, err
}
done = true
ans, err := f.finish(a, "download", hits, ActAnswer{Done: "wrote " + path, Paths: []string{path}, Trashed: trashed})
return DownloadAnswer{ActAnswer: ans, Size: res.Size, SHA256: res.SHA256}, err
}
modules/mesh-files/cmd/mesh-files/transfer_test.go
+301
View File
@@ -0,0 +1,301 @@
package main
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
)
// fakeStore stands for the object store behind signed links: a PUT keeps the body under its path, a GET answers it.
// objectlink (mesh-sdk go/objectlink) is the real client of both.
type fakeStore struct {
mu sync.Mutex
objects map[string][]byte
puts []string
srv *httptest.Server
}
func newFakeStore(t *testing.T) *fakeStore {
t.Helper()
s := &fakeStore{objects: map[string][]byte{}}
s.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
s.mu.Lock()
defer s.mu.Unlock()
switch r.Method {
case http.MethodPut:
b, _ := io.ReadAll(r.Body)
s.objects[r.URL.Path] = b
s.puts = append(s.puts, r.URL.Path)
case http.MethodGet:
b, ok := s.objects[r.URL.Path]
if !ok {
w.WriteHeader(404)
return
}
w.Write(b)
}
}))
t.Cleanup(s.srv.Close)
return s
}
// linkVerb is the fake object-store.link: it records each ask and answers a link to the fake store.
type linkVerb struct {
store *fakeStore
asks []string
err error
}
func (l *linkVerb) ask(bucket, key, method string, seconds int) (string, error) {
l.asks = append(l.asks, method+" "+bucket+"/"+key)
if l.err != nil {
return "", l.err
}
return l.store.srv.URL + "/" + bucket + "/" + key + "?sig=1", nil
}
func transferRig(t *testing.T, places ...string) (*rig, *fakeStore, *linkVerb) {
t.Helper()
g := testFiles(t)
g.Refuser.Places = places
st := newFakeStore(t)
lv := &linkVerb{store: st}
g.Link = lv.ask
return g, st, lv
}
func sum(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) }
// 1597/AC1: the store receives exactly the file's bytes and the answer is {key: SHA-256, size, sha256}.
func TestUploadSendsTheFileUnderItsHash(t *testing.T) {
g, st, lv := transferRig(t)
body := bytes.Repeat([]byte("a document\n"), 1000)
write(t, g.path("home/ops/doc.pdf"), string(body))
ans, err := g.Upload(g.path("home/ops/doc.pdf"), "mesh-shanks-brother", "")
if err != nil {
t.Fatal(err)
}
want := sum(body)
if ans.Key != want || ans.SHA256 != want || ans.Size != int64(len(body)) {
t.Fatalf("answer %+v", ans)
}
if !bytes.Equal(st.objects["/mesh-shanks-brother/"+want], body) {
t.Fatal("the store did not receive exactly the file's bytes")
}
if len(lv.asks) != 1 || lv.asks[0] != "put mesh-shanks-brother/"+want {
t.Fatalf("link asks %v", lv.asks)
}
}
// 1597/AC2: a key given is the key it is stored under.
func TestUploadStoresUnderTheKeyGiven(t *testing.T) {
g, st, _ := transferRig(t)
write(t, g.path("a.txt"), "hello")
ans, err := g.Upload(g.path("a.txt"), "b", "scans/a.txt")
if err != nil {
t.Fatal(err)
}
if ans.Key != "scans/a.txt" || string(st.objects["/b/scans/a.txt"]) != "hello" || ans.SHA256 != sum([]byte("hello")) {
t.Fatalf("answer %+v, store %v", ans, st.puts)
}
}
// 1597/AC3: places a read refuses are refused by name, and no link is asked and nothing reaches the store.
func TestUploadRefusesWhatAReadRefuses(t *testing.T) {
dir := t.TempDir()
// A vault directory and a module's own-secret directory are the controller's secret places (MESH_SECRET_PLACES).
places := []string{filepath.Join(dir, "var/lib/vault"), filepath.Join(dir, "var/lib/brother")}
g, st, lv := transferRig(t, places...)
secret := map[string]string{
"vault": filepath.Join(dir, "var/lib/vault/box"),
"own-secret": filepath.Join(dir, "var/lib/brother/admin-password.secret"),
"key file": filepath.Join(dir, "srv/tls/server.key"),
"ssh key": filepath.Join(dir, "home/ops/.ssh/id_ed25519"),
"known profile": filepath.Join(dir, "home/ops/.mozilla/firefox/logins.json"),
}
for name, p := range secret {
write(t, p, "SECRET")
if _, err := g.Upload(p, "b", ""); err == nil || !strings.Contains(err.Error(), "refused by name") {
t.Errorf("%s: not refused by name: %v", name, err)
}
}
link := filepath.Join(dir, "home/ops/innocent.txt")
if err := os.Symlink(secret["ssh key"], link); err != nil {
t.Fatal(err)
}
if _, err := g.Upload(link, "b", ""); err == nil || !strings.Contains(err.Error(), "leads to") {
t.Errorf("a link to a secret: %v", err)
}
if len(lv.asks) != 0 || len(st.puts) != 0 {
t.Fatalf("a link was asked %v or the store was reached %v", lv.asks, st.puts)
}
}
// 1597/AC4: a file over 100 MiB is refused before a link is asked.
func TestUploadRefusesOverOneHundredMiB(t *testing.T) {
g, st, lv := transferRig(t)
p := g.path("big.bin")
f, err := os.Create(p)
if err != nil {
t.Fatal(err)
}
if err := f.Truncate(100<<20 + 1); err != nil { // a sparse file: no bytes are written
t.Fatal(err)
}
f.Close()
if _, err := g.Upload(p, "b", ""); err == nil || !strings.Contains(err.Error(), "over the") {
t.Fatalf("not refused: %v", err)
}
if len(lv.asks) != 0 || len(st.puts) != 0 {
t.Fatal("a link was asked for a file over the limit")
}
}
// 1597/AC5: download writes the object to the path; a module-owned path is refused without an override; a hash that
// does not match leaves no file.
func TestDownloadWritesTheObject(t *testing.T) {
g, st, lv := transferRig(t)
st.objects["/b/k.txt"] = []byte("from the store")
ans, err := g.Download(by, "b", "k.txt", g.path("out/k.txt"), "")
if err == nil {
t.Fatal("the folder does not exist, yet download succeeded")
}
if err := os.MkdirAll(g.path("out"), 0o755); err != nil {
t.Fatal(err)
}
ans, err = g.Download(by, "b", "k.txt", g.path("out/k.txt"), "")
if err != nil {
t.Fatal(err)
}
if mustRead(t, g.path("out/k.txt")) != "from the store" || ans.SHA256 != sum([]byte("from the store")) || ans.Size != 14 {
t.Fatalf("answer %+v", ans)
}
if len(lv.asks) != 1 || lv.asks[0] != "get b/k.txt" {
t.Fatalf("asks %v", lv.asks)
}
if lines := actLines(t, g); len(lines) != 1 || lines[0]["verb"] != "download" {
t.Fatalf("act log %v", lines)
}
}
func TestDownloadRefusesAModuleOwnedPathWithoutOverride(t *testing.T) {
g, st, lv := transferRig(t)
st.objects["/b/k"] = []byte("x")
owned := g.path("etc/brother.conf")
write(t, owned, "old")
g.Owned[owned] = "brother (settings)"
_, err := g.Download(by, "b", "k", owned, "")
if err == nil || !strings.Contains(err.Error(), "is owned by brother") {
t.Fatalf("not refused: %v", err)
}
if mustRead(t, owned) != "old" || len(lv.asks) != 0 {
t.Fatal("the owned file changed or a link was asked")
}
if _, err := g.Download(Act{Caller: by.Caller, Reason: "a test", Override: "because"}, "b", "k", owned, ""); err != nil {
t.Fatalf("with an override: %v", err)
}
}
func TestDownloadWithAWrongHashLeavesNoFile(t *testing.T) {
g, st, _ := transferRig(t)
st.objects["/b/"+sum([]byte("expected"))] = []byte("something else") // the key says a hash the bytes do not have
dir := g.path("out")
os.MkdirAll(dir, 0o755)
write(t, filepath.Join(dir, "doc"), "old content")
_, err := g.Download(by, "b", sum([]byte("expected")), filepath.Join(dir, "doc"), "")
if err == nil || !strings.Contains(err.Error(), "hashes to") {
t.Fatalf("not refused: %v", err)
}
if mustRead(t, filepath.Join(dir, "doc")) != "old content" {
t.Fatal("the old file was replaced")
}
entries, _ := os.ReadDir(dir)
if len(entries) != 1 {
t.Fatalf("a temporary file is left: %v", entries)
}
// An explicit sha256 is judged too, for a key that is not a hash.
st.objects["/b/plain"] = []byte("abc")
if _, err := g.Download(by, "b", "plain", filepath.Join(dir, "plain"), sum([]byte("other"))); err == nil {
t.Fatal("a wrong sha256 was accepted")
}
if _, err := os.Stat(filepath.Join(dir, "plain")); err == nil {
t.Fatal("a file was left for a wrong sha256")
}
}
// 1597/AC6: upload then download keeps the SHA-256 across the round trip.
func TestUploadThenDownloadKeepsTheHash(t *testing.T) {
g, _, _ := transferRig(t)
body := bytes.Repeat([]byte{0, 1, 2, 3, 250}, 50000)
write(t, g.path("in.bin"), string(body))
up, err := g.Upload(g.path("in.bin"), "b", "")
if err != nil {
t.Fatal(err)
}
down, err := g.Download(by, "b", up.Key, g.path("out.bin"), "")
if err != nil {
t.Fatal(err)
}
if up.SHA256 != down.SHA256 || sum([]byte(mustRead(t, g.path("out.bin")))) != up.SHA256 {
t.Fatalf("up %+v down %+v", up, down)
}
}
// 1597/AC1: the verbs are listed on the seat with their inputs.
func TestTheSeatListsUploadAndDownload(t *testing.T) {
g, _, _ := transferRig(t)
names := map[string]bool{}
for _, tool := range tools(g.Files) {
names[tool.Name] = true
}
if !names[Seat+".upload"] || !names[Seat+".download"] {
t.Fatalf("tools %v", names)
}
}
// 1597/AC9: a transfer slower than its bound is refused naming it, and no object or file is left.
func TestASlowTransferIsRefusedNamingTheBoundAndKeepsNothing(t *testing.T) {
g := testFiles(t)
g.Bound = 150 * time.Millisecond
release := make(chan struct{})
slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet {
w.Write([]byte("half of the document"))
w.(http.Flusher).Flush()
}
select { // a store that does not finish: nothing is kept of a PUT whose body it never took whole
case <-release:
case <-r.Context().Done():
}
}))
t.Cleanup(func() { close(release); slow.Close() })
g.Link = func(bucket, key, method string, seconds int) (string, error) {
return slow.URL + "/" + bucket + "/" + key, nil
}
write(t, g.path("in.bin"), "a document")
if _, err := g.Upload(g.path("in.bin"), "b", "k"); err == nil || !strings.Contains(err.Error(), "did not finish within 150ms") ||
!strings.Contains(err.Error(), "nothing is kept") {
t.Fatalf("upload error = %v", err)
}
if err := os.MkdirAll(g.path("out"), 0o755); err != nil {
t.Fatal(err)
}
if _, err := g.Download(by, "b", "k", g.path("out/doc"), ""); err == nil || !strings.Contains(err.Error(), "did not finish within 150ms") {
t.Fatalf("download error = %v", err)
}
if e, _ := os.ReadDir(g.path("out")); len(e) != 0 {
t.Errorf("a file is left: %v", e)
}
if TransferTime != 25*time.Second {
t.Errorf("the bound is %s, want 25s", TransferTime)
}
}
modules/mesh-files/go.mod
+2 -2
View File
@@ -1,5 +1,5 @@
module mesh-files
go 1.25.0
go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f
modules/mesh-files/go.sum
+2
View File
@@ -2,3 +2,5 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261012122749-67bfa22132f2 h1:1E5iapAO
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261012122749-67bfa22132f2/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44 h1:r5xIo5qWOXQdCSMxvqJy8kV4U+P/QWXFK6971Ha6wU8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f h1:7J/9jOfSfjfYpf8MOj96HwQRViQUkBNMb71RHP2mjn8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
modules/mesh-files/module.json
+5 -2
View File
@@ -20,12 +20,15 @@
"chmod",
"chown",
"trash",
"restore"
"restore",
"upload",
"download"
]
}
],
"invokes": [
"seat:operator-channel.tell"
"seat:operator-channel.tell",
"seat:object-store.link"
],
"resources": [
{
modules/minio/client.ts
+19 -24
View File
@@ -5,7 +5,7 @@
//
// It speaks two planes with node built-ins only (never the `minio` npm package):
// - the S3 data plane over `fetch`, signed with AWS Signature V4 (node:crypto) — bucket and object
// operations, and presigned URLs;
// operations (links are signed by the object-store seat's Go tool with a link key, never here);
// - the admin plane through the `mc` CLI (node:child_process) — scoped service accounts, whose
// creation the MinIO admin REST API guards behind an encrypted payload `fetch` cannot form.
// This mirrors hal's MinIOClient/MinIOAdmin split, folded into one client the module builds from env.
@@ -193,26 +193,6 @@ export class MinioClient {
};
}
/** A time-limited URL for GET (download) or PUT (upload) of one object — query-string SigV4. */
presignedUrl(method: "GET" | "PUT", bucket: string, object: string, expires = 86400): string {
const { amzDate, dateStamp } = this.stamp();
const scope = `${dateStamp}/${this.region}/s3/aws4_request`;
const host = new URL(this.baseUrl).host;
const params: Record<string, string> = {
"X-Amz-Algorithm": "AWS4-HMAC-SHA256",
"X-Amz-Credential": `${this.rootUser}/${scope}`,
"X-Amz-Date": amzDate,
"X-Amz-Expires": String(expires),
"X-Amz-SignedHeaders": "host",
};
const path = uriEncode(`/${bucket}/${object}`, false);
const canonicalQuery = encodeQuery(params);
const canonicalRequest = [method, path, canonicalQuery, `host:${host}\n`, "host", "UNSIGNED-PAYLOAD"].join("\n");
const stringToSign = ["AWS4-HMAC-SHA256", amzDate, scope, sha256hex(canonicalRequest)].join("\n");
const signature = hmac(this.signingKey(dateStamp), stringToSign).toString("hex");
return `${this.baseUrl}${path}?${canonicalQuery}&X-Amz-Signature=${signature}`;
}
// --- admin plane (mc CLI) ------------------------------------------------
/**
@@ -228,9 +208,14 @@ export class MinioClient {
* but the machine's process table shows it to every local user for the second `mc` runs (novox/hq
* issue 282, open): replacing it means the admin API's encrypted request made here.
*/
async createAccessKey(bucket: string, accessKey: string, secretKey: string): Promise<AccessKey> {
async createAccessKey(
bucket: string,
accessKey: string,
secretKey: string,
policy: string = bucketPolicy(bucket),
): Promise<AccessKey> {
const policyPath = join(this.mcConfigDir, `policy-${accessKey}.json`);
writeFileSync(policyPath, bucketPolicy(bucket), { mode: 0o600 });
writeFileSync(policyPath, policy, { mode: 0o600 });
try {
await this.mc(
"admin", "user", "svcacct", "add", "mesh", this.rootUser,
@@ -248,6 +233,16 @@ export class MinioClient {
await this.mc("admin", "user", "svcacct", "rm", "mesh", accessKey);
}
/** Whether a service account of this access key exists (`mc admin user svcacct info` fails when it does not). */
async accessKeyExists(accessKey: string): Promise<boolean> {
try {
await this.mc("admin", "user", "svcacct", "info", "mesh", accessKey);
return true;
} catch {
return false;
}
}
/**
* Run `mc` against the alias `mesh`, which it is given as MC_HOST_mesh in its environment — the
* root user and password inside the URL — rather than by an `mc alias set` that carried the root
@@ -320,7 +315,7 @@ export class MinioClient {
// is a line of this module's manifest, filled per consumer and delivered to both ends). Both
// survived with no callers, which is the state a rule comes back from; they are gone.
function bucketPolicy(bucket: string): string {
export function bucketPolicy(bucket: string): string {
return JSON.stringify({
Version: "2012-10-17",
Statement: [{
modules/minio/cmd/object-store/main.go
+52
View File
@@ -0,0 +1,52 @@
// minio's object-store seat (novox/hq ADR 0468 §3, §8): the holder of `object-store`, serving `link`. A party that
// needs to put a document into a bucket, or get one out, asks for one short link and moves the bytes over HTTP
// itself; no document ever travels on the bus.
//
// The link is signed with the bucket's link key (issue 1594), which only reads and writes that bucket's objects,
// never with the root account. Who may have a link is judged here, at each call, from the grant file the mesh
// delivered to minio: the bucket's consumer, and whoever its `shares` name for that method.
//
// stdout is the MCP channel; what this module says, it says on stderr.
package main
import (
"fmt"
"os"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// Words the manifest gives this bundle (modules/minio/module.json).
const (
ConfigWord = "OBJECT_STORE_CONFIG" // the store as the machine reaches it: files/object-store.json, rendered
LinksWord = "MESH_MINIO_LINKS" // the directory of the link keys' secrets (issue 1594)
ReceivesWord = "MESH_RECEIVES" // the grant file
)
func main() {
s, err := newServer(os.Getenv(ConfigWord), os.Getenv(LinksWord), os.Getenv(ReceivesWord))
if err != nil {
fmt.Fprintf(os.Stderr, "[object-store] %v\n", err)
os.Exit(1)
}
if err := stdio.Serve("", tools(s)); err != nil {
fmt.Fprintf(os.Stderr, "[object-store] %v\n", err)
os.Exit(1)
}
}
func tools(s *server) []stdio.Tool {
return []stdio.Tool{{
Name: "object-store.link",
Description: "A signed link to one object of a bucket, for one method, that lives at most ten minutes: the caller " +
"moves the bytes over HTTP to it and never over the bus. Allowed to the bucket's own consumer and to whoever the " +
"consumer's shares name for the method; the link is signed with a key that reaches that bucket's objects only.",
Input: map[string]any{
"bucket": map[string]any{"type": "string", "description": "the bucket"},
"key": map[string]any{"type": "string", "description": "the object's key"},
"method": map[string]any{"type": "string", "description": "get to read the object, put to write it"},
"seconds": map[string]any{"type": "integer", "description": "how long the link lives, 1 to 600 (default 300)"},
},
Run: func(args map[string]any) (any, error) { return s.link(stdio.CallerOf(args), args) },
}}
}
modules/minio/cmd/object-store/manifest_test.go
+107
View File
@@ -0,0 +1,107 @@
package main
import (
"encoding/json"
"os"
"strings"
"testing"
)
func manifest(t *testing.T) map[string]any {
t.Helper()
raw, err := os.ReadFile("../../module.json")
if err != nil {
t.Fatal(err)
}
var m map[string]any
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
return m
}
// 1516/AC6: minio claims object-store (scope mesh) serving link, invokes seat:mesh-controller.seats, builds the Go
// bundle, and no tool named minio_presigned_url remains.
func TestManifestClaimsTheSeat(t *testing.T) {
m := manifest(t)
seats, _ := m["seats"].([]any)
if len(seats) != 1 || seats[0].(map[string]any)["name"] != "object-store" || seats[0].(map[string]any)["scope"] != "mesh" ||
len(seats[0].(map[string]any)["serves"].([]any)) != 1 || seats[0].(map[string]any)["serves"].([]any)[0] != "link" {
t.Fatalf("the seat object-store is not declared in seats (the mesh defines no seat a module does not declare): %v", seats)
}
claims := m["claims"].([]any)
if len(claims) != 1 {
t.Fatalf("claims: %v", claims)
}
c := claims[0].(map[string]any)
serves := c["serves"].([]any)
if c["name"] != "object-store" || c["scope"] != "mesh" || len(serves) != 1 || serves[0] != "link" {
t.Fatalf("claim: %v", c)
}
inv := m["invokes"].([]any)
if len(inv) != 1 || inv[0] != "seat:mesh-controller.seats" {
t.Fatalf("invokes: %v", inv)
}
var goBundle map[string]any
for _, a := range m["build"].(map[string]any)["artifacts"].([]any) {
if a.(map[string]any)["from"] == "cmd/object-store" {
goBundle = a.(map[string]any)
}
}
if goBundle == nil || goBundle["language"] != "go" || goBundle["kind"] != "bundle" {
t.Fatalf("no Go bundle for cmd/object-store: %v", goBundle)
}
env := goBundle["env"].(map[string]any)
for _, word := range []string{ConfigWord, LinksWord, ReceivesWord} {
if env[word] == nil {
t.Errorf("the Go bundle's env lacks %s", word)
}
}
for _, f := range []string{"../../tools/index.ts", "../../client.ts"} {
raw, err := os.ReadFile(f)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(raw), "minio_presigned_url") || strings.Contains(string(raw), "presignedUrl") {
t.Errorf("%s still has the root-signed link", f)
}
}
}
// 1516/AC1: the file the Go tool reads is rendered from files/object-store.json with the facts the s3-bucket binding
// renders for a consumer: at is the providing machine's `at`, port the S3 endpoint's port (brother's files/store.json
// is the writer of that shape: scheme, at, port, region).
func TestStoreFileSaysWhatTheBindingSays(t *testing.T) {
raw, err := os.ReadFile("../../files/object-store.json")
if err != nil {
t.Fatal(err)
}
var f map[string]string
if err := json.Unmarshal(raw, &f); err != nil {
t.Fatal(err)
}
if f["at"] != "${machine:at}" || f["port"] != "${port:9000}" || f["scheme"] != "http" || f["region"] != "eu-west" {
t.Fatalf("object-store.json: %v", f)
}
brother, err := os.ReadFile("../../../brother/files/store.json")
if err != nil {
t.Fatal(err)
}
var b map[string]string
if err := json.Unmarshal(brother, &b); err != nil {
t.Fatal(err)
}
for _, k := range []string{"scheme", "at", "port", "region"} {
if _, ok := b[k]; !ok {
t.Errorf("brother's store.json has no %s: the shape moved", k)
}
if _, ok := f[k]; !ok {
t.Errorf("object-store.json has no %s", k)
}
}
// The same values: minio's serves.s3-bucket.port is the port its S3 listen declares.
m := manifest(t)
if port := m["serves"].(map[string]any)["s3-bucket"].(map[string]any)["port"]; port != float64(9000) {
t.Errorf("serves port %v", port)
}
}
modules/minio/cmd/object-store/server.go
+306
View File
@@ -0,0 +1,306 @@
package main
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/objectlink"
)
// DefaultSeconds is the life of a link when none is asked; MaxSeconds is the longest (ADR 0468 §3).
const (
DefaultSeconds = 300
MaxSeconds = 600
holdersFor = 60 * time.Second
)
// storeConfig is the store as this machine reaches it. The file is rendered by the mesh from files/object-store.json
// with the same facts the s3-bucket binding renders for a consumer (`at` is the providing machine's, `port` the one
// it gave the S3 endpoint), so a link names the host and port every consumer's own file names.
type storeConfig struct {
Scheme string `json:"scheme"`
At string `json:"at"`
Port string `json:"port"`
Region string `json:"region"`
}
func (c storeConfig) endpoint() string { return c.Scheme + "://" + c.At + ":" + c.Port }
func readConfig(path string) (storeConfig, error) {
var c storeConfig
raw, err := os.ReadFile(path)
if err != nil {
return c, fmt.Errorf("the store's address is unreadable (%s): %w", path, err)
}
if err := json.Unmarshal(raw, &c); err != nil {
return c, fmt.Errorf("the store's address file %s is not JSON: %w", path, err)
}
if c.Scheme == "" || c.At == "" || c.Port == "" {
return c, fmt.Errorf("the store's address file %s lacks a scheme, an at or a port", path)
}
return c, nil
}
// linkKeyID is the access key id of a bucket's link key, as the provisioner makes it (modules/minio/links.ts).
func linkKeyID(bucket string) string {
sum := sha256.Sum256([]byte(bucket))
return "link" + hex.EncodeToString(sum[:])[:16]
}
type server struct {
configPath, linksDir, receives string
now func() time.Time
// seats asks the controller for its seats; a test replaces it.
seats func() (json.RawMessage, error)
mu sync.Mutex
holders map[string]bool // "<seat>/<node>.<module>"
readAt time.Time
haveSet bool
}
func newServer(config, links, receives string) (*server, error) {
for word, v := range map[string]string{ConfigWord: config, LinksWord: links, ReceivesWord: receives} {
if strings.TrimSpace(v) == "" {
return nil, fmt.Errorf("%s is not given: the manifest names it", word)
}
}
return &server{configPath: config, linksDir: links, receives: receives, now: time.Now, seats: askSeats}, nil
}
func askSeats() (json.RawMessage, error) {
return stdio.Ask("seat:mesh-controller.seats", map[string]any{})
}
// link answers one ask.
func (s *server) link(caller string, args map[string]any) (any, error) {
bucket, _ := args["bucket"].(string)
key, _ := args["key"].(string)
method, _ := args["method"].(string)
if bucket == "" {
return nil, errors.New("link: bucket is needed")
}
if key == "" {
return nil, errors.New("link: key is needed")
}
if method != "get" && method != "put" {
return nil, fmt.Errorf("link: method %q is not get or put", method)
}
seconds := DefaultSeconds
if v, given := args["seconds"]; given {
f, ok := v.(float64)
if !ok || f != float64(int(f)) {
return nil, fmt.Errorf("link: seconds %v is not a whole number", v)
}
seconds = int(f)
}
if seconds < 1 || seconds > MaxSeconds {
return nil, fmt.Errorf("link: seconds %d is outside 1 to %d: a link lives at most ten minutes (ADR 0468 §3)", seconds, MaxSeconds)
}
node, module, ok := splitCaller(caller)
if !ok {
return nil, fmt.Errorf("link: the caller is %q: the runtime did not prove who asks, and a link is not given to an unproven caller", caller)
}
grant, err := s.grantOf(bucket)
if err != nil {
return nil, err
}
if err := s.allowed(grant, node, module, method); err != nil {
return nil, fmt.Errorf("link: %s.%s may not %s in bucket %s: %w", node, module, method, bucket, err)
}
cfg, err := readConfig(s.configPath)
if err != nil {
return nil, err
}
secret, err := os.ReadFile(filepath.Join(s.linksDir, bucket+".secret"))
if err != nil || strings.TrimSpace(string(secret)) == "" {
return nil, fmt.Errorf("link: bucket %s has no link key secret yet (%s); minio's provisioner makes it", bucket, filepath.Join(s.linksDir, bucket+".secret"))
}
now := s.now()
url, err := objectlink.Presign(objectlink.Request{
Method: strings.ToUpper(method), Endpoint: cfg.endpoint(), Bucket: bucket, Key: key,
AccessKey: linkKeyID(bucket), SecretKey: strings.TrimSpace(string(secret)), Region: cfg.Region,
Expires: time.Duration(seconds) * time.Second, Now: now,
})
if err != nil {
return nil, err
}
return map[string]any{"link": url, "method": method, "seconds": seconds, "expires-at": now.Add(time.Duration(seconds) * time.Second).UTC().Format(time.RFC3339)}, nil
}
// splitCaller splits the bus user `<node>.<module>` the runtime proved.
func splitCaller(caller string) (node, module string, ok bool) {
if caller == "" || caller == stdio.Unproven {
return "", "", false
}
node, module, ok = strings.Cut(caller, ".")
if !ok || node == "" || module == "" || strings.Contains(module, ".") {
return "", "", false
}
return node, module, true
}
// grant is what the mesh delivered for one bucket: its consumer and the consumer's shares.
type grant struct {
consumerModule, consumerNode string
shares []share
}
type share struct {
To string `json:"to"`
Methods []string `json:"methods"`
}
// grantOf reads the grant file now (never cached, so a change of shares applies to the next ask) and finds the entry
// whose derived bucket is this one. The entry's shape is the mesh's Contribution (mesh-sdk go/catalogue/declaration.go
// `Contribution`: from, node, as, values, derived) inside `given` (receivedFile in the same file).
func (s *server) grantOf(bucket string) (grant, error) {
raw, err := os.ReadFile(s.receives)
if err != nil {
return grant{}, fmt.Errorf("link: the grant file is unreadable (%s): %w", s.receives, err)
}
var doc struct {
Given []struct {
From string `json:"from"`
Node string `json:"node"`
Values struct {
Shares []share `json:"shares"`
} `json:"values"`
Derived struct {
Bucket string `json:"bucket"`
} `json:"derived"`
} `json:"given"`
}
if err := json.Unmarshal(raw, &doc); err != nil {
return grant{}, fmt.Errorf("link: the grant file %s is not JSON: %w", s.receives, err)
}
for _, g := range doc.Given {
if g.Derived.Bucket == bucket {
return grant{consumerModule: g.From, consumerNode: g.Node, shares: g.Values.Shares}, nil
}
}
return grant{}, fmt.Errorf("link: no grant names bucket %s: nobody is given it", bucket)
}
// allowed judges the caller: the bucket's consumer, or a share for this method naming the caller's module, or a seat
// the caller's module holds.
func (s *server) allowed(g grant, node, module, method string) error {
if g.consumerModule == module && (g.consumerNode == "" || g.consumerNode == node) {
return nil
}
// A share names the caller when its `to` is the caller's module, or a seat the caller's module holds. The
// controller is asked only for a seat share, and only when nothing earlier decided.
names := func(sh share) (bool, error) {
if sh.To == module {
return true, nil
}
if seat, isSeat := strings.CutPrefix(sh.To, "seat:"); isSeat {
held, err := s.holds(seat, node, module)
if err != nil {
return false, fmt.Errorf("a share names seat %s and the controller's seats cannot be read to tell whether this caller holds it: %w", seat, err)
}
return held, nil
}
return false, nil
}
var unread error
for _, sh := range g.shares {
if !has(sh.Methods, method) {
continue
}
named, err := names(sh)
if err != nil {
unread = err
continue
}
if named {
return nil
}
}
if unread != nil {
return unread
}
for _, sh := range g.shares {
if has(sh.Methods, method) {
continue
}
if named, err := names(sh); err == nil && named {
return fmt.Errorf("the bucket's shares give this caller %s, not %s", strings.Join(sh.Methods, " and "), method)
}
}
return errors.New("it is not the bucket's consumer and no share of the bucket names it for that method")
}
func has(list []string, v string) bool {
for _, x := range list {
if x == v {
return true
}
}
return false
}
// holds says whether node.module holds the seat, from the controller's seats, read at most once a minute. A read that
// failed is not cached: the next ask reads again.
func (s *server) holds(seat, node, module string) (bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
if !s.haveSet || s.now().Sub(s.readAt) >= holdersFor {
raw, err := s.seats()
if err != nil {
return false, err
}
set, err := holdersIn(raw)
if err != nil {
return false, err
}
s.holders, s.readAt, s.haveSet = set, s.now(), true
}
return s.holders[seat+"/"+node+"."+module], nil
}
// holdersIn reads the controller's `seats` answer (answer.seats[].holders[] of {node, module}, and answer.outside[] of
// {Claim, Node, Module}) into "<seat>/<node>.<module>". The answer may arrive wrapped in {"answer": ...}.
func holdersIn(raw json.RawMessage) (map[string]bool, error) {
var doc struct {
Answer json.RawMessage `json:"answer"`
Seats []struct {
Seat string `json:"seat"`
Holders []struct {
Node string `json:"node"`
Module string `json:"module"`
} `json:"holders"`
} `json:"seats"`
Outside []struct {
Claim, Node, Module string
} `json:"outside"`
}
if err := json.Unmarshal(raw, &doc); err != nil {
return nil, fmt.Errorf("the controller's seats answer is not JSON: %w", err)
}
if len(doc.Answer) > 0 && string(doc.Answer) != "null" {
return holdersIn(doc.Answer)
}
if doc.Seats == nil && doc.Outside == nil {
return nil, errors.New("the controller's seats answer holds neither seats nor outside claims")
}
out := map[string]bool{}
for _, s := range doc.Seats {
for _, h := range s.Holders {
out[s.Seat+"/"+h.Node+"."+h.Module] = true
}
}
for _, o := range doc.Outside {
out[o.Claim+"/"+o.Node+"."+o.Module] = true
}
return out, nil
}
modules/minio/cmd/object-store/server_test.go
+217
View File
@@ -0,0 +1,217 @@
package main
import (
"encoding/json"
"errors"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
// The grant entry's shape is the mesh's Contribution (mesh-sdk go/catalogue/declaration.go: from, node, as, values,
// derived), written in a `given` list by receivedFile in the same file; `shares` is written into values by
// go/catalogue/bucket_shares.go withShares: [{"to": ..., "methods": [...]}].
const grantFile = `{
"contributions": 1,
"requirement": "s3-bucket",
"given": [
{"from": "brother", "node": "shanks", "as": "brother-key",
"values": {"shares": [{"to": "seat:node-files", "methods": ["put"]}, {"to": "print-and-scan", "methods": ["get"]}]},
"derived": {"bucket": "mesh-shanks-brother"}},
{"from": "nextcloud", "node": "novox", "as": "nextcloud-key", "derived": {"bucket": "mesh-novox-nextcloud"}}
]
}`
// The seats answer is the controller's (mesh-controller.seats read on 2026-10-21): seats[].holders[] of {node, module},
// and outside[] of {Claim, Node, Module}.
const seatsAnswer = `{"answer": {"seats": [
{"seat": "node-files", "scope": "node", "holders": [{"node": "g14", "module": "mesh-files"}, {"node": "shanks", "module": "mesh-files"}]},
{"seat": "node-backup", "scope": "node", "holders": [{"node": "g14", "module": "restic"}]}
], "outside": [{"Claim": "printer", "Scope": "mesh", "Node": "shanks", "Module": "brother", "Site": "home", "Kind": ""}]}}`
type rig struct {
s *server
dir string
now time.Time
seatsCalls int
seatsErr error
}
func newRig(t *testing.T) *rig {
t.Helper()
dir := t.TempDir()
r := &rig{dir: dir, now: time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)}
write := func(name, content string) string {
p := filepath.Join(dir, name)
if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
return p
}
// What the mesh renders from files/object-store.json for a store on novox.internal, port 20001.
cfg := write("object-store.json", `{"scheme":"http","at":"novox.internal","port":"20001","region":"eu-west"}`)
write("links/mesh-shanks-brother.secret", "brother-link-secret\n")
write("links/mesh-novox-nextcloud.secret", "nextcloud-link-secret\n")
grants := write("grants/mesh.json", grantFile)
s, err := newServer(cfg, filepath.Join(dir, "links"), grants)
if err != nil {
t.Fatal(err)
}
s.now = func() time.Time { return r.now }
s.seats = func() (json.RawMessage, error) {
r.seatsCalls++
if r.seatsErr != nil {
return nil, r.seatsErr
}
return json.RawMessage(seatsAnswer), nil
}
r.s = s
return r
}
func ask(bucket, method string, seconds any) map[string]any {
a := map[string]any{"bucket": bucket, "key": "doc.pdf", "method": method}
if seconds != nil {
a["seconds"] = seconds
}
return a
}
func linkOf(t *testing.T, got any) *url.URL {
t.Helper()
m := got.(map[string]any)
u, err := url.Parse(m["link"].(string))
if err != nil {
t.Fatal(err)
}
return u
}
// 1516/AC1: brother's own ask gets a link signed with the bucket's link key, for the store's host and port, 300 s.
func TestLinkIsSignedWithTheBucketsLinkKey(t *testing.T) {
r := newRig(t)
got, err := r.s.link("shanks.brother", ask("mesh-shanks-brother", "get", float64(300)))
if err != nil {
t.Fatal(err)
}
u := linkOf(t, got)
if u.Host != "novox.internal:20001" || u.Scheme != "http" || u.Path != "/mesh-shanks-brother/doc.pdf" {
t.Fatalf("link goes to %s", u)
}
q := u.Query()
if cred := q.Get("X-Amz-Credential"); !strings.HasPrefix(cred, linkKeyID("mesh-shanks-brother")+"/20261021/eu-west/s3/") {
t.Fatalf("credential %q is not the bucket's link key", cred)
}
if strings.Contains(u.String(), "meshroot") {
t.Fatal("the root account signed a link")
}
if q.Get("X-Amz-Expires") != "300" || q.Get("X-Amz-Date") != "20261021T120000Z" {
t.Fatalf("expiry %q at %q", q.Get("X-Amz-Expires"), q.Get("X-Amz-Date"))
}
}
// 1516/AC1: the link key id is the one the provisioner makes (modules/minio/links.ts linkKeyId, tested there with the
// same literal).
func TestLinkKeyIDMatchesTheProvisioners(t *testing.T) {
if got := linkKeyID("brother"); got != "link0b4bd77cec705cf5" {
t.Fatalf("linkKeyID = %s", got)
}
}
// 1516/AC2: a share to seat:node-files for put, mesh-files holding node-files on g14: put given, get refused naming the method.
func TestSeatShareGivesTheHoldersTheSharedMethodOnly(t *testing.T) {
r := newRig(t)
if _, err := r.s.link("g14.mesh-files", ask("mesh-shanks-brother", "put", nil)); err != nil {
t.Fatalf("put: %v", err)
}
_, err := r.s.link("g14.mesh-files", ask("mesh-shanks-brother", "get", nil))
if err == nil || !strings.Contains(err.Error(), "put, not get") {
t.Fatalf("get was not refused naming the method: %v", err)
}
// A module share is judged by name alone.
if _, err := r.s.link("novox.print-and-scan", ask("mesh-shanks-brother", "get", nil)); err != nil {
t.Fatalf("print-and-scan get: %v", err)
}
}
// 1516/AC3: each of these is refused with its reason.
func TestRefusals(t *testing.T) {
r := newRig(t)
cases := []struct {
name, caller string
args map[string]any
want string
}{
{"a stranger", "g14.restic", ask("mesh-shanks-brother", "get", nil), "not the bucket's consumer"},
{"a seat it does not hold", "g14.restic", ask("mesh-shanks-brother", "put", nil), "not the bucket's consumer"},
{"unproven", "unproven", ask("mesh-shanks-brother", "get", nil), "did not prove"},
{"no grant", "shanks.brother", ask("no-such-bucket", "get", nil), "no grant names bucket"},
{"seconds over 600", "shanks.brother", ask("mesh-shanks-brother", "get", float64(601)), "at most ten minutes"},
{"seconds zero", "shanks.brother", ask("mesh-shanks-brother", "get", float64(0)), "outside 1 to 600"},
{"method delete", "shanks.brother", ask("mesh-shanks-brother", "delete", nil), "not get or put"},
{"another consumer's bucket", "shanks.brother", ask("mesh-novox-nextcloud", "get", nil), "not the bucket's consumer"},
}
for _, c := range cases {
_, err := r.s.link(c.caller, c.args)
if err == nil || !strings.Contains(err.Error(), c.want) {
t.Errorf("%s: want a refusal containing %q, got %v", c.name, c.want, err)
}
}
}
// 1516/AC4: with the controller unreadable a seat share is refused naming why; a good read serves a second ask within
// 60 s from the cache and is read again after.
func TestSeatsUnreadableAndCached(t *testing.T) {
r := newRig(t)
r.seatsErr = errors.New("the controller did not answer")
_, err := r.s.link("g14.mesh-files", ask("mesh-shanks-brother", "put", nil))
if err == nil || !strings.Contains(err.Error(), "the controller did not answer") || !strings.Contains(err.Error(), "node-files") {
t.Fatalf("not refused naming why: %v", err)
}
r.seatsErr = nil
put := func() {
t.Helper()
if _, err := r.s.link("g14.mesh-files", ask("mesh-shanks-brother", "put", nil)); err != nil {
t.Fatal(err)
}
}
put() // a failed read is not cached: this reads again
put() // served from the cache
if r.seatsCalls != 2 {
t.Fatalf("the controller was asked %d times, want 2 (one failure, one good read)", r.seatsCalls)
}
r.now = r.now.Add(59 * time.Second)
put()
if r.seatsCalls != 2 {
t.Fatalf("an ask within 60 s read the seats again (%d calls)", r.seatsCalls)
}
r.now = r.now.Add(2 * time.Second)
put()
if r.seatsCalls != 3 {
t.Fatalf("the seats were not read again after 60 s (%d calls)", r.seatsCalls)
}
}
// 1516/AC5: the grant file is read at each call.
func TestGrantFileIsReadAtEachCall(t *testing.T) {
r := newRig(t)
if _, err := r.s.link("novox.print-and-scan", ask("mesh-shanks-brother", "get", nil)); err != nil {
t.Fatal(err)
}
changed := strings.Replace(grantFile, `{"to": "print-and-scan", "methods": ["get"]}`, `{"to": "someone-else", "methods": ["get"]}`, 1)
if err := os.WriteFile(r.s.receives, []byte(changed), 0o600); err != nil {
t.Fatal(err)
}
if _, err := r.s.link("novox.print-and-scan", ask("mesh-shanks-brother", "get", nil)); err == nil {
t.Fatal("the second call judged by the old shares")
}
if _, err := r.s.link("novox.someone-else", ask("mesh-shanks-brother", "get", nil)); err != nil {
t.Fatal(err)
}
}
modules/minio/files/object-store.json
+6
View File
@@ -0,0 +1,6 @@
{
"scheme": "http",
"at": "${machine:at}",
"port": "${port:9000}",
"region": "eu-west"
}
modules/minio/go.mod
+5
View File
@@ -0,0 +1,5 @@
module minio
go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f
modules/minio/go.sum
+2
View File
@@ -0,0 +1,2 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f h1:7J/9jOfSfjfYpf8MOj96HwQRViQUkBNMb71RHP2mjn8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
modules/minio/links.ts
+64
View File
@@ -0,0 +1,64 @@
// The link key — a second service account per bucket, beside the consumer's own key (novox/hq ADR 0468 §8,
// issue 1594). The object-store seat signs the links (issue 1516) with this key and never with the root
// account: its policy allows only s3:GetObject and s3:PutObject on the bucket's objects, so a link can read or
// write an object of that bucket and do nothing else. Its secret is minio's own: a file under the state
// directory (links/<bucket>.secret, 0600), never in a grant, never on the bus, never in an emitted event.
//
// This file imports nothing of the SDK, so its tests run without the toolchain's packages.
import { createHash, randomBytes } from "node:crypto";
import { chmodSync, existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
import { join } from "node:path";
/** What the link key needs of the admin client (MinioClient has all three). */
export interface LinkAdmin {
createAccessKey(bucket: string, accessKey: string, secretKey: string, policy?: string): Promise<unknown>;
removeAccessKey(accessKey: string): Promise<void>;
accessKeyExists(accessKey: string): Promise<boolean>;
}
/** The access key id of a bucket's link key: "link" and the first 16 hex digits of the SHA-256 of the bucket
* name (20 characters, the longest an access key may be; a bucket name may be 63). The signer of issue 1516
* derives the same id. */
export function linkKeyId(bucket: string): string {
return "link" + createHash("sha256").update(bucket, "utf8").digest("hex").slice(0, 16);
}
/** Only get and put of the bucket's objects: no list, no delete, no bucket-level action. */
export function linkPolicy(bucket: string): string {
return JSON.stringify({
Version: "2012-10-17",
Statement: [{
Effect: "Allow",
Action: ["s3:GetObject", "s3:PutObject"],
Resource: [`arn:aws:s3:::${bucket}/*`],
}],
});
}
export function linkSecretPath(dir: string, bucket: string): string {
return join(dir, `${bucket}.secret`);
}
/** Make the bucket's link key if it is not there: the secret is kept if its file exists (so a key lost in
* minio is made again under the same secret), made fresh otherwise. Idempotent: a key that exists is left. */
export async function ensureLinkKey(admin: LinkAdmin, dir: string, bucket: string): Promise<void> {
const id = linkKeyId(bucket);
const path = linkSecretPath(dir, bucket);
mkdirSync(dir, { recursive: true, mode: 0o700 });
let secret = existsSync(path) ? readFileSync(path, "utf8").trim() : "";
if (secret !== "" && (await admin.accessKeyExists(id))) return;
if (secret === "") {
secret = randomBytes(32).toString("hex");
writeFileSync(path, secret + "\n", { mode: 0o600 });
chmodSync(path, 0o600);
}
try { await admin.removeAccessKey(id); } catch { /* none yet */ }
await admin.createAccessKey(bucket, id, secret, linkPolicy(bucket));
}
/** Revoke the bucket's link key; the bucket and the secret file's absence are left as they are. */
export async function revokeLinkKey(admin: LinkAdmin, dir: string, bucket: string): Promise<void> {
try { await admin.removeAccessKey(linkKeyId(bucket)); } catch { /* already gone */ }
try { unlinkSync(linkSecretPath(dir, bucket)); } catch { /* none */ }
}
modules/minio/module.json
+45
View File
@@ -18,6 +18,27 @@
"requires": [
"route"
],
"seats": [
{
"name": "object-store",
"scope": "mesh",
"serves": [
"link"
]
}
],
"claims": [
{
"name": "object-store",
"scope": "mesh",
"serves": [
"link"
]
}
],
"invokes": [
"seat:mesh-controller.seats"
],
"contributes": {
"route": {
"api": {
@@ -108,6 +129,13 @@
"mode": "0600",
"template": "files/root.env"
},
{
"id": "object-store",
"type": "file",
"path": "${dir:state}/object-store.json",
"mode": "0600",
"template": "files/object-store.json"
},
{
"id": "data",
"type": "directory",
@@ -173,11 +201,28 @@
"MESH_MINIO_ROOT_USER": "meshroot",
"MESH_MINIO_ROOT_PASSWORD_FILE": "${dir:state}/root.secret",
"MESH_MINIO_REGION": "eu-west",
"MESH_MINIO_LINKS": "${dir:state}/links",
"MESH_MINIO_MC_BIN": "mcli",
"MESH_MINIO_MC_CONFIG": "${dir:state}/mc",
"MESH_RECEIVES": "${dir:grants}/mesh.json"
}
},
{
"name": "tools-go",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/object-store",
"binary": "object-store",
"loads": [
"object-store"
],
"env": {
"OBJECT_STORE_CONFIG": "${dir:state}/object-store.json",
"MESH_MINIO_LINKS": "${dir:state}/links",
"MESH_RECEIVES": "${dir:grants}/mesh.json"
}
},
{
"name": "files",
"kind": "archive",
modules/minio/package.json
+3
View File
@@ -4,6 +4,9 @@
"description": "minio — S3-compatible object store. Its admin client, tools, provisioner and events live here (novox/hq ADR 0039).",
"type": "module",
"private": true,
"scripts": {
"test": "node --test --experimental-strip-types 'test/*.test.ts'"
},
"dependencies": {
"@novox/mesh-sdk": "^0.1.7"
},
modules/minio/provisioner/handlers.ts
+78
View File
@@ -0,0 +1,78 @@
// The s3-bucket provisioner's create, remove and holds, with what they touch handed in, so a unit test drives
// them with a fake admin client (the provisioner's index.ts runs them against minio).
import type { Provision } from "@novox/mesh-sdk/provisioner";
import { ensureLinkKey, revokeLinkKey, type LinkAdmin } from "../links.js";
export interface Admin extends LinkAdmin {
bucketExists(bucket: string): Promise<boolean>;
createBucket(bucket: string): Promise<void>;
canReachAs(bucket: string, accessKey: string, secretKey: string): Promise<boolean>;
readonly baseUrl: string;
}
export interface Deps {
minio: Admin;
linksDir: string;
announce(type: string, body: unknown): Promise<void>;
}
export function handlers({ minio, linksDir, announce }: Deps) {
return {
async create(p: Provision): Promise<void> {
const bucket = bucketNamed(p.derived);
const accessKeyId = p.as;
if (!(await minio.bucketExists(bucket))) await minio.createBucket(bucket);
// Re-mint the scoped key idempotently: drop any prior one under this id, then add it back with
// the mesh's secret.
try { await minio.removeAccessKey(accessKeyId); } catch { /* none yet — first provision */ }
await minio.createAccessKey(bucket, accessKeyId, p.password);
await ensureLinkKey(minio, linksDir, bucket);
await announce("bucket.created", {
bucket,
consumer: p.consumer ?? "",
accessKey: accessKeyId,
endpoint: minio.baseUrl,
});
},
async remove(p: { as: string; derived: Readonly<Record<string, unknown>> }): Promise<void> {
const bucket = bucketNamed(p.derived);
// Revoking the key is what cuts the consumer's access, and the bucket is kept, empty or not
// (novox/hq issue 241: a withdrawal never destroys a consumer's data — on 2026-10-04 a misread grants file withdrew every consumer at once). A bucket is removed by a person, never by this loop.
try { await minio.removeAccessKey(p.as); } catch { /* already gone */ }
await revokeLinkKey(minio, linksDir, bucket);
console.error(`[minio] ${p.as} withdrawn: access key and link key revoked, bucket ${bucket} kept`);
await announce("bucket.removed", { bucket, accessKey: p.as });
},
// Asked every minute by the harness: whether the backend still holds this consumer exactly as
// the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120).
async holds(p: Provision): Promise<boolean> {
return minio.canReachAs(bucketNamed(p.derived), p.as, p.password);
},
};
}
/** The bucket the mesh derived for this consumer.
*
* Absent means this module is running against a control plane that does not fill `${consumer:…}`
* yet, or a manifest whose `serves` block lost the line. Both are the same mistake from here —
* nobody said which bucket — and both are said rather than guessed: a provisioner that fell back
* to deriving one would restore the second rule and hide the fault behind a bucket that happens
* to be right. */
function bucketNamed(derived: Readonly<Record<string, unknown>>): string {
const bucket = derived.bucket;
if (typeof bucket !== "string" || bucket === "") {
throw new Error(
"the mesh did not say which bucket this consumer gets: minio's manifest must serve " +
"`bucket` under s3-bucket (novox/hq ADR 0202)",
);
}
return bucket;
}
modules/minio/provisioner/index.ts
+18 -53
View File
@@ -22,63 +22,28 @@
import { runProvisioner, type Provision } from "@novox/mesh-sdk/provisioner";
import { emit } from "@novox/mesh-sdk/events";
import { MinioClient } from "../client.js";
import { ensureLinkKey } from "../links.js";
import { handlers } from "./handlers.js";
const minio = MinioClient.fromEnv();
// minio's own link keys (issue 1594): the secrets live under the state directory, nowhere else.
const linksDir = process.env.MESH_MINIO_LINKS ?? "";
runProvisioner("s3-bucket", {
async create(p: Provision): Promise<void> {
const bucket = bucketNamed(p.derived);
const accessKeyId = p.as;
if (!(await minio.bucketExists(bucket))) await minio.createBucket(bucket);
// Re-mint the scoped key idempotently: drop any prior one under this id, then add it back with
// the mesh's secret.
try { await minio.removeAccessKey(accessKeyId); } catch { /* none yet — first provision */ }
await minio.createAccessKey(bucket, accessKeyId, p.password);
await announce("bucket.created", {
bucket,
consumer: p.consumer ?? "",
accessKey: accessKeyId,
endpoint: minio.baseUrl,
});
},
async remove(p: { as: string; derived: Readonly<Record<string, unknown>> }): Promise<void> {
const bucket = bucketNamed(p.derived);
// Revoking the key is what cuts the consumer's access, and the bucket is kept, empty or not
// (novox/hq issue 241: a withdrawal never destroys a consumer's data — on 2026-10-04 a misread grants file withdrew every consumer at once). A bucket is removed by a person, never by this loop.
try { await minio.removeAccessKey(p.as); } catch { /* already gone */ }
console.error(`[minio] ${p.as} withdrawn: access key revoked, bucket ${bucket} kept`);
await announce("bucket.removed", { bucket, accessKey: p.as });
},
// Asked every minute by the harness: whether the backend still holds this consumer exactly as
// the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120).
async holds(p: Provision): Promise<boolean> {
return minio.canReachAs(bucketNamed(p.derived), p.as, p.password);
},
});
/** The bucket the mesh derived for this consumer.
*
* Absent means this module is running against a control plane that does not fill `${consumer:…}`
* yet, or a manifest whose `serves` block lost the line. Both are the same mistake from here —
* nobody said which bucket — and both are said rather than guessed: a provisioner that fell back
* to deriving one would restore the second rule and hide the fault behind a bucket that happens
* to be right. */
function bucketNamed(derived: Readonly<Record<string, unknown>>): string {
const bucket = derived.bucket;
if (typeof bucket !== "string" || bucket === "") {
throw new Error(
"the mesh did not say which bucket this consumer gets: minio's manifest must serve " +
"`bucket` under s3-bucket (novox/hq ADR 0202)",
);
// At start every bucket that exists gets its link key too, those made before link keys were (best effort:
// a bucket that fails is said, the provisioner still serves).
void (async () => {
if (linksDir === "") { console.error("[minio] MESH_MINIO_LINKS is not set: no link keys made"); return; }
try {
for (const b of await minio.listBuckets()) {
try { await ensureLinkKey(minio, linksDir, b.name); }
catch (err) { console.error(`[minio] link key for ${b.name}: ${err}`); }
}
} catch (err) {
console.error(`[minio] could not list buckets for link keys: ${err}`);
}
return bucket;
}
})();
runProvisioner("s3-bucket", handlers({ minio, linksDir, announce }));
/** Emit best-effort: a broker hiccup is logged and dropped, never allowed to throw back and fail a
* bucket that was made. */
modules/minio/test/links.test.ts
+117
View File
@@ -0,0 +1,117 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { mkdtempSync, readFileSync, statSync, existsSync, readdirSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
// Issue 1594: a link key per bucket (novox/hq ADR 0468 §8). The fake admin below stands for minio's service
// accounts; the real client's policies come from client.ts's bucketPolicy, which the tests import.
import { bucketPolicy } from "../client.ts";
import { ensureLinkKey, linkKeyId, linkSecretPath } from "../links.ts";
import { handlers } from "../provisioner/handlers.ts";
class FakeAdmin {
baseUrl = "http://127.0.0.1:9000";
buckets = new Set<string>();
keys = new Map<string, { bucket: string; secret: string; policy: string }>();
created: string[] = [];
async bucketExists(b: string) { return this.buckets.has(b); }
async createBucket(b: string) { this.buckets.add(b); }
async canReachAs() { return true; }
async accessKeyExists(id: string) { return this.keys.has(id); }
async removeAccessKey(id: string) {
if (!this.keys.delete(id)) throw new Error("no such key");
}
async createAccessKey(bucket: string, id: string, secret: string, policy: string = bucketPolicy(bucket)) {
this.keys.set(id, { bucket, secret, policy });
this.created.push(id);
return { accessKey: id, secretKey: secret };
}
}
function setup() {
const minio = new FakeAdmin();
const dir = join(mkdtempSync(join(process.env.TMPDIR ?? tmpdir(), "minio-links-")), "links");
const events: Array<{ type: string; body: unknown }> = [];
const h = handlers({ minio, linksDir: dir, announce: async (type, body) => { events.push({ type, body }); } });
return { minio, dir, events, h };
}
const provision = { as: "brother-key", password: "consumer-secret", consumer: "brother", derived: { bucket: "brother" } } as any;
test("creating a consumer's bucket makes a link key allowed only get and put of the bucket's objects, its secret in links/<bucket>.secret at 0600", async () => {
// 1594/AC1
const { minio, dir, h } = setup();
await h.create(provision);
const link = minio.keys.get(linkKeyId("brother"));
assert.ok(link, "a link key exists for the bucket");
const policy = JSON.parse(link.policy);
assert.equal(policy.Statement.length, 1);
assert.deepEqual(policy.Statement[0].Action, ["s3:GetObject", "s3:PutObject"]);
assert.deepEqual(policy.Statement[0].Resource, ["arn:aws:s3:::brother/*"]);
const path = linkSecretPath(dir, "brother");
assert.equal(statSync(path).mode & 0o777, 0o600);
assert.equal(readFileSync(path, "utf8").trim(), link.secret);
assert.ok(link.secret.length >= 32);
});
test("buckets that existed before get a link key at start, and a second start makes none twice", async () => {
// 1594/AC2
const { minio, dir } = setup();
for (const b of ["brother", "mesh-mail", "nextcloud", "invoicing"]) minio.buckets.add(b);
for (const b of minio.buckets) await ensureLinkKey(minio, dir, b);
assert.equal(minio.keys.size, 4);
const first = [...minio.created];
const secret = readFileSync(linkSecretPath(dir, "nextcloud"), "utf8");
for (const b of minio.buckets) await ensureLinkKey(minio, dir, b);
assert.deepEqual(minio.created, first, "no key made a second time");
assert.equal(readFileSync(linkSecretPath(dir, "nextcloud"), "utf8"), secret, "the secret is kept");
});
test("a link key lost in minio is made again under the kept secret", async () => {
// 1594/AC2
const { minio, dir } = setup();
await ensureLinkKey(minio, dir, "brother");
const secret = minio.keys.get(linkKeyId("brother"))!.secret;
minio.keys.clear();
await ensureLinkKey(minio, dir, "brother");
assert.equal(minio.keys.get(linkKeyId("brother"))!.secret, secret);
});
test("removing a consumer revokes its bucket's link key and keeps the bucket", async () => {
// 1594/AC3
const { minio, dir, h } = setup();
await h.create(provision);
assert.ok(minio.keys.has(linkKeyId("brother")));
await h.remove({ as: "brother-key", derived: { bucket: "brother" } });
assert.ok(!minio.keys.has(linkKeyId("brother")), "link key revoked");
assert.ok(!minio.keys.has("brother-key"), "consumer key revoked");
assert.ok(minio.buckets.has("brother"), "bucket kept");
assert.ok(!existsSync(linkSecretPath(dir, "brother")), "secret file removed");
});
test("the consumer's own key keeps its s3:* policy, and the link key's secret is in no event", async () => {
// 1594/AC4
const { minio, dir, events, h } = setup();
await h.create(provision);
const own = minio.keys.get("brother-key")!;
assert.equal(own.secret, "consumer-secret");
assert.equal(own.policy, bucketPolicy("brother"));
assert.deepEqual(JSON.parse(own.policy).Statement[0].Action, ["s3:*"]);
const linkSecret = minio.keys.get(linkKeyId("brother"))!.secret;
await h.remove({ as: "brother-key", derived: { bucket: "brother" } });
assert.ok(events.length >= 2);
for (const e of events) assert.ok(!JSON.stringify(e).includes(linkSecret), `${e.type} carries no link secret`);
assert.deepEqual(readdirSync(dir), [], "nothing else is kept in the links directory");
});
test("a link key id fits an access key (20 characters) whatever the bucket name", () => {
assert.equal(linkKeyId("a".repeat(63)).length, 20);
assert.notEqual(linkKeyId("brother"), linkKeyId("mesh-mail"));
});
test("the link key id is the one the object-store tool derives (cmd/object-store server_test.go holds the same literal)", () => {
// 1516/AC1
assert.equal(linkKeyId("brother"), "link0b4bd77cec705cf5");
});
modules/minio/tools/index.ts
-16
View File
@@ -50,22 +50,6 @@ export function getMinioTools(minio: MinioClient): ToolDefinition[] {
input: { bucket: { type: "string", description: "the bucket name" } },
run: async (args) => minio.bucketInfo(String(args.bucket)),
},
{
name: "minio_presigned_url",
description: "A time-limited URL to download (GET) or upload (PUT) one object without credentials.",
input: {
bucket: { type: "string", description: "the bucket name" },
object: { type: "string", description: "the object key" },
method: { type: "string", description: "'GET' to download (default) or 'PUT' to upload" },
expires: { type: "number", description: "seconds until the URL expires (default 86400 = 24h)" },
},
run: async (args) => {
const method = String(args.method ?? "GET").toUpperCase() === "PUT" ? "PUT" : "GET";
const expires = args.expires ? Number(args.expires) : 86400;
const url = minio.presignedUrl(method, String(args.bucket), String(args.object), expires);
return { method, bucket: String(args.bucket), object: String(args.object), expiresInSeconds: expires, url };
},
},
];
}
modules/minio/tsconfig.json
+3 -1
View File
@@ -10,7 +10,9 @@
},
"include": [
"client.ts",
"links.ts",
"tools/index.ts",
"provisioner/index.ts"
"provisioner/index.ts",
"provisioner/handlers.ts"
]
}
modules/only-office/cmd/only-office/convert.go
+204
View File
@@ -0,0 +1,204 @@
package main
import (
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"fmt"
"net/url"
"regexp"
"sort"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// The ask the verb queues for the converter process (ADR 0468 §7; the shape of brother's device asks, ADR 0436 §6):
// on the work queue convert-asks, whose one consumer is the module's process `converter`, which writes its answer
// under the ask's id in the state convert-answers.
const (
// EventAsked is the event on the work queue; the stream convert-asks carries it.
EventAsked = "convert.asked"
// AnswersState is the state the converter writes its answers in.
AnswersState = "convert-answers"
// ConvertWait is how long one call waits for the converter. The tool runner cuts every tool call at 30 s, so a
// call answers within 25 s: a conversion that is not done by then goes on, and is read with convert {ask}.
ConvertWait = 25 * time.Second
// TakeUpWithin is how long an ask may wait on the queue before the converter takes it up: brother's device process
// follows a conversion for 3 minutes.
TakeUpWithin = 3 * time.Minute
)
// Types are the document types the converter takes (ADR 0468 §7). Any other is refused by name.
var Types = []string{"docx", "doc", "xlsx", "xls", "pptx", "ppt", "odt", "ods", "odp", "rtf", "txt", "html"}
// Ask is what the verb queues: the operation, when it was asked, and the operation's arguments.
type Ask struct {
ID string `json:"id"`
Op string `json:"op"`
At time.Time `json:"asked-at"`
Deadline time.Time `json:"deadline"`
// Source is the signed GET link of the document, Type its type, Target the signed PUT link of the PDF.
Source string `json:"source"`
Type string `json:"type"`
Target string `json:"target"`
// SHA256 is the source's SHA-256 when the asker knows it; the converter then names the conversion by it.
SHA256 string `json:"sha256,omitempty"`
}
var sha256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
// OpConvert is the one operation.
const OpConvert = "convert"
// AskBus is how the tool reaches the converter: it queues an ask and reads an answer back.
type AskBus interface {
Publish(id string, body any) error
Answer(id string) (json.RawMessage, error)
}
type sdkAskBus struct{}
func (sdkAskBus) Publish(id string, body any) error {
return stdio.Emit(EventAsked, body, stdio.EmitOptions{MessageID: id})
}
func (sdkAskBus) Answer(id string) (json.RawMessage, error) {
e, err := stdio.State(AnswersState).Get(id)
if err != nil || e == nil {
return nil, err
}
return e.Value, nil
}
// Service is the tool side.
type Service struct {
Asks AskBus
Now func() time.Time
// Wait is how long an answer is waited for, Poll how often it is looked for.
Wait, Poll time.Duration
}
func newService() *Service {
return &Service{Asks: sdkAskBus{}, Now: time.Now, Wait: ConvertWait, Poll: 500 * time.Millisecond}
}
func newAskID() string {
b := make([]byte, 12)
rand.Read(b)
return hex.EncodeToString(b)
}
// cleanType is a type as asked, lower-case and without a leading dot, when it is one the converter takes.
func cleanType(raw string) (string, error) {
t := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(raw), "."))
for _, ok := range Types {
if t == ok {
return t, nil
}
}
taken := append([]string(nil), Types...)
sort.Strings(taken)
return "", fmt.Errorf("type %q is not one the converter takes: %s", raw, strings.Join(taken, ", "))
}
func checkLink(name, raw string) error {
u, err := url.Parse(strings.TrimSpace(raw))
if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
return fmt.Errorf("%s is a signed http(s) link, such as one object-store.link gives", name)
}
return nil
}
// Convert queues one ask and waits for the converter's answer {size, sha256} (the PDF it put to target). Nothing is
// queued for a type the converter does not take or a link that is not one.
func (s *Service) Convert(ctx context.Context, source, typ, target, sha256 string) (map[string]any, error) {
t, err := cleanType(typ)
if err != nil {
return nil, err
}
if err := checkLink("source", source); err != nil {
return nil, err
}
if err := checkLink("target", target); err != nil {
return nil, err
}
sha256 = strings.ToLower(strings.TrimSpace(sha256))
if sha256 != "" && !sha256Pattern.MatchString(sha256) {
return nil, fmt.Errorf("sha256 is the source's SHA-256, 64 hex digits, not %q", sha256)
}
at := s.Now().UTC()
ask := Ask{ID: newAskID(), Op: OpConvert, At: at, Deadline: at.Add(TakeUpWithin), Source: strings.TrimSpace(source), Type: t, Target: strings.TrimSpace(target), SHA256: sha256}
if err := s.Asks.Publish(ask.ID, ask); err != nil {
return nil, fmt.Errorf("the ask for the converter could not be queued: %w", err)
}
return s.await(ctx, ask.ID)
}
// Read answers the ask of an earlier call: the converter's answer, or that it is still converting.
func (s *Service) Read(ctx context.Context, id string) (map[string]any, error) {
if !askPattern.MatchString(id) {
return nil, fmt.Errorf("ask %q is not an ask id: the 24 hex digits convert answered", id)
}
return s.await(ctx, id)
}
var askPattern = regexp.MustCompile(`^[0-9a-f]{24}$`)
// await waits at most Wait for the converter's answer under the id. A conversion still running is answered as
// converting with its ask id, never as a failure: it goes on, and convert {ask} reads its answer.
func (s *Service) await(ctx context.Context, id string) (map[string]any, error) {
ctx, cancel := context.WithTimeout(ctx, s.Wait)
defer cancel()
for {
raw, err := s.Asks.Answer(id)
if err != nil {
return nil, fmt.Errorf("the converter's answers cannot be read (ask %s): %w", id, err)
}
if raw != nil {
var out map[string]any
if err := json.Unmarshal(raw, &out); err != nil {
return nil, fmt.Errorf("the converter's answer is not JSON (ask %s): %w", id, err)
}
if msg, bad := out["error"].(string); bad {
return nil, fmt.Errorf("%s (ask %s)", msg, id)
}
out["ask"] = id
return out, nil
}
select {
case <-ctx.Done():
return map[string]any{"state": "converting", "ask": id,
"note": "the conversion goes on; call convert again with only ask to read its answer (kept 10 minutes)"}, nil
case <-time.After(s.Poll):
}
}
}
func tools(s *Service) []stdio.Tool {
return []stdio.Tool{{
Name: "document-converter.convert",
Description: "Turn a Word, spreadsheet, slide, text or web document into a PDF: the document is read from a signed GET " +
"link and the PDF is put to a signed PUT link, so no document crosses the bus. Takes " + strings.Join(Types, ", ") +
"; any other type is refused by name. Answers the PDF's size and SHA-256 or the document server's error; a conversion not done within 25 s is answered as converting with its ask id (a tool call is cut at 30 s), and convert with only ask reads it.",
Input: map[string]any{
"source": map[string]any{"type": "string", "description": "a signed GET link of the document"},
"type": map[string]any{"type": "string", "description": "the document's type: " + strings.Join(Types, ", ")},
"target": map[string]any{"type": "string", "description": "a signed PUT link for the PDF"},
"sha256": map[string]any{"type": "string", "description": "the document's SHA-256, if known (optional)"},
"ask": map[string]any{"type": "string", "description": "the ask id of a conversion that was answered as converting: alone, it reads that conversion's answer"},
},
Run: func(a map[string]any) (any, error) {
str := func(k string) string { v, _ := a[k].(string); return v }
if ask := str("ask"); ask != "" {
if str("source") != "" || str("type") != "" || str("target") != "" || str("sha256") != "" {
return nil, fmt.Errorf("ask is read on its own: give ask alone to read an earlier conversion, or source, type and target to start one")
}
return s.Read(context.Background(), ask)
}
return s.Convert(context.Background(), str("source"), str("type"), str("target"), str("sha256"))
},
}}
}
modules/only-office/cmd/only-office/convert_test.go
+221
View File
@@ -0,0 +1,221 @@
package main
import (
"context"
"encoding/json"
"os"
"strings"
"sync"
"testing"
"time"
)
// fakeBus is the work queue and the state: it records each ask and lets a test write the answer, as the converter
// process would under the ask's id.
type fakeBus struct {
mu sync.Mutex
asks []Ask
answers map[string]json.RawMessage
answerOn func(Ask) json.RawMessage // answers an ask as it is published
}
func (b *fakeBus) Publish(id string, body any) error {
b.mu.Lock()
defer b.mu.Unlock()
raw, _ := json.Marshal(body)
var a Ask
_ = json.Unmarshal(raw, &a)
b.asks = append(b.asks, a)
if b.answerOn != nil {
if ans := b.answerOn(a); ans != nil {
b.answers[id] = ans
}
}
return nil
}
func (b *fakeBus) Answer(id string) (json.RawMessage, error) {
b.mu.Lock()
defer b.mu.Unlock()
return b.answers[id], nil
}
func rig(wait time.Duration) (*Service, *fakeBus) {
b := &fakeBus{answers: map[string]json.RawMessage{}}
return &Service{Asks: b, Now: time.Now, Wait: wait, Poll: time.Millisecond}, b
}
const get, put = "http://novox.internal:20001/b/doc.docx?X-Amz-Signature=a", "http://novox.internal:20001/b/doc.pdf?X-Amz-Signature=b"
// 1595/AC1: one ask with those fields is published on convert-asks, and the verb answers what the converter wrote under
// the ask's id.
func TestConvertQueuesOneAskAndAnswersWhatTheConverterWrote(t *testing.T) {
s, b := rig(time.Second)
b.answerOn = func(a Ask) json.RawMessage { return json.RawMessage(`{"size": 4096, "sha256": "ab12"}`) }
got, err := s.Convert(context.Background(), get, "DOCX", put, "")
if err != nil {
t.Fatal(err)
}
if len(b.asks) != 1 {
t.Fatalf("%d asks queued", len(b.asks))
}
a := b.asks[0]
if a.Op != OpConvert || a.Source != get || a.Type != "docx" || a.Target != put || a.ID == "" {
t.Fatalf("ask %+v", a)
}
if got["size"] != float64(4096) || got["sha256"] != "ab12" || got["ask"] != a.ID {
t.Fatalf("answer %v", got)
}
}
// 1595/AC1: an error the converter wrote is the verb's error.
func TestConvertAnswersTheConvertersError(t *testing.T) {
s, b := rig(time.Second)
b.answerOn = func(a Ask) json.RawMessage {
return json.RawMessage(`{"error": "-4: the document server could not download the file"}`)
}
if _, err := s.Convert(context.Background(), get, "doc", put, ""); err == nil || !strings.Contains(err.Error(), "-4: the document server") {
t.Fatalf("error %v", err)
}
}
// 1595/AC2: a type outside the twelve is refused naming the types taken, and nothing is queued.
func TestConvertRefusesOtherTypes(t *testing.T) {
s, b := rig(time.Second)
for _, typ := range []string{"exe", "pdf", "", "docm", "jpg"} {
_, err := s.Convert(context.Background(), get, typ, put, "")
if err == nil || !strings.Contains(err.Error(), "docx") || !strings.Contains(err.Error(), "txt") {
t.Errorf("type %q: %v", typ, err)
}
}
if _, err := s.Convert(context.Background(), "file:///etc/passwd", "docx", put, ""); err == nil {
t.Error("a file link was taken")
}
if len(b.asks) != 0 {
t.Fatalf("%d asks queued", len(b.asks))
}
if len(Types) != 12 {
t.Fatalf("%d types", len(Types))
}
}
// 1595/AC3, AC6: with no answer within the call's bound the verb answers that it is converting, with the ask id,
// and does not fail; convert {ask} then returns the converter's answer once it is written, or converting again.
func TestAConversionNotDoneInTimeIsAnsweredAsConvertingAndRead(t *testing.T) {
s, b := rig(30 * time.Millisecond)
got, err := s.Convert(context.Background(), get, "docx", put, "")
if err != nil {
t.Fatalf("a slow converter failed the call: %v", err)
}
id := b.asks[0].ID
if got["state"] != "converting" || got["ask"] != id {
t.Fatalf("answer %v", got)
}
again, err := s.Read(context.Background(), id)
if err != nil || again["state"] != "converting" || again["ask"] != id {
t.Fatalf("read while converting: %v, %v", again, err)
}
b.mu.Lock()
b.answers[id] = json.RawMessage(`{"size": 77, "sha256": "cd34"}`)
b.mu.Unlock()
done, err := s.Read(context.Background(), id)
if err != nil || done["size"] != float64(77) || done["sha256"] != "cd34" || done["state"] != nil {
t.Fatalf("read after: %v, %v", done, err)
}
if len(b.asks) != 1 {
t.Errorf("%d asks queued, want the one", len(b.asks))
}
if ConvertWait != 25*time.Second {
t.Errorf("a call waits %s, want 25s", ConvertWait)
}
// Through the tool: ask alone reads; ask with anything else is refused; a made-up ask id is refused.
tl := tools(s)[0]
if r, err := tl.Run(map[string]any{"ask": id}); err != nil || r.(map[string]any)["size"] != float64(77) {
t.Errorf("tool read: %v, %v", r, err)
}
if _, err := tl.Run(map[string]any{"ask": id, "source": get}); err == nil {
t.Error("ask with a source was accepted")
}
if _, err := tl.Run(map[string]any{"ask": "../x"}); err == nil {
t.Error("a made-up ask id was accepted")
}
}
// 1595/AC4: the tool side opens no credential: its code never names the JWT or a secret file, and the manifest's
// tools bundle is given no env naming one.
func TestTheToolSideHoldsNoCredential(t *testing.T) {
// The tool side is these two files; the converter process (issue 1596) is code of its own, which alone reads the token.
for _, f := range []string{"main.go", "convert.go"} {
raw, err := os.ReadFile(f)
if err != nil {
t.Fatal(err)
}
for _, word := range []string{"jwt", "JWT", ".secret", "os.ReadFile", "os.Open"} {
if strings.Contains(string(raw), word) {
t.Errorf("%s mentions %q", f, word)
}
}
}
m := manifest(t)
for _, a := range m["build"].(map[string]any)["artifacts"].([]any) {
if env, ok := a.(map[string]any)["env"]; ok {
t.Errorf("an artifact is given env %v", env)
}
}
}
func manifest(t *testing.T) map[string]any {
t.Helper()
raw, err := os.ReadFile("../../module.json")
if err != nil {
t.Fatal(err)
}
var m map[string]any
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
return m
}
// 1595/AC5: only-office claims document-converter (scope mesh) serving convert, declares the stream convert-asks as a
// work queue and the state convert-answers (the shape of brother's device-asks and device-answers).
func TestManifestClaimsTheSeat(t *testing.T) {
m := manifest(t)
seats, _ := m["seats"].([]any)
if len(seats) != 1 || seats[0].(map[string]any)["name"] != "document-converter" || seats[0].(map[string]any)["scope"] != "mesh" ||
len(seats[0].(map[string]any)["serves"].([]any)) != 1 || seats[0].(map[string]any)["serves"].([]any)[0] != "convert" {
t.Fatalf("the seat document-converter is not declared in seats (the mesh defines no seat a module does not declare): %v", seats)
}
claims := m["claims"].([]any)
c := claims[0].(map[string]any)
if len(claims) != 1 || c["name"] != "document-converter" || c["scope"] != "mesh" ||
len(c["serves"].([]any)) != 1 || c["serves"].([]any)[0] != "convert" {
t.Fatalf("claims %v", claims)
}
streams := m["streams"].([]any)
s := streams[0].(map[string]any)
if len(streams) != 1 || s["name"] != "convert-asks" || s["retention"] != "work-queue" || s["max-deliveries"] != float64(3) ||
s["events"].([]any)[0] != EventAsked {
t.Fatalf("streams %v", streams)
}
state := m["state"].([]any)
st := state[0].(map[string]any)
if len(state) != 1 || st["name"] != AnswersState || st["ttl-seconds"] != float64(600) {
t.Fatalf("state %v", state)
}
found := false
for _, a := range m["build"].(map[string]any)["artifacts"].([]any) {
found = found || a.(map[string]any)["from"] == "cmd/only-office"
}
if !found {
t.Error("no Go bundle")
}
}
func TestTheVerbIsListedUnderTheSeat(t *testing.T) {
s, _ := rig(time.Second)
ts := tools(s)
if len(ts) != 1 || ts[0].Name != "document-converter.convert" {
t.Fatalf("tools %v", ts)
}
}
modules/only-office/cmd/only-office/converter.go
+311
View File
@@ -0,0 +1,311 @@
package main
// The converter process (novox/hq ADR 0468 §7, ADR 0436 §1, §3, §6; issue 1596). It runs under an account of its own
// and a bus account of its own, is the one consumer of the work queue convert-asks, serves no verb, and alone reads
// the document server's token (the JWT). For each ask it has the document server turn the source into a PDF, fetches
// the PDF from the document server and puts it to the ask's target link, and writes {size, sha256} -- or the error --
// under the ask's id in the state convert-answers, where the tool side reads it.
import (
"bytes"
"context"
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"os/signal"
"strings"
"syscall"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/objectlink"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
)
const (
// AsksStream is the work-queue stream's bus name: the manifest's convert-asks, capitalised with its hyphen an underscore.
AsksStream = "CONVERT_ASKS"
// ConvertPath is the document server's conversion service, reached on the machine's own published port. The path is
// the one the conversion API's request page names, https://api.onlyoffice.com/docs/docs-api/additional-api/conversion-api/request/
// (read 2026-10-21): POST to https://documentserver/converter.
ConvertPath = "/converter"
// StaleAfter is how old an ask may be when the converter takes it: older than twice the longest wait, nobody waits.
StaleAfter = 6 * time.Minute
// ConvertRun is how long one conversion, its fetch and its put may take.
ConvertRun = 4 * time.Minute
)
// errorMeanings are the conversion service's error codes. Read from the conversion API's documentation, page
// https://api.onlyoffice.com/docs/docs-api/additional-api/conversion-api/error-codes/ on 2026-10-21.
var errorMeanings = map[int]string{
-1: "unknown error",
-2: "conversion timeout error",
-3: "conversion error",
-4: "error while downloading the document file to be converted",
-5: "incorrect password",
-6: "error while accessing the conversion result database",
-7: "input error",
-8: "invalid token",
-9: "the converter cannot determine the output file format",
-10: "size limit exceeded",
}
// meaningOf is an error code with its meaning, "-4: error while ...".
func meaningOf(code int) string {
if m, ok := errorMeanings[code]; ok {
return fmt.Sprintf("%d: %s", code, m)
}
return fmt.Sprintf("%d: a code the conversion API does not document", code)
}
// jwtHS256 signs a payload as a JSON Web Token with HMAC-SHA256.
func jwtHS256(secret string, payload []byte) string {
enc := base64.RawURLEncoding
signing := enc.EncodeToString([]byte(`{"alg":"HS256","typ":"JWT"}`)) + "." + enc.EncodeToString(payload)
mac := hmac.New(sha256.New, []byte(secret))
mac.Write([]byte(signing))
return signing + "." + enc.EncodeToString(mac.Sum(nil))
}
type answerStore interface {
Create(key string, value any) (uint64, error)
}
type action int
const (
actionAck action = iota
actionTerm
actionRetry
)
// converterWorker answers asks. Everything it touches is handed in, so a test drives it with fakes.
type converterWorker struct {
// JWT reads the document server's secret; only this process calls it.
JWT func() (string, error)
// Server is the document server's address, "http://127.0.0.1:9070".
Server string
// Docs talks to the document server, Store to the object store.
Docs, Store *http.Client
Answers answerStore
Now func() time.Time
Logf func(string, ...any)
NewKey func() string
}
func (w *converterWorker) logf(f string, a ...any) {
if w.Logf != nil {
w.Logf(f, a...)
}
}
// Handle takes one queued ask: it answers it under its id and says how the message is settled.
func (w *converterWorker) Handle(body []byte) (action, time.Duration) {
var ask Ask
if err := json.Unmarshal(body, &ask); err != nil || ask.ID == "" {
w.logf("[only-office] converter: a queued ask could not be read: %v", err)
return actionTerm, 0
}
if age := w.Now().Sub(ask.At); age > StaleAfter {
w.logf("[only-office] converter: ask %s is %s old; nobody waits for it", ask.ID, age.Round(time.Second))
return actionAck, 0
}
if !ask.Deadline.IsZero() && w.Now().After(ask.Deadline) {
w.answer(ask.ID, map[string]any{"error": "the ask was taken up after its deadline: nothing was converted for it; ask again"})
return actionTerm, 0
}
ctx, cancel := context.WithTimeout(context.Background(), ConvertRun)
defer cancel()
out, err := w.convert(ctx, ask)
if err != nil {
out = map[string]any{"error": err.Error()}
}
if err := w.answer(ask.ID, out); err != nil {
return actionRetry, 5 * time.Second
}
return actionAck, 0
}
func (w *converterWorker) answer(id string, v map[string]any) error {
if _, err := w.Answers.Create(id, v); err != nil && !errors.Is(err, stdio.ErrChanged) {
w.logf("[only-office] converter: the answer to %s could not be written: %v", id, err)
var api *jetstream.APIError
if errors.As(err, &api) {
return nil // a refused value would be refused again
}
return err
}
return nil
}
// convert does one ask: the document server converts, then the PDF is put to the target.
func (w *converterWorker) convert(ctx context.Context, ask Ask) (map[string]any, error) {
if ask.Op != OpConvert {
return nil, fmt.Errorf("the converter does not know the operation %q", ask.Op)
}
if _, err := cleanType(ask.Type); err != nil {
return nil, err
}
secret, err := w.JWT()
if err != nil {
return nil, fmt.Errorf("the document server's token cannot be read: %w", err)
}
key := ask.SHA256
if key == "" {
key = w.NewKey()
}
req := map[string]any{"async": false, "filetype": ask.Type, "key": key, "outputtype": "pdf", "url": ask.Source}
payload, _ := json.Marshal(req)
req["token"] = jwtHS256(secret, payload) // JWT_IN_BODY: the document server reads the token from the body
reqBody, _ := json.Marshal(req)
hreq, err := http.NewRequestWithContext(ctx, http.MethodPost, w.Server+ConvertPath, bytes.NewReader(reqBody))
if err != nil {
return nil, err
}
hreq.Header.Set("Content-Type", "application/json")
hreq.Header.Set("Accept", "application/json")
resp, err := w.Docs.Do(hreq)
if err != nil {
return nil, fmt.Errorf("the document server cannot be reached: %w", err)
}
defer resp.Body.Close()
raw, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if err != nil {
return nil, fmt.Errorf("the document server's answer cannot be read: %w", err)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("the document server answered %d", resp.StatusCode)
}
var out struct {
EndConvert bool `json:"endConvert"`
FileURL string `json:"fileUrl"`
Error int `json:"error"`
}
if err := json.Unmarshal(raw, &out); err != nil {
return nil, fmt.Errorf("the document server's answer is not JSON: %w", err)
}
if out.Error != 0 {
return nil, fmt.Errorf("the document server refused the conversion, error %s", meaningOf(out.Error))
}
if !out.EndConvert || out.FileURL == "" {
return nil, errors.New("the document server did not finish the conversion: its answer has no file")
}
return w.deliver(ctx, out.FileURL, ask.Target)
}
// deliver fetches the PDF from the document server into a temporary file and puts it to the target.
func (w *converterWorker) deliver(ctx context.Context, fileURL, target string) (map[string]any, error) {
greq, err := http.NewRequestWithContext(ctx, http.MethodGet, fileURL, nil)
if err != nil {
return nil, err
}
gresp, err := w.Docs.Do(greq)
if err != nil {
return nil, fmt.Errorf("the converted file cannot be fetched from the document server: %w", err)
}
defer gresp.Body.Close()
if gresp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("the document server answered %d for the converted file", gresp.StatusCode)
}
tmp, err := os.CreateTemp("", "converted-*.pdf")
if err != nil {
return nil, fmt.Errorf("a temporary file cannot be made: %w", err)
}
defer os.Remove(tmp.Name())
defer tmp.Close()
n, err := io.Copy(tmp, io.LimitReader(gresp.Body, objectlink.MaxDocument+1))
if err != nil {
return nil, fmt.Errorf("the converted file cannot be read: %w", err)
}
if n > objectlink.MaxDocument {
return nil, fmt.Errorf("the converted file is over %d bytes (novox/hq ADR 0468 §3)", objectlink.MaxDocument)
}
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
return nil, err
}
res, err := objectlink.Put(ctx, w.Store, target, tmp, n, objectlink.MaxDocument)
if err != nil {
return nil, fmt.Errorf("the store did not take the PDF: %w", err)
}
return map[string]any{"size": res.Size, "sha256": res.SHA256}, nil
}
func newKey() string {
b := make([]byte, 16)
rand.Read(b)
return hex.EncodeToString(b)
}
func readSecret(path string) func() (string, error) {
return func() (string, error) {
raw, err := os.ReadFile(path)
if err != nil {
return "", err
}
s := strings.TrimSpace(string(raw))
if s == "" {
return "", fmt.Errorf("%s is empty", path)
}
return s, nil
}
}
// converter is `only-office converter`: it reads its own bus account, connects, and consumes convert-asks until stopped.
func converter() int {
brokerFile, jwtFile, server := os.Getenv("ONLY_OFFICE_BROKER"), os.Getenv("ONLY_OFFICE_JWT"), os.Getenv("ONLY_OFFICE_SERVER")
if brokerFile == "" || jwtFile == "" || server == "" {
fmt.Fprintln(os.Stderr, "[only-office] converter was started without ONLY_OFFICE_BROKER, ONLY_OFFICE_JWT or ONLY_OFFICE_SERVER")
return 2
}
account, err := process.ReadAccount(brokerFile)
if err != nil {
fmt.Fprintf(os.Stderr, "[only-office] converter: %v\n", err)
return 1
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
w := &converterWorker{JWT: readSecret(jwtFile), Server: strings.TrimRight(server, "/"),
Docs: &http.Client{Timeout: ConvertRun}, Store: &http.Client{Timeout: ConvertRun},
Now: time.Now, NewKey: newKey, Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, f+"\n", a...) }}
fmt.Fprintf(os.Stderr, "[only-office] converter is consuming %s as %s\n", AsksStream, account.User)
if err := runConverter(ctx, account, w); err != nil && ctx.Err() == nil {
fmt.Fprintf(os.Stderr, "[only-office] converter stopped: %v\n", err)
return 1
}
return 0
}
// runConverter connects as the converter's own account and consumes convert-asks until ctx ends, answering each ask in
// the state convert-answers. Apart from converter so a test can drive it over a bus of its own.
func runConverter(ctx context.Context, account process.Account, w *converterWorker) error {
conn, err := process.Connect(account)
if err != nil {
return fmt.Errorf("the bus cannot be reached: %w", err)
}
defer conn.Close()
w.Answers = conn.State(AnswersState)
return conn.Consume(ctx, AsksStream, func(msg *process.Message) {
act, delay := w.Handle(msg.Body)
var err error
switch act {
case actionAck:
err = msg.Ack()
case actionRetry:
err = msg.NakWithDelay(delay)
case actionTerm:
err = msg.Term()
}
if err != nil {
w.logf("[only-office] converter: settling %s: %v", msg.Subject, err)
}
})
}
modules/only-office/cmd/only-office/converter_integration_test.go
+98
View File
@@ -0,0 +1,98 @@
//go:build integration
package main
// 1596/AC6: the tool side and the converter over a real bus (the shape of brother's device_integration_test.go,
// ADR 0436 §7): a JetStream server of the test's own, the work-queue stream convert-asks and its consumer, the state
// convert-answers, and a fake document server. The tool side's Convert queues an ask; the converter takes it and
// answers; Convert returns the answer. Compiled by the merge gate's `go vet -tags integration`; never run on this machine.
import (
"context"
"encoding/json"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
)
// convertAskedSubject is the wire subject the controller composes for the stream's one event:
// "mesh.stream." + module + ".event." + event (mesh-controller internal/broker/modstreams.go).
const convertAskedSubject = "mesh.stream.only-office.event." + EventAsked
// natsBus is the tool side's bus: the ask published on the stream, the answer read from the state's bucket.
type natsBus struct {
js jetstream.JetStream
kv jetstream.KeyValue
}
func (b natsBus) Publish(id string, body any) error {
raw, _ := json.Marshal(body)
_, err := b.js.Publish(context.Background(), convertAskedSubject, raw, jetstream.WithMsgID(id))
return err
}
func (b natsBus) Answer(id string) (json.RawMessage, error) {
e, err := b.kv.Get(context.Background(), id)
if err != nil {
return nil, nil
}
return e.Value(), nil
}
func TestTheToolSideAndTheConverterConvertOverARealBus(t *testing.T) {
opts := &server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true}
s, err := server.NewServer(opts)
if err != nil {
t.Fatal(err)
}
go s.Start()
if !s.ReadyForConnections(30 * time.Second) {
t.Fatal("a bus for this test did not come up within 30s")
}
t.Cleanup(func() { s.Shutdown(); s.WaitForShutdown() })
ctx := context.Background()
nc, err := nats.Connect(s.ClientURL())
if err != nil {
t.Fatal(err)
}
defer nc.Close()
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: AsksStream, Subjects: []string{convertAskedSubject},
Retention: jetstream.WorkQueuePolicy, Discard: jetstream.DiscardNew, MaxAge: 24 * time.Hour, MaxBytes: 8 << 20}); err != nil {
t.Fatal(err)
}
account := process.Account{URL: s.ClientURL(), Node: "novox", Module: "only-office", User: "novox.only-office.converter", Password: "pw"}
if _, err := js.CreateConsumer(ctx, AsksStream, jetstream.ConsumerConfig{Durable: account.QueueConsumerName(),
AckPolicy: jetstream.AckExplicitPolicy, AckWait: 30 * time.Second, MaxDeliver: 3}); err != nil {
t.Fatal(err)
}
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: account.Module + "_" + AnswersState, TTL: 10 * time.Minute})
if err != nil {
t.Fatal(err)
}
d, tgt := newFakeDocs(t), newFakeTarget(t)
w, _ := worker(d)
runCtx, cancel := context.WithCancel(ctx)
defer cancel()
go runConverter(runCtx, account, w)
svc := &Service{Asks: natsBus{js: js, kv: kv}, Now: time.Now, Wait: 20 * time.Second, Poll: 50 * time.Millisecond}
got, err := svc.Convert(ctx, "http://novox.internal:20001/b/in.docx?X-Amz-Signature=a", "docx", tgt.srv.URL+"/b/in.pdf?sig=1", "")
if err != nil {
t.Fatal(err)
}
if got["size"] != float64(len(d.pdf)) || string(tgt.got) != string(d.pdf) {
t.Fatalf("answer %v, target got %q", got, tgt.got)
}
if len(d.requests) != 1 {
t.Fatalf("%d conversion requests", len(d.requests))
}
}
modules/only-office/cmd/only-office/converter_test.go
+273
View File
@@ -0,0 +1,273 @@
package main
import (
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"os"
"strings"
"sync"
"testing"
"time"
)
const jwtSecret = "module-jwt-secret"
// fakeDocs stands for the document server's conversion service (the request and answer shapes are the conversion API's:
// https://api.onlyoffice.com/docs/docs-api/additional-api/conversion-api/request/ and .../error-codes/, read
// 2026-10-21) and the file it serves back.
type fakeDocs struct {
mu sync.Mutex
requests []map[string]any
accept string
answer string // the JSON the service answers; {fileUrl} is replaced by the file's address
pdf []byte
srv *httptest.Server
}
func newFakeDocs(t *testing.T) *fakeDocs {
d := &fakeDocs{pdf: []byte("%PDF-1.7 converted"), answer: `{"endConvert": true, "fileUrl": "{fileUrl}", "percent": 100}`}
d.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
d.mu.Lock()
defer d.mu.Unlock()
switch r.URL.Path {
case ConvertPath:
var m map[string]any
json.NewDecoder(r.Body).Decode(&m)
d.requests = append(d.requests, m)
d.accept = r.Header.Get("Accept")
io.WriteString(w, strings.ReplaceAll(d.answer, "{fileUrl}", "http://"+r.Host+"/cache/out.pdf"))
case "/cache/out.pdf":
w.Write(d.pdf)
default:
w.WriteHeader(404)
}
}))
t.Cleanup(d.srv.Close)
return d
}
type fakeTarget struct {
mu sync.Mutex
got []byte
puts int
status int
srv *httptest.Server
}
func newFakeTarget(t *testing.T) *fakeTarget {
f := &fakeTarget{status: 200}
f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
f.mu.Lock()
defer f.mu.Unlock()
if f.status != 200 {
w.WriteHeader(f.status)
io.WriteString(w, `<Error><Code>AccessDenied</Code><Message>Access Denied.</Message></Error>`)
return
}
f.got, _ = io.ReadAll(r.Body)
f.puts++
}))
t.Cleanup(f.srv.Close)
return f
}
type answers struct {
mu sync.Mutex
m map[string]map[string]any
}
func (a *answers) Create(key string, value any) (uint64, error) {
a.mu.Lock()
defer a.mu.Unlock()
raw, _ := json.Marshal(value)
var v map[string]any
json.Unmarshal(raw, &v)
a.m[key] = v
return 1, nil
}
func worker(d *fakeDocs) (*converterWorker, *answers) {
a := &answers{m: map[string]map[string]any{}}
return &converterWorker{JWT: func() (string, error) { return jwtSecret, nil }, Server: d.srv.URL,
Docs: http.DefaultClient, Store: http.DefaultClient, Answers: a, Now: time.Now,
NewKey: func() string { return "fresh-key" }}, a
}
func askBody(t *testing.T, target string, sha string) []byte {
raw, err := json.Marshal(Ask{ID: "ask1", Op: OpConvert, At: time.Now(), Deadline: time.Now().Add(time.Minute),
Source: "http://novox.internal:20001/b/in.docx?X-Amz-Signature=a", Type: "docx", Target: target, SHA256: sha})
if err != nil {
t.Fatal(err)
}
return raw
}
// verifies a JWT the way a document server does: three parts, HS256 over the first two with the secret.
func verifiesJWT(t *testing.T, token string) map[string]any {
t.Helper()
parts := strings.Split(token, ".")
if len(parts) != 3 {
t.Fatalf("token %q", token)
}
mac := hmac.New(sha256.New, []byte(jwtSecret))
mac.Write([]byte(parts[0] + "." + parts[1]))
if base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) != parts[2] {
t.Fatal("the token does not verify with the module's secret")
}
payload, err := base64.RawURLEncoding.DecodeString(parts[1])
if err != nil {
t.Fatal(err)
}
var m map[string]any
json.Unmarshal(payload, &m)
return m
}
// 1596/AC1: the server receives one conversion request with the source link, the type, outputtype pdf and a JWT that
// verifies with the module's secret, and the PDF it returns is PUT to target.
func TestTheConverterPostsTheRequestAndPutsThePDF(t *testing.T) {
d, tgt := newFakeDocs(t), newFakeTarget(t)
w, _ := worker(d)
act, _ := w.Handle(askBody(t, tgt.srv.URL+"/b/in.pdf?sig=1", strings.Repeat("ab", 32)))
if act != actionAck {
t.Fatalf("action %v", act)
}
if len(d.requests) != 1 {
t.Fatalf("%d conversion requests", len(d.requests))
}
r := d.requests[0]
if r["url"] != "http://novox.internal:20001/b/in.docx?X-Amz-Signature=a" || r["filetype"] != "docx" || r["outputtype"] != "pdf" ||
r["async"] != false || r["key"] != strings.Repeat("ab", 32) || d.accept != "application/json" {
t.Fatalf("request %v accept %q", r, d.accept)
}
claims := verifiesJWT(t, r["token"].(string))
for _, k := range []string{"url", "filetype", "outputtype", "key"} {
if claims[k] != r[k] {
t.Errorf("the token's %s is %v, the body's %v", k, claims[k], r[k])
}
}
if string(tgt.got) != string(d.pdf) || tgt.puts != 1 {
t.Fatalf("the target got %q in %d puts", tgt.got, tgt.puts)
}
}
// 1596/AC1: without a known SHA-256 the key is a fresh one.
func TestTheKeyIsFreshWhenTheSourceHashIsNotKnown(t *testing.T) {
d, tgt := newFakeDocs(t), newFakeTarget(t)
w, _ := worker(d)
w.Handle(askBody(t, tgt.srv.URL+"/x", ""))
if d.requests[0]["key"] != "fresh-key" {
t.Fatalf("key %v", d.requests[0]["key"])
}
}
// 1596/AC2: the answer under the ask id carries the PUT body's size and SHA-256.
func TestTheAnswerCarriesTheSizeAndHashOfWhatWasPut(t *testing.T) {
d, tgt := newFakeDocs(t), newFakeTarget(t)
w, a := worker(d)
w.Handle(askBody(t, tgt.srv.URL+"/x", ""))
sum := sha256.Sum256(d.pdf)
got := a.m["ask1"]
if got["size"] != float64(len(d.pdf)) || got["sha256"] != hex.EncodeToString(sum[:]) || got["error"] != nil {
t.Fatalf("answer %v", got)
}
}
// 1596/AC3: a server answer with error -4 is answered with the code and its meaning, and nothing is PUT.
func TestAServerErrorIsAnsweredWithItsMeaningAndNothingIsPut(t *testing.T) {
d, tgt := newFakeDocs(t), newFakeTarget(t)
d.answer = `{"error": -4}`
w, a := worker(d)
w.Handle(askBody(t, tgt.srv.URL+"/x", ""))
msg, _ := a.m["ask1"]["error"].(string)
if !strings.Contains(msg, "-4: error while downloading the document file to be converted") {
t.Fatalf("answer %v", a.m["ask1"])
}
if tgt.puts != 0 {
t.Fatal("something was put")
}
if got := meaningOf(-8); got != "-8: invalid token" {
t.Fatalf("-8 reads %q", got)
}
}
// 1596/AC4: a failed PUT is answered as a failure naming the store's refusal; the ask is not answered as done.
func TestAFailedPutIsAFailureNamingTheStoresRefusal(t *testing.T) {
d, tgt := newFakeDocs(t), newFakeTarget(t)
tgt.status = 403
w, a := worker(d)
w.Handle(askBody(t, tgt.srv.URL+"/x", ""))
got := a.m["ask1"]
msg, _ := got["error"].(string)
if !strings.Contains(msg, "403") || !strings.Contains(msg, "AccessDenied") || got["size"] != nil || got["sha256"] != nil {
t.Fatalf("answer %v", got)
}
}
// An ask whose deadline passed is not converted, and an unreadable ask is ended.
func TestALateAskIsNotConverted(t *testing.T) {
d, tgt := newFakeDocs(t), newFakeTarget(t)
w, a := worker(d)
w.Now = func() time.Time { return time.Now().Add(2 * time.Minute) }
if act, _ := w.Handle(askBody(t, tgt.srv.URL+"/x", "")); act != actionTerm {
t.Fatalf("action %v", act)
}
if len(d.requests) != 0 || a.m["ask1"]["error"] == nil {
t.Fatalf("requests %d, answer %v", len(d.requests), a.m["ask1"])
}
if act, _ := w.Handle([]byte("nonsense")); act != actionTerm {
t.Fatal("an unreadable ask was kept")
}
}
// 1596/AC5: the process converter has no serves, is the one consumer of convert-asks and writes convert-answers, and
// its account owns the JWT file.
func TestManifestDeclaresTheConverterProcess(t *testing.T) {
m := manifest(t)
var proc, account map[string]any
for _, r := range m["resources"].([]any) {
res := r.(map[string]any)
if res["type"] == "process" && res["name"] == "only-office-converter" {
proc = res
}
if res["type"] == "user" && res["name"] == "only-office-converter" {
account = res
}
}
if proc == nil || account == nil {
t.Fatalf("process %v account %v", proc, account)
}
if _, serves := proc["serves"]; serves {
t.Error("the converter serves")
}
bus := proc["bus"].(map[string]any)
if q := bus["queues"].([]any); len(q) != 1 || q[0] != "convert-asks" {
t.Errorf("queues %v", q)
}
if s := bus["state"].([]any); len(s) != 1 || s[0] != AnswersState {
t.Errorf("state %v", s)
}
if proc["user"] != "only-office-converter" || account["shell"] != "/usr/bin/nologin" {
t.Errorf("user %v shell %v", proc["user"], account["shell"])
}
jwt := m["own-secrets"].(map[string]any)["jwt"].(map[string]any)
if jwt["owner"] != "only-office-converter" || jwt["path"] != "${dir:state}/jwt.secret" {
t.Errorf("jwt %v", jwt)
}
broker := m["own-secrets"].(map[string]any)["broker-converter"].(map[string]any)
env := proc["env"].(map[string]any)
if broker["owner"] != "only-office-converter" || env["ONLY_OFFICE_BROKER"] != broker["path"] || env["ONLY_OFFICE_JWT"] != jwt["path"] {
t.Errorf("broker %v env %v", broker, env)
}
// The container still gets the token: server.env names it.
raw, _ := os.ReadFile("../../files/server.env")
if !strings.Contains(string(raw), "JWT_SECRET=${secret:jwt}") {
t.Error("the document server is no longer given the token")
}
}
modules/only-office/cmd/only-office/main.go
+24
View File
@@ -0,0 +1,24 @@
// only-office's Go part (novox/hq ADR 0468 §7): the tool side of the seat `document-converter`. A caller asks to
// have a document turned into a PDF, by a signed link to read it from and a signed link to put the PDF to; the
// tool side queues the ask for the module's `converter` process (issue 1596) and reads its answer. It holds no
// credential: the document server's token is the process's alone (ADR 0436 §12).
//
// stdout is the MCP channel; what this module says, it says on stderr.
package main
import (
"fmt"
"os"
stdio "git.novox.be/novox/mesh-sdk/go"
)
func main() {
if len(os.Args) > 1 && os.Args[1] == "converter" {
os.Exit(converter())
}
if err := stdio.Serve("", tools(newService())); err != nil {
fmt.Fprintf(os.Stderr, "[only-office] %v\n", err)
os.Exit(1)
}
}
modules/only-office/go.mod
+22
View File
@@ -0,0 +1,22 @@
module only-office
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
require (
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/klauspost/compress v1.20.0 // indirect
github.com/minio/highwayhash v1.0.4 // indirect
github.com/nats-io/jwt/v2 v2.8.1 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
modules/only-office/go.sum
+27
View File
@@ -0,0 +1,27 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f h1:7J/9jOfSfjfYpf8MOj96HwQRViQUkBNMb71RHP2mjn8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
modules/only-office/module.json
+90 -2
View File
@@ -14,11 +14,55 @@
"endpoint": "web"
}
},
"seats": [
{
"name": "document-converter",
"scope": "mesh",
"serves": [
"convert"
]
}
],
"claims": [
{
"name": "document-converter",
"scope": "mesh",
"serves": [
"convert"
]
}
],
"streams": [
{
"name": "convert-asks",
"events": [
"convert.asked"
],
"max-age-days": 1,
"max-mib": 8,
"retention": "work-queue",
"max-deliveries": 3,
"duplicates": 24
}
],
"state": [
{
"name": "convert-answers",
"ttl-seconds": 600
}
],
"binds": {
"route": "${dir:state}/route.json"
},
"own-secrets": {
"jwt": "${dir:state}/jwt.secret"
"jwt": {
"path": "${dir:state}/jwt.secret",
"owner": "only-office-converter"
},
"broker-converter": {
"path": "${dir:state}/broker-converter.secret",
"owner": "only-office-converter"
}
},
"listens": [
{
@@ -79,10 +123,16 @@
{
"id": "state",
"type": "directory",
"mode": "0700",
"mode": "0711",
"place": ".",
"unknown": "program"
},
{
"id": "converter-account",
"type": "user",
"name": "only-office-converter",
"shell": "/usr/bin/nologin"
},
{
"id": "server-env",
"type": "file",
@@ -159,10 +209,48 @@
"${dir:fonts}:/usr/share/fonts/truetype/custom"
],
"secrets-in-environment": "run-document-server.sh regenerates JWT_SECRET from the environment on every start and overwrites local.json; not convertible"
},
{
"id": "converter",
"type": "process",
"name": "only-office-converter",
"artifact": "tools",
"run": [
"./only-office",
"converter"
],
"user": "only-office-converter",
"env": {
"ONLY_OFFICE_BROKER": "${dir:state}/broker-converter.secret",
"ONLY_OFFICE_JWT": "${dir:state}/jwt.secret",
"ONLY_OFFICE_SERVER": "http://127.0.0.1:9070"
},
"bus": {
"queues": [
"convert-asks"
],
"state": [
"convert-answers"
]
},
"health": {
"kind": "unit"
}
}
],
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/only-office",
"binary": "only-office",
"loads": [
"only-office"
]
},
{
"name": "files",
"kind": "archive",