Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 mesh-admin
|
86bcbc48ad |
+2
-8
@@ -8,22 +8,16 @@ modules/messenger/cmd/messenger/messenger
|
||||
modules/mesh-watcher/cmd/mesh-watcher/mesh-watcher
|
||||
modules/claude-code/cmd/claude-code/claude-code
|
||||
modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq-tools
|
||||
modules/mesh-delivery/cmd/mesh-delivery/mesh-delivery
|
||||
# ...and the same built from the module's root (go build ./cmd/<name>), which names it after the module.
|
||||
modules/messenger/messenger
|
||||
modules/claude-code/claude-code
|
||||
modules/mesh-watcher/mesh-watcher
|
||||
modules/dnsmasq/dnsmasq
|
||||
modules/mesh-delivery/mesh-delivery
|
||||
modules/mesh-issues/cmd/mesh-issues/mesh-issues
|
||||
modules/mounts/cmd/mounts/mounts
|
||||
modules/mesh-knowledge/cmd/mesh-knowledge/mesh-knowledge
|
||||
modules/mesh-knowledge/mesh-knowledge
|
||||
modules/mesh-reads/mesh-reads
|
||||
modules/mesh-reads/cmd/mesh-reads/mesh-reads
|
||||
modules/mesh-files/mesh-files
|
||||
modules/mesh-files/cmd/mesh-files/mesh-files
|
||||
|
||||
# The provisioner client built in place (issue 671).
|
||||
modules/mesh-provisioner-client/cmd/mesh-provisioner-client/mesh-provisioner-client
|
||||
modules/brother/cmd/brother/brother
|
||||
modules/mesh-build-go/cmd/mesh-build-go/mesh-build-go
|
||||
modules/mesh-config/mesh-config
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
# Release 57: config and provisioning
|
||||
|
||||
This branch carries the work of hq design 57 (ADR 0348, ADR 0349) in this repository. Each package
|
||||
merges here through its own reviewed pull request; the branch reaches main as one pull request, in one
|
||||
delivery group with the branch of the same name in mesh-controller, mesh-host, mesh-sdk, mesh-catalog
|
||||
and hq, after a rehearsal in the lab. This file is deleted by the last commit before that pull request.
|
||||
@@ -1,3 +0,0 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/config
|
||||
|
||||
go 1.22
|
||||
@@ -1,201 +0,0 @@
|
||||
// Command config refuses the settings form's names in a module that has moved to config (novox/hq ADR 0348
|
||||
// clauses 1 and 11, design 57 package K6, issue 665).
|
||||
//
|
||||
// "Config" replaces "settings": a manifest declares its values under `config`, a mergeable file names the keys it
|
||||
// takes under `takes-config`, and a file asks for a value with `${config:<key>}`. A mergeable file naming no keys at
|
||||
// all is the settings form's last shape: it takes every key of its module's layer and declares none, so mesh-config
|
||||
// could set none of them (ADR 0348 §10, ADR 0326 §3); it is found here like an old name (issue 735). The old names — `settings`,
|
||||
// `takes-settings` and `${setting:<key>}` — stay accepted by the controller as aliases until every module has
|
||||
// moved, and are then refused (ADR 0348 §1). The modules not yet moved are listed in the file `unmoved` beside this
|
||||
// check, one per line: the catalogue's list ADR 0348 §11 names. Every other module says no old name, in its
|
||||
// manifest or in a template file its manifest names; a module on the list that says none is taken off it; and a
|
||||
// line the trunk's copy (the merge base's, given with -base) does not hold is refused, so the list only shrinks;
|
||||
// while the trunk holds no copy, the modules in seed are the trunk's.
|
||||
// When it is empty, the controller refuses the old names itself (design 57 package R1).
|
||||
//
|
||||
// Prose is not judged: a comment in a file's text that says "settings" composes into the machine's file, and
|
||||
// rewording it would change what a node is sent.
|
||||
//
|
||||
// go run . [-base <the trunk's unmoved>] <catalogue root>
|
||||
//
|
||||
// Exits 1 on a finding.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The settings form's names (ADR 0348 §1). A key of a JSON object is a quoted string followed by a colon; a
|
||||
// string value is never followed by one.
|
||||
var (
|
||||
settingRef = regexp.MustCompile(`\$\{setting:[a-z0-9][a-z0-9_.-]*\}`)
|
||||
takesSettingsKey = regexp.MustCompile(`"takes-settings"\s*:`)
|
||||
oldPlaceholder = "${setting:…}"
|
||||
oldTakes = "takes-settings"
|
||||
oldDeclaration = "settings"
|
||||
wholeLayer = "taking its whole layer"
|
||||
listFile = "unmoved"
|
||||
listPathInCatalog = "checks/config/" + listFile
|
||||
)
|
||||
|
||||
// Check answers, per module that says any, the settings form's names its manifest and its template files say.
|
||||
func Check(root string) (map[string][]string, error) {
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(manifests)
|
||||
out := map[string][]string{}
|
||||
for _, path := range manifests {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Settings json.RawMessage `json:"settings"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
var said []string
|
||||
if len(m.Settings) > 0 && string(m.Settings) != "null" {
|
||||
said = append(said, oldDeclaration)
|
||||
}
|
||||
if takesSettingsKey.Match(raw) {
|
||||
said = append(said, oldTakes)
|
||||
}
|
||||
if settingRef.Match(raw) {
|
||||
said = append(said, oldPlaceholder)
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
// A mergeable file naming no keys takes every key of the layer (ADR 0326 §3), declaring none of them.
|
||||
if how, _ := r["merge"].(string); how != "" {
|
||||
_, oldList := r["takes-settings"]
|
||||
_, newList := r["takes-config"]
|
||||
if !oldList && !newList {
|
||||
said = append(said, fmt.Sprintf("%v %s", r["id"], wholeLayer))
|
||||
}
|
||||
}
|
||||
name, _ := r["template"].(string)
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
text, err := os.ReadFile(filepath.Join(filepath.Dir(path), filepath.FromSlash(name)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s names the template %s: %w", m.Module, name, err)
|
||||
}
|
||||
if settingRef.Match(text) {
|
||||
said = append(said, oldPlaceholder+" in "+name)
|
||||
}
|
||||
}
|
||||
if len(said) > 0 {
|
||||
sort.Strings(said)
|
||||
out[m.Module] = said
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// seed is the list as it was first written, standing for the trunk's copy until the trunk holds one.
|
||||
// The modules taking a whole layer were found after the list was first written and before the trunk held a copy
|
||||
// (issue 735), so they stand in it too.
|
||||
var seed = map[string]bool{
|
||||
"betterbird": true, "fail2ban": true, "influxdb": true, "keycloak": true, "mailu": true, "searxng": true,
|
||||
"baserow": true, "confluence": true, "gitea": true, "gitlab": true, "grafana": true, "home-assistant": true,
|
||||
"icecast": true, "jira": true, "letta": true, "matrix": true, "nodered": true, "supabase": true, "unifi": true,
|
||||
}
|
||||
|
||||
// Unmoved is the modules a list names, one per line; a line starting with # and a blank line are not modules.
|
||||
func Unmoved(path string) (map[string]bool, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
out := map[string]bool{}
|
||||
s := bufio.NewScanner(f)
|
||||
for s.Scan() {
|
||||
line := strings.TrimSpace(s.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
out[line] = true
|
||||
}
|
||||
return out, s.Err()
|
||||
}
|
||||
|
||||
// Judge answers the modules that say an old name and are not on the list (refused), the modules on the list that
|
||||
// say none (stale), and the modules on the list that the trunk's list does not hold (added).
|
||||
func Judge(found map[string][]string, unmoved, base map[string]bool) (refused, stale, added []string) {
|
||||
for m := range found {
|
||||
if !unmoved[m] {
|
||||
refused = append(refused, m)
|
||||
}
|
||||
}
|
||||
for m := range unmoved {
|
||||
if _, says := found[m]; !says {
|
||||
stale = append(stale, m)
|
||||
}
|
||||
if !base[m] {
|
||||
added = append(added, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(refused)
|
||||
sort.Strings(stale)
|
||||
sort.Strings(added)
|
||||
return refused, stale, added
|
||||
}
|
||||
|
||||
func main() {
|
||||
basePath := flag.String("base", "", "the trunk's copy of unmoved (the merge base's); seed when absent")
|
||||
flag.Parse()
|
||||
root := "."
|
||||
if flag.NArg() > 0 {
|
||||
root = flag.Arg(0)
|
||||
}
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
unmoved, err := Unmoved(listFile)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
base := seed
|
||||
if *basePath != "" {
|
||||
if base, err = Unmoved(*basePath); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
refused, stale, added := Judge(found, unmoved, base)
|
||||
for _, m := range refused {
|
||||
fmt.Printf("%s says %s, the settings form's names, and is not on %s: write config, takes-config and "+
|
||||
"${config:<key>} throughout, and name the keys each mergeable file takes (novox/hq ADR 0348 §1, §11)\n", m, strings.Join(found[m], ", "), listPathInCatalog)
|
||||
}
|
||||
for _, m := range stale {
|
||||
fmt.Printf("%s names %s, which says no old name: take it off the list\n", listPathInCatalog, m)
|
||||
}
|
||||
for _, m := range added {
|
||||
fmt.Printf("%s gains %s, which the trunk's copy does not hold: the list only shrinks — move the module to "+
|
||||
"config instead (novox/hq ADR 0348 §11)\n", listPathInCatalog, m)
|
||||
}
|
||||
if len(refused) > 0 || len(stale) > 0 || len(added) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Printf("config: no moved module says settings, takes-settings or ${setting:<key>}, or takes a whole layer "+
|
||||
"(%d not yet moved)\n",
|
||||
len(unmoved))
|
||||
}
|
||||
@@ -1,131 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func module(t *testing.T, root, name, body string, files map[string]string) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(root, "modules", name)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for path, text := range files {
|
||||
full := filepath.Join(dir, filepath.FromSlash(path))
|
||||
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(full, []byte(text), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Each of the settings form's names is found where it stands: the declaration, a mergeable file's keys, a
|
||||
// placeholder in the manifest, and a placeholder in a template file the manifest names (novox/hq ADR 0348 §1).
|
||||
func TestEveryOldNameIsFound(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
module(t, root, "declares", `{"module":"declares","settings":{"size":{"kind":"preference","default":1,"why":"w"}},`+
|
||||
`"resources":[{"id":"f","type":"file","content":"${setting:size}"}]}`, nil)
|
||||
module(t, root, "takes", `{"module":"takes","resources":[{"id":"f","type":"file","merge":"json",`+
|
||||
`"takes-settings":["a"],"content":"{}"}]}`, nil)
|
||||
module(t, root, "asks", `{"module":"asks","serves":{"x":{"d":"${setting:domain}"}}}`, nil)
|
||||
module(t, root, "templated", `{"module":"templated","config":{"size":{"type":"number","description":"d",`+
|
||||
`"default":1}},"resources":[{"id":"f","type":"file","template":"files/conf"}]}`,
|
||||
map[string]string{"files/conf": "size = ${setting:size}\n"})
|
||||
module(t, root, "moved", `{"module":"moved","config":{"size":{"type":"number","description":"d","default":1}},`+
|
||||
`"resources":[{"id":"f","type":"file","template":"files/conf"},{"id":"g","type":"file","merge":"json",`+
|
||||
`"takes-config":[],"content":"{\"settings\": 1}"}]}`,
|
||||
map[string]string{"files/conf": "size = ${config:size}\n# change the settings, never this file\n"})
|
||||
module(t, root, "none", `{"module":"none","resources":[{"id":"f","type":"file","content":"no settings here"}]}`, nil)
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string][]string{
|
||||
"asks": {"${setting:…}"},
|
||||
"declares": {"${setting:…}", "settings"},
|
||||
"takes": {"takes-settings"},
|
||||
"templated": {"${setting:…} in files/conf"},
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("found %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A module not on the list says no old name; one on the list that says none is taken off it; a line the trunk's
|
||||
// list does not hold is refused, so the list only shrinks until it is empty (novox/hq ADR 0348 §11).
|
||||
func TestTheListOfModulesNotYetMovedOnlyShrinks(t *testing.T) {
|
||||
found := map[string][]string{"old": {"settings"}, "forgotten": {"takes-settings"}}
|
||||
unmoved := map[string]bool{"old": true, "done": true, "new": true}
|
||||
base := map[string]bool{"old": true, "done": true, "forgotten": true}
|
||||
refused, stale, added := Judge(found, unmoved, base)
|
||||
if !reflect.DeepEqual(refused, []string{"forgotten"}) || !reflect.DeepEqual(stale, []string{"done", "new"}) ||
|
||||
!reflect.DeepEqual(added, []string{"new"}) {
|
||||
t.Fatalf("refused %v, stale %v, added %v", refused, stale, added)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheListIsReadWithoutItsComments(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "unmoved")
|
||||
if err := os.WriteFile(path, []byte("# why\n\nmailu\n searxng \n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := Unmoved(path)
|
||||
if err != nil || !reflect.DeepEqual(got, map[string]bool{"mailu": true, "searxng": true}) {
|
||||
t.Fatalf("%v %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue itself: every moved module says config alone, and the list names exactly the modules that
|
||||
// still say settings.
|
||||
func TestTheCatalogueSaysConfigInEveryMovedModule(t *testing.T) {
|
||||
found, err := Check(filepath.Join("..", ".."))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unmoved, err := Unmoved("unmoved")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
refused, stale, added := Judge(found, unmoved, seed)
|
||||
for _, m := range refused {
|
||||
t.Errorf("%s is not on checks/config/unmoved and says %s", m, strings.Join(found[m], ", "))
|
||||
}
|
||||
for _, m := range stale {
|
||||
t.Errorf("checks/config/unmoved names %s, which says no old name", m)
|
||||
}
|
||||
for _, m := range added {
|
||||
t.Errorf("checks/config/unmoved gains %s, which its first copy did not hold", m)
|
||||
}
|
||||
}
|
||||
|
||||
// A mergeable file that names no keys takes every key of its module's layer (novox/hq ADR 0326 §3): the settings
|
||||
// form's last shape, which declares none of what it takes, so mesh-config could set none of it (ADR 0348 §10). It
|
||||
// is found like an old name, in a module that says no other (issue 735); one naming its keys, an empty list
|
||||
// included, is not.
|
||||
func TestAMergeableFileTakingTheWholeLayerIsFound(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
module(t, root, "whole", `{"module":"whole","resources":[{"id":"runtime-config","type":"file","merge":"json",`+
|
||||
`"template":"files/config.json"}]}`, map[string]string{"files/config.json": "{}\n"})
|
||||
module(t, root, "names", `{"module":"names","config":{"url":{"type":"string","description":"d","default":""}},`+
|
||||
`"resources":[{"id":"runtime-config","type":"file","merge":"json","takes-config":["url"],"content":"{}"}]}`, nil)
|
||||
module(t, root, "none", `{"module":"none","resources":[{"id":"c","type":"file","merge":"json",`+
|
||||
`"takes-config":[],"content":"{}"}]}`, nil)
|
||||
module(t, root, "plain", `{"module":"plain","resources":[{"id":"c","type":"file","content":"{}"}]}`, nil)
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string][]string{"whole": {"runtime-config taking its whole layer"}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("found %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
# The modules still written in the settings form (`settings`, `takes-settings`, `${setting:<key>}`, or a mergeable
|
||||
# file naming no keys, which takes its whole layer), one per line: the catalogue's list of modules not yet moved to
|
||||
# config (novox/hq ADR 0348 §11). Every other module says config alone. This list only shrinks: a module that says no
|
||||
# old name is taken off, and a line the trunk's copy does not hold is refused. When it is empty, the controller
|
||||
# refuses the old names (design 57 package R1).
|
||||
@@ -1,3 +0,0 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/mailstore
|
||||
|
||||
go 1.22
|
||||
@@ -1,81 +0,0 @@
|
||||
// Command mailstore refuses any manifest but modules/mesh-mail claiming the mail-store seat (novox/hq ADR 0378
|
||||
// §3, issue 800 part A): the operator's mail is read, searched and sent through this one seat, and a second
|
||||
// claimant would be a second path to the same mail that ADR 0378 never accounted for.
|
||||
//
|
||||
// go run . <catalogue root>
|
||||
//
|
||||
// Exits 1 on a finding.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// MailStoreSeat is the seat only mesh-mail may claim.
|
||||
const MailStoreSeat = "mail-store"
|
||||
|
||||
// MailStoreModule is the one module allowed to claim it.
|
||||
const MailStoreModule = "mesh-mail"
|
||||
|
||||
// Finding is one manifest refused: a module other than mesh-mail claiming mail-store.
|
||||
type Finding struct {
|
||||
Module string
|
||||
}
|
||||
|
||||
// Check reads every manifest under root/modules and answers which ones it refuses.
|
||||
func Check(root string) ([]Finding, error) {
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(manifests)
|
||||
var out []Finding
|
||||
for _, path := range manifests {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Claims []struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"claims"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
if m.Module == MailStoreModule {
|
||||
continue
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
if c.Name == MailStoreSeat {
|
||||
out = append(out, Finding{Module: m.Module})
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
root := "."
|
||||
if len(os.Args) > 1 {
|
||||
root = os.Args[1]
|
||||
}
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
for _, f := range found {
|
||||
fmt.Printf("%s claims the mail-store seat: only %s may (novox/hq ADR 0378 §3)\n", f.Module, MailStoreModule)
|
||||
}
|
||||
if len(found) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("mail-store: claimed only by mesh-mail")
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func manifest(t *testing.T, root, module, claims string) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(root, "modules", module)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := `{"module":"` + module + `","claims":[` + claims + `]}`
|
||||
if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnotherModuleClaimingMailStoreIsRefused(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
manifest(t, root, "mesh-mail", `{"name":"mail-store","scope":"mesh"}`)
|
||||
manifest(t, root, "betterbird", `{"name":"mail-store","scope":"mesh"}`)
|
||||
manifest(t, root, "dunst", `{"name":"node-notifier","scope":"node"}`)
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 1 || found[0].Module != "betterbird" {
|
||||
t.Fatalf("refused %+v", found)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheCatalogueHasNone(t *testing.T) {
|
||||
found, err := Check("../..")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 0 {
|
||||
t.Fatalf("%+v", found)
|
||||
}
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/owned
|
||||
|
||||
go 1.22
|
||||
@@ -1,242 +0,0 @@
|
||||
// Command owned judges how the catalogue's directories and archives treat files they did not write (novox/hq
|
||||
// ADR 0422 §8, issue 1024). It refuses, naming the module, the resource and the reason:
|
||||
//
|
||||
// 1. a directory or archive without `unknown`, or with a value but move-out, module or program;
|
||||
//
|
||||
// 2. `move-out` on a path not under ${machine:account-home} or ${machine:agent-home};
|
||||
//
|
||||
// 3. `knows` with an empty, absolute, `.`/`..`/empty-component or repeated path;
|
||||
//
|
||||
// 4. a path one module declares (a resource's path, or a fact with "home": true) lying inside a directory of
|
||||
// another module, after ${dir:<id>} is resolved within each module and both homes are read as one root.
|
||||
// Archives are not containers for this rule. This holds for ~/.ssh what checks/sshdir held (ADR 0415 §6).
|
||||
//
|
||||
// go run . <catalogue root>
|
||||
//
|
||||
// Exits 1 on a finding.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Home is the one root both machine homes are read as.
|
||||
const Home = "${home}"
|
||||
|
||||
var (
|
||||
ofDir = regexp.MustCompile(`\$\{dir:([a-z0-9][a-z0-9_-]*)\}`)
|
||||
homeVar = strings.NewReplacer("${machine:account-home}", Home, "${machine:agent-home}", Home)
|
||||
values = map[string]bool{"move-out": true, "module": true, "program": true}
|
||||
)
|
||||
|
||||
// Finding is one refusal.
|
||||
type Finding struct {
|
||||
Rule int
|
||||
Module, ID, Why string
|
||||
}
|
||||
|
||||
type resource struct {
|
||||
ID, Type, Path, Unknown string
|
||||
Knows []string
|
||||
HasKnows bool
|
||||
}
|
||||
|
||||
type parsed struct {
|
||||
Module string
|
||||
Resources []resource
|
||||
Facts []string // paths of facts with "home": true, under the home
|
||||
}
|
||||
|
||||
func load(file string) (*parsed, error) {
|
||||
raw, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Resources []struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Path string `json:"path"`
|
||||
Unknown *string `json:"unknown"`
|
||||
Knows []string `json:"knows"`
|
||||
} `json:"resources"`
|
||||
Facts map[string]struct {
|
||||
Path string `json:"path"`
|
||||
Home bool `json:"home"`
|
||||
} `json:"facts"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", file, err)
|
||||
}
|
||||
out := &parsed{Module: m.Module}
|
||||
if out.Module == "" {
|
||||
out.Module = filepath.Base(filepath.Dir(file))
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
x := resource{ID: r.ID, Type: r.Type, Path: r.Path, Knows: r.Knows, HasKnows: r.Knows != nil}
|
||||
if r.Unknown != nil {
|
||||
x.Unknown = *r.Unknown
|
||||
if x.Unknown == "" {
|
||||
x.Unknown = "(empty)"
|
||||
}
|
||||
}
|
||||
out.Resources = append(out.Resources, x)
|
||||
}
|
||||
keys := make([]string, 0, len(m.Facts))
|
||||
for k := range m.Facts {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if f := m.Facts[k]; f.Home && f.Path != "" {
|
||||
out.Facts = append(out.Facts, f.Path)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// norm resolves ${dir:<id>} from the module's own resources, reads both homes as one root and cleans the path.
|
||||
func norm(p string, dirs map[string]string) string {
|
||||
for i := 0; i < 8 && ofDir.MatchString(p); i++ {
|
||||
p = ofDir.ReplaceAllStringFunc(p, func(w string) string {
|
||||
if to, ok := dirs[ofDir.FindStringSubmatch(w)[1]]; ok {
|
||||
return to
|
||||
}
|
||||
return w
|
||||
})
|
||||
}
|
||||
p = homeVar.Replace(p)
|
||||
if p == "" {
|
||||
return ""
|
||||
}
|
||||
return path.Clean(p)
|
||||
}
|
||||
|
||||
func underHome(p string) bool { return strings.HasPrefix(p, Home+"/") }
|
||||
|
||||
// badKnows says why a knows entry is refused, or "".
|
||||
func badKnows(k string) string {
|
||||
if k == "" {
|
||||
return "is empty"
|
||||
}
|
||||
if strings.HasPrefix(k, "/") {
|
||||
return "is absolute"
|
||||
}
|
||||
for _, c := range strings.Split(strings.TrimSuffix(k, "/"), "/") {
|
||||
if c == "" || c == "." || c == ".." {
|
||||
return "has an empty, . or .. component"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type declared struct {
|
||||
module, id, path string
|
||||
}
|
||||
|
||||
// Check reads every manifest under root/modules and answers what it refuses.
|
||||
func Check(root string) ([]Finding, error) {
|
||||
files, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(files)
|
||||
var out []Finding
|
||||
var paths []declared
|
||||
type container struct{ module, path string }
|
||||
var containers []container
|
||||
for _, file := range files {
|
||||
m, err := load(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dirs := map[string]string{}
|
||||
for _, r := range m.Resources {
|
||||
switch {
|
||||
case r.ID != "" && r.Path != "":
|
||||
dirs[r.ID] = r.Path
|
||||
case r.ID != "" && r.Type == "directory":
|
||||
// A directory the mesh places for the module (`place`): where it lies is the node-engine's, but it is
|
||||
// the module's own, so a path under it belongs to this module and to no other's directory of the same
|
||||
// id (issue 1484: two runs-as modules each keep a .ssh in their own state directory).
|
||||
dirs[r.ID] = "/placed/" + m.Module + "/" + r.ID
|
||||
}
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
p := norm(r.Path, dirs)
|
||||
if p != "" {
|
||||
paths = append(paths, declared{m.Module, r.ID, p})
|
||||
}
|
||||
if r.Type != "directory" && r.Type != "archive" {
|
||||
continue
|
||||
}
|
||||
if r.Type == "directory" && p != "" {
|
||||
containers = append(containers, container{m.Module, p})
|
||||
}
|
||||
add := func(rule int, why string) { out = append(out, Finding{rule, m.Module, r.ID, why}) }
|
||||
switch {
|
||||
case r.Unknown == "":
|
||||
add(1, "a "+r.Type+" without `unknown` (move-out, module or program)")
|
||||
case !values[r.Unknown]:
|
||||
add(1, fmt.Sprintf("`unknown` is %q, not move-out, module or program", r.Unknown))
|
||||
case r.Unknown == "move-out" && !underHome(p):
|
||||
add(2, fmt.Sprintf("`move-out` on %q, which is not under ${machine:account-home} or ${machine:agent-home}", r.Path))
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, k := range r.Knows {
|
||||
if why := badKnows(k); why != "" {
|
||||
add(3, fmt.Sprintf("`knows` entry %q %s", k, why))
|
||||
continue
|
||||
}
|
||||
c := strings.TrimSuffix(k, "/")
|
||||
if seen[c] {
|
||||
add(3, fmt.Sprintf("`knows` entry %q is repeated", k))
|
||||
}
|
||||
seen[c] = true
|
||||
}
|
||||
if r.HasKnows && len(r.Knows) == 0 {
|
||||
add(3, "`knows` is empty")
|
||||
}
|
||||
}
|
||||
for _, f := range m.Facts {
|
||||
if p := norm(Home+"/"+f, nil); p != "" {
|
||||
paths = append(paths, declared{m.Module, "fact " + f, p})
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, d := range paths {
|
||||
for _, c := range containers {
|
||||
if c.module != d.module && (d.path == c.path || strings.HasPrefix(d.path, c.path+"/")) {
|
||||
out = append(out, Finding{4, d.module, d.id, fmt.Sprintf("declares %q, inside a directory of %s (%q)", d.path, c.module, c.path)})
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
root := "."
|
||||
if len(os.Args) > 1 {
|
||||
root = os.Args[1]
|
||||
}
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
for _, f := range found {
|
||||
fmt.Printf("%s: %s: %s (rule %d, novox/hq ADR 0422 §8, issue 1024)\n", f.Module, f.ID, f.Why, f.Rule)
|
||||
}
|
||||
if len(found) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("owned paths: every directory and archive says how it treats unknown files; no module declares inside another's directory")
|
||||
}
|
||||
@@ -1,147 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func manifest(t *testing.T, root, module, body string) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(root, "modules", module)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func check(t *testing.T, root string) []Finding {
|
||||
t.Helper()
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
// only asserts exactly one finding, of the rule, naming the module and the resource (issue 1024).
|
||||
func only(t *testing.T, found []Finding, rule int, module, id string) {
|
||||
t.Helper()
|
||||
if len(found) != 1 || found[0].Rule != rule || found[0].Module != module || found[0].ID != id || found[0].Why == "" {
|
||||
t.Fatalf("found %+v; want one finding of rule %d for %s/%s with a reason", found, rule, module, id)
|
||||
}
|
||||
}
|
||||
|
||||
func dir(id, path, rest string) string {
|
||||
return `{"id":"` + id + `","type":"directory","path":"` + path + `"` + rest + `}`
|
||||
}
|
||||
|
||||
func mod(name string, resources ...string) string {
|
||||
return `{"module":"` + name + `","resources":[` + strings.Join(resources, ",") + `]}`
|
||||
}
|
||||
|
||||
func TestMissingOrUnknownValueIsRefused(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
manifest(t, root, "a", mod("a", dir("d", "/var/lib/a", "")))
|
||||
only(t, check(t, root), 1, "a", "d")
|
||||
|
||||
root = t.TempDir()
|
||||
manifest(t, root, "a", mod("a", `{"id":"z","type":"archive","path":"/opt/a","unknown":"delete"}`))
|
||||
only(t, check(t, root), 1, "a", "z")
|
||||
}
|
||||
|
||||
func TestMoveOutOutsideAHomeIsRefused(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
manifest(t, root, "a", mod("a", dir("d", "/etc/a.d", `,"unknown":"move-out"`)))
|
||||
only(t, check(t, root), 2, "a", "d")
|
||||
}
|
||||
|
||||
func TestMoveOutUnderAHomeIsAllowed(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
manifest(t, root, "a", mod("a",
|
||||
dir("d", "${machine:account-home}/.config/a", `,"unknown":"move-out"`),
|
||||
dir("e", "${dir:d}/sub", `,"unknown":"move-out"`),
|
||||
dir("f", "${machine:agent-home}/x", `,"unknown":"move-out"`),
|
||||
dir("g", "/var/lib/a", `,"unknown":"program"`),
|
||||
dir("h", "${machine:account-home}/.ssh", `,"unknown":"module"`)))
|
||||
if found := check(t, root); len(found) != 0 {
|
||||
t.Fatalf("refused %+v", found)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBadKnowsIsRefused(t *testing.T) {
|
||||
for _, knows := range []string{`[""]`, `["/abs"]`, `["a/../b"]`, `["./a"]`, `["a//b"]`, `["a","a"]`, `["a","a/"]`} {
|
||||
root := t.TempDir()
|
||||
manifest(t, root, "a", mod("a", dir("d", "${machine:account-home}/.config/a", `,"unknown":"move-out","knows":`+knows)))
|
||||
only(t, check(t, root), 3, "a", "d")
|
||||
}
|
||||
root := t.TempDir()
|
||||
manifest(t, root, "a", mod("a", dir("d", "${machine:account-home}/.config/a", `,"unknown":"move-out","knows":["forge.credentials","kept/"]`)))
|
||||
if found := check(t, root); len(found) != 0 {
|
||||
t.Fatalf("refused %+v", found)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnotherModulesPathInsideADirectoryIsRefused(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
manifest(t, root, "owner", mod("owner", dir("ssh", "${machine:account-home}/.ssh", `,"unknown":"module"`)))
|
||||
manifest(t, root, "intruder", mod("intruder", `{"id":"f","type":"file","path":"${machine:account-home}/.ssh/config.d/20"}`))
|
||||
manifest(t, root, "viahome", mod("viahome", `{"id":"g","type":"file","path":"${machine:agent-home}/.ssh/x"}`))
|
||||
manifest(t, root, "facty", `{"module":"facty","facts":{"hosts":{"path":".ssh/config.d/00","home":true}}}`)
|
||||
manifest(t, root, "outside", mod("outside", `{"id":"o","type":"file","path":"${machine:account-home}/.sshrc"}`))
|
||||
got := map[string]int{}
|
||||
for _, f := range check(t, root) {
|
||||
if f.Rule != 4 || f.Why == "" {
|
||||
t.Fatalf("finding %+v", f)
|
||||
}
|
||||
got[f.Module]++
|
||||
}
|
||||
if len(got) != 3 || got["intruder"] != 1 || got["viahome"] != 1 || got["facty"] != 1 {
|
||||
t.Fatalf("refused %v; want intruder, viahome, facty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirReferencesAreResolvedPerModule(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
manifest(t, root, "owner", mod("owner", dir("base", "${machine:account-home}/.config/o", `,"unknown":"move-out"`)))
|
||||
manifest(t, root, "other", mod("other",
|
||||
dir("base", "${machine:account-home}/.config/p", `,"unknown":"move-out"`),
|
||||
`{"id":"f","type":"file","path":"${dir:base}/x"}`))
|
||||
if found := check(t, root); len(found) != 0 {
|
||||
t.Fatalf("refused %+v", found)
|
||||
}
|
||||
manifest(t, root, "third", mod("third",
|
||||
dir("d", "${machine:account-home}/.config/t", `,"unknown":"move-out"`),
|
||||
`{"id":"f","type":"file","path":"${dir:d}/../o/x"}`))
|
||||
only(t, check(t, root), 4, "third", "f")
|
||||
}
|
||||
|
||||
func TestAnArchiveIsNotAContainerAndSameModuleIsFine(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
manifest(t, root, "a", mod("a",
|
||||
`{"id":"arc","type":"archive","path":"/opt/a","unknown":"program"}`,
|
||||
dir("d", "/opt/a2", `,"unknown":"program"`),
|
||||
`{"id":"f","type":"file","path":"/opt/a2/x"}`))
|
||||
manifest(t, root, "b", mod("b", `{"id":"f","type":"file","path":"/opt/a/x"}`))
|
||||
if found := check(t, root); len(found) != 0 {
|
||||
t.Fatalf("refused %+v", found)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1484: a directory the mesh places for a module (no path, `place`) is that module's own: another module's
|
||||
// directory of the same id, and a path under it, are not inside it.
|
||||
func TestPlacedDirectoriesOfTwoModulesAreTheirOwn(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for _, name := range []string{"first", "second"} {
|
||||
manifest(t, root, name, mod(name,
|
||||
`{"id":"state","type":"directory","place":".","unknown":"program"}`,
|
||||
dir("ssh", "${dir:state}/.ssh", `,"unknown":"program"`),
|
||||
`{"id":"conf","type":"file","path":"${dir:ssh}/config"}`))
|
||||
}
|
||||
if found := check(t, root); len(found) != 0 {
|
||||
t.Fatalf("refused %+v", found)
|
||||
}
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/predefined
|
||||
|
||||
go 1.22
|
||||
@@ -1,153 +0,0 @@
|
||||
// Command predefined refuses a change to predefined.json that removes a key a module still names (novox/hq ADR
|
||||
// 0348 clause 7, design 57 package K2, issue 661).
|
||||
//
|
||||
// Mesh config and node config hold values only for the keys on the catalogue's list of predefined keys, at the
|
||||
// root of the repository; a module's config value may fall back to one, `"predefined": "<key>"`. A key removed
|
||||
// while a manifest still names it would leave that module falling back to nothing: the controller refuses its
|
||||
// build then, at the next merge that touches it. So the removal is refused here, in the pull request that makes
|
||||
// it, naming each module that names the key.
|
||||
//
|
||||
// **What it cannot know.** Whether a key still holds a value at the mesh or a node is mesh-config's to say (design
|
||||
// 57 package K3): it holds the values, and the catalogue's check reads no store. Until mesh-config refuses that
|
||||
// itself, removing a key nobody names is let through here, and said.
|
||||
//
|
||||
// The list it compares with is the trunk's at the merge base, given with -base; without one, nothing was removed.
|
||||
// The controller the mesh runs reads the list itself (type, validation, every manifest against it) in the
|
||||
// module check; this check reads only its keys.
|
||||
//
|
||||
// go run . [-base <the trunk's predefined.json>] <catalogue root>
|
||||
//
|
||||
// Exits 1 on a finding.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// File is the list's file at the root of the catalogue.
|
||||
const File = "predefined.json"
|
||||
|
||||
// Keys is the keys of a list, read as a JSON object; an absent file has none.
|
||||
func Keys(path string) (map[string]bool, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return map[string]bool{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var keys map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &keys); err != nil {
|
||||
return nil, fmt.Errorf("%s is not a JSON object of keys: %w", path, err)
|
||||
}
|
||||
out := map[string]bool{}
|
||||
for k := range keys {
|
||||
out[k] = true
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Named is, per predefined key, the modules whose manifests name it: under `config`, or its alias `settings`.
|
||||
func Named(root string) (map[string][]string, error) {
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(manifests)
|
||||
named := map[string][]string{}
|
||||
for _, path := range manifests {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Config map[string]json.RawMessage `json:"config"`
|
||||
Settings map[string]json.RawMessage `json:"settings"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
for _, declared := range []map[string]json.RawMessage{m.Config, m.Settings} {
|
||||
for _, d := range declared {
|
||||
var value struct {
|
||||
Predefined string `json:"predefined"`
|
||||
}
|
||||
if json.Unmarshal(d, &value) == nil && value.Predefined != "" {
|
||||
named[value.Predefined] = append(named[value.Predefined], m.Module)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return named, nil
|
||||
}
|
||||
|
||||
// Removed is each key the base list holds and the tree's does not, with the modules still naming it; a removed
|
||||
// key nobody names is answered with none.
|
||||
func Removed(base, tree map[string]bool, named map[string][]string) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
for k := range base {
|
||||
if !tree[k] {
|
||||
out[k] = named[k]
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func main() {
|
||||
basePath := flag.String("base", "", "the trunk's predefined.json at the merge base")
|
||||
flag.Parse()
|
||||
if flag.NArg() != 1 {
|
||||
fmt.Fprintln(os.Stderr, "usage: predefined [-base <the trunk's predefined.json>] <catalogue root>")
|
||||
os.Exit(2)
|
||||
}
|
||||
root := flag.Arg(0)
|
||||
tree, err := Keys(filepath.Join(root, File))
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if *basePath == "" {
|
||||
fmt.Printf("%s: %d key(s); no trunk list given, so none was removed\n", File, len(tree))
|
||||
return
|
||||
}
|
||||
base, err := Keys(*basePath)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
named, err := Named(root)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
removed := Removed(base, tree, named)
|
||||
keys := make([]string, 0, len(removed))
|
||||
for k := range removed {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
failed := false
|
||||
for _, k := range keys {
|
||||
if modules := removed[k]; len(modules) > 0 {
|
||||
failed = true
|
||||
fmt.Printf("%s: the key %q is removed and still named by %s: move each off it first (novox/hq ADR 0348 "+
|
||||
"clause 7)\n", File, k, strings.Join(modules, ", "))
|
||||
} else {
|
||||
fmt.Printf("%s: the key %q is removed and no module names it; whether the mesh or a node still holds a "+
|
||||
"value for it is mesh-config's to refuse, not this check's\n", File, k)
|
||||
}
|
||||
}
|
||||
if failed {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Printf("%s: %d key(s), none removed that a module names\n", File, len(tree))
|
||||
}
|
||||
@@ -1,100 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A catalogue whose dunst falls back to font-size, and the trunk's list holding font-size and
|
||||
// agent-managed-settings.
|
||||
func aCatalogue(t *testing.T, list string) (root, base string) {
|
||||
t.Helper()
|
||||
root = t.TempDir()
|
||||
write := func(path, text string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(text), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
write(filepath.Join(root, "modules", "dunst", "module.json"), `{"module": "dunst", "config": {"font-size":
|
||||
{"type": "number", "description": "points", "default": 10, "predefined": "font-size"}}}`)
|
||||
write(filepath.Join(root, "modules", "claude-code", "module.json"), `{"module": "claude-code", "settings": {
|
||||
"managed_settings": {"kind": "preference", "default": "{}", "why": "none"}}}`)
|
||||
write(filepath.Join(root, "modules", "plain", "module.json"), `{"module": "plain"}`)
|
||||
if list != "" {
|
||||
write(filepath.Join(root, File), list)
|
||||
}
|
||||
base = filepath.Join(t.TempDir(), "base.json")
|
||||
write(base, `{"font-size": {"type": "number"}, "agent-managed-settings": {"type": "object"}}`)
|
||||
return root, base
|
||||
}
|
||||
|
||||
func run(t *testing.T, args ...string) (string, error) {
|
||||
t.Helper()
|
||||
out, err := exec.Command("go", append([]string{"run", "."}, args...)...).CombinedOutput()
|
||||
return string(out), err
|
||||
}
|
||||
|
||||
func TestARemovalOfAKeyAModuleStillNamesIsRefused(t *testing.T) {
|
||||
// The planted removal: font-size goes while dunst still falls back to it.
|
||||
root, base := aCatalogue(t, `{"agent-managed-settings": {"type": "object"}}`)
|
||||
out, err := run(t, "-base", base, root)
|
||||
if err == nil || !strings.Contains(out, `the key "font-size" is removed and still named by dunst`) {
|
||||
t.Fatalf("a removal of a named key passed:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARemovalNobodyNamesPassesAndSaysWhatItCannotKnow(t *testing.T) {
|
||||
root, base := aCatalogue(t, `{"font-size": {"type": "number"}}`)
|
||||
out, err := run(t, "-base", base, root)
|
||||
if err != nil || !strings.Contains(out, `"agent-managed-settings" is removed and no module names it`) ||
|
||||
!strings.Contains(out, "mesh-config's to refuse") {
|
||||
t.Fatalf("a removal nobody names: %v\n%s", err, out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAListUnchangedOrGrownPasses(t *testing.T) {
|
||||
for _, list := range []string{
|
||||
`{"font-size": {"type": "number"}, "agent-managed-settings": {"type": "object"}}`,
|
||||
`{"font-size": {"type": "number"}, "agent-managed-settings": {"type": "object"}, "accent": {"type": "string"}}`,
|
||||
} {
|
||||
root, base := aCatalogue(t, list)
|
||||
if out, err := run(t, "-base", base, root); err != nil || !strings.Contains(out, "none removed") {
|
||||
t.Fatalf("%s: %v\n%s", list, err, out)
|
||||
}
|
||||
}
|
||||
// No trunk list given: nothing was removed.
|
||||
root, _ := aCatalogue(t, `{}`)
|
||||
if out, err := run(t, root); err != nil || !strings.Contains(out, "none was removed") {
|
||||
t.Fatalf("without a base: %v\n%s", err, out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheFileRemovedWholeIsARemovalOfEveryKey(t *testing.T) {
|
||||
root, base := aCatalogue(t, "")
|
||||
out, err := run(t, "-base", base, root)
|
||||
if err == nil || !strings.Contains(out, `"font-size" is removed and still named by dunst`) {
|
||||
t.Fatalf("the list removed whole passed:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNamedReadsBothForms(t *testing.T) {
|
||||
root, _ := aCatalogue(t, "")
|
||||
if err := os.WriteFile(filepath.Join(root, "modules", "plain", "module.json"), []byte(`{"module": "plain",
|
||||
"settings": {"size": {"predefined": "font-size"}}}`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
named, err := Named(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strings.Join(named["font-size"], ","); got != "dunst,plain" {
|
||||
t.Fatalf("font-size named by %s", got)
|
||||
}
|
||||
}
|
||||
+1
-11
@@ -7,8 +7,7 @@
|
||||
// until somebody removed it by hand. The controller refuses a setting of a key the file keeps as `protected`,
|
||||
// naming the layer and the key, never the value. This check holds every such file to that: each top-level key
|
||||
// whose value holds `${secret:…}`, anywhere below it, is either listed in the file's `protected`, or left out
|
||||
// of the file's `takes-settings` (novox/hq ADR 0326), so that no layer reaches it. A file whose text is a template
|
||||
// in files/ is judged by the template's text.
|
||||
// of the file's `takes-settings` (novox/hq ADR 0326), so that no layer reaches it.
|
||||
//
|
||||
// A secret nested below a key the file takes for its other values (`server.secret_key` below `server`) cannot
|
||||
// be protected alone today: the controller protects top-level keys. Such a file is named in the file `known`
|
||||
@@ -65,15 +64,6 @@ func Check(root string) ([]Finding, error) {
|
||||
}
|
||||
id, _ := r["id"].(string)
|
||||
content, _ := r["content"].(string)
|
||||
// A file whose text is a template in its module's files/ (novox/hq ADR 0349 §2) is judged by that text,
|
||||
// exactly as it would be inline.
|
||||
if name, _ := r["template"].(string); name != "" {
|
||||
text, err := os.ReadFile(filepath.Join(filepath.Dir(path), filepath.FromSlash(name)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %s names the template %s: %w", m.Module, id, name, err)
|
||||
}
|
||||
content = string(text)
|
||||
}
|
||||
var doc map[string]any
|
||||
if strings.TrimSpace(content) == "" {
|
||||
continue
|
||||
|
||||
@@ -51,35 +51,6 @@ func TestASecretASettingCouldReplaceIsRefused(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A mergeable file whose text is a template in files/ (novox/hq ADR 0349 §2) is judged by the template's text,
|
||||
// exactly as the same text inline; a template the module's directory lacks is an error, never a silent pass.
|
||||
func TestAMergeableFilesTemplateIsJudgedAsItsText(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
manifest(t, root, "a", `{"id":"cfg","type":"file","merge":"json","template":"files/cfg.json"}`)
|
||||
manifest(t, root, "c", `{"id":"cfg","type":"file","merge":"json","protected":["password"],"template":"files/cfg.json"}`)
|
||||
for _, m := range []string{"a", "c"} {
|
||||
dir := filepath.Join(root, "modules", m, "files")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "cfg.json"), []byte(`{"password": "${secret:admin}"}`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 1 || found[0].String() != "a cfg password" {
|
||||
t.Fatalf("found %v, want [a cfg password]", found)
|
||||
}
|
||||
|
||||
manifest(t, root, "g", `{"id":"cfg","type":"file","merge":"json","template":"files/gone.json"}`)
|
||||
if _, err := Check(root); err == nil {
|
||||
t.Fatal("a template the module's directory lacks passed the check")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheKnownListOnlyShrinks(t *testing.T) {
|
||||
found := []Finding{{"b", "cfg", "server.secret_key"}, {"a", "cfg", "password"}}
|
||||
refused, stale := Judge(found, map[string]bool{"b cfg server.secret_key": true, "gone cfg token": true})
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/sshdir
|
||||
|
||||
go 1.22
|
||||
@@ -1,161 +0,0 @@
|
||||
// Command sshdir refuses any manifest but modules/ssh-client naming a path under ~/.ssh (novox/hq ADR 0415 §6,
|
||||
// issue 1020): ssh-client owns ~/.ssh and sweeps what it does not know, so a path another module placed there
|
||||
// would be moved out at the next sweep. A module that needs something there provides it to ssh-client instead.
|
||||
//
|
||||
// go run . <catalogue root>
|
||||
//
|
||||
// Every string of a manifest is read, wherever it stands, except prose and a file's text (Prose): a path, a
|
||||
// place, a volume; a path inside a directory of the module's own (`${dir:<id>}`) is not ~/.ssh. Exits 1 on a finding.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// SSHModule is the one module allowed to name a path under ~/.ssh.
|
||||
const SSHModule = "ssh-client"
|
||||
|
||||
// underSSH is a path under ~/.ssh, or ~/.ssh itself, however the home is spelled: ${machine:account-home}/.ssh,
|
||||
// ~/.ssh, .ssh/config, or a volume's host side (…/.ssh:/root/.ssh).
|
||||
var underSSH = regexp.MustCompile(`(^|[/~])\.ssh($|[/:])`)
|
||||
|
||||
// Prose are the keys whose strings are words or a file's text, not a path: a template may say ~/.ssh in a comment.
|
||||
var Prose = map[string]bool{"description": true, "why": true, "does": true, "template": true, "content": true, "title": true}
|
||||
|
||||
// Finding is one path under ~/.ssh in a manifest other than ssh-client's.
|
||||
type Finding struct {
|
||||
Module string
|
||||
Key string
|
||||
Value string
|
||||
}
|
||||
|
||||
// Check reads every manifest under root/modules and answers which paths it refuses.
|
||||
func Check(root string) ([]Finding, error) {
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(manifests)
|
||||
var out []Finding
|
||||
for _, path := range manifests {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m any
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
module := filepath.Base(filepath.Dir(path))
|
||||
if obj, ok := m.(map[string]any); ok {
|
||||
if name, ok := obj["module"].(string); ok && name != "" {
|
||||
module = name
|
||||
}
|
||||
}
|
||||
if module == SSHModule {
|
||||
continue
|
||||
}
|
||||
walk(m, "", dirsOf(m), func(key, value string) {
|
||||
out = append(out, Finding{Module: module, Key: key, Value: value})
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// walk calls found for every string under v, outside Prose, that names a path under ~/.ssh.
|
||||
func walk(v any, key string, dirs map[string]string, found func(key, value string)) {
|
||||
switch x := v.(type) {
|
||||
case map[string]any:
|
||||
keys := make([]string, 0, len(x))
|
||||
for k := range x {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if !Prose[k] {
|
||||
walk(x[k], k, dirs, found)
|
||||
}
|
||||
}
|
||||
case []any:
|
||||
for _, e := range x {
|
||||
walk(e, key, dirs, found)
|
||||
}
|
||||
case string:
|
||||
// A path under a directory the mesh places for the module (${dir:<id>} of a directory with a place and no path,
|
||||
// as a runs-as module's home) is that account's own .ssh, not the operator's ~/.ssh that ssh-client sweeps
|
||||
// (issue 1484, ADR 0466 §4). Any other ${dir:<id>} is resolved to the path its resource declares and judged.
|
||||
if resolved, placed := resolveDir(x, dirs); !placed && underSSH.MatchString(resolved) {
|
||||
found(key, x)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var ofDir = regexp.MustCompile(`^\$\{dir:([a-z0-9][a-z0-9_-]*)\}`)
|
||||
|
||||
// placedDir marks a directory the mesh places for the module: it has no path of its own.
|
||||
const placedDir = "\x00placed"
|
||||
|
||||
// dirsOf are the module's directories by id: the path each declares, or placedDir for one with a place and no path.
|
||||
func dirsOf(m any) map[string]string {
|
||||
dirs := map[string]string{}
|
||||
obj, _ := m.(map[string]any)
|
||||
list, _ := obj["resources"].([]any)
|
||||
for _, r := range list {
|
||||
res, _ := r.(map[string]any)
|
||||
id, _ := res["id"].(string)
|
||||
path, _ := res["path"].(string)
|
||||
switch {
|
||||
case id == "":
|
||||
case path != "":
|
||||
dirs[id] = path
|
||||
case res["type"] == "directory":
|
||||
dirs[id] = placedDir
|
||||
}
|
||||
}
|
||||
return dirs
|
||||
}
|
||||
|
||||
// resolveDir follows a leading ${dir:<id>} through the module's directories. It answers the path with the references
|
||||
// resolved, and whether it ends under a placed directory.
|
||||
func resolveDir(x string, dirs map[string]string) (string, bool) {
|
||||
for i := 0; i < 8; i++ {
|
||||
m := ofDir.FindStringSubmatch(x)
|
||||
if m == nil {
|
||||
return x, false
|
||||
}
|
||||
to, ok := dirs[m[1]]
|
||||
if !ok {
|
||||
return x, false
|
||||
}
|
||||
if to == placedDir {
|
||||
return x, true
|
||||
}
|
||||
x = to + strings.TrimPrefix(x, m[0])
|
||||
}
|
||||
return x, false
|
||||
}
|
||||
|
||||
func main() {
|
||||
root := "."
|
||||
if len(os.Args) > 1 {
|
||||
root = os.Args[1]
|
||||
}
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
for _, f := range found {
|
||||
fmt.Printf("%s names %q (%s) under ~/.ssh: only %s may, and the sweep moves anything else out (novox/hq ADR 0415 §6)\n", f.Module, f.Value, f.Key, SSHModule)
|
||||
}
|
||||
if len(found) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("~/.ssh: named only by ssh-client")
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func manifest(t *testing.T, root, module, body string) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(root, "modules", module)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnotherModuleNamingSSHIsRefused(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
manifest(t, root, "ssh-client", `{"module":"ssh-client","resources":[{"path":"${machine:account-home}/.ssh/config"}]}`)
|
||||
manifest(t, root, "forge-account", `{"module":"forge-account","resources":[{"path":"${machine:account-home}/.ssh/config.d/20-forge"}]}`)
|
||||
manifest(t, root, "roster", `{"module":"roster","facts":{"hosts":{"path":".ssh/known_hosts"}}}`)
|
||||
manifest(t, root, "box", `{"module":"box","containers":[{"volumes":["~/.ssh:/root/.ssh:ro"]}]}`)
|
||||
manifest(t, root, "quiet", `{"module":"quiet","description":"reads ~/.ssh/config","resources":[{"path":"${machine:account-home}/.sshrc-not","template":"# see ~/.ssh/config"},{"path":"/etc/ssh/sshd_config"}]}`)
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := map[string]bool{}
|
||||
for _, f := range found {
|
||||
got[f.Module] = true
|
||||
}
|
||||
if len(found) != 3 || !got["forge-account"] || !got["roster"] || !got["box"] {
|
||||
t.Fatalf("refused %+v; want forge-account, roster and box", found)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1484: a runs-as module's own account has a .ssh of its own, inside its home (a directory of the module's);
|
||||
// that is not the operator's ~/.ssh, which only ssh-client names. A path outside any ${dir:} is still refused.
|
||||
func TestAModulesOwnDirectoryMayHoldADotSSH(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
manifest(t, root, "builder", `{"module":"builder","resources":[{"id":"state","type":"directory","place":"."},{"id":"ssh-dir","type":"directory","path":"${dir:state}/.ssh"},{"path":"${dir:ssh-dir}/config"}]}`)
|
||||
manifest(t, root, "other", `{"module":"other","resources":[{"path":"${machine:account-home}/.ssh/config"}]}`)
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 1 || found[0].Module != "other" {
|
||||
t.Fatalf("refused %+v; want only other", found)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1484 review: only a directory the mesh places is the module's own home. A ${dir:} that resolves to the
|
||||
// operator's home is judged by the path it resolves to.
|
||||
func TestADirResolvingToTheOperatorsHomeIsStillRefused(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
manifest(t, root, "escape", `{"module":"escape","resources":[{"id":"h","type":"directory","path":"${machine:account-home}","unknown":"module"},{"path":"${dir:h}/.ssh/config"}]}`)
|
||||
manifest(t, root, "placed", `{"module":"placed","resources":[{"id":"state","type":"directory","place":".","unknown":"program"},{"id":"s","type":"directory","path":"${dir:state}/.ssh"},{"path":"${dir:s}/config"}]}`)
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 1 || found[0].Module != "escape" {
|
||||
t.Fatalf("refused %+v; want only escape", found)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheCatalogueHasNone(t *testing.T) {
|
||||
found, err := Check("../..")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 0 {
|
||||
t.Fatalf("the catalogue names paths under ~/.ssh outside ssh-client: %+v", found)
|
||||
}
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/templates
|
||||
|
||||
go 1.22
|
||||
@@ -1,9 +0,0 @@
|
||||
# The modules that still keep a file's text inline in module.json (`content`), one per line: the catalogue's list of
|
||||
# modules whose file text has not yet moved to a template in files/ (novox/hq ADR 0349 §2, design 57), or in the
|
||||
# templates directory its manifest names (`templates`, issue 731). Every other module's file resources name their
|
||||
# template. This list only shrinks: a module that keeps no file text inline is taken off, and a line the trunk's copy
|
||||
# does not hold is refused. When it is empty, no manifest in the catalogue carries file text (design 57 package T5,
|
||||
# issue 658).
|
||||
#
|
||||
# A window-manager addition (`contributions`, such as files/i3/*.conf) is not a file resource and takes no template;
|
||||
# it moves with the collections (design 57 package L2, issue 681) and is not judged here.
|
||||
@@ -1,178 +0,0 @@
|
||||
// Command templates refuses a file's text kept inline in a manifest, outside the modules still to move (novox/hq
|
||||
// ADR 0349 §2, design 57 package T5, issue 658).
|
||||
//
|
||||
// A file resource names its text as a template in its module's templates directory (`"template": "<dir>/<name>"`;
|
||||
// the directory is the manifest's `"templates"`, files/ when it says none), which the module's build packs into an
|
||||
// archive; the controller reads each template back into the file's content, so the
|
||||
// mesh holds and composes what it would for the same text inline. A module whose file text has not yet moved is
|
||||
// listed in the file `inline` beside this check, one per line. Every other module's file resources say no
|
||||
// `content`; a module on the list that keeps none inline is taken off it; and a line the trunk's copy (the merge
|
||||
// base's, given with -base) does not hold is refused, so the list only shrinks; while the trunk holds no copy, the
|
||||
// modules in seed are the trunk's. When it is empty, no manifest in the catalogue carries file text.
|
||||
//
|
||||
// Only file resources are judged: a window-manager addition (`contributions`) takes no template and moves with the
|
||||
// collections (design 57 package L2, issue 681).
|
||||
//
|
||||
// go run . [-base <the trunk's inline>] <catalogue root>
|
||||
//
|
||||
// Every refusal names the refused module's own templates directory. Exits 1 on a finding.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
listFile = "inline"
|
||||
listPathInCatalog = "checks/templates/" + listFile
|
||||
)
|
||||
|
||||
// Check answers, per module that keeps any, the ids of its file resources that say their text inline.
|
||||
func Check(root string) (map[string][]string, error) {
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(manifests)
|
||||
out := map[string][]string{}
|
||||
for _, path := range manifests {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if _, inline := r["content"]; inline && r["type"] == "file" {
|
||||
out[m.Module] = append(out[m.Module], fmt.Sprint(r["id"]))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// TemplatesDir answers the directory a module keeps its templates in, with a trailing slash: its manifest's
|
||||
// `templates`, files/ when the manifest says none or cannot be read here.
|
||||
func TemplatesDir(root, module string) string {
|
||||
dir := "files/"
|
||||
raw, err := os.ReadFile(filepath.Join(root, "modules", module, "module.json"))
|
||||
if err != nil {
|
||||
return dir
|
||||
}
|
||||
var m struct {
|
||||
Templates string `json:"templates"`
|
||||
}
|
||||
if json.Unmarshal(raw, &m) == nil && m.Templates != "" {
|
||||
dir = strings.TrimSuffix(m.Templates, "/") + "/"
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
// Refusal is the finding for a module that keeps file text inline and is not on the list.
|
||||
func Refusal(module string, ids []string, dir string) string {
|
||||
return fmt.Sprintf("%s keeps the text of %s inline and is not on %s: keep each file's text in %s and name it as "+
|
||||
"the file's template, packed by the build's archive from %s (novox/hq ADR 0349 §2)",
|
||||
module, strings.Join(ids, ", "), listPathInCatalog, dir, dir)
|
||||
}
|
||||
|
||||
// Gained is the finding for a module the list gains against the trunk's copy.
|
||||
func Gained(module, dir string) string {
|
||||
return fmt.Sprintf("%s gains %s, which the trunk's copy does not hold: the list only shrinks — move its file text "+
|
||||
"to %s instead (novox/hq ADR 0349 §2)", listPathInCatalog, module, dir)
|
||||
}
|
||||
|
||||
// seed is the list as it was first written, standing for the trunk's copy until the trunk holds one.
|
||||
var seed = map[string]bool{"asus-zephyrus-g14": true, "power": true, "showcase": true}
|
||||
|
||||
// Listed is the modules a list names, one per line; a line starting with # and a blank line are not modules.
|
||||
func Listed(path string) (map[string]bool, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
out := map[string]bool{}
|
||||
s := bufio.NewScanner(f)
|
||||
for s.Scan() {
|
||||
line := strings.TrimSpace(s.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
out[line] = true
|
||||
}
|
||||
return out, s.Err()
|
||||
}
|
||||
|
||||
// Judge answers the modules that keep file text inline and are not on the list (refused), the modules on the list
|
||||
// that keep none (stale), and the modules on the list that the trunk's list does not hold (added).
|
||||
func Judge(found map[string][]string, listed, base map[string]bool) (refused, stale, added []string) {
|
||||
for m := range found {
|
||||
if !listed[m] {
|
||||
refused = append(refused, m)
|
||||
}
|
||||
}
|
||||
for m := range listed {
|
||||
if _, keeps := found[m]; !keeps {
|
||||
stale = append(stale, m)
|
||||
}
|
||||
if !base[m] {
|
||||
added = append(added, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(refused)
|
||||
sort.Strings(stale)
|
||||
sort.Strings(added)
|
||||
return refused, stale, added
|
||||
}
|
||||
|
||||
func main() {
|
||||
basePath := flag.String("base", "", "the trunk's copy of inline (the merge base's); seed when absent")
|
||||
flag.Parse()
|
||||
root := "."
|
||||
if flag.NArg() > 0 {
|
||||
root = flag.Arg(0)
|
||||
}
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
listed, err := Listed(listFile)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
base := seed
|
||||
if *basePath != "" {
|
||||
if base, err = Listed(*basePath); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
refused, stale, added := Judge(found, listed, base)
|
||||
for _, m := range refused {
|
||||
fmt.Println(Refusal(m, found[m], TemplatesDir(root, m)))
|
||||
}
|
||||
for _, m := range stale {
|
||||
fmt.Printf("%s names %s, which keeps no file text inline: take it off the list\n", listPathInCatalog, m)
|
||||
}
|
||||
for _, m := range added {
|
||||
fmt.Println(Gained(m, TemplatesDir(root, m)))
|
||||
}
|
||||
if len(refused) > 0 || len(stale) > 0 || len(added) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Printf("templates: no module keeps a file's text inline but the %d listed\n", len(listed))
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func module(t *testing.T, root, name, body string) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(root, "modules", name)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// A file resource that says its text inline is found, by module and resource; one naming its template, a
|
||||
// resource that is not a file and a window-manager addition are not (novox/hq ADR 0349 §2).
|
||||
func TestEveryInlineFileTextIsFound(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
module(t, root, "inline", `{"module":"inline","resources":[{"id":"a","type":"file","content":"x"},`+
|
||||
`{"id":"b","type":"file","template":"files/b"},{"id":"c","type":"file","content":""}]}`)
|
||||
module(t, root, "templated", `{"module":"templated","resources":[{"id":"a","type":"file","template":"files/a"},`+
|
||||
`{"id":"d","type":"directory","path":"/x"}],"contributions":{"i3":[{"content":"bindsym x"}]}}`)
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string][]string{"inline": {"a", "c"}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("found %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A module not on the list keeps no file text inline; one on the list that keeps none is taken off it; a line the
|
||||
// trunk's list does not hold is refused, so the list only shrinks until it is empty.
|
||||
func TestTheListOfModulesKeepingTextInlineOnlyShrinks(t *testing.T) {
|
||||
found := map[string][]string{"old": {"a"}, "forgotten": {"b"}}
|
||||
inline := map[string]bool{"old": true, "done": true, "new": true}
|
||||
base := map[string]bool{"old": true, "done": true, "forgotten": true}
|
||||
refused, stale, added := Judge(found, inline, base)
|
||||
if !reflect.DeepEqual(refused, []string{"forgotten"}) || !reflect.DeepEqual(stale, []string{"done", "new"}) ||
|
||||
!reflect.DeepEqual(added, []string{"new"}) {
|
||||
t.Fatalf("refused %v, stale %v, added %v", refused, stale, added)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheListIsReadWithoutItsComments(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "inline")
|
||||
if err := os.WriteFile(path, []byte("# why\n\npower\n showcase \n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := Listed(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, map[string]bool{"power": true, "showcase": true}) {
|
||||
t.Fatalf("read %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue itself: no module keeps a file's text inline but those the list names, and the list names none
|
||||
// that keeps it no longer (design 57 package T5, issue 658: the ten modules without a build moved).
|
||||
func TestTheCatalogueKeepsFileTextInlineOnlyWhereListed(t *testing.T) {
|
||||
found, err := Check("../..")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inline, err := Listed(listFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
refused, stale, _ := Judge(found, inline, inline)
|
||||
if len(refused) > 0 || len(stale) > 0 {
|
||||
t.Fatalf("keeping file text inline and not listed: %v; listed and keeping none: %v", refused, stale)
|
||||
}
|
||||
}
|
||||
|
||||
// A refusal names the refused module's own templates directory, not files/ when the module names another (issue 809).
|
||||
func TestARefusalNamesTheModulesOwnTemplatesDirectory(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
module(t, root, "own", `{"module":"own","templates":"t/","resources":[{"id":"a","type":"file","content":"x"}]}`)
|
||||
module(t, root, "plain", `{"module":"plain","resources":[{"id":"a","type":"file","content":"x"}]}`)
|
||||
module(t, root, "bare", `{"module":"bare","templates":"templates","resources":[]}`)
|
||||
for module, dir := range map[string]string{"own": "t/", "plain": "files/", "bare": "templates/"} {
|
||||
if got := TemplatesDir(root, module); got != dir {
|
||||
t.Fatalf("%s: directory %q, want %q", module, got, dir)
|
||||
}
|
||||
}
|
||||
msg := Refusal("own", []string{"a"}, TemplatesDir(root, "own"))
|
||||
if !strings.Contains(msg, "t/") || strings.Contains(msg, "files/") {
|
||||
t.Fatalf("refusal %q", msg)
|
||||
}
|
||||
if msg := Gained("own", TemplatesDir(root, "own")); !strings.Contains(msg, "t/") || strings.Contains(msg, "files/") {
|
||||
t.Fatalf("gained %q", msg)
|
||||
}
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/toolchains
|
||||
|
||||
go 1.22
|
||||
@@ -1,104 +0,0 @@
|
||||
// Command toolchains refuses any manifest but the owner's declaring a language's toolchain package (novox/hq
|
||||
// ADR 0477 clause 4, issue 1573): the package go only in modules/go, and nodejs and npm only in modules/nodejs.
|
||||
// Everything that compiles or runs that language requires the owner's provision instead (go-toolchain,
|
||||
// node-toolchain), so the package is declared once, and a node without the owner refuses the module that
|
||||
// needs it at resolution rather than failing at its first build. The owners must declare their packages.
|
||||
//
|
||||
// go run . <catalogue root>
|
||||
//
|
||||
// Exits 1 on a finding.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// Owners names, for each toolchain package, the one module that declares it.
|
||||
var Owners = map[string]string{
|
||||
"go": "go",
|
||||
"nodejs": "nodejs",
|
||||
"npm": "nodejs",
|
||||
}
|
||||
|
||||
// Finding is one refusal: Module declares Package, which Owner alone may, or (Missing) Owner does not declare
|
||||
// the Package it owns.
|
||||
type Finding struct {
|
||||
Module, Package, Owner string
|
||||
Missing bool
|
||||
}
|
||||
|
||||
// Check reads every manifest under root/modules and answers what it refuses.
|
||||
func Check(root string) ([]Finding, error) {
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(manifests)
|
||||
var out []Finding
|
||||
declared := map[string]map[string]bool{} // module -> packages
|
||||
for _, path := range manifests {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Resources []struct {
|
||||
Type string `json:"type"`
|
||||
Package string `json:"package"`
|
||||
} `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
declared[m.Module] = map[string]bool{}
|
||||
for _, r := range m.Resources {
|
||||
if r.Type != "package" {
|
||||
continue
|
||||
}
|
||||
declared[m.Module][r.Package] = true
|
||||
if owner, ok := Owners[r.Package]; ok && owner != m.Module {
|
||||
out = append(out, Finding{Module: m.Module, Package: r.Package, Owner: owner})
|
||||
}
|
||||
}
|
||||
}
|
||||
var pkgs []string
|
||||
for p := range Owners {
|
||||
pkgs = append(pkgs, p)
|
||||
}
|
||||
sort.Strings(pkgs)
|
||||
for _, p := range pkgs {
|
||||
if !declared[Owners[p]][p] {
|
||||
out = append(out, Finding{Module: Owners[p], Package: p, Owner: Owners[p], Missing: true})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
root := "."
|
||||
if len(os.Args) > 1 {
|
||||
root = os.Args[1]
|
||||
}
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
for _, f := range found {
|
||||
if f.Missing {
|
||||
fmt.Printf("%s does not declare the package %s it owns (novox/hq ADR 0477 clause 4)\n", f.Owner, f.Package)
|
||||
continue
|
||||
}
|
||||
fmt.Printf("%s declares the package %s: only %s may; require its provision instead (novox/hq ADR 0477 clause 4)\n",
|
||||
f.Module, f.Package, f.Owner)
|
||||
}
|
||||
if len(found) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("toolchains: go, nodejs and npm are declared only by their owners")
|
||||
}
|
||||
@@ -1,64 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func catalogue(t *testing.T, manifests map[string]string) string {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
for name, body := range manifests {
|
||||
dir := filepath.Join(root, "modules", name)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
func TestOnlyTheOwnerDeclaresAToolchainPackage(t *testing.T) {
|
||||
root := catalogue(t, map[string]string{
|
||||
"go": `{"module":"go","resources":[{"id":"go","type":"package","package":"go"}]}`,
|
||||
"nodejs": `{"module":"nodejs","resources":[{"id":"nodejs","type":"package","package":"nodejs"},{"id":"npm","type":"package","package":"npm"}]}`,
|
||||
"lab": `{"module":"lab","resources":[{"id":"git","type":"package","package":"git"},{"id":"go","type":"package","package":"go"}]}`,
|
||||
"builder": `{"module":"builder","resources":[{"id":"n","type":"package","package":"nodejs"},{"id":"dir","type":"directory","path":"/go"}]}`,
|
||||
})
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []Finding{{Module: "builder", Package: "nodejs", Owner: "nodejs"}, {Module: "lab", Package: "go", Owner: "go"}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("found %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheOwnersMustExistAndDeclareTheirPackages(t *testing.T) {
|
||||
root := catalogue(t, map[string]string{
|
||||
"go": `{"module":"go","resources":[]}`,
|
||||
})
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 3 {
|
||||
t.Errorf("an owner missing its packages gave %v, want three findings (go, nodejs, npm)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue as it stands: this is the check itself, run here too.
|
||||
func TestTheCatalogueGivesEachToolchainPackageToItsOwnerAlone(t *testing.T) {
|
||||
got, err := Check("../..")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Errorf("the catalogue declares a toolchain package outside its owner: %v", got)
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
59
|
||||
66
|
||||
The long-running resources of this catalogue that do not say how they are ready (`health`, novox/hq ADR 0240
|
||||
rule 8), as `module check` counts them. It may only go down: merge-check.sh fails a change that raises it, and
|
||||
one that lowers it writes the new number on the first line. From 2026-11-18, or once it is 0, a long-running
|
||||
|
||||
@@ -23,51 +23,9 @@
|
||||
#
|
||||
# By hand: MESH_CHECK_MODULES=gitea sh merge-check-typescript.sh, in the toolchain image (it reads
|
||||
# /app/node_modules), or with TOOLCHAIN_MODULES pointing at a node_modules that holds the same.
|
||||
#
|
||||
# **In an agent's session, outside the toolchain image** (novox/hq issue 742), the same node_modules is
|
||||
# assembled first, in a throwaway directory under the check's own workspace, the way the image's `deps` stage
|
||||
# assembles it: the toolchain's own package.json and lock (mesh-tools' node-tools/, read from the forge this
|
||||
# clone came from, so the list is never copied here) installed with `npm ci`, then the SDK the mesh last
|
||||
# published (`@novox/mesh-sdk@latest`), the @novox scope from the forge's package registry, read without a
|
||||
# credential. Node and npm are the dev-tools module's; nothing is installed on the machine, no npm
|
||||
# configuration or cache of the account's is read or written. Only when neither TOOLCHAIN_MODULES nor
|
||||
# /app/node_modules is there: the forge's check, in the image, fetches nothing, as before. The forge is
|
||||
# MESH_FORGE (default: the origin remote's), the owner MESH_FORGE_OWNER (default: the origin's owner).
|
||||
set -eu
|
||||
|
||||
work=$(mktemp -d)
|
||||
trap 'rm -rf "$work"' EXIT
|
||||
|
||||
deps="${TOOLCHAIN_MODULES:-/app/node_modules}"
|
||||
if [ -z "${TOOLCHAIN_MODULES:-}" ] && [ ! -f "$deps/typescript/bin/tsc" ]; then
|
||||
origin=$(git remote get-url origin 2>/dev/null || true)
|
||||
forge="${MESH_FORGE:-$(printf '%s\n' "$origin" | sed -n 's#^\(https\{0,1\}://[^/]*\)/.*#\1#p')}"
|
||||
owner="${MESH_FORGE_OWNER:-$(printf '%s\n' "$origin" | sed -n 's#^https\{0,1\}://[^/]*/\([^/]*\)/.*#\1#p')}"
|
||||
if [ -z "$forge" ] || [ -z "$owner" ]; then
|
||||
echo "NOT CHECKED: no TypeScript toolchain at $deps, and no forge to assemble one from (origin is '$origin'; set MESH_FORGE and MESH_FORGE_OWNER, or TOOLCHAIN_MODULES)"
|
||||
exit 1
|
||||
fi
|
||||
echo "no TypeScript toolchain at $deps: assembling one from $forge/$owner/mesh-tools node-tools/ and $owner's package registry (novox/hq issue 742)"
|
||||
tc="$work/toolchain"
|
||||
mkdir -p "$tc"
|
||||
printf '@%s:registry=%s/api/packages/%s/npm/\n' "$owner" "$forge" "$owner" >"$work/npmrc"
|
||||
# Read with Node's own fetch (dev-tools declares Node, not curl); a status that is not 2xx fails.
|
||||
if ! { node -e '
|
||||
const [base, dir] = process.argv.slice(1);
|
||||
Promise.all(["package.json", "package-lock.json"].map(async (f) => {
|
||||
const r = await fetch(base + f);
|
||||
if (!r.ok) throw new Error(base + f + ": " + r.status);
|
||||
require("fs").writeFileSync(dir + "/" + f, await r.text());
|
||||
})).catch((e) => { console.error(e.message); process.exit(1); });' \
|
||||
"$forge/$owner/mesh-tools/raw/branch/main/node-tools/" "$tc" &&
|
||||
(cd "$tc" && export npm_config_userconfig="$work/npmrc" npm_config_cache="$work/npm-cache" &&
|
||||
npm ci --ignore-scripts --no-audit --no-fund --loglevel=error &&
|
||||
npm install --ignore-scripts --no-audit --no-fund --no-save --loglevel=error "@$owner/mesh-sdk@latest"); }; then
|
||||
echo "NOT CHECKED: the TypeScript toolchain could not be assembled from $forge (novox/hq issue 742)"
|
||||
exit 1
|
||||
fi
|
||||
deps="$tc/node_modules"
|
||||
fi
|
||||
tsc="$deps/typescript/bin/tsc"
|
||||
if [ ! -f "$tsc" ]; then
|
||||
echo "NOT CHECKED: no TypeScript compiler at $tsc — this part runs in the TypeScript toolchain"
|
||||
@@ -81,6 +39,8 @@ touched=$({
|
||||
printf '%s\n' "${MESH_CHECK_CHANGED:-}" | tr ',' '\n' | sed -n 's#^modules/\([^/]*\)/.*#\1#p'
|
||||
} | sed '/^$/d' | sort -u)
|
||||
|
||||
work=$(mktemp -d)
|
||||
trap 'rm -rf "$work"' EXIT
|
||||
failed=""
|
||||
for m in $touched; do
|
||||
d="modules/$m"
|
||||
|
||||
+3
-67
@@ -17,8 +17,7 @@
|
||||
# toolchain has a C compiler. A requirement of the mesh's own forge the module cache does not hold (a
|
||||
# pinned SDK commit) is taken from its repository's clone beside the check (checks/deps); a module whose
|
||||
# dependencies still cannot be had **fails the check**, by name and why — its tests could not run, and a
|
||||
# test that could not run is a failure, never a note (novox/hq issue 385). The integration tests (`//go:build
|
||||
# integration`) run here, with -tags integration, and only here. A module written in
|
||||
# test that could not run is a failure, never a note (novox/hq issue 385). A module written in
|
||||
# TypeScript is type-checked and tested by
|
||||
# merge-check-typescript.sh, declared above, in the TypeScript toolchain (novox/hq issues 302, 311);
|
||||
# 3. the words (novox/hq ADR 0244): no word the glossary retired for the tools, as copied in
|
||||
@@ -34,36 +33,11 @@
|
||||
# 6. the secrets (novox/hq issue 342): no mergeable file fills a key with a vault secret that a setting could
|
||||
# replace — the key is `protected`, or left out of the file's `takes-settings`. checks/secrets/known names
|
||||
# the nested ones the controller cannot protect alone yet, and only shrinks.
|
||||
# 7. the list of predefined keys (novox/hq ADR 0348 clause 7): a change to predefined.json removes no key a
|
||||
# module still names, judged against the trunk's copy at the merge base. Whether a removed key still holds a
|
||||
# value at the mesh or a node is mesh-config's to refuse; this check reads no store. The list itself and
|
||||
# every manifest against it are judged by the controller's module check in step 1.
|
||||
# 8. the config form (novox/hq ADR 0348 clauses 1 and 11): no module says `settings`, `takes-settings` or
|
||||
# `${setting:<key>}`, in its manifest or a template it names, or has a mergeable file naming no keys (which
|
||||
# takes its whole layer), unless checks/config/unmoved lists it as not yet moved; that list only shrinks,
|
||||
# judged against the trunk's copy at the merge base.
|
||||
# 9. the file text (novox/hq ADR 0349 §2): no module keeps a file's text inline in its manifest (`content`)
|
||||
# unless checks/templates/inline lists it as not yet moved; each names its template in files/. That list
|
||||
# only shrinks, judged against the trunk's copy at the merge base.
|
||||
# 10. the mail-store seat (novox/hq ADR 0378 §3, issue 800): no manifest but modules/mesh-mail claims it.
|
||||
# 11. owned paths (novox/hq ADR 0422 §8, issue 1024; it holds ~/.ssh for ssh-client as ADR 0415 §6 asked):
|
||||
# every directory and archive says `unknown` (move-out, module or program) and a valid `knows`; `move-out`
|
||||
# only under a home; and no module declares a path inside a directory of another module.
|
||||
# checks/sshdir (ADR 0415 §6) stays while main runs it; it goes under issue 1035 once main runs checks/owned.
|
||||
set -eu
|
||||
|
||||
(cd checks/words && go test -count=1 . && go run . ../..)
|
||||
# The toolchains (novox/hq ADR 0477 clause 4): the package go only in modules/go, nodejs and npm only in modules/nodejs.
|
||||
(cd checks/toolchains && go test -count=1 . && go run . ../..)
|
||||
# The agent's home (novox/hq ADR 0266): no archive, and no file written into, under it.
|
||||
(cd checks/agenthome && go test -count=1 . && go run . ../..)
|
||||
# The mail-store seat (novox/hq ADR 0378 §3): only mesh-mail may claim it.
|
||||
(cd checks/mailstore && go test -count=1 . && go run . ../..)
|
||||
# ~/.ssh (novox/hq ADR 0415 §6): only ssh-client may name a path under it. Main's merge check runs this line, so
|
||||
# checks/sshdir stays until main runs checks/owned; it goes under issue 1035.
|
||||
(cd checks/sshdir && go test -count=1 . && go run . ../..)
|
||||
# Owned paths (novox/hq ADR 0422 §8): unknown files named per directory; no path inside another module's directory.
|
||||
(cd checks/owned && go test -count=1 . && go run . ../..)
|
||||
# The secrets (novox/hq issue 342): a password is not a setting, so no setting can replace the vault's. The
|
||||
# known list only shrinks: judged against the trunk's copy at the merge base, a line it does not hold is refused.
|
||||
secrets_base=""
|
||||
@@ -77,42 +51,6 @@ else
|
||||
echo "NOT COMPARED WITH THE TRUNK: no merge base with $secrets_ref; checks/secrets/known is held to the list it was first written with"
|
||||
fi
|
||||
(cd checks/secrets && go test -count=1 . && if [ -n "$secrets_base" ]; then go run . -base "$secrets_base" ../..; else go run . ../..; fi)
|
||||
# The list of predefined keys (novox/hq ADR 0348 clause 7): no key removed that a module still names. The trunk's
|
||||
# list at the merge base is what it is judged against; a trunk without the file had none.
|
||||
predefined_base=""
|
||||
if merge_base=$(git merge-base HEAD "origin/${MESH_CHECK_BASE:-main}" 2>/dev/null); then
|
||||
predefined_base=$(mktemp)
|
||||
if ! git show "$merge_base:predefined.json" >"$predefined_base" 2>/dev/null; then
|
||||
echo '{}' >"$predefined_base"
|
||||
fi
|
||||
else
|
||||
echo "NOT COMPARED WITH THE TRUNK: no merge base with origin/${MESH_CHECK_BASE:-main}; no removal from predefined.json can be seen"
|
||||
fi
|
||||
(cd checks/predefined && go test -count=1 . && if [ -n "$predefined_base" ]; then go run . -base "$predefined_base" ../..; else go run . ../..; fi)
|
||||
# The config form (novox/hq ADR 0348 §1, §11): a moved module says no old name, and the list of modules not yet moved
|
||||
# only shrinks, judged against the trunk's copy at the merge base; a trunk without one is the list's first copy.
|
||||
unmoved_base=""
|
||||
if merge_base=$(git merge-base HEAD "origin/${MESH_CHECK_BASE:-main}" 2>/dev/null); then
|
||||
if git cat-file -e "$merge_base:checks/config/unmoved" 2>/dev/null; then
|
||||
unmoved_base=$(mktemp)
|
||||
git show "$merge_base:checks/config/unmoved" >"$unmoved_base"
|
||||
fi
|
||||
else
|
||||
echo "NOT COMPARED WITH THE TRUNK: no merge base with origin/${MESH_CHECK_BASE:-main}; checks/config/unmoved is held to the list it was first written with"
|
||||
fi
|
||||
(cd checks/config && go test -count=1 . && if [ -n "$unmoved_base" ]; then go run . -base "$unmoved_base" ../..; else go run . ../..; fi)
|
||||
# The file text (novox/hq ADR 0349 §2): a file's text is a template in files/, outside the modules not yet moved; the
|
||||
# list of those only shrinks, judged against the trunk's copy at the merge base, a trunk without one being its first.
|
||||
inline_base=""
|
||||
if merge_base=$(git merge-base HEAD "origin/${MESH_CHECK_BASE:-main}" 2>/dev/null); then
|
||||
if git cat-file -e "$merge_base:checks/templates/inline" 2>/dev/null; then
|
||||
inline_base=$(mktemp)
|
||||
git show "$merge_base:checks/templates/inline" >"$inline_base"
|
||||
fi
|
||||
else
|
||||
echo "NOT COMPARED WITH THE TRUNK: no merge base with origin/${MESH_CHECK_BASE:-main}; checks/templates/inline is held to the list it was first written with"
|
||||
fi
|
||||
(cd checks/templates && go test -count=1 . && if [ -n "$inline_base" ]; then go run . -base "$inline_base" ../..; else go run . ../..; fi)
|
||||
# The lists that narrow the paths check may not grow unseen: the base branch's copies are what they are
|
||||
# judged against (MESH_CHECK_BASE, main when unset), read from the clone the check runs in.
|
||||
base_lists=""
|
||||
@@ -186,10 +124,8 @@ for m in $touched; do
|
||||
fi
|
||||
cgo=0
|
||||
[ -n "$race" ] && cgo=1
|
||||
# The integration tests (a file whose first line is `//go:build integration`: a test that starts a bus, a store or
|
||||
# another process) run here and nowhere else: the unit suite stays under a second (novox/hq playbook 07).
|
||||
(cd "modules/$m" && CGO_ENABLED=$cgo GOPRIVATE=git.novox.be go vet -tags integration ${modfile:+"-modfile=$modfile"} ./... &&
|
||||
CGO_ENABLED=$cgo GOPRIVATE=git.novox.be go test -tags integration ${modfile:+"-modfile=$modfile"} $race -count=1 ./...)
|
||||
(cd "modules/$m" && CGO_ENABLED=$cgo GOPRIVATE=git.novox.be go vet ${modfile:+"-modfile=$modfile"} ./... &&
|
||||
CGO_ENABLED=$cgo GOPRIVATE=git.novox.be go test ${modfile:+"-modfile=$modfile"} $race -count=1 ./...)
|
||||
done
|
||||
if [ -n "$untestable" ]; then
|
||||
echo "the check fails: the tests of$untestable could not run (novox/hq issue 385)"
|
||||
|
||||
@@ -22,11 +22,11 @@ for GTK, Qt, the portal and the cursor, dark by default. It claims no seat, beca
|
||||
|
||||
| path | class | from |
|
||||
|---|---|---|
|
||||
| `~/.config/gtk-3.0/settings.ini` | owned (the found file kept once) | [`files/gtk-settings.ini`](files/gtk-settings.ini) |
|
||||
| `~/.config/gtk-3.0/settings.ini` | owned (the found file kept once) | [`config/gtk-settings.ini`](config/gtk-settings.ini) |
|
||||
| `~/.config/gtk-4.0/settings.ini` | owned | the same file |
|
||||
| `~/.config/qt6ct/qt6ct.conf` | owned | [`files/qt6ct.conf`](files/qt6ct.conf) |
|
||||
| `~/.config/xdg-desktop-portal/portals.conf` | owned | [`files/portals.conf`](files/portals.conf) |
|
||||
| `~/.icons/default/index.theme` | owned | [`files/cursor-index.theme`](files/cursor-index.theme): the cursor for programs that read neither `XCURSOR_THEME` nor the resources |
|
||||
| `~/.config/qt6ct/qt6ct.conf` | owned | [`config/qt6ct.conf`](config/qt6ct.conf) |
|
||||
| `~/.config/xdg-desktop-portal/portals.conf` | owned | [`config/portals.conf`](config/portals.conf) |
|
||||
| `~/.icons/default/index.theme` | owned | [`config/cursor-index.theme`](config/cursor-index.theme): the cursor for programs that read neither `XCURSOR_THEME` nor the resources |
|
||||
|
||||
**`qt6ct.conf` is the mesh's now.** A change made in qt6ct's own window is replaced at the next push,
|
||||
and the window's saved geometry goes with it. The theme is this module's to say. Settings will make it
|
||||
|
||||
@@ -18,18 +18,10 @@ type manifest struct {
|
||||
Environment struct {
|
||||
Variables map[string]string `json:"variables"`
|
||||
} `json:"environment"`
|
||||
Items []struct {
|
||||
Collection string `json:"collection"`
|
||||
Attributes map[string]string `json:"attributes"`
|
||||
Template string `json:"template"`
|
||||
} `json:"items"`
|
||||
Shell []struct {
|
||||
For, Slot, Code string
|
||||
} `json:"shell"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
Build struct {
|
||||
Artifacts []map[string]any `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
|
||||
func readManifest(t *testing.T) manifest {
|
||||
@@ -41,23 +33,6 @@ func readManifest(t *testing.T) manifest {
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The module's code and variables are the templates of its items (design 57 L3, issues 795 and 807).
|
||||
for _, it := range m.Items {
|
||||
text, err := os.ReadFile(filepath.Join("..", "..", it.Template))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parts := strings.Split(it.Collection, ":")
|
||||
switch {
|
||||
case len(parts) == 3 && parts[0] == "shell":
|
||||
m.Shell = append(m.Shell, struct{ For, Slot, Code string }{parts[1], parts[2], string(text)})
|
||||
case it.Collection == "environment" && it.Attributes["kind"] == "variable":
|
||||
if m.Environment.Variables == nil {
|
||||
m.Environment.Variables = map[string]string{}
|
||||
}
|
||||
m.Environment.Variables[it.Attributes["name"]] = strings.TrimSuffix(string(text), "\n")
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
@@ -121,10 +96,9 @@ func TestItOwnsTheFilesItsSourcesHoldAndNoQt5Duplicate(t *testing.T) {
|
||||
pkgs = append(pkgs, r["package"].(string))
|
||||
continue
|
||||
}
|
||||
_, inline := r["content"]
|
||||
if _, err := os.Stat(filepath.Join("..", "..", "files", sources[id])); err != nil || inline ||
|
||||
r["template"] != "files/"+sources[id] || r["into"] != nil || r["owner"] != "${machine:account}" {
|
||||
t.Errorf("%s is not files/%s, whole, as its template, and the account's", id, sources[id])
|
||||
raw, _ := os.ReadFile(filepath.Join("..", "..", "config", sources[id]))
|
||||
if r["content"] != string(raw) || r["into"] != nil || r["owner"] != "${machine:account}" {
|
||||
t.Errorf("%s is not config/%s, whole and the account's", id, sources[id])
|
||||
}
|
||||
if strings.Contains(r["path"].(string), "qt5ct") {
|
||||
t.Error("qt5ct: both workstations run Qt 6 programs through qt6ct; a second tool is a duplicate")
|
||||
@@ -133,7 +107,7 @@ func TestItOwnsTheFilesItsSourcesHoldAndNoQt5Duplicate(t *testing.T) {
|
||||
if strings.Join(pkgs, ",") != "gnome-themes-extra,adwaita-icon-theme,adwaita-cursors,qt6ct,xdg-desktop-portal-gtk" {
|
||||
t.Fatalf("%v", pkgs)
|
||||
}
|
||||
qt := readINI(filepath.Join("..", "..", "files", "qt6ct.conf"))
|
||||
qt := readINI(filepath.Join("..", "..", "config", "qt6ct.conf"))
|
||||
if qt["Appearance"]["style"] != "Fusion" || qt["Appearance"]["custom_palette"] != "true" || !strings.Contains(qt["Fonts"]["general"], "Inter,11") {
|
||||
t.Fatalf("%v", qt)
|
||||
}
|
||||
@@ -142,7 +116,7 @@ func TestItOwnsTheFilesItsSourcesHoldAndNoQt5Duplicate(t *testing.T) {
|
||||
t.Fatalf("qt6ct ships no %s", qt["Appearance"]["color_scheme_path"])
|
||||
}
|
||||
}
|
||||
gtk := readINI(filepath.Join("..", "..", "files", "gtk-settings.ini"))["Settings"]
|
||||
gtk := readINI(filepath.Join("..", "..", "config", "gtk-settings.ini"))["Settings"]
|
||||
if gtk["gtk-theme-name"] != "Adwaita" || gtk["gtk-application-prefer-dark-theme"] != "1" || gtk["gtk-font-name"] != "Inter 11" {
|
||||
t.Fatalf("%v", gtk)
|
||||
}
|
||||
@@ -270,31 +244,3 @@ func asJSON(v any) string {
|
||||
b, _ := json.Marshal(v)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// No file resource carries its text inline: each names its template in files/, and the build packs files/
|
||||
// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2).
|
||||
func TestEveryFileTakesItsTextFromATemplate(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
for _, r := range m.Resources {
|
||||
if _, inline := r["content"]; inline {
|
||||
t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"])
|
||||
}
|
||||
if name, ok := r["template"].(string); ok {
|
||||
if _, err := os.Stat(filepath.Join("..", "..", name)); err != nil {
|
||||
t.Errorf("resource %v names %s: %v", r["id"], name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
packs := 0
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a["kind"] == "archive" && a["from"] == "files" {
|
||||
packs++
|
||||
if a["name"] != "files" {
|
||||
t.Errorf("the templates archive is named %v, not files", a["name"])
|
||||
}
|
||||
}
|
||||
}
|
||||
if packs != 1 {
|
||||
t.Errorf("the build packs files/ into %d archives, not one", packs)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
Adwaita:dark
|
||||
@@ -1 +0,0 @@
|
||||
/usr/share/themes/Adwaita-dark/gtk-2.0/gtkrc
|
||||
@@ -1 +0,0 @@
|
||||
qt6ct
|
||||
@@ -1 +0,0 @@
|
||||
6
|
||||
@@ -1 +0,0 @@
|
||||
Fusion
|
||||
@@ -1 +0,0 @@
|
||||
24
|
||||
@@ -1 +0,0 @@
|
||||
Adwaita
|
||||
@@ -1,10 +0,0 @@
|
||||
# The appearance (module adwaita): GSettings is where the portal reads dark or light, and the portal is
|
||||
# the only way it reaches Electron, Chromium, Firefox and flatpaks. Set at every session start to the
|
||||
# module's default; adwaita_appearance switches it for a session.
|
||||
gsettings set org.gnome.desktop.interface color-scheme 'prefer-dark' || true
|
||||
gsettings set org.gnome.desktop.interface gtk-theme 'Adwaita' || true
|
||||
gsettings set org.gnome.desktop.interface icon-theme 'Adwaita' || true
|
||||
gsettings set org.gnome.desktop.interface cursor-theme 'Adwaita' || true
|
||||
gsettings set org.gnome.desktop.interface cursor-size 24 || true
|
||||
gsettings set org.gnome.desktop.interface font-name 'Inter 11' || true
|
||||
gsettings set org.gnome.desktop.interface monospace-font-name 'JetBrainsMono Nerd Font 11' || true
|
||||
@@ -1,3 +0,0 @@
|
||||
! adwaita: the cursor, for X programs that take it from the resources.
|
||||
Xcursor.theme: Adwaita
|
||||
Xcursor.size: 24
|
||||
+23
-81
@@ -1,7 +1,6 @@
|
||||
{
|
||||
"module": "adwaita",
|
||||
"version": "1",
|
||||
"moved": true,
|
||||
"capabilities": [
|
||||
"package-manager"
|
||||
],
|
||||
@@ -11,79 +10,27 @@
|
||||
"adwaita_icons",
|
||||
"adwaita_portal_check"
|
||||
],
|
||||
"items": [
|
||||
"environment": {
|
||||
"variables": {
|
||||
"GTK_THEME": "Adwaita:dark",
|
||||
"GTK2_RC_FILES": "/usr/share/themes/Adwaita-dark/gtk-2.0/gtkrc",
|
||||
"QT_QPA_PLATFORMTHEME": "qt6ct",
|
||||
"QT_STYLE_OVERRIDE": "Fusion",
|
||||
"QT_SELECT": "6",
|
||||
"XCURSOR_THEME": "Adwaita",
|
||||
"XCURSOR_SIZE": "24"
|
||||
}
|
||||
},
|
||||
"shell": [
|
||||
{
|
||||
"collection": "environment",
|
||||
"name": "gtk-theme",
|
||||
"attributes": {
|
||||
"kind": "variable",
|
||||
"name": "GTK_THEME"
|
||||
},
|
||||
"template": "files/env-gtk-theme"
|
||||
"for": "xresources",
|
||||
"slot": "normal",
|
||||
"code": "! adwaita: the cursor, for X programs that take it from the resources.\nXcursor.theme: Adwaita\nXcursor.size: 24\n"
|
||||
},
|
||||
{
|
||||
"collection": "environment",
|
||||
"name": "gtk2-rc-files",
|
||||
"attributes": {
|
||||
"kind": "variable",
|
||||
"name": "GTK2_RC_FILES"
|
||||
},
|
||||
"template": "files/env-gtk2-rc-files"
|
||||
},
|
||||
{
|
||||
"collection": "environment",
|
||||
"name": "qt-qpa-platformtheme",
|
||||
"attributes": {
|
||||
"kind": "variable",
|
||||
"name": "QT_QPA_PLATFORMTHEME"
|
||||
},
|
||||
"template": "files/env-qt-qpa-platformtheme"
|
||||
},
|
||||
{
|
||||
"collection": "environment",
|
||||
"name": "qt-style-override",
|
||||
"attributes": {
|
||||
"kind": "variable",
|
||||
"name": "QT_STYLE_OVERRIDE"
|
||||
},
|
||||
"template": "files/env-qt-style-override"
|
||||
},
|
||||
{
|
||||
"collection": "environment",
|
||||
"name": "qt-select",
|
||||
"attributes": {
|
||||
"kind": "variable",
|
||||
"name": "QT_SELECT"
|
||||
},
|
||||
"template": "files/env-qt-select"
|
||||
},
|
||||
{
|
||||
"collection": "environment",
|
||||
"name": "xcursor-theme",
|
||||
"attributes": {
|
||||
"kind": "variable",
|
||||
"name": "XCURSOR_THEME"
|
||||
},
|
||||
"template": "files/env-xcursor-theme"
|
||||
},
|
||||
{
|
||||
"collection": "environment",
|
||||
"name": "xcursor-size",
|
||||
"attributes": {
|
||||
"kind": "variable",
|
||||
"name": "XCURSOR_SIZE"
|
||||
},
|
||||
"template": "files/env-xcursor-size"
|
||||
},
|
||||
{
|
||||
"collection": "shell:xresources:normal",
|
||||
"name": "xresources-normal",
|
||||
"template": "files/xresources-normal"
|
||||
},
|
||||
{
|
||||
"collection": "shell:xinitrc:normal",
|
||||
"name": "xinitrc-normal",
|
||||
"template": "files/xinitrc-normal"
|
||||
"for": "xinitrc",
|
||||
"slot": "normal",
|
||||
"code": "# The appearance (module adwaita): GSettings is where the portal reads dark or light, and the portal is\n# the only way it reaches Electron, Chromium, Firefox and flatpaks. Set at every session start to the\n# module's default; adwaita_appearance switches it for a session.\ngsettings set org.gnome.desktop.interface color-scheme 'prefer-dark' || true\ngsettings set org.gnome.desktop.interface gtk-theme 'Adwaita' || true\ngsettings set org.gnome.desktop.interface icon-theme 'Adwaita' || true\ngsettings set org.gnome.desktop.interface cursor-theme 'Adwaita' || true\ngsettings set org.gnome.desktop.interface cursor-size 24 || true\ngsettings set org.gnome.desktop.interface font-name 'Inter 11' || true\ngsettings set org.gnome.desktop.interface monospace-font-name 'JetBrainsMono Nerd Font 11' || true\n"
|
||||
}
|
||||
],
|
||||
"resources": [
|
||||
@@ -118,7 +65,7 @@
|
||||
"path": "${machine:account-home}/.config/gtk-3.0/settings.ini",
|
||||
"owner": "${machine:account}",
|
||||
"mode": "0644",
|
||||
"template": "files/gtk-settings.ini"
|
||||
"content": "# Written by the mesh (module adwaita, novox/hq ADR 0208), for GTK 3 and GTK 4 alike. Replaced at\n# every push; adwaita_appearance switches dark and light for the running session.\n[Settings]\ngtk-theme-name=Adwaita\ngtk-icon-theme-name=Adwaita\ngtk-cursor-theme-name=Adwaita\ngtk-cursor-theme-size=24\ngtk-font-name=Inter 11\ngtk-application-prefer-dark-theme=1\n"
|
||||
},
|
||||
{
|
||||
"id": "gtk4",
|
||||
@@ -126,7 +73,7 @@
|
||||
"path": "${machine:account-home}/.config/gtk-4.0/settings.ini",
|
||||
"owner": "${machine:account}",
|
||||
"mode": "0644",
|
||||
"template": "files/gtk-settings.ini"
|
||||
"content": "# Written by the mesh (module adwaita, novox/hq ADR 0208), for GTK 3 and GTK 4 alike. Replaced at\n# every push; adwaita_appearance switches dark and light for the running session.\n[Settings]\ngtk-theme-name=Adwaita\ngtk-icon-theme-name=Adwaita\ngtk-cursor-theme-name=Adwaita\ngtk-cursor-theme-size=24\ngtk-font-name=Inter 11\ngtk-application-prefer-dark-theme=1\n"
|
||||
},
|
||||
{
|
||||
"id": "qt6ct",
|
||||
@@ -134,7 +81,7 @@
|
||||
"path": "${machine:account-home}/.config/qt6ct/qt6ct.conf",
|
||||
"owner": "${machine:account}",
|
||||
"mode": "0644",
|
||||
"template": "files/qt6ct.conf"
|
||||
"content": "[Appearance]\ncolor_scheme_path=/usr/share/qt6ct/colors/darker.conf\ncustom_palette=true\nicon_theme=Adwaita\nstandard_dialogs=default\nstyle=Fusion\n\n[Fonts]\nfixed=\"JetBrainsMono Nerd Font,11,-1,5,50,0,0,0,0,0\"\ngeneral=\"Inter,11,-1,5,50,0,0,0,0,0\"\n\n[Interface]\nactivate_item_on_single_click=1\nbuttonbox_layout=0\ncursor_flash_time=1000\ndialog_buttons_have_icons=1\ndouble_click_interval=400\ngui_effects=@Invalid()\nkeyboard_scheme=2\nmenus_have_icons=true\nshow_shortcuts_in_context_menus=true\nstylesheets=@Invalid()\ntoolbutton_style=4\nunderline_shortcut=1\nwheel_scroll_lines=3\n\n[Troubleshooting]\nforce_raster_widgets=1\nignored_applications=@Invalid()\n"
|
||||
},
|
||||
{
|
||||
"id": "portals",
|
||||
@@ -142,7 +89,7 @@
|
||||
"path": "${machine:account-home}/.config/xdg-desktop-portal/portals.conf",
|
||||
"owner": "${machine:account}",
|
||||
"mode": "0644",
|
||||
"template": "files/portals.conf"
|
||||
"content": "# Written by the mesh (module adwaita, novox/hq ADR 0208). Replaced at every push.\n#\n# Which portal backend answers each interface. i3 is not a desktop xdg-desktop-portal knows, so with\n# no preference it uses whichever backend happens to be installed: fine while gtk is the only one,\n# wrong the day another arrives as somebody else's dependency. Named instead. gtk also serves\n# org.freedesktop.appearance (dark or light) from GSettings, which the session's start sets.\n[preferred]\ndefault=gtk\n# Secrets for sandboxed programs come from the keyring's backend. Without this line no backend answers\n# the interface: gtk does not implement it, and gnome-keyring's names only GNOME as its desktop.\norg.freedesktop.impl.portal.Secret=gnome-keyring\n"
|
||||
},
|
||||
{
|
||||
"id": "cursor",
|
||||
@@ -150,7 +97,7 @@
|
||||
"path": "${machine:account-home}/.icons/default/index.theme",
|
||||
"owner": "${machine:account}",
|
||||
"mode": "0644",
|
||||
"template": "files/cursor-index.theme"
|
||||
"content": "# Written by the mesh (module adwaita, novox/hq ADR 0208): the default cursor theme, for programs that\n# read neither XCURSOR_THEME nor the X resources.\n[Icon Theme]\nName=Default\nInherits=Adwaita\n"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
@@ -165,11 +112,6 @@
|
||||
"loads": [
|
||||
"adwaita-tools"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -109,9 +109,3 @@ store's current or an older name, a definition not read said first, escalation t
|
||||
tools served are exactly the manifest's `tools` and its `invokes` exactly the two controller verbs and the
|
||||
catalogue's two tools.
|
||||
Any command other than the two read-only scripts fails the test.
|
||||
|
||||
## Files it does not know
|
||||
|
||||
Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416).
|
||||
|
||||
- `${dir:state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The module's directory, where its manifest and its files/ are.
|
||||
const moduleDir = "../.."
|
||||
|
||||
// templateText is the text a file resource places on the machine: its template's, in the module's files/
|
||||
// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test.
|
||||
func templateText(t *testing.T, r map[string]any) string {
|
||||
t.Helper()
|
||||
name, _ := r["template"].(string)
|
||||
if name == "" {
|
||||
t.Fatalf("resource %v names no template", r["id"])
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name)))
|
||||
if err != nil {
|
||||
t.Fatalf("resource %v names %s: %v", r["id"], name, err)
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
// No file resource carries its text inline: each names its template in files/, and the build packs files/
|
||||
// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2).
|
||||
func TestEveryFileTakesItsTextFromATemplate(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Resources []map[string]any `json:"resources"`
|
||||
Build struct {
|
||||
Artifacts []map[string]any `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if _, inline := r["content"]; inline {
|
||||
t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"])
|
||||
}
|
||||
if r["type"] == "file" {
|
||||
templateText(t, r)
|
||||
}
|
||||
}
|
||||
packs := 0
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a["kind"] == "archive" && a["from"] == "files" {
|
||||
packs++
|
||||
if a["name"] != "files" {
|
||||
t.Errorf("the templates archive is named %v, not files", a["name"])
|
||||
}
|
||||
}
|
||||
}
|
||||
if packs != 1 {
|
||||
t.Errorf("the build packs files/ into %d archives, not one", packs)
|
||||
}
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
# The names the artifact store had before, from the setting older-store-names, written by the mesh.
|
||||
# artifact_store_references reads every definition under these too (novox/hq issue 323).
|
||||
# Replaced on every push; change the setting, never this file.
|
||||
older=${config:older-store-names}
|
||||
@@ -13,21 +13,19 @@
|
||||
"artifact_store_references",
|
||||
"artifact_store_collect"
|
||||
],
|
||||
"config": {
|
||||
"older-store-names": {
|
||||
"type": "string",
|
||||
"description": "The names the artifact store had before, as host names or host:port separated by spaces, or none when it had no other. artifact_store_references reads every definition under these as well as under the store's current name, so an image a definition pins under one is said to be named by a definition (novox/hq issue 323).",
|
||||
"pattern": "none|[A-Za-z0-9.:-]+( [A-Za-z0-9.:-]+)*",
|
||||
"default": "none"
|
||||
}
|
||||
},
|
||||
"settings": {
|
||||
"older-store-names": {
|
||||
"kind": "preference",
|
||||
"default": "none",
|
||||
"why": "the names the artifact store had before, as a definition may still pin an image under them: host names or host:port, separated by spaces; none when it had no other. artifact_store_references reads every definition under these as well as under the store's current name, so an image a definition pins under one is said to be named by a definition (novox/hq issue 323)"
|
||||
}
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
"place": "."
|
||||
},
|
||||
{
|
||||
"id": "store-names",
|
||||
@@ -35,7 +33,7 @@
|
||||
"path": "${dir:state}/store-names",
|
||||
"mode": "0600",
|
||||
"trusted": false,
|
||||
"template": "files/store-names"
|
||||
"content": "# The names the artifact store had before, from the setting older-store-names, written by the mesh.\n# artifact_store_references reads every definition under these too (novox/hq issue 323).\n# Replaced on every push; change the setting, never this file.\nolder=${setting:older-store-names}\n"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
@@ -54,11 +52,6 @@
|
||||
"MESH_ARTIFACT_STORE_CONTAINER": "mesh-registry",
|
||||
"MESH_ARTIFACT_STORE_NAMES_FILE": "${dir:state}/store-names"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -385,15 +385,3 @@ The module no longer writes a file into i3's `config.d`. Its window-manager line
|
||||
under `files/i3/` where it had one) are a contribution to `node-display-session`. The i3 module places
|
||||
them in its own configuration under a `# <module>` line, so this module depends on a window manager
|
||||
being assigned beside it.
|
||||
|
||||
## Files it does not know
|
||||
|
||||
Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416).
|
||||
|
||||
- `/usr/local/lib/asus-zephyrus-g14`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
- `/etc/systemd/system/systemd-suspend.service.d`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
- `/etc/systemd/system/systemd-hibernate.service.d`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
- `/etc/systemd/system/systemd-suspend-then-hibernate.service.d`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
- `/etc/systemd/system/nvidia-powerd.service.d`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
- `/etc/UPower/UPower.conf.d`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
- `/etc/X11/xorg.conf.d`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
|
||||
@@ -34,9 +34,11 @@ func TestKeysListsTriggersI3LinesAndFirmwareKeys(t *testing.T) {
|
||||
home := "/home/operator"
|
||||
t.Setenv("MESH_OPERATOR_HOME", home)
|
||||
// The i3 module's file as the controller composes it: other modules' lines around this one's.
|
||||
own := m.i3Lines()
|
||||
if own == "" || len(m.Contributions) != 0 {
|
||||
t.Fatalf("the lines for the window manager are one item, and no contribution is left: %q %v", own, m.Contributions)
|
||||
var own string
|
||||
for _, c := range m.Contributions {
|
||||
if c.Seat == "node-display-session" {
|
||||
own += c.Content
|
||||
}
|
||||
}
|
||||
put(t, f.root, home+"/"+I3Config, "set $mod Mod4\n# rofi\nbindsym $mod+d exec rofi\n# "+ModuleName+"\n"+own+
|
||||
"# triggerhappy-not-here\nbindsym $mod+Shift+z exec other\n#########################################\ninclude x\n")
|
||||
|
||||
@@ -15,41 +15,19 @@ type manifest struct {
|
||||
Tools []string `json:"tools"`
|
||||
Emits []string `json:"emits"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
// Items of collections (novox/hq ADR 0349 clause 13): the vendor keys are an item of the node-hotkeys
|
||||
// triggers, its text the template file beside the manifest.
|
||||
Items []struct {
|
||||
Collection string `json:"collection"`
|
||||
Template string `json:"template"`
|
||||
} `json:"items"`
|
||||
// Old-form contributions to other modules' seats (novox/hq ADR 0212): none is left.
|
||||
Contributions []any `json:"contributions"`
|
||||
// Contributions to other modules' seats (novox/hq ADR 0212): the vendor keys go to node-hotkeys.
|
||||
Contributions []struct {
|
||||
Seat string `json:"seat"`
|
||||
Kind string `json:"kind"`
|
||||
Content string `json:"content"`
|
||||
} `json:"contributions"`
|
||||
}
|
||||
|
||||
// i3Lines is the module's lines for the window manager: the text of the template of its item of the collection
|
||||
// contribution:node-display-session:config (design 57 L3, issue 848).
|
||||
func (m manifest) i3Lines() string {
|
||||
for _, it := range m.Items {
|
||||
if it.Collection == "contribution:node-display-session:config" {
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", it.Template))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// triggers is the module's contribution to node-hotkeys: its vendor keys, the text of the template of its
|
||||
// item of the collection contribution:node-hotkeys:trigger.
|
||||
// triggers is the module's contribution to node-hotkeys: its vendor keys.
|
||||
func (m manifest) triggers() string {
|
||||
for _, it := range m.Items {
|
||||
if it.Collection == "contribution:node-hotkeys:trigger" {
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", it.Template))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(raw)
|
||||
for _, c := range m.Contributions {
|
||||
if c.Seat == "node-hotkeys" && c.Kind == "trigger" {
|
||||
return c.Content
|
||||
}
|
||||
}
|
||||
return ""
|
||||
|
||||
@@ -1,83 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Each file's text is a template in templates/, the directory the manifest names (novox/hq ADR 0349 §2, issue 731),
|
||||
// never in files/: files/ is the archive the machine unpacks, and stays exactly what the machine is given.
|
||||
func TestEveryFileTakesItsTextFromATemplateOutsideTheUnpackedFiles(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Templates string `json:"templates"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
Build struct {
|
||||
Artifacts []map[string]any `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.Templates != "templates" {
|
||||
t.Fatalf("the manifest names the templates directory %q, not templates", m.Templates)
|
||||
}
|
||||
files := 0
|
||||
for _, r := range m.Resources {
|
||||
if _, inline := r["content"]; inline {
|
||||
t.Errorf("resource %v keeps its text inline: keep it in templates/ and name it as the template", r["id"])
|
||||
}
|
||||
if r["type"] != "file" {
|
||||
continue
|
||||
}
|
||||
files++
|
||||
name, _ := r["template"].(string)
|
||||
if !strings.HasPrefix(name, "templates/") {
|
||||
t.Errorf("resource %v names the template %q, not one in templates/", r["id"], name)
|
||||
continue
|
||||
}
|
||||
if st, err := os.Lstat(filepath.Join("../..", name)); err != nil || !st.Mode().IsRegular() {
|
||||
t.Errorf("resource %v names the template %s, which is not a plain file: %v", r["id"], name, err)
|
||||
}
|
||||
}
|
||||
if files == 0 {
|
||||
t.Fatal("no file resource")
|
||||
}
|
||||
packs := 0
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a["kind"] == "archive" && a["from"] == "templates" {
|
||||
packs++
|
||||
if a["name"] != "templates" {
|
||||
t.Errorf("the templates archive is named %v, not templates", a["name"])
|
||||
}
|
||||
}
|
||||
}
|
||||
if packs != 1 {
|
||||
t.Fatalf("the build packs templates/ into %d archives, not one", packs)
|
||||
}
|
||||
// The templates archive is read by the controller, never unpacked onto the machine.
|
||||
for _, r := range m.Resources {
|
||||
if r["artifact"] == "templates" {
|
||||
t.Errorf("resource %v places the templates archive on the machine", r["id"])
|
||||
}
|
||||
}
|
||||
// And files/ holds what the machine runs, no template.
|
||||
err = filepath.WalkDir("../../files", func(at string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !d.IsDir() && !strings.HasPrefix(filepath.ToSlash(at), "../../files/bin/") {
|
||||
t.Errorf("files/ holds %s, which is not one of the scripts the machine unpacks", at)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,6 @@
|
||||
{
|
||||
"module": "asus-zephyrus-g14",
|
||||
"version": "1",
|
||||
"moved": true,
|
||||
"templates": "templates",
|
||||
"capabilities": [
|
||||
"package-manager",
|
||||
"service-manager"
|
||||
@@ -54,85 +52,80 @@
|
||||
"id": "scripts",
|
||||
"type": "archive",
|
||||
"path": "/usr/local/lib/asus-zephyrus-g14",
|
||||
"artifact": "scripts",
|
||||
"unknown": "program"
|
||||
"artifact": "scripts"
|
||||
},
|
||||
{
|
||||
"id": "nvidia-options",
|
||||
"type": "file",
|
||||
"path": "/etc/modprobe.d/g14-nvidia-power.conf",
|
||||
"mode": "0644",
|
||||
"template": "templates/modprobe.d/g14-nvidia-power.conf"
|
||||
"content": "# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.\n#\n# The discrete GPU's driver options on the ROG Zephyrus G14 (GA403, RTX 40 series, hybrid graphics).\n#\n# NVreg_DynamicPowerManagement=0x00 turns runtime D3 off. With it on, a change of power source sends\n# the driver an ACPI notification it fails to handle on this model (\"RmHandleDNotifierEvent: Failed to\n# handle ACPI D-Notifier event, status=0x62\"), and the GPU stops making progress until the machine is\n# powered off. Off costs a few idle watts in hybrid mode and keeps the machine up.\n#\n# NVreg_PreserveVideoMemoryAllocations=1 saves video memory across suspend, so what used the GPU still\n# works after waking. It needs nvidia-suspend, -hibernate and -resume to run around a sleep, which this\n# module's drop-ins on the sleep services ask for.\n#\n# A change here applies when the driver next loads: at the next boot.\noptions nvidia NVreg_PreserveVideoMemoryAllocations=1\noptions nvidia NVreg_DynamicPowerManagement=0x00\n"
|
||||
},
|
||||
{
|
||||
"id": "video-options",
|
||||
"type": "file",
|
||||
"path": "/etc/modprobe.d/video-brightness-switch.conf",
|
||||
"mode": "0644",
|
||||
"template": "templates/modprobe.d/video-brightness-switch.conf"
|
||||
"content": "# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.\n#\n# The ACPI video driver does not change the backlight itself on the brightness keys: on this model it\n# moves the wrong one. The keys are triggerhappy's (see /etc/triggerhappy/triggers.d/asus-g14.conf).\n# Applies when the module next loads: at the next boot.\noptions video brightness_switch_enabled=0\n"
|
||||
},
|
||||
{
|
||||
"id": "suspend-drop-ins",
|
||||
"type": "directory",
|
||||
"path": "/etc/systemd/system/systemd-suspend.service.d",
|
||||
"mode": "0755",
|
||||
"unknown": "program"
|
||||
"mode": "0755"
|
||||
},
|
||||
{
|
||||
"id": "nvidia-on-suspend",
|
||||
"type": "file",
|
||||
"path": "/etc/systemd/system/systemd-suspend.service.d/asus-zephyrus-g14-nvidia.conf",
|
||||
"mode": "0644",
|
||||
"template": "templates/systemd/system/systemd-suspend.service.d/asus-zephyrus-g14-nvidia.conf"
|
||||
"content": "# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.\n#\n# The NVIDIA driver's own sleep actions, asked for by the sleep itself rather than enabled as\n# links: the mesh declares files and never makes links (novox/hq ADR 0012), and the host's service\n# shape must not start these units by hand, which would put the GPU to sleep with the machine awake.\n[Unit]\nWants=nvidia-suspend.service nvidia-resume.service\n"
|
||||
},
|
||||
{
|
||||
"id": "hibernate-drop-ins",
|
||||
"type": "directory",
|
||||
"path": "/etc/systemd/system/systemd-hibernate.service.d",
|
||||
"mode": "0755",
|
||||
"unknown": "program"
|
||||
"mode": "0755"
|
||||
},
|
||||
{
|
||||
"id": "nvidia-on-hibernate",
|
||||
"type": "file",
|
||||
"path": "/etc/systemd/system/systemd-hibernate.service.d/asus-zephyrus-g14-nvidia.conf",
|
||||
"mode": "0644",
|
||||
"template": "templates/systemd/system/systemd-hibernate.service.d/asus-zephyrus-g14-nvidia.conf"
|
||||
"content": "# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.\n#\n# The NVIDIA driver's own sleep actions, asked for by the sleep itself rather than enabled as\n# links: the mesh declares files and never makes links (novox/hq ADR 0012), and the host's service\n# shape must not start these units by hand, which would put the GPU to sleep with the machine awake.\n[Unit]\nWants=nvidia-hibernate.service nvidia-resume.service\n"
|
||||
},
|
||||
{
|
||||
"id": "suspend-then-hibernate-drop-ins",
|
||||
"type": "directory",
|
||||
"path": "/etc/systemd/system/systemd-suspend-then-hibernate.service.d",
|
||||
"mode": "0755",
|
||||
"unknown": "program"
|
||||
"mode": "0755"
|
||||
},
|
||||
{
|
||||
"id": "nvidia-on-suspend-then-hibernate",
|
||||
"type": "file",
|
||||
"path": "/etc/systemd/system/systemd-suspend-then-hibernate.service.d/asus-zephyrus-g14-nvidia.conf",
|
||||
"mode": "0644",
|
||||
"template": "templates/systemd/system/systemd-suspend-then-hibernate.service.d/asus-zephyrus-g14-nvidia.conf"
|
||||
"content": "# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.\n#\n# The NVIDIA driver's own sleep actions, asked for by the sleep itself rather than enabled as\n# links: the mesh declares files and never makes links (novox/hq ADR 0012), and the host's service\n# shape must not start these units by hand, which would put the GPU to sleep with the machine awake.\n[Unit]\nWants=nvidia-suspend-then-hibernate.service nvidia-resume.service\n"
|
||||
},
|
||||
{
|
||||
"id": "powerd-drop-ins",
|
||||
"type": "directory",
|
||||
"path": "/etc/systemd/system/nvidia-powerd.service.d",
|
||||
"mode": "0755",
|
||||
"unknown": "program"
|
||||
"mode": "0755"
|
||||
},
|
||||
{
|
||||
"id": "powerd-opt-in",
|
||||
"type": "file",
|
||||
"path": "/etc/systemd/system/nvidia-powerd.service.d/asus-zephyrus-g14.conf",
|
||||
"mode": "0644",
|
||||
"template": "templates/systemd/system/nvidia-powerd.service.d/asus-zephyrus-g14.conf"
|
||||
"content": "# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.\n#\n# nvidia-powerd (Dynamic Boost) was the first error in the chain that hung this model's GPU on a change\n# of power source, and asusd starts it on mains. It runs only when the kernel command line says\n# zephyrus.nvidia-powerd — an explicit opt-in, at boot.\n[Unit]\nConditionKernelCommandLine=zephyrus.nvidia-powerd\n"
|
||||
},
|
||||
{
|
||||
"id": "backlight-rule",
|
||||
"type": "file",
|
||||
"path": "/etc/udev/rules.d/90-backlight.rules",
|
||||
"mode": "0644",
|
||||
"template": "templates/udev/rules.d/90-backlight.rules"
|
||||
"content": "# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.\n#\n# The backlights are writable by the video group, so the brightness keys and the module's brightness tool\n# move the panel without root.\nACTION==\"add\", SUBSYSTEM==\"backlight\", RUN+=\"/usr/bin/chgrp video /sys/class/backlight/%k/brightness\", RUN+=\"/usr/bin/chmod g+w /sys/class/backlight/%k/brightness\"\n"
|
||||
},
|
||||
{
|
||||
"id": "udev",
|
||||
@@ -151,15 +144,14 @@
|
||||
"id": "upower-drop-ins",
|
||||
"type": "directory",
|
||||
"path": "/etc/UPower/UPower.conf.d",
|
||||
"mode": "0755",
|
||||
"unknown": "program"
|
||||
"mode": "0755"
|
||||
},
|
||||
{
|
||||
"id": "low-battery",
|
||||
"type": "file",
|
||||
"path": "/etc/UPower/UPower.conf.d/50-asus-zephyrus-g14.conf",
|
||||
"mode": "0644",
|
||||
"template": "templates/UPower/UPower.conf.d/50-asus-zephyrus-g14.conf"
|
||||
"content": "# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.\n#\n# On low battery the machine suspends rather than powering off, at 7 % — s2idle still draws a little,\n# so it leaves headroom. A drop-in over the package's own UPower.conf, which stays the package's.\n[UPower]\nUsePercentageForPolicy=true\nPercentageLow=15.0\nPercentageCritical=10.0\nPercentageAction=7.0\nCriticalPowerAction=Suspend\nAllowRiskyCriticalPowerAction=true\n"
|
||||
},
|
||||
{
|
||||
"id": "upower",
|
||||
@@ -175,30 +167,14 @@
|
||||
"id": "xorg-drop-ins",
|
||||
"type": "directory",
|
||||
"path": "/etc/X11/xorg.conf.d",
|
||||
"mode": "0755",
|
||||
"unknown": "program"
|
||||
"mode": "0755"
|
||||
},
|
||||
{
|
||||
"id": "touchpad",
|
||||
"type": "file",
|
||||
"path": "/etc/X11/xorg.conf.d/30-asus-zephyrus-g14-touchpad.conf",
|
||||
"mode": "0644",
|
||||
"template": "templates/X11/xorg.conf.d/30-asus-zephyrus-g14-touchpad.conf"
|
||||
}
|
||||
],
|
||||
"items": [
|
||||
{
|
||||
"collection": "shell:after-wake:normal",
|
||||
"template": "templates/after-wake"
|
||||
},
|
||||
{
|
||||
"collection": "contribution:node-hotkeys:trigger",
|
||||
"template": "templates/hotkeys-triggers"
|
||||
},
|
||||
{
|
||||
"collection": "contribution:node-display-session:config",
|
||||
"name": "display-session",
|
||||
"template": "templates/display-session"
|
||||
"content": "# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.\n#\n# The touchpad's settings, applied by X every time the device appears — at login and after every\n# resume, when the device is initialised again. This replaces the predecessor's sleep hook, which ran\n# xinput after a resume as a named person on a guessed display.\nSection \"InputClass\"\n Identifier \"asus-zephyrus-g14 touchpad\"\n MatchIsTouchpad \"on\"\n Option \"Tapping\" \"on\"\n Option \"NaturalScrolling\" \"true\"\n Option \"AccelSpeed\" \"0.15\"\nEndSection\n"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
@@ -218,12 +194,31 @@
|
||||
"name": "scripts",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
},
|
||||
{
|
||||
"name": "templates",
|
||||
"kind": "archive",
|
||||
"from": "templates"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"contributions": [
|
||||
{
|
||||
"seat": "node-hotkeys",
|
||||
"kind": "trigger",
|
||||
"content": "# The ROG Zephyrus G14's vendor keys, which reach no X client: media (the M-keys), panel brightness,\n# and the touchpad key. Each runs this module's own script, as the operator's account.\nKEY_PROG1\t1\t/usr/local/lib/asus-zephyrus-g14/bin/zephyrus-session /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-media play-pause\nKEY_PROG3\t1\t/usr/local/lib/asus-zephyrus-g14/bin/zephyrus-session /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-media previous\nKEY_PROG4\t1\t/usr/local/lib/asus-zephyrus-g14/bin/zephyrus-session /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-media next\nKEY_BRIGHTNESSDOWN\t1\t/usr/local/lib/asus-zephyrus-g14/bin/zephyrus-backlight -\nKEY_BRIGHTNESSDOWN\t2\t/usr/local/lib/asus-zephyrus-g14/bin/zephyrus-backlight -\nKEY_BRIGHTNESSUP\t1\t/usr/local/lib/asus-zephyrus-g14/bin/zephyrus-backlight +\nKEY_BRIGHTNESSUP\t2\t/usr/local/lib/asus-zephyrus-g14/bin/zephyrus-backlight +\nKEY_F21\t1\t/usr/local/lib/asus-zephyrus-g14/bin/zephyrus-touchpad reset\n"
|
||||
},
|
||||
{
|
||||
"seat": "node-display-session",
|
||||
"kind": "config",
|
||||
"content": "# The laptop's own lines in i3 (module asus-zephyrus-g14, novox/hq ADR 0208, ADR 0210). Owned by the\n# mesh: replaced at every push. Once the controller places contributions to node-display-session\n# (ADR 0210), these become the module's contribution instead of a file in i3's directory.\n#\n# The keys the firmware turns into ordinary key presses. The vendor keys that reach no X client are\n# triggerhappy's (/etc/triggerhappy/triggers.d/asus-g14.conf): M4 and Fn+F4/F5 for media, Fn+F7/F8 for\n# the panel, Fn+F10 for the touchpad. The keyboard backlight (Fn+F2/F3) is the firmware's and asusd's.\n\n# Fn+F6, the screenshot key: the firmware sends Super+Shift+S. Released before it runs, because the\n# screenshot grabs the pointer to select a region, which fails while the key is still held.\nbindsym --release $mod+Shift+s exec --no-startup-id $XDG_CONFIG_HOME/i3/scripts/screenshot.sh\n\n# Fn+F9, the display key: the firmware sends Super+P. Odd workspaces to the panel, even ones to the\n# external output.\nbindsym $mod+p exec --no-startup-id /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-display order\n\n# The keyboard backlight's level, shown when it changes.\nexec --no-startup-id /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-kbd-notify\n"
|
||||
},
|
||||
{
|
||||
"seat": "node-display-session",
|
||||
"kind": "config",
|
||||
"content": "# The laptop's own lines in i3 (module asus-zephyrus-g14, novox/hq ADR 0208, ADR 0210). Owned by the\n# mesh: replaced at every push. Once the controller places contributions to node-display-session\n# (ADR 0210), these become the module's contribution instead of a file in i3's directory.\n#\n# The model's panel and touchpad.\n\n# The internal panel is the primary output, where the bars' tray goes. Which monitors are on and where\n# is the display server's (autorandr, run at every session start).\nexec --no-startup-id /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-display primary\n\n# The touchpad's settings again, by hand. X applies them itself whenever the device appears.\nbindsym $mod+Shift+x exec --no-startup-id /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-touchpad reset\n"
|
||||
}
|
||||
],
|
||||
"shell": [
|
||||
{
|
||||
"for": "after-wake",
|
||||
"slot": "normal",
|
||||
"code": "# The touchpad's settings again after waking, in the operator's session: X applies the module's\n# input class when the device appears, and this covers a wake that does not initialise it again.\n# Runs as root from the power module; the reset itself runs as the session's owner.\nsleep 2\nowner=$(ps -o user= -C i3 | head -n 1)\n[ -n \"$owner\" ] && runuser -u \"$owner\" -- /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-touchpad reset\ntrue\n"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.
|
||||
#
|
||||
# On low battery the machine suspends rather than powering off, at 7 % — s2idle still draws a little,
|
||||
# so it leaves headroom. A drop-in over the package's own UPower.conf, which stays the package's.
|
||||
[UPower]
|
||||
UsePercentageForPolicy=true
|
||||
PercentageLow=15.0
|
||||
PercentageCritical=10.0
|
||||
PercentageAction=7.0
|
||||
CriticalPowerAction=Suspend
|
||||
AllowRiskyCriticalPowerAction=true
|
||||
-12
@@ -1,12 +0,0 @@
|
||||
# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.
|
||||
#
|
||||
# The touchpad's settings, applied by X every time the device appears — at login and after every
|
||||
# resume, when the device is initialised again. This replaces the predecessor's sleep hook, which ran
|
||||
# xinput after a resume as a named person on a guessed display.
|
||||
Section "InputClass"
|
||||
Identifier "asus-zephyrus-g14 touchpad"
|
||||
MatchIsTouchpad "on"
|
||||
Option "Tapping" "on"
|
||||
Option "NaturalScrolling" "true"
|
||||
Option "AccelSpeed" "0.15"
|
||||
EndSection
|
||||
@@ -1,7 +0,0 @@
|
||||
# The touchpad's settings again after waking, in the operator's session: X applies the module's
|
||||
# input class when the device appears, and this covers a wake that does not initialise it again.
|
||||
# Runs as root from the power module; the reset itself runs as the session's owner.
|
||||
sleep 2
|
||||
owner=$(ps -o user= -C i3 | head -n 1)
|
||||
[ -n "$owner" ] && runuser -u "$owner" -- /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-touchpad reset
|
||||
true
|
||||
@@ -1,30 +0,0 @@
|
||||
# The laptop's own lines in i3 (module asus-zephyrus-g14, novox/hq ADR 0208, ADR 0210). Owned by the
|
||||
# mesh: replaced at every push. Once the controller places contributions to node-display-session
|
||||
# (ADR 0210), these become the module's contribution instead of a file in i3's directory.
|
||||
#
|
||||
# The keys the firmware turns into ordinary key presses. The vendor keys that reach no X client are
|
||||
# triggerhappy's (/etc/triggerhappy/triggers.d/asus-g14.conf): M4 and Fn+F4/F5 for media, Fn+F7/F8 for
|
||||
# the panel, Fn+F10 for the touchpad. The keyboard backlight (Fn+F2/F3) is the firmware's and asusd's.
|
||||
|
||||
# Fn+F6, the screenshot key: the firmware sends Super+Shift+S. Released before it runs, because the
|
||||
# screenshot grabs the pointer to select a region, which fails while the key is still held.
|
||||
bindsym --release $mod+Shift+s exec --no-startup-id $XDG_CONFIG_HOME/i3/scripts/screenshot.sh
|
||||
|
||||
# Fn+F9, the display key: the firmware sends Super+P. Odd workspaces to the panel, even ones to the
|
||||
# external output.
|
||||
bindsym $mod+p exec --no-startup-id /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-display order
|
||||
|
||||
# The keyboard backlight's level, shown when it changes.
|
||||
exec --no-startup-id /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-kbd-notify
|
||||
# The laptop's own lines in i3 (module asus-zephyrus-g14, novox/hq ADR 0208, ADR 0210). Owned by the
|
||||
# mesh: replaced at every push. Once the controller places contributions to node-display-session
|
||||
# (ADR 0210), these become the module's contribution instead of a file in i3's directory.
|
||||
#
|
||||
# The model's panel and touchpad.
|
||||
|
||||
# The internal panel is the primary output, where the bars' tray goes. Which monitors are on and where
|
||||
# is the display server's (autorandr, run at every session start).
|
||||
exec --no-startup-id /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-display primary
|
||||
|
||||
# The touchpad's settings again, by hand. X applies them itself whenever the device appears.
|
||||
bindsym $mod+Shift+x exec --no-startup-id /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-touchpad reset
|
||||
@@ -1,10 +0,0 @@
|
||||
# The ROG Zephyrus G14's vendor keys, which reach no X client: media (the M-keys), panel brightness,
|
||||
# and the touchpad key. Each runs this module's own script, as the operator's account.
|
||||
KEY_PROG1 1 /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-session /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-media play-pause
|
||||
KEY_PROG3 1 /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-session /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-media previous
|
||||
KEY_PROG4 1 /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-session /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-media next
|
||||
KEY_BRIGHTNESSDOWN 1 /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-backlight -
|
||||
KEY_BRIGHTNESSDOWN 2 /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-backlight -
|
||||
KEY_BRIGHTNESSUP 1 /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-backlight +
|
||||
KEY_BRIGHTNESSUP 2 /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-backlight +
|
||||
KEY_F21 1 /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-touchpad reset
|
||||
@@ -1,16 +0,0 @@
|
||||
# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.
|
||||
#
|
||||
# The discrete GPU's driver options on the ROG Zephyrus G14 (GA403, RTX 40 series, hybrid graphics).
|
||||
#
|
||||
# NVreg_DynamicPowerManagement=0x00 turns runtime D3 off. With it on, a change of power source sends
|
||||
# the driver an ACPI notification it fails to handle on this model ("RmHandleDNotifierEvent: Failed to
|
||||
# handle ACPI D-Notifier event, status=0x62"), and the GPU stops making progress until the machine is
|
||||
# powered off. Off costs a few idle watts in hybrid mode and keeps the machine up.
|
||||
#
|
||||
# NVreg_PreserveVideoMemoryAllocations=1 saves video memory across suspend, so what used the GPU still
|
||||
# works after waking. It needs nvidia-suspend, -hibernate and -resume to run around a sleep, which this
|
||||
# module's drop-ins on the sleep services ask for.
|
||||
#
|
||||
# A change here applies when the driver next loads: at the next boot.
|
||||
options nvidia NVreg_PreserveVideoMemoryAllocations=1
|
||||
options nvidia NVreg_DynamicPowerManagement=0x00
|
||||
@@ -1,6 +0,0 @@
|
||||
# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.
|
||||
#
|
||||
# The ACPI video driver does not change the backlight itself on the brightness keys: on this model it
|
||||
# moves the wrong one. The keys are triggerhappy's (see /etc/triggerhappy/triggers.d/asus-g14.conf).
|
||||
# Applies when the module next loads: at the next boot.
|
||||
options video brightness_switch_enabled=0
|
||||
-7
@@ -1,7 +0,0 @@
|
||||
# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.
|
||||
#
|
||||
# nvidia-powerd (Dynamic Boost) was the first error in the chain that hung this model's GPU on a change
|
||||
# of power source, and asusd starts it on mains. It runs only when the kernel command line says
|
||||
# zephyrus.nvidia-powerd — an explicit opt-in, at boot.
|
||||
[Unit]
|
||||
ConditionKernelCommandLine=zephyrus.nvidia-powerd
|
||||
-7
@@ -1,7 +0,0 @@
|
||||
# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.
|
||||
#
|
||||
# The NVIDIA driver's own sleep actions, asked for by the sleep itself rather than enabled as
|
||||
# links: the mesh declares files and never makes links (novox/hq ADR 0012), and the host's service
|
||||
# shape must not start these units by hand, which would put the GPU to sleep with the machine awake.
|
||||
[Unit]
|
||||
Wants=nvidia-hibernate.service nvidia-resume.service
|
||||
-7
@@ -1,7 +0,0 @@
|
||||
# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.
|
||||
#
|
||||
# The NVIDIA driver's own sleep actions, asked for by the sleep itself rather than enabled as
|
||||
# links: the mesh declares files and never makes links (novox/hq ADR 0012), and the host's service
|
||||
# shape must not start these units by hand, which would put the GPU to sleep with the machine awake.
|
||||
[Unit]
|
||||
Wants=nvidia-suspend-then-hibernate.service nvidia-resume.service
|
||||
-7
@@ -1,7 +0,0 @@
|
||||
# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.
|
||||
#
|
||||
# The NVIDIA driver's own sleep actions, asked for by the sleep itself rather than enabled as
|
||||
# links: the mesh declares files and never makes links (novox/hq ADR 0012), and the host's service
|
||||
# shape must not start these units by hand, which would put the GPU to sleep with the machine awake.
|
||||
[Unit]
|
||||
Wants=nvidia-suspend.service nvidia-resume.service
|
||||
@@ -1,5 +0,0 @@
|
||||
# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.
|
||||
#
|
||||
# The backlights are writable by the video group, so the brightness keys and the module's brightness tool
|
||||
# move the panel without root.
|
||||
ACTION=="add", SUBSYSTEM=="backlight", RUN+="/usr/bin/chgrp video /sys/class/backlight/%k/brightness", RUN+="/usr/bin/chmod g+w /sys/class/backlight/%k/brightness"
|
||||
@@ -1,11 +0,0 @@
|
||||
# audit-logger
|
||||
|
||||
The `audit-logger` module of the mesh catalogue: declares directories on the node.
|
||||
|
||||
## Files it does not know
|
||||
|
||||
Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416).
|
||||
|
||||
- `${dir:state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
- `${dir:trail}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
- `${dir:spool}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
@@ -47,20 +47,17 @@
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
"place": "."
|
||||
},
|
||||
{
|
||||
"id": "trail",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"unknown": "program"
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "spool",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"unknown": "program"
|
||||
"mode": "0700"
|
||||
}
|
||||
],
|
||||
"capabilities": [
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
# baserow
|
||||
|
||||
The `baserow` module of the mesh catalogue: runs the container image `baserow/baserow`.
|
||||
|
||||
## Files it does not know
|
||||
|
||||
Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416).
|
||||
|
||||
- `${dir:mesh-state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
- `${dir:state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
- `${dir:data}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
@@ -1,4 +0,0 @@
|
||||
{
|
||||
"password": "${secret:admin}",
|
||||
"host": "${bound:route:name}"
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
DATABASE_HOST=${bound:postgres-database:at}
|
||||
DATABASE_PORT=${bound:postgres-database:port}
|
||||
DATABASE_NAME=${bound:postgres-database:as}
|
||||
DATABASE_USER=${bound:postgres-database:as}
|
||||
DATABASE_PASSWORD_FILE=/run/secrets/database
|
||||
DISABLE_EMBEDDED_PSQL=true
|
||||
BASEROW_PUBLIC_URL=https://${bound:route:name}
|
||||
@@ -46,54 +46,31 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"config": {
|
||||
"host": {
|
||||
"type": "string",
|
||||
"description": "The Host header the module's tools send to Baserow, a host name; the file's default is the name the module's route was given.",
|
||||
"trusted": true,
|
||||
"default": "${bound:route:name}"
|
||||
},
|
||||
"email": {
|
||||
"type": "string",
|
||||
"description": "The email address of the Baserow account the module's tools sign in as; unset, the code falls back to its environment.",
|
||||
"trusted": true,
|
||||
"default": ""
|
||||
},
|
||||
"url": {
|
||||
"type": "string",
|
||||
"description": "The address the module's tools reach Baserow at, as a URL; unset, the code falls back to its environment, else this machine's Baserow on port 80.",
|
||||
"trusted": true,
|
||||
"default": ""
|
||||
}
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "mesh-state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"place": "mesh",
|
||||
"unknown": "program"
|
||||
"place": "mesh"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
"place": "."
|
||||
},
|
||||
{
|
||||
"id": "data",
|
||||
"type": "directory",
|
||||
"mode": "0755",
|
||||
"owner": "9999:9999",
|
||||
"unknown": "program"
|
||||
"owner": "9999:9999"
|
||||
},
|
||||
{
|
||||
"id": "server-env",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/server.env",
|
||||
"mode": "0600",
|
||||
"template": "files/server.env"
|
||||
"content": "DATABASE_HOST=${bound:postgres-database:at}\nDATABASE_PORT=${bound:postgres-database:port}\nDATABASE_NAME=${bound:postgres-database:as}\nDATABASE_USER=${bound:postgres-database:as}\nDATABASE_PASSWORD_FILE=/run/secrets/database\nDISABLE_EMBEDDED_PSQL=true\nBASEROW_PUBLIC_URL=https://${bound:route:name}\n"
|
||||
},
|
||||
{
|
||||
"id": "net",
|
||||
@@ -125,9 +102,8 @@
|
||||
"type": "file",
|
||||
"path": "${dir:mesh-state}/config.json",
|
||||
"mode": "0600",
|
||||
"template": "files/config.json",
|
||||
"content": "{\n \"password\": \"${secret:admin}\",\n \"host\": \"${bound:route:name}\"\n}\n",
|
||||
"merge": "json",
|
||||
"takes-config": ["host", "email", "url"],
|
||||
"protected": ["password"]
|
||||
}
|
||||
],
|
||||
@@ -147,11 +123,6 @@
|
||||
"MESH_BASEROW_URL": "http://127.0.0.1:${port:80}",
|
||||
"MESH_BASEROW_CONFIG_FILE": "${dir:mesh-state}/config.json"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -208,7 +208,7 @@ node set up from nothing, the declared accounts are written at the second start
|
||||
|
||||
Five more settings, mesh-wide with a node's layer over each (an entry set to `null` in a node's layer is not on
|
||||
that node). `identities`, `calendars`, `address-books` and `filters` reach the machine as
|
||||
`~/.config/betterbird/configuration.json`, trusted as `accounts` is; `tags` as `tags.json`, trusted too since issue 751 (the operator's decision), so a change to it is the operator's to approve.
|
||||
`~/.config/betterbird/configuration.json`, trusted as `accounts` is; `tags` as `tags.json`, not trusted (§5).
|
||||
**No value holds a password, a token or a cookie**: a key shaped like one is refused, and so is an address whose
|
||||
user part carries a password (§2). A setting holds no line break, so a signature is a list of its lines.
|
||||
|
||||
@@ -347,14 +347,3 @@ outside the mesh. With this version, a push to a node (each waits for the operat
|
||||
- The controller's `settings` verb, for every proposal and to read a layer as it stands; its `nodes` and `node`
|
||||
verbs, for which machines hold the module; and the module's own `betterbird_derive` on those machines
|
||||
(`invokes`).
|
||||
|
||||
## Files it does not know
|
||||
|
||||
Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416).
|
||||
|
||||
- `~/.config/betterbird`: holds only what this module declares and its own tools' records: `accounts-record.json` (`cmd/betterbird-tools/adoption.go:36`), `configuration-record.json` (`owned.go:33`) and the `kept/` folder of replaced filter files (`filters.go:261`); anything else is moved out by the node-engine to `~/.local/state/mesh-moved/betterbird/` with a list, never deleted (ADR 0422 §4).
|
||||
- `/opt/betterbird`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
- `/opt/betterbird/owl`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
- `/opt/betterbird/distribution`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
- `/opt/betterbird/defaults/pref`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
- `/usr/local/lib/mesh-betterbird`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
|
||||
@@ -7,8 +7,7 @@ package main
|
||||
//
|
||||
// - ~/.config/betterbird/configuration.json: identities, calendars, address-books and filters, trusted as
|
||||
// accounts is (§5), because each names where a password, a reply or forwarded mail goes;
|
||||
// - ~/.config/betterbird/tags.json: tags, which names no server and sends no mail; trusted too since issue 751,
|
||||
// so a change to it is the operator's to approve.
|
||||
// - ~/.config/betterbird/tags.json: tags, which names no server and sends no mail, and is not trusted.
|
||||
//
|
||||
// An entry is keyed by a name of the operator's (lower-case letters and digits, as an account's), a tag by its
|
||||
// own key. A node's layer may set an entry to null to take it out on that node, and may give an identity, a
|
||||
|
||||
@@ -25,7 +25,7 @@ type manifest struct {
|
||||
Upgrade map[string]string `json:"upgrade"`
|
||||
Requires []string `json:"requires"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Config map[string]any `json:"config"`
|
||||
Settings map[string]any `json:"settings"`
|
||||
Tools []string `json:"tools"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
Build struct {
|
||||
@@ -114,21 +114,13 @@ func TestItPlacesBetterbirdWhereTheCommunityPackageDidAndOnlyAfterItIsGone(t *te
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0329 §4–§6, ADR 0326, issue 586, ADR 0348: every value reaches the machine as a mergeable file of the
|
||||
// account's that names the keys it takes, and every one is trusted, tags too since issue 751 (the operator's
|
||||
// decision); each is an object whose default is the file's own, empty, never required, since a node without the
|
||||
// operator's setup still runs Betterbird; and each file's default declares nothing.
|
||||
func TestTheValuesAreTrustedMergeableFilesThatNameTheirKeys(t *testing.T) {
|
||||
// ADR 0329 §4–§6, ADR 0326, issue 586: every setting reaches the machine as a mergeable file of the account's that
|
||||
// names the keys it takes; four kinds and accounts and prefs are trusted, tags is not; no setting has a default in the
|
||||
// definition, since each is the operator's own value; and each file's default declares nothing.
|
||||
func TestTheSettingsAreMergeableFilesThatNameTheirKeysAndTagsAloneIsUntrusted(t *testing.T) {
|
||||
m, _ := readManifest(t)
|
||||
if _, said := m.Config["tags"]; !said || len(m.Config) != 7 {
|
||||
t.Fatalf("the seven values are declared under config: %v", m.Config)
|
||||
}
|
||||
for key, raw := range m.Config {
|
||||
d, _ := raw.(map[string]any)
|
||||
def, _ := d["default"].(map[string]any)
|
||||
if d["type"] != "object" || d["trusted"] != true || d["required"] != nil || def == nil || len(def) != 0 {
|
||||
t.Errorf("%s is a trusted object whose default is empty and which is not required: %v", key, d)
|
||||
}
|
||||
if len(m.Settings) != 0 {
|
||||
t.Fatalf("no setting has a default in the definition: the operator's setup is the settings' mesh-wide layer (issue 586, ADR 0262): %v", m.Settings)
|
||||
}
|
||||
files := map[string]struct {
|
||||
path, mode string
|
||||
@@ -138,7 +130,7 @@ func TestTheValuesAreTrustedMergeableFilesThatNameTheirKeys(t *testing.T) {
|
||||
"prefs": {prefsFile, "0644", []any{"prefs"}, true},
|
||||
"accounts": {accountsFile, "0600", []any{"accounts"}, true},
|
||||
"configuration": {configurationFile, "0600", []any{"identities", "calendars", "address-books", "filters"}, true},
|
||||
"tags": {tagsFile, "0644", []any{"tags"}, true},
|
||||
"tags": {tagsFile, "0644", []any{"tags"}, false},
|
||||
}
|
||||
merging := 0
|
||||
for _, r := range m.Resources {
|
||||
@@ -152,27 +144,27 @@ func TestTheValuesAreTrustedMergeableFilesThatNameTheirKeys(t *testing.T) {
|
||||
for id, want := range files {
|
||||
r := resource(t, m, id)
|
||||
if r["path"] != "${machine:account-home}/"+want.path || r["merge"] != "json" || r["trusted"] != want.trusted ||
|
||||
r["mode"] != want.mode || r["owner"] != "${machine:account}" || !reflect.DeepEqual(r["takes-config"], want.takes) {
|
||||
r["mode"] != want.mode || r["owner"] != "${machine:account}" || !reflect.DeepEqual(r["takes-settings"], want.takes) {
|
||||
t.Errorf("%s: %v", id, r)
|
||||
}
|
||||
var content map[string]any
|
||||
if err := json.Unmarshal([]byte(templateText(t, r)), &content); err != nil || len(content) != len(want.takes) {
|
||||
if err := json.Unmarshal([]byte(r["content"].(string)), &content); err != nil || len(content) != len(want.takes) {
|
||||
t.Errorf("%s's default holds each key it takes, empty: %v %v", id, content, err)
|
||||
}
|
||||
}
|
||||
got, err := parseAccounts([]byte(templateText(t, resource(t, m, "accounts"))))
|
||||
got, err := parseAccounts([]byte(resource(t, m, "accounts")["content"].(string)))
|
||||
if err != nil || len(got) != 0 {
|
||||
t.Fatalf("its default declares no account: %v %v", got, err)
|
||||
}
|
||||
prefs, err := parseDeclaredPrefs([]byte(templateText(t, resource(t, m, "prefs"))))
|
||||
prefs, err := parseDeclaredPrefs([]byte(resource(t, m, "prefs")["content"].(string)))
|
||||
if err != nil || len(prefs) != 0 {
|
||||
t.Fatalf("its default declares no preference: %v %v", prefs, err)
|
||||
}
|
||||
var c Configuration
|
||||
if err := decodeInto([]byte(templateText(t, resource(t, m, "configuration"))), configurationFile, &c); err != nil {
|
||||
if err := decodeInto([]byte(resource(t, m, "configuration")["content"].(string)), configurationFile, &c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := decodeInto([]byte(templateText(t, resource(t, m, "tags"))), tagsFile, &c); err != nil {
|
||||
if err := decodeInto([]byte(resource(t, m, "tags")["content"].(string)), tagsFile, &c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -276,13 +268,13 @@ func TestThePolicyTurnsTheUpdaterOffAndInstallsTheMeshsOwlEverywhere(t *testing.
|
||||
func TestTheStartPreparesThenStartsBetterbirdAndTheLauncherUsesIt(t *testing.T) {
|
||||
m, _ := readManifest(t)
|
||||
start := resource(t, m, "start")
|
||||
content := templateText(t, start)
|
||||
content := start["content"].(string)
|
||||
if start["path"] != "/usr/local/bin/betterbird" || start["mode"] != "0755" ||
|
||||
!strings.Contains(content, "\n"+preparation+" prepare ||") || !strings.HasSuffix(content, "\nexec "+startBin+" \"$@\"\n") {
|
||||
t.Fatalf("the start runs the preparation, says when it did not complete, and starts Betterbird either way: %v", start)
|
||||
}
|
||||
launcher := resource(t, m, "launcher")
|
||||
lc := templateText(t, launcher)
|
||||
lc := launcher["content"].(string)
|
||||
if launcher["path"] != "/usr/local/share/applications/eu.betterbird.Betterbird.desktop" ||
|
||||
!strings.Contains(lc, "\nExec=/usr/local/bin/betterbird %u\n") || !strings.Contains(lc, "\nIcon="+installDir+"/") ||
|
||||
!strings.Contains(lc, "\nStartupWMClass=eu.betterbird.Betterbird\n") {
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The module's directory, where its manifest and its files/ are.
|
||||
const moduleDir = "../.."
|
||||
|
||||
// templateText is the text a file resource places on the machine: its template's, in the module's files/
|
||||
// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test.
|
||||
func templateText(t *testing.T, r map[string]any) string {
|
||||
t.Helper()
|
||||
name, _ := r["template"].(string)
|
||||
if name == "" {
|
||||
t.Fatalf("resource %v names no template", r["id"])
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name)))
|
||||
if err != nil {
|
||||
t.Fatalf("resource %v names %s: %v", r["id"], name, err)
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
// No file resource carries its text inline: each names its template in files/, and the build packs files/
|
||||
// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2).
|
||||
func TestEveryFileTakesItsTextFromATemplate(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Resources []map[string]any `json:"resources"`
|
||||
Build struct {
|
||||
Artifacts []map[string]any `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if _, inline := r["content"]; inline {
|
||||
t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"])
|
||||
}
|
||||
if r["type"] == "file" {
|
||||
templateText(t, r)
|
||||
}
|
||||
}
|
||||
packs := 0
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a["kind"] == "archive" && a["from"] == "files" {
|
||||
packs++
|
||||
if a["name"] != "files" {
|
||||
t.Errorf("the templates archive is named %v, not files", a["name"])
|
||||
}
|
||||
}
|
||||
}
|
||||
if packs != 1 {
|
||||
t.Errorf("the build packs files/ into %d archives, not one", packs)
|
||||
}
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
{
|
||||
"accounts": {}
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
#!/bin/sh
|
||||
# Betterbird's start, written by the mesh (module betterbird, novox/hq ADR 0316). Replaced at every push.
|
||||
#
|
||||
# Before Betterbird starts, the module's tools place its user.js in the profile (the settings prefs, accounts, identities, calendars,
|
||||
# address-books and tags) and write what Betterbird rewrites itself: the account lists and each declared account's filters. They do
|
||||
# it only while no Betterbird of this account runs, because Betterbird rewrites its saved preferences when it quits; a
|
||||
# second start while one runs only hands over to it. What the preparation could not do it says here, and
|
||||
# betterbird_check says it too. Betterbird starts either way: its configuration never holds the mail back.
|
||||
/usr/local/lib/mesh-betterbird/betterbird-tools prepare || echo "betterbird: the mesh's preparation did not complete; betterbird_check says why" >&2
|
||||
exec /opt/betterbird/betterbird "$@"
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"identities": {},
|
||||
"calendars": {},
|
||||
"address-books": {},
|
||||
"filters": {}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
[Desktop Entry]
|
||||
Version=1.0
|
||||
Type=Application
|
||||
Name=Betterbird
|
||||
GenericName=Mail Client
|
||||
Comment=Send and receive mail with Betterbird
|
||||
Exec=/usr/local/bin/betterbird %u
|
||||
Icon=/opt/betterbird/chrome/icons/default/default128.png
|
||||
Terminal=false
|
||||
Categories=Network;Email;
|
||||
MimeType=message/rfc822;x-scheme-handler/mailto;application/x-xpinstall;application/x-extension-ics;text/calendar;text/vcard;text/x-vcard;x-scheme-handler/webcal;x-scheme-handler/webcals;x-scheme-handler/mid;
|
||||
StartupNotify=true
|
||||
StartupWMClass=eu.betterbird.Betterbird
|
||||
Keywords=Email;E-mail;Calendar;Thunderbird;
|
||||
Actions=ComposeMessage;OpenAddressBook;
|
||||
# Written by the mesh (module betterbird, novox/hq ADR 0316 §3), in place of the community package's entry:
|
||||
# it starts Betterbird through the module's start, and names the icon in the installation, with no link.
|
||||
|
||||
[Desktop Action ComposeMessage]
|
||||
Name=Write new message
|
||||
Exec=/usr/local/bin/betterbird -compose
|
||||
|
||||
[Desktop Action OpenAddressBook]
|
||||
Name=Open address book
|
||||
Exec=/usr/local/bin/betterbird -addressbook
|
||||
@@ -1,3 +0,0 @@
|
||||
{
|
||||
"prefs": {}
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
{
|
||||
"tags": {}
|
||||
}
|
||||
@@ -33,59 +33,13 @@
|
||||
"betterbird_drift",
|
||||
"betterbird_reminder_test"
|
||||
],
|
||||
"config": {
|
||||
"prefs": {
|
||||
"type": "object",
|
||||
"description": "Betterbird's preferences the operator sets, by their own names (mailnews.*, calendar.* and the like), laid into prefs.json; an account's, a server's and a tag's are refused, as accounts, the four kinds and tags own them (novox/hq ADR 0329 §6).",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"accounts": {
|
||||
"type": "object",
|
||||
"description": "The mail accounts Betterbird is set up with, each by a name of the operator's: its address, its sender's name and its incoming and outgoing servers; no password, token or cookie (novox/hq ADR 0329 §2).",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"identities": {
|
||||
"type": "object",
|
||||
"description": "The identities beside an account's own, each by a name of the operator's: the address and name it sends as, its signature as a list of lines, and the profile's key to adopt on a node.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"calendars": {
|
||||
"type": "object",
|
||||
"description": "The calendars Betterbird shows, each by a name of the operator's: where it is and how it is reached, and the profile's key to adopt on a node.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"address-books": {
|
||||
"type": "object",
|
||||
"description": "The address books Betterbird uses, each by a name of the operator's: where it is and how it is reached, and the profile's key to adopt on a node.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"filters": {
|
||||
"type": "object",
|
||||
"description": "The message filters of each account, by account: what each matches and what it does with the mail.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"tags": {
|
||||
"type": "object",
|
||||
"description": "The tags Betterbird offers, each by its own key: its name, colour and order; trusted since issue 751, so a change to it is the operator's to approve.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
}
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "configuration-dir",
|
||||
"type": "directory",
|
||||
"path": "${machine:account-home}/.config/betterbird",
|
||||
"owner": "${machine:account}",
|
||||
"mode": "0755",
|
||||
"unknown": "move-out",
|
||||
"knows": ["accounts-record.json", "configuration-record.json", "kept/"]
|
||||
"mode": "0755"
|
||||
},
|
||||
{
|
||||
"id": "prefs",
|
||||
@@ -94,10 +48,10 @@
|
||||
"owner": "${machine:account}",
|
||||
"mode": "0644",
|
||||
"merge": "json",
|
||||
"takes-config": [
|
||||
"takes-settings": [
|
||||
"prefs"
|
||||
],
|
||||
"template": "files/prefs.json",
|
||||
"content": "{\n \"prefs\": {}\n}\n",
|
||||
"trusted": true
|
||||
},
|
||||
{
|
||||
@@ -107,10 +61,10 @@
|
||||
"owner": "${machine:account}",
|
||||
"mode": "0600",
|
||||
"merge": "json",
|
||||
"takes-config": [
|
||||
"takes-settings": [
|
||||
"accounts"
|
||||
],
|
||||
"template": "files/accounts.json",
|
||||
"content": "{\n \"accounts\": {}\n}\n",
|
||||
"trusted": true
|
||||
},
|
||||
{
|
||||
@@ -120,13 +74,13 @@
|
||||
"owner": "${machine:account}",
|
||||
"mode": "0600",
|
||||
"merge": "json",
|
||||
"takes-config": [
|
||||
"takes-settings": [
|
||||
"identities",
|
||||
"calendars",
|
||||
"address-books",
|
||||
"filters"
|
||||
],
|
||||
"template": "files/configuration.json",
|
||||
"content": "{\n \"identities\": {},\n \"calendars\": {},\n \"address-books\": {},\n \"filters\": {}\n}\n",
|
||||
"trusted": true
|
||||
},
|
||||
{
|
||||
@@ -136,11 +90,11 @@
|
||||
"owner": "${machine:account}",
|
||||
"mode": "0644",
|
||||
"merge": "json",
|
||||
"takes-config": [
|
||||
"takes-settings": [
|
||||
"tags"
|
||||
],
|
||||
"template": "files/tags.json",
|
||||
"trusted": true
|
||||
"content": "{\n \"tags\": {}\n}\n",
|
||||
"trusted": false
|
||||
},
|
||||
{
|
||||
"id": "community-package",
|
||||
@@ -152,50 +106,45 @@
|
||||
"id": "installation",
|
||||
"type": "archive",
|
||||
"path": "/opt/betterbird",
|
||||
"artifact": "archive",
|
||||
"unknown": "program"
|
||||
"artifact": "archive"
|
||||
},
|
||||
{
|
||||
"id": "owl",
|
||||
"type": "archive",
|
||||
"path": "/opt/betterbird/owl",
|
||||
"artifact": "owl",
|
||||
"unknown": "program"
|
||||
"artifact": "owl"
|
||||
},
|
||||
{
|
||||
"id": "policies",
|
||||
"type": "archive",
|
||||
"path": "/opt/betterbird/distribution",
|
||||
"artifact": "distribution",
|
||||
"unknown": "program"
|
||||
"artifact": "distribution"
|
||||
},
|
||||
{
|
||||
"id": "spelling",
|
||||
"type": "archive",
|
||||
"path": "/opt/betterbird/defaults/pref",
|
||||
"artifact": "defaults",
|
||||
"unknown": "program"
|
||||
"artifact": "defaults"
|
||||
},
|
||||
{
|
||||
"id": "preparation",
|
||||
"type": "archive",
|
||||
"path": "/usr/local/lib/mesh-betterbird",
|
||||
"artifact": "tools",
|
||||
"unknown": "program"
|
||||
"artifact": "tools"
|
||||
},
|
||||
{
|
||||
"id": "start",
|
||||
"type": "file",
|
||||
"path": "/usr/local/bin/betterbird",
|
||||
"mode": "0755",
|
||||
"template": "files/betterbird"
|
||||
"content": "#!/bin/sh\n# Betterbird's start, written by the mesh (module betterbird, novox/hq ADR 0316). Replaced at every push.\n#\n# Before Betterbird starts, the module's tools place its user.js in the profile (the settings prefs, accounts, identities, calendars,\n# address-books and tags) and write what Betterbird rewrites itself: the account lists and each declared account's filters. They do\n# it only while no Betterbird of this account runs, because Betterbird rewrites its saved preferences when it quits; a\n# second start while one runs only hands over to it. What the preparation could not do it says here, and\n# betterbird_check says it too. Betterbird starts either way: its configuration never holds the mail back.\n/usr/local/lib/mesh-betterbird/betterbird-tools prepare || echo \"betterbird: the mesh's preparation did not complete; betterbird_check says why\" >&2\nexec /opt/betterbird/betterbird \"$@\"\n"
|
||||
},
|
||||
{
|
||||
"id": "launcher",
|
||||
"type": "file",
|
||||
"path": "/usr/local/share/applications/eu.betterbird.Betterbird.desktop",
|
||||
"mode": "0644",
|
||||
"template": "files/eu.betterbird.Betterbird.desktop"
|
||||
"content": "[Desktop Entry]\nVersion=1.0\nType=Application\nName=Betterbird\nGenericName=Mail Client\nComment=Send and receive mail with Betterbird\nExec=/usr/local/bin/betterbird %u\nIcon=/opt/betterbird/chrome/icons/default/default128.png\nTerminal=false\nCategories=Network;Email;\nMimeType=message/rfc822;x-scheme-handler/mailto;application/x-xpinstall;application/x-extension-ics;text/calendar;text/vcard;text/x-vcard;x-scheme-handler/webcal;x-scheme-handler/webcals;x-scheme-handler/mid;\nStartupNotify=true\nStartupWMClass=eu.betterbird.Betterbird\nKeywords=Email;E-mail;Calendar;Thunderbird;\nActions=ComposeMessage;OpenAddressBook;\n# Written by the mesh (module betterbird, novox/hq ADR 0316 §3), in place of the community package's entry:\n# it starts Betterbird through the module's start, and names the icon in the installation, with no link.\n\n[Desktop Action ComposeMessage]\nName=Write new message\nExec=/usr/local/bin/betterbird -compose\n\n[Desktop Action OpenAddressBook]\nName=Open address book\nExec=/usr/local/bin/betterbird -addressbook\n"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
@@ -242,11 +191,6 @@
|
||||
"name": "defaults",
|
||||
"kind": "archive",
|
||||
"from": "install/defaults-pref"
|
||||
},
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,144 +0,0 @@
|
||||
# brother — the Brother multifunction device: scan, printer state, admin pages
|
||||
|
||||
One module for one device (novox/hq ADR 0467, issue 1453): the scanner and the printer are one machine on the
|
||||
network, so one module serves both. It claims two seats, each held once for the mesh: `scanner` (verbs `scan`,
|
||||
`status`, `list`, `document`) and `printer` (verbs `status`, `settings`, `print`, `upload`; `restart` follows in a later round). It replaces the
|
||||
`scan` module of issue 1440 (ADR 0464), which ran on no node and is gone from the catalogue.
|
||||
|
||||
## The scanner seat
|
||||
|
||||
Built for issue 1440 as novox/hq ADR 0464 decides. It scans the glass of a network scanner (any scanner that
|
||||
speaks eSCL, also called AirScan or Mopria), cuts the document out of the lid's white, files it as a one-page PDF
|
||||
in a folder and says `scan.added` on the bus, so other modules can react. A scan happens when a tool is called:
|
||||
the module runs no process of its own, and it runs as the node's operator account.
|
||||
|
||||
## What a scan does
|
||||
|
||||
1. Asks the scanner for its sizes and puts one job to it (`POST <scanner>/ScanJobs`) at the configured resolution
|
||||
and colour mode, as a JPEG: from the glass (`source` glass, the default) or from the document feeder (`source`
|
||||
feeder, with `duplex` true for both sides of each sheet; refused on the glass). The region is A4, cut to what the
|
||||
input can scan (the glass is 3507 high, not A4's 3508). The body is written on one line: the Brother ignores the
|
||||
whole body, and runs its default job, when it holds a line break (issue 1452). The glass page is fetched once;
|
||||
a feeder job is asked `NextDocument` until the scanner answers 404, at most 100 pages. A job that fails is deleted.
|
||||
2. Reads the resolution from the image's own JFIF density, never from the request: a scanner may answer a
|
||||
300 dpi request with 200 dpi, and the page size and the event's millimetres follow what it gave.
|
||||
3. Crops the glass page (a feeder page is a whole sheet and is never cropped): the lid's level is measured in the corner farthest from where documents lie, a pixel is document when
|
||||
it is darker than the lid, and the rows and columns holding enough of them give the box. No document, or one
|
||||
filling nearly the whole glass, keeps the whole image (`cropped` false), and so does `crop` false.
|
||||
With `apart` true (opt-in; without it the glass gives one document, as it always did), several documents lying apart on the glass are each cut out and filed as a one-page PDF of its own (own
|
||||
object, own `scan.added`, own file); the answer then lists them as `documents` (each with its `path`) and
|
||||
`count`, in reading order, and one that could not be kept or filed is named in `failed` without holding back
|
||||
the others. The rule (`Boxes` in `crop.go`): the picture is cut into blocks of 1 mm; a block is document when a
|
||||
quarter of its pixels are darker than the lid; blocks not at least three across (dust, JPEG ringing) are dropped;
|
||||
blocks within 10 mm of each other are one region, so a white band narrower than that does not split a document;
|
||||
a region under 20 mm on a side is a stain, not a document; regions whose boxes overlap are one document (a frame
|
||||
and the text inside it). One document, or none, is cropped exactly as before.
|
||||
4. Files a PDF holding the JPEGs, one page for the glass and one per image for the feeder (so six for three
|
||||
sheets both sides), each page as large as its image really is: written whole beside its place as `.scan-*.tmp` and renamed to
|
||||
`scan-YYYY-MM-DD-HHMMSS.pdf` (local time); a taken name gets `-2`, `-3`, and no file is ever overwritten.
|
||||
5. Says `scan.added` once, after the rename, under the file's sha256 as its message id. It never carries the image.
|
||||
|
||||
A scanner that refuses or gives no page files nothing and says nothing on the bus; the tool's error says why.
|
||||
|
||||
## Config
|
||||
|
||||
| key | meaning | default |
|
||||
|---|---|---|
|
||||
| `scanner` | the scanner's eSCL address, such as `http://192.0.2.10/eSCL` (required, trusted) | none |
|
||||
| `scan-folder` | the absolute path the PDFs are filed in (required, trusted) | none |
|
||||
| `printer` | the printer's host, such as `192.0.2.10`, for its admin page and IPP (required, trusted) | none |
|
||||
| `printer-certificate` | the SHA-256 fingerprint of the certificate the admin page may serve, `AA:BB:...`; several, separated by commas, when it serves more than one (required, trusted) | none |
|
||||
| `resolution` | dpi asked: 100, 200, 300 or 600 | 300 |
|
||||
| `color-mode` | `color` or `gray` | `color` |
|
||||
| `crop` | cut the document out of the lid's white | true |
|
||||
|
||||
The values reach the code in one file, `settings.json` in the module's state directory, which the mesh writes
|
||||
from the config; the code reads it at every call. With `scanner` or `scan-folder` unset, every scanner verb says which and
|
||||
writes and asks nothing; `printer` and `printer-certificate` are the printer verbs' to say.
|
||||
|
||||
The scan folder is not a manifest resource: a directory's path cannot take a config value (issue 1441), so the code
|
||||
makes it (mode 0755) when it files the first scan.
|
||||
|
||||
## Tools
|
||||
|
||||
| verb (tool) | does |
|
||||
|---|---|
|
||||
| `scanner.scan` | `source` (`glass`, default, or `feeder`), `duplex` (feeder only) and `apart` (glass only): asks the device process to scan, crop (glass only), keep the PDF in the bucket under its sha256 and say `scan.added`; then fetches it in parts, checks the sha256 and files it in `scan-folder`, in the background, so a scan is filed whether or not the caller still waits. A glass scan is waited for up to 180 s. Several glass pages in one PDF (a page turned over, a stapled document): `document` `start` scans page 1 into a new document and answers its id (nothing is filed yet), `document` with that id scans the next page, and `document` with the id and `finish` true closes it into one multi-page PDF, filed once with one `scan.added` (nothing is scanned). The pages are kept in the bucket under `pending/<id>/` (not in memory, so a restart of the device process loses none); a document that gets no page for 30 minutes is closed and filed so by the tool process (a restart of the tool process forgets that timer; `finish` still works). The pages in the bucket are not deleted, the bucket having no delete. A feeder scan is waited for 25 s (the mesh console gives up after 30 s); if it is not done, the answer is `started: true` with its `id`, and `scanner.scan {id}` reads the outcome (`state` running, done with the event's body and `path`, or failed with `why`). The device process ends a feeder job at 20 s a page (at least 3 min, at most 30 min). A scan whose PDF the tool could not fetch (over 100 MiB) is refused before it is kept or announced. Answers the event's body and `path` |
|
||||
| `scanner.document` | one part of a kept scan (`object`, `part` from 1): `part`, `parts`, `size`, `sha256` (of the whole), `data` (base64). Join the parts and check the sha256 |
|
||||
| `scanner.status` | whether the scanner answers, its state (Idle or Processing) and its document feeder's state; a scanner that does not answer is `answers: false` with why. |
|
||||
| `brother_health` | the module's health, asked by the node-engine every 60 s (ADR 0240) as the tool check of the `watch` process, beside the device process's unit check: the certificates the printer serves, one for each key type (ECDSA and RSA, told apart by the cipher suites offered under TLS 1.2), read with no login and compared with the pinned `printer-certificate` list. Unhealthy when one is not pinned, `why` naming the new fingerprint, so the pin can be changed before the admin pages are needed. With `printer` or `printer-certificate` not given it answers `state: waits` with `waits` naming each setting (the SDK's health.Waiting, ADR 0283), not an error. Each handshake is bounded by 3 s and the whole check by 7 s. A printer that does not answer is healthy, with `why` saying nothing was compared (it may be switched off). The process's liveness is judged by the node-engine without a declaration |
|
||||
| `scanner.list` | the newest filed scans, newest first (`limit`, default 20). Read-only |
|
||||
|
||||
## The printer seat
|
||||
|
||||
| verb (tool) | does |
|
||||
|---|---|
|
||||
| `printer.status` | without a login: over IPP (`Get-Printer-Attributes`, encoded by hand, on port 631) the state (idle, processing, stopped), the state reasons (such as `toner-low-warning`) and the markers' names, levels (percent; below 0 is unknown) and colours; over eSCL the scanner's state and its feeder's. A part that does not answer is `answers: false` with why. Needs `printer` and `scanner` |
|
||||
| `printer.upload` | one part of a document to print later (`upload` id, `part`, `parts`, `sha256` of the whole, `data` base64): stored as `uploads/<upload>.pdf` or `.jpg` once all parts came and the sha256 matches; a wrong sha256, a part out of range or a document that is neither PDF nor JPEG stores nothing |
|
||||
| `printer.print` | prints a file on the machine the tool runs on (`path`, absolute, a regular file under the tool account's home folder after symbolic links are resolved; any other path is refused with the one message "path refused: only a file under the home folder", which says nothing of whether it exists: the tool reads it, uploads it in parts and prints it in one call; PDF or JPEG by its bytes, never its name), a kept scan (`object`) or a completed upload (`upload`, for modules that send bytes over the bus) as an IPP Print-Job (operation 0x0002, port 631) in its own format, `copies` 1 to 99 (default 1), `sides` one-sided, two-sided-long-edge or two-sided-short-edge (default one-sided), `media` (default `iso_a4_210x297mm`); answers the job's id and state (with `path`, the upload id and the size when `path` was given) |
|
||||
| `printer.settings` | asks the device process (below) to log in, read one admin page and log out, and waits up to 60 s for its answer. Argument `page`: a path on the admin site, starting with `/` and ending in `.html` (default `/general/status.html`). Answers the page's title and forms with their fields, each with its current value (a radio group as one field with the checked button's value, a checkbox with `checked`, a select with the selected option) and `options`; the page's links only with `links: true`, hidden fields only with `hidden: true`; a password field's value is never given. Read-only: it sends the login and the logout and nothing else |
|
||||
|
||||
### Trust and the password
|
||||
|
||||
The admin page is HTTPS with a self-signed certificate. The module verifies no chain: it accepts the one certificate
|
||||
whose SHA-256 fingerprint is one of those in `printer-certificate` and refuses any other, in the handshake, so no
|
||||
password is sent to it; the error names the served and the pinned fingerprints. Give each as 32 upper-case hex pairs
|
||||
joined by colons, several separated by commas. The printer serves two self-signed certificates for one name, an
|
||||
ECDSA one and an RSA one, and which it serves depends on the signature algorithms the client offers, so pin both
|
||||
(issue 1503): read them with `openssl s_client -connect <printer>:443 -sigalgs ECDSA+SHA256` and `-sigalgs
|
||||
RSA-PSS+SHA256`, each piped to `openssl x509 -noout -fingerprint -sha256`.
|
||||
|
||||
The admin password is the module's own secret `admin-password` (ADR 0467 clause 4; ADR 0283): the operator gives it
|
||||
at their desk, and the mesh never generates or rotates it. It is owned by the account `brother-device`, which the
|
||||
module makes, because the tools run as the operator account and must never read it (ADR 0436 §1, §12; the shape of
|
||||
mesh-mail's sender). The file is `${dir:state}/admin-password.secret`, mode 0600, and `BROTHER_ADMIN_PASSWORD` names
|
||||
it for the device process alone; the tools' bundle is given no such name, and `admin_boundary_test.go` checks that
|
||||
no tool file imports the package that reads it. While it is not given, `printer.settings` answers that the secret
|
||||
is not given, and the printer is not contacted.
|
||||
|
||||
### Documents in parts
|
||||
|
||||
The bus refuses a message over 1 MiB, so a document crosses it in parts of at most 700 KiB (both the size this module
|
||||
cuts and the most it accepts, as ADR 0468 clause 3 says),
|
||||
counted from 1, each carried base64-encoded in JSON. A caller joins the parts and checks the SHA-256. The bucket
|
||||
(`s3-bucket`, from `minio`) is spoken to by the device process alone, from `store.json` (owner `brother-device`, mode
|
||||
0600), with the S3 client mesh-mail uses; a scan is kept once as `<sha256>.pdf` and an upload as `uploads/<id>.<ext>`.
|
||||
|
||||
### The device process
|
||||
|
||||
`brother device` runs as `brother-device` with a bus account of its own (the own secret `broker-device`), never a
|
||||
seat, and does all the device and bucket work: the scan (and `scan.added`, emitted by it with the object's key and no
|
||||
path, since filing in the scan folder is the tool's), the documents, the printing, and the admin pages. It consumes the work-queue stream `device-asks` (event `device.asked`, body `id`, `page`, `asked-at`) and
|
||||
writes each answer, once, into the state `device-answers` under the ask's id (ten minutes' life): the page's
|
||||
title, links and form fields, or `error`. It alone reads the password, pins the certificate, logs in, reads the
|
||||
page and logs out. An ask older than two minutes is dropped without contacting the printer; at most three
|
||||
deliveries. It reads `settings.json` (mode 0644, no secret in it) and the state directory is 0711 so that it can
|
||||
reach its own files; both are the choices that let its account read what it needs. The device process is judged by its unit; `brother_health` (the printer's certificates against the pins) is the tool check of the `watch` process beside it.
|
||||
|
||||
The login reads the form each time: the device generates the password field's name, so the code takes the name from
|
||||
the form, posts it with the form's hidden fields, and keeps the session cookie. A login is failed when the answer is
|
||||
the login page again. The logout follows a logout link or form on the page; a page with none is said in the answer
|
||||
(`logout`), and the session is left to expire on the device.
|
||||
|
||||
## The event
|
||||
|
||||
`scan.added`, said by the device process once the PDF is stored, before the file is written (the tool side files it afterwards, from the store, by a link the device signed; the verb's answer adds the file's `path`), on the wire `mesh.mod.brother.event.scan.added`; a consumer declares `brother.scan.added`. Its body:
|
||||
`object` (the key of the stored PDF), `node`, `bytes`, `sha256`, `width-mm`, `height-mm`, `resolution` (dpi, as the scanner gave it),
|
||||
`color-mode`, `cropped`, `pages` (1 for the glass), `scanned-at` (RFC 3339, UTC).
|
||||
|
||||
## Files it does not know
|
||||
|
||||
Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416).
|
||||
|
||||
- `${dir:state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
- The scan folder (`scan-folder`) is declared by no manifest. The module writes only the `scan-*.pdf` files it
|
||||
files and their `.scan-*.tmp` while they are written; it never changes, moves or removes any other file there,
|
||||
and `scanner.list` only reads.
|
||||
|
||||
## Tests
|
||||
|
||||
The unit tests use the captured answers of a real scanner (`cmd/brother/testdata/escl`, captured 2026-10-20) served
|
||||
by a test HTTP server, the captured IPP exchange (`testdata/ipp`) replayed, and the captured login page
|
||||
(`testdata/admin/login-page.html`) served by a test TLS server with its own certificate. The pages behind the login
|
||||
are synthetic (`testdata/admin/synthetic-*.html`) until round 2 captures the real ones. No test reaches the real
|
||||
device.
|
||||
@@ -1,231 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Admin is a session on the printer's admin web page, over HTTPS to the one certificate pinned (ADR 0467
|
||||
// clauses 3 and 4). The device names the password field itself, so every login reads the form first.
|
||||
type Admin struct {
|
||||
Host string
|
||||
Password string
|
||||
HTTP *http.Client
|
||||
}
|
||||
|
||||
// loginStart is where a login starts: the root, which the printer answers with a redirect to the page it serves
|
||||
// its login form on (/home/status.html). Asking for a page behind the login instead is answered by a redirect
|
||||
// to an error page, and a loginurl naming a page this firmware lacks is a 404 after the login (issue 1514).
|
||||
const loginStart = "/"
|
||||
|
||||
// maxPage bounds a page of the admin site.
|
||||
const maxPage = 4 << 20
|
||||
|
||||
// NewAdmin is a client for the printer's host that trusts the certificate with this fingerprint alone, with a
|
||||
// cookie jar for the session.
|
||||
func NewAdmin(host, fingerprint, password string) (*Admin, error) {
|
||||
jar, err := cookiejar.New(nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Admin{
|
||||
Host: host,
|
||||
Password: password,
|
||||
HTTP: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
Jar: jar,
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
return checkRedirect(host, req, via)
|
||||
},
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: pinnedTLS(fingerprint),
|
||||
DisableKeepAlives: true,
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// checkRedirect follows a redirect only on https on the printer's own host, and never with a body: a 307 or 308
|
||||
// answer to the login post would make the client send the post again, the password with it (issue 1515).
|
||||
func checkRedirect(host string, req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 10 {
|
||||
return errors.New("the printer redirected more than ten times")
|
||||
}
|
||||
if req.URL.Scheme != "https" || req.URL.Host != host {
|
||||
return fmt.Errorf("the printer redirected to %s, not to https on the printer %s: not followed", req.URL.Scheme+"://"+req.URL.Host, host)
|
||||
}
|
||||
if req.Method != http.MethodGet && req.Method != http.MethodHead {
|
||||
return fmt.Errorf("the printer redirected a %s with its body (a 307 or 308): not followed, so the password is not sent again", via[len(via)-1].Method)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *Admin) url(path string) string { return "https://" + a.Host + path }
|
||||
|
||||
// fetch is one request; the body is read whole, up to maxPage. The final URL is the one after redirects.
|
||||
func (a *Admin) fetch(ctx context.Context, method, rawURL string, form url.Values) ([]byte, *url.URL, error) {
|
||||
var req *http.Request
|
||||
var err error
|
||||
if form != nil {
|
||||
req, err = http.NewRequestWithContext(ctx, method, rawURL, strings.NewReader(form.Encode()))
|
||||
if err == nil {
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
}
|
||||
} else {
|
||||
req, err = http.NewRequestWithContext(ctx, method, rawURL, nil)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
resp, err := a.HTTP.Do(req)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
buf, err := io.ReadAll(io.LimitReader(resp.Body, maxPage+1))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the page %s cannot be read: %w", rawURL, err)
|
||||
}
|
||||
if len(buf) > maxPage {
|
||||
return nil, nil, fmt.Errorf("the page %s is larger than %d bytes", rawURL, maxPage)
|
||||
}
|
||||
if resp.StatusCode >= 400 {
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
return nil, nil, ¬FoundError{Path: resp.Request.URL.Path}
|
||||
}
|
||||
return nil, nil, fmt.Errorf("the printer answered %s for %s", resp.Status, resp.Request.URL.Path)
|
||||
}
|
||||
return buf, resp.Request.URL, nil
|
||||
}
|
||||
|
||||
// passwordForm is the form of a page that holds a password field, or nil.
|
||||
func passwordForm(p Page) (*Form, *Field) {
|
||||
for i := range p.Forms {
|
||||
for j := range p.Forms[i].Fields {
|
||||
if p.Forms[i].Fields[j].Type == "password" {
|
||||
return &p.Forms[i], &p.Forms[i].Fields[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func hasField(f *Form, name string) bool {
|
||||
for _, fl := range f.Fields {
|
||||
if fl.Name == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Login asks for the printer's login page, reads its login form, and posts the form's own fields with the
|
||||
// password in the field the form names and the form's loginurl set to the default page, which every firmware has; the page asked for is a request of its own afterwards. A login
|
||||
// is failed when the answer is a login page again.
|
||||
func (a *Admin) Login(ctx context.Context) error {
|
||||
body, at, err := a.fetch(ctx, http.MethodGet, a.url(loginStart), nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the printer's admin page cannot be asked for its login: %w", err)
|
||||
}
|
||||
form, pw := passwordForm(ParsePage(body))
|
||||
if form == nil {
|
||||
return fmt.Errorf("the printer's answer to %s holds no login form with a password field", at.Path)
|
||||
}
|
||||
target, err := at.Parse(form.Action)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the login form's action %q is not a URL: %w", form.Action, err)
|
||||
}
|
||||
if target.Scheme != "https" || target.Host != a.Host {
|
||||
return fmt.Errorf("the login form posts to %s, not to https on the printer %s: the password is not sent", target.Scheme+"://"+target.Host, a.Host)
|
||||
}
|
||||
vals := url.Values{}
|
||||
for _, f := range form.Fields {
|
||||
if f.Type == "hidden" {
|
||||
vals.Set(f.Name, f.Value)
|
||||
}
|
||||
}
|
||||
if _, ok := vals["loginurl"]; ok {
|
||||
vals.Set("loginurl", DefaultPage)
|
||||
}
|
||||
vals.Set(pw.Name, a.Password)
|
||||
answer, _, err := a.fetch(ctx, http.MethodPost, target.String(), vals)
|
||||
if err != nil {
|
||||
// The post may have opened a session before its answer failed: end it if there is one.
|
||||
if home, hat, herr := a.Get(context.Background(), DefaultPage); herr == nil {
|
||||
a.Logout(context.Background(), home, hat)
|
||||
}
|
||||
return fmt.Errorf("the login was not answered: %w", err)
|
||||
}
|
||||
// A login page again is a form with a password field and the loginurl field; a page that was asked for may
|
||||
// hold a password field of its own.
|
||||
if again, _ := passwordForm(ParsePage(answer)); again != nil && hasField(again, "loginurl") {
|
||||
return fmt.Errorf("the login failed: the printer showed its login page again, so the secret admin-password is not its password")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get is a page of the admin site, after a login.
|
||||
func (a *Admin) Get(ctx context.Context, path string) ([]byte, *url.URL, error) {
|
||||
return a.fetch(ctx, http.MethodGet, a.url(path), nil)
|
||||
}
|
||||
|
||||
// Logout ends the session through the logout link or form of a page of it, and says what it did. A page with
|
||||
// none leaves the session to expire, and the answer says so.
|
||||
func (a *Admin) Logout(ctx context.Context, page []byte, at *url.URL) string {
|
||||
p := ParsePage(page)
|
||||
isLogout := func(s string) bool {
|
||||
s = strings.ToLower(strings.ReplaceAll(s, " ", ""))
|
||||
return strings.Contains(s, "logout") || strings.Contains(s, "logoff")
|
||||
}
|
||||
for _, f := range p.Forms {
|
||||
if !isLogout(f.Action) {
|
||||
continue
|
||||
}
|
||||
target, err := at.Parse(f.Action)
|
||||
if err != nil || target.Host != a.Host {
|
||||
continue
|
||||
}
|
||||
vals := url.Values{}
|
||||
for _, fl := range f.Fields {
|
||||
if fl.Type == "hidden" {
|
||||
vals.Set(fl.Name, fl.Value)
|
||||
}
|
||||
}
|
||||
method := http.MethodPost
|
||||
var send url.Values = vals
|
||||
if f.Method == "get" {
|
||||
method, send = http.MethodGet, nil
|
||||
target.RawQuery = vals.Encode()
|
||||
}
|
||||
if _, _, err := a.fetch(ctx, method, target.String(), send); err != nil {
|
||||
return "logout failed: " + err.Error()
|
||||
}
|
||||
return "logged out"
|
||||
}
|
||||
for _, l := range p.Links {
|
||||
if !isLogout(l.Href) && !isLogout(l.Text) {
|
||||
continue
|
||||
}
|
||||
target, err := at.Parse(l.Href)
|
||||
if err != nil || target.Host != a.Host {
|
||||
continue
|
||||
}
|
||||
if _, _, err := a.fetch(ctx, http.MethodGet, target.String(), nil); err != nil {
|
||||
return "logout failed: " + err.Error()
|
||||
}
|
||||
return "logged out"
|
||||
}
|
||||
return "no logout link or form on the page: the session is left to expire on the printer"
|
||||
}
|
||||
|
||||
// notFoundError is the printer's 404 for a page.
|
||||
type notFoundError struct{ Path string }
|
||||
|
||||
func (e *notFoundError) Error() string { return "the printer answered 404 Not Found for " + e.Path }
|
||||
@@ -1,351 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fakePrinter is a TLS server standing in for the printer's admin page. Its login page is the captured one
|
||||
// (testdata/admin/login-page.html, 2026-10-20) with the password field's name changed to the one the test
|
||||
// chooses, as the real device generates it. The pages behind the login are SYNTHETIC (testdata/admin/
|
||||
// synthetic-*.html): the real ones arrive in round 2 of issue 1453.
|
||||
type fakePrinter struct {
|
||||
*httptest.Server
|
||||
field string // the password field's name the login form gives
|
||||
password string
|
||||
mu sync.Mutex
|
||||
conns int // connections opened to it
|
||||
requests []string // method and path of every request
|
||||
posted []url.Values // every form body posted
|
||||
session string // the cookie value of the one logged-in session
|
||||
loggedIn map[string]bool // cookie values that are logged in
|
||||
pages map[string]string // path -> body behind the login
|
||||
}
|
||||
|
||||
func adminFixture(t *testing.T, name string) string {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(filepath.Join("testdata", "admin", name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func newFakePrinter(t *testing.T, field, password string) *fakePrinter {
|
||||
t.Helper()
|
||||
f := &fakePrinter{field: field, password: password, loggedIn: map[string]bool{}, pages: map[string]string{}}
|
||||
login := strings.ReplaceAll(adminFixture(t, "login-page.html"), "B1891", field)
|
||||
// The login page the printer serves at /home/status.html carries that page as its loginurl (captured live
|
||||
// 2026-10-20, testdata/admin/live/home_status_html.body); the page it serves at /etc/loginerror.html carries
|
||||
// the page that was asked for.
|
||||
homeLogin := strings.ReplaceAll(login, `value="/general/status.html"`, `value="/home/status.html"`)
|
||||
synthetic := adminFixture(t, "synthetic-status.html")
|
||||
// This firmware has no /general/status.html behind the login (issue 1514): asking for it is a 404.
|
||||
f.pages["/home/status.html"] = synthetic
|
||||
f.pages["/general/other.html"] = adminFixture(t, "synthetic-settings.html")
|
||||
authed := func(r *http.Request) bool {
|
||||
c, err := r.Cookie("session")
|
||||
return err == nil && f.loggedIn[c.Value]
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/etc/loginerror.html", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
w.Write([]byte(login))
|
||||
})
|
||||
mux.HandleFunc("/home/status.html", func(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if r.Method != http.MethodPost {
|
||||
if authed(r) {
|
||||
w.Write([]byte(synthetic))
|
||||
return
|
||||
}
|
||||
w.Write([]byte(homeLogin))
|
||||
return
|
||||
}
|
||||
r.ParseForm()
|
||||
f.posted = append(f.posted, r.PostForm)
|
||||
if r.PostForm.Get(field) != password || r.PostForm.Get("loginurl") == "" {
|
||||
w.Write([]byte(login))
|
||||
return
|
||||
}
|
||||
f.session = "s" + field
|
||||
f.loggedIn[f.session] = true
|
||||
http.SetCookie(w, &http.Cookie{Name: "session", Value: f.session, Path: "/"})
|
||||
// Like Brother's form, a good login goes on to the page named in loginurl.
|
||||
http.Redirect(w, r, r.PostForm.Get("loginurl"), http.StatusFound)
|
||||
})
|
||||
mux.HandleFunc("/home/logout.html", func(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if c, err := r.Cookie("session"); err == nil {
|
||||
delete(f.loggedIn, c.Value)
|
||||
}
|
||||
w.Write([]byte(login))
|
||||
})
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if r.URL.Path == "/" {
|
||||
http.Redirect(w, r, "/home/status.html", http.StatusMovedPermanently)
|
||||
return
|
||||
}
|
||||
f.requests = append(f.requests, r.Method+" "+r.URL.Path)
|
||||
// Captured: a page behind the login is answered by a redirect to the login page whether it exists or not.
|
||||
if !authed(r) {
|
||||
http.Redirect(w, r, "/etc/loginerror.html?url="+url.QueryEscape(r.URL.Path), http.StatusFound)
|
||||
return
|
||||
}
|
||||
body, ok := f.pages[r.URL.Path]
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Write([]byte(body))
|
||||
})
|
||||
f.Server = httptest.NewUnstartedServer(mux)
|
||||
f.Config.ConnState = func(_ net.Conn, st http.ConnState) {
|
||||
if st == http.StateNew {
|
||||
f.mu.Lock()
|
||||
f.conns++
|
||||
f.mu.Unlock()
|
||||
}
|
||||
}
|
||||
f.StartTLS()
|
||||
t.Cleanup(f.Close)
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *fakePrinter) host() string { return strings.TrimPrefix(f.URL, "https://") }
|
||||
|
||||
// print is the fingerprint of the certificate the server serves.
|
||||
func (f *fakePrinter) print() string { return Fingerprint(f.TLS.Certificates[0].Certificate[0]) }
|
||||
|
||||
func (f *fakePrinter) connsNow() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.conns
|
||||
}
|
||||
|
||||
func (f *fakePrinter) loggedInNow() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.loggedIn)
|
||||
}
|
||||
|
||||
// 1453/AC2: a certificate whose fingerprint is not the pinned one is refused, the error names both
|
||||
// fingerprints, and the server saw no request at all, so no password was sent.
|
||||
func TestACertificateOtherThanThePinnedOneIsRefusedBeforeAnythingIsSent(t *testing.T) {
|
||||
f := newFakePrinter(t, "B1891", "hunter2")
|
||||
other := strings.Repeat("AA:", 31) + "AA"
|
||||
a, err := NewAdmin(f.host(), other, "hunter2")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = a.Login(context.Background())
|
||||
if err == nil {
|
||||
t.Fatal("a certificate that is not the pinned one was accepted")
|
||||
}
|
||||
for _, want := range []string{f.print(), other} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("error %q does not name the fingerprint %s", err, want)
|
||||
}
|
||||
}
|
||||
if len(f.requests) != 0 || len(f.posted) != 0 {
|
||||
t.Errorf("the server was sent %v and the forms %v: the password may have gone", f.requests, f.posted)
|
||||
}
|
||||
}
|
||||
|
||||
// 1453/AC2: the pinned certificate itself is accepted, in either case of hex digits.
|
||||
func TestThePinnedCertificateIsAccepted(t *testing.T) {
|
||||
f := newFakePrinter(t, "B1891", "hunter2")
|
||||
a, _ := NewAdmin(f.host(), strings.ToLower(f.print()), "hunter2")
|
||||
if err := a.Login(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1453/AC3: the login posts the password in the field the form names, whatever the name is (the test changes
|
||||
// it), along with the form's hidden fields, and the session cookie it gets is the one later pages are asked with.
|
||||
func TestLoginUsesThePasswordFieldNameTheFormGives(t *testing.T) {
|
||||
for _, name := range []string{"B1891", "Zq77x"} {
|
||||
f := newFakePrinter(t, name, "s3cret pass")
|
||||
a, _ := NewAdmin(f.host(), f.print(), "s3cret pass")
|
||||
ctx := context.Background()
|
||||
if err := a.Login(ctx); err != nil {
|
||||
t.Fatalf("field %s: %v", name, err)
|
||||
}
|
||||
if len(f.posted) != 1 {
|
||||
t.Fatalf("field %s: %d forms posted", name, len(f.posted))
|
||||
}
|
||||
if got := f.posted[0].Get(name); got != "s3cret pass" {
|
||||
t.Errorf("field %s: the password field holds %q", name, got)
|
||||
}
|
||||
if f.posted[0].Get("loginurl") != DefaultPage {
|
||||
t.Errorf("field %s: the form's hidden loginurl was %q", name, f.posted[0].Get("loginurl"))
|
||||
}
|
||||
if len(f.posted[0]) != 2 {
|
||||
t.Errorf("field %s: posted %v, want the password and loginurl only", name, f.posted[0])
|
||||
}
|
||||
body, _, err := a.Get(ctx, "/general/other.html")
|
||||
if err != nil || !strings.Contains(string(body), "Synthetic settings") {
|
||||
t.Errorf("field %s: a page behind the login was not given with the session cookie: %v %.60s", name, err, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1453/AC4: a wrong password is a failed login, said so: the login page comes back.
|
||||
func TestAWrongPasswordIsAFailedLogin(t *testing.T) {
|
||||
f := newFakePrinter(t, "B1891", "right")
|
||||
a, _ := NewAdmin(f.host(), f.print(), "wrong")
|
||||
err := a.Login(context.Background())
|
||||
if err == nil || !strings.Contains(err.Error(), "login failed") {
|
||||
t.Fatalf("error = %v, want a failed login", err)
|
||||
}
|
||||
if f.loggedInNow() != 0 {
|
||||
t.Errorf("a session was opened")
|
||||
}
|
||||
}
|
||||
|
||||
// 1453: the logout link on a page ends the session; a page with none says so.
|
||||
func TestLogoutEndsTheSessionAndAPageWithoutOneSaysSo(t *testing.T) {
|
||||
f := newFakePrinter(t, "B1891", "pw")
|
||||
a, _ := NewAdmin(f.host(), f.print(), "pw")
|
||||
ctx := context.Background()
|
||||
if err := a.Login(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, at, err := a.Get(ctx, DefaultPage)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := a.Logout(ctx, body, at); got != "logged out" {
|
||||
t.Errorf("logout = %q", got)
|
||||
}
|
||||
if f.loggedInNow() != 0 {
|
||||
t.Errorf("the session is still open")
|
||||
}
|
||||
if got := a.Logout(ctx, []byte("<html><body>nothing</body></html>"), at); !strings.Contains(got, "no logout") {
|
||||
t.Errorf("logout without a link = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1453: the module never sends the password to a host other than the printer, even if the form says so.
|
||||
func TestThePasswordIsNotPostedToAnotherHost(t *testing.T) {
|
||||
// A login form whose action names another host.
|
||||
srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(`<form method="post" action="https://elsewhere.example/x"><input type="password" name="p"/></form>`))
|
||||
}))
|
||||
srv.TLS = &tls.Config{}
|
||||
srv.StartTLS()
|
||||
defer srv.Close()
|
||||
host := strings.TrimPrefix(srv.URL, "https://")
|
||||
a, _ := NewAdmin(host, Fingerprint(srv.TLS.Certificates[0].Certificate[0]), "pw")
|
||||
err := a.Login(context.Background())
|
||||
if err == nil || !strings.Contains(err.Error(), "not sent") {
|
||||
t.Fatalf("error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1514 review: a page the printer does not have leaves no session open, and the error says the page is missing,
|
||||
// not that the login failed.
|
||||
func TestAMissingPageLeavesNoSessionOpenAndSaysSo(t *testing.T) {
|
||||
f := newFakePrinter(t, "B1891", "pw")
|
||||
_, run := printerTools(t, f, "pw")
|
||||
_, err := run(map[string]any{"page": "/general/nope.html"})
|
||||
if err == nil || !strings.Contains(err.Error(), "the printer has no page /general/nope.html") {
|
||||
t.Fatalf("error = %v, want the printer has no page", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "login was not answered") {
|
||||
t.Errorf("a missing page is said to be a failed login: %v", err)
|
||||
}
|
||||
if f.loggedInNow() != 0 {
|
||||
t.Errorf("%d sessions are still open on the printer", f.loggedInNow())
|
||||
}
|
||||
}
|
||||
|
||||
// 1514 review: the password is posted to https on the printer's host only. A form whose action is plain http
|
||||
// on the same host, or a root that redirects to plain http, is refused and nothing is posted.
|
||||
func TestThePasswordIsNotPostedOverPlainHTTP(t *testing.T) {
|
||||
var posted int
|
||||
mux := http.NewServeMux()
|
||||
page := func(action string) string {
|
||||
return `<html><body><form method="post" action="` + action + `"><input type="password" name="B1891"/><input type="hidden" name="loginurl" value="/home/status.html"/></form></body></html>`
|
||||
}
|
||||
var host string
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodPost {
|
||||
posted++
|
||||
}
|
||||
w.Write([]byte(page("http://" + host + "/home/status.html")))
|
||||
})
|
||||
srv := httptest.NewTLSServer(mux)
|
||||
defer srv.Close()
|
||||
host = strings.TrimPrefix(srv.URL, "https://")
|
||||
a, _ := NewAdmin(host, Fingerprint(srv.TLS.Certificates[0].Certificate[0]), "pw")
|
||||
err := a.Login(context.Background())
|
||||
if err == nil || !strings.Contains(err.Error(), "password is not sent") {
|
||||
t.Errorf("error = %v, want the plain http action refused", err)
|
||||
}
|
||||
if posted != 0 {
|
||||
t.Errorf("%d posts reached the server", posted)
|
||||
}
|
||||
}
|
||||
|
||||
// 1515: a 307 or 308 after the login post would make the HTTP client send the post again, the password with
|
||||
// it, to the place the answer names; a redirect off https or off the printer's host is refused as well.
|
||||
func TestNoRedirectResendsThePasswordOrLeavesHTTPSOnThePrinter(t *testing.T) {
|
||||
const loginPage = `<html><body><form action="/login" method="post"><input type="password" name="pw"><input type="hidden" name="loginurl" value="/x"></form></body></html>`
|
||||
cases := []struct {
|
||||
name string
|
||||
status int
|
||||
location string // "SAME" is a path on the printer itself
|
||||
}{
|
||||
{"307 on the printer", 307, "/again"},
|
||||
{"308 on the printer", 308, "/again"},
|
||||
{"307 to plain http", 307, "http://192.0.2.1/steal"},
|
||||
{"302 to another host", 302, "https://192.0.2.1/steal"},
|
||||
{"302 to plain http", 302, "http://192.0.2.1/steal"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var posts []string
|
||||
srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
switch {
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/":
|
||||
w.Write([]byte(loginPage))
|
||||
case r.Method == http.MethodPost:
|
||||
r.ParseForm()
|
||||
posts = append(posts, r.URL.Path+" "+r.PostForm.Get("pw"))
|
||||
http.Redirect(w, r, c.location, c.status)
|
||||
default:
|
||||
w.Write([]byte("ok"))
|
||||
}
|
||||
}))
|
||||
srv.StartTLS()
|
||||
defer srv.Close()
|
||||
a, _ := NewAdmin(strings.TrimPrefix(srv.URL, "https://"), Fingerprint(srv.TLS.Certificates[0].Certificate[0]), "hunter2")
|
||||
err := a.Login(context.Background())
|
||||
if err == nil {
|
||||
t.Fatalf("the login went on after a %d to %s", c.status, c.location)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(posts) != 1 {
|
||||
t.Errorf("the password was posted %d times: %v", len(posts), posts)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,182 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"image/jpeg"
|
||||
"math"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The device process's scan (novox/hq ADR 0468 clause 4): the glass scanned over eSCL, the document cut out of
|
||||
// the platen's white and built into a one-page PDF, or the feeder's sheets (both sides on request) into one multi-page PDF. Keeping it in the bucket and saying scan.added are the
|
||||
// device process's (deviceproc.go); filing it in the scan folder is the tool's (scan.go).
|
||||
|
||||
func mm(pixels, dpi int) float64 {
|
||||
return math.Round(float64(pixels)/float64(dpi)*25.4*10) / 10
|
||||
}
|
||||
|
||||
// Capture is a scanned page or pages as a PDF, with what the event says of it. Width, height and resolution are
|
||||
// the first page's.
|
||||
type Capture struct {
|
||||
PDF []byte
|
||||
SHA256 string
|
||||
WidthMM float64
|
||||
HeightMM float64
|
||||
Resolution int
|
||||
Cropped bool
|
||||
Pages int
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// ScanRequest is what one scan asks beyond the configuration: the feeder instead of the glass, and both sides.
|
||||
type ScanRequest struct {
|
||||
Feeder bool
|
||||
Duplex bool
|
||||
// Apart cuts each of several documents lying apart on the glass out as its own PDF (issue 1528); without it
|
||||
// the glass gives one document, as it always did.
|
||||
Apart bool
|
||||
}
|
||||
|
||||
// ParseSource reads the scanner.scan arguments source (glass, the default, or feeder) and duplex. Duplex is for the
|
||||
// feeder: the glass holds one sheet.
|
||||
func ParseSource(source string, duplex bool) (ScanRequest, error) {
|
||||
return ParseScan(source, duplex, false)
|
||||
}
|
||||
|
||||
// ParseScan is ParseSource with apart: several documents on the glass, each its own file; a feeder scan is of
|
||||
// sheets and has none lying apart.
|
||||
func ParseScan(source string, duplex, apart bool) (ScanRequest, error) {
|
||||
req, err := parseSource(source, duplex)
|
||||
if err == nil && apart && req.Feeder {
|
||||
return ScanRequest{}, fmt.Errorf("apart is for the glass: the feeder takes each sheet as its own page already")
|
||||
}
|
||||
req.Apart = apart
|
||||
return req, err
|
||||
}
|
||||
|
||||
func parseSource(source string, duplex bool) (ScanRequest, error) {
|
||||
switch source {
|
||||
case "", "glass":
|
||||
if duplex {
|
||||
return ScanRequest{}, fmt.Errorf("duplex is for the feeder: the glass holds one page (set source to feeder)")
|
||||
}
|
||||
return ScanRequest{}, nil
|
||||
case "feeder":
|
||||
return ScanRequest{Feeder: true, Duplex: duplex}, nil
|
||||
}
|
||||
return ScanRequest{}, fmt.Errorf("source %q is not glass or feeder", source)
|
||||
}
|
||||
|
||||
// Capture scans the glass or the feeder and builds the PDFs: one for a feeder job (every page of it), and for the
|
||||
// glass one for each separate document lying on the platen when crop is set (issue 1528), in reading order. A glass
|
||||
// page is cropped when the settings say so; a feeder page is a whole sheet and never cropped. Nothing is kept and
|
||||
// nothing is said here; an error says why the scanner gave no page. The resolution is the image's own, never the
|
||||
// one asked (1440/AC2).
|
||||
func (s *Service) Capture(ctx context.Context, req ScanRequest) ([]Capture, error) {
|
||||
at := s.Now()
|
||||
scans, err := NewClient(s.Settings.Scanner).ScanPages(ctx, ScanOptions{Resolution: s.Settings.Resolution,
|
||||
ColorMode: s.Settings.ColorMode, Feeder: req.Feeder, Duplex: req.Duplex})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !req.Feeder {
|
||||
dpi, err := Density(scans[0])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("page 1: %w", err)
|
||||
}
|
||||
cuts, err := Crop(scans[0], s.Settings.Crop)
|
||||
one := []Cropped{cuts}
|
||||
if req.Apart {
|
||||
one, err = CropAll(scans[0], true, dpi)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Capture, len(one))
|
||||
for i, cut := range one {
|
||||
if out[i], err = captureOf([]PDFPage{{JPEG: cut.JPEG, DPI: dpi}}, cut, dpi, at); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
pages := make([]PDFPage, len(scans))
|
||||
var first Cropped
|
||||
var firstDPI int
|
||||
for i, scan := range scans {
|
||||
dpi, err := Density(scan)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("page %d: %w", i+1, err)
|
||||
}
|
||||
cfg, err := jpeg.DecodeConfig(bytes.NewReader(scan))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("page %d: the scanned image cannot be read: %w", i+1, err)
|
||||
}
|
||||
pages[i] = PDFPage{JPEG: scan, DPI: dpi}
|
||||
if i == 0 {
|
||||
first, firstDPI = Cropped{JPEG: scan, Width: cfg.Width, Height: cfg.Height}, dpi
|
||||
}
|
||||
}
|
||||
c, err := captureOf(pages, first, firstDPI, at)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []Capture{c}, nil
|
||||
}
|
||||
|
||||
// captureOf is the PDF of the pages, with what the event says of it from the first page.
|
||||
func captureOf(pages []PDFPage, first Cropped, firstDPI int, at time.Time) (Capture, error) {
|
||||
pdf, err := PDFPages(pages)
|
||||
if err != nil {
|
||||
return Capture{}, err
|
||||
}
|
||||
return Capture{PDF: pdf, SHA256: sha256Hex(pdf), WidthMM: mm(first.Width, firstDPI), HeightMM: mm(first.Height, firstDPI),
|
||||
Resolution: firstDPI, Cropped: first.Cropped, Pages: len(pages), At: at}, nil
|
||||
}
|
||||
|
||||
// Body is the event scan.added's body (ADR 0464 §5 with the object key of ADR 0468 clause 2): never the image,
|
||||
// and no path, since filing in a folder is the tool's, after the event.
|
||||
func (c Capture) Body(node, colorMode string) map[string]any {
|
||||
pages := max(c.Pages, 1)
|
||||
return map[string]any{
|
||||
"object": c.SHA256 + ".pdf",
|
||||
"node": node,
|
||||
"bytes": len(c.PDF),
|
||||
"sha256": c.SHA256,
|
||||
"width-mm": c.WidthMM,
|
||||
"height-mm": c.HeightMM,
|
||||
"resolution": c.Resolution,
|
||||
"color-mode": colorMode,
|
||||
"cropped": c.Cropped,
|
||||
"pages": pages,
|
||||
"scanned-at": c.At.UTC().Format(time.RFC3339),
|
||||
}
|
||||
}
|
||||
|
||||
// GlassImage is a page scanned from the glass.
|
||||
type GlassImage struct {
|
||||
JPEG []byte
|
||||
DPI, Width, Height int
|
||||
Cropped bool
|
||||
}
|
||||
|
||||
// GlassPage scans one page from the glass for a document of several pages: cropped to its document when the settings
|
||||
// say so (the first document, never split in several), with the resolution the scanner gave.
|
||||
func (s *Service) GlassPage(ctx context.Context) (page GlassImage, err error) {
|
||||
scans, err := NewClient(s.Settings.Scanner).ScanPages(ctx, ScanOptions{Resolution: s.Settings.Resolution, ColorMode: s.Settings.ColorMode})
|
||||
if err != nil {
|
||||
return page, err
|
||||
}
|
||||
dpi, err := Density(scans[0])
|
||||
if err != nil {
|
||||
return page, err
|
||||
}
|
||||
cut, err := Crop(scans[0], s.Settings.Crop)
|
||||
if err != nil {
|
||||
return page, err
|
||||
}
|
||||
page.JPEG, page.DPI, page.Width, page.Height, page.Cropped = cut.JPEG, dpi, cut.Width, cut.Height, cut.Cropped
|
||||
return page, nil
|
||||
}
|
||||
@@ -1,143 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"net"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// The module's health (issue 1510, novox/hq ADR 0240): the node-engine asks the tool brother_health, and the tool
|
||||
// asks the printer for the certificates it serves, one for each key type, with no login, and compares them with
|
||||
// the pinned list (the config value printer-certificate). A certificate that is not pinned (the printer was reset,
|
||||
// new certificates were made, a firmware update changed them) is unhealthy and names the new fingerprint, so the
|
||||
// operator can approve the new pin before the admin pages are needed. A printer that does not answer is not
|
||||
// unhealthy (it is switched off, or asleep): nothing can be compared, and the answer says so.
|
||||
|
||||
// keyKinds are the two certificates the printer serves for one name. Which one it serves depends on the cipher
|
||||
// suites the client offers, so each handshake offers only one kind's, under TLS 1.2 (where the suite decides).
|
||||
var keyKinds = []struct {
|
||||
Name string
|
||||
Suites []uint16
|
||||
}{
|
||||
{"ecdsa", []uint16{
|
||||
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
|
||||
tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
|
||||
}},
|
||||
{"rsa", []uint16{
|
||||
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
|
||||
tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
|
||||
tls.TLS_RSA_WITH_AES_128_GCM_SHA256, tls.TLS_RSA_WITH_AES_256_GCM_SHA384, tls.TLS_RSA_WITH_AES_128_CBC_SHA, tls.TLS_RSA_WITH_AES_256_CBC_SHA,
|
||||
}},
|
||||
}
|
||||
|
||||
// certDial bounds one handshake and certTotal the whole check (two handshakes), so the node-engine's look, which has
|
||||
// ten seconds, is always answered (issue 1510).
|
||||
var (
|
||||
certDial = 3 * time.Second
|
||||
certTotal = 7 * time.Second
|
||||
)
|
||||
|
||||
// Served is a certificate the printer served to a handshake of one key type.
|
||||
type Served struct {
|
||||
Key string `json:"key"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
Pinned bool `json:"pinned"`
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
// CertHealth is brother_health's answer: healthy, why when it is not (or what could not be compared), and what the
|
||||
// printer served.
|
||||
type CertHealth struct {
|
||||
Healthy bool `json:"healthy"`
|
||||
State string `json:"state"` // healthy, unhealthy or waits (ADR 0283)
|
||||
Why string `json:"why"`
|
||||
Served []Served `json:"served"`
|
||||
}
|
||||
|
||||
// CertWaits is the answer while printer or printer-certificate is not given: the module waits for the operator,
|
||||
// which is not a fault (ADR 0283), and names each setting as the SDK's health.Wait asks.
|
||||
func CertWaits(s Settings) (health.Answer, bool) {
|
||||
var waits []health.Wait
|
||||
var names []string
|
||||
for _, c := range []struct{ name, value, what string }{
|
||||
{"printer", s.Printer, "the printer's address"},
|
||||
{"printer-certificate", s.PrinterCertificate, "the SHA-256 fingerprints of the certificates the printer serves"},
|
||||
} {
|
||||
if c.value == "" {
|
||||
waits = append(waits, health.Wait{Part: "the printer's certificate check", Setting: c.name, What: c.what})
|
||||
names = append(names, c.name)
|
||||
}
|
||||
}
|
||||
if len(waits) == 0 {
|
||||
return health.Answer{}, false
|
||||
}
|
||||
return health.Waiting("waits for the operator to give the config value "+strings.Join(names, " and "), waits...), true
|
||||
}
|
||||
|
||||
// serveAddr is the host with the HTTPS port when it names none.
|
||||
func serveAddr(host string) string {
|
||||
if _, _, err := net.SplitHostPort(host); err == nil {
|
||||
return host
|
||||
}
|
||||
return net.JoinHostPort(strings.Trim(host, "[]"), "443")
|
||||
}
|
||||
|
||||
// served is the leaf's fingerprint the printer gives a handshake that offers only these suites, or why it gave none.
|
||||
func served(ctx context.Context, addr string, suites []uint16) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, certDial)
|
||||
defer cancel()
|
||||
d := tls.Dialer{NetDialer: &net.Dialer{Timeout: certDial}, Config: &tls.Config{
|
||||
MinVersion: tls.VersionTLS12, MaxVersion: tls.VersionTLS12, CipherSuites: suites,
|
||||
// Nothing is sent over this connection, and the certificate is only read, to be compared with the pins.
|
||||
InsecureSkipVerify: true,
|
||||
}}
|
||||
conn, err := d.DialContext(ctx, "tcp", addr)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer conn.Close()
|
||||
state := conn.(*tls.Conn).ConnectionState()
|
||||
if len(state.PeerCertificates) == 0 {
|
||||
return "", fmt.Errorf("the printer served no certificate")
|
||||
}
|
||||
return Fingerprint(state.PeerCertificates[0].Raw), nil
|
||||
}
|
||||
|
||||
// CheckCertificates compares what the printer serves with the pins. The login is never used.
|
||||
func CheckCertificates(ctx context.Context, host, pins string) CertHealth {
|
||||
ctx, cancel := context.WithTimeout(ctx, certTotal)
|
||||
defer cancel()
|
||||
want := pinList(pins)
|
||||
addr := serveAddr(host)
|
||||
h := CertHealth{Healthy: true, State: "healthy"}
|
||||
var changed, unreachable []string
|
||||
for _, k := range keyKinds {
|
||||
s := Served{Key: k.Name}
|
||||
fp, err := served(ctx, addr, k.Suites)
|
||||
if err != nil {
|
||||
s.Note = err.Error()
|
||||
unreachable = append(unreachable, k.Name+": "+err.Error())
|
||||
} else {
|
||||
s.Fingerprint, s.Pinned = fp, slices.Contains(want, fp)
|
||||
if !s.Pinned {
|
||||
changed = append(changed, fmt.Sprintf("the %s certificate is now %s", k.Name, fp))
|
||||
}
|
||||
}
|
||||
h.Served = append(h.Served, s)
|
||||
}
|
||||
switch {
|
||||
case len(changed) > 0:
|
||||
h.Healthy, h.State = false, "unhealthy"
|
||||
h.Why = fmt.Sprintf("the printer serves a certificate that is not pinned: %s; the pinned printer-certificate is %s. If the printer was reset or given new certificates, set printer-certificate to the new fingerprint(s) (with the old ones kept if they still serve)",
|
||||
strings.Join(changed, "; "), strings.Join(want, ", "))
|
||||
case len(unreachable) == len(keyKinds):
|
||||
h.Why = "the printer gave no certificate, so none was compared (it may be switched off or asleep): " + strings.Join(unreachable, "; ")
|
||||
}
|
||||
return h
|
||||
}
|
||||
@@ -1,160 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/json"
|
||||
"math/big"
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// selfSigned is a self-signed certificate over the key.
|
||||
func selfSigned(t *testing.T, key any, pub any) tls.Certificate {
|
||||
t.Helper()
|
||||
tpl := &x509.Certificate{SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: "printer.test"},
|
||||
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(24 * time.Hour), DNSNames: []string{"printer.test"}}
|
||||
der, err := x509.CreateCertificate(rand.Reader, tpl, tpl, pub, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}
|
||||
}
|
||||
|
||||
// twoCertPrinter is a TLS server with an ECDSA and an RSA certificate, as the printer has; it answers its address
|
||||
// and the two fingerprints.
|
||||
func twoCertPrinter(t *testing.T) (addr, ecdsaFP, rsaFP string) {
|
||||
t.Helper()
|
||||
ek, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
rk, _ := rsa.GenerateKey(rand.Reader, 2048)
|
||||
ec, rc := selfSigned(t, ek, &ek.PublicKey), selfSigned(t, rk, &rk.PublicKey)
|
||||
ln, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{ec, rc}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { ln.Close() })
|
||||
go func() {
|
||||
for {
|
||||
c, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go func() { c.(*tls.Conn).Handshake(); c.Close() }()
|
||||
}
|
||||
}()
|
||||
return ln.Addr().String(), Fingerprint(ec.Certificate[0]), Fingerprint(rc.Certificate[0])
|
||||
}
|
||||
|
||||
// 1510: both certificates pinned is healthy; a changed one is unhealthy and names the new fingerprint; none
|
||||
// pinned names both.
|
||||
func TestTheHealthComparesBothServedCertificatesWithThePins(t *testing.T) {
|
||||
addr, e, r := twoCertPrinter(t)
|
||||
ctx := context.Background()
|
||||
if h := CheckCertificates(ctx, addr, e+", "+r); !h.Healthy || len(h.Served) != 2 || !h.Served[0].Pinned || !h.Served[1].Pinned {
|
||||
t.Errorf("both pinned: %+v", h)
|
||||
}
|
||||
old := strings.Repeat("AB:", 31) + "AB"
|
||||
h := CheckCertificates(ctx, addr, e+", "+old) // the RSA certificate changed
|
||||
if h.Healthy || !strings.Contains(h.Why, r) || strings.Contains(h.Why, "ecdsa certificate is now") {
|
||||
t.Errorf("rsa changed: %+v", h)
|
||||
}
|
||||
h = CheckCertificates(ctx, addr, old+", "+r) // the ECDSA certificate changed
|
||||
if h.Healthy || !strings.Contains(h.Why, e) || strings.Contains(h.Why, "rsa certificate is now") {
|
||||
t.Errorf("ecdsa changed: %+v", h)
|
||||
}
|
||||
h = CheckCertificates(ctx, addr, old)
|
||||
if h.Healthy || !strings.Contains(h.Why, e) || !strings.Contains(h.Why, r) || !strings.Contains(h.Why, old) {
|
||||
t.Errorf("none pinned: %+v", h)
|
||||
}
|
||||
}
|
||||
|
||||
// 1510: a printer that does not answer is not an unhealthy module: nothing can be compared, and the answer says so.
|
||||
func TestAPrinterThatDoesNotAnswerIsNotUnhealthy(t *testing.T) {
|
||||
ln, _ := net.Listen("tcp", "127.0.0.1:0")
|
||||
addr := ln.Addr().String()
|
||||
ln.Close()
|
||||
h := CheckCertificates(context.Background(), addr, strings.Repeat("AB:", 31)+"AB")
|
||||
if !h.Healthy || !strings.Contains(h.Why, "none was compared") {
|
||||
t.Errorf("%+v", h)
|
||||
}
|
||||
}
|
||||
|
||||
// 1510: brother_health is a tool of the module, read from the settings at each call.
|
||||
func TestBrotherHealthIsAToolThatReadsTheSettings(t *testing.T) {
|
||||
addr, e, r := twoCertPrinter(t)
|
||||
js := `{"printer":"` + addr + `","printer-certificate":"` + e + `, ` + r + `"}`
|
||||
run := tool(t, newRig(t, js, "pw").tl, "brother_health").Run
|
||||
got, err := run(map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if h, ok := got.(CertHealth); !ok || !h.Healthy {
|
||||
t.Errorf("answer = %+v", got)
|
||||
}
|
||||
bad := `{"printer":"` + addr + `","printer-certificate":"` + strings.Repeat("AB:", 31) + `AB"}`
|
||||
got, _ = tool(t, newRig(t, bad, "pw").tl, "brother_health").Run(map[string]any{})
|
||||
if h := got.(CertHealth); h.Healthy {
|
||||
t.Errorf("a wrong pin is healthy: %+v", h)
|
||||
}
|
||||
}
|
||||
|
||||
// 1510: with the printer not given the answer is waits, read by the SDK as the engine reads it: state waits, each
|
||||
// wait naming exactly one setting, a part and what is wanted; not an error and not unhealthy.
|
||||
func TestHealthWaitsForTheSettingsTheOperatorHasYetToGive(t *testing.T) {
|
||||
for js, want := range map[string][]string{`{}`: {"printer", "printer-certificate"}, `{"printer":"192.0.2.10"}`: {"printer-certificate"}} {
|
||||
got, err := tool(t, newRig(t, js, "pw").tl, "brother_health").Run(map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(got)
|
||||
a, err := health.Read(raw)
|
||||
if err != nil || a.State != health.StateWaits || len(a.Waits) != len(want) {
|
||||
t.Fatalf("%s: read as %+v, %v", js, a, err)
|
||||
}
|
||||
for i, w := range a.Waits {
|
||||
if w.Setting != want[i] || w.Check() != nil {
|
||||
t.Errorf("%s: wait %d = %+v (%v)", js, i, w, w.Check())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1510: each handshake is bounded by 3 s and the whole check by 7 s, and a printer that accepts and never answers
|
||||
// ends the check at the bound with the module healthy (nothing compared).
|
||||
func TestTheCheckIsBoundedAndAHungPrinterEndsItAtTheBound(t *testing.T) {
|
||||
if certDial != 3*time.Second || certTotal != 7*time.Second {
|
||||
t.Fatalf("bounds = %s and %s, want 3 s and 7 s", certDial, certTotal)
|
||||
}
|
||||
ln, _ := net.Listen("tcp", "127.0.0.1:0")
|
||||
defer ln.Close()
|
||||
go func() {
|
||||
for {
|
||||
c, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer c.Close() // held open, never answered
|
||||
}
|
||||
}()
|
||||
oldD, oldT := certDial, certTotal
|
||||
certDial, certTotal = 200*time.Millisecond, 500*time.Millisecond
|
||||
t.Cleanup(func() { certDial, certTotal = oldD, oldT })
|
||||
start := time.Now()
|
||||
h := CheckCertificates(context.Background(), ln.Addr().String(), strings.Repeat("AB:", 31)+"AB")
|
||||
if took := time.Since(start); took > 700*time.Millisecond {
|
||||
t.Errorf("the check took %s", took)
|
||||
}
|
||||
if !h.Healthy || !strings.Contains(h.Why, "none was compared") {
|
||||
t.Errorf("%+v", h)
|
||||
}
|
||||
}
|
||||
@@ -1,207 +0,0 @@
|
||||
package main
|
||||
|
||||
// Conversion before printing (novox/hq ADR 0468 clause 7, issue 1600). The printer takes PDF and JPEG. A stored
|
||||
// document of a type the module only-office converts (seat document-converter) is turned into a PDF first: the device
|
||||
// process signs a GET link on the document and a PUT link on <sha>.pdf with the binding's key, calls
|
||||
// document-converter.convert, checks that the PDF the converter says it put is the PDF the bucket holds, and prints
|
||||
// that. Any other type is refused, naming the types taken.
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/objectlink"
|
||||
)
|
||||
|
||||
// ConvertFollow is how long the device process follows one conversion, in calls of under 25 s each.
|
||||
const ConvertFollow = 3 * time.Minute
|
||||
|
||||
// ConvertTypes are the types document-converter takes (clause 7).
|
||||
var ConvertTypes = []string{"docx", "doc", "xlsx", "xls", "pptx", "ppt", "odt", "ods", "odp", "rtf", "txt", "html"}
|
||||
|
||||
// contentTypes are the media types of a fetched document, by the type the converter is asked for.
|
||||
var contentTypes = map[string]string{
|
||||
"text/html": "html", "application/xhtml+xml": "html", "text/plain": "txt", "application/rtf": "rtf", "text/rtf": "rtf",
|
||||
"application/msword": "doc", "application/vnd.ms-excel": "xls", "application/vnd.ms-powerpoint": "ppt",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation": "pptx",
|
||||
"application/vnd.oasis.opendocument.text": "odt",
|
||||
"application/vnd.oasis.opendocument.spreadsheet": "ods",
|
||||
"application/vnd.oasis.opendocument.presentation": "odp",
|
||||
}
|
||||
|
||||
// the first bytes of an OLE compound file (the legacy Word, Excel and PowerPoint formats), and the stream names that
|
||||
// tell them apart, as UTF-16 (MS-CFB: directory entry names are UTF-16LE).
|
||||
var oleMagic = []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1}
|
||||
|
||||
func utf16le(s string) []byte {
|
||||
var b []byte
|
||||
for _, r := range s {
|
||||
b = append(b, byte(r), byte(r>>8))
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// ConvertibleType is the type a document is, among the converter's twelve, told by its bytes and, for a document that
|
||||
// came from a web server, by its Content-Type; "" when it is none of them. A document that is a PDF or a JPEG is
|
||||
// not asked about: the printer takes it as it is.
|
||||
func ConvertibleType(doc []byte, contentType string) string {
|
||||
if i := strings.IndexByte(contentType, ';'); i >= 0 {
|
||||
contentType = contentType[:i]
|
||||
}
|
||||
byHeader := contentTypes[strings.ToLower(strings.TrimSpace(contentType))]
|
||||
switch {
|
||||
case bytes.HasPrefix(doc, []byte("PK\x03\x04")):
|
||||
if t := zipType(doc); t != "" {
|
||||
return t
|
||||
}
|
||||
return ""
|
||||
case bytes.HasPrefix(doc, oleMagic):
|
||||
for name, t := range map[string]string{"WordDocument": "doc", "Workbook": "xls", "PowerPoint Document": "ppt"} {
|
||||
if bytes.Contains(doc, utf16le(name)) {
|
||||
return t
|
||||
}
|
||||
}
|
||||
return ""
|
||||
case bytes.HasPrefix(doc, []byte(`{\rtf`)):
|
||||
return "rtf"
|
||||
}
|
||||
if byHeader == "html" || looksLikeHTML(doc) {
|
||||
if utf8.Valid(doc) && !bytes.Contains(doc, []byte{0}) {
|
||||
return "html"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
if utf8.Valid(doc) && !bytes.Contains(doc, []byte{0}) && len(doc) > 0 {
|
||||
return "txt"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func looksLikeHTML(doc []byte) bool {
|
||||
head := doc
|
||||
if len(head) > 512 {
|
||||
head = head[:512]
|
||||
}
|
||||
s := strings.ToLower(strings.TrimLeft(strings.TrimPrefix(string(head), "\xef\xbb\xbf"), " \t\r\n"))
|
||||
return strings.HasPrefix(s, "<!doctype html") || strings.HasPrefix(s, "<html")
|
||||
}
|
||||
|
||||
// zipType tells the packages apart by their parts: OOXML by its main part (ECMA-376 Part 2), OpenDocument by its
|
||||
// first file `mimetype` (ODF 1.2 Part 3 §3.3).
|
||||
func zipType(doc []byte) string {
|
||||
zr, err := zip.NewReader(bytes.NewReader(doc), int64(len(doc)))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
for _, f := range zr.File {
|
||||
switch f.Name {
|
||||
case "word/document.xml":
|
||||
return "docx"
|
||||
case "xl/workbook.xml":
|
||||
return "xlsx"
|
||||
case "ppt/presentation.xml":
|
||||
return "pptx"
|
||||
case "mimetype":
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
mt, _ := io.ReadAll(io.LimitReader(rc, 256))
|
||||
rc.Close()
|
||||
switch strings.TrimSpace(string(mt)) {
|
||||
case "application/vnd.oasis.opendocument.text":
|
||||
return "odt"
|
||||
case "application/vnd.oasis.opendocument.spreadsheet":
|
||||
return "ods"
|
||||
case "application/vnd.oasis.opendocument.presentation":
|
||||
return "odp"
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// toPDF turns a stored document of a converter type into a stored PDF and answers the PDF and its key. contentType is
|
||||
// the media type a web server gave, or "".
|
||||
func (w *deviceWorker) toPDF(ctx context.Context, objects Objects, key string, doc []byte, contentType string) ([]byte, string, error) {
|
||||
typ := ConvertibleType(doc, contentType)
|
||||
if typ == "" {
|
||||
return nil, "", fmt.Errorf("the document is neither a PDF nor a JPEG, nor a type the converter takes (%s): nothing is converted or printed",
|
||||
strings.Join(ConvertTypes, ", "))
|
||||
}
|
||||
signer, ok := objects.(Presigner)
|
||||
if !ok {
|
||||
return nil, "", errors.New("the bucket cannot sign a link, so the document cannot be converted")
|
||||
}
|
||||
if w.Convert == nil {
|
||||
return nil, "", errors.New("this process has no way to call the document converter")
|
||||
}
|
||||
sum := sha256Hex(doc)
|
||||
pdfKey := sum + ".pdf"
|
||||
source, err := signer.Link(http.MethodGet, key, objectlink.MaxExpires)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
target, err := signer.Link(http.MethodPut, pdfKey, objectlink.MaxExpires)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
raw, err := w.Convert(map[string]any{"source": source, "type": typ, "target": target, "sha256": sum})
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("the document (%s) could not be converted: %w", typ, err)
|
||||
}
|
||||
// The converter answers within 25 s, the tool runner cutting every call at 30 s; a conversion not done by then is
|
||||
// answered as converting with its ask id, and is read again with convert {ask}, each call under 25 s, for as long
|
||||
// as the device process follows a conversion.
|
||||
follow := w.ConvertFollow
|
||||
if follow <= 0 {
|
||||
follow = ConvertFollow
|
||||
}
|
||||
until := time.Now().Add(follow)
|
||||
var ans struct {
|
||||
SHA256 string `json:"sha256"`
|
||||
Error string `json:"error"`
|
||||
State string `json:"state"`
|
||||
Ask string `json:"ask"`
|
||||
}
|
||||
for {
|
||||
ans.State, ans.Ask, ans.SHA256, ans.Error = "", "", "", ""
|
||||
if err := json.Unmarshal(raw, &ans); err != nil {
|
||||
return nil, "", fmt.Errorf("the converter's answer is not JSON: %w", err)
|
||||
}
|
||||
if ans.State != "converting" {
|
||||
break
|
||||
}
|
||||
if ans.Ask == "" || !time.Now().Before(until) {
|
||||
return nil, "", fmt.Errorf("the document (%s) was still converting after %s (ask %s): nothing is printed", typ, follow, ans.Ask)
|
||||
}
|
||||
if raw, err = w.Convert(map[string]any{"ask": ans.Ask}); err != nil {
|
||||
return nil, "", fmt.Errorf("the document (%s) could not be converted: %w", typ, err)
|
||||
}
|
||||
}
|
||||
if ans.Error != "" {
|
||||
return nil, "", fmt.Errorf("the document (%s) could not be converted: %s", typ, ans.Error)
|
||||
}
|
||||
pdf, err := objects.Get(ctx, pdfKey)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("the converter answered, but no PDF is kept as %s: %w", pdfKey, err)
|
||||
}
|
||||
if got := sha256Hex(pdf); !strings.EqualFold(got, ans.SHA256) {
|
||||
return nil, "", fmt.Errorf("the converter says it put a PDF of SHA-256 %s, but the bucket holds one of %s as %s: nothing is printed", ans.SHA256, got, pdfKey)
|
||||
}
|
||||
if _, _, err := SniffFormat(pdf); err != nil {
|
||||
return nil, "", fmt.Errorf("the converter's result %s: %w", pdfKey, err)
|
||||
}
|
||||
return pdf, pdfKey, nil
|
||||
}
|
||||
@@ -1,268 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// docx builds a minimal Word package: the parts ECMA-376 Part 2 requires of one ([Content_Types].xml, the package
|
||||
// relationships and the main part word/document.xml). No real Word file is captured; the layout is the standard's.
|
||||
func docx(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
for name, body := range map[string]string{
|
||||
"[Content_Types].xml": `<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"/>`,
|
||||
"_rels/.rels": `<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"/>`,
|
||||
"word/document.xml": `<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body/></w:document>`,
|
||||
} {
|
||||
w, err := zw.Create(name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.Write([]byte(body))
|
||||
}
|
||||
zw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// converterFake stands for document-converter.convert: it records the call and, as the converter process would, puts a
|
||||
// PDF in the bucket under the key the target link names and answers its size and SHA-256.
|
||||
type converterFake struct {
|
||||
r *printerRig
|
||||
calls []map[string]any
|
||||
pdf []byte
|
||||
answer func(body map[string]any, stored []byte) (json.RawMessage, error)
|
||||
}
|
||||
|
||||
func withConverter(r *printerRig, pdf []byte) *converterFake {
|
||||
c := &converterFake{r: r, pdf: pdf}
|
||||
r.worker.Convert = func(body map[string]any) (json.RawMessage, error) {
|
||||
c.calls = append(c.calls, body)
|
||||
if body["target"] != nil { // a start puts the PDF; a read of an ask (convert {ask}) puts nothing
|
||||
target, _ := url.Parse(body["target"].(string))
|
||||
r.objects.Put(context.Background(), strings.TrimPrefix(target.Path, "/"), "application/pdf", c.pdf)
|
||||
}
|
||||
if c.answer != nil {
|
||||
return c.answer(body, c.pdf)
|
||||
}
|
||||
return json.Marshal(map[string]any{"size": len(c.pdf), "sha256": sha256Hex(c.pdf)})
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// 1600/AC1: a stored .docx is converted: document-converter.convert is called with a GET link on the source and a PUT
|
||||
// link on <sha>.pdf, both signed in the bucket and expiring within ten minutes, and the PDF is printed.
|
||||
func TestAStoredDocxIsConvertedThenPrinted(t *testing.T) {
|
||||
fake := newIPPFake(t, 51, 5)
|
||||
r := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
|
||||
doc := docx(t)
|
||||
key := sha256Hex(doc)
|
||||
r.objects.Put(t.Context(), key, "application/octet-stream", doc)
|
||||
pdf := pdfOf(2500)
|
||||
conv := withConverter(r, pdf)
|
||||
got, err := r.call(t, "printer.print", map[string]any{"object": key})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(conv.calls) != 1 {
|
||||
t.Fatalf("%d converter calls", len(conv.calls))
|
||||
}
|
||||
call := conv.calls[0]
|
||||
src, _ := url.Parse(call["source"].(string))
|
||||
dst, _ := url.Parse(call["target"].(string))
|
||||
if src.Path != "/"+key || dst.Path != "/"+key+".pdf" || call["type"] != "docx" || call["sha256"] != key {
|
||||
t.Errorf("call = %v", call)
|
||||
}
|
||||
for _, u := range []*url.URL{src, dst} {
|
||||
if secs, _ := strconv.Atoi(u.Query().Get("X-Amz-Expires")); secs < 1 || time.Duration(secs)*time.Second > 10*time.Minute {
|
||||
t.Errorf("%s lives %d s", u, secs)
|
||||
}
|
||||
}
|
||||
if strings.Join(r.objects.links, ",") != "GET "+key+" 600,PUT "+key+".pdf 600" {
|
||||
t.Errorf("links signed: %v", r.objects.links)
|
||||
}
|
||||
m := got.(map[string]any)
|
||||
if m["job-id"] != float64(51) || m["format"] != "application/pdf" || m["object"] != key+".pdf" || m["converted-from"] != key {
|
||||
t.Errorf("answer = %v", m)
|
||||
}
|
||||
if _, sent := fake.request(t); string(sent) != string(pdf) {
|
||||
t.Error("the printer was not sent the converted PDF")
|
||||
}
|
||||
}
|
||||
|
||||
// 1600/AC2: a converter answer whose SHA-256 differs from the stored PDF's is refused and nothing is printed.
|
||||
func TestAConverterAnswerOfAnotherHashPrintsNothing(t *testing.T) {
|
||||
fake := newIPPFake(t, 1, 3)
|
||||
r := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
|
||||
doc := docx(t)
|
||||
r.objects.Put(t.Context(), sha256Hex(doc), "application/octet-stream", doc)
|
||||
conv := withConverter(r, pdfOf(900))
|
||||
conv.answer = func(map[string]any, []byte) (json.RawMessage, error) {
|
||||
return json.Marshal(map[string]any{"size": 900, "sha256": sha256Hex([]byte("another pdf"))})
|
||||
}
|
||||
_, err := r.call(t, "printer.print", map[string]any{"object": sha256Hex(doc)})
|
||||
if err == nil || !strings.Contains(err.Error(), "nothing is printed") {
|
||||
t.Fatalf("error = %v", err)
|
||||
}
|
||||
if fake.body != nil {
|
||||
t.Error("something was printed")
|
||||
}
|
||||
}
|
||||
|
||||
// 1600/AC3: a converter error is answered with its code and meaning.
|
||||
func TestAConverterErrorIsAnsweredWithItsCodeAndMeaning(t *testing.T) {
|
||||
fake := newIPPFake(t, 1, 3)
|
||||
r := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
|
||||
doc := docx(t)
|
||||
r.objects.Put(t.Context(), sha256Hex(doc), "application/octet-stream", doc)
|
||||
conv := withConverter(r, pdfOf(10))
|
||||
conv.answer = func(map[string]any, []byte) (json.RawMessage, error) {
|
||||
return nil, errors.New("the document server refused the conversion, error -4: error while downloading the document file to be converted")
|
||||
}
|
||||
_, err := r.call(t, "printer.print", map[string]any{"object": sha256Hex(doc)})
|
||||
if err == nil || !strings.Contains(err.Error(), "-4: error while downloading the document file to be converted") {
|
||||
t.Fatalf("error = %v", err)
|
||||
}
|
||||
// An error the converter wrote in its answer is the same.
|
||||
conv.answer = func(map[string]any, []byte) (json.RawMessage, error) {
|
||||
return json.RawMessage(`{"error": "-3: conversion error"}`), nil
|
||||
}
|
||||
if _, err := r.call(t, "printer.print", map[string]any{"object": sha256Hex(doc)}); err == nil || !strings.Contains(err.Error(), "-3: conversion error") {
|
||||
t.Fatalf("error = %v", err)
|
||||
}
|
||||
if fake.body != nil {
|
||||
t.Error("something was printed")
|
||||
}
|
||||
}
|
||||
|
||||
// 1600/AC4: a type that is neither PDF, JPEG nor one of the converter's twelve is refused naming the types taken, and
|
||||
// convert is not called.
|
||||
func TestAnotherTypeIsRefusedNamingTheTypesTaken(t *testing.T) {
|
||||
fake := newIPPFake(t, 1, 3)
|
||||
r := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
|
||||
exe := []byte("MZ\x90\x00\x03\x00\x00\x00\x04\x00\x00\x00\xff\xff\x00\x00")
|
||||
r.objects.Put(t.Context(), sha256Hex(exe), "application/octet-stream", exe)
|
||||
conv := withConverter(r, pdfOf(10))
|
||||
_, err := r.call(t, "printer.print", map[string]any{"object": sha256Hex(exe)})
|
||||
if err == nil || !strings.Contains(err.Error(), "docx, doc, xlsx, xls, pptx, ppt, odt, ods, odp, rtf, txt, html") {
|
||||
t.Fatalf("error = %v", err)
|
||||
}
|
||||
if len(conv.calls) != 0 || fake.body != nil {
|
||||
t.Errorf("%d converter calls, printed %v", len(conv.calls), fake.body != nil)
|
||||
}
|
||||
}
|
||||
|
||||
// 1600: each of the twelve types is told by its bytes (and a web page also by its Content-Type).
|
||||
func TestTheTypesAreToldByTheirBytes(t *testing.T) {
|
||||
ole := func(stream string) []byte {
|
||||
return append(append([]byte{}, oleMagic...), append(make([]byte, 64), utf16le(stream)...)...)
|
||||
}
|
||||
odf := func(mime string) []byte {
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
w, _ := zw.Create("mimetype")
|
||||
w.Write([]byte(mime))
|
||||
zw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
ooxml := func(part string) []byte {
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
w, _ := zw.Create(part)
|
||||
w.Write([]byte("<x/>"))
|
||||
zw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
for want, c := range map[string]struct {
|
||||
doc []byte
|
||||
ct string
|
||||
}{
|
||||
"docx": {docx(t), ""}, "xlsx": {ooxml("xl/workbook.xml"), ""}, "pptx": {ooxml("ppt/presentation.xml"), ""},
|
||||
"doc": {ole("WordDocument"), ""}, "xls": {ole("Workbook"), ""}, "ppt": {ole("PowerPoint Document"), ""},
|
||||
"odt": {odf("application/vnd.oasis.opendocument.text"), ""}, "ods": {odf("application/vnd.oasis.opendocument.spreadsheet"), ""},
|
||||
"odp": {odf("application/vnd.oasis.opendocument.presentation"), ""},
|
||||
"rtf": {[]byte(`{\rtf1\ansi hello}`), ""}, "txt": {[]byte("just some words\n"), ""},
|
||||
"html": {[]byte("<!DOCTYPE html><html><body>hi</body></html>"), ""}, "html ": {[]byte("<p>a fragment</p>"), "text/html; charset=utf-8"},
|
||||
} {
|
||||
if got := ConvertibleType(c.doc, c.ct); got != strings.TrimSpace(want) {
|
||||
t.Errorf("%s: told as %q", want, got)
|
||||
}
|
||||
}
|
||||
for name, doc := range map[string][]byte{"empty": {}, "binary": {0, 1, 2, 3}, "a zip of nothing known": ooxml("other.xml"), "ole of nothing known": ole("Other")} {
|
||||
if got := ConvertibleType(doc, ""); got != "" {
|
||||
t.Errorf("%s: told as %q", name, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1600/AC5: the device process invokes seat:document-converter.convert.
|
||||
func TestTheDeviceProcessInvokesTheConverter(t *testing.T) {
|
||||
for _, r := range readManifest(t).Resources {
|
||||
if r["id"] == "device" {
|
||||
inv, _ := r["bus"].(map[string]any)["invokes"].([]any)
|
||||
if len(inv) != 1 || inv[0] != "seat:document-converter.convert" {
|
||||
t.Errorf("invokes = %v", inv)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("no device process")
|
||||
}
|
||||
|
||||
// 1600 (review round 1): a conversion answered as converting is followed with convert {ask}, each call its own, until
|
||||
// it is done; one still converting after the device process's bound is not printed, and the process.CallTimeout of the
|
||||
// SDK is not touched.
|
||||
func TestAConversionAnsweredAsConvertingIsFollowedWithItsAsk(t *testing.T) {
|
||||
fake := newIPPFake(t, 61, 5)
|
||||
r := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
|
||||
doc := docx(t)
|
||||
key := sha256Hex(doc)
|
||||
r.objects.Put(t.Context(), key, "application/octet-stream", doc)
|
||||
pdf := pdfOf(700)
|
||||
conv := withConverter(r, pdf)
|
||||
rounds := 0
|
||||
conv.answer = func(body map[string]any, stored []byte) (json.RawMessage, error) {
|
||||
if body["ask"] != nil {
|
||||
rounds++
|
||||
if rounds < 3 {
|
||||
return json.RawMessage(`{"state": "converting", "ask": "` + body["ask"].(string) + `"}`), nil
|
||||
}
|
||||
return json.Marshal(map[string]any{"size": len(stored), "sha256": sha256Hex(stored)})
|
||||
}
|
||||
return json.RawMessage(`{"state": "converting", "ask": "a1b2c3a1b2c3a1b2c3a1b2c3"}`), nil
|
||||
}
|
||||
if _, err := r.call(t, "printer.print", map[string]any{"object": key}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(conv.calls) != 4 || conv.calls[1]["ask"] != "a1b2c3a1b2c3a1b2c3a1b2c3" || conv.calls[1]["source"] != nil {
|
||||
t.Errorf("calls = %v", conv.calls)
|
||||
}
|
||||
if _, sent := fake.request(t); string(sent) != string(pdf) {
|
||||
t.Error("the converted PDF was not printed")
|
||||
}
|
||||
|
||||
// Always converting: given up at the bound, nothing printed.
|
||||
fake.body = nil
|
||||
r.worker.ConvertFollow = 30 * time.Millisecond
|
||||
conv.answer = func(body map[string]any, _ []byte) (json.RawMessage, error) {
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
return json.RawMessage(`{"state": "converting", "ask": "a1b2c3a1b2c3a1b2c3a1b2c3"}`), nil
|
||||
}
|
||||
if _, err := r.call(t, "printer.print", map[string]any{"object": key}); err == nil || !strings.Contains(err.Error(), "still converting") || fake.body != nil {
|
||||
t.Errorf("error = %v, printed %v", err, fake.body != nil)
|
||||
}
|
||||
src, _ := os.ReadFile("deviceproc.go")
|
||||
if strings.Contains(string(src), "CallTimeout") {
|
||||
t.Error("deviceproc.go sets the SDK's call timeout")
|
||||
}
|
||||
}
|
||||
@@ -1,343 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// Cropping cuts the document out of the platen. The scanner sends the whole glass, a document in one corner
|
||||
// and the lid's white everywhere else. The lid's level is measured where no document can be (the corner
|
||||
// farthest from the origin), a pixel is document when it is darker than the lid by a margin, and the rows and
|
||||
// columns holding enough document pixels give the box (ADR 0464).
|
||||
const (
|
||||
// lidBlock is the side of the corner block the lid's level is measured on.
|
||||
lidBlock = 300
|
||||
// darker is how much darker than the lid a pixel is document: paper on a white lid is a level or two
|
||||
// under it, an edge shadow far more.
|
||||
darker = 1
|
||||
// rowShare is the fraction of a row's (column's) length that must be document for the row (column) to
|
||||
// count: specks of dust stay under it.
|
||||
rowShare = 0.05
|
||||
// margin is the pixels kept around the box.
|
||||
margin = 6
|
||||
// wholeShare is the fraction of the platen a box may fill before it is no crop at all.
|
||||
wholeShare = 0.9
|
||||
// jpegQuality is the quality the cropped image is encoded with.
|
||||
jpegQuality = 90
|
||||
)
|
||||
|
||||
// gray is the pixel's level, 0 to 255.
|
||||
func grayAt(img image.Image, x, y int) int {
|
||||
switch m := img.(type) {
|
||||
case *image.YCbCr:
|
||||
return int(m.Y[m.YOffset(x, y)])
|
||||
case *image.Gray:
|
||||
return int(m.Pix[m.PixOffset(x, y)])
|
||||
}
|
||||
return int(color.GrayModel.Convert(img.At(x, y)).(color.Gray).Y)
|
||||
}
|
||||
|
||||
// Box finds the document's box in the image, and whether there is one: false when no document is found or
|
||||
// when it fills (almost) the platen.
|
||||
func Box(img image.Image) (image.Rectangle, bool) {
|
||||
b := img.Bounds()
|
||||
w, h := b.Dx(), b.Dy()
|
||||
if w < 2*margin || h < 2*margin {
|
||||
return b, false
|
||||
}
|
||||
limit := lidLimit(img)
|
||||
|
||||
cols := make([]int, w)
|
||||
rows := make([]int, h)
|
||||
for y := b.Min.Y; y < b.Max.Y; y++ {
|
||||
for x := b.Min.X; x < b.Max.X; x++ {
|
||||
if grayAt(img, x, y) < limit {
|
||||
cols[x-b.Min.X]++
|
||||
rows[y-b.Min.Y]++
|
||||
}
|
||||
}
|
||||
}
|
||||
x0, x1 := span(cols, int(rowShare*float64(h)))
|
||||
y0, y1 := span(rows, int(rowShare*float64(w)))
|
||||
if x0 < 0 || y0 < 0 {
|
||||
return b, false
|
||||
}
|
||||
box := image.Rect(max(0, x0-margin), max(0, y0-margin), min(w, x1+1+margin), min(h, y1+1+margin)).Add(b.Min)
|
||||
if float64(box.Dx()*box.Dy()) >= wholeShare*float64(w*h) {
|
||||
return b, false
|
||||
}
|
||||
return box, true
|
||||
}
|
||||
|
||||
// lidLimit is the level under which a pixel is document: the lid's level, measured in the corner block farthest
|
||||
// from the origin, less darker.
|
||||
func lidLimit(img image.Image) int {
|
||||
b := img.Bounds()
|
||||
w, h := b.Dx(), b.Dy()
|
||||
side := lidBlock
|
||||
if side > w/2 {
|
||||
side = w / 2
|
||||
}
|
||||
if side > h/2 {
|
||||
side = h / 2
|
||||
}
|
||||
sum := 0
|
||||
for y := b.Max.Y - side; y < b.Max.Y; y++ {
|
||||
for x := b.Max.X - side; x < b.Max.X; x++ {
|
||||
sum += grayAt(img, x, y)
|
||||
}
|
||||
}
|
||||
lid := int(float64(sum)/float64(side*side) + 0.5)
|
||||
return lid - darker
|
||||
}
|
||||
|
||||
// span is the first and last index whose count is over the floor, or -1 when none is.
|
||||
func span(counts []int, floor int) (int, int) {
|
||||
first, last := -1, -1
|
||||
for i, c := range counts {
|
||||
if c > floor {
|
||||
if first < 0 {
|
||||
first = i
|
||||
}
|
||||
last = i
|
||||
}
|
||||
}
|
||||
return first, last
|
||||
}
|
||||
|
||||
// Cropped is what cropping a scan gave.
|
||||
type Cropped struct {
|
||||
// JPEG is the image to file: the scanner's own bytes when kept whole, a re-encoded crop otherwise.
|
||||
JPEG []byte
|
||||
Width, Height int
|
||||
Cropped bool
|
||||
}
|
||||
|
||||
// Crop cuts the document out of a scanned JPEG when crop is set and a document is found; otherwise the whole
|
||||
// image is kept, its bytes untouched, and Cropped is false.
|
||||
func Crop(scan []byte, crop bool) (Cropped, error) {
|
||||
img, err := jpeg.Decode(bytes.NewReader(scan))
|
||||
if err != nil {
|
||||
return Cropped{}, fmt.Errorf("the scanned image cannot be read: %w", err)
|
||||
}
|
||||
b := img.Bounds()
|
||||
whole := Cropped{JPEG: scan, Width: b.Dx(), Height: b.Dy()}
|
||||
if !crop {
|
||||
return whole, nil
|
||||
}
|
||||
box, found := Box(img)
|
||||
if !found {
|
||||
return whole, nil
|
||||
}
|
||||
sub, ok := img.(interface {
|
||||
SubImage(image.Rectangle) image.Image
|
||||
})
|
||||
if !ok {
|
||||
return whole, nil
|
||||
}
|
||||
var out bytes.Buffer
|
||||
if err := jpeg.Encode(&out, sub.SubImage(box), &jpeg.Options{Quality: jpegQuality}); err != nil {
|
||||
return Cropped{}, fmt.Errorf("the cropped image cannot be encoded: %w", err)
|
||||
}
|
||||
return Cropped{JPEG: out.Bytes(), Width: box.Dx(), Height: box.Dy(), Cropped: true}, nil
|
||||
}
|
||||
|
||||
// Several documents on the glass (issue 1528). Where the rows-and-columns box of Box would be one image around
|
||||
// all of them, Boxes finds each document apart from the others: the picture is cut into blocks of about a
|
||||
// millimetre, a block is document when a quarter of its pixels are darker than the lid (dust is not), blocks
|
||||
// within gapMM of each other are one region (so a white area inside a document, or a gap in it, does not split
|
||||
// it), and a region of at least minMM on both sides is a document. Regions whose boxes touch or overlap
|
||||
// (a text block inside a document's frame) are one document.
|
||||
const (
|
||||
// blockMM is the side of a block, in millimetres.
|
||||
blockMM = 1.0
|
||||
// blockShare is the fraction of a block's pixels that must be document for the block to count.
|
||||
blockShare = 0.25
|
||||
// gapMM is how near two regions must be to be one document, in millimetres: documents lying farther apart than
|
||||
// this are separate ones.
|
||||
gapMM = 10.0
|
||||
// minMM is the least a region may measure, on both sides, to be a document and not a speck or a stain.
|
||||
minMM = 20.0
|
||||
)
|
||||
|
||||
// Boxes is the box of each separate document on the platen, in reading order (top to bottom by the box's top,
|
||||
// then left to right), at the image's resolution dpi. It is Box's white-detection (the lid's level and darker) on
|
||||
// a grid of blocks. A box that fills (almost) the platen is no crop: then the whole image is the answer, as one
|
||||
// box with found false.
|
||||
func Boxes(img image.Image, dpi int) ([]image.Rectangle, bool) {
|
||||
b := img.Bounds()
|
||||
w, h := b.Dx(), b.Dy()
|
||||
if dpi <= 0 || w < 2*margin || h < 2*margin {
|
||||
return []image.Rectangle{b}, false
|
||||
}
|
||||
bs := max(2, int(blockMM*float64(dpi)/25.4+0.5))
|
||||
gw, gh := (w+bs-1)/bs, (h+bs-1)/bs
|
||||
limit := lidLimit(img)
|
||||
doc := make([]bool, gw*gh)
|
||||
for gy := 0; gy < gh; gy++ {
|
||||
for gx := 0; gx < gw; gx++ {
|
||||
dark, total := 0, 0
|
||||
for y := b.Min.Y + gy*bs; y < min(b.Min.Y+(gy+1)*bs, b.Max.Y); y++ {
|
||||
for x := b.Min.X + gx*bs; x < min(b.Min.X+(gx+1)*bs, b.Max.X); x++ {
|
||||
total++
|
||||
if grayAt(img, x, y) < limit {
|
||||
dark++
|
||||
}
|
||||
}
|
||||
}
|
||||
doc[gy*gw+gx] = float64(dark) >= blockShare*float64(total)
|
||||
}
|
||||
}
|
||||
// Dust and the ringing of the JPEG around it make single blocks, or a pair, of "document": an opening (every
|
||||
// block must have all eight neighbours document to survive, and survivors give back their neighbours that are
|
||||
// document) removes what is not at least three blocks across.
|
||||
eroded := make([]bool, gw*gh)
|
||||
for gy := 1; gy < gh-1; gy++ {
|
||||
for gx := 1; gx < gw-1; gx++ {
|
||||
all := true
|
||||
for y := gy - 1; y <= gy+1 && all; y++ {
|
||||
for x := gx - 1; x <= gx+1; x++ {
|
||||
if !doc[y*gw+x] {
|
||||
all = false
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
eroded[gy*gw+gx] = all
|
||||
}
|
||||
}
|
||||
opened := make([]bool, gw*gh)
|
||||
for gy := 0; gy < gh; gy++ {
|
||||
for gx := 0; gx < gw; gx++ {
|
||||
if !eroded[gy*gw+gx] {
|
||||
continue
|
||||
}
|
||||
for y := gy - 1; y <= gy+1; y++ {
|
||||
for x := gx - 1; x <= gx+1; x++ {
|
||||
opened[y*gw+x] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
doc = opened
|
||||
// Join the blocks within the gap: a block is near when it is within gapBlocks/2 of a document block in each
|
||||
// direction (so two blocks up to gapBlocks apart meet), then the nearby blocks are labelled in 4-connected
|
||||
// regions; each region's box is that of the document blocks in it.
|
||||
radius := int(gapMM * float64(dpi) / 25.4 / float64(bs) / 2)
|
||||
near := make([]bool, gw*gh)
|
||||
for gy := 0; gy < gh; gy++ {
|
||||
for gx := 0; gx < gw; gx++ {
|
||||
if !doc[gy*gw+gx] {
|
||||
continue
|
||||
}
|
||||
for y := max(0, gy-radius); y <= min(gh-1, gy+radius); y++ {
|
||||
for x := max(0, gx-radius); x <= min(gw-1, gx+radius); x++ {
|
||||
near[y*gw+x] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
seen := make([]bool, gw*gh)
|
||||
var boxes []image.Rectangle
|
||||
for start := range near {
|
||||
if !near[start] || seen[start] {
|
||||
continue
|
||||
}
|
||||
minX, minY, maxX, maxY := gw, gh, -1, -1
|
||||
stack := []int{start}
|
||||
seen[start] = true
|
||||
for len(stack) > 0 {
|
||||
c := stack[len(stack)-1]
|
||||
stack = stack[:len(stack)-1]
|
||||
cx, cy := c%gw, c/gw
|
||||
if doc[c] {
|
||||
minX, minY, maxX, maxY = min(minX, cx), min(minY, cy), max(maxX, cx), max(maxY, cy)
|
||||
}
|
||||
for _, d := range [4][2]int{{1, 0}, {-1, 0}, {0, 1}, {0, -1}} {
|
||||
x, y := cx+d[0], cy+d[1]
|
||||
if x < 0 || y < 0 || x >= gw || y >= gh || !near[y*gw+x] || seen[y*gw+x] {
|
||||
continue
|
||||
}
|
||||
seen[y*gw+x] = true
|
||||
stack = append(stack, y*gw+x)
|
||||
}
|
||||
}
|
||||
if maxX < 0 {
|
||||
continue
|
||||
}
|
||||
r := image.Rect(minX*bs, minY*bs, (maxX+1)*bs, (maxY+1)*bs)
|
||||
if float64(r.Dx())/float64(dpi)*25.4 < minMM || float64(r.Dy())/float64(dpi)*25.4 < minMM {
|
||||
continue
|
||||
}
|
||||
boxes = append(boxes, r)
|
||||
}
|
||||
// Boxes that touch or overlap are one document: a frame and the text inside it, or a faint paper whose
|
||||
// parts the lid's white hides, are one however far apart their blocks lie.
|
||||
kept := boxes
|
||||
for merged := true; merged; {
|
||||
merged = false
|
||||
for i := 0; i < len(kept) && !merged; i++ {
|
||||
for j := i + 1; j < len(kept); j++ {
|
||||
if kept[i].Overlaps(kept[j]) {
|
||||
kept[i] = kept[i].Union(kept[j])
|
||||
kept = append(kept[:j], kept[j+1:]...)
|
||||
merged = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(kept) == 0 {
|
||||
return []image.Rectangle{b}, false
|
||||
}
|
||||
out := make([]image.Rectangle, len(kept))
|
||||
for i, r := range kept {
|
||||
out[i] = image.Rect(max(0, r.Min.X-margin), max(0, r.Min.Y-margin), min(w, r.Max.X+margin), min(h, r.Max.Y+margin)).Add(b.Min)
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
if out[i].Min.Y != out[j].Min.Y {
|
||||
return out[i].Min.Y < out[j].Min.Y
|
||||
}
|
||||
return out[i].Min.X < out[j].Min.X
|
||||
})
|
||||
if len(out) == 1 && float64(out[0].Dx()*out[0].Dy()) >= wholeShare*float64(w*h) {
|
||||
return []image.Rectangle{b}, false
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
// CropAll cuts each separate document out of a scanned glass JPEG when crop is set and there are two or more; one
|
||||
// document, none or crop off is Crop's answer, exactly as before. dpi is the scan's resolution.
|
||||
func CropAll(scan []byte, crop bool, dpi int) ([]Cropped, error) {
|
||||
one, err := Crop(scan, crop)
|
||||
if err != nil || !crop {
|
||||
return []Cropped{one}, err
|
||||
}
|
||||
img, err := jpeg.Decode(bytes.NewReader(scan))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the scanned image cannot be read: %w", err)
|
||||
}
|
||||
boxes, found := Boxes(img, dpi)
|
||||
if !found || len(boxes) < 2 {
|
||||
return []Cropped{one}, nil
|
||||
}
|
||||
sub, ok := img.(interface {
|
||||
SubImage(image.Rectangle) image.Image
|
||||
})
|
||||
if !ok {
|
||||
return []Cropped{one}, nil
|
||||
}
|
||||
out := make([]Cropped, 0, len(boxes))
|
||||
for _, box := range boxes {
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, sub.SubImage(box), &jpeg.Options{Quality: jpegQuality}); err != nil {
|
||||
return nil, fmt.Errorf("the cropped image cannot be encoded: %w", err)
|
||||
}
|
||||
out = append(out, Cropped{JPEG: buf.Bytes(), Width: box.Dx(), Height: box.Dy(), Cropped: true})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -1,229 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/draw"
|
||||
"image/jpeg"
|
||||
"math/rand"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// lidWhite is a w x h gray image of the lid's white, with a few specks of dust in one patch.
|
||||
func lidWhite(w, h int, seed int64) *image.Gray {
|
||||
g := image.NewGray(image.Rect(0, 0, w, h))
|
||||
draw.Draw(g, g.Bounds(), &image.Uniform{color.Gray{255}}, image.Point{}, draw.Src)
|
||||
r := rand.New(rand.NewSource(seed))
|
||||
for i := 0; i < 40; i++ {
|
||||
g.SetGray(100+r.Intn(w/4), 100+r.Intn(h/4), color.Gray{120}) // all in one patch, so counting them would make a box
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
func encode(t *testing.T, img image.Image) []byte {
|
||||
t.Helper()
|
||||
var b bytes.Buffer
|
||||
if err := jpeg.Encode(&b, img, &jpeg.Options{Quality: 95}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
// 1440/AC3: the captured receipt, top-left of a 1696x2203 scan at 200 dpi.
|
||||
func TestCropFindsTheCapturedReceipt(t *testing.T) {
|
||||
c, err := Crop(fixture(t, "scan.jpg"), true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !c.Cropped {
|
||||
t.Fatal("cropped = false for a scan with a receipt on it")
|
||||
}
|
||||
img, err := jpeg.Decode(bytes.NewReader(fixture(t, "scan.jpg")))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
box, found := Box(img)
|
||||
if !found {
|
||||
t.Fatal("no box found")
|
||||
}
|
||||
if box.Min.X > 10 || box.Min.Y > 10 {
|
||||
t.Errorf("box starts at %v, want within 10 px of (0,0)", box.Min)
|
||||
}
|
||||
if box.Max.X < 630 || box.Max.X > 660 {
|
||||
t.Errorf("right edge %d, want 630..660", box.Max.X)
|
||||
}
|
||||
if box.Max.Y < 1730 || box.Max.Y > 1810 {
|
||||
t.Errorf("bottom %d, want 1730..1810", box.Max.Y)
|
||||
}
|
||||
if c.Width != box.Dx() || c.Height != box.Dy() {
|
||||
t.Errorf("cropped image is %dx%d, the box %dx%d", c.Width, c.Height, box.Dx(), box.Dy())
|
||||
}
|
||||
got, err := jpeg.DecodeConfig(bytes.NewReader(c.JPEG))
|
||||
if err != nil || got.Width != box.Dx() || got.Height != box.Dy() {
|
||||
t.Errorf("the cropped JPEG is %+v (%v), want %dx%d", got, err, box.Dx(), box.Dy())
|
||||
}
|
||||
// About 80-84 mm wide at 200 dpi.
|
||||
if mm := float64(c.Width) / 200 * 25.4; mm < 80 || mm > 84 {
|
||||
t.Errorf("%.1f mm wide, want 80-84", mm)
|
||||
}
|
||||
}
|
||||
|
||||
// 1440/AC4: a plain lid with specks has no document.
|
||||
func TestCropKeepsAPlainLidWhole(t *testing.T) {
|
||||
scan := encode(t, lidWhite(800, 1100, 1))
|
||||
c, err := Crop(scan, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.Cropped || c.Width != 800 || c.Height != 1100 || !bytes.Equal(c.JPEG, scan) {
|
||||
t.Fatalf("a plain lid was cropped: %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// 1440/AC4: a document that fills the platen is no crop either.
|
||||
func TestCropKeepsAFullPlatenDocumentWhole(t *testing.T) {
|
||||
g := lidWhite(800, 1100, 2)
|
||||
draw.Draw(g, image.Rect(0, 0, 790, 1090), &image.Uniform{color.Gray{200}}, image.Point{}, draw.Src)
|
||||
scan := encode(t, g)
|
||||
c, err := Crop(scan, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.Cropped || c.Width != 800 || c.Height != 1100 {
|
||||
t.Fatalf("a document filling the platen was cropped: %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// 1440/AC4: crop=false keeps the captured scan whole, though it has a document.
|
||||
func TestCropOffKeepsTheWholeImage(t *testing.T) {
|
||||
scan := fixture(t, "scan.jpg")
|
||||
c, err := Crop(scan, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.Cropped || c.Width != 1696 || c.Height != 2203 || !bytes.Equal(c.JPEG, scan) {
|
||||
t.Fatalf("crop=false cropped: %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// 1440/AC3: a document away from the origin is found where it is, not assumed in the corner.
|
||||
func TestCropFindsADocumentInTheMiddleOfTheGlass(t *testing.T) {
|
||||
g := lidWhite(800, 1100, 3)
|
||||
draw.Draw(g, image.Rect(200, 300, 500, 700), &image.Uniform{color.Gray{240}}, image.Point{}, draw.Src)
|
||||
img, err := jpeg.Decode(bytes.NewReader(encode(t, g)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
box, found := Box(img)
|
||||
if !found {
|
||||
t.Fatal("no box")
|
||||
}
|
||||
if box.Min.X < 190 || box.Min.X > 205 || box.Max.X < 495 || box.Max.X > 510 || box.Min.Y < 290 || box.Min.Y > 305 || box.Max.Y < 695 || box.Max.Y > 710 {
|
||||
t.Fatalf("box = %v, want about (200,300)-(500,700)", box)
|
||||
}
|
||||
}
|
||||
|
||||
// onWhite is a lid-white image of w x h with the rectangles drawn on it in gray (generated, not captured: the
|
||||
// thresholds of Boxes are in millimetres, and this is 100 dpi, 1 mm about 4 px).
|
||||
func onWhite(t *testing.T, seed int64, rects ...image.Rectangle) []byte {
|
||||
t.Helper()
|
||||
g := lidWhite(800, 1100, seed)
|
||||
for _, r := range rects {
|
||||
draw.Draw(g, r, &image.Uniform{color.Gray{200}}, image.Point{}, draw.Src)
|
||||
}
|
||||
return encode(t, g)
|
||||
}
|
||||
|
||||
// 1528: documents lying apart on the white are each cut out, in reading order; one document, as before, is one.
|
||||
func TestCropAllCutsEachDocumentApartFromTheOthers(t *testing.T) {
|
||||
a := image.Rect(100, 100, 300, 300) // 51 mm square
|
||||
b := image.Rect(450, 100, 650, 400) // 38 mm right of a
|
||||
c := image.Rect(100, 600, 400, 900)
|
||||
for name, tc := range map[string]struct {
|
||||
rects []image.Rectangle
|
||||
want []image.Rectangle
|
||||
}{
|
||||
"one": {[]image.Rectangle{a}, nil},
|
||||
"two": {[]image.Rectangle{b, a}, []image.Rectangle{a, b}},
|
||||
"three": {[]image.Rectangle{c, b, a}, []image.Rectangle{a, b, c}},
|
||||
} {
|
||||
got, err := CropAll(onWhite(t, 5, tc.rects...), true, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tc.want == nil {
|
||||
one, _ := Crop(onWhite(t, 5, a), true)
|
||||
if len(got) != 1 || !bytes.Equal(got[0].JPEG, one.JPEG) {
|
||||
t.Errorf("%s: one document is not Crop's answer", name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if len(got) != len(tc.want) {
|
||||
t.Fatalf("%s: %d documents, want %d", name, len(got), len(tc.want))
|
||||
}
|
||||
for i, w := range tc.want {
|
||||
// about the rectangle, within the margin and a block of rounding
|
||||
if !got[i].Cropped || got[i].Width < w.Dx() || got[i].Width > w.Dx()+2*margin+2*4 || got[i].Height < w.Dy() || got[i].Height > w.Dy()+2*margin+2*4 {
|
||||
t.Errorf("%s: document %d is %dx%d, want about %dx%d", name, i, got[i].Width, got[i].Height, w.Dx(), w.Dy())
|
||||
}
|
||||
cfg, err := jpeg.DecodeConfig(bytes.NewReader(got[i].JPEG))
|
||||
if err != nil || cfg.Width != got[i].Width {
|
||||
t.Errorf("%s: document %d JPEG %+v %v", name, i, cfg, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1528: one document with a white area inside it is not split: a frame with an island inside it, a sheet with a
|
||||
// white band across it narrower than the gap, and specks of dust beside it.
|
||||
func TestCropAllDoesNotSplitOneDocumentWithWhiteInsideIt(t *testing.T) {
|
||||
g := lidWhite(800, 1100, 6)
|
||||
frame := image.Rect(100, 100, 600, 800)
|
||||
draw.Draw(g, frame, &image.Uniform{color.Gray{200}}, image.Point{}, draw.Src)
|
||||
draw.Draw(g, frame.Inset(50), &image.Uniform{color.Gray{255}}, image.Point{}, draw.Src) // white inside the frame
|
||||
draw.Draw(g, image.Rect(300, 400, 400, 500), &image.Uniform{color.Gray{200}}, image.Point{}, draw.Src) // an island in it
|
||||
got, err := CropAll(encode(t, g), true, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("a frame with an island inside was cut into %d documents", len(got))
|
||||
}
|
||||
|
||||
g = lidWhite(800, 1100, 7)
|
||||
draw.Draw(g, image.Rect(100, 100, 500, 500), &image.Uniform{color.Gray{200}}, image.Point{}, draw.Src)
|
||||
draw.Draw(g, image.Rect(100, 530, 500, 900), &image.Uniform{color.Gray{200}}, image.Point{}, draw.Src) // 30 px = 7.6 mm of white between
|
||||
got, err = CropAll(encode(t, g), true, 100)
|
||||
if err != nil || len(got) != 1 {
|
||||
t.Fatalf("a sheet with a narrow white band across it: %d documents, %v", len(got), err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1528: a speck smaller than the minimum is no document, and crop off files the whole platen as one.
|
||||
func TestCropAllIgnoresSpecksAndCropOffIsOneWholeImage(t *testing.T) {
|
||||
scan := onWhite(t, 8, image.Rect(100, 100, 300, 300), image.Rect(600, 800, 620, 815)) // 5 x 4 mm stain
|
||||
got, err := CropAll(scan, true, 100)
|
||||
if err != nil || len(got) != 1 {
|
||||
t.Fatalf("a stain made %d documents (%v)", len(got), err)
|
||||
}
|
||||
two := onWhite(t, 9, image.Rect(100, 100, 300, 300), image.Rect(450, 100, 650, 400))
|
||||
got, err = CropAll(two, false, 100)
|
||||
if err != nil || len(got) != 1 || got[0].Cropped {
|
||||
t.Fatalf("crop off: %d documents, cropped %v", len(got), got[0].Cropped)
|
||||
}
|
||||
}
|
||||
|
||||
// 1528: the captured scan (one receipt) is one document, the same bytes Crop gives.
|
||||
func TestCropAllOfTheCapturedReceiptIsCropsAnswer(t *testing.T) {
|
||||
scan := fixture(t, "scan.jpg")
|
||||
got, err := CropAll(scan, true, 200)
|
||||
one, _ := Crop(scan, true)
|
||||
if err != nil || len(got) != 1 || !bytes.Equal(got[0].JPEG, one.JPEG) {
|
||||
t.Fatalf("%d documents (%v), want Crop's one", len(got), err)
|
||||
}
|
||||
img, _ := jpeg.Decode(bytes.NewReader(scan))
|
||||
if boxes, _ := Boxes(img, 200); len(boxes) != 1 {
|
||||
t.Errorf("the receipt is %d regions: %v", len(boxes), boxes)
|
||||
}
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"sort"
|
||||
"strconv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 1528: a tool reads only arguments its input schema declares; the console refuses an undeclared
|
||||
// one before it reaches the tool (scanner.scan read apart, which its schema never named).
|
||||
func TestEveryArgumentAToolReadsIsDeclared(t *testing.T) {
|
||||
fs := token.NewFileSet()
|
||||
f, err := parser.ParseFile(fs, "main.go", nil, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checked := 0
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
lit, ok := n.(*ast.CompositeLit)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
var name string
|
||||
var input, run ast.Expr
|
||||
for _, e := range lit.Elts {
|
||||
kv, ok := e.(*ast.KeyValueExpr)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
switch k, _ := kv.Key.(*ast.Ident); {
|
||||
case k == nil:
|
||||
case k.Name == "Name":
|
||||
if b, ok := kv.Value.(*ast.BasicLit); ok {
|
||||
name, _ = strconv.Unquote(b.Value)
|
||||
}
|
||||
case k.Name == "Input":
|
||||
input = kv.Value
|
||||
case k.Name == "Run":
|
||||
run = kv.Value
|
||||
}
|
||||
}
|
||||
if name == "" || input == nil || run == nil {
|
||||
return true
|
||||
}
|
||||
declared := map[string]bool{}
|
||||
ast.Inspect(input, func(m ast.Node) bool {
|
||||
kv, ok := m.(*ast.KeyValueExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if k, ok := kv.Key.(*ast.BasicLit); ok && k.Value == `"properties"` {
|
||||
if props, ok := kv.Value.(*ast.CompositeLit); ok {
|
||||
for _, p := range props.Elts {
|
||||
if pkv, ok := p.(*ast.KeyValueExpr); ok {
|
||||
if b, ok := pkv.Key.(*ast.BasicLit); ok {
|
||||
s, _ := strconv.Unquote(b.Value)
|
||||
declared[s] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
})
|
||||
var missing []string
|
||||
ast.Inspect(run, func(m ast.Node) bool {
|
||||
ix, ok := m.(*ast.IndexExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if id, ok := ix.X.(*ast.Ident); ok && id.Name == "args" {
|
||||
if b, ok := ix.Index.(*ast.BasicLit); ok && b.Kind == token.STRING {
|
||||
s, _ := strconv.Unquote(b.Value)
|
||||
if !declared[s] {
|
||||
missing = append(missing, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
checked++
|
||||
if len(missing) > 0 {
|
||||
sort.Strings(missing)
|
||||
t.Errorf("%s reads %v, which its input schema does not declare", name, missing)
|
||||
}
|
||||
return false
|
||||
})
|
||||
if checked < 5 {
|
||||
t.Fatalf("checked %d tools; the test no longer finds the tool literals", checked)
|
||||
}
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
package main
|
||||
|
||||
// The tools never open the admin password (novox/hq ADR 0436 §12, ADR 0467 clause 4): only the device process
|
||||
// reads it, as its own account, brother-device. The build seat runs every test under one account, so no
|
||||
// permission test can show the split; this is the file-level proxy mesh-mail's secret_boundary_test.go uses
|
||||
// instead: an AST check that the tools' files import no password-reading package and name no password file.
|
||||
|
||||
import (
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const secretPackage = "brother/cmd/brother/internal/secret"
|
||||
|
||||
// toolFiles are the files the tools' bundle runs (the scanner's, the printer.status's and the printer.settings'
|
||||
// ask), as opposed to the device process's: deviceproc.go, admin.go, pin.go and html.go.
|
||||
var toolFiles = []string{"main.go", "printer.go", "scan.go", "settings.go", "ipp.go", "escl.go", "file.go", "crop.go", "pdf.go", "parts.go", "certcheck.go"}
|
||||
|
||||
// 1453: none of the tools' files imports the password-reading package or names the password's file.
|
||||
func TestTheToolsNeverOpenTheAdminPassword(t *testing.T) {
|
||||
for _, file := range toolFiles {
|
||||
fset := token.NewFileSet()
|
||||
f, err := parser.ParseFile(fset, file, nil, parser.ImportsOnly)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", file, err)
|
||||
}
|
||||
for _, imp := range f.Imports {
|
||||
if strings.Trim(imp.Path.Value, `"`) == secretPackage {
|
||||
t.Errorf("%s imports %s, which only the device process may (ADR 0436 §12)", file, secretPackage)
|
||||
}
|
||||
}
|
||||
src, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, bad := range []string{`Getenv("BROTHER_ADMIN_PASSWORD")`, `Getenv("BROTHER_STORE")`, "ReadStoreFacts", "NewS3", "StoreFacts"} {
|
||||
if strings.Contains(string(src), bad) {
|
||||
t.Errorf("%s names %s: the password's and the bucket key's files are the device process's alone (ADR 0468 clause 1)", file, bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1453: the device process does read it through the package, and the package is the one place a password file is
|
||||
// opened.
|
||||
func TestTheAdminProcessReadsThePasswordThroughThePackage(t *testing.T) {
|
||||
fset := token.NewFileSet()
|
||||
f, err := parser.ParseFile(fset, "deviceproc.go", nil, parser.ImportsOnly)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := false
|
||||
for _, imp := range f.Imports {
|
||||
if strings.Trim(imp.Path.Value, `"`) == secretPackage {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("deviceproc.go does not import %s", secretPackage)
|
||||
}
|
||||
for _, file := range append(append([]string{}, toolFiles...), "admin.go", "deviceproc.go", "pin.go", "html.go", "objects.go", "capture.go") {
|
||||
src, _ := os.ReadFile(file)
|
||||
if strings.Contains(string(src), "os.ReadFile") && file != "settings.go" && file != "file.go" && file != "escl.go" && file != "objects.go" {
|
||||
t.Errorf("%s reads a file: only settings.json (settings.go) and the scan folder's PDFs are read outside the secret package", file)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
// The device process over a real bus (novox/hq ADR 0467 clause 4; the shape of mesh-mail's
|
||||
// sender_integration_test.go, ADR 0436 §7): a JetStream server of the test's own, the work-queue stream
|
||||
// device-asks and its consumer, the state bucket device-answers, and a fake printer. An ask published on the
|
||||
// stream is answered in the state under its id and acknowledged. Compiled by the merge gate's
|
||||
// `go vet -tags integration`; never run on this machine.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"brother/cmd/brother/internal/secret"
|
||||
)
|
||||
|
||||
// printerAskedSubject is the wire subject the controller composes for the stream's one event:
|
||||
// "mesh.stream." + module + ".event." + event (mesh-controller internal/broker/modstreams.go).
|
||||
const printerAskedSubject = "mesh.stream.brother.event." + EventAsked
|
||||
|
||||
func newTestBus(t *testing.T) string {
|
||||
t.Helper()
|
||||
opts := &server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, JetStream: true, StoreDir: t.TempDir(),
|
||||
NoLog: true, NoSigs: true}
|
||||
s, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatalf("a bus for this test could not be made: %v", err)
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(30 * time.Second) {
|
||||
s.Shutdown()
|
||||
t.Fatal("a bus for this test did not come up within 30s")
|
||||
}
|
||||
t.Cleanup(func() { s.Shutdown(); s.WaitForShutdown() })
|
||||
return s.ClientURL()
|
||||
}
|
||||
|
||||
// 1453: an ask on device-asks is answered under its id in device-answers by the device process, over the
|
||||
// printer's pinned HTTPS and the login, and acknowledged; stopping it leaves nothing in hand.
|
||||
func TestTheAdminProcessAnswersAnAskOverARealBus(t *testing.T) {
|
||||
url := newTestBus(t)
|
||||
ctx := context.Background()
|
||||
nc, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: AsksStream, Subjects: []string{printerAskedSubject},
|
||||
Retention: jetstream.WorkQueuePolicy, Discard: jetstream.DiscardNew, MaxAge: 24 * time.Hour, MaxBytes: 8 << 20}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
account := process.Account{URL: url, Node: "g14", Module: "brother", User: "g14.brother." + DeviceProcessID, Password: "pw"}
|
||||
if _, err := js.CreateConsumer(ctx, AsksStream, jetstream.ConsumerConfig{Durable: account.QueueConsumerName(),
|
||||
AckPolicy: jetstream.AckExplicitPolicy, AckWait: 30 * time.Second, MaxDeliver: 3}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: account.Module + "_" + AnswersState, TTL: 10 * time.Minute})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
f := newFakePrinter(t, "B1891", "pw")
|
||||
dir := t.TempDir()
|
||||
settings := filepath.Join(dir, "settings.json")
|
||||
os.WriteFile(settings, []byte(`{"printer":"`+f.host()+`","printer-certificate":"`+f.print()+`"}`), 0o600)
|
||||
pwFile := filepath.Join(dir, "admin-password.secret")
|
||||
os.WriteFile(pwFile, []byte("pw\n"), 0o600)
|
||||
w := &deviceWorker{
|
||||
Settings: func() (Settings, error) { return LoadSettings(settings) },
|
||||
Password: func() (string, error) { return secret.Read(pwFile) },
|
||||
Now: time.Now,
|
||||
}
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- runDevice(runCtx, account, w) }()
|
||||
|
||||
body, _ := json.Marshal(Ask{ID: "ask-1", Op: OpSettings, Page: "/general/other.html", At: time.Now()})
|
||||
if _, err := js.Publish(ctx, printerAskedSubject, body, jetstream.WithMsgID("ask-1")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var answer string
|
||||
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := kv.Get(ctx, "ask-1"); err == nil {
|
||||
answer = string(e.Value())
|
||||
break
|
||||
}
|
||||
}
|
||||
if !strings.Contains(answer, "Synthetic settings") {
|
||||
t.Fatalf("the answer under ask-1 is %q", answer)
|
||||
}
|
||||
if strings.Contains(answer, "shouldnotshow") {
|
||||
t.Errorf("a password field's value is in the answer")
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil {
|
||||
t.Fatal("runDevice returned nil, want ctx.Err() once its context was cancelled")
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("runDevice did not stop once its context was cancelled")
|
||||
}
|
||||
cons, err := js.Consumer(ctx, AsksStream, account.QueueConsumerName())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := cons.Info(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.NumPending != 0 || info.NumAckPending != 0 {
|
||||
t.Errorf("the ask was not acknowledged: %d pending, %d awaiting an acknowledgement", info.NumPending, info.NumAckPending)
|
||||
}
|
||||
}
|
||||
@@ -1,816 +0,0 @@
|
||||
package main
|
||||
|
||||
// mesh brother device (novox/hq ADR 0467 clause 4, in the shape of ADR 0436 §7): the device process, a process
|
||||
// with a bus account of its own (brother-device), never a seat. It alone of the module's parts reads the admin
|
||||
// password and logs in to the printer's admin page; the tools queue an ask on the work-queue stream device-asks
|
||||
// and read its answer in the state device-answers. The admin password is never in an answer, an event or a log.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/objectlink"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"brother/cmd/brother/internal/secret"
|
||||
)
|
||||
|
||||
// Answers is the state the worker writes an answer in, once.
|
||||
type Answers interface {
|
||||
Create(key string, value any) (uint64, error)
|
||||
}
|
||||
|
||||
// action is what is done to the queued message.
|
||||
type action int
|
||||
|
||||
const (
|
||||
actionAck action = iota
|
||||
actionRetry
|
||||
actionTerm
|
||||
// actionSettled says the message was acknowledged already, before the work began: what is not safe to do twice
|
||||
// (a scan, a print) is acknowledged first, so a slow one is not delivered again after the bus's wait.
|
||||
actionSettled
|
||||
)
|
||||
|
||||
// pendingUpload is the parts of an upload that have come.
|
||||
type pendingUpload struct {
|
||||
parts map[int][]byte
|
||||
n int
|
||||
sha string
|
||||
at time.Time
|
||||
}
|
||||
|
||||
// pendingFor is how long an incomplete upload's parts are kept.
|
||||
const pendingFor = 15 * time.Minute
|
||||
|
||||
// deviceWorker answers the asks: it alone talks to the device (the scanner, the printer's pages and IPP) and to
|
||||
// the bucket. Settings and the password are read at each ask, so a changed config value or a password given later
|
||||
// is the next ask's.
|
||||
type deviceWorker struct {
|
||||
Settings func() (Settings, error)
|
||||
Password func() (string, error)
|
||||
Objects func() (Objects, error)
|
||||
// Emit says scan.added on the bus under a message id.
|
||||
Emit func(body map[string]any, id string) error
|
||||
Answers Answers
|
||||
Node string
|
||||
// Capture scans; nil is the device's own (a test puts a stand-in).
|
||||
Capture func(ctx context.Context, req ScanRequest) ([]Capture, error)
|
||||
// Convert calls the seat document-converter's convert verb as this process's own account (clause 7); nil: none.
|
||||
Convert func(body map[string]any) (json.RawMessage, error)
|
||||
// ConvertFollow is how long a conversion is followed; zero is ConvertFollow (a test makes it shorter).
|
||||
ConvertFollow time.Duration
|
||||
// Fetcher is the guarded fetch of a web link (fetch.go); it reads its facts when it is first needed.
|
||||
Fetcher func() (*Fetcher, error)
|
||||
// answerPart is the raw size of a document part in an answer; zero is AnswerPartSize (a test makes it larger).
|
||||
answerPart int
|
||||
Now func() time.Time
|
||||
Logf func(format string, a ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
pending map[string]*pendingUpload
|
||||
// docs serialises the work on open documents.
|
||||
docs sync.Mutex
|
||||
}
|
||||
|
||||
func (w *deviceWorker) logf(format string, a ...any) {
|
||||
if w.Logf != nil {
|
||||
w.Logf(format, a...)
|
||||
}
|
||||
}
|
||||
|
||||
// tooLargeCode is the bus's error code for a message over the maximum allowed (the state refuses a value so).
|
||||
const tooLargeCode = 10054
|
||||
|
||||
// Handle answers one queued ask. An ask that cannot be read is terminated; an ask too old for anyone to wait for
|
||||
// is acknowledged and the device not touched; every other outcome, a failure included, is written as the answer
|
||||
// under the ask's id and the message acknowledged; an answer the state refuses as too large is
|
||||
// replaced by a small error answer and the message terminated; an answer that could not be written for another
|
||||
// reason (the store away, a timeout) is redelivered (the bus's limit ends that). early acknowledges the message; it is called before a scan or a print, which are not
|
||||
// done twice.
|
||||
func (w *deviceWorker) Handle(body []byte, early func()) (action, time.Duration) {
|
||||
var ask Ask
|
||||
if err := json.Unmarshal(body, &ask); err != nil || ask.ID == "" {
|
||||
w.logf("[brother] device: a queued ask could not be read: %v", err)
|
||||
return actionTerm, 0
|
||||
}
|
||||
if age := w.Now().Sub(ask.At); age > StaleAfter {
|
||||
w.logf("[brother] device: ask %s is %s old; nobody waits for it, the device is not touched", ask.ID, age.Round(time.Second))
|
||||
return actionAck, 0
|
||||
}
|
||||
if !ask.Deadline.IsZero() && w.Now().After(ask.Deadline) {
|
||||
// The asker has stopped waiting for this ask to be taken up: nothing is scanned or printed for it (issue
|
||||
// 1539). The reason is its answer, for a late reader, and the message ends here.
|
||||
w.logf("[brother] device: ask %s was taken up after its deadline %s; the device is not touched", ask.ID, ask.Deadline.Format(time.RFC3339))
|
||||
if _, err := w.Answers.Create(ask.ID, map[string]any{"error": "the ask was taken up after its deadline (the device process was busy): nothing was done for it; ask again"}); err != nil && !errors.Is(err, stdio.ErrChanged) {
|
||||
w.logf("[brother] device: the answer to %s could not be written: %v", ask.ID, err)
|
||||
}
|
||||
return actionTerm, 0
|
||||
}
|
||||
settled := false
|
||||
if (ask.Op == OpScan || ask.Op == OpPrint) && early != nil {
|
||||
early()
|
||||
settled = true
|
||||
}
|
||||
// A scan of the feeder may run as long as FeederMaxBound (the scan itself ends sooner by FeederBound, which
|
||||
// scales with its pages) and the PDF is then stored; every other ask is done in 5 minutes.
|
||||
bound := 5 * time.Minute
|
||||
if ask.Op == OpScan && ask.Source == "feeder" {
|
||||
bound = FeederMaxBound + 5*time.Minute
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), bound)
|
||||
defer cancel()
|
||||
var answer map[string]any
|
||||
if got, err := w.run(ctx, ask); err != nil {
|
||||
answer = map[string]any{"error": err.Error()}
|
||||
} else {
|
||||
answer = got
|
||||
}
|
||||
if _, err := w.Answers.Create(ask.ID, answer); err != nil && !errors.Is(err, stdio.ErrChanged) {
|
||||
w.logf("[brother] device: the answer to %s could not be written: %v", ask.ID, err)
|
||||
var api *jetstream.APIError
|
||||
if !errors.As(err, &api) || api.ErrorCode != tooLargeCode {
|
||||
// The store is away, timed out or cut: the answer may be written next time (a scan is settled already).
|
||||
if settled {
|
||||
return actionSettled, 0
|
||||
}
|
||||
return actionRetry, 5 * time.Second
|
||||
}
|
||||
// The ask ends here: a refused value would be refused again.
|
||||
// Say the failure as a small error answer, so the tool's wait ends with the reason (issue 1535).
|
||||
msg := fmt.Sprintf("the answer to this ask could not be written in the state %s: %v", AnswersState, err)
|
||||
if _, err2 := w.Answers.Create(ask.ID, map[string]any{"error": msg}); err2 != nil && !errors.Is(err2, stdio.ErrChanged) {
|
||||
w.logf("[brother] device: the failure of %s could not be written either: %v", ask.ID, err2)
|
||||
}
|
||||
if settled {
|
||||
return actionSettled, 0
|
||||
}
|
||||
return actionTerm, 0
|
||||
}
|
||||
if settled {
|
||||
return actionSettled, 0
|
||||
}
|
||||
return actionAck, 0
|
||||
}
|
||||
|
||||
// run is the operation an ask names.
|
||||
func (w *deviceWorker) run(ctx context.Context, ask Ask) (map[string]any, error) {
|
||||
switch ask.Op {
|
||||
case OpSettings:
|
||||
return w.read(ctx, ask.Page, ask.Links, ask.Hidden)
|
||||
case OpScan:
|
||||
return w.scan(ctx, ask)
|
||||
case OpDocument:
|
||||
return w.document(ctx, ask)
|
||||
case OpUpload:
|
||||
return w.upload(ctx, ask)
|
||||
case OpPrint:
|
||||
return w.print(ctx, ask)
|
||||
case OpOpen:
|
||||
return w.openDocuments(ctx)
|
||||
}
|
||||
return nil, fmt.Errorf("the ask %s names an operation, %q, the device process does not do", ask.ID, ask.Op)
|
||||
}
|
||||
|
||||
// scan scans the glass (cropped) or the feeder (whole sheets, one PDF of all pages), builds the PDF, keeps it in the bucket under its SHA-256 and says scan.added with
|
||||
// the object's key (ADR 0468 clause 2), in that order: nothing is said of a scan that is not kept, and nothing is
|
||||
// kept nor said when the scanner gives no page (1440/AC6). The answer is the event's body with key, size and the
|
||||
// number of parts.
|
||||
func (w *deviceWorker) scan(ctx context.Context, ask Ask) (map[string]any, error) {
|
||||
req, err := ParseScan(ask.Source, ask.Duplex, ask.Apart)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
set, err := w.Settings()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if set.Scanner == "" {
|
||||
return nil, unsetError("scanner")
|
||||
}
|
||||
objects, err := w.Objects()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
svc := &Service{Settings: set, Node: w.Node, Now: w.Now}
|
||||
if ask.Document != "" {
|
||||
return w.scanDocument(ctx, ask, svc, objects)
|
||||
}
|
||||
capture := svc.Capture
|
||||
if w.Capture != nil {
|
||||
capture = w.Capture
|
||||
}
|
||||
caps, err := capture(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return w.keep(ctx, objects, set, caps)
|
||||
}
|
||||
|
||||
// keep stores each captured PDF in the bucket and says scan.added for it, in that order. One document is answered
|
||||
// as today: the event's body with key, size and the number of parts. Several (documents lying apart on the glass)
|
||||
// are answered as documents, a list of those, each filed on its own by the tool. A document that could not be kept
|
||||
// or announced is named in failed and does not hold back the others: what is announced is always filed.
|
||||
func (w *deviceWorker) keep(ctx context.Context, objects Objects, set Settings, caps []Capture) (map[string]any, error) {
|
||||
if len(caps) == 0 {
|
||||
return nil, fmt.Errorf("the scanner gave no document")
|
||||
}
|
||||
var docs []map[string]any
|
||||
var failed []string
|
||||
for i, c := range caps {
|
||||
one, err := w.keepOne(ctx, objects, set, c)
|
||||
if err != nil {
|
||||
if len(caps) == 1 {
|
||||
return nil, err
|
||||
}
|
||||
failed = append(failed, fmt.Sprintf("document %d: %v", i+1, err))
|
||||
continue
|
||||
}
|
||||
docs = append(docs, one)
|
||||
}
|
||||
if len(docs) == 0 {
|
||||
return nil, fmt.Errorf("no document could be kept: %s", strings.Join(failed, "; "))
|
||||
}
|
||||
if len(caps) == 1 {
|
||||
return docs[0], nil
|
||||
}
|
||||
out := map[string]any{"documents": docs}
|
||||
if len(failed) > 0 {
|
||||
out["failed"] = failed
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// keepOne keeps one captured PDF and announces it.
|
||||
func (w *deviceWorker) keepOne(ctx context.Context, objects Objects, set Settings, c Capture) (map[string]any, error) {
|
||||
// A scan over a document's limit is not kept nor announced: an announced scan is always filed (issue 1539).
|
||||
if len(c.PDF) > int(objectlink.MaxDocument) {
|
||||
return nil, fmt.Errorf("the scan's PDF is %d bytes, over the %d a document may be (novox/hq ADR 0468 §3): it is not kept; scan with a lower resolution or fewer sheets",
|
||||
len(c.PDF), objectlink.MaxDocument)
|
||||
}
|
||||
signer, ok := objects.(Presigner)
|
||||
if !ok {
|
||||
return nil, errors.New("the bucket cannot sign a link: the scan is not kept")
|
||||
}
|
||||
body := c.Body(w.Node, set.ColorMode)
|
||||
key := body["object"].(string)
|
||||
if err := Store(ctx, objects, key, "application/pdf", c.PDF); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := w.Emit(body, c.SHA256); err != nil {
|
||||
return nil, fmt.Errorf("the scan is kept as %s, but %s could not be said on the bus: %w", key, EventAdded, err)
|
||||
}
|
||||
// The answer carries a link to read the stored PDF, signed with the binding's key (novox/hq ADR 0468 §4): the tool
|
||||
// side reads the scan by it and files it; no byte of the document is in the answer.
|
||||
link, err := signer.Link(http.MethodGet, key, objectlink.MaxExpires)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the scan is kept as %s, but a link to it could not be signed: %w", key, err)
|
||||
}
|
||||
answer := map[string]any{"key": key, "size": len(c.PDF), "link": link, "link-seconds": int(objectlink.MaxExpires / time.Second)}
|
||||
for k, v := range body {
|
||||
answer[k] = v
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// document is one part of a scan kept in the bucket.
|
||||
func (w *deviceWorker) document(ctx context.Context, ask Ask) (map[string]any, error) {
|
||||
if err := checkDocument(ask); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
objects, err := w.Objects()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, err := objects.Get(ctx, ask.Object)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
each := w.answerPart
|
||||
if each <= 0 {
|
||||
each = AnswerPartSize
|
||||
}
|
||||
part, n, err := cutPartOf(data, ask.Part, each)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"part": ask.Part, "parts": n, "size": len(data), "sha256": sha256Hex(data),
|
||||
"data": base64.StdEncoding.EncodeToString(part)}, nil
|
||||
}
|
||||
|
||||
// upload keeps a part, and once all have come joins them, checks the SHA-256 and the format and stores the document
|
||||
// as uploads/<upload>.<ext>. A wrong SHA-256 or a document that is neither PDF nor JPEG stores nothing, and the
|
||||
// parts are dropped; a part out of range is refused.
|
||||
func (w *deviceWorker) upload(ctx context.Context, ask Ask) (map[string]any, error) {
|
||||
if err := checkUpload(ask); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, _ := base64.StdEncoding.DecodeString(ask.Data)
|
||||
w.mu.Lock()
|
||||
if w.pending == nil {
|
||||
w.pending = map[string]*pendingUpload{}
|
||||
}
|
||||
for id, p := range w.pending {
|
||||
if w.Now().Sub(p.at) > pendingFor {
|
||||
delete(w.pending, id)
|
||||
}
|
||||
}
|
||||
p := w.pending[ask.Upload]
|
||||
if p == nil {
|
||||
p = &pendingUpload{parts: map[int][]byte{}, n: ask.Parts, sha: ask.SHA256}
|
||||
w.pending[ask.Upload] = p
|
||||
}
|
||||
if p.n != ask.Parts || p.sha != ask.SHA256 {
|
||||
w.mu.Unlock()
|
||||
return nil, fmt.Errorf("upload %s was begun with %d parts and SHA-256 %s: its parts all say the same", ask.Upload, p.n, p.sha)
|
||||
}
|
||||
p.parts[ask.Part] = raw
|
||||
p.at = w.Now()
|
||||
if len(p.parts) < p.n {
|
||||
got := len(p.parts)
|
||||
w.mu.Unlock()
|
||||
return map[string]any{"upload": ask.Upload, "complete": false, "received": got, "parts": ask.Parts}, nil
|
||||
}
|
||||
delete(w.pending, ask.Upload)
|
||||
w.mu.Unlock()
|
||||
|
||||
ordered := make([][]byte, p.n)
|
||||
for i := range ordered {
|
||||
ordered[i] = p.parts[i+1]
|
||||
}
|
||||
doc, err := joinParts(ordered, p.sha)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("upload %s: %w", ask.Upload, err)
|
||||
}
|
||||
format, ext, err := SniffFormat(doc)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("upload %s: %w", ask.Upload, err)
|
||||
}
|
||||
objects, err := w.Objects()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key := "uploads/" + ask.Upload + "." + ext
|
||||
if err := objects.Put(ctx, key, format, doc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"upload": ask.Upload, "complete": true, "object": key, "size": len(doc), "sha256": p.sha}, nil
|
||||
}
|
||||
|
||||
// print fetches a scan or a completed upload from the bucket and sends it to the printer as an IPP Print-Job in
|
||||
// its own format, and answers the job's id and state. The printer's host is its IPP address; nothing is fetched
|
||||
// or sent before the ask is checked.
|
||||
func (w *deviceWorker) print(ctx context.Context, ask Ask) (map[string]any, error) {
|
||||
if err := checkPrint(&ask); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
set, err := w.Settings()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if set.Printer == "" {
|
||||
return nil, unsetError("printer")
|
||||
}
|
||||
objects, err := w.Objects()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key := ask.Object
|
||||
var doc []byte
|
||||
contentType := ""
|
||||
switch {
|
||||
case ask.URL != "":
|
||||
fetched, err := w.fetch(ctx, ask.URL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The fetched document is kept once under its SHA-256, as a scan is, and printed as that object.
|
||||
doc, key, contentType = fetched.Body, sha256Hex(fetched.Body), fetched.ContentType
|
||||
if err := Store(ctx, objects, key, fetched.ContentType, doc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case key == "":
|
||||
for _, ext := range []string{"pdf", "jpg"} {
|
||||
k := "uploads/" + ask.Upload + "." + ext
|
||||
if has, err := objects.Has(ctx, k); err != nil {
|
||||
return nil, err
|
||||
} else if has {
|
||||
key = k
|
||||
break
|
||||
}
|
||||
}
|
||||
if key == "" {
|
||||
return nil, fmt.Errorf("upload %s is not complete: no document is kept under it", ask.Upload)
|
||||
}
|
||||
fallthrough
|
||||
default:
|
||||
var err error
|
||||
if doc, err = objects.Get(ctx, key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
format, _, err := SniffFormat(doc)
|
||||
var convertedFrom string
|
||||
if err != nil {
|
||||
// Not a PDF or a JPEG: a type the converter takes becomes a PDF first (clause 7), any other is refused.
|
||||
pdf, pdfKey, cerr := w.toPDF(ctx, objects, key, doc, contentType)
|
||||
if cerr != nil {
|
||||
return nil, cerr
|
||||
}
|
||||
doc, convertedFrom, key = pdf, key, pdfKey
|
||||
format = "application/pdf"
|
||||
}
|
||||
// The printer is asked what it takes before the job, and a format it does not list is refused with the list
|
||||
// (novox/hq ADR 0468 clause 5): a printer that cannot be asked prints nothing.
|
||||
listed, err := FormatsOf(ctx, set.Printer)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the printer's document formats could not be read, so nothing is sent: %w", err)
|
||||
}
|
||||
if !slices.Contains(listed, format) {
|
||||
return nil, fmt.Errorf("the printer does not take %s: its document-format-supported lists %s", format, strings.Join(listed, ", "))
|
||||
}
|
||||
res, err := PrintJob(ctx, set.Printer, PrintOptions{Format: format, Copies: ask.Copies, Sides: ask.Sides, Media: ask.Media, Name: "brother print"}, doc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]any{"job-id": res.JobID, "job-state": res.State, "job-state-code": res.StateCode,
|
||||
"job-state-reasons": res.Reasons, "format": format, "copies": ask.Copies, "sides": ask.Sides, "media": ask.Media, "object": key}
|
||||
if convertedFrom != "" {
|
||||
out["converted-from"] = convertedFrom
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// fetch gets a web link under the rules of fetch.go.
|
||||
func (w *deviceWorker) fetch(ctx context.Context, link string) (*Fetched, error) {
|
||||
if w.Fetcher == nil {
|
||||
return nil, errors.New("this process has no way to fetch a web link")
|
||||
}
|
||||
f, err := w.Fetcher()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.Fetch(ctx, link)
|
||||
}
|
||||
|
||||
// read logs in, reads one page of the admin site and logs out. It is read-only: the login (and the logout) are
|
||||
// the only things it sends, and a password field's value is never in the answer. The printer is not contacted
|
||||
// before the settings and the password are known to be given (1453/AC4).
|
||||
func (w *deviceWorker) read(ctx context.Context, page string, links, hidden bool) (map[string]any, error) {
|
||||
if page == "" {
|
||||
page = DefaultPage
|
||||
}
|
||||
if !validPage(page) {
|
||||
return nil, fmt.Errorf("page %q is not a page of the admin site: it starts with / and ends in .html", page)
|
||||
}
|
||||
set, err := w.Settings()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := set.MissingPrinter(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pw, err := w.Password()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := NewAdmin(set.Printer, set.PrinterCertificate, pw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := a.Login(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Log out with a context of its own: the ask's may be what ended this.
|
||||
lctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
body, at, err := a.Get(ctx, page)
|
||||
if err != nil {
|
||||
if home, hat, herr := a.Get(lctx, DefaultPage); herr == nil {
|
||||
a.Logout(lctx, home, hat)
|
||||
}
|
||||
var nf *notFoundError
|
||||
if errors.As(err, &nf) {
|
||||
return nil, fmt.Errorf("the printer has no page %s", page)
|
||||
}
|
||||
return nil, fmt.Errorf("page %s cannot be read: %w", page, err)
|
||||
}
|
||||
logout := a.Logout(lctx, body, at)
|
||||
p := ParsePage(body)
|
||||
out := map[string]any{
|
||||
"page": page,
|
||||
"title": p.Title,
|
||||
"forms": nonNilForms(withoutHidden(p.Forms, hidden)),
|
||||
"logout": logout,
|
||||
}
|
||||
if links {
|
||||
out["links"] = nonNil(p.Links)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// withoutHidden leaves the hidden fields (the CSRF token and the like, noise to a reader) out of the forms
|
||||
// unless they are asked for.
|
||||
func withoutHidden(forms []Form, keep bool) []Form {
|
||||
if keep {
|
||||
return forms
|
||||
}
|
||||
out := make([]Form, 0, len(forms))
|
||||
for _, f := range forms {
|
||||
var fields []Field
|
||||
for _, fl := range f.Fields {
|
||||
if fl.Type != "hidden" {
|
||||
fields = append(fields, fl)
|
||||
}
|
||||
}
|
||||
if len(fields) == 0 && len(f.Fields) > 0 {
|
||||
continue // a form of nothing but hidden fields (the logout form) has nothing to read
|
||||
}
|
||||
f.Fields = fields
|
||||
out = append(out, f)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func nonNil(l []Link) []Link {
|
||||
if l == nil {
|
||||
return []Link{}
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
func nonNilForms(f []Form) []Form {
|
||||
if f == nil {
|
||||
return []Form{}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// device is `brother device`: it reads its own bus account, connects, and consumes device-asks until stopped.
|
||||
// It refuses to start naming what is missing when the account cannot be read, and never logs a password.
|
||||
func device() int {
|
||||
settings, passwordFile, brokerFile := os.Getenv("BROTHER_SETTINGS"), os.Getenv("BROTHER_ADMIN_PASSWORD"), os.Getenv("BROTHER_BROKER")
|
||||
store, fetchFile := os.Getenv("BROTHER_STORE"), os.Getenv("BROTHER_FETCH")
|
||||
if brokerFile == "" {
|
||||
fmt.Fprintln(os.Stderr, "[brother] device was started without BROTHER_BROKER, its bus account's file")
|
||||
return 2
|
||||
}
|
||||
account, err := process.ReadAccount(brokerFile)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[brother] device: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
w := &deviceWorker{
|
||||
Settings: func() (Settings, error) { return LoadSettings(settings) },
|
||||
Password: func() (string, error) { return secret.Read(passwordFile) },
|
||||
Objects: func() (Objects, error) {
|
||||
facts, err := ReadStoreFacts(store)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NewS3(facts), nil
|
||||
},
|
||||
Fetcher: func() (*Fetcher, error) {
|
||||
r, err := ReadFetchFacts(fetchFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NewFetcher(r), nil
|
||||
},
|
||||
Now: time.Now,
|
||||
Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, f+"\n", a...) },
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "[brother] device is consuming %s as %s\n", AsksStream, account.User)
|
||||
err = runDevice(ctx, account, w)
|
||||
if err != nil && ctx.Err() == nil {
|
||||
fmt.Fprintf(os.Stderr, "[brother] device stopped: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// runDevice connects as the device process's own account and consumes device-asks until ctx ends, answering each
|
||||
// ask in the state device-answers on that connection and then settling the message as Handle says. Apart from
|
||||
// device so a test can drive it over a bus of its own.
|
||||
func runDevice(ctx context.Context, account process.Account, w *deviceWorker) error {
|
||||
conn, err := process.Connect(account)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bus cannot be reached: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
w.Answers = conn.State(AnswersState)
|
||||
w.Convert = func(body map[string]any) (json.RawMessage, error) {
|
||||
return conn.Call("seat:document-converter.convert", body)
|
||||
}
|
||||
if w.Node == "" {
|
||||
w.Node = account.Node
|
||||
}
|
||||
w.Emit = func(body map[string]any, id string) error {
|
||||
return conn.Emit(EventAdded, body, stdio.EmitOptions{MessageID: id})
|
||||
}
|
||||
return conn.Consume(ctx, AsksStream, func(msg *process.Message) {
|
||||
var settleErr error
|
||||
act, delay := w.Handle(msg.Body, func() { settleErr = msg.Ack() })
|
||||
if settleErr != nil {
|
||||
w.logf("[brother] device: acknowledging %s early: %v", msg.Subject, settleErr)
|
||||
}
|
||||
var aerr error
|
||||
switch act {
|
||||
case actionAck:
|
||||
aerr = msg.Ack()
|
||||
case actionRetry:
|
||||
aerr = msg.NakWithDelay(delay)
|
||||
case actionTerm:
|
||||
aerr = msg.Term()
|
||||
}
|
||||
if aerr != nil {
|
||||
w.logf("[brother] device: settling %s: %v", msg.Subject, aerr)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// An open document (issue 1538) is kept in the bucket, not in memory, so that a restart of the device process
|
||||
// loses none of its pages: each page's JPEG under pending/<id>/<n>.jpg and a manifest, pending/<id>/manifest.json,
|
||||
// that says how many pages there are and, once it is closed, the key of its PDF. The pages are not deleted when it
|
||||
// is closed (the bucket has no delete); they are the bucket's to age out.
|
||||
const mostDocumentPages = 100
|
||||
|
||||
type docPage struct {
|
||||
Key string `json:"key"`
|
||||
DPI int `json:"dpi"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
Cropped bool `json:"cropped"`
|
||||
}
|
||||
|
||||
type docManifest struct {
|
||||
Started time.Time `json:"started"`
|
||||
Updated time.Time `json:"updated"` // the last page's time
|
||||
Pages []docPage `json:"pages"`
|
||||
Finished string `json:"finished,omitempty"` // the closed document's object key
|
||||
}
|
||||
|
||||
func manifestKey(id string) string { return "pending/" + id + "/manifest.json" }
|
||||
|
||||
func loadManifest(ctx context.Context, o Objects, id string) (*docManifest, error) {
|
||||
has, err := o.Has(ctx, manifestKey(id))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !has {
|
||||
return nil, fmt.Errorf("no document %s is open: its id is wrong, or it was closed", id)
|
||||
}
|
||||
raw, err := o.Get(ctx, manifestKey(id))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m docManifest
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("document %s: its manifest cannot be read: %w", id, err)
|
||||
}
|
||||
if m.Finished != "" {
|
||||
return nil, fmt.Errorf("document %s is closed already, as %s", id, m.Finished)
|
||||
}
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
func saveManifest(ctx context.Context, o Objects, id string, m *docManifest) error {
|
||||
raw, _ := json.Marshal(m)
|
||||
return o.Put(ctx, manifestKey(id), "application/json", raw)
|
||||
}
|
||||
|
||||
// scanDocument adds a glass page to an open document (document start opens one, with the ask's id as its id), or
|
||||
// with finish closes the document into one multi-page PDF, kept and announced once like any scan.
|
||||
func (w *deviceWorker) scanDocument(ctx context.Context, ask Ask, svc *Service, objects Objects) (map[string]any, error) {
|
||||
if ask.Source == "feeder" || ask.Duplex {
|
||||
return nil, fmt.Errorf("a document of several pages is scanned from the glass, page by page; the feeder takes the whole stack in one scan")
|
||||
}
|
||||
id := ask.Document
|
||||
if id == "start" {
|
||||
if ask.Finish {
|
||||
return nil, fmt.Errorf("finish closes an open document: name its id, not start")
|
||||
}
|
||||
id = ask.ID
|
||||
}
|
||||
if !uploadPattern.MatchString(id) {
|
||||
return nil, fmt.Errorf("document %q is not an id: letters, digits, - and _, at most 64", id)
|
||||
}
|
||||
w.docs.Lock()
|
||||
defer w.docs.Unlock()
|
||||
var m *docManifest
|
||||
if ask.Document == "start" {
|
||||
m = &docManifest{Started: w.Now().UTC(), Updated: w.Now().UTC()}
|
||||
} else {
|
||||
var err error
|
||||
if m, err = loadManifest(ctx, objects, id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if ask.Finish {
|
||||
return w.finishDocument(ctx, id, m, svc, objects)
|
||||
}
|
||||
if len(m.Pages) >= mostDocumentPages {
|
||||
return nil, fmt.Errorf("document %s has %d pages, the most one takes: close it", id, len(m.Pages))
|
||||
}
|
||||
page, err := svc.GlassPage(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key := fmt.Sprintf("pending/%s/%03d.jpg", id, len(m.Pages)+1)
|
||||
if err := objects.Put(ctx, key, "image/jpeg", page.JPEG); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.Updated = w.Now().UTC()
|
||||
m.Pages = append(m.Pages, docPage{Key: key, DPI: page.DPI, Width: page.Width, Height: page.Height, Cropped: page.Cropped})
|
||||
if err := saveManifest(ctx, objects, id, m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"document": id, "pages": len(m.Pages), "state": "open",
|
||||
"next": "scanner.scan {document: \"" + id + "\"} adds a page; scanner.scan {document: \"" + id + "\", finish: true} closes it into one PDF"}, nil
|
||||
}
|
||||
|
||||
// finishDocument builds the open document's pages into one PDF and keeps and announces it.
|
||||
func (w *deviceWorker) finishDocument(ctx context.Context, id string, m *docManifest, svc *Service, objects Objects) (map[string]any, error) {
|
||||
if len(m.Pages) == 0 {
|
||||
return nil, fmt.Errorf("document %s has no page", id)
|
||||
}
|
||||
pages := make([]PDFPage, len(m.Pages))
|
||||
for i, p := range m.Pages {
|
||||
jpg, err := objects.Get(ctx, p.Key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("document %s: page %d is not kept: %w", id, i+1, err)
|
||||
}
|
||||
pages[i] = PDFPage{JPEG: jpg, DPI: p.DPI}
|
||||
}
|
||||
first := m.Pages[0]
|
||||
c, err := captureOf(pages, Cropped{Width: first.Width, Height: first.Height, Cropped: first.Cropped}, first.DPI, w.Now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
set, err := w.Settings()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer, err := w.keepOne(ctx, objects, set, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.Finished = answer["key"].(string)
|
||||
if err := saveManifest(ctx, objects, id, m); err != nil {
|
||||
return nil, fmt.Errorf("document %s is kept as %s, but could not be marked closed: %w", id, m.Finished, err)
|
||||
}
|
||||
// The pages are not needed any more: delete them. A bucket that refuses leaves them, and the answer says so.
|
||||
for _, p := range m.Pages {
|
||||
if err := objects.Delete(ctx, p.Key); err != nil {
|
||||
answer["pending-kept"] = fmt.Sprintf("the pages of document %s could not be deleted from the bucket and stay under pending/%s/: %v", id, id, err)
|
||||
break
|
||||
}
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// openDocuments lists the documents of several pages that are open in the bucket, with the time of each one's last
|
||||
// page, so that the tool can arm their expiry again after it was restarted.
|
||||
func (w *deviceWorker) openDocuments(ctx context.Context) (map[string]any, error) {
|
||||
objects, err := w.Objects()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys, err := objects.List(ctx, "pending/")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
open := []map[string]any{}
|
||||
for _, k := range keys {
|
||||
rest, ok := strings.CutPrefix(k, "pending/")
|
||||
id, name, _ := strings.Cut(rest, "/")
|
||||
if !ok || name != "manifest.json" || !uploadPattern.MatchString(id) {
|
||||
continue
|
||||
}
|
||||
m, err := loadManifest(ctx, objects, id)
|
||||
if err != nil {
|
||||
continue // closed, or unreadable: not open
|
||||
}
|
||||
at := m.Updated
|
||||
if at.IsZero() {
|
||||
at = m.Started
|
||||
}
|
||||
open = append(open, map[string]any{"id": id, "pages": len(m.Pages), "updated": at.UTC().Format(time.RFC3339)})
|
||||
}
|
||||
return map[string]any{"open": open}, nil
|
||||
}
|
||||
@@ -1,574 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// pdfOf is a document that is a PDF to the sniff and size bytes long, its bytes varying so that a part out of
|
||||
// place changes the join.
|
||||
func pdfOf(size int) []byte {
|
||||
b := make([]byte, size)
|
||||
copy(b, "%PDF-1.4\n")
|
||||
for i := 9; i < size; i++ {
|
||||
b[i] = byte(i*7 + i/251)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// 1453/AC10: a document is cut into parts of at most PartSize, under the limit, and the parts joined in
|
||||
// order are the document, with the SHA-256 checked; a wrong SHA-256 or a part missing is refused; a part out of
|
||||
// range is refused naming the range.
|
||||
func TestADocumentIsCutIntoPartsAndJoinedWithItsSha256Checked(t *testing.T) {
|
||||
doc := pdfOf(2*AnswerPartSize + 1234)
|
||||
n := answerPartCount(len(doc))
|
||||
if n != 3 {
|
||||
t.Fatalf("%d parts, want 3", n)
|
||||
}
|
||||
var parts [][]byte
|
||||
for i := 1; i <= n; i++ {
|
||||
p, total, err := cutPart(doc, i)
|
||||
if err != nil || total != n {
|
||||
t.Fatalf("part %d: %v (of %d)", i, err, total)
|
||||
}
|
||||
if len(p) > AnswerPartSize {
|
||||
t.Errorf("part %d is %d bytes, over %d", i, len(p), AnswerPartSize)
|
||||
}
|
||||
parts = append(parts, p)
|
||||
}
|
||||
got, err := joinParts(parts, sha256Hex(doc))
|
||||
if err != nil || !bytes.Equal(got, doc) {
|
||||
t.Fatalf("the join is not the document: %v", err)
|
||||
}
|
||||
if _, err := joinParts(parts, sha256Hex([]byte("other"))); err == nil || !strings.Contains(err.Error(), sha256Hex(doc)) {
|
||||
t.Errorf("a wrong SHA-256 was accepted or the error does not name the joined one: %v", err)
|
||||
}
|
||||
if _, err := joinParts(parts[:2], sha256Hex(doc)); err == nil {
|
||||
t.Errorf("a join with a part missing passed its SHA-256")
|
||||
}
|
||||
for _, bad := range []int{0, -1, n + 1} {
|
||||
if _, _, err := cutPart(doc, bad); err == nil || !strings.Contains(err.Error(), "out of range") {
|
||||
t.Errorf("part %d: error = %v", bad, err)
|
||||
}
|
||||
}
|
||||
if partCount(0) != 1 {
|
||||
t.Errorf("an empty document has one part")
|
||||
}
|
||||
}
|
||||
|
||||
// 1453/AC10: scanner.document gives each part of a kept scan with the part count, the size and the SHA-256, each
|
||||
// answer within the bus's 1 MiB; the parts joined are the scan. A part out of range, or a key that is no scan's, is
|
||||
// refused, the first before anything is queued.
|
||||
func TestScannerDocumentGivesEachPartOfAKeptScan(t *testing.T) {
|
||||
r := newRig(t, `{}`, "pw")
|
||||
doc := pdfOf(AnswerPartSize + 5000)
|
||||
key := sha256Hex(doc) + ".pdf"
|
||||
r.objects.Put(t.Context(), key, "application/pdf", doc)
|
||||
var joined []byte
|
||||
for i := 1; i <= 2; i++ {
|
||||
got, err := r.call(t, "scanner.document", map[string]any{"object": key, "part": float64(i)})
|
||||
if err != nil {
|
||||
t.Fatalf("part %d: %v", i, err)
|
||||
}
|
||||
m := got.(map[string]any)
|
||||
if m["parts"] != float64(2) || m["size"] != float64(len(doc)) || m["sha256"] != sha256Hex(doc) || m["part"] != float64(i) {
|
||||
t.Errorf("part %d answer = %v", i, m)
|
||||
}
|
||||
raw, err := base64.StdEncoding.DecodeString(m["data"].(string))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
joined = append(joined, raw...)
|
||||
}
|
||||
if !bytes.Equal(joined, doc) {
|
||||
t.Errorf("the parts fetched are not the scan")
|
||||
}
|
||||
before := r.bus.asks()
|
||||
if _, err := r.call(t, "scanner.document", map[string]any{"object": key, "part": float64(0)}); err == nil {
|
||||
t.Errorf("part 0 was asked for")
|
||||
}
|
||||
if r.bus.asks() != before {
|
||||
t.Errorf("an ask was queued for a part the tool could refuse")
|
||||
}
|
||||
if _, err := r.call(t, "scanner.document", map[string]any{"object": key, "part": float64(3)}); err == nil || !strings.Contains(err.Error(), "out of range") {
|
||||
t.Errorf("part 3: error = %v", err)
|
||||
}
|
||||
for _, k := range []string{"../etc/passwd", "uploads/x.pdf", strings.ToUpper(key)} {
|
||||
if _, err := r.call(t, "scanner.document", map[string]any{"object": k, "part": float64(1)}); err == nil || !strings.Contains(err.Error(), "not a scan's key") {
|
||||
t.Errorf("object %q: error = %v", k, err)
|
||||
}
|
||||
}
|
||||
if _, err := r.call(t, "scanner.document", map[string]any{"object": sha256Hex([]byte("none")) + ".pdf", "part": float64(1)}); err == nil {
|
||||
t.Errorf("a scan that is not kept was given")
|
||||
}
|
||||
}
|
||||
|
||||
// 1453/AC10, 1601: a scan the size of several answer parts is read whole from the store by scanner.scan, checked and filed.
|
||||
func TestScanNowFetchesAScanOfSeveralParts(t *testing.T) {
|
||||
f := newFakeScanner(t, fixture(t, "scan.jpg"))
|
||||
r, _ := scanRig(t, f, Settings{Crop: false})
|
||||
// Everything a scan is made of but its size is the real one; the stored object is what the parts carry, so a
|
||||
// bigger object stands for a bigger scan: replace what the device keeps with a three-part document.
|
||||
big := pdfOf(2*AnswerPartSize + 99)
|
||||
bigKey := sha256Hex(big) + ".pdf"
|
||||
r.objects.Put(t.Context(), bigKey, "application/pdf", big)
|
||||
got := make([][]byte, 0, 3)
|
||||
for i := 1; i <= 3; i++ {
|
||||
p, err := r.worker.document(t.Context(), Ask{Op: OpDocument, Object: bigKey, Part: i})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := base64.StdEncoding.DecodeString(p["data"].(string))
|
||||
got = append(got, raw)
|
||||
}
|
||||
if joined, err := joinParts(got, sha256Hex(big)); err != nil || !bytes.Equal(joined, big) {
|
||||
t.Fatalf("three parts do not join: %v", err)
|
||||
}
|
||||
body, err := r.scan(t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if body["path"] == nil {
|
||||
t.Errorf("no path: %v", body)
|
||||
}
|
||||
}
|
||||
|
||||
func upload(t *testing.T, r *printerRig, id string, doc []byte, sha string, part, parts int, piece []byte) (any, error) {
|
||||
t.Helper()
|
||||
return r.call(t, "printer.upload", map[string]any{"upload": id, "part": float64(part), "parts": float64(parts), "sha256": sha,
|
||||
"data": base64.StdEncoding.EncodeToString(piece)})
|
||||
}
|
||||
|
||||
// 1453/AC11: an upload in parts, in any order, is stored as uploads/<id>.pdf once all have come and the joined
|
||||
// SHA-256 matches; until then it says how many came and stores nothing.
|
||||
func TestAnUploadIsStoredOnceAllItsPartsCameAndTheSha256Matches(t *testing.T) {
|
||||
r := newRig(t, `{}`, "pw")
|
||||
doc := pdfOf(PartSize + 700)
|
||||
sum := sha256Hex(doc)
|
||||
p1, _, _ := cutPartOf(doc, 1, PartSize)
|
||||
p2, _, _ := cutPartOf(doc, 2, PartSize)
|
||||
got, err := upload(t, r, "up-1", doc, sum, 2, 2, p2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m := got.(map[string]any); m["complete"] != false || m["received"] != float64(1) {
|
||||
t.Errorf("after one part: %v", m)
|
||||
}
|
||||
if len(r.objects.keys()) != 0 {
|
||||
t.Errorf("a document was stored before its parts all came: %v", r.objects.keys())
|
||||
}
|
||||
got, err = upload(t, r, "up-1", doc, sum, 1, 2, p1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := got.(map[string]any)
|
||||
if m["complete"] != true || m["object"] != "uploads/up-1.pdf" || m["sha256"] != sum || m["size"] != float64(len(doc)) {
|
||||
t.Errorf("after the last part: %v", m)
|
||||
}
|
||||
if kept, err := r.objects.Get(t.Context(), "uploads/up-1.pdf"); err != nil || !bytes.Equal(kept, doc) {
|
||||
t.Errorf("the stored document is not the upload (%v)", err)
|
||||
}
|
||||
// A JPEG is stored with its own extension.
|
||||
jpg := append([]byte{0xFF, 0xD8, 0xFF, 0xE0}, bytes.Repeat([]byte{7}, 100)...)
|
||||
if _, err := upload(t, r, "pic", jpg, sha256Hex(jpg), 1, 1, jpg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.objects.Get(t.Context(), "uploads/pic.jpg"); err != nil {
|
||||
t.Errorf("the JPEG is not stored as uploads/pic.jpg: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1453/AC11: a wrong SHA-256 on the joined document stores nothing and is said; a document that is neither PDF nor
|
||||
// JPEG stores nothing; a part out of range, a part larger than the limit, a part that is not base64 and parts that
|
||||
// disagree are refused; the upload can then be sent again whole.
|
||||
func TestAnUploadWithAWrongSha256OrAnOutOfRangePartStoresNothing(t *testing.T) {
|
||||
r := newRig(t, `{}`, "pw")
|
||||
doc := pdfOf(300)
|
||||
wrong := sha256Hex([]byte("not it"))
|
||||
if _, err := upload(t, r, "bad", doc, wrong, 1, 1, doc); err == nil || !strings.Contains(err.Error(), "refused") || !strings.Contains(err.Error(), sha256Hex(doc)) {
|
||||
t.Errorf("a wrong SHA-256: error = %v", err)
|
||||
}
|
||||
if len(r.objects.keys()) != 0 || r.objects.puts != 0 {
|
||||
t.Errorf("a document with a wrong SHA-256 was stored: %v", r.objects.keys())
|
||||
}
|
||||
text := []byte("plain text, no PDF")
|
||||
if _, err := upload(t, r, "txt", text, sha256Hex(text), 1, 1, text); err == nil || !strings.Contains(err.Error(), "neither a PDF nor a JPEG") {
|
||||
t.Errorf("a text upload: error = %v", err)
|
||||
}
|
||||
for _, c := range []struct{ part, parts int }{{0, 1}, {2, 1}, {1, 0}, {1, MostParts + 1}} {
|
||||
if _, err := upload(t, r, "range", doc, sha256Hex(doc), c.part, c.parts, doc); err == nil {
|
||||
t.Errorf("part %d of %d was taken", c.part, c.parts)
|
||||
}
|
||||
}
|
||||
if _, err := upload(t, r, "range", doc, sha256Hex(doc), 2, 1, doc); err == nil || !strings.Contains(err.Error(), "out of range") {
|
||||
t.Errorf("part 2 of 1: error = %v", err)
|
||||
}
|
||||
if _, err := r.call(t, "printer.upload", map[string]any{"upload": "x", "part": float64(1), "parts": float64(1), "sha256": sha256Hex(doc), "data": "!!"}); err == nil {
|
||||
t.Errorf("a part that is not base64 was taken")
|
||||
}
|
||||
if _, err := upload(t, r, "big", doc, sha256Hex(doc), 1, 1, make([]byte, MaxPart+1)); err == nil || !strings.Contains(err.Error(), "at most") {
|
||||
t.Errorf("a part over the limit: error = %v", err)
|
||||
}
|
||||
if r.bus.asks() != 2 {
|
||||
// Only the two uploads that were complete in form reached the device process; every refusal above them
|
||||
// was the tool's.
|
||||
t.Errorf("%d asks queued, want the 2 whose form was right", r.bus.asks())
|
||||
}
|
||||
// Parts that disagree about the whole.
|
||||
p := pdfOf(PartSize + 10)
|
||||
p1, _, _ := cutPartOf(p, 1, PartSize)
|
||||
if _, err := upload(t, r, "mix", p, sha256Hex(p), 1, 2, p1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := upload(t, r, "mix", p, sha256Hex(p), 2, 3, p1); err == nil || !strings.Contains(err.Error(), "all say the same") {
|
||||
t.Errorf("parts that disagree: error = %v", err)
|
||||
}
|
||||
if len(r.objects.keys()) != 0 {
|
||||
t.Errorf("objects stored: %v", r.objects.keys())
|
||||
}
|
||||
}
|
||||
|
||||
// 1453/AC11: an incomplete upload's parts are dropped after fifteen minutes.
|
||||
func TestAnIncompleteUploadIsDroppedAfterAWhile(t *testing.T) {
|
||||
r := newRig(t, `{}`, "pw")
|
||||
doc := pdfOf(PartSize + 10)
|
||||
p1, _, _ := cutPartOf(doc, 1, PartSize)
|
||||
p2, _, _ := cutPartOf(doc, 2, PartSize)
|
||||
now := fixedNow
|
||||
r.worker.Now = func() time.Time { return now }
|
||||
if _, err := upload(t, r, "late", doc, sha256Hex(doc), 1, 2, p1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now = now.Add(pendingFor + time.Minute)
|
||||
// Asked of the worker directly: the tool's own clock is the fixed one, and an ask this old would be stale.
|
||||
got, err := r.worker.upload(t.Context(), Ask{Op: OpUpload, Upload: "late", Part: 2, Parts: 2, SHA256: sha256Hex(doc),
|
||||
Data: base64.StdEncoding.EncodeToString(p2)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got["complete"] != false {
|
||||
t.Errorf("the dropped part was still there: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// ippFake is a printer's IPP port: it records the request and answers a Print-Job answer with a job id and state.
|
||||
type ippFake struct {
|
||||
url string
|
||||
body []byte // the Print-Job request
|
||||
status int
|
||||
// formats is document-format-supported, as the printer lists it; asked counts the Get-Printer-Attributes requests.
|
||||
formats []string
|
||||
asked int
|
||||
// delay is how long a Print-Job takes to be answered.
|
||||
delay time.Duration
|
||||
}
|
||||
|
||||
func newIPPFake(t *testing.T, jobID, jobState int) *ippFake {
|
||||
t.Helper()
|
||||
f := &ippFake{formats: []string{"application/octet-stream", "application/pdf", "image/jpeg"}}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
if binary.BigEndian.Uint16(body[2:4]) == uint16(ippGetPrinterAttributes) {
|
||||
// Get-Printer-Attributes (RFC 8011 §4.2.5): the formats are one attribute of several values, a value
|
||||
// after the first carrying an empty name (§4.1.4), each of the mimeMediaType syntax.
|
||||
f.asked++
|
||||
ans := []byte{0x02, 0x00, 0x00, 0x00, 0, 0, 0, 1, ippOperation}
|
||||
ans = appendAttr(ans, ippCharset, "attributes-charset", "utf-8")
|
||||
ans = append(ans, 0x04) // printer-attributes-tag, RFC 8010 §3.5.1
|
||||
for i, format := range f.formats {
|
||||
name := "document-format-supported"
|
||||
if i > 0 {
|
||||
name = ""
|
||||
}
|
||||
ans = appendAttr(ans, ippMimeType, name, format)
|
||||
}
|
||||
ans = append(ans, ippEnd)
|
||||
w.Header().Set("Content-Type", "application/ipp")
|
||||
w.Write(ans)
|
||||
return
|
||||
}
|
||||
f.body = body
|
||||
time.Sleep(f.delay)
|
||||
ans := []byte{0x02, 0x00, byte(f.status >> 8), byte(f.status), 0, 0, 0, 1, ippOperation}
|
||||
ans = appendAttr(ans, ippCharset, "attributes-charset", "utf-8")
|
||||
ans = append(ans, ippJob)
|
||||
ans = appendAttr(ans, ippInteger, "job-id", string(binary.BigEndian.AppendUint32(nil, uint32(jobID))))
|
||||
ans = appendAttr(ans, ippEnum, "job-state", string(binary.BigEndian.AppendUint32(nil, uint32(jobState))))
|
||||
ans = appendAttr(ans, ippKeyword, "job-state-reasons", "none")
|
||||
ans = append(ans, ippEnd)
|
||||
w.Header().Set("Content-Type", "application/ipp")
|
||||
w.Write(ans)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
f.url = srv.URL
|
||||
old := ippURL
|
||||
ippURL = func(string) string { return srv.URL }
|
||||
t.Cleanup(func() { ippURL = old })
|
||||
return f
|
||||
}
|
||||
|
||||
// request splits what the fake was sent into its attributes and its document.
|
||||
func (f *ippFake) request(t *testing.T) (IPPAnswer, []byte) {
|
||||
t.Helper()
|
||||
a, off, err := decodeIPP(f.body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return a, f.body[off:]
|
||||
}
|
||||
|
||||
// 1453/AC12: printer.print sends a kept scan to the printer as an IPP Print-Job (operation 0x0002): the document
|
||||
// bytes after the attributes, document-format by sniffing, copies, sides and media as asked, to the printer's own
|
||||
// IPP address; the answer is the job's id and state.
|
||||
func TestPrinterPrintSendsAPrintJobAndAnswersTheJob(t *testing.T) {
|
||||
fake := newIPPFake(t, 77, 5)
|
||||
r := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
|
||||
doc := pdfOf(5000)
|
||||
key := sha256Hex(doc) + ".pdf"
|
||||
r.objects.Put(t.Context(), key, "application/pdf", doc)
|
||||
got, err := r.call(t, "printer.print", map[string]any{"object": key, "copies": float64(2), "sides": "two-sided-long-edge", "media": "na_letter_8.5x11in"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := got.(map[string]any)
|
||||
if m["job-id"] != float64(77) || m["job-state"] != "processing" || m["format"] != "application/pdf" {
|
||||
t.Errorf("answer = %v", m)
|
||||
}
|
||||
if binary.BigEndian.Uint16(fake.body[2:4]) != 0x0002 {
|
||||
t.Errorf("operation = 0x%04x, want Print-Job 0x0002", binary.BigEndian.Uint16(fake.body[2:4]))
|
||||
}
|
||||
attrs, sent := fake.request(t)
|
||||
if !bytes.Equal(sent, doc) {
|
||||
t.Errorf("the document sent is not the kept one (%d bytes, want %d)", len(sent), len(doc))
|
||||
}
|
||||
text := func(n string) string { return strings.Join(attrs.texts(n), ",") }
|
||||
if text("document-format") != "application/pdf" || text("sides") != "two-sided-long-edge" || text("media") != "na_letter_8.5x11in" ||
|
||||
text("printer-uri") != "ipp://192.0.2.10/ipp/port1" {
|
||||
t.Errorf("attributes = %v", attrs.Attributes)
|
||||
}
|
||||
if c := attrs.Attributes["copies"]; len(c) != 1 || c[0].Integer != 2 {
|
||||
t.Errorf("copies = %v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// 1453/AC12: the defaults are one copy, one-sided, A4; a JPEG is sent as image/jpeg; a document stored under its
|
||||
// SHA-256 (as node-files.upload keeps one) is printed by its key.
|
||||
func TestPrinterPrintDefaultsAndFormatsAndStoredDocuments(t *testing.T) {
|
||||
fake := newIPPFake(t, 8, 3)
|
||||
r := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
|
||||
jpg := append([]byte{0xFF, 0xD8, 0xFF, 0xE0}, bytes.Repeat([]byte{9}, 200)...)
|
||||
r.objects.Put(t.Context(), sha256Hex(jpg), "image/jpeg", jpg)
|
||||
got, err := r.call(t, "printer.print", map[string]any{"object": sha256Hex(jpg)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.(map[string]any)["job-state"] != "pending" {
|
||||
t.Errorf("answer = %v", got)
|
||||
}
|
||||
attrs, sent := fake.request(t)
|
||||
if !bytes.Equal(sent, jpg) || strings.Join(attrs.texts("document-format"), "") != "image/jpeg" ||
|
||||
strings.Join(attrs.texts("sides"), "") != "one-sided" || strings.Join(attrs.texts("media"), "") != DefaultMedia || attrs.Attributes["copies"][0].Integer != 1 {
|
||||
t.Errorf("attributes = %v, %d bytes sent", attrs.Attributes, len(sent))
|
||||
}
|
||||
if _, err := r.call(t, "printer.print", map[string]any{"object": sha256Hex([]byte("never stored"))}); err == nil {
|
||||
t.Errorf("an object never stored was printed")
|
||||
}
|
||||
}
|
||||
|
||||
// 1453/AC12: a print that is not well formed is refused before anything is queued; the printer's refusal is an
|
||||
// error naming its IPP status; a printer not set is said by name.
|
||||
func TestPrinterPrintRefusesWhatIsNotWellFormed(t *testing.T) {
|
||||
fake := newIPPFake(t, 1, 3)
|
||||
r := newRig(t, `{"printer":"192.0.2.10"}`, "pw")
|
||||
key := sha256Hex([]byte("x")) + ".pdf"
|
||||
for name, args := range map[string]map[string]any{
|
||||
"neither": {},
|
||||
"both": {"object": key, "upload": "u"},
|
||||
"copies 0": {"object": key, "copies": float64(-1)},
|
||||
"copies 100": {"object": key, "copies": float64(100)},
|
||||
"sides": {"object": key, "sides": "sideways"},
|
||||
"media": {"object": key, "media": "A4 paper"},
|
||||
"not a key": {"object": "../x.pdf"},
|
||||
"copies float": {"object": key, "copies": 1.5},
|
||||
} {
|
||||
if _, err := r.call(t, "printer.print", args); err == nil {
|
||||
t.Errorf("%s: accepted", name)
|
||||
}
|
||||
}
|
||||
if r.bus.asks() != 0 || fake.body != nil {
|
||||
t.Errorf("%d asks queued, the printer was sent %d bytes", r.bus.asks(), len(fake.body))
|
||||
}
|
||||
doc := pdfOf(100)
|
||||
dk := sha256Hex(doc) + ".pdf"
|
||||
r.objects.Put(t.Context(), dk, "application/pdf", doc)
|
||||
fake.status = 0x0400
|
||||
if _, err := r.call(t, "printer.print", map[string]any{"object": dk}); err == nil || !strings.Contains(err.Error(), "0x0400") {
|
||||
t.Errorf("a refused job: error = %v", err)
|
||||
}
|
||||
// A kept object that is no document the printer takes, nor one the converter takes (1600), is not sent.
|
||||
fake.status, fake.body = 0, nil
|
||||
r.objects.Put(t.Context(), key, "application/octet-stream", []byte{0x00, 0x01, 0xFE, 0xFF, 0x02})
|
||||
if _, err := r.call(t, "printer.print", map[string]any{"object": key}); err == nil || !strings.Contains(err.Error(), "neither a PDF nor a JPEG") || fake.body != nil {
|
||||
t.Errorf("a text object: error = %v, sent %d bytes", err, len(fake.body))
|
||||
}
|
||||
un := newRig(t, `{}`, "pw")
|
||||
if _, err := un.call(t, "printer.print", map[string]any{"object": key}); err == nil || !strings.Contains(err.Error(), "printer is not set") || un.bus.asks() != 0 {
|
||||
t.Errorf("no printer: error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The Print-Job request is the one RFC 8011 gives: header, operation attributes in order, the job's own, the
|
||||
// end-of-attributes tag, then the document.
|
||||
func TestThePrintJobRequestIsWellFormed(t *testing.T) {
|
||||
doc := []byte("%PDF-1.4 hello")
|
||||
b := EncodePrintJob("ipp://192.0.2.10/ipp/port1", PrintOptions{Format: "application/pdf", Copies: 3, Sides: "one-sided", Media: DefaultMedia, Name: "n"}, doc)
|
||||
if !bytes.Equal(b[:8], []byte{2, 0, 0, 2, 0, 0, 0, 1}) || b[8] != ippOperation {
|
||||
t.Errorf("header = %x", b[:9])
|
||||
}
|
||||
a, off, err := decodeIPP(b)
|
||||
if err != nil || !bytes.Equal(b[off:], doc) {
|
||||
t.Fatalf("decode: %v, document %q", err, b[off:])
|
||||
}
|
||||
if first := string(b[9:]); !strings.HasPrefix(first[3:], "attributes-charset") {
|
||||
t.Errorf("the first attribute is not attributes-charset")
|
||||
}
|
||||
if len(a.Attributes["copies"]) != 1 || a.Attributes["copies"][0].Integer != 3 {
|
||||
t.Errorf("copies = %v", a.Attributes["copies"])
|
||||
}
|
||||
}
|
||||
|
||||
// 1453: a scan and a print are acknowledged before the work (a slow one is not delivered again and done twice);
|
||||
// the other operations are acknowledged after it.
|
||||
func TestAScanAndAPrintAreAcknowledgedBeforeTheWork(t *testing.T) {
|
||||
f := newFakeScanner(t, fixture(t, "scan.jpg"))
|
||||
r, _ := scanRig(t, f, Settings{Crop: true})
|
||||
body, _ := json.Marshal(Ask{ID: "s1", Op: OpScan, At: fixedNow})
|
||||
calls := 0
|
||||
var postsAtAck int
|
||||
act, _ := r.worker.Handle(body, func() { calls++; postsAtAck = f.posts })
|
||||
if calls != 1 || postsAtAck != 0 || act != actionSettled || f.posts != 1 {
|
||||
t.Errorf("early called %d times with %d scans begun; action %v; %d scans in all", calls, postsAtAck, act, f.posts)
|
||||
}
|
||||
body, _ = json.Marshal(Ask{ID: "s2", Op: OpSettings, Page: DefaultPage, At: fixedNow})
|
||||
calls = 0
|
||||
if act, _ := r.worker.Handle(body, func() { calls++ }); calls != 0 || act != actionAck {
|
||||
t.Errorf("settings: early called %d times, action %v", calls, act)
|
||||
}
|
||||
}
|
||||
|
||||
// 1453/AC11: a part of 700 KiB is taken and one of a byte more is refused, by the module's own message naming
|
||||
// 700 KiB, before it is queued; the limit and the size the module cuts are one.
|
||||
func TestAPartOverSevenHundredKiBIsRefusedByTheModulesOwnMessage(t *testing.T) {
|
||||
if MaxPart != PartSize || PartSize != 700<<10 {
|
||||
t.Fatalf("MaxPart %d, PartSize %d: both are 700 KiB", MaxPart, PartSize)
|
||||
}
|
||||
r := newRig(t, `{}`, "pw")
|
||||
big := make([]byte, 700<<10+1)
|
||||
_, err := upload(t, r, "over", big, sha256Hex(big), 1, 1, big)
|
||||
if err == nil || !strings.Contains(err.Error(), "700 KiB") {
|
||||
t.Fatalf("a part of 700 KiB + 1 byte: error = %v, want the module's own message naming 700 KiB", err)
|
||||
}
|
||||
if r.bus.asks() != 0 {
|
||||
t.Errorf("an oversized part was queued")
|
||||
}
|
||||
ok := pdfOf(700 << 10)
|
||||
if _, err := upload(t, r, "exact", ok, sha256Hex(ok), 1, 1, ok); err != nil {
|
||||
t.Errorf("a part of exactly 700 KiB: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1535: a scan larger than the state's value limit (256 KiB) is filed whole, its SHA-256 equal, and the answer to
|
||||
// the scan itself is small: it carries the event's body and never the document. Every part answer fits the state.
|
||||
func TestAScanLargerThanTheStatesValueLimitIsFiledWholeAndItsAnswerIsSmall(t *testing.T) {
|
||||
f := newFakeScanner(t, fixture(t, "scan.jpg"))
|
||||
r, folder := scanRig(t, f, Settings{Crop: false})
|
||||
// The real capture is small; the worker keeps a stand-in of 3 parts' size under the key the scan names.
|
||||
big := pdfOf(222388 + 2*AnswerPartSize)
|
||||
if len(big) <= StateMaxValue {
|
||||
t.Fatal("the stand-in is not larger than the state's value limit")
|
||||
}
|
||||
r.worker.Capture = func(ctx context.Context, req ScanRequest) ([]Capture, error) {
|
||||
return []Capture{{PDF: big, SHA256: sha256Hex(big), Resolution: 300, At: fixedNow}}, nil
|
||||
}
|
||||
body, err := r.scan(t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path, _ := body["path"].(string)
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil || sha256Hex(got) != sha256Hex(big) || filepath.Dir(path) != folder {
|
||||
t.Fatalf("filed %s: %v, SHA-256 equal %v", path, err, err == nil && sha256Hex(got) == sha256Hex(big))
|
||||
}
|
||||
for id, raw := range r.answers.m {
|
||||
if len(raw) > StateMaxValue {
|
||||
t.Errorf("answer %s is %d bytes, over the state's %d", id, len(raw), StateMaxValue)
|
||||
}
|
||||
}
|
||||
for _, a := range r.bus.published {
|
||||
if a.Op == OpScan {
|
||||
if raw := r.answers.get(a.ID); len(raw) > 4<<10 || bytes.Contains(raw, []byte(`"data"`)) {
|
||||
t.Errorf("the scan's answer is %d bytes and may carry the document: %s", len(raw), raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1535: an answer the state refuses ends the ask: a small error answer says why, and the queued message is
|
||||
// terminated, never redelivered into the same failure; a scan's message, acknowledged early, stays settled.
|
||||
func TestARefusedAnswerEndsTheAskWithAnErrorAndNoRedelivery(t *testing.T) {
|
||||
r := newRig(t, `{}`, "pw")
|
||||
big := pdfOf(300 << 10)
|
||||
key := sha256Hex(big) + ".pdf"
|
||||
r.objects.Put(t.Context(), key, "application/pdf", big)
|
||||
// A part answer of 700 KiB, as the old cut made it, is over the state's limit; the worker must not loop on it.
|
||||
r.worker.answerPart = PartSize
|
||||
ask, _ := json.Marshal(Ask{ID: "big1", Op: OpDocument, Object: key, Part: 1, At: fixedNow})
|
||||
act, _ := r.worker.Handle(ask, nil)
|
||||
if act != actionTerm {
|
||||
t.Errorf("action = %v, want term", act)
|
||||
}
|
||||
raw := r.answers.get("big1")
|
||||
if raw == nil || !bytes.Contains(raw, []byte(`"error"`)) || len(raw) > 4<<10 {
|
||||
t.Errorf("no small error answer: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// 1535: only a refused value (error code 10054, message too large) ends the ask; a store that is away, a timeout
|
||||
// or a cut connection is redelivered, a document fetch being safe to repeat.
|
||||
func TestOnlyATooLargeAnswerEndsTheAskATransientFailureIsRetried(t *testing.T) {
|
||||
r := newRig(t, `{}`, "pw")
|
||||
big := pdfOf(300 << 10)
|
||||
key := sha256Hex(big) + ".pdf"
|
||||
r.objects.Put(t.Context(), key, "application/pdf", big)
|
||||
ask, _ := json.Marshal(Ask{ID: "t1", Op: OpDocument, Object: key, Part: 1, At: fixedNow})
|
||||
r.answers.fail = fmt.Errorf("nats: timeout")
|
||||
if act, d := r.worker.Handle(ask, nil); act != actionRetry || d <= 0 {
|
||||
t.Errorf("a timeout: action %v delay %v, want retry", act, d)
|
||||
}
|
||||
r.answers.fail = nil
|
||||
r.worker.answerPart = PartSize
|
||||
ask2, _ := json.Marshal(Ask{ID: "t2", Op: OpDocument, Object: key, Part: 1, At: fixedNow})
|
||||
if act, _ := r.worker.Handle(ask2, nil); act != actionTerm {
|
||||
t.Errorf("a too large answer: action %v, want term", act)
|
||||
}
|
||||
}
|
||||
|
||||
// home makes a home folder for the tool account and returns it.
|
||||
func home(t *testing.T) string {
|
||||
t.Helper()
|
||||
h := t.TempDir()
|
||||
t.Setenv("HOME", h)
|
||||
return h
|
||||
}
|
||||
@@ -1,454 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// eSCL (the AirScan protocol) over HTTP: POST <base>/ScanJobs answers 201 and the job's Location, GET
|
||||
// <Location>/NextDocument answers the image, and a second NextDocument answers 404 once the job is done (that is what closes it; DELETE of the job does too).
|
||||
|
||||
// maxImage bounds what a scanner may send: a 600 dpi A4 JPEG is a few tens of megabytes.
|
||||
const maxImage = 256 << 20
|
||||
|
||||
// Client speaks eSCL to one scanner.
|
||||
type Client struct {
|
||||
Base string
|
||||
HTTP *http.Client
|
||||
RetryWait time.Duration // the wait before the one retry of a busy scanner (default 4 s)
|
||||
Logf func(format string, a ...any)
|
||||
}
|
||||
|
||||
// NewClient is a client for the eSCL base URL, with a timeout that outlasts a slow scan.
|
||||
func NewClient(base string) *Client {
|
||||
return &Client{Base: strings.TrimRight(base, "/"), HTTP: &http.Client{Timeout: 3 * time.Minute}}
|
||||
}
|
||||
|
||||
// Size is the largest scan region of one input, in the 300ths of an inch the region is asked in.
|
||||
type Size struct{ MaxWidth, MaxHeight int }
|
||||
|
||||
// Capabilities are the largest scan regions of the platen and of the document feeder (simplex and duplex). A
|
||||
// scanner with no feeder has zero sizes for it.
|
||||
type Capabilities struct {
|
||||
MaxWidth, MaxHeight int // the platen
|
||||
Feeder, FeederDuplex Size
|
||||
}
|
||||
|
||||
type capabilitiesXML struct {
|
||||
MaxWidth int `xml:"Platen>PlatenInputCaps>MaxWidth"`
|
||||
MaxHeight int `xml:"Platen>PlatenInputCaps>MaxHeight"`
|
||||
AdfWidth int `xml:"Adf>AdfSimplexInputCaps>MaxWidth"`
|
||||
AdfHeight int `xml:"Adf>AdfSimplexInputCaps>MaxHeight"`
|
||||
DuplexWidth int `xml:"Adf>AdfDuplexInputCaps>MaxWidth"`
|
||||
DuplexHeight int `xml:"Adf>AdfDuplexInputCaps>MaxHeight"`
|
||||
}
|
||||
|
||||
// Status is what /ScannerStatus says.
|
||||
type Status struct {
|
||||
State string `xml:"State" json:"state"`
|
||||
AdfState string `xml:"AdfState" json:"adf-state"`
|
||||
}
|
||||
|
||||
func (c *Client) get(ctx context.Context, rawURL string) (*http.Response, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c.HTTP.Do(req)
|
||||
}
|
||||
|
||||
func readBody(resp *http.Response, limit int64) ([]byte, error) {
|
||||
defer resp.Body.Close()
|
||||
return io.ReadAll(io.LimitReader(resp.Body, limit))
|
||||
}
|
||||
|
||||
// Capabilities asks the scanner how large its glass is.
|
||||
func (c *Client) Capabilities(ctx context.Context) (Capabilities, error) {
|
||||
resp, err := c.get(ctx, c.Base+"/ScannerCapabilities")
|
||||
if err != nil {
|
||||
return Capabilities{}, fmt.Errorf("the scanner does not answer: %w", err)
|
||||
}
|
||||
body, err := readBody(resp, 1<<20)
|
||||
if err != nil {
|
||||
return Capabilities{}, fmt.Errorf("the scanner's capabilities cannot be read: %w", err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return Capabilities{}, fmt.Errorf("the scanner refused its capabilities: %s", resp.Status)
|
||||
}
|
||||
return parseCapabilities(body)
|
||||
}
|
||||
|
||||
func parseCapabilities(body []byte) (Capabilities, error) {
|
||||
var x capabilitiesXML
|
||||
if err := xml.Unmarshal(body, &x); err != nil {
|
||||
return Capabilities{}, fmt.Errorf("the scanner's capabilities are not XML: %w", err)
|
||||
}
|
||||
if x.MaxWidth <= 0 || x.MaxHeight <= 0 {
|
||||
return Capabilities{}, fmt.Errorf("the scanner's capabilities name no platen size")
|
||||
}
|
||||
return Capabilities{MaxWidth: x.MaxWidth, MaxHeight: x.MaxHeight,
|
||||
Feeder: Size{x.AdfWidth, x.AdfHeight}, FeederDuplex: Size{x.DuplexWidth, x.DuplexHeight}}, nil
|
||||
}
|
||||
|
||||
// Status asks what the scanner is doing.
|
||||
func (c *Client) Status(ctx context.Context) (Status, error) {
|
||||
resp, err := c.get(ctx, c.Base+"/ScannerStatus")
|
||||
if err != nil {
|
||||
return Status{}, fmt.Errorf("the scanner does not answer: %w", err)
|
||||
}
|
||||
body, err := readBody(resp, 1<<20)
|
||||
if err != nil {
|
||||
return Status{}, fmt.Errorf("the scanner's status cannot be read: %w", err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return Status{}, fmt.Errorf("the scanner refused its status: %s", resp.Status)
|
||||
}
|
||||
var s Status
|
||||
if err := xml.Unmarshal(body, &s); err != nil {
|
||||
return Status{}, fmt.Errorf("the scanner's status is not XML: %w", err)
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// colorModes maps the config's words to eSCL's.
|
||||
var colorModes = map[string]string{"color": "RGB24", "gray": "Grayscale8"}
|
||||
|
||||
// ScanOptions is what one job asks: the resolution and colour mode of the configuration, and the input. Duplex is
|
||||
// for the feeder.
|
||||
type ScanOptions struct {
|
||||
Resolution int
|
||||
ColorMode string
|
||||
Feeder bool
|
||||
Duplex bool
|
||||
// Bound is how long a feeder job may have run when it has this many pages and is asked for the next; nil is
|
||||
// FeederBound. The job is deleted and an error given when it has run longer.
|
||||
Bound func(pages int) time.Duration
|
||||
}
|
||||
|
||||
// FeederBound is how long a feeder job may run when it has this many pages (issue 1539): 20 seconds for each,
|
||||
// at least 3 minutes and at most 30. The scanner gives a duplex sheet about every 3.4 seconds, so this is generous
|
||||
// and only ends a job that is stuck.
|
||||
func FeederBound(pages int) time.Duration {
|
||||
return min(max(time.Duration(pages)*20*time.Second, 3*time.Minute), FeederMaxBound)
|
||||
}
|
||||
|
||||
// FeederMaxBound is the longest a feeder job may run.
|
||||
const FeederMaxBound = 30 * time.Minute
|
||||
|
||||
// A4 in the 300ths of an inch the region is asked in; a region is cut to what the input can scan.
|
||||
const a4Width, a4Height = 2480, 3508
|
||||
|
||||
// maxFeederPages bounds one feeder job: the feeder holds 50 sheets, which duplex is 100 pages. After them the
|
||||
// scanner has to answer 404.
|
||||
const maxFeederPages = 100
|
||||
|
||||
// ScanSettings is the request body, on ONE line: the Brother ignores the whole body, and runs its default job (the
|
||||
// platen, 200 dpi), when there is a line break anywhere before the closing root tag (issue 1452). The region is A4
|
||||
// cut to the input's largest region.
|
||||
func ScanSettings(c Capabilities, o ScanOptions) (string, error) {
|
||||
escl, ok := colorModes[o.ColorMode]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("color-mode %q is not color or gray", o.ColorMode)
|
||||
}
|
||||
size, source := Size{c.MaxWidth, c.MaxHeight}, "Platen"
|
||||
if o.Duplex && !o.Feeder {
|
||||
return "", fmt.Errorf("duplex is for the feeder")
|
||||
}
|
||||
if o.Feeder {
|
||||
source, size = "Feeder", c.Feeder
|
||||
if o.Duplex {
|
||||
size = c.FeederDuplex
|
||||
}
|
||||
if size.MaxWidth <= 0 || size.MaxHeight <= 0 {
|
||||
return "", fmt.Errorf("the scanner names no document feeder%s", map[bool]string{true: " that scans both sides", false: ""}[o.Duplex])
|
||||
}
|
||||
}
|
||||
duplex := ""
|
||||
if o.Duplex {
|
||||
duplex = "<scan:Duplex>true</scan:Duplex>"
|
||||
}
|
||||
return fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?><scan:ScanSettings xmlns:scan="http://schemas.hp.com/imaging/escl/2011/05/03" xmlns:pwg="http://www.pwg.org/schemas/2010/12/sm">`+
|
||||
`<pwg:Version>2.0</pwg:Version><scan:Intent>Document</scan:Intent>`+
|
||||
`<pwg:ScanRegions><pwg:ScanRegion><pwg:ContentRegionUnits>escl:ThreeHundredthsOfInches</pwg:ContentRegionUnits><pwg:XOffset>0</pwg:XOffset><pwg:YOffset>0</pwg:YOffset><pwg:Width>%d</pwg:Width><pwg:Height>%d</pwg:Height></pwg:ScanRegion></pwg:ScanRegions>`+
|
||||
`<pwg:InputSource>%s</pwg:InputSource><scan:ColorMode>%s</scan:ColorMode><pwg:DocumentFormat>image/jpeg</pwg:DocumentFormat>`+
|
||||
`<scan:XResolution>%d</scan:XResolution><scan:YResolution>%d</scan:YResolution>%s</scan:ScanSettings>`,
|
||||
min(a4Width, size.MaxWidth), min(a4Height, size.MaxHeight), source, escl, o.Resolution, o.Resolution, duplex), nil
|
||||
}
|
||||
|
||||
// Scan puts one job for the glass to the scanner and returns the JPEG it sends. The resolution asked is only a
|
||||
// request: the real one is the image's own density (Density).
|
||||
//
|
||||
// A job stays open on the scanner (pwg:State Processing, and 503 to the next ScanJobs) until NextDocument has
|
||||
// answered 404 or the job is DELETEd, so Scan always ends the job it made: after the page it asks NextDocument
|
||||
// again, and on any failure after the job was created it deletes the job.
|
||||
func (c *Client) Scan(ctx context.Context, resolution int, colorMode string) ([]byte, error) {
|
||||
pages, err := c.ScanPages(ctx, ScanOptions{Resolution: resolution, ColorMode: colorMode})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pages[0], nil
|
||||
}
|
||||
|
||||
// ScanPages puts one job to the scanner, the glass or the feeder, and returns its pages in order. The glass holds
|
||||
// one page (a second is an error); the feeder is asked NextDocument until it answers 404, at most maxFeederPages
|
||||
// times. Either way the job is ended, and deleted on any failure after it was created.
|
||||
func (c *Client) ScanPages(ctx context.Context, o ScanOptions) ([][]byte, error) {
|
||||
caps, err := c.Capabilities(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
settings, err := ScanSettings(caps, o)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
job, err := c.createJob(ctx, settings)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
doc := job + "/NextDocument"
|
||||
if o.Feeder {
|
||||
pages, err := c.feederPages(ctx, doc, o.Bound)
|
||||
if err != nil {
|
||||
c.deleteJob(job)
|
||||
return nil, err
|
||||
}
|
||||
return pages, nil
|
||||
}
|
||||
image, err := c.page(ctx, doc)
|
||||
if err != nil {
|
||||
c.deleteJob(job)
|
||||
return nil, err
|
||||
}
|
||||
closed, extra := c.closeJob(ctx, doc)
|
||||
if !closed {
|
||||
c.deleteJob(job)
|
||||
}
|
||||
if extra > 0 {
|
||||
return nil, fmt.Errorf("the scanner gave more pages than the glass holds: %d", 1+extra)
|
||||
}
|
||||
return [][]byte{image}, nil
|
||||
}
|
||||
|
||||
// feederPages asks NextDocument until the scanner answers 404, which ends the job. The first page missing means no
|
||||
// paper was in the feeder.
|
||||
func (c *Client) feederPages(ctx context.Context, doc string, bound func(int) time.Duration) ([][]byte, error) {
|
||||
if bound == nil {
|
||||
bound = FeederBound
|
||||
}
|
||||
start := time.Now()
|
||||
var pages [][]byte
|
||||
for len(pages) <= maxFeederPages {
|
||||
if len(pages) > 0 {
|
||||
if run, limit := time.Since(start), bound(len(pages)); run > limit {
|
||||
return nil, fmt.Errorf("the feeder scan has run %s and has %d pages, past its bound of %s: it is stopped", run.Round(time.Second), len(pages), limit)
|
||||
}
|
||||
}
|
||||
got, err := c.get(ctx, doc)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the scanned page %d cannot be fetched: %w", len(pages)+1, err)
|
||||
}
|
||||
image, err := readBody(got, maxImage)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the scanned page %d cannot be read: %w", len(pages)+1, err)
|
||||
}
|
||||
switch got.StatusCode {
|
||||
case http.StatusOK:
|
||||
case http.StatusNotFound:
|
||||
if len(pages) == 0 {
|
||||
return nil, fmt.Errorf("the scanner gave no page: is paper in the feeder?")
|
||||
}
|
||||
return pages, nil
|
||||
case http.StatusGone:
|
||||
return nil, fmt.Errorf("the scanner dropped the job: %s", got.Status)
|
||||
default:
|
||||
return nil, fmt.Errorf("the scanner gave no page %d: %s", len(pages)+1, got.Status)
|
||||
}
|
||||
if len(pages) == maxFeederPages {
|
||||
break
|
||||
}
|
||||
if ct := got.Header.Get("Content-Type"); !strings.HasPrefix(ct, "image/jpeg") {
|
||||
return nil, fmt.Errorf("the scanner sent %q, not image/jpeg", ct)
|
||||
}
|
||||
pages = append(pages, image)
|
||||
}
|
||||
return nil, fmt.Errorf("the feeder gave more than %d pages and the scanner did not end the job", maxFeederPages)
|
||||
}
|
||||
|
||||
// leftOpen is the job URL, per scanner, that brother could neither finish nor delete.
|
||||
var leftOpen sync.Map
|
||||
|
||||
func (c *Client) logf(format string, a ...any) {
|
||||
if c.Logf != nil {
|
||||
c.Logf(format, a...)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, format+"\n", a...)
|
||||
}
|
||||
|
||||
func (c *Client) retryWait() time.Duration {
|
||||
if c.RetryWait > 0 {
|
||||
return c.RetryWait
|
||||
}
|
||||
return 4 * time.Second
|
||||
}
|
||||
|
||||
// createJob posts the settings and returns the job's URL. A 503 is a scanner busy with another job; when the job
|
||||
// brother itself left open on this scanner is now closed, the post goes once more, never more than once.
|
||||
func (c *Client) createJob(ctx context.Context, settings string) (string, error) {
|
||||
resp, reply, err := c.postJob(ctx, settings)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if resp.StatusCode == http.StatusServiceUnavailable {
|
||||
if left, ok := leftOpen.Load(c.Base); ok {
|
||||
select {
|
||||
case <-time.After(c.retryWait()):
|
||||
case <-ctx.Done():
|
||||
return "", ctx.Err()
|
||||
}
|
||||
if c.deleteJob(left.(string)) {
|
||||
if resp, reply, err = c.postJob(ctx, settings); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
}
|
||||
if resp.StatusCode == http.StatusServiceUnavailable {
|
||||
return "", fmt.Errorf("the scanner is busy with another job: %s %s", resp.Status, strings.TrimSpace(string(reply)))
|
||||
}
|
||||
}
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
return "", fmt.Errorf("the scanner refused the scan job: %s %s", resp.Status, strings.TrimSpace(string(reply)))
|
||||
}
|
||||
loc := resp.Header.Get("Location")
|
||||
if loc == "" {
|
||||
return "", fmt.Errorf("the scanner took the scan job but named no Location")
|
||||
}
|
||||
base, _ := url.Parse(c.Base + "/")
|
||||
ref, err := url.Parse(loc)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the scan job's Location %q is not a URL: %w", loc, err)
|
||||
}
|
||||
return strings.TrimRight(base.ResolveReference(ref).String(), "/"), nil
|
||||
}
|
||||
|
||||
func (c *Client) postJob(ctx context.Context, settings string) (*http.Response, []byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.Base+"/ScanJobs", strings.NewReader(settings))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "text/xml")
|
||||
resp, err := c.HTTP.Do(req)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the scanner does not answer: %w", err)
|
||||
}
|
||||
reply, _ := readBody(resp, 1<<20)
|
||||
return resp, reply, nil
|
||||
}
|
||||
|
||||
// page fetches the job's one page.
|
||||
func (c *Client) page(ctx context.Context, doc string) ([]byte, error) {
|
||||
got, err := c.get(ctx, doc)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the scanned page cannot be fetched: %w", err)
|
||||
}
|
||||
image, err := readBody(got, maxImage)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the scanned page cannot be read: %w", err)
|
||||
}
|
||||
if got.StatusCode == http.StatusGone {
|
||||
return nil, fmt.Errorf("the scanner dropped the job: %s", got.Status)
|
||||
}
|
||||
if got.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("the scanner gave no page: %s", got.Status)
|
||||
}
|
||||
if ct := got.Header.Get("Content-Type"); !strings.HasPrefix(ct, "image/jpeg") {
|
||||
return nil, fmt.Errorf("the scanner sent %q, not image/jpeg", ct)
|
||||
}
|
||||
return image, nil
|
||||
}
|
||||
|
||||
// maxClosing bounds the NextDocument requests that ask for the end of a job.
|
||||
const maxClosing = 1
|
||||
|
||||
// closeJob asks NextDocument until the scanner answers 404, which ends the job; true when it did (or the job is
|
||||
// gone already), and how many further pages it sent instead; any is an error for the caller, for a Platen job holds
|
||||
// one page. It stops at the first further page, and the job is then deleted.
|
||||
func (c *Client) closeJob(ctx context.Context, doc string) (closed bool, extra int) {
|
||||
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
|
||||
defer cancel()
|
||||
for i := 0; i < maxClosing; i++ {
|
||||
resp, err := c.get(ctx, doc)
|
||||
if err != nil {
|
||||
c.logf("brother: closing the scan job: %v", err)
|
||||
return false, 0
|
||||
}
|
||||
readBody(resp, maxImage)
|
||||
switch resp.StatusCode {
|
||||
case http.StatusNotFound, http.StatusGone:
|
||||
return true, 0
|
||||
case http.StatusOK:
|
||||
c.logf("brother: the scanner sent a further page after the glass's one")
|
||||
return false, 1
|
||||
}
|
||||
c.logf("brother: closing the scan job: NextDocument answered %s", resp.Status)
|
||||
return false, 0
|
||||
}
|
||||
return false, 0
|
||||
}
|
||||
|
||||
// deleteJob deletes the job, best effort; true when the job is gone. A job that cannot be deleted is remembered, so
|
||||
// a 503 on the next scan can be told from another client's job.
|
||||
func (c *Client) deleteJob(job string) bool {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodDelete, job, nil)
|
||||
if err == nil {
|
||||
var resp *http.Response
|
||||
if resp, err = c.HTTP.Do(req); err == nil {
|
||||
readBody(resp, 1<<20)
|
||||
if resp.StatusCode/100 == 2 || resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone {
|
||||
leftOpen.Delete(c.Base)
|
||||
return true
|
||||
}
|
||||
err = fmt.Errorf("answered %s", resp.Status)
|
||||
}
|
||||
}
|
||||
c.logf("brother: deleting the scan job %s: %v", job, err)
|
||||
leftOpen.Store(c.Base, job)
|
||||
return false
|
||||
}
|
||||
|
||||
// Density is the image's own resolution in dpi, from its JFIF header (units dpi, or dots per cm).
|
||||
func Density(jpegBytes []byte) (int, error) {
|
||||
b := jpegBytes
|
||||
if len(b) < 4 || b[0] != 0xFF || b[1] != 0xD8 {
|
||||
return 0, fmt.Errorf("the image is not a JPEG")
|
||||
}
|
||||
for i := 2; i+4 <= len(b) && b[i] == 0xFF; {
|
||||
marker, n := b[i+1], int(b[i+2])<<8|int(b[i+3])
|
||||
if marker == 0xE0 && n >= 16 && i+4+12 <= len(b) && string(b[i+4:i+9]) == "JFIF\x00" {
|
||||
units := b[i+11]
|
||||
x := int(b[i+12])<<8 | int(b[i+13])
|
||||
switch {
|
||||
case units == 1 && x > 0:
|
||||
return x, nil
|
||||
case units == 2 && x > 0:
|
||||
return int(float64(x)*2.54 + 0.5), nil
|
||||
}
|
||||
return 0, fmt.Errorf("the image's JFIF header gives no resolution")
|
||||
}
|
||||
if marker == 0xDA || n < 2 {
|
||||
break
|
||||
}
|
||||
i += 2 + n
|
||||
}
|
||||
return 0, fmt.Errorf("the image has no JFIF header, so its resolution is unknown")
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user