Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 jschoubben
|
021e5a96ff |
+36
-229
@@ -1,236 +1,43 @@
|
||||
node_modules/
|
||||
dist/
|
||||
|
||||
# Go tool bundles built in place (go build in a module's cmd/<name>-tools) are build output.
|
||||
modules/slack/cmd/slack-tools/slack-tools
|
||||
modules/jetbrains-toolbox/cmd/toolbox-tools/toolbox-tools
|
||||
modules/messenger/cmd/messenger/messenger
|
||||
modules/mesh-watcher/cmd/mesh-watcher/mesh-watcher
|
||||
modules/claude-code/cmd/claude-code/claude-code
|
||||
modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq-tools
|
||||
# ...and the same built from the module's root (go build ./cmd/<name>), which names it after the module.
|
||||
modules/messenger/messenger
|
||||
modules/claude-code/claude-code
|
||||
modules/mesh-watcher/mesh-watcher
|
||||
modules/dnsmasq/dnsmasq
|
||||
modules/mesh-issues/cmd/mesh-issues/mesh-issues
|
||||
modules/mesh-issues/mesh-issues
|
||||
modules/mounts/cmd/mounts/mounts
|
||||
modules/mesh-knowledge/cmd/mesh-knowledge/mesh-knowledge
|
||||
modules/mesh-knowledge/mesh-knowledge
|
||||
modules/mesh-reads/mesh-reads
|
||||
modules/mesh-reads/cmd/mesh-reads/mesh-reads
|
||||
modules/mesh-files/mesh-files
|
||||
modules/mesh-files/cmd/mesh-files/mesh-files
|
||||
|
||||
# The provisioner client built in place (issue 671).
|
||||
modules/mesh-provisioner-client/cmd/mesh-provisioner-client/mesh-provisioner-client
|
||||
modules/brother/cmd/brother/brother
|
||||
modules/mesh-build-go/cmd/mesh-build-go/mesh-build-go
|
||||
modules/mesh-config/mesh-config
|
||||
modules/mesh-build-image/cmd/mesh-build-image/mesh-build-image
|
||||
*.test
|
||||
|
||||
# Every program a local `go build` leaves, in its main package's directory and at its Go module's root, each anchored to
|
||||
# its one path so it never hides source (issue 1577: twice in one milestone a module's program was committed). The
|
||||
# checks/gitignored test refuses a main package whose program no ignore file names: add its line here.
|
||||
/checks/agenthome/agenthome
|
||||
/checks/config/config
|
||||
/checks/deps/deps
|
||||
/checks/environment/environment
|
||||
/checks/imagebuild/imagebuild
|
||||
/checks/imageplace/imageplace
|
||||
/checks/locks/locks
|
||||
/checks/mailstore/mailstore
|
||||
/checks/noplacement/noplacement
|
||||
/checks/owned/owned
|
||||
/checks/particulars/particulars
|
||||
# binaries a local go build leaves in a module root
|
||||
/checks/paths/paths
|
||||
/checks/predefined/predefined
|
||||
/checks/readable/readable
|
||||
/checks/removedpackages/removedpackages
|
||||
/checks/secrets/secrets
|
||||
/checks/sshdir/sshdir
|
||||
/checks/templates/templates
|
||||
/checks/toolchains/toolchains
|
||||
/checks/words/words
|
||||
/modules/adwaita/adwaita-tools
|
||||
/modules/adwaita/cmd/adwaita-tools/adwaita-tools
|
||||
/modules/artifact-store-tools/artifact-store-tools
|
||||
/modules/artifact-store-tools/cmd/artifact-store-tools/artifact-store-tools
|
||||
/modules/asus-zephyrus-g14/cmd/zephyrus/zephyrus
|
||||
/modules/asus-zephyrus-g14/zephyrus
|
||||
/modules/avahi/avahi-tools
|
||||
/modules/avahi/cmd/avahi-tools/avahi-tools
|
||||
/modules/blueman/blueman-tools
|
||||
/modules/blueman/cmd/blueman-tools/blueman-tools
|
||||
/modules/bluetooth/bluetooth-tools
|
||||
/modules/bluetooth/cmd/bluetooth-tools/bluetooth-tools
|
||||
/modules/brother/brother
|
||||
/modules/brother/cmd/brother/brother
|
||||
/modules/claude-code/claude-code
|
||||
/modules/claude-code/cmd/claude-code/claude-code
|
||||
/modules/claude-licence-manager/claude-licence-manager
|
||||
/modules/claude-licence-manager/cmd/claude-licence-manager/claude-licence-manager
|
||||
/modules/clipmenu/clipmenu-tools
|
||||
/modules/clipmenu/cmd/clipmenu-tools/clipmenu-tools
|
||||
/modules/cups/cmd/cups-tools/cups-tools
|
||||
/modules/cups/cups-tools
|
||||
/modules/dbus/cmd/dbus-tools/dbus-tools
|
||||
/modules/dbus/dbus-tools
|
||||
/modules/desk-channel/cmd/desk-channel/desk-channel
|
||||
/modules/desk-channel/desk-channel
|
||||
/modules/disk-load/cmd/disk-load/disk-load
|
||||
/modules/disk-load/disk-load
|
||||
/modules/distribution/cmd/distribution-tools/distribution-tools
|
||||
/modules/distribution/cmd/distribution/distribution
|
||||
/modules/distribution/distribution
|
||||
/modules/distribution/distribution-tools
|
||||
/modules/dmenu/cmd/dmenu-tools/dmenu-tools
|
||||
/modules/dmenu/dmenu-tools
|
||||
/modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq-tools
|
||||
/modules/dnsmasq/dnsmasq-tools
|
||||
/modules/docker-compose/cmd/docker-compose-tools/docker-compose-tools
|
||||
/modules/docker-compose/docker-compose-tools
|
||||
/modules/docker/cmd/docker-tools/docker-tools
|
||||
/modules/docker/docker-tools
|
||||
/modules/dunst/cmd/dunst-tools/dunst-tools
|
||||
/modules/dunst/dunst-tools
|
||||
/modules/fail2ban/cmd/fail2ban-tools/fail2ban-tools
|
||||
/modules/fail2ban/fail2ban-tools
|
||||
/modules/feh/cmd/feh-tools/feh-tools
|
||||
/modules/feh/feh-tools
|
||||
/modules/flatpak/cmd/flatpak-tools/flatpak-tools
|
||||
/modules/flatpak/flatpak-tools
|
||||
/modules/fonts/cmd/fonts-tools/fonts-tools
|
||||
/modules/fonts/fonts-tools
|
||||
/modules/forticlient/cmd/forticlient-tools/forticlient-tools
|
||||
/modules/forticlient/forticlient-tools
|
||||
/modules/gitea/agent-teams
|
||||
/modules/gitea/agent-token
|
||||
/modules/gitea/cmd/agent-teams/agent-teams
|
||||
/modules/gitea/cmd/agent-token/agent-token
|
||||
/modules/gitea/cmd/forge-account-provider/forge-account-provider
|
||||
/modules/gitea/cmd/forge/forge
|
||||
/modules/gitea/cmd/package-registry/package-registry
|
||||
/modules/gitea/cmd/repositories/repositories
|
||||
/modules/gitea/forge
|
||||
/modules/gitea/forge-account-provider
|
||||
/modules/gitea/package-registry
|
||||
/modules/gitea/repositories
|
||||
/modules/gnome-keyring/cmd/gnome-keyring-tools/gnome-keyring-tools
|
||||
/modules/gnome-keyring/gnome-keyring-tools
|
||||
/modules/hostname/cmd/hostname-tools/hostname-tools
|
||||
/modules/hostname/hostname-tools
|
||||
/modules/i3/cmd/i3-tools/i3-tools
|
||||
/modules/i3/i3-tools
|
||||
/modules/i3status-rust/cmd/i3status-rust-tools/i3status-rust-tools
|
||||
/modules/i3status-rust/i3status-rust-tools
|
||||
/modules/kernel/cmd/kernel-tools/kernel-tools
|
||||
/modules/kernel/kernel-tools
|
||||
/modules/keycloak/cmd/keycloak-provider/keycloak-provider
|
||||
/modules/keycloak/keycloak-provider
|
||||
/modules/lemurs/cmd/lemurs-tools/lemurs-tools
|
||||
/modules/lemurs/lemurs-tools
|
||||
/modules/localization/cmd/localization-tools/localization-tools
|
||||
/modules/localization/localization-tools
|
||||
/modules/logrotate/cmd/logrotate-tools/logrotate-tools
|
||||
/modules/logrotate/logrotate-tools
|
||||
/modules/mailu/automx
|
||||
/modules/mailu/cmd/automx/automx
|
||||
/modules/memory-pressure/cmd/memory-pressure/memory-pressure
|
||||
/modules/memory-pressure/memory-pressure
|
||||
/modules/mesh-build-go/buildstep/cmd/mesh-pack/mesh-pack
|
||||
/modules/mesh-build-go/buildstep/cmd/seatgen/seatgen
|
||||
/modules/mesh-build-go/buildstep/mesh-pack
|
||||
/modules/mesh-build-go/cmd/mesh-build-go/mesh-build-go
|
||||
/modules/mesh-build-go/mesh-build-go
|
||||
/modules/mesh-build-image/cmd/mesh-build-image/mesh-build-image
|
||||
/modules/mesh-build-image/mesh-build-image
|
||||
/modules/mesh-build-node/cmd/mesh-build-node/mesh-build-node
|
||||
/modules/mesh-build-node/mesh-build-node
|
||||
/modules/mesh-cli/cmd/mesh-cli/mesh-cli
|
||||
/modules/mesh-cli/mesh-cli
|
||||
/modules/mesh-conditions/cmd/mesh-conditions/mesh-conditions
|
||||
/modules/mesh-conditions/mesh-conditions
|
||||
/modules/mesh-config/cmd/mesh-config/mesh-config
|
||||
/modules/mesh-config/mesh-config
|
||||
/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/mesh-delivery-gate
|
||||
/modules/mesh-delivery-gate/mesh-delivery-gate
|
||||
/modules/mesh-files/cmd/mesh-files/mesh-files
|
||||
/modules/mesh-files/mesh-files
|
||||
/modules/mesh-hand-acts/cmd/mesh-hand-acts/mesh-hand-acts
|
||||
/modules/mesh-hand-acts/mesh-hand-acts
|
||||
/modules/mesh-healers/cmd/mesh-healers/mesh-healers
|
||||
/modules/mesh-healers/mesh-healers
|
||||
/modules/mesh-issues/cmd/mesh-issues/mesh-issues
|
||||
/modules/mesh-issues/mesh-issues
|
||||
/modules/mesh-knowledge/cmd/mesh-knowledge/mesh-knowledge
|
||||
/modules/mesh-knowledge/mesh-knowledge
|
||||
/modules/mesh-mail-notifier/cmd/mesh-mail-notifier/mesh-mail-notifier
|
||||
/modules/mesh-mail-notifier/mesh-mail-notifier
|
||||
/modules/mesh-mail-reader/cmd/mesh-mail-reader/mesh-mail-reader
|
||||
/modules/mesh-mail-reader/mesh-mail-reader
|
||||
/modules/mesh-mail-web/cmd/mesh-mail-web/mesh-mail-web
|
||||
/modules/mesh-mail-web/mesh-mail-web
|
||||
/modules/mesh-mail/cmd/mesh-mail/mesh-mail
|
||||
/modules/mesh-mail/mesh-mail
|
||||
/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/mesh-provisioner-client
|
||||
/modules/mesh-provisioner-client/mesh-provisioner-client
|
||||
/modules/mesh-provisioner/cmd/mesh-provisioner/mesh-provisioner
|
||||
/modules/mesh-provisioner/mesh-provisioner
|
||||
/modules/mesh-reads/cmd/mesh-reads/mesh-reads
|
||||
/modules/mesh-reads/mesh-reads
|
||||
/modules/mesh-self-check/cmd/mesh-self-check/mesh-self-check
|
||||
/modules/mesh-self-check/mesh-self-check
|
||||
/modules/mesh-vault/cmd/mesh-vault/mesh-vault
|
||||
/modules/mesh-vault/mesh-vault
|
||||
/modules/mesh-watcher/cmd/mesh-watcher/mesh-watcher
|
||||
/modules/mesh-watcher/mesh-watcher
|
||||
/modules/messenger/cmd/messenger/messenger
|
||||
/modules/messenger/messenger
|
||||
/modules/minio/cmd/object-store/object-store
|
||||
/modules/minio/object-store
|
||||
/modules/mounts/cmd/mounts/mounts
|
||||
/modules/mounts/mounts
|
||||
/modules/nats/cmd/mesh-nats-snapshot/mesh-nats-snapshot
|
||||
/modules/nats/cmd/nats-tools/nats-tools
|
||||
/modules/nats/mesh-nats-snapshot
|
||||
/modules/nats/nats-tools
|
||||
/modules/netcheck/cmd/netcheck/netcheck
|
||||
/modules/netcheck/netcheck
|
||||
/modules/networkmanager/cmd/uplink-tools/uplink-tools
|
||||
/modules/networkmanager/uplink-tools
|
||||
/modules/nextcloud-client/cmd/nextcloud-client-tools/nextcloud-client-tools
|
||||
/modules/nextcloud-client/nextcloud-client-tools
|
||||
/modules/nfs-server/cmd/nfs-server/nfs-server
|
||||
/modules/nfs-server/nfs-server
|
||||
/modules/nm-applet/cmd/nm-applet-tools/nm-applet-tools
|
||||
/modules/nm-applet/nm-applet-tools
|
||||
/modules/only-office/cmd/only-office/only-office
|
||||
/modules/only-office/only-office
|
||||
/modules/openrazer/cmd/openrazer-tools/openrazer-tools
|
||||
/modules/openrazer/openrazer-tools
|
||||
/modules/pacman/cmd/pacman-tools/pacman-tools
|
||||
/modules/pacman/pacman-tools
|
||||
/modules/picom/cmd/picom-tools/picom-tools
|
||||
/modules/picom/picom-tools
|
||||
/modules/polychromatic/cmd/polychromatic-tools/polychromatic-tools
|
||||
/modules/polychromatic/polychromatic-tools
|
||||
/modules/postgres/cmd/postgres-provider/postgres-provider
|
||||
/modules/postgres/postgres-provider
|
||||
/modules/power/cmd/power/power
|
||||
/modules/power/power
|
||||
/modules/records/cmd/records/records
|
||||
/modules/records/records
|
||||
/modules/restic/cmd/restic-backups/restic-backups
|
||||
/modules/restic/restic-backups
|
||||
/modules/rofi/cmd/rofi-tools/rofi-tools
|
||||
/modules/rofi/rofi-tools
|
||||
/modules/route-adapter/cmd/route-provider/route-provider
|
||||
/modules/route-adapter/route-provider
|
||||
/modules/screen-lock/cmd/screen-lock-tools/screen-lock-tools
|
||||
/modules/screen-lock/screen-lock-tools
|
||||
/modules/sensors/cmd/sensors/sensors
|
||||
/modules/sensors/sensors
|
||||
/modules/slack/cmd/slack-tools/slack-tools
|
||||
/modules/slack/slack-tools
|
||||
/modules/snapd/cmd/snapd-tools/snapd-tools
|
||||
/modules/snapd/snapd-tools
|
||||
/modules/ssh-client/cmd/ssh-client-tools/ssh-client-tools
|
||||
/modules/ssh-client/ssh-client-tools
|
||||
/modules/sudo/cmd/sudo-tools/sudo-tools
|
||||
/modules/sudo/sudo-tools
|
||||
/modules/systemd-networkd/cmd/uplink-tools/uplink-tools
|
||||
/modules/systemd-networkd/uplink-tools
|
||||
/modules/systemd-resolved/cmd/resolver-tools/resolver-tools
|
||||
/modules/systemd-resolved/resolver-tools
|
||||
/modules/systemd/cmd/systemd-tools/systemd-tools
|
||||
/modules/systemd/systemd-tools
|
||||
/modules/telegram/cmd/telegram/telegram
|
||||
/modules/telegram/telegram
|
||||
/modules/time-sync/cmd/time-sync-tools/time-sync-tools
|
||||
/modules/time-sync/time-sync-tools
|
||||
/modules/triggerhappy/cmd/triggerhappy-tools/triggerhappy-tools
|
||||
/modules/triggerhappy/triggerhappy-tools
|
||||
/modules/xclip/cmd/xclip-tools/xclip-tools
|
||||
/modules/xclip/xclip-tools
|
||||
/modules/xdg/cmd/xdg-tools/xdg-tools
|
||||
/modules/xdg/xdg-tools
|
||||
/modules/xorg/cmd/xorg/xorg
|
||||
/modules/xorg/xorg
|
||||
/modules/xterm/cmd/xterm-tools/xterm-tools
|
||||
/modules/xterm/xterm-tools
|
||||
/modules/zfs/cmd/zfs-tools/zfs-tools
|
||||
/modules/zfs/zfs-tools
|
||||
modules/mesh-config/cmd/mesh-config/mesh-config
|
||||
|
||||
# Built by their modules' build step (issue 1577), never committed.
|
||||
modules/forticlient/cmd/forticlient-tools/forticlient-tools
|
||||
modules/networkmanager/uplink-tools
|
||||
checks/words/words
|
||||
|
||||
@@ -2,6 +2,6 @@ module git.novox.be/novox/mesh-catalog/checks/config
|
||||
|
||||
go 1.22
|
||||
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0-00010101000000-000000000000
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0
|
||||
|
||||
replace git.novox.be/novox/mesh-catalog/checks/trunk => ../trunk
|
||||
|
||||
@@ -1,164 +0,0 @@
|
||||
// Package controllerdeps is the catalogue's dependence check on the controller (novox/hq ADR 0590 §5, issue 1126): the
|
||||
// controller is not placed, so no module of this tree reads, requires, invokes, consumes, waits on, uses or otherwise
|
||||
// names it. Nothing is kept: not the lease, not the entries once kept to its write (ADR 0571 §13, superseded with ADR
|
||||
// 0582). A later decision that places a controller again must first lift this check.
|
||||
//
|
||||
// It refuses, over the tree at root:
|
||||
//
|
||||
// - a directory modules/mesh-controller: the controller is no module of the catalogue;
|
||||
// - a manifest string naming the controller's seat, a state, verb or event of it, or one of its buckets
|
||||
// (`mesh-controller.<name>`, `seat:mesh-controller.<verb>`, `mesh-controller_<bucket>`), anywhere in the manifest;
|
||||
// - a manifest string that is the bare module name `mesh-controller` outside the module's own `config`, where a value
|
||||
// is the module's data (an issue owner naming the controller's repository is no dependency), not an address;
|
||||
// - a go.mod of a module or of a check requiring the controller's repository (its module path, github.com/novox/mesh-controller,
|
||||
// or the forge's git.novox.be/novox/mesh-controller): the repository is archived once its last module is delivered
|
||||
// elsewhere (novox/hq ADR 0593 §11), so nothing here may be built from it;
|
||||
// - a line of any module's file naming one of the controller's buckets (`mesh-controller_`): its buckets are removed
|
||||
// after the controller's end (ADR 0590 §6), so a reader of one reads nothing. A test (`_test.go`, `.test.ts`) reads
|
||||
// no live bucket, its names are its own data, and a `testdata` directory is a copy of text from elsewhere (a design,
|
||||
// a backlog): both are skipped.
|
||||
//
|
||||
// The rule is said once, here, for the catalogue: decoding stays the manifests check's (a manifest that is no JSON is
|
||||
// an error here, never judged).
|
||||
package controllerdeps
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Module is the controller's module name, and Bucket the prefix every bucket of its carries.
|
||||
const (
|
||||
Module = "mesh-controller"
|
||||
Bucket = Module + "_"
|
||||
)
|
||||
|
||||
// address is a string naming the controller's seat, one of its states, verbs or events, or one of its buckets.
|
||||
var address = regexp.MustCompile(`(^|[^A-Za-z0-9_-])(seat:)?mesh-controller([._][A-Za-z0-9])`)
|
||||
|
||||
// repository is a go.mod line naming the controller's repository by either of its module paths.
|
||||
var repository = regexp.MustCompile(`(github\.com|git\.novox\.be)/novox/mesh-controller(/|\s|$)`)
|
||||
|
||||
// Refusals is each reason the check gives over the tree at root, sorted; none when it passes.
|
||||
func Refusals(root string) ([]string, error) {
|
||||
var out []string
|
||||
if _, err := os.Lstat(filepath.Join(root, "modules", Module)); err == nil {
|
||||
out = append(out, Module+": the controller is no module of the catalogue: it is not placed and nothing may depend on it (novox/hq ADR 0590 §5)")
|
||||
}
|
||||
files, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, f := range files {
|
||||
raw, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var doc any
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", f, err)
|
||||
}
|
||||
module := filepath.Base(filepath.Dir(f))
|
||||
for _, s := range names(doc, false, nil) {
|
||||
out = append(out, fmt.Sprintf("%s: names %q: the controller is not placed, so no manifest reads, requires, invokes, "+
|
||||
"consumes, waits on or uses it; name the seat that holds the job now (novox/hq ADR 0590 §5)", module, s))
|
||||
}
|
||||
}
|
||||
mods, err := filepath.Glob(filepath.Join(root, "*", "*", "go.mod"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(mods)
|
||||
for _, f := range mods {
|
||||
if rel, _ := filepath.Rel(root, f); !strings.HasPrefix(rel, "modules"+string(filepath.Separator)) && !strings.HasPrefix(rel, "checks"+string(filepath.Separator)) {
|
||||
continue
|
||||
}
|
||||
raw, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for n, l := range strings.Split(string(raw), "\n") {
|
||||
if repository.MatchString(l) {
|
||||
rel, _ := filepath.Rel(root, f)
|
||||
out = append(out, fmt.Sprintf("%s:%d: requires the controller's repository: it is archived once its last module is built elsewhere, so nothing here is built from it (novox/hq ADR 0593 §11)", filepath.ToSlash(rel), n+1))
|
||||
}
|
||||
}
|
||||
}
|
||||
buckets, err := bucketLines(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, buckets...)
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// names collects the strings of a decoded manifest that name the controller; inConfig is under the module's `config`.
|
||||
func names(v any, inConfig bool, into []string) []string {
|
||||
switch v := v.(type) {
|
||||
case string:
|
||||
if address.MatchString(v) || (!inConfig && strings.TrimSpace(v) == Module) {
|
||||
into = append(into, v)
|
||||
}
|
||||
case []any:
|
||||
for _, e := range v {
|
||||
into = names(e, inConfig, into)
|
||||
}
|
||||
case map[string]any:
|
||||
keys := make([]string, 0, len(v))
|
||||
for k := range v {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
into = names(v[k], inConfig || k == "config", into)
|
||||
}
|
||||
}
|
||||
return into
|
||||
}
|
||||
|
||||
// bucketLines is every line of a module's file, outside tests and testdata, that names one of the controller's buckets.
|
||||
func bucketLines(root string) ([]string, error) {
|
||||
var out []string
|
||||
modules := filepath.Join(root, "modules")
|
||||
err := filepath.WalkDir(modules, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) && path == modules {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
if n := d.Name(); n == "testdata" || n == "node_modules" || n == ".git" {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !d.Type().IsRegular() || strings.HasSuffix(d.Name(), "_test.go") || strings.HasSuffix(d.Name(), ".test.ts") {
|
||||
return nil
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
sc := bufio.NewScanner(f)
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 16*1024*1024)
|
||||
rel, _ := filepath.Rel(root, path)
|
||||
for n := 1; sc.Scan(); n++ {
|
||||
if strings.Contains(sc.Text(), Bucket) {
|
||||
out = append(out, fmt.Sprintf("%s:%d: names the controller's bucket (%s…): its buckets are removed after its "+
|
||||
"end (novox/hq ADR 0590 §6); read the seat that holds the job now", filepath.ToSlash(rel), n, Bucket))
|
||||
}
|
||||
}
|
||||
return sc.Err()
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
@@ -1,64 +0,0 @@
|
||||
package controllerdeps
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// ADR 0590 §5 (issue 1126 B10): no manifest of the catalogue at this head reads, invokes, consumes, waits on, uses or
|
||||
// otherwise names the controller's seat, states or buckets, and no module's code names a bucket of the controller's.
|
||||
func TestNothingInTheCatalogueNamesTheController(t *testing.T) {
|
||||
got, err := Refusals("../..")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range got {
|
||||
t.Error(r)
|
||||
}
|
||||
}
|
||||
|
||||
// The check refuses every name of the controller, the lease and the entries once kept to its write among them, on a
|
||||
// tree of its own; a repository named mesh-controller (an issue's owner) and a word in prose are no dependency.
|
||||
func TestRefusalsJudgeATree(t *testing.T) {
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
files map[string]string
|
||||
want string // empty: passes
|
||||
}{
|
||||
{"the lease is refused like any read", map[string]string{"modules/a/module.json": `{"module":"a","reads":["mesh-controller.lease"]}`}, `a: names "mesh-controller.lease"`},
|
||||
{"a verb once kept to the write", map[string]string{"modules/b/module.json": `{"module":"b","invokes":["seat:mesh-controller.collect"]}`}, `b: names "seat:mesh-controller.collect"`},
|
||||
{"an event consumed", map[string]string{"modules/c/module.json": `{"module":"c","consumes":[{"event":"mesh-controller.artifacts-gone"}]}`}, "mesh-controller.artifacts-gone"},
|
||||
{"a requirement", map[string]string{"modules/d/module.json": `{"module":"d","requires":[{"name":"x","from":"mesh-controller"}]}`}, `d: names "mesh-controller"`},
|
||||
{"the controller's own module", map[string]string{"modules/mesh-controller/module.json": `{"module":"mesh-controller"}`}, "mesh-controller: the controller is no module of the catalogue"},
|
||||
{"a bucket in code", map[string]string{"modules/e/cmd/e/main.go": "package main\nconst b = \"mesh-controller_assignments\"\n"}, "modules/e/cmd/e/main.go:2: names the controller's bucket"},
|
||||
{"a requirement of the controller's repository", map[string]string{"modules/h/go.mod": "module h\n\nrequire github.com/novox/mesh-controller v0.0.0\n"}, "modules/h/go.mod:3: requires the controller's repository"},
|
||||
{"its forge path in a check", map[string]string{"checks/i/go.mod": "module i\n\nrequire (\n\tgit.novox.be/novox/mesh-controller/modules/route-proxy v0.1.0\n)\n"}, "checks/i/go.mod:4"},
|
||||
{"another repository of the forge passes", map[string]string{"modules/j/go.mod": "module j\n\nrequire git.novox.be/novox/mesh-sdk/go v0.1.0\n"}, ""},
|
||||
{"a bucket in a test is its data", map[string]string{"modules/e/cmd/e/main_test.go": "package main\n// mesh-controller_lease\n"}, ""},
|
||||
{"a fixture copied from elsewhere", map[string]string{"modules/e/cmd/e/testdata/x.md": "mesh-controller_rollouts\n"}, ""},
|
||||
{"an owner and prose pass", map[string]string{"modules/f/module.json": `{"module":"f","config":{"owners":{"default":["mesh-controller","mesh-controller modules/route-proxy"]}},"why":"the mesh-controller module is gone"}`}, ""},
|
||||
{"the seat's successors pass", map[string]string{"modules/g/module.json": `{"module":"g","reads":["mesh-module-assigner.assignments"],"invokes":["seat:mesh-hand-acts.hand-act"]}`}, ""},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for path, text := range row.files {
|
||||
p := filepath.Join(root, path)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(text), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := Refusals(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if joined := strings.Join(got, "\n"); (row.want == "") != (joined == "") || !strings.Contains(joined, row.want) {
|
||||
t.Errorf("refusals %q, want %q", joined, row.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
module checks/controllerdeps
|
||||
|
||||
go 1.22
|
||||
@@ -1,3 +0,0 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (issue 1936).
|
||||
# every module's directory: its manifest, for a name of the controller, and its files, for one of the controller's buckets.
|
||||
modules
|
||||
@@ -2,5 +2,4 @@ module checks/credentialform
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
|
||||
+2
-13
@@ -111,24 +111,13 @@ func forgeRequirements(dir string) ([]requirement, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("its go.mod cannot be read: %s", lastLine(out))
|
||||
}
|
||||
var mod struct {
|
||||
Require []requirement
|
||||
Replace []struct{ Old, New requirement }
|
||||
}
|
||||
var mod struct{ Require []requirement }
|
||||
if err := json.Unmarshal(out, &mod); err != nil {
|
||||
return nil, fmt.Errorf("its go.mod cannot be read: %w", err)
|
||||
}
|
||||
// A requirement the go.mod replaces to a directory is a package of the module's own repository (novox/hq ADR 0593 §1): it
|
||||
// is in the tree under check, so it is fetched from nowhere.
|
||||
local := map[string]bool{}
|
||||
for _, r := range mod.Replace {
|
||||
if r.New.Version == "" {
|
||||
local[r.Old.Path] = true
|
||||
}
|
||||
}
|
||||
var on []requirement
|
||||
for _, r := range mod.Require {
|
||||
if strings.HasPrefix(r.Path, forge) && !local[r.Path] {
|
||||
if strings.HasPrefix(r.Path, forge) {
|
||||
on = append(on, r)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -175,28 +175,3 @@ func TestTheRefOfAVersion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0593 §1: a module requires a sibling package of its own repository and replaces it to the directory beside it; that
|
||||
// package is no fetch from the forge (it is in the tree under check), so it is never looked for in a clone beside the check,
|
||||
// and the module builds with the SDK taken from there.
|
||||
func TestASiblingPackageReplacedToItsDirectoryIsNotFetched(t *testing.T) {
|
||||
offline(t)
|
||||
beside, commit := sdkBeside(t)
|
||||
version := "v0.1.14-0.20261012082113-" + commit[:12]
|
||||
root := t.TempDir()
|
||||
sibling := "git.novox.be/novox/mesh-catalog/modules/builder/runner"
|
||||
write(t, filepath.Join(root, "builder", "runner", "go.mod"), "module "+sibling+"\n\ngo 1.22\n\nrequire git.novox.be/novox/mesh-sdk/go "+version+"\n")
|
||||
write(t, filepath.Join(root, "builder", "runner", "run.go"), "package runner\n\nimport \"git.novox.be/novox/mesh-sdk/go/sdk\"\n\nfunc Run() string { return sdk.Hello() }\n")
|
||||
dir := filepath.Join(root, "user")
|
||||
write(t, filepath.Join(dir, "go.mod"), "module git.novox.be/novox/mesh-catalog/modules/user\n\ngo 1.22\n\nrequire (\n\t"+sibling+" v0.0.0\n"+
|
||||
"\tgit.novox.be/novox/mesh-sdk/go "+version+"\n)\n\nreplace "+sibling+" => ../builder/runner\n")
|
||||
write(t, filepath.Join(dir, "main.go"), "package main\n\nimport \""+sibling+"\"\n\nfunc main() { println(runner.Run()) }\n")
|
||||
modfile, err := resolve(dir, beside, t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("resolve: %v", err)
|
||||
}
|
||||
if modfile == "" {
|
||||
t.Fatal("resolve said the module fetches its dependencies, with nothing to fetch the SDK from")
|
||||
}
|
||||
builds(t, dir, modfile)
|
||||
}
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/environment
|
||||
|
||||
go 1.22
|
||||
@@ -1,252 +0,0 @@
|
||||
// Command environment refuses a module's code reading a MESH_* word of its environment that nothing gives it: not its
|
||||
// manifest, and not the node-engine or the tool runner for every process (the lead's ruling on A3n, issue 1966).
|
||||
//
|
||||
// A MESH_* word is the mesh's contract with a module's process: the manifest sets it (an `env` of a build artifact or a
|
||||
// resource, an `environment`'s `variables`, a `NAME=` line of a file the module ships for an env-file), or the mesh gives
|
||||
// it to every process (Given). A word read and set by nothing is a contract nobody keeps: the program runs on its
|
||||
// default or on nothing, and says so to nobody (MESH_REGISTRY_REACH answered every published reference without its
|
||||
// registry; MESH_CONFIG_CALLERS held every window shut). Found here, it is refused: the code reads what the mesh gives
|
||||
// (a manifest word, a config value through its tools, a seat's name), or the manifest sets it.
|
||||
//
|
||||
// A read is `os.Getenv` or `os.LookupEnv` of a literal or of a constant of the same directory (Go), and `env.MESH_*` or
|
||||
// `env["MESH_*"]` (TypeScript, `process.env` included); tests, test directories, node_modules and dist are not read. The
|
||||
// modules still reading such a word are listed in the file `unset` beside this check, `<module> <WORD>` a line, until
|
||||
// each is fixed: a line no longer found is refused, so the list only shrinks.
|
||||
//
|
||||
// go run . <catalogue root>
|
||||
//
|
||||
// Exits 1 on a finding.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Given is every MESH_* word the mesh gives a module's processes whatever their manifest says, and who gives it.
|
||||
var Given = map[string]string{
|
||||
// The tool runner, to every bundle and to a module's own runtime (mesh-tools node-tools runtime.BundleEnv).
|
||||
"MESH_MODULE": "the tool runner",
|
||||
"MESH_SERVED_MODULE": "the tool runner",
|
||||
"MESH_NODE": "the tool runner",
|
||||
// The node-engine, to the tool runner it runs as the operator's account; its bundles inherit them (mesh-host
|
||||
// compose/resolution runtime.go).
|
||||
"MESH_OPERATOR_ACCOUNT": "the node-engine",
|
||||
"MESH_OPERATOR_HOME": "the node-engine",
|
||||
// The node-engine, to every module that yields to the node's builds (ADR 0553 §3; catalogue.BuildCPUShareWord).
|
||||
"MESH_BUILD_CPU_SHARE": "the node-engine",
|
||||
}
|
||||
|
||||
const listFile = "unset"
|
||||
|
||||
var (
|
||||
goRead = regexp.MustCompile(`os\.(?:Getenv|LookupEnv)\(\s*(?:"(MESH_[A-Z0-9_]+)"|([A-Za-z_][A-Za-z0-9_]*))\s*\)`)
|
||||
goConst = regexp.MustCompile(`\b([A-Za-z_][A-Za-z0-9_]*)\s*(?:string\s*)?=\s*"(MESH_[A-Z0-9_]+)"`)
|
||||
tsRead = regexp.MustCompile(`\benv(?:\.|\[\s*["'])(MESH_[A-Z0-9_]+)`)
|
||||
envLine = regexp.MustCompile(`(?m)^\s*(?:export\s+)?(MESH_[A-Z0-9_]+)=`)
|
||||
)
|
||||
|
||||
// skipped are the directories whose code no process of the module runs.
|
||||
var skipped = map[string]bool{"node_modules": true, "dist": true, "test": true, "testdata": true, ".git": true}
|
||||
|
||||
// Check answers, per module, each MESH_* word its code reads that its manifest does not set and the mesh does not
|
||||
// give, with the files reading it.
|
||||
func Check(root string) (map[string]map[string][]string, error) {
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(manifests)
|
||||
out := map[string]map[string][]string{}
|
||||
for _, path := range manifests {
|
||||
dir := filepath.Dir(path)
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var manifest any
|
||||
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
set := map[string]bool{}
|
||||
setBy(manifest, set)
|
||||
reads := map[string]map[string]bool{}
|
||||
consts := map[string]map[string]string{} // directory -> constant -> word
|
||||
var goFiles []string
|
||||
err = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
if p != dir && skipped[d.Name()] {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
name := d.Name()
|
||||
text, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel, _ := filepath.Rel(dir, p)
|
||||
rel = filepath.ToSlash(rel)
|
||||
switch {
|
||||
case strings.HasSuffix(name, "_test.go"), strings.HasSuffix(name, ".test.ts"), strings.HasSuffix(name, ".d.ts"):
|
||||
case strings.HasSuffix(name, ".go"):
|
||||
goFiles = append(goFiles, p)
|
||||
c := consts[filepath.Dir(p)]
|
||||
if c == nil {
|
||||
c = map[string]string{}
|
||||
consts[filepath.Dir(p)] = c
|
||||
}
|
||||
for _, m := range goConst.FindAllStringSubmatch(string(text), -1) {
|
||||
c[m[1]] = m[2]
|
||||
}
|
||||
case strings.HasSuffix(name, ".ts"):
|
||||
for _, m := range tsRead.FindAllStringSubmatch(string(text), -1) {
|
||||
read(reads, m[1], rel)
|
||||
}
|
||||
case name != "module.json":
|
||||
// A file the module ships: a NAME= line is a word an env-file gives.
|
||||
for _, m := range envLine.FindAllStringSubmatch(string(text), -1) {
|
||||
set[m[1]] = true
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, p := range goFiles {
|
||||
text, _ := os.ReadFile(p)
|
||||
rel, _ := filepath.Rel(dir, p)
|
||||
for _, m := range goRead.FindAllStringSubmatch(string(text), -1) {
|
||||
word := m[1]
|
||||
if word == "" {
|
||||
word = consts[filepath.Dir(p)][m[2]]
|
||||
}
|
||||
if word != "" {
|
||||
read(reads, word, filepath.ToSlash(rel))
|
||||
}
|
||||
}
|
||||
}
|
||||
module := filepath.Base(dir)
|
||||
for word, files := range reads {
|
||||
if set[word] || Given[word] != "" {
|
||||
continue
|
||||
}
|
||||
if out[module] == nil {
|
||||
out[module] = map[string][]string{}
|
||||
}
|
||||
for f := range files {
|
||||
out[module][word] = append(out[module][word], f)
|
||||
}
|
||||
sort.Strings(out[module][word])
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func read(reads map[string]map[string]bool, word, file string) {
|
||||
if reads[word] == nil {
|
||||
reads[word] = map[string]bool{}
|
||||
}
|
||||
reads[word][file] = true
|
||||
}
|
||||
|
||||
// setBy collects the words a manifest sets: every key of an `env` object and of an `environment`'s `variables`.
|
||||
func setBy(v any, set map[string]bool) {
|
||||
switch v := v.(type) {
|
||||
case map[string]any:
|
||||
for k, e := range v {
|
||||
if m, ok := e.(map[string]any); ok && (k == "env" || k == "variables") {
|
||||
for word := range m {
|
||||
set[word] = true
|
||||
}
|
||||
}
|
||||
setBy(e, set)
|
||||
}
|
||||
case []any:
|
||||
for _, e := range v {
|
||||
setBy(e, set)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Listed is the list's `<module> <WORD>` lines; a line starting with # and a blank line are not entries.
|
||||
func Listed(path string) (map[string]bool, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
out := map[string]bool{}
|
||||
s := bufio.NewScanner(f)
|
||||
for s.Scan() {
|
||||
line := strings.Join(strings.Fields(s.Text()), " ")
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
out[line] = true
|
||||
}
|
||||
return out, s.Err()
|
||||
}
|
||||
|
||||
// Judge answers the reads not on the list (refused) and the list's lines no read stands for (stale).
|
||||
func Judge(found map[string]map[string][]string, listed map[string]bool) (refused, stale []string) {
|
||||
seen := map[string]bool{}
|
||||
for module, words := range found {
|
||||
for word := range words {
|
||||
line := module + " " + word
|
||||
seen[line] = true
|
||||
if !listed[line] {
|
||||
refused = append(refused, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
for line := range listed {
|
||||
if !seen[line] {
|
||||
stale = append(stale, line)
|
||||
}
|
||||
}
|
||||
sort.Strings(refused)
|
||||
sort.Strings(stale)
|
||||
return refused, stale
|
||||
}
|
||||
|
||||
func main() {
|
||||
root := "."
|
||||
if len(os.Args) > 1 {
|
||||
root = os.Args[1]
|
||||
}
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
listed, err := Listed(listFile)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
refused, stale := Judge(found, listed)
|
||||
for _, line := range refused {
|
||||
module, word, _ := strings.Cut(line, " ")
|
||||
fmt.Printf("%s reads %s (%s), which its manifest does not set and the mesh gives no process: set it in the "+
|
||||
"manifest, or read what the mesh gives\n", module, word, strings.Join(found[module][word], ", "))
|
||||
}
|
||||
for _, line := range stale {
|
||||
fmt.Printf("checks/environment/%s names %q, which no code reads unset any more: take it off the list\n", listFile, line)
|
||||
}
|
||||
if len(refused) > 0 || len(stale) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Printf("environment: every MESH_* word a module reads is set by its manifest or given by the mesh (%d still listed)\n", len(listed))
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// firstCount is how many lines the list held when it was first written: it only shrinks.
|
||||
const firstCount = 63
|
||||
|
||||
func module(t *testing.T, root, name, manifest string, files map[string]string) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(root, "modules", name)
|
||||
for path, text := range map[string]string{"module.json": manifest} {
|
||||
files[path] = text
|
||||
}
|
||||
for path, text := range files {
|
||||
full := filepath.Join(dir, filepath.FromSlash(path))
|
||||
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(full, []byte(text), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A word read and set by nothing is found, in Go (a literal, a constant of the same directory) and in TypeScript; one
|
||||
// the manifest sets (an env, an environment's variables, an env-file line it ships) or the mesh gives every process is
|
||||
// not; tests, node_modules and dist are not read.
|
||||
func TestAWordReadThatNothingSetsIsFound(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
module(t, root, "reader", `{"module":"reader","build":{"artifacts":[{"name":"tools","env":{"MESH_SET":"x"}}]},`+
|
||||
`"resources":[{"id":"p","type":"process","environment":{"variables":{"MESH_VARIABLE":"y"}}}]}`, map[string]string{
|
||||
"cmd/a/main.go": `package main; const reach = "MESH_REACH"; var _ = os.Getenv(reach); var _ = os.Getenv("MESH_SET")`,
|
||||
"cmd/a/other.go": `package main; var _, _ = os.LookupEnv("MESH_LITERAL"); var _ = os.Getenv("MESH_MODULE")`,
|
||||
"cmd/a/write.go": `package main; var _ = []string{"MESH_WRITTEN_FOR_A_CHILD=1"}`,
|
||||
"cmd/a/a_test.go": `package main; var _ = os.Getenv("MESH_IN_A_TEST")`,
|
||||
"client.ts": `const a = process.env.MESH_TS ?? "d"; const b = env["MESH_BRACKET"]; const c = env.MESH_VARIABLE;`,
|
||||
"files/env": "MESH_FROM_FILE=${config:x}\n",
|
||||
"index.ts": `const f = env.MESH_FROM_FILE;`,
|
||||
"test/x.test.ts": `env.MESH_IN_A_TEST_DIR`,
|
||||
"node_modules/x.ts": `env.MESH_VENDORED`,
|
||||
})
|
||||
module(t, root, "quiet", `{"module":"quiet"}`, map[string]string{"main.go": `package main; var _ = os.Getenv("MESH_NODE")`})
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]map[string][]string{"reader": {
|
||||
"MESH_REACH": {"cmd/a/main.go"},
|
||||
"MESH_LITERAL": {"cmd/a/other.go"},
|
||||
"MESH_TS": {"client.ts"},
|
||||
"MESH_BRACKET": {"client.ts"},
|
||||
}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("found %v, want %v", got, want)
|
||||
}
|
||||
refused, stale := Judge(got, map[string]bool{"reader MESH_REACH": true, "reader MESH_GONE": true})
|
||||
if !reflect.DeepEqual(refused, []string{"reader MESH_BRACKET", "reader MESH_LITERAL", "reader MESH_TS"}) ||
|
||||
!reflect.DeepEqual(stale, []string{"reader MESH_GONE"}) {
|
||||
t.Errorf("refused %v, stale %v", refused, stale)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue itself: every word a module reads is set or given, but the list's, and the list only shrinks.
|
||||
func TestTheCatalogueReadsOnlyWordsTheMeshSets(t *testing.T) {
|
||||
found, err := Check(filepath.Join("..", ".."))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
listed, err := Listed(listFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
refused, stale := Judge(found, listed)
|
||||
for _, line := range refused {
|
||||
t.Errorf("%s: read, set by nothing, and not on checks/environment/unset", line)
|
||||
}
|
||||
for _, line := range stale {
|
||||
t.Errorf("checks/environment/unset names %q, which no code reads unset any more", line)
|
||||
}
|
||||
if len(listed) > firstCount {
|
||||
t.Errorf("checks/environment/unset holds %d lines, more than its first %d: it only shrinks", len(listed), firstCount)
|
||||
}
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# Check("../.."): every manifest and the code and files of its module.
|
||||
modules
|
||||
@@ -1,67 +0,0 @@
|
||||
# The MESH_* words a module's code still reads that its manifest does not set and the mesh gives no process,
|
||||
# `<module> <WORD>` a line, as checks/environment first found them (issue 1966). Each is a contract nobody keeps: fix
|
||||
# it (the manifest sets it, or the code reads what the mesh gives) and take its line off. A line no code reads unset any
|
||||
# more is refused, so this list only shrinks; a new read set by nothing is refused, never added here.
|
||||
baserow MESH_BASEROW_EMAIL
|
||||
baserow MESH_BASEROW_HOST
|
||||
baserow MESH_BASEROW_PASSWORD
|
||||
claude-code MESH_AGENT_ACCOUNT
|
||||
claude-code MESH_AGENT_HOME
|
||||
claude-licence-manager MESH_ANTHROPIC_TOKEN_ENDPOINT
|
||||
claude-licence-manager MESH_ANTHROPIC_USAGE_ENDPOINT
|
||||
confluence MESH_CONFLUENCE_EMAIL
|
||||
confluence MESH_CONFLUENCE_TOKEN
|
||||
confluence MESH_CONFLUENCE_URL
|
||||
disk-load MESH_DISK_MEASURE_EVERY
|
||||
gitlab MESH_GITLAB_TOKEN
|
||||
gitlab MESH_GITLAB_URL
|
||||
grafana MESH_GRAFANA_PASSWORD
|
||||
grafana MESH_GRAFANA_TOKEN
|
||||
grafana MESH_GRAFANA_USER
|
||||
home-assistant MESH_HOMEASSISTANT_TOKEN
|
||||
home-assistant MESH_HOMEASSISTANT_WAIT_SECONDS
|
||||
home-assistant MESH_HOMEASSISTANT_WATCH
|
||||
icecast MESH_ICECAST_ADMIN_PASSWORD
|
||||
icecast MESH_ICECAST_ADMIN_USER
|
||||
influxdb MESH_INFLUXDB_ORG
|
||||
influxdb MESH_INFLUXDB_TOKEN
|
||||
jira MESH_JIRA_EMAIL
|
||||
jira MESH_JIRA_TOKEN
|
||||
jira MESH_JIRA_URL
|
||||
letta MESH_LETTA_PASSWORD
|
||||
mailu MESH_MAILU_API_KEY
|
||||
mailu MESH_MAILU_DOMAIN
|
||||
mesh-cli MESH_CONSOLE
|
||||
mesh-cli MESH_VERB
|
||||
mesh-files MESH_OWNED_PATHS
|
||||
mesh-files MESH_SECRET_PLACES
|
||||
mesh-reads MESH_SECRET_PLACES
|
||||
minio MESH_MINIO_ROOT_PASSWORD
|
||||
mongodb MESH_MONGODB_AUTHSOURCE
|
||||
mongodb MESH_MONGODB_HOST
|
||||
mongodb MESH_MONGODB_PASSWORD
|
||||
mongodb MESH_MONGODB_PORT
|
||||
mongodb MESH_MONGODB_USER
|
||||
mosquitto MESH_MQTT_ADMIN_USER
|
||||
mosquitto MESH_MQTT_BROKER_GID
|
||||
mosquitto MESH_MQTT_BROKER_UID
|
||||
mosquitto MESH_MQTT_HOST
|
||||
mosquitto MESH_MQTT_PASSWORD
|
||||
mosquitto MESH_MQTT_PORT
|
||||
mssql MESH_MSSQL_HOST
|
||||
mssql MESH_MSSQL_PASSWORD
|
||||
mssql MESH_MSSQL_PORT
|
||||
mssql MESH_MSSQL_READER_PASSWORD
|
||||
mssql MESH_MSSQL_USER
|
||||
nextcloud MESH_NEXTCLOUD_ADMIN_PASSWORD
|
||||
nextcloud MESH_NEXTCLOUD_CONTAINER
|
||||
nodered MESH_NODERED_TOKEN
|
||||
nodered MESH_NODERED_WAIT_SECONDS
|
||||
redis MESH_REDIS_HOST
|
||||
redis MESH_REDIS_PASSWORD
|
||||
redis MESH_REDIS_PORT
|
||||
redis MESH_REDIS_USERNAME
|
||||
restic MESH_BACKUP_HOUR
|
||||
unifi MESH_UNIFI_PASSWORD
|
||||
unifi MESH_UNIFI_SITE
|
||||
unifi MESH_UNIFI_USERNAME
|
||||
@@ -1,4 +1,4 @@
|
||||
// Package failurekit refuses a build module whose tests do not run the build runner's conformance kit (issue 1901, its thorough
|
||||
// Package failurekit refuses a build module whose tests do not run mesh-sdk's conformance kit (issue 1901, its thorough
|
||||
// fix): a module that claims a node-build-* seat and serves a test step must hold a test that calls conformance.Run, so
|
||||
// its parse of the test runner's output is held to the failure contract (a failing test named with its cause, never a
|
||||
// passing line in its place). A build module with no test step (mesh-build-image builds images and runs no tests) is
|
||||
@@ -71,7 +71,7 @@ func Judge(root string) ([]Judged, error) {
|
||||
}
|
||||
if !runs {
|
||||
refused = append(refused, fmt.Sprintf("it claims %s and serves the test step %s, and no test of it calls conformance.Run "+
|
||||
"(mesh-build-go buildstep/conformance): run the kit with the module's own parse path (issue 1901)", seat, test))
|
||||
"(mesh-sdk go/buildstep/conformance): run the kit with the module's own parse path (issue 1901)", seat, test))
|
||||
}
|
||||
}
|
||||
// Every build module keeps steps, so every one runs the left-running case, test step or not (issue 1890).
|
||||
@@ -81,7 +81,7 @@ func Judge(root string) ([]Judged, error) {
|
||||
}
|
||||
if !sweeps {
|
||||
refused = append(refused, fmt.Sprintf("it claims %s, and no test of it calls conformance.LeftRunning "+
|
||||
"(mesh-build-go buildstep/conformance): run the kit's left-running case with the module's own start sweep (issue 1890)", seat))
|
||||
"(mesh-sdk go/buildstep/conformance): run the kit's left-running case with the module's own start sweep (issue 1890)", seat))
|
||||
}
|
||||
j.Refused = strings.Join(refused, "; ")
|
||||
out = append(out, j)
|
||||
@@ -121,7 +121,7 @@ func runsTheKit(dir, call string) (bool, error) {
|
||||
return err
|
||||
}
|
||||
found = strings.Contains(string(raw), call) &&
|
||||
strings.Contains(string(raw), `"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep/conformance"`)
|
||||
strings.Contains(string(raw), `"git.novox.be/novox/mesh-sdk/go/buildstep/conformance"`)
|
||||
return nil
|
||||
})
|
||||
return found, err
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
// cases is refused, and a module that claims no node-build-* seat is not judged.
|
||||
func TestABuildModuleWithATestStepMustRunTheKit(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
const imp = "package main\n\nimport \"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep/conformance\"\n\n"
|
||||
const imp = "package main\n\nimport \"git.novox.be/novox/mesh-sdk/go/buildstep/conformance\"\n\n"
|
||||
const sweep = "func TestSweep(t *testing.T) { conformance.LeftRunning(t, nil) }\n"
|
||||
kit := imp + "func TestKit(t *testing.T) { conformance.Run(t, conformance.GoTest, nil) }\n" + sweep
|
||||
for path, body := range map[string]string{
|
||||
|
||||
@@ -1,292 +0,0 @@
|
||||
// Package gitignored refuses an ignore pattern that hides a module's source (catalogue review F1, and its kind): the
|
||||
// bare pattern `mesh-build-node` in a module's .gitignore, meant for the binary `go build` leaves, also matched the
|
||||
// directory cmd/mesh-build-node, so git ignored every new file there and publish.go was never committed. A path under
|
||||
// a module's cmd/ or internal/ is source: no .gitignore, the module's or the catalogue's, may ignore one, tracked or
|
||||
// new. A binary is ignored by an anchored pattern (`/mesh-build-node`, `/cmd/mesh-build-node/mesh-build-node`).
|
||||
package gitignored
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Probe is the file name asked of each source directory: a Go file a change might add there.
|
||||
const Probe = "added_by_a_change.go"
|
||||
|
||||
// Hidden is one source path an ignore pattern hides, and the pattern (its file, line and text, as git names it).
|
||||
type Hidden struct {
|
||||
Path, Pattern string
|
||||
}
|
||||
|
||||
// Sources are the source directories of root's modules, relative to root: every directory under modules/*/cmd and
|
||||
// modules/*/internal (node_modules left out).
|
||||
func Sources(root string) ([]string, error) {
|
||||
mods, err := filepath.Glob(filepath.Join(root, "modules", "*"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for _, m := range mods {
|
||||
for _, top := range []string{"cmd", "internal"} {
|
||||
base := filepath.Join(m, top)
|
||||
if st, err := os.Stat(base); err != nil || !st.IsDir() {
|
||||
continue
|
||||
}
|
||||
err := filepath.WalkDir(base, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if d.Name() == "node_modules" {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
rel, _ := filepath.Rel(root, p)
|
||||
out = append(out, filepath.ToSlash(rel))
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// isSource is whether a file is source a module's build reads: Go and TypeScript files and a Go module's own files.
|
||||
// What a build or a test script leaves beside them (a binary, compiled JavaScript) may be ignored.
|
||||
func isSource(name string) bool {
|
||||
switch filepath.Ext(name) {
|
||||
case ".go", ".ts", ".tsx":
|
||||
return true
|
||||
}
|
||||
return name == "go.mod" || name == "go.sum"
|
||||
}
|
||||
|
||||
// Find asks git which of the source paths root's ignore files hide: each source directory, a Go file added in it, and
|
||||
// every source file there (isSource). It reads the .gitignore files of root's tree through a repository of its own, so it judges a git
|
||||
// checkout and a plain folder alike, and no one's global ignores count.
|
||||
func Find(root string, dirs []string) ([]Hidden, error) {
|
||||
gitDir, err := os.MkdirTemp("", "gitignored-")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.RemoveAll(gitDir)
|
||||
if out, err := exec.Command("git", "init", "-q", "--bare", gitDir).CombinedOutput(); err != nil {
|
||||
return nil, fmt.Errorf("git init: %v: %s", err, out)
|
||||
}
|
||||
var ask bytes.Buffer
|
||||
for _, d := range dirs {
|
||||
ask.WriteString(d + "/\x00" + d + "/" + Probe + "\x00")
|
||||
entries, err := os.ReadDir(filepath.Join(root, d))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() && isSource(e.Name()) {
|
||||
ask.WriteString(d + "/" + e.Name() + "\x00")
|
||||
}
|
||||
}
|
||||
}
|
||||
abs, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cmd := exec.Command("git", "--git-dir="+gitDir, "--work-tree="+abs, "-c", "core.excludesFile=/dev/null",
|
||||
"check-ignore", "--no-index", "-v", "-z", "--stdin")
|
||||
cmd.Dir = abs
|
||||
cmd.Stdin = &ask
|
||||
out, err := cmd.Output()
|
||||
if ee, ok := err.(*exec.ExitError); ok && ee.ExitCode() == 1 {
|
||||
return nil, nil // nothing is ignored
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("git check-ignore: %w", err)
|
||||
}
|
||||
// -v -z answers source, line, pattern and path, each ended by NUL.
|
||||
f := strings.Split(strings.TrimRight(string(out), "\x00"), "\x00")
|
||||
var hidden []Hidden
|
||||
for i := 0; i+3 < len(f); i += 4 {
|
||||
if f[i+2] == "" || strings.HasPrefix(f[i+2], "!") {
|
||||
continue // a negation keeps the path
|
||||
}
|
||||
hidden = append(hidden, Hidden{Path: f[i+3], Pattern: fmt.Sprintf("%s:%s: %s", f[i], f[i+1], f[i+2])})
|
||||
}
|
||||
return hidden, nil
|
||||
}
|
||||
|
||||
// Programs are the paths `go build` writes a compiled program to, for every main package of every Go module under
|
||||
// root (issue 1577, met twice in one milestone: a module's program committed with `git add -A`): `go build` in the
|
||||
// package's directory writes it there, and `go build ./<dir>` from the module's root writes it at the root, each named
|
||||
// after the package's import path. Relative to root, sorted; directories named node_modules, testdata or vendor, and
|
||||
// .git, are left out, and so is a path that is a directory of the tree.
|
||||
func Programs(root string) ([]string, error) {
|
||||
var mods []string
|
||||
err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() && skipped(d.Name()) && p != root {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if !d.IsDir() && d.Name() == "go.mod" {
|
||||
mods = append(mods, filepath.Dir(p))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, m := range mods {
|
||||
modPath, err := modulePath(filepath.Join(m, "go.mod"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = filepath.WalkDir(m, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if p != m {
|
||||
if skipped(d.Name()) {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(p, "go.mod")); err == nil {
|
||||
return filepath.SkipDir // a module of its own, walked on its own
|
||||
}
|
||||
}
|
||||
isMain, err := mainPackage(p)
|
||||
if err != nil || !isMain {
|
||||
return err
|
||||
}
|
||||
name := filepath.Base(p)
|
||||
if p == m {
|
||||
name = pathBase(modPath)
|
||||
}
|
||||
for _, dir := range []string{p, m} {
|
||||
out := filepath.Join(dir, name)
|
||||
if st, err := os.Stat(out); err == nil && st.IsDir() {
|
||||
continue // go build refuses to write over a directory, so no program lands there
|
||||
}
|
||||
rel, _ := filepath.Rel(root, out)
|
||||
seen[filepath.ToSlash(rel)] = true
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(seen))
|
||||
for p := range seen {
|
||||
out = append(out, p)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Unignored names each of programs (paths from root) that root's ignore files do not ignore: a program a local build
|
||||
// leaves there that `git add -A` would commit.
|
||||
func Unignored(root string, programs []string) ([]string, error) {
|
||||
gitDir, err := os.MkdirTemp("", "gitignored-")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.RemoveAll(gitDir)
|
||||
if out, err := exec.Command("git", "init", "-q", "--bare", gitDir).CombinedOutput(); err != nil {
|
||||
return nil, fmt.Errorf("git init: %v: %s", err, out)
|
||||
}
|
||||
abs, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cmd := exec.Command("git", "--git-dir="+gitDir, "--work-tree="+abs, "-c", "core.excludesFile=/dev/null",
|
||||
"check-ignore", "--no-index", "-v", "-n", "-z", "--stdin")
|
||||
cmd.Dir = abs
|
||||
cmd.Stdin = strings.NewReader(strings.Join(programs, "\x00") + "\x00")
|
||||
out, err := cmd.Output()
|
||||
if ee, ok := err.(*exec.ExitError); !(err == nil || ok && ee.ExitCode() == 1) {
|
||||
return nil, fmt.Errorf("git check-ignore: %w", err)
|
||||
}
|
||||
// -v -n -z answers source, line, pattern and path for every path asked; the source is empty for one no pattern
|
||||
// matches, and a pattern starting with ! keeps the path.
|
||||
f := strings.Split(strings.TrimRight(string(out), "\x00"), "\x00")
|
||||
var open []string
|
||||
for i := 0; i+3 < len(f); i += 4 {
|
||||
if f[i+2] == "" || strings.HasPrefix(f[i+2], "!") {
|
||||
open = append(open, f[i+3])
|
||||
}
|
||||
}
|
||||
return open, nil
|
||||
}
|
||||
|
||||
func skipped(name string) bool {
|
||||
switch name {
|
||||
case ".git", "node_modules", "testdata", "vendor":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// modulePath reads the module line of a go.mod.
|
||||
func modulePath(goMod string) (string, error) {
|
||||
b, err := os.ReadFile(goMod)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, l := range strings.Split(string(b), "\n") {
|
||||
if f := strings.Fields(l); len(f) >= 2 && f[0] == "module" {
|
||||
return strings.Trim(f[1], `"`), nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("%s names no module", goMod)
|
||||
}
|
||||
|
||||
// pathBase is the name `go build` gives a program at a module's root: its path's last element, a major version
|
||||
// suffix (/v2) left out.
|
||||
func pathBase(modPath string) string {
|
||||
parts := strings.Split(modPath, "/")
|
||||
last := parts[len(parts)-1]
|
||||
if len(parts) > 1 && len(last) > 1 && last[0] == 'v' && strings.Trim(last[1:], "0123456789") == "" {
|
||||
last = parts[len(parts)-2]
|
||||
}
|
||||
return last
|
||||
}
|
||||
|
||||
// mainPackage is whether dir holds a Go file, not a test, of package main.
|
||||
func mainPackage(dir string) (bool, error) {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
n := e.Name()
|
||||
if e.IsDir() || filepath.Ext(n) != ".go" || strings.HasSuffix(n, "_test.go") {
|
||||
continue
|
||||
}
|
||||
b, err := os.ReadFile(filepath.Join(dir, n))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, l := range strings.Split(string(b), "\n") {
|
||||
if f := strings.Fields(l); len(f) >= 2 && f[0] == "package" {
|
||||
if f[1] == "main" {
|
||||
return true, nil
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
@@ -1,158 +0,0 @@
|
||||
package gitignored
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The kind of catalogue review F1: a bare pattern hides the command directory it shares a name with; an anchored one
|
||||
// hides the binary alone; a glob that ignores source is refused too; a negation keeps a path.
|
||||
func TestFindNamesEachPatternThatHidesSource(t *testing.T) {
|
||||
for name, c := range map[string]struct {
|
||||
ignore string
|
||||
hidden []string
|
||||
}{
|
||||
"a bare name hides cmd/<name>": {"nfs-server\n", []string{"modules/m/cmd/nfs-server/", "modules/m/cmd/nfs-server/" + Probe, "modules/m/cmd/nfs-server/main.go"}},
|
||||
"anchored to the binary": {"/nfs-server\n/cmd/nfs-server/nfs-server\n", nil},
|
||||
"a glob ignoring Go source": {"*.go\n!main.go\n", []string{"modules/m/cmd/" + Probe, "modules/m/cmd/nfs-server/" + Probe,
|
||||
"modules/m/internal/" + Probe, "modules/m/internal/x/" + Probe, "modules/m/internal/x/x.go"}},
|
||||
"a pattern over internal/": {"internal/\n", []string{"modules/m/internal/", "modules/m/internal/" + Probe, "modules/m/internal/x/",
|
||||
"modules/m/internal/x/" + Probe, "modules/m/internal/x/x.go"}},
|
||||
"nothing but generated JavaScript": {"*.js\n", nil},
|
||||
} {
|
||||
root := t.TempDir()
|
||||
for f, b := range map[string]string{
|
||||
"modules/m/.gitignore": c.ignore,
|
||||
"modules/m/cmd/nfs-server/main.go": "package main\n",
|
||||
"modules/m/cmd/nfs-server/run.js": "",
|
||||
"modules/m/internal/x/x.go": "package x\n",
|
||||
"modules/m/node_modules/y/index.go": "",
|
||||
"modules/other/cmd/other/other.go": "package main\n",
|
||||
} {
|
||||
p := filepath.Join(root, f)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(b), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
dirs, err := Sources(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := Find(root, dirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var paths []string
|
||||
for _, h := range got {
|
||||
paths = append(paths, h.Path)
|
||||
if !strings.HasPrefix(h.Pattern, "modules/m/.gitignore:") {
|
||||
t.Errorf("%s: %s is named hidden by %q, want the module's .gitignore", name, h.Path, h.Pattern)
|
||||
}
|
||||
}
|
||||
if strings.Join(paths, ",") != strings.Join(c.hidden, ",") {
|
||||
t.Errorf("%s: hidden %v, want %v", name, paths, c.hidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue's own tree: no ignore pattern hides a path under a module's cmd/ or internal/.
|
||||
func TestNoIgnorePatternHidesAModulesSource(t *testing.T) {
|
||||
root := filepath.Join("..", "..")
|
||||
dirs, err := Sources(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(dirs) == 0 {
|
||||
t.Fatal("no module source directory found: the check reads nothing")
|
||||
}
|
||||
hidden, err := Find(root, dirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, h := range hidden {
|
||||
t.Errorf("%s is hidden by %s: a module's source is never ignored; anchor the pattern to the binary (/<name>, /cmd/<name>/<name>)", h.Path, h.Pattern)
|
||||
}
|
||||
}
|
||||
|
||||
// The kind of issue 1577, met twice in one milestone: a module's program `go build` left in its folder was committed.
|
||||
// Every program a main package builds, in its own directory and at its module's root, is found, and one no ignore file
|
||||
// names is open; a directory of the same name is no program's place.
|
||||
func TestEveryProgramABuildLeavesIsIgnored(t *testing.T) {
|
||||
files := map[string]string{
|
||||
"modules/m/go.mod": "module example.com/m/v2\n",
|
||||
"modules/m/main.go": "// a program\npackage main\n",
|
||||
"modules/m/cmd/m-tools/main.go": "package main\n",
|
||||
"modules/m/cmd/m-tools/main_test.go": "package main\n",
|
||||
"modules/m/internal/x/x.go": "package x\n",
|
||||
"modules/m/internal/x/x_test.go": "package main\n",
|
||||
"modules/m/node_modules/y/y.go": "package main\n",
|
||||
"modules/m/tool/go.mod": "module tool\n",
|
||||
"modules/m/tool/cmd/tool/main.go": "package main\n",
|
||||
"modules/m/tool/tool/source.go": "package tool\n",
|
||||
"checks/c/go.mod": "module example.com/checks/c\n",
|
||||
"checks/c/main.go": "package main\n",
|
||||
}
|
||||
for name, c := range map[string]struct {
|
||||
ignore string
|
||||
open []string
|
||||
}{
|
||||
"nothing ignored": {"", []string{"checks/c/c", "modules/m/cmd/m-tools/m-tools", "modules/m/m", "modules/m/m-tools",
|
||||
"modules/m/tool/cmd/tool/tool"}},
|
||||
"each anchored": {"/checks/c/c\n/modules/m/cmd/m-tools/m-tools\n/modules/m/m\n/modules/m/m-tools\n/modules/m/tool/cmd/tool/tool\n", nil},
|
||||
"one kept by a negation": {"/checks/c/c\n/modules/m/*\n!/modules/m/m\n/modules/m/cmd/m-tools/m-tools\n/modules/m/tool/cmd/tool/tool\n",
|
||||
[]string{"modules/m/m"}},
|
||||
} {
|
||||
root := t.TempDir()
|
||||
for f, b := range files {
|
||||
p := filepath.Join(root, f)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(b), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, ".gitignore"), []byte(c.ignore), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
programs, err := Programs(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := "checks/c/c,modules/m/cmd/m-tools/m-tools,modules/m/m,modules/m/m-tools,modules/m/tool/cmd/tool/tool"
|
||||
if got := strings.Join(programs, ","); got != want {
|
||||
t.Fatalf("%s: programs %s, want %s", name, got, want)
|
||||
}
|
||||
open, err := Unignored(root, programs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(open, ",") != strings.Join(c.open, ",") {
|
||||
t.Errorf("%s: open %v, want %v", name, open, c.open)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue's own tree: every program a local build leaves is ignored, so `git add -A` never takes one.
|
||||
func TestNoProgramABuildLeavesIsCommitted(t *testing.T) {
|
||||
root := filepath.Join("..", "..")
|
||||
programs, err := Programs(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(programs) == 0 {
|
||||
t.Fatal("no main package found: the check reads nothing")
|
||||
}
|
||||
open, err := Unignored(root, programs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, p := range open {
|
||||
t.Errorf("%s is the program a local go build leaves, and no ignore file names it: add /%s to the catalogue's .gitignore", p, p)
|
||||
}
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/gitignored
|
||||
|
||||
go 1.22
|
||||
@@ -1,3 +0,0 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/gofmt
|
||||
|
||||
go 1.22
|
||||
@@ -1,58 +0,0 @@
|
||||
// Package gofmt refuses a Go file of the catalogue that gofmt would change (the second of a kind in one milestone:
|
||||
// buildstep's generated.go and claude-code's config_test.go reached a join unformatted). A check here holds every
|
||||
// module to it, where a module's own tests hold only themselves.
|
||||
package gofmt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"go/format"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Files are the Go files under root, relative to it: every .go file but those in .git, node_modules and a testdata
|
||||
// directory (a test's fixture may be anything).
|
||||
func Files(root string) ([]string, error) {
|
||||
var out []string
|
||||
err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
switch d.Name() {
|
||||
case ".git", "node_modules", "testdata":
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if strings.HasSuffix(d.Name(), ".go") && d.Type().IsRegular() {
|
||||
rel, _ := filepath.Rel(root, p)
|
||||
out = append(out, filepath.ToSlash(rel))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
sort.Strings(out)
|
||||
return out, err
|
||||
}
|
||||
|
||||
// Unformatted are the files gofmt would change, and why for one that does not parse.
|
||||
func Unformatted(root string, files []string) (map[string]string, error) {
|
||||
out := map[string]string{}
|
||||
for _, f := range files {
|
||||
src, err := os.ReadFile(filepath.Join(root, f))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
got, err := format.Source(src)
|
||||
switch {
|
||||
case err != nil:
|
||||
out[f] = "it does not parse: " + err.Error()
|
||||
case !bytes.Equal(got, src):
|
||||
out[f] = "gofmt changes it: run gofmt -w " + f
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
package gofmt
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUnformattedNamesWhatGofmtChanges(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for f, src := range map[string]string{
|
||||
"modules/a/a.go": "package a\n",
|
||||
"modules/b/b.go": "package b\nimport (\n\t\"os\"\n\t\"fmt\"\n)\nvar _, _ = fmt.Sprint, os.Exit\n",
|
||||
"modules/c/testdata/c.go": "package c",
|
||||
"modules/d/node_modules/d.go": "package d",
|
||||
} {
|
||||
p := filepath.Join(root, f)
|
||||
_ = os.MkdirAll(filepath.Dir(p), 0o755)
|
||||
if err := os.WriteFile(p, []byte(src), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
files, err := Files(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := Unformatted(root, files)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(files) != 2 || len(got) != 1 || got["modules/b/b.go"] == "" {
|
||||
t.Errorf("files %v, unformatted %v; want a.go and b.go read, b.go named", files, got)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue's own Go files are as gofmt writes them.
|
||||
func TestTheCataloguesGoIsFormatted(t *testing.T) {
|
||||
root := filepath.Join("..", "..")
|
||||
files, err := Files(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(files) == 0 {
|
||||
t.Fatal("no Go file found: the check reads nothing")
|
||||
}
|
||||
got, err := Unformatted(root, files)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for f, why := range got {
|
||||
t.Errorf("%s: %s", f, why)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
# The calls of the controller's hand-act log a manifest of this catalogue may still invoke, each `<module> <verb> <why>`:
|
||||
# the fallback of a recorder (mesh-sdk handacts.Recorder) while the seat mesh-hand-acts has no holder that answers, until
|
||||
# the controller's lease is written to a holder that is no controller. The check of this directory (handacts) refuses any
|
||||
# other, and an entry whose module no longer invokes it: the list only shrinks, and it is empty before the lease write
|
||||
# (novox/hq ADR 0565 §11; issue 1953, issue 774).
|
||||
mesh-config hand-act fallback until the lease write: its sets, clears and held values record through mesh-hand-acts first
|
||||
mesh-config warranted fallback until the lease write: its warrants record through mesh-hand-acts first
|
||||
mesh-healers drill fallback until the lease write: a drill records through mesh-hand-acts first
|
||||
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/handacts
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,109 @@
|
||||
// Command handacts refuses a manifest that records into the controller's hand-act log (novox/hq ADR 0565 §11): every
|
||||
// recorder records through the seat mesh-hand-acts from its delivery, so no manifest invokes
|
||||
// `seat:mesh-controller.hand-act`, `.warranted` or `.drill`. The recorders' fallbacks to the controller's log while the
|
||||
// seat has no holder are named in checks/handacts/fallbacks, each module and verb with why; that list only shrinks, and
|
||||
// an entry whose module no longer invokes the verb is refused too.
|
||||
//
|
||||
// go run . <catalog root>
|
||||
//
|
||||
// Exits 1 on a finding.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Verbs are the controller's hand-act verbs, as a manifest invokes them, and Allowed the list of named fallbacks.
|
||||
var Verbs = []string{"hand-act", "warranted", "drill"}
|
||||
|
||||
const (
|
||||
invokePrefix = "seat:mesh-controller."
|
||||
Allowed = "checks/handacts/fallbacks"
|
||||
)
|
||||
|
||||
// Findings is every refusal in the catalog at root, sorted.
|
||||
func Findings(root string) ([]string, error) {
|
||||
allowed := map[string]bool{}
|
||||
raw, err := os.ReadFile(filepath.Join(root, Allowed))
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return nil, err
|
||||
}
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
if f := strings.Fields(line); len(f) >= 3 && !strings.HasPrefix(f[0], "#") {
|
||||
allowed[f[0]+" "+f[1]] = true
|
||||
} else if len(f) > 0 && !strings.HasPrefix(f[0], "#") {
|
||||
return nil, fmt.Errorf("%s: %q is not `<module> <verb> <why>`", Allowed, line)
|
||||
}
|
||||
}
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
invoked := map[string]bool{}
|
||||
for _, path := range manifests {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Invokes []string `json:"invokes"`
|
||||
}
|
||||
if json.Unmarshal(raw, &m) != nil || m.Module == "mesh-controller" {
|
||||
continue
|
||||
}
|
||||
for _, inv := range m.Invokes {
|
||||
verb, ok := strings.CutPrefix(strings.TrimSpace(inv), invokePrefix)
|
||||
if !ok || !contains(Verbs, verb) {
|
||||
continue
|
||||
}
|
||||
invoked[m.Module+" "+verb] = true
|
||||
if !allowed[m.Module+" "+verb] {
|
||||
out = append(out, fmt.Sprintf("%s invokes %s%s: record through seat:mesh-hand-acts.%s (novox/hq ADR 0565 §11), "+
|
||||
"or name the fallback in %s", m.Module, invokePrefix, verb, verb, Allowed))
|
||||
}
|
||||
}
|
||||
}
|
||||
for entry := range allowed {
|
||||
if !invoked[entry] {
|
||||
out = append(out, fmt.Sprintf("%s names %q, which is no longer invoked: take it off the list, which only shrinks",
|
||||
Allowed, entry))
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func contains(list []string, s string) bool {
|
||||
for _, x := range list {
|
||||
if x == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 2 {
|
||||
fmt.Fprintln(os.Stderr, "usage: handacts <catalog root>")
|
||||
os.Exit(2)
|
||||
}
|
||||
found, err := Findings(os.Args[1])
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
for _, f := range found {
|
||||
fmt.Println(f)
|
||||
}
|
||||
if len(found) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("no manifest records into the controller's hand-act log but the fallbacks named")
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// No manifest invokes the controller's hand-act verbs but a named fallback; a named fallback no longer invoked is
|
||||
// refused (novox/hq ADR 0565 §11's catalog test).
|
||||
func TestFindings(t *testing.T) {
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
files map[string]string
|
||||
want string // empty: clean
|
||||
}{
|
||||
{"clean", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-hand-acts.hand-act"]}`}, ""},
|
||||
{"an unnamed call", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.warranted"]}`},
|
||||
"a invokes seat:mesh-controller.warranted"},
|
||||
{"a named fallback, and the controller's own", map[string]string{
|
||||
"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.drill"]}`,
|
||||
"modules/mesh-controller/module.json": `{"module":"mesh-controller","invokes":["seat:mesh-controller.hand-act"]}`,
|
||||
Allowed: "# why\na drill fallback until the lease write\n"}, ""},
|
||||
{"a fallback no longer invoked", map[string]string{Allowed: "gone hand-act fallback\n"}, `names "gone hand-act"`},
|
||||
{"another verb of the controller", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.conditions"]}`}, ""},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for path, text := range row.files {
|
||||
p := filepath.Join(root, path)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(text), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := Findings(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if all := strings.Join(got, "\n"); (row.want == "") != (all == "") || !strings.Contains(all, row.want) {
|
||||
t.Fatalf("findings = %q, want one containing %q", all, row.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The run of this check over the real tree.
|
||||
func TestTheCatalogRecordsOnlyThroughTheSeat(t *testing.T) {
|
||||
got, err := Findings(filepath.Join("..", ".."))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range got {
|
||||
t.Error(f)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (issue 1936).
|
||||
# every manifest, for what it invokes.
|
||||
modules/*/module.json
|
||||
@@ -2,5 +2,4 @@ module checks/health
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
|
||||
@@ -2,5 +2,4 @@ module checks/manifests
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Package manifests reads every module's manifest as the catalogue's tree holds it and refuses what no judge could
|
||||
// read: a module.json that does not decode, a module.json giving one field twice, and two manifests that name one module.
|
||||
// read: a module.json that does not decode, and two manifests that name one module.
|
||||
//
|
||||
// It judges no manifest (novox/hq ADR 0578 §2): the checker judges every change's manifests with the indexer's judge, the
|
||||
// one judge, so the field, seat, bus and catalogue rules are said once, there. Until 2026-11 this check parsed each
|
||||
@@ -9,8 +9,6 @@
|
||||
package manifests
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -20,9 +18,7 @@ import (
|
||||
)
|
||||
|
||||
// Problems is what is wrong with the tree's manifests under root/modules, one line each: a module.json whose bytes are
|
||||
// no manifest (catalogue.Decode: not JSON, or a field of the wrong shape), one giving a field twice (a decoder keeps the
|
||||
// last and drops the first without a word: a merge of two branches that each added the field writes it so), then a module
|
||||
// named by two manifests.
|
||||
// no manifest (catalogue.Decode: not JSON, or a field of the wrong shape), then a module named by two manifests.
|
||||
func Problems(root string) ([]string, error) {
|
||||
files, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
@@ -42,10 +38,6 @@ func Problems(root string) ([]string, error) {
|
||||
out = append(out, fmt.Sprintf("%s: %v", rel, err))
|
||||
continue
|
||||
}
|
||||
if twice := givenTwice(raw); twice != "" {
|
||||
out = append(out, fmt.Sprintf("%s: the field %s is given twice; a judge reads only the last", rel, twice))
|
||||
continue
|
||||
}
|
||||
if named[m.Module] {
|
||||
out = append(out, fmt.Sprintf("%s: %s was already given; two manifests name one module", rel, m.Module))
|
||||
continue
|
||||
@@ -54,47 +46,3 @@ func Problems(root string) ([]string, error) {
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// givenTwice is the first field an object of raw gives twice, as its path (`invokes`, `build.artifacts[0].env`), or empty.
|
||||
func givenTwice(raw []byte) string {
|
||||
d := json.NewDecoder(bytes.NewReader(raw))
|
||||
var walk func(path string) string
|
||||
walk = func(path string) string {
|
||||
t, err := d.Token()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
switch t {
|
||||
case json.Delim('{'):
|
||||
seen := map[string]bool{}
|
||||
for d.More() {
|
||||
k, err := d.Token()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
key, _ := k.(string)
|
||||
at := key
|
||||
if path != "" {
|
||||
at = path + "." + key
|
||||
}
|
||||
if seen[key] {
|
||||
return at
|
||||
}
|
||||
seen[key] = true
|
||||
if twice := walk(at); twice != "" {
|
||||
return twice
|
||||
}
|
||||
}
|
||||
_, _ = d.Token()
|
||||
case json.Delim('['):
|
||||
for i := 0; d.More(); i++ {
|
||||
if twice := walk(fmt.Sprintf("%s[%d]", path, i)); twice != "" {
|
||||
return twice
|
||||
}
|
||||
}
|
||||
_, _ = d.Token()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
return walk("")
|
||||
}
|
||||
|
||||
@@ -5,12 +5,10 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// The tree's manifests are ones a judge can read, and no module is named twice. A row is a catalogue of its own: a
|
||||
// manifest that is no JSON, one giving a field twice and two manifests of one module are refused, naming the file; a manifest with a field this
|
||||
// manifest that is no JSON and two manifests of one module are refused, naming the file; a manifest with a field this
|
||||
// check does not know is not (it judges nothing: the checker does, ADR 0578 §2). The catalogue itself passes.
|
||||
func TestEveryManifestDecodesAndNamesOneModule(t *testing.T) {
|
||||
for _, row := range []struct {
|
||||
@@ -21,10 +19,6 @@ func TestEveryManifestDecodesAndNamesOneModule(t *testing.T) {
|
||||
{name: "a manifest the judge can read", files: map[string]string{"a": `{"module":"a","version":"1"}`}},
|
||||
{name: "a field no part knows is the judge's to refuse", files: map[string]string{"a": `{"module":"a","version":"1","unheard":true}`}},
|
||||
{name: "bytes that are no manifest", files: map[string]string{"a": `{"module":`}, refused: "modules/a/module.json"},
|
||||
{name: "a field given twice", files: map[string]string{"a": `{"module":"a","version":"1","invokes":["x.y"],"build":{"artifacts":[{}]},"invokes":["z.w"]}`},
|
||||
refused: "modules/a/module.json: the field invokes is given twice"},
|
||||
{name: "a field given twice deeper", files: map[string]string{"a": `{"module":"a","version":"1","build":{"artifacts":[{"name":"t","name":"u"}]}}`},
|
||||
refused: "the field build.artifacts[0].name is given twice"},
|
||||
{name: "two manifests of one module", files: map[string]string{"a": `{"module":"x","version":"1"}`, "b": `{"module":"x","version":"1"}`},
|
||||
refused: "two manifests name one module"},
|
||||
} {
|
||||
@@ -54,45 +48,3 @@ func TestEveryManifestDecodesAndNamesOneModule(t *testing.T) {
|
||||
t.Fatalf("the catalogue holds manifests no judge can read:\n%s", strings.Join(real, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
// No manifest of this catalogue claims the seat mesh-catalog: the indexer of the delivery holds it, and the module that
|
||||
// held it before has dropped its claim (ADR 0604 §6). A row is a tree of its own: a manifest claiming it is named.
|
||||
func TestNoManifestClaimsTheSeatMeshCatalog(t *testing.T) {
|
||||
claimants := func(root string) []string {
|
||||
files, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []string
|
||||
for _, f := range files {
|
||||
raw, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.Decode(raw)
|
||||
if err != nil {
|
||||
continue // Problems names it
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
if c.Name == "mesh-catalog" {
|
||||
out = append(out, m.Module)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
row := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(row, "modules", "a"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(row, "modules", "a", "module.json"),
|
||||
[]byte(`{"module":"a","version":"1","claims":[{"name":"mesh-catalog","scope":"mesh"}]}`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := claimants(row); len(got) != 1 || got[0] != "a" {
|
||||
t.Fatalf("a manifest claiming the seat is named, got %v", got)
|
||||
}
|
||||
if got := claimants(filepath.Join("..", "..")); len(got) != 0 {
|
||||
t.Errorf("the catalogue's manifests %v claim the seat mesh-catalog: the indexer of the delivery holds it", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
# The modules of this catalogue that still read the controller's assignments (mesh-controller.assignments), each with
|
||||
# why. The check of this directory (noplacement) refuses any other module that reads them, and an entry here whose module no longer
|
||||
# does: the list only shrinks, and it is empty before the controller's lease is written to a holder that is no
|
||||
# controller (novox/hq ADR 0549 §3, §6; issue 774).
|
||||
@@ -1,3 +1,7 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/noplacement
|
||||
|
||||
go 1.22
|
||||
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0
|
||||
|
||||
replace git.novox.be/novox/mesh-catalog/checks/trunk => ../trunk
|
||||
|
||||
+76
-13
@@ -1,6 +1,6 @@
|
||||
// Command noplacement refuses what the repository path of placement left behind (novox/hq ADR 0567 §10, §12).
|
||||
//
|
||||
// Placement, which module runs on which node, is bus state held by the seat mesh-module-assigner; no file of the
|
||||
// Placement, which module runs on which node, is bus state held by the seat mesh-placement; no file of the
|
||||
// catalogue says it, and no manifest reads it from the delivery indexer, which no longer keeps it. It refuses:
|
||||
//
|
||||
// 1. a `placement` directory (or file) at the root of the catalogue;
|
||||
@@ -8,11 +8,13 @@
|
||||
// 2. a `particulars` file at the root (the mesh's node names come from the checker's run, §16);
|
||||
//
|
||||
// 3. a manifest naming a state, stream, event or bucket of the indexer's placement (`mesh-delivery-indexer.placement…`
|
||||
// or `mesh-delivery-indexer_placement…`), or the assigner's retired state placement (`mesh-delivery-assigner.placement…`,
|
||||
// `mesh-module-assigner.placement…`, its views and its read tool), in any string value, such as `reads`;
|
||||
// or `mesh-delivery-indexer_placement…`) in any string value, such as `reads`;
|
||||
//
|
||||
// The controller's assignments, which item 4 once listed readers of, are refused with every other name of the
|
||||
// controller by checks/controllerdeps (novox/hq ADR 0590 §5).
|
||||
// 4. a manifest, other than mesh-controller's, that reads the controller's assignments (`mesh-controller.assignments`):
|
||||
// every reader takes placement from the seat mesh-placement (novox/hq ADR 0549 §3, ADR 0567), and the lease write is
|
||||
// refused while one still reads them (ADR 0549 §6; issue 774). The modules that still read them for what the
|
||||
// placement does not carry are named in checks/noplacement/assignments-readers, each with why; that list only
|
||||
// shrinks: an entry whose module no longer reads them is refused too.
|
||||
//
|
||||
// go run . <catalogue root>
|
||||
//
|
||||
@@ -28,17 +30,14 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Prefixes are the names of the indexer's placement parts and the retired names of the assigner's state, now
|
||||
// assignments (novox/hq ADR 0641); a manifest string starting with one names them.
|
||||
var Prefixes = []string{"mesh-delivery-indexer.placement", "mesh-delivery-indexer_placement",
|
||||
"mesh-delivery-assigner.placement", "mesh-delivery-assigner_placement", "mesh-delivery-assigner.views.placement",
|
||||
"mesh-delivery-assigner.assigner_read_placement", "mesh-module-assigner.placement"}
|
||||
// Prefixes are the names of the indexer's placement parts; a manifest string starting with one names them.
|
||||
var Prefixes = []string{"mesh-delivery-indexer.placement", "mesh-delivery-indexer_placement"}
|
||||
|
||||
// Findings is every refusal in the catalogue at root, sorted.
|
||||
func Findings(root string) ([]string, error) {
|
||||
var out []string
|
||||
if _, err := os.Lstat(filepath.Join(root, "placement")); err == nil {
|
||||
out = append(out, "placement: the repository holds no placement; it is bus state of the seat mesh-module-assigner (novox/hq ADR 0567 §10)")
|
||||
out = append(out, "placement: the repository holds no placement; it is bus state of the seat mesh-placement (novox/hq ADR 0567 §10)")
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Join(root, "particulars")); err == nil {
|
||||
out = append(out, "particulars: the repository lists no node names; the checker hands them to the check (novox/hq ADR 0567 §16)")
|
||||
@@ -59,13 +58,77 @@ func Findings(root string) ([]string, error) {
|
||||
}
|
||||
module := filepath.Base(filepath.Dir(file))
|
||||
for _, s := range named(doc, nil) {
|
||||
out = append(out, fmt.Sprintf("%s: names %q, a name that is retired: the indexer keeps no placement (novox/hq ADR 0567 §12) and the seat mesh-module-assigner's state is assignments (novox/hq ADR 0641)", module, s))
|
||||
out = append(out, fmt.Sprintf("%s: names %q, which the indexer no longer keeps; placement is the seat mesh-placement's (novox/hq ADR 0567 §12)", module, s))
|
||||
}
|
||||
}
|
||||
readers, err := assignmentsReaders(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, readers...)
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// AssignmentsRead is the controller's assignments as a manifest reads them, and Allowed the file, relative to the
|
||||
// catalogue's root, that names the modules still reading them: `<module> <why>` per line, `#` a comment.
|
||||
const (
|
||||
AssignmentsRead = "mesh-controller.assignments"
|
||||
Allowed = "checks/noplacement/assignments-readers"
|
||||
)
|
||||
|
||||
// assignmentsReaders is every finding on the reads of the controller's assignments: a module other than the controller
|
||||
// reading them that Allowed does not name, and a module Allowed names that no longer reads them (issue 774).
|
||||
func assignmentsReaders(root string) ([]string, error) {
|
||||
allowed := map[string]bool{}
|
||||
raw, err := os.ReadFile(filepath.Join(root, Allowed))
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return nil, err
|
||||
}
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
if f := strings.Fields(line); len(f) > 0 && !strings.HasPrefix(f[0], "#") {
|
||||
allowed[f[0]] = true
|
||||
}
|
||||
}
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var found []string
|
||||
reads := map[string]bool{}
|
||||
for _, path := range manifests {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Reads []string `json:"reads"`
|
||||
}
|
||||
if json.Unmarshal(raw, &m) != nil || m.Module == "mesh-controller" {
|
||||
continue // a manifest that does not read is the manifest checks' to refuse
|
||||
}
|
||||
for _, r := range m.Reads {
|
||||
if strings.TrimSpace(r) != AssignmentsRead {
|
||||
continue
|
||||
}
|
||||
reads[m.Module] = true
|
||||
if !allowed[m.Module] {
|
||||
found = append(found, fmt.Sprintf("%s reads %s: read where modules run from mesh-delivery-assigner.placement, "+
|
||||
"the seat mesh-placement's (novox/hq ADR 0549 §3, ADR 0567); the lease write is refused while a module reads "+
|
||||
"the assignments (ADR 0549 §6)", m.Module, AssignmentsRead))
|
||||
}
|
||||
}
|
||||
}
|
||||
for module := range allowed {
|
||||
if !reads[module] {
|
||||
found = append(found, fmt.Sprintf("%s names %s, which no longer reads %s: take it off the list, which only shrinks",
|
||||
Allowed, module, AssignmentsRead))
|
||||
}
|
||||
}
|
||||
return found, nil
|
||||
}
|
||||
|
||||
// named collects the strings of a decoded manifest that start with one of the Prefixes.
|
||||
func named(v any, into []string) []string {
|
||||
switch v := v.(type) {
|
||||
@@ -103,5 +166,5 @@ func main() {
|
||||
if len(found) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("no placement directory, and no manifest names the retired placement")
|
||||
fmt.Println("no placement directory, no manifest reads the indexer's placement, and only the modules listed read the controller's assignments")
|
||||
}
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-catalog/checks/trunk"
|
||||
)
|
||||
|
||||
func TestFindingsRefuseWhatThePlacementPathLeft(t *testing.T) {
|
||||
@@ -13,17 +15,19 @@ func TestFindingsRefuseWhatThePlacementPathLeft(t *testing.T) {
|
||||
files map[string]string
|
||||
want string // empty: clean
|
||||
}{
|
||||
{"clean", map[string]string{"modules/a/module.json": `{"module":"a","reads":["mesh-delivery-assigner.assignments"]}`}, ""},
|
||||
{"clean", map[string]string{"modules/a/module.json": `{"module":"a","reads":["mesh-delivery-assigner.placement"]}`}, ""},
|
||||
{"a placement directory", map[string]string{"placement/ace.json": `{}`}, "placement: the repository holds no placement"},
|
||||
{"a particulars file", map[string]string{"particulars": "ace\n"}, "particulars: the repository lists no node names"},
|
||||
{"a read of the indexer's placement", map[string]string{"modules/a/module.json": `{"module":"a","reads":["mesh-delivery-indexer.placement"]}`}, `a: names "mesh-delivery-indexer.placement"`},
|
||||
{"a read of its candidates", map[string]string{"modules/b/module.json": `{"module":"b","reads":["mesh-delivery-indexer.placement-candidates"]}`}, "placement-candidates"},
|
||||
{"its event", map[string]string{"modules/c/module.json": `{"module":"c","on":[{"event":"mesh-delivery-indexer.placement-changed"}]}`}, "placement-changed"},
|
||||
{"the seat's own state is allowed", map[string]string{"modules/d/module.json": `{"module":"d","reads":["mesh-module-assigner.assignments"]}`}, ""},
|
||||
{"the assigner's retired state", map[string]string{"modules/e/module.json": `{"module":"e","reads":["mesh-delivery-assigner.placement"]}`}, `e: names "mesh-delivery-assigner.placement"`},
|
||||
{"its retired views family", map[string]string{"modules/f/module.json": `{"module":"f","consumes":["mesh-delivery-assigner.views.placement"]}`}, "views.placement"},
|
||||
{"its retired read tool", map[string]string{"modules/g/module.json": `{"module":"g","invokes":["mesh-delivery-assigner.assigner_read_placement"]}`}, "assigner_read_placement"},
|
||||
{"the seat's retired verb", map[string]string{"modules/h/module.json": `{"module":"h","invokes":["mesh-module-assigner.placement"]}`}, "mesh-module-assigner.placement"},
|
||||
// Issue 774 (ADR 0549 §3, §6): a reader of the controller's assignments is refused unless the list names it, the
|
||||
// controller's own manifest is not judged, and a name on the list whose module no longer reads them is refused.
|
||||
{"an unlisted reader of the assignments", map[string]string{"modules/r/module.json": `{"module":"r","reads":["mesh-controller.assignments"]}`}, "r reads mesh-controller.assignments"},
|
||||
{"a listed reader and the controller", map[string]string{"modules/l/module.json": `{"module":"l","reads":["mesh-controller.assignments"]}`,
|
||||
"modules/mesh-controller/module.json": `{"module":"mesh-controller","reads":["mesh-controller.assignments"]}`, Allowed: "# why\nl its plan\n"}, ""},
|
||||
{"a listed module that reads them no more", map[string]string{Allowed: "gone no longer reads\n"}, "names gone, which no longer reads"},
|
||||
{"the seat's own state is allowed", map[string]string{"modules/d/module.json": `{"module":"d","reads":["mesh-placement.placement"]}`}, ""},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
@@ -56,4 +60,29 @@ func TestTheCatalogueHoldsNoPlacementOfTheRepositoryPath(t *testing.T) {
|
||||
for _, f := range got {
|
||||
t.Error(f)
|
||||
}
|
||||
// Issue 774 (ADR 0549 §3, §6; the comparison of novox/hq ADR 0529 clause 2): the list of modules still reading the
|
||||
// controller's assignments gains none the trunk's copy at the merge base does not hold. Its first copy gains nothing.
|
||||
path, ok := trunk.Copy(t, Allowed)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
names := func(path string) map[string]bool {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := map[string]bool{}
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
if f := strings.Fields(line); len(f) > 0 && !strings.HasPrefix(f[0], "#") {
|
||||
out[f[0]] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
base := names(path)
|
||||
for module := range names(filepath.Join("..", "..", Allowed)) {
|
||||
if !base[module] {
|
||||
t.Errorf("%s gains %s, which the trunk's copy does not hold: the list only shrinks", Allowed, module)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -2,6 +2,6 @@ module git.novox.be/novox/mesh-catalog/checks/owned
|
||||
|
||||
go 1.22
|
||||
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0-00010101000000-000000000000
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0
|
||||
|
||||
replace git.novox.be/novox/mesh-catalog/checks/trunk => ../trunk
|
||||
|
||||
+1
-1
@@ -2,6 +2,6 @@ module git.novox.be/novox/mesh-catalog/checks/paths
|
||||
|
||||
go 1.22
|
||||
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0-00010101000000-000000000000
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0
|
||||
|
||||
replace git.novox.be/novox/mesh-catalog/checks/trunk => ../trunk
|
||||
|
||||
@@ -2,6 +2,6 @@ module git.novox.be/novox/mesh-catalog/checks/predefined
|
||||
|
||||
go 1.22
|
||||
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0-00010101000000-000000000000
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0
|
||||
|
||||
replace git.novox.be/novox/mesh-catalog/checks/trunk => ../trunk
|
||||
|
||||
+1
-12
@@ -2,15 +2,4 @@ module checks/readable
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-catalog/modules/restic/backupgood v0.0.0-00010101000000-000000000000
|
||||
)
|
||||
|
||||
replace git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings => ../../modules/disk-load/diskreadings
|
||||
|
||||
replace git.novox.be/novox/mesh-catalog/modules/restic/backupgood => ../../modules/restic/backupgood
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 // indirect
|
||||
)
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
// the owner's manifest went with clause 5, and announcetest holds a module to its own list, not to its readers.
|
||||
//
|
||||
// An owner announces from its code. A Go module announces when its code imports mesh-sdk's go/announce: every
|
||||
// announce.State literal there is read, its Name a string, a constant of the package, or a constant of an owner's package this
|
||||
// announce.State literal there is read, its Name a string, a constant of the package, or a constant of mesh-sdk this
|
||||
// check knows (Known). A TypeScript module announces when its code says the event `states` or calls the SDK's keeps:
|
||||
// every `{ name: "...", readable: true }` there is read. An owner that announces nothing from code is read as before
|
||||
// on the mesh, and is not judged; neither is an owner outside the catalogue (another repository's).
|
||||
@@ -29,15 +29,13 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
|
||||
"git.novox.be/novox/mesh-catalog/modules/restic/backupgood"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// Known are the constants of the owners' published packages an announcement may name a state by, by the selector the
|
||||
// code writes: disk-load's readings and restic's last good night (ADR 0634 §4: a reading's layout is its owner's).
|
||||
// Known are the mesh-sdk constants an announcement may name a state by, by the selector the code writes.
|
||||
var Known = map[string]string{
|
||||
"backupgood.GoodState": backupgood.GoodState,
|
||||
"diskreadings.ReadingsState": diskreadings.ReadingsState,
|
||||
"health.GoodState": health.GoodState,
|
||||
"health.ReadingsState": health.ReadingsState,
|
||||
}
|
||||
|
||||
const announcePkg = `"git.novox.be/novox/mesh-sdk/go/announce"`
|
||||
|
||||
@@ -30,7 +30,7 @@ func TestFindings(t *testing.T) {
|
||||
{"go, a known mesh-sdk name and readers", map[string]string{
|
||||
"modules/r/module.json": `{"module":"r","reads":["o.good","o.s"]}`,
|
||||
"modules/o/module.json": `{"module":"o"}`,
|
||||
"modules/o/cmd/o/main.go": goOwner + "var _ = []announce.State{{Name: backupgood.GoodState, Readable: true}, {Name: \"s\", Readers: []string{\"r\"}}}\n"}, ""},
|
||||
"modules/o/cmd/o/main.go": goOwner + "var _ = []announce.State{{Name: health.GoodState, Readable: true}, {Name: \"s\", Readers: []string{\"r\"}}}\n"}, ""},
|
||||
{"go, a name that cannot be read", map[string]string{
|
||||
"modules/r/module.json": `{"module":"r","reads":["o.s"]}`,
|
||||
"modules/o/module.json": `{"module":"o"}`,
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/removedpackages
|
||||
|
||||
go 1.22
|
||||
@@ -1,166 +0,0 @@
|
||||
// Command removedpackages refuses a Go source of the catalogue that imports what mesh-sdk gave up in the release that
|
||||
// narrowed it (novox/hq ADR 0593 §1): a package that left for its owner, and the build runner's names that left
|
||||
// mesh-sdk's buildstep for mesh-build-go's package.
|
||||
//
|
||||
// A module compiles against the mesh-sdk version it pins, so a source that still names a removed package builds until the
|
||||
// pin moves, and then breaks in a delivery of its own; and a later change could bring the import back with the old pin.
|
||||
// This check refuses it at once, from the source alone:
|
||||
//
|
||||
// - an import of a package in Removed (or one under it);
|
||||
// - in a file importing mesh-sdk's buildstep (under any name), a selector naming anything that is not in
|
||||
// BuildstepContract: what is left of that package is what the delivery parts read, and the runner is
|
||||
// mesh-build-go's package (git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep).
|
||||
//
|
||||
// It reads the non-test and test Go files under modules/ and checks/ (testdata, vendor and node_modules left out).
|
||||
//
|
||||
// go run . <catalogue root>
|
||||
//
|
||||
// Exits 1 on a finding. The lists are the release's: a package that goes later is added here with the change.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const sdk = "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
// Removed are the packages of mesh-sdk that left it, each with where its logic is now.
|
||||
var Removed = map[string]string{
|
||||
"buildcpu": "the build runner's package, git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep/buildcpu",
|
||||
"buildstep/conformance": "the build runner's package, git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep/conformance",
|
||||
"delivery/buildrun": "the builder's package (mesh-delivery-builder), imported by its repository's modules",
|
||||
"compose": "the node-engine, which alone composes (ADR 0593 §6)",
|
||||
"compose/frombus": "the node-engine, which alone composes (ADR 0593 §6)",
|
||||
"compose/fromruntime": "the node-engine, which alone composes (ADR 0593 §6)",
|
||||
"compose/judge": "the assigner's own package",
|
||||
"takein": "the node-engine, which alone composes (ADR 0593 §6)",
|
||||
"validate": "the node-engine, which alone composes (ADR 0593 §6)",
|
||||
"facts": "nothing: the controller they served is ended (ADR 0590)",
|
||||
"example": "nothing: no module imports it (ADR 0593 §12)",
|
||||
"packageguard": "nothing: a test helper of mesh-sdk's own",
|
||||
}
|
||||
|
||||
// BuildstepContract are the names mesh-sdk's buildstep keeps: the failure contract, the fetch request, the stale-step sweep,
|
||||
// the step id and the module tree shape the delivery parts read.
|
||||
var BuildstepContract = map[string]bool{
|
||||
"AskWhat": true, "Boot": true, "By": true, "CapacitySetting": true, "Clip": true, "ConsoleFailure": true, "CutLine": true,
|
||||
"EndOne": true, "EndStale": true, "EndStaleAtStart": true, "EndStaleRound": true, "Failure": true, "FetchRequest": true,
|
||||
"FetchRequestProperties": true, "Hidden": true, "Launchers": true, "LauncherSuffix": true, "LineBytes": true,
|
||||
"ModuleTree": true, "NewID": true, "OutputBytes": true, "Owner": true, "PackagesArg": true, "ParseFetchRequest": true,
|
||||
"PassingLine": true, "RestartReason": true, "Scan": true, "ScanOf": true, "StaleAttempts": true, "StaleRetry": true,
|
||||
"StaleServed": true,
|
||||
}
|
||||
|
||||
// Findings is every refusal in the Go sources under root, sorted.
|
||||
func Findings(root string) ([]string, error) {
|
||||
var out []string
|
||||
for _, top := range []string{"modules", "checks"} {
|
||||
err := filepath.WalkDir(filepath.Join(root, top), func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
switch d.Name() {
|
||||
case "testdata", "vendor", "node_modules", ".mesh-build":
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !strings.HasSuffix(path, ".go") {
|
||||
return nil
|
||||
}
|
||||
rel, _ := filepath.Rel(root, path)
|
||||
found, err := fileFindings(path, filepath.ToSlash(rel))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out = append(out, found...)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func fileFindings(path, rel string) ([]string, error) {
|
||||
file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.SkipObjectResolution)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", rel, err)
|
||||
}
|
||||
var out []string
|
||||
buildstep := ""
|
||||
for _, imp := range file.Imports {
|
||||
p, err := strconv.Unquote(imp.Path.Value)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
sub, ok := strings.CutPrefix(p, sdk+"/")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for pkg, to := range Removed {
|
||||
if sub == pkg || strings.HasPrefix(sub, pkg+"/") {
|
||||
out = append(out, fmt.Sprintf("%s: imports %s, which left mesh-sdk; it is %s (novox/hq ADR 0593 §1)", rel, p, to))
|
||||
break
|
||||
}
|
||||
}
|
||||
if sub == "buildstep" {
|
||||
buildstep = "buildstep"
|
||||
if imp.Name != nil {
|
||||
buildstep = imp.Name.Name
|
||||
}
|
||||
}
|
||||
}
|
||||
if buildstep == "" || buildstep == "_" || buildstep == "." {
|
||||
return out, nil
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
ast.Inspect(file, func(n ast.Node) bool {
|
||||
sel, ok := n.(*ast.SelectorExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
id, ok := sel.X.(*ast.Ident)
|
||||
if !ok || id.Name != buildstep || !sel.Sel.IsExported() || BuildstepContract[sel.Sel.Name] || seen[sel.Sel.Name] {
|
||||
return true
|
||||
}
|
||||
seen[sel.Sel.Name] = true
|
||||
out = append(out, fmt.Sprintf("%s: uses %s.%s, which is the build runner's, no longer mesh-sdk's buildstep; import git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep (novox/hq ADR 0593 §1)", rel, buildstep, sel.Sel.Name))
|
||||
return true
|
||||
})
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 2 {
|
||||
fmt.Fprintln(os.Stderr, "usage: removedpackages <catalogue root>")
|
||||
os.Exit(2)
|
||||
}
|
||||
found, err := Findings(os.Args[1])
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
for _, f := range found {
|
||||
fmt.Println(f)
|
||||
}
|
||||
if len(found) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("no source imports a package mesh-sdk gave up")
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// ADR 0593 §1: a source that imports a package mesh-sdk gave up, or names the build runner through mesh-sdk's buildstep, is
|
||||
// refused naming where it went; the contract that stays, the runner's own package and a testdata copy are not.
|
||||
func TestFindingsRefuseWhatMeshSDKGaveUp(t *testing.T) {
|
||||
const head = "package x\n\nimport (\n"
|
||||
for _, row := range []struct {
|
||||
name, file, src string
|
||||
want string // empty: clean
|
||||
}{
|
||||
{"clean", "modules/a/x.go", head + "\t\"git.novox.be/novox/mesh-sdk/go/delivery\"\n)\n\nvar _ = delivery.StepsState\n", ""},
|
||||
{"a removed package", "modules/a/x.go", head + "\t\"git.novox.be/novox/mesh-sdk/go/compose\"\n)\n\nvar _ = compose.Resolve\n", "imports git.novox.be/novox/mesh-sdk/go/compose, which left mesh-sdk; it is the node-engine"},
|
||||
{"under a removed package", "modules/a/x_test.go", head + "\t\"git.novox.be/novox/mesh-sdk/go/compose/frombus\"\n)\n", "the node-engine"},
|
||||
{"buildcpu", "checks/c/x.go", head + "\t\"git.novox.be/novox/mesh-sdk/go/buildcpu\"\n)\n", "buildstep/buildcpu"},
|
||||
{"the runner through mesh-sdk's buildstep", "modules/a/x.go", head + "\t\"git.novox.be/novox/mesh-sdk/go/buildstep\"\n)\n\nvar _ = buildstep.RunAsk\n", "uses buildstep.RunAsk, which is the build runner's"},
|
||||
{"the runner through an alias", "modules/a/x.go", head + "\tsdkstep \"git.novox.be/novox/mesh-sdk/go/buildstep\"\n)\n\nvar _ = sdkstep.Puller{}\n", "uses sdkstep.Puller"},
|
||||
{"the contract of buildstep", "modules/a/x.go", head + "\t\"git.novox.be/novox/mesh-sdk/go/buildstep\"\n)\n\nvar _ = buildstep.NewID\nvar _ buildstep.Failure\n", ""},
|
||||
{"the runner's own package", "modules/a/x.go", head + "\t\"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep\"\n)\n\nvar _ = buildstep.RunAsk\n", ""},
|
||||
{"a copy in testdata", "modules/a/testdata/x.go", head + "\t\"git.novox.be/novox/mesh-sdk/go/compose\"\n)\n", ""},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
p := filepath.Join(root, row.file)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(row.src), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := Findings(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if all := strings.Join(got, "\n"); (row.want == "") != (all == "") || !strings.Contains(all, row.want) {
|
||||
t.Fatalf("findings = %q, want one containing %q", all, row.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The script's run of this check over the real tree (novox/hq ADR 0529 clause 2).
|
||||
func TestTheCatalogueImportsNothingMeshSDKGaveUp(t *testing.T) {
|
||||
got, err := Findings(filepath.Join("..", ".."))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range got {
|
||||
t.Error(f)
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# every Go source of the modules and of the checks, whose imports are read.
|
||||
modules
|
||||
checks
|
||||
@@ -36,7 +36,13 @@ import (
|
||||
// composition is every function or table the check lets stay in mesh-sdk, as "path-from-go/:name", and why. It names no
|
||||
// manifest judgement: each is the composer's refusal of composition itself (ADR 0578 clause 5).
|
||||
var composition = map[string]string{
|
||||
"catalogue/collections.go:itemTemplateProblem": "composition: filling an item's template",
|
||||
"catalogue/consumers.go:consumersPlaceholderProblems": "composition: rendering a consumer's template",
|
||||
"catalogue/collections.go:collectedPlaceholderProblems": "composition: placeholders a contribution renders",
|
||||
"catalogue/environment_into.go:placeholderProblems": "composition: placeholders a contribution renders",
|
||||
"catalogue/seat_contributions.go:seatPlaceholderProblems": "composition: placeholders a contribution renders",
|
||||
"catalogue/collections.go:itemTemplateProblem": "composition: filling an item's template",
|
||||
"catalogue/setting_takes.go:takenSettingsProblems": "composition: whether stored settings still compose",
|
||||
"catalogue/template.go:HeldTemplateProblem": "composition: the take-in refusal of a resolved build",
|
||||
}
|
||||
|
||||
// Problems is what mesh-sdk's Go tree under root (its go/ directory) holds of a manifest judge, one line each, but for
|
||||
|
||||
@@ -2,6 +2,6 @@ module git.novox.be/novox/mesh-catalog/checks/secrets
|
||||
|
||||
go 1.22
|
||||
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0-00010101000000-000000000000
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0
|
||||
|
||||
replace git.novox.be/novox/mesh-catalog/checks/trunk => ../trunk
|
||||
|
||||
@@ -2,6 +2,6 @@ module git.novox.be/novox/mesh-catalog/checks/templates
|
||||
|
||||
go 1.22
|
||||
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0-00010101000000-000000000000
|
||||
require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0
|
||||
|
||||
replace git.novox.be/novox/mesh-catalog/checks/trunk => ../trunk
|
||||
|
||||
+1
-37
@@ -4,9 +4,7 @@
|
||||
// own code that is not a test (a tool's description, the notes and errors it answers with) and every
|
||||
// description in its manifest. Comments are not read: an agent never sees them. And no daemon's line
|
||||
// (novox/hq ADR 0272, lines.go): the copy in retired-lines, the lines a person would otherwise be told to
|
||||
// type of the node-engine's or the controller's binary, where the operator's command is `nox`. And no address of the
|
||||
// controller (novox/hq ADR 0590 §5): it is not placed, so a text naming one of its verbs, states or buckets sends its
|
||||
// reader nowhere.
|
||||
// type of the node-engine's or the controller's binary, where the operator's command is `nox`.
|
||||
//
|
||||
// A word is matched whole and in any case, a space in it matching any run of white space, and its plural
|
||||
// (`s` or `es` on its last part) is matched as the word. A vendor word is
|
||||
@@ -91,45 +89,11 @@ func run(root string) int {
|
||||
len(findings))
|
||||
return 1
|
||||
}
|
||||
// And no address of the controller (novox/hq ADR 0590 §5): it is not placed, so a text that sends an agent or the
|
||||
// operator to one of its verbs, states or buckets sends them nowhere.
|
||||
if findings := checkAddresses(texts); len(findings) > 0 {
|
||||
for _, f := range findings {
|
||||
fmt.Println(f)
|
||||
}
|
||||
fmt.Printf("words: %d address(es) of the controller in what a module shows — name the seat that holds the job now (novox/hq ADR 0590)\n",
|
||||
len(findings))
|
||||
return 1
|
||||
}
|
||||
fmt.Printf("words: %d retired words and %d daemons' lines, none in the %d strings of %d files\n", len(words), len(lines),
|
||||
len(texts), files)
|
||||
return 0
|
||||
}
|
||||
|
||||
// controllerAddress is an address of the controller: its seat's verb or state (`mesh-controller.<name>`, with or without
|
||||
// `seat:`) or one of its buckets (`mesh-controller_<bucket>`). The module's name in prose is no address.
|
||||
var controllerAddress = regexp.MustCompile(`(^|[^A-Za-z0-9_-])(seat:)?mesh-controller[._][A-Za-z0-9<]`)
|
||||
|
||||
// checkAddresses is every text that names an address of the controller (novox/hq ADR 0590 §5).
|
||||
func checkAddresses(texts []text) []string {
|
||||
var out []string
|
||||
for _, t := range texts {
|
||||
if !strings.Contains(t.s, "mesh-controller") {
|
||||
continue
|
||||
}
|
||||
if m := controllerAddress.FindStringIndex(t.s); m != nil {
|
||||
line := t.line
|
||||
if line > 0 {
|
||||
line += strings.Count(t.s[:m[0]], "\n")
|
||||
}
|
||||
out = append(out, fmt.Sprintf("%s:%d: names an address of the controller, which is not placed — in %q", t.file, line,
|
||||
excerpt(t.s, m[0], m[1])))
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// collect reads every text a module can show under root: the strings of its own code that is not a test, and every string of
|
||||
// its manifests.
|
||||
func collect(root string) ([]text, int, error) {
|
||||
|
||||
@@ -95,25 +95,3 @@ func TestTheCatalogueHoldsNoRetiredWordAndNoDaemonsLine(t *testing.T) {
|
||||
t.Fatalf("the check of the real catalogue exits %d (see what it printed above)", code)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0590 §5 (issue 1126 B10): no text a module shows names an address of the controller, which is not placed; prose
|
||||
// naming it passes.
|
||||
func TestCheckAddressesRefusesTheControllersAddresses(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
s string
|
||||
want bool
|
||||
}{
|
||||
{"push this machine (mesh-controller.push)", true},
|
||||
{"ask seat:mesh-controller.nodes", true},
|
||||
{"the bucket mesh-controller_assignments", true},
|
||||
{"`mesh-controller.<verb>`", true},
|
||||
{"the mesh-controller module is gone.", false},
|
||||
{"located in mesh-controller#246", false},
|
||||
{"ask mesh-module-assigner.assignments", false},
|
||||
} {
|
||||
got := checkAddresses([]text{{file: "m/a.go", line: 1, s: c.s}})
|
||||
if (len(got) == 1) != c.want {
|
||||
t.Errorf("checkAddresses(%q) = %q, want a finding: %v", c.s, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
56
|
||||
58
|
||||
The long-running resources of this catalogue that do not say how they are ready (`health`, novox/hq ADR 0240
|
||||
rule 8), as `module check` counts them. It may only go down: checks/health fails a change that raises it, and
|
||||
one that lowers it writes the new number on the first line. From 2026-11-18, or once it is 0, a long-running
|
||||
|
||||
@@ -2,10 +2,10 @@ module adwaita
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
|
||||
require (
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/nats-io/nats.go v1.54.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
|
||||
@@ -6,34 +6,38 @@ directory and the file the mesh writes there from its setting `older-store-names
|
||||
|
||||
## Why a module beside the store
|
||||
|
||||
The store is the `distribution` module, the registry seats' holder. These tools became a module of their own when the
|
||||
indexer's judge refused a store holder that builds an artifact; that rule is gone (issue 1966), and the sweep stays its
|
||||
own module, holding the seat `mesh-artifact-sweep`. It requires no provision: what it holds, releases and removes it
|
||||
asks the registry seats `oci-artifact-registry` and `container-image-registry` (see *The sweep through the registry
|
||||
seats*), and what the store holds it reads through the same seats' `list` (novox/hq ADR 0632 §4): it reads neither the
|
||||
store's files nor its read side, so it runs on any machine.
|
||||
The store is the `distribution` module. A module that provides the artifact store cannot also build an
|
||||
artifact: building publishes to the store, so the store would be needed to create itself, and the
|
||||
controller's module check refuses that manifest. So the store's tools are a second module, as the
|
||||
controller's refusal says to do. **Assign it to the machine that holds the store**: its tools read the
|
||||
store's files through the store's own container, and anywhere else they answer that the container is not
|
||||
on that machine.
|
||||
|
||||
## Tools
|
||||
|
||||
| tool | | what |
|
||||
|---|---|---|
|
||||
| `artifact_store_repositories` | r | every repository: tags and what each names, how many manifests (tagged or not), size |
|
||||
| `artifact_store_usage` | r | the store's size, each manifest's as its seat says it, and the largest repositories |
|
||||
| `artifact_store_usage` | r | the store's size, the largest repositories, bytes shared between repositories, bytes no manifest marks (what the nightly collector frees next) |
|
||||
| `artifact_store_references` | r | what the controller's records and the catalogue's definitions say of each manifest, counted and sized by state and repository; each manifest listed when one repository is asked; what the records keep that the store does not hold |
|
||||
| `artifact_store_collect` | a | a dry run unless `dry_run` is false: what the controller would let go of, and the bytes the nightly collector would free then and now. A real run needs `why` and asks the controller's `collect` |
|
||||
|
||||
### How the store is read
|
||||
|
||||
The store's read side lists repositories and tags, but not a manifest no tag names, and the mesh pins every
|
||||
machine by digest, so that is most of them. So the bundle asks the two registry seats' `list`
|
||||
(`container-image-registry` for images and indexes, `oci-artifact-registry` for bundles and archives), which the
|
||||
store's holder answers from the store's own files: each manifest's digest, whether it is an index and the manifests it
|
||||
names, the tags naming it, its size and whether a hold keeps it (novox/hq ADR 0632 §4).
|
||||
The store's door lists repositories and tags, but not a manifest no tag names, and the mesh pins every
|
||||
machine by digest, so that is most of them. So the bundle reads the store's own files through its own
|
||||
container (`docker exec mesh-registry`, busybox `find`, `stat` and `cat`), and only reads them: every
|
||||
blob with its size, every manifest each repository holds, what each tag names, and each manifest's
|
||||
content, four hundred to an exec. What it read it keeps (a manifest is named by its content, so it never
|
||||
changes), and the next call reads only what is new. A manifest that could not be read is counted and
|
||||
said; what it marks beyond itself is then unknown, so sizes may read low and freed bytes high, and the
|
||||
answer says so.
|
||||
|
||||
A manifest's size is its own content and every blob it names, as its seat says it, so a layer two manifests share
|
||||
is counted in each, and the bytes the store's nightly collector frees are the sizes of the manifests that go, an
|
||||
upper bound. The address the bundle says the store is at comes from `${seat:container-image-registry:reach}`, the
|
||||
image seat holder's listen (ADR 0632 §1), and is used for the catalogue's definitions only, never to reach the store.
|
||||
docker runs as the tool runner's account; a socket that refuses it is asked again through `sudo -n`,
|
||||
never with a prompt, as the container runtime's own tools do.
|
||||
|
||||
A manifest *marks* its own content, its configuration and its layers, and an index marks the manifests
|
||||
it lists. The store's collector removes every blob no manifest marks, so these numbers are its own.
|
||||
|
||||
### The states of a manifest
|
||||
|
||||
@@ -57,7 +61,7 @@ record names leaves its platform manifests unrecorded with it.
|
||||
**What the definitions pin** (hq issue 323). A definition may pin an image of the store the mesh did not
|
||||
make: pushed before the mesh built it, and pinned under the store's current name or a name it had before.
|
||||
No record of the controller names it, yet a machine runs it, so `artifact_store_references` reads every
|
||||
definition the catalogue holds (the seat mesh-catalog's verbs `modules`, then `module` for each, eight at
|
||||
definition the catalogue holds (`mesh-catalog.catalog_modules`, then `catalog_module` for each, eight at
|
||||
a time and within twenty seconds) and finds every `<name>/<repository>@sha256:…` in any of its strings,
|
||||
under each of the store's names: `artifact-store://`, the store's address as the controller answers it,
|
||||
and the names it had before. Those are the mesh's setting `older-store-names` (space-separated, `none` by
|
||||
@@ -95,27 +99,19 @@ manifest that stays and would lose a platform it names — to be read before a r
|
||||
go test ./...
|
||||
```
|
||||
|
||||
Against a fake of the two registry seats' `list`: the store read through it (an index and the manifests it names, a
|
||||
bundle's holder, a hold, the seat's sizes), every state, what the records keep that the store lacks, bytes freed as the
|
||||
sizes of what goes, a dry run never confirming, a real run without `why` refused before anything is asked,
|
||||
Against a fake container that prints what the two scripts would: the listing parsed (a repository name
|
||||
with slashes, a cut listing refused), manifests read with one absent or unreadable said as unread, a
|
||||
digest never becoming a path outside the blobs, the mark (an index, a shared blob, a blob nothing marks),
|
||||
every state, what the records keep that the store lacks, bytes freed counting a blob shared with a kept
|
||||
manifest as kept, a dry run never confirming, a real run without `why` refused before anything is asked,
|
||||
the fall-back when the references let go of cannot be carried, a manifest a definition pins under the
|
||||
store's current or an older name, a definition not read said first, and that the
|
||||
store's current or an older name, a definition not read said first, escalation through `sudo -n`, and that the
|
||||
tools served are exactly the manifest's `tools` and its `invokes` exactly the two controller verbs and the
|
||||
catalogue's two tools.
|
||||
Any ask other than a seat's `list` fails the read tools' tests.
|
||||
Any command other than the two read-only scripts fails the test.
|
||||
|
||||
## Files it does not know
|
||||
|
||||
Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416).
|
||||
|
||||
- `${dir:state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
|
||||
### The sweep through the registry seats
|
||||
|
||||
The sweep writes nothing to the store's door (CAT-AS1, ADR 0617 §2): it asks the registry seats the store's holder
|
||||
serves. Each kept bundle or archive is held on `oci-artifact-registry` and each kept copy of an upstream base on
|
||||
`container-image-registry` (`hold`, with why it is kept) before anything goes; each eligible one is released, then
|
||||
removed for `retention` on its seat (`release`, `remove`). It reads a reference in both forms, `artifact-store://<path>`
|
||||
and `<seat>/<module>/<artifact>@<digest>`, as one artifact, and says artifacts-gone in the form the build named it. A
|
||||
call a seat refuses stops the sweep and writes the finding `sweep-refused` in the module's findings; the next sweep no
|
||||
seat refuses clears it (`sweepseats.go`).
|
||||
|
||||
@@ -23,11 +23,10 @@ import (
|
||||
// manifest's.
|
||||
const NamesFileEnv = "MESH_ARTIFACT_STORE_NAMES_FILE"
|
||||
|
||||
// CatalogueModules and CatalogueModule are the verbs of the seat mesh-catalog these definitions are read through
|
||||
// (ADR 0604 §3); the answers are catalogue.CatalogModules and catalogue.CatalogOne.
|
||||
// CatalogueModules and CatalogueModule are the catalog's tools these definitions are read through.
|
||||
const (
|
||||
CatalogueModules = "seat:mesh-catalog.modules"
|
||||
CatalogueModule = "seat:mesh-catalog.module"
|
||||
CatalogueModules = "mesh-catalog.catalog_modules"
|
||||
CatalogueModule = "mesh-catalog.catalog_module"
|
||||
)
|
||||
|
||||
// DefinitionsTimeout bounds reading every definition, below the runtime's thirty-second call limit; what
|
||||
@@ -280,8 +279,11 @@ func (c *Catalogue) definition(m catalogueEntry, names StoreNames, nameKey strin
|
||||
return pins, ""
|
||||
}
|
||||
}
|
||||
// The verb takes no commit: it answers the trunk's module, and the answer names the commit it is at.
|
||||
raw, err := c.Ask(CatalogueModule, map[string]any{"module": m.Module})
|
||||
args := map[string]any{"module": m.Module}
|
||||
if m.Commit != "" {
|
||||
args["commit"] = m.Commit
|
||||
}
|
||||
raw, err := c.Ask(CatalogueModule, args)
|
||||
if err != nil {
|
||||
return nil, "could not be asked: " + err.Error()
|
||||
}
|
||||
@@ -291,7 +293,6 @@ func (c *Catalogue) definition(m catalogueEntry, names StoreNames, nameKey strin
|
||||
}
|
||||
var one struct {
|
||||
Module *struct {
|
||||
Commit string `json:"commit"`
|
||||
Manifest json.RawMessage `json:"manifest"`
|
||||
} `json:"module"`
|
||||
Error string `json:"error"`
|
||||
@@ -300,11 +301,9 @@ func (c *Catalogue) definition(m catalogueEntry, names StoreNames, nameKey strin
|
||||
return nil, "not readable: " + firstOf(one.Error, errText(err), "no manifest in the answer")
|
||||
}
|
||||
pins := PinsIn(one.Module.Manifest, names)
|
||||
// Kept by the commit the answer names, which may be newer than the list's: a definition and its commit never
|
||||
// change together.
|
||||
if at := firstOf(one.Module.Commit, m.Commit); at != "" {
|
||||
if m.Commit != "" {
|
||||
c.mu.Lock()
|
||||
c.kept[m.Module+"@"+at+"|"+nameKey] = pins
|
||||
c.kept[key] = pins
|
||||
c.mu.Unlock()
|
||||
}
|
||||
return pins, ""
|
||||
|
||||
@@ -152,8 +152,8 @@ func TestOlderNamesNotReadAreSaidFirst(t *testing.T) {
|
||||
// references is one artifact_store_references tool, kept across calls as the runtime keeps the bundle.
|
||||
func references(t *testing.T, a *asked) func(map[string]any) (any, error) {
|
||||
f := world()
|
||||
s := &Store{Ask: f.ask(t)}
|
||||
for _, tl := range Tools(s, recordsAsked{a}, &Catalogue{Ask: a.ask}) {
|
||||
s := &Store{Container: "mesh-registry", Run: f.run(t)}
|
||||
for _, tl := range Tools(s, Controller{Ask: a.ask}, &Catalogue{Ask: a.ask}) {
|
||||
if tl.Name == "artifact_store_references" {
|
||||
return tl.Run
|
||||
}
|
||||
@@ -211,7 +211,7 @@ func TestReferencesReadTheDefinitionsUnderTheStoresOlderNames(t *testing.T) {
|
||||
}
|
||||
}
|
||||
if before != 2 || after != 2 {
|
||||
t.Errorf("asked the seat's module %d times, then %d: each definition at its commit is read once", before, after)
|
||||
t.Errorf("asked catalog_module %d times, then %d: each definition at its commit is read once", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,57 +1,41 @@
|
||||
package main
|
||||
|
||||
// The artifact store, through its seats, and the bus, as the holder reaches them through this module's runtime.
|
||||
// The artifact store's door and the bus, as the holder reaches them through this module's runtime.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
"artifactstoretools/internal/registry"
|
||||
)
|
||||
|
||||
// Gone is the answer when the store does not hold it: the outcome wanted, already true.
|
||||
var Gone = errors.New("the store does not hold it")
|
||||
// door is registry.Store at the store's address as this machine reaches it.
|
||||
type door struct{ s registry.Store }
|
||||
|
||||
// ErrNotOurs is a reference this sweep will not address: not the mesh's own, or naming nothing the store holds by digest.
|
||||
//
|
||||
// **A fact about the record, not about the store** (novox/hq issue 226). The two deserve opposite responses (skip one and
|
||||
// go on, abandon the sweep for the other), and collapsing them into "an error" is how a cautious loop became one that did
|
||||
// nothing while reporting the right number.
|
||||
var ErrNotOurs = errors.New("not a reference into the mesh's artifact store")
|
||||
|
||||
// door is the store as its seats' list answers it (ADR 0632 §4), at the address this machine reaches it by: it reads
|
||||
// only; what the sweep holds and removes it asks the registry seats too (sweepseats.go).
|
||||
type door struct {
|
||||
address string
|
||||
store *Store
|
||||
func (d *door) Address() string { return d.s.Address }
|
||||
func (d *door) Hold(ctx context.Context, r string) (bool, error) { return d.s.Hold(ctx, r) }
|
||||
func (d *door) Holds(ctx context.Context, r string) error { return d.s.Holds(ctx, r) }
|
||||
func (d *door) IsIndex(ctx context.Context, r string) (bool, error) {
|
||||
return d.s.IsIndex(ctx, r)
|
||||
}
|
||||
|
||||
func (d *door) Address() string { return d.address }
|
||||
|
||||
// Holds is whether the store holds what a reference names: an image's manifest, a bundle's or an archive's blob. Gone when
|
||||
// its seat lists none (ADR 0526 clause 3: a registration by hand is refused at bytes that are not there).
|
||||
func (d *door) Holds(_ context.Context, r string) error {
|
||||
_, err := d.store.listed(r)
|
||||
func (d *door) LetGo(ctx context.Context, r string) error { return d.s.LetGo(ctx, r) }
|
||||
func (d *door) SpareKept(ctx context.Context, keptImages, touching []string) error {
|
||||
spare, err := d.s.SpareKeptIndexes(ctx, keptImages, touching)
|
||||
if err == nil {
|
||||
d.s.Spare = spare
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// IsIndex is whether a reference is an index the store holds, as its seat lists it. Gone when it does not hold it.
|
||||
func (d *door) IsIndex(_ context.Context, r string) (bool, error) {
|
||||
e, err := d.store.listed(r)
|
||||
return e.Index, err
|
||||
}
|
||||
|
||||
// DoorAt reads the store's address (host:port) from the file the manifest fills from the seat container-image-registry's
|
||||
// reach, each time, so a store that moved is said where it is now; the store is read through s.
|
||||
func DoorAt(file string, s *Store) func() (Door, error) {
|
||||
// DoorAt reads the store's address (host:port) from the file the manifest fills from the seat mesh-artifact-store's
|
||||
// reach, each time, so a store that moved is reached where it is now.
|
||||
func DoorAt(file string) func() (Door, error) {
|
||||
return func() (Door, error) {
|
||||
if file == "" {
|
||||
return nil, fmt.Errorf("this bundle was given no file naming the artifact store's address")
|
||||
@@ -64,27 +48,15 @@ func DoorAt(file string, s *Store) func() (Door, error) {
|
||||
if address == "" {
|
||||
return nil, fmt.Errorf("the artifact store's address is empty in %s", file)
|
||||
}
|
||||
return &door{address: address, store: s}, nil
|
||||
return &door{s: registry.Store{Address: address}}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// runtimeBus is the holder's bus through the runtime: its reads, its findings and its seat's event.
|
||||
type runtimeBus struct{ views *views.Reader }
|
||||
type runtimeBus struct{}
|
||||
|
||||
// followed says whether the state is the indexer's or the assigner's, which the holder asks once and follows by events
|
||||
// (ADR 0615) rather than reads.
|
||||
func (b runtimeBus) followed(state string) bool { return b.views != nil && b.views.Follows(state) }
|
||||
|
||||
func (b runtimeBus) Keys(state string) ([]string, error) {
|
||||
if b.followed(state) {
|
||||
return b.views.Keys(state)
|
||||
}
|
||||
return stdio.State(state).Keys()
|
||||
}
|
||||
func (b runtimeBus) Get(state, key string) ([]byte, bool, error) {
|
||||
if b.followed(state) {
|
||||
return b.views.Get(state, key)
|
||||
}
|
||||
func (runtimeBus) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
|
||||
func (runtimeBus) Get(state, key string) ([]byte, bool, error) {
|
||||
e, err := stdio.State(state).Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, false, err
|
||||
@@ -116,9 +88,9 @@ func (runtimeBus) Publish(subject string, body any) error {
|
||||
return stdio.SeatPublish(subject, body, "")
|
||||
}
|
||||
|
||||
// follow watches the builder's done keys and the wants: a build's end sweeps and
|
||||
// follow watches the builder's done keys and the wants: a build's end sweeps (once the lease names no controller) and
|
||||
// observes again; a want changed observes again. Event-driven: nothing reads them on a timer.
|
||||
func follow(h *Holder, v *views.Reader) {
|
||||
func follow(h *Holder) {
|
||||
<-stdio.Served()
|
||||
watch := func(state string, handle func(stdio.StateChange) error) {
|
||||
for {
|
||||
@@ -147,22 +119,11 @@ func follow(h *Holder, v *views.Reader) {
|
||||
}
|
||||
return nil
|
||||
})
|
||||
// The wants are the assigner's: the first view then its events (ADR 0615). What Watch hands over before it returns
|
||||
// is the first view, not a change.
|
||||
var current atomic.Bool
|
||||
current.Store(true)
|
||||
for {
|
||||
err := v.Watch(WantsRead, func(string, []byte, bool) {
|
||||
if !current.Load() {
|
||||
go observe()
|
||||
}
|
||||
})
|
||||
if err == nil {
|
||||
break
|
||||
watch(WantsRead, func(c stdio.StateChange) error {
|
||||
if !c.Current {
|
||||
go observe()
|
||||
}
|
||||
h.logf("watching %s: %v; asked again in 10s", WantsRead, err)
|
||||
time.Sleep(10 * time.Second)
|
||||
}
|
||||
current.Store(false)
|
||||
return nil
|
||||
})
|
||||
observe()
|
||||
}
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
sweep "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep"
|
||||
)
|
||||
|
||||
// The holders whose states this module reads, one constant each (a reader names its holder once; ADR 0634 §4).
|
||||
const (
|
||||
assignerModule = "mesh-delivery-assigner"
|
||||
indexerModule = "mesh-delivery-indexer"
|
||||
)
|
||||
|
||||
// The seat mesh-artifact-sweep types the answer of its verb held and the body of its event artifacts-gone; this module
|
||||
// holds the seat, so it answers and says them in those types.
|
||||
type (
|
||||
// HeldAnswer is the answer of the seat's verb held.
|
||||
HeldAnswer = sweep.HeldAnswer
|
||||
// ArtifactsGone is the body of the seat's event artifacts-gone.
|
||||
ArtifactsGone = sweep.ArtifactsGone
|
||||
)
|
||||
|
||||
// GoneSubject is where the seat says artifacts-gone.
|
||||
const GoneSubject = sweep.SubjectArtifactsGone
|
||||
|
||||
// EncodeArtifactsGone is the event's body, the references sorted; it refuses one naming no module, no fingerprint or no
|
||||
// gone reference.
|
||||
func EncodeArtifactsGone(e ArtifactsGone) ([]byte, error) {
|
||||
e.Event = sweep.EventArtifactsGone
|
||||
e.Gone = append([]string(nil), e.Gone...)
|
||||
sort.Strings(e.Gone)
|
||||
if e.Module == "" || e.Fingerprint == "" || len(e.Gone) == 0 {
|
||||
return nil, fmt.Errorf("%s names a module, a fingerprint and at least one gone reference", sweep.EventArtifactsGone)
|
||||
}
|
||||
return json.Marshal(e)
|
||||
}
|
||||
@@ -9,8 +9,8 @@ package main
|
||||
// At each build's end (a done key written) it sweeps as the controller's take-in did: every kept archive held first
|
||||
// (ADR 0251 §8), then what is eligible let go of, oldest first, within the same bound per sweep, an index left for a
|
||||
// `collect` a person confirms (ADR 0251 §7), each deletion recorded in the hand-act log (seat mesh-hand-acts) and said
|
||||
// as artifacts-gone. **No lease turns it** (issue 1126): the controller ended by its unassignment, so its take-in sweeps
|
||||
// nothing and this seat is the one sweep. It also observes S25, image-missing: a want's
|
||||
// as artifacts-gone. **Nothing is let go until the controller's lease names a holder that is no controller**: until
|
||||
// then the controller's take-in sweeps, and two sweeps never run at once. It also observes S25, image-missing: a want's
|
||||
// image the store does not hold, a finding in its own state `findings` the condition keeper reads.
|
||||
|
||||
import (
|
||||
@@ -18,37 +18,35 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
engineclient "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
catalogclient "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
||||
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
artifactref "git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient/reference"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
"artifactstoretools/internal/registry"
|
||||
)
|
||||
|
||||
// What the holder reads and keeps.
|
||||
const (
|
||||
WantsRead = assignerModule + ".wants"
|
||||
BuildsRead = builds.BuilderModule + ".builds"
|
||||
WantsRead = delivery.AssignerModule + ".wants"
|
||||
BuildsRead = delivery.BuilderModule + ".builds"
|
||||
HavesRead = "node-engine.have"
|
||||
// ManifestsRead is the indexer's manifests: the images a manifest pins by itself.
|
||||
ManifestsRead = indexerModule + "." + catalog.StateManifests
|
||||
// FindingsState is the holder's own findings (meshconditions.StateFindings): image-missing.
|
||||
FindingsState = meshconditions.StateFindings
|
||||
ManifestsRead = delivery.IndexerModule + ".manifests"
|
||||
LeaseRead = buscompose.ControllerLease
|
||||
// FindingsState is the holder's own findings (health.FindingsState): image-missing.
|
||||
FindingsState = health.FindingsState
|
||||
// Module is this module, the observer its findings name.
|
||||
Module = "artifact-store-tools"
|
||||
// GoneSubject is where the seat says artifacts-gone.
|
||||
GoneSubject = "mesh.seat." + catalogue.ArtifactSweepSeat + ".event." + delivery.ArtifactsGoneEvent
|
||||
)
|
||||
|
||||
// The sweep's bounds, the controller's (cmd/mesh-controller/collect.go mostPerSweep, sweepBudget).
|
||||
@@ -60,7 +58,7 @@ const (
|
||||
ImageMissing = "image-missing"
|
||||
)
|
||||
|
||||
// Bus is what the holder reads and writes on the bus: its read states (wants, builds, haves), its own
|
||||
// Bus is what the holder reads and writes on the bus: its read states (wants, builds, haves, the lease), its own
|
||||
// findings, and its seat's event. The runtime in main; a test's in memory.
|
||||
type Bus interface {
|
||||
Keys(state string) ([]string, error)
|
||||
@@ -73,25 +71,26 @@ type Bus interface {
|
||||
Publish(subject string, body any) error
|
||||
}
|
||||
|
||||
// Door is the artifact store as its seats' list answers it, read only, behind an interface so a test needs no store. What
|
||||
// the sweep holds and removes it asks the registry seats (Seats, sweepseats.go).
|
||||
// Door is the artifact store's own door: registry.Store, behind an interface so a test needs no store.
|
||||
type Door interface {
|
||||
Address() string
|
||||
Hold(ctx context.Context, reference string) (bool, error)
|
||||
Holds(ctx context.Context, reference string) error
|
||||
IsIndex(ctx context.Context, reference string) (bool, error)
|
||||
LetGo(ctx context.Context, reference string) error
|
||||
// SpareKept sets the platform manifests a kept index names aside before an index is let go of (ADR 0257).
|
||||
SpareKept(ctx context.Context, keptImages, touching []string) error
|
||||
}
|
||||
|
||||
// Holder is the seat's holder.
|
||||
type Holder struct {
|
||||
Bus Bus
|
||||
Door func() (Door, error)
|
||||
// Listed is the store as its seats list it now (Store.View); nil reads none, and then nothing is said gone or collected.
|
||||
// Listed is the store's own listing now (Store.View); nil reads none, and then nothing is said gone or collected.
|
||||
Listed func(ctx context.Context) (*View, error)
|
||||
Acts meshhandactsclient.Recorder
|
||||
// Seats asks a registry seat's verb, `seat:<seat>.<verb>` (hold, remove): the runtime's stdio.Ask in main.
|
||||
Seats func(key string, body any) (json.RawMessage, error)
|
||||
Now func() time.Time
|
||||
Logf func(string, ...any)
|
||||
Acts handacts.Recorder
|
||||
Now func() time.Time
|
||||
Logf func(string, ...any)
|
||||
|
||||
sweep sync.Mutex // one sweep at a time
|
||||
mu sync.Mutex
|
||||
@@ -102,11 +101,18 @@ type Holder struct {
|
||||
type Build struct {
|
||||
Module, FP, Key string
|
||||
At time.Time
|
||||
Artifacts []builds.BuiltArtifact
|
||||
Artifacts []delivery.BuiltArtifact
|
||||
// Manifest is the digest of the manifest it built, in the indexer's `manifests`.
|
||||
Manifest string
|
||||
}
|
||||
|
||||
// LeaseAway says the controller's lease names a holder that is no controller: then the seat sweeps.
|
||||
func (h *Holder) LeaseAway() bool {
|
||||
raw, found, err := h.Bus.Get(LeaseRead, buscompose.ControllerLeaseKey)
|
||||
away, _ := buscompose.LeaseHandedAway(raw, found, err)
|
||||
return away
|
||||
}
|
||||
|
||||
// Builds is every done build the builder's keys hold, each with its end.
|
||||
func (h *Holder) Builds() ([]Build, error) {
|
||||
keys, err := h.Bus.Keys(BuildsRead)
|
||||
@@ -115,7 +121,7 @@ func (h *Holder) Builds() ([]Build, error) {
|
||||
}
|
||||
var out []Build
|
||||
for _, k := range keys {
|
||||
module, fp, err := builds.ParseBuilderBuildKey(k)
|
||||
module, fp, err := delivery.ParseBuilderBuildKey(k)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -126,8 +132,8 @@ func (h *Holder) Builds() ([]Build, error) {
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
b, err := builds.DecodeBuilderBuild(raw)
|
||||
if err != nil || b.State != builds.BuildDone {
|
||||
b, err := delivery.DecodeBuilderBuild(raw)
|
||||
if err != nil || b.State != delivery.BuildDone {
|
||||
continue
|
||||
}
|
||||
at, err := h.endOf(k)
|
||||
@@ -152,7 +158,7 @@ func (h *Holder) endOf(key string) (time.Time, error) {
|
||||
return time.Time{}, fmt.Errorf("the history of the builder's build %s: %w", key, err)
|
||||
}
|
||||
for _, r := range revs {
|
||||
if b, err := builds.DecodeBuilderBuild(r.Value); err == nil && b.State == builds.BuildDone && r.At.After(at) {
|
||||
if b, err := delivery.DecodeBuilderBuild(r.Value); err == nil && b.State == delivery.BuildDone && r.At.After(at) {
|
||||
at = r.At
|
||||
}
|
||||
}
|
||||
@@ -184,11 +190,11 @@ func (h *Holder) Wants() ([]Want, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the want %s: %w", k, err)
|
||||
}
|
||||
var w meshwants.WantsByWantNodeModuleValue
|
||||
if !found || json.Unmarshal(raw, &w) != nil || w.Removed || w.Fp == "" {
|
||||
var w delivery.Want
|
||||
if !found || json.Unmarshal(raw, &w) != nil || w.Removed || w.FP == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, Want{Node: node, Module: module, FP: w.Fp})
|
||||
out = append(out, Want{Node: node, Module: module, FP: w.FP})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -220,8 +226,8 @@ func KeepSet(builds []Build, wants []Want) map[string]*Ref {
|
||||
out := map[string]*Ref{}
|
||||
for _, b := range builds {
|
||||
for _, a := range b.Artifacts {
|
||||
reference := canonical(registryForms.Recorded(a.Reference))
|
||||
if _, ours := registryForms.InArtifactStore(reference); !ours || a.Kind == "package" {
|
||||
reference := catalogue.Recorded(a.Reference)
|
||||
if _, ours := catalogue.InArtifactStore(reference); !ours || a.Kind == "package" {
|
||||
continue
|
||||
}
|
||||
r := out[reference]
|
||||
@@ -355,17 +361,17 @@ func (h *Holder) judge(ctx context.Context) (*judged, error) {
|
||||
|
||||
// against is what each build the builder's log says ended stood on, by build key: the bases' references.
|
||||
func (h *Holder) against() (map[string][]string, error) {
|
||||
entries, err := h.Bus.LogSince(builds.BuildLogRead, h.now().Add(-builds.BuilderLogDays*24*time.Hour))
|
||||
entries, err := h.Bus.LogSince(delivery.BuildLogRead, h.now().Add(-delivery.BuilderLogDays*24*time.Hour))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string][]string{}
|
||||
for _, raw := range entries {
|
||||
e, err := builds.DecodeBuildLogEntry(raw.Value)
|
||||
if err != nil || e.Event != builds.BuildEndedEvent {
|
||||
e, err := delivery.DecodeBuildLogEntry(raw.Value)
|
||||
if err != nil || e.Event != delivery.BuildEndedEvent {
|
||||
continue
|
||||
}
|
||||
key, err := builds.BuilderBuildKey(e.Module, e.FP)
|
||||
key, err := delivery.BuilderBuildKey(e.Module, e.FP)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -386,7 +392,7 @@ func Mirrors(v *View, builds []Build, kept map[string][]string, against map[stri
|
||||
continue
|
||||
}
|
||||
for d := range v.L.Repos[name].Revisions {
|
||||
r := artifactref.ArtifactStoreScheme + name + "@" + d
|
||||
r := catalogue.ArtifactStoreScheme + name + "@" + d
|
||||
out[r] = &Ref{Reference: r, Kind: "image"}
|
||||
}
|
||||
}
|
||||
@@ -400,7 +406,7 @@ func Mirrors(v *View, builds []Build, kept map[string][]string, against map[stri
|
||||
continue
|
||||
}
|
||||
for _, base := range bases {
|
||||
if m := out[registryForms.Recorded(base)]; m != nil {
|
||||
if m := out[catalogue.Recorded(base)]; m != nil {
|
||||
m.Builds = append(m.Builds, b)
|
||||
why := kept[b.Key]
|
||||
if len(why) > 0 {
|
||||
@@ -448,7 +454,7 @@ func (j *judged) records(repository string, collected bool) *Records {
|
||||
}
|
||||
}
|
||||
r.Counts[state]++
|
||||
if (repository != "" && !strings.HasPrefix(strings.TrimPrefix(ref.Reference, artifactref.ArtifactStoreScheme), repository)) ||
|
||||
if (repository != "" && !strings.HasPrefix(strings.TrimPrefix(ref.Reference, catalogue.ArtifactStoreScheme), repository)) ||
|
||||
(state == "collected" && !collected) {
|
||||
continue
|
||||
}
|
||||
@@ -489,6 +495,10 @@ func (h *Holder) Collect(why string, confirm bool, most int) (*CollectAnswer, er
|
||||
if most > MostAtAll {
|
||||
return nil, fmt.Errorf("collect lets go of at most %d at once, not %d", MostAtAll, most)
|
||||
}
|
||||
if confirm && !h.LeaseAway() {
|
||||
return nil, errors.New("the controller holds its lease, and its take-in sweeps the store until it hands it away: " +
|
||||
"this seat lets nothing go before then (novox/hq ADR 0571 clause 2); ask the controller's collect")
|
||||
}
|
||||
return h.Sweep(context.Background(), Sweeping{Confirmed: confirm, DryRun: !confirm, Most: most, Why: why, By: "collect"})
|
||||
}
|
||||
|
||||
@@ -514,13 +524,13 @@ func (h *Holder) Sweep(ctx context.Context, s Sweeping) (*CollectAnswer, error)
|
||||
return nil, err
|
||||
}
|
||||
a := &CollectAnswer{DryRun: s.DryRun, WouldLetGo: []string{}, LetGo: []string{}}
|
||||
var keptArchives, eligible []string
|
||||
var keptArchives, keptImages, eligible []string
|
||||
for _, r := range j.sorted() {
|
||||
switch {
|
||||
case r.Kept && (r.Kind == "archive" || isCopy(r.Reference)):
|
||||
// What retention would remove unless held: a bundle or an archive, and a copied base (ADR 0617 §2).
|
||||
case r.Kept && r.Kind == "archive":
|
||||
keptArchives = append(keptArchives, r.Reference)
|
||||
case r.Kept:
|
||||
keptImages = append(keptImages, r.Reference)
|
||||
default:
|
||||
if held, known := inStore(j.view, r.Reference); known && !held {
|
||||
continue // collected already
|
||||
@@ -546,25 +556,33 @@ func (h *Holder) Sweep(ctx context.Context, s Sweeping) (*CollectAnswer, error)
|
||||
a.Store = door.Address()
|
||||
within, stop := context.WithTimeout(ctx, SweepBudget)
|
||||
defer stop()
|
||||
// Hold before letting go (ADR 0251 §8, ADR 0617 §2): each kept archive and copied base is held on its seat; one that
|
||||
// cannot be held stops the sweep before any removal and writes the sweep's finding.
|
||||
// Hold before letting go (ADR 0251 §8): a kept archive that cannot be held stops the sweep before any delete.
|
||||
var missing []string
|
||||
for _, r := range keptArchives {
|
||||
if held, known := inStore(j.view, r); known && !held {
|
||||
wrote, err := door.Hold(within, r)
|
||||
switch {
|
||||
case err == nil:
|
||||
if wrote {
|
||||
a.HoldersWritten++
|
||||
}
|
||||
case errors.Is(err, registry.Gone):
|
||||
missing = append(missing, r)
|
||||
continue
|
||||
}
|
||||
if _, err := h.holdOnSeat(r, keptWhy(j.refs[r])); err != nil {
|
||||
case errors.Is(err, registry.ErrNotOurs):
|
||||
default:
|
||||
a.Stopped = fmt.Sprintf("not every kept archive could be held, so nothing was let go: holding %s: %v", r, err)
|
||||
a.Left = len(eligible)
|
||||
h.sweepFinding("the hold of "+r, err)
|
||||
return a, nil
|
||||
}
|
||||
a.HoldersWritten++
|
||||
}
|
||||
a.Held, a.Missing = len(keptArchives)-len(missing), len(missing)
|
||||
h.sayGone(j, missing, "a kept archive the store does not hold")
|
||||
refused := false
|
||||
if s.Confirmed {
|
||||
if err := door.SpareKept(within, keptImages, eligible); err != nil {
|
||||
a.Stopped = fmt.Sprintf("which platform manifests a kept index names could not be read, so nothing was let go: %v", err)
|
||||
a.Left = len(eligible)
|
||||
return a, nil
|
||||
}
|
||||
}
|
||||
for i, r := range eligible {
|
||||
if i >= s.Most || within.Err() != nil {
|
||||
a.Left = len(eligible) - i
|
||||
@@ -578,7 +596,7 @@ func (h *Holder) Sweep(ctx context.Context, s Sweeping) (*CollectAnswer, error)
|
||||
if !s.Confirmed {
|
||||
// An index waits for a confirmed collect (ADR 0251 §7, ADR 0257 §4).
|
||||
index, err := door.IsIndex(within, r)
|
||||
if err != nil && !errors.Is(err, Gone) && !errors.Is(err, ErrNotOurs) {
|
||||
if err != nil && !errors.Is(err, registry.Gone) && !errors.Is(err, registry.ErrNotOurs) {
|
||||
a.Stopped = fmt.Sprintf("the store could not say whether %s is an index, so nothing more was asked: %v", r, err)
|
||||
a.Left = len(eligible) - i
|
||||
break
|
||||
@@ -588,35 +606,37 @@ func (h *Holder) Sweep(ctx context.Context, s Sweeping) (*CollectAnswer, error)
|
||||
continue
|
||||
}
|
||||
}
|
||||
// Removed through its seat (ADR 0617 §2), recorded in the hand-act log, each one (ADR 0565 §10; the operator's
|
||||
// "option A").
|
||||
err := door.LetGo(within, r)
|
||||
switch {
|
||||
case err == nil || errors.Is(err, registry.Gone):
|
||||
case errors.Is(err, registry.ErrNotOurs):
|
||||
a.Skipped++
|
||||
continue
|
||||
default:
|
||||
a.Stopped = fmt.Sprintf("the store refused to let go of %s, so nothing more was asked: %v", r, err)
|
||||
a.Left = len(eligible) - i
|
||||
}
|
||||
if a.Stopped != "" {
|
||||
break
|
||||
}
|
||||
// Recorded in the hand-act log, each one (ADR 0565 §10; the operator's "option A").
|
||||
why := s.Why
|
||||
if why == "" {
|
||||
why = "eligible: no want names it and it is not among its module's " + fmt.Sprint(KeptBuilds) + " newest builds"
|
||||
}
|
||||
if err := h.removeOnSeat(r, why); err != nil {
|
||||
a.Stopped = fmt.Sprintf("a registry seat refused to remove %s, so nothing more was asked: %v", r, err)
|
||||
a.Left = len(eligible) - i
|
||||
h.sweepFinding("the removal of "+r, err)
|
||||
refused = true
|
||||
break
|
||||
}
|
||||
if _, err := h.Acts.Record(meshhandactsclient.Request{What: "let go of " + r + " (" + s.By + ")", Why: why,
|
||||
if _, err := h.Acts.Record(handacts.Request{What: "let go of " + r + " (" + s.By + ")", Why: why,
|
||||
Cause: "artifact-store-sweep", Verb: "collect"}); err != nil {
|
||||
h.logf("%s was let go of and could not be recorded in the hand-act log: %v", r, err)
|
||||
}
|
||||
a.LetGo = append(a.LetGo, r)
|
||||
}
|
||||
h.sayGone(j, a.LetGo, "let go of")
|
||||
if !refused {
|
||||
h.sweepClean()
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// sayGone says artifacts-gone for each build that named one of refs: the builder builds a key again whose artifact is gone.
|
||||
func (h *Holder) sayGone(j *judged, refs []string, how string) {
|
||||
byKey := map[string]*ArtifactsGone{}
|
||||
byKey := map[string]*delivery.ArtifactsGone{}
|
||||
var keys []string
|
||||
for _, r := range refs {
|
||||
ref := j.refs[r]
|
||||
@@ -626,17 +646,16 @@ func (h *Holder) sayGone(j *judged, refs []string, how string) {
|
||||
for _, b := range ref.Builds {
|
||||
g := byKey[b.Key]
|
||||
if g == nil {
|
||||
at := h.now().UTC()
|
||||
g = &ArtifactsGone{Module: b.Module, Key: b.Key, Fingerprint: b.FP, At: &at}
|
||||
g = &delivery.ArtifactsGone{Module: b.Module, Key: b.Key, Fingerprint: b.FP, At: h.now().UTC()}
|
||||
byKey[b.Key] = g
|
||||
keys = append(keys, b.Key)
|
||||
}
|
||||
g.Gone = append(g.Gone, namedBy(b, r))
|
||||
g.Gone = append(g.Gone, r)
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
raw, err := EncodeArtifactsGone(*byKey[k])
|
||||
raw, err := delivery.EncodeArtifactsGone(*byKey[k])
|
||||
if err == nil {
|
||||
err = h.Bus.Publish(GoneSubject, json.RawMessage(raw))
|
||||
}
|
||||
@@ -646,19 +665,11 @@ func (h *Holder) sayGone(j *judged, refs []string, how string) {
|
||||
}
|
||||
}
|
||||
|
||||
// namedBy is the reference b names r by, in the form its key holds (the older one or the seat's): the builder matches
|
||||
// artifacts-gone against its own key.
|
||||
func namedBy(b Build, r string) string {
|
||||
for _, a := range b.Artifacts {
|
||||
if rec := registryForms.Recorded(a.Reference); canonical(rec) == r {
|
||||
return rec
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// AfterBuild is the sweep at a build's end.
|
||||
// AfterBuild is the sweep at a build's end: only once the lease names no controller.
|
||||
func (h *Holder) AfterBuild(key string) {
|
||||
if !h.LeaseAway() {
|
||||
return
|
||||
}
|
||||
a, err := h.Sweep(context.Background(), Sweeping{Most: MostPerSweep, By: "after the build " + key})
|
||||
if err != nil {
|
||||
h.logf("the sweep after %s: %v", key, err)
|
||||
@@ -670,30 +681,30 @@ func (h *Holder) AfterBuild(key string) {
|
||||
}
|
||||
|
||||
// Held is the seat's `held`: each reference, held, gone or unknown with why. Holds and writes nothing.
|
||||
func (h *Holder) Held(refs []string) (HeldAnswer, error) {
|
||||
func (h *Holder) Held(refs []string) (delivery.HeldAnswer, error) {
|
||||
if len(refs) == 0 {
|
||||
return HeldAnswer{}, errors.New("held judges the references given, and none was")
|
||||
return delivery.HeldAnswer{}, errors.New("held judges the references given, and none was")
|
||||
}
|
||||
if len(refs) > 64 {
|
||||
return HeldAnswer{}, fmt.Errorf("held judges at most 64 references at once, and %d were given", len(refs))
|
||||
return delivery.HeldAnswer{}, fmt.Errorf("held judges at most 64 references at once, and %d were given", len(refs))
|
||||
}
|
||||
for _, r := range refs {
|
||||
if _, ours := registryForms.InArtifactStore(r); !ours {
|
||||
return HeldAnswer{}, fmt.Errorf("%s is not a reference into the mesh's artifact store (artifact-store://...): nothing was asked", r)
|
||||
if _, ours := catalogue.InArtifactStore(r); !ours {
|
||||
return delivery.HeldAnswer{}, fmt.Errorf("%s is not a reference into the mesh's artifact store (artifact-store://...): nothing was asked", r)
|
||||
}
|
||||
}
|
||||
door, err := h.Door()
|
||||
if err != nil {
|
||||
return HeldAnswer{}, err
|
||||
return delivery.HeldAnswer{}, err
|
||||
}
|
||||
asking, stop := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer stop()
|
||||
a := HeldAnswer{Store: door.Address(), Held: []string{}, Gone: []string{}}
|
||||
a := delivery.HeldAnswer{Store: door.Address(), Held: []string{}, Gone: []string{}}
|
||||
for _, r := range refs {
|
||||
switch err := door.Holds(asking, canonical(r)); {
|
||||
switch err := door.Holds(asking, r); {
|
||||
case err == nil:
|
||||
a.Held = append(a.Held, r)
|
||||
case errors.Is(err, Gone):
|
||||
case errors.Is(err, registry.Gone):
|
||||
a.Gone = append(a.Gone, r)
|
||||
default:
|
||||
if a.Unknown == nil {
|
||||
@@ -729,7 +740,7 @@ func (h *Holder) ImagesOf(node string, address string) (Images, error) {
|
||||
}
|
||||
byImage := map[string]*Image{}
|
||||
add := func(module, fp, in string) error {
|
||||
key, err := builds.BuilderBuildKey(module, fp)
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
@@ -737,14 +748,14 @@ func (h *Holder) ImagesOf(node string, address string) (Images, error) {
|
||||
if err != nil || !found {
|
||||
return err
|
||||
}
|
||||
b, err := builds.DecodeBuilderBuild(raw)
|
||||
b, err := delivery.DecodeBuilderBuild(raw)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
images := []string{}
|
||||
for _, a := range b.Artifacts {
|
||||
if (a.Kind == "image" || a.Kind == "upstream") && a.Reference != "" {
|
||||
images = append(images, registryForms.Routed(registryForms.Recorded(a.Reference), address))
|
||||
images = append(images, catalogue.Routed(catalogue.Recorded(a.Reference), address))
|
||||
}
|
||||
}
|
||||
pinned, err := h.Pinned(b.Manifest)
|
||||
@@ -778,7 +789,7 @@ func (h *Holder) ImagesOf(node string, address string) (Images, error) {
|
||||
return out, fmt.Errorf("the haves: %w", err)
|
||||
}
|
||||
for _, k := range keys {
|
||||
n, module, isNode, err := engineclient.ParseHaveKey(k)
|
||||
n, module, isNode, err := delivery.ParseHaveKey(k)
|
||||
if err != nil || isNode || n != node {
|
||||
continue
|
||||
}
|
||||
@@ -786,12 +797,12 @@ func (h *Holder) ImagesOf(node string, address string) (Images, error) {
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
var have nodeengine.HaveByNodeModuleValue
|
||||
if !found || json.Unmarshal(raw, &have) != nil || have.Fp == "" {
|
||||
var have delivery.Have
|
||||
if !found || json.Unmarshal(raw, &have) != nil || have.FP == "" {
|
||||
continue
|
||||
}
|
||||
out.SentKnown = true
|
||||
if err := add(module, have.Fp, "sent"); err != nil {
|
||||
if err := add(module, have.FP, "sent"); err != nil {
|
||||
return out, err
|
||||
}
|
||||
}
|
||||
@@ -810,7 +821,7 @@ func (h *Holder) Pinned(manifest string) ([]string, error) {
|
||||
if manifest == "" {
|
||||
return nil, nil
|
||||
}
|
||||
key, err := catalogclient.ManifestsKey(manifest)
|
||||
key, err := delivery.ManifestsKey(manifest)
|
||||
if err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -818,14 +829,14 @@ func (h *Holder) Pinned(manifest string) ([]string, error) {
|
||||
if err != nil || !found {
|
||||
return nil, err
|
||||
}
|
||||
m, err := catalogclient.DecodeManifest(raw)
|
||||
m, err := delivery.DecodeManifest(raw)
|
||||
if err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
var parsed struct {
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if json.Unmarshal([]byte(m.Manifest), &parsed) != nil {
|
||||
if json.Unmarshal([]byte(m.JSON), &parsed) != nil {
|
||||
return nil, nil
|
||||
}
|
||||
var out []string
|
||||
@@ -845,8 +856,8 @@ func inStoreAt(image, address string) string {
|
||||
if address == "" || !strings.HasPrefix(image, address+"/") || !strings.Contains(image, "@sha256:") {
|
||||
return ""
|
||||
}
|
||||
r := registryForms.Recorded(image)
|
||||
if _, ours := registryForms.InArtifactStore(r); !ours {
|
||||
r := catalogue.Recorded(image)
|
||||
if _, ours := catalogue.InArtifactStore(r); !ours {
|
||||
return ""
|
||||
}
|
||||
return r
|
||||
@@ -870,7 +881,7 @@ func (h *Holder) Observe(ctx context.Context) error {
|
||||
for _, b := range j.built {
|
||||
builds[b.Module+" "+b.FP] = b
|
||||
}
|
||||
now := map[string]meshconditions.FindingsByKeyValue{}
|
||||
now := map[string]health.Finding{}
|
||||
for _, w := range j.wants {
|
||||
b, ok := builds[w.Module+" "+w.FP]
|
||||
if !ok {
|
||||
@@ -879,7 +890,7 @@ func (h *Holder) Observe(ctx context.Context) error {
|
||||
var refs []string
|
||||
for _, a := range b.Artifacts {
|
||||
if a.Kind == "image" {
|
||||
refs = append(refs, registryForms.Recorded(a.Reference))
|
||||
refs = append(refs, catalogue.Recorded(a.Reference))
|
||||
}
|
||||
}
|
||||
// And what its manifest pins by itself under the store's address, as the controller judged S25.
|
||||
@@ -904,16 +915,7 @@ func (h *Holder) Observe(ctx context.Context) error {
|
||||
return fmt.Errorf("the findings: %w", err)
|
||||
}
|
||||
for _, k := range keys {
|
||||
if k == sweepFindingKey {
|
||||
continue // the sweep's own finding, which the sweep clears
|
||||
}
|
||||
if _, still := now[k]; !still {
|
||||
// A finding of a first view unanswered is the reader's (views.Reader), cleared by its answer, not by a sweep.
|
||||
if raw, found, err := h.Bus.Get(FindingsState, k); err == nil && found {
|
||||
if f, err := meshconditions.DecodeFindingsByKeyValue(raw); err == nil && f.Kind == views.UnansweredKind {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := h.Bus.Delete(FindingsState, k); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -925,11 +927,11 @@ func (h *Holder) Observe(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
if found {
|
||||
if was, err := meshconditions.DecodeFindingsByKeyValue(raw); err == nil && was.Summary == f.Summary {
|
||||
if was, err := health.DecodeFinding(raw); err == nil && was.Summary == f.Summary {
|
||||
continue // written only when what it says changes
|
||||
}
|
||||
}
|
||||
if err := meshconditionsclient.CheckFinding(f); err != nil {
|
||||
if _, err := health.EncodeFinding(f); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := h.Bus.Put(FindingsState, k, f); err != nil {
|
||||
@@ -940,11 +942,11 @@ func (h *Holder) Observe(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// imageMissing is S25's finding, in the controller's words (cmd/mesh-controller/image_missing.go watchMissingImages).
|
||||
func imageMissing(node, module, image string, at time.Time) meshconditions.FindingsByKeyValue {
|
||||
func imageMissing(node, module, image string, at time.Time) health.Finding {
|
||||
id := module + "." + node
|
||||
return meshconditions.FindingsByKeyValue{V: meshconditionsclient.FindingVersion, Key: meshconditionsclient.Key(meshconditionsclient.ScopeModule, id, ImageMissing), Kind: ImageMissing,
|
||||
Subject: meshconditions.Subject{Scope: meshconditionsclient.ScopeModule, ID: id, Machine: node},
|
||||
Severity: meshconditionsclient.Warning, Resolver: meshconditionsclient.ResolverOperator,
|
||||
return health.Finding{V: health.FindingVersion, Key: health.FindingKey(health.ScopeModule, id, ImageMissing), Kind: ImageMissing,
|
||||
Subject: health.Subject{Scope: health.ScopeModule, ID: id, Machine: node},
|
||||
Severity: health.Warning, Resolver: health.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s is left out of every send to %s: its declaration names an image that the artifact store does "+
|
||||
"not hold (the evidence names it); the running container is untouched", module, node),
|
||||
Headline: capital(module) + " is not updated on " + node,
|
||||
@@ -952,7 +954,7 @@ func imageMissing(node, module, image string, at time.Time) meshconditions.Findi
|
||||
"was not sent to %s. What runs there is untouched, and the rest of %s is sent as usual.", module, node, node),
|
||||
Needs: fmt.Sprintf("build %s again; the details name the image.", module),
|
||||
Resolved: "Resolved: the store holds the image of " + module + " again",
|
||||
Evidence: []meshconditions.Evidence{{At: at.UTC(), Said: "image " + image + " is not in the store"}},
|
||||
Evidence: []health.Evidence{{At: at.UTC(), Said: "image " + image + " is not in the store"}},
|
||||
Source: "S25", Observer: Module, At: at.UTC()}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
package main
|
||||
|
||||
// The seat mesh-artifact-sweep's holder whole against a real bus (novox/hq ADR 0571 clause 2, issue 1955): the builder's done
|
||||
// keys and the wants in a real nats-server's key-value buckets, no lease key anywhere (issue 1126), a build's end heard by a watch
|
||||
// keys, the wants and the controller's lease in a real nats-server's key-value buckets, a build's end heard by a watch
|
||||
// of the builds as the runtime hands it over, each deletion recorded by a request to the seat mesh-hand-acts on the bus,
|
||||
// artifacts-gone said on the seat's subject, and S25's finding written to the holder's own bucket. Only the store's
|
||||
// door is in memory: its HTTP is the copied registry package's, tested there.
|
||||
@@ -19,9 +19,9 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
meshhandacts "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
)
|
||||
|
||||
@@ -117,7 +117,7 @@ func (b realBus) Publish(subject string, body any) error {
|
||||
return b.nc.Publish(subject, raw)
|
||||
}
|
||||
|
||||
func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) {
|
||||
func TestTheSeatSweepsAtABuildsEndOnARealBusOnceTheLeaseIsAway(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
nc, err := nats.Connect(testbus.URL(t))
|
||||
@@ -129,7 +129,7 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range []string{BuildsRead, WantsRead, HavesRead, ManifestsRead, "artifact-store-tools." + FindingsState} {
|
||||
for _, s := range []string{BuildsRead, WantsRead, HavesRead, LeaseRead, ManifestsRead, "artifact-store-tools." + FindingsState} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucketOf(s), History: 5}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -137,15 +137,9 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) {
|
||||
bus := realBus{ctx: ctx, nc: nc, js: js}
|
||||
|
||||
// The mesh as aMesh lays it, copied onto the real buckets: letta's seven builds, a want, the store's listing.
|
||||
mem, memb, door, _ := aMesh(t)
|
||||
// The builder's log as written today: aMesh dates it from 2026-10-01 for its fixed clock, and this holder runs on the
|
||||
// real clock, so read as dated the entries fall out of the log's 30 days and every copy is kept as unrecorded.
|
||||
shift := time.Since(memb.log[len(memb.log)-1].At)
|
||||
for _, e := range memb.log {
|
||||
e.At = e.At.Add(shift)
|
||||
bus.log = append(bus.log, e)
|
||||
}
|
||||
for _, s := range []string{BuildsRead, WantsRead, ManifestsRead} {
|
||||
mem, memb, door, _ := aMesh(t, &buscompose.LeaseHolder{Instance: "c1", Host: "novox", Epoch: 3})
|
||||
bus.log = memb.log
|
||||
for _, s := range []string{BuildsRead, WantsRead, LeaseRead, ManifestsRead} {
|
||||
keys, _ := memb.Keys(s)
|
||||
for _, k := range keys[:min(len(keys), 6)] { // the seventh build ends later, on the bus
|
||||
raw, _, _ := memb.Get(s, k)
|
||||
@@ -159,7 +153,7 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) {
|
||||
// The seat mesh-hand-acts, answering every act it records; artifacts-gone heard on the seat's subject.
|
||||
var mu sync.Mutex
|
||||
var acts, gone []string
|
||||
if _, err := nc.Subscribe("mesh.seat."+meshhandacts.Seat+".tool.hand-act", func(m *nats.Msg) {
|
||||
if _, err := nc.Subscribe("mesh.seat."+handacts.Seat+".tool.hand-act", func(m *nats.Msg) {
|
||||
mu.Lock()
|
||||
acts = append(acts, string(m.Data))
|
||||
mu.Unlock()
|
||||
@@ -174,32 +168,15 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) {
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The registry seats' holder on the bus (CAT-AS1): the sweep holds and removes through their verbs.
|
||||
for _, seat := range []string{"oci-artifact-registry", "container-image-registry"} {
|
||||
if _, err := nc.Subscribe("mesh.seat."+seat+".tool.*", func(m *nats.Msg) {
|
||||
var body map[string]any
|
||||
_ = json.Unmarshal(m.Data, &body)
|
||||
mu.Lock()
|
||||
out, err := door.seat("seat:"+seat+"."+m.Subject[strings.LastIndex(m.Subject, ".")+1:], body)
|
||||
mu.Unlock()
|
||||
if err != nil {
|
||||
out, _ = json.Marshal(map[string]string{"error": err.Error()})
|
||||
}
|
||||
_ = m.Respond(out)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ask := func(key string, body any) (json.RawMessage, error) {
|
||||
h := &Holder{Bus: findingsAt{realBus: bus}, Door: mem.Door, Listed: mem.Listed, Now: time.Now, Logf: t.Logf}
|
||||
h.Acts = handacts.Recorder{LeaseAway: h.LeaseAway, Ask: func(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
m, err := nc.Request("mesh.seat."+strings.TrimPrefix(strings.Replace(key, ".", ".tool.", 1), "seat:"), raw, 5*time.Second)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return m.Data, nil
|
||||
}
|
||||
h := &Holder{Bus: findingsAt{realBus: bus}, Door: mem.Door, Listed: mem.Listed, Seats: ask, Now: time.Now, Logf: t.Logf}
|
||||
h.Acts = meshhandactsclient.Recorder{Ask: ask}
|
||||
}}
|
||||
|
||||
// A build's end heard from the builds bucket, as follow hands it over.
|
||||
kv, _ := bus.kv(BuildsRead)
|
||||
@@ -221,10 +198,18 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// No lease key: the build's end holds the kept archives and lets go of letta's 2nd, which the 7th moved past the five
|
||||
// newest and no want names (the want keeps the 1st), recording each deletion.
|
||||
// While the lease names the controller, a build's end lets nothing go.
|
||||
ended()
|
||||
deadline := time.Now().Add(30 * time.Second)
|
||||
if len(door.asked) != 0 {
|
||||
t.Fatalf("before the lease write the store was asked %v", door.asked)
|
||||
}
|
||||
// The lease handed away: the next build's end holds the kept archives and lets go of letta's 2nd, which the 7th moved
|
||||
// past the five newest and no want names (the want keeps the 1st), recording each deletion.
|
||||
if err := bus.Put(LeaseRead, buscompose.ControllerLeaseKey, buscompose.LeaseHolder{HandedTo: "mesh-grants", Why: "the lease write"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ended()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
mu.Lock()
|
||||
n := len(gone)
|
||||
|
||||
@@ -3,28 +3,20 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
catalogclient "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
||||
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
|
||||
artifactref "git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient/reference"
|
||||
reg "git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry"
|
||||
ociseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry"
|
||||
"artifactstoretools/internal/registry"
|
||||
)
|
||||
|
||||
// memBus is the holder's bus in memory: states by name, each key's history, and what was published.
|
||||
@@ -39,7 +31,7 @@ type memBus struct {
|
||||
func (b *memBus) LogSince(read string, at time.Time) ([]stdio.LogEntry, error) {
|
||||
var out []stdio.LogEntry
|
||||
for _, e := range b.log {
|
||||
if read == builds.BuildLogRead && !e.At.Before(at) {
|
||||
if read == delivery.BuildLogRead && !e.At.Before(at) {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
@@ -94,63 +86,37 @@ func (b *memBus) Publish(subject string, body any) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// memDoor is the store's door in memory, read only, and the registry seats' holder: what it holds, which references are
|
||||
// indexes, what the seats were asked (asked: "hold <ref>", "let go <ref>"; calls: each seat call as its key and name), the
|
||||
// holds asked (held), and the calls it refuses (refuse: by "<verb> <ref>").
|
||||
// memDoor is the store's door in memory: what it holds, which references are indexes, and what was asked.
|
||||
type memDoor struct {
|
||||
holds map[string]bool
|
||||
indexes map[string]bool
|
||||
asked []string
|
||||
calls []string
|
||||
held map[string]bool
|
||||
refuse map[string]error
|
||||
}
|
||||
|
||||
// seat is the registry seats' holder, as Holder.Seats calls it.
|
||||
func (d *memDoor) seat(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
var in struct{ Name, Digest, Reason string }
|
||||
_ = json.Unmarshal(raw, &in)
|
||||
seat, verb, _ := strings.Cut(strings.TrimPrefix(key, "seat:"), ".")
|
||||
ref := artifactref.ArtifactStoreScheme + in.Name + "@" + in.Digest
|
||||
if seat == "oci-artifact-registry" {
|
||||
ref = artifactref.ArtifactStoreScheme + in.Name + "/blobs/" + in.Digest
|
||||
}
|
||||
d.calls = append(d.calls, key+" "+in.Name+"@"+in.Digest+" "+in.Reason)
|
||||
if err := d.refuse[verb+" "+ref]; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if d.held == nil {
|
||||
d.held = map[string]bool{}
|
||||
}
|
||||
switch verb {
|
||||
case "hold":
|
||||
d.asked = append(d.asked, "hold "+ref)
|
||||
d.held[ref] = true
|
||||
return json.Marshal(map[string]any{"held": true})
|
||||
case "release":
|
||||
released := d.held[ref] // what is not held answers released false, as the seat's definition says
|
||||
delete(d.held, ref)
|
||||
return json.Marshal(map[string]any{"released": released})
|
||||
case "remove":
|
||||
if d.held[ref] {
|
||||
return nil, fmt.Errorf("remove: refused: %s@%s is held; release it first", in.Name, in.Digest)
|
||||
}
|
||||
d.asked = append(d.asked, "let go "+ref)
|
||||
delete(d.holds, ref)
|
||||
return json.Marshal(map[string]any{"removed": true})
|
||||
}
|
||||
return nil, fmt.Errorf("no verb %s", key)
|
||||
}
|
||||
|
||||
func (d *memDoor) Address() string { return "10.0.0.5:5000" }
|
||||
func (d *memDoor) Hold(_ context.Context, r string) (bool, error) {
|
||||
d.asked = append(d.asked, "hold "+r)
|
||||
if !d.holds[r] {
|
||||
return false, registry.Gone
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
func (d *memDoor) Holds(_ context.Context, r string) error {
|
||||
if !d.holds[r] {
|
||||
return Gone
|
||||
return registry.Gone
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (d *memDoor) IsIndex(_ context.Context, r string) (bool, error) { return d.indexes[r], nil }
|
||||
func (d *memDoor) LetGo(_ context.Context, r string) error {
|
||||
d.asked = append(d.asked, "let go "+r)
|
||||
delete(d.holds, r)
|
||||
return nil
|
||||
}
|
||||
func (d *memDoor) SpareKept(context.Context, []string, []string) error {
|
||||
d.asked = append(d.asked, "spare")
|
||||
return nil
|
||||
}
|
||||
|
||||
// mirror is a copy of an upstream base in the store, the n-th.
|
||||
func mirror(n int) string {
|
||||
@@ -173,16 +139,16 @@ func archiveRef(module string, n int) string {
|
||||
}
|
||||
|
||||
// A mesh with seven done builds of letta, the n-th ending n hours after a day began, each with an image and an archive;
|
||||
// a want names the first on ace; the store lists them all and one image no build names; no lease key (issue 1126).
|
||||
func aMesh(t *testing.T) (*Holder, *memBus, *memDoor, *[]string) {
|
||||
// a want names the first on ace; the store lists them all and one image no build names; the lease as given.
|
||||
func aMesh(t *testing.T, lease *buscompose.LeaseHolder) (*Holder, *memBus, *memDoor, *[]string) {
|
||||
t.Helper()
|
||||
day := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
|
||||
b := &memBus{states: map[string]map[string][]byte{}, history: map[string][]stdio.StateRevision{}}
|
||||
door := &memDoor{holds: map[string]bool{}, indexes: map[string]bool{}}
|
||||
listing := &Listing{Blobs: map[string]int64{}, Repos: map[string]*Repo{}}
|
||||
for n := 1; n <= 7; n++ {
|
||||
key, _ := builds.BuilderBuildKey("letta", fp(n))
|
||||
raw, err := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: strings.Repeat("d", 64), Artifacts: []builds.BuiltArtifact{
|
||||
key, _ := delivery.BuilderBuildKey("letta", fp(n))
|
||||
raw, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: strings.Repeat("d", 64), Artifacts: []delivery.BuiltArtifact{
|
||||
{Name: "image", Kind: "image", Digest: digestOf(imageRef("letta", n)), Reference: imageRef("letta", n)},
|
||||
{Name: "files", Kind: "archive", Digest: digestOf(archiveRef("letta", n)), Reference: archiveRef("letta", n)},
|
||||
{Name: "pkg", Kind: "package", Digest: "sha256:" + strings.Repeat("f", 64), Reference: "letta@1.0." + fmt.Sprint(n)}}})
|
||||
@@ -210,7 +176,7 @@ func aMesh(t *testing.T) (*Holder, *memBus, *memDoor, *[]string) {
|
||||
door.holds[mirror(n)] = true
|
||||
}
|
||||
for n := 1; n <= 7; n++ {
|
||||
e := builds.BuildLogEntry{Event: builds.BuildEndedEvent, Module: "letta", FP: fp(n), Repository: "novox/x", Commit: "c",
|
||||
e := delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: "letta", FP: fp(n), Repository: "novox/x", Commit: "c",
|
||||
Node: "ace", At: day.Add(time.Duration(n) * time.Hour), Outcome: "done"}
|
||||
switch n {
|
||||
case 2:
|
||||
@@ -218,33 +184,40 @@ func aMesh(t *testing.T) (*Holder, *memBus, *memDoor, *[]string) {
|
||||
case 7:
|
||||
e.Against = []string{mirror(2)}
|
||||
}
|
||||
raw, _ := builds.EncodeBuildLogEntry(e)
|
||||
raw, _ := delivery.EncodeBuildLogEntry(e)
|
||||
b.log = append(b.log, stdio.LogEntry{Sequence: uint64(n), Key: "letta", Value: raw, At: e.At})
|
||||
}
|
||||
manifest, _ := catalogclient.EncodeManifest(catalog.ManifestsByDigestValue{Module: "letta",
|
||||
Manifest: `{"module":"letta","resources":[{"id":"app","type":"container","image":"` + pinnedImage + `"},` +
|
||||
manifest, _ := delivery.EncodeManifest(delivery.Manifest{Module: "letta",
|
||||
JSON: `{"module":"letta","resources":[{"id":"app","type":"container","image":"` + pinnedImage + `"},` +
|
||||
`{"id":"built","type":"container","image":"${artifact:image}"}]}`})
|
||||
mk, _ := catalogclient.ManifestsKey(strings.Repeat("d", 64))
|
||||
mk, _ := delivery.ManifestsKey(strings.Repeat("d", 64))
|
||||
_ = b.Put(ManifestsRead, mk, json.RawMessage(manifest))
|
||||
want, _ := json.Marshal(meshwants.WantsByWantNodeModuleValue{V: 1, Fp: fp(1), Self: true})
|
||||
want, _ := json.Marshal(delivery.Want{V: delivery.Version, FP: fp(1), Self: true})
|
||||
_ = b.Put(WantsRead, "want.ace.letta", json.RawMessage(want))
|
||||
if lease != nil {
|
||||
_ = b.Put(LeaseRead, buscompose.ControllerLeaseKey, lease)
|
||||
}
|
||||
var acts []string
|
||||
h := &Holder{Bus: b, Door: func() (Door, error) { return door, nil }, Seats: door.seat,
|
||||
h := &Holder{Bus: b, Door: func() (Door, error) { return door, nil },
|
||||
Listed: func(context.Context) (*View, error) { return &View{L: listing}, nil },
|
||||
Now: func() time.Time { return day.Add(48 * time.Hour) },
|
||||
Acts: meshhandactsclient.Recorder{Ask: func(key string, body any) (json.RawMessage, error) {
|
||||
Acts: handacts.Recorder{Ask: func(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
acts = append(acts, key+" "+string(raw))
|
||||
return json.RawMessage(`{}`), nil
|
||||
}}}
|
||||
h.Acts.LeaseAway = h.LeaseAway
|
||||
return h, b, door, &acts
|
||||
}
|
||||
|
||||
// ADR 0571 clause 2, 1126/A4/AC2: the keep set from the store's listing, the wants and the five newest builds; with no
|
||||
// lease key, a build's end and a confirmed collect let go of the eligible within the bound, leave an index for a
|
||||
// confirmed collect, record each deletion as a hand act and say artifacts-gone.
|
||||
func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsWithNoLease(t *testing.T) {
|
||||
h, _, _, _ := aMesh(t)
|
||||
// ADR 0571 clause 2: the keep set from the store's listing, the wants and the five newest builds; nothing let go of while
|
||||
// the lease names a controller; once it names another holder, a build's end lets go of the eligible within the bound,
|
||||
// leaves an index for a confirmed collect, records each deletion and says artifacts-gone.
|
||||
func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsOnlyOnceTheLeaseIsAway(t *testing.T) {
|
||||
controller := &buscompose.LeaseHolder{Instance: "c1", Host: "novox", Epoch: 3}
|
||||
handed := &buscompose.LeaseHolder{HandedTo: "mesh-grants", HandedBy: "person.jochen", Why: "the lease write"}
|
||||
|
||||
h, _, _, _ := aMesh(t, controller)
|
||||
r, err := h.Artifacts("")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -271,121 +244,56 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsWithNoLease(t *testing.T)
|
||||
t.Errorf("the references are %d, counted %v, at %q", len(r.References), r.Counts, r.Store)
|
||||
}
|
||||
|
||||
all := []string{archiveRef("letta", 2), imageRef("letta", 2), mirror(1)}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
how string // "build" (a build's end), "collect" (a confirmed collect), or a sweep asked with most
|
||||
lease *buscompose.LeaseHolder
|
||||
index bool
|
||||
most int
|
||||
letGo []string
|
||||
left int
|
||||
skipped int
|
||||
unlog int
|
||||
// held is how many kept archives and copies are held (7 when 0); seatForm names letta's 2nd and 3rd builds in the
|
||||
// seat's form; refuse is the seat call refused, "<verb> <ref>".
|
||||
held int
|
||||
seatForm bool
|
||||
refuse string
|
||||
}{
|
||||
{name: "a build's end", how: "build", letGo: all},
|
||||
// ADR 0605 §9: a build that names its artifacts in the seat's form is the same artifact: removed and held under the
|
||||
// seat's name and digest, and said gone in its own form.
|
||||
{name: "the seat's form", most: MostPerSweep, letGo: all, seatForm: true},
|
||||
// A seat that refuses a removal stops the sweep; the finding sweep-refused is written, and cleared by a clean run.
|
||||
{name: "a refused removal", most: MostPerSweep, letGo: []string{archiveRef("letta", 2)}, left: 2, refuse: "remove " + imageRef("letta", 2)},
|
||||
{name: "a confirmed collect", how: "collect", letGo: all},
|
||||
{name: "a sweep", most: MostPerSweep, letGo: all},
|
||||
{name: "an index waits", index: true, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), mirror(1)}, skipped: 1},
|
||||
{name: "bounded", most: 1, letGo: []string{archiveRef("letta", 2)}, left: 2},
|
||||
{name: "the lease names a controller", lease: controller},
|
||||
{name: "no lease key", lease: nil},
|
||||
{name: "handed away", lease: handed, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2), mirror(1)}},
|
||||
{name: "an index waits", lease: handed, index: true, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), mirror(1)}, skipped: 1},
|
||||
{name: "bounded", lease: handed, most: 1, letGo: []string{archiveRef("letta", 2)}, left: 2},
|
||||
// A kept build the log no longer holds: what it stood on is unknown, so no copy goes.
|
||||
{name: "a kept build's bases unknown", most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2)}, unlog: 3, held: 8},
|
||||
{name: "a kept build's bases unknown", lease: handed, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2)}, unlog: 3},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
h, b, door, acts := aMesh(t)
|
||||
h, b, door, acts := aMesh(t, c.lease)
|
||||
door.indexes[imageRef("letta", 2)] = c.index
|
||||
if c.held == 0 {
|
||||
c.held = 7
|
||||
}
|
||||
if c.seatForm {
|
||||
for _, n := range []int{2, 3} {
|
||||
key, _ := builds.BuilderBuildKey("letta", fp(n))
|
||||
raw, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: strings.Repeat("d", 64), Artifacts: []builds.BuiltArtifact{
|
||||
{Name: "image", Kind: "image", Digest: digestOf(imageRef("letta", n)), Reference: artifactref.SeatReference(reg.Seat, "letta", "image", digestOf(imageRef("letta", n)))},
|
||||
{Name: "files", Kind: "archive", Digest: digestOf(archiveRef("letta", n)), Reference: artifactref.SeatReference(ociseat.Seat, "letta", "files", digestOf(archiveRef("letta", n)))}}})
|
||||
_ = b.Put(BuildsRead, key, json.RawMessage(raw))
|
||||
}
|
||||
}
|
||||
if c.refuse != "" {
|
||||
door.refuse = map[string]error{c.refuse: errors.New("the store is read only")}
|
||||
}
|
||||
if c.unlog > 0 {
|
||||
b.log = append(b.log[:c.unlog-1:c.unlog-1], b.log[c.unlog:]...)
|
||||
}
|
||||
switch c.how {
|
||||
case "build":
|
||||
if c.most == 0 {
|
||||
h.AfterBuild("letta.x")
|
||||
var gone []string
|
||||
for _, x := range door.asked {
|
||||
if r, ok := strings.CutPrefix(x, "let go "); ok {
|
||||
gone = append(gone, r)
|
||||
}
|
||||
if _, err := h.Collect("tidy", true, 0); err == nil {
|
||||
t.Error("a confirmed collect was let through while the controller sweeps")
|
||||
}
|
||||
if fmt.Sprint(gone) != fmt.Sprint(c.letGo) {
|
||||
t.Fatalf("a build's end let go of %v, want %v", gone, c.letGo)
|
||||
}
|
||||
default:
|
||||
var a *CollectAnswer
|
||||
var err error
|
||||
if c.how == "collect" {
|
||||
a, err = h.Collect("tidy", true, 0)
|
||||
} else {
|
||||
a, err = h.Sweep(context.Background(), Sweeping{Most: c.most, By: "after the build letta.x"})
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fmt.Sprint(a.LetGo) != fmt.Sprint(c.letGo) || a.Left != c.left || a.Skipped != c.skipped || a.Held != c.held || a.Missing != 0 {
|
||||
t.Fatalf("let go of %v (left %d, skipped %d, held %d, missing %d), want %v", a.LetGo, a.Left, a.Skipped, a.Held, a.Missing, c.letGo)
|
||||
}
|
||||
}
|
||||
// Every kept archive held before the first deletion.
|
||||
held := 0
|
||||
for _, x := range door.asked {
|
||||
if strings.HasPrefix(x, "let go ") {
|
||||
break
|
||||
}
|
||||
if strings.HasPrefix(x, "hold ") {
|
||||
held++
|
||||
}
|
||||
}
|
||||
if held != c.held {
|
||||
t.Errorf("the seats were asked %v: the %d kept archives and copies are held first", door.asked, c.held)
|
||||
}
|
||||
// Each removal asks the seat that holds it, by its name and digest, for retention, after a release.
|
||||
for _, call := range door.calls {
|
||||
if strings.Contains(call, ".remove ") && !strings.HasSuffix(call, " "+Retention) {
|
||||
t.Errorf("a removal asked %q, not for retention", call)
|
||||
}
|
||||
}
|
||||
if c.letGo[0] == archiveRef("letta", 2) && !slices.Contains(door.calls, "seat:oci-artifact-registry.remove letta/files@"+digestOf(archiveRef("letta", 2))+" "+Retention) {
|
||||
t.Errorf("the seats were called %v", door.calls)
|
||||
}
|
||||
_, finding, _ := b.Get(FindingsState, sweepFindingKey)
|
||||
if finding != (c.refuse != "") {
|
||||
t.Errorf("the sweep's finding is written: %v, want %v", finding, c.refuse != "")
|
||||
}
|
||||
if c.refuse != "" {
|
||||
door.refuse = nil
|
||||
if _, err := h.Sweep(context.Background(), Sweeping{Most: MostPerSweep, By: "again"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, still, _ := b.Get(FindingsState, sweepFindingKey); still {
|
||||
t.Error("a clean sweep left the sweep's finding")
|
||||
if len(door.asked) != 0 || len(*acts) != 0 || len(b.published) != 0 {
|
||||
t.Fatalf("before the lease write the store was asked %v, acts %v, said %v", door.asked, *acts, b.published)
|
||||
}
|
||||
return
|
||||
}
|
||||
if c.seatForm && !strings.Contains(b.published[0], `"container-image-registry/letta/image@`+digestOf(imageRef("letta", 2))+`"`) {
|
||||
t.Errorf("artifacts-gone does not name the build's own form: %v", b.published)
|
||||
a, err := h.Sweep(context.Background(), Sweeping{Most: c.most, By: "after the build letta.x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fmt.Sprint(a.LetGo) != fmt.Sprint(c.letGo) || a.Left != c.left || a.Skipped != c.skipped || a.Held != 6 || a.Missing != 0 {
|
||||
t.Fatalf("let go of %v (left %d, skipped %d, held %d, missing %d), want %v", a.LetGo, a.Left, a.Skipped, a.Held, a.Missing, c.letGo)
|
||||
}
|
||||
// Every kept archive held before the first deletion.
|
||||
first := -1
|
||||
for i, x := range door.asked {
|
||||
if strings.HasPrefix(x, "let go ") && first < 0 {
|
||||
first = i
|
||||
}
|
||||
}
|
||||
if first != 6 {
|
||||
t.Errorf("the store was asked %v: the six kept archives are held first", door.asked)
|
||||
}
|
||||
if len(*acts) != len(c.letGo) || !strings.HasPrefix((*acts)[0], "seat:mesh-hand-acts.hand-act ") ||
|
||||
!strings.Contains((*acts)[0], c.letGo[0]) {
|
||||
@@ -402,7 +310,7 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsWithNoLease(t *testing.T)
|
||||
// S25 (image-missing): a want whose build names an image the store does not hold is a finding in the seat's own state,
|
||||
// written once and deleted once the store holds it again; images names what a machine's wants and haves name.
|
||||
func TestAWantedImageTheStoreLacksIsAFindingAndImagesNamesWantsAndHaves(t *testing.T) {
|
||||
h, b, _, _ := aMesh(t)
|
||||
h, b, _, _ := aMesh(t, nil)
|
||||
view, _ := h.Listed(context.Background())
|
||||
repo, digest, _ := splitReference(imageRef("letta", 1))
|
||||
delete(view.L.Repos[repo].Revisions, digest)
|
||||
@@ -410,9 +318,9 @@ func TestAWantedImageTheStoreLacksIsAFindingAndImagesNamesWantsAndHaves(t *testi
|
||||
if err := h.Observe(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "letta.ace", ImageMissing)
|
||||
key := health.FindingKey(health.ScopeModule, "letta.ace", ImageMissing)
|
||||
raw, found, _ := b.Get(FindingsState, key)
|
||||
f, err := meshconditions.DecodeFindingsByKeyValue(raw)
|
||||
f, err := health.DecodeFinding(raw)
|
||||
if !found || err != nil || f.Subject.Machine != "ace" || f.Source != "S25" || !strings.Contains(f.Evidence[0].Said, imageRef("letta", 1)) {
|
||||
t.Fatalf("the finding is %+v (%v, %v)", f, found, err)
|
||||
}
|
||||
@@ -432,7 +340,7 @@ func TestAWantedImageTheStoreLacksIsAFindingAndImagesNamesWantsAndHaves(t *testi
|
||||
t.Fatalf("the pinnedImage image missing is %s", raw)
|
||||
}
|
||||
|
||||
have, _ := json.Marshal(nodeengine.HaveByNodeModuleValue{V: 1, Fp: fp(7), Phase: "running"})
|
||||
have, _ := json.Marshal(delivery.Have{V: delivery.Version, FP: fp(7), Phase: "running"})
|
||||
_ = b.Put(HavesRead, "ace.letta", json.RawMessage(have))
|
||||
images, err := h.ImagesOf("ace", "10.0.0.5:5000")
|
||||
if err != nil || len(images.Images) != 3 || !images.SentKnown || images.Images[0].Image != "10.0.0.5:5000/letta/image@sha256:"+strings.Repeat("1", 64) ||
|
||||
|
||||
@@ -1,166 +0,0 @@
|
||||
package main
|
||||
|
||||
// **The store, read through its seats** (novox/hq ADR 0632 §4): the read tools and the sweep read what the store holds
|
||||
// from the seats' `list`, container-image-registry's images and indexes and oci-artifact-registry's bundles and archives,
|
||||
// never from the store's files or its read side. What a manifest marks beyond itself is the seat's size of it, so a
|
||||
// blob two manifests share is counted in each, and what the store's nightly collector frees is not known before it runs.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
ociseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry"
|
||||
|
||||
reg "git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry"
|
||||
)
|
||||
|
||||
// Repo is one repository as the seats' list answers it.
|
||||
type Repo struct {
|
||||
Name string
|
||||
// Revisions are the digests of every manifest it holds, tagged or not.
|
||||
Revisions map[string]bool
|
||||
// Tags are what each tag names now.
|
||||
Tags map[string]string
|
||||
}
|
||||
|
||||
// Listing is the store as the seats' list answers it: every repository, and a size for each manifest and each
|
||||
// bundle's or archive's blob, keyed by its digest. A manifest's size is its own content and every blob it names, as the
|
||||
// seat answers it, so a layer two manifests share is counted in each.
|
||||
type Listing struct {
|
||||
Blobs map[string]int64
|
||||
Repos map[string]*Repo
|
||||
}
|
||||
|
||||
func (l *Listing) repo(name string) *Repo {
|
||||
r := l.Repos[name]
|
||||
if r == nil {
|
||||
r = &Repo{Name: name, Revisions: map[string]bool{}, Tags: map[string]string{}}
|
||||
l.Repos[name] = r
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// RepoNames are the repositories in order.
|
||||
func (l *Listing) RepoNames() []string {
|
||||
out := make([]string, 0, len(l.Repos))
|
||||
for n := range l.Repos {
|
||||
out = append(out, n)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// Manifest is what one manifest names.
|
||||
type Manifest struct {
|
||||
MediaType string
|
||||
ConfigMedia string
|
||||
// Refs are the blobs it names: its configuration and its layers.
|
||||
Refs []string
|
||||
// Layers are its layers alone, in order.
|
||||
Layers []string
|
||||
// Children are the manifests an index names.
|
||||
Children []string
|
||||
}
|
||||
|
||||
// Holder is the shape of a manifest that keeps one blob: an empty configuration and one layer. The
|
||||
// controller holds every archive the mesh keeps this way (novox/hq issue 253), and keeps a named
|
||||
// document the same way, under a tag.
|
||||
func (m *Manifest) Holder() bool {
|
||||
return m != nil && m.ConfigMedia == "application/vnd.oci.empty.v1+json" && len(m.Layers) == 1
|
||||
}
|
||||
|
||||
// Key names one manifest in one repository.
|
||||
func Key(repo, digest string) string { return repo + "@" + digest }
|
||||
|
||||
// Store is the artifact store as its two seats' `list` answers it: Ask is the runtime's stdio.Ask in main.
|
||||
type Store struct {
|
||||
Ask func(key string, body any) (json.RawMessage, error)
|
||||
}
|
||||
|
||||
// ListOf asks seat's list, of one repository or, with name empty, of every one.
|
||||
func (s *Store) ListOf(seat, name string) ([]reg.Repository, error) {
|
||||
if s == nil || s.Ask == nil {
|
||||
return nil, fmt.Errorf("this process reaches no registry seat")
|
||||
}
|
||||
raw, err := s.Ask("seat:"+seat+"."+reg.VerbList, reg.ListInput{Name: name})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s's list: %w", seat, err)
|
||||
}
|
||||
var a reg.ListAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("%s's answer to list: %w", seat, err)
|
||||
}
|
||||
return a.Repositories, nil
|
||||
}
|
||||
|
||||
// View reads the store now, through both seats.
|
||||
func (s *Store) View(context.Context) (*View, error) {
|
||||
images, err := s.ListOf(reg.Seat, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
artifacts, err := s.ListOf(ociseat.Seat, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ViewOf(images, artifacts), nil
|
||||
}
|
||||
|
||||
// ViewOf is the store as the two seats list it: each manifest a repository's revision, with its tags; an index naming its
|
||||
// platform manifests; a bundle's or archive's manifest keeping its one blob, so it reads as the holder of that archive.
|
||||
func ViewOf(seats ...[]reg.Repository) *View {
|
||||
l := &Listing{Blobs: map[string]int64{}, Repos: map[string]*Repo{}}
|
||||
m := map[string]*Manifest{}
|
||||
held := map[string]bool{}
|
||||
for _, repos := range seats {
|
||||
for _, r := range repos {
|
||||
repo := l.repo(r.Name)
|
||||
for _, e := range r.Manifests {
|
||||
repo.Revisions[e.Digest] = true
|
||||
for _, t := range e.Tags {
|
||||
repo.Tags[t] = e.Digest
|
||||
}
|
||||
doc := &Manifest{Children: append([]string(nil), e.Platforms...)}
|
||||
l.Blobs[e.Digest] = e.Size
|
||||
if e.Artifact != "" {
|
||||
// The holder's size is the archive's with it: the blob itself counts nothing more.
|
||||
doc.ConfigMedia, doc.Layers, doc.Refs = emptyConfig, []string{e.Artifact}, []string{e.Artifact}
|
||||
if _, sized := l.Blobs[e.Artifact]; !sized {
|
||||
l.Blobs[e.Artifact] = 0
|
||||
}
|
||||
}
|
||||
m[Key(r.Name, e.Digest)] = doc
|
||||
if e.Held {
|
||||
held[Key(r.Name, e.Digest)] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return &View{L: l, M: m, Unread: map[string]string{}, Held: held}
|
||||
}
|
||||
|
||||
// emptyConfig is the configuration media type of a manifest keeping one blob.
|
||||
const emptyConfig = "application/vnd.oci.empty.v1+json"
|
||||
|
||||
// listed is the one entry of a reference's seat list naming it: by its manifest's digest, or a bundle's or an archive's by
|
||||
// the blob its reference names. Gone when the seat lists none.
|
||||
func (s *Store) listed(reference string) (reg.Listed, error) {
|
||||
seat, name, digest, err := seatOf(reference)
|
||||
if err != nil {
|
||||
return reg.Listed{}, fmt.Errorf("%w: %v", ErrNotOurs, err)
|
||||
}
|
||||
repos, err := s.ListOf(seat, name)
|
||||
if err != nil {
|
||||
return reg.Listed{}, err
|
||||
}
|
||||
for _, r := range repos {
|
||||
for _, e := range r.Manifests {
|
||||
if (seat == ociseat.Seat && e.Artifact == digest) || (seat != ociseat.Seat && e.Digest == digest) {
|
||||
return e, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return reg.Listed{}, Gone
|
||||
}
|
||||
@@ -1,46 +1,39 @@
|
||||
// The artifact-store-tools module's Go bundle (novox/hq ADR 0251 §1–3, ADR 0571 clause 2, to-be 51): a process the
|
||||
// node's runtime launches and speaks MCP over stdio to. It says what the artifact store holds — its repositories, its
|
||||
// size, and what the mesh keeps of each manifest — read through the registry seats' list (novox/hq ADR 0632 §4), and it
|
||||
// holds the seat mesh-artifact-sweep: the keep set, the sweep at each build's end, and S25's image-missing.
|
||||
// size, and what the mesh keeps of each manifest — read from the store's own files through the store's own container,
|
||||
// and it holds the seat mesh-artifact-sweep: the keep set, the sweep at each build's end once the controller's lease names
|
||||
// no controller, and S25's image-missing.
|
||||
//
|
||||
// **A module beside the store, not the store's own.** It requires no provision: what it reads, holds and removes it asks
|
||||
// the registry seats (listed.go, sweepseats.go), never the store's files or its read side. stdout is the protocol; what this bundle says, it says on stderr.
|
||||
// **A module beside the store, not the store's own.** A module that provides the artifact store cannot also build an
|
||||
// artifact: building publishes to the store, so the store would be needed to create itself. So these tools are a
|
||||
// second module, assigned to the machine that holds the store. stdout is the protocol; what this bundle says, it says
|
||||
// on stderr.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
)
|
||||
|
||||
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
|
||||
|
||||
func main() {
|
||||
s := &Store{Ask: stdio.Ask}
|
||||
v := &views.Reader{Module: "artifact-store-tools", Logf: logf, Findings: meshconditionsclient.FindingWriter{State: stdio.State("findings")}, Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}}
|
||||
v.Follow(assignerModule, "wants", WantsTool)
|
||||
v.Follow(indexerModule, "manifests", ManifestsTool)
|
||||
h := &Holder{Bus: runtimeBus{views: v}, Door: DoorAt(os.Getenv("MESH_ARTIFACT_STORE_ADDRESS_FILE"), s), Listed: s.View,
|
||||
Seats: stdio.Ask, Now: time.Now, Logf: logf}
|
||||
h.Acts = meshhandactsclient.Recorder{Ask: stdio.Ask}
|
||||
v.Start(context.Background())
|
||||
go follow(h, v)
|
||||
s := &Store{
|
||||
Container: os.Getenv("MESH_ARTIFACT_STORE_CONTAINER"),
|
||||
Run: ExecRunner,
|
||||
UID: os.Getuid(),
|
||||
}
|
||||
h := &Holder{Bus: runtimeBus{}, Door: DoorAt(os.Getenv("MESH_ARTIFACT_STORE_ADDRESS_FILE")), Listed: s.View,
|
||||
Now: time.Now, Logf: logf}
|
||||
h.Acts = handacts.Recorder{Ask: stdio.Ask, LeaseAway: h.LeaseAway}
|
||||
go follow(h)
|
||||
// An empty name serves as the module the runtime names (MESH_SERVED_MODULE): artifact-store-tools.
|
||||
if err := stdio.Serve("", append(append(Tools(s, h, &Catalogue{Ask: stdio.Ask}), SeatTools(h, s)...), v.Tools("artifact_store")...)); err != nil {
|
||||
if err := stdio.Serve("", append(Tools(s, h, &Catalogue{Ask: stdio.Ask}), SeatTools(h, s)...)); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// The owners' read tools of the states the holder follows, which the manifest invokes (ADR 0615).
|
||||
const (
|
||||
WantsTool = assignerModule + ".assigner_read_wants"
|
||||
ManifestsTool = indexerModule + ".indexer_read_manifests"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// readScript prints each manifest named on its command line, each after a line naming its path. A
|
||||
// manifest is JSON, which holds no raw line break inside a string, so no line of one starts with the
|
||||
// marker. One the store does not have is said, not skipped.
|
||||
const readScript = `cd ` + storageRoot + `
|
||||
for p in "$@"; do
|
||||
echo "#manifest $p"
|
||||
if [ -f "$p" ]; then cat "$p"; echo; else echo "#absent"; fi
|
||||
done
|
||||
echo '#end'`
|
||||
|
||||
// readBatch is how many manifests one exec reads: well under any command line's limit.
|
||||
const readBatch = 400
|
||||
|
||||
// Store reaches the artifact store's own files, through its own container, and only reads them.
|
||||
type Store struct {
|
||||
Container string
|
||||
Run Runner
|
||||
UID int
|
||||
|
||||
mu sync.Mutex
|
||||
// cache holds every manifest read: a manifest is named by its content's digest, so it never changes.
|
||||
cache map[string]*Manifest
|
||||
}
|
||||
|
||||
// List reads the store's files.
|
||||
func (s *Store) List(ctx context.Context) (*Listing, error) {
|
||||
if s.Container == "" {
|
||||
return nil, fmt.Errorf("this bundle was not told the artifact store's container (MESH_ARTIFACT_STORE_CONTAINER)")
|
||||
}
|
||||
out, err := docker(ctx, s.Run, s.UID, "exec", s.Container, "sh", "-c", listScript)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ParseListing(out)
|
||||
}
|
||||
|
||||
// Key names one manifest in one repository.
|
||||
func Key(repo, digest string) string { return repo + "@" + digest }
|
||||
|
||||
// blobPath is where the store keeps a blob, under storageRoot.
|
||||
func blobPath(digest string) string {
|
||||
algorithm, hex, _ := strings.Cut(digest, ":")
|
||||
if algorithm == "" || len(hex) < 2 || strings.ContainsAny(digest, "/ \n") {
|
||||
return ""
|
||||
}
|
||||
return "blobs/" + algorithm + "/" + hex[:2] + "/" + hex + "/data"
|
||||
}
|
||||
|
||||
// Manifests reads every manifest the listing names, from the cache or the store's files. Answers each
|
||||
// one read, and each one not read with why.
|
||||
func (s *Store) Manifests(ctx context.Context, l *Listing) (map[string]*Manifest, map[string]string) {
|
||||
need := map[string]bool{}
|
||||
s.mu.Lock()
|
||||
if s.cache == nil {
|
||||
s.cache = map[string]*Manifest{}
|
||||
}
|
||||
for _, name := range l.RepoNames() {
|
||||
for d := range l.Repos[name].Revisions {
|
||||
if _, ok := s.cache[d]; !ok {
|
||||
need[d] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
digests := make([]string, 0, len(need))
|
||||
for d := range need {
|
||||
digests = append(digests, d)
|
||||
}
|
||||
sort.Strings(digests)
|
||||
failed := map[string]string{}
|
||||
for start := 0; start < len(digests); start += readBatch {
|
||||
batch := digests[start:min(start+readBatch, len(digests))]
|
||||
read, err := s.read(ctx, batch)
|
||||
s.mu.Lock()
|
||||
for _, d := range batch {
|
||||
switch m := read[d]; {
|
||||
case err != nil:
|
||||
failed[d] = err.Error()
|
||||
case m == nil:
|
||||
failed[d] = "the store holds no manifest content for it"
|
||||
default:
|
||||
s.cache[d] = m
|
||||
}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
got := map[string]*Manifest{}
|
||||
unread := map[string]string{}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for _, name := range l.RepoNames() {
|
||||
for d := range l.Repos[name].Revisions {
|
||||
if m, ok := s.cache[d]; ok {
|
||||
got[Key(name, d)] = m
|
||||
} else if why, ok := failed[d]; ok {
|
||||
unread[Key(name, d)] = why
|
||||
} else {
|
||||
unread[Key(name, d)] = "not read"
|
||||
}
|
||||
}
|
||||
}
|
||||
return got, unread
|
||||
}
|
||||
|
||||
// read reads one batch of manifests. A blob that is not a manifest, or not there, is answered as nil;
|
||||
// an exec that fails fails the batch.
|
||||
func (s *Store) read(ctx context.Context, digests []string) (map[string]*Manifest, error) {
|
||||
args := []string{"exec", s.Container, "sh", "-c", readScript, "sh"}
|
||||
byPath := map[string]string{}
|
||||
for _, d := range digests {
|
||||
if p := blobPath(d); p != "" {
|
||||
byPath[p] = d
|
||||
args = append(args, p)
|
||||
}
|
||||
}
|
||||
out, err := docker(ctx, s.Run, s.UID, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseManifests(out, byPath)
|
||||
}
|
||||
|
||||
// parseManifests reads readScript's output.
|
||||
func parseManifests(out string, byPath map[string]string) (map[string]*Manifest, error) {
|
||||
got := map[string]*Manifest{}
|
||||
current := ""
|
||||
var body strings.Builder
|
||||
flush := func() {
|
||||
if d := byPath[current]; d != "" {
|
||||
m, err := ParseManifest([]byte(body.String()))
|
||||
if err != nil {
|
||||
m = nil
|
||||
}
|
||||
got[d] = m
|
||||
}
|
||||
current = ""
|
||||
body.Reset()
|
||||
}
|
||||
ended := false
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
switch {
|
||||
case strings.HasPrefix(line, "#manifest "):
|
||||
flush()
|
||||
current = strings.TrimPrefix(line, "#manifest ")
|
||||
case line == "#absent":
|
||||
if d := byPath[current]; d != "" {
|
||||
got[d] = nil
|
||||
}
|
||||
current = ""
|
||||
body.Reset()
|
||||
case line == "#end":
|
||||
flush()
|
||||
ended = true
|
||||
default:
|
||||
body.WriteString(line)
|
||||
body.WriteString("\n")
|
||||
}
|
||||
}
|
||||
if !ended {
|
||||
return nil, fmt.Errorf("reading the store's manifests was cut short (it never reached its end)")
|
||||
}
|
||||
return got, nil
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestManifestsAreReadFromTheStoresFilesAndAMissingOneIsSaid(t *testing.T) {
|
||||
got, err := parseManifests("#manifest blobs/sha256/aa/aa1/data\n"+image(d("cfg"), d("l"))+"\n\n#manifest blobs/sha256/bb/bb1/data\n#absent\n#manifest blobs/sha256/cc/cc1/data\nnot json\n#end\n",
|
||||
map[string]string{"blobs/sha256/aa/aa1/data": "sha256:aa1", "blobs/sha256/bb/bb1/data": "sha256:bb1", "blobs/sha256/cc/cc1/data": "sha256:cc1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m := got["sha256:aa1"]; m == nil || len(m.Layers) != 1 || m.Layers[0] != d("l") {
|
||||
t.Errorf("aa1 %+v", m)
|
||||
}
|
||||
if got["sha256:bb1"] != nil || got["sha256:cc1"] != nil {
|
||||
t.Error("an absent or unreadable manifest was read as one")
|
||||
}
|
||||
if _, err := parseManifests("#manifest x\n{}\n", map[string]string{"x": "sha256:x"}); err == nil {
|
||||
t.Error("a cut read was accepted")
|
||||
}
|
||||
|
||||
f := world()
|
||||
delete(f.manifests, d("m-old"))
|
||||
v := view(t, f)
|
||||
if v.Unread[Key("app/server", d("m-old"))] == "" {
|
||||
t.Error("a manifest whose content is gone was not said as unread")
|
||||
}
|
||||
if out := Usage(v, 3); out["unread"] == nil {
|
||||
t.Error("the answer did not say a manifest was unread")
|
||||
}
|
||||
}
|
||||
|
||||
func TestADigestNeverBecomesAPathOutsideTheBlobs(t *testing.T) {
|
||||
for _, bad := range []string{"sha256:", "nocolon", "sha256:a/../../etc", "sha256:ab cd"} {
|
||||
if p := blobPath(bad); p != "" {
|
||||
t.Errorf("%q became %q", bad, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,16 +2,22 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// What the records say, in the shape the seat's holder answers them (Holder; novox/hq ADR 0251 §2, ADR 0571 clause 2):
|
||||
// the holder decides what the mesh keeps; the tools only set that beside what the store holds.
|
||||
// What the records say, asked of the controller and never copied (novox/hq ADR 0251 §2). The
|
||||
// controller decides what the mesh keeps; this bundle only sets that beside what the store holds.
|
||||
|
||||
// Asking is how a tool on the mesh is asked: stdio.Ask, or a test's.
|
||||
type Asking func(key string, body any) (json.RawMessage, error)
|
||||
|
||||
// Record is one reference the mesh recorded making, as the seat's `artifacts` answers it.
|
||||
// ControllerSeat is the seat whose verbs the controller serves.
|
||||
const ControllerSeat = "mesh-controller"
|
||||
|
||||
// Record is one reference the mesh recorded making, as the controller's `artifacts` answers it.
|
||||
type Record struct {
|
||||
Reference string `json:"reference"`
|
||||
Repository string `json:"repository,omitempty"`
|
||||
@@ -21,7 +27,7 @@ type Record struct {
|
||||
Why []string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
// Records is the seat's `artifacts` answer.
|
||||
// Records is the controller's `artifacts` answer.
|
||||
type Records struct {
|
||||
Store string `json:"store"`
|
||||
KeptBuilds int `json:"kept_builds"`
|
||||
@@ -33,7 +39,7 @@ type Records struct {
|
||||
Note string `json:"-"`
|
||||
}
|
||||
|
||||
// CollectAnswer is the seat's `collect` answer.
|
||||
// CollectAnswer is the controller's `collect` answer.
|
||||
type CollectAnswer struct {
|
||||
DryRun bool `json:"dry_run"`
|
||||
Store string `json:"store"`
|
||||
@@ -47,12 +53,15 @@ type CollectAnswer struct {
|
||||
Skipped int `json:"skipped"`
|
||||
Left int `json:"left"`
|
||||
Stopped string `json:"stopped"`
|
||||
// Rest is whatever else the sweep said, passed on as it said it.
|
||||
// Rest is whatever else the controller said, passed on as it said it.
|
||||
Rest map[string]json.RawMessage `json:"-"`
|
||||
}
|
||||
|
||||
// answerOf reads a verb's answer whatever wraps it: {output, ok, answer}, a text the runtime handed over, or the
|
||||
// protocol's content list.
|
||||
// Controller asks the controller's seat through the runtime.
|
||||
type Controller struct{ Ask Asking }
|
||||
|
||||
// answerOf reads a verb's answer whatever wraps it: the controller's {output, ok, answer}, a text the
|
||||
// runtime handed over, or the protocol's content list.
|
||||
func answerOf(raw json.RawMessage) (json.RawMessage, string, bool) {
|
||||
var s string
|
||||
if json.Unmarshal(raw, &s) == nil {
|
||||
@@ -86,11 +95,89 @@ func answerOf(raw json.RawMessage) (json.RawMessage, string, bool) {
|
||||
return raw, string(raw), true
|
||||
}
|
||||
|
||||
func (c Controller) verb(name string, args map[string]any) (json.RawMessage, error) {
|
||||
if c.Ask == nil {
|
||||
return nil, errors.New("this bundle has no way to ask the controller")
|
||||
}
|
||||
raw, err := c.Ask("seat:"+ControllerSeat+"."+name, args)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the controller's %s could not be asked: %w", name, err)
|
||||
}
|
||||
answer, output, ok := answerOf(raw)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the controller refused %s: %s", name, lastLine(output))
|
||||
}
|
||||
if len(answer) == 0 || string(answer) == "null" {
|
||||
return nil, fmt.Errorf("the controller's %s answered no data: %s", name, lastLine(output))
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
func lastLine(s string) string {
|
||||
lines := strings.Split(strings.TrimSpace(s), "\n")
|
||||
return strings.TrimSpace(lines[len(lines)-1])
|
||||
}
|
||||
|
||||
// Artifacts asks what the records say, with the references already let go of when the answer can carry
|
||||
// them; when it cannot, without, and says so.
|
||||
func (c Controller) Artifacts(repository string) (*Records, error) {
|
||||
args := map[string]any{"collected": "true"}
|
||||
if repository != "" {
|
||||
args["repository"] = repository
|
||||
}
|
||||
raw, err := c.verb("artifacts", args)
|
||||
collected := true
|
||||
note := ""
|
||||
if err != nil {
|
||||
delete(args, "collected")
|
||||
var again error
|
||||
raw, again = c.verb("artifacts", args)
|
||||
if again != nil {
|
||||
return nil, again
|
||||
}
|
||||
collected = false
|
||||
note = "the references the mesh already let go of could not be asked for (" + err.Error() +
|
||||
"), so a manifest the mesh let go of that the store still holds is counted as unrecorded"
|
||||
}
|
||||
var r Records
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
return nil, fmt.Errorf("the controller's artifacts answer is not readable: %v", err)
|
||||
}
|
||||
for i := range r.References {
|
||||
fill(&r.References[i])
|
||||
}
|
||||
r.CollectedAsked, r.Note = collected, note
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
// Collect asks the controller's sweep: a dry run unless confirm, which needs why.
|
||||
func (c Controller) Collect(why string, confirm bool, most int) (*CollectAnswer, error) {
|
||||
args := map[string]any{}
|
||||
if confirm {
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return nil, errors.New("a real collection needs why: nothing was asked")
|
||||
}
|
||||
args["why"] = why
|
||||
args["confirm"] = "true"
|
||||
}
|
||||
if most > 0 {
|
||||
args["most"] = strconv.Itoa(most)
|
||||
}
|
||||
raw, err := c.verb("collect", args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a CollectAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("the controller's collect answer is not readable: %v", err)
|
||||
}
|
||||
_ = json.Unmarshal(raw, &a.Rest)
|
||||
if !confirm && !a.DryRun && len(a.LetGo) > 0 {
|
||||
return nil, fmt.Errorf("the controller let go of %d artifacts when only a dry run was asked", len(a.LetGo))
|
||||
}
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
// fill reads the repository, digest and kind out of a reference when the answer left them out.
|
||||
func fill(r *Record) {
|
||||
repo, digest, kind := splitReference(r.Reference)
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Ran is what a command did: its output, its exit status, and why it never ran to an answer.
|
||||
type Ran struct {
|
||||
Stdout string
|
||||
Stderr string
|
||||
Status int
|
||||
// Err is "ENOENT" when the program is not installed, or says it was ended for taking too long.
|
||||
Err string
|
||||
}
|
||||
|
||||
// Runner runs one command, so every tool can be tested without a daemon.
|
||||
type Runner func(ctx context.Context, name string, args ...string) Ran
|
||||
|
||||
// ListTimeout is how long listing the store's files may take: below the runtime's thirty-second call
|
||||
// limit, so a store that hangs is answered as such rather than as a call the runtime gave up on.
|
||||
const ListTimeout = 15 * time.Second
|
||||
|
||||
// ExecRunner runs a command on this machine, bounded by ListTimeout.
|
||||
func ExecRunner(ctx context.Context, name string, args ...string) Ran {
|
||||
ctx, cancel := context.WithTimeout(ctx, ListTimeout)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
var out, errb bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &out, &errb
|
||||
err := cmd.Run()
|
||||
r := Ran{Stdout: out.String(), Stderr: errb.String()}
|
||||
var exitErr *exec.ExitError
|
||||
switch {
|
||||
case errors.Is(ctx.Err(), context.DeadlineExceeded):
|
||||
r.Status, r.Err = 124, fmt.Sprintf("no answer within %d s", int(ListTimeout/time.Second))
|
||||
case errors.Is(err, exec.ErrNotFound):
|
||||
r.Status, r.Err = 127, "ENOENT"
|
||||
case errors.As(err, &exitErr):
|
||||
r.Status = exitErr.ExitCode()
|
||||
case err != nil:
|
||||
r.Status, r.Err = 1, err.Error()
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
var socketRefused = regexp.MustCompile(`(?i)permission denied.*docker.*sock|docker\.sock.*permission denied`)
|
||||
|
||||
// docker runs one docker command as this account and answers its stdout. A socket that refuses the
|
||||
// account is asked again through `sudo -n`, as the container runtime's own tools do; never as root
|
||||
// otherwise, and never with a prompt.
|
||||
func docker(ctx context.Context, run Runner, uid int, args ...string) (string, error) {
|
||||
r := run(ctx, "docker", args...)
|
||||
program := "docker"
|
||||
if r.Status != 0 && r.Err == "" && uid != 0 && socketRefused.MatchString(r.Stderr) {
|
||||
program = "sudo"
|
||||
r = run(ctx, "sudo", append([]string{"-n", "docker"}, args...)...)
|
||||
}
|
||||
if r.Status == 0 && r.Err == "" {
|
||||
return r.Stdout, nil
|
||||
}
|
||||
said := strings.TrimSpace(r.Stderr + "\n" + r.Stdout)
|
||||
switch {
|
||||
case r.Err == "ENOENT" && program == "sudo":
|
||||
return "", errors.New("the runtime's socket refused this account, and sudo is not installed here to escalate with")
|
||||
case r.Err == "ENOENT":
|
||||
return "", errors.New("docker is not installed on this machine, so the store's files cannot be listed")
|
||||
case r.Err != "":
|
||||
return "", fmt.Errorf("listing the store's files did not answer: %s", r.Err)
|
||||
case program == "sudo" && strings.HasPrefix(said, "sudo:"):
|
||||
return "", fmt.Errorf("the runtime's socket refused this account and it may not escalate without a prompt: %s", firstLine(said))
|
||||
case strings.Contains(said, "No such container"):
|
||||
return "", fmt.Errorf("the store's container is not on this machine: %s", firstLine(said))
|
||||
case strings.Contains(said, "is not running"):
|
||||
return "", fmt.Errorf("the store's container is not running: %s", firstLine(said))
|
||||
}
|
||||
if l := firstLine(said); l != "" {
|
||||
return "", fmt.Errorf("listing the store's files failed (%d): %s", r.Status, l)
|
||||
}
|
||||
return "", fmt.Errorf("listing the store's files failed with status %d", r.Status)
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
for _, l := range strings.Split(s, "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" {
|
||||
return l
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
package main
|
||||
|
||||
// The seat mesh-artifact-sweep's verbs (novox/hq ADR 0571 clause 2; sweep.Seat), each a tool named
|
||||
// The seat mesh-artifact-sweep's verbs (novox/hq ADR 0571 clause 2; catalogue.ArtifactSweepSeat), each a tool named
|
||||
// `artifact-store.<verb>` as the runtime serves a seat's verb, answering as the controller's verb answered: {output,
|
||||
// ok, answer}.
|
||||
|
||||
@@ -8,8 +8,8 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
sweep "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// verbAnswer is the controller's verbs' answer shape, which the builder and docker read (answerOf).
|
||||
@@ -21,7 +21,7 @@ type verbAnswer struct {
|
||||
|
||||
// SeatTools are the seat's verbs over h; s reads the store's listing for mirrors.
|
||||
func SeatTools(h *Holder, s *Store) []stdio.Tool {
|
||||
seat := sweep.Seat + "."
|
||||
seat := catalogue.ArtifactSweepSeat + "."
|
||||
return []stdio.Tool{
|
||||
{Name: seat + "artifacts", Description: "Every artifact the mesh made that the store holds: kept and why, eligible, " +
|
||||
"or with collected those it no longer holds.",
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// What the artifact store holds, read from its own files, and nothing written to them.
|
||||
//
|
||||
// **Why its files.** The store's door lists repositories and the tags in each, but not a manifest no tag
|
||||
// names — and since the mesh pins every machine by digest, that is almost every manifest the store
|
||||
// holds. The files list them all, with every blob's size, and hold every manifest's content. They are
|
||||
// read through the store's own container, which is the only process that has them, and only read
|
||||
// (novox/hq ADR 0251 §1). So this module runs on the machine that holds the store; anywhere else its
|
||||
// tools say the container is not there.
|
||||
|
||||
// storageRoot is where the registry keeps its files, inside its container.
|
||||
const storageRoot = "/var/lib/registry/docker/registry/v2"
|
||||
|
||||
// listScript lists, in three sections, every blob with its size, every manifest each repository holds,
|
||||
// and what each tag names now. Busybox's find and stat (the store's image is Alpine): no -printf. An
|
||||
// upload in progress is not listed, and nor are a tag's former targets.
|
||||
const listScript = `set -e
|
||||
cd ` + storageRoot + `
|
||||
echo '#blobs'
|
||||
if [ -d blobs ]; then find blobs -type f -name data | xargs -r stat -c '%s %n'; fi
|
||||
echo '#revisions'
|
||||
if [ -d repositories ]; then find repositories -type f -path '*/_manifests/revisions/sha256/*/link'; fi
|
||||
echo '#tags'
|
||||
if [ -d repositories ]; then find repositories -type f -path '*/_manifests/tags/*/current/link' -exec grep -H . {} + || true; fi
|
||||
echo '#end'`
|
||||
|
||||
// Repo is one repository as the store's files list it.
|
||||
type Repo struct {
|
||||
Name string
|
||||
// Revisions are the digests of every manifest it holds, tagged or not.
|
||||
Revisions map[string]bool
|
||||
// Tags are what each tag names now.
|
||||
Tags map[string]string
|
||||
}
|
||||
|
||||
// Listing is the store's files: every blob with its size, and every repository.
|
||||
type Listing struct {
|
||||
Blobs map[string]int64
|
||||
Repos map[string]*Repo
|
||||
}
|
||||
|
||||
func (l *Listing) repo(name string) *Repo {
|
||||
r := l.Repos[name]
|
||||
if r == nil {
|
||||
r = &Repo{Name: name, Revisions: map[string]bool{}, Tags: map[string]string{}}
|
||||
l.Repos[name] = r
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// RepoNames are the repositories in order.
|
||||
func (l *Listing) RepoNames() []string {
|
||||
out := make([]string, 0, len(l.Repos))
|
||||
for n := range l.Repos {
|
||||
out = append(out, n)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// ParseListing reads listScript's output. A section that never ended means the listing was cut short,
|
||||
// which is an error: a partial listing would make held things look absent.
|
||||
func ParseListing(out string) (*Listing, error) {
|
||||
l := &Listing{Blobs: map[string]int64{}, Repos: map[string]*Repo{}}
|
||||
section := ""
|
||||
ended := false
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "#") {
|
||||
section = line
|
||||
ended = line == "#end"
|
||||
continue
|
||||
}
|
||||
switch section {
|
||||
case "#blobs":
|
||||
size, path, ok := strings.Cut(line, " ")
|
||||
n, err := strconv.ParseInt(size, 10, 64)
|
||||
if !ok || err != nil {
|
||||
return nil, fmt.Errorf("the store's listing has a blob line it cannot read: %q", line)
|
||||
}
|
||||
// blobs/sha256/ab/<hex>/data
|
||||
parts := strings.Split(strings.TrimPrefix(path, "./"), "/")
|
||||
if len(parts) != 5 || parts[0] != "blobs" || parts[4] != "data" {
|
||||
return nil, fmt.Errorf("the store's listing has a blob at an unexpected place: %q", path)
|
||||
}
|
||||
l.Blobs[parts[1]+":"+parts[3]] = n
|
||||
case "#revisions":
|
||||
// repositories/<repo>/_manifests/revisions/sha256/<hex>/link
|
||||
name, rest, ok := strings.Cut(strings.TrimPrefix(strings.TrimPrefix(line, "./"), "repositories/"), "/_manifests/revisions/")
|
||||
parts := strings.Split(rest, "/")
|
||||
if !ok || len(parts) != 3 || parts[2] != "link" {
|
||||
return nil, fmt.Errorf("the store's listing has a manifest at an unexpected place: %q", line)
|
||||
}
|
||||
l.repo(name).Revisions[parts[0]+":"+parts[1]] = true
|
||||
case "#tags":
|
||||
// repositories/<repo>/_manifests/tags/<tag>/current/link:sha256:<hex>
|
||||
path, digest, ok := strings.Cut(line, ":sha256:")
|
||||
name, rest, ok2 := strings.Cut(strings.TrimPrefix(strings.TrimPrefix(path, "./"), "repositories/"), "/_manifests/tags/")
|
||||
tag, _, ok3 := strings.Cut(rest, "/current/link")
|
||||
if !ok || !ok2 || !ok3 || tag == "" {
|
||||
return nil, fmt.Errorf("the store's listing has a tag line it cannot read: %q", line)
|
||||
}
|
||||
l.repo(name).Tags[tag] = "sha256:" + digest
|
||||
default:
|
||||
return nil, fmt.Errorf("the store's listing has a line outside any section: %q", line)
|
||||
}
|
||||
}
|
||||
if !ended {
|
||||
return nil, fmt.Errorf("the store's listing was cut short (it never reached its end)")
|
||||
}
|
||||
return l, nil
|
||||
}
|
||||
|
||||
// Manifest is what one manifest names.
|
||||
type Manifest struct {
|
||||
MediaType string
|
||||
ConfigMedia string
|
||||
// Refs are the blobs it names: its configuration and its layers.
|
||||
Refs []string
|
||||
// Layers are its layers alone, in order.
|
||||
Layers []string
|
||||
// Children are the manifests an index names.
|
||||
Children []string
|
||||
}
|
||||
|
||||
// Holder is the shape of a manifest that keeps one blob: an empty configuration and one layer. The
|
||||
// controller holds every archive the mesh keeps this way (novox/hq issue 253), and keeps a named
|
||||
// document the same way, under a tag.
|
||||
func (m *Manifest) Holder() bool {
|
||||
return m != nil && m.ConfigMedia == "application/vnd.oci.empty.v1+json" && len(m.Layers) == 1
|
||||
}
|
||||
|
||||
// ParseManifest reads a manifest's content, of any of the shapes a registry holds.
|
||||
func ParseManifest(raw []byte) (*Manifest, error) {
|
||||
type desc struct {
|
||||
MediaType string `json:"mediaType"`
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
var v struct {
|
||||
MediaType string `json:"mediaType"`
|
||||
Config *desc `json:"config"`
|
||||
Layers []desc `json:"layers"`
|
||||
Manifests []desc `json:"manifests"`
|
||||
FSLayers []struct {
|
||||
BlobSum string `json:"blobSum"`
|
||||
} `json:"fsLayers"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return nil, fmt.Errorf("not a manifest: %v", err)
|
||||
}
|
||||
m := &Manifest{MediaType: v.MediaType}
|
||||
if v.Config != nil && v.Config.Digest != "" {
|
||||
m.ConfigMedia = v.Config.MediaType
|
||||
m.Refs = append(m.Refs, v.Config.Digest)
|
||||
}
|
||||
for _, d := range v.Layers {
|
||||
m.Refs = append(m.Refs, d.Digest)
|
||||
m.Layers = append(m.Layers, d.Digest)
|
||||
}
|
||||
for _, d := range v.FSLayers {
|
||||
m.Refs = append(m.Refs, d.BlobSum)
|
||||
m.Layers = append(m.Layers, d.BlobSum)
|
||||
}
|
||||
for _, d := range v.Manifests {
|
||||
m.Children = append(m.Children, d.Digest)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
@@ -1,20 +1,14 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
reg "git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry"
|
||||
sweep "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist/seatlisttest"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -27,72 +21,66 @@ func d(name string) string {
|
||||
|
||||
func hexOf(digest string) string { return strings.TrimPrefix(digest, "sha256:") }
|
||||
|
||||
// fakeStore is a store's blobs and manifests, as the two registry seats' list answers them (ADR 0632 §4).
|
||||
// fakeStore is a store's files and its manifests: what listScript and readScript would print.
|
||||
type fakeStore struct {
|
||||
blobs map[string]int64
|
||||
revisions map[string][]string // repo -> digests
|
||||
tags map[string]map[string]string
|
||||
manifests map[string]string // digest -> content
|
||||
held map[string]bool // Key(repo, digest) -> held
|
||||
}
|
||||
|
||||
// ask is the seats as the runtime asks them: list on each, every repository or one; anything else fails the test, as
|
||||
// these tools only read.
|
||||
func (f *fakeStore) ask(t *testing.T) func(string, any) (json.RawMessage, error) {
|
||||
return func(key string, body any) (json.RawMessage, error) {
|
||||
seat, verb, _ := strings.Cut(strings.TrimPrefix(key, "seat:"), ".")
|
||||
if verb != "list" || (seat != "container-image-registry" && seat != "oci-artifact-registry") {
|
||||
t.Fatalf("asked %s", key)
|
||||
func (f *fakeStore) listing() string {
|
||||
var b strings.Builder
|
||||
b.WriteString("#blobs\n")
|
||||
for dg, n := range f.blobs {
|
||||
fmt.Fprintf(&b, "%d blobs/sha256/%s/%s/data\n", n, hexOf(dg)[:2], hexOf(dg))
|
||||
}
|
||||
b.WriteString("#revisions\n")
|
||||
for repo, ds := range f.revisions {
|
||||
for _, dg := range ds {
|
||||
fmt.Fprintf(&b, "repositories/%s/_manifests/revisions/sha256/%s/link\n", repo, hexOf(dg))
|
||||
}
|
||||
raw, _ := json.Marshal(body)
|
||||
var in reg.ListInput
|
||||
_ = json.Unmarshal(raw, &in)
|
||||
repos := []reg.Repository{}
|
||||
names := make([]string, 0, len(f.revisions))
|
||||
for n := range f.revisions {
|
||||
names = append(names, n)
|
||||
}
|
||||
b.WriteString("#tags\n")
|
||||
for repo, ts := range f.tags {
|
||||
for t, dg := range ts {
|
||||
fmt.Fprintf(&b, "repositories/%s/_manifests/tags/%s/current/link:%s\n", repo, t, dg)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
if in.Name != "" && in.Name != name {
|
||||
continue
|
||||
}
|
||||
var ms []reg.Listed
|
||||
for _, dg := range f.revisions[name] {
|
||||
var doc struct {
|
||||
Config *struct{ MediaType, Digest string }
|
||||
Layers []struct{ Digest string }
|
||||
Manifests []struct{ Digest string }
|
||||
}
|
||||
_ = json.Unmarshal([]byte(f.manifests[dg]), &doc)
|
||||
e := reg.Listed{Digest: dg, Size: f.blobs[dg], Held: f.held[Key(name, dg)]}
|
||||
if doc.Config != nil {
|
||||
e.Size += f.blobs[doc.Config.Digest]
|
||||
}
|
||||
for _, l := range doc.Layers {
|
||||
e.Size += f.blobs[l.Digest]
|
||||
}
|
||||
for _, c := range doc.Manifests {
|
||||
e.Platforms = append(e.Platforms, c.Digest)
|
||||
}
|
||||
e.Index = len(e.Platforms) > 0
|
||||
if doc.Config != nil && doc.Config.MediaType == "application/vnd.oci.empty.v1+json" && len(doc.Layers) == 1 {
|
||||
e.Artifact = doc.Layers[0].Digest
|
||||
}
|
||||
for tag, at := range f.tags[name] {
|
||||
if at == dg {
|
||||
e.Tags = append(e.Tags, tag)
|
||||
}
|
||||
b.WriteString("#end\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// run is the store's container as docker exec reaches it: the listing, or the manifests asked for, as
|
||||
// the two scripts print them. Anything else is a test failure: these tools only read.
|
||||
func (f *fakeStore) run(t *testing.T) Runner {
|
||||
return func(_ context.Context, name string, args ...string) Ran {
|
||||
if name != "docker" || len(args) < 5 || args[0] != "exec" || args[1] != "mesh-registry" || args[2] != "sh" || args[3] != "-c" {
|
||||
t.Fatalf("ran %s %v", name, args)
|
||||
}
|
||||
switch args[4] {
|
||||
case listScript:
|
||||
return Ran{Stdout: f.listing()}
|
||||
case readScript:
|
||||
var b strings.Builder
|
||||
for _, p := range args[6:] {
|
||||
fmt.Fprintf(&b, "#manifest %s\n", p)
|
||||
found := false
|
||||
for dg, body := range f.manifests {
|
||||
if blobPath(dg) == p {
|
||||
b.WriteString(body + "\n")
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if (e.Artifact != "") == (seat == "oci-artifact-registry") {
|
||||
ms = append(ms, e)
|
||||
if !found {
|
||||
b.WriteString("#absent\n")
|
||||
}
|
||||
}
|
||||
if len(ms) > 0 {
|
||||
repos = append(repos, reg.Repository{Name: name, Manifests: ms})
|
||||
}
|
||||
b.WriteString("#end\n")
|
||||
return Ran{Stdout: b.String()}
|
||||
}
|
||||
return json.Marshal(reg.ListAnswer{Repositories: repos})
|
||||
t.Fatalf("ran a script these tools do not have: %q", args[4])
|
||||
return Ran{}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -164,7 +152,7 @@ func records() *Records {
|
||||
|
||||
func view(t *testing.T, f *fakeStore) *View {
|
||||
t.Helper()
|
||||
s := &Store{Ask: f.ask(t)}
|
||||
s := &Store{Container: "mesh-registry", UID: 1000, Run: f.run(t)}
|
||||
v, err := s.View(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -179,34 +167,50 @@ func withFill(r *Records) *Records {
|
||||
return r
|
||||
}
|
||||
|
||||
// The store is read through its two seats' list (ADR 0632 §4): every manifest of every repository, tagged or not, an
|
||||
// index with the manifests it names, a bundle's or archive's manifest as the holder of its blob; each manifest's size is
|
||||
// the seat's, so nothing is unmarked before a manifest goes, and a manifest gone frees its size.
|
||||
func TestTheStoreIsReadThroughItsSeats(t *testing.T) {
|
||||
f := world()
|
||||
f.held = map[string]bool{Key("app/tools", d("h-kept")): true}
|
||||
v := view(t, f)
|
||||
server := v.L.Repos["app/server"]
|
||||
if server == nil || len(server.Revisions) != 4 || server.Tags["latest"] != d("m-stranger") {
|
||||
t.Fatalf("app/server reads %+v", server)
|
||||
func TestTheListingIsRead(t *testing.T) {
|
||||
l, err := ParseListing("#blobs\n12 blobs/sha256/ab/abcd/data\n#revisions\nrepositories/a/b/c/_manifests/revisions/sha256/ff/link\n#tags\nrepositories/a/b/c/_manifests/tags/v1/current/link:sha256:ff\n#end\n")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m := v.M[Key("app/server", d("idx"))]; m == nil || !reflect.DeepEqual(m.Children, []string{d("m-kept")}) {
|
||||
t.Errorf("the index names %+v", m)
|
||||
if l.Blobs["sha256:abcd"] != 12 {
|
||||
t.Errorf("blobs %v", l.Blobs)
|
||||
}
|
||||
if m := v.M[Key("app/tools", d("h-kept"))]; !m.Holder() || m.Layers[0] != d("a-kept") {
|
||||
t.Errorf("the holder of a-kept reads %+v", m)
|
||||
r := l.Repos["a/b/c"]
|
||||
if r == nil || !r.Revisions["sha256:ff"] || r.Tags["v1"] != "sha256:ff" {
|
||||
t.Errorf("repository with slashes misread: %+v", r)
|
||||
}
|
||||
if _, held := v.L.Blobs[d("a-kept")]; !held || !v.Held[Key("app/tools", d("h-kept"))] {
|
||||
t.Errorf("a-kept is not read as held by the store, or its hold is not read: %v", v.Held)
|
||||
}
|
||||
|
||||
func TestACutListingIsRefused(t *testing.T) {
|
||||
for _, out := range []string{"#blobs\n12 blobs/sha256/ab/abcd/data\n", "#blobs\nnot a line\n#end\n", "stray\n#end\n"} {
|
||||
if _, err := ParseListing(out); err == nil {
|
||||
t.Errorf("accepted %q", out)
|
||||
}
|
||||
}
|
||||
if want := int64(len(f.manifests[d("m-old")])) + 10 + 1000 + 500; v.L.Blobs[d("m-old")] != want {
|
||||
t.Errorf("m-old's size %d, want %d (its content and every blob it names)", v.L.Blobs[d("m-old")], want)
|
||||
}
|
||||
|
||||
func TestTheMarkIsTheCollectorsIncludingAnIndexAndSharedBlobs(t *testing.T) {
|
||||
v := view(t, world())
|
||||
marked := v.Marked(nil)
|
||||
for _, name := range []string{"cfg", "shared", "old-only", "stranger-layer", "a-kept", "a-old", "doc-body", "empty", "m-kept", "idx"} {
|
||||
if !marked[d(name)] {
|
||||
t.Errorf("%s not marked", name)
|
||||
}
|
||||
}
|
||||
if freed, n := v.Unmarked(nil); freed != 0 || n != 0 {
|
||||
t.Errorf("unmarked %d in %d before anything goes", freed, n)
|
||||
if marked[d("orphan")] {
|
||||
t.Error("a blob no manifest names is marked")
|
||||
}
|
||||
if freed, n := v.Unmarked(map[string]bool{Key("app/server", d("idx")): true}); freed != v.L.Blobs[d("idx")] || n != 1 {
|
||||
t.Errorf("the index gone frees %d in %d, want its size", freed, n)
|
||||
freed, n := v.Unmarked(nil)
|
||||
if freed != 7000 || n != 1 {
|
||||
t.Errorf("the collector frees %d in %d, want 7000 in 1", freed, n)
|
||||
}
|
||||
// The index alone gone frees its own content and nothing m-kept still marks.
|
||||
_, n = v.Unmarked(map[string]bool{Key("app/server", d("idx")): true})
|
||||
if n != 2 {
|
||||
t.Errorf("without the index %d blobs unmarked, want 2 (orphan and the index itself)", n)
|
||||
}
|
||||
if sh := v.Shared(); !sh[d("empty")] || sh[d("cfg")] {
|
||||
t.Error("shared misjudged")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -250,15 +254,16 @@ func TestAStateThisBundleDoesNotKnowIsKept(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBytesFreedAreTheSizesOfWhatGoes(t *testing.T) {
|
||||
func TestBytesFreedCountASharedBlobAsKept(t *testing.T) {
|
||||
v := view(t, world())
|
||||
a := &CollectAnswer{DryRun: true, Eligible: 2, WouldLetGo: []string{
|
||||
"artifact-store://app/server@" + d("m-old"), "artifact-store://app/tools/blobs/" + d("a-old")}}
|
||||
out := Collected(v, a, true, nil)
|
||||
now, after := out["collector_frees_now_bytes"].(int64), out["collector_frees_after_bytes"].(int64)
|
||||
// m-old and the holder of a-old go, each with the size its seat says.
|
||||
if want := v.L.Blobs[d("m-old")] + v.L.Blobs[d("h-old")]; now != 0 || after != want {
|
||||
t.Errorf("now %d after %d, want 0 and %d", now, after, want)
|
||||
// m-old's own content, old-only and a-old and its holder go; shared and cfg stay with m-kept.
|
||||
want := int64(7000) + 500 + 4000 + v.L.Blobs[d("m-old")] + v.L.Blobs[d("h-old")]
|
||||
if now != 7000 || after != want {
|
||||
t.Errorf("now %d after %d, want 7000 and %d", now, after, want)
|
||||
}
|
||||
if out["manifests_gone"].(int) != 2 {
|
||||
t.Errorf("gone %v", out["manifests_gone"])
|
||||
@@ -269,12 +274,13 @@ func TestOnlyTheirBytesLeaveOutWhatAnotherStateMarks(t *testing.T) {
|
||||
v := view(t, world())
|
||||
entries := Classify(v, withFill(records()), nil)
|
||||
sums := Summarise(v, entries, entries)
|
||||
if el := sums[StateEligible]; el.OnlyTheirBytes != v.L.Blobs[d("m-old")] {
|
||||
el := sums[StateEligible]
|
||||
if el.OnlyTheirBytes != 500+v.L.Blobs[d("m-old")] {
|
||||
t.Errorf("eligible only-theirs %d", el.OnlyTheirBytes)
|
||||
}
|
||||
}
|
||||
|
||||
// asked records every ask, and answers as the seat and the catalogue would.
|
||||
// asked records every ask, and answers as the controller would.
|
||||
type asked struct {
|
||||
mu sync.Mutex
|
||||
calls []string
|
||||
@@ -287,51 +293,6 @@ type asked struct {
|
||||
noCatalogue bool
|
||||
}
|
||||
|
||||
// recordsAsked is the tools' Recorded in a test: each call asked of asked, as `artifacts` or `collect`, with the
|
||||
// arguments a real run gives the seat's sweep.
|
||||
type recordsAsked struct{ a *asked }
|
||||
|
||||
func (r recordsAsked) Artifacts(repository string) (*Records, error) {
|
||||
args := map[string]any{"collected": "true"}
|
||||
if repository != "" {
|
||||
args["repository"] = repository
|
||||
}
|
||||
raw, err := r.a.ask("artifacts", args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var recs Records
|
||||
answer, _, _ := answerOf(raw)
|
||||
if err := json.Unmarshal(answer, &recs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range recs.References {
|
||||
fill(&recs.References[i])
|
||||
}
|
||||
recs.CollectedAsked = true
|
||||
return &recs, nil
|
||||
}
|
||||
|
||||
func (r recordsAsked) Collect(why string, confirm bool, most int) (*CollectAnswer, error) {
|
||||
args := map[string]any{}
|
||||
if confirm {
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return nil, fmt.Errorf("a real collection needs why: nothing was asked")
|
||||
}
|
||||
args["why"], args["confirm"] = why, "true"
|
||||
}
|
||||
if most > 0 {
|
||||
args["most"] = strconv.Itoa(most)
|
||||
}
|
||||
raw, err := r.a.ask("collect", args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var c CollectAnswer
|
||||
answer, _, _ := answerOf(raw)
|
||||
return &c, json.Unmarshal(answer, &c)
|
||||
}
|
||||
|
||||
func (a *asked) ask(key string, body any) (json.RawMessage, error) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
@@ -343,7 +304,7 @@ func (a *asked) ask(key string, body any) (json.RawMessage, error) {
|
||||
}
|
||||
var answer any
|
||||
switch key {
|
||||
case "artifacts":
|
||||
case "seat:mesh-controller.artifacts":
|
||||
answer = records()
|
||||
case CatalogueModules:
|
||||
if a.noCatalogue {
|
||||
@@ -359,8 +320,8 @@ func (a *asked) ask(key string, body any) (json.RawMessage, error) {
|
||||
if a.failing[name] {
|
||||
return nil, fmt.Errorf("no answer")
|
||||
}
|
||||
if _, has := args["commit"]; has {
|
||||
return nil, fmt.Errorf("asked %s at %v: the seat's verb takes no commit", name, args["commit"])
|
||||
if args["commit"] != "c-"+name {
|
||||
return nil, fmt.Errorf("asked %s at %v", name, args["commit"])
|
||||
}
|
||||
manifest, has := a.definitions[name]
|
||||
if !has {
|
||||
@@ -368,10 +329,9 @@ func (a *asked) ask(key string, body any) (json.RawMessage, error) {
|
||||
break
|
||||
}
|
||||
// The catalogue's tool answers through the runtime as the protocol's content list.
|
||||
raw, _ := json.Marshal(catalogue.CatalogOne{Module: catalogue.CatalogModule{
|
||||
CatalogEntry: catalogue.CatalogEntry{Module: name, Commit: "c-" + name}, Manifest: json.RawMessage(manifest)}})
|
||||
raw, _ := json.Marshal(map[string]any{"module": map[string]any{"module": name, "manifest": json.RawMessage(manifest)}})
|
||||
return json.Marshal(map[string]any{"content": []map[string]any{{"type": "text", "text": string(raw)}}})
|
||||
case "collect":
|
||||
case "seat:mesh-controller.collect":
|
||||
if args["confirm"] == "true" {
|
||||
answer = CollectAnswer{LetGo: []string{"artifact-store://app/server@" + d("m-old")}}
|
||||
} else {
|
||||
@@ -386,8 +346,8 @@ func (a *asked) ask(key string, body any) (json.RawMessage, error) {
|
||||
|
||||
func tool(t *testing.T, a *asked, name string) func(map[string]any) (any, error) {
|
||||
f := world()
|
||||
s := &Store{Ask: f.ask(t)}
|
||||
for _, tl := range Tools(s, recordsAsked{a}, &Catalogue{Ask: a.ask}) {
|
||||
s := &Store{Container: "mesh-registry", Run: f.run(t)}
|
||||
for _, tl := range Tools(s, Controller{Ask: a.ask}, &Catalogue{Ask: a.ask}) {
|
||||
if tl.Name == name {
|
||||
return tl.Run
|
||||
}
|
||||
@@ -430,18 +390,31 @@ func TestARealRunWithoutWhyIsRefusedAndAsksNothing(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferencesFallBackWhenCollectedCannotBeCarried(t *testing.T) {
|
||||
a := &asked{fail: map[string]bool{"collected": true}}
|
||||
out, err := tool(t, a, "artifact_store_references")(map[string]any{"repository": "app/server"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := out.(map[string]any)
|
||||
if m["note"] == nil || m["manifests"] == nil {
|
||||
t.Errorf("answer %v", m)
|
||||
}
|
||||
if _, err := tool(t, a, "artifact_store_references")(map[string]any{"repository": "nope"}); err == nil {
|
||||
t.Error("an unknown repository was answered")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Tools []string `json:"tools"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Reads []string `json:"reads"`
|
||||
Consumes []string `json:"consumes"`
|
||||
Requires []any `json:"requires"`
|
||||
Claims []struct {
|
||||
Tools []string `json:"tools"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Reads []string `json:"reads"`
|
||||
Claims []struct {
|
||||
Name string `json:"name"`
|
||||
Serves []string `json:"serves"`
|
||||
} `json:"claims"`
|
||||
@@ -450,7 +423,7 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var served []string
|
||||
for _, tl := range append(Tools(&Store{}, recordsAsked{&asked{}}, &Catalogue{}), (&views.Reader{}).Tools("artifact_store")...) {
|
||||
for _, tl := range Tools(&Store{}, Controller{}, &Catalogue{}) {
|
||||
if !strings.HasPrefix(tl.Name, "artifact_store_") || tl.Description == "" || tl.Run == nil {
|
||||
t.Errorf("tool %q", tl.Name)
|
||||
}
|
||||
@@ -463,48 +436,49 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
|
||||
t.Fatalf("served %v, manifest %v", served, listed)
|
||||
}
|
||||
// ADR 0571 clause 2: it holds the seat mesh-artifact-sweep and serves its verbs, and calls no verb of the controller's;
|
||||
// it reads no controller state, the lease neither (issue 1126).
|
||||
// CAT-AS1: what it holds and removes it asks the two registry seats the artifact store's holder serves.
|
||||
want := []string{"seat:mesh-hand-acts.hand-act", CatalogueModules, CatalogueModule, WantsTool, ManifestsTool,
|
||||
"seat:oci-artifact-registry.hold", "seat:oci-artifact-registry.release", "seat:oci-artifact-registry.remove",
|
||||
"seat:container-image-registry.hold", "seat:container-image-registry.release", "seat:container-image-registry.remove",
|
||||
// ADR 0632 §4: what the store holds it reads through the two seats' list.
|
||||
"seat:container-image-registry.list", "seat:oci-artifact-registry.list"}
|
||||
// the lease is the one controller state it reads (clause 10's exemption).
|
||||
want := []string{"seat:mesh-hand-acts.hand-act", CatalogueModules, CatalogueModule}
|
||||
if !reflect.DeepEqual(m.Invokes, want) {
|
||||
t.Errorf("invokes %v", m.Invokes)
|
||||
}
|
||||
// ADR 0615: the assigner's wants and the indexer's manifests are asked once and followed by their events, not read.
|
||||
if !reflect.DeepEqual(m.Consumes, []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.views.manifests"}) ||
|
||||
slices.Contains(m.Reads, WantsRead) || slices.Contains(m.Reads, ManifestsRead) {
|
||||
t.Errorf("consumes %v, reads %v", m.Consumes, m.Reads)
|
||||
}
|
||||
// The operator, 2026-11-04 (issue 1966): it reaches the store through the registry seats' verbs, and requires no
|
||||
// provision artifact-store.
|
||||
if len(m.Requires) != 0 {
|
||||
t.Errorf("requires %v, want nothing: the store is reached through the registry seats", m.Requires)
|
||||
}
|
||||
// Its address file is filled from the image seat holder's listen (ADR 0632 §1), never by the retired provision.
|
||||
address, err := os.ReadFile("../../files/store-address")
|
||||
if err != nil || strings.TrimSpace(string(address)) != "${seat:container-image-registry:reach}" {
|
||||
t.Errorf("the store's address file reads %q (%v), want the image seat's reach", address, err)
|
||||
}
|
||||
var verbs []string
|
||||
for _, tl := range SeatTools(&Holder{}, &Store{}) {
|
||||
verbs = append(verbs, strings.TrimPrefix(tl.Name, sweep.Seat+"."))
|
||||
verbs = append(verbs, strings.TrimPrefix(tl.Name, catalogue.ArtifactSweepSeat+"."))
|
||||
}
|
||||
catalogue.UseSeats(catalogue.SeatsFromDefinitions(seatlisttest.Definitions()))
|
||||
seat, _ := catalogue.SeatNamed(sweep.Seat)
|
||||
seat, _ := catalogue.SeatNamed(catalogue.ArtifactSweepSeat)
|
||||
var contract []string
|
||||
for _, v := range seat.Serves {
|
||||
contract = append(contract, v.Name)
|
||||
}
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != sweep.Seat || !reflect.DeepEqual(m.Claims[0].Serves, verbs) ||
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != catalogue.ArtifactSweepSeat || !reflect.DeepEqual(m.Claims[0].Serves, verbs) ||
|
||||
!reflect.DeepEqual(verbs, contract) {
|
||||
t.Errorf("claims %+v, serves %v, the seat's verbs %v", m.Claims, verbs, contract)
|
||||
}
|
||||
for _, r := range m.Reads {
|
||||
if strings.HasPrefix(r, "mesh-controller.") {
|
||||
if strings.HasPrefix(r, "mesh-controller.") && r != LeaseRead {
|
||||
t.Errorf("reads the controller's %s", r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestASocketRefusalIsAskedAgainThroughSudo(t *testing.T) {
|
||||
var ran []string
|
||||
run := func(_ context.Context, name string, args ...string) Ran {
|
||||
ran = append(ran, name)
|
||||
if name == "docker" {
|
||||
return Ran{Status: 1, Stderr: "permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock"}
|
||||
}
|
||||
return Ran{Stdout: "#end\n"}
|
||||
}
|
||||
if _, err := docker(context.Background(), run, 1000, "exec", "x"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(ran, []string{"docker", "sudo"}) {
|
||||
t.Errorf("ran %v", ran)
|
||||
}
|
||||
if _, err := docker(context.Background(), func(context.Context, string, ...string) Ran {
|
||||
return Ran{Status: 1, Stderr: "Error: No such container: mesh-registry"}
|
||||
}, 1000, "exec"); err == nil || !strings.Contains(err.Error(), "not on this machine") {
|
||||
t.Errorf("err %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,153 +0,0 @@
|
||||
package main
|
||||
|
||||
// **The sweep asks the registry seats** (CAT-AS1; novox/hq ADR 0617 §2, design 51): the artifact store's write side is
|
||||
// its holder's, so the sweep no longer deletes or holds over the store's door. What it keeps from retention it asks the
|
||||
// seat's `hold` for (a kept bundle or archive on oci-artifact-registry, a kept copy of an upstream base on
|
||||
// container-image-registry), and what is eligible it asks the seat's `remove` for, with the sweep's reason; both verbs
|
||||
// are in the manifest's `invokes`, and only this module is on their caller lists. A reference is read in both forms
|
||||
// (ADR 0605 §9): the older `artifact-store://<path>` and the seat's `<seat>/<module>/<artifact>@<digest>`; the keep set
|
||||
// keys each by the older form, so the two meet as one artifact. A call a seat refuses stops the sweep and writes the
|
||||
// sweep's finding, sweep-refused; the next sweep with no refusal clears it.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
artifactref "git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient/reference"
|
||||
ociseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
reg "git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry"
|
||||
)
|
||||
|
||||
// SweepRefused is the sweep's finding: a seat refused a hold or a remove the sweep asked.
|
||||
const SweepRefused = "sweep-refused"
|
||||
|
||||
// sweepFindingKey is the one key of the sweep's finding in the module's findings.
|
||||
var sweepFindingKey = meshconditionsclient.Key(meshconditionsclient.ScopeModule, Module, SweepRefused)
|
||||
|
||||
// canonical is a kept reference in the older form, `artifact-store://<path>`, whichever form it came in; another is as
|
||||
// it was.
|
||||
func canonical(reference string) string {
|
||||
path, kept := registryForms.InArtifactStore(reference)
|
||||
if !kept {
|
||||
return reference
|
||||
}
|
||||
return catalogue.ArtifactStoreScheme + path
|
||||
}
|
||||
|
||||
// seatOf is the seat, name and digest a kept reference is held under: a bundle or an archive on oci-artifact-registry,
|
||||
// an image or a copy of an upstream base on container-image-registry.
|
||||
func seatOf(reference string) (seat, name, digest string, err error) {
|
||||
path, kept := registryForms.InArtifactStore(reference)
|
||||
if !kept {
|
||||
return "", "", "", fmt.Errorf("%s is no reference into the mesh's registries", reference)
|
||||
}
|
||||
if name, digest, ok := strings.Cut(path, "/blobs/"); ok {
|
||||
return ociseat.Seat, name, digest, nil
|
||||
}
|
||||
if name, digest, ok := strings.Cut(path, "@"); ok {
|
||||
return reg.Seat, name, digest, nil
|
||||
}
|
||||
return "", "", "", fmt.Errorf("%s names no digest", reference)
|
||||
}
|
||||
|
||||
// isCopy is whether a kept reference is a copy of an upstream base.
|
||||
func isCopy(reference string) bool {
|
||||
path, _ := registryForms.InArtifactStore(reference)
|
||||
return strings.HasPrefix(path, "upstream/")
|
||||
}
|
||||
|
||||
// askSeat asks verb of the seat reference is held on, with body built from its name and digest.
|
||||
func (h *Holder) askSeat(reference, verb string, body func(name, digest string) any, answer any) error {
|
||||
seat, name, digest, err := seatOf(reference)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if h.Seats == nil {
|
||||
return fmt.Errorf("this process reaches no registry seat")
|
||||
}
|
||||
raw, err := h.Seats("seat:"+seat+"."+verb, body(name, digest))
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s's %s of %s@%s: %w", seat, verb, name, digest, err)
|
||||
}
|
||||
if answer != nil && len(raw) > 0 {
|
||||
if err := json.Unmarshal(raw, answer); err != nil {
|
||||
return fmt.Errorf("%s's answer to %s: %w", seat, verb, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// holdOnSeat asks the seat to keep reference from retention, for why.
|
||||
func (h *Holder) holdOnSeat(reference, why string) (bool, error) {
|
||||
var a reg.HoldAnswer
|
||||
err := h.askSeat(reference, reg.VerbHold, func(name, digest string) any {
|
||||
return reg.HoldInput{Name: name, Digest: digest, Reason: why}
|
||||
}, &a)
|
||||
if err == nil && !a.Held {
|
||||
err = fmt.Errorf("%s: the seat answered it is not held", reference)
|
||||
}
|
||||
return a.Held, err
|
||||
}
|
||||
|
||||
// Retention is the reason of a removal by digest, the sweep's alone (design 51 *Who may call which verb*): the holder
|
||||
// removes an untagged thing only for it. The seat's definition names it.
|
||||
const Retention = reg.RemoveInputReasonRetention
|
||||
|
||||
// removeOnSeat lets reference go: a hold an earlier sweep asked while it was kept is released first (the holder
|
||||
// refuses to remove a held thing; one not held answers released false), then it is removed for retention. why goes with
|
||||
// the release; the hand act records it.
|
||||
func (h *Holder) removeOnSeat(reference, why string) error {
|
||||
if err := h.askSeat(reference, reg.VerbRelease, func(name, digest string) any {
|
||||
return reg.ReleaseInput{Name: name, Digest: digest, Reason: why}
|
||||
}, ®.ReleaseAnswer{}); err != nil {
|
||||
return err
|
||||
}
|
||||
return h.askSeat(reference, reg.VerbRemove, func(name, digest string) any {
|
||||
return reg.RemoveInput{Name: name, Digest: digest, Reason: Retention}
|
||||
}, ®.RemoveAnswer{})
|
||||
}
|
||||
|
||||
// sweepFinding writes the sweep's finding: what the seat refused, and why.
|
||||
func (h *Holder) sweepFinding(what string, err error) {
|
||||
at := h.now().UTC()
|
||||
f := meshconditions.FindingsByKeyValue{V: meshconditionsclient.FindingVersion, Key: sweepFindingKey, Kind: SweepRefused,
|
||||
Subject: meshconditions.Subject{Scope: meshconditionsclient.ScopeModule, ID: Module},
|
||||
Severity: meshconditionsclient.Warning, Resolver: meshconditionsclient.ResolverOperator,
|
||||
Summary: fmt.Sprintf("the artifact sweep stopped: a registry seat refused %s (the evidence names why)", what),
|
||||
Headline: "The artifact sweep stopped",
|
||||
Explanation: "The sweep keeps what the mesh still needs and removes the rest through the registry seats. A seat " +
|
||||
"refused one of its asks, so it stopped before anything more was removed; what it kept stays kept.",
|
||||
Needs: "read the seat holder's journal for the refusal; the next sweep asks again.",
|
||||
Resolved: "Resolved: the artifact sweep ran through without a refusal",
|
||||
Evidence: []meshconditions.Evidence{{At: at.UTC(), Said: fmt.Sprintf("%s: %v", what, err)}},
|
||||
Source: "CAT-AS1", Observer: Module, At: at}
|
||||
if werr := h.Bus.Put(FindingsState, sweepFindingKey, f); werr != nil {
|
||||
h.logf("the sweep's finding (%s: %v) could not be written: %v", what, err, werr)
|
||||
}
|
||||
}
|
||||
|
||||
// sweepClean clears the sweep's finding after a sweep no seat refused.
|
||||
func (h *Holder) sweepClean() {
|
||||
if _, found, err := h.Bus.Get(FindingsState, sweepFindingKey); err == nil && found {
|
||||
if err := h.Bus.Delete(FindingsState, sweepFindingKey); err != nil {
|
||||
h.logf("the sweep's finding could not be cleared: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// keptWhy is why a kept reference is held, as the seat's hold takes it.
|
||||
func keptWhy(r *Ref) string {
|
||||
if r == nil || len(r.Why) == 0 {
|
||||
return "kept by the artifact sweep"
|
||||
}
|
||||
return "kept: " + strings.Join(r.Why, "; ")
|
||||
}
|
||||
|
||||
// registryForms reads and routes a reference of the registry form, with the two registry seats' names from their contracts
|
||||
// (the registry clients' reference forms, novox/hq ADR 0605 §9, ADR 0634 §4).
|
||||
var registryForms = artifactref.Seats{Image: reg.Seat, Object: ociseat.Seat}
|
||||
@@ -33,8 +33,7 @@ func TestEveryFileTakesItsTextFromATemplate(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Resources []map[string]any `json:"resources"`
|
||||
Packages map[string]string `json:"packages"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
Build struct {
|
||||
Artifacts []map[string]any `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
@@ -42,10 +41,6 @@ func TestEveryFileTakesItsTextFromATemplate(t *testing.T) {
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The builder's keys are read through its published package, taken by its series (ADR 0634 §1).
|
||||
if m.Packages["builds"] != "0.1" {
|
||||
t.Errorf("packages %v: want builds of series 0.1", m.Packages)
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if _, inline := r["content"]; inline {
|
||||
t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"])
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
)
|
||||
|
||||
// Recorded is what says which references the mesh keeps and lets go of them: the seat's holder (Holder, novox/hq ADR
|
||||
// 0571 clause 2), or a test's stand-in.
|
||||
// 0571 clause 2), or before it the controller (Controller), each in the controller's answer's shape.
|
||||
type Recorded interface {
|
||||
Artifacts(repository string) (*Records, error)
|
||||
Collect(why string, confirm bool, most int) (*CollectAnswer, error)
|
||||
@@ -27,9 +27,9 @@ func Tools(s *Store, c Recorded, cat *Catalogue) []stdio.Tool {
|
||||
{
|
||||
Name: "artifact_store_repositories",
|
||||
Description: "Every repository the artifact store holds: its tags and what each names, how many manifests it holds " +
|
||||
"(tagged or not — the mesh pins by digest, so most are untagged) and its size, each manifest's as its seat says it. " +
|
||||
"A layer two manifests share is counted in each. Read through the registry seats' list (ADR 0632 §4); changes " +
|
||||
"nothing. Replaces curl /v2/_catalog and /v2/<name>/tags/list. (r)",
|
||||
"(tagged or not — the mesh pins by digest, so most are untagged) and its size, the blobs its manifests mark. " +
|
||||
"A blob two repositories share is counted in each, and shared_bytes says how much of a size that is. Read from " +
|
||||
"the store's own files; changes nothing. Replaces curl /v2/_catalog and /v2/<name>/tags/list. (r)",
|
||||
Input: map[string]any{"repository": repoArg},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
v, err := s.View(ctx)
|
||||
@@ -41,9 +41,9 @@ func Tools(s *Store, c Recorded, cat *Catalogue) []stdio.Tool {
|
||||
},
|
||||
{
|
||||
Name: "artifact_store_usage",
|
||||
Description: "How large the artifact store is: every manifest's size together, as its seat says it (a layer two " +
|
||||
"manifests share is counted in each), and the largest repositories. Read through the registry seats' list (ADR " +
|
||||
"0632 §4); changes nothing. Replaces du on the store's directory. (r)",
|
||||
Description: "How large the artifact store is: every blob's bytes together, the largest repositories, the bytes " +
|
||||
"more than one repository's manifests mark, and the bytes no manifest marks — what the store's nightly collector " +
|
||||
"frees next. Read from the store's own files; changes nothing. Replaces du on the store's directory. (r)",
|
||||
Input: map[string]any{"top": map[string]any{"type": "integer", "description": "how many of the largest repositories to name (default 10, at most 200)"}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
top, err := count(args, "top", 10, 1, 200)
|
||||
@@ -139,6 +139,16 @@ func Tools(s *Store, c Recorded, cat *Catalogue) []stdio.Tool {
|
||||
}
|
||||
}
|
||||
|
||||
// View reads the store now.
|
||||
func (s *Store) View(ctx context.Context) (*View, error) {
|
||||
l, err := s.List(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m, unread := s.Manifests(ctx, l)
|
||||
return &View{L: l, M: m, Unread: unread}, nil
|
||||
}
|
||||
|
||||
// unreadNote says what manifests that could not be read do to the numbers, when there are any.
|
||||
func unreadNote(v *View) map[string]any {
|
||||
if len(v.Unread) == 0 {
|
||||
|
||||
@@ -4,15 +4,13 @@ import (
|
||||
"sort"
|
||||
)
|
||||
|
||||
// View is the store at one moment, as its seats list it (listed.go).
|
||||
// View is the store at one moment: its files, and every manifest's content that could be read.
|
||||
type View struct {
|
||||
L *Listing
|
||||
// M is each manifest read, by Key.
|
||||
M map[string]*Manifest
|
||||
// Unread is each manifest that could not be read, by Key, with why.
|
||||
Unread map[string]string
|
||||
// Held is each manifest a hold on its seat keeps from retention, by Key.
|
||||
Held map[string]bool
|
||||
}
|
||||
|
||||
// Marks are the blobs one manifest keeps from the store's collector: its own content, its
|
||||
|
||||
@@ -1 +1 @@
|
||||
${seat:container-image-registry:reach}
|
||||
${seat:mesh-artifact-store:reach}
|
||||
|
||||
@@ -3,40 +3,20 @@ module artifactstoretools
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep v0.0.0-00010101000000-000000000000
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.2 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.16.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
|
||||
github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
@@ -21,5 +23,5 @@ golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZ
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
|
||||
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package registry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// blobBound is the most GetBlob reads: a templates archive is a few kilobytes; anything near this is no archive a
|
||||
// take-in reads text from.
|
||||
const blobBound = 64 << 20
|
||||
|
||||
// GetBlob is the archive a recorded reference names (`artifact-store://<repository>/blobs/sha256:<hex>`), read whole
|
||||
// and checked against the digest it names: bytes that hash to another digest are refused, never read (R4e: the
|
||||
// controller reads a templates archive by the reference of a builder's done key). Gone when the store does not hold
|
||||
// it; ErrNotOurs for a reference outside the store or one that is not an archive.
|
||||
func (s Store) GetBlob(ctx context.Context, reference string) ([]byte, error) {
|
||||
repository, digest, archive, err := s.archive(reference)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !archive {
|
||||
return nil, fmt.Errorf("%w: %s is not an archive", ErrNotOurs, reference)
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url(repository, "blobs", digest), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
switch response.StatusCode {
|
||||
case http.StatusOK:
|
||||
case http.StatusNotFound:
|
||||
return nil, Gone
|
||||
default:
|
||||
return nil, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(response.Body, blobBound+1))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading %s/blobs/%s: %w", repository, digest, err)
|
||||
}
|
||||
if len(body) > blobBound {
|
||||
return nil, fmt.Errorf("%s/blobs/%s is larger than %d bytes, which no archive read here is", repository, digest, blobBound)
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest {
|
||||
return nil, fmt.Errorf("the artifact store answered %s/blobs/%s with bytes of the digest %s: refused", repository,
|
||||
digest, got)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package registry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A take-in reads a templates archive by its reference, and only bytes of the digest the reference names.
|
||||
func TestGetBlobReadsOnlyTheBytesOfItsDigest(t *testing.T) {
|
||||
good := []byte("the archive")
|
||||
sum := sha256.Sum256(good)
|
||||
digest := "sha256:" + hex.EncodeToString(sum[:])
|
||||
liar := "sha256:" + strings.Repeat("0", 64)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case strings.HasSuffix(r.URL.Path, "/x/files/blobs/"+digest), strings.HasSuffix(r.URL.Path, "/x/files/blobs/"+liar):
|
||||
_, _ = w.Write(good)
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
s := Store{Address: strings.TrimPrefix(server.URL, "http://")}
|
||||
ctx := context.Background()
|
||||
body, err := s.GetBlob(ctx, "artifact-store://x/files/blobs/"+digest)
|
||||
if err != nil || string(body) != string(good) {
|
||||
t.Fatalf("the archive: %q %v", body, err)
|
||||
}
|
||||
if _, err := s.GetBlob(ctx, "artifact-store://x/files/blobs/"+liar); err == nil || !strings.Contains(err.Error(), "refused") {
|
||||
t.Fatalf("bytes of another digest are read: %v", err)
|
||||
}
|
||||
if _, err := s.GetBlob(ctx, "artifact-store://x/other/blobs/"+digest); !errors.Is(err, Gone) {
|
||||
t.Fatalf("a blob the store does not hold: %v", err)
|
||||
}
|
||||
if _, err := s.GetBlob(ctx, "registry.example/x@"+digest); !errors.Is(err, ErrNotOurs) {
|
||||
t.Fatalf("a reference outside the store: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,345 @@
|
||||
package registry
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// Every archive the store keeps is held by a manifest (novox/hq issue 253, ADR 0189).
|
||||
//
|
||||
// **The store's collector marks only from manifests.** The mesh's store is a stock registry, and
|
||||
// its nightly `registry garbage-collect` walks every manifest in every repository, marks the blobs
|
||||
// those manifests name, and deletes every blob it did not mark. An image is a manifest, so what
|
||||
// the mesh keeps of an image survives. An archive was not: the builder put it in the store as a
|
||||
// bare blob — upload, then `PUT ?digest=` — and nothing in the store names it. To the collector a
|
||||
// bare blob is unreferenced, so the first real collection would have deleted every archive the
|
||||
// mesh holds, kept or not, and every machine pinning a bundle would have found it gone. The
|
||||
// collector runs `--dry-run` until this is true.
|
||||
//
|
||||
// **So each archive gets a holder**: the smallest OCI image manifest that names it — the empty
|
||||
// config, one layer, nothing else — put in the archive's own repository, by digest, untagged. The
|
||||
// collector marks it and so keeps the archive; the sweep lets go of an archive by deleting its
|
||||
// holder first, which is what lets the bytes go at the next collection.
|
||||
//
|
||||
// **Nothing a machine reads changes.** The recorded reference stays
|
||||
// `artifact-store://<module>/<artifact>/blobs/sha256:…`, and machines fetch the blob exactly as
|
||||
// before. The holder is the store's bookkeeping, not a second way to reach anything.
|
||||
//
|
||||
// **Deterministic, so it never needs recording.** The holder is composed from the archive's digest
|
||||
// and size alone, in a fixed field order with no timestamps or annotations, so the sweep can
|
||||
// compute which manifest holds any archive from the reference it already has plus one HEAD for the
|
||||
// size. No schema change, no second record that could disagree with the store.
|
||||
|
||||
const (
|
||||
// mediaManifest is the type a holder is put and asked for as.
|
||||
mediaManifest = "application/vnd.oci.image.manifest.v1+json"
|
||||
// mediaEmpty is the OCI empty descriptor's type: a config that says nothing, for a manifest
|
||||
// whose only purpose is to name its layer.
|
||||
mediaEmpty = "application/vnd.oci.empty.v1+json"
|
||||
// emptyDigest is the digest of `{}`, the empty config's content, fixed by the OCI spec.
|
||||
emptyDigest = "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"
|
||||
// mediaArchive is the layer type an archive is held as. Every archive the builder publishes is
|
||||
// `pack`'s gzipped tar, so this is the true type and not a placeholder — and it is a constant,
|
||||
// not read from anywhere, because the holder must be recomputable from the reference alone.
|
||||
mediaArchive = "application/vnd.oci.image.layer.v1.tar+gzip"
|
||||
)
|
||||
|
||||
// emptyConfig is the content emptyDigest names.
|
||||
var emptyConfig = []byte("{}")
|
||||
|
||||
// manifestAccept is what a manifest is asked for as. A registry answers a manifest HEAD only in a
|
||||
// type the caller named, and answers 404 to a bare one for a manifest it holds perfectly well
|
||||
// (measured 2026-09-28; internal/builder/registry.go says how that was found).
|
||||
var manifestAccept = []string{
|
||||
mediaManifest,
|
||||
"application/vnd.docker.distribution.manifest.v2+json",
|
||||
}
|
||||
|
||||
type descriptor struct {
|
||||
MediaType string `json:"mediaType"`
|
||||
Digest string `json:"digest"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
type holderManifest struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
MediaType string `json:"mediaType"`
|
||||
Config descriptor `json:"config"`
|
||||
Layers []descriptor `json:"layers"`
|
||||
}
|
||||
|
||||
// Holder is the manifest that holds an archive in the store, and its digest.
|
||||
//
|
||||
// A pure function of the archive's digest and size: the same two in give the same bytes out,
|
||||
// always, because `encoding/json` writes a struct's fields in their declared order and there is
|
||||
// nothing here that varies by when or where it was composed.
|
||||
func Holder(digest string, size int64) (body []byte, holder string) {
|
||||
body, err := json.Marshal(holderManifest{
|
||||
SchemaVersion: 2,
|
||||
MediaType: mediaManifest,
|
||||
Config: descriptor{MediaType: mediaEmpty, Digest: emptyDigest, Size: int64(len(emptyConfig))},
|
||||
Layers: []descriptor{{MediaType: mediaArchive, Digest: digest, Size: size}},
|
||||
})
|
||||
if err != nil {
|
||||
// Marshalling a struct of strings and integers cannot fail.
|
||||
panic(err)
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
return body, "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// Hold makes sure the store holds this archive by a manifest, and says whether it had to write one.
|
||||
//
|
||||
// Takes a reference as the mesh records it. An image is its own manifest and needs no holder, so
|
||||
// it answers false and nothing is asked. Idempotent: a holder already there is left alone, which
|
||||
// is what lets the sweep run it over every kept archive on every build and so backfill the bare
|
||||
// blobs published before holders existed (novox/hq issue 253).
|
||||
//
|
||||
// Gone when the store does not hold the archive at all: there is nothing to hold, and that is a
|
||||
// fact the caller reports rather than one this invents a remedy for.
|
||||
func (s Store) Hold(ctx context.Context, reference string) (bool, error) {
|
||||
repository, digest, archive, err := s.archive(reference)
|
||||
if err != nil || !archive {
|
||||
return false, err
|
||||
}
|
||||
size, err := s.blobSize(ctx, repository, digest)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return s.HoldBlob(ctx, repository, digest, size)
|
||||
}
|
||||
|
||||
// Held is whether the store holds this archive by its manifest. Asks and changes nothing — the
|
||||
// question an operator needs answered with "none unheld" before the collector is let loose.
|
||||
//
|
||||
// An image answers true: it is its own manifest. An archive the store does not have answers Gone.
|
||||
func (s Store) Held(ctx context.Context, reference string) (bool, error) {
|
||||
repository, digest, archive, err := s.archive(reference)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !archive {
|
||||
return true, nil
|
||||
}
|
||||
size, err := s.blobSize(ctx, repository, digest)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
_, holder := Holder(digest, size)
|
||||
return s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...)
|
||||
}
|
||||
|
||||
// HoldBlob puts the holder for a blob of this digest and size into its repository, unless it is
|
||||
// there already. Answers whether it wrote one.
|
||||
//
|
||||
// The builder calls this with the size it has just uploaded; the sweep, through Hold, with the size
|
||||
// the store reports. Both arrive at the same holder, which is the point of composing it.
|
||||
func (s Store) HoldBlob(ctx context.Context, repository, digest string, size int64) (bool, error) {
|
||||
if s.Address == "" {
|
||||
return false, fmt.Errorf("this mesh has no artifact store on its network to hold %s/%s in", repository, digest)
|
||||
}
|
||||
body, holder := Holder(digest, size)
|
||||
there, err := s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if there {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// The config must be in the repository before a manifest naming it is accepted: a registry
|
||||
// refuses a manifest whose blobs it cannot find there, which is the property that makes a
|
||||
// holder mean something.
|
||||
if err := s.putBlob(ctx, repository, emptyDigest, emptyConfig); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// **By digest, never by tag.** A tag would be one more name to move and one more thing the
|
||||
// collector's `--delete-untagged` would read as meaningful; the mesh names nothing by tag that
|
||||
// it pins by digest, and an untagged manifest is kept by plain collection.
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPut,
|
||||
s.url(repository, "manifests", holder), bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
request.Header.Set("Content-Type", mediaManifest)
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
said, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||
return false, fmt.Errorf("the artifact store refused to hold %s/%s: %s %s",
|
||||
repository, digest, response.Status, strings.TrimSpace(string(said)))
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// letGoOfHolder deletes the manifest holding an archive, before the archive's own link goes.
|
||||
//
|
||||
// **Holder first.** Deleting the blob link alone leaves a manifest still naming the blob, and the
|
||||
// collector would keep its bytes for ever on the strength of it — the sweep would record the
|
||||
// archive collected while the disk said otherwise. Deleting the holder first and failing before
|
||||
// the link goes leaves an unheld archive that the next sweep still offers, which is safe.
|
||||
//
|
||||
// A store that no longer has the blob answers Gone: without its size the holder cannot be named,
|
||||
// and without the blob there is nothing left for a holder to keep. A store that never had a holder
|
||||
// for it — an archive published before holders, never backfilled — answers 404 to the delete, and
|
||||
// that is the outcome wanted.
|
||||
func (s Store) letGoOfHolder(ctx context.Context, repository, digest string) error {
|
||||
size, err := s.blobSize(ctx, repository, digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, holder := Holder(digest, size)
|
||||
err = s.remove(ctx, s.url(repository, "manifests", holder), repository+"/manifests/"+holder)
|
||||
if err == Gone {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// archive reads a recorded reference into its repository and digest, and whether it is an archive
|
||||
// at all. Refuses as ErrNotOurs anything the mesh did not put in its own store.
|
||||
func (s Store) archive(reference string) (repository, digest string, archive bool, err error) {
|
||||
path, kept := catalogue.InArtifactStore(reference)
|
||||
if !kept {
|
||||
return "", "", false, fmt.Errorf("%w: %s", ErrNotOurs, reference)
|
||||
}
|
||||
if s.Address == "" {
|
||||
return "", "", false, fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference)
|
||||
}
|
||||
repository, kind, digest, err := split(path)
|
||||
if err != nil {
|
||||
return "", "", false, err
|
||||
}
|
||||
return repository, digest, kind == "blobs", nil
|
||||
}
|
||||
|
||||
// blobSize is how large the store says a blob is; Gone when it does not have it.
|
||||
func (s Store) blobSize(ctx context.Context, repository, digest string) (int64, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodHead, s.url(repository, "blobs", digest), nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
switch response.StatusCode {
|
||||
case http.StatusOK:
|
||||
case http.StatusNotFound:
|
||||
return 0, Gone
|
||||
default:
|
||||
return 0, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest)
|
||||
}
|
||||
// Read from the header rather than ContentLength: a HEAD's ContentLength is what the response
|
||||
// says it would have sent, which Go reports faithfully, but a proxy in between is free to drop
|
||||
// it, and the header is what the registry itself wrote.
|
||||
if length := response.Header.Get("Content-Length"); length != "" {
|
||||
if n, err := strconv.ParseInt(length, 10, 64); err == nil && n >= 0 {
|
||||
return n, nil
|
||||
}
|
||||
}
|
||||
if response.ContentLength >= 0 {
|
||||
return response.ContentLength, nil
|
||||
}
|
||||
return 0, fmt.Errorf("the artifact store holds %s/blobs/%s and will not say how large it is", repository, digest)
|
||||
}
|
||||
|
||||
// putBlob uploads a small blob unless the repository already has it: ask where, then put it there
|
||||
// naming the digest — the registry's own two steps, the same the builder takes for an archive.
|
||||
func (s Store) putBlob(ctx context.Context, repository, digest string, body []byte) error {
|
||||
if there, err := s.has(ctx, s.url(repository, "blobs", digest)); err != nil {
|
||||
return err
|
||||
} else if there {
|
||||
return nil
|
||||
}
|
||||
start, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
"http://"+s.Address+"/v2/"+repository+"/blobs/uploads/", nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
begun, err := s.client().Do(start)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot start an upload to %s: %w", repository, err)
|
||||
}
|
||||
begun.Body.Close()
|
||||
if begun.StatusCode != http.StatusAccepted {
|
||||
return fmt.Errorf("the artifact store answered %s when asked where to put a blob in %s", begun.Status, repository)
|
||||
}
|
||||
where := begun.Header.Get("Location")
|
||||
if where == "" {
|
||||
return fmt.Errorf("the artifact store accepted an upload to %s and said nowhere to put it", repository)
|
||||
}
|
||||
if strings.HasPrefix(where, "/") {
|
||||
where = "http://" + s.Address + where
|
||||
}
|
||||
separator := "?"
|
||||
if strings.Contains(where, "?") {
|
||||
separator = "&"
|
||||
}
|
||||
put, err := http.NewRequestWithContext(ctx, http.MethodPut, where+separator+"digest="+digest, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
put.Header.Set("Content-Type", "application/octet-stream")
|
||||
done, err := s.client().Do(put)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer done.Body.Close()
|
||||
if done.StatusCode != http.StatusCreated {
|
||||
said, _ := io.ReadAll(io.LimitReader(done.Body, 4096))
|
||||
return fmt.Errorf("the artifact store refused a blob in %s: %s %s", repository, done.Status, strings.TrimSpace(string(said)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// has is whether the store answers 200 for a HEAD at that URL.
|
||||
func (s Store) has(ctx context.Context, url string, accept ...string) (bool, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, media := range accept {
|
||||
request.Header.Add("Accept", media)
|
||||
}
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
switch response.StatusCode {
|
||||
case http.StatusOK:
|
||||
return true, nil
|
||||
case http.StatusNotFound:
|
||||
return false, nil
|
||||
default:
|
||||
return false, fmt.Errorf("the artifact store answered %s for %s", response.Status, url)
|
||||
}
|
||||
}
|
||||
|
||||
func (s Store) url(repository, kind, digest string) string {
|
||||
return "http://" + s.Address + "/v2/" + repository + "/" + kind + "/" + digest
|
||||
}
|
||||
|
||||
func (s Store) client() *http.Client {
|
||||
if s.HTTP != nil {
|
||||
return s.HTTP
|
||||
}
|
||||
return &http.Client{Timeout: 30 * time.Second}
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
package registry
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// Every kept archive is held by a manifest (novox/hq issue 253, ADR 0189).
|
||||
//
|
||||
// Against an in-memory registry that keeps blobs and manifests per repository and refuses what a
|
||||
// registry refuses — a blob whose digest does not match, a manifest whose digest does not match
|
||||
// or whose blobs the repository does not have, a manifest asked for without an Accept naming its
|
||||
// type. What is asserted is this side's decisions; the live test below asserts the registry's.
|
||||
|
||||
type memRegistry struct {
|
||||
mu sync.Mutex
|
||||
blobs map[string][]byte // repository + "@" + digest
|
||||
manifests map[string][]byte // repository + "@" + digest
|
||||
writes []string // every PUT and DELETE, as "METHOD path"
|
||||
}
|
||||
|
||||
func digestOf(body []byte) string {
|
||||
sum := sha256.Sum256(body)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func (m *memRegistry) serve(t *testing.T) Store {
|
||||
t.Helper()
|
||||
m.blobs = map[string][]byte{}
|
||||
m.manifests = map[string][]byte{}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
path := strings.TrimPrefix(r.URL.Path, "/v2/")
|
||||
if r.Method == http.MethodPut || r.Method == http.MethodDelete {
|
||||
m.writes = append(m.writes, r.Method+" "+r.URL.Path)
|
||||
}
|
||||
switch {
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(path, "/blobs/uploads/"):
|
||||
repository := strings.TrimSuffix(path, "/blobs/uploads/")
|
||||
w.Header().Set("Location", "/upload/"+repository+"?state=x")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
case r.Method == http.MethodPut && strings.HasPrefix(r.URL.Path, "/upload/"):
|
||||
repository := strings.TrimPrefix(r.URL.Path, "/upload/")
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
digest := r.URL.Query().Get("digest")
|
||||
if digest != digestOf(body) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
m.blobs[repository+"@"+digest] = body
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
case strings.Contains(path, "/blobs/"):
|
||||
repository, digest, _ := strings.Cut(path, "/blobs/")
|
||||
key := repository + "@" + digest
|
||||
body, ok := m.blobs[key]
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
switch r.Method {
|
||||
case http.MethodHead:
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case http.MethodDelete:
|
||||
delete(m.blobs, key)
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
default:
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
}
|
||||
case strings.Contains(path, "/manifests/"):
|
||||
repository, digest, _ := strings.Cut(path, "/manifests/")
|
||||
key := repository + "@" + digest
|
||||
switch r.Method {
|
||||
case http.MethodHead:
|
||||
if _, ok := m.manifests[key]; !ok || !strings.Contains(r.Header.Get("Accept"), mediaManifest) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case http.MethodPut:
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
if digest != digestOf(body) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
var named holderManifest
|
||||
if err := json.Unmarshal(body, &named); err != nil {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
for _, d := range append([]descriptor{named.Config}, named.Layers...) {
|
||||
if _, ok := m.blobs[repository+"@"+d.Digest]; !ok {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
fmt.Fprintf(w, "MANIFEST_BLOB_UNKNOWN %s", d.Digest)
|
||||
return
|
||||
}
|
||||
}
|
||||
m.manifests[key] = body
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
case http.MethodDelete:
|
||||
if _, ok := m.manifests[key]; !ok {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
delete(m.manifests, key)
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
}
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return Store{Address: strings.TrimPrefix(server.URL, "http://")}
|
||||
}
|
||||
|
||||
// bare puts an archive in the store the way the builder did before holders: a blob, nothing more.
|
||||
func (m *memRegistry) bare(repository string, body []byte) string {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
digest := digestOf(body)
|
||||
m.blobs[repository+"@"+digest] = body
|
||||
return catalogue.ArtifactStoreScheme + repository + "/blobs/" + digest
|
||||
}
|
||||
|
||||
func TestTheHolderIsComposedFromTheDigestAndSizeAlone(t *testing.T) {
|
||||
// The sweep must arrive at the very manifest the builder wrote, with nothing recorded between
|
||||
// them. Same inputs, same bytes — and a different size is a different holder, so a holder can
|
||||
// never be mistaken for one of a different blob.
|
||||
digest := "sha256:" + strings.Repeat("a", 64)
|
||||
one, first := Holder(digest, 42)
|
||||
two, second := Holder(digest, 42)
|
||||
if !bytes.Equal(one, two) || first != second {
|
||||
t.Fatalf("the same archive composed two holders:\n%s\n%s", one, two)
|
||||
}
|
||||
if _, other := Holder(digest, 43); other == first {
|
||||
t.Fatal("a different size composed the same holder")
|
||||
}
|
||||
want := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json",` +
|
||||
`"config":{"mediaType":"application/vnd.oci.empty.v1+json",` +
|
||||
`"digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2},` +
|
||||
`"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"` + digest + `","size":42}]}`
|
||||
if string(one) != want {
|
||||
t.Fatalf("the holder is\n%s\nwant\n%s", one, want)
|
||||
}
|
||||
if digestOf(emptyConfig) != emptyDigest {
|
||||
t.Fatalf("the empty config's digest is %s, not %s", digestOf(emptyConfig), emptyDigest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHoldBackfillsABareArchiveAndIsIdempotent(t *testing.T) {
|
||||
// The archives published before this have no holder. Hold, run over every kept archive on
|
||||
// every sweep, writes one the first time and nothing after.
|
||||
m := &memRegistry{}
|
||||
store := m.serve(t)
|
||||
ctx := context.Background()
|
||||
body := []byte("a theme")
|
||||
reference := m.bare("shell/config", body)
|
||||
|
||||
if held, err := store.Held(ctx, reference); err != nil || held {
|
||||
t.Fatalf("a bare blob reads as held=%v (%v)", held, err)
|
||||
}
|
||||
wrote, err := store.Hold(ctx, reference)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !wrote {
|
||||
t.Fatal("holding a bare archive wrote nothing")
|
||||
}
|
||||
_, holder := Holder(digestOf(body), int64(len(body)))
|
||||
if _, ok := m.manifests["shell/config@"+holder]; !ok {
|
||||
t.Fatalf("the store holds manifests %v; want %s", m.manifests, holder)
|
||||
}
|
||||
if held, err := store.Held(ctx, reference); err != nil || !held {
|
||||
t.Fatalf("after holding, held=%v (%v)", held, err)
|
||||
}
|
||||
|
||||
writes := len(m.writes)
|
||||
wrote, err = store.Hold(ctx, reference)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if wrote || len(m.writes) != writes {
|
||||
t.Fatalf("holding again wrote %v", m.writes[writes:])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnImageNeedsNoHolderAndAMissingArchiveIsGone(t *testing.T) {
|
||||
m := &memRegistry{}
|
||||
store := m.serve(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// An image is its own manifest: nothing is asked.
|
||||
wrote, err := store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/app@sha256:"+strings.Repeat("b", 64))
|
||||
if err != nil || wrote || len(m.writes) != 0 {
|
||||
t.Fatalf("holding an image wrote=%v err=%v writes=%v", wrote, err, m.writes)
|
||||
}
|
||||
// An archive the store does not have is a fact to report, not something to invent a holder for.
|
||||
_, err = store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/config/blobs/sha256:"+strings.Repeat("c", 64))
|
||||
if !errors.Is(err, Gone) {
|
||||
t.Fatalf("holding a missing archive answered %v, want Gone", err)
|
||||
}
|
||||
// And a reference that is not the mesh's is refused as such.
|
||||
if _, err := store.Hold(ctx, "docker.io/library/registry@sha256:abc"); !errors.Is(err, ErrNotOurs) {
|
||||
t.Fatalf("holding a vendor's image answered %v, want ErrNotOurs", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLettingGoOfAnArchiveDeletesItsHolderFirst(t *testing.T) {
|
||||
// A holder left behind would keep the bytes through every collection while the record said
|
||||
// collected; the link deleted first and the holder failing after would be that exactly.
|
||||
m := &memRegistry{}
|
||||
store := m.serve(t)
|
||||
ctx := context.Background()
|
||||
body := []byte("an old theme")
|
||||
reference := m.bare("shell/config", body)
|
||||
if _, err := store.Hold(ctx, reference); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.writes = nil
|
||||
|
||||
if err := store.LetGo(ctx, reference); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, holder := Holder(digestOf(body), int64(len(body)))
|
||||
want := []string{
|
||||
"DELETE /v2/shell/config/manifests/" + holder,
|
||||
"DELETE /v2/shell/config/blobs/" + digestOf(body),
|
||||
}
|
||||
if strings.Join(m.writes, "\n") != strings.Join(want, "\n") {
|
||||
t.Fatalf("the store was asked\n%s\nwant\n%s", strings.Join(m.writes, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
if len(m.manifests) != 0 {
|
||||
t.Fatalf("a holder survived: %v", m.manifests)
|
||||
}
|
||||
// Asked again, the archive is already gone, which is the outcome wanted.
|
||||
if err := store.LetGo(ctx, reference); !errors.Is(err, Gone) {
|
||||
t.Fatalf("letting go twice answered %v, want Gone", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLettingGoOfAnUnheldArchiveStillDeletesIt(t *testing.T) {
|
||||
// An archive published before holders and let go of before any sweep held it: the holder's
|
||||
// delete answers 404, which is the outcome wanted, and the blob still goes.
|
||||
m := &memRegistry{}
|
||||
store := m.serve(t)
|
||||
reference := m.bare("shell/config", []byte("never held"))
|
||||
if err := store.LetGo(context.Background(), reference); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(m.blobs) != 0 {
|
||||
t.Fatalf("the blob survived: %v", m.blobs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package registry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// An index is let go of with the platform manifests it names, except those a kept index of the same
|
||||
// repository names too (novox/hq ADR 0257). Each platform manifest is a manifest of the repository in
|
||||
// its own right, and the store's collector keeps every manifest a repository holds — so an index let go
|
||||
// of alone frees nothing of the images it names.
|
||||
|
||||
func digestN(n int) string { return fmt.Sprintf("sha256:%064x", n) }
|
||||
|
||||
// indexStore holds indexes and images in one repository, answers GETs with their documents, and records
|
||||
// every delete.
|
||||
type indexStore struct {
|
||||
mu sync.Mutex
|
||||
indexes map[string][]string // digest → the platform manifests it names
|
||||
images map[string]bool
|
||||
deleted []string
|
||||
}
|
||||
|
||||
func (s *indexStore) serve(t *testing.T) Store {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
digest := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
if children, ok := s.indexes[digest]; ok {
|
||||
var named []string
|
||||
for _, c := range children {
|
||||
named = append(named, `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"`+c+`","size":1}`)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json")
|
||||
_, _ = w.Write([]byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[` +
|
||||
strings.Join(named, ",") + `]}`))
|
||||
return
|
||||
}
|
||||
if s.images[digest] {
|
||||
w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
|
||||
_, _ = w.Write([]byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","layers":[]}`))
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
case http.MethodHead:
|
||||
if _, ok := s.indexes[digest]; ok || s.images[digest] {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
case http.MethodDelete:
|
||||
if _, ok := s.indexes[digest]; !ok && !s.images[digest] {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
s.deleted = append(s.deleted, digest)
|
||||
delete(s.indexes, digest)
|
||||
delete(s.images, digest)
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
default:
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return Store{Address: strings.TrimPrefix(server.URL, "http://")}
|
||||
}
|
||||
|
||||
func TestAnIndexGoesWithItsPlatformsAndAKeptIndexKeepsItsOwn(t *testing.T) {
|
||||
// Two indexes of one image's repository: the old one names platforms 11 and 12, the kept one 12 and 13.
|
||||
s := &indexStore{
|
||||
indexes: map[string][]string{digestN(1): {digestN(11), digestN(12)}, digestN(2): {digestN(12), digestN(13)}},
|
||||
images: map[string]bool{digestN(11): true, digestN(12): true, digestN(13): true},
|
||||
}
|
||||
store := s.serve(t)
|
||||
repository := "upstream/docker.io/library/golang"
|
||||
old := catalogue.ArtifactStoreScheme + repository + "@" + digestN(1)
|
||||
kept := catalogue.ArtifactStoreScheme + repository + "@" + digestN(2)
|
||||
spare, err := store.SpareKeptIndexes(context.Background(), []string{kept}, []string{old})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store.Spare = spare
|
||||
|
||||
if err := store.LetGo(context.Background(), old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Its own platform first, the index last; the platform the kept index names is left in place.
|
||||
if !slices.Equal(s.deleted, []string{digestN(11), digestN(1)}) {
|
||||
t.Fatalf("deleted %v; want the old index's own platform, then the index", s.deleted)
|
||||
}
|
||||
if !s.images[digestN(12)] || !s.images[digestN(13)] {
|
||||
t.Fatal("a platform a kept index names was let go of")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithoutSparingAnIndexGoesAloneAndAnImageNamesNoPlatforms(t *testing.T) {
|
||||
s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true, digestN(5): true}}
|
||||
store := s.serve(t)
|
||||
// No Spare: as before ADR 0257, the index alone.
|
||||
if err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(s.deleted, []string{digestN(1)}) {
|
||||
t.Fatalf("without a Spare, deleted %v", s.deleted)
|
||||
}
|
||||
// An image is one delete, with a Spare or without.
|
||||
s.deleted = nil
|
||||
spare, err := store.SpareKeptIndexes(context.Background(), nil, []string{catalogue.ArtifactStoreScheme + "web/app@" + digestN(5)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store.Spare = spare
|
||||
if err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@"+digestN(5)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(s.deleted, []string{digestN(5)}) {
|
||||
t.Fatalf("an image was let go of as %v", s.deleted)
|
||||
}
|
||||
// An index the store no longer holds is Gone, and nothing is deleted on its account.
|
||||
s.deleted = nil
|
||||
err = store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(9))
|
||||
if !errors.Is(err, Gone) || len(s.deleted) != 0 {
|
||||
t.Fatalf("an index the store does not hold: %v, deleted %v", err, s.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASpareThatCannotBeReadKeepsTheIndex(t *testing.T) {
|
||||
s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true}}
|
||||
store := s.serve(t)
|
||||
store.Spare = func(context.Context, string) (map[string]bool, error) { return nil, fmt.Errorf("the store went away") }
|
||||
err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(1))
|
||||
if err == nil || !strings.Contains(err.Error(), "was kept") {
|
||||
t.Fatalf("a spare that could not be read: %v", err)
|
||||
}
|
||||
if len(s.deleted) != 0 {
|
||||
t.Fatalf("deleted %v without knowing what a kept index names", s.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
// The spare list is read whole before the sweep: a kept index that cannot be read is an error before
|
||||
// anything is deleted, and a repository not read before keeps its index.
|
||||
func TestASpareListIsReadWholeBeforeAnythingGoes(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
store := Store{Address: strings.TrimPrefix(server.URL, "http://")}
|
||||
kept := catalogue.ArtifactStoreScheme + "web/base@" + digestN(2)
|
||||
old := catalogue.ArtifactStoreScheme + "web/base@" + digestN(1)
|
||||
if _, err := store.SpareKeptIndexes(context.Background(), []string{kept}, []string{old}); err == nil {
|
||||
t.Fatal("a kept index the store would not answer for was taken as naming nothing")
|
||||
}
|
||||
|
||||
s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true}}
|
||||
good := s.serve(t)
|
||||
spare, err := good.SpareKeptIndexes(context.Background(), nil, []string{catalogue.ArtifactStoreScheme + "other/repo@" + digestN(3)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
good.Spare = spare
|
||||
if err := good.LetGo(context.Background(), old); err == nil || len(s.deleted) != 0 {
|
||||
t.Fatalf("an index in a repository not read before the sweep was let go of: %v, deleted %v", err, s.deleted)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,325 @@
|
||||
// Package registry speaks to the mesh's artifact store over its own door.
|
||||
//
|
||||
// Only what the mesh needs that nothing else does: letting go of something it put there
|
||||
// (novox/hq ADR 0189, issue 108), and holding every archive it keeps by a manifest so the store's
|
||||
// own collector does not take it (novox/hq issue 253). Pushing is the builder's, through the container runtime; reading
|
||||
// is every machine's, through its runtime. This is the one operation that belongs to the thing
|
||||
// holding the records, because it is the only one that is a decision rather than a transfer.
|
||||
//
|
||||
// Copied from mesh-controller internal/artifacts at e04846e9 (novox/hq ADR 0571 clause 2): the seat mesh-artifact-sweep lets
|
||||
// go of what it no longer keeps exactly as the controller's sweep did; the controller's copy goes with its lease.
|
||||
package registry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// Store is the artifact store at an address, as this machine reaches it.
|
||||
type Store struct {
|
||||
// Address is `host:port` — the store as the caller reaches it now, composed and never
|
||||
// recorded (novox/hq 04-ISSUES/102).
|
||||
Address string
|
||||
// HTTP is the client used; nil is a client with a modest timeout.
|
||||
HTTP *http.Client
|
||||
// Spare says, for one repository, which manifests a kept index there names: the platform
|
||||
// manifests an index being let go of must leave in place (novox/hq ADR 0257). Nil spares every
|
||||
// platform manifest — an index is then let go of alone, as it was before.
|
||||
Spare func(ctx context.Context, repository string) (map[string]bool, error)
|
||||
}
|
||||
|
||||
// Gone is the answer when the store does not hold it: the outcome wanted, already true.
|
||||
var Gone = errors.New("the store does not hold it")
|
||||
|
||||
// ErrNotOurs is a reference this sweep will not address: not the mesh's own, or naming nothing
|
||||
// the store holds by digest.
|
||||
//
|
||||
// **A fact about the record, not about the store** (novox/hq issue 226). The two deserve opposite
|
||||
// responses — skip one and go on, abandon the sweep for the other — and collapsing them into "an
|
||||
// error" is how a cautious loop became one that did nothing while reporting the right number.
|
||||
var ErrNotOurs = errors.New("not a reference into the mesh's artifact store")
|
||||
|
||||
// LetGo asks the store to drop one artifact the mesh recorded making.
|
||||
//
|
||||
// Takes a reference as the mesh records it — `artifact-store://<module>/<artifact>@sha256:…` for
|
||||
// an image, `…/blobs/sha256:…` for an archive — because that is the identity every record uses,
|
||||
// and composes the address here at the moment of use.
|
||||
//
|
||||
// An archive is let go of in two deletes, its holder manifest and then the blob's link (novox/hq
|
||||
// issue 253); an image in one.
|
||||
//
|
||||
// Returns Gone when the store answers that it does not have it. That is not a failure: the sweep
|
||||
// wants the artifact absent, and it is. It is distinguished from success only so a caller can say
|
||||
// which of the two happened.
|
||||
func (s Store) LetGo(ctx context.Context, reference string) error {
|
||||
// **Strict, and deliberately** (novox/hq issue 226). Only a reference the mesh keeps in its
|
||||
// own vocabulary is addressed here. `Recorded` would read `docker.io/library/registry@sha256:…`
|
||||
// as the mesh's too — it cannot tell one registry host from another — so normalising belongs
|
||||
// where the provenance is known, which is the sweep reading its own build records, not here
|
||||
// where the only job is to refuse anything that is not plainly ours.
|
||||
path, kept := catalogue.InArtifactStore(reference)
|
||||
if !kept {
|
||||
// Nothing the mesh put in its own store. Refused rather than attempted: composing a
|
||||
// delete for a reference of unknown shape is how a sweep reaches something that is not
|
||||
// the mesh's. Distinguished from a store that refuses, so a sweep skips this and goes on.
|
||||
return fmt.Errorf("%w: %s", ErrNotOurs, reference)
|
||||
}
|
||||
if s.Address == "" {
|
||||
return fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference)
|
||||
}
|
||||
repository, kind, digest, err := split(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if kind == "blobs" {
|
||||
// **An archive's holder goes before the archive** (novox/hq issue 253): a manifest left
|
||||
// naming the blob would keep its bytes through every collection while the record said
|
||||
// collected. Gone here means the store has no such blob, so there is nothing to let go.
|
||||
if err := s.letGoOfHolder(ctx, repository, digest); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if kind == "manifests" && s.Spare != nil {
|
||||
// **An index's platform manifests go before the index** (novox/hq ADR 0257). Each was put
|
||||
// under its own digest, so each is a manifest of the repository in its own right, and the
|
||||
// store's collector keeps every manifest a repository holds: an index let go of alone frees
|
||||
// no byte of the images it names. Before, not after, so a sweep cut short leaves the index
|
||||
// to be asked about again, and with it the list of what is still to go.
|
||||
if err := s.letGoOfPlatforms(ctx, repository, digest, reference); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return s.remove(ctx, s.url(repository, kind, digest), reference)
|
||||
}
|
||||
|
||||
// letGoOfPlatforms deletes the manifests an index names, except those a kept index of the same
|
||||
// repository also names. A manifest that is not an index names none.
|
||||
func (s Store) letGoOfPlatforms(ctx context.Context, repository, digest, reference string) error {
|
||||
children, err := s.Platforms(ctx, repository, digest)
|
||||
if err != nil || len(children) == 0 {
|
||||
// Gone: there is no index to read, and the delete that follows says so.
|
||||
if errors.Is(err, Gone) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
spared, err := s.Spare(ctx, repository)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot say which platform manifests of %s a kept index names, so %s was kept: %w",
|
||||
repository, reference, err)
|
||||
}
|
||||
for _, child := range children {
|
||||
if spared[child] {
|
||||
continue
|
||||
}
|
||||
if err := s.remove(ctx, s.url(repository, "manifests", child), reference+" (its "+child+")"); err != nil && !errors.Is(err, Gone) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// anyManifest is every manifest the store may hold, indexes included, so it answers with the
|
||||
// document as it is.
|
||||
var anyManifest = []string{
|
||||
"application/vnd.oci.image.index.v1+json",
|
||||
"application/vnd.docker.distribution.manifest.list.v2+json",
|
||||
"application/vnd.oci.image.manifest.v1+json",
|
||||
"application/vnd.docker.distribution.manifest.v2+json",
|
||||
}
|
||||
|
||||
// Platforms is the manifests an index names, by digest; none for a manifest that is not an index.
|
||||
// Gone when the store does not hold it.
|
||||
func (s Store) Platforms(ctx context.Context, repository, digest string) ([]string, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url(repository, "manifests", digest), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
request.Header.Set("Accept", strings.Join(anyManifest, ", "))
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
switch response.StatusCode {
|
||||
case http.StatusOK:
|
||||
case http.StatusNotFound:
|
||||
return nil, Gone
|
||||
default:
|
||||
return nil, fmt.Errorf("the artifact store answered %s when asked for %s@%s", response.Status, repository, digest)
|
||||
}
|
||||
var document struct {
|
||||
Manifests []struct {
|
||||
Digest string `json:"digest"`
|
||||
} `json:"manifests"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(response.Body, 4<<20)).Decode(&document); err != nil {
|
||||
return nil, fmt.Errorf("%s@%s is not a manifest: %w", repository, digest, err)
|
||||
}
|
||||
var out []string
|
||||
for _, m := range document.Manifests {
|
||||
if strings.HasPrefix(m.Digest, "sha256:") {
|
||||
out = append(out, m.Digest)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SpareKeptIndexes is a Spare for a sweep, read whole before it starts: for every repository an
|
||||
// image reference of `touching` is in, each kept manifest there and every platform manifest a kept
|
||||
// index there names. Any read that fails is an error, and the sweep lets nothing go. A repository not
|
||||
// read here is answered with an error, so an index in it is kept. A kept index the store no longer
|
||||
// holds names nothing.
|
||||
func (s Store) SpareKeptIndexes(ctx context.Context, kept, touching []string) (func(context.Context, string) (map[string]bool, error), error) {
|
||||
imageIn := func(reference string) (repository, digest string, ok bool) {
|
||||
path, ours := catalogue.InArtifactStore(reference)
|
||||
if !ours {
|
||||
return "", "", false
|
||||
}
|
||||
repository, kind, digest, err := split(path)
|
||||
if err != nil || kind != "manifests" {
|
||||
return "", "", false
|
||||
}
|
||||
return repository, digest, true
|
||||
}
|
||||
read := map[string]map[string]bool{}
|
||||
for _, reference := range touching {
|
||||
if repository, _, ok := imageIn(reference); ok {
|
||||
read[repository] = map[string]bool{}
|
||||
}
|
||||
}
|
||||
for _, reference := range kept {
|
||||
repository, digest, ok := imageIn(reference)
|
||||
spared, touched := read[repository]
|
||||
if !ok || !touched {
|
||||
continue
|
||||
}
|
||||
spared[digest] = true
|
||||
children, err := s.Platforms(ctx, repository, digest)
|
||||
if errors.Is(err, Gone) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading the kept %s@%s: %w", repository, digest, err)
|
||||
}
|
||||
for _, c := range children {
|
||||
spared[c] = true
|
||||
}
|
||||
}
|
||||
return func(_ context.Context, repository string) (map[string]bool, error) {
|
||||
spared, done := read[repository]
|
||||
if !done {
|
||||
return nil, fmt.Errorf("%s was not read before the sweep began", repository)
|
||||
}
|
||||
return spared, nil
|
||||
}, nil
|
||||
}
|
||||
|
||||
// remove asks the store to delete what is at url. Gone when it has no such thing.
|
||||
func (s Store) remove(ctx context.Context, url, what string) error {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
switch response.StatusCode {
|
||||
case http.StatusAccepted, http.StatusOK, http.StatusNoContent:
|
||||
return nil
|
||||
case http.StatusNotFound:
|
||||
return Gone
|
||||
case http.StatusMethodNotAllowed:
|
||||
// The registry was started without deletion enabled. Said plainly, because the remedy is
|
||||
// a setting on the store's module and not anything about this artifact.
|
||||
return fmt.Errorf(
|
||||
"the artifact store refuses deletion: its server was started without it enabled "+
|
||||
"(REGISTRY_STORAGE_DELETE_ENABLED), so nothing can be collected until the store "+
|
||||
"module is applied again (novox/hq ADR 0189). Asking about %s", what)
|
||||
default:
|
||||
return fmt.Errorf("the artifact store answered %s for %s", response.Status, what)
|
||||
}
|
||||
}
|
||||
|
||||
// split reads a recorded path into the repository, which endpoint names the thing, and the digest.
|
||||
//
|
||||
// Two shapes, which are the two the mesh records: `<repository>@sha256:<hex>` is a manifest, and
|
||||
// `<repository>/blobs/sha256:<hex>` is a blob.
|
||||
func split(path string) (repository, kind, digest string, err error) {
|
||||
if before, after, ok := strings.Cut(path, "@sha256:"); ok {
|
||||
return before, "manifests", "sha256:" + after, nil
|
||||
}
|
||||
if before, after, ok := strings.Cut(path, "/blobs/sha256:"); ok {
|
||||
return before, "blobs", "sha256:" + after, nil
|
||||
}
|
||||
return "", "", "", fmt.Errorf("%w: %q names nothing the store holds by digest", ErrNotOurs, path)
|
||||
}
|
||||
|
||||
// IsIndex is whether a recorded reference is an index the store holds: one that names manifests. A
|
||||
// blob, or a manifest that names none, is not. Gone when the store does not hold it.
|
||||
func (s Store) IsIndex(ctx context.Context, reference string) (bool, error) {
|
||||
path, ours := catalogue.InArtifactStore(reference)
|
||||
if !ours {
|
||||
return false, fmt.Errorf("%w: %s", ErrNotOurs, reference)
|
||||
}
|
||||
repository, kind, digest, err := split(path)
|
||||
if err != nil || kind != "manifests" {
|
||||
return false, err
|
||||
}
|
||||
children, err := s.Platforms(ctx, repository, digest)
|
||||
return len(children) > 0, err
|
||||
}
|
||||
|
||||
// HoldsManifest is whether the store holds the manifest a recorded image reference names.
|
||||
func (s Store) HoldsManifest(ctx context.Context, reference string) (bool, error) {
|
||||
path, ours := catalogue.InArtifactStore(reference)
|
||||
if !ours {
|
||||
return false, fmt.Errorf("%w: %s", ErrNotOurs, reference)
|
||||
}
|
||||
repository, kind, digest, err := split(path)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if kind != "manifests" {
|
||||
return false, fmt.Errorf("%w: %s names a blob, not a manifest", ErrNotOurs, reference)
|
||||
}
|
||||
return s.has(ctx, s.url(repository, kind, digest), anyManifest...)
|
||||
}
|
||||
|
||||
// Holds is whether the store holds what a recorded reference names: the blob of an archive, the manifest of an image.
|
||||
// Gone when it does not; asks and changes nothing (ADR 0526 clause 3: a registration by hand is refused at bytes that
|
||||
// are not there).
|
||||
func (s Store) Holds(ctx context.Context, reference string) error {
|
||||
path, ours := catalogue.InArtifactStore(reference)
|
||||
if !ours {
|
||||
return fmt.Errorf("%w: %s", ErrNotOurs, reference)
|
||||
}
|
||||
repository, kind, digest, err := split(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if s.Address == "" {
|
||||
return fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference)
|
||||
}
|
||||
if kind == "blobs" {
|
||||
_, err := s.blobSize(ctx, repository, digest)
|
||||
return err
|
||||
}
|
||||
held, err := s.has(ctx, s.url(repository, kind, digest), anyManifest...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !held {
|
||||
return Gone
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
package registry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// Asking the store to let go of what the mesh no longer keeps (novox/hq ADR 0189, issue 108).
|
||||
//
|
||||
// A fake store records what it was asked to delete, so what is asserted is the mesh's decision
|
||||
// and the shape of the request — not the registry's behaviour, which is the registry's to test.
|
||||
|
||||
func fakeStore(t *testing.T, answer int) (Store, *[]string) {
|
||||
t.Helper()
|
||||
var asked []string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodHead && strings.Contains(r.URL.Path, "/blobs/") {
|
||||
// An archive's size, asked so its holder can be named (novox/hq issue 253). A store
|
||||
// that does not have the thing does not have its blob either.
|
||||
if answer == http.StatusNotFound {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Length", "7")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
if r.Method != http.MethodDelete {
|
||||
t.Errorf("the store was asked %s %s; collecting is a delete", r.Method, r.URL.Path)
|
||||
}
|
||||
asked = append(asked, r.URL.Path)
|
||||
w.WriteHeader(answer)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return Store{Address: strings.TrimPrefix(server.URL, "http://")}, &asked
|
||||
}
|
||||
|
||||
func TestAnImageAndAnArchiveAreAskedForAtTheirOwnEndpoints(t *testing.T) {
|
||||
// The two shapes the mesh records: a manifest by digest, and a blob by digest. They are
|
||||
// different endpoints, and asking at the wrong one answers 404 — which this would then
|
||||
// record as collected, leaving the bytes on disk for ever while the record says otherwise.
|
||||
store, asked := fakeStore(t, http.StatusAccepted)
|
||||
ctx := context.Background()
|
||||
|
||||
image := catalogue.ArtifactStoreScheme + "web/app@sha256:abc123"
|
||||
archive := catalogue.ArtifactStoreScheme + "web/config/blobs/sha256:def456"
|
||||
if err := store.LetGo(ctx, image); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.LetGo(ctx, archive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The archive's holder goes first, then the archive (novox/hq issue 253).
|
||||
_, holder := Holder("sha256:def456", 7)
|
||||
want := []string{
|
||||
"/v2/web/app/manifests/sha256:abc123",
|
||||
"/v2/web/config/manifests/" + holder,
|
||||
"/v2/web/config/blobs/sha256:def456",
|
||||
}
|
||||
if strings.Join(*asked, " ") != strings.Join(want, " ") {
|
||||
t.Fatalf("the store was asked %v; want %v", *asked, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStoreThatDoesNotHaveItAnswersGone(t *testing.T) {
|
||||
// The outcome wanted, already true. Told apart from success only so the sweep can say which
|
||||
// happened; both are recorded, because retrying for ever is the thing to avoid.
|
||||
store, _ := fakeStore(t, http.StatusNotFound)
|
||||
err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@sha256:abc123")
|
||||
if !errors.Is(err, Gone) {
|
||||
t.Fatalf("a store that does not hold it answered %v, want Gone", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStoreWithDeletionOffSaysSoAndNamesTheRemedy(t *testing.T) {
|
||||
// The registry answers 405 when it was started without deletion enabled. The remedy is a
|
||||
// setting on the store's module, and saying "405" would send somebody to the wrong place.
|
||||
store, _ := fakeStore(t, http.StatusMethodNotAllowed)
|
||||
err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@sha256:abc123")
|
||||
if err == nil {
|
||||
t.Fatal("a store that refuses deletion was read as success")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "REGISTRY_STORAGE_DELETE_ENABLED") {
|
||||
t.Fatalf("the refusal does not name the remedy: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReferenceThatIsNotTheMeshsOwnIsNeverAsked(t *testing.T) {
|
||||
// The whole safety of the sweep is that it names only what the mesh recorded putting there.
|
||||
// A reference of another shape — a vendor's image, a package version — is refused rather
|
||||
// than composed into a delete somewhere that is not the mesh's store.
|
||||
store, asked := fakeStore(t, http.StatusAccepted)
|
||||
for _, reference := range []string{
|
||||
"docker.io/library/registry@sha256:abc123",
|
||||
"registry@sha256:abc123",
|
||||
"1.4.2",
|
||||
} {
|
||||
if err := store.LetGo(context.Background(), reference); err == nil {
|
||||
t.Errorf("%s was asked about; it is not a reference into the mesh's store", reference)
|
||||
}
|
||||
}
|
||||
if len(*asked) != 0 {
|
||||
t.Fatalf("the store was asked about %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// A reference this sweep will not address says so as ErrNotOurs, which is a fact about the
|
||||
// record and not about the store (novox/hq issue 226).
|
||||
//
|
||||
// The sweep skips one and abandons itself for the other, so they cannot be the same error. The
|
||||
// first live run met a reference recorded with the store's old address, read the refusal as "the
|
||||
// store refuses everything", and collected none of the 1681 it had found.
|
||||
func TestAReferenceThisSweepWillNotAddressIsToldApartFromAStoreRefusing(t *testing.T) {
|
||||
store, asked := fakeStore(t, http.StatusAccepted)
|
||||
for _, reference := range []string{
|
||||
"docker.io/library/registry@sha256:abc123",
|
||||
"127.0.0.1:5100/mesh-tools/build@sha256:abc123",
|
||||
"1.4.2",
|
||||
} {
|
||||
err := store.LetGo(context.Background(), reference)
|
||||
if !errors.Is(err, ErrNotOurs) {
|
||||
t.Errorf("%s answered %v; a sweep must be able to skip it and go on", reference, err)
|
||||
}
|
||||
}
|
||||
if len(*asked) != 0 {
|
||||
t.Fatalf("the store was asked about %v", *asked)
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,6 @@
|
||||
{
|
||||
"module": "artifact-store-tools",
|
||||
"version": "1",
|
||||
"packages": {
|
||||
"builds": "0.1"
|
||||
},
|
||||
"claims": [
|
||||
{
|
||||
"name": "mesh-artifact-sweep",
|
||||
@@ -17,29 +14,21 @@
|
||||
]
|
||||
}
|
||||
],
|
||||
"requires": [
|
||||
"artifact-store"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"seat:mesh-catalog.modules",
|
||||
"seat:mesh-catalog.module",
|
||||
"mesh-delivery-assigner.assigner_read_wants",
|
||||
"mesh-delivery-indexer.indexer_read_manifests",
|
||||
"seat:oci-artifact-registry.hold",
|
||||
"seat:oci-artifact-registry.release",
|
||||
"seat:oci-artifact-registry.remove",
|
||||
"seat:container-image-registry.hold",
|
||||
"seat:container-image-registry.release",
|
||||
"seat:container-image-registry.remove",
|
||||
"seat:container-image-registry.list",
|
||||
"seat:oci-artifact-registry.list"
|
||||
"mesh-catalog.catalog_modules",
|
||||
"mesh-catalog.catalog_module"
|
||||
],
|
||||
"reads": [
|
||||
"mesh-delivery-assigner.wants",
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-delivery-builder.build-log",
|
||||
"node-engine.have"
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-delivery-assigner.views.wants",
|
||||
"mesh-delivery-indexer.views.manifests"
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"node-engine.have",
|
||||
"mesh-controller.lease"
|
||||
],
|
||||
"state": [
|
||||
"findings"
|
||||
@@ -48,9 +37,7 @@
|
||||
"artifact_store_repositories",
|
||||
"artifact_store_usage",
|
||||
"artifact_store_references",
|
||||
"artifact_store_collect",
|
||||
"artifact_store_first_views",
|
||||
"artifact_store_first_view_again"
|
||||
"artifact_store_collect"
|
||||
],
|
||||
"config": {
|
||||
"older-store-names": {
|
||||
@@ -98,6 +85,7 @@
|
||||
"artifact-store-tools"
|
||||
],
|
||||
"env": {
|
||||
"MESH_ARTIFACT_STORE_CONTAINER": "mesh-registry",
|
||||
"MESH_ARTIFACT_STORE_NAMES_FILE": "${dir:state}/store-names",
|
||||
"MESH_ARTIFACT_STORE_ADDRESS_FILE": "${dir:state}/store-address"
|
||||
}
|
||||
|
||||
@@ -171,7 +171,7 @@ func TestTheSwitcherDoesNotActOnAnotherModel(t *testing.T) {
|
||||
go func() { sw.Run(context.Background()); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(30 * time.Second):
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("the switcher ran on another model")
|
||||
}
|
||||
if r := sw.Report(); r.Running || r.Disabled == "" || len(f.calls) != 0 {
|
||||
|
||||
@@ -2,10 +2,10 @@ module asuszephyrusg14
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
|
||||
require (
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/nats-io/nats.go v1.54.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
|
||||
@@ -2,10 +2,10 @@ module avahi
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
|
||||
require (
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/nats-io/nats.go v1.54.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
|
||||
@@ -2,10 +2,10 @@ module blueman
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
|
||||
require (
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/nats-io/nats.go v1.54.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
|
||||
@@ -2,10 +2,10 @@ module bluetooth
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
|
||||
require (
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/nats-io/nats.go v1.54.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
|
||||
@@ -181,7 +181,7 @@ func standInCapture(release <-chan struct{}, fail error) func(ctx context.Contex
|
||||
// waitFor polls until the condition holds.
|
||||
func waitFor(t *testing.T, what string, cond func() bool) {
|
||||
t.Helper()
|
||||
for i := 0; i < 3000; i++ {
|
||||
for i := 0; i < 400; i++ {
|
||||
if cond() {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -181,7 +181,7 @@ func TestAFinishedGlassDocumentIsNotFiledAgainByItsExpiry(t *testing.T) {
|
||||
if _, err := r.call(t, "scanner.scan", docArgs(id, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFor(t, "the expired document to be closed and filed", func() bool { e, _ := os.ReadDir(folder); return len(e) >= 1 })
|
||||
time.Sleep(400 * time.Millisecond)
|
||||
if e, _ := os.ReadDir(folder); len(e) != 1 {
|
||||
t.Errorf("%d files, want 1", len(e))
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user