novox/mesh-controller
18
0
Fork 0
Code Issues Pull Requests 3 Releases Wiki Activity

Vendor the dependencies again for R4g's go.mod (mesh-sdk 9205222a, the mesh-host replace as before), so build-agent compiles the controller offline in the group's first tier (issue 1305)

Browse Source
This commit is contained in:
jschoubben
2026-10-23 16:38:31 +02:00
parent c5b8fca8bd
commit 9222575ae9
45 changed files with 3440 additions and 3508 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vendor/git.novox.be/novox/mesh-sdk/go/buildstep/askrun.go
Vendored
+409
View File
@@ -0,0 +1,409 @@
package buildstep
import (
"context"
"encoding/json"
"fmt"
"path"
"slices"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// **One ask, one whole job** (novox/hq ADR 0525 §3). A build module runs a pulled ask with its own step tools' bodies,
// the same ones its request and reply tools run (ADR 0525 §8), in the order the job names: a build ask fetches, then
// builds or copies each artifact; a check ask fetches, digests the level, takes the kept verdicts and tests the rest,
// the packages whose tests the change touches first. The answers are the tools' own, whole, so the asker judges them as
// it judged the tools' answers.
// StepFunc is one step tool's body: its arguments in, its answer out.
type StepFunc func(ctx context.Context, args map[string]any) (any, error)
// AskSteps are the step tools a build module runs an ask with. Copy is the image module's alone; Digest the Go module's.
type AskSteps struct {
Fetch, Build, Copy, Digest, Test StepFunc
// Toolchain is the toolchain the module builds with now; a build ask naming another is refused. Nil checks nothing
// (an image or an archive names none).
Toolchain func(ctx context.Context) (delivery.ToolchainUsed, error)
// Kept looks up a kept verdict by level and digest (the checker's kept bucket, delivery.KeptKey): whether one is
// kept and how many tests it counted. Nil takes none.
Kept func(ctx context.Context, level, digest string) (tests int, found bool, err error)
// Release removes a workspace once the ask's steps on it ended (the module's release tool's body); nil leaves it to
// the module's sweep.
Release func(workspace string) error
Logf func(format string, args ...any)
}
// RunAsk runs ask's job with steps and answers a delivery.BuildAnswer or a delivery.CheckAnswer.
func RunAsk(ctx context.Context, steps AskSteps, ask delivery.StepAsk) (any, error) {
if ask.Build != nil {
return steps.build(ctx, *ask.Build)
}
if ask.Check != nil {
return steps.check(ctx, *ask.Check)
}
return nil, Refuse("the ask %s holds no job", ask.ID)
}
func raw(v any) (json.RawMessage, error) {
if r, ok := v.(json.RawMessage); ok {
return r, nil
}
return json.Marshal(v)
}
// fetched is what an ask reads of a fetch's answer.
type fetched struct {
Workspace string `json:"workspace"`
Merge string `json:"merge"`
Conflicts []string `json:"conflicts"`
Changed []string `json:"changed"`
Modules []string `json:"modules"`
}
func (s AskSteps) fetch(ctx context.Context, args map[string]any) (json.RawMessage, fetched, error) {
if s.Fetch == nil {
return nil, fetched{}, Refuse("this build module fetches nothing")
}
out, err := s.Fetch(ctx, args)
if err != nil {
return nil, fetched{}, err
}
r, err := raw(out)
if err != nil {
return nil, fetched{}, err
}
var f fetched
if err := json.Unmarshal(r, &f); err != nil || f.Workspace == "" {
return r, f, fmt.Errorf("the fetch answered no workspace: %s", r)
}
return r, f, nil
}
func (s AskSteps) build(ctx context.Context, job delivery.BuildJob) (any, error) {
answer := delivery.BuildAnswer{}
if job.Toolchain != nil && s.Toolchain != nil {
tc, err := s.Toolchain(ctx)
if err != nil {
return nil, err
}
if tc.Version != job.Toolchain.Version || tc.Recipe != job.Toolchain.Recipe {
return nil, Refuse("the ask is for %s (recipe %s) and this node builds with %s (recipe %s)",
job.Toolchain.Version, job.Toolchain.Recipe, tc.Version, tc.Recipe)
}
answer.Toolchain = &tc
}
var f fetched
defer func() { s.release(f.Workspace) }()
for _, a := range job.Artifacts {
if a.From == "" && f.Workspace == "" {
r, got, err := s.fetch(ctx, job.Fetch)
if err != nil {
return nil, err
}
if _, candidate := job.Fetch["base"]; candidate {
// A candidate is built from its head merged onto the trunk; a head that does not merge leaves the tree at the
// trunk, and building that would store the trunk's build under the candidate's fingerprint.
if len(got.Conflicts) > 0 {
return nil, fmt.Errorf("the head does not merge into the trunk: files in conflict: %s", strings.Join(got.Conflicts, ", "))
}
if got.Merge == "" {
return nil, fmt.Errorf("the fetch with a base answered no merge: %s", r)
}
}
f = got
}
var (
out any
err error
)
if a.From != "" {
if s.Copy == nil {
return nil, Refuse("this build module copies no artifact (%s)", a.Artifact)
}
out, err = s.Copy(ctx, map[string]any{"module": job.Module, "artifact": a.Artifact, "from": a.From})
} else {
if s.Build == nil {
return nil, Refuse("this build module builds no artifact (%s)", a.Artifact)
}
args := map[string]any{"workspace": f.Workspace, "module": job.Path, "artifact": a.Artifact}
if job.Publish {
args["publish"] = true
}
out, err = s.Build(ctx, args)
}
if err != nil {
return nil, fmt.Errorf("%s: %w", a.Artifact, err)
}
r, err := raw(out)
if err != nil {
return nil, err
}
var built struct {
Digest string `json:"digest"`
Reference string `json:"reference"`
}
if err := json.Unmarshal(r, &built); err != nil || built.Digest == "" {
return nil, fmt.Errorf("the build of %s answered no digest: %s", a.Artifact, r)
}
answer.Artifacts = append(answer.Artifacts, delivery.AskBuilt{Artifact: a.Artifact, Digest: built.Digest, Reference: built.Reference})
}
return answer, nil
}
func (s AskSteps) check(ctx context.Context, job delivery.CheckJob) (any, error) {
r, f, err := s.fetch(ctx, job.Fetch)
if err != nil {
return nil, err
}
defer s.release(f.Workspace)
answer := delivery.CheckAnswer{Fetch: r}
if len(f.Conflicts) > 0 {
return answer, nil // the asker reads the conflict from the fetch's answer: a verdict, not an error
}
if s.Test == nil {
return nil, Refuse("this build module tests nothing")
}
runner := ""
if job.Runner != "" && (job.RunnerWhen == "" || changesUnder(f.Changed, job.RunnerWhen)) {
runner = job.Runner
}
for _, level := range job.Levels {
la := delivery.LevelAnswer{Level: level}
passed, err := s.level(ctx, job, level, f, runner, &la)
answer.Levels = append(answer.Levels, la)
if err != nil {
return nil, err
}
if !passed {
return answer, nil // the first level that did not pass ends the check (ADR 0484 clause 13)
}
}
return answer, nil
}
// level runs one level of a check ask and says whether it passed.
func (s AskSteps) level(ctx context.Context, job delivery.CheckJob, level string, f fetched, runner string, la *delivery.LevelAnswer) (bool, error) {
withRunner := func(args map[string]any) map[string]any {
if runner != "" {
args["runner"] = runner
}
return args
}
test := func(args map[string]any) (bool, error) {
out, err := s.Test(ctx, args)
if err != nil {
return false, err
}
r, err := raw(out)
if err != nil {
return false, err
}
la.Tests = append(la.Tests, r)
var t struct {
Passed bool `json:"passed"`
}
_ = json.Unmarshal(r, &t)
return t.Passed, nil
}
if s.Digest == nil {
// A module with no digest (Node) tests each module directory of the tree, those whose tests changed first.
for _, priority := range []bool{true, false} {
for _, dir := range f.Modules {
if changesTestIn(f.Changed, dir) != priority {
continue
}
passed, err := test(withRunner(map[string]any{"workspace": f.Workspace, "module": dir, "level": level}))
if err != nil {
return false, fmt.Errorf("the %s level of %s: %w", level, dir, err)
}
if !passed {
return false, nil
}
}
}
return true, nil
}
todo, all, keepRunner := s.plan(ctx, job, level, f, runner, withRunner, la)
if !keepRunner {
runner = "" // an installed module that ignores the runner judges the level as it judges any other (ADR 0535 §2)
}
for _, priority := range []bool{true, false} {
args := withRunner(map[string]any{"workspace": f.Workspace, "level": level})
if job.Run != "" {
args["run"] = job.Run
}
if all {
if priority {
continue // nothing to put first: the level runs whole after
}
} else {
var run []string
for _, p := range todo {
if changesTestOf(f.Changed, p) == priority {
run = append(run, p)
}
}
if len(run) == 0 {
continue
}
args["packages"] = run
}
passed, err := test(args)
if err != nil {
return false, fmt.Errorf("the %s level: %w", level, err)
}
if !passed {
return false, nil
}
}
return true, nil
}
// release removes an ask's workspace, saying a failure: the module's sweep removes it later.
func (s AskSteps) release(workspace string) {
if workspace == "" || s.Release == nil {
return
}
if err := s.Release(workspace); err != nil && s.Logf != nil {
s.Logf("the workspace %s is left to the sweep: %v", workspace, err)
}
}
// plan digests the level and takes the kept verdicts: the packages left to test, or all when the level runs whole, and
// whether the runner stays. A try of some tests runs whole, digesting nothing (a partial run proves no package). Under the
// candidate's runner the digest is asked as the candidate: when the candidate answers it, every package runs and nothing
// is digested for keeping (ADR 0535 §4); when the installed module answers it (it does not take the runner), the level is
// planned as any other and the runner is dropped. The level runs whole and takes nothing when the digest names no package
// or does not cover the siblings the fetch placed (ADR 0444, L5).
func (s AskSteps) plan(ctx context.Context, job delivery.CheckJob, level string, f fetched, runner string, withRunner func(map[string]any) map[string]any, la *delivery.LevelAnswer) ([]string, bool, bool) {
if job.Run != "" {
return nil, true, true
}
out, err := s.Digest(ctx, withRunner(map[string]any{"workspace": f.Workspace, "level": level}))
if err != nil {
if s.Logf != nil {
s.Logf("no digest of the %s level, so it runs whole: %v", level, err)
}
return nil, true, true
}
r, err := raw(out)
if err != nil {
return nil, true, true
}
var d struct {
Packages map[string]string `json:"packages"`
Beside []string `json:"beside"`
JudgedBy string `json:"judged_by"`
}
if err := json.Unmarshal(r, &d); err != nil {
return nil, true, true
}
if runner != "" {
if strings.HasPrefix(d.JudgedBy, "candidate ") {
la.Judged = d.JudgedBy
return nil, true, true // the candidate decides which verdicts would be taken: all run, nothing kept
}
runner = ""
}
if len(d.Packages) == 0 || !slices.Equal(d.Beside, besideOf(job.Fetch)) {
return nil, true, runner != "" // a digest that names nothing or not the siblings keys no verdict: the level runs whole
}
la.Digest = r
pkgs := make([]string, 0, len(d.Packages))
for p := range d.Packages {
pkgs = append(pkgs, p)
}
sort.Strings(pkgs)
var todo []string
for _, p := range pkgs {
if job.Kept && s.Kept != nil {
tests, found, err := s.Kept(ctx, level, d.Packages[p])
if err != nil && s.Logf != nil {
s.Logf("the kept verdict of %s at %s cannot be read, so it runs: %v", p, level, err)
}
if err == nil && found {
la.Kept = append(la.Kept, delivery.KeptTaken{Package: p, Digest: d.Packages[p], Tests: tests})
continue
}
}
todo = append(todo, p)
}
return todo, false, false
}
// besideOf are the siblings a fetch's arguments place, as a digest echoes them: `<owner/repo> <commit>`, sorted.
func besideOf(fetch map[string]any) []string {
raw, err := json.Marshal(fetch["beside"])
if err != nil {
return nil
}
var refs []struct {
Repository string `json:"repository"`
Commit string `json:"commit"`
}
_ = json.Unmarshal(raw, &refs)
var out []string
for _, r := range refs {
out = append(out, r.Repository+" "+r.Commit)
}
sort.Strings(out)
return out
}
// changesUnder says whether a path the merge changes is under prefix.
func changesUnder(changed []string, prefix string) bool {
for _, p := range changed {
if strings.HasPrefix(p, prefix) {
return true
}
}
return false
}
// isTestFile says whether a repository-relative path is a test file: Go's _test.go, and a TypeScript module's .test.
// and .spec. files and test directories.
func isTestFile(p string) bool {
switch {
case strings.HasSuffix(p, "_test.go"):
return true
case strings.Contains(p, ".test.") || strings.Contains(p, ".spec."):
return true
}
for _, seg := range strings.Split(path.Dir(p), "/") {
if seg == "__tests__" || seg == "test" || seg == "tests" {
return true
}
}
return false
}
// changesTestOf says whether the merge changes a test file of the Go package with the import path pkg: the file's
// directory and the import path agree on their last segments.
func changesTestOf(changed []string, pkg string) bool {
for _, f := range changed {
if !strings.HasSuffix(f, "_test.go") {
continue
}
dir := path.Dir(f)
if dir == pkg || strings.HasSuffix(pkg, "/"+dir) || strings.HasSuffix(dir, "/"+pkg) {
return true
}
}
return false
}
// changesTestIn says whether the merge changes a test file inside the directory of a Node module.
func changesTestIn(changed []string, dir string) bool {
for _, f := range changed {
if !isTestFile(f) || strings.HasSuffix(f, "_test.go") {
continue
}
if dir == "." || dir == "" || strings.HasPrefix(f, strings.TrimSuffix(dir, "/")+"/") {
return true
}
}
return false
}
vendor/git.novox.be/novox/mesh-sdk/go/buildstep/capacity.go
Vendored
+52
View File
@@ -0,0 +1,52 @@
package buildstep
import (
"encoding/json"
"fmt"
"os"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// CapacitySetting is the build modules' setting of each node's build capacity (novox/hq ADR 0525 §5): `<node>=<asks at
// once>` pairs separated by commas, as the builder's was (ADR 0485 §1). Each build module declares it with
// DefaultCapacities as its default, and the controller lays it into the module's settings file.
const CapacitySetting = "capacity"
// DefaultCapacities is the setting's default, the builder's before R4g: the control-node (novox) and the laptop (g14)
// build nothing.
const DefaultCapacities = "shanks=3,ace=2,g14=0,novox=0"
// NodeCapacity is node's build capacity, read from the module's settings file at path at every call, so a change takes
// effect at the next pull; with no path it reads fallback. A node the setting does not name runs one ask at a time (as
// the builder's did). A file or a setting that cannot be read gives 0 and says why: pulling on a setting nobody can read
// could build on a node that must not.
func NodeCapacity(path, fallback, node string, logf func(string, ...any)) func() int {
say := func(format string, args ...any) {
if logf != nil {
logf(format, args...)
}
}
return func() int {
setting := fallback
if path != "" {
raw, err := os.ReadFile(path)
if err != nil {
say("the build capacity cannot be read, so %s pulls nothing: %v", node, err)
return 0
}
var s map[string]any
if err := json.Unmarshal(raw, &s); err != nil {
say("the settings file %s is not JSON, so %s pulls nothing: %v", path, node, err)
return 0
}
setting, _ = s[CapacitySetting].(string)
}
caps, err := buildrun.ParseCapacity(setting)
if err != nil {
say("%v, so %s pulls nothing", fmt.Errorf("the build capacity: %w", err), node)
return 0
}
return (&buildrun.Places{Capacity: caps}).CapacityOf(node)
}
}
vendor/git.novox.be/novox/mesh-sdk/go/buildstep/fetch.go
Vendored
+88 -1
View File
@@ -4,6 +4,8 @@ import (
"context"
"fmt"
"net/url"
"os"
"path/filepath"
"regexp"
"strings"
@@ -13,12 +15,37 @@ import (
// Option changes FetchCommit.
type Option func(*fetchOptions)
type fetchOptions struct{ allowFile bool }
type fetchOptions struct {
allowFile bool
mirror string
}
// AllowFileRemote lets FetchCommit take a file:// remote. It is for a test's local repository; production
// code does not pass it, and FetchCommit then refuses anything but ssh://.
func AllowFileRemote() Option { return func(o *fetchOptions) { o.allowFile = true } }
// WithMirror makes FetchCommit (and MergeInto) borrow objects from mirror, a bare repository that UpdateMirror keeps
// in the build module's own cache (novox/hq ADR 0536 §2; issue 890): the workspace's .git/objects/info/alternates
// names the mirror's objects directory, so git transfers only the objects the mirror lacks. It is a plain file, never
// a symlink. mirror must be an absolute path of an existing directory; the workspace never writes to it.
func WithMirror(mirror string) Option { return func(o *fetchOptions) { o.mirror = mirror } }
// useMirror writes the alternates file of the repository in dir, when a mirror is set.
func useMirror(o fetchOptions, dir string) error {
if o.mirror == "" {
return nil
}
info := filepath.Join(dir, ".git", "objects", "info")
if err := os.MkdirAll(info, 0o755); err != nil {
return fmt.Errorf("buildstep: the mirror cannot be used: %w", err)
}
line := filepath.Join(o.mirror, "objects") + "\n"
if err := os.WriteFile(filepath.Join(info, "alternates"), []byte(line), 0o644); err != nil {
return fmt.Errorf("buildstep: the mirror cannot be used: %w", err)
}
return nil
}
var (
commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
refOK = regexp.MustCompile(`^refs/[A-Za-z0-9._/-]+$`)
@@ -51,6 +78,9 @@ func FetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit,
if _, err := git("init", "-q"); err != nil {
return err
}
if err := useMirror(o, dir); err != nil {
return err
}
if _, err := git("remote", "add", "origin", remote); err != nil {
return err
}
@@ -78,10 +108,67 @@ func FetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit,
return nil
}
// UpdateMirror brings the bare repository mirror to commit: it makes the mirror when it is absent (git init --bare,
// origin set to remote), then fetches the commit, or the ref when only the ref holds it, with the same rules as
// FetchCommit (no prompt, no credential helper, ssh:// or an allowed file:// remote, a 40-hex commit). The mirror
// keeps every commit it was brought to, so later fetches through WithMirror transfer only what is new. Callers
// serialise updates of one mirror.
func UpdateMirror(ctx context.Context, run forgekey.Runner, mirror, remote, commit, ref string, opts ...Option) error {
var o fetchOptions
for _, opt := range opts {
opt(&o)
}
if !filepath.IsAbs(mirror) {
return fmt.Errorf("buildstep: the mirror %q is not an absolute path", mirror)
}
if err := checkFetch(o, mirror, remote, commit, ref); err != nil {
return err
}
env := []string{"GIT_TERMINAL_PROMPT=0"}
git := func(args ...string) error {
r := run(ctx, env, "git", append([]string{"-c", "credential.helper="}, args...)...)
if r.Status != 0 || r.Err != "" {
return fmt.Errorf("git %s: %s", args[0], firstLine(r.Err+" "+r.Stderr))
}
return nil
}
in := func(args ...string) error { return git(append([]string{"-C", mirror}, args...)...) }
if _, err := os.Stat(filepath.Join(mirror, "HEAD")); err != nil {
if err := git("init", "-q", "--bare", mirror); err != nil {
return err
}
if err := in("remote", "add", "origin", remote); err != nil {
return err
}
} else if err := in("remote", "set-url", "origin", remote); err != nil {
return err
}
if err := in("fetch", "--no-tags", "origin", commit); err != nil {
if ref == "" {
return err
}
if rerr := in("fetch", "--no-tags", "origin", ref); rerr != nil {
return fmt.Errorf("%v; then the ref %s: %w", err, ref, rerr)
}
}
if err := in("cat-file", "-e", commit+"^{commit}"); err != nil {
return fmt.Errorf("the commit %s is not in the mirror: %w", commit, err)
}
return nil
}
func checkFetch(o fetchOptions, dir, remote, commit, ref string) error {
if dir == "" {
return fmt.Errorf("buildstep: no workspace directory")
}
if o.mirror != "" {
if !filepath.IsAbs(o.mirror) {
return fmt.Errorf("buildstep: the mirror %q is not an absolute path", o.mirror)
}
if fi, err := os.Stat(o.mirror); err != nil || !fi.IsDir() {
return fmt.Errorf("buildstep: the mirror %q is not an existing directory", o.mirror)
}
}
if !commitOK.MatchString(commit) {
return fmt.Errorf("buildstep: %q is not a full commit (40 hex digits)", commit)
}
vendor/git.novox.be/novox/mesh-sdk/go/buildstep/merge.go
Vendored
+3
View File
@@ -44,6 +44,9 @@ func MergeInto(ctx context.Context, run forgekey.Runner, dir, remote, base, comm
if err := checkFetch(o, dir, remote, commit, ""); err != nil {
return res, err
}
if err := useMirror(o, dir); err != nil {
return res, err
}
env := []string{"GIT_TERMINAL_PROMPT=0"}
git := func(args ...string) (string, error) {
full := append([]string{"-c", "credential.helper=", "-C", dir}, args...)
vendor/git.novox.be/novox/mesh-sdk/go/buildstep/pull.go
Vendored
+335
View File
@@ -0,0 +1,335 @@
package buildstep
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// **The pull loop the build modules share** (novox/hq ADR 0525 §4-§6, ADR 0536 §4; issue 1887). A build module on a node
// whose build capacity is above zero pulls the asks of its language's seat: from the delivery worker first, then check,
// then recheck, holding at most its capacity at once. Each ask runs as a step under the ask's own id, so its end is the
// key <node>.<ask id> in the module's steps state, which the asker watches; the ask is acknowledged when that end is
// written, kept alive while it runs by an in-progress signal, and redelivered by the stream to another node if this one
// stops. No part keeps a timer for an ask: the pull's own wait and the stream's acknowledgement wait are the bus's.
// Pulled is one ask a source handed over, with what decides its fate on the bus.
type Pulled struct {
// Kind is the worker it came from, filled by the puller.
Kind string
Data []byte
// Delivered is how many times the bus has handed it over, this time included.
Delivered uint64
// Ack ends it: it is never delivered again. InProgress tells the bus it is still being worked, restarting its
// acknowledgement wait. Nak hands it back to be delivered again at once (to this node or another). Term drops it.
Ack, InProgress, Nak, Term func() error
}
// AskSource is where a puller takes asks from: the seat's workers on the bus (JetStreamAsks) or a test double
// (FakeAsks).
type AskSource interface {
// Next takes one ask of kind: with wait 0 only one already waiting, else waiting for one at most wait. It answers
// nil and no error when none came.
Next(ctx context.Context, kind string, wait time.Duration) (*Pulled, error)
}
// Defaults of a Puller.
var (
// DefaultPullWait is how long one pull of the delivery worker waits for an ask while every worker is empty: the
// pull's own wait (as process.Consume's), after which the check and recheck workers are looked at again.
DefaultPullWait = 5 * time.Second
// DefaultHeartbeat is how often a running ask tells the bus it is still being worked; under the seat worker's
// acknowledgement wait of 60 s (mesh-controller internal/broker HolderConsumerFor).
DefaultHeartbeat = 20 * time.Second
)
// Puller pulls a build seat's asks and runs them.
type Puller struct {
Source AskSource
Language string // the seat's language: delivery.AskGo, AskNode or AskImage
Node string // the node this holder runs on, the first part of its step keys
Put Put // writes the module's steps state
// Capacity is how many asks the node runs at once; read before every pull, so a change takes effect at the next
// one. 0 pulls nothing.
Capacity func() int
// Run runs one ask's job; a *Refusal ends its step refused. Its result is the step's result.
Run func(ctx context.Context, ask delivery.StepAsk) (any, error)
// PullWait and Heartbeat default to DefaultPullWait and DefaultHeartbeat.
PullWait time.Duration
Heartbeat time.Duration
Logf func(format string, args ...any)
// Load reads one key of the module's steps state (nil and no error when it is absent). When set, Pull reads the
// node's pause key (delivery.PauseKey) before it pulls, and Pause and Resume write it with Put, so a pause outlives
// a restart.
Load func(key string) ([]byte, error)
mu sync.Mutex
inHand int
running map[string]RunningAsk
paused bool
wake chan struct{}
once sync.Once
}
func (p *Puller) init() {
p.once.Do(func() { p.wake = make(chan struct{}, 1) })
}
func (p *Puller) logf(format string, args ...any) {
if p.Logf != nil {
p.Logf(format, args...)
}
}
// Wake makes a waiting puller look again: after a capacity change, a resume, or a step's end.
func (p *Puller) Wake() {
p.init()
select {
case p.wake <- struct{}{}:
default:
}
}
// Pause stops the puller taking asks; the asks it holds run to their end. by names who paused it. With Load set the
// pause is written on the node's pause key first, and an error there leaves the puller as it was.
func (p *Puller) Pause(by string) error { return p.setPaused(true, by) }
// Resume lets the puller take asks again, written as Pause's is.
func (p *Puller) Resume(by string) error { return p.setPaused(false, by) }
func (p *Puller) setPaused(paused bool, by string) error {
if p.Load != nil {
key, err := delivery.PauseKey(p.Node)
if err != nil {
return err
}
raw, err := delivery.EncodePause(delivery.Pause{Paused: paused, By: by, At: now()})
if err != nil {
return err
}
if _, err := p.Put(key, json.RawMessage(raw)); err != nil {
return fmt.Errorf("buildstep: writing the pause of %s: %w", p.Node, err)
}
}
p.mu.Lock()
p.paused = paused
p.mu.Unlock()
p.Wake()
return nil
}
// restore reads the node's pause key, when Load is set.
func (p *Puller) restore() error {
if p.Load == nil {
return nil
}
key, err := delivery.PauseKey(p.Node)
if err != nil {
return err
}
raw, err := p.Load(key)
if err != nil {
return fmt.Errorf("buildstep: reading the pause of %s: %w", p.Node, err)
}
if raw == nil {
return nil
}
v, err := delivery.DecodePause(raw)
if err != nil {
return fmt.Errorf("buildstep: the pause of %s: %w", p.Node, err)
}
p.mu.Lock()
p.paused = v.Paused
p.mu.Unlock()
return nil
}
// Paused says whether the puller takes no asks.
func (p *Puller) Paused() bool {
p.mu.Lock()
defer p.mu.Unlock()
return p.paused
}
// InHand is how many asks run now.
func (p *Puller) InHand() int {
p.mu.Lock()
defer p.mu.Unlock()
return p.inHand
}
// RunningAsk is an ask a puller runs now, as its seat's current verb says it.
type RunningAsk struct {
ID string `json:"id"`
Kind string `json:"kind"`
Module string `json:"module,omitempty"`
Started time.Time `json:"started"`
}
// Running are the asks the puller runs now, oldest first.
func (p *Puller) Running() []RunningAsk {
p.mu.Lock()
defer p.mu.Unlock()
out := make([]RunningAsk, 0, len(p.running))
for _, r := range p.running {
out = append(out, r)
}
sort.Slice(out, func(i, j int) bool { return out[i].Started.Before(out[j].Started) })
return out
}
// room says whether the puller may take one more ask now.
func (p *Puller) room() bool {
capacity := 0
if p.Capacity != nil {
capacity = p.Capacity()
}
p.mu.Lock()
defer p.mu.Unlock()
return !p.paused && p.inHand < capacity
}
// Pull takes and runs asks until ctx ends. An ask still running then is neither ended nor acknowledged: the stream
// delivers it again, to another node or to this one restarted.
func (p *Puller) Pull(ctx context.Context) error {
p.init()
if p.Source == nil || p.Put == nil || p.Run == nil {
return errors.New("buildstep: a puller needs its source, its steps' put and its run")
}
if err := p.restore(); err != nil {
return err // pulling on a pause that cannot be read could build on a node paused on purpose
}
wait := p.PullWait
if wait <= 0 {
wait = DefaultPullWait
}
for {
if err := ctx.Err(); err != nil {
return err
}
if !p.room() {
select {
case <-ctx.Done():
return ctx.Err()
case <-p.wake:
}
continue
}
msg, err := p.next(ctx, wait)
if err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
p.logf("the %s asks cannot be pulled: %v", p.Language, err)
select { // the pull's own wait before the next try, as an empty pull's
case <-ctx.Done():
return ctx.Err()
case <-p.wake:
case <-time.After(wait):
}
continue
}
if msg == nil {
continue
}
p.take(ctx, msg)
}
}
// next is one ask from the workers in their order: one already waiting on delivery, check or recheck, and when none
// waits, a pull of the delivery worker that waits for one.
func (p *Puller) next(ctx context.Context, wait time.Duration) (*Pulled, error) {
for _, kind := range delivery.AskKinds {
msg, err := p.Source.Next(ctx, kind, 0)
if err != nil || msg != nil {
return withKind(msg, kind), err
}
}
msg, err := p.Source.Next(ctx, delivery.AskDelivery, wait)
return withKind(msg, delivery.AskDelivery), err
}
func withKind(msg *Pulled, kind string) *Pulled {
if msg != nil {
msg.Kind = kind
}
return msg
}
// take starts one ask: it decodes it, refuses one that is not of this seat, and runs it as a step under its own id.
func (p *Puller) take(ctx context.Context, msg *Pulled) {
kind := msg.Kind
ask, err := delivery.DecodeStepAsk(msg.Data)
if err == nil && (ask.Language != p.Language || (kind != "" && ask.Kind != kind)) {
err = fmt.Errorf("the ask %s is a %s ask of %s, pulled from the %s %s worker", ask.ID, ask.Kind, ask.Language, p.Language, kind)
}
if err != nil {
// An ask that is not one runs nowhere: delivering it again would only fail again.
p.logf("an ask dropped: %v", err)
_ = msg.Term()
return
}
step, key, err := begin(p.Put, p.Node, ask.Kind+" ask", ask.ID)
if err != nil {
p.logf("the ask %s cannot start on %s: %v", ask.ID, p.Node, err)
_ = msg.Nak()
return
}
module := ""
if ask.Build != nil {
module = ask.Build.Module
}
p.mu.Lock()
p.inHand++
if p.running == nil {
p.running = map[string]RunningAsk{}
}
p.running[ask.ID] = RunningAsk{ID: ask.ID, Kind: ask.Kind, Module: module, Started: step.Started}
p.mu.Unlock()
go p.run(ctx, msg, ask, key, step)
}
func (p *Puller) run(ctx context.Context, msg *Pulled, ask delivery.StepAsk, key string, step delivery.Step) {
defer func() {
p.mu.Lock()
p.inHand--
delete(p.running, ask.ID)
p.mu.Unlock()
p.Wake()
}()
beat := p.Heartbeat
if beat <= 0 {
beat = DefaultHeartbeat
}
done := make(chan struct{})
go func() {
t := time.NewTicker(beat)
defer t.Stop()
for {
select {
case <-done:
return
case <-t.C:
if msg.InProgress != nil {
_ = msg.InProgress()
}
}
}
}()
result, err := protect(ctx, func(ctx context.Context) (any, error) { return p.Run(ctx, ask) })
close(done)
if ctx.Err() != nil {
// The process stops: the ask is neither ended nor acknowledged here, so the stream gives it to a node that runs.
return
}
finish(p.Put, key, step, result, err)
if aerr := msg.Ack(); aerr != nil {
p.logf("the ask %s ended on %s and its acknowledgement failed: %v", ask.ID, p.Node, aerr)
}
}
vendor/git.novox.be/novox/mesh-sdk/go/buildstep/pull_bus.go
Vendored
+102
View File
@@ -0,0 +1,102 @@
package buildstep
import (
"context"
"fmt"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// JetStreamAsks is a build seat's workers on the bus: the shared pull consumer of each kind on the seat's stream
// (delivery.AskStream, AskWorker), which the broker makes when the seat is declared; a holder only binds them.
type JetStreamAsks struct {
JS jetstream.JetStream
Language string
mu sync.Mutex
cons map[string]jetstream.Consumer
}
func (s *JetStreamAsks) consumer(ctx context.Context, kind string) (jetstream.Consumer, error) {
s.mu.Lock()
defer s.mu.Unlock()
if c, ok := s.cons[kind]; ok {
return c, nil
}
stream, err := delivery.AskStream(s.Language)
if err != nil {
return nil, err
}
worker, err := delivery.AskWorker(s.Language, kind)
if err != nil {
return nil, err
}
c, err := s.JS.Consumer(ctx, stream, worker)
if err != nil {
return nil, fmt.Errorf("binding the %s worker %s on %s: %w", kind, worker, stream, err)
}
if s.cons == nil {
s.cons = map[string]jetstream.Consumer{}
}
s.cons[kind] = c
return c, nil
}
// Next is AskSource's: a fetch of one with no wait when wait is 0, else one that waits at most wait.
func (s *JetStreamAsks) Next(ctx context.Context, kind string, wait time.Duration) (*Pulled, error) {
c, err := s.consumer(ctx, kind)
if err != nil {
return nil, err
}
var batch jetstream.MessageBatch
if wait <= 0 {
batch, err = c.FetchNoWait(1)
} else {
batch, err = c.Fetch(1, jetstream.FetchMaxWait(wait))
}
if err != nil {
return nil, err
}
select {
case msg, ok := <-batch.Messages():
if !ok {
return nil, batch.Error()
}
var delivered uint64 = 1
if meta, err := msg.Metadata(); err == nil && meta != nil {
delivered = meta.NumDelivered
}
return &Pulled{Data: append([]byte(nil), msg.Data()...), Delivered: delivered,
Ack: msg.Ack, InProgress: msg.InProgress, Nak: msg.Nak, Term: msg.Term}, nil
case <-ctx.Done():
return nil, ctx.Err()
}
}
// DeclareAskWorkers makes language's seat stream and its three workers on a bus with no broker: a test's (the build
// modules' integration tests start one). On the mesh the broker declares them from the seat, as it declares every
// seat's stream (mesh-controller internal/broker SeatStreams); a holder never does.
func DeclareAskWorkers(ctx context.Context, js jetstream.JetStream, language string, ackWait time.Duration) error {
stream, err := delivery.AskStream(language)
if err != nil {
return err
}
seat, _ := delivery.BuildSeat(language)
if _, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{"mesh.seat." + seat + ".accept.>"},
Retention: jetstream.WorkQueuePolicy}); err != nil {
return fmt.Errorf("the stream %s: %w", stream, err)
}
for _, kind := range delivery.AskKinds {
worker, _ := delivery.AskWorker(language, kind)
subject, _ := delivery.AskSubject(language, kind)
if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: worker, FilterSubject: subject,
AckPolicy: jetstream.AckExplicitPolicy, AckWait: ackWait, MaxDeliver: 5}); err != nil {
return fmt.Errorf("the worker %s: %w", worker, err)
}
}
return nil
}
vendor/git.novox.be/novox/mesh-sdk/go/buildstep/pull_fake.go
Vendored
+107
View File
@@ -0,0 +1,107 @@
package buildstep
import (
"context"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// FakeAsks is a test double of a seat's workers for the build modules' unit tests: Publish puts an ask on a kind's
// worker, Next hands them out in their order, and the fates the puller gives them are counted. Safe for concurrent use.
type FakeAsks struct {
mu sync.Mutex
waiting map[string][][]byte
added chan struct{}
// Acked, Termed and Naked are the asks' ids or data by fate, InProgress how many heartbeats were given.
Acked, Termed, Naked []string
InProgress int
}
// NewFakeAsks is an empty double.
func NewFakeAsks() *FakeAsks {
return &FakeAsks{waiting: map[string][][]byte{}, added: make(chan struct{}, 1)}
}
// Publish puts an encoded ask on kind's worker.
func (f *FakeAsks) Publish(kind string, data []byte) {
f.mu.Lock()
f.waiting[kind] = append(f.waiting[kind], data)
f.mu.Unlock()
select {
case f.added <- struct{}{}:
default:
}
}
// PublishAsk encodes a and puts it on its kind's worker.
func (f *FakeAsks) PublishAsk(a delivery.StepAsk) error {
raw, err := delivery.EncodeStepAsk(a)
if err != nil {
return err
}
f.Publish(a.Kind, raw)
return nil
}
// Waiting is how many asks wait on kind's worker.
func (f *FakeAsks) Waiting(kind string) int {
f.mu.Lock()
defer f.mu.Unlock()
return len(f.waiting[kind])
}
// Fates is a copy of the acked, termed and naked lists.
func (f *FakeAsks) Fates() (acked, termed, naked []string) {
f.mu.Lock()
defer f.mu.Unlock()
return append([]string(nil), f.Acked...), append([]string(nil), f.Termed...), append([]string(nil), f.Naked...)
}
// Next is AskSource's.
func (f *FakeAsks) Next(ctx context.Context, kind string, wait time.Duration) (*Pulled, error) {
var deadline <-chan time.Time
if wait > 0 {
t := time.NewTimer(wait)
defer t.Stop()
deadline = t.C
}
for {
f.mu.Lock()
if q := f.waiting[kind]; len(q) > 0 {
data := q[0]
f.waiting[kind] = q[1:]
f.mu.Unlock()
return f.pulled(data), nil
}
f.mu.Unlock()
if wait <= 0 {
return nil, nil
}
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-deadline:
return nil, nil
case <-f.added:
}
}
}
func (f *FakeAsks) pulled(data []byte) *Pulled {
id := string(data)
if a, err := delivery.DecodeStepAsk(data); err == nil {
id = a.ID
}
fate := func(list *[]string) func() error {
return func() error {
f.mu.Lock()
*list = append(*list, id)
f.mu.Unlock()
return nil
}
}
return &Pulled{Data: data, Delivered: 1, Ack: fate(&f.Acked), Nak: fate(&f.Naked), Term: fate(&f.Termed),
InProgress: func() error { f.mu.Lock(); f.InProgress++; f.mu.Unlock(); return nil }}
}
vendor/git.novox.be/novox/mesh-sdk/go/buildstep/pull_seat.go
Vendored
+44
View File
@@ -0,0 +1,44 @@
package buildstep
import (
"context"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// SeatAsks is a build seat's workers as a tools bundle reaches them: through its runtime's seat pull and settle
// (stdio.SeatPull, SeatSettle; issue 1887 A11), one message per pull, only from the workers the module's membership
// lists.
type SeatAsks struct {
Language string
// Pull and Settle are stdio.SeatPull and stdio.SeatSettle when nil; a test gives its own.
Pull func(worker string, wait time.Duration) (*stdio.Pulled, error)
Settle func(id, fate string) error
}
// Next is AskSource's.
func (s SeatAsks) Next(ctx context.Context, kind string, wait time.Duration) (*Pulled, error) {
pull, settle := s.Pull, s.Settle
if pull == nil {
pull = stdio.SeatPull
}
if settle == nil {
settle = stdio.SeatSettle
}
worker, err := delivery.AskWorker(s.Language, kind)
if err != nil {
return nil, err
}
if err := ctx.Err(); err != nil {
return nil, err
}
p, err := pull(worker, wait)
if err != nil || p == nil {
return nil, err
}
fate := func(f string) func() error { return func() error { return settle(p.ID, f) } }
return &Pulled{Data: p.Body, Delivered: p.Delivered, Ack: fate(stdio.SettleAck), InProgress: fate(stdio.SettleProgress),
Nak: fate(stdio.SettleNak), Term: fate(stdio.SettleTerm)}, nil
}
vendor/git.novox.be/novox/mesh-sdk/go/buildstep/refusal.go
Vendored
+5
View File
@@ -1,5 +1,7 @@
package buildstep
import "fmt"
// Refusal is a step that is not run on purpose, as against one that failed: the toolchain installed is not the
// recipe's, the integration level's postgres major is not the one it is built for (novox/hq ADR 0469 §1, ADR 0470
// §4; issue 1473). A step's run returns one and the runner ends the step refused with its text.
@@ -11,3 +13,6 @@ type Refusal struct {
}
func (r *Refusal) Error() string { return r.Text }
// Refuse is a Refusal with its text formatted.
func Refuse(format string, args ...any) *Refusal { return &Refusal{Text: fmt.Sprintf(format, args...)} }
vendor/git.novox.be/novox/mesh-sdk/go/buildstep/seat_tools.go
Vendored
+40
View File
@@ -0,0 +1,40 @@
package buildstep
import (
stdio "git.novox.be/novox/mesh-sdk/go"
)
// SeatTools are a build seat's verbs on one node, as its holder serves them (novox/hq ADR 0525 §10; mesh-sdk catalogue
// buildSeatVerbs): `<seat>.current` (what the node runs, its capacity, whether it is paused), `<seat>.pause` and
// `<seat>.resume`, the pause kept on the node's pause key by the puller. what names the asks in the descriptions ("Go").
func SeatTools(seat, what string, p *Puller) []stdio.Tool {
verb := func(name, description string, run func() (any, error)) stdio.Tool {
return stdio.Tool{Name: seat + "." + name, Description: description,
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) { return run() }}
}
current := func() (any, error) {
capacity := 0
if p.Capacity != nil {
capacity = p.Capacity()
}
return map[string]any{"node": p.Node, "capacity": capacity, "running": p.Running(), "paused": p.Paused()}, nil
}
return []stdio.Tool{
verb("current", "The "+what+" asks this node runs now (each one's id, kind and when it started), its build capacity, "+
"and whether it is paused.", current),
verb("pause", "Take no new "+what+" ask on this node until resumed; an ask running here ends in its place. Kept "+
"across a restart (the node's pause key).", func() (any, error) {
if err := p.Pause("the seat's pause verb"); err != nil {
return nil, err
}
return current()
}),
verb("resume", "Take "+what+" asks again on this node.", func() (any, error) {
if err := p.Resume("the seat's resume verb"); err != nil {
return nil, err
}
return current()
}),
}
}
vendor/git.novox.be/novox/mesh-sdk/go/buildstep/step.go
Vendored
+48 -31
View File
@@ -42,43 +42,60 @@ func NewID() string {
// the error's text, or refused when the error is a *Refusal; a panic in run ends the step failed, naming it, and
// the process goes on. If the running write fails, nothing was started and Start answers the error.
func Start(ctx context.Context, put Put, node, tool string, run func(ctx context.Context) (result any, err error)) (string, error) {
step := delivery.Step{ID: NewID(), Tool: tool, Node: node, State: delivery.StepRunning, Started: now().UTC()}
key, err := delivery.StepKey(node, step.ID)
step, key, err := begin(put, node, tool, NewID())
if err != nil {
return "", err
}
if err := write(put, key, step); err != nil {
return "", fmt.Errorf("buildstep: writing the running step %s: %w", key, err)
}
go func() {
result, err := protect(ctx, run)
step.Ended = now().UTC()
switch ref := (*Refusal)(nil); {
case err == nil:
step.State = delivery.StepPassed
if result != nil {
raw, merr := json.Marshal(result)
if merr != nil {
step.State, step.Reason = delivery.StepFailed, "the result is not JSON: "+merr.Error()
} else {
step.Result = raw
}
}
case errors.As(err, &ref):
step.State, step.Reason = delivery.StepRefused, err.Error()
default:
step.State, step.Reason = delivery.StepFailed, err.Error()
}
for attempt := 1; ; attempt++ {
if werr := write(put, key, step); werr == nil || attempt >= EndAttempts {
return
}
time.Sleep(EndRetry)
}
}()
go end(ctx, put, key, step, run)
return step.ID, nil
}
// begin writes the step running under id.
func begin(put Put, node, tool, id string) (delivery.Step, string, error) {
step := delivery.Step{ID: id, Tool: tool, Node: node, State: delivery.StepRunning, Started: now().UTC()}
key, err := delivery.StepKey(node, step.ID)
if err != nil {
return delivery.Step{}, "", err
}
if err := write(put, key, step); err != nil {
return delivery.Step{}, "", fmt.Errorf("buildstep: writing the running step %s: %w", key, err)
}
return step, key, nil
}
// end runs run and writes the step's end.
func end(ctx context.Context, put Put, key string, step delivery.Step, run func(ctx context.Context) (any, error)) {
result, err := protect(ctx, run)
finish(put, key, step, result, err)
}
// finish writes the step's end from run's result or error: passed, failed or refused (a *Refusal).
func finish(put Put, key string, step delivery.Step, result any, err error) {
step.Ended = now().UTC()
switch ref := (*Refusal)(nil); {
case err == nil:
step.State = delivery.StepPassed
if result != nil {
raw, merr := json.Marshal(result)
if merr != nil {
step.State, step.Reason = delivery.StepFailed, "the result is not JSON: "+merr.Error()
} else {
step.Result = raw
}
}
case errors.As(err, &ref):
step.State, step.Reason = delivery.StepRefused, err.Error()
default:
step.State, step.Reason = delivery.StepFailed, err.Error()
}
for attempt := 1; ; attempt++ {
if werr := write(put, key, step); werr == nil || attempt >= EndAttempts {
return
}
time.Sleep(EndRetry)
}
}
// protect is run with a panic turned into the error that ends the step.
func protect(ctx context.Context, run func(ctx context.Context) (any, error)) (result any, err error) {
defer func() {
vendor/git.novox.be/novox/mesh-sdk/go/buildstep/workspaces.go
Vendored
+160
View File
@@ -0,0 +1,160 @@
package buildstep
// A build module's room to work and its workspaces (novox/hq issue 1839, ADR 0509): a step is refused while the
// filesystem it writes to has less free than the floor, and a workspace is removed when its caller releases it, or,
// unheld, once its last use is past a bound (a lost caller's), never while a running step holds it.
import (
"fmt"
"os"
"path/filepath"
"regexp"
"sort"
"sync"
"syscall"
"time"
)
// FreeSpaceFloor is the free space a build module needs on its work and cache filesystems to take a step: the floor
// build-agent keeps (mesh-controller internal/builder/floor.go, issue 1834).
const FreeSpaceFloor uint64 = 20 << 30
// UnheldBound is how long a workspace no step holds is kept after its last use, when its caller never released it
// (ADR 0509: a lost caller's workspace goes after two hours).
const UnheldBound = 2 * time.Hour
// FreeBytes reads the bytes an unprivileged writer can still use on the filesystem holding path.
func FreeBytes(path string) (uint64, error) {
var s syscall.Statfs_t
if err := syscall.Statfs(path, &s); err != nil {
return 0, err
}
return s.Bavail * uint64(s.Bsize), nil
}
// Floor refuses a step while a filesystem it writes to has less free than Bytes. A zero Bytes checks nothing.
type Floor struct {
// Node is named in the refusal.
Node string
Bytes uint64
// Free reads a path's free bytes; FreeBytes when nil. A test answers for a disk it does not have.
Free func(path string) (uint64, error)
}
// Check answers a *Refusal, "refused: <node> has N GiB free under the floor of F GiB (on <path>)", when any path's
// filesystem has less free than the floor, or its free space cannot be read; nil otherwise. A path not made yet is
// read at its nearest folder that is; an empty path is skipped.
func (f Floor) Check(paths ...string) error {
if f.Bytes == 0 {
return nil
}
free := f.Free
if free == nil {
free = FreeBytes
}
for _, p := range paths {
if p == "" {
continue
}
at := p
for {
if _, err := os.Stat(at); err == nil || filepath.Dir(at) == at {
break
}
at = filepath.Dir(at)
}
n, err := free(at)
if err != nil {
return &Refusal{Text: fmt.Sprintf("refused: the free space of %s on %s cannot be read: %v", p, f.Node, err)}
}
if n < f.Bytes {
return &Refusal{Text: fmt.Sprintf("refused: %s has %d GiB free under the floor of %d GiB (on %s)", f.Node, n>>30, f.Bytes>>30, p),
Want: fmt.Sprintf("%d GiB", f.Bytes>>30), Have: fmt.Sprintf("%d GiB", n>>30)}
}
}
return nil
}
var workspaceID = regexp.MustCompile(`^[a-z0-9]+$`)
// Workspaces are a build module's workspaces, each <Dir>/<id>, and which of them running steps hold.
type Workspaces struct {
Dir string
mu sync.Mutex
held map[string]int
}
// Hold marks the workspace held by a running step until the answered func is called; both mark it used now, so the
// sweep counts its bound from the step's end. An id that is not a workspace's holds nothing.
func (w *Workspaces) Hold(id string) func() {
if !workspaceID.MatchString(id) {
return func() {}
}
w.mu.Lock()
if w.held == nil {
w.held = map[string]int{}
}
w.held[id]++
w.mu.Unlock()
w.touch(id)
var once sync.Once
return func() {
once.Do(func() {
w.touch(id)
w.mu.Lock()
if w.held[id]--; w.held[id] <= 0 {
delete(w.held, id)
}
w.mu.Unlock()
})
}
}
func (w *Workspaces) touch(id string) {
now := time.Now()
_ = os.Chtimes(filepath.Join(w.Dir, id), now, now) // a workspace not made yet has nothing to mark
}
// Release removes the workspace at once: its caller's chain of steps has ended. One already gone is no error; one a
// running step holds is refused and kept; an id that is not a workspace's is refused.
func (w *Workspaces) Release(id string) error {
if !workspaceID.MatchString(id) {
return fmt.Errorf("%q is not a workspace id", id)
}
w.mu.Lock()
defer w.mu.Unlock()
if w.held[id] > 0 {
return fmt.Errorf("the workspace %s is held by a running step", id)
}
if err := os.RemoveAll(filepath.Join(w.Dir, id)); err != nil {
return fmt.Errorf("the workspace %s could not be removed: %w", id, err)
}
return nil
}
// Sweep removes every workspace no step holds whose last use is older than bound, and answers the ids it removed,
// sorted. It is the safety net for a caller that never released its workspace.
func (w *Workspaces) Sweep(now time.Time, bound time.Duration) []string {
entries, err := os.ReadDir(w.Dir)
if err != nil {
return nil
}
removed := []string{}
w.mu.Lock()
defer w.mu.Unlock()
for _, e := range entries {
if !e.IsDir() || !workspaceID.MatchString(e.Name()) || w.held[e.Name()] > 0 {
continue
}
info, err := e.Info()
if err != nil || now.Sub(info.ModTime()) < bound {
continue
}
if os.RemoveAll(filepath.Join(w.Dir, e.Name())) == nil {
removed = append(removed, e.Name())
}
}
sort.Strings(removed)
return removed
}
vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seats.go
Vendored
+41 -38
View File
@@ -45,6 +45,10 @@ type Seat struct {
// today — which the bus refuses, because a namespace belongs to who it is named for.
Accepts []string
Emits []string
// PulledByKind says each accepted kind is a worker of its own on the seat's stream, which its holders pull in the
// order Accepts lists them (novox/hq ADR 0525 §2, ADR 0536 §4): the build seats'. Compiled, never stored, like
// Receives.
PulledByKind bool
// Serves carries each verb in full — name, description, schema — because a role's tools are the
// mesh's to define and an agent's to call (novox/hq ADR 0132, design 33 §2).
Serves []Verb
@@ -238,31 +242,19 @@ var defaultSeats = append([]Seat{
// They keep their names until that migration is done deliberately, apart from the node-* pass.
{Name: "npm-package-registry", Scope: ScopeMesh, Delivers: "npm-package-registry", Decision: "novox/hq ADR 0109"},
{Name: "git", Scope: ScopeMesh, Delivers: "git", Decision: "novox/hq ADR 0111"},
// A build is work submitted to this role and its outcome is the role's own event (ADR 0129).
// One publish reaches whoever asked, the controller that records it, and the catalogue that
// places it in the graph — what the old bus's shared exchange did for free.
// A build says what it does as it does it (novox/hq ADR 0157): `started` when work is taken,
// `log.<build id>` for every line, `built` for the outcome. The log's tail token is the build's
// id, so a reader follows one build by subject alone.
// **Node-scoped, and every holder takes from one queue** (novox/hq ADR 0190): a build is asked of
// the role, and whichever machine holding the seat is idle pulls it. One holder per machine is
// what the scope says; sharing the work is what a seat's queue has always done.
//
// **And its holder answers for the build it is running** (novox/hq ADR 0219): what it is
// building, kill it, take nothing new, take again. The queue as a whole is the controller's to
// show and change (`queue`, `cancel`, `clear`); what one machine does with an ask it already
// took only that machine can do. `paused.<node>` is each holder saying whether it takes work, so
// the controller can tell a plan waiting on a paused seat from one that is late.
{Name: "node-build-agent", Scope: ScopeNode,
Accepts: []string{"build"}, Emits: []string{"started", "built", "log.*", "paused.*"},
Serves: buildAgentVerbs(),
Decision: "novox/hq ADR 0190, ADR 0219"},
// **Retired by ADR 0190, kept while a manifest still claims it.** The one build machine's seat.
// A claim to a seat the mesh no longer defines is refused, and the module holding this one is
// assigned on a live machine until build-agent replaces it — removing the row first would make
// that machine unresolvable in the meantime. Deleted once no registered manifest claims it.
{Name: "mesh-build-machine", Scope: ScopeMesh,
Accepts: []string{"build"}, Emits: []string{"started", "built", "log.*"}, Decision: "novox/hq ADR 0190"},
// **A build node's work, one seat per build language** (novox/hq ADR 0525 §2, §10; ADR 0536 §4; issue 1887). The
// build module of each language claims its seat on every node it runs on; the builder and the checker put their
// asks on the seat's accepted kinds, which the broker derives as one work queue with a shared worker per kind, and
// each holder pulls delivery first, then check, then recheck, at most its node's build capacity at once. Its verbs
// say what the node runs and pause or resume it there: every holder pulls from the same worker, so a node's pause is
// its holder taking nothing, kept on the module's own pause key (issue 1887's design limit 2). They replace
// node-build-agent, retired with build-agent (ADR 0524 §5).
{Name: "node-build-go", Scope: ScopeNode, Accepts: buildAskKinds(), PulledByKind: true, Serves: buildSeatVerbs("Go"),
Decision: "novox/hq ADR 0525, ADR 0536"},
{Name: "node-build-node", Scope: ScopeNode, Accepts: buildAskKinds(), PulledByKind: true, Serves: buildSeatVerbs("Node"),
Decision: "novox/hq ADR 0525, ADR 0536"},
{Name: "node-build-image", Scope: ScopeNode, Accepts: buildAskKinds(), PulledByKind: true, Serves: buildSeatVerbs("image"),
Decision: "novox/hq ADR 0525, ADR 0536"},
// **The mesh's resolvers** (novox/hq ADR 0194, 0196, 0223): every node's internal domain, and
// every node and container asks them and nothing else. Replicated since ADR 0223: held on more
// than one machine, each answering the same names from the same roster, and every machine's
@@ -498,6 +490,7 @@ func UseSeats(s []Seat) {
row.Receives = d.Receives
row.Paths = d.Paths
row.Replicated = d.Replicated
row.PulledByKind = d.PulledByKind
row.Serves = optionalAsCompiled(row.Serves, d.Serves)
}
merged = append(merged, row)
@@ -623,6 +616,10 @@ func claimProblems(m Manifest) []string {
// after UseSeats has replaced the set with the store's.
continue
}
if replaced, retired := RetiredSeats[c.Name]; retired {
problems = append(problems, fmt.Sprintf("%s claims the seat %q, which is retired: its work is %s", m.Module, c.Name, replaced))
continue
}
if isSystemSeatName(c.Name) {
problems = append(problems, fmt.Sprintf(
"%s claims %q, which is a seat in the mesh's own namespace (mesh-*/node-*) that it "+
@@ -903,22 +900,28 @@ func loginShellVerbs() []Verb {
}
}
// buildAgentVerbs are what a holder of node-build-agent answers on its own machine (novox/hq ADR
// 0219). One build at a time per holder (ADR 0190), so "the build running here" is one or none.
func buildAgentVerbs() []Verb {
// buildAskKinds are a build seat's accepted kinds, in the order a holder pulls them (ADR 0536 §4; mesh-sdk
// delivery.AskKinds).
func buildAskKinds() []string { return []string{"delivery", "check", "recheck"} }
// RetiredSeats are seats the mesh no longer defines, each with what replaced it: a claim or a use of one is refused
// naming its replacement, not as a seat nobody declares.
var RetiredSeats = map[string]string{
"node-build-agent": "node-build-go, node-build-node and node-build-image, held by the build modules (novox/hq ADR 0525 §10)",
"mesh-build-machine": "node-build-go, node-build-node and node-build-image, held by the build modules (novox/hq ADR 0190, ADR 0525 §10)",
}
// buildSeatVerbs are what a build module answers on its own node for its seat (ADR 0525 §10).
func buildSeatVerbs(language string) []Verb {
return []Verb{
{Name: "current", Description: "The build this machine is running — its id, repository, path, " +
"the step it is at, when it started and for how long — or none; and whether this machine is " +
"paused, taking no new build.",
{Name: "current", Description: "The " + language + " asks this node runs now (each ask's id, kind, module and when it " +
"started), its build capacity, and whether it is paused, taking no new ask.",
Input: schema(map[string]string{}, nil)},
{Name: "kill", Description: "End the build with this id, running here: its commands and the " +
"containers it started are stopped, and its outcome is announced as failed, killed by hand — " +
"settled, so it is not handed to another machine.",
Input: schema(map[string]string{"id": "the build's id, as `queue` or `current` says it"}, []string{"id"})},
{Name: "pause", Description: "Take no new build on this machine until resumed; a build running " +
"here finishes. Kept across a restart of the holder.",
{Name: "pause", Description: "Take no new " + language + " ask on this node until resumed; an ask running here " +
"ends in its place. Kept across a restart of the build module (its pause key). A capacity of 0 is the " +
"lasting way to keep a node from building.",
Input: schema(map[string]string{}, nil)},
{Name: "resume", Description: "Take builds again on this machine.",
{Name: "resume", Description: "Take " + language + " asks again on this node.",
Input: schema(map[string]string{}, nil)},
}
}
vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seats_declared.go
Vendored
+8
View File
@@ -364,6 +364,10 @@ func CatalogueProblems(shelf Shelf) []string {
// A `uses` naming nothing is where ADR 0110's guarantee lands under a derived set: the
// same refusal, at the same moment, from a set nobody maintains by hand.
for _, u := range m.Uses {
if replaced, retired := RetiredSeats[u]; retired && !exists(u) {
problems = append(problems, fmt.Sprintf("%s uses the seat %q, which is retired: its work is %s", module, u, replaced))
continue
}
if !exists(u) {
problems = append(problems, fmt.Sprintf(
"%s uses the seat %q, "+UndeclaredSeat,
@@ -372,6 +376,10 @@ func CatalogueProblems(shelf Shelf) []string {
}
for _, c := range m.Claims {
if replaced, retired := RetiredSeats[c.Name]; retired && !exists(c.Name) {
problems = append(problems, fmt.Sprintf("%s claims the seat %q, which is retired: its work is %s", module, c.Name, replaced))
continue
}
if !exists(c.Name) {
problems = append(problems, fmt.Sprintf(
"%s claims the seat %q, "+UndeclaredSeat,
vendor/git.novox.be/novox/mesh-sdk/go/catalogue/verbs.go
Vendored
+12 -33
View File
@@ -350,36 +350,24 @@ var ControllerVerbs = []Verb{
"its shell — `node show ace`, `module list`, `plans`, `conditions show <key>` — and answer what it " +
"printed. The generic verb beside the named ones (novox/hq ADR 0154), and since ADR 0266 it only reads: " +
"status, version, help, seats, healers, hand-acts, durations, collection, images, artifacts, data, builds, " +
"queue, plan, plans (not stop/close/go), doctor (not run), conditions list/show/history, node list/show, " +
"plan, plans (not stop/close/go), doctor (not run), conditions list/show/history, node list/show, " +
"module list, settings show/preferences, retire list, cleanup list, delivery plan/walks, bus, mirrors (not " +
"--record/--confirm). What writes has its named verb; what sets a key, issues a credential or a token, or " +
"accepts, recovers or exports a secret is the controller's terminal's alone.",
Input: schema(map[string]string{
"command": "the command line, as the controller's binary takes it; quotes group a word with spaces",
}, []string{"command"})},
// The build queue, controlled by hand (novox/hq ADR 0219). Every verb that drops an ask leaves a
// failed outcome for it, so a plan waiting on it fails visibly instead of hanging.
{Name: "queue", Description: "Every ask in the build queue: waiting, in flight (on which machine, for how long), " +
"and dead (handed out as often as allowed and never settled) — each with its id, repository, path, ref and when it was asked. " +
"Answers the table as text and, as data, each ask's state, kind (check or build), repository, commit, machine, since " +
"and position among the waiting (novox/hq issue 455).",
// The build queue went with build-agent (novox/hq ADR 0525 §9): asks are the build seats' streams, their backlog the
// streams' own state, shown in the delivery view. The verbs are refused by name for one release, naming that.
{Name: "queue", Description: "Removed in R4g (ADR 0525 §9): refused by name, naming its replacement: the build seats' backlog in the delivery view",
Input: schema(nil, nil)},
{Name: "cancel", Description: "Drop one waiting or dead ask from the build queue; its outcome is recorded failed, " +
"cancelled by hand, and a plan that asked for it fails. One in flight is refused: `kill` ends it where it runs.",
Input: schema(map[string]string{"id": "the ask's build id, as `queue` lists it"}, []string{"id"})},
{Name: "clear", Description: "Cancel every waiting ask in the build queue — and with dead, every dead one too — each " +
"recorded failed, cancelled by hand. Never touches one in flight.",
Input: schema(map[string]string{"dead": "\"true\" to cancel the dead asks as well"}, nil, "dead")},
{Name: "rebuild", Description: "Removed in R4e (ADR 0501 §5 / ADR 0503 §3): refused by name, naming its replacement",
Input: schema(nil, nil)},
{Name: "replay", Description: "Ask a recorded build's repository and path again at the commit it built, under a new id: " +
"a dry run only, nothing recorded or registered (novox/hq ADR 0429 §18: a fingerprint's key is built again by the " +
"builder, never registered by hand).",
Input: schema(map[string]string{"id": "the build's id"}, []string{"id"})},
{Name: "cancel", Description: "Removed in R4g (ADR 0525 §9): refused by name, naming its replacement", Input: schema(nil, nil)},
{Name: "clear", Description: "Removed in R4g (ADR 0525 §9): refused by name, naming its replacement", Input: schema(nil, nil)},
{Name: "replay", Description: "Removed in R4g (ADR 0525 §9): refused by name, naming its replacement", Input: schema(nil, nil)},
// novox/hq issue 686: a builder ran a branch's store tests for the first time in its pull request's merge check.
{Name: "try", Description: "Try a pushed branch or commit before its pull request (novox/hq issue 686): the " +
"checker (mesh-delivery-checker) merges the head onto the repository's trunk and runs its levels on the merge, " +
"as it judges a pull request (ADR 0503 §3); no merge-check.sh is run and build-agent is not asked. Answers at " +
"as it judges a pull request (ADR 0503 §3), its levels asks on the build seats (ADR 0525 §1); no merge-check.sh is run. Answers at " +
"once with an id; asked again with that id it waits up to 20 s and answers pass, fail or error, how long it " +
"took and the lines naming what failed — or that it is still waiting or running, to ask again. Sets no " +
"status on the forge, registers and builds nothing for delivery. One try of a commit at a time — a second " +
@@ -388,23 +376,14 @@ var ControllerVerbs = []Verb{
"repository": "the repository on the forge, owner/name; with ref",
"ref": "the branch or commit to try, as pushed to the forge",
"base": "the branch it is merged into (the repository's default branch when none is named)",
"tests": "only the tests this `go test -run` pattern names, where the repository's merge-check.sh reads " +
"MESH_CHECK_TESTS; a script that does not runs whole",
"tests": "only the tests this `go test -run` pattern names",
"store-only": "\"true\": only the integration level",
"id": "instead of a repository: the id a try answered, to read what came of it",
}, nil, "store-only"),
Replaces: []string{"sh merge-check.sh", "docker run postgres"}},
// novox/hq ADR 0350 §5: asked by the delivery's owner when the forge says a pull request closed unmerged.
{Name: "forget-checks", Description: "Removed in R4e (ADR 0501 §5 / ADR 0503 §3): refused by name, naming its replacement",
Input: schema(nil, nil)},
{Name: "kill", Description: "End a build where it runs: the machine that took it stops its commands and containers and " +
"announces it failed, killed by hand — settled, never handed to another machine.",
Input: schema(map[string]string{"id": "the build's id"}, []string{"id"})},
{Name: "pause", Description: "The build seat's holder on one machine — or every holder — takes no new build until resumed; " +
"a build running finishes. Kept across a restart of the holder. A plan waiting on a paused seat says so and is not late.",
Input: schema(map[string]string{"node": "one machine; every holder when absent"}, nil)},
{Name: "resume", Description: "The build seat's holder on one machine — or every holder — takes builds again.",
Input: schema(map[string]string{"node": "one machine; every holder when absent"}, nil)},
{Name: "kill", Description: "Removed in R4g (ADR 0525 §9): refused by name, naming its replacement: a build seat's ask ends where it runs", Input: schema(nil, nil)},
{Name: "pause", Description: "Removed in R4g (ADR 0525 §9): refused by name, naming its replacement: <node>/node-build-<language>.pause", Input: schema(nil, nil)},
{Name: "resume", Description: "Removed in R4g (ADR 0525 §9): refused by name, naming its replacement: <node>/node-build-<language>.resume", Input: schema(nil, nil)},
// Acts done by hand, and what the bounds are set from (novox/hq to-be 45 §7, Phase 0).
{Name: "hand-act", Description: "Record an act done by hand outside the mesh — a container restarted, a file " +
"edited, a service started on a machine — with why and its cause, in the hand-act log beside the pushes and " +
vendor/git.novox.be/novox/mesh-sdk/go/delivery/assigner.go
Vendored
+408
View File
@@ -0,0 +1,408 @@
package delivery
import (
"encoding/json"
"fmt"
"reflect"
"strings"
)
// The assigner's keys (novox/hq ADR 0528 clauses 1 and 4, ADR 0429 §6, issue 1297). The assigner writes, by the six
// rules of ADR 0429 §6, what each node should run: `want.<node>.<module>` in its bucket wants, and beside it
// `held.<module>.<fingerprint>` for a fingerprint it holds after a failed health check. A module that leaves a node's
// modules gets its want written removed, never deleted (ADR 0408 §4). For each node and module it compares the want with
// what the controller sends (the build the controller's assignment names) and writes every difference under
// `<node>.<module>` in its bucket differences, with the rule that explains it or unexplained. In release 5 wants are
// only compared; the node agent applying them is release 6 (ADR 0429 §7). Only the assigner's credential writes these
// buckets; the watchman, the view and the controller read them.
//
// **The bucket is the first word, the key the rest** (as the builder's `builds.<module>.<fp>` is the key
// `<module>.<fp>` in its builds bucket): the differences bucket's keys are `<node>.<module>`. The wants bucket holds
// two kinds of key, so each keeps its word: `want.<node>.<module>` and `held.<module>.<fingerprint hex>`.
const (
// AssignerModule is the module of the assigner part.
AssignerModule = "mesh-delivery-assigner"
// AssignerWantsBucket is the assigner's bucket of wants and holds, `<owner module>_<name>`.
AssignerWantsBucket = AssignerModule + "_wants"
// AssignerWantsHistory is the revisions the wants bucket keeps per key, so a want's history shows each move and
// the operator's manual write (ADR 0429 §6, the one escape).
AssignerWantsHistory = 20
// AssignerDifferencesBucket is the assigner's bucket of differences between a want and what the controller sends.
AssignerDifferencesBucket = AssignerModule + "_differences"
// AssignerDifferencesHistory is the revisions the differences bucket keeps per key: acceptance test 17 of design 47
// reads a day of them.
AssignerDifferencesHistory = 64
)
// The words that open the wants bucket's two kinds of key.
const (
wantWord = "want"
heldWord = "held"
)
// WantKey is the key, in AssignerWantsBucket, of a node's want for a module: `want.<node>.<module>`. A module named
// `node` is refused, since `want.<node>.node` is the node's own key (ADR 0429 §4), as HaveKey refuses it.
func WantKey(node, module string) (string, error) {
have, err := HaveKey(node, module)
if err != nil {
return "", err
}
return wantWord + "." + have, nil
}
// ParseWantKey is the inverse of WantKey; it refuses a held key, a have key and any key of another bucket, naming it.
func ParseWantKey(key string) (node, module string, err error) {
rest, ok := strings.CutPrefix(key, wantWord+".")
if !ok {
return "", "", fmt.Errorf("delivery: %q is not a want key, want.<node>.<module>", key)
}
node, module, isNode, err := ParseHaveKey(rest)
if err != nil || isNode {
return "", "", fmt.Errorf("delivery: %q is not a want key, want.<node>.<module>", key)
}
return node, module, nil
}
// HeldKey is the key, in AssignerWantsBucket, of a module's fingerprint the assigner holds: `held.<module>.<hex>`,
// the hex being the fingerprint's key part (FingerprintKeyPart).
func HeldKey(module, fingerprint string) (string, error) {
key, err := BuilderBuildKey(module, fingerprint)
if err != nil {
return "", err
}
return heldWord + "." + key, nil
}
// ParseHeldKey is the inverse of HeldKey: the module and the fingerprint (`src2:` and the hex); it refuses a want key
// and any key of another bucket, naming it.
func ParseHeldKey(key string) (module, fingerprint string, err error) {
rest, ok := strings.CutPrefix(key, heldWord+".")
if !ok {
return "", "", fmt.Errorf("delivery: %q is not a held key, held.<module>.<fingerprint hex>", key)
}
if module, fingerprint, err = ParseBuilderBuildKey(rest); err != nil {
return "", "", fmt.Errorf("delivery: %q is not a held key: %w", key, err)
}
return module, fingerprint, nil
}
// DifferenceKey is the key, in AssignerDifferencesBucket, of the difference for a module on a node: `<node>.<module>`.
func DifferenceKey(node, module string) (string, error) {
return HaveKey(node, module)
}
// ParseDifferenceKey is the inverse of DifferenceKey; it refuses a want or held key and a node's own key, naming it.
func ParseDifferenceKey(key string) (node, module string, err error) {
node, module, isNode, err := ParseHaveKey(key)
if err != nil || isNode {
return "", "", fmt.Errorf("delivery: %q is not a difference key, <node>.<module>", key)
}
return node, module, nil
}
// The six rules of ADR 0429 §6, by number, as a want or a difference names the one that holds a node back.
const (
RuleBuildDone = 1 // the build of F is done
RuleNotHeld = 2 // F is not held
RuleCanary = 3 // the node is the module's canary, or the canary runs F healthy
RuleStep = 4 // at most step other nodes are ahead (step 0: no limit, ADR 0528 §3)
RuleProviders = 5 // every provider the module requires runs healthy on that node, or at its canary
RuleGrant = 6 // the module's credential for F exists (grants.<module>.<fp>)
)
func checkRule(rule int, of string) error {
if rule < RuleBuildDone || rule > RuleGrant {
return fmt.Errorf("delivery: %s names the rule %d, which is not one of the six (1 to 6)", of, rule)
}
return nil
}
func checkFingerprint(fp, of string) error {
if _, err := FingerprintKeyPart(fp); err != nil {
return fmt.Errorf("delivery: %s: %w", of, err)
}
return nil
}
// Want is a node's want for a module: the fingerprint FP the node should run, or Removed for a module that left the
// node's modules (then FP is empty). When the module's newest fingerprint is not yet allowed on this node, Newest names
// it and Rule the rule that holds the node back; both are empty when FP is the newest. Words say why in a sentence.
// The entry's own time says since when.
type Want struct {
V int `json:"v"`
FP string `json:"fp,omitempty"`
Removed bool `json:"removed,omitempty"`
Newest string `json:"newest,omitempty"`
Rule int `json:"rule,omitempty"`
Words string `json:"words,omitempty"`
}
func (w Want) check() error {
if w.Removed {
if w.FP != "" || w.Newest != "" || w.Rule != 0 {
return fmt.Errorf("delivery: a removed want names no fingerprint and no rule")
}
return nil
}
if err := checkFingerprint(w.FP, "a want's fingerprint"); err != nil {
return err
}
if w.Newest == "" && w.Rule == 0 {
return nil
}
if err := checkFingerprint(w.Newest, "a want's newest fingerprint"); err != nil {
return err
}
if w.Newest == w.FP {
return fmt.Errorf("delivery: a want at its newest fingerprint names no rule holding it back")
}
return checkRule(w.Rule, "a want held back")
}
// Held is a fingerprint the assigner holds: the node whose health check failed on it, and the check's words.
type Held struct {
V int `json:"v"`
Node string `json:"node"`
Words string `json:"words,omitempty"`
}
func (h Held) check() error {
return checkName("node", h.Node)
}
// Difference is where a want and what the controller sends differ for a module on a node (ADR 0528 clause 4): the
// want's fingerprint (Want, empty when the node has no want for the module, WantRemoved when the want is removed), the
// fingerprint of the build the controller's assignment names (Sent, empty when it sends none or the build names no
// fingerprint; SentBuild names that build), and the rule that explains the difference, or Unexplained.
type Difference struct {
V int `json:"v"`
Want string `json:"want,omitempty"`
WantRemoved bool `json:"want-removed,omitempty"`
Sent string `json:"sent,omitempty"`
SentBuild string `json:"sent-build,omitempty"`
Rule int `json:"rule,omitempty"`
Unexplained bool `json:"unexplained,omitempty"`
Words string `json:"words,omitempty"`
}
func (d Difference) check() error {
if d.WantRemoved && d.Want != "" {
return fmt.Errorf("delivery: a difference with a removed want names no wanted fingerprint")
}
if d.Want != "" {
if err := checkFingerprint(d.Want, "a difference's want"); err != nil {
return err
}
}
if d.Sent != "" {
if err := checkFingerprint(d.Sent, "a difference's sent fingerprint"); err != nil {
return err
}
}
if !d.WantRemoved && d.Want == d.Sent && (d.Want != "" || d.SentBuild == "") {
return fmt.Errorf("delivery: a difference whose want and send are both %q is no difference", d.Want)
}
if d.Unexplained {
if d.Rule != 0 {
return fmt.Errorf("delivery: an unexplained difference names no rule")
}
return nil
}
return checkRule(d.Rule, "a difference")
}
// EncodeWant is w as JSON at Version; it refuses a fingerprint not of the new form, a removed want naming one, and a
// want held back without its rule.
func EncodeWant(w Want) ([]byte, error) {
if err := w.check(); err != nil {
return nil, err
}
w.V = Version
return json.Marshal(w)
}
// DecodeWant reads a want; it refuses an unknown version (ErrUnknownVersion) and what EncodeWant refuses.
func DecodeWant(data []byte) (Want, error) {
var w Want
if err := decode(data, &w); err != nil {
return Want{}, err
}
if err := w.check(); err != nil {
return Want{}, err
}
return w, nil
}
// SameWant says whether two wants are the same value, so the assigner writes only when they differ.
func SameWant(a, b Want) bool { return a == b }
// EncodeHeld is h as JSON at Version; it refuses a node name a key refuses.
func EncodeHeld(h Held) ([]byte, error) {
if err := h.check(); err != nil {
return nil, err
}
h.V = Version
return json.Marshal(h)
}
// DecodeHeld reads a hold; it refuses an unknown version (ErrUnknownVersion) and what EncodeHeld refuses.
func DecodeHeld(data []byte) (Held, error) {
var h Held
if err := decode(data, &h); err != nil {
return Held{}, err
}
if err := h.check(); err != nil {
return Held{}, err
}
return h, nil
}
// SameHeld says whether two holds are the same value.
func SameHeld(a, b Held) bool { return a == b }
// EncodeDifference is d as JSON at Version; it refuses a difference with neither a rule nor unexplained, one with both,
// a rule outside the six, and a want equal to the send.
func EncodeDifference(d Difference) ([]byte, error) {
if err := d.check(); err != nil {
return nil, err
}
d.V = Version
return json.Marshal(d)
}
// DecodeDifference reads a difference; it refuses an unknown version (ErrUnknownVersion) and what EncodeDifference
// refuses.
func DecodeDifference(data []byte) (Difference, error) {
var d Difference
if err := decode(data, &d); err != nil {
return Difference{}, err
}
if err := d.check(); err != nil {
return Difference{}, err
}
return d, nil
}
// SameDifference says whether two differences are the same value.
func SameDifference(a, b Difference) bool { return reflect.DeepEqual(a, b) }
// The assigner's composition check (novox/hq ADR 0446 clauses 1 to 5, ADR 0528 clause 5, ADR 0482 clauses 3 and 9): for
// each group the checker judges, the assigner composes the machines the change touches and writes its verdict under
// the checker's key of that check, `<escaped group>.<heads hash>` (CheckerVerdictKey's form), in its bucket
// compositions. The checker watches that key and reads it into its verdict's composition (Composition); a key not yet
// written is a composition still running.
const (
// AssignerCompositionsBucket is the assigner's bucket of composition verdicts.
AssignerCompositionsBucket = AssignerModule + "_compositions"
// AssignerCompositionsHistory is the revisions the bucket keeps per key.
AssignerCompositionsHistory = 5
// CompositionWaits is the state of a composition that waits for what it composes from (ADR 0446 clause 3: the
// indexer holds no candidates for the group yet), its Words saying what; it is no verdict.
CompositionWaits = "waits"
// CompositionRebuildWarning is the most modules a merge may rebuild before the composition warns (ADR 0446 clause 1).
CompositionRebuildWarning = 12
)
// CompositionKey is the key, in AssignerCompositionsBucket, of the composition of a group's check at its heads: the
// checker's verdict key of the same check (CheckerVerdictKey).
func CompositionKey(group, headsHash string) (string, error) {
return CheckerVerdictKey(group, headsHash)
}
// ParseCompositionKey is the inverse of CompositionKey; it refuses a want, held or difference key, naming it.
func ParseCompositionKey(key string) (group, headsHash string, err error) {
if group, headsHash, err = ParseCheckerVerdictKey(key); err != nil {
return "", "", fmt.Errorf("delivery: %q is not a composition key, <group>.<heads hash>: %w", key, err)
}
return group, headsHash, nil
}
// CompositionVerdict is the value of a composition key: the group and its heads hash, the State (pass, fail or waits),
// the Words of a refusal or of what it waits for, the mesh-sdk commit it composed with (the assigner's own, or the
// group's mesh-sdk member's head, ADR 0482 clause 3), the machines composed, the first machine that failed, the
// warnings that fail nothing (more than twelve modules rebuilt; what was already broken before the change, ADR 0446
// clause 1), and the time it took in integer milliseconds.
type CompositionVerdict struct {
V int `json:"v"`
Group string `json:"group"`
Heads string `json:"heads"`
State string `json:"state"`
Words string `json:"words,omitempty"`
SDK string `json:"sdk,omitempty"`
Machines []string `json:"machines,omitempty"`
Machine string `json:"machine,omitempty"`
Warnings []string `json:"warnings,omitempty"`
Took int64 `json:"took,omitempty"`
}
func (c CompositionVerdict) check() error {
if c.Group == "" {
return fmt.Errorf("delivery: a composition names no group")
}
if !isHex12(c.Heads) {
return fmt.Errorf("delivery: the composition's heads hash %q is not 12 lowercase hex digits", c.Heads)
}
if c.Took < 0 {
return fmt.Errorf("delivery: the composition's time is negative")
}
switch c.State {
case CompositionPass:
if c.SDK == "" {
return fmt.Errorf("delivery: a composition that passed names no mesh-sdk commit")
}
if c.Machine != "" || c.Words != "" {
return fmt.Errorf("delivery: a composition that passed names no failure")
}
case CompositionFail:
if c.Words == "" {
return fmt.Errorf("delivery: a composition that failed gives no words")
}
if c.SDK == "" {
return fmt.Errorf("delivery: a composition that failed names no mesh-sdk commit")
}
case CompositionWaits:
if c.Words == "" {
return fmt.Errorf("delivery: a composition that waits does not say for what")
}
if c.Machine != "" || len(c.Machines) > 0 {
return fmt.Errorf("delivery: a composition that waits composed no machine")
}
default:
return fmt.Errorf("delivery: the composition state %q is not %s, %s or %s", c.State, CompositionPass, CompositionFail, CompositionWaits)
}
return nil
}
// Composition is what the checker writes into its verdict from c: nil while c waits (the check is still running).
func (c CompositionVerdict) Composition() *Composition {
if c.State == CompositionWaits {
return nil
}
return &Composition{State: c.State, Words: c.Words, SDK: c.SDK, Machines: c.Machines}
}
// EncodeCompositionVerdict is c as JSON at Version; it refuses an unknown state, a pass or fail naming no mesh-sdk
// commit, a fail without words, a pass naming a failure, and a wait that composed a machine.
func EncodeCompositionVerdict(c CompositionVerdict) ([]byte, error) {
if err := c.check(); err != nil {
return nil, err
}
c.V = Version
return json.Marshal(c)
}
// DecodeCompositionVerdict reads a composition; it refuses an unknown version (ErrUnknownVersion) and what
// EncodeCompositionVerdict refuses.
func DecodeCompositionVerdict(data []byte) (CompositionVerdict, error) {
var c CompositionVerdict
if err := decode(data, &c); err != nil {
return CompositionVerdict{}, err
}
if err := c.check(); err != nil {
return CompositionVerdict{}, err
}
return c, nil
}
// SameCompositionVerdict says whether two compositions are the same value.
func SameCompositionVerdict(a, b CompositionVerdict) bool { return reflect.DeepEqual(a, b) }
vendor/git.novox.be/novox/mesh-sdk/go/delivery/builder.go
Vendored
+15
View File
@@ -184,6 +184,10 @@ type BuilderBuild struct {
// does not yet hold: the key is not claimed until that package's trunk key is done and published (novox/hq ADR 0500
// clause 6). A reader that does not know the field reads the key as queued.
Waits string `json:"waits,omitempty"`
// Asks are, on a building key, the asks the builder put for it by build seat language (AskGo, AskNode, AskImage), each
// its id (novox/hq ADR 0525 §1; issue 1887 A4b): a builder that takes the key over at its start waits for those asks'
// steps and never puts them again. A reader that does not know the field reads the key as before.
Asks map[string]string `json:"asks,omitempty"`
}
func (b BuilderBuild) check() error {
@@ -216,6 +220,17 @@ func (b BuilderBuild) check() error {
return fmt.Errorf("delivery: the reference %q of the %s %q %s", a.Reference, a.Kind, a.Name, p)
}
}
for language, id := range b.Asks {
if b.State != BuildBuilding {
return fmt.Errorf("delivery: a %s build names the ask %s; only a building key does", b.State, id)
}
if err := checkAskLanguage(language); err != nil {
return err
}
if err := checkAskID(id); err != nil {
return err
}
}
if b.Waits != "" {
if b.State != BuildQueued {
return fmt.Errorf("delivery: a %s build waits for %q; only a queued key waits for a package", b.State, b.Waits)
vendor/git.novox.be/novox/mesh-sdk/go/delivery/buildrun/asks.go
Vendored
+220
View File
@@ -0,0 +1,220 @@
package buildrun
import (
"context"
"errors"
"fmt"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// **An asker puts an ask on its build seat's stream and waits for its step** (novox/hq ADR 0525 §1, §6; issue 1887 A4,
// A5). It chooses no node: whichever build node pulls the ask writes `<node>.<ask id>` in its build module's steps state,
// and the asker reads the first such key that is not running. The builder and the checker share it.
// AskPublisher puts an encoded ask on subject, de-duplicated by its id.
type AskPublisher interface {
PublishAsk(ctx context.Context, subject, id string, data []byte) error
}
// AskWaiter waits for an ask's step: the first key `*.<ask id>` of the build module's steps state whose state is not
// running. A watch, not a poll; it returns when the step ends or ctx does.
type AskWaiter interface {
WaitAsk(ctx context.Context, module, askID string) (delivery.Step, error)
}
// Asker puts asks and waits for their steps.
type Asker struct {
Publisher AskPublisher
Waiter AskWaiter
// Bound is how long the asker waits for an ask's step before it stops waiting; zero waits as long as ctx.
Bound time.Duration
}
// Put puts a on its stream: the seat's accept subject of its language and kind.
func (k Asker) Put(ctx context.Context, a delivery.StepAsk) error {
raw, err := delivery.EncodeStepAsk(a)
if err != nil {
return err
}
subject, err := delivery.AskSubject(a.Language, a.Kind)
if err != nil {
return err
}
if err := k.Publisher.PublishAsk(ctx, subject, a.ID, raw); err != nil {
return fmt.Errorf("putting the %s ask %s: %w", a.Kind, a.ID, err)
}
return nil
}
// Wait waits for the step of an ask already put (a builder that started again takes over its own asks with it, never
// putting them twice) and answers it when it ended, whatever its state.
func (k Asker) Wait(ctx context.Context, a delivery.StepAsk) (delivery.Step, error) {
module, err := delivery.BuildModuleOf(a.Language)
if err != nil {
return delivery.Step{}, err
}
wait, cancel := ctx, context.CancelFunc(func() {})
if k.Bound > 0 {
wait, cancel = context.WithTimeoutCause(ctx, k.Bound, fmt.Errorf("the ask %s's step did not end within %s", a.ID, k.Bound))
}
defer cancel()
step, err := k.Waiter.WaitAsk(wait, module, a.ID)
if err != nil {
if cause := context.Cause(wait); cause != nil && ctx.Err() == nil {
err = cause
}
return delivery.Step{}, err
}
return step, nil
}
// Ask puts a and waits for its step; a step that did not pass is an error naming its node, its state and its reason.
func (k Asker) Ask(ctx context.Context, a delivery.StepAsk) (delivery.Step, error) {
if err := k.Put(ctx, a); err != nil {
return delivery.Step{}, err
}
return k.Passed(k.Wait(ctx, a))
}
// Passed is step when it passed, else an error naming where it ended and why.
func (k Asker) Passed(step delivery.Step, err error) (delivery.Step, error) {
if err != nil {
return step, err
}
if step.State != delivery.StepPassed {
return step, fmt.Errorf("the ask %s on %s %s: %s", step.ID, step.Node, step.State, step.Reason)
}
return step, nil
}
// JetStreamAsks is the bus side of an Asker: a JetStream publish de-duplicated by the ask's id (Nats-Msg-Id), and a watch
// of `*.<ask id>` in the build module's steps bucket.
type JetStreamAsks struct {
JS jetstream.JetStream
}
// PublishAsk is AskPublisher's.
func (s JetStreamAsks) PublishAsk(ctx context.Context, subject, id string, data []byte) error {
msg := &nats.Msg{Subject: subject, Data: data, Header: nats.Header{}}
msg.Header.Set(jetstream.MsgIDHeader, id)
_, err := s.JS.PublishMsg(ctx, msg)
return err
}
// WaitAsk is AskWaiter's: the watch hands over what the keys hold now, so a step that ended before it began is read at
// once, and every change after.
func (s JetStreamAsks) WaitAsk(ctx context.Context, module, askID string) (delivery.Step, error) {
pattern, err := delivery.AskStepWatch(askID)
if err != nil {
return delivery.Step{}, err
}
kv, err := s.JS.KeyValue(ctx, delivery.StepsBucket(module))
if err != nil {
return delivery.Step{}, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
}
w, err := kv.Watch(ctx, pattern)
if err != nil {
return delivery.Step{}, err
}
defer w.Stop()
for {
select {
case <-ctx.Done():
return delivery.Step{}, ctx.Err()
case e, ok := <-w.Updates():
if !ok {
return delivery.Step{}, errors.New("the watch of the ask's step ended")
}
if e == nil || e.Operation() != jetstream.KeyValuePut {
continue
}
step, err := delivery.DecodeStep(e.Value())
if err != nil {
continue // not a step (a node's pause key never matches an ask id; anything else is not this ask's)
}
if step.State != delivery.StepRunning {
return step, nil
}
}
}
}
// FakeAsks is a test double of the bus for an asker's tests: each ask put is kept and, when Answer is set, its step is
// ended at once with what Answer says a build node wrote. Safe for concurrent use.
type FakeAsks struct {
Answer func(a delivery.StepAsk) delivery.Step
mu sync.Mutex
asks []delivery.StepAsk
steps map[string]delivery.Step
waits map[string]chan struct{}
}
// PublishAsk is AskPublisher's.
func (f *FakeAsks) PublishAsk(_ context.Context, _, id string, data []byte) error {
a, err := delivery.DecodeStepAsk(data)
if err != nil {
return err
}
f.mu.Lock()
f.asks = append(f.asks, a)
f.mu.Unlock()
if f.Answer != nil {
f.End(f.Answer(a))
}
return nil
}
// End writes an ask's step end, as a build node does.
func (f *FakeAsks) End(step delivery.Step) {
f.mu.Lock()
if f.steps == nil {
f.steps, f.waits = map[string]delivery.Step{}, map[string]chan struct{}{}
}
f.steps[step.ID] = step
ch := f.waits[step.ID]
delete(f.waits, step.ID)
f.mu.Unlock()
if ch != nil {
close(ch)
}
}
// WaitAsk is AskWaiter's.
func (f *FakeAsks) WaitAsk(ctx context.Context, _ string, askID string) (delivery.Step, error) {
f.mu.Lock()
if f.steps == nil {
f.steps, f.waits = map[string]delivery.Step{}, map[string]chan struct{}{}
}
if s, ok := f.steps[askID]; ok {
f.mu.Unlock()
return s, nil
}
ch := f.waits[askID]
if ch == nil {
ch = make(chan struct{})
f.waits[askID] = ch
}
f.mu.Unlock()
select {
case <-ctx.Done():
return delivery.Step{}, ctx.Err()
case <-ch:
}
f.mu.Lock()
defer f.mu.Unlock()
return f.steps[askID], nil
}
// Asks are the asks put so far.
func (f *FakeAsks) Asks() []delivery.StepAsk {
f.mu.Lock()
defer f.mu.Unlock()
return append([]delivery.StepAsk(nil), f.asks...)
}
vendor/git.novox.be/novox/mesh-sdk/go/delivery/buildrun/choose.go
Vendored
+88
View File
@@ -0,0 +1,88 @@
package buildrun
import (
"fmt"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// ChooseNodes picks up to want build nodes for a language (ADR 0478 clause 3): the nodes whose toolchains key reports the
// version the nodes agree on, the ones with room under their build capacity first (places), then the fewest lanes the
// part has running or chosen there, a tie broken by the node's name. A place is reserved on each node chosen, in the
// same step as the choice, so builds choosing at once do not choose the same full node while another has room; the
// caller takes the place (Places.Acquire) or gives it up (Places.Cancel). A node at its capacity is still chosen when
// the others are too (fewer than want have room): its lane waits for a place. A node qualifies only when qualifies is true
// of it (online and its build module healthy; nil means every node does). It answers the toolchain all the reporting nodes
// agree on, qualifying or not, as the indexer computes it. When nothing is reported, or the nodes report different
// versions (so the indexer agreed none), it fails naming the language and why; when no reporting node qualifies it fails
// naming the nodes. A node whose build module refuses on its toolchain refuses the step, and the build fails naming it.
func ChooseNodes(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, delivery.Toolchain2, error) {
tc, none := delivery.AgreedToolchain(entries)
if none != "" {
return nil, delivery.Toolchain2{}, fmt.Errorf("no %s build node qualifies: %s", language, none)
}
nodes, err := choose(language, entries, places, qualifies, want)
return nodes, tc, err
}
// ChooseAnyNodes is ChooseNodes for artifacts that need no toolchain, such as an archive (ADR 0478 clause 1): the
// build module's nodes agree on nothing, so any node of the build module that qualifies is chosen, by capacity and load only.
func ChooseAnyNodes(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, error) {
return choose(language, entries, places, qualifies, want)
}
func choose(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, error) {
if places == nil {
places = &Places{}
}
nodes := make([]string, 0, len(entries))
var reporting []string
seen := map[string]bool{}
for _, e := range entries {
if seen[e.Node] {
continue
}
seen[e.Node] = true
reporting = append(reporting, e.Node)
if places.CapacityOf(e.Node) > 0 && (qualifies == nil || qualifies(e.Node)) { // a capacity of 0 keeps the part off a node
nodes = append(nodes, e.Node)
}
}
if len(nodes) == 0 {
sort.Strings(reporting)
return nil, fmt.Errorf("no %s build node qualifies: %s report the toolchain, and none is online with a healthy build module and a build capacity", language, strings.Join(reporting, ", "))
}
return places.choose(nodes, want), nil
}
// DigestsDifferPrefix opens the error of a build whose nodes built an artifact to different digests (ADR 0478 clause 4):
// such a key is not built again as a new revision, since the same source on the same toolchains gave two answers and
// building again settles nothing; it waits for another fingerprint.
const DigestsDifferPrefix = "digests differ:"
// Agree is the one digest every node built an artifact to (ADR 0478 clause 4), or an error naming each node and its
// digest when they differ. One node's digest agrees with itself.
func Agree(artifact string, byNode map[string]string) (string, error) {
nodes := make([]string, 0, len(byNode))
for n := range byNode {
nodes = append(nodes, n)
}
sort.Strings(nodes)
var first string
for i, n := range nodes {
if i == 0 {
first = byNode[n]
continue
}
if byNode[n] != first {
msg := fmt.Sprintf("%s the nodes built %s to different digests:", DigestsDifferPrefix, artifact)
for _, n := range nodes {
msg += fmt.Sprintf(" %s %s;", n, byNode[n])
}
return "", fmt.Errorf("%s", msg[:len(msg)-1])
}
}
return first, nil
}
vendor/git.novox.be/novox/mesh-sdk/go/delivery/buildrun/doc.go
Vendored
+5
View File
@@ -0,0 +1,5 @@
// Package buildrun holds the parts every module that runs steps on the build modules shares: the caller, the steps
// watch and the runner of a long step (ADR 0469), the sets of presence, toolchains and healthy build modules, the
// places a node offers by capacity, and the choice of build nodes (ADR 0478). The builder and the checker of the
// delivery use them; they moved here from the builder's own package so neither keeps a copy (issue 1671, ADR 0484).
package buildrun
vendor/git.novox.be/novox/mesh-sdk/go/delivery/buildrun/have.go
Vendored
+56
View File
@@ -0,0 +1,56 @@
package buildrun
import (
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// HaveSet holds what each node's node-engine says of each module on it (node-engine_have, key `<node>.<module>`): the
// builder reads it to know whether a node's build module is running and healthy, since a build module whose installed
// toolchain is not the one its recipe builds with is unhealthy (mesh-build-go build_go_health) and its steps would
// refuse (ADR 0478 clause 3). A module the node-engine has written no key of is not healthy. Safe for use from the watch
// and from a build at once.
type HaveSet struct {
mu sync.Mutex
healthy map[string]bool // by `<node>.<module>`
}
// Put takes a change of the have bucket in and answers whether a module's healthy state changed. A node's own key
// (`<node>.node`) and a value that is not a have are left out.
func (s *HaveSet) Put(key string, value []byte, deleted bool) bool {
node, module, isNode, err := delivery.ParseHaveKey(key)
if err != nil || isNode {
return false
}
ok := false
if !deleted {
h, err := delivery.DecodeHave(value)
if err != nil {
return false
}
ok = h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK
}
k := node + "." + module
s.mu.Lock()
defer s.mu.Unlock()
if s.healthy[k] == ok {
return false
}
if s.healthy == nil {
s.healthy = map[string]bool{}
}
if ok {
s.healthy[k] = true
} else {
delete(s.healthy, k)
}
return true
}
// Healthy is whether the module on the node was last seen running and healthy.
func (s *HaveSet) Healthy(node, module string) bool {
s.mu.Lock()
defer s.mu.Unlock()
return s.healthy[node+"."+module]
}
vendor/git.novox.be/novox/mesh-sdk/go/delivery/buildrun/modules.go
Vendored
+7
View File
@@ -0,0 +1,7 @@
package buildrun
// The build modules the mesh runs, by the language each builds (ADR 0469).
const (
GoBuildModule = "mesh-build-go"
NodeBuildModule = "mesh-build-node"
)
vendor/git.novox.be/novox/mesh-sdk/go/delivery/buildrun/places.go
Vendored
+171
View File
@@ -0,0 +1,171 @@
package buildrun
import (
"context"
"fmt"
"sort"
"strconv"
"strings"
"sync"
)
// DefaultCapacity is the build capacity of a node that has no setting: one build step at a time.
const DefaultCapacity = 1
// ParseCapacity reads a capacity setting: `node=steps` pairs separated by commas, such as
// `shanks=3,ace=2`. A node without a pair has DefaultCapacity. A pair with no node, a count that is not a whole number of
// at least 0, and a node named twice are refused. A count of 0 keeps the part off the node: it is never chosen.
func ParseCapacity(s string) (map[string]int, error) {
out := map[string]int{}
for _, pair := range strings.Split(s, ",") {
pair = strings.TrimSpace(pair)
if pair == "" {
continue
}
node, count, ok := strings.Cut(pair, "=")
node, count = strings.TrimSpace(node), strings.TrimSpace(count)
n, err := strconv.Atoi(count)
if !ok || node == "" || err != nil || n < 0 {
return nil, fmt.Errorf("the capacity %q is not <node>=<steps at once, 0 or more>", pair)
}
if _, dup := out[node]; dup {
return nil, fmt.Errorf("the capacity names %s twice", node)
}
out[node] = n
}
return out, nil
}
// Places counts the build steps a part runs on each node and holds a lane back while its node is at its
// capacity (the operator's decision of 2026-10-21; ADR 0478 clause 3). A lane is the fetch and the builds of one build
// module on one node, run one step at a time, so a lane holds one place while it runs. A node is chosen for a lane by
// reserving a place on it at the moment of the choice (ChooseNodes), so that builds choosing at once see each other's
// choices and none waits on a full node while another has room. A lane waits for its place by watching the ends of the
// other lanes, not by a timer: ending a lane wakes the waiting ones.
type Places struct {
// Capacity is the number of steps at once by node; a node not in it has DefaultCapacity.
Capacity map[string]int
mu sync.Mutex
running map[string]int // lanes holding a place
reserved map[string]int // lanes chosen for the node that do not hold a place yet
changed chan struct{} // closed, and replaced, each time a lane ends
}
// CapacityOf is how many steps the part runs on the node at once: DefaultCapacity for a node the setting does not
// name, and 0 for one it names so, where the part never builds.
func (p *Places) CapacityOf(node string) int {
if n, ok := p.Capacity[node]; ok && n >= 0 {
return n
}
return DefaultCapacity
}
// Running is the steps in flight now, by node.
func (p *Places) Running() map[string]int {
p.mu.Lock()
defer p.mu.Unlock()
out := make(map[string]int, len(p.running))
for n, c := range p.running {
if c > 0 {
out[n] = c
}
}
return out
}
// Reserved is the lanes chosen for a node that hold no place yet, by node.
func (p *Places) Reserved() map[string]int {
p.mu.Lock()
defer p.mu.Unlock()
out := make(map[string]int, len(p.reserved))
for n, c := range p.reserved {
if c > 0 {
out[n] = c
}
}
return out
}
func (p *Places) init() {
if p.running == nil {
p.running = map[string]int{}
}
if p.reserved == nil {
p.reserved = map[string]int{}
}
if p.changed == nil {
p.changed = make(chan struct{})
}
}
// Acquire takes a place on the node, waiting while the node is at its capacity, and answers the function that gives it
// back. A place reserved for the node by a choice is the one taken. It returns an error when ctx ends first, and then
// gives up the reservation too.
func (p *Places) Acquire(ctx context.Context, node string) (release func(), err error) {
for {
p.mu.Lock()
p.init()
if p.running[node] < p.CapacityOf(node) {
p.running[node]++
if p.reserved[node] > 0 {
p.reserved[node]--
}
p.mu.Unlock()
var once sync.Once
return func() {
once.Do(func() {
p.mu.Lock()
p.running[node]--
close(p.changed)
p.changed = make(chan struct{})
p.mu.Unlock()
})
}, nil
}
wait := p.changed
p.mu.Unlock()
select {
case <-wait:
case <-ctx.Done():
p.Cancel(node)
return nil, fmt.Errorf("waiting for a place on %s: %w", node, ctx.Err())
}
}
}
// Cancel gives up a place reserved for the node that will not be taken.
func (p *Places) Cancel(node string) {
p.mu.Lock()
defer p.mu.Unlock()
p.init()
if p.reserved[node] > 0 {
p.reserved[node]--
}
}
// choose orders the nodes (those with room first, counting the lanes running and the lanes reserved, then the fewest of
// them, then by name), keeps the first want and reserves a place on each, all at once.
func (p *Places) choose(nodes []string, want int) []string {
p.mu.Lock()
defer p.mu.Unlock()
p.init()
load := func(n string) int { return p.running[n] + p.reserved[n] }
sort.SliceStable(nodes, func(i, j int) bool {
ri, rj := load(nodes[i]) < p.CapacityOf(nodes[i]), load(nodes[j]) < p.CapacityOf(nodes[j])
if ri != rj {
return ri
}
if load(nodes[i]) != load(nodes[j]) {
return load(nodes[i]) < load(nodes[j])
}
return nodes[i] < nodes[j]
})
if len(nodes) > want {
nodes = nodes[:want]
}
for _, n := range nodes {
p.reserved[n]++
}
return nodes
}
vendor/git.novox.be/novox/mesh-sdk/go/delivery/buildrun/presence.go
Vendored
+53
View File
@@ -0,0 +1,53 @@
package buildrun
import (
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// PresenceSet holds which nodes are online, as the watchman writes it (mesh-delivery-watchman_presence, key `<node>`, the
// node-engine's connection): a node asleep or off the bus is offline and its build module cannot answer. A node the
// watchman has no key of is not online. Safe for use from the watch and from a build at once.
type PresenceSet struct {
mu sync.Mutex
online map[string]bool
}
// Put takes a change of the presence bucket in and answers whether a node's online state changed. A part's key and a
// value that is not a presence are left out.
func (s *PresenceSet) Put(key string, value []byte, deleted bool) bool {
node, part, _, err := delivery.ParsePresenceKey(key)
if err != nil || part != "" {
return false
}
on := false
if !deleted {
p, err := delivery.DecodePresence(value)
if err != nil {
return false
}
on = p.Online
}
s.mu.Lock()
defer s.mu.Unlock()
if s.online[node] == on {
return false
}
if s.online == nil {
s.online = map[string]bool{}
}
if on {
s.online[node] = true
} else {
delete(s.online, node)
}
return true
}
// Online is whether the watchman last saw the node online.
func (s *PresenceSet) Online(node string) bool {
s.mu.Lock()
defer s.mu.Unlock()
return s.online[node]
}
vendor/git.novox.be/novox/mesh-sdk/go/delivery/buildrun/steps.go
Vendored
+67
View File
@@ -0,0 +1,67 @@
package buildrun
import (
"context"
"encoding/json"
"fmt"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Caller calls a tool of a module on one named machine (mesh-sdk process.Conn.CallOn; ADR 0469 clause 3).
type Caller interface {
CallOn(node, key string, body any) (json.RawMessage, error)
}
// Steps waits for the end of a build module's long step: the value of its key `<node>.<id>` in the module's steps state
// (ADR 0469 clause 2) once its state is not running. It is a watch of that key, not a poll; it returns when the step
// ends or ctx does.
type Steps interface {
Wait(ctx context.Context, module, key string) (delivery.Step, error)
}
// Runner runs a build module's long steps: the call answers at once with the step's key, and the result is read from that
// key when the step ends (ADR 0469 clause 2).
type Runner struct {
Caller Caller
Steps Steps
// Bound is how long a step may run before the builder stops waiting for it.
Bound time.Duration
}
// Run calls tool of the build module on node with args, waits for the step's end and answers its result. A step that
// failed or was refused is an error naming the node, the state and the reason.
func (r Runner) Run(ctx context.Context, module, node, tool string, args map[string]any) (json.RawMessage, error) {
raw, err := r.Caller.CallOn(node, module+"."+tool, args)
if err != nil {
return nil, fmt.Errorf("%s on %s: %w", tool, node, err)
}
var answer struct {
Step string `json:"step"`
Key string `json:"key"`
State string `json:"state"`
}
if err := json.Unmarshal(raw, &answer); err != nil {
return nil, fmt.Errorf("%s on %s answered what is not a step: %w", tool, node, err)
}
if answer.Key == "" {
return nil, fmt.Errorf("%s on %s answered no step key: %s", tool, node, raw)
}
if n, _, err := delivery.ParseStepKey(answer.Key); err != nil || n != node {
return nil, fmt.Errorf("%s on %s answered the step key %q, which is not a key of that node", tool, node, answer.Key)
}
wait, cancel := context.WithTimeoutCause(ctx, r.Bound, fmt.Errorf("the step %s did not end within %s", answer.Key, r.Bound))
defer cancel()
step, err := r.Steps.Wait(wait, module, answer.Key)
if err != nil {
if cause := context.Cause(wait); cause != nil && ctx.Err() == nil {
err = cause
}
return nil, fmt.Errorf("%s on %s: %w", tool, node, err)
}
if step.State != delivery.StepPassed {
return nil, fmt.Errorf("%s on %s: the step %s %s: %s", tool, node, answer.Key, step.State, step.Reason)
}
return step.Result, nil
}
vendor/git.novox.be/novox/mesh-sdk/go/delivery/buildrun/toolchains.go
Vendored
+64
View File
@@ -0,0 +1,64 @@
package buildrun
import (
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// LanguageOf is the language each build module's toolchains state holds (the indexer's ToolchainModules, reversed).
var LanguageOf = map[string]string{GoBuildModule: "go", NodeBuildModule: "typescript"}
// ToolchainSet holds what the build modules' toolchains buckets hold: each node's entry of a language, as the watches
// put them in. Safe for use from the watches and from a build at once.
type ToolchainSet struct {
mu sync.Mutex
by map[string]map[string]delivery.Toolchain // language -> node -> entry
}
// Put takes a change of a build module's toolchains bucket in: a put of a node's entry (key <node>.<language>) or its
// delete. It answers whether the language's entries changed (a node's version or recipe, or a node appearing or going).
// A key or a value that is not an entry of the module's language is left out.
func (s *ToolchainSet) Put(module, key string, value []byte, deleted bool) bool {
language, ok := LanguageOf[module]
if !ok {
return false
}
node, keyLanguage, err := delivery.ParseToolchainKey(key)
if err != nil || keyLanguage != language {
return false
}
s.mu.Lock()
defer s.mu.Unlock()
if deleted {
if _, had := s.by[language][node]; !had {
return false
}
delete(s.by[language], node)
return true
}
tc, err := delivery.DecodeToolchain(value)
if err != nil || tc.Language != language {
return false
}
if s.by == nil {
s.by = map[string]map[string]delivery.Toolchain{}
}
if s.by[language] == nil {
s.by[language] = map[string]delivery.Toolchain{}
}
cur, had := s.by[language][node]
s.by[language][node] = tc
return !had || cur.Version != tc.Version || cur.Recipe != tc.Recipe
}
// Entries are the nodes' entries of a language, in no particular order.
func (s *ToolchainSet) Entries(language string) []delivery.NodeToolchain {
s.mu.Lock()
defer s.mu.Unlock()
var out []delivery.NodeToolchain
for node, tc := range s.by[language] {
out = append(out, delivery.NodeToolchain{Node: node, Toolchain: tc})
}
return out
}
vendor/git.novox.be/novox/mesh-sdk/go/delivery/checker.go
Vendored
+8 -2
View File
@@ -96,10 +96,16 @@ type LevelResult struct {
// The states of a Composition (ADR 0484 clause 6): the composition check is the assigner's, so until it delivers a
// verdict says CompositionNotRun, and a line never says the composition passed on that.
//
// A composition not run may say why in its Words: AssignerNotRunning when no instance of the assigner is online, so a
// check never waits for a composition nobody is making (issue 1297).
const (
CompositionNotRun = "not-run"
CompositionPass = "pass"
CompositionFail = "fail"
// AssignerNotRunning is the Words of a composition not run because no instance of the assigner is online.
AssignerNotRunning = "the assigner is not running"
)
// Composition is what the check of the machines' composition made of a verdict: its State (not-run, pass or fail), the
@@ -156,8 +162,8 @@ func (c *Composition) check() error {
}
switch c.State {
case CompositionNotRun:
if c.Words != "" || c.SDK != "" || len(c.Machines) > 0 {
return fmt.Errorf("delivery: the verdict's composition is not run and says nothing of words, the mesh-sdk commit or machines")
if c.SDK != "" || len(c.Machines) > 0 {
return fmt.Errorf("delivery: the verdict's composition is not run and names no mesh-sdk commit or machines")
}
case CompositionPass:
if c.SDK == "" {
vendor/git.novox.be/novox/mesh-sdk/go/delivery/grants.go
Vendored
+75
View File
@@ -0,0 +1,75 @@
package delivery
import (
"encoding/json"
"fmt"
"reflect"
"strings"
)
// The controller's grants (novox/hq ADR 0528 clause 2, ADR 0429 §6 rule 6): when the controller has issued a module's
// bus credential for a fingerprint, it writes `<module>.<fingerprint hex>` in its bucket grants. The assigner reads
// it: a node's want moves to a fingerprint only once its grant exists. Only the controller writes the bucket.
//
// The names say Controller, since the catalogue's GrantKey (catalogue/streams.go) is a stream's grant pattern, unrelated.
const (
// ControllerGrantsBucket is the controller's bucket of issued credentials by fingerprint.
ControllerGrantsBucket = "mesh-controller_grants"
// ControllerGrantsHistory is the revisions the bucket keeps per key.
ControllerGrantsHistory = 5
)
// ControllerGrantKey is the key, in ControllerGrantsBucket, of a module's credential issued for a fingerprint:
// `<module>.<hex>`, the hex being the fingerprint's key part (FingerprintKeyPart).
func ControllerGrantKey(module, fingerprint string) (string, error) {
return BuilderBuildKey(module, fingerprint)
}
// ParseControllerGrantKey is the inverse of ControllerGrantKey; it refuses a want or held key, naming it.
func ParseControllerGrantKey(key string) (module, fingerprint string, err error) {
if module, fingerprint, err = ParseBuilderBuildKey(key); err != nil {
return "", "", fmt.Errorf("delivery: %q is not a grant key, <module>.<fingerprint hex>: %w", key, err)
}
return module, fingerprint, nil
}
// ControllerGrant is the value of a grant key: the bus accounts (their public keys) the module's processes were issued
// for that fingerprint, none for a module whose grants the tool runner's account carries, so a reissue for another
// account is a new revision. The entry's own time says when it was issued.
type ControllerGrant struct {
V int `json:"v"`
Accounts []string `json:"accounts,omitempty"`
}
func (g ControllerGrant) check() error {
for _, a := range g.Accounts {
if strings.TrimSpace(a) == "" || strings.ContainsAny(a, " \t\n") {
return fmt.Errorf("delivery: a grant names an empty or spaced account %q", a)
}
}
return nil
}
// EncodeControllerGrant is g as JSON at Version; it refuses an empty or spaced account.
func EncodeControllerGrant(g ControllerGrant) ([]byte, error) {
if err := g.check(); err != nil {
return nil, err
}
g.V = Version
return json.Marshal(g)
}
// DecodeControllerGrant reads a grant; it refuses an unknown version (ErrUnknownVersion) and an empty or spaced account.
func DecodeControllerGrant(data []byte) (ControllerGrant, error) {
var g ControllerGrant
if err := decode(data, &g); err != nil {
return ControllerGrant{}, err
}
if err := g.check(); err != nil {
return ControllerGrant{}, err
}
return g, nil
}
// SameControllerGrant says whether two grants are the same value.
func SameControllerGrant(a, b ControllerGrant) bool { return reflect.DeepEqual(a, b) }
vendor/git.novox.be/novox/mesh-sdk/go/delivery/source.go
Vendored
+4
View File
@@ -271,6 +271,10 @@ type CandidatePull struct {
Fps map[string]CandidateModule `json:"fps,omitempty"`
}
// CandidateNoneNoModule is the None2 of a candidate's module whose directory the merge preview names no manifest for:
// the change removes the module. The indexer writes it and the assigner reads it as a removal (issue 1297).
const CandidateNoneNoModule = "the merge preview names no manifest for the module's directory"
// CandidateModule is one touched module's new-form fingerprint FP2 (`src2:` and 64 hex digits), or None2, the reason
// it has none; exactly one of them. Manifest is the digest of the module's manifest as read at the preview.
type CandidateModule struct {
vendor/git.novox.be/novox/mesh-sdk/go/delivery/stepasks.go
Vendored
+427
View File
@@ -0,0 +1,427 @@
package delivery
import (
"encoding/json"
"fmt"
"regexp"
"strings"
"time"
)
// **Asks: a build or a check step is a message a build node pulls when it has a free slot** (novox/hq ADR 0525, as
// ADR 0536 §4 changes it; issue 1887). The builder puts a trunk fingerprint's build on the delivery stream of the
// artifact's language and a candidate's on the check stream; the checker puts a check of one level on the check stream,
// and a re-judge after a trunk move on the recheck stream. Each build module claims the node seat of its language
// (`node-build-go`, `node-build-node`, `node-build-image`), whose accepted kinds the broker derives as the work queue
// `SEAT_NODE_BUILD_<LANGUAGE>` with one shared pull worker per kind; a holder pulls delivery first, then check, then
// recheck, at most its node's build capacity at once (ADR 0525 §4, §5).
//
// **An ask is one whole job on one node** (ADR 0525 §3): a build ask is the fetch, the build (or copy) of each artifact
// and the push; a check ask is the fetch, the digest and the tests of one level. Its end is written on the build module's
// step key `<node>.<ask id>` (StepsBucket), which the asker watches with AskStepWatch (ADR 0525 §6).
// The kinds of ask: the seat's accepted kinds, in the order a holder pulls them.
const (
AskDelivery = "delivery"
AskCheck = "check"
AskRecheck = "recheck"
)
// AskKinds are the accepted kinds of a build seat, in the order a holder pulls them (ADR 0536 §4).
var AskKinds = []string{AskDelivery, AskCheck, AskRecheck}
// The languages of the build seats: each is one build module's (mesh-build-<language>) and one seat's
// (node-build-<language>).
const (
AskGo = "go"
AskNode = "node"
AskImage = "image"
)
// AskLanguages are the build seats' languages.
var AskLanguages = []string{AskGo, AskNode, AskImage}
// StepAskVersion is the version of an Ask value; a reader refuses another (ErrUnknownVersion).
const StepAskVersion = 1
func checkAskKind(kind string) error {
for _, k := range AskKinds {
if kind == k {
return nil
}
}
return fmt.Errorf("delivery: the ask kind %q is not delivery, check or recheck", kind)
}
func checkAskLanguage(language string) error {
for _, l := range AskLanguages {
if language == l {
return nil
}
}
return fmt.Errorf("delivery: the ask language %q is not go, node or image", language)
}
// BuildSeat is the node seat a build module of language claims: node-build-<language>.
func BuildSeat(language string) (string, error) {
if err := checkAskLanguage(language); err != nil {
return "", err
}
return "node-build-" + language, nil
}
// BuildModuleOf is the build module that holds language's seat: mesh-build-<language>.
func BuildModuleOf(language string) (string, error) {
if err := checkAskLanguage(language); err != nil {
return "", err
}
return "mesh-build-" + language, nil
}
// AskSubject is the subject an ask of kind for language is published on: the seat's accept subject,
// `mesh.seat.node-build-<language>.accept.<kind>` (mesh-controller internal/broker SeatStreams).
func AskSubject(language, kind string) (string, error) {
seat, err := BuildSeat(language)
if err != nil {
return "", err
}
if err := checkAskKind(kind); err != nil {
return "", err
}
return "mesh.seat." + seat + ".accept." + kind, nil
}
// ParseAskSubject is the inverse of AskSubject; it refuses a subject of another seat or kind.
func ParseAskSubject(subject string) (language, kind string, err error) {
rest, ok := strings.CutPrefix(subject, "mesh.seat.node-build-")
if !ok {
return "", "", fmt.Errorf("delivery: %q is not a build seat's accept subject", subject)
}
language, kind, ok = strings.Cut(rest, ".accept.")
if !ok {
return "", "", fmt.Errorf("delivery: %q is not a build seat's accept subject", subject)
}
if want, err := AskSubject(language, kind); err != nil || want != subject {
return "", "", fmt.Errorf("delivery: %q is not a build seat's accept subject: %v", subject, err)
}
return language, kind, nil
}
// AskStream is the work-queue stream of language's seat, as the broker names a seat's stream: SEAT_NODE_BUILD_<LANGUAGE>.
func AskStream(language string) (string, error) {
seat, err := BuildSeat(language)
if err != nil {
return "", err
}
return "SEAT_" + strings.ToUpper(strings.ReplaceAll(seat, "-", "_")), nil
}
// AskWorker is the shared pull consumer of one kind on language's stream, filtered on that kind's subject:
// SEAT_NODE_BUILD_<LANGUAGE>_<KIND>_worker. Every holder of the seat pulls from it, so the work is shared without a
// holder knowing the others (ADR 0190), and the kinds' workers are apart so a holder can pull them in their order.
func AskWorker(language, kind string) (string, error) {
stream, err := AskStream(language)
if err != nil {
return "", err
}
if err := checkAskKind(kind); err != nil {
return "", err
}
return stream + "_" + strings.ToUpper(kind) + "_worker", nil
}
// An ask id is a step id of the form buildstep.NewID gives: the time in milliseconds in base 36 (8 or 9 digits) and six
// random hex digits. Its form keeps it apart from a module's or a node's name in a key.
var askIDOK = regexp.MustCompile(`^[0-9a-z]{8,9}[0-9a-f]{6}$`)
func checkAskID(id string) error {
if !askIDOK.MatchString(id) {
return fmt.Errorf("delivery: %q is not an ask id (base-36 milliseconds and six hex digits)", id)
}
return nil
}
// StepAsk is the message on a build seat's stream: who asked what, of which kind and language, and the one whole job.
// Exactly one of Build and Check is set: a delivery ask is a build; a check ask a build (a candidate's fingerprint) or a
// check; a recheck ask a check; an image ask is always a build.
type StepAsk struct {
V int `json:"v"`
ID string `json:"id"`
Kind string `json:"kind"`
Language string `json:"language"`
// Asker is the module that put the ask (mesh-delivery-builder or mesh-delivery-checker); the stream's grant is what
// makes it true, this field only names it to a reader.
Asker string `json:"asker"`
At time.Time `json:"at"`
Build *BuildJob `json:"build,omitempty"`
Check *CheckJob `json:"check,omitempty"`
}
// BuildJob is a build ask's job: fetch the commit (unless every artifact is a copy), then build or copy each artifact,
// pushing it when Publish says so, and answer each artifact's digest and reference (BuildAnswer).
type BuildJob struct {
// Module is the module's name; Path its directory in the repository (the build tool's `module`, "." for its root).
Module string `json:"module"`
Path string `json:"path,omitempty"`
// FP is the fingerprint the build is of (the builder's key), named for a reader of the ask.
FP string `json:"fp,omitempty"`
// Fetch is the fetch tool's arguments, as the build module's fetch tool takes them (buildstep.FetchRequest's Args);
// absent when every artifact is a copy.
Fetch map[string]any `json:"fetch,omitempty"`
// Artifacts are built in this order.
Artifacts []AskArtifact `json:"artifacts"`
// Publish says the build pushes an image or publishes a package: a trunk build only (ADR 0487 clause 12).
Publish bool `json:"publish,omitempty"`
// Toolchain is the toolchain the fingerprint was computed with; a build module running another refuses the ask.
Toolchain *ToolchainUsed `json:"toolchain,omitempty"`
}
// AskArtifact is one artifact of a build ask: built from the fetched tree, or copied from From, an upstream pin (ADR
// 0498 clause 4).
type AskArtifact struct {
Artifact string `json:"artifact"`
From string `json:"from,omitempty"`
}
// CheckJob is a check ask's job, the levels of one check (ADR 0525 §3): fetch the head merged onto its trunk once, then
// for each level in order digest its packages, take the kept verdicts for the digests that have one, and test the rest,
// the packages the change's own tests are in first, stopping at the first level that does not pass (CheckAnswer).
type CheckJob struct {
// Fetch is the fetch tool's arguments (repository, commit, ref, base, and the build module's own: beside, package).
Fetch map[string]any `json:"fetch"`
// Levels are run in this order (LevelUnit, LevelIntegration); a try may name one.
Levels []string `json:"levels"`
// Kept says the build module takes the kept verdicts of the checker's kept bucket (CheckerKeptBucket, by KeptKey)
// for the packages whose digest has one and covers the siblings the fetch placed, and tests only the others (issue
// 1887's design limit 1). A try that runs some tests (Run), and a level the candidate's own build module judges
// (Runner), keep nothing.
Kept bool `json:"kept,omitempty"`
// Runner is the test step's runner ("workspace": the candidate's own build module judges it, ADR 0535 §4), used
// when the merge changes a path under RunnerWhen (every merge when RunnerWhen is empty).
Runner string `json:"runner,omitempty"`
RunnerWhen string `json:"runner_when,omitempty"`
// Run is a try's test pattern (go test -run): each level runs it alone, digesting nothing.
Run string `json:"run,omitempty"`
}
func (a StepAsk) check() error {
if err := checkAskID(a.ID); err != nil {
return err
}
if err := checkAskKind(a.Kind); err != nil {
return err
}
if err := checkAskLanguage(a.Language); err != nil {
return err
}
if err := checkName("asker", a.Asker); err != nil {
return err
}
if a.At.IsZero() {
return fmt.Errorf("delivery: the ask %s names no time", a.ID)
}
switch {
case (a.Build == nil) == (a.Check == nil):
return fmt.Errorf("delivery: the ask %s holds a build or a check, exactly one", a.ID)
case a.Build != nil && a.Kind == AskRecheck:
return fmt.Errorf("delivery: the ask %s is a build on the recheck stream, which carries checks alone", a.ID)
case a.Check != nil && a.Kind == AskDelivery:
return fmt.Errorf("delivery: the ask %s is a check on the delivery stream, which carries trunk builds alone", a.ID)
case a.Check != nil && a.Language == AskImage:
return fmt.Errorf("delivery: the ask %s is a check of an image, which has no tests", a.ID)
}
if a.Build != nil {
return a.Build.check(a.ID)
}
return a.Check.check(a.ID)
}
func (b BuildJob) check(id string) error {
if err := checkName("module", b.Module); err != nil {
return err
}
if len(b.Artifacts) == 0 {
return fmt.Errorf("delivery: the build ask %s names no artifact", id)
}
fetch := false
for _, a := range b.Artifacts {
if err := checkName("artifact", a.Artifact); err != nil {
return err
}
fetch = fetch || a.From == ""
}
if fetch {
if b.Path == "" {
return fmt.Errorf("delivery: the build ask %s names no directory (path) for the module %s", id, b.Module)
}
if err := checkFetchArgs(id, b.Fetch); err != nil {
return err
}
}
return nil
}
func (c CheckJob) check(id string) error {
if len(c.Levels) == 0 {
return fmt.Errorf("delivery: the check ask %s names no level", id)
}
for _, l := range c.Levels {
if err := checkLevel(l); err != nil {
return err
}
}
return checkFetchArgs(id, c.Fetch)
}
var askCommitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
// checkFetchArgs refuses fetch arguments with no repository or no full commit; the rest is the build module's fetch to
// judge, as a direct call's is.
func checkFetchArgs(id string, args map[string]any) error {
repository, _ := args["repository"].(string)
commit, _ := args["commit"].(string)
if repository == "" || !strings.Contains(repository, "/") {
return fmt.Errorf("delivery: the ask %s fetches no repository <owner>/<name>", id)
}
if !askCommitOK.MatchString(commit) {
return fmt.Errorf("delivery: the ask %s fetches no full commit", id)
}
return nil
}
// EncodeStepAsk is a as JSON at StepAskVersion; it refuses what DecodeStepAsk refuses.
func EncodeStepAsk(a StepAsk) ([]byte, error) {
a.V = StepAskVersion
a.At = a.At.UTC()
if err := a.check(); err != nil {
return nil, err
}
return json.Marshal(a)
}
// DecodeStepAsk reads an ask; it refuses an unknown version (ErrUnknownVersion), an unknown kind or language, an id that
// is not an ask id, a build and a check together or neither, a job on a stream that does not carry it, and a fetch with
// no repository or no full commit.
func DecodeStepAsk(data []byte) (StepAsk, error) {
var probe struct {
V *int `json:"v"`
}
if err := json.Unmarshal(data, &probe); err != nil {
return StepAsk{}, fmt.Errorf("delivery: an ask is not JSON: %w", err)
}
if probe.V == nil || *probe.V != StepAskVersion {
return StepAsk{}, fmt.Errorf("%w: an ask of version %v, this reader reads %d", ErrUnknownVersion, probe.V, StepAskVersion)
}
var a StepAsk
if err := json.Unmarshal(data, &a); err != nil {
return StepAsk{}, fmt.Errorf("delivery: an ask does not fit its shape: %w", err)
}
if err := a.check(); err != nil {
return StepAsk{}, err
}
return a, nil
}
// BuildAnswer is a build ask's step result: each artifact's digest and, for a pushed one, its reference.
type BuildAnswer struct {
Artifacts []AskBuilt `json:"artifacts"`
Toolchain *ToolchainUsed `json:"toolchain,omitempty"`
}
// AskBuilt is one artifact a build ask built.
type AskBuilt struct {
Artifact string `json:"artifact"`
Digest string `json:"digest"`
Reference string `json:"reference,omitempty"`
}
// CheckAnswer is a check ask's step result, each part the build module's own tool's answer whole: the fetch's, then one
// LevelAnswer per level run, in order; a level that did not pass is the last. The checker judges and keeps from it as it
// did from the tools' answers.
type CheckAnswer struct {
Fetch json.RawMessage `json:"fetch"`
Levels []LevelAnswer `json:"levels,omitempty"`
}
// LevelAnswer is one level of a check ask: the digest's answer (absent when nothing was digested: the level ran whole and
// keeps nothing), the kept verdicts taken, and one test result per test step run, in the order run.
type LevelAnswer struct {
Level string `json:"level"`
// Judged is what the digest under the candidate's runner said of the program that ran it ("candidate <merge>"),
// when the candidate judged the level (ADR 0535 §4, §5); empty otherwise.
Judged string `json:"judged,omitempty"`
Digest json.RawMessage `json:"digest,omitempty"`
Kept []KeptTaken `json:"kept,omitempty"`
Tests []json.RawMessage `json:"tests,omitempty"`
}
// KeptTaken is a package whose kept verdict the check took instead of testing it.
type KeptTaken struct {
Package string `json:"package"`
Digest string `json:"digest"`
Tests int `json:"tests"`
}
// AskStepKey is the key, in the build module's StepsBucket, of the step that runs an ask on node: `<node>.<ask id>`
// (ADR 0525 §6). The node is the one that pulled the ask, which the asker learns from the key.
func AskStepKey(node, askID string) (string, error) {
if err := checkAskID(askID); err != nil {
return "", err
}
return StepKey(node, askID)
}
// AskStepWatch is the pattern an asker watches its ask's step with, the node a wildcard: `*.<ask id>`.
func AskStepWatch(askID string) (string, error) {
if err := checkAskID(askID); err != nil {
return "", err
}
return "*." + askID, nil
}
// ParseAskStepKey reads the node and the ask id from a step key of an ask; it refuses a key that is not `<node>.<ask
// id>`, such as a have key `<node>.<module>` or a step key a direct tool call wrote under another id form.
func ParseAskStepKey(key string) (node, askID string, err error) {
node, askID, err = ParseStepKey(key)
if err != nil {
return "", "", err
}
if err := checkAskID(askID); err != nil {
return "", "", fmt.Errorf("delivery: %q is not an ask's step key: %w", key, err)
}
return node, askID, nil
}
// PauseKey is the key, in a build module's StepsBucket, that keeps whether the module on node takes asks:
// `paused.<node>` (issue 1887's design limit 2: every holder pulls from one shared worker, so a node's pause is its holder
// not fetching, kept here so a restart keeps it). A pause key is never an ask's step key: its second part is a node.
func PauseKey(node string) (string, error) {
if err := checkName("node", node); err != nil {
return "", err
}
return "paused." + node, nil
}
// Pause is the value of a pause key: whether the node's module takes no asks, who said so and when.
type Pause struct {
V int `json:"v"`
Paused bool `json:"paused"`
By string `json:"by,omitempty"`
At time.Time `json:"at"`
}
// EncodePause is p as JSON at Version.
func EncodePause(p Pause) ([]byte, error) {
p.V = Version
p.At = p.At.UTC()
return json.Marshal(p)
}
// DecodePause reads a pause value; it refuses an unknown version (ErrUnknownVersion).
func DecodePause(data []byte) (Pause, error) {
var p Pause
if err := decode(data, &p); err != nil {
return Pause{}, err
}
return p, nil
}
vendor/git.novox.be/novox/mesh-sdk/go/delivery/view_answers.go
Vendored
+261
View File
@@ -0,0 +1,261 @@
package delivery
import (
"encoding/json"
"fmt"
"strings"
"git.novox.be/novox/mesh-sdk/go/asks"
)
// The asks the view may answer, and its answers (novox/hq ADR 0528 clause 6, ADR 0429 §3 and §6, ADR 0437 §10). The
// gate's merge ask (`merge.<group>.<heads hash>`, MergeAskKey) needs proof of who answered (ADR 0421 §1), so it is
// answered on the phone alone until issue 953 closes. The assigner's asks need no such proof, and the view may
// answer them: the hold of a fingerprint whose health check failed (`hold.<module>.<fingerprint hex>`: release it or
// keep it held) and the restore of a data module set back (`restore.<node>.<module>.<fingerprint hex>`: restore and
// lose newer writes, or leave it stopped, ADR 0429 §6). The assigner writes them in its bucket asks; the view writes
// its answer `<ask key>.view` in its bucket answers, and the teller, which alone fills an ask's answer, takes it from
// there as it takes a warrant from the phone.
const (
// ViewModule is the module of the view part.
ViewModule = "mesh-delivery-view"
// ViewAnswersBucket is the view's bucket of answers, the only state the view writes.
ViewAnswersBucket = ViewModule + "_answers"
// ViewAnswersHistory is the revisions the bucket keeps per key.
ViewAnswersHistory = 5
// AssignerAsksBucket is the assigner's bucket of asks that need no proof of who answered.
AssignerAsksBucket = AssignerModule + "_asks"
// AssignerAsksHistory is the revisions the bucket keeps per key.
AssignerAsksHistory = 5
// AskKindHold is the kind of the assigner's ask about a held fingerprint.
AskKindHold = "hold"
// AskKindRestore is the kind of the assigner's ask about a data module set back.
AskKindRestore = "restore"
viewWord = "view"
)
// AskKey is an ask's key read whole: its Kind, and the parts its kind has: a merge's Group and heads Hash, a hold's
// Module and fingerprint FP, a restore's Node, Module and FP.
type AskKey struct {
Kind string
Group string
Hash string
Node string
Module string
FP string
}
// AskNeedsProof says whether an ask of this kind must prove who answered it, so the view may not answer it: the
// merge's Approve does (ADR 0421 §1, issue 953), the assigner's hold and restore do not. An unknown kind is refused.
func AskNeedsProof(kind string) (bool, error) {
switch kind {
case AskKindMerge:
return true, nil
case AskKindHold, AskKindRestore:
return false, nil
}
return false, fmt.Errorf("delivery: %q is not a kind of ask, %s, %s or %s", kind, AskKindMerge, AskKindHold, AskKindRestore)
}
// HoldAskKey is the key, in AssignerAsksBucket, of the ask about a module's held fingerprint: `hold.<module>.<hex>`.
func HoldAskKey(module, fingerprint string) (string, error) {
key, err := BuilderBuildKey(module, fingerprint)
if err != nil {
return "", err
}
return AskKindHold + "." + key, nil
}
// RestoreAskKey is the key, in AssignerAsksBucket, of the ask about a data module set back on a node from a
// fingerprint: `restore.<node>.<module>.<hex>`.
func RestoreAskKey(node, module, fingerprint string) (string, error) {
if err := checkName("node", node); err != nil {
return "", err
}
key, err := BuilderBuildKey(module, fingerprint)
if err != nil {
return "", err
}
return AskKindRestore + "." + node + "." + key, nil
}
// ParseAnyAskKey reads an ask key of any kind (MergeAskKey, HoldAskKey, RestoreAskKey); it refuses an unknown kind
// and a key whose parts do not fit its kind, naming the key.
func ParseAnyAskKey(key string) (AskKey, error) {
kind, rest, _ := strings.Cut(key, ".")
refused := func(err error) (AskKey, error) {
return AskKey{}, fmt.Errorf("delivery: %q is not an ask key: %w", key, err)
}
switch kind {
case AskKindMerge:
_, group, hash, err := ParseAskKey(key)
if err != nil {
return AskKey{}, err
}
return AskKey{Kind: kind, Group: group, Hash: hash}, nil
case AskKindHold:
module, fp, err := ParseBuilderBuildKey(rest)
if err != nil {
return refused(err)
}
return AskKey{Kind: kind, Module: module, FP: fp}, nil
case AskKindRestore:
node, build, ok := strings.Cut(rest, ".")
if !ok {
return refused(fmt.Errorf("a restore ask is restore.<node>.<module>.<fingerprint hex>"))
}
if err := checkName("node", node); err != nil {
return refused(err)
}
module, fp, err := ParseBuilderBuildKey(build)
if err != nil {
return refused(err)
}
return AskKey{Kind: kind, Node: node, Module: module, FP: fp}, nil
}
_, err := AskNeedsProof(kind)
return refused(err)
}
// ViewAnswerKey is the key, in ViewAnswersBucket, of the view's answer to the ask at askKey: `<ask key>.view`. An ask
// that needs proof of who answered is refused with its reason: it is answered on the phone (issue 953).
func ViewAnswerKey(askKey string) (string, error) {
ask, err := ParseAnyAskKey(askKey)
if err != nil {
return "", err
}
if proof, _ := AskNeedsProof(ask.Kind); proof {
return "", fmt.Errorf("delivery: the ask %s needs proof of who answered: answer it on the phone (issue 953)", askKey)
}
return askKey + "." + viewWord, nil
}
// ParseViewAnswerKey is the inverse of ViewAnswerKey: the ask's key. It refuses a key without `.view`, an ask key of
// a kind that needs proof, and a key of another bucket, naming it.
func ParseViewAnswerKey(key string) (askKey string, err error) {
askKey, ok := strings.CutSuffix(key, "."+viewWord)
if !ok {
return "", fmt.Errorf("delivery: %q is not a view answer key, <ask key>.view", key)
}
if _, err := ViewAnswerKey(askKey); err != nil {
return "", fmt.Errorf("delivery: %q is not a view answer key: %w", key, err)
}
return askKey, nil
}
// AssignerAsk is the value of a key in AssignerAsksBucket: the router's ask whole, as the assigner composes it, so the
// teller publishes it unchanged. Asking counts the times the assigner wrote the ask at this key, as the gate's Ask
// does: 0 or 1 for the first, n for the n-th, and the ask's id is AskingID(key, Asking), so each asking has its own id
// and an answer to an earlier asking is never taken for a later one.
type AssignerAsk struct {
V int `json:"v"`
Ask asks.Ask `json:"ask"`
Asking int `json:"asking,omitempty"`
}
func (a AssignerAsk) check(key string) error {
ask, err := ParseAnyAskKey(key)
if err != nil {
return err
}
if proof, _ := AskNeedsProof(ask.Kind); proof {
return fmt.Errorf("delivery: the assigner asks no %s ask: it needs proof of who answered", ask.Kind)
}
if a.Asking < 0 {
return fmt.Errorf("delivery: the ask's asking %d is negative", a.Asking)
}
if want := AskingID(key, a.Asking); a.Ask.ID != want {
return fmt.Errorf("delivery: the ask's id %q is not %q, the id its key and asking %d give", a.Ask.ID, want, a.Asking)
}
if len(a.Ask.Options) == 0 {
return fmt.Errorf("delivery: the ask has no option")
}
return nil
}
// EncodeAssignerAsk is a, the ask at key, as JSON at Version; it refuses a key of a kind that needs proof or of no
// kind, an id other than the one the key and the asking give, and an ask with no option.
func EncodeAssignerAsk(key string, a AssignerAsk) ([]byte, error) {
if err := a.check(key); err != nil {
return nil, err
}
a.V = Version
return json.Marshal(a)
}
// DecodeAssignerAsk reads the ask at key; it refuses an unknown version (ErrUnknownVersion) and what EncodeAssignerAsk
// refuses.
func DecodeAssignerAsk(key string, data []byte) (AssignerAsk, error) {
var a AssignerAsk
if err := decode(data, &a); err != nil {
return AssignerAsk{}, err
}
if err := a.check(key); err != nil {
return AssignerAsk{}, err
}
return a, nil
}
// SameAssignerAsk says whether two asks are the same value.
func SameAssignerAsk(a, b AssignerAsk) bool {
return a.V == b.V && a.Asking == b.Asking && a.Ask.Digest() == b.Ask.Digest()
}
// ViewAnswer is the view's answer to an ask: the id of the asking it answers (the ask's id, AskingID of its key and
// asking), the option the operator chose and their words. The entry's own time says when.
type ViewAnswer struct {
V int `json:"v"`
Asking string `json:"asking"`
Option string `json:"option"`
Words string `json:"words,omitempty"`
}
func (a ViewAnswer) check() error {
if strings.TrimSpace(a.Asking) == "" {
return fmt.Errorf("delivery: a view answer names no asking")
}
if strings.TrimSpace(a.Option) == "" {
return fmt.Errorf("delivery: a view answer names no option")
}
return nil
}
// ViewAnswerFits says whether the view's answer a answers the ask as it stands now: it refuses an answer to another
// asking of the ask (a stale answer to an earlier writing of the same key) and an option the ask does not offer,
// naming both.
func ViewAnswerFits(a ViewAnswer, ask AssignerAsk) error {
if a.Asking != ask.Ask.ID {
return fmt.Errorf("delivery: the view's answer is to the asking %s, and the ask stands at the asking %s", a.Asking, ask.Ask.ID)
}
if _, ok := ask.Ask.Option(a.Option); !ok {
return fmt.Errorf("delivery: the view's answer chose %q, which the asking %s does not offer", a.Option, ask.Ask.ID)
}
return nil
}
// EncodeViewAnswer is a as JSON at Version; it refuses an answer naming no asking or no option.
func EncodeViewAnswer(a ViewAnswer) ([]byte, error) {
if err := a.check(); err != nil {
return nil, err
}
a.V = Version
return json.Marshal(a)
}
// DecodeViewAnswer reads a view answer; it refuses an unknown version (ErrUnknownVersion) and an answer naming no
// asking or no option.
func DecodeViewAnswer(data []byte) (ViewAnswer, error) {
var a ViewAnswer
if err := decode(data, &a); err != nil {
return ViewAnswer{}, err
}
if err := a.check(); err != nil {
return ViewAnswer{}, err
}
return a, nil
}
// SameViewAnswer says whether two view answers are the same value.
func SameViewAnswer(a, b ViewAnswer) bool { return a == b }
vendor/git.novox.be/novox/mesh-sdk/go/merge-check.sh
Vendored
-136
View File
@@ -1,136 +0,0 @@
#!/bin/sh
# mesh-check-toolchain: go
#
# The Go part of the SDK's own check (novox/hq ADR 0237 as amended, ADR 0272): the Go binding, run by the build
# seat in the mesh's Go toolchain, its own container, because the SDK's merge-check.sh declares it
# (`# mesh-check-also: go go/merge-check.sh`) — and by hand, from anywhere: `sh go/merge-check.sh`.
#
# Formatted, vetted (with and without the build tag `integration`), and run at two levels (novox/hq ADR 0391):
#
# 1. the unit level: `go test ./...` untagged, no bus or other process, timed and held to a bound
# (unit_bound below); over it the check fails and names the slowest tests (issue 895);
# 2. the integration level: the whole suite with `-tags integration`, each test against a bus of its
# own (testbus, which issue 1171 brings with the first test that needs a bus) so nothing is
# serialised. With no tagged test yet, the level runs the unit tests again and says their count.
#
# **A test that starts a bus or another process carries `//go:build integration`** as the first line of
# its file; a file that held both kinds is two files (`<name>_integration_test.go` beside `<name>_test.go`).
# Each level says its test count and its time.
#
# **A test that needs bubblewrap carries `//go:build integration && isolation`** (issue 1309's proof that a run step
# is shut off): this script vets it but does not run it, since the toolchain it runs in has no bubblewrap; the checker
# runs the `isolation` tag on the build nodes, which have it (R4e).
#
# No dependency is committed (no vendor directory: .gitignore; the operator, 2026-10-19; issue 1304). Every dependency
# is resolved by the version and checksum go.sum names (novox/hq ADR 0405 §5), from its public source: this binding
# requires nothing of the mesh's own forge. `go mod verify` fails by module name when a downloaded dependency does not
# match go.sum, and -mod=readonly fails a build whose go.sum lacks an entry instead of writing it. gofmt reads only
# the binding's own files.
#
# Exit 0 is a pass; anything else fails `mesh/repo-check` with the last line printed.
set -eu
cd "$(dirname "$0")"
# The toolchain the check runs in sets -mod=vendor; there is no vendor directory, so it is replaced.
GOFLAGS="$(printf '%s' "${GOFLAGS:-}" | sed 's/-mod=[a-z]*//g') -mod=readonly"
export GOFLAGS
export GOPRIVATE=git.novox.be
# **The unit level's bound, in seconds** (issue 895): the slowest package's own test time of the untagged run,
# the compile not counted. A run over it fails the check and names the slowest tests. The SDK's unit suite is
# small (its slowest package measured well under a second on a quiet machine), so 2 s leaves room for a busier
# build node without hiding a test that should carry `integration` instead.
unit_bound=2
# judge_unit_run says the unit level as `go test -v` reported it and fails when it was over the bound: $1 the
# file holding the output of the untagged `go test -v` of every package (packages one after another, each
# ending in its `ok <package> <time>` line), $2 the bound in seconds. Packages run side by side, so the
# level's time is that of its slowest package; the five slowest tests of the run are named when it is over,
# so that the bound is kept by making them cheap or by tagging them `integration`, and never by raising it
# (issue 895).
judge_unit_run() {
out=$1 bound=$2
awk '
/^--- (PASS|FAIL|SKIP): / {
name = $3; took = $4; gsub(/[()s]/, "", took)
pending[++n] = took "\t" name
if ($2 == "SKIP:") skipped++
next
}
/^(ok|FAIL)[ \t]/ && NF >= 3 && $3 ~ /^[0-9.]+s$/ {
t = $3; sub(/s$/, "", t)
for (i = 1; i <= n; i++) print "T\t" pending[i] "\t" $2
n = 0
print "P\t" t "\t" $2
}
END { print "S\t" skipped + 0 }
' "$out" >"$out.judged"
tests=$(grep -c '^T' "$out.judged" || true)
skipped=$(awk -F'\t' '$1 == "S" {print $2}' "$out.judged")
level=$(awk -F'\t' '$1 == "P" && $2 + 0 > m {m = $2 + 0} END {printf "%.2f", m}' "$out.judged")
echo "unit level: $tests tests ($skipped skipped), $level s (bound $bound s)"
if awk -v t="$level" -v b="$bound" 'BEGIN {exit !(t + 0 > b + 0)}'; then
echo "the unit level took $level s, over its bound of $bound s; the slowest tests:"
awk -F'\t' '$1 == "T" {print $2 "\t" $4 "\t" $3}' "$out.judged" | sort -t"$(printf '\t')" -k1,1 -nr | head -n 5 |
awk -F'\t' '{printf " %s s %s %s\n", $1, $3, $2}'
echo "make them cheap (no wait, no process), or give their files //go:build integration if they start a bus or another process"
return 1
fi
}
# `sh go/merge-check.sh judge-unit-run <output> <bound>` runs only the judging of the unit level, on a
# recorded output.
if [ "${1:-}" = judge-unit-run ]; then
shift
judge_unit_run "$@"
exit
fi
if ! fetched=$(go mod download 2>&1); then
echo "FAILED: the dependencies cannot be fetched: $(printf '%s\n' "$fetched" | tail -n 1)"
exit 1
fi
if ! verified=$(go mod verify 2>&1); then
echo "FAILED: a dependency does not match go.sum: $verified"
exit 1
fi
unformatted=$(find . -name '*.go' -not -path './vendor/*' | xargs gofmt -l)
if [ -n "$unformatted" ]; then
echo "not gofmt'd:"
echo "$unformatted"
exit 1
fi
go vet ./...
# The files tagged `integration` are vetted too, or nothing would read them.
go vet -tags integration ./...
go vet -tags integration,isolation ./...
# NOT RACE-CHECKED here: the SDK's suite is small and carries no store or other shared state between
# packages, so the race detector is not required of this check (novox/hq issue 1175); a module that needs
# it asks for it in its own merge-check.sh.
echo "NOT RACE-CHECKED: the Go binding's check runs without the race detector"
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT
# The unit level first and alone: every package untagged, no bus or other process (judged above).
status=0
go test -count=1 -v ./... >"$work/unit.out" 2>&1 || status=$?
if [ "$status" -ne 0 ]; then
grep -vE '^(=== |[[:space:]]*--- PASS)' "$work/unit.out"
echo "the unit level failed (go test exited $status)"
exit 1
fi
judge_unit_run "$work/unit.out" "$unit_bound"
# The integration level: the whole suite with the tag, each test against its own bus (testbus).
integration_start=$(date +%s)
status=0
go test -tags integration -count=1 -v ./... >"$work/integration.out" 2>&1 || status=$?
grep -vE '^(=== |[[:space:]]*--- PASS)' "$work/integration.out" || true
tests=$(grep -cE '^--- (PASS|FAIL|SKIP): ' "$work/integration.out" || true)
skipped=$(grep -cE '^--- SKIP: ' "$work/integration.out" || true)
echo "integration level: $tests tests ($skipped skipped), $(($(date +%s) - integration_start)) s"
if [ "$status" -ne 0 ]; then
echo "the integration level failed (go test exited $status)"
exit 1
fi
vendor/git.novox.be/novox/mesh-sdk/go/seat.go
Vendored
+55
View File
@@ -69,6 +69,61 @@ func SeatTake(worker string, handle func(Work) error) error {
return err
}
// Pulled is one message a bundle pulled from a seat's worker (SeatPull), named by the runtime's id for it, which
// SeatSettle gives its fate.
type Pulled struct {
ID string `json:"id"`
Worker string `json:"worker"`
Subject string `json:"subject"`
Body json.RawMessage `json:"body"`
Headers map[string]string `json:"headers,omitempty"`
Delivered uint64 `json:"delivered"`
}
// The fates SeatSettle gives a pulled message: done, handed back to be delivered again, dropped, or still being worked.
const (
SettleAck = "ack"
SettleNak = "nak"
SettleTerm = "term"
SettleProgress = "progress"
)
// SeatPull pulls at most one message from a held seat's worker the module's membership lists, waiting at most wait
// (none when 0, at most 25 s): nil when none came. Unlike SeatTake nothing is fetched ahead: a build module pulls an
// ask only when it has a free slot (novox/hq ADR 0525 §4; issue 1887 A11). A message never settled is delivered again
// by the worker's acknowledgement wait.
func SeatPull(worker string, wait time.Duration) (*Pulled, error) {
c, err := live()
if err != nil {
return nil, err
}
raw, err := c.ask("mesh/seat.pull", map[string]any{"worker": worker, "wait_ms": wait.Milliseconds()}, seatTimeout)
if err != nil {
return nil, err
}
if len(raw) == 0 || string(raw) == "null" {
return nil, nil
}
var p Pulled
if err := json.Unmarshal(raw, &p); err != nil {
return nil, err
}
if p.ID == "" {
return nil, nil
}
return &p, nil
}
// SeatSettle gives the message pulled under id its fate (SettleAck, SettleNak, SettleTerm, SettleProgress).
func SeatSettle(id, fate string) error {
c, err := live()
if err != nil {
return err
}
_, err = c.ask("mesh/seat.settle", map[string]any{"id": id, "fate": fate}, seatTimeout)
return err
}
// SeatRecord reads one key of a record a seat keeps for this module, under its own name, or nil.
func SeatRecord(bucket, key string) (*StateEntry, error) {
c, err := live()
vendor/github.com/therootcompany/xz/AUTHORS
Generated Vendored
-8
View File
@@ -1,8 +0,0 @@
# Package xz authors
Michael Cross <https://github.com/xi2>
# XZ Embedded authors
Lasse Collin <lasse.collin@tukaani.org>
Igor Pavlov <http://7-zip.org/>
vendor/github.com/therootcompany/xz/LICENSE
Generated Vendored
-86
View File
@@ -1,86 +0,0 @@
Creative Commons CC0 1.0 Universal
CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE LEGAL
SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN ATTORNEY-CLIENT
RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS INFORMATION ON AN "AS-IS" BASIS.
CREATIVE COMMONS MAKES NO WARRANTIES REGARDING THE USE OF THIS DOCUMENT OR THE
INFORMATION OR WORKS PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES
RESULTING FROM THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
HEREUNDER.
Statement of Purpose
The laws of most jurisdictions throughout the world automatically confer
exclusive Copyright and Related Rights (defined below) upon the creator and
subsequent owner(s) (each and all, an "owner") of an original work of
authorship and/or a database (each, a "Work").
Certain owners wish to permanently relinquish those rights to a Work for the
purpose of contributing to a commons of creative, cultural and scientific
works ("Commons") that the public can reliably and without fear of later
claims of infringement build upon, modify, incorporate in other works, reuse
and redistribute as freely as possible in any form whatsoever and for any
purposes, including without limitation commercial purposes. These owners may
contribute to the Commons to promote the ideal of a free culture and the
further production of creative, cultural and scientific works, or to gain
reputation or greater distribution for their Work in part through the use and
efforts of others.
For these and/or other purposes and motivations, and without any expectation
of additional consideration or compensation, the person associating CC0 with a
Work (the "Affirmer"), to the extent that he or she is an owner of Copyright
and Related Rights in the Work, voluntarily elects to apply CC0 to the Work
and publicly distribute the Work under its terms, with knowledge of his or her
Copyright and Related Rights in the Work and the meaning and intended legal
effect of CC0 on those rights.
1. Copyright and Related Rights. A Work made available under CC0 may be protected by copyright and related or neighboring rights ("Copyright and Related Rights"). Copyright and Related Rights include, but are not limited to, the following:
i. the right to reproduce, adapt, distribute, perform, display, communicate,
and translate a Work;
ii. moral rights retained by the original author(s) and/or performer(s);
iii. publicity and privacy rights pertaining to a person&apos;s image or
likeness depicted in a Work;
iv. rights protecting against unfair competition in regards to a Work, subject
to the limitations in paragraph 4(a), below;
v. rights protecting the extraction, dissemination, use and reuse of data in a
Work;
vi. database rights (such as those arising under Directive 96/9/EC of the
European Parliament and of the Council of 11 March 1996 on the legal
protection of databases, and under any national implementation thereof,
including any amended or successor version of such directive); and
vii. other similar, equivalent or corresponding rights throughout the world
based on applicable law or treaty, and any national implementations thereof.
2. Waiver. To the greatest extent permitted by, but not in contravention of, applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and unconditionally waives, abandons, and surrenders all of Affirmer&apos;s Copyright and Related Rights and associated claims and causes of action, whether now known or unknown (including existing as well as future claims and causes of action), in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each member of the public at large and to the detriment of Affirmer&apos;s heirs and successors, fully intending that such Waiver shall not be subject to revocation, rescission, cancellation, termination, or any other legal or equitable action to disrupt the quiet enjoyment of the Work by the public as contemplated by Affirmer&apos;s express Statement of Purpose.
3. Public License Fallback. Should any part of the Waiver for any reason be judged legally invalid or ineffective under applicable law, then the Waiver shall be preserved to the maximum extent permitted taking into account Affirmer&apos;s express Statement of Purpose. In addition, to the extent the Waiver is so judged Affirmer hereby grants to each affected person a royalty-free, non transferable, non sublicensable, non exclusive, irrevocable and unconditional license to exercise Affirmer&apos;s Copyright and Related Rights in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "License"). The License shall be deemed effective as of the date CC0 was applied by Affirmer to the Work. Should any part of the License for any reason be judged legally invalid or ineffective under applicable law, such partial invalidity or ineffectiveness shall not invalidate the remainder of the License, and in such case Affirmer hereby affirms that he or she will not (i) exercise any of his or her remaining Copyright and Related Rights in the Work or (ii) assert any associated claims and causes of action with respect to the Work, in either case contrary to Affirmer&apos;s express Statement of Purpose.
4. Limitations and Disclaimers.
a. No trademark or patent rights held by Affirmer are waived, abandoned,
surrendered, licensed or otherwise affected by this document.
b. Affirmer offers the Work as-is and makes no representations or warranties
of any kind concerning the Work, express, implied, statutory or otherwise,
including without limitation warranties of title, merchantability, fitness for
a particular purpose, non infringement, or the absence of latent or other
defects, accuracy, or the present or absence of errors, whether or not
discoverable, all to the greatest extent permissible under applicable law.
c. Affirmer disclaims responsibility for clearing rights of other persons that
may apply to the Work or any use thereof, including without limitation any
person&apos;s Copyright and Related Rights in the Work. Further, Affirmer
disclaims responsibility for obtaining any necessary consents, permissions or
other rights required for any use of the Work.
d. Affirmer understands and acknowledges that Creative Commons is not a party
to this document and has no duty or obligation with respect to this CC0 or use
of the Work.
vendor/github.com/therootcompany/xz/README.md
Generated Vendored
-19
View File
@@ -1,19 +0,0 @@
# Xz
Package xz implements XZ decompression natively in Go.
Documentation at <https://pkg.go.dev/github.com/therootcompany/xz>.
Download and install with `go get github.com/xi2/xz`.
If you need compression as well as decompression, you might want to
look at <https://github.com/ulikunitz/xz>.
# LICENSE
This was originally released into the public domain by the AUTHORS.
Here it is licensed more explicitly as Creative Commons CC0 1.0 Universal
(CC0-1.0) so that it can be detected by automated tooling, and satisfy the legal
requirements for vendor integration for cases in which a "public domain"
statement is not sufficient.
vendor/github.com/therootcompany/xz/dec_bcj.go
Generated Vendored
-461
View File
@@ -1,461 +0,0 @@
/*
* Branch/Call/Jump (BCJ) filter decoders
*
* Authors: Lasse Collin <lasse.collin@tukaani.org>
* Igor Pavlov <http://7-zip.org/>
*
* Translation to Go: Michael Cross <https://github.com/xi2>
*
* This file has been put into the public domain.
* You can do whatever you want with this file.
*/
package xz
/* from linux/lib/xz/xz_dec_bcj.c *************************************/
type xzDecBCJ struct {
/* Type of the BCJ filter being used */
typ xzFilterID
/*
* Return value of the next filter in the chain. We need to preserve
* this information across calls, because we must not call the next
* filter anymore once it has returned xzStreamEnd
*/
ret xzRet
/*
* Absolute position relative to the beginning of the uncompressed
* data (in a single .xz Block).
*/
pos int
/* x86 filter state */
x86PrevMask uint32
/* Temporary space to hold the variables from xzBuf */
out []byte
outPos int
temp struct {
/* Amount of already filtered data in the beginning of buf */
filtered int
/*
* Buffer to hold a mix of filtered and unfiltered data. This
* needs to be big enough to hold Alignment + 2 * Look-ahead:
*
* Type Alignment Look-ahead
* x86 1 4
* PowerPC 4 0
* IA-64 16 0
* ARM 4 0
* ARM-Thumb 2 2
* SPARC 4 0
*/
buf []byte // slice buf will be backed by bufArray
bufArray [16]byte
}
}
/*
* This is used to test the most significant byte of a memory address
* in an x86 instruction.
*/
func bcjX86TestMSByte(b byte) bool {
return b == 0x00 || b == 0xff
}
func bcjX86Filter(s *xzDecBCJ, buf []byte) int {
var maskToAllowedStatus = []bool{
true, true, true, false, true, false, false, false,
}
var maskToBitNum = []byte{0, 1, 2, 2, 3, 3, 3, 3}
var i int
var prevPos int = -1
var prevMask uint32 = s.x86PrevMask
var src uint32
var dest uint32
var j uint32
var b byte
if len(buf) <= 4 {
return 0
}
for i = 0; i < len(buf)-4; i++ {
if buf[i]&0xfe != 0xe8 {
continue
}
prevPos = i - prevPos
if prevPos > 3 {
prevMask = 0
} else {
prevMask = (prevMask << (uint(prevPos) - 1)) & 7
if prevMask != 0 {
b = buf[i+4-int(maskToBitNum[prevMask])]
if !maskToAllowedStatus[prevMask] || bcjX86TestMSByte(b) {
prevPos = i
prevMask = prevMask<<1 | 1
continue
}
}
}
prevPos = i
if bcjX86TestMSByte(buf[i+4]) {
src = getLE32(buf[i+1:])
for {
dest = src - uint32(s.pos+i+5)
if prevMask == 0 {
break
}
j = uint32(maskToBitNum[prevMask]) * 8
b = byte(dest >> (24 - j))
if !bcjX86TestMSByte(b) {
break
}
src = dest ^ (1<<(32-j) - 1)
}
dest &= 0x01FFFFFF
dest |= 0 - dest&0x01000000
putLE32(dest, buf[i+1:])
i += 4
} else {
prevMask = prevMask<<1 | 1
}
}
prevPos = i - prevPos
if prevPos > 3 {
s.x86PrevMask = 0
} else {
s.x86PrevMask = prevMask << (uint(prevPos) - 1)
}
return i
}
func bcjPowerPCFilter(s *xzDecBCJ, buf []byte) int {
var i int
var instr uint32
for i = 0; i+4 <= len(buf); i += 4 {
instr = getBE32(buf[i:])
if instr&0xFC000003 == 0x48000001 {
instr &= 0x03FFFFFC
instr -= uint32(s.pos + i)
instr &= 0x03FFFFFC
instr |= 0x48000001
putBE32(instr, buf[i:])
}
}
return i
}
var bcjIA64BranchTable = [...]byte{
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
4, 4, 6, 6, 0, 0, 7, 7,
4, 4, 0, 0, 4, 4, 0, 0,
}
func bcjIA64Filter(s *xzDecBCJ, buf []byte) int {
var branchTable = bcjIA64BranchTable[:]
/*
* The local variables take a little bit stack space, but it's less
* than what LZMA2 decoder takes, so it doesn't make sense to reduce
* stack usage here without doing that for the LZMA2 decoder too.
*/
/* Loop counters */
var i int
var j int
/* Instruction slot (0, 1, or 2) in the 128-bit instruction word */
var slot uint32
/* Bitwise offset of the instruction indicated by slot */
var bitPos uint32
/* bit_pos split into byte and bit parts */
var bytePos uint32
var bitRes uint32
/* Address part of an instruction */
var addr uint32
/* Mask used to detect which instructions to convert */
var mask uint32
/* 41-bit instruction stored somewhere in the lowest 48 bits */
var instr uint64
/* Instruction normalized with bit_res for easier manipulation */
var norm uint64
for i = 0; i+16 <= len(buf); i += 16 {
mask = uint32(branchTable[buf[i]&0x1f])
for slot, bitPos = 0, 5; slot < 3; slot, bitPos = slot+1, bitPos+41 {
if (mask>>slot)&1 == 0 {
continue
}
bytePos = bitPos >> 3
bitRes = bitPos & 7
instr = 0
for j = 0; j < 6; j++ {
instr |= uint64(buf[i+j+int(bytePos)]) << (8 * uint(j))
}
norm = instr >> bitRes
if (norm>>37)&0x0f == 0x05 && (norm>>9)&0x07 == 0 {
addr = uint32((norm >> 13) & 0x0fffff)
addr |= (uint32(norm>>36) & 1) << 20
addr <<= 4
addr -= uint32(s.pos + i)
addr >>= 4
norm &= ^(uint64(0x8fffff) << 13)
norm |= uint64(addr&0x0fffff) << 13
norm |= uint64(addr&0x100000) << (36 - 20)
instr &= 1<<bitRes - 1
instr |= norm << bitRes
for j = 0; j < 6; j++ {
buf[i+j+int(bytePos)] = byte(instr >> (8 * uint(j)))
}
}
}
}
return i
}
func bcjARMFilter(s *xzDecBCJ, buf []byte) int {
var i int
var addr uint32
for i = 0; i+4 <= len(buf); i += 4 {
if buf[i+3] == 0xeb {
addr = uint32(buf[i]) | uint32(buf[i+1])<<8 |
uint32(buf[i+2])<<16
addr <<= 2
addr -= uint32(s.pos + i + 8)
addr >>= 2
buf[i] = byte(addr)
buf[i+1] = byte(addr >> 8)
buf[i+2] = byte(addr >> 16)
}
}
return i
}
func bcjARMThumbFilter(s *xzDecBCJ, buf []byte) int {
var i int
var addr uint32
for i = 0; i+4 <= len(buf); i += 2 {
if buf[i+1]&0xf8 == 0xf0 && buf[i+3]&0xf8 == 0xf8 {
addr = uint32(buf[i+1]&0x07)<<19 |
uint32(buf[i])<<11 |
uint32(buf[i+3]&0x07)<<8 |
uint32(buf[i+2])
addr <<= 1
addr -= uint32(s.pos + i + 4)
addr >>= 1
buf[i+1] = byte(0xf0 | (addr>>19)&0x07)
buf[i] = byte(addr >> 11)
buf[i+3] = byte(0xf8 | (addr>>8)&0x07)
buf[i+2] = byte(addr)
i += 2
}
}
return i
}
func bcjSPARCFilter(s *xzDecBCJ, buf []byte) int {
var i int
var instr uint32
for i = 0; i+4 <= len(buf); i += 4 {
instr = getBE32(buf[i:])
if instr>>22 == 0x100 || instr>>22 == 0x1ff {
instr <<= 2
instr -= uint32(s.pos + i)
instr >>= 2
instr = (0x40000000 - instr&0x400000) |
0x40000000 | (instr & 0x3FFFFF)
putBE32(instr, buf[i:])
}
}
return i
}
/*
* Apply the selected BCJ filter. Update *pos and s.pos to match the amount
* of data that got filtered.
*/
func bcjApply(s *xzDecBCJ, buf []byte, pos *int) {
var filtered int
buf = buf[*pos:]
switch s.typ {
case idBCJX86:
filtered = bcjX86Filter(s, buf)
case idBCJPowerPC:
filtered = bcjPowerPCFilter(s, buf)
case idBCJIA64:
filtered = bcjIA64Filter(s, buf)
case idBCJARM:
filtered = bcjARMFilter(s, buf)
case idBCJARMThumb:
filtered = bcjARMThumbFilter(s, buf)
case idBCJSPARC:
filtered = bcjSPARCFilter(s, buf)
default:
/* Never reached */
}
*pos += filtered
s.pos += filtered
}
/*
* Flush pending filtered data from temp to the output buffer.
* Move the remaining mixture of possibly filtered and unfiltered
* data to the beginning of temp.
*/
func bcjFlush(s *xzDecBCJ, b *xzBuf) {
var copySize int
copySize = len(b.out) - b.outPos
if copySize > s.temp.filtered {
copySize = s.temp.filtered
}
copy(b.out[b.outPos:], s.temp.buf[:copySize])
b.outPos += copySize
s.temp.filtered -= copySize
copy(s.temp.buf, s.temp.buf[copySize:])
s.temp.buf = s.temp.buf[:len(s.temp.buf)-copySize]
}
/*
* Decode raw stream which has a BCJ filter as the first filter.
*
* The BCJ filter functions are primitive in sense that they process the
* data in chunks of 1-16 bytes. To hide this issue, this function does
* some buffering.
*/
func xzDecBCJRun(s *xzDecBCJ, b *xzBuf, chain func(*xzBuf) xzRet) xzRet {
var outStart int
/*
* Flush pending already filtered data to the output buffer. Return
* immediately if we couldn't flush everything, or if the next
* filter in the chain had already returned xzStreamEnd.
*/
if s.temp.filtered > 0 {
bcjFlush(s, b)
if s.temp.filtered > 0 {
return xzOK
}
if s.ret == xzStreamEnd {
return xzStreamEnd
}
}
/*
* If we have more output space than what is currently pending in
* temp, copy the unfiltered data from temp to the output buffer
* and try to fill the output buffer by decoding more data from the
* next filter in the chain. Apply the BCJ filter on the new data
* in the output buffer. If everything cannot be filtered, copy it
* to temp and rewind the output buffer position accordingly.
*
* This needs to be always run when len(temp.buf) == 0 to handle a special
* case where the output buffer is full and the next filter has no
* more output coming but hasn't returned xzStreamEnd yet.
*/
if len(s.temp.buf) < len(b.out)-b.outPos || len(s.temp.buf) == 0 {
outStart = b.outPos
copy(b.out[b.outPos:], s.temp.buf)
b.outPos += len(s.temp.buf)
s.ret = chain(b)
if s.ret != xzStreamEnd && s.ret != xzOK {
return s.ret
}
bcjApply(s, b.out[:b.outPos], &outStart)
/*
* As an exception, if the next filter returned xzStreamEnd,
* we can do that too, since the last few bytes that remain
* unfiltered are meant to remain unfiltered.
*/
if s.ret == xzStreamEnd {
return xzStreamEnd
}
s.temp.buf = s.temp.bufArray[:b.outPos-outStart]
b.outPos -= len(s.temp.buf)
copy(s.temp.buf, b.out[b.outPos:])
/*
* If there wasn't enough input to the next filter to fill
* the output buffer with unfiltered data, there's no point
* to try decoding more data to temp.
*/
if b.outPos+len(s.temp.buf) < len(b.out) {
return xzOK
}
}
/*
* We have unfiltered data in temp. If the output buffer isn't full
* yet, try to fill the temp buffer by decoding more data from the
* next filter. Apply the BCJ filter on temp. Then we hopefully can
* fill the actual output buffer by copying filtered data from temp.
* A mix of filtered and unfiltered data may be left in temp; it will
* be taken care on the next call to this function.
*/
if b.outPos < len(b.out) {
/* Make b.out temporarily point to s.temp. */
s.out = b.out
s.outPos = b.outPos
b.out = s.temp.bufArray[:]
b.outPos = len(s.temp.buf)
s.ret = chain(b)
s.temp.buf = s.temp.bufArray[:b.outPos]
b.out = s.out
b.outPos = s.outPos
if s.ret != xzOK && s.ret != xzStreamEnd {
return s.ret
}
bcjApply(s, s.temp.buf, &s.temp.filtered)
/*
* If the next filter returned xzStreamEnd, we mark that
* everything is filtered, since the last unfiltered bytes
* of the stream are meant to be left as is.
*/
if s.ret == xzStreamEnd {
s.temp.filtered = len(s.temp.buf)
}
bcjFlush(s, b)
if s.temp.filtered > 0 {
return xzOK
}
}
return s.ret
}
/*
* Allocate memory for BCJ decoders. xzDecBCJReset must be used before
* calling xzDecBCJRun.
*/
func xzDecBCJCreate() *xzDecBCJ {
return new(xzDecBCJ)
}
/*
* Decode the Filter ID of a BCJ filter and check the start offset is
* valid. Returns xzOK if the given Filter ID and offset is
* supported. Otherwise xzOptionsError is returned.
*/
func xzDecBCJReset(s *xzDecBCJ, id xzFilterID, offset int) xzRet {
switch id {
case idBCJX86:
case idBCJPowerPC:
case idBCJIA64:
case idBCJARM:
case idBCJARMThumb:
case idBCJSPARC:
default:
/* Unsupported Filter ID */
return xzOptionsError
}
// check offset is a multiple of alignment
switch id {
case idBCJPowerPC, idBCJARM, idBCJSPARC:
if offset%4 != 0 {
return xzOptionsError
}
case idBCJIA64:
if offset%16 != 0 {
return xzOptionsError
}
case idBCJARMThumb:
if offset%2 != 0 {
return xzOptionsError
}
}
s.typ = id
s.ret = xzOK
s.pos = offset
s.x86PrevMask = 0
s.temp.filtered = 0
s.temp.buf = nil
return xzOK
}
vendor/github.com/therootcompany/xz/dec_delta.go
Generated Vendored
-55
View File
@@ -1,55 +0,0 @@
/*
* Delta decoder
*
* Author: Lasse Collin <lasse.collin@tukaani.org>
*
* Translation to Go: Michael Cross <https://github.com/xi2>
*
* This file has been put into the public domain.
* You can do whatever you want with this file.
*/
package xz
type xzDecDelta struct {
delta [256]byte
pos byte
distance int // in range [1, 256]
}
/*
* Decode raw stream which has a delta filter as the first filter.
*/
func xzDecDeltaRun(s *xzDecDelta, b *xzBuf, chain func(*xzBuf) xzRet) xzRet {
outStart := b.outPos
ret := chain(b)
for i := outStart; i < b.outPos; i++ {
tmp := b.out[i] + s.delta[byte(s.distance+int(s.pos))]
s.delta[s.pos] = tmp
b.out[i] = tmp
s.pos--
}
return ret
}
/*
* Allocate memory for a delta decoder. xzDecDeltaReset must be used
* before calling xzDecDeltaRun.
*/
func xzDecDeltaCreate() *xzDecDelta {
return new(xzDecDelta)
}
/*
* Returns xzOK if the given distance is valid. Otherwise
* xzOptionsError is returned.
*/
func xzDecDeltaReset(s *xzDecDelta, distance int) xzRet {
if distance < 1 || distance > 256 {
return xzOptionsError
}
s.delta = [256]byte{}
s.pos = 0
s.distance = distance
return xzOK
}
vendor/github.com/therootcompany/xz/dec_lzma2.go
Generated Vendored
-1235
View File
File diff suppressed because it is too large Load Diff
vendor/github.com/therootcompany/xz/dec_stream.go
Generated Vendored
-932
View File
@@ -1,932 +0,0 @@
/*
* .xz Stream decoder
*
* Author: Lasse Collin <lasse.collin@tukaani.org>
*
* Translation to Go: Michael Cross <https://github.com/xi2>
*
* This file has been put into the public domain.
* You can do whatever you want with this file.
*/
package xz
import (
"bytes"
"crypto/sha256"
"hash"
"hash/crc32"
"hash/crc64"
)
/* from linux/lib/xz/xz_stream.h **************************************/
/*
* See the .xz file format specification at
* http://tukaani.org/xz/xz-file-format.txt
* to understand the container format.
*/
const (
streamHeaderSize = 12
headerMagic = "\xfd7zXZ\x00"
footerMagic = "YZ"
)
/*
* Variable-length integer can hold a 63-bit unsigned integer or a special
* value indicating that the value is unknown.
*/
type vliType uint64
const (
vliUnknown vliType = ^vliType(0)
/* Maximum encoded size of a VLI */
vliBytesMax = 8 * 8 / 7 // (Sizeof(vliType) * 8 / 7)
)
/* from linux/lib/xz/xz_dec_stream.c **********************************/
/* Hash used to validate the Index field */
type xzDecHash struct {
unpadded vliType
uncompressed vliType
sha256 hash.Hash
}
// type of xzDec.sequence
type xzDecSeq int
const (
seqStreamHeader xzDecSeq = iota
seqBlockStart
seqBlockHeader
seqBlockUncompress
seqBlockPadding
seqBlockCheck
seqIndex
seqIndexPadding
seqIndexCRC32
seqStreamFooter
)
// type of xzDec.index.sequence
type xzDecIndexSeq int
const (
seqIndexCount xzDecIndexSeq = iota
seqIndexUnpadded
seqIndexUncompressed
)
/**
* xzDec - Opaque type to hold the XZ decoder state
*/
type xzDec struct {
/* Position in decMain */
sequence xzDecSeq
/* Position in variable-length integers and Check fields */
pos int
/* Variable-length integer decoded by decVLI */
vli vliType
/* Saved inPos and outPos */
inStart int
outStart int
/* CRC32 checksum hash used in Index */
crc32 hash.Hash
/* Hashes used in Blocks */
checkCRC32 hash.Hash
checkCRC64 hash.Hash
checkSHA256 hash.Hash
/* for checkTypes CRC32/CRC64/SHA256, check is one of the above 3 hashes */
check hash.Hash
/* Embedded stream header struct containing CheckType */
*Header
/*
* True if the next call to xzDecRun is allowed to return
* xzBufError.
*/
allowBufError bool
/* Information stored in Block Header */
blockHeader struct {
/*
* Value stored in the Compressed Size field, or
* vliUnknown if Compressed Size is not present.
*/
compressed vliType
/*
* Value stored in the Uncompressed Size field, or
* vliUnknown if Uncompressed Size is not present.
*/
uncompressed vliType
/* Size of the Block Header field */
size int
}
/* Information collected when decoding Blocks */
block struct {
/* Observed compressed size of the current Block */
compressed vliType
/* Observed uncompressed size of the current Block */
uncompressed vliType
/* Number of Blocks decoded so far */
count vliType
/*
* Hash calculated from the Block sizes. This is used to
* validate the Index field.
*/
hash xzDecHash
}
/* Variables needed when verifying the Index field */
index struct {
/* Position in decIndex */
sequence xzDecIndexSeq
/* Size of the Index in bytes */
size vliType
/* Number of Records (matches block.count in valid files) */
count vliType
/*
* Hash calculated from the Records (matches block.hash in
* valid files).
*/
hash xzDecHash
}
/*
* Temporary buffer needed to hold Stream Header, Block Header,
* and Stream Footer. The Block Header is the biggest (1 KiB)
* so we reserve space according to that. bufArray has to be aligned
* to a multiple of four bytes; the variables before it
* should guarantee this.
*/
temp struct {
pos int
buf []byte // slice buf will be backed by bufArray
bufArray [1024]byte
}
// chain is the function (or to be more precise, closure) which
// does the decompression and will call into the lzma2 and other
// filter code as needed. It is constructed by decBlockHeader
chain func(b *xzBuf) xzRet
// lzma2 holds the state of the last filter (which must be LZMA2)
lzma2 *xzDecLZMA2
// pointers to allocated BCJ/Delta filters
bcjs []*xzDecBCJ
deltas []*xzDecDelta
// number of currently in use BCJ/Delta filters from the above
bcjsUsed int
deltasUsed int
}
/* Sizes of the Check field with different Check IDs */
var checkSizes = [...]byte{
0,
4, 4, 4,
8, 8, 8,
16, 16, 16,
32, 32, 32,
64, 64, 64,
}
/*
* Fill s.temp by copying data starting from b.in[b.inPos]. Caller
* must have set s.temp.pos to indicate how much data we are supposed
* to copy into s.temp.buf. Return true once s.temp.pos has reached
* len(s.temp.buf).
*/
func fillTemp(s *xzDec, b *xzBuf) bool {
copySize := len(b.in) - b.inPos
tempRemaining := len(s.temp.buf) - s.temp.pos
if copySize > tempRemaining {
copySize = tempRemaining
}
copy(s.temp.buf[s.temp.pos:], b.in[b.inPos:])
b.inPos += copySize
s.temp.pos += copySize
if s.temp.pos == len(s.temp.buf) {
s.temp.pos = 0
return true
}
return false
}
/* Decode a variable-length integer (little-endian base-128 encoding) */
func decVLI(s *xzDec, in []byte, inPos *int) xzRet {
var byte byte
if s.pos == 0 {
s.vli = 0
}
for *inPos < len(in) {
byte = in[*inPos]
*inPos++
s.vli |= vliType(byte&0x7f) << uint(s.pos)
if byte&0x80 == 0 {
/* Don't allow non-minimal encodings. */
if byte == 0 && s.pos != 0 {
return xzDataError
}
s.pos = 0
return xzStreamEnd
}
s.pos += 7
if s.pos == 7*vliBytesMax {
return xzDataError
}
}
return xzOK
}
/*
* Decode the Compressed Data field from a Block. Update and validate
* the observed compressed and uncompressed sizes of the Block so that
* they don't exceed the values possibly stored in the Block Header
* (validation assumes that no integer overflow occurs, since vliType
* is uint64). Update s.check if presence of the CRC32/CRC64/SHA256
* field was indicated in Stream Header.
*
* Once the decoding is finished, validate that the observed sizes match
* the sizes possibly stored in the Block Header. Update the hash and
* Block count, which are later used to validate the Index field.
*/
func decBlock(s *xzDec, b *xzBuf) xzRet {
var ret xzRet
s.inStart = b.inPos
s.outStart = b.outPos
ret = s.chain(b)
s.block.compressed += vliType(b.inPos - s.inStart)
s.block.uncompressed += vliType(b.outPos - s.outStart)
/*
* There is no need to separately check for vliUnknown since
* the observed sizes are always smaller than vliUnknown.
*/
if s.block.compressed > s.blockHeader.compressed ||
s.block.uncompressed > s.blockHeader.uncompressed {
return xzDataError
}
switch s.CheckType {
case CheckCRC32, CheckCRC64, CheckSHA256:
_, _ = s.check.Write(b.out[s.outStart:b.outPos])
}
if ret == xzStreamEnd {
if s.blockHeader.compressed != vliUnknown &&
s.blockHeader.compressed != s.block.compressed {
return xzDataError
}
if s.blockHeader.uncompressed != vliUnknown &&
s.blockHeader.uncompressed != s.block.uncompressed {
return xzDataError
}
s.block.hash.unpadded +=
vliType(s.blockHeader.size) + s.block.compressed
s.block.hash.unpadded += vliType(checkSizes[s.CheckType])
s.block.hash.uncompressed += s.block.uncompressed
var buf [2 * 8]byte // 2*Sizeof(vliType)
putLE64(uint64(s.block.hash.unpadded), buf[:])
putLE64(uint64(s.block.hash.uncompressed), buf[8:])
_, _ = s.block.hash.sha256.Write(buf[:])
s.block.count++
}
return ret
}
/* Update the Index size and the CRC32 hash. */
func indexUpdate(s *xzDec, b *xzBuf) {
inUsed := b.inPos - s.inStart
s.index.size += vliType(inUsed)
_, _ = s.crc32.Write(b.in[s.inStart : s.inStart+inUsed])
}
/*
* Decode the Number of Records, Unpadded Size, and Uncompressed Size
* fields from the Index field. That is, Index Padding and CRC32 are not
* decoded by this function.
*
* This can return xzOK (more input needed), xzStreamEnd (everything
* successfully decoded), or xzDataError (input is corrupt).
*/
func decIndex(s *xzDec, b *xzBuf) xzRet {
var ret xzRet
for {
ret = decVLI(s, b.in, &b.inPos)
if ret != xzStreamEnd {
indexUpdate(s, b)
return ret
}
switch s.index.sequence {
case seqIndexCount:
s.index.count = s.vli
/*
* Validate that the Number of Records field
* indicates the same number of Records as
* there were Blocks in the Stream.
*/
if s.index.count != s.block.count {
return xzDataError
}
s.index.sequence = seqIndexUnpadded
case seqIndexUnpadded:
s.index.hash.unpadded += s.vli
s.index.sequence = seqIndexUncompressed
case seqIndexUncompressed:
s.index.hash.uncompressed += s.vli
var buf [2 * 8]byte // 2*Sizeof(vliType)
putLE64(uint64(s.index.hash.unpadded), buf[:])
putLE64(uint64(s.index.hash.uncompressed), buf[8:])
_, _ = s.index.hash.sha256.Write(buf[:])
s.index.count--
s.index.sequence = seqIndexUnpadded
}
if !(s.index.count > 0) {
break
}
}
return xzStreamEnd
}
/*
* Validate that the next 4 bytes match s.crc32.Sum(nil). s.pos must
* be zero when starting to validate the first byte.
*/
func crcValidate(s *xzDec, b *xzBuf) xzRet {
sum := s.crc32.Sum(nil)
// CRC32 - reverse slice
sum[0], sum[1], sum[2], sum[3] = sum[3], sum[2], sum[1], sum[0]
for {
if b.inPos == len(b.in) {
return xzOK
}
if sum[s.pos] != b.in[b.inPos] {
return xzDataError
}
b.inPos++
s.pos++
if !(s.pos < 4) {
break
}
}
s.crc32.Reset()
s.pos = 0
return xzStreamEnd
}
/*
* Validate that the next 4/8/32 bytes match s.check.Sum(nil). s.pos
* must be zero when starting to validate the first byte.
*/
func checkValidate(s *xzDec, b *xzBuf) xzRet {
sum := s.check.Sum(nil)
if s.CheckType == CheckCRC32 || s.CheckType == CheckCRC64 {
// CRC32/64 - reverse slice
for i, j := 0, len(sum)-1; i < j; i, j = i+1, j-1 {
sum[i], sum[j] = sum[j], sum[i]
}
}
for {
if b.inPos == len(b.in) {
return xzOK
}
if sum[s.pos] != b.in[b.inPos] {
return xzDataError
}
b.inPos++
s.pos++
if !(s.pos < len(sum)) {
break
}
}
s.check.Reset()
s.pos = 0
return xzStreamEnd
}
/*
* Skip over the Check field when the Check ID is not supported.
* Returns true once the whole Check field has been skipped over.
*/
func checkSkip(s *xzDec, b *xzBuf) bool {
for s.pos < int(checkSizes[s.CheckType]) {
if b.inPos == len(b.in) {
return false
}
b.inPos++
s.pos++
}
s.pos = 0
return true
}
/* polynomial table used in decStreamHeader below */
var xzCRC64Table = crc64.MakeTable(crc64.ECMA)
/* Decode the Stream Header field (the first 12 bytes of the .xz Stream). */
func decStreamHeader(s *xzDec) xzRet {
if string(s.temp.buf[:len(headerMagic)]) != headerMagic {
return xzFormatError
}
if crc32.ChecksumIEEE(s.temp.buf[len(headerMagic):len(headerMagic)+2]) !=
getLE32(s.temp.buf[len(headerMagic)+2:]) {
return xzDataError
}
if s.temp.buf[len(headerMagic)] != 0 {
return xzOptionsError
}
/*
* Of integrity checks, we support none (Check ID = 0),
* CRC32 (Check ID = 1), CRC64 (Check ID = 4) and SHA256 (Check ID = 10)
* However, we will accept other check types too, but then the check
* won't be verified and a warning (xzUnsupportedCheck) will be given.
*/
s.CheckType = CheckID(s.temp.buf[len(headerMagic)+1])
if s.CheckType > checkMax {
return xzOptionsError
}
switch s.CheckType {
case CheckNone:
// CheckNone: no action needed
case CheckCRC32:
if s.checkCRC32 == nil {
s.checkCRC32 = crc32.NewIEEE()
} else {
s.checkCRC32.Reset()
}
s.check = s.checkCRC32
case CheckCRC64:
if s.checkCRC64 == nil {
s.checkCRC64 = crc64.New(xzCRC64Table)
} else {
s.checkCRC64.Reset()
}
s.check = s.checkCRC64
case CheckSHA256:
if s.checkSHA256 == nil {
s.checkSHA256 = sha256.New()
} else {
s.checkSHA256.Reset()
}
s.check = s.checkSHA256
default:
return xzUnsupportedCheck
}
return xzOK
}
/* Decode the Stream Footer field (the last 12 bytes of the .xz Stream) */
func decStreamFooter(s *xzDec) xzRet {
if string(s.temp.buf[10:10+len(footerMagic)]) != footerMagic {
return xzDataError
}
if crc32.ChecksumIEEE(s.temp.buf[4:10]) != getLE32(s.temp.buf) {
return xzDataError
}
/*
* Validate Backward Size. Note that we never added the size of the
* Index CRC32 field to s->index.size, thus we use s->index.size / 4
* instead of s->index.size / 4 - 1.
*/
if s.index.size>>2 != vliType(getLE32(s.temp.buf[4:])) {
return xzDataError
}
if s.temp.buf[8] != 0 || CheckID(s.temp.buf[9]) != s.CheckType {
return xzDataError
}
/*
* Use xzStreamEnd instead of xzOK to be more convenient
* for the caller.
*/
return xzStreamEnd
}
/* Decode the Block Header and initialize the filter chain. */
func decBlockHeader(s *xzDec) xzRet {
var ret xzRet
/*
* Validate the CRC32. We know that the temp buffer is at least
* eight bytes so this is safe.
*/
crc := getLE32(s.temp.buf[len(s.temp.buf)-4:])
s.temp.buf = s.temp.buf[:len(s.temp.buf)-4]
if crc32.ChecksumIEEE(s.temp.buf) != crc {
return xzDataError
}
s.temp.pos = 2
/*
* Catch unsupported Block Flags.
*/
if s.temp.buf[1]&0x3C != 0 {
return xzOptionsError
}
/* Compressed Size */
if s.temp.buf[1]&0x40 != 0 {
if decVLI(s, s.temp.buf, &s.temp.pos) != xzStreamEnd {
return xzDataError
}
if s.vli >= 1<<63-8 {
// the whole block must stay smaller than 2^63 bytes
// the block header cannot be smaller than 8 bytes
return xzDataError
}
if s.vli == 0 {
// compressed size must be non-zero
return xzDataError
}
s.blockHeader.compressed = s.vli
} else {
s.blockHeader.compressed = vliUnknown
}
/* Uncompressed Size */
if s.temp.buf[1]&0x80 != 0 {
if decVLI(s, s.temp.buf, &s.temp.pos) != xzStreamEnd {
return xzDataError
}
s.blockHeader.uncompressed = s.vli
} else {
s.blockHeader.uncompressed = vliUnknown
}
// get total number of filters (1-4)
filterTotal := int(s.temp.buf[1]&0x03) + 1
// slice to hold decoded filters
filterList := make([]struct {
id xzFilterID
props uint32
}, filterTotal)
// decode the non-last filters which cannot be LZMA2
for i := 0; i < filterTotal-1; i++ {
/* Valid Filter Flags always take at least two bytes. */
if len(s.temp.buf)-s.temp.pos < 2 {
return xzDataError
}
s.temp.pos += 2
switch id := xzFilterID(s.temp.buf[s.temp.pos-2]); id {
case idDelta:
// delta filter
if s.temp.buf[s.temp.pos-1] != 0x01 {
return xzOptionsError
}
/* Filter Properties contains distance - 1 */
if len(s.temp.buf)-s.temp.pos < 1 {
return xzDataError
}
props := uint32(s.temp.buf[s.temp.pos])
s.temp.pos++
filterList[i] = struct {
id xzFilterID
props uint32
}{id: id, props: props}
case idBCJX86, idBCJPowerPC, idBCJIA64,
idBCJARM, idBCJARMThumb, idBCJSPARC:
// bcj filter
var props uint32
switch s.temp.buf[s.temp.pos-1] {
case 0x00:
props = 0
case 0x04:
if len(s.temp.buf)-s.temp.pos < 4 {
return xzDataError
}
props = getLE32(s.temp.buf[s.temp.pos:])
s.temp.pos += 4
default:
return xzOptionsError
}
filterList[i] = struct {
id xzFilterID
props uint32
}{id: id, props: props}
default:
return xzOptionsError
}
}
/*
* decode the last filter which must be LZMA2
*/
if len(s.temp.buf)-s.temp.pos < 2 {
return xzDataError
}
/* Filter ID = LZMA2 */
if xzFilterID(s.temp.buf[s.temp.pos]) != idLZMA2 {
return xzOptionsError
}
s.temp.pos++
/* Size of Properties = 1-byte Filter Properties */
if s.temp.buf[s.temp.pos] != 0x01 {
return xzOptionsError
}
s.temp.pos++
/* Filter Properties contains LZMA2 dictionary size. */
if len(s.temp.buf)-s.temp.pos < 1 {
return xzDataError
}
props := uint32(s.temp.buf[s.temp.pos])
s.temp.pos++
filterList[filterTotal-1] = struct {
id xzFilterID
props uint32
}{id: idLZMA2, props: props}
/*
* Process the filter list and create s.chain, going from last
* filter (LZMA2) to first filter
*
* First, LZMA2.
*/
ret = xzDecLZMA2Reset(s.lzma2, byte(filterList[filterTotal-1].props))
if ret != xzOK {
return ret
}
s.chain = func(b *xzBuf) xzRet {
return xzDecLZMA2Run(s.lzma2, b)
}
/*
* Now the non-last filters
*/
for i := filterTotal - 2; i >= 0; i-- {
switch id := filterList[i].id; id {
case idDelta:
// delta filter
var delta *xzDecDelta
if s.deltasUsed < len(s.deltas) {
delta = s.deltas[s.deltasUsed]
} else {
delta = xzDecDeltaCreate()
s.deltas = append(s.deltas, delta)
}
s.deltasUsed++
ret = xzDecDeltaReset(delta, int(filterList[i].props)+1)
if ret != xzOK {
return ret
}
chain := s.chain
s.chain = func(b *xzBuf) xzRet {
return xzDecDeltaRun(delta, b, chain)
}
case idBCJX86, idBCJPowerPC, idBCJIA64,
idBCJARM, idBCJARMThumb, idBCJSPARC:
// bcj filter
var bcj *xzDecBCJ
if s.bcjsUsed < len(s.bcjs) {
bcj = s.bcjs[s.bcjsUsed]
} else {
bcj = xzDecBCJCreate()
s.bcjs = append(s.bcjs, bcj)
}
s.bcjsUsed++
ret = xzDecBCJReset(bcj, id, int(filterList[i].props))
if ret != xzOK {
return ret
}
chain := s.chain
s.chain = func(b *xzBuf) xzRet {
return xzDecBCJRun(bcj, b, chain)
}
}
}
/* The rest must be Header Padding. */
for s.temp.pos < len(s.temp.buf) {
if s.temp.buf[s.temp.pos] != 0x00 {
return xzOptionsError
}
s.temp.pos++
}
s.temp.pos = 0
s.block.compressed = 0
s.block.uncompressed = 0
return xzOK
}
func decMain(s *xzDec, b *xzBuf) xzRet {
var ret xzRet
/*
* Store the start position for the case when we are in the middle
* of the Index field.
*/
s.inStart = b.inPos
for {
switch s.sequence {
case seqStreamHeader:
/*
* Stream Header is copied to s.temp, and then
* decoded from there. This way if the caller
* gives us only little input at a time, we can
* still keep the Stream Header decoding code
* simple. Similar approach is used in many places
* in this file.
*/
if !fillTemp(s, b) {
return xzOK
}
/*
* If decStreamHeader returns
* xzUnsupportedCheck, it is still possible
* to continue decoding. Thus, update s.sequence
* before calling decStreamHeader.
*/
s.sequence = seqBlockStart
ret = decStreamHeader(s)
if ret != xzOK {
return ret
}
fallthrough
case seqBlockStart:
/* We need one byte of input to continue. */
if b.inPos == len(b.in) {
return xzOK
}
/* See if this is the beginning of the Index field. */
if b.in[b.inPos] == 0 {
s.inStart = b.inPos
b.inPos++
s.sequence = seqIndex
break
}
/*
* Calculate the size of the Block Header and
* prepare to decode it.
*/
s.blockHeader.size = (int(b.in[b.inPos]) + 1) * 4
s.temp.buf = s.temp.bufArray[:s.blockHeader.size]
s.temp.pos = 0
s.sequence = seqBlockHeader
fallthrough
case seqBlockHeader:
if !fillTemp(s, b) {
return xzOK
}
ret = decBlockHeader(s)
if ret != xzOK {
return ret
}
s.sequence = seqBlockUncompress
fallthrough
case seqBlockUncompress:
ret = decBlock(s, b)
if ret != xzStreamEnd {
return ret
}
s.sequence = seqBlockPadding
fallthrough
case seqBlockPadding:
/*
* Size of Compressed Data + Block Padding
* must be a multiple of four. We don't need
* s->block.compressed for anything else
* anymore, so we use it here to test the size
* of the Block Padding field.
*/
for s.block.compressed&3 != 0 {
if b.inPos == len(b.in) {
return xzOK
}
if b.in[b.inPos] != 0 {
return xzDataError
}
b.inPos++
s.block.compressed++
}
s.sequence = seqBlockCheck
fallthrough
case seqBlockCheck:
switch s.CheckType {
case CheckCRC32, CheckCRC64, CheckSHA256:
ret = checkValidate(s, b)
if ret != xzStreamEnd {
return ret
}
default:
if !checkSkip(s, b) {
return xzOK
}
}
s.sequence = seqBlockStart
case seqIndex:
ret = decIndex(s, b)
if ret != xzStreamEnd {
return ret
}
s.sequence = seqIndexPadding
fallthrough
case seqIndexPadding:
for (s.index.size+vliType(b.inPos-s.inStart))&3 != 0 {
if b.inPos == len(b.in) {
indexUpdate(s, b)
return xzOK
}
if b.in[b.inPos] != 0 {
return xzDataError
}
b.inPos++
}
/* Finish the CRC32 value and Index size. */
indexUpdate(s, b)
/* Compare the hashes to validate the Index field. */
if !bytes.Equal(
s.block.hash.sha256.Sum(nil), s.index.hash.sha256.Sum(nil)) {
return xzDataError
}
s.sequence = seqIndexCRC32
fallthrough
case seqIndexCRC32:
ret = crcValidate(s, b)
if ret != xzStreamEnd {
return ret
}
s.temp.buf = s.temp.bufArray[:streamHeaderSize]
s.sequence = seqStreamFooter
fallthrough
case seqStreamFooter:
if !fillTemp(s, b) {
return xzOK
}
return decStreamFooter(s)
}
}
/* Never reached */
}
/**
* xzDecRun - Run the XZ decoder
* @s: Decoder state allocated using xzDecInit
* @b: Input and output buffers
*
* See xzRet for details of return values.
*
* xzDecRun is a wrapper for decMain to handle some special cases.
*
* We must return xzBufError when it seems clear that we are not
* going to make any progress anymore. This is to prevent the caller
* from calling us infinitely when the input file is truncated or
* otherwise corrupt. Since zlib-style API allows that the caller
* fills the input buffer only when the decoder doesn't produce any
* new output, we have to be careful to avoid returning xzBufError
* too easily: xzBufError is returned only after the second
* consecutive call to xzDecRun that makes no progress.
*/
func xzDecRun(s *xzDec, b *xzBuf) xzRet {
inStart := b.inPos
outStart := b.outPos
ret := decMain(s, b)
if ret == xzOK && inStart == b.inPos && outStart == b.outPos {
if s.allowBufError {
ret = xzBufError
}
s.allowBufError = true
} else {
s.allowBufError = false
}
return ret
}
/**
* xzDecInit - Allocate and initialize a XZ decoder state
* @dictMax: Maximum size of the LZMA2 dictionary (history buffer) for
* decoding. LZMA2 dictionary is always 2^n bytes
* or 2^n + 2^(n-1) bytes (the latter sizes are less common
* in practice), so other values for dictMax don't make sense.
*
* dictMax specifies the maximum allowed dictionary size that xzDecRun
* may allocate once it has parsed the dictionary size from the stream
* headers. This way excessive allocations can be avoided while still
* limiting the maximum memory usage to a sane value to prevent running the
* system out of memory when decompressing streams from untrusted sources.
*
* xzDecInit returns a pointer to an xzDec, which is ready to be used with
* xzDecRun.
*/
func xzDecInit(dictMax uint32, header *Header) *xzDec {
s := new(xzDec)
s.crc32 = crc32.NewIEEE()
s.Header = header
s.block.hash.sha256 = sha256.New()
s.index.hash.sha256 = sha256.New()
s.lzma2 = xzDecLZMA2Create(dictMax)
xzDecReset(s)
return s
}
/**
* xzDecReset - Reset an already allocated decoder state
* @s: Decoder state allocated using xzDecInit
*
* This function can be used to reset the decoder state without
* reallocating memory with xzDecInit.
*/
func xzDecReset(s *xzDec) {
s.sequence = seqStreamHeader
s.allowBufError = false
s.pos = 0
s.crc32.Reset()
s.check = nil
s.CheckType = checkUnset
s.block.compressed = 0
s.block.uncompressed = 0
s.block.count = 0
s.block.hash.unpadded = 0
s.block.hash.uncompressed = 0
s.block.hash.sha256.Reset()
s.index.sequence = seqIndexCount
s.index.size = 0
s.index.count = 0
s.index.hash.unpadded = 0
s.index.hash.uncompressed = 0
s.index.hash.sha256.Reset()
s.temp.pos = 0
s.temp.buf = s.temp.bufArray[:streamHeaderSize]
s.chain = nil
s.bcjsUsed = 0
s.deltasUsed = 0
}
vendor/github.com/therootcompany/xz/dec_util.go
Generated Vendored
-52
View File
@@ -1,52 +0,0 @@
/*
* XZ decompressor utility functions
*
* Author: Michael Cross <https://github.com/xi2>
*
* This file has been put into the public domain.
* You can do whatever you want with this file.
*/
package xz
func getLE32(buf []byte) uint32 {
return uint32(buf[0]) |
uint32(buf[1])<<8 |
uint32(buf[2])<<16 |
uint32(buf[3])<<24
}
func getBE32(buf []byte) uint32 {
return uint32(buf[0])<<24 |
uint32(buf[1])<<16 |
uint32(buf[2])<<8 |
uint32(buf[3])
}
func putLE32(val uint32, buf []byte) {
buf[0] = byte(val)
buf[1] = byte(val >> 8)
buf[2] = byte(val >> 16)
buf[3] = byte(val >> 24)
return
}
func putBE32(val uint32, buf []byte) {
buf[0] = byte(val >> 24)
buf[1] = byte(val >> 16)
buf[2] = byte(val >> 8)
buf[3] = byte(val)
return
}
func putLE64(val uint64, buf []byte) {
buf[0] = byte(val)
buf[1] = byte(val >> 8)
buf[2] = byte(val >> 16)
buf[3] = byte(val >> 24)
buf[4] = byte(val >> 32)
buf[5] = byte(val >> 40)
buf[6] = byte(val >> 48)
buf[7] = byte(val >> 56)
return
}
vendor/github.com/therootcompany/xz/dec_xz.go
Generated Vendored
-124
View File
@@ -1,124 +0,0 @@
/*
* XZ decompressor
*
* Authors: Lasse Collin <lasse.collin@tukaani.org>
* Igor Pavlov <http://7-zip.org/>
*
* Translation to Go: Michael Cross <https://github.com/xi2>
*
* This file has been put into the public domain.
* You can do whatever you want with this file.
*/
package xz
/* from linux/include/linux/xz.h **************************************/
/**
* xzRet - Return codes
* @xzOK: Everything is OK so far. More input or more
* output space is required to continue.
* @xzStreamEnd: Operation finished successfully.
* @xzUnSupportedCheck: Integrity check type is not supported. Decoding
* is still possible by simply calling xzDecRun
* again.
* @xzMemlimitError: A bigger LZMA2 dictionary would be needed than
* allowed by the dictMax argument given to
* xzDecInit.
* @xzFormatError: File format was not recognized (wrong magic
* bytes).
* @xzOptionsError: This implementation doesn't support the requested
* compression options. In the decoder this means
* that the header CRC32 matches, but the header
* itself specifies something that we don't support.
* @xzDataError: Compressed data is corrupt.
* @xzBufError: Cannot make any progress.
*
* xzBufError is returned when two consecutive calls to XZ code cannot
* consume any input and cannot produce any new output. This happens
* when there is no new input available, or the output buffer is full
* while at least one output byte is still pending. Assuming your code
* is not buggy, you can get this error only when decoding a
* compressed stream that is truncated or otherwise corrupt.
*/
type xzRet int
const (
xzOK xzRet = iota
xzStreamEnd
xzUnsupportedCheck
xzMemlimitError
xzFormatError
xzOptionsError
xzDataError
xzBufError
)
/**
* xzBuf - Passing input and output buffers to XZ code
* @in: Input buffer.
* @inPos: Current position in the input buffer. This must not exceed
* input buffer size.
* @out: Output buffer.
* @outPos: Current position in the output buffer. This must not exceed
* output buffer size.
*
* Only the contents of the output buffer from out[outPos] onward, and
* the variables inPos and outPos are modified by the XZ code.
*/
type xzBuf struct {
in []byte
inPos int
out []byte
outPos int
}
/* All XZ filter IDs */
type xzFilterID int64
const (
idDelta xzFilterID = 0x03
idBCJX86 xzFilterID = 0x04
idBCJPowerPC xzFilterID = 0x05
idBCJIA64 xzFilterID = 0x06
idBCJARM xzFilterID = 0x07
idBCJARMThumb xzFilterID = 0x08
idBCJSPARC xzFilterID = 0x09
idLZMA2 xzFilterID = 0x21
)
// CheckID is the type of the data integrity check in an XZ stream
// calculated from the uncompressed data.
type CheckID int
func (id CheckID) String() string {
switch id {
case CheckNone:
return "None"
case CheckCRC32:
return "CRC32"
case CheckCRC64:
return "CRC64"
case CheckSHA256:
return "SHA256"
default:
return "Unknown"
}
}
const (
CheckNone CheckID = 0x00
CheckCRC32 CheckID = 0x01
CheckCRC64 CheckID = 0x04
CheckSHA256 CheckID = 0x0A
checkMax CheckID = 0x0F
checkUnset CheckID = -1
)
// An XZ stream contains a stream header which holds information about
// the stream. That information is exposed as fields of the
// Reader. Currently it contains only the stream's data integrity
// check type.
type Header struct {
CheckType CheckID // type of the stream's data integrity check
}
vendor/github.com/therootcompany/xz/doc.go
Generated Vendored
-35
View File
@@ -1,35 +0,0 @@
// Package xz implements XZ decompression natively in Go.
//
// Usage
//
// For ease of use, this package is designed to have a similar API to
// compress/gzip. See the examples for further details.
//
// Implementation
//
// This package is a translation from C to Go of XZ Embedded
// (http://tukaani.org/xz/embedded.html) with enhancements made so as
// to implement all mandatory and optional parts of the XZ file format
// specification v1.0.4. It supports all filters and block check
// types, supports multiple streams, and performs index verification
// using SHA-256 as recommended by the specification.
//
// Speed
//
// On the author's Intel Ivybridge i5, decompression speed is about
// half that of the standard XZ Utils (tested with a recent linux
// kernel tarball).
//
// Thanks
//
// Thanks are due to Lasse Collin and Igor Pavlov, the authors of XZ
// Embedded, on whose code package xz is based. It would not exist
// without their decision to allow others to modify and reuse their
// code.
//
// Bug reports
//
// For bug reports relating to this package please contact the author
// through https://github.com/xi2/xz/issues, and not the authors of XZ
// Embedded.
package xz
vendor/github.com/therootcompany/xz/reader.go
Generated Vendored
-256
View File
@@ -1,256 +0,0 @@
/*
* Package xz Go Reader API
*
* Author: Michael Cross <https://github.com/xi2>
*
* This file has been put into the public domain.
* You can do whatever you want with this file.
*/
package xz
import (
"errors"
"io"
)
// Package specific errors.
var (
ErrUnsupportedCheck = errors.New("xz: integrity check type not supported")
ErrMemlimit = errors.New("xz: LZMA2 dictionary size exceeds max")
ErrFormat = errors.New("xz: file format not recognized")
ErrOptions = errors.New("xz: compression options not supported")
ErrData = errors.New("xz: data is corrupt")
ErrBuf = errors.New("xz: data is truncated or corrupt")
)
// DefaultDictMax is the default maximum dictionary size in bytes used
// by the decoder. This value is sufficient to decompress files
// created with XZ Utils "xz -9".
const DefaultDictMax = 1 << 26 // 64 MiB
// inBufSize is the input buffer size used by the decoder.
const inBufSize = 1 << 13 // 8 KiB
// A Reader is an io.Reader that can be used to retrieve uncompressed
// data from an XZ file.
//
// In general, an XZ file can be a concatenation of other XZ
// files. Reads from the Reader return the concatenation of the
// uncompressed data of each.
type Reader struct {
Header
r io.Reader // the wrapped io.Reader
multistream bool // true if reader is in multistream mode
rEOF bool // true after io.EOF received on r
dEOF bool // true after decoder has completed
padding int // bytes of stream padding read (or -1)
in [inBufSize]byte // backing array for buf.in
buf *xzBuf // decoder input/output buffers
dec *xzDec // decoder state
err error // the result of the last decoder call
}
// NewReader creates a new Reader reading from r. The decompressor
// will use an LZMA2 dictionary size up to dictMax bytes in
// size. Passing a value of zero sets dictMax to DefaultDictMax. If
// an individual XZ stream requires a dictionary size greater than
// dictMax in order to decompress, Read will return ErrMemlimit.
//
// If NewReader is passed a value of nil for r then a Reader is
// created such that all read attempts will return io.EOF. This is
// useful if you just want to allocate memory for a Reader which will
// later be initialized with Reset.
//
// Due to internal buffering, the Reader may read more data than
// necessary from r.
func NewReader(r io.Reader, dictMax uint32) (*Reader, error) {
if dictMax == 0 {
dictMax = DefaultDictMax
}
z := &Reader{
r: r,
multistream: true,
padding: -1,
buf: &xzBuf{},
}
if r == nil {
z.rEOF, z.dEOF = true, true
}
z.dec = xzDecInit(dictMax, &z.Header)
var err error
if r != nil {
_, err = z.Read(nil) // read stream header
}
return z, err
}
// decode is a wrapper around xzDecRun that additionally handles
// stream padding. It treats the padding as a kind of stream that
// decodes to nothing.
//
// When decoding padding, z.padding >= 0
// When decoding a real stream, z.padding == -1
func (z *Reader) decode() (ret xzRet) {
if z.padding >= 0 {
// read all padding in input buffer
for z.buf.inPos < len(z.buf.in) &&
z.buf.in[z.buf.inPos] == 0 {
z.buf.inPos++
z.padding++
}
switch {
case z.buf.inPos == len(z.buf.in) && z.rEOF:
// case: out of padding. no more input data available
if z.padding%4 != 0 {
ret = xzDataError
} else {
ret = xzStreamEnd
}
case z.buf.inPos == len(z.buf.in):
// case: read more padding next loop iteration
ret = xzOK
default:
// case: out of padding. more input data available
if z.padding%4 != 0 {
ret = xzDataError
} else {
xzDecReset(z.dec)
ret = xzStreamEnd
}
}
} else {
ret = xzDecRun(z.dec, z.buf)
}
return
}
func (z *Reader) Read(p []byte) (n int, err error) {
// restore err
err = z.err
// set decoder output buffer to p
z.buf.out = p
z.buf.outPos = 0
for {
// update n
n = z.buf.outPos
// if last call to decoder ended with an error, return that error
if err != nil {
break
}
// if decoder has finished, return with err == io.EOF
if z.dEOF {
err = io.EOF
break
}
// if p full, return with err == nil, unless we have not yet
// read the stream header with Read(nil)
if n == len(p) && z.CheckType != checkUnset {
break
}
// if needed, read more data from z.r
if z.buf.inPos == len(z.buf.in) && !z.rEOF {
rn, e := z.r.Read(z.in[:])
if e != nil && e != io.EOF {
// read error
err = e
break
}
if e == io.EOF {
z.rEOF = true
}
// set new input buffer in z.buf
z.buf.in = z.in[:rn]
z.buf.inPos = 0
}
// decode more data
ret := z.decode()
switch ret {
case xzOK:
// no action needed
case xzStreamEnd:
if z.padding >= 0 {
z.padding = -1
if !z.multistream || z.rEOF {
z.dEOF = true
}
} else {
z.padding = 0
}
case xzUnsupportedCheck:
err = ErrUnsupportedCheck
case xzMemlimitError:
err = ErrMemlimit
case xzFormatError:
err = ErrFormat
case xzOptionsError:
err = ErrOptions
case xzDataError:
err = ErrData
case xzBufError:
err = ErrBuf
}
// save err
z.err = err
}
return
}
// Multistream controls whether the reader is operating in multistream
// mode.
//
// If enabled (the default), the Reader expects the input to be a
// sequence of XZ streams, possibly interspersed with stream padding,
// which it reads one after another. The effect is that the
// concatenation of a sequence of XZ streams or XZ files is
// treated as equivalent to the compressed result of the concatenation
// of the sequence. This is standard behaviour for XZ readers.
//
// Calling Multistream(false) disables this behaviour; disabling the
// behaviour can be useful when reading file formats that distinguish
// individual XZ streams. In this mode, when the Reader reaches the
// end of the stream, Read returns io.EOF. To start the next stream,
// call z.Reset(nil) followed by z.Multistream(false). If there is no
// next stream, z.Reset(nil) will return io.EOF.
func (z *Reader) Multistream(ok bool) {
z.multistream = ok
}
// Reset, for non-nil values of io.Reader r, discards the Reader z's
// state and makes it equivalent to the result of its original state
// from NewReader, but reading from r instead. This permits reusing a
// Reader rather than allocating a new one.
//
// If you wish to leave r unchanged use z.Reset(nil). This keeps r
// unchanged and ensures internal buffering is preserved. If the
// Reader was at the end of a stream it is then ready to read any
// follow on streams. If there are no follow on streams z.Reset(nil)
// returns io.EOF. If the Reader was not at the end of a stream then
// z.Reset(nil) does nothing.
func (z *Reader) Reset(r io.Reader) error {
switch {
case r == nil:
z.multistream = true
if !z.dEOF {
return nil
}
if z.rEOF {
return io.EOF
}
z.dEOF = false
_, err := z.Read(nil) // read stream header
return err
default:
z.r = r
z.multistream = true
z.rEOF = false
z.dEOF = false
z.padding = -1
z.buf.in = nil
z.buf.inPos = 0
xzDecReset(z.dec)
z.err = nil
_, err := z.Read(nil) // read stream header
return err
}
}
vendor/modules.txt
Vendored
+2 -4
View File
@@ -1,4 +1,4 @@
# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9
# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023141925-9205222a98bb
## explicit; go 1.26.0
git.novox.be/novox/mesh-sdk/go
git.novox.be/novox/mesh-sdk/go/announce
@@ -8,6 +8,7 @@ git.novox.be/novox/mesh-sdk/go/catalogue
git.novox.be/novox/mesh-sdk/go/compose
git.novox.be/novox/mesh-sdk/go/declaration
git.novox.be/novox/mesh-sdk/go/delivery
git.novox.be/novox/mesh-sdk/go/delivery/buildrun
git.novox.be/novox/mesh-sdk/go/facts
git.novox.be/novox/mesh-sdk/go/forgekey
git.novox.be/novox/mesh-sdk/go/membership
@@ -95,9 +96,6 @@ github.com/nats-io/nuid
# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5
## explicit; go 1.26.0
github.com/novox/mesh-host/rootsearch
# github.com/therootcompany/xz v1.0.1
## explicit; go 1.15
github.com/therootcompany/xz
# golang.org/x/crypto v0.57.0
## explicit; go 1.26.0
golang.org/x/crypto/bcrypt