Compare commits
50
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cbac703e81 | ||
|
|
ee1d7e5cae | ||
|
|
43cab56452 | ||
|
|
56d6313262 | ||
|
|
7f3ff32361 | ||
|
|
21ac8fd858 | ||
|
|
2819915e8e | ||
|
|
7892019d9e | ||
|
|
e63af0df17 | ||
|
|
961f2e82ea | ||
|
|
e727589e48 | ||
|
|
2f157148c6 | ||
|
|
13be75eb65 | ||
|
|
13711fb5b9 | ||
|
|
fc573f951a | ||
|
|
c0c379af2b | ||
|
|
1ded7dff34 | ||
|
|
50c55a773e | ||
|
|
daf52b3aeb | ||
|
|
c6e234455f | ||
|
|
4dbd4066ca | ||
|
|
45a1783ba9 | ||
|
|
84deb530ee | ||
|
|
b3787e4d8e | ||
|
|
a681227af3 | ||
|
|
d8f36a8fb7 | ||
|
|
f0fd7fe886 | ||
|
|
41a9ff15dd | ||
|
|
ba685b17f5 | ||
|
|
d170491256 | ||
|
|
6bb1c9b768 | ||
|
|
f8daec01d3 | ||
|
|
f3c15e00af | ||
|
|
9038d4b0d3 | ||
|
|
b538397055 | ||
|
|
dd10889209 | ||
|
|
286257d802 | ||
|
|
67d54d9d09 | ||
|
|
7953d2889d | ||
|
|
24347ea5a0 | ||
|
|
5ad36127ee | ||
|
|
a1a6f6493d | ||
|
|
e9f060e0be | ||
|
|
a3bf5da9b1 | ||
|
|
d20b52cb19 | ||
|
|
7ca5a45ecc | ||
|
|
539a1c16cd | ||
|
|
05eccbf22b | ||
|
|
01fc9e1d73 | ||
|
|
10f76cafc4 |
@@ -16,8 +16,7 @@ RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' -o /mesh-builder ./cmd/mes
|
||||
FROM ${ALPINE_BASE}
|
||||
# git to clone what it is asked to build, and the docker client to build and push it. The daemon
|
||||
# is the machine's, reached through its socket — a build machine shares the runtime it was given
|
||||
# rather than running one inside itself. xz to read an upstream release an archive is pinned to
|
||||
# (novox/hq ADR 0316): Go's standard library reads gzip and bzip2 and not xz.
|
||||
RUN apk add --no-cache git docker-cli xz
|
||||
# rather than running one inside itself.
|
||||
RUN apk add --no-cache git docker-cli
|
||||
COPY --from=build /mesh-builder /usr/local/bin/mesh-builder
|
||||
ENTRYPOINT ["/usr/local/bin/mesh-builder"]
|
||||
|
||||
@@ -66,6 +66,11 @@ broker to take work from or a registry to publish into:
|
||||
|
||||
Without --registry the artifacts stay in this machine's container runtime, named by the
|
||||
digest of their own configuration. The result is printed as JSON.
|
||||
|
||||
mesh-builder ready
|
||||
|
||||
exits 0 once the agent running in this container is bound to its build work and pulls it, and
|
||||
1 before: its container's health check.
|
||||
`
|
||||
|
||||
func run() error {
|
||||
@@ -76,12 +81,18 @@ func run() error {
|
||||
return nil
|
||||
case "build":
|
||||
return buildOnce(context.Background(), os.Args[2:])
|
||||
case "ready":
|
||||
// The container's health check (novox/hq issue 544): exits non-zero until the agent takes work.
|
||||
return ready()
|
||||
default:
|
||||
fmt.Print(usage)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if err := notReady(); err != nil {
|
||||
return err
|
||||
}
|
||||
credential, err := brokerFrom()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -134,7 +145,8 @@ func run() error {
|
||||
// **What an earlier holder here left goes before anything is taken** (novox/hq issue 306): a holder
|
||||
// recreated mid-check left the check's store and bus running, and the ask went to another machine.
|
||||
h.removeLeftBehind()
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused})
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused,
|
||||
Taking: func() { taking(seat) }})
|
||||
defer machine.Close()
|
||||
|
||||
// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// **The build agent says it is ready** (novox/hq issue 544). It listens on nothing, so no http or tcp
|
||||
// check can see it work; it writes a mark once it is bound to its worker and pulls build work, and its
|
||||
// container's health check runs `mesh-builder ready`, which passes only while the mark is there. Without
|
||||
// it the node-engine could judge the agent only alive, and every walk that moved it waited out the whole
|
||||
// grace on each machine.
|
||||
//
|
||||
// The mark lives in the container's own temporary directory, never in the workspace the machine keeps:
|
||||
// a new container starts without it, and a restarted one removes it before it binds again.
|
||||
func readyMark() string { return filepath.Join(os.TempDir(), "mesh-builder.ready") }
|
||||
|
||||
// notReady removes the mark: a holder starting is not taking work until it says so again.
|
||||
func notReady() error {
|
||||
if err := os.Remove(readyMark()); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("cannot remove the mark that this agent takes build work (%s): %w", readyMark(), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// taking writes the mark. A mark that cannot be written is said, and the agent goes on building: its
|
||||
// check then says it is not ready, which is the truth the gate should hear.
|
||||
func taking(seat string) {
|
||||
if err := os.WriteFile(readyMark(), []byte(seat+"\n"), 0o644); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "cannot mark that this agent takes build work (%s): %v\n", readyMark(), err)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "taking build work from %s\n", seat)
|
||||
}
|
||||
|
||||
// ready is the health check: nil when this container's agent is bound and pulling build work.
|
||||
func ready() error {
|
||||
if _, err := os.Stat(readyMark()); err != nil {
|
||||
return fmt.Errorf("not taking build work yet: %s is not there", readyMark())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The health check passes only between the agent saying it takes work and its next start (novox/hq
|
||||
// issue 544).
|
||||
func TestTheAgentIsReadyOnlyOnceItTakesWork(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
if err := ready(); err == nil {
|
||||
t.Fatal("an agent that never took work is ready")
|
||||
}
|
||||
taking("mesh-build-machine")
|
||||
if err := ready(); err != nil {
|
||||
t.Fatalf("an agent taking work is not ready: %v", err)
|
||||
}
|
||||
// Started again, it is not ready until it binds again.
|
||||
if err := notReady(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ready(); err == nil {
|
||||
t.Fatal("an agent starting again is ready before it takes work")
|
||||
}
|
||||
if err := notReady(); err != nil {
|
||||
t.Fatalf("removing a mark that is not there failed: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(readyMark()); err == nil {
|
||||
t.Fatal("the mark is still there")
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,11 @@ type actor struct {
|
||||
// the highest epoch issued, and what was said of it; zero when it was not (S12).
|
||||
reset time.Time
|
||||
resetSaid string
|
||||
// renewEvery and poll, when set, are how often the serving controller renews the lease it holds and
|
||||
// looks again at one another holds, in place of the lease's own: a test's, so two controllers hand the
|
||||
// lease over in a second rather than in a renewal of five (novox/hq issue 591). A renewal is given
|
||||
// the lease's own RenewEvery to answer however often it is made, so a slow bus does not lose it.
|
||||
renewEvery, poll time.Duration
|
||||
}
|
||||
|
||||
// theLease is this process's standing to the lease.
|
||||
@@ -223,7 +228,7 @@ func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory
|
||||
}
|
||||
say := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
|
||||
l, err := lease.Open(ctx, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
|
||||
Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) {
|
||||
RenewEvery: a.renewEvery, RenewWithin: lease.RenewEvery, Poll: a.poll, Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.reset = time.Now()
|
||||
|
||||
@@ -1109,6 +1109,16 @@ const resolverSeat = "mesh-dns-resolver"
|
||||
// classOf is a walk's class (novox/hq ADR 0282 decision 1): core when it walks a module on the controller's own
|
||||
// path or one holding the mesh's resolver, leaf otherwise.
|
||||
func classOf(walk inventory.Plan, entries []inventory.Entry) string {
|
||||
modules := make([]string, 0, len(walk.Modules))
|
||||
for m := range walk.Modules {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
return classOfModules(modules, entries)
|
||||
}
|
||||
|
||||
// classOfModules is the class of modules moved together: core when one is on the controller's own path or holds
|
||||
// the mesh's resolver, leaf otherwise (ADR 0282 decision 1).
|
||||
func classOfModules(modules []string, entries []inventory.Entry) string {
|
||||
resolvers := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
for _, c := range e.Manifest.Claims {
|
||||
@@ -1117,7 +1127,7 @@ func classOf(walk inventory.Plan, entries []inventory.Entry) string {
|
||||
}
|
||||
}
|
||||
}
|
||||
for m := range walk.Modules {
|
||||
for _, m := range modules {
|
||||
if _, own := onTheControllersPath[m]; own || resolvers[m] {
|
||||
return inventory.ClassCore
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -105,9 +106,32 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento
|
||||
p.Machines = append(p.Machines, *machines[name])
|
||||
}
|
||||
p.Summary = summaryOf(p)
|
||||
p.Class = changeClassOf(p, entries)
|
||||
return p
|
||||
}
|
||||
|
||||
// classNone is the class of a change that moves no module: nothing for a walk to move (novox/hq ADR 0313 §2).
|
||||
const classNone = "none"
|
||||
|
||||
// changeClassOf is a change plan's class (novox/hq ADR 0313 §2): that of the slowest-budgeted module it moves,
|
||||
// itself or as a dependent — core when one is on the controller's own path or holds the mesh's resolver, leaf
|
||||
// otherwise — and none when it moves nothing. A module it adds a directory for counts by its own class: the
|
||||
// catalogue names a module's directory by the module, so the directory's last element is its name. The graph holds
|
||||
// no manifest of it yet, so a new module is core by its name alone (on the controller's own path), never by a
|
||||
// resolver claim its manifest makes.
|
||||
func changeClassOf(p link.ChangePlan, entries []inventory.Entry) string {
|
||||
if len(p.Moved) == 0 && len(p.Dependents) == 0 && len(p.New) == 0 {
|
||||
return classNone
|
||||
}
|
||||
modules := append(append([]string{}, p.Moved...), p.Dependents...)
|
||||
for _, dir := range p.New {
|
||||
if name := path.Base(strings.Trim(dir, "/")); name != "." && name != "/" && name != "" {
|
||||
modules = append(modules, name)
|
||||
}
|
||||
}
|
||||
return classOfModules(modules, entries)
|
||||
}
|
||||
|
||||
// grantsStepWord names the grants step (novox/hq issue 490): the bus's user list sent alone to the machine
|
||||
// holding the bus ahead of a walk's gate. Not the bus step, which replaces the bus itself.
|
||||
const grantsStepWord = "grants step"
|
||||
|
||||
@@ -429,6 +429,20 @@ func checkedOf(result link.BuildResult) link.Checked {
|
||||
l.Verdict = "error"
|
||||
}
|
||||
}
|
||||
// **A check run again says so** (novox/hq issue 594). The summaries are the lines the pull request
|
||||
// shows, clipped by the forge to a few words, so they get only a short marker after the verdict's own
|
||||
// words; the report gets the whole sentence. Layers copied, so the build's own outcome is left as it came.
|
||||
if mark := link.RunMarker(result.Delivery); mark != "" {
|
||||
c.Summary += " " + mark
|
||||
for _, l := range []**link.CheckLayer{&c.Gate, &c.RepoCheck} {
|
||||
if *l != nil {
|
||||
layer := **l
|
||||
layer.Summary += " " + mark
|
||||
*l = &layer
|
||||
}
|
||||
}
|
||||
c.Report = link.Restarted(result.Delivery) + "\n\n" + c.Report
|
||||
}
|
||||
if len(c.Report) > maxCheckReport {
|
||||
c.Report = "…" + c.Report[len(c.Report)-maxCheckReport:]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// **No send but a person's `push <node>` and a gate's own put-back moves the mesh's own core to a build older
|
||||
// than the one its machine was last sent** (novox/hq issue 201). Every send made through sendToEach keeps it — a
|
||||
// plan's, a release's, a healer's, a rotation's, and the bus step too, which is a person's act but composes under
|
||||
// sendKeeps like the rest. On 2026-10-02 two merges a minute apart made two plans;
|
||||
// the controller's plan rolled the new controller, which widened its seat's row, and the node-engine's plan then
|
||||
// sent the control node the controller build before it. The older binary read the row it could not serve and the
|
||||
// mesh lost its voice for ten minutes — the thing that would have corrected it was the thing that was down. ADR
|
||||
// 0185 took the outage out of it (a controller behind its row serves what it can); the race is closed here,
|
||||
// without needing its trigger: whatever made the mesh's record older, the send keeps the build.
|
||||
//
|
||||
// **Kept, not refused.** The send composes the controller or the node-engine at the build the machine was last
|
||||
// sent, as a recorded module is kept (ADR 0245), and carries everything else: no other module, secret, healer or
|
||||
// walk waits on it, and nothing is retried. The kept move is no move for a gate either: ungatedIn and gatedSend
|
||||
// leave it out (withoutKept), so a gate judges the other moves of its send. A plan whose own build is the older one finds its gate superseded by
|
||||
// the build kept there (issue 352) and ends. The mesh's record stays older, and a warning condition says so until
|
||||
// a send carries the record — a person's `push <node>` accepts the older build — or the record moves forward: a
|
||||
// newer merge, or `rebuild mesh-controller`.
|
||||
//
|
||||
// Older is by the builds' registration order — when each was asked (novox/hq issue 219) — never by the commits'
|
||||
// dates. So a retry or a rebuild of an old commit, asked anew, counts as newer: it is registered as the module's
|
||||
// current build by the same order, and the guard does not stop that move. A build that failed its gate is never
|
||||
// ahead: the machine was sent it and the record was put back before it (gateFailed), and if the put-back's own
|
||||
// send was refused, the next send completes it rather than keeping the failed build there. A gate's own put-back
|
||||
// carries its modules back whatever the order (sendScope.puttingBack); the controller's is held to the store's
|
||||
// schema already (issue 352).
|
||||
|
||||
// kindCoreDowngrade is a core module kept at the build its machine was last sent, over an older record.
|
||||
const kindCoreDowngrade = "core-downgrade-kept"
|
||||
|
||||
// coreDowngradeSource is the condition's source: the sends, which raise and clear it.
|
||||
const coreDowngradeSource = "sends"
|
||||
|
||||
// downgradeGuarded is the core components a send may not move backwards: the controller, which reads the
|
||||
// store and its seat's row, and the node-engine, which reads what the controller declares.
|
||||
var downgradeGuarded = []string{lease.ComponentController, lease.ComponentEngine}
|
||||
|
||||
// askedBuild is what the order needs of one build of a module: when it was asked, and whether it failed its gate.
|
||||
type askedBuild struct {
|
||||
asked time.Time
|
||||
failedGate bool
|
||||
}
|
||||
|
||||
// buildAsked is the newest build of a module from a commit that worked — when it was asked (or recorded, when
|
||||
// the request time is not known), and whether it failed its gate — and false when the records hold none. A
|
||||
// variable, so a test reads its own order.
|
||||
var buildAsked = func(ctx context.Context, inv *inventory.Inventory, module, commit string) (askedBuild, bool, error) {
|
||||
builds, err := inv.Builds(ctx, module, askedLookBack)
|
||||
if err != nil {
|
||||
return askedBuild{}, false, err
|
||||
}
|
||||
for _, b := range builds {
|
||||
if b.Worked() && sameCommit(b.Commit, commit) {
|
||||
failed, err := inv.GateFailed(ctx, b.ID)
|
||||
if err != nil {
|
||||
return askedBuild{}, false, err
|
||||
}
|
||||
return askedBuild{asked: b.AskedOrAt(), failedGate: failed}, true, nil
|
||||
}
|
||||
}
|
||||
return askedBuild{}, false, nil
|
||||
}
|
||||
|
||||
// askedLookBack is how many of a module's newest builds are read for when a build was asked.
|
||||
const askedLookBack = 50
|
||||
|
||||
// coreDowngrade is one core module a machine was last sent a build of that is newer than the mesh's record.
|
||||
type coreDowngrade struct {
|
||||
module, node, component string
|
||||
// sent is the build the machine was last sent, record the one the mesh holds.
|
||||
sent, record string
|
||||
sentAsked, recordAsked time.Time
|
||||
}
|
||||
|
||||
func (d coreDowngrade) String() string {
|
||||
return fmt.Sprintf("%s was last sent %s %s, and the mesh holds the older %s (asked %s, before %s): a send there "+
|
||||
"keeps %s; `push %s` sends the older build, and a newer merge or `rebuild %s` moves the record forward "+
|
||||
"(novox/hq issue 201)", d.node, d.module, short(d.sent), short(d.record),
|
||||
d.recordAsked.UTC().Format(time.RFC3339), d.sentAsked.UTC().Format(time.RFC3339), short(d.sent), d.node,
|
||||
d.module)
|
||||
}
|
||||
|
||||
// coreDowngrades is, for one machine, each guarded core module whose record is older than the build the machine
|
||||
// was last sent, leaving out what a gate's own put-back puts back and a sent build that failed its gate. A move
|
||||
// whose either build the records do not hold is not proved older and is left to the gate that judges it. Pure
|
||||
// but for asked.
|
||||
func coreDowngrades(node string, sent map[string]string, record map[string]string, scope sendScope,
|
||||
asked func(module, commit string) (askedBuild, bool, error)) ([]coreDowngrade, error) {
|
||||
var out []coreDowngrade
|
||||
for module, was := range sent {
|
||||
component := coreComponent(module)
|
||||
now := record[module]
|
||||
if !guarded(component) || was == "" || now == "" || sameCommit(was, now) {
|
||||
continue
|
||||
}
|
||||
if scope.puttingBack && scope.modules[module] {
|
||||
continue
|
||||
}
|
||||
from, known, err := asked(module, was)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(was), err)
|
||||
}
|
||||
if !known || from.failedGate {
|
||||
continue
|
||||
}
|
||||
to, known, err := asked(module, now)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(now), err)
|
||||
}
|
||||
if known && to.asked.Before(from.asked) {
|
||||
out = append(out, coreDowngrade{module: module, node: node, component: component, sent: was, record: now,
|
||||
sentAsked: from.asked, recordAsked: to.asked})
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].module < out[j].module })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func guarded(component string) bool {
|
||||
for _, c := range downgradeGuarded {
|
||||
if c == component {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// coreKept is, for a send that keeps (sendKeeps), each guarded core module of a machine to keep at the build it
|
||||
// was last sent, module → commit, each said and raised as a condition.
|
||||
func coreKept(ctx context.Context, inv *inventory.Inventory, node string, sent map[string]string,
|
||||
current map[string]inventory.CurrentBuild) (map[string]string, error) {
|
||||
record := map[string]string{}
|
||||
for m, c := range current {
|
||||
record[m] = c.Commit
|
||||
}
|
||||
found, err := coreDowngrades(node, sent, record, scopeOf(ctx), func(module, commit string) (askedBuild, bool, error) {
|
||||
return buildAsked(ctx, inv, module, commit)
|
||||
})
|
||||
if err != nil || len(found) == 0 {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, d := range found {
|
||||
out[d.module] = d.sent
|
||||
sayCoreDowngrade(ctx, d)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// coreDowngradeObservation is a kept core build as a condition: `core.<component>.<machine>.<kind>`.
|
||||
func coreDowngradeObservation(component, node, summary string) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeCore, ID: component + "." + node, Kind: kindCoreDowngrade,
|
||||
Token: kindCoreDowngrade, Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource,
|
||||
Summary: summary}
|
||||
}
|
||||
|
||||
func sayCoreDowngrade(ctx context.Context, d coreDowngrade) {
|
||||
fmt.Printf("KEPT: %s\n", d)
|
||||
keeper := conditionsFrom
|
||||
if keeper == nil {
|
||||
return
|
||||
}
|
||||
if _, err := keeper.Observe(ctx, coreDowngradeObservation(d.component, d.node, d.String())); err != nil {
|
||||
fmt.Printf("the condition for %s kept on %s could not be raised: %v\n", d.module, d.node, err)
|
||||
}
|
||||
}
|
||||
|
||||
// settleCoreDowngrades clears, after a send to a machine is away, the condition of each guarded core module the
|
||||
// send carried at the mesh's record — a person's push that accepted the older build, or a send after the record
|
||||
// moved forward — or did not carry at all. A send that kept the newer build leaves it open.
|
||||
func settleCoreDowngrades(ctx context.Context, inv *inventory.Inventory, node string, carried map[string]string) {
|
||||
keeper := conditionsFrom
|
||||
if keeper == nil || carried == nil {
|
||||
return
|
||||
}
|
||||
var open []string
|
||||
for _, c := range downgradeGuarded {
|
||||
if _, found, err := keeper.Get(ctx, coreDowngradeObservation(c, node, "").Key()); err == nil && found {
|
||||
open = append(open, c)
|
||||
}
|
||||
}
|
||||
if len(open) == 0 {
|
||||
return
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("whether %s still keeps a core build over the mesh's record cannot be read: %v\n", node, err)
|
||||
return
|
||||
}
|
||||
for module, now := range current {
|
||||
component := coreComponent(module)
|
||||
if !guarded(component) {
|
||||
continue
|
||||
}
|
||||
if was, has := carried[module]; has && !sameCommit(was, now.Commit) {
|
||||
continue
|
||||
}
|
||||
key := coreDowngradeObservation(component, node, "").Key()
|
||||
if _, err := keeper.Clear(ctx, key, node+" was sent the "+component+" the mesh holds"); err != nil {
|
||||
fmt.Printf("the condition %s could not be cleared: %v\n", key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1042,13 +1042,13 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
return fmt.Errorf("delivery %s: plan, order, check, go, stop or walks", sub)
|
||||
}
|
||||
|
||||
// appliedFrom answers a machine's first report after a send from the controller's `apply` durations; a lookup
|
||||
// that fails is a report not read, which leaves the walk's end unknown rather than wrong.
|
||||
// appliedFrom answers a machine's own report of the send a walk made to it from the controller's `apply` durations
|
||||
// (novox/hq issue 582); a lookup that fails is a report not read, which leaves the walk's end unknown rather than wrong.
|
||||
func appliedFrom(ctx context.Context, inv *inventory.Inventory) inventory.AppliedLookup {
|
||||
return func(node string, sent time.Time) (inventory.AppliedReport, bool) {
|
||||
r, ok, err := inv.FirstAppliedAfter(ctx, node, sent)
|
||||
return func(node string, send inventory.SentDeclaration, walkSent time.Time) (inventory.AppliedReport, bool) {
|
||||
r, ok, err := inv.FirstAppliedAfter(ctx, node, send, walkSent)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "the report of %s after %s could not be read: %v\n", node, sent.Format(time.RFC3339), err)
|
||||
fmt.Fprintf(os.Stderr, "the report of %s after %s could not be read: %v\n", node, walkSent.Format(time.RFC3339), err)
|
||||
return inventory.AppliedReport{}, false
|
||||
}
|
||||
return r, ok
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A change plan says its class** (novox/hq ADR 0313 §2): the issue tracker holds a fix to the budget of the
|
||||
// slowest-budgeted module the fix moves, and a fix that moves no module has no fix budget. The plan says core,
|
||||
// leaf or none, so mesh-delivery can pass it on with the fix's delivery.
|
||||
func TestAChangePlanSaysTheClassOfWhatItMoves(t *testing.T) {
|
||||
const repo = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
entry := func(name, path string) inventory.Entry {
|
||||
e := fromRepo(name, repo, path)
|
||||
e.Source.BuiltFrom = "old"
|
||||
e.On = []string{"anchor"}
|
||||
return e
|
||||
}
|
||||
gitea := entry("gitea", "modules/gitea")
|
||||
resolver := entry("dnsmasq", "modules/dnsmasq")
|
||||
resolver.Manifest.Claims = []catalogue.Claim{{Name: resolverSeat}}
|
||||
delivery := entry(catalogue.DeliverySeat, "modules/mesh-delivery")
|
||||
webApp := entry("photos", "modules/photos")
|
||||
entries := []inventory.Entry{gitea, resolver, delivery, webApp}
|
||||
// photos stands on gitea: a change to gitea moves photos as its dependent.
|
||||
edges := []inventory.Edge{{From: "photos", To: "gitea", Kind: inventory.EdgeStandsOn}}
|
||||
policy := func(string) (inventory.Upgrade, bool) { return inventory.Upgrade{RollOut: true}, true }
|
||||
plan := func(paths ...string) link.ChangePlan {
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths}
|
||||
return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, edges), entries, policy)
|
||||
}
|
||||
|
||||
for _, c := range []struct {
|
||||
why string
|
||||
paths []string
|
||||
want string
|
||||
}{
|
||||
{"a leaf module and its dependent", []string{"modules/gitea/index.ts"}, inventory.ClassLeaf},
|
||||
{"the delivery's owner, on the controller's own path", []string{"modules/mesh-delivery/cmd/x.go"}, inventory.ClassCore},
|
||||
{"the mesh's resolver", []string{"modules/dnsmasq/module.json"}, inventory.ClassCore},
|
||||
{"a leaf and a core module together: the slowest-budgeted", []string{"modules/gitea/index.ts", "modules/dnsmasq/module.json"}, inventory.ClassCore},
|
||||
{"nothing any module's build reads", []string{"README.md"}, "none"},
|
||||
{"a new leaf module alone", []string{"modules/weather/module.json"}, inventory.ClassLeaf},
|
||||
{"a new module on the controller's own path, by its own class", []string{"next/mesh-delivery/module.json"}, inventory.ClassCore},
|
||||
{"a new leaf module beside a core one moved", []string{"modules/weather/module.json", "modules/dnsmasq/module.json"}, inventory.ClassCore},
|
||||
} {
|
||||
if got := plan(c.paths...).Class; got != c.want {
|
||||
t.Errorf("%s: class %q, want %q", c.why, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1039,7 +1039,7 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str
|
||||
}
|
||||
return
|
||||
}
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines)
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}, puttingBack: true}), open, g.Machines)
|
||||
if err != nil {
|
||||
notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+
|
||||
"sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0]))
|
||||
@@ -1132,7 +1132,7 @@ func sendRollbacks(ctx context.Context, open *stores, p *inventory.Plan, b *roll
|
||||
}
|
||||
inv := open.inventory
|
||||
sort.Strings(b.machines)
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules}), open, b.machines)
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules, puttingBack: true}), open, b.machines)
|
||||
var back []string
|
||||
for _, r := range b.pending {
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// novox/hq issue 201: on 2026-10-02 a plan sent the control node a controller older than the one running there,
|
||||
// whose seat row the newer one had already widened; the older binary refused the row and the mesh went quiet for
|
||||
// ten minutes. No send but a person's push and a gate's own put-back moves the controller or the node-engine back
|
||||
// behind the build its machine was last sent: the send keeps it there, and carries the rest.
|
||||
|
||||
// askedIn answers a build from a fixed table: module@commit → when it was asked; failed names those that failed
|
||||
// their gate.
|
||||
func askedIn(table map[string]time.Time, failed ...string) func(module, commit string) (askedBuild, bool, error) {
|
||||
return func(module, commit string) (askedBuild, bool, error) {
|
||||
at, known := table[module+"@"+commit]
|
||||
b := askedBuild{asked: at}
|
||||
for _, f := range failed {
|
||||
if f == module+"@"+commit {
|
||||
b.failedGate = true
|
||||
}
|
||||
}
|
||||
return b, known, nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestACoreModuleIsKeptOverAnOlderRecord(t *testing.T) {
|
||||
old, now := t0, t0.Add(time.Minute)
|
||||
table := map[string]time.Time{
|
||||
"mesh-controller@c1": old, "mesh-controller@c2": now,
|
||||
"mesh-host@h1": old, "mesh-host@h2": now,
|
||||
"app@a1": old, "app@a2": now,
|
||||
}
|
||||
sent := map[string]string{"mesh-controller": "c2", "mesh-host": "h2", "app": "a2"}
|
||||
record := map[string]string{"mesh-controller": "c1", "mesh-host": "h1", "app": "a1"}
|
||||
found, err := coreDowngrades("anchor", sent, record, sendScope{judged: map[string]bool{"anchor": true}}, askedIn(table))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 2 || found[0].module != "mesh-controller" || found[0].component != lease.ComponentController ||
|
||||
found[0].sent != "c2" || found[1].module != "mesh-host" || found[1].component != lease.ComponentEngine {
|
||||
t.Fatalf("the controller and the node-engine over an older record are kept, and nothing else: %+v", found)
|
||||
}
|
||||
if said := found[0].String(); !strings.Contains(said, "anchor was last sent mesh-controller c2") ||
|
||||
!strings.Contains(said, "`push anchor`") || !strings.Contains(said, "`rebuild mesh-controller`") {
|
||||
t.Fatalf("it says the machine, the build kept and the two ways out: %s", said)
|
||||
}
|
||||
|
||||
// The record forward, the same build, or a build the records do not hold: nothing kept.
|
||||
for _, c := range [][2]string{{"c1", "c2"}, {"c2", "c2"}, {"c2", "c9"}, {"c9", "c1"}} {
|
||||
found, err := coreDowngrades("anchor", map[string]string{"mesh-controller": c[0]},
|
||||
map[string]string{"mesh-controller": c[1]}, sendScope{}, askedIn(table))
|
||||
if err != nil || len(found) != 0 {
|
||||
t.Fatalf("sent %s, record %s: no downgrade proved, nothing kept: %+v %v", c[0], c[1], found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A sent build that failed its gate is never ahead: the record put back before it is sent.
|
||||
if found, _ := coreDowngrades("anchor", sent, record, sendScope{}, askedIn(table, "mesh-controller@c2")); len(found) != 1 ||
|
||||
found[0].module != "mesh-host" {
|
||||
t.Fatalf("a controller that failed its gate was kept over the build put back: %+v", found)
|
||||
}
|
||||
|
||||
// A gate's own put-back carries its own modules back, and nothing else.
|
||||
putBack := sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}
|
||||
if found, _ := coreDowngrades("anchor", sent, record, putBack, askedIn(table)); len(found) != 1 || found[0].module != "mesh-host" {
|
||||
t.Fatalf("a put-back of the controller carries it back, and not the node-engine: %+v", found)
|
||||
}
|
||||
if found, _ := coreDowngrades("anchor", sent, record, sendScope{modules: map[string]bool{"mesh-controller": true}},
|
||||
askedIn(table)); len(found) != 2 {
|
||||
t.Fatalf("a send for the controller that is no put-back carries it back: %+v", found)
|
||||
}
|
||||
}
|
||||
|
||||
// core201 is a mesh whose control node, anchor, was sent the newer controller c2 while the mesh's record of the
|
||||
// controller is the build before it, c1, which passed its gate when it rolled out.
|
||||
func core201(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
|
||||
record := func(commit string, asked time.Time) inventory.Build {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"})
|
||||
b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit,
|
||||
Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}}
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
before := record("c1aaaaaaaaaa", old)
|
||||
if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: before.Commit,
|
||||
Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record("c2bbbbbbbbbb", now)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RestoreModule(ctx, before); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wasMoves := machineMoves
|
||||
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
|
||||
modules, err := open.inventory.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sent, known, err := open.inventory.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.moves(node, modules, sent, known, all), nil
|
||||
}
|
||||
t.Cleanup(func() { machineMoves = wasMoves })
|
||||
return open
|
||||
}
|
||||
|
||||
// TestReplay201 replays the race on a store: the control node runs the newer controller and the mesh's record is
|
||||
// the build before it. A plan's send there — the node-engine's plan of 2026-10-02 — goes, keeps the newer
|
||||
// controller, and says so as a condition; the gate's own put-back carries it back; a send that carries the record,
|
||||
// as a person's push does, clears the condition.
|
||||
func TestReplay201(t *testing.T) {
|
||||
open := core201(t)
|
||||
ctx := t.Context()
|
||||
k, _ := withConditionsInMemory(t)
|
||||
|
||||
judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}})
|
||||
if kept, err := ungatedIn(judged, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
|
||||
t.Fatalf("a plan's send to the machine is made, the rest of it with it: %v %v", kept, err)
|
||||
}
|
||||
keeps, err := recordedKept(keepingRecorded(judged), open, "anchor")
|
||||
if err != nil || keeps["mesh-controller"] != "c2bbbbbbbbbb" {
|
||||
t.Fatalf("the send keeps the newer controller there: %v %v", keeps, err)
|
||||
}
|
||||
shelf := map[string]catalogue.Manifest{"mesh-controller": {Module: "mesh-controller", Version: "old"}}
|
||||
if _, err := keepRecorded(keepingRecorded(judged), open, "anchor", shelf); err != nil || shelf["mesh-controller"].Version != "1" {
|
||||
t.Fatalf("the send composes the kept build's manifest: %+v %v", shelf, err)
|
||||
}
|
||||
key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key()
|
||||
if c, found, err := k.Get(ctx, key); err != nil || !found || !strings.Contains(c.Summary, "c2bbbbbb") {
|
||||
t.Fatalf("the condition %s is raised, saying the build kept: %+v %v %v", key, c, found, err)
|
||||
}
|
||||
|
||||
putBack := keepingRecorded(withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}))
|
||||
if keeps, err := recordedKept(putBack, open, "anchor"); err != nil || len(keeps) != 0 {
|
||||
t.Fatalf("the gate's own put-back of the controller carries it back: %v %v", keeps, err)
|
||||
}
|
||||
|
||||
settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"})
|
||||
if _, found, _ := k.Get(ctx, key); !found {
|
||||
t.Fatal("a send that kept the newer controller cleared the condition")
|
||||
}
|
||||
settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c1aaaaaaaaaa"})
|
||||
if _, found, _ := k.Get(ctx, key); found {
|
||||
t.Fatal("a send carrying the mesh's record — a person's push — leaves the condition open")
|
||||
}
|
||||
}
|
||||
|
||||
// The review of #273: a gate failed the newer controller, put the record back, and the put-back's own send was
|
||||
// refused. The next automatic send — a healer's, which judges nothing — completes the put-back: a build that
|
||||
// failed its gate is never kept.
|
||||
func TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend(t *testing.T) {
|
||||
open := core201(t)
|
||||
ctx := t.Context()
|
||||
k, _ := withConditionsInMemory(t)
|
||||
if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: "build-mesh-controller-c2bbbbbbbbbb",
|
||||
Module: "mesh-controller", Commit: "c2bbbbbbbbbb", Previous: "c1aaaaaaaaaa", Machines: []string{"anchor"},
|
||||
Verdict: inventory.GateFailed, Rollback: inventory.NotRolledBack}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
healer := withScope(ctx, sendScope{})
|
||||
if kept, err := ungatedIn(healer, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
|
||||
t.Fatalf("the healer's send is made: %v %v", kept, err)
|
||||
}
|
||||
if keeps, err := recordedKept(keepingRecorded(healer), open, "anchor"); err != nil || len(keeps) != 0 {
|
||||
t.Fatalf("the failed controller was kept over the build put back: %v %v", keeps, err)
|
||||
}
|
||||
if open, _ := k.Open(ctx); len(open) != 0 {
|
||||
t.Fatalf("a completed put-back raised a condition: %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// Review 91 of #273: a gated send to a machine whose controller is kept over an older record leaves the kept
|
||||
// controller out of what its gate judges. Judged as moving back to the record, the gate would read the send's own
|
||||
// kept controller as another build sent there after it (issue 352) and be superseded, judging nothing.
|
||||
func TestAGateJudgesTheOtherMovesBesideAKeptCore(t *testing.T) {
|
||||
open := core201(t)
|
||||
ctx := t.Context()
|
||||
withConditionsInMemory(t)
|
||||
inv := open.inventory
|
||||
old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
|
||||
for _, b := range []struct {
|
||||
commit string
|
||||
at time.Time
|
||||
}{{"a1aaaaaaaaaa", old}, {"a2bbbbbbbbbb", now}} {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-" + b.commit, Module: "app", Commit: b.commit,
|
||||
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: b.at, At: b.at,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-" + b.commit}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: b.commit, Head: b.commit,
|
||||
Asked: b.at}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-2",
|
||||
map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a1aaaaaaaaaa"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The send as it is made: the controller kept at c2, the app moved to the record.
|
||||
wasSend := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
return names, open.inventory.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-3",
|
||||
map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a2bbbbbbbbbb"})
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = wasSend })
|
||||
|
||||
moves, sent, _, err := gatedSend(ctx, open, "anchor", nil)
|
||||
if err != nil || len(sent) != 1 {
|
||||
t.Fatalf("the gated send is made: %v %v", sent, err)
|
||||
}
|
||||
if len(moves) != 1 || moves[0].Module != "app" || moves[0].To != "a2bbbbbbbbbb" {
|
||||
t.Fatalf("the gate judges the app's move and not the kept controller: %+v", moves)
|
||||
}
|
||||
builds, _, err := inv.SentBuilds(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g := &inventory.PlanGate{Carried: moves}
|
||||
f := gateFacts{commits: judgedCommits(g, nil), sentBuilds: map[string]map[string]string{"anchor": builds}}
|
||||
for _, m := range []string{"app", "mesh-controller"} {
|
||||
if why, superseded := f.supersededOn(m, "anchor"); superseded {
|
||||
t.Fatalf("the gate is superseded by its own send: %s", why)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -55,8 +55,14 @@ func TestTwoControllersOneActs(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
ctx := t.Context()
|
||||
|
||||
// Each renews five times and looks again ten times as often as a serving controller does, so the
|
||||
// handover and the loss below are seen in a second, not in a renewal of five (novox/hq issue 591): what
|
||||
// is checked is the order of events, which does not change with the interval. Each renewal still has a
|
||||
// serving controller's five seconds to answer (acting.go), so a slow bus does not lose A the lease.
|
||||
const renewEvery, poll = lease.RenewEvery / 5, lease.Poll / 10
|
||||
|
||||
// Controller A takes the lease.
|
||||
a := &actor{}
|
||||
a := &actor{renewEvery: renewEvery, poll: poll}
|
||||
aCtx, stopA := context.WithCancel(ctx)
|
||||
defer stopA()
|
||||
lostA, err := a.serveUnderTheLease(aCtx, inv, url)
|
||||
@@ -69,7 +75,7 @@ func TestTwoControllersOneActs(t *testing.T) {
|
||||
}
|
||||
|
||||
// Controller B starts while A holds it, and waits — acting on nothing meanwhile.
|
||||
b := &actor{}
|
||||
b := &actor{renewEvery: renewEvery, poll: poll}
|
||||
bCtx, stopB := context.WithCancel(ctx)
|
||||
defer stopB()
|
||||
tookB := make(chan (<-chan struct{}), 1)
|
||||
@@ -82,10 +88,11 @@ func TestTwoControllersOneActs(t *testing.T) {
|
||||
}
|
||||
tookB <- lost
|
||||
}()
|
||||
// B looks fifteen times, across one of A's renewals at least.
|
||||
select {
|
||||
case <-tookB:
|
||||
t.Fatal("B took the lease while A held it")
|
||||
case <-time.After(3 * time.Second):
|
||||
case <-time.After(15 * poll):
|
||||
}
|
||||
if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) {
|
||||
t.Fatalf("B, waiting, may act: %v", err)
|
||||
@@ -125,7 +132,7 @@ func TestTwoControllersOneActs(t *testing.T) {
|
||||
}
|
||||
select {
|
||||
case <-lostB:
|
||||
case <-time.After(2 * lease.RenewEvery):
|
||||
case <-time.After(2 * renewEvery):
|
||||
t.Fatal("B was not told it lost the lease")
|
||||
}
|
||||
if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) {
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// machineLines is what a machine said it is made of (novox/hq issue 160), as `node` shows it: every fact in
|
||||
// one block with its age, the architecture and kernel the profile always carried among them, and each fact
|
||||
// the machine could not determine said as unknown with its why — never as a zero, and never left out.
|
||||
func machineLines(a inventory.MachineAccount, had bool, now time.Time) []string {
|
||||
if !had {
|
||||
return []string{" what it is made of: never said — this machine has not reported a profile"}
|
||||
}
|
||||
m := a.Machine
|
||||
if m == nil {
|
||||
return []string{
|
||||
" what it is made of: not said — its node-engine is older than the one that says it",
|
||||
" " + factLine("architecture", orUnsaid(a.Architecture)),
|
||||
" " + factLine("kernel", orUnsaid(a.Kernel)),
|
||||
}
|
||||
}
|
||||
age := roughly(now.Sub(m.ObservedAt)) + " ago"
|
||||
if m.ObservedAt.IsZero() {
|
||||
// Said with no time: its age is not known, and is never read as two thousand years.
|
||||
age = "at a time it did not say (age unknown)"
|
||||
}
|
||||
out := []string{fmt.Sprintf(" what it is made of, as it said %s:", age)}
|
||||
add := func(name, value string) { out = append(out, " "+factLine(name, value)) }
|
||||
unknown := map[string]string{}
|
||||
for k, v := range m.Unknown {
|
||||
unknown[k] = v
|
||||
}
|
||||
// said is the value, or the fact's own reason it is unknown; a fact neither said nor unknown is a
|
||||
// node-engine that broke its own rule, and is said so rather than shown blank.
|
||||
said := func(fact, value string) string {
|
||||
if value != "" {
|
||||
delete(unknown, fact)
|
||||
return value
|
||||
}
|
||||
if why, ok := unknown[fact]; ok {
|
||||
delete(unknown, fact)
|
||||
return "unknown: " + why
|
||||
}
|
||||
return "unknown: the machine said neither it nor why"
|
||||
}
|
||||
|
||||
add("architecture", orUnsaid(a.Architecture))
|
||||
release := said("kernel release", m.KernelRelease)
|
||||
if m.KernelRelease == "" {
|
||||
release = "(release " + release + ")"
|
||||
}
|
||||
add("kernel", orUnsaid(a.Kernel)+" "+release)
|
||||
dist := said("distribution", m.Distribution)
|
||||
if m.Distribution != "" {
|
||||
dist += " " + said("version", m.Version)
|
||||
}
|
||||
add("distribution", dist)
|
||||
add("processor", said("processor", m.Processor))
|
||||
add("cores", said("cores", countWords(m.Cores)))
|
||||
add("threads", said("threads", countWords(m.Threads)))
|
||||
add("memory", said("memory", bytesWords(m.MemoryBytes)))
|
||||
add("virtual", said("virtual or physical", m.Virtual))
|
||||
up := ""
|
||||
if m.BootedAt != nil {
|
||||
up = fmt.Sprintf("%s, since %s", roughly(now.Sub(*m.BootedAt)), m.BootedAt.Local().Format("2006-01-02 15:04"))
|
||||
}
|
||||
add("up", said("uptime", up))
|
||||
add("timezone", said("timezone", m.Timezone))
|
||||
if len(m.Filesystems) == 0 {
|
||||
add("filesystems", said("filesystems", ""))
|
||||
} else {
|
||||
for i, f := range m.Filesystems {
|
||||
name := ""
|
||||
if i == 0 {
|
||||
name = "filesystems"
|
||||
}
|
||||
add(name, fmt.Sprintf("%s %s, %s free (%s %s)", f.Mount, bytesWords(f.SizeBytes),
|
||||
bytesWords(f.FreeBytes), f.Type, f.Device))
|
||||
}
|
||||
if why, ok := unknown["filesystems"]; ok {
|
||||
add("", "and some unknown: "+why)
|
||||
delete(unknown, "filesystems")
|
||||
}
|
||||
}
|
||||
add("public address", said("public address", ""))
|
||||
// Any other fact a newer node-engine named unknown, so nothing it said is dropped.
|
||||
rest := make([]string, 0, len(unknown))
|
||||
for k := range unknown {
|
||||
rest = append(rest, k)
|
||||
}
|
||||
sort.Strings(rest)
|
||||
for _, k := range rest {
|
||||
add(k, "unknown: "+unknown[k])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func factLine(name, value string) string { return fmt.Sprintf("%-15s %s", name, value) }
|
||||
|
||||
func orUnsaid(s string) string {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return "not said"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// countWords is a number a machine said, or nothing when it said none (its unknown says why).
|
||||
func countWords(n int) string {
|
||||
if n <= 0 {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprint(n)
|
||||
}
|
||||
|
||||
// bytesWords is an amount of bytes a person reads, in binary units; nothing when none was said.
|
||||
func bytesWords(n int64) string {
|
||||
if n <= 0 {
|
||||
return ""
|
||||
}
|
||||
units := []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB"}
|
||||
v, u := float64(n), 0
|
||||
for v >= 1024 && u < len(units)-1 {
|
||||
v /= 1024
|
||||
u++
|
||||
}
|
||||
if u == 0 {
|
||||
return fmt.Sprintf("%d B", n)
|
||||
}
|
||||
return fmt.Sprintf("%.1f %s", v, units[u])
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// Defends novox/hq issue 160: `node` shows every fact a machine said it is made of, in one block with its
|
||||
// age, the architecture and kernel among them; and a fact it could not determine as unknown with why.
|
||||
func TestNodeShowsWhatAMachineIsMadeOfWithItsAge(t *testing.T) {
|
||||
now := time.Date(2026, 10, 12, 15, 0, 0, 0, time.UTC)
|
||||
booted := now.Add(-75 * time.Hour)
|
||||
a := inventory.MachineAccount{Architecture: "amd64", Kernel: "linux", Machine: &inventory.MadeOf{
|
||||
ObservedAt: now.Add(-3 * time.Hour), KernelRelease: "7.2.8-arch1-2", Distribution: "Arch Linux",
|
||||
Version: "rolling", Processor: "AMD Ryzen 9 8945HS", Cores: 8, Threads: 16, MemoryBytes: 32891494400,
|
||||
Virtual: "physical", BootedAt: &booted, Timezone: "Europe/Brussels",
|
||||
Filesystems: []inventory.MountedFilesystem{{Mount: "/", Device: "/dev/nvme0n1p3", Type: "ext4",
|
||||
SizeBytes: 987978522624, FreeBytes: 31830110208}},
|
||||
Unknown: map[string]string{"public address": "how it learns it is not decided yet"},
|
||||
}}
|
||||
got := strings.Join(machineLines(a, true, now), "\n")
|
||||
t.Log("\n" + got)
|
||||
for _, want := range []string{
|
||||
"what it is made of, as it said 3h ago:",
|
||||
"architecture amd64",
|
||||
"kernel linux 7.2.8-arch1-2",
|
||||
"distribution Arch Linux rolling",
|
||||
"processor AMD Ryzen 9 8945HS",
|
||||
"cores 8",
|
||||
"threads 16",
|
||||
"memory 30.6 GiB",
|
||||
"virtual physical",
|
||||
"up 3d, since",
|
||||
"timezone Europe/Brussels",
|
||||
"filesystems / 920.1 GiB, 29.6 GiB free (ext4 /dev/nvme0n1p3)",
|
||||
"public address unknown: how it learns it is not decided yet",
|
||||
} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("missing %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFactAMachineCouldNotReadIsShownUnknownNeverZero(t *testing.T) {
|
||||
now := time.Date(2026, 10, 12, 15, 0, 0, 0, time.UTC)
|
||||
a := inventory.MachineAccount{Architecture: "arm64", Kernel: "linux", Machine: &inventory.MadeOf{
|
||||
ObservedAt: now, Unknown: map[string]string{"memory": "MemTotal not found in /proc/meminfo",
|
||||
"cores": "/proc/cpuinfo says no core id", "kernel release": "permission denied"}}}
|
||||
got := strings.Join(machineLines(a, true, now), "\n")
|
||||
t.Log("\n" + got)
|
||||
for _, want := range []string{
|
||||
"memory unknown: MemTotal not found in /proc/meminfo",
|
||||
"cores unknown: /proc/cpuinfo says no core id",
|
||||
"kernel linux (release unknown: permission denied)",
|
||||
// Neither said nor named unknown: the node-engine broke its rule, and that is shown.
|
||||
"timezone unknown: the machine said neither it nor why",
|
||||
} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("missing %q", want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(got, " 0 ") || strings.Contains(got, "0 B") {
|
||||
t.Error("a fact the machine could not read was shown as a zero")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOlderNodeEngineStillShowsArchitectureAndKernel(t *testing.T) {
|
||||
got := strings.Join(machineLines(inventory.MachineAccount{Architecture: "amd64", Kernel: "linux"}, true,
|
||||
time.Now()), "\n")
|
||||
if !strings.Contains(got, "older than the one that says it") || !strings.Contains(got, "architecture amd64") ||
|
||||
!strings.Contains(got, "kernel linux") {
|
||||
t.Errorf("%s", got)
|
||||
}
|
||||
if got := strings.Join(machineLines(inventory.MachineAccount{}, false, time.Now()), "\n"); !strings.Contains(got, "never said") {
|
||||
t.Errorf("%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAccountWithoutItsTimeSaysAgeUnknown(t *testing.T) {
|
||||
got := machineLines(inventory.MachineAccount{Architecture: "amd64", Kernel: "linux",
|
||||
Machine: &inventory.MadeOf{KernelRelease: "7.2"}}, true, time.Now())[0]
|
||||
if !strings.Contains(got, "age unknown") || strings.Contains(got, "d ago") {
|
||||
t.Errorf("%s", got)
|
||||
}
|
||||
}
|
||||
@@ -875,6 +875,17 @@ func showNode(ctx context.Context, stored *stores, name string) error {
|
||||
}
|
||||
}
|
||||
|
||||
// What it is made of, in one block with its age (novox/hq issue 160): collected and shown in the same
|
||||
// breath, so no fact the machine says is kept and read by nothing.
|
||||
if account, had, err := inv.MachineOf(ctx, name); err != nil {
|
||||
fmt.Printf("\n what it is made of could NOT be read: %v\n", err)
|
||||
} else {
|
||||
fmt.Println()
|
||||
for _, line := range machineLines(account, had, time.Now()) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
|
||||
held, err := inv.Profile(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -182,6 +182,20 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
"register it, so it was not made.", module, m),
|
||||
Resolved: "Resolved: " + module + " on " + m + " is settled"}
|
||||
}),
|
||||
// The controller or the node-engine kept at the build a machine was last sent, over an older record (novox/hq
|
||||
// issue 201).
|
||||
kindCoreDowngrade: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
component := idPart(o, 0)
|
||||
if o.Scope != conditions.ScopeCore || !guarded(component) {
|
||||
component = "core component"
|
||||
}
|
||||
return words{Headline: m + " keeps a newer " + component + " than the mesh holds",
|
||||
Needs: fmt.Sprintf("push %s to take the older build, or a newer merge or rebuild moves the record forward.", m),
|
||||
Explanation: fmt.Sprintf("The mesh's record of its own %s is older than the build %s was last sent, so "+
|
||||
"the mesh's sends keep the newer one there rather than put it back.", component, m),
|
||||
Resolved: "Resolved: " + m + " runs the " + component + " the mesh holds"}
|
||||
}),
|
||||
kindBindingKept: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "A module's data source is held on " + m,
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
// planID is every id the mesh gives a plan: a merge's walk or batch (plan-<nanoseconds>) or a release
|
||||
// (release-<nanoseconds>).
|
||||
var planID = regexp.MustCompile(`^(plan|release)-[0-9]+$`)
|
||||
|
||||
// planIDArguments are the arguments of plans that name a plan. wait is among them for when it is declared
|
||||
// (novox/hq issue 229); an argument not declared is refused before this is asked.
|
||||
var planIDArguments = []string{"id", "wait", "stop", "close", "retry", "go"}
|
||||
|
||||
// plainPlanIDs refuses a plans call naming a plan by anything but a plan's id (novox/hq issue 561): checked
|
||||
// once, where the call becomes a command line, so nothing else a caller sent reaches a command or the store.
|
||||
func plainPlanIDs(given map[string]string) error {
|
||||
for _, arg := range planIDArguments {
|
||||
if id, named := given[arg]; named && !planID.MatchString(id) {
|
||||
return fmt.Errorf("plans %s: %q is not a plan's id (plan-<digits> or release-<digits>, as `plans` "+
|
||||
"lists them); nothing was done", arg, id)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **A plan's id is checked once, where a call becomes a command line** (novox/hq issue 561): every argument of
|
||||
// plans that names a plan is refused unless it is an id the mesh gives a plan, so a quote never reaches a
|
||||
// command, nor the store behind it.
|
||||
func TestAPlanIdThatIsNoPlansIdIsRefusedBeforeACommandStarts(t *testing.T) {
|
||||
for _, arg := range []string{"id", "stop", "close", "retry", "go"} {
|
||||
for _, bad := range []string{"x' or '1'='1", "plan-1'", "plan-", "plan-1 2", "batch-1", "PLAN-1"} {
|
||||
args := map[string]any{arg: bad}
|
||||
if arg == "stop" || arg == "close" || arg == "go" {
|
||||
args["why"] = "a test"
|
||||
}
|
||||
argv, err := argvFor("plans", args)
|
||||
if err == nil {
|
||||
t.Errorf("plans %s %q was composed: %v", arg, bad, argv)
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(err.Error(), "is not a plan's id") {
|
||||
t.Errorf("plans %s %q was refused for another reason: %v", arg, bad, err)
|
||||
}
|
||||
}
|
||||
for _, good := range []string{"plan-1791813626034291775", "release-1791811938267404408"} {
|
||||
args := map[string]any{arg: good}
|
||||
if arg == "stop" || arg == "close" || arg == "go" {
|
||||
args["why"] = "a test"
|
||||
}
|
||||
if _, err := argvFor("plans", args); err != nil {
|
||||
t.Errorf("plans %s %q was refused: %v", arg, good, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Following a plan (novox/hq issue 229): `plans wait <id>` holds its answer until the plan moves, reaches its
|
||||
// next tier or ends, or until its bound passes, and says which. Before it, an agent following a rollout had
|
||||
// `plans` answer once, in prose, and wrote a loop of its own round the mesh's tools to be told the plan moved.
|
||||
//
|
||||
// **No wait is longer than a call is answered within** (link.LongestAnswer, inside the console's thirty
|
||||
// seconds): a longer one would answer "still running" and leave its caller to ask `calls` — the loop again.
|
||||
// So a caller following a plan to its end asks again; each call answers the moment the plan moves.
|
||||
|
||||
// maxPlanWait is the longest bound a wait takes, and the bound when none is given: what a call is answered
|
||||
// within, less the time a verb's command takes to start and read the store.
|
||||
var maxPlanWait = link.LongestAnswer - planWaitStart
|
||||
|
||||
// planWaitStart is what the verb's window allows beyond the bound: starting the command and opening the stores.
|
||||
// A store slow enough to push the read and the answer past it degrades as any long call does: the caller is
|
||||
// answered that the call is still running, and `calls` with its id says what came of it.
|
||||
const planWaitStart = 5 * time.Second
|
||||
|
||||
// planWaitEvery is how often a wait reads the plan again. A variable so a test reads it more often
|
||||
// (novox/hq issue 591).
|
||||
var planWaitEvery = time.Second
|
||||
|
||||
// The things a wait waits until.
|
||||
var planWaitUntil = []string{"moved", "tier", "done"}
|
||||
|
||||
// planWaitAsked checks what a wait is asked for: until (moved when empty) and a bound (a duration, or a number
|
||||
// of seconds; maxPlanWait when empty).
|
||||
func planWaitAsked(until, bound string) (string, time.Duration, error) {
|
||||
if until == "" {
|
||||
until = "moved"
|
||||
}
|
||||
known := false
|
||||
for _, u := range planWaitUntil {
|
||||
known = known || u == until
|
||||
}
|
||||
if !known {
|
||||
return "", 0, fmt.Errorf("a wait is until %s, not %q", strings.Join(planWaitUntil, ", "), until)
|
||||
}
|
||||
d := maxPlanWait
|
||||
if bound = strings.TrimSpace(bound); bound != "" {
|
||||
var err error
|
||||
if secs, isNumber := strconv.Atoi(bound); isNumber == nil {
|
||||
d = time.Duration(secs) * time.Second
|
||||
} else if d, err = time.ParseDuration(bound); err != nil {
|
||||
return "", 0, fmt.Errorf("the bound %q is not a duration (such as 20s) or a number of seconds", bound)
|
||||
}
|
||||
if d <= 0 || d > maxPlanWait {
|
||||
return "", 0, fmt.Errorf("the bound is at most %s, the longest a call is answered within; %s was asked "+
|
||||
"— to follow a plan longer, ask again: each call answers the moment the plan moves", maxPlanWait, d)
|
||||
}
|
||||
}
|
||||
return until, d, nil
|
||||
}
|
||||
|
||||
// planEnded is a plan that will not move again: done, failed or superseded. A batch being assembled is not one.
|
||||
func planEnded(p inventory.Plan) bool { return !p.Open() && !p.Batch() }
|
||||
|
||||
// waitOnPlan reads the plan until what it waits for happens or the bound passes, and answers the plan as last
|
||||
// read and what ended the wait: moved (any save of it), tier (its tier changed), done (it ended — which ends
|
||||
// every wait, since nothing comes after it), or bound. Moved and tier are judged against the plan as first read.
|
||||
func waitOnPlan(ctx context.Context, read func(context.Context) (inventory.Plan, error), until string,
|
||||
bound, every time.Duration) (inventory.Plan, string, error) {
|
||||
first, err := read(ctx)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, "", err
|
||||
}
|
||||
if planEnded(first) {
|
||||
return first, "done", nil
|
||||
}
|
||||
deadline := time.Now().Add(bound)
|
||||
tick := time.NewTicker(every)
|
||||
defer tick.Stop()
|
||||
last := first
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return last, "", ctx.Err()
|
||||
case <-tick.C:
|
||||
}
|
||||
if !time.Now().Before(deadline) {
|
||||
return last, "bound", nil
|
||||
}
|
||||
p, err := read(ctx)
|
||||
if err != nil {
|
||||
return last, "", err
|
||||
}
|
||||
last = p
|
||||
switch {
|
||||
case planEnded(p):
|
||||
return p, "done", nil
|
||||
case until == "moved" && p.Revision != first.Revision:
|
||||
return p, "moved", nil
|
||||
case until == "tier" && p.Tier != first.Tier:
|
||||
return p, "tier", nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// planWaited is what `plans wait` answers: the plan, what ended the wait, and the line a person reads (`text`,
|
||||
// which the verb answers as its output and not again in the data).
|
||||
type planWaited struct {
|
||||
// Came is what ended the wait: moved, tier, done, or bound when nothing did.
|
||||
Came string `json:"came"`
|
||||
Until string `json:"until"`
|
||||
Waited string `json:"waited"`
|
||||
Plan planStatus `json:"plan"`
|
||||
// Revision is the plan's as read: a caller asking again with the same plan sees whether it moved since.
|
||||
Revision int64 `json:"revision"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
func planWaitedDoc(p inventory.Plan, st planStatus, until, came string, waited time.Duration, line string) planWaited {
|
||||
said := map[string]string{
|
||||
"moved": "moved", "tier": "reached another tier", "done": "ended",
|
||||
"bound": "did not " + map[string]string{"moved": "move", "tier": "reach another tier", "done": "end"}[until],
|
||||
}[came]
|
||||
text := fmt.Sprintf("%s %s in %s (waited until %s)\n%s — %s\n", p.ID, said, waited.Round(time.Second), until,
|
||||
p.ID, line)
|
||||
return planWaited{Came: came, Until: until, Waited: waited.Round(time.Millisecond).String(), Plan: st,
|
||||
Revision: p.Revision, Text: text}
|
||||
}
|
||||
|
||||
// planWaitCommand is `plans wait <id>`: the plan followed until it moves, or the bound passes.
|
||||
func planWaitCommand(ctx context.Context, inv *inventory.Inventory, id, untilAsked, boundAsked string) error {
|
||||
until, bound, err := planWaitAsked(untilAsked, boundAsked)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
began := time.Now()
|
||||
p, came, err := waitOnPlan(ctx, func(ctx context.Context) (inventory.Plan, error) {
|
||||
return inv.PlanByID(ctx, id)
|
||||
}, until, bound, planWaitEvery)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Said as `plans <id>` says its first line, and as `status` says the plan.
|
||||
now, one := time.Now(), []inventory.Plan{p}
|
||||
bounds, pause := readTierBounds(ctx, inv, now), buildSeatPause(ctx, inv, one)
|
||||
line := planLineSaid(p, now, pause, bounds.of(p.Repository), func(module string) []string {
|
||||
on, _ := inv.Running(ctx, module)
|
||||
return on
|
||||
}, waitsOf(ctx, inv, one))
|
||||
doc := planWaitedDoc(p, planStatuses(one, now, pause, bounds)[0], until, came, now.Sub(began), line)
|
||||
return json.NewEncoder(os.Stdout).Encode(doc)
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A plan as the store would answer it at each read: the reads are counted, and each answers the next.
|
||||
type planReads struct {
|
||||
plans []inventory.Plan
|
||||
n int
|
||||
}
|
||||
|
||||
func (r *planReads) read(context.Context) (inventory.Plan, error) {
|
||||
p := r.plans[min(r.n, len(r.plans)-1)]
|
||||
r.n++
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func followedPlan(rev int64, state string, tier int) inventory.Plan {
|
||||
return inventory.Plan{ID: "plan-1", Repository: "novox/x", Commit: "abc", State: state, Tier: tier,
|
||||
Tiers: [][]string{{"a"}, {"b"}}, Revision: rev}
|
||||
}
|
||||
|
||||
// **`plans` waits for a plan, and says what ended the wait** (novox/hq issue 229): an agent following a
|
||||
// rollout is told when the plan moved, reached its next tier or ended, or that the bound passed — in one
|
||||
// call, without a loop of its own.
|
||||
func TestAWaitForAPlanEndsWhenWhatItWaitsForHappens(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
until string
|
||||
reads []inventory.Plan
|
||||
came string
|
||||
times int
|
||||
}{
|
||||
{"moved: any save", "moved", []inventory.Plan{
|
||||
followedPlan(1, inventory.PlanBuilding, 0), followedPlan(1, inventory.PlanBuilding, 0),
|
||||
followedPlan(2, inventory.PlanBuilding, 0)}, "moved", 3},
|
||||
{"tier: a save inside the tier is not enough", "tier", []inventory.Plan{
|
||||
followedPlan(1, inventory.PlanBuilding, 0), followedPlan(2, inventory.PlanBuilding, 0),
|
||||
followedPlan(3, inventory.PlanRolling, 1)}, "tier", 3},
|
||||
{"tier: a plan that ended has no next tier to wait for", "tier", []inventory.Plan{
|
||||
followedPlan(1, inventory.PlanBuilding, 0), followedPlan(2, inventory.PlanFailed, 0)}, "done", 2},
|
||||
{"done: a new tier is not the end", "done", []inventory.Plan{
|
||||
followedPlan(1, inventory.PlanBuilding, 0), followedPlan(2, inventory.PlanRolling, 1),
|
||||
followedPlan(3, inventory.PlanDone, 1)}, "done", 3},
|
||||
{"done: a plan already ended answers at once", "done", []inventory.Plan{
|
||||
followedPlan(4, inventory.PlanSuperseded, 1)}, "done", 1},
|
||||
{"the bound passes", "moved", []inventory.Plan{followedPlan(1, inventory.PlanBuilding, 0)}, "bound", -1},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
r := &planReads{plans: c.reads}
|
||||
p, came, err := waitOnPlan(context.Background(), r.read, c.until, 50*time.Millisecond, time.Millisecond)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if came != c.came {
|
||||
t.Fatalf("the wait ended as %q, want %q", came, c.came)
|
||||
}
|
||||
if c.times > 0 && r.n != c.times {
|
||||
t.Errorf("read the plan %d times, want %d: it answered before or after what it waited for", r.n, c.times)
|
||||
}
|
||||
if want := c.reads[min(r.n, len(c.reads))-1]; p.Revision != want.Revision {
|
||||
t.Errorf("answered revision %d, not the plan as it was last read (%d)", p.Revision, want.Revision)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// What a wait is asked for is checked before anything is read: what it waits until, and a bound no longer than
|
||||
// a call is answered within.
|
||||
func TestAWaitIsRefusedWhatItCannotDo(t *testing.T) {
|
||||
for _, bad := range []struct{ until, bound string }{{"landed", ""}, {"", "1h"}, {"", "-3s"}, {"", "soon"}} {
|
||||
if _, _, err := planWaitAsked(bad.until, bad.bound); err == nil {
|
||||
t.Errorf("until %q, bound %q was taken", bad.until, bad.bound)
|
||||
}
|
||||
}
|
||||
until, bound, err := planWaitAsked("", "")
|
||||
if err != nil || until != "moved" || bound != maxPlanWait {
|
||||
t.Fatalf("the defaults are %q %s %v, want moved and %s", until, bound, err, maxPlanWait)
|
||||
}
|
||||
if _, bound, _ := planWaitAsked("tier", "12"); bound != 12*time.Second {
|
||||
t.Errorf("a bare number is seconds; read %s", bound)
|
||||
}
|
||||
}
|
||||
|
||||
// **The answer is data, with its prose as a field beside it** (novox/hq issue 229): the verb's `answer` carries
|
||||
// the plan and what ended the wait, and its output is the line a person reads — never JSON inside a string.
|
||||
func TestAWaitForAPlanAnswersAsData(t *testing.T) {
|
||||
p := followedPlan(7, inventory.PlanRolling, 1)
|
||||
st := planStatuses([]inventory.Plan{p}, time.Now(), pauseView{}, tierBounds{})[0]
|
||||
doc := planWaitedDoc(p, st, "tier", "tier", 3*time.Second, "rolling, tier 2 of 2")
|
||||
stdout, _ := json.Marshal(doc)
|
||||
a := answerOf([]string{"plans", "wait", "plan-1"}, stdout, "", true)
|
||||
data, ok := a.Answer.(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("the answer is not data: %v", a.Answer)
|
||||
}
|
||||
if data["came"] != "tier" || data["revision"] != float64(7) {
|
||||
t.Errorf("the data does not say what ended the wait: %v", data)
|
||||
}
|
||||
if plan, _ := data["plan"].(map[string]any); plan["state"] != "rolling" || plan["tier"] != float64(1) {
|
||||
t.Errorf("the data does not carry the plan: %v", data["plan"])
|
||||
}
|
||||
if _, text := data["text"]; text {
|
||||
t.Errorf("the prose is in the data too: %v", data)
|
||||
}
|
||||
if strings.Contains(a.Output, "{") || !strings.Contains(a.Output, "plan-1") || !strings.Contains(a.Output, "tier") {
|
||||
t.Errorf("the output is not the line a person reads: %q", a.Output)
|
||||
}
|
||||
}
|
||||
|
||||
// The verb composes the wait, and asks for a window as long as its bound.
|
||||
func TestThePlansVerbComposesAWait(t *testing.T) {
|
||||
argv, err := argvFor("plans", map[string]any{"wait": "plan-1", "until": "done", "bound": "15s"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strings.Join(argv, " "); got != "plans wait plan-1 --until done --bound 15s" {
|
||||
t.Fatalf("composed %q", got)
|
||||
}
|
||||
if _, err := argvFor("plans", map[string]any{"until": "done"}); err == nil {
|
||||
t.Error("until without wait was taken")
|
||||
}
|
||||
if _, err := argvFor("plans", map[string]any{"wait": "plan-1", "bound": "1h"}); err == nil {
|
||||
t.Error("a bound past what a call can wait was taken")
|
||||
}
|
||||
}
|
||||
|
||||
// Live, with the store: `plans wait` run as the verb runs it answers when the plan reaches its next tier, as
|
||||
// data, and says the bound passed when nothing moved (novox/hq issue 229).
|
||||
func TestAPlanFollowedFromTheStoreAnswersWhenItMoves(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
// Read every 50ms rather than every second, so the wait answers as soon as the plan moves and the bound
|
||||
// below passes in a fraction of a second (novox/hq issue 591); the move still waits 1.5s, long enough
|
||||
// for the wait to have read the plan before it.
|
||||
was := planWaitEvery
|
||||
planWaitEvery = 50 * time.Millisecond
|
||||
t.Cleanup(func() { planWaitEvery = was })
|
||||
now := time.Now().UTC()
|
||||
walk := inventory.Plan{ID: "plan-229", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c229c229",
|
||||
Created: now, State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {}, "b": {}}}
|
||||
if err := open.inventory.SavePlan(ctx, &walk); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
moved := make(chan error, 1)
|
||||
go func() {
|
||||
time.Sleep(1500 * time.Millisecond)
|
||||
p, err := open.inventory.PlanByID(ctx, walk.ID)
|
||||
if err == nil {
|
||||
p.Tier, p.State = 1, inventory.PlanRolling
|
||||
err = open.inventory.SavePlan(ctx, &p)
|
||||
}
|
||||
moved <- err
|
||||
}()
|
||||
began := time.Now()
|
||||
var got planWaited
|
||||
out := printed(t, func() error {
|
||||
return plansCommand(ctx, []string{"wait", walk.ID, "--until", "tier", "--bound", "15s"})
|
||||
})
|
||||
if err := <-moved; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal([]byte(out), &got); err != nil {
|
||||
t.Fatalf("plans wait did not answer data: %v: %q", err, out)
|
||||
}
|
||||
if got.Came != "tier" || got.Plan.Tier != 1 || got.Plan.State != inventory.PlanRolling {
|
||||
t.Fatalf("the wait answered %+v", got)
|
||||
}
|
||||
if waited := time.Since(began); waited > 3*time.Second {
|
||||
t.Errorf("it answered after %s, not when the plan moved", waited)
|
||||
}
|
||||
out = printed(t, func() error {
|
||||
return plansCommand(ctx, []string{"wait", walk.ID, "--bound", "300ms"})
|
||||
})
|
||||
if err := json.Unmarshal([]byte(out), &got); err != nil || got.Came != "bound" || got.Until != "moved" {
|
||||
t.Fatalf("a plan that did not move answered %+v (%v)", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The verb's schema says the longest bound in words, and the words are the value (review 75 on #266): the
|
||||
// catalogue cannot read maxPlanWait, so this holds them together.
|
||||
func TestThePlansSchemaSaysTheLongestBoundThereIs(t *testing.T) {
|
||||
v, ok := controllerVerb("plans")
|
||||
if !ok {
|
||||
t.Fatal("no plans verb")
|
||||
}
|
||||
props, _ := v.Input["properties"].(map[string]any)
|
||||
bound, _ := props["bound"].(map[string]any)
|
||||
said, _ := bound["description"].(string)
|
||||
if !strings.Contains(said, maxPlanWait.String()+", the default, is the longest") {
|
||||
t.Fatalf("the schema says %q; the longest bound is %s", said, maxPlanWait)
|
||||
}
|
||||
}
|
||||
@@ -1027,6 +1027,9 @@ func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (stri
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// A send that carried the mesh's record of the controller and the node-engine — a person's push included —
|
||||
// ends the condition of one kept over it (novox/hq issue 201).
|
||||
settleCoreDowngrades(ctx, b.open.inventory, s.node, s.declared.Builds)
|
||||
if len(s.declared.Bindings) > 0 {
|
||||
// And where it bound each consumer of a provision that keeps its data (novox/hq ADR 0232):
|
||||
// what the next resolution keeps it at. On the same outliving context as the send's record.
|
||||
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
@@ -81,6 +80,30 @@ func listQueue(ctx context.Context, args []string, w io.Writer) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// keepingTheQueueInOrder runs the build queue's passes while this controller acts (novox/hq issue 558):
|
||||
// the one process that puts the queue in order, so no two passes pause and resume it over each other.
|
||||
// The build seat is read from the catalogue at most every half minute.
|
||||
func keepingTheQueueInOrder(ctx context.Context, open *stores, js *broker.JetStream) {
|
||||
if js == nil {
|
||||
return
|
||||
}
|
||||
seat, readAt := link.TheBuildMachine, time.Time{}
|
||||
seatNow := func() string {
|
||||
if time.Since(readAt) < watchEvery {
|
||||
return seat
|
||||
}
|
||||
if entries, err := open.inventory.Catalogued(ctx); err == nil {
|
||||
seat, readAt = buildSeatAmong(entries), time.Now()
|
||||
} else {
|
||||
fmt.Printf("the build seat is read as %s: the catalogue cannot be read: %v\n", seat, err)
|
||||
}
|
||||
return seat
|
||||
}
|
||||
link.KeepingInOrder(ctx, js, seatNow, link.Holding, func(format string, args ...any) {
|
||||
fmt.Printf(format+"\n", args...)
|
||||
})
|
||||
}
|
||||
|
||||
// queueAnswer is the queue as data (novox/hq issue 455): what a caller reads to say how far back an ask
|
||||
// waits — "waiting in the build queue behind 2" — which the table alone could not tell it. Text is the
|
||||
// table, written from the same read: the verb answers it as its output and the rest as its answer, so the
|
||||
@@ -100,7 +123,8 @@ type queueAnswer struct {
|
||||
type queuedAnswer struct {
|
||||
ID string `json:"id"`
|
||||
State string `json:"state"`
|
||||
// Kind is "check" for a pull request's merge check, "build" for anything else.
|
||||
// Kind is "delivery" for a build a walk waits for, "check" for a pull request's check and any ask
|
||||
// whose kind cannot be told (link.KindOf, novox/hq issue 558).
|
||||
Kind string `json:"kind"`
|
||||
Repository string `json:"repository"`
|
||||
Path string `json:"path,omitempty"`
|
||||
@@ -117,6 +141,9 @@ type queuedAnswer struct {
|
||||
// Position is its place among the waiting, 1 being handed out next; 0 for an ask not waiting.
|
||||
Position int `json:"position,omitempty"`
|
||||
Seq uint64 `json:"seq"`
|
||||
// Delivery is the how-manyth time an ask in flight was handed out; more than 1 is a restart
|
||||
// (novox/hq issue 594).
|
||||
Delivery int `json:"delivery,omitempty"`
|
||||
}
|
||||
|
||||
// aCommitRef is a ref that names a commit by its full hash (SHA-1 or SHA-256).
|
||||
@@ -129,10 +156,7 @@ func queueAnswerOf(q link.Queue, now time.Time) queueAnswer {
|
||||
out := queueAnswer{Seat: q.Seat, Waiting: len(waiting), InFlight: len(running), Dead: len(dead),
|
||||
MaxDeliver: q.MaxDeliver, Asks: []queuedAnswer{}, Text: queueText(q, now)}
|
||||
one := func(a link.QueuedAsk, position int) queuedAnswer {
|
||||
kind := "build"
|
||||
if a.Request.Check != nil {
|
||||
kind = "check"
|
||||
}
|
||||
kind := link.KindOf(a.Request)
|
||||
since := a.AskedAt
|
||||
if a.State == link.AskInFlight && a.On != "" && !a.Started.IsZero() {
|
||||
since = a.Started
|
||||
@@ -143,7 +167,7 @@ func queueAnswerOf(q link.Queue, now time.Time) queueAnswer {
|
||||
}
|
||||
return queuedAnswer{ID: a.ID, State: a.State, Kind: kind, Repository: link.WithoutPassword(a.Repository), Path: a.Path,
|
||||
Ref: a.Ref, Commit: commit, Machine: a.On, WasOn: a.Was, AskedAt: a.AskedAt, Since: since,
|
||||
Position: position, Seq: a.Seq}
|
||||
Position: position, Seq: a.Seq, Delivery: a.Delivery}
|
||||
}
|
||||
for _, a := range running {
|
||||
out.Asks = append(out.Asks, one(a, 0))
|
||||
@@ -186,22 +210,27 @@ func queueText(q link.Queue, now time.Time) string {
|
||||
switch {
|
||||
case a.On != "":
|
||||
where = fmt.Sprintf("on %s for %s", a.On, now.Sub(a.Started).Round(time.Second))
|
||||
// Started again after an earlier taker left it (novox/hq issue 594).
|
||||
if a.Delivery > 1 {
|
||||
where += fmt.Sprintf(", handed out %d times", a.Delivery)
|
||||
}
|
||||
case a.Was != "":
|
||||
where = fmt.Sprintf("handed back by %s, to be handed out again", a.Was)
|
||||
}
|
||||
fmt.Fprintf(&b, " %-26s %s — %s, %s (seq %d)\n", a.ID, what(a), where, ago(a.AskedAt), a.Seq)
|
||||
fmt.Fprintf(&b, " %-26s %-8s %s — %s, %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), where,
|
||||
ago(a.AskedAt), a.Seq)
|
||||
}
|
||||
}
|
||||
if len(waiting) > 0 {
|
||||
fmt.Fprintln(&b, "\nwaiting:")
|
||||
fmt.Fprintln(&b, "\nwaiting, in the order they are handed out:")
|
||||
for _, a := range waiting {
|
||||
fmt.Fprintf(&b, " %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
|
||||
fmt.Fprintf(&b, " %-26s %-8s %s — %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), ago(a.AskedAt), a.Seq)
|
||||
}
|
||||
}
|
||||
if len(dead) > 0 {
|
||||
fmt.Fprintf(&b, "\ndead — handed out as often as the worker allows (%d) and never settled, still held:\n", q.MaxDeliver)
|
||||
for _, a := range dead {
|
||||
fmt.Fprintf(&b, " %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
|
||||
fmt.Fprintf(&b, " %-26s %-8s %s — %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), ago(a.AskedAt), a.Seq)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
@@ -275,6 +304,15 @@ func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat str
|
||||
}
|
||||
}
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
|
||||
// **Where it waits now** (novox/hq issue 558): moved into order since it was read, a waiting ask has a
|
||||
// later sequence, and the worker handing out what was behind its old one is not it being taken.
|
||||
if ask.State == link.AskWaiting {
|
||||
if q, err := link.ReadQueue(ctx, js, seat); err == nil {
|
||||
if now, ok := q.Find(ask.ID); ok && now.State == link.AskWaiting {
|
||||
ask.Seq = now.Seq
|
||||
}
|
||||
}
|
||||
}
|
||||
switch ask.State {
|
||||
case link.AskWaiting:
|
||||
if taken, err := takenSince(js, worker, ask.Seq); err != nil {
|
||||
@@ -296,14 +334,25 @@ func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat str
|
||||
"ends it there", ask.ID, started, ask.ID)
|
||||
}
|
||||
}
|
||||
if err := js.Context().DeleteMsg(worker.Stream, ask.Seq); err != nil && !errors.Is(err, nats.ErrMsgNotFound) &&
|
||||
!errors.Is(err, jetstream.ErrMsgNotFound) && !strings.Contains(err.Error(), "no message found") {
|
||||
// Found by its id when it was moved into order since it was read (novox/hq issue 558).
|
||||
removed, err := link.DeleteAsk(ctx, js, seat, ask)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s is marked cancelled and could not be deleted from the queue (seq %d): %w — a "+
|
||||
"holder taking it ends it as cancelled", ask.ID, ask.Seq, err)
|
||||
}
|
||||
recordCancelled(ctx, open, ask)
|
||||
return fmt.Sprintf("cancelled %s (%s, %s): deleted from the %s queue and recorded failed, %s — a plan "+
|
||||
"that asked for it fails with that", ask.ID, ask.Repository, ask.State, seat, link.CancelledByHand), nil
|
||||
return cancelSaid(ask, seat, removed), nil
|
||||
}
|
||||
|
||||
// cancelSaid is what a cancel says: deleted from the queue only when it was.
|
||||
func cancelSaid(ask link.QueuedAsk, seat string, removed uint64) string {
|
||||
where := fmt.Sprintf("deleted from the %s queue", seat)
|
||||
if removed == 0 {
|
||||
where = fmt.Sprintf("no longer in the %s queue to delete, and marked cancelled, so a holder taking it "+
|
||||
"ends it", seat)
|
||||
}
|
||||
return fmt.Sprintf("cancelled %s (%s, %s): %s and recorded failed, %s — a plan "+
|
||||
"that asked for it fails with that", ask.ID, ask.Repository, ask.State, where, link.CancelledByHand)
|
||||
}
|
||||
|
||||
// holderLooks is how long a cancel waits for a holder that took the ask before the mark to say it
|
||||
|
||||
@@ -24,7 +24,7 @@ func aQueueOfEachKind(now time.Time) link.Queue {
|
||||
State: link.AskInFlight, On: "ace", Started: started,
|
||||
Request: link.BuildRequest{ID: "build-running", Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-host"}}},
|
||||
{Seq: 3, ID: "build-first", Repository: "ssh://forge/novox/mesh-catalog", Ref: "main", AskedAt: asked,
|
||||
State: link.AskWaiting},
|
||||
State: link.AskWaiting, Request: link.BuildRequest{ID: "build-first", Repository: "ssh://forge/novox/mesh-catalog"}},
|
||||
{Seq: 4, ID: "build-second", Repository: "ssh://forge/novox/mesh-catalog", Path: "modules/gitea", Ref: aCommit,
|
||||
AskedAt: now.Add(-time.Minute), State: link.AskWaiting,
|
||||
Request: link.BuildRequest{ID: "build-second", Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog"}}},
|
||||
@@ -51,7 +51,7 @@ func TestTheQueueVerbAnswersEachAskAsData(t *testing.T) {
|
||||
t.Errorf("in flight: %+v", running)
|
||||
}
|
||||
first, second := byID["build-first"], byID["build-second"]
|
||||
if first.Position != 1 || first.Kind != "build" || first.Commit != "" || first.Ref != "main" || first.Machine != "" ||
|
||||
if first.Position != 1 || first.Kind != "delivery" || first.Commit != "" || first.Ref != "main" || first.Machine != "" ||
|
||||
!first.Since.Equal(now.Add(-5*time.Minute)) {
|
||||
t.Errorf("first waiting: %+v", first)
|
||||
}
|
||||
@@ -62,6 +62,13 @@ func TestTheQueueVerbAnswersEachAskAsData(t *testing.T) {
|
||||
if dead := byID["build-dead"]; dead.Position != 0 || dead.State != link.AskDead {
|
||||
t.Errorf("dead: %+v", dead)
|
||||
}
|
||||
// And the table says each ask's kind too (novox/hq issue 558).
|
||||
for _, line := range []string{"build-running check ", "build-first delivery ",
|
||||
"build-second check ", "build-dead check "} {
|
||||
if !strings.Contains(got.Text, line) {
|
||||
t.Errorf("the table does not say %q:\n%s", line, got.Text)
|
||||
}
|
||||
}
|
||||
// In the table's order: in flight, then the waiting by position, then the dead.
|
||||
var order []string
|
||||
for _, a := range got.Asks {
|
||||
@@ -135,3 +142,16 @@ func TestTheQueueNeverShowsARepositorysPassword(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A cancel says it deleted the ask only when it did: moved into order and gone since, it says so (novox/hq
|
||||
// issue 558).
|
||||
func TestACancelSaysDeletedOnlyWhenItDeleted(t *testing.T) {
|
||||
ask := link.QueuedAsk{ID: "build-1", Repository: "novox/x", State: link.AskWaiting, Seq: 4}
|
||||
if said := cancelSaid(ask, link.TheBuildMachine, 9); !strings.Contains(said, "deleted from the") {
|
||||
t.Errorf("removed, it says: %s", said)
|
||||
}
|
||||
if said := cancelSaid(ask, link.TheBuildMachine, 0); strings.Contains(said, "deleted from") ||
|
||||
!strings.Contains(said, "no longer in the") {
|
||||
t.Errorf("gone already, it says: %s", said)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,6 +135,17 @@ func recordedKept(ctx context.Context, open *stores, node string) (map[string]st
|
||||
}
|
||||
out[m] = was
|
||||
}
|
||||
// **And the controller and the node-engine at the build the machine was last sent, over an older record**
|
||||
// (novox/hq issue 201, core_downgrade.go). Under keepingEveryBuild they are kept already.
|
||||
if !every {
|
||||
core, err := coreKept(ctx, inv, node, sent, current)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for m, was := range core {
|
||||
out[m] = was
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -159,6 +170,12 @@ func keepRecorded(ctx context.Context, open *stores, node string, shelf map[stri
|
||||
return nil, fmt.Errorf("%s runs %s's build %s, which the build records no longer hold: the bus's user "+
|
||||
"list cannot be sent there alone with it kept (novox/hq issue 490)", node, m, short(kept[m]))
|
||||
}
|
||||
if !found && guarded(coreComponent(m)) {
|
||||
return nil, fmt.Errorf("%s was last sent %s's build %s, newer than the one the mesh holds, and the build records "+
|
||||
"no longer hold it: this send cannot keep it and does not move it back — `push %s` sends the older build on "+
|
||||
"a person's word, and a newer merge or `rebuild %s` moves the record forward (novox/hq issue 201)",
|
||||
node, m, short(kept[m]), node, m)
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("%s records rather than rolls out, and %s runs its build %s, which the build "+
|
||||
"records no longer hold: this send cannot keep it and does not move it — `push %s` sends the new "+
|
||||
|
||||
@@ -51,6 +51,9 @@ type sendScope struct {
|
||||
modules map[string]bool
|
||||
// person is a person's act: it carries what it carries.
|
||||
person bool
|
||||
// puttingBack is a gate's own put-back of its modules: the one send that may carry a core module to a build
|
||||
// older than the one its machine runs (novox/hq issue 201).
|
||||
puttingBack bool
|
||||
}
|
||||
|
||||
type sendScopeKey struct{}
|
||||
@@ -237,6 +240,32 @@ var machineMoves = func(ctx context.Context, open *stores, f moveFacts, node str
|
||||
return f.moves(node, modules, sent, known, all), nil
|
||||
}
|
||||
|
||||
// withoutKept is a machine's moves less those its send keeps at the build the machine was last sent, and so does
|
||||
// not make: the grants step keeps every module (novox/hq issue 490), and every send keeps the controller and the
|
||||
// node-engine over an older record (issue 201). Read as the send composes it (sendKeeps, set after the checks that
|
||||
// call this), so a move it could not keep is still counted.
|
||||
func withoutKept(ctx context.Context, open *stores, node string, moves []inventory.CarriedMove) ([]inventory.CarriedMove, error) {
|
||||
if len(moves) == 0 {
|
||||
return moves, nil
|
||||
}
|
||||
keeping := ctx
|
||||
if _, on := keptExcept(ctx); !on {
|
||||
keeping = keepingRecorded(ctx)
|
||||
}
|
||||
keeps, err := recordedKept(keeping, open, node)
|
||||
if err != nil || len(keeps) == 0 {
|
||||
return moves, err
|
||||
}
|
||||
var out []inventory.CarriedMove
|
||||
for _, mv := range moves {
|
||||
if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) {
|
||||
continue
|
||||
}
|
||||
out = append(out, mv)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ungatedIn refuses a send whose machines would move a build no gate has seen, outside its scope: the
|
||||
// machines named, and why; the machine holding the bus, added only for its user list, is left instead.
|
||||
func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder string) ([]string, error) {
|
||||
@@ -254,19 +283,11 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The grants step composes every module at the build its machine was last sent (novox/hq issue 490):
|
||||
// a move it keeps is not made. Read, not assumed, so a move it could not keep is still refused here.
|
||||
var keeps map[string]string
|
||||
if everyBuildKept(ctx, n) {
|
||||
if keeps, err = recordedKept(ctx, open, n); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if moves, err = withoutKept(ctx, open, n, moves); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var waiting []string
|
||||
for _, mv := range moves {
|
||||
if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) {
|
||||
continue
|
||||
}
|
||||
if !scope.judged[n] && !scope.modules[mv.Module] {
|
||||
waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To)))
|
||||
}
|
||||
@@ -308,6 +329,11 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
// A move the send keeps is not made, and is no move for the gate to judge (novox/hq issue 201): a core module
|
||||
// kept over an older record judged as moving back to it would read the gate as superseded by its own send.
|
||||
if moves, err = withoutKept(ctx, open, node, moves); err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
own := func(module string) bool {
|
||||
return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module })
|
||||
}
|
||||
|
||||
@@ -1463,6 +1463,10 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
modules := set.String("modules", "", "or the modules it would change, comma-separated")
|
||||
moduleDirs := set.String("module-dirs", "", "with --paths: the directories holding a module.json at the commit, "+
|
||||
"comma-separated, as the forge's announcer says them (novox/hq issue 278)")
|
||||
// Following a plan (novox/hq issue 229): `plans wait <id>`.
|
||||
until := set.String("until", "", "with wait: what to wait for — moved (any change), tier (its next tier) or done (its end)")
|
||||
bound := set.String("bound", "", "with wait: how long to wait at most, as 15s or seconds (default and at most the "+
|
||||
"longest a call is answered within)")
|
||||
// Ending a plan by hand is a repair, and says why (novox/hq to-be 45 §7).
|
||||
why := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args)
|
||||
@@ -1482,6 +1486,9 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
now := time.Now()
|
||||
if len(positionals) == 2 && positionals[0] == "wait" {
|
||||
return planWaitCommand(ctx, inv, positionals[1], *until, *bound)
|
||||
}
|
||||
if len(positionals) == 1 {
|
||||
p, err := inv.PlanByID(ctx, positionals[0])
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// TestReplay201ThePlansSend replays novox/hq issue 201 (2026-10-02): two merges a minute apart made two plans; the
|
||||
// controller's plan rolled the newer controller onto the control node, and the node-engine's plan then sent that
|
||||
// machine the controller build before it, whose binary refused the seat row the newer one had widened. Here the
|
||||
// control node, anchor, was last sent the newer controller c2 while the mesh's record of the controller is the
|
||||
// build before it, c1, which passed its gate. The outcome asserted: a send that is no person's push — composed as a
|
||||
// plan's send composes it (keepingRecorded, what sendKeeps gives every send through sendToEach) — carries the
|
||||
// controller at c2, the build the machine was last sent, and not the older record; a person's push still carries
|
||||
// the record. Written with only what the controller had before its fix (mesh-controller #273) — the inventory's
|
||||
// build, gate, assignment and send records, keepingRecorded, planFor and declarationFor — so it is laid over the
|
||||
// commit before; the fix's own TestReplay201 reads what the fix added. Needs the store (MESH_TEST_POSTGRES), and
|
||||
// is named in testdata/store-tests.
|
||||
func TestReplay201ThePlansSend(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
const older, newer = "c1aaaaaaaaaa", "c2bbbbbbbbbb"
|
||||
old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
|
||||
record := func(commit string, asked time.Time) inventory.Build {
|
||||
t.Helper()
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"})
|
||||
b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit,
|
||||
Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}}
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// The older controller rolled out and passed its gate; the newer one was asked after it and registered.
|
||||
before := record(older, old)
|
||||
if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: older,
|
||||
Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record(newer, now)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-controller", BuiltFrom: newer, Head: newer, Asked: now}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The controller's plan sent anchor the newer controller.
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": newer}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// And the mesh's record of the controller is the build before it — whatever made it so.
|
||||
if err := inv.RestoreModule(ctx, before); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The controller each send to anchor carries: a plan's send keeps recorded builds (keepingRecorded, what
|
||||
// sendKeeps gives every send but a person's push); a person's push composes the mesh's record.
|
||||
controllerSent := func(sends string, keeping bool) string {
|
||||
t.Helper()
|
||||
c := ctx
|
||||
if keeping {
|
||||
c = keepingRecorded(ctx)
|
||||
}
|
||||
plan, settings, err := planFor(c, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatalf("%s to anchor did not compose: %v", sends, err)
|
||||
}
|
||||
declared, err := declarationFor(c, open, "anchor", plan, settings)
|
||||
if err != nil {
|
||||
t.Fatalf("%s to anchor did not compose: %v", sends, err)
|
||||
}
|
||||
return declared.Builds["mesh-controller"]
|
||||
}
|
||||
|
||||
if got := controllerSent("a plan's send", true); got != newer {
|
||||
t.Fatalf("a plan's send carried anchor the controller %s, older than the %s anchor was last sent: the "+
|
||||
"older binary would read the row its successor widened", short(got), short(newer))
|
||||
}
|
||||
if got := controllerSent("a person's push", false); got != older {
|
||||
t.Fatalf("a person's push carried the controller %s, not the mesh's record %s", short(got), short(older))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// TestReplay401 replays novox/hq issue 401 (2026-10-10): the mounts module's smb-credentials, an own secret
|
||||
// only an outside party issues, was given by nobody, and the send made a random value for it and placed it on
|
||||
// the machine; the module read it as a credential given and wrong. The outcome asserted (ADR 0228): the send
|
||||
// makes no value for an own secret issued outside the mesh that nobody gave and places none, and a value the
|
||||
// mesh made for it before is not placed either; once a person gives one, that value is placed. Written with
|
||||
// only what the controller had before its fix (mesh-controller #262) — register, assign, planFor,
|
||||
// declarationFor and the inventory's secret reads — so it is laid over the commit before. Needs the store
|
||||
// (MESH_TEST_POSTGRES), and is named in testdata/store-tests.
|
||||
func TestReplay401(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
const made, outside = "/var/lib/mesh/mounts/password", "/var/lib/mesh/mounts/smb-credentials"
|
||||
register(t, open, catalogue.Manifest{Module: "mounts", Version: "1",
|
||||
OwnSecrets: catalogue.OwnSecrets{
|
||||
"password": {Path: made, Taken: catalogue.TakenAtStart},
|
||||
"smb-credentials": {Path: outside, Taken: catalogue.TakenAtStart, IssuedBy: catalogue.IssuedOutside},
|
||||
},
|
||||
Resources: []map[string]any{
|
||||
{"id": "state", "type": "directory", "path": "/var/lib/mesh/mounts", "mode": "0700"},
|
||||
}})
|
||||
if _, err := assign(ctx, open, "laptop", "mounts"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sent := func() []map[string]any {
|
||||
t.Helper()
|
||||
plan, settings, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, "laptop", plan, settings)
|
||||
if err != nil {
|
||||
t.Fatalf("the send to laptop did not compose: %v", err)
|
||||
}
|
||||
return declared.Resources
|
||||
}
|
||||
placed := func(resources []map[string]any, path string) map[string]any {
|
||||
for _, r := range resources {
|
||||
if r["path"] == path && r["sealed"] != nil {
|
||||
return r
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
first := sent()
|
||||
if placed(first, made) == nil {
|
||||
t.Fatal("the secret the mesh makes was not placed")
|
||||
}
|
||||
if got := placed(first, outside); got != nil {
|
||||
t.Errorf("the send placed a value for smb-credentials, which nobody gave: %v", got)
|
||||
}
|
||||
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "mounts", "smb-credentials"); err != nil || held {
|
||||
t.Fatalf("the send made a value for a secret issued outside the mesh (held %v, %v)", held, err)
|
||||
}
|
||||
|
||||
// A value the mesh made for it before the rule is not one a person gave: it is not placed.
|
||||
if _, err := open.inventory.SecretForModule(ctx, "laptop", "mounts", "smb-credentials"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := placed(sent(), outside); got != nil {
|
||||
t.Fatalf("a value the mesh made for a secret issued outside was placed: %v", got)
|
||||
}
|
||||
|
||||
// Given by a person, it is placed, as given.
|
||||
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "mounts", "smb-credentials", "the share's credentials"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
given, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "mounts", "smb-credentials")
|
||||
if err != nil || !held {
|
||||
t.Fatalf("the value given is not held (%v)", err)
|
||||
}
|
||||
if got := placed(sent(), outside); got == nil || got["sealed"] != given {
|
||||
t.Fatalf("the value given is placed as %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A check that ran again says so on the pull request** (novox/hq issue 594): a build agent replaced
|
||||
// mid-check leaves the ask to be handed out again, and the verdict that finally comes is from a second
|
||||
// run. Each layer's summary — what the forge shows as the check's line — names the restart.
|
||||
func TestARestartedCheckSaysSoOnEveryLayer(t *testing.T) {
|
||||
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-controller", Number: 279}
|
||||
gate := &link.CheckLayer{Verdict: "pass", Summary: "every node composes"}
|
||||
repo := &link.CheckLayer{Verdict: "pass", Summary: "merge-check.sh passed"}
|
||||
c := checkedOf(link.BuildResult{ID: "b", Ref: "abc", On: "shanks", Checked: asked, Delivery: 2,
|
||||
Check: &link.CheckOutcome{Verdict: "pass", Summary: "every node composes", Gate: gate, RepoCheck: repo}})
|
||||
for name, said := range map[string]string{"summary": c.Summary, "gate": c.Gate.Summary,
|
||||
"repository check": c.RepoCheck.Summary} {
|
||||
// The verdict's own words first, then the marker: the forge clips a status to a few words.
|
||||
if !strings.HasSuffix(said, " (run 2)") || strings.HasPrefix(said, "(run") {
|
||||
t.Errorf("the %s says %q: a check run twice reads as run once, or its marker pushes its words out", name, said)
|
||||
}
|
||||
}
|
||||
if !strings.HasPrefix(c.Report, "restarted: handed out 2 times") {
|
||||
t.Errorf("the report does not open with the restart: %q", c.Report)
|
||||
}
|
||||
if gate.Summary != "every node composes" {
|
||||
t.Errorf("the build's own outcome was changed under it: %q", gate.Summary)
|
||||
}
|
||||
|
||||
// Taken once, nothing is added.
|
||||
once := checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Delivery: 1,
|
||||
Check: &link.CheckOutcome{Verdict: "pass", Summary: "every node composes"}})
|
||||
if strings.Contains(once.Summary, "(run") || strings.Contains(once.Report, "restarted") {
|
||||
t.Errorf("a check run once says %q", once.Summary)
|
||||
}
|
||||
}
|
||||
|
||||
// And the queue says it of the ask in flight.
|
||||
func TestTheQueueSaysAnAskInFlightRestarted(t *testing.T) {
|
||||
now := time.Date(2026, 10, 13, 0, 16, 0, 0, time.UTC)
|
||||
q := link.Queue{Seat: link.TheBuildMachine, Delivered: 1, AckPending: 1, MaxDeliver: 5, Asks: []link.QueuedAsk{
|
||||
{Seq: 1, ID: "build-279", Repository: "ssh://forge/novox/mesh-controller", AskedAt: now.Add(-7 * time.Minute),
|
||||
State: link.AskInFlight, On: "shanks", Started: now.Add(-time.Minute), Delivery: 2},
|
||||
}}
|
||||
text := queueText(q, now)
|
||||
if !strings.Contains(text, "handed out 2 times") {
|
||||
t.Fatalf("the queue does not say the ask was handed out again:\n%s", text)
|
||||
}
|
||||
if a := queueAnswerOf(q, now).Asks[0]; a.Delivery != 2 {
|
||||
t.Errorf("the queue's data says delivery %d, want 2", a.Delivery)
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,14 @@ import (
|
||||
// shape fails the suite, naming its source. The keeper would say such a summary in words at run time;
|
||||
// this is where the producer is made to say it rightly in the first place.
|
||||
func TestMain(m *testing.M) {
|
||||
// **A command line this suite runs ends when it is done** (novox/hq issue 591). Many tests run this test
|
||||
// binary as a verb's command line, and a binary built with the race detector sleeps a second before it
|
||||
// exits (GORACE's atexit_sleep_ms), which was more than the rest of each such test. The children still
|
||||
// run under the race detector and still fail on a race they find; they only stop waiting at their end.
|
||||
// Set before any child starts, and only where nobody chose GORACE.
|
||||
if _, chosen := os.LookupEnv("GORACE"); !chosen {
|
||||
_ = os.Setenv("GORACE", "atexit_sleep_ms=0")
|
||||
}
|
||||
// The process a mesh-cli test runs as a command line: it says the verb and the caller it was given, and
|
||||
// ends (meshcli_test.go).
|
||||
// The process a mesh-cli test runs as `secret accept`: it reads its value exactly as `secret accept` does
|
||||
|
||||
@@ -34,8 +34,10 @@ type seatRow struct {
|
||||
Delivers string `json:"delivers,omitempty"`
|
||||
Decision string `json:"decision"`
|
||||
// Replicated says the seat may be held on several machines at once (novox/hq ADR 0223).
|
||||
Replicated bool `json:"replicated,omitempty"`
|
||||
Holders []seatHolder `json:"holders"`
|
||||
Replicated bool `json:"replicated,omitempty"`
|
||||
// Paths are the paths on its machine the seat's holder owns (novox/hq ADR 0287).
|
||||
Paths []catalogue.SeatPath `json:"paths,omitempty"`
|
||||
Holders []seatHolder `json:"holders"`
|
||||
}
|
||||
|
||||
// seatsHeld is every seat the mesh defines with its holders, and every claim held that names no
|
||||
@@ -49,7 +51,7 @@ func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []cata
|
||||
rows := make([]seatRow, 0, len(seats))
|
||||
for _, s := range seats {
|
||||
row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
|
||||
Replicated: s.Replicated, Holders: []seatHolder{}}
|
||||
Replicated: s.Replicated, Paths: s.Paths, Holders: []seatHolder{}}
|
||||
seen := map[seatHolder]bool{}
|
||||
for _, h := range held {
|
||||
// Resolve the held claim to a seat rather than comparing names, so a record naming a
|
||||
@@ -255,12 +257,16 @@ func seatsCommand(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
rows, outside := seatsHeld(catalogue.Seats(), world.Held)
|
||||
// Which module owns each seat's path on each machine (novox/hq ADR 0287): the seat naming it, and
|
||||
// that seat's holder there, with the stepping back applied.
|
||||
owners := catalogue.PathOwners(catalogue.Seats(), world.Held)
|
||||
|
||||
if *asJSON {
|
||||
out := struct {
|
||||
Seats []seatRow `json:"seats"`
|
||||
Outside []catalogue.Held `json:"outside,omitempty"`
|
||||
}{rows, outside}
|
||||
Seats []seatRow `json:"seats"`
|
||||
Owners []catalogue.PathOwner `json:"path-owners,omitempty"`
|
||||
Outside []catalogue.Held `json:"outside,omitempty"`
|
||||
}{rows, owners, outside}
|
||||
body, err := json.MarshalIndent(out, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -270,7 +276,7 @@ func seatsCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
|
||||
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintln(w, "SEAT\tSCOPE\tDELIVERS\tHELD BY")
|
||||
fmt.Fprintln(w, "SEAT\tSCOPE\tDELIVERS\tPATHS\tHELD BY")
|
||||
for _, r := range rows {
|
||||
delivers := r.Delivers
|
||||
if delivers == "" {
|
||||
@@ -284,11 +290,22 @@ func seatsCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
holders = strings.Join(parts, ", ")
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", r.Seat, r.Scope, delivers, holders)
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", r.Seat, r.Scope, delivers, seatPathsText(r.Paths), holders)
|
||||
}
|
||||
if err := w.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(owners) > 0 {
|
||||
fmt.Println("\nwho owns each seat's path, by machine (novox/hq ADR 0287):")
|
||||
w = tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintln(w, "PATH\tMACHINE\tSEAT\tMODULE")
|
||||
for _, o := range owners {
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", o.Path, o.Node, o.Seat, o.Module)
|
||||
}
|
||||
if err := w.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(outside) > 0 {
|
||||
fmt.Println("\nheld, and not a seat this mesh defines (registered before the set closed — novox/hq ADR 0110):")
|
||||
for _, h := range outside {
|
||||
@@ -297,3 +314,19 @@ func seatsCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// seatPathsText is a seat's paths in one column: each path, and the seat it steps back for, if any.
|
||||
func seatPathsText(paths []catalogue.SeatPath) string {
|
||||
if len(paths) == 0 {
|
||||
return "—"
|
||||
}
|
||||
parts := make([]string, 0, len(paths))
|
||||
for _, p := range paths {
|
||||
if p.StepsBackFor != "" {
|
||||
parts = append(parts, p.Path+" (steps back for "+p.StepsBackFor+")")
|
||||
continue
|
||||
}
|
||||
parts = append(parts, p.Path)
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
@@ -74,3 +74,22 @@ func TestAKindedBenchIsNotHandedOver(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The listing shows each seat's paths, and the seat a path steps back for (novox/hq ADR 0287).
|
||||
func TestTheSeatsListingShowsEachSeatsPaths(t *testing.T) {
|
||||
rows, _ := seatsHeld(catalogue.Seats(), nil)
|
||||
got := map[string]string{}
|
||||
for _, r := range rows {
|
||||
got[r.Seat] = seatPathsText(r.Paths)
|
||||
}
|
||||
for seat, want := range map[string]string{
|
||||
"node-hostname": "/etc/hosts, /etc/hostname",
|
||||
"node-uplink": "/etc/resolv.conf (steps back for node-resolver)",
|
||||
"node-resolver": "/etc/resolv.conf",
|
||||
"node-packet-filter": "—",
|
||||
} {
|
||||
if got[seat] != want {
|
||||
t.Errorf("%s lists its paths as %q, not %q", seat, got[seat], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -336,6 +336,9 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
}
|
||||
return argv, nil
|
||||
case "plans":
|
||||
if err := plainPlanIDs(a.given); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r := str("repository"); r != "" {
|
||||
argv := []string{"plans", "--what-if", r}
|
||||
if p := str("paths"); p != "" {
|
||||
@@ -365,6 +368,21 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
return argv, nil
|
||||
}
|
||||
}
|
||||
if id := str("wait"); id != "" {
|
||||
// Following a plan (novox/hq issue 229): checked here too, so a call asking what no wait can do is
|
||||
// refused before a command is started.
|
||||
if _, _, err := planWaitAsked(a.given["until"], a.given["bound"]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"plans", "wait", id}
|
||||
if u := str("until"); u != "" {
|
||||
argv = append(argv, "--until", u)
|
||||
}
|
||||
if b := str("bound"); b != "" {
|
||||
argv = append(argv, "--bound", b)
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
if id := str("id"); id != "" {
|
||||
return []string{"plans", id}, nil
|
||||
}
|
||||
@@ -985,12 +1003,14 @@ var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "co
|
||||
// The delivery's owner's verbs answer JSON where they read (plan, order, check, walks) — novox/hq ADR 0239.
|
||||
"delivery": true,
|
||||
// The build queue, each ask with its position among the waiting (novox/hq issue 455).
|
||||
"queue": true}
|
||||
"queue": true,
|
||||
// A plan followed (novox/hq issue 229): `plans wait` answers JSON; its other forms say text, and so answer no data.
|
||||
"plans": true}
|
||||
|
||||
// tableBesideVerbs are the JSON verbs whose document carries the table a person reads as its `text`: the
|
||||
// verb answers that table as its output, as it did before it answered data, and the rest as `answer` —
|
||||
// both from the one read, so they never disagree (novox/hq issue 455).
|
||||
var tableBesideVerbs = map[string]bool{"queue": true}
|
||||
var tableBesideVerbs = map[string]bool{"queue": true, "plans": true}
|
||||
|
||||
// overviewVerbs are the JSON verbs whose answer is said once, as data, and not again as the text the
|
||||
// command printed: the overviews, which grow with the mesh (novox/hq issue 314). Every reader of theirs
|
||||
@@ -1264,6 +1284,16 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
// Whatever it did, `status` is composed again once it has.
|
||||
defer statusFrom.nudge()
|
||||
}
|
||||
if verb == "plans" && len(argv) > 1 && argv[1] == "wait" {
|
||||
// A wait holds its answer up to its bound (novox/hq issue 229), and is answered in full inside it.
|
||||
asked := ""
|
||||
if i := slices.Index(argv, "--bound"); i > 0 && i+1 < len(argv) {
|
||||
asked = argv[i+1]
|
||||
}
|
||||
if _, bound, err := planWaitAsked("", asked); err == nil {
|
||||
link.AnswersWithin(ctx, bound+planWaitStart)
|
||||
}
|
||||
}
|
||||
if answer, read := readHere(ctx, argv); read {
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
@@ -739,6 +739,9 @@ func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus li
|
||||
// and the answer chosen is performed on its warrant.
|
||||
startAsking(watching, open, server, bus.Conn, keeper)
|
||||
go forgettingOldHeals(watching, open.inventory)
|
||||
// And the build queue's order (novox/hq issue 558): a check that has waited its while goes ahead of the
|
||||
// deliveries asked after it even when nothing new is asked.
|
||||
go keepingTheQueueInOrder(watching, open, server.JetStream())
|
||||
fmt.Printf("watching the mesh: %d signal(s) every %s, %d probe(s) every %s; what is wrong is kept in %s "+
|
||||
"and said as %s events\n", len(watchedRows()), watchEvery, len(runnableProbes()), doctorEvery,
|
||||
broker.ConditionsBucket, conditions.Seat)
|
||||
|
||||
@@ -8,6 +8,7 @@ require (
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
github.com/novox/mesh-host v0.0.0
|
||||
github.com/therootcompany/xz v1.0.1
|
||||
golang.org/x/crypto v0.57.0
|
||||
golang.org/x/net v0.58.0
|
||||
golang.org/x/sys v0.48.0
|
||||
|
||||
@@ -38,6 +38,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
|
||||
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||
|
||||
BIN
Binary file not shown.
@@ -14,13 +14,13 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/therootcompany/xz"
|
||||
)
|
||||
|
||||
// An archive whose source is somebody else's published file, pinned by URL and sha256 (novox/hq ADR 0316
|
||||
@@ -183,7 +183,6 @@ func unpackUpstream(ctx context.Context, file, into, within string) error {
|
||||
buffered := bufio.NewReader(f)
|
||||
magic, _ := buffered.Peek(6)
|
||||
var stream io.Reader
|
||||
var wait func() error
|
||||
switch {
|
||||
case bytes.HasPrefix(magic, []byte{0x1f, 0x8b}):
|
||||
zipped, err := gzip.NewReader(buffered)
|
||||
@@ -195,53 +194,33 @@ func unpackUpstream(ctx context.Context, file, into, within string) error {
|
||||
case bytes.HasPrefix(magic, []byte("BZh")):
|
||||
stream = bzip2.NewReader(buffered)
|
||||
case bytes.HasPrefix(magic, []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}):
|
||||
// Go's standard library reads no xz; the build seat's image carries xz (cmd/mesh-builder/Dockerfile).
|
||||
if _, err := exec.LookPath("xz"); err != nil {
|
||||
return errors.New("it is compressed with xz, and this build seat has no xz to read it")
|
||||
// Read in Go (github.com/therootcompany/xz, a port of xz-embedded), never by a tool the build seat's image
|
||||
// may not carry: the seat's image is the catalogue's build-agent recipe, which installs git and docker-cli
|
||||
// alone, and a release pinned as .tar.xz failed there on 2026-10-12 (hq issue 345). xzDictMax is a real
|
||||
// maximum: a block whose LZMA2 dictionary is larger is refused before its dictionary is allocated
|
||||
// (ErrMemlimit). The reader keeps a filter decoder for every block of a stream, so the file's framing is
|
||||
// walked first and a file of more blocks than xzBlocksMax is refused before anything is decoded
|
||||
// (xz_framing.go). What the stream unpacks to is bounded by untar (maxUpstream, maxEntries).
|
||||
if err := xzWithinLimits(file); err != nil {
|
||||
return err
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "xz", "-dc")
|
||||
cmd.Stdin = buffered
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
out, err := cmd.StdoutPipe()
|
||||
zipped, err := xz.NewReader(buffered, xzDictMax)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("it is not an xz stream this build seat reads (a dictionary of %d MiB at most): %v",
|
||||
xzDictMax>>20, err)
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
stream = out
|
||||
wait = func() error {
|
||||
if err := cmd.Wait(); err != nil {
|
||||
return fmt.Errorf("xz could not read it: %v %s", err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
defer func() {
|
||||
if wait != nil {
|
||||
_ = cmd.Process.Kill()
|
||||
_ = cmd.Wait()
|
||||
}
|
||||
}()
|
||||
stream = zipped
|
||||
default:
|
||||
return errors.New("it is not a tar archive compressed with gzip, bzip2 or xz; a single file is kept " +
|
||||
"whole by naming it in `file`")
|
||||
}
|
||||
if err := untar(stream, into, within); err != nil {
|
||||
return err
|
||||
}
|
||||
if wait != nil {
|
||||
// Whatever follows the tar's end is read before xz is waited for: xz blocks on a full pipe otherwise.
|
||||
if _, err := io.Copy(io.Discard, stream); err != nil {
|
||||
return err
|
||||
}
|
||||
w := wait
|
||||
wait = nil
|
||||
return w()
|
||||
}
|
||||
return nil
|
||||
return untar(stream, into, within)
|
||||
}
|
||||
|
||||
// xzDictMax is the largest xz dictionary read: xz's own presets use at most 64 MiB (-9); a stream asking for
|
||||
// more is refused rather than given the memory.
|
||||
const xzDictMax = 64 << 20
|
||||
|
||||
// untar writes a tar stream's files and directories within `within` into a directory.
|
||||
func untar(stream io.Reader, into, within string) error {
|
||||
reader := tar.NewReader(stream)
|
||||
|
||||
@@ -6,8 +6,10 @@ import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
@@ -15,6 +17,7 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -383,3 +386,84 @@ func TestARedirectOffHttpsIsNotFollowed(t *testing.T) {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnXzReleaseIsReadWithoutAnyXzToolOnTheBuildSeat(t *testing.T) {
|
||||
// The build seat's image (the catalogue's build-agent recipe) carries git and docker-cli alone: Betterbird's
|
||||
// .tar.xz failed there on 2026-10-12 with "this build seat has no xz to read it" (hq issue 345). It is
|
||||
// read in Go, with nothing on PATH.
|
||||
t.Setenv("PATH", "")
|
||||
body, err := os.ReadFile(filepath.Join("testdata", "release.tar.xz"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v := &vendor{files: map[string][]byte{"/r.tar.xz": body}}
|
||||
base := serveVendor(t, v)
|
||||
packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
|
||||
URL: base + "/r.tar.xz", SHA256: sumHex(body), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e := entriesOf(t, packed)
|
||||
if len(e) != 2 || e["betterbird"] != "#!/bin/sh\n -rwxr-xr-x" || e["defaults/p.js"] != "pref(\"a\", 1);\n -rw-r--r--" {
|
||||
t.Fatalf("%v", e)
|
||||
}
|
||||
}
|
||||
|
||||
// withDictionary is an xz stream whose first block's LZMA2 dictionary property is props, its header's CRC32
|
||||
// recomputed: a stream asking for a dictionary of its own choosing.
|
||||
func withDictionary(t *testing.T, stream []byte, props byte) []byte {
|
||||
t.Helper()
|
||||
out := append([]byte{}, stream...)
|
||||
const at = 12 // the stream header
|
||||
size := (int(out[at]) + 1) * 4
|
||||
header := out[at : at+size]
|
||||
flags := header[1]
|
||||
i := 2
|
||||
readVarint := func() {
|
||||
for header[i]&0x80 != 0 {
|
||||
i++
|
||||
}
|
||||
i++
|
||||
}
|
||||
if flags&0x40 != 0 {
|
||||
readVarint()
|
||||
}
|
||||
if flags&0x80 != 0 {
|
||||
readVarint()
|
||||
}
|
||||
if header[i] != 0x21 || header[i+1] != 0x01 {
|
||||
t.Fatalf("the fixture's first filter is not LZMA2: % x", header)
|
||||
}
|
||||
header[i+2] = props
|
||||
binary.LittleEndian.PutUint32(header[size-4:], crc32.ChecksumIEEE(header[:size-4]))
|
||||
return out
|
||||
}
|
||||
|
||||
func TestAnXzStreamAskingForTooLargeADictionaryIsRefused(t *testing.T) {
|
||||
body, err := os.ReadFile(filepath.Join("testdata", "release.tar.xz"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 1 GiB (props 36) and 4 GiB less one (props 40): over the 64 MiB maximum, refused before allocation.
|
||||
for _, props := range []byte{36, 40} {
|
||||
crafted := withDictionary(t, body, props)
|
||||
v := &vendor{files: map[string][]byte{"/r.tar.xz": crafted}}
|
||||
base := serveVendor(t, v)
|
||||
var before, after runtime.MemStats
|
||||
runtime.GC()
|
||||
runtime.ReadMemStats(&before)
|
||||
_, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
|
||||
URL: base + "/r.tar.xz", SHA256: sumHex(crafted), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
|
||||
runtime.ReadMemStats(&after)
|
||||
if err == nil || !strings.Contains(err.Error(), "dictionary") {
|
||||
t.Errorf("props %d: a dictionary over the maximum was not refused: %v", props, err)
|
||||
}
|
||||
if grew := after.TotalAlloc - before.TotalAlloc; grew > 256<<20 {
|
||||
t.Errorf("props %d: %d bytes were allocated for a refused stream", props, grew)
|
||||
}
|
||||
}
|
||||
// The fixture as it is (a small dictionary) still reads.
|
||||
if crafted := withDictionary(t, body, 0); len(crafted) == 0 {
|
||||
t.Fatal("no fixture")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
// xzBlocksMax and xzStreamsMax are the most blocks and the most concatenated streams an xz file may hold for
|
||||
// the build seat to read it.
|
||||
//
|
||||
// **Why a block count is limited at all.** The xz reader (github.com/therootcompany/xz v1.0.1) keeps one
|
||||
// filter decoder for every block that names a delta or BCJ filter, and frees them only when a stream ends:
|
||||
// its block counters are reset per stream, not per block. A stream of many tiny blocks therefore grows the
|
||||
// heap with its length: an `xz --block-size=1 --delta` file of 6 MB kept about 67 MiB live, which comes to
|
||||
// about 5.6 GiB at the 512 MiB download cap (review 68 on mesh-controller pull request 264, hq issue 345).
|
||||
//
|
||||
// **Why 1,024.** At about 340 bytes of decoder a block, 1,024 blocks keep at most about 0.35 MiB live. A
|
||||
// real release is far below it: Betterbird's archive is 1 stream with 1 block, and multi-threaded xz cuts
|
||||
// blocks of three times the dictionary (24 MiB at the default preset), so even 512 MiB unpacked in blocks
|
||||
// of 512 KiB stays within it. Streams are limited to 16 because a release is one stream; the decoders are
|
||||
// reused from one stream to the next, so the stream limit only bounds the work of this walk.
|
||||
const (
|
||||
xzBlocksMax = 1024
|
||||
xzStreamsMax = 16
|
||||
)
|
||||
|
||||
// xzWithinLimits walks an xz file's framing, without decompressing anything, and refuses one holding more
|
||||
// blocks or streams than the build seat reads.
|
||||
//
|
||||
// **The framing, not the index.** Each stream's index at its end lists its blocks, but the reader compares
|
||||
// the index with the blocks only after it has decoded them all, so a file whose index lies would still be
|
||||
// decoded block by block. This walk follows the same framing the reader follows: stream header, each block
|
||||
// header, the block's LZMA2 chunks (each chunk header carries its own compressed size, so a chunk is skipped
|
||||
// without being decoded), its padding and check, then the index and the footer. It counts the blocks the
|
||||
// reader would decode, and it also refuses an index whose count of blocks is not the count walked.
|
||||
func xzWithinLimits(file string) error {
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
_, _, err = xzFraming(bufio.NewReader(f))
|
||||
if errors.Is(err, errXzTooManyBlocks) || errors.Is(err, errXzTooManyStreams) {
|
||||
return err
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("it is not an xz file this build seat reads: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
errXzTooManyBlocks = fmt.Errorf("it holds more than %d xz blocks, the most this build seat reads: "+
|
||||
"its xz reader keeps a filter decoder for every block (hq issue 345)", xzBlocksMax)
|
||||
errXzTooManyStreams = fmt.Errorf("it holds more than %d concatenated xz streams, the most this build "+
|
||||
"seat reads", xzStreamsMax)
|
||||
)
|
||||
|
||||
// xzFraming counts the streams and blocks of an xz file, checking its framing as it goes. It stops with
|
||||
// errXzTooManyBlocks or errXzTooManyStreams at the first block or stream over the limit.
|
||||
func xzFraming(r *bufio.Reader) (streams, blocks int, err error) {
|
||||
full := func(n int) ([]byte, error) {
|
||||
b := make([]byte, n)
|
||||
if _, err := io.ReadFull(r, b); err != nil {
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
skip := func(n int) error {
|
||||
if got, err := r.Discard(n); err != nil || got != n {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for {
|
||||
header, err := full(12)
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: header: %v", streams+1, err)
|
||||
}
|
||||
if !bytes.Equal(header[:6], []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}) {
|
||||
return streams, blocks, fmt.Errorf("stream %d: no xz stream header", streams+1)
|
||||
}
|
||||
if header[6] != 0 || header[7]&0xf0 != 0 ||
|
||||
crc32.ChecksumIEEE(header[6:8]) != binary.LittleEndian.Uint32(header[8:12]) {
|
||||
return streams, blocks, fmt.Errorf("stream %d: a corrupt stream header", streams+1)
|
||||
}
|
||||
if streams++; streams > xzStreamsMax {
|
||||
return streams, blocks, errXzTooManyStreams
|
||||
}
|
||||
checkSize := 0
|
||||
if check := int(header[7]); check != 0 {
|
||||
checkSize = 4 << ((check - 1) / 3)
|
||||
}
|
||||
inStream := 0
|
||||
for {
|
||||
first, err := r.ReadByte()
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: no index: %v", streams, io.ErrUnexpectedEOF)
|
||||
}
|
||||
if first == 0 {
|
||||
break // the index indicator
|
||||
}
|
||||
rest, err := full(int(first)*4 + 3)
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: header: %v", streams, inStream+1, err)
|
||||
}
|
||||
if err := xzBlockHeader(append([]byte{first}, rest...)); err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: %v", streams, inStream+1, err)
|
||||
}
|
||||
inStream++
|
||||
if blocks++; blocks > xzBlocksMax {
|
||||
return streams, blocks, errXzTooManyBlocks
|
||||
}
|
||||
// The block's LZMA2 chunks, up to the end marker.
|
||||
size := 0
|
||||
for {
|
||||
control, err := r.ReadByte()
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: %v", streams, inStream, io.ErrUnexpectedEOF)
|
||||
}
|
||||
size++
|
||||
if control == 0 {
|
||||
break
|
||||
}
|
||||
switch {
|
||||
case control == 1 || control == 2: // stored: a 16-bit size less one, then the bytes
|
||||
b, err := full(2)
|
||||
if err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
n := int(binary.BigEndian.Uint16(b)) + 1
|
||||
if err := skip(n); err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
size += 2 + n
|
||||
case control >= 0x80: // LZMA: unpacked and packed sizes, the properties byte on a reset
|
||||
b, err := full(4)
|
||||
if err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
n := int(binary.BigEndian.Uint16(b[2:])) + 1
|
||||
size += 4
|
||||
if control >= 0xc0 {
|
||||
if err := skip(1); err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
size++
|
||||
}
|
||||
if err := skip(n); err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
size += n
|
||||
default:
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: an LZMA2 chunk of kind %#x",
|
||||
streams, inStream, control)
|
||||
}
|
||||
}
|
||||
if pad := (4 - size%4) % 4; pad > 0 {
|
||||
b, err := full(pad)
|
||||
if err != nil || !bytes.Equal(b, make([]byte, pad)) {
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: corrupt block padding", streams, inStream)
|
||||
}
|
||||
}
|
||||
if err := skip(checkSize); err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
}
|
||||
// The index: its count of records, two numbers each, padding, CRC32.
|
||||
used := 1
|
||||
count, n, err := xzVLI(r)
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err)
|
||||
}
|
||||
used += n
|
||||
if count != uint64(inStream) {
|
||||
return streams, blocks, fmt.Errorf("stream %d: its index lists %d blocks, but it holds %d",
|
||||
streams, count, inStream)
|
||||
}
|
||||
for i := 0; i < 2*inStream; i++ {
|
||||
_, n, err := xzVLI(r)
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err)
|
||||
}
|
||||
used += n
|
||||
}
|
||||
if err := skip((4-used%4)%4 + 4); err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err)
|
||||
}
|
||||
footer, err := full(12)
|
||||
if err != nil || footer[10] != 'Y' || footer[11] != 'Z' {
|
||||
return streams, blocks, fmt.Errorf("stream %d: no stream footer", streams)
|
||||
}
|
||||
// Stream padding: zero bytes, a multiple of four, then the end or another stream.
|
||||
zeros := 0
|
||||
for {
|
||||
b, err := r.ReadByte()
|
||||
if err == io.EOF {
|
||||
if zeros%4 != 0 {
|
||||
return streams, blocks, errors.New("stream padding not a multiple of four bytes")
|
||||
}
|
||||
return streams, blocks, nil
|
||||
}
|
||||
if err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
if b != 0 {
|
||||
_ = r.UnreadByte()
|
||||
break
|
||||
}
|
||||
zeros++
|
||||
}
|
||||
if zeros%4 != 0 {
|
||||
return streams, blocks, errors.New("stream padding not a multiple of four bytes")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// xzBlockHeader checks one block header: its CRC32, its flags, and that its last filter is LZMA2, the only
|
||||
// one whose chunks this walk can skip (and the only last filter the reader decodes).
|
||||
func xzBlockHeader(h []byte) error {
|
||||
size := len(h)
|
||||
if crc32.ChecksumIEEE(h[:size-4]) != binary.LittleEndian.Uint32(h[size-4:]) {
|
||||
return errors.New("a corrupt block header")
|
||||
}
|
||||
flags := h[1]
|
||||
if flags&0x3c != 0 {
|
||||
return errors.New("a block header with reserved flags")
|
||||
}
|
||||
body := bytes.NewReader(h[2 : size-4])
|
||||
if flags&0x40 != 0 {
|
||||
if _, _, err := xzVLI(body); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if flags&0x80 != 0 {
|
||||
if _, _, err := xzVLI(body); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
var last uint64
|
||||
for i := 0; i <= int(flags&3); i++ {
|
||||
id, _, err := xzVLI(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
props, _, err := xzVLI(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if props > uint64(body.Len()) {
|
||||
return errors.New("a corrupt block header")
|
||||
}
|
||||
if _, err := body.Seek(int64(props), io.SeekCurrent); err != nil {
|
||||
return err
|
||||
}
|
||||
last = id
|
||||
}
|
||||
if last != 0x21 {
|
||||
return fmt.Errorf("a block whose last filter is %#x, not LZMA2", last)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// xzVLI reads one of xz's variable-length integers (at most 9 bytes, 7 bits each), and how many bytes it took.
|
||||
func xzVLI(r io.ByteReader) (uint64, int, error) {
|
||||
var v uint64
|
||||
for i := 0; i < 9; i++ {
|
||||
b, err := r.ReadByte()
|
||||
if err != nil {
|
||||
return 0, i, io.ErrUnexpectedEOF
|
||||
}
|
||||
v |= uint64(b&0x7f) << (7 * i)
|
||||
if b&0x80 == 0 {
|
||||
if i > 0 && b == 0 {
|
||||
return 0, i + 1, errors.New("a number not in its shortest form")
|
||||
}
|
||||
return v, i + 1, nil
|
||||
}
|
||||
}
|
||||
return 0, 9, errors.New("a number longer than nine bytes")
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"hash/crc32"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// xzStream is one xz stream (no integrity check) of data cut into blocks of blockSize bytes, each block's
|
||||
// LZMA2 data stored uncompressed, each block a delta filter before LZMA2 when delta is set: the shape of
|
||||
// `xz --block-size=N --filters=delta,lzma2`, built without any xz tool. listed is how many blocks its index
|
||||
// lists (-1: as many as it holds), so a stream whose index lies can be built too.
|
||||
func xzStream(data []byte, blockSize int, delta bool, listed int) []byte {
|
||||
var out bytes.Buffer
|
||||
flags := []byte{0, 0}
|
||||
out.Write([]byte{0xfd, '7', 'z', 'X', 'Z', 0x00})
|
||||
out.Write(flags)
|
||||
out.Write(binary.LittleEndian.AppendUint32(nil, crc32.ChecksumIEEE(flags)))
|
||||
type record struct{ unpadded, uncompressed uint64 }
|
||||
var records []record
|
||||
for at := 0; at < len(data); at += blockSize {
|
||||
block := data[at:min(at+blockSize, len(data))]
|
||||
if delta {
|
||||
// Delta-encoded with a distance of one; the decoder's history starts empty at each block.
|
||||
encoded := make([]byte, len(block))
|
||||
for i := range block {
|
||||
encoded[i] = block[i]
|
||||
if i > 0 {
|
||||
encoded[i] -= block[i-1]
|
||||
}
|
||||
}
|
||||
block = encoded
|
||||
}
|
||||
header := []byte{2, 0} // 12 bytes
|
||||
if delta {
|
||||
header[1] = 1 // two filters
|
||||
header = append(header, 0x03, 0x01, 0x00)
|
||||
}
|
||||
header = append(header, 0x21, 0x01, 0x00) // LZMA2, a 4 KiB dictionary
|
||||
for len(header) < 8 {
|
||||
header = append(header, 0)
|
||||
}
|
||||
header = binary.LittleEndian.AppendUint32(header, crc32.ChecksumIEEE(header))
|
||||
out.Write(header)
|
||||
size := 0
|
||||
for i, chunk := 0, block; len(chunk) > 0; i++ {
|
||||
n := min(len(chunk), 1<<16)
|
||||
control := byte(2)
|
||||
if i == 0 {
|
||||
control = 1 // the first chunk of a block resets the dictionary
|
||||
}
|
||||
out.WriteByte(control)
|
||||
out.Write(binary.BigEndian.AppendUint16(nil, uint16(n-1)))
|
||||
out.Write(chunk[:n])
|
||||
size += 3 + n
|
||||
chunk = chunk[n:]
|
||||
}
|
||||
out.WriteByte(0)
|
||||
size++
|
||||
out.Write(make([]byte, (4-size%4)%4))
|
||||
records = append(records, record{uint64(len(header) + size), uint64(len(block))})
|
||||
}
|
||||
if listed >= 0 {
|
||||
for len(records) < listed {
|
||||
records = append(records, records[0])
|
||||
}
|
||||
records = records[:listed]
|
||||
}
|
||||
vli := func(b []byte, v uint64) []byte {
|
||||
for v >= 0x80 {
|
||||
b = append(b, byte(v)|0x80)
|
||||
v >>= 7
|
||||
}
|
||||
return append(b, byte(v))
|
||||
}
|
||||
index := vli([]byte{0}, uint64(len(records)))
|
||||
for _, r := range records {
|
||||
index = vli(vli(index, r.unpadded), r.uncompressed)
|
||||
}
|
||||
for len(index)%4 != 0 {
|
||||
index = append(index, 0)
|
||||
}
|
||||
index = binary.LittleEndian.AppendUint32(index, crc32.ChecksumIEEE(index))
|
||||
out.Write(index)
|
||||
footer := binary.LittleEndian.AppendUint32(nil, uint32(len(index)/4-1))
|
||||
footer = append(footer, flags...)
|
||||
out.Write(binary.LittleEndian.AppendUint32(nil, crc32.ChecksumIEEE(footer)))
|
||||
out.Write(footer)
|
||||
out.WriteString("YZ")
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
// tarHolding is a tar holding betterbird/data with body.
|
||||
func tarHolding(t *testing.T, body []byte) []byte {
|
||||
t.Helper()
|
||||
var out bytes.Buffer
|
||||
w := tar.NewWriter(&out)
|
||||
if err := w.WriteHeader(&tar.Header{Name: "betterbird/data", Mode: 0o644, Size: int64(len(body)),
|
||||
Typeflag: tar.TypeReg}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.Write(body)
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
func writeTemp(t *testing.T, body []byte) string {
|
||||
t.Helper()
|
||||
file := filepath.Join(t.TempDir(), "r.tar.xz")
|
||||
if err := os.WriteFile(file, body, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return file
|
||||
}
|
||||
|
||||
func TestAnXzFileOfAFewDeltaBlocksAndTwoStreamsIsRead(t *testing.T) {
|
||||
// The streams this test builds are ones the reader decodes, so what the walk refuses below is the
|
||||
// same framing the reader follows.
|
||||
body := bytes.Repeat([]byte("betterbird "), 300)
|
||||
whole := tarHolding(t, body)
|
||||
half := len(whole) / 2
|
||||
for name, file := range map[string][]byte{
|
||||
"one stream of delta blocks": xzStream(whole, 512, true, -1),
|
||||
"two streams with padding": append(append(xzStream(whole[:half], 700, true, -1), 0, 0, 0, 0),
|
||||
xzStream(whole[half:], 100, false, -1)...),
|
||||
} {
|
||||
path := writeTemp(t, file)
|
||||
if xzPath, err := exec.LookPath("xz"); err == nil {
|
||||
if out, err := exec.Command(xzPath, "-t", path).CombinedOutput(); err != nil {
|
||||
t.Fatalf("%s: xz itself refuses the built stream: %v %s", name, err, out)
|
||||
}
|
||||
}
|
||||
into := t.TempDir()
|
||||
if err := unpackUpstream(context.Background(), path, into, "betterbird"); err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
got, err := os.ReadFile(filepath.Join(into, "data"))
|
||||
if err != nil || !bytes.Equal(got, body) {
|
||||
t.Fatalf("%s: unpacked %d bytes, want %d: %v", name, len(got), len(body), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnXzFileOfManyDeltaBlocksIsRefusedBeforeItIsDecoded(t *testing.T) {
|
||||
// Review 68 on pull request 264: `xz --block-size=1 --delta` of a 200 kB tar kept about 67 MiB live in
|
||||
// the reader, one delta decoder per block. Its index here lists one block, as a file crafted to pass a
|
||||
// check of the index alone would; the walk counts the blocks the reader would decode.
|
||||
body := make([]byte, 200_000)
|
||||
for i := range body {
|
||||
body[i] = byte(i * 7)
|
||||
}
|
||||
crafted := xzStream(tarHolding(t, body), 1, true, 1)
|
||||
path := writeTemp(t, crafted)
|
||||
var before, after runtime.MemStats
|
||||
runtime.GC()
|
||||
runtime.ReadMemStats(&before)
|
||||
err := unpackUpstream(context.Background(), path, t.TempDir(), "betterbird")
|
||||
runtime.ReadMemStats(&after)
|
||||
if err == nil || !strings.Contains(err.Error(), "more than 1024 xz blocks") {
|
||||
t.Fatalf("a file of %d bytes in blocks of one byte was not refused for its blocks: %v", len(crafted), err)
|
||||
}
|
||||
if grew := after.TotalAlloc - before.TotalAlloc; grew > 4<<20 {
|
||||
t.Fatalf("%d bytes were allocated for a refused file", grew)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheXzBlockLimitIsExact(t *testing.T) {
|
||||
data := make([]byte, xzBlocksMax+1)
|
||||
streams, blocks, err := xzFraming(bufio.NewReader(bytes.NewReader(xzStream(data[:xzBlocksMax], 1, true, -1))))
|
||||
if err != nil || streams != 1 || blocks != xzBlocksMax {
|
||||
t.Fatalf("%d blocks: %d streams, %d blocks, %v", xzBlocksMax, streams, blocks, err)
|
||||
}
|
||||
if _, _, err := xzFraming(bufio.NewReader(bytes.NewReader(xzStream(data, 1, true, -1)))); !errors.Is(err, errXzTooManyBlocks) {
|
||||
t.Fatalf("%d blocks: %v", xzBlocksMax+1, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnXzFileOfTooManyStreamsIsRefused(t *testing.T) {
|
||||
var file []byte
|
||||
for i := 0; i <= xzStreamsMax; i++ {
|
||||
file = append(file, xzStream([]byte("x"), 1, false, -1)...)
|
||||
}
|
||||
err := unpackUpstream(context.Background(), writeTemp(t, file), t.TempDir(), "")
|
||||
if err == nil || !strings.Contains(err.Error(), "more than 16 concatenated xz streams") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnXzIndexThatDoesNotListItsBlocksIsRefused(t *testing.T) {
|
||||
file := xzStream(tarHolding(t, []byte("x")), 512, false, 1)
|
||||
err := unpackUpstream(context.Background(), writeTemp(t, file), t.TempDir(), "betterbird")
|
||||
if err == nil || !strings.Contains(err.Error(), "its index lists 1 blocks, but it holds 4") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
@@ -1693,6 +1693,8 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
problems = append(problems, LogProblems(m)...)
|
||||
// And the defaults it gives its settings (setting_defaults.go, novox/hq ADR 0262).
|
||||
problems = append(problems, SettingProblems(m)...)
|
||||
// And which setting keys each mergeable file takes (setting_takes.go, novox/hq ADR 0326).
|
||||
problems = append(problems, TakenSettingsProblems(m)...)
|
||||
wellFormed := true
|
||||
for _, c := range m.Claims {
|
||||
if !name.MatchString(c.Name) {
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package catalogue
|
||||
|
||||
import "path"
|
||||
|
||||
// The machine's own resolver (novox/hq ADR 0247).
|
||||
//
|
||||
// **Most machines have none.** Every machine lists the mesh's resolvers in /etc/resolv.conf and nothing
|
||||
@@ -66,43 +68,39 @@ func holdsSeat(m Manifest, seat string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// rendersResolverFile says whether a module asks the mesh to render the machine's resolver file.
|
||||
func rendersResolverFile(m Manifest) bool {
|
||||
for _, f := range m.Facts {
|
||||
if !f.Home && f.Path == ResolverFile {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ResolverFileOwner is the module that writes the machine's resolver file among the modules on one
|
||||
// machine (novox/hq ADR 0247): the holder of node-resolver when one renders it, else nobody is named here
|
||||
// and the file is whoever's it always was — the uplink holder's (ADR 0223).
|
||||
func ResolverFileOwner(modules []Manifest) string {
|
||||
for _, m := range modules {
|
||||
if holdsSeat(m, ResolverSeat) && rendersResolverFile(m) {
|
||||
return m.Module
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// stepsBack is the module as it composes on a machine whose resolver file is the node-resolver holder's:
|
||||
// **the uplink holder's rendering of that file is left out**, and nothing else of it changes. Only the
|
||||
// uplink's holder steps back — any other module rendering the file beside the resolver's is still two
|
||||
// owners of one path, and is refused as before.
|
||||
// stepsBack is the module as it composes among the modules of one machine (novox/hq ADR 0287,
|
||||
// ADR 0247):
|
||||
// **where it holds a seat that names a path stepping back for another seat, and that other seat's holder
|
||||
// is on the machine and declares the path, its own rendering of the path is left out**, and nothing else
|
||||
// of it changes. Only a file the mesh renders for it steps back, as before: a resource on the path is
|
||||
// composed as declared (the declaration copies a module's resources whole), so an uplink holder declaring
|
||||
// the resolver file as a resource beside the resolver's holder is two owners of one path, and refused. The stepping back is the seat's definition — node-uplink stepping back from the resolver
|
||||
// file for node-resolver — so any other module declaring the path beside the owner's is still two owners
|
||||
// of one path, and is refused as before.
|
||||
func stepsBack(m Manifest, modules []Manifest) Manifest {
|
||||
if !holdsSeat(m, "node-uplink") || !rendersResolverFile(m) {
|
||||
return m
|
||||
drop := map[string]bool{}
|
||||
for _, c := range m.Claims {
|
||||
seat, known := SeatNamed(c.Name)
|
||||
if !known {
|
||||
continue
|
||||
}
|
||||
for _, p := range seat.Paths {
|
||||
if p.StepsBackFor == "" {
|
||||
continue
|
||||
}
|
||||
for _, o := range modules {
|
||||
if o.Module != m.Module && holdsSeat(o, p.StepsBackFor) && declaresPath(o, p.Path) {
|
||||
drop[p.Path] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
owner := ResolverFileOwner(modules)
|
||||
if owner == "" || owner == m.Module {
|
||||
if len(drop) == 0 {
|
||||
return m
|
||||
}
|
||||
facts := make(map[string]RosterFile, len(m.Facts))
|
||||
for name, f := range m.Facts {
|
||||
if !f.Home && f.Path == ResolverFile {
|
||||
if !f.Home && drop[path.Clean(f.Path)] {
|
||||
continue
|
||||
}
|
||||
facts[name] = f
|
||||
@@ -113,9 +111,6 @@ func stepsBack(m Manifest, modules []Manifest) Manifest {
|
||||
|
||||
// steppedBack is every module of one machine as it composes there (stepsBack, each).
|
||||
func steppedBack(modules []Manifest) []Manifest {
|
||||
if ResolverFileOwner(modules) == "" {
|
||||
return modules
|
||||
}
|
||||
out := make([]Manifest, len(modules))
|
||||
for i, m := range modules {
|
||||
out[i] = stepsBack(m, modules)
|
||||
|
||||
@@ -957,7 +957,9 @@ func checkResources(modules []Manifest) []string {
|
||||
// Judged as the machine composes them: where the node-resolver holder writes the resolver file, the
|
||||
// uplink holder's rendering of it is not composed, so it is not a second owner (novox/hq ADR 0247).
|
||||
modules = steppedBack(modules)
|
||||
var problems []string
|
||||
// And only a claimant of a seat naming a path declares it — the controller's generated resources
|
||||
// included, which are judged here at composition (novox/hq ADR 0287).
|
||||
problems := seatPathsOnMachine(modules)
|
||||
owner := map[string]string{}
|
||||
ownedPath := map[string]string{} // path → owning module, for the access check below
|
||||
packagePresent, packageAbsent := map[string][]string{}, map[string][]string{}
|
||||
|
||||
@@ -0,0 +1,315 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The paths a seat's holder owns (novox/hq ADR 0287).
|
||||
//
|
||||
// **Who owns a machine-wide file is the role's question, answered once.** Until this, which module owned
|
||||
// /etc/hosts or /etc/resolv.conf was decided record by record (ADRs 0199, 0222, 0223, 0241) and enforced
|
||||
// only after the fact, by checkResources finding two modules on one machine declaring the same path: a
|
||||
// module that did not hold node-hostname could declare /etc/hosts on a machine where nobody held it, and
|
||||
// nothing refused it. Now a seat's definition names the paths the role owns on any machine, and three
|
||||
// refusals follow from it, each where the mesh can still say no:
|
||||
//
|
||||
// - a claim whose module does not declare every path its seat names, by CanHold, at registration and at
|
||||
// handover;
|
||||
// - a module declaring a path a seat names while claiming none of the seats that name it, at
|
||||
// registration (CatalogueProblems), and what the controller generates for a machine, at composition;
|
||||
// - a module's own seat naming a path another seat already names, at registration — the second loses.
|
||||
//
|
||||
// Reading a seat's path is access, never a resource, so it stays allowed.
|
||||
|
||||
// SeatPath is one path a seat's holder owns: absolute, literal, on the machine where the holder runs.
|
||||
type SeatPath struct {
|
||||
Path string `json:"path"`
|
||||
// StepsBackFor is a node seat whose holder owns the path instead, on a machine where that seat is held
|
||||
// (novox/hq ADR 0287, ADR 0247): this seat's holder still declares the path, and its declaration is not
|
||||
// composed there. Only the mesh's own seats step back, each by a record.
|
||||
StepsBackFor string `json:"steps-back-for,omitempty"`
|
||||
}
|
||||
|
||||
// seatPathProblem is what is wrong with one path a seat names, judged alone: absolute, written literally
|
||||
// with no placeholder, and clean, so it compares with what a module declares byte for byte.
|
||||
func seatPathProblem(seat, scope, p string) string {
|
||||
switch {
|
||||
case scope != ScopeNode:
|
||||
return fmt.Sprintf("the seat %s names the path %q and is a %s seat; only a node seat names paths, "+
|
||||
"because a path is a machine's (novox/hq ADR 0287)", seat, p, scope)
|
||||
case strings.Contains(p, "${"):
|
||||
return fmt.Sprintf("the seat %s names the path %q, which has a placeholder; a seat's path is written "+
|
||||
"literally (novox/hq ADR 0287)", seat, p)
|
||||
case !strings.HasPrefix(p, "/") || path.Clean(p) != p:
|
||||
return fmt.Sprintf("the seat %s names the path %q, which is not a clean absolute path (novox/hq ADR 0287)",
|
||||
seat, p)
|
||||
case path.Dir(p) == "/":
|
||||
// The root or a top-level directory is every role's, never one role's own (ADR 0287).
|
||||
return fmt.Sprintf("the seat %s names the path %q, the root or a top-level directory, which no one "+
|
||||
"role owns (novox/hq ADR 0287)", seat, p)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// SeatPathProblems is what is wrong with the paths a set of the mesh's own seats names (novox/hq ADR
|
||||
// 0287): only a node seat names one, each absolute and literal; every path has exactly one seat naming it
|
||||
// without stepping back; and a seat steps back only for another node seat naming the same path without
|
||||
// stepping back, so the stepping back cannot cycle.
|
||||
func SeatPathProblems(set []Seat) []string {
|
||||
var problems []string
|
||||
owners := map[string][]string{} // path → the seats naming it without stepping back
|
||||
stepping := map[string][]SeatPath{} // seat → the paths it names stepping back
|
||||
byName := map[string]Seat{}
|
||||
for _, s := range set {
|
||||
byName[s.Name] = s
|
||||
for _, p := range s.Paths {
|
||||
if why := seatPathProblem(s.Name, s.Scope, p.Path); why != "" {
|
||||
problems = append(problems, why)
|
||||
}
|
||||
if p.StepsBackFor == "" {
|
||||
owners[p.Path] = append(owners[p.Path], s.Name)
|
||||
} else {
|
||||
stepping[s.Name] = append(stepping[s.Name], p)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, s := range set {
|
||||
for _, p := range stepping[s.Name] {
|
||||
other, known := byName[p.StepsBackFor]
|
||||
switch {
|
||||
case !known || other.Scope != ScopeNode:
|
||||
problems = append(problems, fmt.Sprintf("the seat %s steps back from %q for %q, which is not a "+
|
||||
"node seat of the mesh's (novox/hq ADR 0287)", s.Name, p.Path, p.StepsBackFor))
|
||||
case !namesWithoutSteppingBack(other, p.Path):
|
||||
problems = append(problems, fmt.Sprintf("the seat %s steps back from %q for %s, which does not "+
|
||||
"own that path itself; a seat steps back only for the one that owns it (novox/hq ADR 0287)",
|
||||
s.Name, p.Path, other.Name))
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, p := range sortedKeys(owners) {
|
||||
if len(owners[p]) > 1 {
|
||||
problems = append(problems, fmt.Sprintf("the path %q is named by %s, each without stepping back; "+
|
||||
"no two seats name one path (novox/hq ADR 0287)", p, strings.Join(owners[p], " and ")))
|
||||
}
|
||||
}
|
||||
for _, s := range set {
|
||||
for _, p := range stepping[s.Name] {
|
||||
if len(owners[p.Path]) == 0 {
|
||||
problems = append(problems, fmt.Sprintf("the path %q has no seat naming it without stepping back "+
|
||||
"(novox/hq ADR 0287)", p.Path))
|
||||
}
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
func namesWithoutSteppingBack(s Seat, p string) bool {
|
||||
for _, sp := range s.Paths {
|
||||
if sp.Path == p && sp.StepsBackFor == "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// declaredPaths is every machine path a module declares it owns: a resource's literal path, and a file
|
||||
// the mesh renders for it outside the operator's home. An access is not here — reading is not owning.
|
||||
//
|
||||
// **Cleaned, because the machine writes the cleaned path**: /etc//hosts, /etc/./hosts, /etc/../etc/hosts and
|
||||
// /etc/hosts/ are all /etc/hosts to it, and a seat's path is clean by its own rule, so a declaration compared
|
||||
// as written would pass for a path nobody named.
|
||||
func declaredPaths(m Manifest) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, r := range m.Resources {
|
||||
if p, ok := r["path"].(string); ok && strings.HasPrefix(p, "/") {
|
||||
out[path.Clean(p)] = true
|
||||
}
|
||||
}
|
||||
for _, f := range m.Facts {
|
||||
if !f.Home && strings.HasPrefix(f.Path, "/") {
|
||||
out[path.Clean(f.Path)] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// declaresPath says whether a module declares one path, as a resource or as a file the mesh renders.
|
||||
func declaresPath(m Manifest, p string) bool { return declaredPaths(m)[p] }
|
||||
|
||||
// missingPaths is each path a seat names that a module does not declare, in the seat's order.
|
||||
func missingPaths(m Manifest, paths []SeatPath) []string {
|
||||
if len(paths) == 0 {
|
||||
return nil
|
||||
}
|
||||
has := declaredPaths(m)
|
||||
var missing []string
|
||||
for _, p := range paths {
|
||||
if !has[p.Path] {
|
||||
missing = append(missing, p.Path)
|
||||
}
|
||||
}
|
||||
return missing
|
||||
}
|
||||
|
||||
// pathSeats is which seats name each path: the mesh's own, and then those modules declare, in the order
|
||||
// given. A module's seat naming a path a seat already names loses it, and the refusal says so (novox/hq
|
||||
// ADR 0287 — as for a seat's name, the second declaration loses).
|
||||
func pathSeats(order []string, shelf Shelf, declaredBy map[string]string) (map[string][]string, []string) {
|
||||
named := map[string][]string{}
|
||||
for _, s := range Seats() {
|
||||
for _, p := range s.Paths {
|
||||
named[p.Path] = append(named[p.Path], s.Name)
|
||||
}
|
||||
}
|
||||
var problems []string
|
||||
for _, module := range order {
|
||||
for _, s := range shelf[module].DefinesSeats {
|
||||
if declaredBy != nil && declaredBy[s.Name] != module {
|
||||
continue // a second declaration of the seat's name, refused for that already
|
||||
}
|
||||
for _, p := range s.Paths {
|
||||
if others := named[p]; len(others) > 0 {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s declares the seat %s naming the path %q, which %s already names; no two seats "+
|
||||
"name one path (novox/hq ADR 0287)", module, s.Name, p, strings.Join(others, " and ")))
|
||||
continue
|
||||
}
|
||||
named[p] = append(named[p], s.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
return named, problems
|
||||
}
|
||||
|
||||
// unclaimedSeatPaths is each path a module declares that a seat names while it claims none of the seats
|
||||
// naming it (novox/hq ADR 0287): only a claimant declares a seat's path, and a module claiming any one of
|
||||
// them may.
|
||||
func unclaimedSeatPaths(m Manifest, named map[string][]string) []string {
|
||||
claims := map[string]bool{}
|
||||
for _, c := range m.Claims {
|
||||
claims[canonicalSeat(c.Name)] = true
|
||||
}
|
||||
var problems []string
|
||||
declared := declaredPaths(m)
|
||||
for _, p := range sortedKeys(declared) {
|
||||
seats := named[p]
|
||||
if len(seats) == 0 {
|
||||
continue
|
||||
}
|
||||
claimsOne := false
|
||||
for _, s := range seats {
|
||||
claimsOne = claimsOne || claims[s]
|
||||
}
|
||||
if !claimsOne {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s declares the path %q, which the seat %s names, and claims none of them; only a holder of "+
|
||||
"a seat naming a path declares it (novox/hq ADR 0287)", m.Module, p, strings.Join(seats, " and ")))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// declaredSeatPathProblems is what one module's own seat declaration can be judged on alone: a path
|
||||
// only on a node seat, absolute, literal and clean.
|
||||
func declaredSeatPathProblems(m Manifest, s SeatDeclaration) []string {
|
||||
var problems []string
|
||||
for _, p := range s.Paths {
|
||||
if why := seatPathProblem(s.Name, s.At(), p); why != "" {
|
||||
problems = append(problems, m.Module+": "+why)
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// asSeatPaths is a module's own seat's paths in the shape the mesh's seats name theirs. A module's seat
|
||||
// never steps back: only the mesh's own seats do, each by a record.
|
||||
func asSeatPaths(paths []string) []SeatPath {
|
||||
out := make([]SeatPath, 0, len(paths))
|
||||
for _, p := range paths {
|
||||
out = append(out, SeatPath{Path: p})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// PathOwner is the module that owns one seat's path on one machine (novox/hq ADR 0287): the seat naming
|
||||
// the path, and that seat's holder there, with the stepping back applied.
|
||||
type PathOwner struct {
|
||||
Path string `json:"path"`
|
||||
Node string `json:"node"`
|
||||
Seat string `json:"seat"`
|
||||
Module string `json:"module"`
|
||||
}
|
||||
|
||||
// PathOwners is, for every path a seat in the set names and every machine where a seat naming it is held,
|
||||
// who owns the path there. A seat that steps back for another held on the same machine is passed over.
|
||||
func PathOwners(set []Seat, held []Held) []PathOwner {
|
||||
// node → seat → holding module
|
||||
holders := map[string]map[string]string{}
|
||||
for _, h := range held {
|
||||
s, ok := SeatNamed(h.Claim)
|
||||
if !ok || h.Scope != ScopeNode || s.Scope != ScopeNode {
|
||||
continue
|
||||
}
|
||||
if holders[h.Node] == nil {
|
||||
holders[h.Node] = map[string]string{}
|
||||
}
|
||||
if _, taken := holders[h.Node][s.Name]; !taken {
|
||||
holders[h.Node][s.Name] = h.Module
|
||||
}
|
||||
}
|
||||
naming := map[string][]Seat{} // path → the seats naming it
|
||||
steps := map[string]map[string]string{} // path → seat → the seat it steps back for
|
||||
for _, s := range set {
|
||||
for _, p := range s.Paths {
|
||||
naming[p.Path] = append(naming[p.Path], s)
|
||||
if p.StepsBackFor != "" {
|
||||
if steps[p.Path] == nil {
|
||||
steps[p.Path] = map[string]string{}
|
||||
}
|
||||
steps[p.Path][s.Name] = p.StepsBackFor
|
||||
}
|
||||
}
|
||||
}
|
||||
var out []PathOwner
|
||||
for _, p := range sortedKeys(naming) {
|
||||
for _, node := range sortedKeys(holders) {
|
||||
for _, s := range naming[p] {
|
||||
module, holds := holders[node][s.Name]
|
||||
if !holds {
|
||||
continue
|
||||
}
|
||||
if forSeat := steps[p][s.Name]; forSeat != "" {
|
||||
if _, there := holders[node][forSeat]; there {
|
||||
continue
|
||||
}
|
||||
}
|
||||
out = append(out, PathOwner{Path: p, Node: node, Seat: s.Name, Module: module})
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// seatPathsOnMachine is each path the modules of one machine declare that a seat names, for a module
|
||||
// claiming none of the seats naming it (novox/hq ADR 0287). Registration refuses the same of a manifest;
|
||||
// this holds what the controller generates for a machine to it too, because a computed module's resources
|
||||
// exist only once composed (checkResources is asked them there). The seats are the mesh's own and those the
|
||||
// machine's modules declare.
|
||||
func seatPathsOnMachine(modules []Manifest) []string {
|
||||
shelf := Shelf{}
|
||||
var order []string
|
||||
for _, m := range modules {
|
||||
shelf[m.Module] = m
|
||||
order = append(order, m.Module)
|
||||
}
|
||||
named, _ := pathSeats(order, shelf, nil)
|
||||
var problems []string
|
||||
for _, m := range modules {
|
||||
problems = append(problems, unclaimedSeatPaths(m, named)...)
|
||||
}
|
||||
return problems
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Defends novox/hq ADR 0287 (issue 412): a seat names the paths its holder owns, a claim without them is
|
||||
// refused, and only a claimant declares a seat's path.
|
||||
|
||||
// The starting set, exactly: node-hostname names /etc/hosts and /etc/hostname, node-uplink the resolver
|
||||
// file stepping back for node-resolver, node-resolver the resolver file, and every other seat names none.
|
||||
func TestTheSeatsThatNamePathsAreTheStartingSet(t *testing.T) {
|
||||
want := map[string]string{
|
||||
"node-hostname": "/etc/hosts /etc/hostname",
|
||||
"node-uplink": "/etc/resolv.conf>node-resolver",
|
||||
ResolverSeat: "/etc/resolv.conf",
|
||||
}
|
||||
for _, s := range DefaultSeats() {
|
||||
var got []string
|
||||
for _, p := range s.Paths {
|
||||
if p.StepsBackFor != "" {
|
||||
got = append(got, p.Path+">"+p.StepsBackFor)
|
||||
} else {
|
||||
got = append(got, p.Path)
|
||||
}
|
||||
}
|
||||
if strings.Join(got, " ") != want[s.Name] {
|
||||
t.Errorf("%s names %v; ADR 0287 gives it %q", s.Name, got, want[s.Name])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The compiled set keeps the rules: node seats only, absolute and literal, one seat per path without
|
||||
// stepping back, and a stepping back that cannot cycle.
|
||||
func TestTheCompiledSeatPathsKeepTheRules(t *testing.T) {
|
||||
if p := SeatPathProblems(DefaultSeats()); len(p) != 0 {
|
||||
t.Fatalf("the compiled set breaks ADR 0287: %v", p)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
why string
|
||||
set []Seat
|
||||
want string
|
||||
}{
|
||||
{"a mesh seat naming a path", []Seat{{Name: "mesh-x", Scope: ScopeMesh, Paths: []SeatPath{{Path: "/etc/x"}}}},
|
||||
"only a node seat names paths"},
|
||||
{"a placeholder", []Seat{{Name: "node-x", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/${setting:x}"}}}},
|
||||
"placeholder"},
|
||||
{"a relative path", []Seat{{Name: "node-x", Scope: ScopeNode, Paths: []SeatPath{{Path: "etc/x"}}}},
|
||||
"not a clean absolute path"},
|
||||
{"two owners", []Seat{
|
||||
{Name: "node-a", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/x"}}},
|
||||
{Name: "node-b", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/x"}}}}, "no two seats name one path"},
|
||||
{"a cycle", []Seat{
|
||||
{Name: "node-a", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/x", StepsBackFor: "node-b"}}},
|
||||
{Name: "node-b", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/x", StepsBackFor: "node-a"}}}},
|
||||
"does not own that path itself"},
|
||||
{"stepping back for nobody", []Seat{
|
||||
{Name: "node-a", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/x", StepsBackFor: "node-z"}}}},
|
||||
"not a node seat"},
|
||||
} {
|
||||
if got := strings.Join(SeatPathProblems(c.set), "; "); !strings.Contains(got, c.want) {
|
||||
t.Errorf("%s: %q does not say %q", c.why, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// What the store's rows carry is never the paths: a row read back keeps the compiled ones.
|
||||
func TestTheSeatPathsSurviveTheStoresRows(t *testing.T) {
|
||||
before := seats
|
||||
t.Cleanup(func() { seats = before })
|
||||
UseSeats([]Seat{{Name: "node-hostname", Scope: ScopeNode}})
|
||||
s, _ := SeatNamed("node-hostname")
|
||||
if len(s.Paths) != 2 {
|
||||
t.Fatalf("node-hostname read back from its row names %v", s.Paths)
|
||||
}
|
||||
}
|
||||
|
||||
func hostnameSeat(t *testing.T) Seat {
|
||||
t.Helper()
|
||||
s, ok := SeatNamed("node-hostname")
|
||||
if !ok {
|
||||
t.Fatal("no node-hostname seat")
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func hostnameHolder(resources ...map[string]any) Manifest {
|
||||
return Manifest{Module: "names", Version: "1", Claims: []Claim{{Name: "node-hostname", Scope: ScopeNode,
|
||||
Serves: []string{"entries", "add", "remove"}}}, Resources: resources}
|
||||
}
|
||||
|
||||
// A holder declares every path its seat names, as a resource or as a file the mesh renders for it, or
|
||||
// CanHold refuses it naming the seat and each missing path — the judgement registration and handover share.
|
||||
func TestAClaimWithoutTheSeatsPathsIsRefused(t *testing.T) {
|
||||
seat := hostnameSeat(t)
|
||||
hosts := map[string]any{"id": "hosts", "type": "file", "path": "/etc/hosts", "content": "x"}
|
||||
name := map[string]any{"id": "name", "type": "file", "path": "/etc/hostname", "content": "x"}
|
||||
|
||||
if err := CanHold(hostnameHolder(hosts, name), seat); err != nil {
|
||||
t.Errorf("a holder declaring both paths as resources was refused: %v", err)
|
||||
}
|
||||
rendered := hostnameHolder(hosts)
|
||||
rendered.Facts = map[string]RosterFile{"name": {Path: "/etc/hostname", Template: "{{.Node}}\n"}}
|
||||
if err := CanHold(rendered, seat); err != nil {
|
||||
t.Errorf("a holder declaring a path as a file the mesh renders was refused: %v", err)
|
||||
}
|
||||
err := CanHold(hostnameHolder(), seat)
|
||||
if err == nil {
|
||||
t.Fatal("a holder declaring neither path was accepted")
|
||||
}
|
||||
for _, want := range []string{"node-hostname", "/etc/hosts", "/etc/hostname", "ADR 0287"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not name %s: %v", want, err)
|
||||
}
|
||||
}
|
||||
// A file under the operator's home is placed per account: it is not the machine's path.
|
||||
home := hostnameHolder(hosts)
|
||||
home.Facts = map[string]RosterFile{"name": {Path: "/etc/hostname", Home: true, Template: "x"}}
|
||||
if err := CanHold(home, seat); err == nil {
|
||||
t.Error("a file in the operator's home counted as the machine's /etc/hostname")
|
||||
}
|
||||
// At registration, the same refusal.
|
||||
if got := strings.Join(CatalogueProblems(Shelf{"names": hostnameHolder(hosts)}), "; "); !strings.Contains(got,
|
||||
"does not declare /etc/hostname") {
|
||||
t.Errorf("registration let a holder without /etc/hostname through: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Only a claimant declares a seat's path: a module claiming no seat that names /etc/hosts and declaring
|
||||
// it is refused at registration naming the seat; the uplink's holder and the resolver's both declaring
|
||||
// the resolver file are each a claimant of a seat naming it. Reading a seat's path is access, and allowed.
|
||||
func TestOnlyAClaimantDeclaresASeatsPath(t *testing.T) {
|
||||
stranger := Manifest{Module: "stranger", Version: "1", Resources: []map[string]any{
|
||||
{"id": "hosts", "type": "file", "path": "/etc/hosts", "into": "block", "content": "10.0.0.1 x\n"}}}
|
||||
got := strings.Join(CatalogueProblems(Shelf{"stranger": stranger}), "; ")
|
||||
if !strings.Contains(got, `stranger declares the path "/etc/hosts", which the seat node-hostname names`) {
|
||||
t.Errorf("a module declaring /etc/hosts without claiming node-hostname was not refused: %q", got)
|
||||
}
|
||||
// However the path is spelled (novox/hq issue 412 review).
|
||||
unclean := Manifest{Module: "unclean", Version: "1", Resources: []map[string]any{
|
||||
{"id": "hosts", "type": "file", "path": "/etc/../etc/hosts", "content": "10.0.0.1 x\n"}}}
|
||||
if got := strings.Join(CatalogueProblems(Shelf{"unclean": unclean}), "; "); !strings.Contains(got, "node-hostname") {
|
||||
t.Errorf("/etc/../etc/hosts from a module not claiming node-hostname was not refused: %q", got)
|
||||
}
|
||||
reader := Manifest{Module: "reader", Version: "1", Accesses: []Access{{ID: "hosts", Path: "/etc/hosts"}}}
|
||||
if got := CatalogueProblems(Shelf{"reader": reader}); len(got) != 0 {
|
||||
t.Errorf("reading /etc/hosts was refused: %v", got)
|
||||
}
|
||||
uplink := Manifest{Module: "uplink", Version: "1", Claims: []Claim{{Name: "node-uplink", Scope: ScopeNode}},
|
||||
Facts: map[string]RosterFile{"resolvers": {Path: ResolverFile, Template: "nameserver 10.42.0.1\n"}}}
|
||||
resolver := localResolver()
|
||||
resolver.Claims[0].Serves = []string{"routes", "route", "unroute"}
|
||||
if got := CatalogueProblems(Shelf{"uplink": uplink, resolver.Module: resolver}); len(got) != 0 {
|
||||
t.Errorf("the uplink's holder and the resolver's declaring the resolver file were refused: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A module's own seat names paths under the same rules: a node seat only, literal, and none another seat
|
||||
// names — the second declaration loses. Its holder declares them, as a mesh seat's does.
|
||||
func TestAModulesOwnSeatNamesPathsUnderTheSameRules(t *testing.T) {
|
||||
meshScoped := Manifest{Module: "a", Version: "1", DefinesSeats: []SeatDeclaration{
|
||||
{Name: "a-seat", Paths: []string{"/etc/a"}}}}
|
||||
if got := strings.Join(declaredSeatProblems(meshScoped), "; "); !strings.Contains(got, "only a node seat names paths") {
|
||||
t.Errorf("a module's mesh seat naming a path was accepted: %q", got)
|
||||
}
|
||||
placeholder := Manifest{Module: "a", Version: "1", DefinesSeats: []SeatDeclaration{
|
||||
{Name: "a-seat", Scope: ScopeNode, Paths: []string{"${dir:state}/a"}}}}
|
||||
if got := strings.Join(declaredSeatProblems(placeholder), "; "); !strings.Contains(got, "placeholder") {
|
||||
t.Errorf("a module's seat naming a path with a placeholder was accepted: %q", got)
|
||||
}
|
||||
|
||||
taken := Manifest{Module: "b", Version: "1", DefinesSeats: []SeatDeclaration{
|
||||
{Name: "b-seat", Scope: ScopeNode, Paths: []string{"/etc/hosts"}}}}
|
||||
if got := strings.Join(CatalogueProblems(Shelf{"b": taken}), "; "); !strings.Contains(got,
|
||||
`b declares the seat b-seat naming the path "/etc/hosts", which node-hostname already names`) {
|
||||
t.Errorf("a module's seat naming /etc/hosts was accepted: %q", got)
|
||||
}
|
||||
|
||||
own := Manifest{Module: "c", Version: "1",
|
||||
DefinesSeats: []SeatDeclaration{{Name: "c-seat", Scope: ScopeNode, Paths: []string{"/etc/c.conf"}}},
|
||||
Claims: []Claim{{Name: "c-seat", Scope: ScopeNode}}}
|
||||
if got := strings.Join(CatalogueProblems(Shelf{"c": own}), "; "); !strings.Contains(got,
|
||||
"c claims c-seat but does not declare /etc/c.conf") {
|
||||
t.Errorf("a holder of a module's seat without its path was accepted: %q", got)
|
||||
}
|
||||
own.Resources = []map[string]any{{"id": "conf", "type": "file", "path": "/etc/c.conf", "content": "x"}}
|
||||
if got := CatalogueProblems(Shelf{"c": own}); len(got) != 0 {
|
||||
t.Errorf("a holder declaring its own seat's path was refused: %v", got)
|
||||
}
|
||||
other := Manifest{Module: "d", Version: "1", Resources: []map[string]any{
|
||||
{"id": "conf", "type": "file", "path": "/etc/c.conf", "content": "x"}}}
|
||||
if got := strings.Join(CatalogueProblems(Shelf{"c": own, "d": other}), "; "); !strings.Contains(got,
|
||||
`d declares the path "/etc/c.conf", which the seat c-seat names`) {
|
||||
t.Errorf("a module not claiming c-seat declared its path: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
type writesHosts struct{}
|
||||
|
||||
func (writesHosts) Resources(string) ([]map[string]any, bool, error) {
|
||||
return []map[string]any{{"id": "hosts", "type": "file", "path": "/etc/hosts", "into": "block",
|
||||
"content": "10.42.0.1 anchor\n"}}, true, nil
|
||||
}
|
||||
|
||||
// The controller is bound too: what it generates for a machine on a seat's path is refused at composition.
|
||||
func TestAGeneratedResourceOnASeatsPathIsRefused(t *testing.T) {
|
||||
r := Resolution{Node: "workstation", Modules: []Manifest{
|
||||
{Module: "mesh-names", Computed: "mesh-names"},
|
||||
}}
|
||||
_, err := r.Declaration(Rendering{Generators: map[string]Generator{"mesh-names": writesHosts{}}})
|
||||
if err == nil {
|
||||
t.Fatal("a generated resource on /etc/hosts was composed")
|
||||
}
|
||||
for _, want := range []string{"mesh-names", "/etc/hosts", "node-hostname"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not name %s: %v", want, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Who owns a seat's path on a machine: the seat naming it and its holder there, with the stepping back
|
||||
// applied — the resolver's holder where node-resolver is held, else the uplink's.
|
||||
func TestPathOwnersApplyTheSteppingBack(t *testing.T) {
|
||||
held := []Held{
|
||||
{Claim: "node-uplink", Scope: ScopeNode, Node: "laptop", Module: "networkmanager"},
|
||||
{Claim: ResolverSeat, Scope: ScopeNode, Node: "laptop", Module: "systemd-resolved"},
|
||||
{Claim: "node-uplink", Scope: ScopeNode, Node: "server", Module: "systemd-networkd"},
|
||||
{Claim: "node-hostname", Scope: ScopeNode, Node: "server", Module: "hostname"},
|
||||
}
|
||||
var got []string
|
||||
for _, o := range PathOwners(Seats(), held) {
|
||||
got = append(got, o.Path+" "+o.Node+" "+o.Seat+" "+o.Module)
|
||||
}
|
||||
want := []string{
|
||||
"/etc/hostname server node-hostname hostname",
|
||||
"/etc/hosts server node-hostname hostname",
|
||||
"/etc/resolv.conf laptop node-resolver systemd-resolved",
|
||||
"/etc/resolv.conf server node-uplink systemd-networkd",
|
||||
}
|
||||
if strings.Join(got, "\n") != strings.Join(want, "\n") {
|
||||
t.Errorf("the owners are\n%s\nnot\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
// The stepping back comes from the seat, and is of a rendered file only: an uplink holder declaring the
|
||||
// resolver file as a resource beside a node-resolver holder is two writers of one path, refused when the
|
||||
// machine is resolved — never composed twice — as before the seat named its paths.
|
||||
func TestAnUplinkDeclaringTheResolverFileAsAResourceIsRefusedBesideTheResolver(t *testing.T) {
|
||||
shelf := resolverShelf(t)
|
||||
uplink := shelf["systemd-networkd"]
|
||||
uplink.Facts = nil
|
||||
uplink.Resources = append(append([]map[string]any(nil), uplink.Resources...),
|
||||
map[string]any{"id": "resolvers", "type": "file", "path": ResolverFile, "content": "nameserver 10.42.0.1\n"})
|
||||
shelf["systemd-networkd"] = uplink
|
||||
shelf["local-resolver"] = localResolver()
|
||||
_, err := Resolve(shelf, []string{"dnsmasq", "systemd-networkd", "local-resolver"}, splitDNSLaptop(), World{})
|
||||
if err == nil || !strings.Contains(err.Error(), `both declare the path "/etc/resolv.conf"`) {
|
||||
t.Fatalf("an uplink declaring the resolver file as a resource beside the resolver was not refused: %v", err)
|
||||
}
|
||||
// Its rendered file steps back, by the seat's definition, and nothing else of it does.
|
||||
rendered := Manifest{Module: "uplink", Version: "1", Claims: []Claim{{Name: "node-uplink"}},
|
||||
Facts: map[string]RosterFile{"resolvers": {Path: "/etc//resolv.conf", Template: "x"},
|
||||
"other": {Path: "/etc/uplink.conf", Template: "x"}}}
|
||||
got := stepsBack(rendered, []Manifest{rendered, localResolver()})
|
||||
if len(got.Facts) != 1 || got.Facts["other"].Path == "" {
|
||||
t.Errorf("the uplink's holder renders %v beside the resolver", got.Facts)
|
||||
}
|
||||
}
|
||||
|
||||
// A path is compared as the machine writes it: cleaned. Each spelling of /etc/hosts is /etc/hosts.
|
||||
func TestAnUncleanSpellingOfASeatsPathIsTheSeatsPath(t *testing.T) {
|
||||
for _, spelled := range []string{"/etc//hosts", "/etc/./hosts", "/etc/../etc/hosts", "/etc/hosts/"} {
|
||||
stranger := Manifest{Module: "stranger", Version: "1", Resources: []map[string]any{
|
||||
{"id": "hosts", "type": "file", "path": spelled, "content": "10.0.0.1 x\n"}}}
|
||||
if got := strings.Join(CatalogueProblems(Shelf{"stranger": stranger}), "; "); !strings.Contains(got,
|
||||
`stranger declares the path "/etc/hosts", which the seat node-hostname names`) {
|
||||
t.Errorf("%s from a module not claiming node-hostname was not refused: %q", spelled, got)
|
||||
}
|
||||
holder := hostnameHolder(map[string]any{"id": "hosts", "type": "file", "path": spelled, "content": "x"},
|
||||
map[string]any{"id": "name", "type": "file", "path": "/etc/hostname", "content": "x"})
|
||||
if err := CanHold(holder, hostnameSeat(t)); err != nil {
|
||||
t.Errorf("a holder declaring %s was refused: %v", spelled, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A seat names a role's own path, never the root or a top-level directory.
|
||||
func TestASeatNamesNoRootOrTopLevelDirectory(t *testing.T) {
|
||||
for _, p := range []string{"/", "/etc"} {
|
||||
m := Manifest{Module: "a", Version: "1", DefinesSeats: []SeatDeclaration{
|
||||
{Name: "a-seat", Scope: ScopeNode, Paths: []string{p}}}}
|
||||
if got := strings.Join(declaredSeatProblems(m), "; "); !strings.Contains(got, "top-level directory") {
|
||||
t.Errorf("a module's seat naming %s was accepted: %q", p, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue as it is passes: no holder of a seat naming a path is missing one, and nobody else
|
||||
// declares one.
|
||||
func TestTheCataloguePassesTheSeatPathRules(t *testing.T) {
|
||||
shelf := Shelf{}
|
||||
for _, m := range theCatalogue(t) {
|
||||
shelf[m.Module] = m
|
||||
}
|
||||
for _, p := range CatalogueProblems(shelf) {
|
||||
if strings.Contains(p, "ADR 0287") {
|
||||
t.Errorf("the catalogue breaks ADR 0287: %s", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -53,6 +53,11 @@ type Seat struct {
|
||||
// ${contribution:<seat>:<kind>}. Compiled, never stored: like the protocol, it is the mesh's
|
||||
// definition of the role, and the store's rows carry no column for it.
|
||||
Receives []Receivable
|
||||
// Paths are the paths on its machine the seat's holder owns (novox/hq ADR 0287): a holder declares
|
||||
// every one, and a module claiming no seat that names a path may not declare it. Only a node seat
|
||||
// names paths. Compiled, never stored, like Receives: adding or taking away a path is a decision,
|
||||
// recorded, and the store's rows carry no column for it.
|
||||
Paths []SeatPath
|
||||
// Decision is the record that made it a seat.
|
||||
Decision string
|
||||
}
|
||||
@@ -178,6 +183,8 @@ var defaultSeats = append([]Seat{
|
||||
// holds none of it. Named node-hosts-file until ADR 0223; the former name resolves to it as an
|
||||
// alias on a mesh that knew it.
|
||||
{Name: "node-hostname", Scope: ScopeNode, Decision: "novox/hq ADR 0199, ADR 0223",
|
||||
// The machine's names are the role's files, whatever program fills it (novox/hq ADR 0287).
|
||||
Paths: []SeatPath{{Path: "/etc/hosts"}, {Path: "/etc/hostname"}},
|
||||
Serves: []Verb{
|
||||
{Name: "entries", Description: "Every line of this machine's /etc/hosts, each marked whose it is: " +
|
||||
"the operator's, or the block of the module or tool that writes it.",
|
||||
@@ -299,12 +306,17 @@ var defaultSeats = append([]Seat{
|
||||
// resolver file as it is and who wrote it, and every link with its addresses, routes and the resolvers
|
||||
// the manager knows for it — the same for every holder, so a question about a machine's names is asked
|
||||
// the same way whatever manages its network. Optional until both holders serve them.
|
||||
{Name: "node-uplink", Scope: ScopeNode, Decision: "novox/hq ADR 0117", Serves: uplinkVerbs()},
|
||||
//
|
||||
// It owns the resolver file, stepping back for node-resolver where that is held (novox/hq ADR 0287,
|
||||
// ADR 0247).
|
||||
{Name: "node-uplink", Scope: ScopeNode, Decision: "novox/hq ADR 0117", Serves: uplinkVerbs(),
|
||||
Paths: []SeatPath{{Path: ResolverFile, StepsBackFor: ResolverSeat}}},
|
||||
// The machine's own resolver (novox/hq ADR 0247): held only where something requires `split-dns` — a
|
||||
// VPN client whose domains must go to its own servers while every other name goes to the mesh's.
|
||||
// Where it is held, its holder writes the resolver file and the uplink's holder steps back from it
|
||||
// (node_resolver.go). It knows nothing of any VPN: its verbs route domains to servers over a link.
|
||||
{Name: ResolverSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0247", Serves: resolverVerbs()},
|
||||
{Name: ResolverSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0247", Serves: resolverVerbs(),
|
||||
Paths: []SeatPath{{Path: ResolverFile}}},
|
||||
// A machine's shares and the shares it mounts (novox/hq ADR 0263): the holder of node-nfs-server
|
||||
// exports a machine's folders to the private network and provides each as `nfs-share`; the holder of
|
||||
// node-mounts writes a mount and an automount unit per share on a machine that asks (shares.go).
|
||||
@@ -364,10 +376,11 @@ func UseSeats(s []Seat) {
|
||||
row.Accepts, row.Emits, row.Serves = d.Accepts, d.Emits, d.Serves
|
||||
}
|
||||
}
|
||||
// What a seat receives and whether it is replicated are never stored (novox/hq ADR 0212, ADR
|
||||
// 0223), so they are always the compiled ones.
|
||||
// What a seat receives, whether it is replicated and the paths it names are never stored
|
||||
// (novox/hq ADR 0212, ADR 0223, ADR 0287), so they are always the compiled ones.
|
||||
if d, known := byName[row.Name]; known {
|
||||
row.Receives = d.Receives
|
||||
row.Paths = d.Paths
|
||||
row.Replicated = d.Replicated
|
||||
row.Serves = optionalAsCompiled(row.Serves, d.Serves)
|
||||
}
|
||||
@@ -541,6 +554,14 @@ func CanHold(m Manifest, seat Seat) error {
|
||||
return fmt.Errorf("%s claims %s, whose holder answers for %q, and %s does not provide %q at %s scope",
|
||||
m.Module, seat.Name, seat.Delivers, m.Module, seat.Delivers, seat.Scope)
|
||||
}
|
||||
// **A holder owns what the role owns** (novox/hq ADR 0287): every path the seat names, declared as
|
||||
// a resource or as a file the mesh renders for it, so which module owns /etc/hosts on a machine is
|
||||
// answered by the seat's definition and not by reading every manifest.
|
||||
if missing := missingPaths(m, seat.Paths); len(missing) > 0 {
|
||||
return fmt.Errorf("%s claims %s but does not declare %s, which that seat's holder owns (novox/hq "+
|
||||
"ADR 0287) — a holder declares every path its seat names, as a resource or as a file the mesh "+
|
||||
"renders for it", m.Module, seat.Name, strings.Join(missing, ", "))
|
||||
}
|
||||
// **Serving the seat's tools is a condition of holding it** (novox/hq ADR 0132). A holder that
|
||||
// does not answer what the role promises is every caller's timeout, found at registration and
|
||||
// at handover instead, naming the verbs rather than the fact that something is missing.
|
||||
|
||||
@@ -67,6 +67,10 @@ type SeatDeclaration struct {
|
||||
// Records are state buckets of the declaring module that each user reads under its own name — the
|
||||
// keys `<user>.…` and no other (ADR 0259 §3).
|
||||
Records []string `json:"records,omitempty"`
|
||||
// Paths are the paths on its machine the seat's holder owns (novox/hq ADR 0287), under the rules the
|
||||
// mesh's own seats name theirs by: a node seat only, each absolute and literal, and none another seat
|
||||
// names. A module's seat never steps back.
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
}
|
||||
|
||||
// KindedBenches are the seats that may be kinded (novox/hq ADR 0234 §2): `channel` sends to the operator,
|
||||
@@ -163,6 +167,7 @@ func declaredSeatProblems(m Manifest) []string {
|
||||
}
|
||||
}
|
||||
problems = append(problems, trafficProblems(m, s)...)
|
||||
problems = append(problems, declaredSeatPathProblems(m, s)...)
|
||||
}
|
||||
|
||||
for _, u := range m.Uses {
|
||||
@@ -271,6 +276,10 @@ func CatalogueProblems(shelf Shelf) []string {
|
||||
return ok
|
||||
}
|
||||
|
||||
// Which seats name each path (novox/hq ADR 0287): a module's seat naming one another seat names loses.
|
||||
named, pathProblems := pathSeats(shelfOrder(shelf), shelf, declaredBy)
|
||||
problems = append(problems, pathProblems...)
|
||||
|
||||
// Who claims each kind of a kinded bench, so a second claim of one kind is refused (ADR 0234 §2).
|
||||
kindsTaken := map[string]string{}
|
||||
|
||||
@@ -327,7 +336,15 @@ func CatalogueProblems(shelf Shelf) []string {
|
||||
"%s claims %s but does not serve %s, which that seat's protocol promises",
|
||||
module, c.Name, strings.Join(missing, ", ")))
|
||||
}
|
||||
// And owns what the seat's holder owns, as a holder of the mesh's own seats does (ADR 0287).
|
||||
if missing := missingPaths(m, asSeatPaths(s.Paths)); len(missing) > 0 {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s claims %s but does not declare %s, which that seat's holder owns (novox/hq ADR 0287)",
|
||||
module, c.Name, strings.Join(missing, ", ")))
|
||||
}
|
||||
}
|
||||
// Only a claimant declares a seat's path (novox/hq ADR 0287).
|
||||
problems = append(problems, unclaimedSeatPaths(m, named)...)
|
||||
}
|
||||
// **A trusted holder runs as its own account** (novox/hq ADR 0259 §8): a module saying warrants, or
|
||||
// speaking for a kind that proves its sender, is never carried by a machine's runtime.
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// A mergeable file names the setting keys it takes (novox/hq ADR 0326, issue 168).
|
||||
//
|
||||
// Before this, every key of a module's layers merged into every one of its mergeable files: the
|
||||
// mesh's own words, a key meant for a `${setting:<key>}` in another file, a key for a second
|
||||
// configuration file. So a module could not own two configurable files, and every receiver had to
|
||||
// ignore what it did not know. ADR 0164 §11 decided that a setting names the files that read it; this
|
||||
// is that relation, stated from the file's side, so the operator's own values stay declared nowhere
|
||||
// (ADR 0262 §2).
|
||||
|
||||
// TakesSettingsField is the key a mergeable file carries to name the top-level setting keys it takes.
|
||||
const TakesSettingsField = "takes-settings"
|
||||
|
||||
// takenKeys is the keys a resource names in takes-settings, and whether it names any list at all. An
|
||||
// empty list is a list: that file takes no setting.
|
||||
func takenKeys(resource map[string]any) (map[string]bool, bool) {
|
||||
raw, said := resource[TakesSettingsField]
|
||||
if !said {
|
||||
return nil, false
|
||||
}
|
||||
keys := map[string]bool{}
|
||||
for _, k := range stringsOf(raw) {
|
||||
keys[k] = true
|
||||
}
|
||||
return keys, true
|
||||
}
|
||||
|
||||
// onlyTaken is the layers with every key the file does not name left out. The file's own defaults and
|
||||
// what the mesh reads itself are not touched here: settle already keeps both out of a file.
|
||||
func onlyTaken(layers []Layer, taken map[string]bool) []Layer {
|
||||
kept := make([]Layer, 0, len(layers))
|
||||
for _, layer := range layers {
|
||||
values := map[string]any{}
|
||||
for key, value := range layer.Values {
|
||||
if taken[key] {
|
||||
values[key] = value
|
||||
}
|
||||
}
|
||||
kept = append(kept, Layer{From: layer.From, Values: values, Default: layer.Default})
|
||||
}
|
||||
return kept
|
||||
}
|
||||
|
||||
// namesItsKeys says whether a module's mergeable files name the keys they take (ADR 0326 §2): true
|
||||
// only when it has one and every one carries takes-settings. A module with a silent mergeable file
|
||||
// keeps today's form (§3), where every key reaches every mergeable file.
|
||||
func namesItsKeys(m Manifest) bool {
|
||||
merging := 0
|
||||
for _, r := range m.Resources {
|
||||
if how, _ := r["merge"].(string); how == "" {
|
||||
continue
|
||||
}
|
||||
merging++
|
||||
if _, said := r[TakesSettingsField]; !said {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return merging > 0
|
||||
}
|
||||
|
||||
// TakenSettingsProblems is every way a definition's takes-settings is wrong (ADR 0326 §2 and §4), in
|
||||
// its own words: on a file that does not merge, not a list of keys, a key named twice, one the mesh
|
||||
// reads itself, one the file keeps for itself, or one mergeable file naming its keys beside another
|
||||
// that names none.
|
||||
func TakenSettingsProblems(m Manifest) []string {
|
||||
// Not yet here: refusing a module that declares a setting and owns a mergeable file naming no keys
|
||||
// (ADR 0326 §3). betterbird is one, and the catalogue cannot name its keys before this controller
|
||||
// runs, so that refusal follows once the catalogue does (novox/hq issue 574).
|
||||
return takenSettingsProblems(m)
|
||||
}
|
||||
|
||||
// takenSettingsProblems is every rule of ADR 0326 §2 and §4, judged where a definition is checked and
|
||||
// where a stored one is composed.
|
||||
func takenSettingsProblems(m Manifest) []string {
|
||||
var problems []string
|
||||
var naming, silent []string
|
||||
for _, r := range m.Resources {
|
||||
id := fmt.Sprint(r["id"])
|
||||
how, _ := r["merge"].(string)
|
||||
raw, said := r[TakesSettingsField]
|
||||
if how == "" {
|
||||
if said {
|
||||
problems = append(problems, fmt.Sprintf("%s's %s says %s and does not merge: only a mergeable "+
|
||||
"file takes settings by key (novox/hq ADR 0326)", m.Module, id, TakesSettingsField))
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !said {
|
||||
silent = append(silent, id)
|
||||
continue
|
||||
}
|
||||
naming = append(naming, id)
|
||||
list, isList := raw.([]any)
|
||||
if !isList {
|
||||
problems = append(problems, fmt.Sprintf("%s's %s: %s is a list of setting keys, and this is %T",
|
||||
m.Module, id, TakesSettingsField, raw))
|
||||
continue
|
||||
}
|
||||
protected := map[string]bool{}
|
||||
for _, k := range stringsOf(r["protected"]) {
|
||||
protected[k] = true
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, item := range list {
|
||||
key, isString := item.(string)
|
||||
switch {
|
||||
case !isString || !usableKey(key):
|
||||
problems = append(problems, fmt.Sprintf("%s's %s: %v in %s is not a usable setting key",
|
||||
m.Module, id, item, TakesSettingsField))
|
||||
case seen[key]:
|
||||
problems = append(problems, fmt.Sprintf("%s's %s names %q twice in %s",
|
||||
m.Module, id, key, TakesSettingsField))
|
||||
case meshWords[key]:
|
||||
problems = append(problems, fmt.Sprintf("%s's %s names %q in %s, and the mesh reads %q itself "+
|
||||
"(novox/hq ADR 0164 §14)", m.Module, id, key, TakesSettingsField, key))
|
||||
case protected[key]:
|
||||
problems = append(problems, fmt.Sprintf("%s's %s names %q in %s and keeps it as protected: a "+
|
||||
"key is the file's or the setting's, never both", m.Module, id, key, TakesSettingsField))
|
||||
}
|
||||
if isString {
|
||||
seen[key] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(naming) > 0 {
|
||||
for _, id := range silent {
|
||||
problems = append(problems, fmt.Sprintf("%s's %s merges and names no %s while %s does: once one "+
|
||||
"mergeable file of a module names the keys it takes, every one does, an empty list for none "+
|
||||
"(novox/hq ADR 0326)", m.Module, id, TakesSettingsField, strings.Join(naming, ", ")))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// usableKey is a key a file can be said to take: not empty, and no space, line end or control
|
||||
// character. Not the lower-case rule of a `${setting:<key>}`: a file's own keys are its software's
|
||||
// spelling, such as `timeZone`.
|
||||
func usableKey(key string) bool {
|
||||
if key == "" {
|
||||
return false
|
||||
}
|
||||
for _, r := range key {
|
||||
if unicode.IsSpace(r) || unicode.IsControl(r) || !unicode.IsPrint(r) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// takenSaid is what a stray key's message adds for a module whose files name their keys: which keys
|
||||
// each file takes, so the person sees where the key would have had to be named.
|
||||
func takenSaid(m Manifest) string {
|
||||
if !namesItsKeys(m) {
|
||||
return ""
|
||||
}
|
||||
var files []string
|
||||
for _, r := range m.Resources {
|
||||
if how, _ := r["merge"].(string); how == "" {
|
||||
continue
|
||||
}
|
||||
keys := stringsOf(r[TakesSettingsField])
|
||||
if len(keys) == 0 {
|
||||
files = append(files, fmt.Sprintf("%v takes none", r["id"]))
|
||||
continue
|
||||
}
|
||||
files = append(files, fmt.Sprintf("%v takes %s", r["id"], strings.Join(keys, ", ")))
|
||||
}
|
||||
return " (" + strings.Join(files, "; ") + ")"
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// twoFiles is a module that owns two configurable files, which issue 168 said no module could:
|
||||
// every key of every layer reached both.
|
||||
func twoFiles(app, sidecar []any) Manifest {
|
||||
appFile := map[string]any{"id": "app", "type": "file", "path": "/etc/app.json", "merge": MergeJSON,
|
||||
"content": `{"server":{"port":8080},"search":{"safe":1}}`}
|
||||
sideFile := map[string]any{"id": "sidecar", "type": "file", "path": "/etc/sidecar.json", "merge": MergeJSON,
|
||||
"content": `{"url":"http://127.0.0.1:8080"}`}
|
||||
if app != nil {
|
||||
appFile[TakesSettingsField] = app
|
||||
}
|
||||
if sidecar != nil {
|
||||
sideFile[TakesSettingsField] = sidecar
|
||||
}
|
||||
return Manifest{Module: "search", Resources: []map[string]any{
|
||||
appFile, sideFile,
|
||||
{"id": "env", "type": "file", "path": "/etc/search.env", "content": "SITE=${setting:site-name}\n"},
|
||||
}}
|
||||
}
|
||||
|
||||
func settledContent(t *testing.T, resource map[string]any, layers []Layer) map[string]any {
|
||||
t.Helper()
|
||||
out, err := ApplySettings(resource, layers)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, kept := out[TakesSettingsField]; kept {
|
||||
t.Errorf("%v still carries %s; the host knows no such field", out["id"], TakesSettingsField)
|
||||
}
|
||||
var parsed map[string]any
|
||||
if err := json.Unmarshal([]byte(out["content"].(string)), &parsed); err != nil {
|
||||
t.Fatalf("%v is not JSON after settling: %v", out["id"], err)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func TestEachMergeableFileTakesOnlyTheKeysItNames(t *testing.T) {
|
||||
// novox/hq ADR 0326 §1, issue 168: two configurable files, each taking its own keys. A key for
|
||||
// the other file, for a ${setting:…} elsewhere or for the mesh itself lands in neither.
|
||||
m := twoFiles([]any{"server", "search"}, []any{"url"})
|
||||
layers := []Layer{
|
||||
{From: "the mesh", Values: map[string]any{
|
||||
"server": map[string]any{"base_url": "https://search.example.tld/"},
|
||||
"url": "http://127.0.0.1:9090", "site-name": "Search",
|
||||
"endpoints": map[string]any{"web": map[string]any{"reach": "public"}}}},
|
||||
{From: "node-a", Values: map[string]any{"search": map[string]any{"safe": 0.0}}},
|
||||
}
|
||||
|
||||
app := settledContent(t, m.Resources[0], layers)
|
||||
for _, stray := range []string{"url", "site-name", "endpoints"} {
|
||||
if _, in := app[stray]; in {
|
||||
t.Errorf("app took %q, which it does not name: %v", stray, app)
|
||||
}
|
||||
}
|
||||
server, _ := app["server"].(map[string]any)
|
||||
if server["base_url"] != "https://search.example.tld/" || server["port"] != 8080.0 {
|
||||
t.Errorf("app's server block is %v; the setting should merge into the file's own", server)
|
||||
}
|
||||
if search, _ := app["search"].(map[string]any); search["safe"] != 0.0 {
|
||||
t.Errorf("app's search block is %v; the node's layer should win", search)
|
||||
}
|
||||
|
||||
side := settledContent(t, m.Resources[1], layers)
|
||||
if side["url"] != "http://127.0.0.1:9090" {
|
||||
t.Errorf("sidecar's url is %v; it names url", side["url"])
|
||||
}
|
||||
if len(side) != 1 {
|
||||
t.Errorf("sidecar took keys it does not name: %v", side)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnEmptyListTakesNoSetting(t *testing.T) {
|
||||
// ADR 0164 §12: a file that names no setting takes none.
|
||||
m := twoFiles([]any{"server"}, []any{})
|
||||
side := settledContent(t, m.Resources[1], []Layer{{From: "the mesh", Values: map[string]any{
|
||||
"server": map[string]any{"port": 1.0}, "url": "x"}}})
|
||||
if len(side) != 1 || side["url"] != "http://127.0.0.1:8080" {
|
||||
t.Errorf("a file taking none was changed: %v", side)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFileThatNamesNothingKeepsTodaysForm(t *testing.T) {
|
||||
// ADR 0326 §3, ADR 0164 §15: a module none of whose files names keys is left as it was.
|
||||
m := twoFiles(nil, nil)
|
||||
app := settledContent(t, m.Resources[0], []Layer{{From: "the mesh", Values: map[string]any{"url": "x"}}})
|
||||
if app["url"] != "x" {
|
||||
t.Errorf("a silent file should still take every key until its module names them: %v", app)
|
||||
}
|
||||
if unused := UnusedSettings(m, []Layer{{From: "the mesh", Values: map[string]any{"anything": 1}}}); len(unused) != 0 {
|
||||
t.Errorf("today's form has nothing stray, and said: %v", unused)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAKeyNoFileNamesIsStray(t *testing.T) {
|
||||
// ADR 0326 §5: once the files name their keys, a key none of them names — and nothing else
|
||||
// takes — is stray, and is said with what each file takes.
|
||||
m := twoFiles([]any{"server", "search"}, []any{"url"})
|
||||
unused := UnusedSettings(m, []Layer{{From: "the mesh", Values: map[string]any{
|
||||
"server": 1, "url": "x", "site-name": "Search", "ui": map[string]any{"theme": "dark"}}}})
|
||||
if len(unused) != 1 || !strings.Contains(unused[0], `"ui"`) {
|
||||
t.Fatalf("only ui reaches nothing; named: %v", unused)
|
||||
}
|
||||
if !strings.Contains(unused[0], "app takes server, search") || !strings.Contains(unused[0], "sidecar takes url") {
|
||||
t.Errorf("the stray key's message does not say what each file takes: %s", unused[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestOneNamingFileMakesEveryMergeableFileName(t *testing.T) {
|
||||
// ADR 0326 §2: a module with one naming and one silent mergeable file is refused, naming the
|
||||
// silent one — the silent one would otherwise still take every key.
|
||||
m := twoFiles([]any{"server"}, nil)
|
||||
problems := TakenSettingsProblems(m)
|
||||
if len(problems) != 1 || !strings.Contains(problems[0], "sidecar merges and names no takes-settings") {
|
||||
t.Fatalf("problems: %v", problems)
|
||||
}
|
||||
if err := JudgeSettings(m, nil, false); err == nil || !strings.Contains(err.Error(), "sidecar") {
|
||||
t.Errorf("composing it should refuse it too, and said: %v", err)
|
||||
}
|
||||
raw, err := json.Marshal(map[string]any{"module": m.Module, "version": "1", "resources": m.Resources})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ParseManifest(raw); err == nil || !strings.Contains(err.Error(), "sidecar merges and names no") {
|
||||
t.Errorf("the module check does not say it: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTakesSettingsIsRefusedWhereItCannotMeanAnything(t *testing.T) {
|
||||
// ADR 0326 §4.
|
||||
cases := map[string]struct {
|
||||
resource map[string]any
|
||||
says string
|
||||
}{
|
||||
"on a file that does not merge": {
|
||||
map[string]any{"id": "plain", "type": "file", "path": "/x", "content": "x",
|
||||
TakesSettingsField: []any{"a"}},
|
||||
"does not merge"},
|
||||
"not a list": {
|
||||
map[string]any{"id": "conf", "type": "file", "path": "/x", "merge": MergeJSON, "content": "{}",
|
||||
TakesSettingsField: "server"},
|
||||
"is a list of setting keys"},
|
||||
"a mesh word": {
|
||||
map[string]any{"id": "conf", "type": "file", "path": "/x", "merge": MergeJSON, "content": "{}",
|
||||
TakesSettingsField: []any{"endpoints"}},
|
||||
"the mesh reads"},
|
||||
"a protected key": {
|
||||
map[string]any{"id": "conf", "type": "file", "path": "/x", "merge": MergeJSON, "content": "{}",
|
||||
"protected": []any{"secret"}, TakesSettingsField: []any{"secret"}},
|
||||
"keeps it as protected"},
|
||||
"named twice": {
|
||||
map[string]any{"id": "conf", "type": "file", "path": "/x", "merge": MergeJSON, "content": "{}",
|
||||
TakesSettingsField: []any{"a", "a"}},
|
||||
"twice"},
|
||||
"not a key": {
|
||||
map[string]any{"id": "conf", "type": "file", "path": "/x", "merge": MergeJSON, "content": "{}",
|
||||
TakesSettingsField: []any{"two words"}},
|
||||
"not a usable setting key"},
|
||||
}
|
||||
for name, c := range cases {
|
||||
problems := TakenSettingsProblems(Manifest{Module: "m", Resources: []map[string]any{c.resource}})
|
||||
if !containsText(problems, c.says) {
|
||||
t.Errorf("%s: want a problem saying %q, got %v", name, c.says, problems)
|
||||
}
|
||||
}
|
||||
// And a file that does not merge is refused where it is settled, not handed to the host.
|
||||
plain := map[string]any{"id": "plain", "type": "file", "path": "/x", "content": "x",
|
||||
TakesSettingsField: []any{"a"}}
|
||||
if _, err := ApplySettings(plain, nil); err == nil {
|
||||
t.Error("a file that does not merge and names keys was settled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestASoftwareSpelledKeyIsUsable(t *testing.T) {
|
||||
// A file's keys are its software's: nodered's timeZone is not lower case and is still a key.
|
||||
m := Manifest{Module: "flows", Resources: []map[string]any{{"id": "settings", "type": "file",
|
||||
"path": "/x", "merge": MergeJSON, "content": "{}", TakesSettingsField: []any{"timeZone"}}}}
|
||||
if problems := TakenSettingsProblems(m); len(problems) != 0 {
|
||||
t.Errorf("timeZone was refused: %v", problems)
|
||||
}
|
||||
}
|
||||
|
||||
func containsText(list []string, text string) bool {
|
||||
for _, s := range list {
|
||||
if strings.Contains(s, text) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -46,11 +46,24 @@ type Layer struct {
|
||||
// rather than by repeating everything the rest already say.
|
||||
func ApplySettings(resource map[string]any, layers []Layer) (map[string]any, error) {
|
||||
how, _ := resource["merge"].(string)
|
||||
taken, names := takenKeys(resource)
|
||||
if how == "" {
|
||||
if names {
|
||||
// Refused by the module check too (TakenSettingsProblems); said here as well, for a
|
||||
// definition stored by an older binary, rather than handed to the host, which knows no
|
||||
// such field.
|
||||
return nil, fmt.Errorf("%v says %s and does not merge: only a mergeable file takes "+
|
||||
"settings by key (novox/hq ADR 0326)", resource["id"], TakesSettingsField)
|
||||
}
|
||||
// Not a mergeable file. Settings for it are ignored rather than silently doing nothing
|
||||
// somewhere else — see UnusedSettings, which is what says so.
|
||||
return resource, nil
|
||||
}
|
||||
if names {
|
||||
// **The file takes the keys it names, and no other** (novox/hq ADR 0326, issue 168): a key
|
||||
// for another file, for a `${setting:<key>}` or for the mesh itself does not land here.
|
||||
layers = onlyTaken(layers, taken)
|
||||
}
|
||||
if how != MergeJSON {
|
||||
return nil, fmt.Errorf(
|
||||
"%v says it merges as %q, and this control plane knows how to merge %q",
|
||||
@@ -91,6 +104,7 @@ func ApplySettings(resource map[string]any, layers []Layer) (map[string]any, err
|
||||
delete(out, "merge")
|
||||
delete(out, "protected")
|
||||
delete(out, TrustedField)
|
||||
delete(out, TakesSettingsField)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -324,8 +338,11 @@ func sortedKeysAny(m map[string]any) []string {
|
||||
// (novox/hq 04-ISSUES/173); and the mesh's own words — `expose`, `ports`, `reach`, `endpoints` —
|
||||
// are read by the mesh. A key none of those takes is stray, and is said so rather than dropped.
|
||||
func UnusedSettings(m Manifest, layers []Layer) []string {
|
||||
named := namesItsKeys(m)
|
||||
for _, r := range m.Resources {
|
||||
if how, _ := r["merge"].(string); how != "" {
|
||||
if how, _ := r["merge"].(string); how != "" && !named {
|
||||
// Today's form (novox/hq ADR 0326 §3, ADR 0164 §15): a mergeable file that names no
|
||||
// keys takes every key, so nothing is stray.
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -337,6 +354,13 @@ func UnusedSettings(m Manifest, layers []Layer) []string {
|
||||
}
|
||||
|
||||
lands := settingKeysUsedBy(m)
|
||||
// A mergeable file takes the keys it names (novox/hq ADR 0326 §5).
|
||||
for _, r := range m.Resources {
|
||||
taken, _ := takenKeys(r)
|
||||
for key := range taken {
|
||||
lands[key] = true
|
||||
}
|
||||
}
|
||||
// A key the module gives a default reaches what asks for it (novox/hq ADR 0262): setting it
|
||||
// overrides the default, which is how a running module gains a setting without a gap between.
|
||||
for key := range m.Settings {
|
||||
@@ -401,8 +425,8 @@ func UnusedSettings(m Manifest, layers []Layer) []string {
|
||||
}
|
||||
unused = append(unused, fmt.Sprintf(
|
||||
"%s sets %q, and %s has no file that merges it, asks for no ${setting:%s}, and "+
|
||||
"declares no %q in what it contributes or serves",
|
||||
layer.From, key, m.Module, key, key))
|
||||
"declares no %q in what it contributes or serves%s",
|
||||
layer.From, key, m.Module, key, key, takenSaid(m)))
|
||||
}
|
||||
}
|
||||
sort.Strings(unused)
|
||||
@@ -512,6 +536,11 @@ func JudgeSettings(m Manifest, layers []Layer, adopted bool) error {
|
||||
if err := settingsHoldOneLine(m.Module, layers); err != nil {
|
||||
return err
|
||||
}
|
||||
// Which keys each mergeable file takes is the definition's to say right (novox/hq ADR 0326);
|
||||
// judged here too for a definition stored by an older binary.
|
||||
if problems := takenSettingsProblems(m); len(problems) > 0 {
|
||||
return fmt.Errorf("%s", problems[0])
|
||||
}
|
||||
// With no layers too: a definition may ask for a setting nobody made — an access placed by
|
||||
// nobody, a file's ${setting:…} nothing sets — and that is the same statement, missing.
|
||||
if _, err := GivenPorts(m, layers); err != nil {
|
||||
|
||||
@@ -33,7 +33,9 @@ func TestTheUplinkSeatPromisesItsVerbsAndRequiresNoneYet(t *testing.T) {
|
||||
if !slices.Equal(names, []string{"resolvers", "links"}) {
|
||||
t.Fatalf("node-uplink promises %v", names)
|
||||
}
|
||||
today := Manifest{Module: "networkmanager", Version: "1", Claims: []Claim{{Name: "node-uplink", Scope: ScopeNode}}}
|
||||
// The resolver file is the seat's path (novox/hq ADR 0287), which every holder of today renders.
|
||||
today := Manifest{Module: "networkmanager", Version: "1", Claims: []Claim{{Name: "node-uplink", Scope: ScopeNode}},
|
||||
Facts: map[string]RosterFile{"resolvers": {Path: ResolverFile, Template: "x"}}}
|
||||
if err := CanHold(today, seat); err != nil {
|
||||
t.Fatalf("a holder that serves no verb yet is refused: %v", err)
|
||||
}
|
||||
@@ -55,7 +57,8 @@ func TestTheUplinkVerbsReadBackFromTheRowStayOptional(t *testing.T) {
|
||||
// The row as seeding stores it: both verbs, the mark not a column.
|
||||
UseSeats([]Seat{{Name: "node-uplink", Scope: ScopeNode, Serves: []Verb{{Name: "resolvers"}, {Name: "links"}}}})
|
||||
seat := uplinkSeat(t)
|
||||
holder := Manifest{Module: "systemd-networkd", Version: "1", Claims: []Claim{{Name: "node-uplink", Scope: ScopeNode}}}
|
||||
holder := Manifest{Module: "systemd-networkd", Version: "1", Claims: []Claim{{Name: "node-uplink", Scope: ScopeNode}},
|
||||
Facts: map[string]RosterFile{"resolvers": {Path: ResolverFile, Template: "x"}}}
|
||||
if err := CanHold(holder, seat); err != nil {
|
||||
t.Fatalf("read back from the row, the verbs are required: %v", err)
|
||||
}
|
||||
|
||||
@@ -116,9 +116,15 @@ var ControllerVerbs = []Verb{
|
||||
"limit": "how many builds to list (default 20); not with log",
|
||||
}, nil)},
|
||||
{Name: "plans", Description: "What the last merges produced and where each stands (novox/hq ADR 0162): " +
|
||||
"the tiers, the tier a plan is at, what it waits for and since when; one plan whole, given its id.",
|
||||
"the tiers, the tier a plan is at, what it waits for and since when; one plan whole, given its id; " +
|
||||
"or, given wait, one plan followed until it moves.",
|
||||
Input: schema(map[string]string{
|
||||
"id": "a plan's id (as `plans` lists them): that plan, tier by tier",
|
||||
"id": "a plan's id (as `plans` lists them): that plan, tier by tier",
|
||||
"wait": "a plan's id: wait until it moves, then answer it as data — the plan, and what ended the wait " +
|
||||
"(moved, tier, done, or bound); ask again to follow it further (novox/hq issue 229)",
|
||||
"until": "with wait: moved (any change, the default), tier (it reaches another tier) or done (it ends)",
|
||||
"bound": "with wait: how long to wait at most, as 15s or seconds — 20s, the default, is the longest, " +
|
||||
"inside what a call is answered within",
|
||||
"stop": "a plan's id: stop it — what was asked still builds, nothing further is asked",
|
||||
"close": "a plan's id: close a plan that will not move again, as failed by hand (novox/hq issue 254)",
|
||||
"retry": "a failed plan's id: ask its failed builds again under new ids, and carry the plan on from that tier (novox/hq ADR 0219)",
|
||||
|
||||
@@ -29,14 +29,13 @@ func ForTest(t *testing.T) *Identity {
|
||||
if len(name) > 60 {
|
||||
name = name[:60]
|
||||
}
|
||||
conn, err := pgx.Connect(t.Context(), admin)
|
||||
// Created already migrated, a copy of a template migrated once (novox/hq issue 591); the Migrate
|
||||
// below then finds nothing pending, which it does only if the copy's record matches every migration.
|
||||
migrations, err := Migrations()
|
||||
if err != nil {
|
||||
t.Fatalf("cannot reach the test PostgreSQL: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := conn.Exec(t.Context(), "create database "+name); err != nil {
|
||||
t.Fatalf("cannot create %s: %v", name, err)
|
||||
}
|
||||
conn.Close(t.Context())
|
||||
store.CreateMigratedForTest(t, admin, name, migrations)
|
||||
|
||||
cut := strings.LastIndex(admin, "/")
|
||||
t.Setenv(store.Variable(Name), admin[:cut]+"/"+name+"?sslmode=disable")
|
||||
@@ -57,10 +56,6 @@ func ForTest(t *testing.T) *Identity {
|
||||
if err := ident.Ready(t.Context(), 20*time.Second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
migrations, err := Migrations()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ident.store.Migrate(t.Context(), migrations); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
@@ -23,6 +24,25 @@ import (
|
||||
// access change would blank every module's password if the mesh kept nothing (design 25 §4, and the
|
||||
// migration beside this).
|
||||
|
||||
// busPasswordCostForTest, when set, is the bcrypt cost a bus password is hashed at instead of the
|
||||
// library's default. Only ForTest sets it, to bcrypt's least (novox/hq issue 591): at the default
|
||||
// cost one hash takes most of a second under the race detector, and a hash at the least cost is the
|
||||
// same kind of hash, checked by the same comparison and by the bus the same way.
|
||||
var busPasswordCostForTest atomic.Int32
|
||||
|
||||
// busPasswordCost is the bcrypt cost a bus password is hashed at: the library's default, but in a test.
|
||||
func busPasswordCost() int {
|
||||
if c := busPasswordCostForTest.Load(); c != 0 {
|
||||
return int(c)
|
||||
}
|
||||
return bcrypt.DefaultCost
|
||||
}
|
||||
|
||||
// hashBusPassword is the hash a bus password is kept as, minted, recorded or seeded alike.
|
||||
func hashBusPassword(password string) ([]byte, error) {
|
||||
return bcrypt.GenerateFromPassword([]byte(password), busPasswordCost())
|
||||
}
|
||||
|
||||
// BusUser is one user of the bus, as the mesh records it.
|
||||
type BusUser struct {
|
||||
Username string
|
||||
@@ -67,7 +87,7 @@ func (i *Inventory) RecordBusPassword(ctx context.Context, u BusUser, password s
|
||||
if password == "" {
|
||||
return errors.New("a bus user needs a password")
|
||||
}
|
||||
hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
|
||||
hash, err := hashBusPassword(password)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot hash a bus password: %w", err)
|
||||
}
|
||||
@@ -101,7 +121,7 @@ func (i *Inventory) MintBusPassword(ctx context.Context, u BusUser) (string, err
|
||||
// The cost the server will pay on every connection. Left at the library's default rather than
|
||||
// raised: a node reconnecting after a network blip pays it, and the mesh's own links reconnect
|
||||
// far more often than a person logs in anywhere.
|
||||
hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
|
||||
hash, err := hashBusPassword(password)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot hash a bus password: %w", err)
|
||||
}
|
||||
@@ -183,7 +203,7 @@ func (i *Inventory) SeedBusUser(ctx context.Context, u BusUser, password string)
|
||||
if u.Username == "" || u.Kind == "" || password == "" {
|
||||
return errors.New("a bus user needs a username, a kind and the credential it is using")
|
||||
}
|
||||
hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
|
||||
hash, err := hashBusPassword(password)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot hash a bus password: %w", err)
|
||||
}
|
||||
|
||||
@@ -158,3 +158,17 @@ func TestACredentialTheMeshDidNotMintIsRecordedOnceAndNotOverwritten(t *testing.
|
||||
t.Fatal("a restart put the bootstrap credential back over a rotated one")
|
||||
}
|
||||
}
|
||||
|
||||
// Outside a test a bus password is hashed at bcrypt's default cost: only ForTest lowers it (novox/hq issue 591).
|
||||
func TestABusPasswordIsHashedAtTheDefaultCostOutsideATest(t *testing.T) {
|
||||
was := busPasswordCostForTest.Load()
|
||||
t.Cleanup(func() { busPasswordCostForTest.Store(was) })
|
||||
busPasswordCostForTest.Store(0)
|
||||
hash, err := hashBusPassword("a password")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := bcrypt.Cost(hash); err != nil || got != bcrypt.DefaultCost {
|
||||
t.Fatalf("a bus password is hashed at cost %d (%v), not bcrypt's default %d", got, err, bcrypt.DefaultCost)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/store"
|
||||
)
|
||||
@@ -30,14 +31,14 @@ func ForTest(t *testing.T) *Inventory {
|
||||
name = name[:60]
|
||||
}
|
||||
|
||||
conn, err := pgx.Connect(t.Context(), admin)
|
||||
// Created already migrated, a copy of a template migrated once (novox/hq issue 591); the Migrate
|
||||
// below then finds nothing pending, which it does only if the copy's record matches every migration.
|
||||
migrations, err := Migrations()
|
||||
if err != nil {
|
||||
t.Fatalf("cannot reach the test PostgreSQL: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := conn.Exec(t.Context(), "create database "+name); err != nil {
|
||||
t.Fatalf("cannot create %s: %v", name, err)
|
||||
}
|
||||
conn.Close(t.Context())
|
||||
store.CreateMigratedForTest(t, admin, name, migrations)
|
||||
busPasswordCostForTest.Store(int32(bcrypt.MinCost))
|
||||
|
||||
cut := strings.LastIndex(admin, "/")
|
||||
t.Setenv(store.Variable(Name), admin[:cut]+"/"+name+"?sslmode=disable")
|
||||
@@ -59,10 +60,6 @@ func ForTest(t *testing.T) *Inventory {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
migrations, err := Migrations()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.store.Migrate(t.Context(), migrations); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// MachineAccount is what a machine said it is made of (novox/hq issue 160), from the profile it last
|
||||
// reported: the architecture and kernel family the profile always carried, and the facts a node-engine that
|
||||
// knows how says beside them. Read by `node`, so what a machine says of itself is visible in one place —
|
||||
// collected and shown in the same breath, not collected and read by nothing (issue 160's pattern).
|
||||
type MachineAccount struct {
|
||||
Architecture string `json:"architecture"`
|
||||
Kernel string `json:"kernel"`
|
||||
// Machine is nil when the node-engine that reported is older than the one that says it.
|
||||
Machine *MadeOf `json:"machine"`
|
||||
}
|
||||
|
||||
// MadeOf is the node-engine's account of what it is made of, as it put it on the wire. A fact it could
|
||||
// not determine is absent here and named in Unknown with why — never a zero.
|
||||
type MadeOf struct {
|
||||
ObservedAt time.Time `json:"observed_at"`
|
||||
KernelRelease string `json:"kernel_release"`
|
||||
Distribution string `json:"distribution"`
|
||||
Version string `json:"version"`
|
||||
Processor string `json:"processor"`
|
||||
Cores int `json:"cores"`
|
||||
Threads int `json:"threads"`
|
||||
MemoryBytes int64 `json:"memory_bytes"`
|
||||
Virtual string `json:"virtual"`
|
||||
BootedAt *time.Time `json:"booted_at"`
|
||||
Timezone string `json:"timezone"`
|
||||
Filesystems []MountedFilesystem `json:"filesystems"`
|
||||
Unknown map[string]string `json:"unknown"`
|
||||
}
|
||||
|
||||
// MountedFilesystem is one local filesystem a machine has mounted.
|
||||
type MountedFilesystem struct {
|
||||
Mount string `json:"mount"`
|
||||
Device string `json:"device"`
|
||||
Type string `json:"type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
FreeBytes int64 `json:"free_bytes"`
|
||||
}
|
||||
|
||||
// MachineOf is what a node last said it is made of. had is false for a node that has never reported a
|
||||
// profile at all, which is a different answer from one whose node-engine does not say the facts yet.
|
||||
func (i *Inventory) MachineOf(ctx context.Context, nodeName string) (account MachineAccount, had bool, err error) {
|
||||
var raw []byte
|
||||
err = i.store.Pool().QueryRow(ctx, `select profile from node where name = $1`, nodeName).Scan(&raw)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return MachineAccount{}, false, fmt.Errorf("%w: %s", ErrNoSuchNode, nodeName)
|
||||
}
|
||||
if err != nil {
|
||||
return MachineAccount{}, false, err
|
||||
}
|
||||
return machineFrom(raw)
|
||||
}
|
||||
|
||||
func machineFrom(raw []byte) (MachineAccount, bool, error) {
|
||||
if len(raw) == 0 {
|
||||
return MachineAccount{}, false, nil
|
||||
}
|
||||
var account MachineAccount
|
||||
if err := json.Unmarshal(raw, &account); err != nil {
|
||||
return MachineAccount{}, false, err
|
||||
}
|
||||
return account, true, nil
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The node-engine's words for a machine's facts (novox/hq issue 160), as mesh-host's profile.Machine puts
|
||||
// them on the wire. A rename on either side fails here rather than as a blank on a real machine.
|
||||
func TestAMachinesAccountIsReadInTheNodeEnginesWords(t *testing.T) {
|
||||
raw := []byte(`{"architecture":"amd64","kernel":"linux","capabilities":[],
|
||||
"machine":{"observed_at":"2026-10-12T15:00:00Z","kernel_release":"7.2.8-arch1-2",
|
||||
"distribution":"Arch Linux","version":"rolling","processor":"AMD Ryzen 9","cores":8,"threads":16,
|
||||
"memory_bytes":32891494400,"virtual":"physical","booted_at":"2026-10-09T08:50:34Z",
|
||||
"timezone":"Europe/Brussels",
|
||||
"filesystems":[{"mount":"/","device":"/dev/nvme0n1p3","type":"ext4","size_bytes":100,"free_bytes":40}],
|
||||
"unknown":{"public address":"not decided"}}}`)
|
||||
a, had, err := machineFrom(raw)
|
||||
if err != nil || !had {
|
||||
t.Fatalf("had %v, err %v", had, err)
|
||||
}
|
||||
m := a.Machine
|
||||
if a.Architecture != "amd64" || a.Kernel != "linux" || m == nil {
|
||||
t.Fatalf("%+v", a)
|
||||
}
|
||||
if m.KernelRelease != "7.2.8-arch1-2" || m.Distribution != "Arch Linux" || m.Version != "rolling" ||
|
||||
m.Processor != "AMD Ryzen 9" || m.Cores != 8 || m.Threads != 16 || m.MemoryBytes != 32891494400 ||
|
||||
m.Virtual != "physical" || m.BootedAt == nil || m.Timezone != "Europe/Brussels" ||
|
||||
len(m.Filesystems) != 1 || m.Filesystems[0].FreeBytes != 40 || m.Unknown["public address"] == "" ||
|
||||
m.ObservedAt.IsZero() {
|
||||
t.Errorf("%+v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAProfileWithoutTheFactsIsAnOlderNodeEngineNotNothing(t *testing.T) {
|
||||
a, had, err := machineFrom([]byte(`{"architecture":"amd64","kernel":"linux","capabilities":[]}`))
|
||||
if err != nil || !had || a.Machine != nil || a.Architecture != "amd64" {
|
||||
t.Errorf("%+v had %v err %v", a, had, err)
|
||||
}
|
||||
if _, had, _ := machineFrom(nil); had {
|
||||
t.Error("a node that never reported read as having reported")
|
||||
}
|
||||
}
|
||||
@@ -1064,9 +1064,14 @@ type Reported struct {
|
||||
// SentDeclaration is what one send carried to a machine, as a gate keeps it: the declaration's digest and
|
||||
// its sequence (novox/hq issue 352). A report about this declaration, or about one sequenced after it, is a
|
||||
// report on what the gate sent — whatever the machine was sent since.
|
||||
//
|
||||
// At is when the controller sent it to that machine: the node's own send time, which is where that machine's
|
||||
// `apply` duration starts. A walk measures its rest from it, not from its own record of the send, made only after
|
||||
// every machine of the rest was sent (novox/hq issue 582). Zero for a send kept before it was recorded.
|
||||
type SentDeclaration struct {
|
||||
Digest string `json:"digest"`
|
||||
Sequence int64 `json:"sequence,omitempty"`
|
||||
Digest string `json:"digest"`
|
||||
Sequence int64 `json:"sequence,omitempty"`
|
||||
At time.Time `json:"at,omitzero"`
|
||||
}
|
||||
|
||||
// ReportsOn says a report is about this send: the declaration itself; one the same machine was sequenced
|
||||
@@ -1085,7 +1090,8 @@ func (s SentDeclaration) ReportsOn(r Reported) bool {
|
||||
func (i *Inventory) SentTo(ctx context.Context, name string) (SentDeclaration, bool, error) {
|
||||
var digest *string
|
||||
var seq *int64
|
||||
err := i.store.Pool().QueryRow(ctx, `select sent, sequence from node where name = $1`, name).Scan(&digest, &seq)
|
||||
var at *time.Time
|
||||
err := i.store.Pool().QueryRow(ctx, `select sent, sequence, sent_at from node where name = $1`, name).Scan(&digest, &seq, &at)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return SentDeclaration{}, false, fmt.Errorf("%w: %s", ErrNoSuchNode, name)
|
||||
}
|
||||
@@ -1096,6 +1102,9 @@ func (i *Inventory) SentTo(ctx context.Context, name string) (SentDeclaration, b
|
||||
if seq != nil {
|
||||
s.Sequence = *seq
|
||||
}
|
||||
if at != nil {
|
||||
s.At = at.UTC()
|
||||
}
|
||||
return s, true, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **A plan's id is a value, never SQL** (novox/hq issue 561): the id reaches PlanByID from a caller of the
|
||||
// controller's plans verb, so an id holding a quote is read as a plain id — no error from the store, and no
|
||||
// other plan answered for it.
|
||||
func TestAPlanIdHoldingAQuoteIsReadAsAPlainId(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
p := Plan{ID: "plan-561", Repository: "novox/mesh-controller", Commit: "abc", Created: time.Now().UTC(),
|
||||
State: PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*PlanModule{"a": {}}}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, id := range []string{
|
||||
"x' or '1'='1", // every plan, when pasted into the query
|
||||
"plan-561' or id like '%", // the plan above, by a pattern
|
||||
"it's", // a quote alone: a syntax error, when pasted
|
||||
} {
|
||||
got, err := inv.PlanByID(ctx, id)
|
||||
if err == nil {
|
||||
t.Fatalf("PlanByID(%q) answered %s, a plan of another id", id, got.ID)
|
||||
}
|
||||
if want := "no plan " + id; err.Error() != want {
|
||||
t.Fatalf("PlanByID(%q) failed with %q, not as a plain id no plan has (%q)", id, err, want)
|
||||
}
|
||||
}
|
||||
// A plan whose id holds a quote is found by it, and by nothing else.
|
||||
quoted := p
|
||||
quoted.ID, quoted.Revision = "plan-'561'", 0
|
||||
if err := inv.SavePlan(ctx, "ed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := inv.PlanByID(ctx, quoted.ID); err != nil || got.ID != quoted.ID {
|
||||
t.Fatalf("the plan with a quote in its id was not read back: %v %+v", err, got.ID)
|
||||
}
|
||||
if recent, err := inv.RecentPlans(ctx, 1); err != nil || len(recent) != 1 || !strings.HasPrefix(recent[0].ID, "plan-") {
|
||||
t.Fatalf("RecentPlans with a parameter: %v %+v", err, recent)
|
||||
}
|
||||
}
|
||||
@@ -623,7 +623,7 @@ func (i *Inventory) PlansSince(ctx context.Context, since time.Time) ([]Plan, er
|
||||
|
||||
// RecentPlans is the last few plans, newest first, open or not — what the overview shows.
|
||||
func (i *Inventory) RecentPlans(ctx context.Context, limit int) ([]Plan, error) {
|
||||
return i.plans(ctx, fmt.Sprintf(`order by created desc limit %d`, limit))
|
||||
return i.plans(ctx, `order by created desc limit $1`, limit)
|
||||
}
|
||||
|
||||
// NewestMergeOf is the plan, in any state, of the newest merge into a repository's branch that the mesh
|
||||
@@ -638,9 +638,10 @@ func (i *Inventory) NewestMergeOf(ctx context.Context, repository, branch string
|
||||
return plans[0], true, nil
|
||||
}
|
||||
|
||||
// PlanByID is one plan.
|
||||
// PlanByID is one plan. The id is a parameter, never part of the query (novox/hq issue 561): it comes from a
|
||||
// caller of the plans verb.
|
||||
func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
|
||||
plans, err := i.plans(ctx, `where id = '`+id+`'`)
|
||||
plans, err := i.plans(ctx, `where id = $1`, id)
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
|
||||
@@ -2,13 +2,10 @@ package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// Where a walk's time went (novox/hq ADR 0282 decision 6, issue 382): from its merge to every machine of its
|
||||
@@ -59,11 +56,14 @@ type WalkPhases struct {
|
||||
TotalMS int64 `json:"total_ms,omitempty"`
|
||||
Total string `json:"total,omitempty"`
|
||||
// UnknownMS is the time within the walk no phase could be measured over.
|
||||
UnknownMS int64 `json:"unknown_ms,omitempty"`
|
||||
Phases []WalkPhase `json:"phases"`
|
||||
Silent []string `json:"silent,omitempty"`
|
||||
Said string `json:"said"`
|
||||
Merges []MergeStart `json:"merges,omitempty"`
|
||||
UnknownMS int64 `json:"unknown_ms,omitempty"`
|
||||
Phases []WalkPhase `json:"phases"`
|
||||
Silent []string `json:"silent,omitempty"`
|
||||
// Rest is each machine of the rest's own report of the send that carried the walk's build there (novox/hq ADR
|
||||
// 0282 decision 6, issue 582): when it was sent, when it said, and what.
|
||||
Rest []RestReport `json:"rest,omitempty"`
|
||||
Said string `json:"said"`
|
||||
Merges []MergeStart `json:"merges,omitempty"`
|
||||
}
|
||||
|
||||
// MergeStart is one merge the walk answers and when it was made: where that merge's delivery time starts.
|
||||
@@ -92,8 +92,34 @@ type AppliedReport struct {
|
||||
Outcome string
|
||||
}
|
||||
|
||||
// AppliedLookup answers a machine's first report after a send to it; false when it has not reported.
|
||||
type AppliedLookup func(node string, sent time.Time) (AppliedReport, bool)
|
||||
// AppliedLookup answers a machine's report of the send the walk made to it: send is what the walk kept of it, and
|
||||
// walkSent the walk's own record of its send to the rest; false when it has not reported.
|
||||
type AppliedLookup func(node string, send SentDeclaration, walkSent time.Time) (AppliedReport, bool)
|
||||
|
||||
// RestReport is one machine of the rest and its own report of a send of the walk (novox/hq issue 582).
|
||||
type RestReport struct {
|
||||
Node string `json:"node"`
|
||||
Modules []string `json:"modules"`
|
||||
// Sent is when that machine was sent, by its own send time where the walk kept it; else the walk's record of
|
||||
// its send to the rest, and SentSaid says so.
|
||||
Sent time.Time `json:"sent"`
|
||||
SentSaid string `json:"sent_said,omitempty"`
|
||||
Sequence int64 `json:"sequence,omitempty"`
|
||||
Reported *time.Time `json:"reported,omitempty"`
|
||||
Took string `json:"took,omitempty"`
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
// State is applied, failed, waiting (not yet heard, within ApplySilentAfter) or silent (not heard within it:
|
||||
// listed, never moving the walk's end).
|
||||
State string `json:"state"`
|
||||
}
|
||||
|
||||
// The states of a machine of the rest.
|
||||
const (
|
||||
RestApplied = "applied"
|
||||
RestFailed = "failed"
|
||||
RestWaiting = "waiting"
|
||||
RestSilent = "silent"
|
||||
)
|
||||
|
||||
// point is one moment of the walk, ending the phase named.
|
||||
type point struct {
|
||||
@@ -307,8 +333,8 @@ func (p Plan) points(now time.Time, applied AppliedLookup, w *WalkPhases) []poin
|
||||
}
|
||||
} else {
|
||||
rest = latest(rest, s.SentAt)
|
||||
for node := range s.Rest {
|
||||
lastSends = append(lastSends, restSend{module: m, node: node, at: *s.SentAt})
|
||||
for node, what := range s.Rest {
|
||||
lastSends = append(lastSends, restSend{module: m, node: node, walkSent: *s.SentAt, what: what})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -347,22 +373,73 @@ func (p Plan) points(now time.Time, applied AppliedLookup, w *WalkPhases) []poin
|
||||
if applied == nil {
|
||||
return append(out, point{phase: PhaseApply, tier: -1, said: "the rest's reports were not read"})
|
||||
}
|
||||
var end *time.Time
|
||||
var waiting, failed []string
|
||||
// Each machine of the rest, by its own report of the send that carried the build there (novox/hq issue 582):
|
||||
// never a report of a send made after it, which another walk's send may be.
|
||||
reports := map[string]*RestReport{}
|
||||
var order []string
|
||||
for _, s := range lastSends {
|
||||
r, ok := applied(s.node, s.at)
|
||||
switch {
|
||||
case !ok && now.Sub(s.at) > ApplySilentAfter, ok && r.At.Sub(s.at) > ApplySilentAfter:
|
||||
w.Silent = appendOnce(w.Silent, s.node)
|
||||
case !ok:
|
||||
waiting = appendOnce(waiting, s.node)
|
||||
case r.Outcome != OutcomeApplied:
|
||||
failed = appendOnce(failed, s.node+" ("+r.Outcome+")")
|
||||
default:
|
||||
at := r.At
|
||||
end = latest(end, &at)
|
||||
at, said := s.walkSent, ""
|
||||
if !s.what.At.IsZero() {
|
||||
at = s.what.At
|
||||
} else {
|
||||
said = "the walk's record of its send to the rest: this machine's own send time was not kept"
|
||||
}
|
||||
key := s.node + "\x00" + s.what.Digest + "\x00" + at.Format(time.RFC3339Nano)
|
||||
if r := reports[key]; r != nil {
|
||||
r.Modules = appendOnce(r.Modules, s.module)
|
||||
continue
|
||||
}
|
||||
rr := &RestReport{Node: s.node, Modules: []string{s.module}, Sent: at, SentSaid: said, Sequence: s.what.Sequence}
|
||||
if r, ok := applied(s.node, s.what, s.walkSent); ok {
|
||||
got := r.At
|
||||
rr.Reported, rr.Outcome, rr.Took = &got, r.Outcome, words(r.At.Sub(at))
|
||||
}
|
||||
reports[key] = rr
|
||||
order = append(order, key)
|
||||
}
|
||||
// A machine sent twice by the walk (one tier's rest, then the next's) that said nothing of the first send has
|
||||
// said it of the second, which carries the first's builds too: both are this walk's.
|
||||
for _, k := range order {
|
||||
rr := reports[k]
|
||||
if rr.Reported != nil {
|
||||
continue
|
||||
}
|
||||
for _, l := range order {
|
||||
later := reports[l]
|
||||
if later.Node == rr.Node && later.Sent.After(rr.Sent) && later.Reported != nil &&
|
||||
(rr.Reported == nil || later.Reported.Before(*rr.Reported)) {
|
||||
got := *later.Reported
|
||||
rr.Reported, rr.Outcome, rr.Took = &got, later.Outcome, words(got.Sub(rr.Sent))
|
||||
}
|
||||
}
|
||||
}
|
||||
var end *time.Time
|
||||
var waiting, failed []string
|
||||
for _, k := range order {
|
||||
rr := reports[k]
|
||||
switch {
|
||||
case rr.Reported == nil && now.Sub(rr.Sent) > ApplySilentAfter, rr.Reported != nil && rr.Reported.Sub(rr.Sent) > ApplySilentAfter:
|
||||
rr.State = RestSilent
|
||||
w.Silent = appendOnce(w.Silent, rr.Node)
|
||||
case rr.Reported == nil:
|
||||
rr.State = RestWaiting
|
||||
waiting = appendOnce(waiting, rr.Node)
|
||||
case rr.Outcome != OutcomeApplied:
|
||||
rr.State = RestFailed
|
||||
failed = appendOnce(failed, rr.Node+" ("+rr.Outcome+")")
|
||||
default:
|
||||
rr.State = RestApplied
|
||||
end = latest(end, rr.Reported)
|
||||
}
|
||||
sort.Strings(rr.Modules)
|
||||
w.Rest = append(w.Rest, *rr)
|
||||
}
|
||||
sort.SliceStable(w.Rest, func(a, b int) bool {
|
||||
if w.Rest[a].Node != w.Rest[b].Node {
|
||||
return w.Rest[a].Node < w.Rest[b].Node
|
||||
}
|
||||
return w.Rest[a].Sent.Before(w.Rest[b].Sent)
|
||||
})
|
||||
switch {
|
||||
case len(failed) > 0:
|
||||
return append(out, point{phase: PhaseApply, tier: -1, said: "not applied on " + strings.Join(failed, ", ")})
|
||||
@@ -393,7 +470,8 @@ func (p Plan) points(now time.Time, applied AppliedLookup, w *WalkPhases) []poin
|
||||
|
||||
type restSend struct {
|
||||
module, node string
|
||||
at time.Time
|
||||
walkSent time.Time
|
||||
what SentDeclaration
|
||||
}
|
||||
|
||||
func askedAnyOf(p Plan, tier []string) bool {
|
||||
@@ -451,28 +529,79 @@ func words(d time.Duration) string {
|
||||
return d.Round(100 * time.Millisecond).String()
|
||||
}
|
||||
|
||||
// FirstAppliedAfter is a machine's first report of a send made at or after a walk's send to it, within
|
||||
// ApplySilentAfter of it — the controller's `apply` durations, which measure every send to its first report (to-be
|
||||
// 45 Phase 0). A declaration sent later carries the build too, so its report counts; one sent past the bound is a
|
||||
// machine that slept, listed as silent and never moving the walk's end (ADR 0282 decision 1).
|
||||
func (i *Inventory) FirstAppliedAfter(ctx context.Context, node string, sent time.Time) (AppliedReport, bool, error) {
|
||||
var started time.Time
|
||||
var ms int64
|
||||
var outcome string
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select started, took_ms, detail from duration
|
||||
// FirstAppliedAfter is a machine's report of the send a walk made to it: the controller's `apply` durations, which
|
||||
// measure every send to its first report (to-be 45 Phase 0), each row starting at the machine's own send time and
|
||||
// naming the declaration it was about. send is what the walk kept of that send; walkSent its own record of the
|
||||
// send to the rest, made after every machine of the rest was sent (novox/hq issue 582). FirstReportOf chooses.
|
||||
func (i *Inventory) FirstAppliedAfter(ctx context.Context, node string, send SentDeclaration, walkSent time.Time) (AppliedReport, bool, error) {
|
||||
from, to := applyRowsBetween(send, walkSent)
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select ref, started, took_ms, detail from duration
|
||||
where kind = $1 and subject = $2 and started >= $3 and started < $4
|
||||
order by started limit 1`,
|
||||
DurationApply, node, sent.Add(-appliedSlack), sent.Add(ApplySilentAfter)).Scan(&started, &ms, &outcome)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return AppliedReport{}, false, nil
|
||||
}
|
||||
order by started`,
|
||||
DurationApply, node, from, to)
|
||||
if err != nil {
|
||||
return AppliedReport{}, false, err
|
||||
}
|
||||
return AppliedReport{At: started.Add(time.Duration(ms) * time.Millisecond).UTC(), Outcome: outcome}, true, nil
|
||||
defer rows.Close()
|
||||
var ds []Duration
|
||||
for rows.Next() {
|
||||
d := Duration{Kind: DurationApply, Subject: node, Node: node}
|
||||
var ms int64
|
||||
if err := rows.Scan(&d.Ref, &d.Started, &ms, &d.Detail); err != nil {
|
||||
return AppliedReport{}, false, err
|
||||
}
|
||||
d.Took = time.Duration(ms) * time.Millisecond
|
||||
ds = append(ds, d)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return AppliedReport{}, false, err
|
||||
}
|
||||
r, ok := FirstReportOf(ds, send, walkSent)
|
||||
return r, ok, nil
|
||||
}
|
||||
|
||||
// appliedSlack is how much earlier than a walk's record of its send the machine's own record of it may be:
|
||||
// the send is recorded on the machine first, then on the walk.
|
||||
// applyRowsBetween is the span of a machine's `apply` durations that may hold its report of a walk's send.
|
||||
func applyRowsBetween(send SentDeclaration, walkSent time.Time) (time.Time, time.Time) {
|
||||
switch {
|
||||
case !send.At.IsZero():
|
||||
return send.At.Add(-time.Millisecond), send.At.Add(time.Millisecond)
|
||||
case send.Digest != "":
|
||||
return walkSent.Add(-restSendBound), walkSent.Add(ApplySilentAfter)
|
||||
}
|
||||
return walkSent.Add(-appliedSlack), walkSent.Add(ApplySilentAfter)
|
||||
}
|
||||
|
||||
// FirstReportOf picks, from a machine's `apply` durations ordered by their start, its report of the send a walk made
|
||||
// to it (novox/hq issue 582):
|
||||
// - with the machine's own send time kept (every walk since issue 582): the duration of that very send, and of the
|
||||
// declaration sent where the walk kept its digest. A report of a later send is never this walk's: a send made
|
||||
// later, by another walk, would end this walk at that walk's send.
|
||||
// - with only the declaration's digest kept (a walk kept before): the first duration about that declaration
|
||||
// within restSendBound before the walk's record of its send, the machine sent first in the rest included.
|
||||
// - with neither (a walk kept before ADR 0282's rest): the first duration started no earlier than appliedSlack
|
||||
// before the walk's record of its send, as it was read before.
|
||||
//
|
||||
// A send past ApplySilentAfter is never read: its machine slept, and is listed silent (ADR 0282 decision 1).
|
||||
func FirstReportOf(ds []Duration, send SentDeclaration, walkSent time.Time) (AppliedReport, bool) {
|
||||
from, to := applyRowsBetween(send, walkSent)
|
||||
for _, d := range ds {
|
||||
if d.Started.Before(from) || !d.Started.Before(to) {
|
||||
continue
|
||||
}
|
||||
if send.Digest != "" && !strings.HasPrefix(d.Ref, send.Digest+"@") {
|
||||
continue
|
||||
}
|
||||
return AppliedReport{At: d.Started.Add(d.Took).UTC(), Outcome: d.Detail}, true
|
||||
}
|
||||
return AppliedReport{}, false
|
||||
}
|
||||
|
||||
// appliedSlack is how much earlier than a walk's record of its send the machine's own record of it may be, for a
|
||||
// walk that kept neither the machine's send time nor what it sent.
|
||||
const appliedSlack = 2 * time.Second
|
||||
|
||||
// restSendBound is how long before a walk's record of its send to the rest a machine of the rest may have been
|
||||
// sent, for a walk that kept what it sent and not when: the send to the rest goes machine by machine, and took
|
||||
// 19 to 28 seconds on 2026-10-12 (novox/hq issue 582).
|
||||
const restSendBound = 5 * time.Minute
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// novox/hq issue 582: the walk's own record of its send to the rest is made after every machine of the rest was
|
||||
// sent, and that send took 19 to 28 s on 2026-10-12. The machine sent first reported 5 s after its own send, long
|
||||
// before the walk's record; read from the walk's record, its report was missed and the walk ended at another walk's
|
||||
// send to it, minutes later. A walk of one tier and three machines of the rest, the send to the rest taking 20 s.
|
||||
type sendRest582 struct {
|
||||
judged, walkSent time.Time
|
||||
walk Plan
|
||||
rows map[string][]Duration
|
||||
}
|
||||
|
||||
func walk582(t *testing.T) sendRest582 {
|
||||
t.Helper()
|
||||
judged := time.Date(2026, 10, 12, 17, 45, 35, 137000000, time.UTC)
|
||||
ownSend := map[string]time.Duration{"g14": 500 * time.Millisecond, "novox": 10 * time.Second, "shanks": 20 * time.Second}
|
||||
walkSent := judged.Add(20200 * time.Millisecond)
|
||||
merged := judged.Add(-10 * time.Minute)
|
||||
built, first := judged.Add(-3*time.Minute), judged.Add(-2*time.Minute)
|
||||
rest := map[string]SentDeclaration{}
|
||||
rows := map[string][]Duration{}
|
||||
took := map[string]time.Duration{"g14": 5 * time.Second, "novox": 13 * time.Second, "shanks": 15 * time.Second}
|
||||
for i, n := range []string{"g14", "novox", "shanks"} {
|
||||
at := judged.Add(ownSend[n])
|
||||
digest := "walk-" + n
|
||||
rest[n] = SentDeclaration{Digest: digest, Sequence: int64(647 + i), At: at}
|
||||
rows[n] = append(rows[n],
|
||||
// This walk's send, and its report.
|
||||
Duration{Kind: DurationApply, Subject: n, Node: n, Ref: digest + "@" + at.Format("2006-01-02T15:04:05.000000"),
|
||||
Started: at, Took: took[n], Detail: OutcomeApplied},
|
||||
// The next walk's send to the same machine, nine minutes later, and its report.
|
||||
Duration{Kind: DurationApply, Subject: n, Node: n, Ref: "next-" + n + "@x",
|
||||
Started: judged.Add(9*time.Minute + ownSend[n]), Took: 4 * time.Second, Detail: OutcomeApplied})
|
||||
}
|
||||
p := Plan{ID: "plan-582", State: PlanDone, Tier: 1, Tiers: [][]string{{"build-agent"}}, Created: merged,
|
||||
Delivery: &PlanDelivery{Merges: []PlanMerge{{Repository: "novox/mesh-controller", Commit: "c", Merged: merged}}},
|
||||
Times: &PlanTimes{Cut: &merged, Class: ClassCore},
|
||||
Modules: map[string]*PlanModule{"build-agent": {State: "built", AskedAt: &merged, BuiltAt: &built,
|
||||
FirstAt: &first, SentAt: &walkSent, Gate: &PlanGate{JudgedAt: &judged}, Rest: rest}}}
|
||||
return sendRest582{judged: judged, walkSent: walkSent, walk: p, rows: rows}
|
||||
}
|
||||
|
||||
// lookup answers from the rows as the controller's durations would.
|
||||
func (w sendRest582) lookup(node string, send SentDeclaration, walkSent time.Time) (AppliedReport, bool) {
|
||||
return FirstReportOf(w.rows[node], send, walkSent)
|
||||
}
|
||||
|
||||
// The apply phase ends at the last machine's own report of this walk's send — shanks, sent at the end of the 20 s
|
||||
// and reporting 15 s later — and never at the next walk's send; every machine's report is shown.
|
||||
func TestTheApplyPhaseEndsAtEachMachinesOwnReport(t *testing.T) {
|
||||
c := walk582(t)
|
||||
w := c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup)
|
||||
sumsUp(t, w)
|
||||
want := c.judged.Add(35 * time.Second) // shanks: sent 20 s after the judgement, reported 15 s later
|
||||
if !w.End.Equal(want) {
|
||||
t.Fatalf("the walk ends at %s, want %s: its rest %+v", w.End, want, w.Rest)
|
||||
}
|
||||
if got := phase(w, PhaseApply, -1); got.State != PhaseMeasured || got.TookMS != want.Sub(c.walkSent).Milliseconds() {
|
||||
t.Fatalf("apply: %+v", got)
|
||||
}
|
||||
if len(w.Rest) != 3 || len(w.Silent) != 0 {
|
||||
t.Fatalf("every machine of the rest is shown with its report: %+v, silent %v", w.Rest, w.Silent)
|
||||
}
|
||||
g14 := w.Rest[0]
|
||||
if g14.Node != "g14" || g14.State != RestApplied || g14.Reported == nil ||
|
||||
!g14.Reported.Equal(c.judged.Add(5500*time.Millisecond)) || g14.Took != "5s" || g14.Sequence != 647 ||
|
||||
!g14.Sent.Equal(c.judged.Add(500*time.Millisecond)) || g14.SentSaid != "" {
|
||||
t.Fatalf("g14, sent first and reporting 5 s after its own send: %+v", g14)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk kept before its machines' own send times were: matched by the declaration it sent, within restSendBound
|
||||
// of its record of the send, so the machine sent first is still found; a walk that kept neither is read as before.
|
||||
func TestAnOlderWalkIsReadByWhatItSent(t *testing.T) {
|
||||
c := walk582(t)
|
||||
for n, s := range c.walk.Modules["build-agent"].Rest {
|
||||
s.At = time.Time{}
|
||||
c.walk.Modules["build-agent"].Rest[n] = s
|
||||
}
|
||||
w := c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup)
|
||||
sumsUp(t, w)
|
||||
if !w.End.Equal(c.judged.Add(35 * time.Second)) {
|
||||
t.Fatalf("by the declaration sent: end %s, rest %+v", w.End, w.Rest)
|
||||
}
|
||||
if !strings.Contains(w.Rest[0].SentSaid, "walk's record") || !w.Rest[0].Sent.Equal(c.walkSent) {
|
||||
t.Fatalf("a send time not kept is said: %+v", w.Rest[0])
|
||||
}
|
||||
// Nothing of the send kept: the first report no earlier than appliedSlack before the walk's record, as before.
|
||||
if r, ok := FirstReportOf(c.rows["shanks"], SentDeclaration{}, c.walkSent); !ok || !r.At.Equal(c.judged.Add(35*time.Second)) {
|
||||
t.Fatalf("a walk that kept nothing of its send: %+v %v", r, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine that said nothing of this walk's send is waiting, then silent past ApplySilentAfter: the report of a
|
||||
// later send to it is never taken as this walk's, and never moves the walk's end.
|
||||
func TestALaterSendsReportIsNeverThisWalks(t *testing.T) {
|
||||
c := walk582(t)
|
||||
c.rows["g14"] = c.rows["g14"][1:] // g14 slept through this walk's send; the next walk's send it reported
|
||||
w := c.walk.WalkPhases(c.judged.Add(5*time.Minute), c.lookup)
|
||||
if w.End != nil || !strings.Contains(phase(w, PhaseApply, -1).Said, "waiting for g14") {
|
||||
t.Fatalf("g14 not heard from yet: end %v, apply %+v", w.End, phase(w, PhaseApply, -1))
|
||||
}
|
||||
w = c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup)
|
||||
sumsUp(t, w)
|
||||
if len(w.Silent) != 1 || w.Silent[0] != "g14" || !w.End.Equal(c.judged.Add(35*time.Second)) {
|
||||
t.Fatalf("g14 silent, the end at shanks' report: silent %v, end %s", w.Silent, w.End)
|
||||
}
|
||||
if w.Rest[0].State != RestSilent || w.Rest[0].Reported != nil {
|
||||
t.Fatalf("g14's later report taken as this walk's: %+v", w.Rest[0])
|
||||
}
|
||||
}
|
||||
|
||||
// A machine sent by two tiers of one walk that said nothing of the first send has said it of the second, which
|
||||
// carries the first tier's build too.
|
||||
func TestAMachineSentTwiceByOneWalkIsHeardByItsLaterSend(t *testing.T) {
|
||||
c := walk582(t)
|
||||
second := c.judged.Add(4 * time.Minute)
|
||||
secondSent := second.Add(time.Second)
|
||||
c.walk.Tiers = append(c.walk.Tiers, []string{"route-proxy"})
|
||||
c.walk.Tier = 2
|
||||
asked, built := c.judged.Add(time.Minute), c.judged.Add(2*time.Minute)
|
||||
first := built.Add(time.Second)
|
||||
c.walk.Modules["route-proxy"] = &PlanModule{State: "built", AskedAt: &asked, BuiltAt: &built, FirstAt: &first,
|
||||
SentAt: &secondSent, Gate: &PlanGate{JudgedAt: &second},
|
||||
Rest: map[string]SentDeclaration{"novox": {Digest: "walk-novox-2", Sequence: 1554, At: second.Add(500 * time.Millisecond)}}}
|
||||
c.rows["novox"] = []Duration{{Kind: DurationApply, Subject: "novox", Ref: "walk-novox-2@x",
|
||||
Started: second.Add(500 * time.Millisecond), Took: 14 * time.Second, Detail: OutcomeApplied}}
|
||||
w := c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup)
|
||||
sumsUp(t, w)
|
||||
want := second.Add(14500 * time.Millisecond)
|
||||
if !w.End.Equal(want) || len(w.Silent) != 0 {
|
||||
t.Fatalf("end %s (want %s), silent %v, rest %+v", w.End, want, w.Silent, w.Rest)
|
||||
}
|
||||
for _, r := range w.Rest {
|
||||
if r.Node == "novox" && (r.State != RestApplied || !r.Reported.Equal(want)) {
|
||||
t.Fatalf("novox, heard by its later send of this walk: %+v", r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The store: a machine's own send time is read with what it was sent, and its `apply` duration is found by it —
|
||||
// the send and the duration start at one moment, the node's sent_at.
|
||||
func TestAWalksRestIsReadFromEachMachinesOwnSend(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
record, err := inv.AddNode(ctx, "g14")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.NextSequence(ctx, record.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, record.ID, "d-582", map[string]string{"build-agent": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sent, found, err := inv.SentTo(ctx, "g14")
|
||||
if err != nil || !found || sent.At.IsZero() || sent.Digest != "d-582" {
|
||||
t.Fatalf("the machine's own send time: %+v %v %v", sent, found, err)
|
||||
}
|
||||
if _, ok, err := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second)); err != nil || ok {
|
||||
t.Fatalf("no report yet: %v %v", ok, err)
|
||||
}
|
||||
if err := inv.RecordApplyDuration(ctx, "g14", "d-582", OutcomeApplied); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The walk's record of its send, 20 s after the machine's own: still found, by the machine's own send.
|
||||
r, ok, err := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second))
|
||||
if err != nil || !ok || r.Outcome != OutcomeApplied || r.At.Before(sent.At) {
|
||||
t.Fatalf("the machine's own report of its send: %+v %v %v", r, ok, err)
|
||||
}
|
||||
// A later send, reported: never this walk's.
|
||||
if err := inv.RecordSent(ctx, record.ID, "d-next", map[string]string{"build-agent": "c2"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordApplyDuration(ctx, "g14", "d-next", OutcomeApplied); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, _, _ := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second))
|
||||
if !again.At.Equal(r.At) {
|
||||
t.Fatalf("a later send's report read as this walk's: %+v, was %+v", again, r)
|
||||
}
|
||||
}
|
||||
@@ -116,7 +116,7 @@ func TestAWalkOfTwoTiersEndsAtItsRestsLastReport(t *testing.T) {
|
||||
}}
|
||||
reports := map[string]AppliedReport{"ace": {At: *at("05:12"), Outcome: OutcomeApplied},
|
||||
"shanks": {At: *at("08:15.25"), Outcome: OutcomeApplied}}
|
||||
w := p.WalkPhases(*at("30:00"), func(node string, _ time.Time) (AppliedReport, bool) {
|
||||
w := p.WalkPhases(*at("30:00"), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) {
|
||||
r, ok := reports[node]
|
||||
return r, ok
|
||||
})
|
||||
@@ -135,7 +135,7 @@ func TestAWalkOfTwoTiersEndsAtItsRestsLastReport(t *testing.T) {
|
||||
}
|
||||
// A report not yet in: the walk has no end, and its apply is unknown — never zero.
|
||||
delete(reports, "shanks")
|
||||
w = p.WalkPhases(*at("10:00"), func(node string, _ time.Time) (AppliedReport, bool) {
|
||||
w = p.WalkPhases(*at("10:00"), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) {
|
||||
r, ok := reports[node]
|
||||
return r, ok
|
||||
})
|
||||
@@ -146,7 +146,7 @@ func TestAWalkOfTwoTiersEndsAtItsRestsLastReport(t *testing.T) {
|
||||
t.Errorf("apply while shanks has not reported: %+v", got)
|
||||
}
|
||||
// Silent past the bound: listed, not waited for.
|
||||
w = p.WalkPhases(at("08:00.5").Add(ApplySilentAfter+time.Second), func(node string, _ time.Time) (AppliedReport, bool) {
|
||||
w = p.WalkPhases(at("08:00.5").Add(ApplySilentAfter+time.Second), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) {
|
||||
r, ok := reports[node]
|
||||
return r, ok
|
||||
})
|
||||
@@ -156,7 +156,7 @@ func TestAWalkOfTwoTiersEndsAtItsRestsLastReport(t *testing.T) {
|
||||
}
|
||||
// A failed report: the walk has no end, said.
|
||||
reports["shanks"] = AppliedReport{At: *at("08:10"), Outcome: OutcomeFailed}
|
||||
w = p.WalkPhases(*at("30:00"), func(node string, _ time.Time) (AppliedReport, bool) {
|
||||
w = p.WalkPhases(*at("30:00"), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) {
|
||||
r, ok := reports[node]
|
||||
return r, ok
|
||||
})
|
||||
@@ -247,7 +247,7 @@ func TestAWalksMomentsAreKeptAndItsRestsReportRead(t *testing.T) {
|
||||
t.Fatal("a merge's time was not kept")
|
||||
}
|
||||
sent := time.Now().UTC().Add(-time.Minute).Truncate(time.Millisecond)
|
||||
if _, ok, err := inv.FirstAppliedAfter(ctx, "ace", sent); err != nil || ok {
|
||||
if _, ok, err := inv.FirstAppliedAfter(ctx, "ace", SentDeclaration{}, sent); err != nil || ok {
|
||||
t.Fatalf("no report yet: %v %v", ok, err)
|
||||
}
|
||||
for _, d := range []Duration{
|
||||
@@ -258,7 +258,7 @@ func TestAWalksMomentsAreKeptAndItsRestsReportRead(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
r, ok, err := inv.FirstAppliedAfter(ctx, "ace", sent)
|
||||
r, ok, err := inv.FirstAppliedAfter(ctx, "ace", SentDeclaration{}, sent)
|
||||
if err != nil || !ok || r.Outcome != OutcomeApplied || !r.At.Equal(sent.Add(11*time.Second)) {
|
||||
t.Fatalf("the report after the send: %+v %v %v", r, ok, err)
|
||||
}
|
||||
@@ -281,7 +281,7 @@ func TestALateReportIsSilentNotTheEnd(t *testing.T) {
|
||||
p := walkOf(t, recordedWalk)
|
||||
p.Modules["mesh-delivery"].SentAt = &sent
|
||||
p.Modules["mesh-delivery"].Rest = map[string]SentDeclaration{"laptop": {Digest: "d"}, "server": {Digest: "e"}}
|
||||
w := p.WalkPhases(sent.Add(3*time.Hour), func(node string, _ time.Time) (AppliedReport, bool) {
|
||||
w := p.WalkPhases(sent.Add(3*time.Hour), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) {
|
||||
if node == "server" {
|
||||
return AppliedReport{At: sent.Add(20 * time.Second), Outcome: OutcomeApplied}, true
|
||||
}
|
||||
@@ -296,7 +296,7 @@ func TestALateReportIsSilentNotTheEnd(t *testing.T) {
|
||||
Started: sent.Add(time.Hour), Took: time.Second, Detail: OutcomeApplied}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok, err := inv.FirstAppliedAfter(ctx, "laptop", sent); err != nil || ok {
|
||||
if _, ok, err := inv.FirstAppliedAfter(ctx, "laptop", SentDeclaration{}, sent); err != nil || ok {
|
||||
t.Fatalf("a send past the bound read as the walk's: %v %v", ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,6 +75,10 @@ type Options struct {
|
||||
Holder Holder
|
||||
// RenewEvery, Margin and Poll default to the package's.
|
||||
RenewEvery, Margin, Poll time.Duration
|
||||
// RenewWithin is how long one renewal may take before it counts as failed: RenewEvery when zero. Apart
|
||||
// from it so a test that renews often still gives a slow bus a serving controller's time to answer
|
||||
// (novox/hq issue 591).
|
||||
RenewWithin time.Duration
|
||||
// Floor is the highest epoch this mesh has issued, from the controller's own store. **An epoch is
|
||||
// never issued twice**: a lease bucket raised again from nothing — a bus whose data was replaced —
|
||||
// starts its revisions over, and every machine that heard epoch 57 would refuse the next controller
|
||||
@@ -138,6 +142,9 @@ func Open(ctx context.Context, js jetstream.JetStream, bucket string, o Options)
|
||||
if o.RenewEvery <= 0 {
|
||||
o.RenewEvery = RenewEvery
|
||||
}
|
||||
if o.RenewWithin <= 0 {
|
||||
o.RenewWithin = o.RenewEvery
|
||||
}
|
||||
if o.Margin <= 0 {
|
||||
o.Margin = Margin
|
||||
}
|
||||
@@ -406,7 +413,7 @@ func (l *Lease) Renew(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return l.lose(err)
|
||||
}
|
||||
renewing, cancel := context.WithTimeout(ctx, l.o.RenewEvery)
|
||||
renewing, cancel := context.WithTimeout(ctx, l.o.RenewWithin)
|
||||
defer cancel()
|
||||
next, err := l.kv.Update(renewing, Key, value, revision)
|
||||
if err != nil {
|
||||
|
||||
@@ -32,14 +32,13 @@ func ForTest(t *testing.T) *Licences {
|
||||
name = name[:60]
|
||||
}
|
||||
|
||||
conn, err := pgx.Connect(t.Context(), admin)
|
||||
// Created already migrated, a copy of a template migrated once (novox/hq issue 591); the Migrate
|
||||
// below then finds nothing pending, which it does only if the copy's record matches every migration.
|
||||
migrations, err := Migrations()
|
||||
if err != nil {
|
||||
t.Fatalf("cannot reach the test PostgreSQL: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := conn.Exec(t.Context(), "create database "+name); err != nil {
|
||||
t.Fatalf("cannot create %s: %v", name, err)
|
||||
}
|
||||
conn.Close(t.Context())
|
||||
store.CreateMigratedForTest(t, admin, name, migrations)
|
||||
|
||||
cut := strings.LastIndex(admin, "/")
|
||||
t.Setenv(store.Variable(Name), admin[:cut]+"/"+name+"?sslmode=disable")
|
||||
@@ -61,10 +60,6 @@ func ForTest(t *testing.T) *Licences {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
migrations, err := Migrations()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := held.store.Migrate(t.Context(), migrations); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -257,6 +257,10 @@ type BuildResult struct {
|
||||
// build; then only identical artifacts make a rebuild no move.
|
||||
SourceFingerprint string `json:"source-fingerprint,omitempty"`
|
||||
|
||||
// Delivery is the how-manyth time the queue handed this ask out (BuildStart.Delivery): more than 1
|
||||
// says the outcome is from a run started again after an earlier taker left it (novox/hq issue 594).
|
||||
Delivery int `json:"delivery,omitempty"`
|
||||
|
||||
// Failed is why, when it did.
|
||||
Failed string `json:"failed,omitempty"`
|
||||
|
||||
|
||||
@@ -92,6 +92,31 @@ type BuildStart struct {
|
||||
Ref string `json:"ref,omitempty"`
|
||||
On string `json:"on"`
|
||||
At string `json:"at"`
|
||||
// Delivery is the how-manyth time the queue handed this ask out: 1 for a first start, more for one
|
||||
// an earlier taker left unfinished — most often a build agent replaced mid-build — and started
|
||||
// again from the beginning (novox/hq issue 594). 0 from a machine that predates it.
|
||||
Delivery int `json:"delivery,omitempty"`
|
||||
}
|
||||
|
||||
// Restarted is how a build handed out again says so (novox/hq issue 594), in its log and in its check's
|
||||
// report, so a second run never reads as the first. Neutral on why: a holder replaced mid-build, a paused
|
||||
// holder handing an ask back as it arrived, and a bus that held its acknowledgements past the wait all
|
||||
// count as a hand-out. Empty for a first start.
|
||||
func Restarted(delivery int) string {
|
||||
if delivery < 2 {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("restarted: handed out %d times; an earlier hand-out did not finish here, so this run "+
|
||||
"started from the beginning", delivery)
|
||||
}
|
||||
|
||||
// RunMarker is the short form of Restarted, for a line clipped to a few words — a check's status on the
|
||||
// forge (novox/hq issue 594): "(run 2)". Empty for a first start.
|
||||
func RunMarker(delivery int) string {
|
||||
if delivery < 2 {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("(run %d)", delivery)
|
||||
}
|
||||
|
||||
// BuildLine is one thing a build said while building.
|
||||
|
||||
@@ -79,6 +79,8 @@ func (b *natsBuilds) Ask(ctx context.Context, request BuildRequest) error {
|
||||
if _, err := b.js.Context().Publish(BuildWorkOf(b.role()), body, nats.Context(publish)); err != nil {
|
||||
return fmt.Errorf("cannot submit a build: %w", err)
|
||||
}
|
||||
// The acting controller puts it in order (novox/hq issue 558); this only says an ask joined.
|
||||
signalAsked()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -113,6 +115,8 @@ func (b *natsBuilds) Submit(ctx context.Context, request BuildRequest,
|
||||
if _, err := b.js.Context().Publish(BuildWorkOf(b.role()), body, nats.Context(publish)); err != nil {
|
||||
return BuildResult{}, fmt.Errorf("%w: cannot submit a build: %w", ErrNotHandedOver, err)
|
||||
}
|
||||
// The acting controller puts it in order (novox/hq issue 558); this only says an ask joined.
|
||||
signalAsked()
|
||||
|
||||
waiting, cancelWait := context.WithTimeout(ctx, wait)
|
||||
defer cancelWait()
|
||||
@@ -167,6 +171,10 @@ type MachineOptions struct {
|
||||
// Paused is asked before every fetch: while it says so, nothing new is taken, and a build
|
||||
// already running finishes. Nil is never paused.
|
||||
Paused func() bool
|
||||
// Taking is called once, when this machine is bound to its worker and pulls work: from then on it
|
||||
// can take a build, which is what its container's health check reads (novox/hq issue 544). Nil is
|
||||
// told nothing.
|
||||
Taking func()
|
||||
}
|
||||
|
||||
// MachineOverNATS takes build work from the current build role.
|
||||
@@ -232,6 +240,9 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
"the mesh's to answer, not this machine's", m.seat, err)
|
||||
}
|
||||
m.sub = sub
|
||||
if m.opts.Taking != nil {
|
||||
m.opts.Taking()
|
||||
}
|
||||
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
@@ -299,7 +310,7 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
// built. A set that cannot be read is said and the ask built: a cancel is a person's
|
||||
// exception, and a holder that refused every build while its grant was missing would
|
||||
// stop the mesh building for it.
|
||||
build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat}
|
||||
build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat, delivery: deliveryOf(msg)}
|
||||
if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err)
|
||||
} else if cancelled {
|
||||
@@ -325,6 +336,18 @@ type natsBuild struct {
|
||||
// seat is the role this build was taken from; what the machine says about it is that role's.
|
||||
seat string
|
||||
seq int
|
||||
// delivery is the how-manyth time the queue handed this ask out (novox/hq issue 594); 0 when the
|
||||
// server did not say.
|
||||
delivery int
|
||||
}
|
||||
|
||||
// deliveryOf is how many times the queue has handed this message out, this time included.
|
||||
func deliveryOf(msg *nats.Msg) int {
|
||||
meta, err := msg.Metadata()
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return int(meta.NumDelivered)
|
||||
}
|
||||
|
||||
func (b *natsBuild) Request() BuildRequest { return b.request }
|
||||
@@ -345,6 +368,11 @@ func (b *natsBuild) Announce(ctx context.Context, result BuildResult) error {
|
||||
if result.Failed == "" && result.Commit != "" {
|
||||
result.Module = ModuleOf(result.Manifest)
|
||||
}
|
||||
// **Which run this is** (novox/hq issue 594): stamped by the transport, which alone knows, so every
|
||||
// outcome carries it whatever the machine's code filled in.
|
||||
if result.Delivery == 0 {
|
||||
result.Delivery = b.delivery
|
||||
}
|
||||
body, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -364,11 +392,16 @@ func (b *natsBuild) Done() error { return b.msg.Ack() }
|
||||
func (b *natsBuild) Began(ctx context.Context) error {
|
||||
body, err := json.Marshal(BuildStart{
|
||||
ID: b.request.ID, Repository: b.request.Repository, Path: b.request.Path,
|
||||
Ref: b.request.Ref, On: b.on, At: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
Ref: b.request.Ref, On: b.on, At: time.Now().UTC().Format(time.RFC3339Nano), Delivery: b.delivery,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// **A start that is a restart says so in the build's log** (novox/hq issue 594): a reader following
|
||||
// the build sees why it began again, never a silent second run.
|
||||
if said := Restarted(b.delivery); said != "" {
|
||||
b.Say("restarted", said)
|
||||
}
|
||||
publish, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
if _, err := b.js.Context().Publish(BuildStartedOf(b.seat), body, nats.Context(publish)); err != nil {
|
||||
|
||||
@@ -368,3 +368,130 @@ func TestNatsAMachineOnTheRetiredBuildRoleTakesThatRolesAsks(t *testing.T) {
|
||||
t.Errorf("the retired role's holder did not answer: %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
// A holder says once that it takes work, and only once it is bound to its worker (novox/hq issue 544):
|
||||
// its container's health check reads that, so a holder that could not bind is never said ready.
|
||||
func TestNatsAHolderSaysItTakesWorkOnceItIsBound(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
said := make(chan struct{}, 4)
|
||||
machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Taking: func() { said <- struct{}{} }})
|
||||
defer machine.Close()
|
||||
go func() { _ = machine.Take(ctx, func(context.Context, Build) {}) }()
|
||||
select {
|
||||
case <-said:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("a holder bound to its worker never said it takes work")
|
||||
}
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if len(said) != 0 {
|
||||
t.Fatalf("a holder said it takes work %d more time(s)", len(said))
|
||||
}
|
||||
|
||||
// A role the mesh raised no worker for cannot be bound, and its holder says nothing.
|
||||
bounded, end := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer end()
|
||||
other := MachineOverNATSWith(js, "anchor", "mesh-no-such-seat", MachineOptions{Taking: func() { said <- struct{}{} }})
|
||||
defer other.Close()
|
||||
if err := other.Take(bounded, func(context.Context, Build) {}); err == nil {
|
||||
t.Fatal("a machine with no worker took work")
|
||||
}
|
||||
if len(said) != 0 {
|
||||
t.Fatal("a machine that could not bind said it takes work")
|
||||
}
|
||||
}
|
||||
|
||||
// **An ask handed out again says so** (novox/hq issue 594). A build agent replaced mid-build leaves its
|
||||
// ask unsettled, and the queue hands it out again to start over from the beginning. That second start
|
||||
// must not look like a first one: the machine that takes it says it restarted — in its started event,
|
||||
// in the build's log and in its outcome — so the queue and the pull request can say it too.
|
||||
func TestNatsAnAskHandedOutAgainSaysItRestarted(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
|
||||
started, err := js.Conn().SubscribeSync(BuildStarted())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = started.Unsubscribe() }()
|
||||
lines, err := js.Conn().SubscribeSync(BuildLog("b-594"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = lines.Unsubscribe() }()
|
||||
outcomes, err := js.Conn().SubscribeSync(BuildOutcome())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = outcomes.Unsubscribe() }()
|
||||
_ = js.Conn().Flush()
|
||||
|
||||
body, _ := json.Marshal(BuildRequest{ID: "b-594", Repository: "/r"})
|
||||
if _, err := js.Context().Publish(BuildWork(), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
machine := MachineOverNATS(js, "anchor")
|
||||
defer machine.Close()
|
||||
var first sync.Once
|
||||
go func() {
|
||||
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
||||
_ = work.Began(ctx)
|
||||
handedBack := false
|
||||
// The first time it is left unfinished — an agent replaced mid-build — and comes back.
|
||||
first.Do(func() { _ = work.Hold(100 * time.Millisecond); handedBack = true })
|
||||
if handedBack {
|
||||
return
|
||||
}
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
|
||||
var starts []BuildStart
|
||||
for len(starts) < 2 {
|
||||
msg, err := started.NextMsg(10 * time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("heard %d start(s) of an ask handed out twice: %v", len(starts), err)
|
||||
}
|
||||
var s BuildStart
|
||||
if err := json.Unmarshal(msg.Data, &s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
starts = append(starts, s)
|
||||
}
|
||||
if starts[0].Delivery != 1 || starts[1].Delivery != 2 {
|
||||
t.Fatalf("the starts say deliveries %d and %d, want 1 and 2: a restart reads as a first start",
|
||||
starts[0].Delivery, starts[1].Delivery)
|
||||
}
|
||||
|
||||
msg, err := outcomes.NextMsg(10 * time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var result BuildResult
|
||||
if err := json.Unmarshal(msg.Data, &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Delivery != 2 {
|
||||
t.Fatalf("the outcome says delivery %d, want 2: the pull request cannot say the check restarted",
|
||||
result.Delivery)
|
||||
}
|
||||
|
||||
sawRestarted := false
|
||||
for {
|
||||
msg, err := lines.NextMsg(time.Second)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
var line BuildLine
|
||||
if json.Unmarshal(msg.Data, &line) == nil && line.Step == "restarted" {
|
||||
sawRestarted = true
|
||||
}
|
||||
}
|
||||
if !sawRestarted {
|
||||
t.Fatal("the build's log never said it restarted")
|
||||
}
|
||||
}
|
||||
|
||||
+47
-10
@@ -601,6 +601,24 @@ func Acknowledge(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// LongestAnswer is the longest a handler that waits on purpose may hold its caller's one answer
|
||||
// (AnswersWithin, novox/hq issue 229): inside the console's thirty seconds with room for the trip back,
|
||||
// and inside the bus's window for an answer (broker.ResponseTTL). A variable so a test need not wait.
|
||||
var LongestAnswer = 25 * time.Second
|
||||
|
||||
type answerWithinKey struct{}
|
||||
|
||||
// AnswersWithin asks that the call's caller be answered in full if the handler ends within d of the
|
||||
// call's start, rather than within AnswerWithin: for a handler that waits on purpose, with a bound —
|
||||
// `plans` waiting for a plan to move (novox/hq issue 229). Past d, or past LongestAnswer whichever is
|
||||
// shorter, its caller is told it is running, as for any call. Never shortens the window; without effect
|
||||
// outside a served call, and once the caller was answered.
|
||||
func AnswersWithin(ctx context.Context, d time.Duration) {
|
||||
if f, ok := ctx.Value(answerWithinKey{}).(func(time.Duration)); ok {
|
||||
f(d)
|
||||
}
|
||||
}
|
||||
|
||||
// running is the interim answer: what a caller reads when the call has not finished.
|
||||
func running(c *Call, within time.Duration, acknowledged bool, follow string) []byte {
|
||||
why := fmt.Sprintf("it has not finished in %s", within)
|
||||
@@ -645,6 +663,13 @@ func (l *CallLog) serveCallWithin(seat, verb string, args json.RawMessage, reply
|
||||
acknowledged := make(chan struct{})
|
||||
var once sync.Once
|
||||
ctx = context.WithValue(ctx, answerNowKey{}, func() { once.Do(func() { close(acknowledged) }) })
|
||||
longer := make(chan time.Duration, 1)
|
||||
ctx = context.WithValue(ctx, answerWithinKey{}, func(d time.Duration) {
|
||||
select {
|
||||
case longer <- d:
|
||||
default:
|
||||
}
|
||||
})
|
||||
|
||||
type outcome struct {
|
||||
body []byte
|
||||
@@ -703,17 +728,29 @@ func (l *CallLog) serveCallWithin(seat, verb string, args json.RawMessage, reply
|
||||
}
|
||||
}
|
||||
}
|
||||
timer := time.NewTimer(AnswerWithin)
|
||||
within, start := AnswerWithin, time.Now()
|
||||
timer := time.NewTimer(within)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case o := <-done:
|
||||
l.finish(c, o.body, o.failed, false)
|
||||
say(o.body)
|
||||
return
|
||||
case <-acknowledged:
|
||||
say(running(c, AnswerWithin, true, l.Follow))
|
||||
case <-timer.C:
|
||||
say(running(c, AnswerWithin, false, l.Follow))
|
||||
waiting:
|
||||
for {
|
||||
select {
|
||||
case o := <-done:
|
||||
l.finish(c, o.body, o.failed, false)
|
||||
say(o.body)
|
||||
return
|
||||
case <-acknowledged:
|
||||
say(running(c, within, true, l.Follow))
|
||||
break waiting
|
||||
case d := <-longer:
|
||||
// A handler waiting on purpose (AnswersWithin): its window is stretched, never past LongestAnswer.
|
||||
if d = min(d, LongestAnswer); d > within {
|
||||
within = d
|
||||
timer.Reset(time.Until(start.Add(d)))
|
||||
}
|
||||
case <-timer.C:
|
||||
say(running(c, within, false, l.Follow))
|
||||
break waiting
|
||||
}
|
||||
}
|
||||
o := <-done
|
||||
l.finish(c, o.body, o.failed, true)
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **A handler that waits on purpose is answered in full when it ends inside the window it asked for**
|
||||
// (novox/hq issue 229): `plans` asked to wait for a plan holds its caller's one answer until the plan
|
||||
// moves or its bound passes, rather than being answered "still running" at AnswerWithin.
|
||||
func TestAHandlerThatWaitsOnPurposeIsAnsweredInFullInsideItsWindow(t *testing.T) {
|
||||
shortWindow(t, 20*time.Millisecond)
|
||||
l, a := NewCallLog(), newAnswers(t)
|
||||
l.serveCall("mesh-controller", "plans", json.RawMessage(`{"wait":"plan-1"}`), "_INBOX.x.3",
|
||||
func(ctx context.Context, _ json.RawMessage) (any, error) {
|
||||
AnswersWithin(ctx, 200*time.Millisecond)
|
||||
time.Sleep(80 * time.Millisecond) // past AnswerWithin, inside what it asked for
|
||||
return "plan-1 moved", nil
|
||||
}, a.respond, nil)
|
||||
if got := a.only()["result"]; got != "plan-1 moved" {
|
||||
t.Fatalf("a waiting handler inside its window was answered %v, not in full", got)
|
||||
}
|
||||
}
|
||||
|
||||
// What a handler may ask for is bounded: past LongestAnswer its caller is told it is running, as any
|
||||
// call is, so no answer is ever later than the console waits.
|
||||
func TestAWaitIsNeverLongerThanTheLongestAnswer(t *testing.T) {
|
||||
shortWindow(t, 10*time.Millisecond)
|
||||
was := LongestAnswer
|
||||
LongestAnswer = 30 * time.Millisecond
|
||||
t.Cleanup(func() { LongestAnswer = was })
|
||||
l, a := NewCallLog(), newAnswers(t)
|
||||
release := make(chan struct{})
|
||||
finished := make(chan struct{})
|
||||
go func() {
|
||||
l.serveCall("mesh-controller", "plans", nil, "_INBOX.x.4", func(ctx context.Context, _ json.RawMessage) (any, error) {
|
||||
AnswersWithin(ctx, time.Hour)
|
||||
<-release
|
||||
return "late", nil
|
||||
}, a.respond, nil)
|
||||
close(finished)
|
||||
}()
|
||||
select {
|
||||
case <-a.sent:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("a handler asking for an hour held its caller past LongestAnswer")
|
||||
}
|
||||
if got, _ := a.only()["result"].(map[string]any); got["running"] != true {
|
||||
t.Fatalf("past LongestAnswer the caller was answered %v, not that it is running", a.only())
|
||||
}
|
||||
close(release)
|
||||
<-finished
|
||||
}
|
||||
@@ -325,6 +325,9 @@ type ChangePlan struct {
|
||||
Steps []string `json:"steps,omitempty"`
|
||||
// Summary is the plan in one line, for a commit status.
|
||||
Summary string `json:"summary"`
|
||||
// Class is the class of the slowest-budgeted module the change moves (novox/hq ADR 0282 §1, ADR 0313 §2):
|
||||
// core, leaf, or none when it moves no module at all — a fix that moves nothing has no fix budget.
|
||||
Class string `json:"class,omitempty"`
|
||||
// Trunk is the trunk commit the head was merged into when it was judged (novox/hq ADR 0323 §3), twelve
|
||||
// characters: set on the plan a verdict carries, which mesh-delivery keeps with its other inputs.
|
||||
Trunk string `json:"trunk,omitempty"`
|
||||
|
||||
+45
-17
@@ -62,13 +62,20 @@ type QueuedAsk struct {
|
||||
Path string `json:"path,omitempty"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
AskedAt time.Time `json:"asked-at,omitempty"`
|
||||
State string `json:"state"`
|
||||
// Stored is when the stream first took it, a move into order kept (novox/hq issue 558).
|
||||
Stored time.Time `json:"stored,omitempty"`
|
||||
State string `json:"state"`
|
||||
// movedFrom is the MovedFromHeader it carries: the sequence of the message a move copied it from.
|
||||
movedFrom string
|
||||
// On and Started are the holder building an in-flight ask and when it started, from its started
|
||||
// event. Was is the holder that started an in-flight ask and is no longer building it — handed
|
||||
// back, to be handed out again — with Started its start there.
|
||||
On string `json:"on,omitempty"`
|
||||
Was string `json:"was-on,omitempty"`
|
||||
Started time.Time `json:"started,omitempty"`
|
||||
// Delivery is the how-manyth time the ask was handed out, as its latest start said: more than 1 is
|
||||
// a build started again after an earlier taker left it (novox/hq issue 594).
|
||||
Delivery int `json:"delivery,omitempty"`
|
||||
}
|
||||
|
||||
// Queue is the seat's work queue as it stands.
|
||||
@@ -158,7 +165,7 @@ func ClassifyQueue(asks []QueuedAsk, delivered uint64, ackPending int, heard Bui
|
||||
}
|
||||
a.State = AskDead
|
||||
if s, ok := current[a.ID]; ok {
|
||||
a.On, a.Started = s.On, startedAt(s)
|
||||
a.On, a.Started, a.Delivery = s.On, startedAt(s), s.Delivery
|
||||
running = append(running, i)
|
||||
continue
|
||||
}
|
||||
@@ -179,7 +186,7 @@ func ClassifyQueue(asks []QueuedAsk, delivered uint64, ackPending int, heard Bui
|
||||
for _, i := range group {
|
||||
if slots == 0 {
|
||||
// Past what the worker counts: handed out as often as it may be.
|
||||
out[i].On, out[i].Started = "", time.Time{}
|
||||
out[i].On, out[i].Started, out[i].Delivery = "", time.Time{}, 0
|
||||
continue
|
||||
}
|
||||
out[i].State = AskInFlight
|
||||
@@ -236,15 +243,7 @@ func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, e
|
||||
if err != nil {
|
||||
return q, fmt.Errorf("reading the %s work queue: %w", seat, err)
|
||||
}
|
||||
ask := QueuedAsk{Seq: msg.Sequence}
|
||||
if json.Unmarshal(msg.Data, &ask.Request) == nil {
|
||||
r := ask.Request
|
||||
ask.ID, ask.Repository, ask.Path, ask.Ref = r.ID, r.Repository, r.Path, r.Ref
|
||||
if at, ok := BuildAskedAt(r.ID); ok {
|
||||
ask.AskedAt = at
|
||||
}
|
||||
}
|
||||
asks = append(asks, ask)
|
||||
asks = append(asks, askOf(msg))
|
||||
seq = msg.Sequence + 1
|
||||
}
|
||||
|
||||
@@ -269,6 +268,26 @@ func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, e
|
||||
return q, nil
|
||||
}
|
||||
|
||||
// askOf is one ask as the stream holds it: what its request says, and when the stream took it.
|
||||
func askOf(msg *jetstream.RawStreamMsg) QueuedAsk {
|
||||
ask := QueuedAsk{Seq: msg.Sequence, Stored: msg.Time}
|
||||
// Moved into order, it keeps when the stream first took it (novox/hq issue 558).
|
||||
if at, err := time.Parse(time.RFC3339Nano, msg.Header.Get(QueuedAtHeader)); err == nil {
|
||||
ask.Stored = at
|
||||
}
|
||||
if json.Unmarshal(msg.Data, &ask.Request) == nil {
|
||||
r := ask.Request
|
||||
ask.ID, ask.Repository, ask.Path, ask.Ref = r.ID, r.Repository, r.Path, r.Ref
|
||||
if at, ok := BuildAskedAt(r.ID); ok {
|
||||
ask.AskedAt = at
|
||||
}
|
||||
} else {
|
||||
// Nothing read is nothing kept: a half-read request would pass for a delivery (novox/hq issue 558).
|
||||
ask.Request = BuildRequest{}
|
||||
}
|
||||
return ask
|
||||
}
|
||||
|
||||
// WithoutPassword is a repository as a listing may say it: a URL's password, should one ever be written
|
||||
// into it, left out (novox/hq issue 455). Only for what is shown: the ask itself keeps what it was given,
|
||||
// which a cancel's outcome and a rebuild's match are read from.
|
||||
@@ -305,11 +324,23 @@ func ReadBuildEvents(ctx context.Context, js *broker.JetStream, seat string, sin
|
||||
return heard, fmt.Errorf("cannot read %s from the bus: %w", subject, err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
for {
|
||||
batch, err := sub.Fetch(500, nats.MaxWait(2*time.Second))
|
||||
// **Read what is there, then stop** (novox/hq issue 591): the consumer says how many events it holds
|
||||
// from the moment asked, and the reading ends once they are read. A fetch for a full batch waits out
|
||||
// its whole wait when fewer are there, so every reading of the queue took two seconds and more.
|
||||
info, err := sub.ConsumerInfo()
|
||||
if err != nil {
|
||||
return heard, fmt.Errorf("cannot read %s from the bus: %w", subject, err)
|
||||
}
|
||||
for remaining := info.NumPending; remaining > 0; {
|
||||
batch, err := sub.Fetch(int(min(remaining, 500)), nats.MaxWait(2*time.Second))
|
||||
if err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.DeadlineExceeded) {
|
||||
return heard, fmt.Errorf("reading %s: %w", subject, err)
|
||||
}
|
||||
// None in the whole wait: what was counted is gone from the stream since (its limits), not coming.
|
||||
if len(batch) == 0 {
|
||||
break
|
||||
}
|
||||
remaining -= uint64(len(batch))
|
||||
for _, msg := range batch {
|
||||
switch msg.Subject {
|
||||
case BuildStartedOf(seat):
|
||||
@@ -326,9 +357,6 @@ func ReadBuildEvents(ctx context.Context, js *broker.JetStream, seat string, sin
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(batch) < 500 {
|
||||
break
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return heard, ctx.Err()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,426 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// The order the build queue hands asks out in (novox/hq issue 558).
|
||||
//
|
||||
// **A build a walk waits for goes before a pull request's check.** The queue was first come, first served,
|
||||
// and a merged change's delivery build waited minutes behind checks while its walk, and every walk after
|
||||
// it, waited on the build. So:
|
||||
//
|
||||
// - a delivery is handed out before any check;
|
||||
// - a check that has waited CheckWaitsAtMost goes before every delivery asked after it, so no check
|
||||
// waits for ever behind a stream of deliveries;
|
||||
// - within a kind, first come, first served, by when it was asked;
|
||||
// - an ask whose kind cannot be told counts as a check: a delivery taken late breaks a budget, a check
|
||||
// taken late breaks nothing.
|
||||
//
|
||||
// **Kept in the stream, never beside it.** The queue is the seat's work queue, which the server hands out
|
||||
// in stream order to whichever holder pulls; nothing else holds the order, so a restarted controller
|
||||
// finds it as it was left. An ask that must go behind another is moved to the tail — deleted from where
|
||||
// it was and published again, its body unchanged, under the same id — while the worker is paused, so no
|
||||
// holder takes it in the moment it moves. Only waiting asks move: what a holder took is never touched.
|
||||
|
||||
// The kinds of ask, as the queue says them.
|
||||
const (
|
||||
KindDelivery = "delivery"
|
||||
KindCheck = "check"
|
||||
)
|
||||
|
||||
// CheckWaitsAtMost is how long a check waits before it goes ahead of the deliveries asked after it.
|
||||
var CheckWaitsAtMost = 10 * time.Minute
|
||||
|
||||
// KindOf is an ask's kind: a check for a pull request's check — a recheck the merge tool asks included —
|
||||
// a dry run, or an ask nothing could be read from; a delivery for any other build.
|
||||
func KindOf(r BuildRequest) string {
|
||||
if r.ID == "" || r.Check != nil || r.DryRun {
|
||||
return KindCheck
|
||||
}
|
||||
return KindDelivery
|
||||
}
|
||||
|
||||
// askedOrStored is when an ask was asked, as its id says, or else when the queue took it.
|
||||
func (a QueuedAsk) askedOrStored() time.Time {
|
||||
if !a.AskedAt.IsZero() {
|
||||
return a.AskedAt
|
||||
}
|
||||
return a.Stored
|
||||
}
|
||||
|
||||
// QueueOrder is the order the waiting asks are to be handed out in, now: the deliveries and every check
|
||||
// that has waited CheckWaitsAtMost, by when each was asked, then the other checks, by when each was asked.
|
||||
func QueueOrder(waiting []QueuedAsk, now time.Time) []QueuedAsk {
|
||||
var first, then []QueuedAsk
|
||||
for _, a := range waiting {
|
||||
if KindOf(a.Request) == KindCheck && now.Sub(a.askedOrStored()) < CheckWaitsAtMost {
|
||||
then = append(then, a)
|
||||
continue
|
||||
}
|
||||
first = append(first, a)
|
||||
}
|
||||
byAsking := func(s []QueuedAsk) {
|
||||
sort.SliceStable(s, func(i, j int) bool {
|
||||
if ti, tj := s[i].askedOrStored(), s[j].askedOrStored(); !ti.Equal(tj) {
|
||||
return ti.Before(tj)
|
||||
}
|
||||
return s[i].Seq < s[j].Seq
|
||||
})
|
||||
}
|
||||
byAsking(first)
|
||||
byAsking(then)
|
||||
return append(first, then...)
|
||||
}
|
||||
|
||||
// toMove is the asks to move to the tail, in this order, for the queue to stand as order says: everything
|
||||
// after the longest start of order that the queue already holds in that order.
|
||||
func toMove(queue, order []QueuedAsk) []QueuedAsk {
|
||||
at := make(map[uint64]int, len(queue))
|
||||
for i, a := range queue {
|
||||
at[a.Seq] = i
|
||||
}
|
||||
kept, last := 0, -1
|
||||
for kept < len(order) {
|
||||
i, ok := at[order[kept].Seq]
|
||||
if !ok || i < last {
|
||||
break
|
||||
}
|
||||
last = i
|
||||
kept++
|
||||
}
|
||||
return order[kept:]
|
||||
}
|
||||
|
||||
// orderPasses keeps one pass at a time. Only the acting controller's loop runs one (KeepingInOrder), so
|
||||
// there is one; the lock holds a test's two at once to that too.
|
||||
var orderPasses sync.Mutex
|
||||
|
||||
// pausedFor bounds a pause the pass could not end itself: a controller that stops mid-pass leaves the
|
||||
// worker paused this long at most. moveMargin is the part of it no move starts in: a pass stops moving
|
||||
// with at least this left, and the next pass moves the rest, so no move runs past the pause.
|
||||
const (
|
||||
pausedFor = 20 * time.Second
|
||||
moveMargin = 5 * time.Second
|
||||
publishWaits = 3 * time.Second
|
||||
)
|
||||
|
||||
// QueuedAtHeader carries, on a moved ask, when the stream first took it: a move keeps an ask's place in
|
||||
// time, its id unread or not.
|
||||
const QueuedAtHeader = "Mesh-Queued-At"
|
||||
|
||||
// MovedFromHeader names, on a move's copy, the sequence of the message it is a copy of: with QueuedAtHeader,
|
||||
// what proves a second message of one id is a move's copy and not an ask delivered again (ADR 0293), whose
|
||||
// headers are its original's — and so never name that original's own sequence.
|
||||
const MovedFromHeader = "Mesh-Moved-From"
|
||||
|
||||
// Pass is what one pass did: the asks it moved, and each second copy it removed, in words.
|
||||
type Pass struct {
|
||||
Moved []string
|
||||
Removed []string
|
||||
}
|
||||
|
||||
// asked is the signal an ask sends the acting controller's pass loop, in this process. An ask asked in
|
||||
// another process — a verb run as its own command — is seen by the loop as the queue's last sequence moving.
|
||||
var asked = make(chan struct{}, 1)
|
||||
|
||||
// signalAsked tells the pass loop an ask joined the queue. Never blocks, never runs a pass.
|
||||
func signalAsked() {
|
||||
select {
|
||||
case asked <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// PutInOrder moves the seat's waiting asks into the order QueueOrder says, and says each it moved. A queue
|
||||
// already in order is read and left alone; the worker is paused only while asks move. Run by the acting
|
||||
// controller alone (KeepingInOrder): two passes in two processes would each resume what the other paused.
|
||||
func PutInOrder(ctx context.Context, js *broker.JetStream, seat string) (Pass, error) {
|
||||
var pass Pass
|
||||
orderPasses.Lock()
|
||||
defer orderPasses.Unlock()
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
return pass, err
|
||||
}
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
|
||||
waiting, extra, delivered, err := readWaiting(ctx, api, worker, seat)
|
||||
if err != nil || (len(extra) == 0 && len(toMove(waiting, QueueOrder(waiting, time.Now()))) == 0) {
|
||||
return pass, err
|
||||
}
|
||||
// Paused, then read again: what the worker handed out up to the pause is not waiting any more.
|
||||
pauseEnds := time.Now().Add(pausedFor)
|
||||
if delivered >= 0 {
|
||||
if _, err := api.PauseConsumer(ctx, worker.Stream, worker.Name, pauseEnds); err != nil {
|
||||
return pass, fmt.Errorf("the %s worker could not be paused to put its queue in order, so it is left "+
|
||||
"first come, first served: %w", seat, err)
|
||||
}
|
||||
defer func() {
|
||||
resuming, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
defer cancel()
|
||||
if _, err := api.ResumeConsumer(resuming, worker.Stream, worker.Name); err != nil {
|
||||
fmt.Printf("the %s worker could not be resumed after its queue was put in order; it resumes by "+
|
||||
"itself by %s: %v\n", seat, pauseEnds.Format(time.RFC3339), err)
|
||||
}
|
||||
}()
|
||||
if waiting, extra, _, err = readWaiting(ctx, api, worker, seat); err != nil {
|
||||
return pass, err
|
||||
}
|
||||
}
|
||||
stream, err := api.Stream(ctx, worker.Stream)
|
||||
if err != nil {
|
||||
return pass, err
|
||||
}
|
||||
// A second copy of one ask — a move that published and stopped before it deleted — goes first.
|
||||
for _, a := range extra {
|
||||
if err := stream.DeleteMsg(ctx, a.Seq); err != nil && !errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
return pass, fmt.Errorf("a second copy of %s (seq %d) could not be removed: %w", a.ID, a.Seq, err)
|
||||
}
|
||||
pass.Removed = append(pass.Removed, fmt.Sprintf("%s at seq %d, a move's second copy of the message at seq %s",
|
||||
a.ID, a.Seq, a.movedFrom))
|
||||
}
|
||||
for _, a := range toMove(waiting, QueueOrder(waiting, time.Now())) {
|
||||
if delivered >= 0 && time.Until(pauseEnds) < moveMargin {
|
||||
// The rest waits for the next pass, under a pause of its own.
|
||||
break
|
||||
}
|
||||
ok, err := moveToTheTail(ctx, js, stream, worker.Name, seat, a)
|
||||
if err != nil {
|
||||
return pass, err
|
||||
}
|
||||
if ok {
|
||||
pass.Moved = append(pass.Moved, a.ID)
|
||||
}
|
||||
}
|
||||
return pass, nil
|
||||
}
|
||||
|
||||
// readWaiting is every ask the worker has not yet handed out, in queue order, but each second copy of one
|
||||
// ask; the second copies, to remove; and the last sequence the worker handed out, -1 when there is no
|
||||
// worker, so nothing is handed out to pause.
|
||||
//
|
||||
// **A second copy is proven, never guessed from an id**: a waiting message whose MovedFromHeader names an
|
||||
// earlier message of the same request id still in the queue, and whose QueuedAtHeader is that message's
|
||||
// first-queued time — a move that published its copy and stopped before it deleted the original. The
|
||||
// original stays, waiting or handed out (a holder's then, and the copy would build it twice). Any other
|
||||
// second message of one id — an ask a dead letter delivered again beside its original (ADR 0293) — is a
|
||||
// real ask and stays.
|
||||
func readWaiting(ctx context.Context, api jetstream.JetStream, worker broker.Consumer, seat string) (
|
||||
waiting, extra []QueuedAsk, delivered int64, err error) {
|
||||
stream, err := api.Stream(ctx, worker.Stream)
|
||||
if err != nil {
|
||||
return nil, nil, 0, fmt.Errorf("the %s work queue (%s) cannot be read: %w", seat, worker.Stream, err)
|
||||
}
|
||||
delivered = -1
|
||||
if consumer, err := stream.Consumer(ctx, worker.Name); err == nil {
|
||||
info, err := consumer.Info(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, 0, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
|
||||
}
|
||||
delivered = int64(info.Delivered.Stream)
|
||||
} else if !errors.Is(err, jetstream.ErrConsumerNotFound) {
|
||||
return nil, nil, 0, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
|
||||
}
|
||||
// From the first: the stream holds only what is unsettled, so this is a handful of reads.
|
||||
// Every message by sequence, with its id and first-queued time, to prove a copy against.
|
||||
type held struct {
|
||||
id string
|
||||
stored time.Time
|
||||
}
|
||||
bySeq := map[string]held{}
|
||||
var seq uint64 = 1
|
||||
for n := 0; n < 10000; n++ {
|
||||
msg, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(BuildWorkOf(seat)))
|
||||
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, 0, fmt.Errorf("reading the %s work queue: %w", seat, err)
|
||||
}
|
||||
seq = msg.Sequence + 1
|
||||
a := askOf(msg)
|
||||
a.movedFrom = msg.Header.Get(MovedFromHeader)
|
||||
from, known := bySeq[a.movedFrom]
|
||||
again := a.ID != "" && a.movedFrom != "" && known && from.id == a.ID && from.stored.Equal(a.Stored)
|
||||
bySeq[strconv.FormatUint(a.Seq, 10)] = held{id: a.ID, stored: a.Stored}
|
||||
switch {
|
||||
case int64(a.Seq) <= delivered:
|
||||
case again:
|
||||
extra = append(extra, a)
|
||||
default:
|
||||
waiting = append(waiting, a)
|
||||
}
|
||||
}
|
||||
return waiting, extra, delivered, nil
|
||||
}
|
||||
|
||||
// moveToTheTail publishes the ask's body again at the queue's tail and then deletes it where it was. **The
|
||||
// copy first**: a publish that fails deletes nothing, and a pass that stops between the two leaves two
|
||||
// copies, which the next pass sees by their one id and makes one (readWaiting). A move never loses an ask.
|
||||
// Looked at once more just before: one the worker has handed out since is a holder's now and stays. False
|
||||
// when it was not moved.
|
||||
func moveToTheTail(ctx context.Context, js *broker.JetStream, stream jetstream.Stream, workerName, seat string,
|
||||
a QueuedAsk) (bool, error) {
|
||||
if consumer, err := stream.Consumer(ctx, workerName); err == nil {
|
||||
info, err := consumer.Info(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if info.Delivered.Stream >= a.Seq {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
msg, err := stream.GetMsg(ctx, a.Seq)
|
||||
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
again := &nats.Msg{Subject: msg.Subject, Data: msg.Data, Header: nats.Header{}}
|
||||
for k, v := range msg.Header {
|
||||
// The server's own headers stay behind: a message id would be taken for a duplicate and dropped.
|
||||
if !strings.HasPrefix(k, "Nats-") {
|
||||
again.Header[k] = v
|
||||
}
|
||||
}
|
||||
// Its place in time goes with it: when the stream first took it, as askOf read it; and the message it
|
||||
// is a copy of, which proves it one.
|
||||
again.Header.Set(QueuedAtHeader, askOf(msg).Stored.UTC().Format(time.RFC3339Nano))
|
||||
again.Header.Set(MovedFromHeader, strconv.FormatUint(a.Seq, 10))
|
||||
// Inside the margin, so the publish ends while the worker is still paused.
|
||||
publishing, cancel := context.WithTimeout(ctx, publishWaits)
|
||||
ack, err := js.Context().PublishMsg(again, nats.Context(publishing))
|
||||
cancel()
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s (seq %d) could not be put behind, and stays where it is: %w", a.ID, a.Seq, err)
|
||||
}
|
||||
// **Handed out meanwhile** — a bus slow enough that the pause ended: the original is a holder's, and the
|
||||
// copy goes instead, so it is built once.
|
||||
if consumer, err := stream.Consumer(ctx, workerName); err == nil {
|
||||
info, err := consumer.Info(ctx)
|
||||
if err != nil || info.Delivered.Stream >= a.Seq {
|
||||
if derr := stream.DeleteMsg(ctx, ack.Sequence); derr != nil && !errors.Is(derr, jetstream.ErrMsgNotFound) {
|
||||
return false, fmt.Errorf("%s was handed out while it moved and its copy (seq %d) could not be "+
|
||||
"removed; the next pass removes it: %w", a.ID, ack.Sequence, derr)
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("whether %s was handed out while it moved cannot be read, so its copy was "+
|
||||
"removed and it stays where it was: %w", a.ID, err)
|
||||
}
|
||||
if info.Delivered.Stream >= ack.Sequence {
|
||||
return false, fmt.Errorf("%s was handed out while it moved, and its copy too: it may be built twice", a.ID)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
if err := stream.DeleteMsg(ctx, a.Seq); err != nil && !errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
return false, fmt.Errorf("%s was copied behind and its original (seq %d) not removed; the next pass "+
|
||||
"removes the second copy: %w", a.ID, a.Seq, err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// KeepingInOrder runs the passes, on the acting controller alone, until ctx ends: when an ask in this
|
||||
// process says so, when the queue's last sequence moved — an ask from another process — and every
|
||||
// passEvery, for a check that comes of age with nothing asked. seat says the build seat to order; acting
|
||||
// whether this controller acts (Holding); said what a pass did.
|
||||
func KeepingInOrder(ctx context.Context, js *broker.JetStream, seat func() string, acting func() bool,
|
||||
said func(format string, args ...any)) {
|
||||
tick := time.NewTicker(orderLooks)
|
||||
defer tick.Stop()
|
||||
var lastSeq uint64
|
||||
var lastPass time.Time
|
||||
for {
|
||||
signalled := false
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-asked:
|
||||
signalled = true
|
||||
case <-tick.C:
|
||||
}
|
||||
if !acting() {
|
||||
continue
|
||||
}
|
||||
name := seat()
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: name, Accepts: []string{"build"}})
|
||||
info, err := js.Context().StreamInfo(worker.Stream)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if !signalled && info.State.LastSeq == lastSeq && time.Since(lastPass) < passEvery {
|
||||
continue
|
||||
}
|
||||
lastSeq, lastPass = info.State.LastSeq, time.Now()
|
||||
passing, cancel := context.WithTimeout(ctx, pausedFor+10*time.Second)
|
||||
pass, err := PutInOrder(passing, js, name)
|
||||
cancel()
|
||||
for _, r := range pass.Removed {
|
||||
said("the %s queue: removed %s", name, r)
|
||||
}
|
||||
if len(pass.Moved) > 0 {
|
||||
said("the %s queue put in order: %s moved behind what goes first", name, strings.Join(pass.Moved, ", "))
|
||||
}
|
||||
if err != nil {
|
||||
said("the %s queue could not be put in order: %v", name, err)
|
||||
}
|
||||
if info, err := js.Context().StreamInfo(worker.Stream); err == nil {
|
||||
// Its own moves are not an ask.
|
||||
lastSeq = info.State.LastSeq
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// orderLooks is how often the loop looks whether an ask joined from another process; passEvery how often
|
||||
// it passes with nothing asked.
|
||||
var (
|
||||
orderLooks = 2 * time.Second
|
||||
passEvery = 30 * time.Second
|
||||
)
|
||||
|
||||
// DeleteAsk removes a waiting or dead ask from the queue: at its sequence, or — moved into order since it
|
||||
// was read — the copy carrying its id. Says the sequence it removed, 0 when neither was there any more.
|
||||
func DeleteAsk(ctx context.Context, js *broker.JetStream, seat string, ask QueuedAsk) (uint64, error) {
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
|
||||
gone := func(err error) bool {
|
||||
return errors.Is(err, nats.ErrMsgNotFound) || errors.Is(err, jetstream.ErrMsgNotFound) ||
|
||||
(err != nil && strings.Contains(err.Error(), "no message found"))
|
||||
}
|
||||
err := js.Context().DeleteMsg(worker.Stream, ask.Seq)
|
||||
if err == nil {
|
||||
return ask.Seq, nil
|
||||
}
|
||||
if !gone(err) {
|
||||
return 0, err
|
||||
}
|
||||
if ask.ID == "" {
|
||||
return 0, nil
|
||||
}
|
||||
q, err := ReadQueue(ctx, js, seat)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, a := range q.Asks {
|
||||
if a.ID == ask.ID && a.Seq != ask.Seq && a.State != AskInFlight {
|
||||
if err := js.Context().DeleteMsg(worker.Stream, a.Seq); err != nil && !gone(err) {
|
||||
return 0, err
|
||||
} else if err == nil {
|
||||
return a.Seq, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
@@ -0,0 +1,441 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// The order the build queue hands asks out in (novox/hq issue 558): a build a walk waits for before any
|
||||
// pull request's check, a check that has waited ten minutes before every delivery asked after it, and
|
||||
// first come, first served within a kind.
|
||||
|
||||
func anAsk(seq uint64, id string, asked time.Time, check bool) QueuedAsk {
|
||||
r := BuildRequest{ID: id, Repository: "/r"}
|
||||
if check {
|
||||
r.Check = &CheckRequest{Owner: "novox", Repo: "r"}
|
||||
}
|
||||
return QueuedAsk{Seq: seq, ID: id, AskedAt: asked, Request: r, State: AskWaiting}
|
||||
}
|
||||
|
||||
func idsOf(asks []QueuedAsk) []string {
|
||||
out := []string{}
|
||||
for _, a := range asks {
|
||||
out = append(out, a.ID)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestADeliveryIsHandedOutBeforeTheChecksAskedBeforeIt(t *testing.T) {
|
||||
now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
|
||||
waiting := []QueuedAsk{
|
||||
anAsk(1, "check-a", now.Add(-3*time.Minute), true),
|
||||
anAsk(2, "check-b", now.Add(-2*time.Minute), true),
|
||||
anAsk(3, "delivery-d", now.Add(-time.Minute), false),
|
||||
}
|
||||
want := []string{"delivery-d", "check-a", "check-b"}
|
||||
if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("handed out as %v, want %v", got, want)
|
||||
}
|
||||
// Only the checks ahead of it move behind it; the delivery stays where it is.
|
||||
if got := idsOf(toMove(waiting, QueueOrder(waiting, now))); !reflect.DeepEqual(got, []string{"check-a", "check-b"}) {
|
||||
t.Fatalf("moved %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACheckThatWaitedTenMinutesGoesBeforeALaterDelivery(t *testing.T) {
|
||||
now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
|
||||
// As the queue stands after a delivery put the young checks behind it, ten minutes ago and since.
|
||||
waiting := []QueuedAsk{
|
||||
anAsk(5, "delivery-early", now.Add(-12*time.Minute), false),
|
||||
anAsk(6, "delivery-later", now.Add(-time.Minute), false),
|
||||
anAsk(7, "check-old", now.Add(-11*time.Minute), true),
|
||||
anAsk(8, "check-young", now.Add(-9*time.Minute), true),
|
||||
}
|
||||
want := []string{"delivery-early", "check-old", "delivery-later", "check-young"}
|
||||
if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("handed out as %v, want %v", got, want)
|
||||
}
|
||||
if got := idsOf(toMove(waiting, QueueOrder(waiting, now))); !reflect.DeepEqual(got, []string{"delivery-later", "check-young"}) {
|
||||
t.Fatalf("moved %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAskWhoseKindCannotBeToldIsACheck(t *testing.T) {
|
||||
for _, r := range []BuildRequest{
|
||||
{}, // unreadable: nothing was read from it
|
||||
{ID: "build-1", Check: &CheckRequest{}}, // a pull request's check, or a recheck the merge tool asked
|
||||
{ID: "build-2", DryRun: true}, // looked at, never delivered
|
||||
} {
|
||||
if got := KindOf(r); got != KindCheck {
|
||||
t.Errorf("%+v is a %s", r, got)
|
||||
}
|
||||
}
|
||||
if got := KindOf(BuildRequest{ID: "build-3", Repository: "/r"}); got != KindDelivery {
|
||||
t.Errorf("a build is a %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Against a real server: two checks wait, a delivery is asked, and the next free holder takes the delivery.
|
||||
func TestNatsADeliveryIsTakenBeforeTheChecksQueuedAheadOfIt(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
now := time.Now()
|
||||
for i, r := range []BuildRequest{
|
||||
{ID: NewBuildID(now.Add(-3 * time.Minute)), Repository: "/a", Check: &CheckRequest{Owner: "novox", Repo: "a"}},
|
||||
{ID: NewBuildID(now.Add(-2 * time.Minute)), Repository: "/b", Check: &CheckRequest{Owner: "novox", Repo: "b"}},
|
||||
{ID: NewBuildID(now), Repository: "/d"},
|
||||
} {
|
||||
if i == 2 {
|
||||
// And one nobody can read, whose kind cannot be told: it waits with the checks.
|
||||
if _, err := js.Context().Publish(BuildWork(), []byte("not an ask")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := asks.Ask(ctx, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The acting controller's loop puts it in order, told by the asks.
|
||||
go KeepingInOrder(ctx, js, func() string { return TheBuildMachine }, func() bool { return true },
|
||||
func(string, ...any) {})
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
took := make(chan string, 4)
|
||||
machine := MachineOverNATS(js, "anchor")
|
||||
defer machine.Close()
|
||||
go func() {
|
||||
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
||||
took <- work.Request().Repository
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
var order []string
|
||||
for len(order) < 3 {
|
||||
select {
|
||||
case r := <-took:
|
||||
order = append(order, r)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatalf("taken so far: %v", order)
|
||||
}
|
||||
}
|
||||
if want := []string{"/d", "/a", "/b"}; !reflect.DeepEqual(order, want) {
|
||||
t.Fatalf("taken as %v, want %v", order, want)
|
||||
}
|
||||
}
|
||||
|
||||
// --- the moves, against a real server ------------------------------------------------------------
|
||||
|
||||
func publishAsk(t *testing.T, js *broker.JetStream, r BuildRequest) {
|
||||
t.Helper()
|
||||
body, _ := json.Marshal(r)
|
||||
if _, err := js.Context().Publish(BuildWork(), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func aCheck(asked time.Time, repository string) BuildRequest {
|
||||
return BuildRequest{ID: NewBuildID(asked), Repository: repository, Check: &CheckRequest{Owner: "novox", Repo: "r"}}
|
||||
}
|
||||
|
||||
func aDelivery(asked time.Time, repository string) BuildRequest {
|
||||
return BuildRequest{ID: NewBuildID(asked), Repository: repository}
|
||||
}
|
||||
|
||||
// queued is the stream's asks, in queue order.
|
||||
func queued(t *testing.T, js *broker.JetStream) []QueuedAsk {
|
||||
t.Helper()
|
||||
q, err := ReadQueue(context.Background(), js, TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return q.Asks
|
||||
}
|
||||
|
||||
// takeAll runs a holder that records how often it was handed each repository.
|
||||
func takeAll(t *testing.T, ctx context.Context, js *broker.JetStream) (chan string, func() map[string]int) {
|
||||
t.Helper()
|
||||
took := make(chan string, 100)
|
||||
var mu sync.Mutex
|
||||
counts := map[string]int{}
|
||||
machine := MachineOverNATS(js, "anchor")
|
||||
t.Cleanup(machine.Close)
|
||||
go func() {
|
||||
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
||||
mu.Lock()
|
||||
counts[work.Request().Repository]++
|
||||
mu.Unlock()
|
||||
took <- work.Request().Repository
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
return took, func() map[string]int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
out := map[string]int{}
|
||||
for k, v := range counts {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
|
||||
// A move that published its copy and stopped before deleting the original leaves two copies of one ask;
|
||||
// the next pass makes them one, and a copy of one a holder already took is removed, never built twice.
|
||||
func TestNatsAMoveCutShortLeavesOneCopyAfterTheNextPass(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx := t.Context()
|
||||
now := time.Now()
|
||||
check := aCheck(now.Add(-2*time.Minute), "/c")
|
||||
publishAsk(t, js, check)
|
||||
publishAsk(t, js, aDelivery(now.Add(-time.Minute), "/d"))
|
||||
moveCopyOf(t, js, check.ID) // the copy, published; the original never deleted
|
||||
if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Removed) != 1 {
|
||||
t.Fatalf("removed %v: %v", pass.Removed, err)
|
||||
}
|
||||
var got []string
|
||||
for _, a := range queued(t, js) {
|
||||
got = append(got, a.Repository)
|
||||
}
|
||||
if want := []string{"/d", "/c"}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("after the pass the queue holds %v, want %v", got, want)
|
||||
}
|
||||
|
||||
// And one a holder took before the copy was published: the copy goes, the holder's stays.
|
||||
taken := aCheck(now, "/taken")
|
||||
publishAsk(t, js, taken)
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}})
|
||||
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
// Take everything waiting up to and including /taken, and keep /taken unsettled.
|
||||
for {
|
||||
msgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second))
|
||||
if err != nil || len(msgs) == 0 {
|
||||
t.Fatalf("nothing to take: %v", err)
|
||||
}
|
||||
var r BuildRequest
|
||||
_ = json.Unmarshal(msgs[0].Data, &r)
|
||||
if r.Repository == "/taken" {
|
||||
break
|
||||
}
|
||||
_ = msgs[0].Ack()
|
||||
}
|
||||
moveCopyOf(t, js, taken.ID)
|
||||
if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n := 0
|
||||
for _, a := range queued(t, js) {
|
||||
if a.ID == taken.ID {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("%d copies of the ask a holder took are in the queue, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
// moveCopyOf publishes what a move publishes for the ask with this id, as a pass stopped before its delete
|
||||
// would leave it.
|
||||
func moveCopyOf(t *testing.T, js *broker.JetStream, id string) {
|
||||
t.Helper()
|
||||
for _, a := range queued(t, js) {
|
||||
if a.ID != id {
|
||||
continue
|
||||
}
|
||||
body, _ := json.Marshal(a.Request)
|
||||
msg := &nats.Msg{Subject: BuildWork(), Data: body, Header: nats.Header{}}
|
||||
msg.Header.Set(QueuedAtHeader, a.Stored.UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set(MovedFromHeader, fmt.Sprint(a.Seq))
|
||||
if _, err := js.Context().PublishMsg(msg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Fatalf("%s is not queued", id)
|
||||
}
|
||||
|
||||
// A dead letter delivered again whose original stayed beside it (ADR 0293: its removal failed) is a real
|
||||
// ask with the same id, and is never removed as a second copy — even when the original was itself a move's
|
||||
// copy, whose headers the delivery carries.
|
||||
func TestNatsAnAskDeliveredAgainBesideItsOriginalStays(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx := t.Context()
|
||||
ask := aCheck(time.Now(), "/again")
|
||||
publishAsk(t, js, aCheck(time.Now().Add(-time.Minute), "/before"))
|
||||
publishAsk(t, js, ask)
|
||||
publishAsk(t, js, aDelivery(time.Now(), "/d"))
|
||||
if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil { // /again becomes a move's copy
|
||||
t.Fatal(err)
|
||||
}
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}})
|
||||
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
var original *nats.Msg
|
||||
for original == nil {
|
||||
msgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second))
|
||||
if err != nil || len(msgs) == 0 {
|
||||
t.Fatalf("nothing to take: %v", err)
|
||||
}
|
||||
var r BuildRequest
|
||||
_ = json.Unmarshal(msgs[0].Data, &r)
|
||||
if r.ID == ask.ID {
|
||||
original = msgs[0] // given up on, never settled: dead, and kept
|
||||
continue
|
||||
}
|
||||
_ = msgs[0].Ack()
|
||||
}
|
||||
// Delivered again as DeliverAgain does: its headers, and its own.
|
||||
again := &nats.Msg{Subject: BuildWork(), Data: original.Data, Header: nats.Header{}}
|
||||
for k, v := range original.Header {
|
||||
again.Header[k] = v
|
||||
}
|
||||
again.Header.Set(AgainHeader, "7")
|
||||
if _, err := js.Context().PublishMsg(again); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pass, err := PutInOrder(ctx, js, TheBuildMachine)
|
||||
if err != nil || len(pass.Removed) != 0 {
|
||||
t.Fatalf("removed %v: %v", pass.Removed, err)
|
||||
}
|
||||
n := 0
|
||||
for _, a := range queued(t, js) {
|
||||
if a.ID == ask.ID {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 2 {
|
||||
t.Fatalf("%d messages of the ask delivered again are queued, want its original and the delivery", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Two passes at once: nothing is built twice, and nothing is lost.
|
||||
func TestNatsTwoPassesAtOnceBuildNothingTwice(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
took, counts := takeAll(t, ctx, js)
|
||||
now := time.Now()
|
||||
const asks = 24
|
||||
var passes sync.WaitGroup
|
||||
for p := 0; p < 2; p++ {
|
||||
passes.Add(1)
|
||||
go func() {
|
||||
defer passes.Done()
|
||||
for i := 0; i < 15; i++ {
|
||||
_, _ = PutInOrder(ctx, js, TheBuildMachine)
|
||||
}
|
||||
}()
|
||||
}
|
||||
for i := 0; i < asks; i++ {
|
||||
repository := fmt.Sprintf("/%02d", i)
|
||||
if i%3 == 2 {
|
||||
publishAsk(t, js, aDelivery(now.Add(time.Duration(i)*time.Millisecond), repository))
|
||||
} else {
|
||||
publishAsk(t, js, aCheck(now.Add(time.Duration(i)*time.Millisecond), repository))
|
||||
}
|
||||
}
|
||||
passes.Wait()
|
||||
for got := 0; got < asks; got++ {
|
||||
select {
|
||||
case <-took:
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatalf("taken %d of %d: %v", got, asks, counts())
|
||||
}
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
c := counts()
|
||||
for i := 0; i < asks; i++ {
|
||||
if n := c[fmt.Sprintf("/%02d", i)]; n != 1 {
|
||||
t.Errorf("/%02d was built %d times", i, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A check that has waited ten minutes goes before a delivery asked after it, even one queued ahead of it.
|
||||
func TestNatsACheckThatWaitedTenMinutesIsTakenBeforeALaterDelivery(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
now := time.Now()
|
||||
publishAsk(t, js, aDelivery(now.Add(-time.Minute), "/delivery"))
|
||||
publishAsk(t, js, aCheck(now.Add(-11*time.Minute), "/old-check"))
|
||||
if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
took, _ := takeAll(t, ctx, js)
|
||||
var order []string
|
||||
for len(order) < 2 {
|
||||
select {
|
||||
case r := <-took:
|
||||
order = append(order, r)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatalf("taken so far: %v", order)
|
||||
}
|
||||
}
|
||||
if want := []string{"/old-check", "/delivery"}; !reflect.DeepEqual(order, want) {
|
||||
t.Fatalf("taken as %v, want %v", order, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A cancel that read an ask before it moved deletes the moved copy, found by its id, and says when
|
||||
// there was nothing left to delete.
|
||||
func TestNatsACancelRacingAMoveDeletesTheMovedCopy(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx := t.Context()
|
||||
now := time.Now()
|
||||
publishAsk(t, js, aCheck(now.Add(-time.Minute), "/c"))
|
||||
read := queued(t, js)[0]
|
||||
publishAsk(t, js, aDelivery(now, "/d"))
|
||||
if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Moved) != 1 {
|
||||
t.Fatalf("moved %v: %v", pass.Moved, err)
|
||||
}
|
||||
removed, err := DeleteAsk(ctx, js, TheBuildMachine, read)
|
||||
if err != nil || removed == 0 || removed == read.Seq {
|
||||
t.Fatalf("removed seq %d (read at %d): %v", removed, read.Seq, err)
|
||||
}
|
||||
for _, a := range queued(t, js) {
|
||||
if a.ID == read.ID {
|
||||
t.Fatalf("the cancelled ask is still queued at seq %d", a.Seq)
|
||||
}
|
||||
}
|
||||
if removed, err := DeleteAsk(ctx, js, TheBuildMachine, read); err != nil || removed != 0 {
|
||||
t.Fatalf("an ask gone already is said removed at %d: %v", removed, err)
|
||||
}
|
||||
}
|
||||
|
||||
// An ask nothing can be read from keeps its place in time when it moves.
|
||||
func TestNatsAnUnreadableAskKeepsItsAgeWhenItMoves(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx := t.Context()
|
||||
if _, err := js.Context().Publish(BuildWork(), []byte("not an ask")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first := queued(t, js)[0].Stored
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
publishAsk(t, js, aDelivery(time.Now(), "/d"))
|
||||
if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Moved) != 1 {
|
||||
t.Fatalf("moved %v: %v", pass.Moved, err)
|
||||
}
|
||||
asks := queued(t, js)
|
||||
if len(asks) != 2 || asks[1].ID != "" || !asks[1].Stored.Equal(first) {
|
||||
t.Fatalf("after the move the unreadable ask was taken at %v, first at %v: %+v", asks[1].Stored, first, asks)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// CreateMigratedForTest creates the database name on the test PostgreSQL that admin reaches, holding
|
||||
// every one of migrations already applied and recorded, as a fresh database migrated by Migrate would.
|
||||
//
|
||||
// **A copy of a template, migrated once** (novox/hq issue 591): a store test used to create an empty
|
||||
// database and run every migration into it, some 90 for the inventory, and with hundreds of store tests
|
||||
// that alone was most of the controller's check. The template is migrated by Migrate itself, once for
|
||||
// each set of migrations, and every test's database is copied from it by PostgreSQL. The caller still
|
||||
// runs Migrate on the copy, which finds nothing pending only if the copy's record matches the migrations
|
||||
// exactly, so a template that disagreed with them would fail the test rather than pass it unseen.
|
||||
//
|
||||
// The template is named after a digest of the migrations, so a changed or added migration is a new
|
||||
// template, never an old one reused. Packages tested at once in separate processes share one: it is
|
||||
// built under an advisory lock, and a template a crashed build left half done is not marked a template
|
||||
// and is built again. It takes a *testing.T, so nothing that is not a test can call it.
|
||||
func CreateMigratedForTest(t *testing.T, admin, name string, migrations []Migration) {
|
||||
t.Helper()
|
||||
template, err := templateFor(admin, migrations)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot raise the migrated template: %v", err)
|
||||
}
|
||||
conn, err := pgx.Connect(t.Context(), admin)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot reach the test PostgreSQL: %v", err)
|
||||
}
|
||||
defer conn.Close(context.Background())
|
||||
// A copy is refused while anything is connected to its template; the template allows no connection
|
||||
// once built, so only the builder's last connection closing can still be in the way.
|
||||
for deadline := time.Now().Add(10 * time.Second); ; {
|
||||
_, err = conn.Exec(t.Context(), "create database "+name+" template "+template)
|
||||
var pgErr *pgconn.PgError
|
||||
if err == nil || !errors.As(err, &pgErr) || pgErr.Code != "55006" || time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("cannot create %s from %s: %v", name, template, err)
|
||||
}
|
||||
}
|
||||
|
||||
// templates are the templates this process has raised or found, by name.
|
||||
var templates sync.Map // name → *templateOnce
|
||||
|
||||
type templateOnce struct {
|
||||
once sync.Once
|
||||
err error
|
||||
}
|
||||
|
||||
// templateFor is the name of the template holding migrations, raised if no process has yet.
|
||||
func templateFor(admin string, migrations []Migration) (string, error) {
|
||||
digest := sha256.New()
|
||||
for _, m := range migrations {
|
||||
fmt.Fprintf(digest, "%d %s %s\n", m.Number, m.Name, m.Checksum)
|
||||
}
|
||||
sum := digest.Sum(nil)
|
||||
name := "tpl_" + hex.EncodeToString(sum[:12])
|
||||
held, _ := templates.LoadOrStore(name, &templateOnce{})
|
||||
once := held.(*templateOnce)
|
||||
once.once.Do(func() {
|
||||
// Not the asking test's context: the template outlives that test, and is raised once for all.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
once.err = raiseTemplate(ctx, admin, name, int64(binary.BigEndian.Uint64(sum[12:20])), migrations)
|
||||
})
|
||||
return name, once.err
|
||||
}
|
||||
|
||||
// raiseTemplate builds the template name under the advisory lock key, unless it stands built.
|
||||
func raiseTemplate(ctx context.Context, admin, name string, key int64, migrations []Migration) error {
|
||||
conn, err := pgx.Connect(ctx, admin)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Close(context.Background())
|
||||
if _, err := conn.Exec(ctx, `select pg_advisory_lock($1)`, key); err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _, _ = conn.Exec(context.Background(), `select pg_advisory_unlock($1)`, key) }()
|
||||
|
||||
var built bool
|
||||
if err := conn.QueryRow(ctx, `select exists (select 1 from pg_database where datname = $1 and datistemplate)`,
|
||||
name).Scan(&built); err != nil {
|
||||
return err
|
||||
}
|
||||
if built {
|
||||
return nil
|
||||
}
|
||||
if _, err := conn.Exec(ctx, "drop database if exists "+name+" with (force)"); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := conn.Exec(ctx, "create database "+name); err != nil {
|
||||
return err
|
||||
}
|
||||
cut := strings.LastIndex(admin, "/")
|
||||
rest := ""
|
||||
if q := strings.Index(admin[cut:], "?"); q >= 0 {
|
||||
rest = admin[cut+q:]
|
||||
}
|
||||
pool, err := pgxpool.New(ctx, admin[:cut]+"/"+name+rest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = (&Store{context: "template", pool: pool}).Migrate(ctx, migrations)
|
||||
pool.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Marked a template only once whole, and closed to connections, so a copy is never refused for one.
|
||||
for _, sql := range []string{
|
||||
"alter database " + name + " with allow_connections false",
|
||||
"select pg_terminate_backend(pid) from pg_stat_activity where datname = '" + name + "'",
|
||||
"alter database " + name + " with is_template true",
|
||||
} {
|
||||
if _, err := conn.Exec(ctx, sql); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Vendored
+8
@@ -7,12 +7,14 @@
|
||||
./cmd/mesh-controller TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt
|
||||
./cmd/mesh-controller TestAStopWithoutAMergeLeavesItOutOfABatch
|
||||
./cmd/mesh-controller TestSavingPlansAndMovingMergesIsOneAct
|
||||
./cmd/mesh-controller TestAPlanFollowedFromTheStoreAnswersWhenItMoves
|
||||
./internal/inventory TestTheLastSendSaysWhatTheMachineAnswered
|
||||
./internal/inventory TestAReportIsTheLastSendsAnswerOnlyWhenItIsAboutThatSend
|
||||
./cmd/mesh-controller TestNodeShowAndStatusReadTheLastSendFromTheStore
|
||||
./internal/inventory TestAnIssuedValueIsSealedToEveryHolderOnceAndNotAgain
|
||||
./internal/inventory TestAnIssuedValueIsSealedAgainAfterAKeyChangeOrAnAcceptance
|
||||
./internal/inventory TestAConditionsLooksAreKeptBesideTheBucket
|
||||
./internal/inventory TestAPlanIdHoldingAQuoteIsReadAsAPlainId
|
||||
./cmd/mesh-controller TestTheConditionsVerbReadsTheLooksTheBucketNoLongerHolds
|
||||
./internal/inventory TestAWalksStartIsKeptAndARefusedSaveLeavesItUnsaid
|
||||
./cmd/mesh-controller TestAWalkLetGoAndStoppedByVerbsIsSaidStartedAndEnded
|
||||
@@ -29,3 +31,9 @@
|
||||
./cmd/mesh-controller TestADirectoryOwnersKeyIsRefusedThroughAVerb
|
||||
./cmd/mesh-controller TestTheSendMakesNoValueForAnOutsideSecretNobodyGave
|
||||
./cmd/mesh-controller TestAQuestionComposesWithoutAnOutsideSecretNobodyGave
|
||||
./cmd/mesh-controller TestReplay201
|
||||
./cmd/mesh-controller TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend
|
||||
./cmd/mesh-controller TestAGateJudgesTheOtherMovesBesideAKeptCore
|
||||
./cmd/mesh-controller TestReplay401
|
||||
./cmd/mesh-controller TestReplay201ThePlansSend
|
||||
./internal/inventory TestAWalksRestIsReadFromEachMachinesOwnSend
|
||||
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
# Package xz authors
|
||||
|
||||
Michael Cross <https://github.com/xi2>
|
||||
|
||||
# XZ Embedded authors
|
||||
|
||||
Lasse Collin <lasse.collin@tukaani.org>
|
||||
Igor Pavlov <http://7-zip.org/>
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
Creative Commons CC0 1.0 Universal
|
||||
|
||||
CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE LEGAL
|
||||
SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN ATTORNEY-CLIENT
|
||||
RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS INFORMATION ON AN "AS-IS" BASIS.
|
||||
CREATIVE COMMONS MAKES NO WARRANTIES REGARDING THE USE OF THIS DOCUMENT OR THE
|
||||
INFORMATION OR WORKS PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES
|
||||
RESULTING FROM THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
|
||||
HEREUNDER.
|
||||
|
||||
Statement of Purpose
|
||||
|
||||
The laws of most jurisdictions throughout the world automatically confer
|
||||
exclusive Copyright and Related Rights (defined below) upon the creator and
|
||||
subsequent owner(s) (each and all, an "owner") of an original work of
|
||||
authorship and/or a database (each, a "Work").
|
||||
|
||||
Certain owners wish to permanently relinquish those rights to a Work for the
|
||||
purpose of contributing to a commons of creative, cultural and scientific
|
||||
works ("Commons") that the public can reliably and without fear of later
|
||||
claims of infringement build upon, modify, incorporate in other works, reuse
|
||||
and redistribute as freely as possible in any form whatsoever and for any
|
||||
purposes, including without limitation commercial purposes. These owners may
|
||||
contribute to the Commons to promote the ideal of a free culture and the
|
||||
further production of creative, cultural and scientific works, or to gain
|
||||
reputation or greater distribution for their Work in part through the use and
|
||||
efforts of others.
|
||||
|
||||
For these and/or other purposes and motivations, and without any expectation
|
||||
of additional consideration or compensation, the person associating CC0 with a
|
||||
Work (the "Affirmer"), to the extent that he or she is an owner of Copyright
|
||||
and Related Rights in the Work, voluntarily elects to apply CC0 to the Work
|
||||
and publicly distribute the Work under its terms, with knowledge of his or her
|
||||
Copyright and Related Rights in the Work and the meaning and intended legal
|
||||
effect of CC0 on those rights.
|
||||
|
||||
1. Copyright and Related Rights. A Work made available under CC0 may be protected by copyright and related or neighboring rights ("Copyright and Related Rights"). Copyright and Related Rights include, but are not limited to, the following:
|
||||
|
||||
i. the right to reproduce, adapt, distribute, perform, display, communicate,
|
||||
and translate a Work;
|
||||
|
||||
ii. moral rights retained by the original author(s) and/or performer(s);
|
||||
|
||||
iii. publicity and privacy rights pertaining to a person's image or
|
||||
likeness depicted in a Work;
|
||||
|
||||
iv. rights protecting against unfair competition in regards to a Work, subject
|
||||
to the limitations in paragraph 4(a), below;
|
||||
|
||||
v. rights protecting the extraction, dissemination, use and reuse of data in a
|
||||
Work;
|
||||
|
||||
vi. database rights (such as those arising under Directive 96/9/EC of the
|
||||
European Parliament and of the Council of 11 March 1996 on the legal
|
||||
protection of databases, and under any national implementation thereof,
|
||||
including any amended or successor version of such directive); and
|
||||
|
||||
vii. other similar, equivalent or corresponding rights throughout the world
|
||||
based on applicable law or treaty, and any national implementations thereof.
|
||||
|
||||
2. Waiver. To the greatest extent permitted by, but not in contravention of, applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and unconditionally waives, abandons, and surrenders all of Affirmer's Copyright and Related Rights and associated claims and causes of action, whether now known or unknown (including existing as well as future claims and causes of action), in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each member of the public at large and to the detriment of Affirmer's heirs and successors, fully intending that such Waiver shall not be subject to revocation, rescission, cancellation, termination, or any other legal or equitable action to disrupt the quiet enjoyment of the Work by the public as contemplated by Affirmer's express Statement of Purpose.
|
||||
|
||||
3. Public License Fallback. Should any part of the Waiver for any reason be judged legally invalid or ineffective under applicable law, then the Waiver shall be preserved to the maximum extent permitted taking into account Affirmer's express Statement of Purpose. In addition, to the extent the Waiver is so judged Affirmer hereby grants to each affected person a royalty-free, non transferable, non sublicensable, non exclusive, irrevocable and unconditional license to exercise Affirmer's Copyright and Related Rights in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "License"). The License shall be deemed effective as of the date CC0 was applied by Affirmer to the Work. Should any part of the License for any reason be judged legally invalid or ineffective under applicable law, such partial invalidity or ineffectiveness shall not invalidate the remainder of the License, and in such case Affirmer hereby affirms that he or she will not (i) exercise any of his or her remaining Copyright and Related Rights in the Work or (ii) assert any associated claims and causes of action with respect to the Work, in either case contrary to Affirmer's express Statement of Purpose.
|
||||
|
||||
4. Limitations and Disclaimers.
|
||||
|
||||
a. No trademark or patent rights held by Affirmer are waived, abandoned,
|
||||
surrendered, licensed or otherwise affected by this document.
|
||||
|
||||
b. Affirmer offers the Work as-is and makes no representations or warranties
|
||||
of any kind concerning the Work, express, implied, statutory or otherwise,
|
||||
including without limitation warranties of title, merchantability, fitness for
|
||||
a particular purpose, non infringement, or the absence of latent or other
|
||||
defects, accuracy, or the present or absence of errors, whether or not
|
||||
discoverable, all to the greatest extent permissible under applicable law.
|
||||
|
||||
c. Affirmer disclaims responsibility for clearing rights of other persons that
|
||||
may apply to the Work or any use thereof, including without limitation any
|
||||
person's Copyright and Related Rights in the Work. Further, Affirmer
|
||||
disclaims responsibility for obtaining any necessary consents, permissions or
|
||||
other rights required for any use of the Work.
|
||||
|
||||
d. Affirmer understands and acknowledges that Creative Commons is not a party
|
||||
to this document and has no duty or obligation with respect to this CC0 or use
|
||||
of the Work.
|
||||
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
# Xz
|
||||
|
||||
Package xz implements XZ decompression natively in Go.
|
||||
|
||||
Documentation at <https://pkg.go.dev/github.com/therootcompany/xz>.
|
||||
|
||||
Download and install with `go get github.com/xi2/xz`.
|
||||
|
||||
If you need compression as well as decompression, you might want to
|
||||
look at <https://github.com/ulikunitz/xz>.
|
||||
|
||||
# LICENSE
|
||||
|
||||
This was originally released into the public domain by the AUTHORS.
|
||||
|
||||
Here it is licensed more explicitly as Creative Commons CC0 1.0 Universal
|
||||
(CC0-1.0) so that it can be detected by automated tooling, and satisfy the legal
|
||||
requirements for vendor integration for cases in which a "public domain"
|
||||
statement is not sufficient.
|
||||
+461
@@ -0,0 +1,461 @@
|
||||
/*
|
||||
* Branch/Call/Jump (BCJ) filter decoders
|
||||
*
|
||||
* Authors: Lasse Collin <lasse.collin@tukaani.org>
|
||||
* Igor Pavlov <http://7-zip.org/>
|
||||
*
|
||||
* Translation to Go: Michael Cross <https://github.com/xi2>
|
||||
*
|
||||
* This file has been put into the public domain.
|
||||
* You can do whatever you want with this file.
|
||||
*/
|
||||
|
||||
package xz
|
||||
|
||||
/* from linux/lib/xz/xz_dec_bcj.c *************************************/
|
||||
|
||||
type xzDecBCJ struct {
|
||||
/* Type of the BCJ filter being used */
|
||||
typ xzFilterID
|
||||
/*
|
||||
* Return value of the next filter in the chain. We need to preserve
|
||||
* this information across calls, because we must not call the next
|
||||
* filter anymore once it has returned xzStreamEnd
|
||||
*/
|
||||
ret xzRet
|
||||
/*
|
||||
* Absolute position relative to the beginning of the uncompressed
|
||||
* data (in a single .xz Block).
|
||||
*/
|
||||
pos int
|
||||
/* x86 filter state */
|
||||
x86PrevMask uint32
|
||||
/* Temporary space to hold the variables from xzBuf */
|
||||
out []byte
|
||||
outPos int
|
||||
temp struct {
|
||||
/* Amount of already filtered data in the beginning of buf */
|
||||
filtered int
|
||||
/*
|
||||
* Buffer to hold a mix of filtered and unfiltered data. This
|
||||
* needs to be big enough to hold Alignment + 2 * Look-ahead:
|
||||
*
|
||||
* Type Alignment Look-ahead
|
||||
* x86 1 4
|
||||
* PowerPC 4 0
|
||||
* IA-64 16 0
|
||||
* ARM 4 0
|
||||
* ARM-Thumb 2 2
|
||||
* SPARC 4 0
|
||||
*/
|
||||
buf []byte // slice buf will be backed by bufArray
|
||||
bufArray [16]byte
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* This is used to test the most significant byte of a memory address
|
||||
* in an x86 instruction.
|
||||
*/
|
||||
func bcjX86TestMSByte(b byte) bool {
|
||||
return b == 0x00 || b == 0xff
|
||||
}
|
||||
|
||||
func bcjX86Filter(s *xzDecBCJ, buf []byte) int {
|
||||
var maskToAllowedStatus = []bool{
|
||||
true, true, true, false, true, false, false, false,
|
||||
}
|
||||
var maskToBitNum = []byte{0, 1, 2, 2, 3, 3, 3, 3}
|
||||
var i int
|
||||
var prevPos int = -1
|
||||
var prevMask uint32 = s.x86PrevMask
|
||||
var src uint32
|
||||
var dest uint32
|
||||
var j uint32
|
||||
var b byte
|
||||
if len(buf) <= 4 {
|
||||
return 0
|
||||
}
|
||||
for i = 0; i < len(buf)-4; i++ {
|
||||
if buf[i]&0xfe != 0xe8 {
|
||||
continue
|
||||
}
|
||||
prevPos = i - prevPos
|
||||
if prevPos > 3 {
|
||||
prevMask = 0
|
||||
} else {
|
||||
prevMask = (prevMask << (uint(prevPos) - 1)) & 7
|
||||
if prevMask != 0 {
|
||||
b = buf[i+4-int(maskToBitNum[prevMask])]
|
||||
if !maskToAllowedStatus[prevMask] || bcjX86TestMSByte(b) {
|
||||
prevPos = i
|
||||
prevMask = prevMask<<1 | 1
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
prevPos = i
|
||||
if bcjX86TestMSByte(buf[i+4]) {
|
||||
src = getLE32(buf[i+1:])
|
||||
for {
|
||||
dest = src - uint32(s.pos+i+5)
|
||||
if prevMask == 0 {
|
||||
break
|
||||
}
|
||||
j = uint32(maskToBitNum[prevMask]) * 8
|
||||
b = byte(dest >> (24 - j))
|
||||
if !bcjX86TestMSByte(b) {
|
||||
break
|
||||
}
|
||||
src = dest ^ (1<<(32-j) - 1)
|
||||
}
|
||||
dest &= 0x01FFFFFF
|
||||
dest |= 0 - dest&0x01000000
|
||||
putLE32(dest, buf[i+1:])
|
||||
i += 4
|
||||
} else {
|
||||
prevMask = prevMask<<1 | 1
|
||||
}
|
||||
}
|
||||
prevPos = i - prevPos
|
||||
if prevPos > 3 {
|
||||
s.x86PrevMask = 0
|
||||
} else {
|
||||
s.x86PrevMask = prevMask << (uint(prevPos) - 1)
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
func bcjPowerPCFilter(s *xzDecBCJ, buf []byte) int {
|
||||
var i int
|
||||
var instr uint32
|
||||
for i = 0; i+4 <= len(buf); i += 4 {
|
||||
instr = getBE32(buf[i:])
|
||||
if instr&0xFC000003 == 0x48000001 {
|
||||
instr &= 0x03FFFFFC
|
||||
instr -= uint32(s.pos + i)
|
||||
instr &= 0x03FFFFFC
|
||||
instr |= 0x48000001
|
||||
putBE32(instr, buf[i:])
|
||||
}
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
var bcjIA64BranchTable = [...]byte{
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
4, 4, 6, 6, 0, 0, 7, 7,
|
||||
4, 4, 0, 0, 4, 4, 0, 0,
|
||||
}
|
||||
|
||||
func bcjIA64Filter(s *xzDecBCJ, buf []byte) int {
|
||||
var branchTable = bcjIA64BranchTable[:]
|
||||
/*
|
||||
* The local variables take a little bit stack space, but it's less
|
||||
* than what LZMA2 decoder takes, so it doesn't make sense to reduce
|
||||
* stack usage here without doing that for the LZMA2 decoder too.
|
||||
*/
|
||||
/* Loop counters */
|
||||
var i int
|
||||
var j int
|
||||
/* Instruction slot (0, 1, or 2) in the 128-bit instruction word */
|
||||
var slot uint32
|
||||
/* Bitwise offset of the instruction indicated by slot */
|
||||
var bitPos uint32
|
||||
/* bit_pos split into byte and bit parts */
|
||||
var bytePos uint32
|
||||
var bitRes uint32
|
||||
/* Address part of an instruction */
|
||||
var addr uint32
|
||||
/* Mask used to detect which instructions to convert */
|
||||
var mask uint32
|
||||
/* 41-bit instruction stored somewhere in the lowest 48 bits */
|
||||
var instr uint64
|
||||
/* Instruction normalized with bit_res for easier manipulation */
|
||||
var norm uint64
|
||||
for i = 0; i+16 <= len(buf); i += 16 {
|
||||
mask = uint32(branchTable[buf[i]&0x1f])
|
||||
for slot, bitPos = 0, 5; slot < 3; slot, bitPos = slot+1, bitPos+41 {
|
||||
if (mask>>slot)&1 == 0 {
|
||||
continue
|
||||
}
|
||||
bytePos = bitPos >> 3
|
||||
bitRes = bitPos & 7
|
||||
instr = 0
|
||||
for j = 0; j < 6; j++ {
|
||||
instr |= uint64(buf[i+j+int(bytePos)]) << (8 * uint(j))
|
||||
}
|
||||
norm = instr >> bitRes
|
||||
if (norm>>37)&0x0f == 0x05 && (norm>>9)&0x07 == 0 {
|
||||
addr = uint32((norm >> 13) & 0x0fffff)
|
||||
addr |= (uint32(norm>>36) & 1) << 20
|
||||
addr <<= 4
|
||||
addr -= uint32(s.pos + i)
|
||||
addr >>= 4
|
||||
norm &= ^(uint64(0x8fffff) << 13)
|
||||
norm |= uint64(addr&0x0fffff) << 13
|
||||
norm |= uint64(addr&0x100000) << (36 - 20)
|
||||
instr &= 1<<bitRes - 1
|
||||
instr |= norm << bitRes
|
||||
for j = 0; j < 6; j++ {
|
||||
buf[i+j+int(bytePos)] = byte(instr >> (8 * uint(j)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
func bcjARMFilter(s *xzDecBCJ, buf []byte) int {
|
||||
var i int
|
||||
var addr uint32
|
||||
for i = 0; i+4 <= len(buf); i += 4 {
|
||||
if buf[i+3] == 0xeb {
|
||||
addr = uint32(buf[i]) | uint32(buf[i+1])<<8 |
|
||||
uint32(buf[i+2])<<16
|
||||
addr <<= 2
|
||||
addr -= uint32(s.pos + i + 8)
|
||||
addr >>= 2
|
||||
buf[i] = byte(addr)
|
||||
buf[i+1] = byte(addr >> 8)
|
||||
buf[i+2] = byte(addr >> 16)
|
||||
}
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
func bcjARMThumbFilter(s *xzDecBCJ, buf []byte) int {
|
||||
var i int
|
||||
var addr uint32
|
||||
for i = 0; i+4 <= len(buf); i += 2 {
|
||||
if buf[i+1]&0xf8 == 0xf0 && buf[i+3]&0xf8 == 0xf8 {
|
||||
addr = uint32(buf[i+1]&0x07)<<19 |
|
||||
uint32(buf[i])<<11 |
|
||||
uint32(buf[i+3]&0x07)<<8 |
|
||||
uint32(buf[i+2])
|
||||
addr <<= 1
|
||||
addr -= uint32(s.pos + i + 4)
|
||||
addr >>= 1
|
||||
buf[i+1] = byte(0xf0 | (addr>>19)&0x07)
|
||||
buf[i] = byte(addr >> 11)
|
||||
buf[i+3] = byte(0xf8 | (addr>>8)&0x07)
|
||||
buf[i+2] = byte(addr)
|
||||
i += 2
|
||||
}
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
func bcjSPARCFilter(s *xzDecBCJ, buf []byte) int {
|
||||
var i int
|
||||
var instr uint32
|
||||
for i = 0; i+4 <= len(buf); i += 4 {
|
||||
instr = getBE32(buf[i:])
|
||||
if instr>>22 == 0x100 || instr>>22 == 0x1ff {
|
||||
instr <<= 2
|
||||
instr -= uint32(s.pos + i)
|
||||
instr >>= 2
|
||||
instr = (0x40000000 - instr&0x400000) |
|
||||
0x40000000 | (instr & 0x3FFFFF)
|
||||
putBE32(instr, buf[i:])
|
||||
}
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
/*
|
||||
* Apply the selected BCJ filter. Update *pos and s.pos to match the amount
|
||||
* of data that got filtered.
|
||||
*/
|
||||
func bcjApply(s *xzDecBCJ, buf []byte, pos *int) {
|
||||
var filtered int
|
||||
buf = buf[*pos:]
|
||||
switch s.typ {
|
||||
case idBCJX86:
|
||||
filtered = bcjX86Filter(s, buf)
|
||||
case idBCJPowerPC:
|
||||
filtered = bcjPowerPCFilter(s, buf)
|
||||
case idBCJIA64:
|
||||
filtered = bcjIA64Filter(s, buf)
|
||||
case idBCJARM:
|
||||
filtered = bcjARMFilter(s, buf)
|
||||
case idBCJARMThumb:
|
||||
filtered = bcjARMThumbFilter(s, buf)
|
||||
case idBCJSPARC:
|
||||
filtered = bcjSPARCFilter(s, buf)
|
||||
default:
|
||||
/* Never reached */
|
||||
}
|
||||
*pos += filtered
|
||||
s.pos += filtered
|
||||
}
|
||||
|
||||
/*
|
||||
* Flush pending filtered data from temp to the output buffer.
|
||||
* Move the remaining mixture of possibly filtered and unfiltered
|
||||
* data to the beginning of temp.
|
||||
*/
|
||||
func bcjFlush(s *xzDecBCJ, b *xzBuf) {
|
||||
var copySize int
|
||||
copySize = len(b.out) - b.outPos
|
||||
if copySize > s.temp.filtered {
|
||||
copySize = s.temp.filtered
|
||||
}
|
||||
copy(b.out[b.outPos:], s.temp.buf[:copySize])
|
||||
b.outPos += copySize
|
||||
s.temp.filtered -= copySize
|
||||
copy(s.temp.buf, s.temp.buf[copySize:])
|
||||
s.temp.buf = s.temp.buf[:len(s.temp.buf)-copySize]
|
||||
}
|
||||
|
||||
/*
|
||||
* Decode raw stream which has a BCJ filter as the first filter.
|
||||
*
|
||||
* The BCJ filter functions are primitive in sense that they process the
|
||||
* data in chunks of 1-16 bytes. To hide this issue, this function does
|
||||
* some buffering.
|
||||
*/
|
||||
func xzDecBCJRun(s *xzDecBCJ, b *xzBuf, chain func(*xzBuf) xzRet) xzRet {
|
||||
var outStart int
|
||||
/*
|
||||
* Flush pending already filtered data to the output buffer. Return
|
||||
* immediately if we couldn't flush everything, or if the next
|
||||
* filter in the chain had already returned xzStreamEnd.
|
||||
*/
|
||||
if s.temp.filtered > 0 {
|
||||
bcjFlush(s, b)
|
||||
if s.temp.filtered > 0 {
|
||||
return xzOK
|
||||
}
|
||||
if s.ret == xzStreamEnd {
|
||||
return xzStreamEnd
|
||||
}
|
||||
}
|
||||
/*
|
||||
* If we have more output space than what is currently pending in
|
||||
* temp, copy the unfiltered data from temp to the output buffer
|
||||
* and try to fill the output buffer by decoding more data from the
|
||||
* next filter in the chain. Apply the BCJ filter on the new data
|
||||
* in the output buffer. If everything cannot be filtered, copy it
|
||||
* to temp and rewind the output buffer position accordingly.
|
||||
*
|
||||
* This needs to be always run when len(temp.buf) == 0 to handle a special
|
||||
* case where the output buffer is full and the next filter has no
|
||||
* more output coming but hasn't returned xzStreamEnd yet.
|
||||
*/
|
||||
if len(s.temp.buf) < len(b.out)-b.outPos || len(s.temp.buf) == 0 {
|
||||
outStart = b.outPos
|
||||
copy(b.out[b.outPos:], s.temp.buf)
|
||||
b.outPos += len(s.temp.buf)
|
||||
s.ret = chain(b)
|
||||
if s.ret != xzStreamEnd && s.ret != xzOK {
|
||||
return s.ret
|
||||
}
|
||||
bcjApply(s, b.out[:b.outPos], &outStart)
|
||||
/*
|
||||
* As an exception, if the next filter returned xzStreamEnd,
|
||||
* we can do that too, since the last few bytes that remain
|
||||
* unfiltered are meant to remain unfiltered.
|
||||
*/
|
||||
if s.ret == xzStreamEnd {
|
||||
return xzStreamEnd
|
||||
}
|
||||
s.temp.buf = s.temp.bufArray[:b.outPos-outStart]
|
||||
b.outPos -= len(s.temp.buf)
|
||||
copy(s.temp.buf, b.out[b.outPos:])
|
||||
/*
|
||||
* If there wasn't enough input to the next filter to fill
|
||||
* the output buffer with unfiltered data, there's no point
|
||||
* to try decoding more data to temp.
|
||||
*/
|
||||
if b.outPos+len(s.temp.buf) < len(b.out) {
|
||||
return xzOK
|
||||
}
|
||||
}
|
||||
/*
|
||||
* We have unfiltered data in temp. If the output buffer isn't full
|
||||
* yet, try to fill the temp buffer by decoding more data from the
|
||||
* next filter. Apply the BCJ filter on temp. Then we hopefully can
|
||||
* fill the actual output buffer by copying filtered data from temp.
|
||||
* A mix of filtered and unfiltered data may be left in temp; it will
|
||||
* be taken care on the next call to this function.
|
||||
*/
|
||||
if b.outPos < len(b.out) {
|
||||
/* Make b.out temporarily point to s.temp. */
|
||||
s.out = b.out
|
||||
s.outPos = b.outPos
|
||||
b.out = s.temp.bufArray[:]
|
||||
b.outPos = len(s.temp.buf)
|
||||
s.ret = chain(b)
|
||||
s.temp.buf = s.temp.bufArray[:b.outPos]
|
||||
b.out = s.out
|
||||
b.outPos = s.outPos
|
||||
if s.ret != xzOK && s.ret != xzStreamEnd {
|
||||
return s.ret
|
||||
}
|
||||
bcjApply(s, s.temp.buf, &s.temp.filtered)
|
||||
/*
|
||||
* If the next filter returned xzStreamEnd, we mark that
|
||||
* everything is filtered, since the last unfiltered bytes
|
||||
* of the stream are meant to be left as is.
|
||||
*/
|
||||
if s.ret == xzStreamEnd {
|
||||
s.temp.filtered = len(s.temp.buf)
|
||||
}
|
||||
bcjFlush(s, b)
|
||||
if s.temp.filtered > 0 {
|
||||
return xzOK
|
||||
}
|
||||
}
|
||||
return s.ret
|
||||
}
|
||||
|
||||
/*
|
||||
* Allocate memory for BCJ decoders. xzDecBCJReset must be used before
|
||||
* calling xzDecBCJRun.
|
||||
*/
|
||||
func xzDecBCJCreate() *xzDecBCJ {
|
||||
return new(xzDecBCJ)
|
||||
}
|
||||
|
||||
/*
|
||||
* Decode the Filter ID of a BCJ filter and check the start offset is
|
||||
* valid. Returns xzOK if the given Filter ID and offset is
|
||||
* supported. Otherwise xzOptionsError is returned.
|
||||
*/
|
||||
func xzDecBCJReset(s *xzDecBCJ, id xzFilterID, offset int) xzRet {
|
||||
switch id {
|
||||
case idBCJX86:
|
||||
case idBCJPowerPC:
|
||||
case idBCJIA64:
|
||||
case idBCJARM:
|
||||
case idBCJARMThumb:
|
||||
case idBCJSPARC:
|
||||
default:
|
||||
/* Unsupported Filter ID */
|
||||
return xzOptionsError
|
||||
}
|
||||
// check offset is a multiple of alignment
|
||||
switch id {
|
||||
case idBCJPowerPC, idBCJARM, idBCJSPARC:
|
||||
if offset%4 != 0 {
|
||||
return xzOptionsError
|
||||
}
|
||||
case idBCJIA64:
|
||||
if offset%16 != 0 {
|
||||
return xzOptionsError
|
||||
}
|
||||
case idBCJARMThumb:
|
||||
if offset%2 != 0 {
|
||||
return xzOptionsError
|
||||
}
|
||||
}
|
||||
s.typ = id
|
||||
s.ret = xzOK
|
||||
s.pos = offset
|
||||
s.x86PrevMask = 0
|
||||
s.temp.filtered = 0
|
||||
s.temp.buf = nil
|
||||
return xzOK
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
* Delta decoder
|
||||
*
|
||||
* Author: Lasse Collin <lasse.collin@tukaani.org>
|
||||
*
|
||||
* Translation to Go: Michael Cross <https://github.com/xi2>
|
||||
*
|
||||
* This file has been put into the public domain.
|
||||
* You can do whatever you want with this file.
|
||||
*/
|
||||
|
||||
package xz
|
||||
|
||||
type xzDecDelta struct {
|
||||
delta [256]byte
|
||||
pos byte
|
||||
distance int // in range [1, 256]
|
||||
}
|
||||
|
||||
/*
|
||||
* Decode raw stream which has a delta filter as the first filter.
|
||||
*/
|
||||
func xzDecDeltaRun(s *xzDecDelta, b *xzBuf, chain func(*xzBuf) xzRet) xzRet {
|
||||
outStart := b.outPos
|
||||
ret := chain(b)
|
||||
for i := outStart; i < b.outPos; i++ {
|
||||
tmp := b.out[i] + s.delta[byte(s.distance+int(s.pos))]
|
||||
s.delta[s.pos] = tmp
|
||||
b.out[i] = tmp
|
||||
s.pos--
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
/*
|
||||
* Allocate memory for a delta decoder. xzDecDeltaReset must be used
|
||||
* before calling xzDecDeltaRun.
|
||||
*/
|
||||
func xzDecDeltaCreate() *xzDecDelta {
|
||||
return new(xzDecDelta)
|
||||
}
|
||||
|
||||
/*
|
||||
* Returns xzOK if the given distance is valid. Otherwise
|
||||
* xzOptionsError is returned.
|
||||
*/
|
||||
func xzDecDeltaReset(s *xzDecDelta, distance int) xzRet {
|
||||
if distance < 1 || distance > 256 {
|
||||
return xzOptionsError
|
||||
}
|
||||
s.delta = [256]byte{}
|
||||
s.pos = 0
|
||||
s.distance = distance
|
||||
return xzOK
|
||||
}
|
||||
+1235
File diff suppressed because it is too large
Load Diff
+932
@@ -0,0 +1,932 @@
|
||||
/*
|
||||
* .xz Stream decoder
|
||||
*
|
||||
* Author: Lasse Collin <lasse.collin@tukaani.org>
|
||||
*
|
||||
* Translation to Go: Michael Cross <https://github.com/xi2>
|
||||
*
|
||||
* This file has been put into the public domain.
|
||||
* You can do whatever you want with this file.
|
||||
*/
|
||||
|
||||
package xz
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"hash"
|
||||
"hash/crc32"
|
||||
"hash/crc64"
|
||||
)
|
||||
|
||||
/* from linux/lib/xz/xz_stream.h **************************************/
|
||||
|
||||
/*
|
||||
* See the .xz file format specification at
|
||||
* http://tukaani.org/xz/xz-file-format.txt
|
||||
* to understand the container format.
|
||||
*/
|
||||
const (
|
||||
streamHeaderSize = 12
|
||||
headerMagic = "\xfd7zXZ\x00"
|
||||
footerMagic = "YZ"
|
||||
)
|
||||
|
||||
/*
|
||||
* Variable-length integer can hold a 63-bit unsigned integer or a special
|
||||
* value indicating that the value is unknown.
|
||||
*/
|
||||
type vliType uint64
|
||||
|
||||
const (
|
||||
vliUnknown vliType = ^vliType(0)
|
||||
/* Maximum encoded size of a VLI */
|
||||
vliBytesMax = 8 * 8 / 7 // (Sizeof(vliType) * 8 / 7)
|
||||
)
|
||||
|
||||
/* from linux/lib/xz/xz_dec_stream.c **********************************/
|
||||
|
||||
/* Hash used to validate the Index field */
|
||||
type xzDecHash struct {
|
||||
unpadded vliType
|
||||
uncompressed vliType
|
||||
sha256 hash.Hash
|
||||
}
|
||||
|
||||
// type of xzDec.sequence
|
||||
type xzDecSeq int
|
||||
|
||||
const (
|
||||
seqStreamHeader xzDecSeq = iota
|
||||
seqBlockStart
|
||||
seqBlockHeader
|
||||
seqBlockUncompress
|
||||
seqBlockPadding
|
||||
seqBlockCheck
|
||||
seqIndex
|
||||
seqIndexPadding
|
||||
seqIndexCRC32
|
||||
seqStreamFooter
|
||||
)
|
||||
|
||||
// type of xzDec.index.sequence
|
||||
type xzDecIndexSeq int
|
||||
|
||||
const (
|
||||
seqIndexCount xzDecIndexSeq = iota
|
||||
seqIndexUnpadded
|
||||
seqIndexUncompressed
|
||||
)
|
||||
|
||||
/**
|
||||
* xzDec - Opaque type to hold the XZ decoder state
|
||||
*/
|
||||
type xzDec struct {
|
||||
/* Position in decMain */
|
||||
sequence xzDecSeq
|
||||
/* Position in variable-length integers and Check fields */
|
||||
pos int
|
||||
/* Variable-length integer decoded by decVLI */
|
||||
vli vliType
|
||||
/* Saved inPos and outPos */
|
||||
inStart int
|
||||
outStart int
|
||||
/* CRC32 checksum hash used in Index */
|
||||
crc32 hash.Hash
|
||||
/* Hashes used in Blocks */
|
||||
checkCRC32 hash.Hash
|
||||
checkCRC64 hash.Hash
|
||||
checkSHA256 hash.Hash
|
||||
/* for checkTypes CRC32/CRC64/SHA256, check is one of the above 3 hashes */
|
||||
check hash.Hash
|
||||
/* Embedded stream header struct containing CheckType */
|
||||
*Header
|
||||
/*
|
||||
* True if the next call to xzDecRun is allowed to return
|
||||
* xzBufError.
|
||||
*/
|
||||
allowBufError bool
|
||||
/* Information stored in Block Header */
|
||||
blockHeader struct {
|
||||
/*
|
||||
* Value stored in the Compressed Size field, or
|
||||
* vliUnknown if Compressed Size is not present.
|
||||
*/
|
||||
compressed vliType
|
||||
/*
|
||||
* Value stored in the Uncompressed Size field, or
|
||||
* vliUnknown if Uncompressed Size is not present.
|
||||
*/
|
||||
uncompressed vliType
|
||||
/* Size of the Block Header field */
|
||||
size int
|
||||
}
|
||||
/* Information collected when decoding Blocks */
|
||||
block struct {
|
||||
/* Observed compressed size of the current Block */
|
||||
compressed vliType
|
||||
/* Observed uncompressed size of the current Block */
|
||||
uncompressed vliType
|
||||
/* Number of Blocks decoded so far */
|
||||
count vliType
|
||||
/*
|
||||
* Hash calculated from the Block sizes. This is used to
|
||||
* validate the Index field.
|
||||
*/
|
||||
hash xzDecHash
|
||||
}
|
||||
/* Variables needed when verifying the Index field */
|
||||
index struct {
|
||||
/* Position in decIndex */
|
||||
sequence xzDecIndexSeq
|
||||
/* Size of the Index in bytes */
|
||||
size vliType
|
||||
/* Number of Records (matches block.count in valid files) */
|
||||
count vliType
|
||||
/*
|
||||
* Hash calculated from the Records (matches block.hash in
|
||||
* valid files).
|
||||
*/
|
||||
hash xzDecHash
|
||||
}
|
||||
/*
|
||||
* Temporary buffer needed to hold Stream Header, Block Header,
|
||||
* and Stream Footer. The Block Header is the biggest (1 KiB)
|
||||
* so we reserve space according to that. bufArray has to be aligned
|
||||
* to a multiple of four bytes; the variables before it
|
||||
* should guarantee this.
|
||||
*/
|
||||
temp struct {
|
||||
pos int
|
||||
buf []byte // slice buf will be backed by bufArray
|
||||
bufArray [1024]byte
|
||||
}
|
||||
// chain is the function (or to be more precise, closure) which
|
||||
// does the decompression and will call into the lzma2 and other
|
||||
// filter code as needed. It is constructed by decBlockHeader
|
||||
chain func(b *xzBuf) xzRet
|
||||
// lzma2 holds the state of the last filter (which must be LZMA2)
|
||||
lzma2 *xzDecLZMA2
|
||||
// pointers to allocated BCJ/Delta filters
|
||||
bcjs []*xzDecBCJ
|
||||
deltas []*xzDecDelta
|
||||
// number of currently in use BCJ/Delta filters from the above
|
||||
bcjsUsed int
|
||||
deltasUsed int
|
||||
}
|
||||
|
||||
/* Sizes of the Check field with different Check IDs */
|
||||
var checkSizes = [...]byte{
|
||||
0,
|
||||
4, 4, 4,
|
||||
8, 8, 8,
|
||||
16, 16, 16,
|
||||
32, 32, 32,
|
||||
64, 64, 64,
|
||||
}
|
||||
|
||||
/*
|
||||
* Fill s.temp by copying data starting from b.in[b.inPos]. Caller
|
||||
* must have set s.temp.pos to indicate how much data we are supposed
|
||||
* to copy into s.temp.buf. Return true once s.temp.pos has reached
|
||||
* len(s.temp.buf).
|
||||
*/
|
||||
func fillTemp(s *xzDec, b *xzBuf) bool {
|
||||
copySize := len(b.in) - b.inPos
|
||||
tempRemaining := len(s.temp.buf) - s.temp.pos
|
||||
if copySize > tempRemaining {
|
||||
copySize = tempRemaining
|
||||
}
|
||||
copy(s.temp.buf[s.temp.pos:], b.in[b.inPos:])
|
||||
b.inPos += copySize
|
||||
s.temp.pos += copySize
|
||||
if s.temp.pos == len(s.temp.buf) {
|
||||
s.temp.pos = 0
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/* Decode a variable-length integer (little-endian base-128 encoding) */
|
||||
func decVLI(s *xzDec, in []byte, inPos *int) xzRet {
|
||||
var byte byte
|
||||
if s.pos == 0 {
|
||||
s.vli = 0
|
||||
}
|
||||
for *inPos < len(in) {
|
||||
byte = in[*inPos]
|
||||
*inPos++
|
||||
s.vli |= vliType(byte&0x7f) << uint(s.pos)
|
||||
if byte&0x80 == 0 {
|
||||
/* Don't allow non-minimal encodings. */
|
||||
if byte == 0 && s.pos != 0 {
|
||||
return xzDataError
|
||||
}
|
||||
s.pos = 0
|
||||
return xzStreamEnd
|
||||
}
|
||||
s.pos += 7
|
||||
if s.pos == 7*vliBytesMax {
|
||||
return xzDataError
|
||||
}
|
||||
}
|
||||
return xzOK
|
||||
}
|
||||
|
||||
/*
|
||||
* Decode the Compressed Data field from a Block. Update and validate
|
||||
* the observed compressed and uncompressed sizes of the Block so that
|
||||
* they don't exceed the values possibly stored in the Block Header
|
||||
* (validation assumes that no integer overflow occurs, since vliType
|
||||
* is uint64). Update s.check if presence of the CRC32/CRC64/SHA256
|
||||
* field was indicated in Stream Header.
|
||||
*
|
||||
* Once the decoding is finished, validate that the observed sizes match
|
||||
* the sizes possibly stored in the Block Header. Update the hash and
|
||||
* Block count, which are later used to validate the Index field.
|
||||
*/
|
||||
func decBlock(s *xzDec, b *xzBuf) xzRet {
|
||||
var ret xzRet
|
||||
s.inStart = b.inPos
|
||||
s.outStart = b.outPos
|
||||
ret = s.chain(b)
|
||||
s.block.compressed += vliType(b.inPos - s.inStart)
|
||||
s.block.uncompressed += vliType(b.outPos - s.outStart)
|
||||
/*
|
||||
* There is no need to separately check for vliUnknown since
|
||||
* the observed sizes are always smaller than vliUnknown.
|
||||
*/
|
||||
if s.block.compressed > s.blockHeader.compressed ||
|
||||
s.block.uncompressed > s.blockHeader.uncompressed {
|
||||
return xzDataError
|
||||
}
|
||||
switch s.CheckType {
|
||||
case CheckCRC32, CheckCRC64, CheckSHA256:
|
||||
_, _ = s.check.Write(b.out[s.outStart:b.outPos])
|
||||
}
|
||||
if ret == xzStreamEnd {
|
||||
if s.blockHeader.compressed != vliUnknown &&
|
||||
s.blockHeader.compressed != s.block.compressed {
|
||||
return xzDataError
|
||||
}
|
||||
if s.blockHeader.uncompressed != vliUnknown &&
|
||||
s.blockHeader.uncompressed != s.block.uncompressed {
|
||||
return xzDataError
|
||||
}
|
||||
s.block.hash.unpadded +=
|
||||
vliType(s.blockHeader.size) + s.block.compressed
|
||||
s.block.hash.unpadded += vliType(checkSizes[s.CheckType])
|
||||
s.block.hash.uncompressed += s.block.uncompressed
|
||||
var buf [2 * 8]byte // 2*Sizeof(vliType)
|
||||
putLE64(uint64(s.block.hash.unpadded), buf[:])
|
||||
putLE64(uint64(s.block.hash.uncompressed), buf[8:])
|
||||
_, _ = s.block.hash.sha256.Write(buf[:])
|
||||
s.block.count++
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
/* Update the Index size and the CRC32 hash. */
|
||||
func indexUpdate(s *xzDec, b *xzBuf) {
|
||||
inUsed := b.inPos - s.inStart
|
||||
s.index.size += vliType(inUsed)
|
||||
_, _ = s.crc32.Write(b.in[s.inStart : s.inStart+inUsed])
|
||||
}
|
||||
|
||||
/*
|
||||
* Decode the Number of Records, Unpadded Size, and Uncompressed Size
|
||||
* fields from the Index field. That is, Index Padding and CRC32 are not
|
||||
* decoded by this function.
|
||||
*
|
||||
* This can return xzOK (more input needed), xzStreamEnd (everything
|
||||
* successfully decoded), or xzDataError (input is corrupt).
|
||||
*/
|
||||
func decIndex(s *xzDec, b *xzBuf) xzRet {
|
||||
var ret xzRet
|
||||
for {
|
||||
ret = decVLI(s, b.in, &b.inPos)
|
||||
if ret != xzStreamEnd {
|
||||
indexUpdate(s, b)
|
||||
return ret
|
||||
}
|
||||
switch s.index.sequence {
|
||||
case seqIndexCount:
|
||||
s.index.count = s.vli
|
||||
/*
|
||||
* Validate that the Number of Records field
|
||||
* indicates the same number of Records as
|
||||
* there were Blocks in the Stream.
|
||||
*/
|
||||
if s.index.count != s.block.count {
|
||||
return xzDataError
|
||||
}
|
||||
s.index.sequence = seqIndexUnpadded
|
||||
case seqIndexUnpadded:
|
||||
s.index.hash.unpadded += s.vli
|
||||
s.index.sequence = seqIndexUncompressed
|
||||
case seqIndexUncompressed:
|
||||
s.index.hash.uncompressed += s.vli
|
||||
var buf [2 * 8]byte // 2*Sizeof(vliType)
|
||||
putLE64(uint64(s.index.hash.unpadded), buf[:])
|
||||
putLE64(uint64(s.index.hash.uncompressed), buf[8:])
|
||||
_, _ = s.index.hash.sha256.Write(buf[:])
|
||||
s.index.count--
|
||||
s.index.sequence = seqIndexUnpadded
|
||||
}
|
||||
if !(s.index.count > 0) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return xzStreamEnd
|
||||
}
|
||||
|
||||
/*
|
||||
* Validate that the next 4 bytes match s.crc32.Sum(nil). s.pos must
|
||||
* be zero when starting to validate the first byte.
|
||||
*/
|
||||
func crcValidate(s *xzDec, b *xzBuf) xzRet {
|
||||
sum := s.crc32.Sum(nil)
|
||||
// CRC32 - reverse slice
|
||||
sum[0], sum[1], sum[2], sum[3] = sum[3], sum[2], sum[1], sum[0]
|
||||
for {
|
||||
if b.inPos == len(b.in) {
|
||||
return xzOK
|
||||
}
|
||||
if sum[s.pos] != b.in[b.inPos] {
|
||||
return xzDataError
|
||||
}
|
||||
b.inPos++
|
||||
s.pos++
|
||||
if !(s.pos < 4) {
|
||||
break
|
||||
}
|
||||
}
|
||||
s.crc32.Reset()
|
||||
s.pos = 0
|
||||
return xzStreamEnd
|
||||
}
|
||||
|
||||
/*
|
||||
* Validate that the next 4/8/32 bytes match s.check.Sum(nil). s.pos
|
||||
* must be zero when starting to validate the first byte.
|
||||
*/
|
||||
func checkValidate(s *xzDec, b *xzBuf) xzRet {
|
||||
sum := s.check.Sum(nil)
|
||||
if s.CheckType == CheckCRC32 || s.CheckType == CheckCRC64 {
|
||||
// CRC32/64 - reverse slice
|
||||
for i, j := 0, len(sum)-1; i < j; i, j = i+1, j-1 {
|
||||
sum[i], sum[j] = sum[j], sum[i]
|
||||
}
|
||||
}
|
||||
for {
|
||||
if b.inPos == len(b.in) {
|
||||
return xzOK
|
||||
}
|
||||
if sum[s.pos] != b.in[b.inPos] {
|
||||
return xzDataError
|
||||
}
|
||||
b.inPos++
|
||||
s.pos++
|
||||
if !(s.pos < len(sum)) {
|
||||
break
|
||||
}
|
||||
}
|
||||
s.check.Reset()
|
||||
s.pos = 0
|
||||
return xzStreamEnd
|
||||
}
|
||||
|
||||
/*
|
||||
* Skip over the Check field when the Check ID is not supported.
|
||||
* Returns true once the whole Check field has been skipped over.
|
||||
*/
|
||||
func checkSkip(s *xzDec, b *xzBuf) bool {
|
||||
for s.pos < int(checkSizes[s.CheckType]) {
|
||||
if b.inPos == len(b.in) {
|
||||
return false
|
||||
}
|
||||
b.inPos++
|
||||
s.pos++
|
||||
}
|
||||
s.pos = 0
|
||||
return true
|
||||
}
|
||||
|
||||
/* polynomial table used in decStreamHeader below */
|
||||
var xzCRC64Table = crc64.MakeTable(crc64.ECMA)
|
||||
|
||||
/* Decode the Stream Header field (the first 12 bytes of the .xz Stream). */
|
||||
func decStreamHeader(s *xzDec) xzRet {
|
||||
if string(s.temp.buf[:len(headerMagic)]) != headerMagic {
|
||||
return xzFormatError
|
||||
}
|
||||
if crc32.ChecksumIEEE(s.temp.buf[len(headerMagic):len(headerMagic)+2]) !=
|
||||
getLE32(s.temp.buf[len(headerMagic)+2:]) {
|
||||
return xzDataError
|
||||
}
|
||||
if s.temp.buf[len(headerMagic)] != 0 {
|
||||
return xzOptionsError
|
||||
}
|
||||
/*
|
||||
* Of integrity checks, we support none (Check ID = 0),
|
||||
* CRC32 (Check ID = 1), CRC64 (Check ID = 4) and SHA256 (Check ID = 10)
|
||||
* However, we will accept other check types too, but then the check
|
||||
* won't be verified and a warning (xzUnsupportedCheck) will be given.
|
||||
*/
|
||||
s.CheckType = CheckID(s.temp.buf[len(headerMagic)+1])
|
||||
if s.CheckType > checkMax {
|
||||
return xzOptionsError
|
||||
}
|
||||
switch s.CheckType {
|
||||
case CheckNone:
|
||||
// CheckNone: no action needed
|
||||
case CheckCRC32:
|
||||
if s.checkCRC32 == nil {
|
||||
s.checkCRC32 = crc32.NewIEEE()
|
||||
} else {
|
||||
s.checkCRC32.Reset()
|
||||
}
|
||||
s.check = s.checkCRC32
|
||||
case CheckCRC64:
|
||||
if s.checkCRC64 == nil {
|
||||
s.checkCRC64 = crc64.New(xzCRC64Table)
|
||||
} else {
|
||||
s.checkCRC64.Reset()
|
||||
}
|
||||
s.check = s.checkCRC64
|
||||
case CheckSHA256:
|
||||
if s.checkSHA256 == nil {
|
||||
s.checkSHA256 = sha256.New()
|
||||
} else {
|
||||
s.checkSHA256.Reset()
|
||||
}
|
||||
s.check = s.checkSHA256
|
||||
default:
|
||||
return xzUnsupportedCheck
|
||||
}
|
||||
return xzOK
|
||||
}
|
||||
|
||||
/* Decode the Stream Footer field (the last 12 bytes of the .xz Stream) */
|
||||
func decStreamFooter(s *xzDec) xzRet {
|
||||
if string(s.temp.buf[10:10+len(footerMagic)]) != footerMagic {
|
||||
return xzDataError
|
||||
}
|
||||
if crc32.ChecksumIEEE(s.temp.buf[4:10]) != getLE32(s.temp.buf) {
|
||||
return xzDataError
|
||||
}
|
||||
/*
|
||||
* Validate Backward Size. Note that we never added the size of the
|
||||
* Index CRC32 field to s->index.size, thus we use s->index.size / 4
|
||||
* instead of s->index.size / 4 - 1.
|
||||
*/
|
||||
if s.index.size>>2 != vliType(getLE32(s.temp.buf[4:])) {
|
||||
return xzDataError
|
||||
}
|
||||
if s.temp.buf[8] != 0 || CheckID(s.temp.buf[9]) != s.CheckType {
|
||||
return xzDataError
|
||||
}
|
||||
/*
|
||||
* Use xzStreamEnd instead of xzOK to be more convenient
|
||||
* for the caller.
|
||||
*/
|
||||
return xzStreamEnd
|
||||
}
|
||||
|
||||
/* Decode the Block Header and initialize the filter chain. */
|
||||
func decBlockHeader(s *xzDec) xzRet {
|
||||
var ret xzRet
|
||||
/*
|
||||
* Validate the CRC32. We know that the temp buffer is at least
|
||||
* eight bytes so this is safe.
|
||||
*/
|
||||
crc := getLE32(s.temp.buf[len(s.temp.buf)-4:])
|
||||
s.temp.buf = s.temp.buf[:len(s.temp.buf)-4]
|
||||
if crc32.ChecksumIEEE(s.temp.buf) != crc {
|
||||
return xzDataError
|
||||
}
|
||||
s.temp.pos = 2
|
||||
/*
|
||||
* Catch unsupported Block Flags.
|
||||
*/
|
||||
if s.temp.buf[1]&0x3C != 0 {
|
||||
return xzOptionsError
|
||||
}
|
||||
/* Compressed Size */
|
||||
if s.temp.buf[1]&0x40 != 0 {
|
||||
if decVLI(s, s.temp.buf, &s.temp.pos) != xzStreamEnd {
|
||||
return xzDataError
|
||||
}
|
||||
if s.vli >= 1<<63-8 {
|
||||
// the whole block must stay smaller than 2^63 bytes
|
||||
// the block header cannot be smaller than 8 bytes
|
||||
return xzDataError
|
||||
}
|
||||
if s.vli == 0 {
|
||||
// compressed size must be non-zero
|
||||
return xzDataError
|
||||
}
|
||||
s.blockHeader.compressed = s.vli
|
||||
} else {
|
||||
s.blockHeader.compressed = vliUnknown
|
||||
}
|
||||
/* Uncompressed Size */
|
||||
if s.temp.buf[1]&0x80 != 0 {
|
||||
if decVLI(s, s.temp.buf, &s.temp.pos) != xzStreamEnd {
|
||||
return xzDataError
|
||||
}
|
||||
s.blockHeader.uncompressed = s.vli
|
||||
} else {
|
||||
s.blockHeader.uncompressed = vliUnknown
|
||||
}
|
||||
// get total number of filters (1-4)
|
||||
filterTotal := int(s.temp.buf[1]&0x03) + 1
|
||||
// slice to hold decoded filters
|
||||
filterList := make([]struct {
|
||||
id xzFilterID
|
||||
props uint32
|
||||
}, filterTotal)
|
||||
// decode the non-last filters which cannot be LZMA2
|
||||
for i := 0; i < filterTotal-1; i++ {
|
||||
/* Valid Filter Flags always take at least two bytes. */
|
||||
if len(s.temp.buf)-s.temp.pos < 2 {
|
||||
return xzDataError
|
||||
}
|
||||
s.temp.pos += 2
|
||||
switch id := xzFilterID(s.temp.buf[s.temp.pos-2]); id {
|
||||
case idDelta:
|
||||
// delta filter
|
||||
if s.temp.buf[s.temp.pos-1] != 0x01 {
|
||||
return xzOptionsError
|
||||
}
|
||||
/* Filter Properties contains distance - 1 */
|
||||
if len(s.temp.buf)-s.temp.pos < 1 {
|
||||
return xzDataError
|
||||
}
|
||||
props := uint32(s.temp.buf[s.temp.pos])
|
||||
s.temp.pos++
|
||||
filterList[i] = struct {
|
||||
id xzFilterID
|
||||
props uint32
|
||||
}{id: id, props: props}
|
||||
case idBCJX86, idBCJPowerPC, idBCJIA64,
|
||||
idBCJARM, idBCJARMThumb, idBCJSPARC:
|
||||
// bcj filter
|
||||
var props uint32
|
||||
switch s.temp.buf[s.temp.pos-1] {
|
||||
case 0x00:
|
||||
props = 0
|
||||
case 0x04:
|
||||
if len(s.temp.buf)-s.temp.pos < 4 {
|
||||
return xzDataError
|
||||
}
|
||||
props = getLE32(s.temp.buf[s.temp.pos:])
|
||||
s.temp.pos += 4
|
||||
default:
|
||||
return xzOptionsError
|
||||
}
|
||||
filterList[i] = struct {
|
||||
id xzFilterID
|
||||
props uint32
|
||||
}{id: id, props: props}
|
||||
default:
|
||||
return xzOptionsError
|
||||
}
|
||||
}
|
||||
/*
|
||||
* decode the last filter which must be LZMA2
|
||||
*/
|
||||
if len(s.temp.buf)-s.temp.pos < 2 {
|
||||
return xzDataError
|
||||
}
|
||||
/* Filter ID = LZMA2 */
|
||||
if xzFilterID(s.temp.buf[s.temp.pos]) != idLZMA2 {
|
||||
return xzOptionsError
|
||||
}
|
||||
s.temp.pos++
|
||||
/* Size of Properties = 1-byte Filter Properties */
|
||||
if s.temp.buf[s.temp.pos] != 0x01 {
|
||||
return xzOptionsError
|
||||
}
|
||||
s.temp.pos++
|
||||
/* Filter Properties contains LZMA2 dictionary size. */
|
||||
if len(s.temp.buf)-s.temp.pos < 1 {
|
||||
return xzDataError
|
||||
}
|
||||
props := uint32(s.temp.buf[s.temp.pos])
|
||||
s.temp.pos++
|
||||
filterList[filterTotal-1] = struct {
|
||||
id xzFilterID
|
||||
props uint32
|
||||
}{id: idLZMA2, props: props}
|
||||
/*
|
||||
* Process the filter list and create s.chain, going from last
|
||||
* filter (LZMA2) to first filter
|
||||
*
|
||||
* First, LZMA2.
|
||||
*/
|
||||
ret = xzDecLZMA2Reset(s.lzma2, byte(filterList[filterTotal-1].props))
|
||||
if ret != xzOK {
|
||||
return ret
|
||||
}
|
||||
s.chain = func(b *xzBuf) xzRet {
|
||||
return xzDecLZMA2Run(s.lzma2, b)
|
||||
}
|
||||
/*
|
||||
* Now the non-last filters
|
||||
*/
|
||||
for i := filterTotal - 2; i >= 0; i-- {
|
||||
switch id := filterList[i].id; id {
|
||||
case idDelta:
|
||||
// delta filter
|
||||
var delta *xzDecDelta
|
||||
if s.deltasUsed < len(s.deltas) {
|
||||
delta = s.deltas[s.deltasUsed]
|
||||
} else {
|
||||
delta = xzDecDeltaCreate()
|
||||
s.deltas = append(s.deltas, delta)
|
||||
}
|
||||
s.deltasUsed++
|
||||
ret = xzDecDeltaReset(delta, int(filterList[i].props)+1)
|
||||
if ret != xzOK {
|
||||
return ret
|
||||
}
|
||||
chain := s.chain
|
||||
s.chain = func(b *xzBuf) xzRet {
|
||||
return xzDecDeltaRun(delta, b, chain)
|
||||
}
|
||||
case idBCJX86, idBCJPowerPC, idBCJIA64,
|
||||
idBCJARM, idBCJARMThumb, idBCJSPARC:
|
||||
// bcj filter
|
||||
var bcj *xzDecBCJ
|
||||
if s.bcjsUsed < len(s.bcjs) {
|
||||
bcj = s.bcjs[s.bcjsUsed]
|
||||
} else {
|
||||
bcj = xzDecBCJCreate()
|
||||
s.bcjs = append(s.bcjs, bcj)
|
||||
}
|
||||
s.bcjsUsed++
|
||||
ret = xzDecBCJReset(bcj, id, int(filterList[i].props))
|
||||
if ret != xzOK {
|
||||
return ret
|
||||
}
|
||||
chain := s.chain
|
||||
s.chain = func(b *xzBuf) xzRet {
|
||||
return xzDecBCJRun(bcj, b, chain)
|
||||
}
|
||||
}
|
||||
}
|
||||
/* The rest must be Header Padding. */
|
||||
for s.temp.pos < len(s.temp.buf) {
|
||||
if s.temp.buf[s.temp.pos] != 0x00 {
|
||||
return xzOptionsError
|
||||
}
|
||||
s.temp.pos++
|
||||
}
|
||||
s.temp.pos = 0
|
||||
s.block.compressed = 0
|
||||
s.block.uncompressed = 0
|
||||
return xzOK
|
||||
}
|
||||
|
||||
func decMain(s *xzDec, b *xzBuf) xzRet {
|
||||
var ret xzRet
|
||||
/*
|
||||
* Store the start position for the case when we are in the middle
|
||||
* of the Index field.
|
||||
*/
|
||||
s.inStart = b.inPos
|
||||
for {
|
||||
switch s.sequence {
|
||||
case seqStreamHeader:
|
||||
/*
|
||||
* Stream Header is copied to s.temp, and then
|
||||
* decoded from there. This way if the caller
|
||||
* gives us only little input at a time, we can
|
||||
* still keep the Stream Header decoding code
|
||||
* simple. Similar approach is used in many places
|
||||
* in this file.
|
||||
*/
|
||||
if !fillTemp(s, b) {
|
||||
return xzOK
|
||||
}
|
||||
/*
|
||||
* If decStreamHeader returns
|
||||
* xzUnsupportedCheck, it is still possible
|
||||
* to continue decoding. Thus, update s.sequence
|
||||
* before calling decStreamHeader.
|
||||
*/
|
||||
s.sequence = seqBlockStart
|
||||
ret = decStreamHeader(s)
|
||||
if ret != xzOK {
|
||||
return ret
|
||||
}
|
||||
fallthrough
|
||||
case seqBlockStart:
|
||||
/* We need one byte of input to continue. */
|
||||
if b.inPos == len(b.in) {
|
||||
return xzOK
|
||||
}
|
||||
/* See if this is the beginning of the Index field. */
|
||||
if b.in[b.inPos] == 0 {
|
||||
s.inStart = b.inPos
|
||||
b.inPos++
|
||||
s.sequence = seqIndex
|
||||
break
|
||||
}
|
||||
/*
|
||||
* Calculate the size of the Block Header and
|
||||
* prepare to decode it.
|
||||
*/
|
||||
s.blockHeader.size = (int(b.in[b.inPos]) + 1) * 4
|
||||
s.temp.buf = s.temp.bufArray[:s.blockHeader.size]
|
||||
s.temp.pos = 0
|
||||
s.sequence = seqBlockHeader
|
||||
fallthrough
|
||||
case seqBlockHeader:
|
||||
if !fillTemp(s, b) {
|
||||
return xzOK
|
||||
}
|
||||
ret = decBlockHeader(s)
|
||||
if ret != xzOK {
|
||||
return ret
|
||||
}
|
||||
s.sequence = seqBlockUncompress
|
||||
fallthrough
|
||||
case seqBlockUncompress:
|
||||
ret = decBlock(s, b)
|
||||
if ret != xzStreamEnd {
|
||||
return ret
|
||||
}
|
||||
s.sequence = seqBlockPadding
|
||||
fallthrough
|
||||
case seqBlockPadding:
|
||||
/*
|
||||
* Size of Compressed Data + Block Padding
|
||||
* must be a multiple of four. We don't need
|
||||
* s->block.compressed for anything else
|
||||
* anymore, so we use it here to test the size
|
||||
* of the Block Padding field.
|
||||
*/
|
||||
for s.block.compressed&3 != 0 {
|
||||
if b.inPos == len(b.in) {
|
||||
return xzOK
|
||||
}
|
||||
if b.in[b.inPos] != 0 {
|
||||
return xzDataError
|
||||
}
|
||||
b.inPos++
|
||||
s.block.compressed++
|
||||
}
|
||||
s.sequence = seqBlockCheck
|
||||
fallthrough
|
||||
case seqBlockCheck:
|
||||
switch s.CheckType {
|
||||
case CheckCRC32, CheckCRC64, CheckSHA256:
|
||||
ret = checkValidate(s, b)
|
||||
if ret != xzStreamEnd {
|
||||
return ret
|
||||
}
|
||||
default:
|
||||
if !checkSkip(s, b) {
|
||||
return xzOK
|
||||
}
|
||||
}
|
||||
s.sequence = seqBlockStart
|
||||
case seqIndex:
|
||||
ret = decIndex(s, b)
|
||||
if ret != xzStreamEnd {
|
||||
return ret
|
||||
}
|
||||
s.sequence = seqIndexPadding
|
||||
fallthrough
|
||||
case seqIndexPadding:
|
||||
for (s.index.size+vliType(b.inPos-s.inStart))&3 != 0 {
|
||||
if b.inPos == len(b.in) {
|
||||
indexUpdate(s, b)
|
||||
return xzOK
|
||||
}
|
||||
if b.in[b.inPos] != 0 {
|
||||
return xzDataError
|
||||
}
|
||||
b.inPos++
|
||||
}
|
||||
/* Finish the CRC32 value and Index size. */
|
||||
indexUpdate(s, b)
|
||||
/* Compare the hashes to validate the Index field. */
|
||||
if !bytes.Equal(
|
||||
s.block.hash.sha256.Sum(nil), s.index.hash.sha256.Sum(nil)) {
|
||||
return xzDataError
|
||||
}
|
||||
s.sequence = seqIndexCRC32
|
||||
fallthrough
|
||||
case seqIndexCRC32:
|
||||
ret = crcValidate(s, b)
|
||||
if ret != xzStreamEnd {
|
||||
return ret
|
||||
}
|
||||
s.temp.buf = s.temp.bufArray[:streamHeaderSize]
|
||||
s.sequence = seqStreamFooter
|
||||
fallthrough
|
||||
case seqStreamFooter:
|
||||
if !fillTemp(s, b) {
|
||||
return xzOK
|
||||
}
|
||||
return decStreamFooter(s)
|
||||
}
|
||||
}
|
||||
/* Never reached */
|
||||
}
|
||||
|
||||
/**
|
||||
* xzDecRun - Run the XZ decoder
|
||||
* @s: Decoder state allocated using xzDecInit
|
||||
* @b: Input and output buffers
|
||||
*
|
||||
* See xzRet for details of return values.
|
||||
*
|
||||
* xzDecRun is a wrapper for decMain to handle some special cases.
|
||||
*
|
||||
* We must return xzBufError when it seems clear that we are not
|
||||
* going to make any progress anymore. This is to prevent the caller
|
||||
* from calling us infinitely when the input file is truncated or
|
||||
* otherwise corrupt. Since zlib-style API allows that the caller
|
||||
* fills the input buffer only when the decoder doesn't produce any
|
||||
* new output, we have to be careful to avoid returning xzBufError
|
||||
* too easily: xzBufError is returned only after the second
|
||||
* consecutive call to xzDecRun that makes no progress.
|
||||
*/
|
||||
func xzDecRun(s *xzDec, b *xzBuf) xzRet {
|
||||
inStart := b.inPos
|
||||
outStart := b.outPos
|
||||
ret := decMain(s, b)
|
||||
if ret == xzOK && inStart == b.inPos && outStart == b.outPos {
|
||||
if s.allowBufError {
|
||||
ret = xzBufError
|
||||
}
|
||||
s.allowBufError = true
|
||||
} else {
|
||||
s.allowBufError = false
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
/**
|
||||
* xzDecInit - Allocate and initialize a XZ decoder state
|
||||
* @dictMax: Maximum size of the LZMA2 dictionary (history buffer) for
|
||||
* decoding. LZMA2 dictionary is always 2^n bytes
|
||||
* or 2^n + 2^(n-1) bytes (the latter sizes are less common
|
||||
* in practice), so other values for dictMax don't make sense.
|
||||
*
|
||||
* dictMax specifies the maximum allowed dictionary size that xzDecRun
|
||||
* may allocate once it has parsed the dictionary size from the stream
|
||||
* headers. This way excessive allocations can be avoided while still
|
||||
* limiting the maximum memory usage to a sane value to prevent running the
|
||||
* system out of memory when decompressing streams from untrusted sources.
|
||||
*
|
||||
* xzDecInit returns a pointer to an xzDec, which is ready to be used with
|
||||
* xzDecRun.
|
||||
*/
|
||||
func xzDecInit(dictMax uint32, header *Header) *xzDec {
|
||||
s := new(xzDec)
|
||||
s.crc32 = crc32.NewIEEE()
|
||||
s.Header = header
|
||||
s.block.hash.sha256 = sha256.New()
|
||||
s.index.hash.sha256 = sha256.New()
|
||||
s.lzma2 = xzDecLZMA2Create(dictMax)
|
||||
xzDecReset(s)
|
||||
return s
|
||||
}
|
||||
|
||||
/**
|
||||
* xzDecReset - Reset an already allocated decoder state
|
||||
* @s: Decoder state allocated using xzDecInit
|
||||
*
|
||||
* This function can be used to reset the decoder state without
|
||||
* reallocating memory with xzDecInit.
|
||||
*/
|
||||
func xzDecReset(s *xzDec) {
|
||||
s.sequence = seqStreamHeader
|
||||
s.allowBufError = false
|
||||
s.pos = 0
|
||||
s.crc32.Reset()
|
||||
s.check = nil
|
||||
s.CheckType = checkUnset
|
||||
s.block.compressed = 0
|
||||
s.block.uncompressed = 0
|
||||
s.block.count = 0
|
||||
s.block.hash.unpadded = 0
|
||||
s.block.hash.uncompressed = 0
|
||||
s.block.hash.sha256.Reset()
|
||||
s.index.sequence = seqIndexCount
|
||||
s.index.size = 0
|
||||
s.index.count = 0
|
||||
s.index.hash.unpadded = 0
|
||||
s.index.hash.uncompressed = 0
|
||||
s.index.hash.sha256.Reset()
|
||||
s.temp.pos = 0
|
||||
s.temp.buf = s.temp.bufArray[:streamHeaderSize]
|
||||
s.chain = nil
|
||||
s.bcjsUsed = 0
|
||||
s.deltasUsed = 0
|
||||
}
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
* XZ decompressor utility functions
|
||||
*
|
||||
* Author: Michael Cross <https://github.com/xi2>
|
||||
*
|
||||
* This file has been put into the public domain.
|
||||
* You can do whatever you want with this file.
|
||||
*/
|
||||
|
||||
package xz
|
||||
|
||||
func getLE32(buf []byte) uint32 {
|
||||
return uint32(buf[0]) |
|
||||
uint32(buf[1])<<8 |
|
||||
uint32(buf[2])<<16 |
|
||||
uint32(buf[3])<<24
|
||||
}
|
||||
|
||||
func getBE32(buf []byte) uint32 {
|
||||
return uint32(buf[0])<<24 |
|
||||
uint32(buf[1])<<16 |
|
||||
uint32(buf[2])<<8 |
|
||||
uint32(buf[3])
|
||||
}
|
||||
|
||||
func putLE32(val uint32, buf []byte) {
|
||||
buf[0] = byte(val)
|
||||
buf[1] = byte(val >> 8)
|
||||
buf[2] = byte(val >> 16)
|
||||
buf[3] = byte(val >> 24)
|
||||
return
|
||||
}
|
||||
|
||||
func putBE32(val uint32, buf []byte) {
|
||||
buf[0] = byte(val >> 24)
|
||||
buf[1] = byte(val >> 16)
|
||||
buf[2] = byte(val >> 8)
|
||||
buf[3] = byte(val)
|
||||
return
|
||||
}
|
||||
|
||||
func putLE64(val uint64, buf []byte) {
|
||||
buf[0] = byte(val)
|
||||
buf[1] = byte(val >> 8)
|
||||
buf[2] = byte(val >> 16)
|
||||
buf[3] = byte(val >> 24)
|
||||
buf[4] = byte(val >> 32)
|
||||
buf[5] = byte(val >> 40)
|
||||
buf[6] = byte(val >> 48)
|
||||
buf[7] = byte(val >> 56)
|
||||
return
|
||||
}
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
/*
|
||||
* XZ decompressor
|
||||
*
|
||||
* Authors: Lasse Collin <lasse.collin@tukaani.org>
|
||||
* Igor Pavlov <http://7-zip.org/>
|
||||
*
|
||||
* Translation to Go: Michael Cross <https://github.com/xi2>
|
||||
*
|
||||
* This file has been put into the public domain.
|
||||
* You can do whatever you want with this file.
|
||||
*/
|
||||
|
||||
package xz
|
||||
|
||||
/* from linux/include/linux/xz.h **************************************/
|
||||
|
||||
/**
|
||||
* xzRet - Return codes
|
||||
* @xzOK: Everything is OK so far. More input or more
|
||||
* output space is required to continue.
|
||||
* @xzStreamEnd: Operation finished successfully.
|
||||
* @xzUnSupportedCheck: Integrity check type is not supported. Decoding
|
||||
* is still possible by simply calling xzDecRun
|
||||
* again.
|
||||
* @xzMemlimitError: A bigger LZMA2 dictionary would be needed than
|
||||
* allowed by the dictMax argument given to
|
||||
* xzDecInit.
|
||||
* @xzFormatError: File format was not recognized (wrong magic
|
||||
* bytes).
|
||||
* @xzOptionsError: This implementation doesn't support the requested
|
||||
* compression options. In the decoder this means
|
||||
* that the header CRC32 matches, but the header
|
||||
* itself specifies something that we don't support.
|
||||
* @xzDataError: Compressed data is corrupt.
|
||||
* @xzBufError: Cannot make any progress.
|
||||
*
|
||||
* xzBufError is returned when two consecutive calls to XZ code cannot
|
||||
* consume any input and cannot produce any new output. This happens
|
||||
* when there is no new input available, or the output buffer is full
|
||||
* while at least one output byte is still pending. Assuming your code
|
||||
* is not buggy, you can get this error only when decoding a
|
||||
* compressed stream that is truncated or otherwise corrupt.
|
||||
*/
|
||||
type xzRet int
|
||||
|
||||
const (
|
||||
xzOK xzRet = iota
|
||||
xzStreamEnd
|
||||
xzUnsupportedCheck
|
||||
xzMemlimitError
|
||||
xzFormatError
|
||||
xzOptionsError
|
||||
xzDataError
|
||||
xzBufError
|
||||
)
|
||||
|
||||
/**
|
||||
* xzBuf - Passing input and output buffers to XZ code
|
||||
* @in: Input buffer.
|
||||
* @inPos: Current position in the input buffer. This must not exceed
|
||||
* input buffer size.
|
||||
* @out: Output buffer.
|
||||
* @outPos: Current position in the output buffer. This must not exceed
|
||||
* output buffer size.
|
||||
*
|
||||
* Only the contents of the output buffer from out[outPos] onward, and
|
||||
* the variables inPos and outPos are modified by the XZ code.
|
||||
*/
|
||||
type xzBuf struct {
|
||||
in []byte
|
||||
inPos int
|
||||
out []byte
|
||||
outPos int
|
||||
}
|
||||
|
||||
/* All XZ filter IDs */
|
||||
type xzFilterID int64
|
||||
|
||||
const (
|
||||
idDelta xzFilterID = 0x03
|
||||
idBCJX86 xzFilterID = 0x04
|
||||
idBCJPowerPC xzFilterID = 0x05
|
||||
idBCJIA64 xzFilterID = 0x06
|
||||
idBCJARM xzFilterID = 0x07
|
||||
idBCJARMThumb xzFilterID = 0x08
|
||||
idBCJSPARC xzFilterID = 0x09
|
||||
idLZMA2 xzFilterID = 0x21
|
||||
)
|
||||
|
||||
// CheckID is the type of the data integrity check in an XZ stream
|
||||
// calculated from the uncompressed data.
|
||||
type CheckID int
|
||||
|
||||
func (id CheckID) String() string {
|
||||
switch id {
|
||||
case CheckNone:
|
||||
return "None"
|
||||
case CheckCRC32:
|
||||
return "CRC32"
|
||||
case CheckCRC64:
|
||||
return "CRC64"
|
||||
case CheckSHA256:
|
||||
return "SHA256"
|
||||
default:
|
||||
return "Unknown"
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
CheckNone CheckID = 0x00
|
||||
CheckCRC32 CheckID = 0x01
|
||||
CheckCRC64 CheckID = 0x04
|
||||
CheckSHA256 CheckID = 0x0A
|
||||
checkMax CheckID = 0x0F
|
||||
checkUnset CheckID = -1
|
||||
)
|
||||
|
||||
// An XZ stream contains a stream header which holds information about
|
||||
// the stream. That information is exposed as fields of the
|
||||
// Reader. Currently it contains only the stream's data integrity
|
||||
// check type.
|
||||
type Header struct {
|
||||
CheckType CheckID // type of the stream's data integrity check
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
// Package xz implements XZ decompression natively in Go.
|
||||
//
|
||||
// Usage
|
||||
//
|
||||
// For ease of use, this package is designed to have a similar API to
|
||||
// compress/gzip. See the examples for further details.
|
||||
//
|
||||
// Implementation
|
||||
//
|
||||
// This package is a translation from C to Go of XZ Embedded
|
||||
// (http://tukaani.org/xz/embedded.html) with enhancements made so as
|
||||
// to implement all mandatory and optional parts of the XZ file format
|
||||
// specification v1.0.4. It supports all filters and block check
|
||||
// types, supports multiple streams, and performs index verification
|
||||
// using SHA-256 as recommended by the specification.
|
||||
//
|
||||
// Speed
|
||||
//
|
||||
// On the author's Intel Ivybridge i5, decompression speed is about
|
||||
// half that of the standard XZ Utils (tested with a recent linux
|
||||
// kernel tarball).
|
||||
//
|
||||
// Thanks
|
||||
//
|
||||
// Thanks are due to Lasse Collin and Igor Pavlov, the authors of XZ
|
||||
// Embedded, on whose code package xz is based. It would not exist
|
||||
// without their decision to allow others to modify and reuse their
|
||||
// code.
|
||||
//
|
||||
// Bug reports
|
||||
//
|
||||
// For bug reports relating to this package please contact the author
|
||||
// through https://github.com/xi2/xz/issues, and not the authors of XZ
|
||||
// Embedded.
|
||||
package xz
|
||||
+256
@@ -0,0 +1,256 @@
|
||||
/*
|
||||
* Package xz Go Reader API
|
||||
*
|
||||
* Author: Michael Cross <https://github.com/xi2>
|
||||
*
|
||||
* This file has been put into the public domain.
|
||||
* You can do whatever you want with this file.
|
||||
*/
|
||||
|
||||
package xz
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
|
||||
// Package specific errors.
|
||||
var (
|
||||
ErrUnsupportedCheck = errors.New("xz: integrity check type not supported")
|
||||
ErrMemlimit = errors.New("xz: LZMA2 dictionary size exceeds max")
|
||||
ErrFormat = errors.New("xz: file format not recognized")
|
||||
ErrOptions = errors.New("xz: compression options not supported")
|
||||
ErrData = errors.New("xz: data is corrupt")
|
||||
ErrBuf = errors.New("xz: data is truncated or corrupt")
|
||||
)
|
||||
|
||||
// DefaultDictMax is the default maximum dictionary size in bytes used
|
||||
// by the decoder. This value is sufficient to decompress files
|
||||
// created with XZ Utils "xz -9".
|
||||
const DefaultDictMax = 1 << 26 // 64 MiB
|
||||
|
||||
// inBufSize is the input buffer size used by the decoder.
|
||||
const inBufSize = 1 << 13 // 8 KiB
|
||||
|
||||
// A Reader is an io.Reader that can be used to retrieve uncompressed
|
||||
// data from an XZ file.
|
||||
//
|
||||
// In general, an XZ file can be a concatenation of other XZ
|
||||
// files. Reads from the Reader return the concatenation of the
|
||||
// uncompressed data of each.
|
||||
type Reader struct {
|
||||
Header
|
||||
r io.Reader // the wrapped io.Reader
|
||||
multistream bool // true if reader is in multistream mode
|
||||
rEOF bool // true after io.EOF received on r
|
||||
dEOF bool // true after decoder has completed
|
||||
padding int // bytes of stream padding read (or -1)
|
||||
in [inBufSize]byte // backing array for buf.in
|
||||
buf *xzBuf // decoder input/output buffers
|
||||
dec *xzDec // decoder state
|
||||
err error // the result of the last decoder call
|
||||
}
|
||||
|
||||
// NewReader creates a new Reader reading from r. The decompressor
|
||||
// will use an LZMA2 dictionary size up to dictMax bytes in
|
||||
// size. Passing a value of zero sets dictMax to DefaultDictMax. If
|
||||
// an individual XZ stream requires a dictionary size greater than
|
||||
// dictMax in order to decompress, Read will return ErrMemlimit.
|
||||
//
|
||||
// If NewReader is passed a value of nil for r then a Reader is
|
||||
// created such that all read attempts will return io.EOF. This is
|
||||
// useful if you just want to allocate memory for a Reader which will
|
||||
// later be initialized with Reset.
|
||||
//
|
||||
// Due to internal buffering, the Reader may read more data than
|
||||
// necessary from r.
|
||||
func NewReader(r io.Reader, dictMax uint32) (*Reader, error) {
|
||||
if dictMax == 0 {
|
||||
dictMax = DefaultDictMax
|
||||
}
|
||||
z := &Reader{
|
||||
r: r,
|
||||
multistream: true,
|
||||
padding: -1,
|
||||
buf: &xzBuf{},
|
||||
}
|
||||
if r == nil {
|
||||
z.rEOF, z.dEOF = true, true
|
||||
}
|
||||
z.dec = xzDecInit(dictMax, &z.Header)
|
||||
var err error
|
||||
if r != nil {
|
||||
_, err = z.Read(nil) // read stream header
|
||||
}
|
||||
return z, err
|
||||
}
|
||||
|
||||
// decode is a wrapper around xzDecRun that additionally handles
|
||||
// stream padding. It treats the padding as a kind of stream that
|
||||
// decodes to nothing.
|
||||
//
|
||||
// When decoding padding, z.padding >= 0
|
||||
// When decoding a real stream, z.padding == -1
|
||||
func (z *Reader) decode() (ret xzRet) {
|
||||
if z.padding >= 0 {
|
||||
// read all padding in input buffer
|
||||
for z.buf.inPos < len(z.buf.in) &&
|
||||
z.buf.in[z.buf.inPos] == 0 {
|
||||
z.buf.inPos++
|
||||
z.padding++
|
||||
}
|
||||
switch {
|
||||
case z.buf.inPos == len(z.buf.in) && z.rEOF:
|
||||
// case: out of padding. no more input data available
|
||||
if z.padding%4 != 0 {
|
||||
ret = xzDataError
|
||||
} else {
|
||||
ret = xzStreamEnd
|
||||
}
|
||||
case z.buf.inPos == len(z.buf.in):
|
||||
// case: read more padding next loop iteration
|
||||
ret = xzOK
|
||||
default:
|
||||
// case: out of padding. more input data available
|
||||
if z.padding%4 != 0 {
|
||||
ret = xzDataError
|
||||
} else {
|
||||
xzDecReset(z.dec)
|
||||
ret = xzStreamEnd
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ret = xzDecRun(z.dec, z.buf)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (z *Reader) Read(p []byte) (n int, err error) {
|
||||
// restore err
|
||||
err = z.err
|
||||
// set decoder output buffer to p
|
||||
z.buf.out = p
|
||||
z.buf.outPos = 0
|
||||
for {
|
||||
// update n
|
||||
n = z.buf.outPos
|
||||
// if last call to decoder ended with an error, return that error
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
// if decoder has finished, return with err == io.EOF
|
||||
if z.dEOF {
|
||||
err = io.EOF
|
||||
break
|
||||
}
|
||||
// if p full, return with err == nil, unless we have not yet
|
||||
// read the stream header with Read(nil)
|
||||
if n == len(p) && z.CheckType != checkUnset {
|
||||
break
|
||||
}
|
||||
// if needed, read more data from z.r
|
||||
if z.buf.inPos == len(z.buf.in) && !z.rEOF {
|
||||
rn, e := z.r.Read(z.in[:])
|
||||
if e != nil && e != io.EOF {
|
||||
// read error
|
||||
err = e
|
||||
break
|
||||
}
|
||||
if e == io.EOF {
|
||||
z.rEOF = true
|
||||
}
|
||||
// set new input buffer in z.buf
|
||||
z.buf.in = z.in[:rn]
|
||||
z.buf.inPos = 0
|
||||
}
|
||||
// decode more data
|
||||
ret := z.decode()
|
||||
switch ret {
|
||||
case xzOK:
|
||||
// no action needed
|
||||
case xzStreamEnd:
|
||||
if z.padding >= 0 {
|
||||
z.padding = -1
|
||||
if !z.multistream || z.rEOF {
|
||||
z.dEOF = true
|
||||
}
|
||||
} else {
|
||||
z.padding = 0
|
||||
}
|
||||
case xzUnsupportedCheck:
|
||||
err = ErrUnsupportedCheck
|
||||
case xzMemlimitError:
|
||||
err = ErrMemlimit
|
||||
case xzFormatError:
|
||||
err = ErrFormat
|
||||
case xzOptionsError:
|
||||
err = ErrOptions
|
||||
case xzDataError:
|
||||
err = ErrData
|
||||
case xzBufError:
|
||||
err = ErrBuf
|
||||
}
|
||||
// save err
|
||||
z.err = err
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Multistream controls whether the reader is operating in multistream
|
||||
// mode.
|
||||
//
|
||||
// If enabled (the default), the Reader expects the input to be a
|
||||
// sequence of XZ streams, possibly interspersed with stream padding,
|
||||
// which it reads one after another. The effect is that the
|
||||
// concatenation of a sequence of XZ streams or XZ files is
|
||||
// treated as equivalent to the compressed result of the concatenation
|
||||
// of the sequence. This is standard behaviour for XZ readers.
|
||||
//
|
||||
// Calling Multistream(false) disables this behaviour; disabling the
|
||||
// behaviour can be useful when reading file formats that distinguish
|
||||
// individual XZ streams. In this mode, when the Reader reaches the
|
||||
// end of the stream, Read returns io.EOF. To start the next stream,
|
||||
// call z.Reset(nil) followed by z.Multistream(false). If there is no
|
||||
// next stream, z.Reset(nil) will return io.EOF.
|
||||
func (z *Reader) Multistream(ok bool) {
|
||||
z.multistream = ok
|
||||
}
|
||||
|
||||
// Reset, for non-nil values of io.Reader r, discards the Reader z's
|
||||
// state and makes it equivalent to the result of its original state
|
||||
// from NewReader, but reading from r instead. This permits reusing a
|
||||
// Reader rather than allocating a new one.
|
||||
//
|
||||
// If you wish to leave r unchanged use z.Reset(nil). This keeps r
|
||||
// unchanged and ensures internal buffering is preserved. If the
|
||||
// Reader was at the end of a stream it is then ready to read any
|
||||
// follow on streams. If there are no follow on streams z.Reset(nil)
|
||||
// returns io.EOF. If the Reader was not at the end of a stream then
|
||||
// z.Reset(nil) does nothing.
|
||||
func (z *Reader) Reset(r io.Reader) error {
|
||||
switch {
|
||||
case r == nil:
|
||||
z.multistream = true
|
||||
if !z.dEOF {
|
||||
return nil
|
||||
}
|
||||
if z.rEOF {
|
||||
return io.EOF
|
||||
}
|
||||
z.dEOF = false
|
||||
_, err := z.Read(nil) // read stream header
|
||||
return err
|
||||
default:
|
||||
z.r = r
|
||||
z.multistream = true
|
||||
z.rEOF = false
|
||||
z.dEOF = false
|
||||
z.padding = -1
|
||||
z.buf.in = nil
|
||||
z.buf.inPos = 0
|
||||
xzDecReset(z.dec)
|
||||
z.err = nil
|
||||
_, err := z.Read(nil) // read stream header
|
||||
return err
|
||||
}
|
||||
}
|
||||
Vendored
+3
@@ -83,6 +83,9 @@ github.com/nats-io/nuid
|
||||
github.com/novox/mesh-host/internal/declaration
|
||||
github.com/novox/mesh-host/rootsearch
|
||||
github.com/novox/mesh-host/validate
|
||||
# github.com/therootcompany/xz v1.0.1
|
||||
## explicit; go 1.15
|
||||
github.com/therootcompany/xz
|
||||
# golang.org/x/crypto v0.57.0
|
||||
## explicit; go 1.26.0
|
||||
golang.org/x/crypto/acme
|
||||
|
||||
Reference in New Issue
Block a user