Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 jochen
|
24347ea5a0 | ||
|
jochen
|
a3bf5da9b1 |
@@ -6,5 +6,3 @@
|
||||
/mesh-controller
|
||||
/postgres-provisioner
|
||||
/redis-provisioner
|
||||
# And where `go build` in cmd/mesh-controller leaves it (novox/hq issue 233).
|
||||
/cmd/mesh-controller/mesh-controller
|
||||
|
||||
@@ -16,7 +16,8 @@ RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' -o /mesh-builder ./cmd/mes
|
||||
FROM ${ALPINE_BASE}
|
||||
# git to clone what it is asked to build, and the docker client to build and push it. The daemon
|
||||
# is the machine's, reached through its socket — a build machine shares the runtime it was given
|
||||
# rather than running one inside itself.
|
||||
RUN apk add --no-cache git docker-cli
|
||||
# rather than running one inside itself. xz to read an upstream release an archive is pinned to
|
||||
# (novox/hq ADR 0316): Go's standard library reads gzip and bzip2 and not xz.
|
||||
RUN apk add --no-cache git docker-cli xz
|
||||
COPY --from=build /mesh-builder /usr/local/bin/mesh-builder
|
||||
ENTRYPOINT ["/usr/local/bin/mesh-builder"]
|
||||
|
||||
@@ -256,13 +256,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
v, err = builder.Check(building, builder.Command, checkSpecOf(request), workspace, registry, forgeFrom(), say)
|
||||
if err == nil {
|
||||
result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report,
|
||||
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo),
|
||||
Head: v.Head, Trunk: v.Trunk, Beside: v.Beside}
|
||||
// What each of a group's heads was judged as (novox/hq ADR 0323 §6).
|
||||
for _, m := range v.Members {
|
||||
result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo,
|
||||
Commit: m.Head, Trunk: m.Trunk})
|
||||
}
|
||||
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo)}
|
||||
}
|
||||
} else if err == nil {
|
||||
// The package-registry credential is a build input, so it is resolved before the clone: a
|
||||
@@ -360,7 +354,7 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
|
||||
}
|
||||
for _, m := range c.Members {
|
||||
spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository,
|
||||
Ref: m.Ref, Paths: m.Paths, Base: m.Base})
|
||||
Ref: m.Ref, Paths: m.Paths})
|
||||
}
|
||||
return spec
|
||||
}
|
||||
|
||||
@@ -1,118 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// kindApplyFailing is S21's kind: a machine whose declarations have been refused, or applied only in part,
|
||||
// for longer than one apply interval (novox/hq issue 233).
|
||||
const kindApplyFailing = "apply-failing"
|
||||
|
||||
// applyEvery is how often a node-engine applies what it was last told when nothing new arrives: its
|
||||
// ReconcileEvery. A failure still there after one more of them is not one the next apply clears.
|
||||
const applyEvery = 5 * time.Minute
|
||||
|
||||
// packageManagerKey is how a node-engine names a package manager that does not answer among what failed
|
||||
// (mesh-host's probe.go): "the package manager (pacman)".
|
||||
const packageManagerKey = "the package manager ("
|
||||
|
||||
// watchApplyFailing is S21: **a machine that does not apply what it was sent is said by the mesh, not only
|
||||
// to someone who asks for `status`** (novox/hq issue 233, issue 187's family). A workstation refused every
|
||||
// declaration for an hour and a half because its package manager did not answer, and the mesh's only word
|
||||
// on it was a line in `status`. Raised once the machine's first report of this failure is older than one apply
|
||||
// interval **while the machine is heard** — judged by time, not by how many reports said it: a node-engine
|
||||
// says a re-apply only when how it went changed, so the same failure is one report however long it lasts.
|
||||
// Names the refusal's or the failure's words; cleared by the clean apply the node-engine then says. Not while
|
||||
// the machine is silent, asleep or not known: S1 says silent, and asleep is not lost.
|
||||
func watchApplyFailing(f *signalFacts) []conditions.Observation {
|
||||
heard := map[string]bool{}
|
||||
for _, m := range f.machines {
|
||||
if !m.asleep() && !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) {
|
||||
heard[m.name] = true
|
||||
}
|
||||
}
|
||||
var out []conditions.Observation
|
||||
for _, d := range f.doing {
|
||||
if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] {
|
||||
continue
|
||||
}
|
||||
out = append(out, applyFailing(d, f.now))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// applyFailing is one machine's S21 condition.
|
||||
func applyFailing(d inventory.Doing, now time.Time) conditions.Observation {
|
||||
m := d.Node
|
||||
since := d.Since.UTC().Format("2006-01-02 15:04 MST")
|
||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m,
|
||||
Severity: conditions.Warning, Said: doingWords(d),
|
||||
Resolved: m + " applies what it is sent again"}
|
||||
how := "refused its declaration whole"
|
||||
if d.Outcome != inventory.OutcomeRefused {
|
||||
how = "applied its declaration only in part"
|
||||
}
|
||||
o.Summary = fmt.Sprintf("%s has %s since %s (bound %s): %s", m, how, since, ago(applyEvery), failedNames(d))
|
||||
o.Headline = m + " is not applying its update"
|
||||
o.Explanation = fmt.Sprintf("%s has %s for %s, and trying again has not helped. The details say what fails.",
|
||||
m, plainHow(d), humanDuration(now.Sub(*d.Since)))
|
||||
if tool := unansweringTool(d); tool != "" {
|
||||
o.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent "+
|
||||
"is applied, since %s (bound %s)", m, tool, since, ago(applyEvery))
|
||||
o.Headline = fmt.Sprintf("%s cannot install packages", m)
|
||||
o.Explanation = fmt.Sprintf("The package manager on %s (%s) has not answered for %s, so no package reaches "+
|
||||
"it. Everything else is still applied, so an update that repairs it can land; if none does, it needs a "+
|
||||
"look on the machine.", m, tool, humanDuration(now.Sub(*d.Since)))
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
// plainHow is how a machine failed, as the operator reads it.
|
||||
func plainHow(d inventory.Doing) string {
|
||||
if d.Outcome == inventory.OutcomeRefused {
|
||||
return "refused every update"
|
||||
}
|
||||
return "applied its updates only in part"
|
||||
}
|
||||
|
||||
// unansweringTool is the package manager a machine said does not answer, or "".
|
||||
func unansweringTool(d inventory.Doing) string {
|
||||
for _, r := range d.Failed {
|
||||
if rest, ok := strings.CutPrefix(r.ID, packageManagerKey); ok {
|
||||
return strings.TrimSuffix(rest, ")")
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// failedNames is what failed, by name: the refusal, or the failed resources' ids.
|
||||
func failedNames(d inventory.Doing) string {
|
||||
if d.Outcome == inventory.OutcomeRefused {
|
||||
return "refused"
|
||||
}
|
||||
ids := make([]string, 0, len(d.Failed))
|
||||
for _, r := range d.Failed {
|
||||
ids = append(ids, r.ID)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return "failed " + strings.Join(ids, ", ")
|
||||
}
|
||||
|
||||
// doingWords is the machine's own words on what failed: the evidence, kept inside the mesh.
|
||||
func doingWords(d inventory.Doing) string {
|
||||
if d.Outcome == inventory.OutcomeRefused {
|
||||
return "refused: " + oneLine(d.Refused)
|
||||
}
|
||||
var parts []string
|
||||
for _, r := range d.Failed {
|
||||
parts = append(parts, r.ID+": "+oneLine(r.Error))
|
||||
}
|
||||
sort.Strings(parts)
|
||||
return strings.Join(parts, "; ")
|
||||
}
|
||||
@@ -1,123 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// unansweringPackageManager is anchor's last report as the workstation's read on 2026-10-04, once its
|
||||
// node-engine applies the rest (novox/hq issue 233): the package failed and the package manager did not answer,
|
||||
// the same failure on times reports since for before now.
|
||||
func unansweringPackageManager(now time.Time, forHowLong time.Duration, times int) inventory.Doing {
|
||||
since := now.Add(-forHowLong)
|
||||
return inventory.Doing{Node: "anchor", Outcome: inventory.OutcomeFailed, At: now.Add(-time.Second),
|
||||
Since: &since, Times: times, Applied: 41, Failed: []inventory.FailedResource{
|
||||
{ID: "apply", Error: `applying "pacman.base": nothing can be said about "base": this is the arch host and ` +
|
||||
"pacman does not answer here. Either this machine is not Arch, or its package database is broken: " +
|
||||
"pacman exited 1: error: config file /etc/pacman.conf could not be read: No such file or directory"},
|
||||
{ID: "the package manager (pacman)", Error: "pacman does not answer here, so every package in this " +
|
||||
"declaration fails and the rest of it is applied (novox/hq issue 233): pacman exited 1: error: config " +
|
||||
"file /etc/pacman.conf could not be read: No such file or directory"}}}
|
||||
}
|
||||
|
||||
// **A machine whose package manager does not answer is raised by the mesh itself, naming the tool and the
|
||||
// probe's words** (novox/hq issue 233): the words in the evidence, the path kept out of the summary.
|
||||
func TestAPackageManagerThatDoesNotAnswerIsRaisedNamingItsWords(t *testing.T) {
|
||||
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
f.doing = []inventory.Doing{unansweringPackageManager(now, 90*time.Minute, 18)}
|
||||
got := watchApplyFailing(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("raised %d conditions, want one: %+v", len(got), got)
|
||||
}
|
||||
o := got[0]
|
||||
if o.Key() != "machine.anchor.apply-failing" || o.Machine != "anchor" {
|
||||
t.Errorf("the condition is not the machine's: %s", o.Key())
|
||||
}
|
||||
if !strings.Contains(o.Summary, "pacman does not answer") || !strings.Contains(o.Summary, "since 2026-10-04 12:30") {
|
||||
t.Errorf("the summary does not name the tool and since when: %s", o.Summary)
|
||||
}
|
||||
if strings.Contains(o.Summary, "/etc/") {
|
||||
t.Errorf("the summary carries a path, which belongs in the evidence: %s", o.Summary)
|
||||
}
|
||||
if !strings.Contains(o.Said, "could not be read") {
|
||||
t.Errorf("the evidence does not carry the probe's words: %s", o.Said)
|
||||
}
|
||||
if !strings.Contains(o.Headline, "cannot install packages") || o.Explanation == "" || o.Resolved == "" {
|
||||
t.Errorf("the plain words do not say it: %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
// **A whole refusal is raised too**, with the node-engine's words: what the workstation's report said before the
|
||||
// node-engine applied the rest.
|
||||
func TestARefusalThatGoesOnIsRaised(t *testing.T) {
|
||||
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
|
||||
since := now.Add(-time.Hour)
|
||||
f := calm(now)
|
||||
f.doing = []inventory.Doing{{Node: "anchor", Outcome: inventory.OutcomeRefused, Since: &since, Times: 12,
|
||||
Refused: "this is the arch host and pacman does not answer here", At: now.Add(-time.Minute)}}
|
||||
got := watchApplyFailing(f)
|
||||
if len(got) != 1 || !strings.Contains(got[0].Summary, "refused its declaration whole") ||
|
||||
!strings.Contains(got[0].Said, "pacman does not answer") {
|
||||
t.Fatalf("a refusal that went on for an hour was not raised with its words: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **Judged by time while the machine is heard, not by how many reports said it** (review of novox/hq issue 233):
|
||||
// a node-engine says the same failure once, so one report an apply interval old is a fault that goes on; one
|
||||
// younger is not yet; and a silent or sleeping machine is S1's.
|
||||
func TestOneReportOlderThanAnApplyIsRaisedAndAQuietMachineIsNot(t *testing.T) {
|
||||
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
f.doing = []inventory.Doing{unansweringPackageManager(now, 6*time.Minute, 1)}
|
||||
if got := watchApplyFailing(f); len(got) != 1 {
|
||||
t.Errorf("one report of a failure six minutes old was not raised: %+v", got)
|
||||
}
|
||||
f.doing = []inventory.Doing{unansweringPackageManager(now, 4*time.Minute, 1)}
|
||||
if got := watchApplyFailing(f); len(got) != 0 {
|
||||
t.Errorf("a failure four minutes old was raised before one apply could clear it: %+v", got)
|
||||
}
|
||||
f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)}
|
||||
f.machines[0].lastHeard = now.Add(-time.Hour)
|
||||
if got := watchApplyFailing(f); len(got) != 0 {
|
||||
t.Errorf("a silent machine was raised for its last report: %+v", got)
|
||||
}
|
||||
f = calm(now)
|
||||
f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)}
|
||||
f.doing[0].Node = "a-machine-the-mesh-does-not-know"
|
||||
if got := watchApplyFailing(f); len(got) != 0 {
|
||||
t.Errorf("a machine not heard of was raised: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A failed re-apply followed by a clean one clears S21** (review of novox/hq issue 233): the node-engine says
|
||||
// the clean re-apply, the store keeps it as applied with no failure since, and the condition the failure raised
|
||||
// is cleared through the keeper on the next tick.
|
||||
func TestACleanReapplyAfterAFailedOneClearsTheCondition(t *testing.T) {
|
||||
now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||
defer k.Close(context.Background())
|
||||
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
||||
|
||||
failing := calm(now)
|
||||
failing.doing = []inventory.Doing{unansweringPackageManager(now, 20*time.Minute, 1)}
|
||||
w.see(t.Context(), failing)
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil || len(open) != 1 || open[0].Kind != kindApplyFailing {
|
||||
t.Fatalf("the failed re-apply raised %+v (%v), want apply-failing", open, err)
|
||||
}
|
||||
|
||||
clean := calm(now.Add(5 * time.Minute))
|
||||
clean.doing = nil // the clean re-apply recorded: NotDoingWhatTheyWereTold no longer lists the machine
|
||||
w.see(t.Context(), clean)
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("the condition stayed open after the clean re-apply: %+v", open)
|
||||
}
|
||||
}
|
||||
@@ -161,7 +161,9 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
return u, err == nil
|
||||
})
|
||||
fmt.Print(planText(plan))
|
||||
direct := directVerdict(p, plan)
|
||||
direct := link.Checked{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Commit: p.Commit,
|
||||
ID: link.NewBuildID(time.Now()), Verdict: "pass", Summary: noModuleTouched,
|
||||
Gate: &link.CheckLayer{Verdict: "pass", Summary: noModuleTouched}, Plan: &plan}
|
||||
switch {
|
||||
case !scope.gated() && !scope.Mesh:
|
||||
fmt.Printf("%s/%s#%d (%.8s): %s, and the repository is not the mesh's: said, nothing run\n",
|
||||
@@ -199,24 +201,6 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// directVerdict is the verdict of a change no build judges — it touches no module, and either the repository is not
|
||||
// the mesh's or the head holds no merge-check.sh. **It names the trunk commit it holds for** (novox/hq ADR 0323 §3):
|
||||
// the base branch's head the forge announced with the head, so the merge tool can weigh it against the trunk's head
|
||||
// and does not refuse it for ever (a verdict naming no trunk is never the trunk's head's). Nothing was built to
|
||||
// judge: the trunk commit is the one the forge said the change was announced against.
|
||||
func directVerdict(p link.PullUpdated, plan link.ChangePlan) link.Checked {
|
||||
trunk := p.BaseSHA
|
||||
if len(trunk) > 12 {
|
||||
trunk = trunk[:12]
|
||||
}
|
||||
if trunk != "" {
|
||||
plan.Trunk = trunk
|
||||
}
|
||||
return link.Checked{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Commit: p.Commit,
|
||||
ID: link.NewBuildID(time.Now()), Verdict: "pass", Summary: noModuleTouched,
|
||||
Gate: &link.CheckLayer{Verdict: "pass", Summary: noModuleTouched}, Plan: &plan, Trunk: trunk}
|
||||
}
|
||||
|
||||
// checkRequestFor is the ask for one pull request's head: the repository as the mesh clones it, the head,
|
||||
// and what is read beside it.
|
||||
func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scope checkScope,
|
||||
@@ -379,9 +363,6 @@ func checkedOf(result link.BuildResult) link.Checked {
|
||||
case result.Check != nil:
|
||||
c.Verdict, c.Summary, c.Report = result.Check.Verdict, result.Check.Summary, result.Check.Report
|
||||
c.Gate, c.RepoCheck = result.Check.Gate, result.Check.RepoCheck
|
||||
// What was judged (novox/hq ADR 0323 §3): the head merged into the trunk commit, and the commits of the
|
||||
// repositories read beside it.
|
||||
c.Trunk, c.Beside, c.Took = result.Check.Trunk, result.Check.Beside, result.Check.Took
|
||||
if c.Gate == nil {
|
||||
// A build seat from before the layers: its verdict is the gate's.
|
||||
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
||||
@@ -407,20 +388,14 @@ func checkedOf(result link.BuildResult) link.Checked {
|
||||
}
|
||||
if result.Checked != nil {
|
||||
c.Plan = result.Checked.Plan
|
||||
if c.Plan != nil && c.Trunk != "" {
|
||||
// The kept delivery plan records the trunk commit with its other inputs (ADR 0323 §3).
|
||||
plan := *c.Plan
|
||||
plan.Trunk = c.Trunk
|
||||
c.Plan = &plan
|
||||
}
|
||||
// A delivery group's composed check (novox/hq ADR 0239): every head it judged, this one first.
|
||||
if g := result.Checked; g.Group != "" {
|
||||
c.Group = g.Group
|
||||
c.Members = append(c.Members, link.CheckedMember{Owner: g.Owner, Repo: g.Repo, Number: g.Number,
|
||||
Commit: result.Ref, Trunk: c.Trunk})
|
||||
Commit: result.Ref})
|
||||
for _, m := range g.Members {
|
||||
c.Members = append(c.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo, Number: m.Number,
|
||||
Commit: m.Ref, Trunk: memberTrunk(result.Check, m)})
|
||||
Commit: m.Ref})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -435,20 +410,6 @@ func checkedOf(result link.BuildResult) link.Checked {
|
||||
return c
|
||||
}
|
||||
|
||||
// memberTrunk is the trunk commit the build seat merged one of a group's heads into (novox/hq ADR 0323 §6), empty
|
||||
// when it said none.
|
||||
func memberTrunk(outcome *link.CheckOutcome, m link.GroupMember) string {
|
||||
if outcome == nil {
|
||||
return ""
|
||||
}
|
||||
for _, j := range outcome.Members {
|
||||
if strings.EqualFold(j.Owner, m.Owner) && strings.EqualFold(j.Repo, m.Repo) && strings.HasPrefix(m.Ref, j.Commit) {
|
||||
return j.Trunk
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func prefixedAll(prefix string, items []string) []string {
|
||||
out := make([]string, 0, len(items))
|
||||
for _, i := range items {
|
||||
@@ -463,12 +424,8 @@ func sayChecked(ctx context.Context, c link.Checked) {
|
||||
if c.RepoCheck != nil {
|
||||
repo = strings.ToUpper(c.RepoCheck.Verdict) + " — " + c.RepoCheck.Summary
|
||||
}
|
||||
judged := ""
|
||||
if c.Trunk != "" {
|
||||
judged = " merged into trunk " + c.Trunk
|
||||
}
|
||||
fmt.Printf("%s: %s/%s#%d at %.8s%s checked on %s: gate %s — %s; repository %s\n", c.ID, c.Owner, c.Repo, c.Number,
|
||||
c.Commit, judged, orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary, repo)
|
||||
fmt.Printf("%s: %s/%s#%d at %.8s checked on %s: gate %s — %s; repository %s\n", c.ID, c.Owner, c.Repo, c.Number,
|
||||
c.Commit, orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary, repo)
|
||||
if checkEvents == nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -210,81 +210,3 @@ func TestABuildOffTheTrunkIsNeverPublished(t *testing.T) {
|
||||
t.Errorf("a build seat that said nothing was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A verdict names the trunk commit it judged** (novox/hq ADR 0323 §3, issue 341): the controller's `checked`
|
||||
// carries the head and the trunk commit the build seat merged it into, the commits of the repositories read beside
|
||||
// it, how long it took, and the kept delivery plan records the trunk commit with its other inputs. A group's
|
||||
// composed check names, per member, the trunk commit that head was merged into (§6).
|
||||
func TestAVerdictNamesTheTrunkCommitItJudged(t *testing.T) {
|
||||
plan := &link.ChangePlan{Repository: "novox/mesh-host", Base: "main", Head: "dfb1415aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
Summary: "builds mesh-host"}
|
||||
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Number: 66, Base: "main", Plan: plan}
|
||||
c := checkedOf(link.BuildResult{ID: "b", Ref: "dfb1415aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Checked: asked,
|
||||
Check: &link.CheckOutcome{Verdict: "pass", Took: "4m12s", Head: "dfb1415aaaaa", Trunk: "d28ca95bbbbb",
|
||||
Gate: &link.CheckLayer{Verdict: "pass"}, RepoCheck: &link.CheckLayer{Verdict: "pass"},
|
||||
Beside: map[string]string{"mesh-catalog": "0123456789ab", "mesh-lab": "ba9876543210"}}})
|
||||
if c.Trunk != "d28ca95bbbbb" || c.Commit != "dfb1415aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" {
|
||||
t.Fatalf("the verdict names head %s and trunk %q", c.Commit, c.Trunk)
|
||||
}
|
||||
if c.Beside["mesh-catalog"] != "0123456789ab" || c.Beside["mesh-lab"] != "ba9876543210" {
|
||||
t.Fatalf("the verdict names the repositories read beside it as %v", c.Beside)
|
||||
}
|
||||
if c.Took != "4m12s" {
|
||||
t.Fatalf("the verdict says it took %q", c.Took)
|
||||
}
|
||||
if c.Plan == nil || c.Plan.Trunk != "d28ca95bbbbb" || c.Plan.Head != plan.Head {
|
||||
t.Fatalf("the kept plan records %+v", c.Plan)
|
||||
}
|
||||
if plan.Trunk != "" {
|
||||
t.Fatal("the plan the controller asked with was changed: the verdict's is a copy")
|
||||
}
|
||||
|
||||
group := &link.CheckRequest{Owner: "novox", Repo: "mesh-controller", Number: 7, Base: "main", Group: "feat/x",
|
||||
Members: []link.GroupMember{{Owner: "novox", Repo: "mesh-host", Number: 8, Ref: "cccccccccccc0000", Base: "main"}}}
|
||||
c = checkedOf(link.BuildResult{ID: "g", Ref: "aaaaaaaaaaaa0000", Checked: group, Check: &link.CheckOutcome{
|
||||
Verdict: "pass", Head: "aaaaaaaaaaaa", Trunk: "111111111111", Gate: &link.CheckLayer{Verdict: "pass"},
|
||||
Members: []link.CheckedMember{{Owner: "novox", Repo: "mesh-host", Commit: "cccccccccccc", Trunk: "222222222222"}}}})
|
||||
if len(c.Members) != 2 || c.Members[0].Trunk != "111111111111" || c.Members[1].Trunk != "222222222222" ||
|
||||
c.Members[1].Commit != "cccccccccccc0000" {
|
||||
t.Fatalf("the composed check names its members %+v", c.Members)
|
||||
}
|
||||
}
|
||||
|
||||
// **The merge made for a check is never registered** (novox/hq ADR 0323 §2): a build of it — a commit on no branch
|
||||
// of the forge — is refused, naming the trunk. The rule of ADR 0238 §1 stands unchanged; this holds it for the
|
||||
// merge commit a check makes.
|
||||
func TestABuildOfACheckMergeIsRefusedNamingTheTrunk(t *testing.T) {
|
||||
merge := link.BuildResult{ID: "b", Commit: "4b5c6d7e8f90aabbccddeeff0011223344556677", Trunk: "main"}
|
||||
err := publishable(merge, "")
|
||||
if !errors.Is(err, errOffTheTrunk) || !strings.Contains(err.Error(), "not on main") {
|
||||
t.Fatalf("a build of a check's merge was answered %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A verdict no build judges names its trunk commit too** (novox/hq ADR 0323 §3): a change touching no module of a
|
||||
// repository with no merge-check.sh (novox/mesh-tools-go) is answered by the controller itself. Naming no trunk, the
|
||||
// merge tool refused it, asked again, got the same verdict, and refused for ever.
|
||||
func TestADirectVerdictNamesTheTrunkTheForgeAnnounced(t *testing.T) {
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "mesh-tools-go", Number: 3, Base: "main", Commit: "1a2b3c4d5e6f7a8b9c0d",
|
||||
BaseSHA: "7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b"}
|
||||
c := directVerdict(p, link.ChangePlan{Repository: "novox/mesh-tools-go", Base: "main", Head: p.Commit})
|
||||
if c.Trunk != "7a8b9c0d1e2f" || c.Plan == nil || c.Plan.Trunk != "7a8b9c0d1e2f" {
|
||||
t.Fatalf("the direct verdict names trunk %q, its plan %+v", c.Trunk, c.Plan)
|
||||
}
|
||||
if c.Verdict != "pass" || c.Commit != p.Commit {
|
||||
t.Fatalf("the direct verdict is %+v", c)
|
||||
}
|
||||
if c := directVerdict(link.PullUpdated{Owner: "novox", Repo: "x", Commit: "abc"}, link.ChangePlan{}); c.Trunk != "" {
|
||||
t.Fatalf("a forge that said no base head gave the trunk %q", c.Trunk)
|
||||
}
|
||||
}
|
||||
|
||||
// A head checked again through mesh-delivery (`delivery check --members`, a recheck) reaches the same direct verdict:
|
||||
// it names the base head the forge last announced, kept on the member, or the merge tool would refuse it for ever.
|
||||
func TestARecheckedHeadsDirectVerdictNamesItsTrunk(t *testing.T) {
|
||||
m := orderMember{ID: "novox/mesh-tools-go@1a2b3c4d5e6f", Repository: "novox/mesh-tools-go", Base: "main",
|
||||
Head: "1a2b3c4d5e6f7a8b9c0d", Number: 3, BaseSHA: "7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b"}
|
||||
if c := directVerdict(m.pull(), link.ChangePlan{}); c.Trunk != "7a8b9c0d1e2f" {
|
||||
t.Fatalf("a rechecked head's direct verdict names trunk %q", c.Trunk)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,215 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// **No send but a person's `push <node>` and a gate's own put-back moves the mesh's own core to a build older
|
||||
// than the one its machine was last sent** (novox/hq issue 201). Every send made through sendToEach keeps it — a
|
||||
// plan's, a release's, a healer's, a rotation's, and the bus step too, which is a person's act but composes under
|
||||
// sendKeeps like the rest. On 2026-10-02 two merges a minute apart made two plans;
|
||||
// the controller's plan rolled the new controller, which widened its seat's row, and the node-engine's plan then
|
||||
// sent the control node the controller build before it. The older binary read the row it could not serve and the
|
||||
// mesh lost its voice for ten minutes — the thing that would have corrected it was the thing that was down. ADR
|
||||
// 0185 took the outage out of it (a controller behind its row serves what it can); the race is closed here,
|
||||
// without needing its trigger: whatever made the mesh's record older, the send keeps the build.
|
||||
//
|
||||
// **Kept, not refused.** The send composes the controller or the node-engine at the build the machine was last
|
||||
// sent, as a recorded module is kept (ADR 0245), and carries everything else: no other module, secret, healer or
|
||||
// walk waits on it, and nothing is retried. The kept move is no move for a gate either: ungatedIn and gatedSend
|
||||
// leave it out (withoutKept), so a gate judges the other moves of its send. A plan whose own build is the older one finds its gate superseded by
|
||||
// the build kept there (issue 352) and ends. The mesh's record stays older, and a warning condition says so until
|
||||
// a send carries the record — a person's `push <node>` accepts the older build — or the record moves forward: a
|
||||
// newer merge, or `rebuild mesh-controller`.
|
||||
//
|
||||
// Older is by the builds' registration order — when each was asked (novox/hq issue 219) — never by the commits'
|
||||
// dates. So a retry or a rebuild of an old commit, asked anew, counts as newer: it is registered as the module's
|
||||
// current build by the same order, and the guard does not stop that move. A build that failed its gate is never
|
||||
// ahead: the machine was sent it and the record was put back before it (gateFailed), and if the put-back's own
|
||||
// send was refused, the next send completes it rather than keeping the failed build there. A gate's own put-back
|
||||
// carries its modules back whatever the order (sendScope.puttingBack); the controller's is held to the store's
|
||||
// schema already (issue 352).
|
||||
|
||||
// kindCoreDowngrade is a core module kept at the build its machine was last sent, over an older record.
|
||||
const kindCoreDowngrade = "core-downgrade-kept"
|
||||
|
||||
// coreDowngradeSource is the condition's source: the sends, which raise and clear it.
|
||||
const coreDowngradeSource = "sends"
|
||||
|
||||
// downgradeGuarded is the core components a send may not move backwards: the controller, which reads the
|
||||
// store and its seat's row, and the node-engine, which reads what the controller declares.
|
||||
var downgradeGuarded = []string{lease.ComponentController, lease.ComponentEngine}
|
||||
|
||||
// askedBuild is what the order needs of one build of a module: when it was asked, and whether it failed its gate.
|
||||
type askedBuild struct {
|
||||
asked time.Time
|
||||
failedGate bool
|
||||
}
|
||||
|
||||
// buildAsked is the newest build of a module from a commit that worked — when it was asked (or recorded, when
|
||||
// the request time is not known), and whether it failed its gate — and false when the records hold none. A
|
||||
// variable, so a test reads its own order.
|
||||
var buildAsked = func(ctx context.Context, inv *inventory.Inventory, module, commit string) (askedBuild, bool, error) {
|
||||
builds, err := inv.Builds(ctx, module, askedLookBack)
|
||||
if err != nil {
|
||||
return askedBuild{}, false, err
|
||||
}
|
||||
for _, b := range builds {
|
||||
if b.Worked() && sameCommit(b.Commit, commit) {
|
||||
failed, err := inv.GateFailed(ctx, b.ID)
|
||||
if err != nil {
|
||||
return askedBuild{}, false, err
|
||||
}
|
||||
return askedBuild{asked: b.AskedOrAt(), failedGate: failed}, true, nil
|
||||
}
|
||||
}
|
||||
return askedBuild{}, false, nil
|
||||
}
|
||||
|
||||
// askedLookBack is how many of a module's newest builds are read for when a build was asked.
|
||||
const askedLookBack = 50
|
||||
|
||||
// coreDowngrade is one core module a machine was last sent a build of that is newer than the mesh's record.
|
||||
type coreDowngrade struct {
|
||||
module, node, component string
|
||||
// sent is the build the machine was last sent, record the one the mesh holds.
|
||||
sent, record string
|
||||
sentAsked, recordAsked time.Time
|
||||
}
|
||||
|
||||
func (d coreDowngrade) String() string {
|
||||
return fmt.Sprintf("%s was last sent %s %s, and the mesh holds the older %s (asked %s, before %s): a send there "+
|
||||
"keeps %s; `push %s` sends the older build, and a newer merge or `rebuild %s` moves the record forward "+
|
||||
"(novox/hq issue 201)", d.node, d.module, short(d.sent), short(d.record),
|
||||
d.recordAsked.UTC().Format(time.RFC3339), d.sentAsked.UTC().Format(time.RFC3339), short(d.sent), d.node,
|
||||
d.module)
|
||||
}
|
||||
|
||||
// coreDowngrades is, for one machine, each guarded core module whose record is older than the build the machine
|
||||
// was last sent, leaving out what a gate's own put-back puts back and a sent build that failed its gate. A move
|
||||
// whose either build the records do not hold is not proved older and is left to the gate that judges it. Pure
|
||||
// but for asked.
|
||||
func coreDowngrades(node string, sent map[string]string, record map[string]string, scope sendScope,
|
||||
asked func(module, commit string) (askedBuild, bool, error)) ([]coreDowngrade, error) {
|
||||
var out []coreDowngrade
|
||||
for module, was := range sent {
|
||||
component := coreComponent(module)
|
||||
now := record[module]
|
||||
if !guarded(component) || was == "" || now == "" || sameCommit(was, now) {
|
||||
continue
|
||||
}
|
||||
if scope.puttingBack && scope.modules[module] {
|
||||
continue
|
||||
}
|
||||
from, known, err := asked(module, was)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(was), err)
|
||||
}
|
||||
if !known || from.failedGate {
|
||||
continue
|
||||
}
|
||||
to, known, err := asked(module, now)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(now), err)
|
||||
}
|
||||
if known && to.asked.Before(from.asked) {
|
||||
out = append(out, coreDowngrade{module: module, node: node, component: component, sent: was, record: now,
|
||||
sentAsked: from.asked, recordAsked: to.asked})
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].module < out[j].module })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func guarded(component string) bool {
|
||||
for _, c := range downgradeGuarded {
|
||||
if c == component {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// coreKept is, for a send that keeps (sendKeeps), each guarded core module of a machine to keep at the build it
|
||||
// was last sent, module → commit, each said and raised as a condition.
|
||||
func coreKept(ctx context.Context, inv *inventory.Inventory, node string, sent map[string]string,
|
||||
current map[string]inventory.CurrentBuild) (map[string]string, error) {
|
||||
record := map[string]string{}
|
||||
for m, c := range current {
|
||||
record[m] = c.Commit
|
||||
}
|
||||
found, err := coreDowngrades(node, sent, record, scopeOf(ctx), func(module, commit string) (askedBuild, bool, error) {
|
||||
return buildAsked(ctx, inv, module, commit)
|
||||
})
|
||||
if err != nil || len(found) == 0 {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, d := range found {
|
||||
out[d.module] = d.sent
|
||||
sayCoreDowngrade(ctx, d)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// coreDowngradeObservation is a kept core build as a condition: `core.<component>.<machine>.<kind>`.
|
||||
func coreDowngradeObservation(component, node, summary string) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeCore, ID: component + "." + node, Kind: kindCoreDowngrade,
|
||||
Token: kindCoreDowngrade, Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource,
|
||||
Summary: summary}
|
||||
}
|
||||
|
||||
func sayCoreDowngrade(ctx context.Context, d coreDowngrade) {
|
||||
fmt.Printf("KEPT: %s\n", d)
|
||||
keeper := conditionsFrom
|
||||
if keeper == nil {
|
||||
return
|
||||
}
|
||||
if _, err := keeper.Observe(ctx, coreDowngradeObservation(d.component, d.node, d.String())); err != nil {
|
||||
fmt.Printf("the condition for %s kept on %s could not be raised: %v\n", d.module, d.node, err)
|
||||
}
|
||||
}
|
||||
|
||||
// settleCoreDowngrades clears, after a send to a machine is away, the condition of each guarded core module the
|
||||
// send carried at the mesh's record — a person's push that accepted the older build, or a send after the record
|
||||
// moved forward — or did not carry at all. A send that kept the newer build leaves it open.
|
||||
func settleCoreDowngrades(ctx context.Context, inv *inventory.Inventory, node string, carried map[string]string) {
|
||||
keeper := conditionsFrom
|
||||
if keeper == nil || carried == nil {
|
||||
return
|
||||
}
|
||||
var open []string
|
||||
for _, c := range downgradeGuarded {
|
||||
if _, found, err := keeper.Get(ctx, coreDowngradeObservation(c, node, "").Key()); err == nil && found {
|
||||
open = append(open, c)
|
||||
}
|
||||
}
|
||||
if len(open) == 0 {
|
||||
return
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("whether %s still keeps a core build over the mesh's record cannot be read: %v\n", node, err)
|
||||
return
|
||||
}
|
||||
for module, now := range current {
|
||||
component := coreComponent(module)
|
||||
if !guarded(component) {
|
||||
continue
|
||||
}
|
||||
if was, has := carried[module]; has && !sameCommit(was, now.Commit) {
|
||||
continue
|
||||
}
|
||||
key := coreDowngradeObservation(component, node, "").Key()
|
||||
if _, err := keeper.Clear(ctx, key, node+" was sent the "+component+" the mesh holds"); err != nil {
|
||||
fmt.Printf("the condition %s could not be cleared: %v\n", key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -580,9 +580,6 @@ type orderMember struct {
|
||||
CloneURL string `json:"clone_url,omitempty"`
|
||||
// After are the repositories its pull request says it goes after (`after: <repository>`).
|
||||
After []string `json:"after,omitempty"`
|
||||
// BaseSHA is the base branch's head the forge last announced the head with (novox/hq ADR 0323): the trunk commit
|
||||
// a verdict no build judges names when the head is checked again through mesh-delivery.
|
||||
BaseSHA string `json:"base_sha,omitempty"`
|
||||
}
|
||||
|
||||
// pull is the member as the forge announced it.
|
||||
@@ -592,7 +589,7 @@ func (m orderMember) pull() link.PullUpdated {
|
||||
if base == "" {
|
||||
base = "main"
|
||||
}
|
||||
return link.PullUpdated{Owner: owner, Repo: repo, Number: m.Number, Base: base, Commit: m.Head, BaseSHA: m.BaseSHA,
|
||||
return link.PullUpdated{Owner: owner, Repo: repo, Number: m.Number, Base: base, Commit: m.Head,
|
||||
CloneURL: m.CloneURL, Paths: m.Paths, PathsTruncated: m.PathsTruncated, Removed: m.Removed,
|
||||
ModuleDirs: m.ModuleDirs, ModuleDirsSaid: m.ModuleDirsSaid}
|
||||
}
|
||||
@@ -1188,7 +1185,7 @@ func askGroupCheck(ctx context.Context, open *stores, group string, members []or
|
||||
return "", fmt.Errorf("%s cannot be cloned for the group's check: %w", m.ID, err)
|
||||
}
|
||||
request.Check.Members = append(request.Check.Members, link.GroupMember{Owner: mp.Owner, Repo: mp.Repo,
|
||||
Number: mp.Number, Repository: url, Ref: mp.Commit, Paths: mp.Paths, Base: mp.Base})
|
||||
Number: mp.Number, Repository: url, Ref: mp.Commit, Paths: mp.Paths})
|
||||
}
|
||||
seat := buildSeatHeld(ctx)
|
||||
ask, err := askOverOn(seat)
|
||||
|
||||
@@ -1039,7 +1039,7 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str
|
||||
}
|
||||
return
|
||||
}
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}, puttingBack: true}), open, g.Machines)
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines)
|
||||
if err != nil {
|
||||
notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+
|
||||
"sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0]))
|
||||
@@ -1132,7 +1132,7 @@ func sendRollbacks(ctx context.Context, open *stores, p *inventory.Plan, b *roll
|
||||
}
|
||||
inv := open.inventory
|
||||
sort.Strings(b.machines)
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules, puttingBack: true}), open, b.machines)
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules}), open, b.machines)
|
||||
var back []string
|
||||
for _, r := range b.pending {
|
||||
if err != nil {
|
||||
|
||||
@@ -1,259 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// novox/hq issue 201: on 2026-10-02 a plan sent the control node a controller older than the one running there,
|
||||
// whose seat row the newer one had already widened; the older binary refused the row and the mesh went quiet for
|
||||
// ten minutes. No send but a person's push and a gate's own put-back moves the controller or the node-engine back
|
||||
// behind the build its machine was last sent: the send keeps it there, and carries the rest.
|
||||
|
||||
// askedIn answers a build from a fixed table: module@commit → when it was asked; failed names those that failed
|
||||
// their gate.
|
||||
func askedIn(table map[string]time.Time, failed ...string) func(module, commit string) (askedBuild, bool, error) {
|
||||
return func(module, commit string) (askedBuild, bool, error) {
|
||||
at, known := table[module+"@"+commit]
|
||||
b := askedBuild{asked: at}
|
||||
for _, f := range failed {
|
||||
if f == module+"@"+commit {
|
||||
b.failedGate = true
|
||||
}
|
||||
}
|
||||
return b, known, nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestACoreModuleIsKeptOverAnOlderRecord(t *testing.T) {
|
||||
old, now := t0, t0.Add(time.Minute)
|
||||
table := map[string]time.Time{
|
||||
"mesh-controller@c1": old, "mesh-controller@c2": now,
|
||||
"mesh-host@h1": old, "mesh-host@h2": now,
|
||||
"app@a1": old, "app@a2": now,
|
||||
}
|
||||
sent := map[string]string{"mesh-controller": "c2", "mesh-host": "h2", "app": "a2"}
|
||||
record := map[string]string{"mesh-controller": "c1", "mesh-host": "h1", "app": "a1"}
|
||||
found, err := coreDowngrades("anchor", sent, record, sendScope{judged: map[string]bool{"anchor": true}}, askedIn(table))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 2 || found[0].module != "mesh-controller" || found[0].component != lease.ComponentController ||
|
||||
found[0].sent != "c2" || found[1].module != "mesh-host" || found[1].component != lease.ComponentEngine {
|
||||
t.Fatalf("the controller and the node-engine over an older record are kept, and nothing else: %+v", found)
|
||||
}
|
||||
if said := found[0].String(); !strings.Contains(said, "anchor was last sent mesh-controller c2") ||
|
||||
!strings.Contains(said, "`push anchor`") || !strings.Contains(said, "`rebuild mesh-controller`") {
|
||||
t.Fatalf("it says the machine, the build kept and the two ways out: %s", said)
|
||||
}
|
||||
|
||||
// The record forward, the same build, or a build the records do not hold: nothing kept.
|
||||
for _, c := range [][2]string{{"c1", "c2"}, {"c2", "c2"}, {"c2", "c9"}, {"c9", "c1"}} {
|
||||
found, err := coreDowngrades("anchor", map[string]string{"mesh-controller": c[0]},
|
||||
map[string]string{"mesh-controller": c[1]}, sendScope{}, askedIn(table))
|
||||
if err != nil || len(found) != 0 {
|
||||
t.Fatalf("sent %s, record %s: no downgrade proved, nothing kept: %+v %v", c[0], c[1], found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A sent build that failed its gate is never ahead: the record put back before it is sent.
|
||||
if found, _ := coreDowngrades("anchor", sent, record, sendScope{}, askedIn(table, "mesh-controller@c2")); len(found) != 1 ||
|
||||
found[0].module != "mesh-host" {
|
||||
t.Fatalf("a controller that failed its gate was kept over the build put back: %+v", found)
|
||||
}
|
||||
|
||||
// A gate's own put-back carries its own modules back, and nothing else.
|
||||
putBack := sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}
|
||||
if found, _ := coreDowngrades("anchor", sent, record, putBack, askedIn(table)); len(found) != 1 || found[0].module != "mesh-host" {
|
||||
t.Fatalf("a put-back of the controller carries it back, and not the node-engine: %+v", found)
|
||||
}
|
||||
if found, _ := coreDowngrades("anchor", sent, record, sendScope{modules: map[string]bool{"mesh-controller": true}},
|
||||
askedIn(table)); len(found) != 2 {
|
||||
t.Fatalf("a send for the controller that is no put-back carries it back: %+v", found)
|
||||
}
|
||||
}
|
||||
|
||||
// core201 is a mesh whose control node, anchor, was sent the newer controller c2 while the mesh's record of the
|
||||
// controller is the build before it, c1, which passed its gate when it rolled out.
|
||||
func core201(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
|
||||
record := func(commit string, asked time.Time) inventory.Build {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"})
|
||||
b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit,
|
||||
Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}}
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
before := record("c1aaaaaaaaaa", old)
|
||||
if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: before.Commit,
|
||||
Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record("c2bbbbbbbbbb", now)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RestoreModule(ctx, before); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wasMoves := machineMoves
|
||||
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
|
||||
modules, err := open.inventory.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sent, known, err := open.inventory.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.moves(node, modules, sent, known, all), nil
|
||||
}
|
||||
t.Cleanup(func() { machineMoves = wasMoves })
|
||||
return open
|
||||
}
|
||||
|
||||
// TestReplay201 replays the race on a store: the control node runs the newer controller and the mesh's record is
|
||||
// the build before it. A plan's send there — the node-engine's plan of 2026-10-02 — goes, keeps the newer
|
||||
// controller, and says so as a condition; the gate's own put-back carries it back; a send that carries the record,
|
||||
// as a person's push does, clears the condition.
|
||||
func TestReplay201(t *testing.T) {
|
||||
open := core201(t)
|
||||
ctx := t.Context()
|
||||
k, _ := withConditionsInMemory(t)
|
||||
|
||||
judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}})
|
||||
if kept, err := ungatedIn(judged, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
|
||||
t.Fatalf("a plan's send to the machine is made, the rest of it with it: %v %v", kept, err)
|
||||
}
|
||||
keeps, err := recordedKept(keepingRecorded(judged), open, "anchor")
|
||||
if err != nil || keeps["mesh-controller"] != "c2bbbbbbbbbb" {
|
||||
t.Fatalf("the send keeps the newer controller there: %v %v", keeps, err)
|
||||
}
|
||||
shelf := map[string]catalogue.Manifest{"mesh-controller": {Module: "mesh-controller", Version: "old"}}
|
||||
if _, err := keepRecorded(keepingRecorded(judged), open, "anchor", shelf); err != nil || shelf["mesh-controller"].Version != "1" {
|
||||
t.Fatalf("the send composes the kept build's manifest: %+v %v", shelf, err)
|
||||
}
|
||||
key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key()
|
||||
if c, found, err := k.Get(ctx, key); err != nil || !found || !strings.Contains(c.Summary, "c2bbbbbb") {
|
||||
t.Fatalf("the condition %s is raised, saying the build kept: %+v %v %v", key, c, found, err)
|
||||
}
|
||||
|
||||
putBack := keepingRecorded(withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}))
|
||||
if keeps, err := recordedKept(putBack, open, "anchor"); err != nil || len(keeps) != 0 {
|
||||
t.Fatalf("the gate's own put-back of the controller carries it back: %v %v", keeps, err)
|
||||
}
|
||||
|
||||
settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"})
|
||||
if _, found, _ := k.Get(ctx, key); !found {
|
||||
t.Fatal("a send that kept the newer controller cleared the condition")
|
||||
}
|
||||
settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c1aaaaaaaaaa"})
|
||||
if _, found, _ := k.Get(ctx, key); found {
|
||||
t.Fatal("a send carrying the mesh's record — a person's push — leaves the condition open")
|
||||
}
|
||||
}
|
||||
|
||||
// The review of #273: a gate failed the newer controller, put the record back, and the put-back's own send was
|
||||
// refused. The next automatic send — a healer's, which judges nothing — completes the put-back: a build that
|
||||
// failed its gate is never kept.
|
||||
func TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend(t *testing.T) {
|
||||
open := core201(t)
|
||||
ctx := t.Context()
|
||||
k, _ := withConditionsInMemory(t)
|
||||
if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: "build-mesh-controller-c2bbbbbbbbbb",
|
||||
Module: "mesh-controller", Commit: "c2bbbbbbbbbb", Previous: "c1aaaaaaaaaa", Machines: []string{"anchor"},
|
||||
Verdict: inventory.GateFailed, Rollback: inventory.NotRolledBack}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
healer := withScope(ctx, sendScope{})
|
||||
if kept, err := ungatedIn(healer, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
|
||||
t.Fatalf("the healer's send is made: %v %v", kept, err)
|
||||
}
|
||||
if keeps, err := recordedKept(keepingRecorded(healer), open, "anchor"); err != nil || len(keeps) != 0 {
|
||||
t.Fatalf("the failed controller was kept over the build put back: %v %v", keeps, err)
|
||||
}
|
||||
if open, _ := k.Open(ctx); len(open) != 0 {
|
||||
t.Fatalf("a completed put-back raised a condition: %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// Review 91 of #273: a gated send to a machine whose controller is kept over an older record leaves the kept
|
||||
// controller out of what its gate judges. Judged as moving back to the record, the gate would read the send's own
|
||||
// kept controller as another build sent there after it (issue 352) and be superseded, judging nothing.
|
||||
func TestAGateJudgesTheOtherMovesBesideAKeptCore(t *testing.T) {
|
||||
open := core201(t)
|
||||
ctx := t.Context()
|
||||
withConditionsInMemory(t)
|
||||
inv := open.inventory
|
||||
old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
|
||||
for _, b := range []struct {
|
||||
commit string
|
||||
at time.Time
|
||||
}{{"a1aaaaaaaaaa", old}, {"a2bbbbbbbbbb", now}} {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-" + b.commit, Module: "app", Commit: b.commit,
|
||||
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: b.at, At: b.at,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-" + b.commit}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: b.commit, Head: b.commit,
|
||||
Asked: b.at}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-2",
|
||||
map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a1aaaaaaaaaa"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The send as it is made: the controller kept at c2, the app moved to the record.
|
||||
wasSend := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
return names, open.inventory.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-3",
|
||||
map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a2bbbbbbbbbb"})
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = wasSend })
|
||||
|
||||
moves, sent, _, err := gatedSend(ctx, open, "anchor", nil)
|
||||
if err != nil || len(sent) != 1 {
|
||||
t.Fatalf("the gated send is made: %v %v", sent, err)
|
||||
}
|
||||
if len(moves) != 1 || moves[0].Module != "app" || moves[0].To != "a2bbbbbbbbbb" {
|
||||
t.Fatalf("the gate judges the app's move and not the kept controller: %+v", moves)
|
||||
}
|
||||
builds, _, err := inv.SentBuilds(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g := &inventory.PlanGate{Carried: moves}
|
||||
f := gateFacts{commits: judgedCommits(g, nil), sentBuilds: map[string]map[string]string{"anchor": builds}}
|
||||
for _, m := range []string{"app", "mesh-controller"} {
|
||||
if why, superseded := f.supersededOn(m, "anchor"); superseded {
|
||||
t.Fatalf("the gate is superseded by its own send: %s", why)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,120 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// An own secret issued outside the mesh that nobody gave is never made (novox/hq issue 401, ADR 0228). Seen on
|
||||
// 2026-10-10: the send made a random value for the mounts module's smb-credentials and placed it, and the module read
|
||||
// it as a credential given and wrong. The send now makes none, places none, and places a value only once a person
|
||||
// gave one — and a value the mesh made for it before is not placed either.
|
||||
|
||||
// anAssistant is a module with a secret the mesh makes and one only an outside party issues.
|
||||
func anAssistant() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "assistant", Version: "1",
|
||||
OwnSecrets: catalogue.OwnSecrets{
|
||||
"password": {Path: "/var/lib/mesh/assistant/password", Taken: catalogue.TakenAtStart},
|
||||
"vendor-key": {Path: "/var/lib/mesh/assistant/vendor-key", Taken: catalogue.TakenAtStart, IssuedBy: catalogue.IssuedOutside},
|
||||
},
|
||||
Resources: []map[string]any{
|
||||
{"id": "state", "type": "directory", "path": "/var/lib/mesh/assistant", "mode": "0700"},
|
||||
}}
|
||||
}
|
||||
|
||||
// sentTo composes what a push sends to a machine, on the send path.
|
||||
func sentTo(t *testing.T, open *stores, node string) []map[string]any {
|
||||
t.Helper()
|
||||
plan, settings, err := planFor(t.Context(), open, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(t.Context(), open, node, plan, settings)
|
||||
if err != nil {
|
||||
t.Fatalf("the send to %s did not compose: %v", node, err)
|
||||
}
|
||||
return declared.Resources
|
||||
}
|
||||
|
||||
func placedAt(resources []map[string]any, path string) map[string]any {
|
||||
for _, r := range resources {
|
||||
if r["path"] == path && r["sealed"] != nil {
|
||||
return r
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestTheSendMakesNoValueForAnOutsideSecretNobodyGave(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, anAssistant())
|
||||
if _, err := assign(ctx, open, "laptop", "assistant"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sent := sentTo(t, open, "laptop")
|
||||
if placedAt(sent, "/var/lib/mesh/assistant/password") == nil {
|
||||
t.Fatal("the secret the mesh makes was not placed")
|
||||
}
|
||||
if got := placedAt(sent, "/var/lib/mesh/assistant/vendor-key"); got != nil {
|
||||
t.Fatalf("a secret nobody gave was placed: %v", got)
|
||||
}
|
||||
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "assistant", "vendor-key"); err != nil || held {
|
||||
t.Fatalf("the send made a value for a secret issued outside the mesh (held %v, %v)", held, err)
|
||||
}
|
||||
|
||||
// A value the mesh made before this rule is not one a person gave: it is not placed.
|
||||
if _, err := open.inventory.SecretForModule(ctx, "laptop", "assistant", "vendor-key"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := placedAt(sentTo(t, open, "laptop"), "/var/lib/mesh/assistant/vendor-key"); got != nil {
|
||||
t.Fatalf("a value the mesh made for a secret issued outside was placed: %v", got)
|
||||
}
|
||||
|
||||
// Given, it is placed.
|
||||
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "assistant", "vendor-key", "the vendor's key"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
given, ok, err := open.inventory.GivenOwnSecret(ctx, "laptop", "assistant", "vendor-key")
|
||||
if err != nil || !ok || !given.Current {
|
||||
t.Fatalf("the value given is not read as given: %+v %v %v", given, ok, err)
|
||||
}
|
||||
placed := placedAt(sentTo(t, open, "laptop"), "/var/lib/mesh/assistant/vendor-key")
|
||||
if placed == nil || placed["sealed"] != given.Sealed {
|
||||
t.Fatalf("the value given is placed as %v", placed)
|
||||
}
|
||||
}
|
||||
|
||||
// The read paths answer as the send does: a question about the machine composes without the secret nobody gave, and
|
||||
// asking writes nothing.
|
||||
func TestAQuestionComposesWithoutAnOutsideSecretNobodyGave(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, anAssistant())
|
||||
if _, err := assign(ctx, open, "laptop", "assistant"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pushedBy(t, open, "laptop") // the mesh's own secret is made by the send
|
||||
plan, settings, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, choosing := range []Choosing{Reading, Foreseeing} {
|
||||
got, err := declarationWith(ctx, open, "laptop", plan, settings, gens, choosing)
|
||||
if err != nil {
|
||||
t.Fatalf("%v: %v", choosing, err)
|
||||
}
|
||||
if placedAt(got.Resources, "/var/lib/mesh/assistant/vendor-key") != nil || len(got.foreseen) != 0 {
|
||||
t.Fatalf("%v: a secret nobody gave is composed (foreseen %v)", choosing, got.foreseen)
|
||||
}
|
||||
}
|
||||
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "assistant", "vendor-key"); err != nil || held {
|
||||
t.Fatalf("a question made a value (held %v, %v)", held, err)
|
||||
}
|
||||
}
|
||||
@@ -81,13 +81,6 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
"It usually catches up by itself.", m, m),
|
||||
Resolved: m + " applied the change"}
|
||||
}),
|
||||
kindApplyFailing: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: m + " is not applying its update",
|
||||
Explanation: fmt.Sprintf("%s has refused or only partly applied what the mesh sent it, again and again. "+
|
||||
"The details say what fails.", conditions.Capital(m)),
|
||||
Resolved: m + " applies what it is sent again"}
|
||||
}),
|
||||
"tools-silent": worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "Tools on " + m + " are not answering",
|
||||
@@ -182,20 +175,6 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
"register it, so it was not made.", module, m),
|
||||
Resolved: "Resolved: " + module + " on " + m + " is settled"}
|
||||
}),
|
||||
// The controller or the node-engine kept at the build a machine was last sent, over an older record (novox/hq
|
||||
// issue 201).
|
||||
kindCoreDowngrade: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
component := idPart(o, 0)
|
||||
if o.Scope != conditions.ScopeCore || !guarded(component) {
|
||||
component = "core component"
|
||||
}
|
||||
return words{Headline: m + " keeps a newer " + component + " than the mesh holds",
|
||||
Needs: fmt.Sprintf("push %s to take the older build, or a newer merge or rebuild moves the record forward.", m),
|
||||
Explanation: fmt.Sprintf("The mesh's record of its own %s is older than the build %s was last sent, so "+
|
||||
"the mesh's sends keep the newer one there rather than put it back.", component, m),
|
||||
Resolved: "Resolved: " + m + " runs the " + component + " the mesh holds"}
|
||||
}),
|
||||
kindBindingKept: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "A module's data source is held on " + m,
|
||||
|
||||
@@ -590,13 +590,6 @@ func busCredentialIssued(ctx context.Context, inv *inventory.Inventory, node, mo
|
||||
return nil
|
||||
}
|
||||
|
||||
// givenUnderAnOldKey is the refusal for a value a person gave — a family's member, or an own secret issued outside
|
||||
// the mesh — sealed to a key the machine no longer holds: the mesh cannot make another, so it is given again.
|
||||
func givenUnderAnOldKey(module, node, name string) error {
|
||||
return fmt.Errorf("%s on %s holds %q, which was given to the mesh rather than made by it, and %s has "+
|
||||
"since generated a new sealing key. The mesh cannot make another; give it again", module, node, name, node)
|
||||
}
|
||||
|
||||
// renderingFor is everything a node's declaration is composed with, and the node's record.
|
||||
func renderingFor(ctx context.Context, open *stores, node string,
|
||||
plan catalogue.Resolution, settings catalogue.SettingsBy,
|
||||
@@ -715,7 +708,10 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
}
|
||||
if !g.Current {
|
||||
if choosing == Allocating {
|
||||
return catalogue.Rendering{}, inventory.Node{}, givenUnderAnOldKey(m.Module, node, g.Name)
|
||||
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf(
|
||||
"%s on %s holds %q, which was given to the mesh rather than made by it, and %s has "+
|
||||
"since generated a new sealing key. The mesh cannot make another; give it again",
|
||||
m.Module, node, g.Name, node)
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -725,31 +721,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
needed[m.Module][g.Name] = g.Sealed
|
||||
}
|
||||
}
|
||||
for name, own := range m.OwnSecrets.Plain() {
|
||||
// **An own secret issued outside the mesh is never made** (novox/hq issue 401, ADR 0228; ADR 0283 for
|
||||
// a family): only a value a person gave is placed. One not given is nothing — no file, and an empty
|
||||
// value where a file of the module names it — so the module can tell it was not given, where a value
|
||||
// the mesh made looked given and wrong. A value the mesh made for it before is not one either.
|
||||
if own.IssuedBy == catalogue.IssuedOutside {
|
||||
g, given, err := inv.GivenOwnSecret(ctx, node, m.Module, name)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
if !given {
|
||||
continue
|
||||
}
|
||||
if !g.Current {
|
||||
if choosing == Allocating {
|
||||
return catalogue.Rendering{}, inventory.Node{}, givenUnderAnOldKey(m.Module, node, name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if needed[m.Module] == nil {
|
||||
needed[m.Module] = map[string]string{}
|
||||
}
|
||||
needed[m.Module][name] = g.Sealed
|
||||
continue
|
||||
}
|
||||
for name := range m.OwnSecrets.Plain() {
|
||||
// Minted on the send path and only read on every other. Making one is an insert, and
|
||||
// a question that writes is a question that can block against the machine it is about.
|
||||
var sealed string
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
// planID is every id the mesh gives a plan: a merge's walk or batch (plan-<nanoseconds>) or a release
|
||||
// (release-<nanoseconds>).
|
||||
var planID = regexp.MustCompile(`^(plan|release)-[0-9]+$`)
|
||||
|
||||
// planIDArguments are the arguments of plans that name a plan. wait is among them for when it is declared
|
||||
// (novox/hq issue 229); an argument not declared is refused before this is asked.
|
||||
var planIDArguments = []string{"id", "wait", "stop", "close", "retry", "go"}
|
||||
|
||||
// plainPlanIDs refuses a plans call naming a plan by anything but a plan's id (novox/hq issue 561): checked
|
||||
// once, where the call becomes a command line, so nothing else a caller sent reaches a command or the store.
|
||||
func plainPlanIDs(given map[string]string) error {
|
||||
for _, arg := range planIDArguments {
|
||||
if id, named := given[arg]; named && !planID.MatchString(id) {
|
||||
return fmt.Errorf("plans %s: %q is not a plan's id (plan-<digits> or release-<digits>, as `plans` "+
|
||||
"lists them); nothing was done", arg, id)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **A plan's id is checked once, where a call becomes a command line** (novox/hq issue 561): every argument of
|
||||
// plans that names a plan is refused unless it is an id the mesh gives a plan, so a quote never reaches a
|
||||
// command, nor the store behind it.
|
||||
func TestAPlanIdThatIsNoPlansIdIsRefusedBeforeACommandStarts(t *testing.T) {
|
||||
for _, arg := range []string{"id", "stop", "close", "retry", "go"} {
|
||||
for _, bad := range []string{"x' or '1'='1", "plan-1'", "plan-", "plan-1 2", "batch-1", "PLAN-1"} {
|
||||
args := map[string]any{arg: bad}
|
||||
if arg == "stop" || arg == "close" || arg == "go" {
|
||||
args["why"] = "a test"
|
||||
}
|
||||
argv, err := argvFor("plans", args)
|
||||
if err == nil {
|
||||
t.Errorf("plans %s %q was composed: %v", arg, bad, argv)
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(err.Error(), "is not a plan's id") {
|
||||
t.Errorf("plans %s %q was refused for another reason: %v", arg, bad, err)
|
||||
}
|
||||
}
|
||||
for _, good := range []string{"plan-1791813626034291775", "release-1791811938267404408"} {
|
||||
args := map[string]any{arg: good}
|
||||
if arg == "stop" || arg == "close" || arg == "go" {
|
||||
args["why"] = "a test"
|
||||
}
|
||||
if _, err := argvFor("plans", args); err != nil {
|
||||
t.Errorf("plans %s %q was refused: %v", arg, good, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,158 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Following a plan (novox/hq issue 229): `plans wait <id>` holds its answer until the plan moves, reaches its
|
||||
// next tier or ends, or until its bound passes, and says which. Before it, an agent following a rollout had
|
||||
// `plans` answer once, in prose, and wrote a loop of its own round the mesh's tools to be told the plan moved.
|
||||
//
|
||||
// **No wait is longer than a call is answered within** (link.LongestAnswer, inside the console's thirty
|
||||
// seconds): a longer one would answer "still running" and leave its caller to ask `calls` — the loop again.
|
||||
// So a caller following a plan to its end asks again; each call answers the moment the plan moves.
|
||||
|
||||
// maxPlanWait is the longest bound a wait takes, and the bound when none is given: what a call is answered
|
||||
// within, less the time a verb's command takes to start and read the store.
|
||||
var maxPlanWait = link.LongestAnswer - planWaitStart
|
||||
|
||||
// planWaitStart is what the verb's window allows beyond the bound: starting the command and opening the stores.
|
||||
// A store slow enough to push the read and the answer past it degrades as any long call does: the caller is
|
||||
// answered that the call is still running, and `calls` with its id says what came of it.
|
||||
const planWaitStart = 5 * time.Second
|
||||
|
||||
// planWaitEvery is how often a wait reads the plan again.
|
||||
const planWaitEvery = time.Second
|
||||
|
||||
// The things a wait waits until.
|
||||
var planWaitUntil = []string{"moved", "tier", "done"}
|
||||
|
||||
// planWaitAsked checks what a wait is asked for: until (moved when empty) and a bound (a duration, or a number
|
||||
// of seconds; maxPlanWait when empty).
|
||||
func planWaitAsked(until, bound string) (string, time.Duration, error) {
|
||||
if until == "" {
|
||||
until = "moved"
|
||||
}
|
||||
known := false
|
||||
for _, u := range planWaitUntil {
|
||||
known = known || u == until
|
||||
}
|
||||
if !known {
|
||||
return "", 0, fmt.Errorf("a wait is until %s, not %q", strings.Join(planWaitUntil, ", "), until)
|
||||
}
|
||||
d := maxPlanWait
|
||||
if bound = strings.TrimSpace(bound); bound != "" {
|
||||
var err error
|
||||
if secs, isNumber := strconv.Atoi(bound); isNumber == nil {
|
||||
d = time.Duration(secs) * time.Second
|
||||
} else if d, err = time.ParseDuration(bound); err != nil {
|
||||
return "", 0, fmt.Errorf("the bound %q is not a duration (such as 20s) or a number of seconds", bound)
|
||||
}
|
||||
if d <= 0 || d > maxPlanWait {
|
||||
return "", 0, fmt.Errorf("the bound is at most %s, the longest a call is answered within; %s was asked "+
|
||||
"— to follow a plan longer, ask again: each call answers the moment the plan moves", maxPlanWait, d)
|
||||
}
|
||||
}
|
||||
return until, d, nil
|
||||
}
|
||||
|
||||
// planEnded is a plan that will not move again: done, failed or superseded. A batch being assembled is not one.
|
||||
func planEnded(p inventory.Plan) bool { return !p.Open() && !p.Batch() }
|
||||
|
||||
// waitOnPlan reads the plan until what it waits for happens or the bound passes, and answers the plan as last
|
||||
// read and what ended the wait: moved (any save of it), tier (its tier changed), done (it ended — which ends
|
||||
// every wait, since nothing comes after it), or bound. Moved and tier are judged against the plan as first read.
|
||||
func waitOnPlan(ctx context.Context, read func(context.Context) (inventory.Plan, error), until string,
|
||||
bound, every time.Duration) (inventory.Plan, string, error) {
|
||||
first, err := read(ctx)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, "", err
|
||||
}
|
||||
if planEnded(first) {
|
||||
return first, "done", nil
|
||||
}
|
||||
deadline := time.Now().Add(bound)
|
||||
tick := time.NewTicker(every)
|
||||
defer tick.Stop()
|
||||
last := first
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return last, "", ctx.Err()
|
||||
case <-tick.C:
|
||||
}
|
||||
if !time.Now().Before(deadline) {
|
||||
return last, "bound", nil
|
||||
}
|
||||
p, err := read(ctx)
|
||||
if err != nil {
|
||||
return last, "", err
|
||||
}
|
||||
last = p
|
||||
switch {
|
||||
case planEnded(p):
|
||||
return p, "done", nil
|
||||
case until == "moved" && p.Revision != first.Revision:
|
||||
return p, "moved", nil
|
||||
case until == "tier" && p.Tier != first.Tier:
|
||||
return p, "tier", nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// planWaited is what `plans wait` answers: the plan, what ended the wait, and the line a person reads (`text`,
|
||||
// which the verb answers as its output and not again in the data).
|
||||
type planWaited struct {
|
||||
// Came is what ended the wait: moved, tier, done, or bound when nothing did.
|
||||
Came string `json:"came"`
|
||||
Until string `json:"until"`
|
||||
Waited string `json:"waited"`
|
||||
Plan planStatus `json:"plan"`
|
||||
// Revision is the plan's as read: a caller asking again with the same plan sees whether it moved since.
|
||||
Revision int64 `json:"revision"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
func planWaitedDoc(p inventory.Plan, st planStatus, until, came string, waited time.Duration, line string) planWaited {
|
||||
said := map[string]string{
|
||||
"moved": "moved", "tier": "reached another tier", "done": "ended",
|
||||
"bound": "did not " + map[string]string{"moved": "move", "tier": "reach another tier", "done": "end"}[until],
|
||||
}[came]
|
||||
text := fmt.Sprintf("%s %s in %s (waited until %s)\n%s — %s\n", p.ID, said, waited.Round(time.Second), until,
|
||||
p.ID, line)
|
||||
return planWaited{Came: came, Until: until, Waited: waited.Round(time.Millisecond).String(), Plan: st,
|
||||
Revision: p.Revision, Text: text}
|
||||
}
|
||||
|
||||
// planWaitCommand is `plans wait <id>`: the plan followed until it moves, or the bound passes.
|
||||
func planWaitCommand(ctx context.Context, inv *inventory.Inventory, id, untilAsked, boundAsked string) error {
|
||||
until, bound, err := planWaitAsked(untilAsked, boundAsked)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
began := time.Now()
|
||||
p, came, err := waitOnPlan(ctx, func(ctx context.Context) (inventory.Plan, error) {
|
||||
return inv.PlanByID(ctx, id)
|
||||
}, until, bound, planWaitEvery)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Said as `plans <id>` says its first line, and as `status` says the plan.
|
||||
now, one := time.Now(), []inventory.Plan{p}
|
||||
bounds, pause := readTierBounds(ctx, inv, now), buildSeatPause(ctx, inv, one)
|
||||
line := planLineSaid(p, now, pause, bounds.of(p.Repository), func(module string) []string {
|
||||
on, _ := inv.Running(ctx, module)
|
||||
return on
|
||||
}, waitsOf(ctx, inv, one))
|
||||
doc := planWaitedDoc(p, planStatuses(one, now, pause, bounds)[0], until, came, now.Sub(began), line)
|
||||
return json.NewEncoder(os.Stdout).Encode(doc)
|
||||
}
|
||||
@@ -1,196 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A plan as the store would answer it at each read: the reads are counted, and each answers the next.
|
||||
type planReads struct {
|
||||
plans []inventory.Plan
|
||||
n int
|
||||
}
|
||||
|
||||
func (r *planReads) read(context.Context) (inventory.Plan, error) {
|
||||
p := r.plans[min(r.n, len(r.plans)-1)]
|
||||
r.n++
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func followedPlan(rev int64, state string, tier int) inventory.Plan {
|
||||
return inventory.Plan{ID: "plan-1", Repository: "novox/x", Commit: "abc", State: state, Tier: tier,
|
||||
Tiers: [][]string{{"a"}, {"b"}}, Revision: rev}
|
||||
}
|
||||
|
||||
// **`plans` waits for a plan, and says what ended the wait** (novox/hq issue 229): an agent following a
|
||||
// rollout is told when the plan moved, reached its next tier or ended, or that the bound passed — in one
|
||||
// call, without a loop of its own.
|
||||
func TestAWaitForAPlanEndsWhenWhatItWaitsForHappens(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
until string
|
||||
reads []inventory.Plan
|
||||
came string
|
||||
times int
|
||||
}{
|
||||
{"moved: any save", "moved", []inventory.Plan{
|
||||
followedPlan(1, inventory.PlanBuilding, 0), followedPlan(1, inventory.PlanBuilding, 0),
|
||||
followedPlan(2, inventory.PlanBuilding, 0)}, "moved", 3},
|
||||
{"tier: a save inside the tier is not enough", "tier", []inventory.Plan{
|
||||
followedPlan(1, inventory.PlanBuilding, 0), followedPlan(2, inventory.PlanBuilding, 0),
|
||||
followedPlan(3, inventory.PlanRolling, 1)}, "tier", 3},
|
||||
{"tier: a plan that ended has no next tier to wait for", "tier", []inventory.Plan{
|
||||
followedPlan(1, inventory.PlanBuilding, 0), followedPlan(2, inventory.PlanFailed, 0)}, "done", 2},
|
||||
{"done: a new tier is not the end", "done", []inventory.Plan{
|
||||
followedPlan(1, inventory.PlanBuilding, 0), followedPlan(2, inventory.PlanRolling, 1),
|
||||
followedPlan(3, inventory.PlanDone, 1)}, "done", 3},
|
||||
{"done: a plan already ended answers at once", "done", []inventory.Plan{
|
||||
followedPlan(4, inventory.PlanSuperseded, 1)}, "done", 1},
|
||||
{"the bound passes", "moved", []inventory.Plan{followedPlan(1, inventory.PlanBuilding, 0)}, "bound", -1},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
r := &planReads{plans: c.reads}
|
||||
p, came, err := waitOnPlan(context.Background(), r.read, c.until, 50*time.Millisecond, time.Millisecond)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if came != c.came {
|
||||
t.Fatalf("the wait ended as %q, want %q", came, c.came)
|
||||
}
|
||||
if c.times > 0 && r.n != c.times {
|
||||
t.Errorf("read the plan %d times, want %d: it answered before or after what it waited for", r.n, c.times)
|
||||
}
|
||||
if want := c.reads[min(r.n, len(c.reads))-1]; p.Revision != want.Revision {
|
||||
t.Errorf("answered revision %d, not the plan as it was last read (%d)", p.Revision, want.Revision)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// What a wait is asked for is checked before anything is read: what it waits until, and a bound no longer than
|
||||
// a call is answered within.
|
||||
func TestAWaitIsRefusedWhatItCannotDo(t *testing.T) {
|
||||
for _, bad := range []struct{ until, bound string }{{"landed", ""}, {"", "1h"}, {"", "-3s"}, {"", "soon"}} {
|
||||
if _, _, err := planWaitAsked(bad.until, bad.bound); err == nil {
|
||||
t.Errorf("until %q, bound %q was taken", bad.until, bad.bound)
|
||||
}
|
||||
}
|
||||
until, bound, err := planWaitAsked("", "")
|
||||
if err != nil || until != "moved" || bound != maxPlanWait {
|
||||
t.Fatalf("the defaults are %q %s %v, want moved and %s", until, bound, err, maxPlanWait)
|
||||
}
|
||||
if _, bound, _ := planWaitAsked("tier", "12"); bound != 12*time.Second {
|
||||
t.Errorf("a bare number is seconds; read %s", bound)
|
||||
}
|
||||
}
|
||||
|
||||
// **The answer is data, with its prose as a field beside it** (novox/hq issue 229): the verb's `answer` carries
|
||||
// the plan and what ended the wait, and its output is the line a person reads — never JSON inside a string.
|
||||
func TestAWaitForAPlanAnswersAsData(t *testing.T) {
|
||||
p := followedPlan(7, inventory.PlanRolling, 1)
|
||||
st := planStatuses([]inventory.Plan{p}, time.Now(), pauseView{}, tierBounds{})[0]
|
||||
doc := planWaitedDoc(p, st, "tier", "tier", 3*time.Second, "rolling, tier 2 of 2")
|
||||
stdout, _ := json.Marshal(doc)
|
||||
a := answerOf([]string{"plans", "wait", "plan-1"}, stdout, "", true)
|
||||
data, ok := a.Answer.(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("the answer is not data: %v", a.Answer)
|
||||
}
|
||||
if data["came"] != "tier" || data["revision"] != float64(7) {
|
||||
t.Errorf("the data does not say what ended the wait: %v", data)
|
||||
}
|
||||
if plan, _ := data["plan"].(map[string]any); plan["state"] != "rolling" || plan["tier"] != float64(1) {
|
||||
t.Errorf("the data does not carry the plan: %v", data["plan"])
|
||||
}
|
||||
if _, text := data["text"]; text {
|
||||
t.Errorf("the prose is in the data too: %v", data)
|
||||
}
|
||||
if strings.Contains(a.Output, "{") || !strings.Contains(a.Output, "plan-1") || !strings.Contains(a.Output, "tier") {
|
||||
t.Errorf("the output is not the line a person reads: %q", a.Output)
|
||||
}
|
||||
}
|
||||
|
||||
// The verb composes the wait, and asks for a window as long as its bound.
|
||||
func TestThePlansVerbComposesAWait(t *testing.T) {
|
||||
argv, err := argvFor("plans", map[string]any{"wait": "plan-1", "until": "done", "bound": "15s"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strings.Join(argv, " "); got != "plans wait plan-1 --until done --bound 15s" {
|
||||
t.Fatalf("composed %q", got)
|
||||
}
|
||||
if _, err := argvFor("plans", map[string]any{"until": "done"}); err == nil {
|
||||
t.Error("until without wait was taken")
|
||||
}
|
||||
if _, err := argvFor("plans", map[string]any{"wait": "plan-1", "bound": "1h"}); err == nil {
|
||||
t.Error("a bound past what a call can wait was taken")
|
||||
}
|
||||
}
|
||||
|
||||
// Live, with the store: `plans wait` run as the verb runs it answers when the plan reaches its next tier, as
|
||||
// data, and says the bound passed when nothing moved (novox/hq issue 229).
|
||||
func TestAPlanFollowedFromTheStoreAnswersWhenItMoves(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
now := time.Now().UTC()
|
||||
walk := inventory.Plan{ID: "plan-229", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c229c229",
|
||||
Created: now, State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {}, "b": {}}}
|
||||
if err := open.inventory.SavePlan(ctx, &walk); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
moved := make(chan error, 1)
|
||||
go func() {
|
||||
time.Sleep(1500 * time.Millisecond)
|
||||
p, err := open.inventory.PlanByID(ctx, walk.ID)
|
||||
if err == nil {
|
||||
p.Tier, p.State = 1, inventory.PlanRolling
|
||||
err = open.inventory.SavePlan(ctx, &p)
|
||||
}
|
||||
moved <- err
|
||||
}()
|
||||
began := time.Now()
|
||||
var got planWaited
|
||||
out := printed(t, func() error {
|
||||
return plansCommand(ctx, []string{"wait", walk.ID, "--until", "tier", "--bound", "15s"})
|
||||
})
|
||||
if err := <-moved; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal([]byte(out), &got); err != nil {
|
||||
t.Fatalf("plans wait did not answer data: %v: %q", err, out)
|
||||
}
|
||||
if got.Came != "tier" || got.Plan.Tier != 1 || got.Plan.State != inventory.PlanRolling {
|
||||
t.Fatalf("the wait answered %+v", got)
|
||||
}
|
||||
if waited := time.Since(began); waited > 10*time.Second {
|
||||
t.Errorf("it answered after %s, not when the plan moved", waited)
|
||||
}
|
||||
out = printed(t, func() error {
|
||||
return plansCommand(ctx, []string{"wait", walk.ID, "--bound", "2"})
|
||||
})
|
||||
if err := json.Unmarshal([]byte(out), &got); err != nil || got.Came != "bound" || got.Until != "moved" {
|
||||
t.Fatalf("a plan that did not move answered %+v (%v)", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The verb's schema says the longest bound in words, and the words are the value (review 75 on #266): the
|
||||
// catalogue cannot read maxPlanWait, so this holds them together.
|
||||
func TestThePlansSchemaSaysTheLongestBoundThereIs(t *testing.T) {
|
||||
v, ok := controllerVerb("plans")
|
||||
if !ok {
|
||||
t.Fatal("no plans verb")
|
||||
}
|
||||
props, _ := v.Input["properties"].(map[string]any)
|
||||
bound, _ := props["bound"].(map[string]any)
|
||||
said, _ := bound["description"].(string)
|
||||
if !strings.Contains(said, maxPlanWait.String()+", the default, is the longest") {
|
||||
t.Fatalf("the schema says %q; the longest bound is %s", said, maxPlanWait)
|
||||
}
|
||||
}
|
||||
@@ -1027,9 +1027,6 @@ func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (stri
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// A send that carried the mesh's record of the controller and the node-engine — a person's push included —
|
||||
// ends the condition of one kept over it (novox/hq issue 201).
|
||||
settleCoreDowngrades(ctx, b.open.inventory, s.node, s.declared.Builds)
|
||||
if len(s.declared.Bindings) > 0 {
|
||||
// And where it bound each consumer of a provision that keeps its data (novox/hq ADR 0232):
|
||||
// what the next resolution keeps it at. On the same outliving context as the send's record.
|
||||
|
||||
@@ -135,17 +135,6 @@ func recordedKept(ctx context.Context, open *stores, node string) (map[string]st
|
||||
}
|
||||
out[m] = was
|
||||
}
|
||||
// **And the controller and the node-engine at the build the machine was last sent, over an older record**
|
||||
// (novox/hq issue 201, core_downgrade.go). Under keepingEveryBuild they are kept already.
|
||||
if !every {
|
||||
core, err := coreKept(ctx, inv, node, sent, current)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for m, was := range core {
|
||||
out[m] = was
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -170,12 +159,6 @@ func keepRecorded(ctx context.Context, open *stores, node string, shelf map[stri
|
||||
return nil, fmt.Errorf("%s runs %s's build %s, which the build records no longer hold: the bus's user "+
|
||||
"list cannot be sent there alone with it kept (novox/hq issue 490)", node, m, short(kept[m]))
|
||||
}
|
||||
if !found && guarded(coreComponent(m)) {
|
||||
return nil, fmt.Errorf("%s was last sent %s's build %s, newer than the one the mesh holds, and the build records "+
|
||||
"no longer hold it: this send cannot keep it and does not move it back — `push %s` sends the older build on "+
|
||||
"a person's word, and a newer merge or `rebuild %s` moves the record forward (novox/hq issue 201)",
|
||||
node, m, short(kept[m]), node, m)
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("%s records rather than rolls out, and %s runs its build %s, which the build "+
|
||||
"records no longer hold: this send cannot keep it and does not move it — `push %s` sends the new "+
|
||||
|
||||
@@ -51,9 +51,6 @@ type sendScope struct {
|
||||
modules map[string]bool
|
||||
// person is a person's act: it carries what it carries.
|
||||
person bool
|
||||
// puttingBack is a gate's own put-back of its modules: the one send that may carry a core module to a build
|
||||
// older than the one its machine runs (novox/hq issue 201).
|
||||
puttingBack bool
|
||||
}
|
||||
|
||||
type sendScopeKey struct{}
|
||||
@@ -240,32 +237,6 @@ var machineMoves = func(ctx context.Context, open *stores, f moveFacts, node str
|
||||
return f.moves(node, modules, sent, known, all), nil
|
||||
}
|
||||
|
||||
// withoutKept is a machine's moves less those its send keeps at the build the machine was last sent, and so does
|
||||
// not make: the grants step keeps every module (novox/hq issue 490), and every send keeps the controller and the
|
||||
// node-engine over an older record (issue 201). Read as the send composes it (sendKeeps, set after the checks that
|
||||
// call this), so a move it could not keep is still counted.
|
||||
func withoutKept(ctx context.Context, open *stores, node string, moves []inventory.CarriedMove) ([]inventory.CarriedMove, error) {
|
||||
if len(moves) == 0 {
|
||||
return moves, nil
|
||||
}
|
||||
keeping := ctx
|
||||
if _, on := keptExcept(ctx); !on {
|
||||
keeping = keepingRecorded(ctx)
|
||||
}
|
||||
keeps, err := recordedKept(keeping, open, node)
|
||||
if err != nil || len(keeps) == 0 {
|
||||
return moves, err
|
||||
}
|
||||
var out []inventory.CarriedMove
|
||||
for _, mv := range moves {
|
||||
if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) {
|
||||
continue
|
||||
}
|
||||
out = append(out, mv)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ungatedIn refuses a send whose machines would move a build no gate has seen, outside its scope: the
|
||||
// machines named, and why; the machine holding the bus, added only for its user list, is left instead.
|
||||
func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder string) ([]string, error) {
|
||||
@@ -283,11 +254,19 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if moves, err = withoutKept(ctx, open, n, moves); err != nil {
|
||||
return nil, err
|
||||
// The grants step composes every module at the build its machine was last sent (novox/hq issue 490):
|
||||
// a move it keeps is not made. Read, not assumed, so a move it could not keep is still refused here.
|
||||
var keeps map[string]string
|
||||
if everyBuildKept(ctx, n) {
|
||||
if keeps, err = recordedKept(ctx, open, n); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var waiting []string
|
||||
for _, mv := range moves {
|
||||
if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) {
|
||||
continue
|
||||
}
|
||||
if !scope.judged[n] && !scope.modules[mv.Module] {
|
||||
waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To)))
|
||||
}
|
||||
@@ -329,11 +308,6 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
// A move the send keeps is not made, and is no move for the gate to judge (novox/hq issue 201): a core module
|
||||
// kept over an older record judged as moving back to it would read the gate as superseded by its own send.
|
||||
if moves, err = withoutKept(ctx, open, node, moves); err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
own := func(module string) bool {
|
||||
return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module })
|
||||
}
|
||||
|
||||
@@ -1463,10 +1463,6 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
modules := set.String("modules", "", "or the modules it would change, comma-separated")
|
||||
moduleDirs := set.String("module-dirs", "", "with --paths: the directories holding a module.json at the commit, "+
|
||||
"comma-separated, as the forge's announcer says them (novox/hq issue 278)")
|
||||
// Following a plan (novox/hq issue 229): `plans wait <id>`.
|
||||
until := set.String("until", "", "with wait: what to wait for — moved (any change), tier (its next tier) or done (its end)")
|
||||
bound := set.String("bound", "", "with wait: how long to wait at most, as 15s or seconds (default and at most the "+
|
||||
"longest a call is answered within)")
|
||||
// Ending a plan by hand is a repair, and says why (novox/hq to-be 45 §7).
|
||||
why := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args)
|
||||
@@ -1486,9 +1482,6 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
now := time.Now()
|
||||
if len(positionals) == 2 && positionals[0] == "wait" {
|
||||
return planWaitCommand(ctx, inv, positionals[1], *until, *bound)
|
||||
}
|
||||
if len(positionals) == 1 {
|
||||
p, err := inv.PlanByID(ctx, positionals[0])
|
||||
if err != nil {
|
||||
|
||||
@@ -1,83 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// TestReplay401 replays novox/hq issue 401 (2026-10-10): the mounts module's smb-credentials, an own secret
|
||||
// only an outside party issues, was given by nobody, and the send made a random value for it and placed it on
|
||||
// the machine; the module read it as a credential given and wrong. The outcome asserted (ADR 0228): the send
|
||||
// makes no value for an own secret issued outside the mesh that nobody gave and places none, and a value the
|
||||
// mesh made for it before is not placed either; once a person gives one, that value is placed. Written with
|
||||
// only what the controller had before its fix (mesh-controller #262) — register, assign, planFor,
|
||||
// declarationFor and the inventory's secret reads — so it is laid over the commit before. Needs the store
|
||||
// (MESH_TEST_POSTGRES), and is named in testdata/store-tests.
|
||||
func TestReplay401(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
const made, outside = "/var/lib/mesh/mounts/password", "/var/lib/mesh/mounts/smb-credentials"
|
||||
register(t, open, catalogue.Manifest{Module: "mounts", Version: "1",
|
||||
OwnSecrets: catalogue.OwnSecrets{
|
||||
"password": {Path: made, Taken: catalogue.TakenAtStart},
|
||||
"smb-credentials": {Path: outside, Taken: catalogue.TakenAtStart, IssuedBy: catalogue.IssuedOutside},
|
||||
},
|
||||
Resources: []map[string]any{
|
||||
{"id": "state", "type": "directory", "path": "/var/lib/mesh/mounts", "mode": "0700"},
|
||||
}})
|
||||
if _, err := assign(ctx, open, "laptop", "mounts"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sent := func() []map[string]any {
|
||||
t.Helper()
|
||||
plan, settings, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, "laptop", plan, settings)
|
||||
if err != nil {
|
||||
t.Fatalf("the send to laptop did not compose: %v", err)
|
||||
}
|
||||
return declared.Resources
|
||||
}
|
||||
placed := func(resources []map[string]any, path string) map[string]any {
|
||||
for _, r := range resources {
|
||||
if r["path"] == path && r["sealed"] != nil {
|
||||
return r
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
first := sent()
|
||||
if placed(first, made) == nil {
|
||||
t.Fatal("the secret the mesh makes was not placed")
|
||||
}
|
||||
if got := placed(first, outside); got != nil {
|
||||
t.Errorf("the send placed a value for smb-credentials, which nobody gave: %v", got)
|
||||
}
|
||||
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "mounts", "smb-credentials"); err != nil || held {
|
||||
t.Fatalf("the send made a value for a secret issued outside the mesh (held %v, %v)", held, err)
|
||||
}
|
||||
|
||||
// A value the mesh made for it before the rule is not one a person gave: it is not placed.
|
||||
if _, err := open.inventory.SecretForModule(ctx, "laptop", "mounts", "smb-credentials"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := placed(sent(), outside); got != nil {
|
||||
t.Fatalf("a value the mesh made for a secret issued outside was placed: %v", got)
|
||||
}
|
||||
|
||||
// Given by a person, it is placed, as given.
|
||||
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "mounts", "smb-credentials", "the share's credentials"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
given, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "mounts", "smb-credentials")
|
||||
if err != nil || !held {
|
||||
t.Fatalf("the value given is not held (%v)", err)
|
||||
}
|
||||
if got := placed(sent(), outside); got == nil || got["sealed"] != given {
|
||||
t.Fatalf("the value given is placed as %v", got)
|
||||
}
|
||||
}
|
||||
@@ -34,8 +34,10 @@ type seatRow struct {
|
||||
Delivers string `json:"delivers,omitempty"`
|
||||
Decision string `json:"decision"`
|
||||
// Replicated says the seat may be held on several machines at once (novox/hq ADR 0223).
|
||||
Replicated bool `json:"replicated,omitempty"`
|
||||
Holders []seatHolder `json:"holders"`
|
||||
Replicated bool `json:"replicated,omitempty"`
|
||||
// Paths are the paths on its machine the seat's holder owns (novox/hq ADR 0287).
|
||||
Paths []catalogue.SeatPath `json:"paths,omitempty"`
|
||||
Holders []seatHolder `json:"holders"`
|
||||
}
|
||||
|
||||
// seatsHeld is every seat the mesh defines with its holders, and every claim held that names no
|
||||
@@ -49,7 +51,7 @@ func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []cata
|
||||
rows := make([]seatRow, 0, len(seats))
|
||||
for _, s := range seats {
|
||||
row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
|
||||
Replicated: s.Replicated, Holders: []seatHolder{}}
|
||||
Replicated: s.Replicated, Paths: s.Paths, Holders: []seatHolder{}}
|
||||
seen := map[seatHolder]bool{}
|
||||
for _, h := range held {
|
||||
// Resolve the held claim to a seat rather than comparing names, so a record naming a
|
||||
@@ -255,12 +257,16 @@ func seatsCommand(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
rows, outside := seatsHeld(catalogue.Seats(), world.Held)
|
||||
// Which module owns each seat's path on each machine (novox/hq ADR 0287): the seat naming it, and
|
||||
// that seat's holder there, with the stepping back applied.
|
||||
owners := catalogue.PathOwners(catalogue.Seats(), world.Held)
|
||||
|
||||
if *asJSON {
|
||||
out := struct {
|
||||
Seats []seatRow `json:"seats"`
|
||||
Outside []catalogue.Held `json:"outside,omitempty"`
|
||||
}{rows, outside}
|
||||
Seats []seatRow `json:"seats"`
|
||||
Owners []catalogue.PathOwner `json:"path-owners,omitempty"`
|
||||
Outside []catalogue.Held `json:"outside,omitempty"`
|
||||
}{rows, owners, outside}
|
||||
body, err := json.MarshalIndent(out, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -270,7 +276,7 @@ func seatsCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
|
||||
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintln(w, "SEAT\tSCOPE\tDELIVERS\tHELD BY")
|
||||
fmt.Fprintln(w, "SEAT\tSCOPE\tDELIVERS\tPATHS\tHELD BY")
|
||||
for _, r := range rows {
|
||||
delivers := r.Delivers
|
||||
if delivers == "" {
|
||||
@@ -284,11 +290,22 @@ func seatsCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
holders = strings.Join(parts, ", ")
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", r.Seat, r.Scope, delivers, holders)
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", r.Seat, r.Scope, delivers, seatPathsText(r.Paths), holders)
|
||||
}
|
||||
if err := w.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(owners) > 0 {
|
||||
fmt.Println("\nwho owns each seat's path, by machine (novox/hq ADR 0287):")
|
||||
w = tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintln(w, "PATH\tMACHINE\tSEAT\tMODULE")
|
||||
for _, o := range owners {
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", o.Path, o.Node, o.Seat, o.Module)
|
||||
}
|
||||
if err := w.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(outside) > 0 {
|
||||
fmt.Println("\nheld, and not a seat this mesh defines (registered before the set closed — novox/hq ADR 0110):")
|
||||
for _, h := range outside {
|
||||
@@ -297,3 +314,19 @@ func seatsCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// seatPathsText is a seat's paths in one column: each path, and the seat it steps back for, if any.
|
||||
func seatPathsText(paths []catalogue.SeatPath) string {
|
||||
if len(paths) == 0 {
|
||||
return "—"
|
||||
}
|
||||
parts := make([]string, 0, len(paths))
|
||||
for _, p := range paths {
|
||||
if p.StepsBackFor != "" {
|
||||
parts = append(parts, p.Path+" (steps back for "+p.StepsBackFor+")")
|
||||
continue
|
||||
}
|
||||
parts = append(parts, p.Path)
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
@@ -74,3 +74,22 @@ func TestAKindedBenchIsNotHandedOver(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The listing shows each seat's paths, and the seat a path steps back for (novox/hq ADR 0287).
|
||||
func TestTheSeatsListingShowsEachSeatsPaths(t *testing.T) {
|
||||
rows, _ := seatsHeld(catalogue.Seats(), nil)
|
||||
got := map[string]string{}
|
||||
for _, r := range rows {
|
||||
got[r.Seat] = seatPathsText(r.Paths)
|
||||
}
|
||||
for seat, want := range map[string]string{
|
||||
"node-hostname": "/etc/hosts, /etc/hostname",
|
||||
"node-uplink": "/etc/resolv.conf (steps back for node-resolver)",
|
||||
"node-resolver": "/etc/resolv.conf",
|
||||
"node-packet-filter": "—",
|
||||
} {
|
||||
if got[seat] != want {
|
||||
t.Errorf("%s lists its paths as %q, not %q", seat, got[seat], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -336,9 +336,6 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
}
|
||||
return argv, nil
|
||||
case "plans":
|
||||
if err := plainPlanIDs(a.given); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r := str("repository"); r != "" {
|
||||
argv := []string{"plans", "--what-if", r}
|
||||
if p := str("paths"); p != "" {
|
||||
@@ -368,21 +365,6 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
return argv, nil
|
||||
}
|
||||
}
|
||||
if id := str("wait"); id != "" {
|
||||
// Following a plan (novox/hq issue 229): checked here too, so a call asking what no wait can do is
|
||||
// refused before a command is started.
|
||||
if _, _, err := planWaitAsked(a.given["until"], a.given["bound"]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"plans", "wait", id}
|
||||
if u := str("until"); u != "" {
|
||||
argv = append(argv, "--until", u)
|
||||
}
|
||||
if b := str("bound"); b != "" {
|
||||
argv = append(argv, "--bound", b)
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
if id := str("id"); id != "" {
|
||||
return []string{"plans", id}, nil
|
||||
}
|
||||
@@ -1003,14 +985,12 @@ var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "co
|
||||
// The delivery's owner's verbs answer JSON where they read (plan, order, check, walks) — novox/hq ADR 0239.
|
||||
"delivery": true,
|
||||
// The build queue, each ask with its position among the waiting (novox/hq issue 455).
|
||||
"queue": true,
|
||||
// A plan followed (novox/hq issue 229): `plans wait` answers JSON; its other forms say text, and so answer no data.
|
||||
"plans": true}
|
||||
"queue": true}
|
||||
|
||||
// tableBesideVerbs are the JSON verbs whose document carries the table a person reads as its `text`: the
|
||||
// verb answers that table as its output, as it did before it answered data, and the rest as `answer` —
|
||||
// both from the one read, so they never disagree (novox/hq issue 455).
|
||||
var tableBesideVerbs = map[string]bool{"queue": true, "plans": true}
|
||||
var tableBesideVerbs = map[string]bool{"queue": true}
|
||||
|
||||
// overviewVerbs are the JSON verbs whose answer is said once, as data, and not again as the text the
|
||||
// command printed: the overviews, which grow with the mesh (novox/hq issue 314). Every reader of theirs
|
||||
@@ -1284,16 +1264,6 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
// Whatever it did, `status` is composed again once it has.
|
||||
defer statusFrom.nudge()
|
||||
}
|
||||
if verb == "plans" && len(argv) > 1 && argv[1] == "wait" {
|
||||
// A wait holds its answer up to its bound (novox/hq issue 229), and is answered in full inside it.
|
||||
asked := ""
|
||||
if i := slices.Index(argv, "--bound"); i > 0 && i+1 < len(argv) {
|
||||
asked = argv[i+1]
|
||||
}
|
||||
if _, bound, err := planWaitAsked("", asked); err == nil {
|
||||
link.AnswersWithin(ctx, bound+planWaitStart)
|
||||
}
|
||||
}
|
||||
if answer, read := readHere(ctx, argv); read {
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
@@ -254,16 +254,6 @@ var signalsTable = []signalRow{
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.bus.waits, func(w busWaitFacts) time.Time { return w.since })
|
||||
}},
|
||||
{Row: "S21", Signal: "a clean apply", Emitter: "node-engine", Trigger: "each report of a declaration refused or applied only in part (novox/hq issue 233)",
|
||||
Bound: "one apply interval (5 min) past the first report of the same failure, judged by time while the machine " +
|
||||
"is heard — the node-engine says a re-apply only when how it went changed; not while it is silent or " +
|
||||
"asleep; naming the refusal, or the package manager that does not answer with its words; cleared by the " +
|
||||
"clean apply it then says",
|
||||
Kind: kindApplyFailing, Severity: conditions.Warning, Phase: 2,
|
||||
needs: func(f *signalFacts) error { return f.doingErr }, watch: watchApplyFailing,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.doing, func(d inventory.Doing) time.Time { return d.At })
|
||||
}},
|
||||
}
|
||||
|
||||
// watchFacts is S14: the snapshot a merge check is fed is older than its bound, or none was kept since
|
||||
|
||||
@@ -186,14 +186,6 @@ var suppressions = map[string]suppression{
|
||||
},
|
||||
past: func(f *signalFacts) { f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-time.Second))} },
|
||||
},
|
||||
// A machine whose package manager does not answer (novox/hq issue 233): said once its first report of the
|
||||
// failure is older than one apply interval while it is heard. Inside: exactly five minutes old.
|
||||
"S21": {
|
||||
inside: func(f *signalFacts) { f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute, 1)} },
|
||||
past: func(f *signalFacts) {
|
||||
f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute+time.Second, 1)}
|
||||
},
|
||||
},
|
||||
// Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not.
|
||||
"S15": {
|
||||
inside: func(f *signalFacts) {
|
||||
|
||||
@@ -59,10 +59,6 @@ type signalFacts struct {
|
||||
// from, each naming its sender (S20, novox/hq issue 234).
|
||||
refusedSends []inventory.Send
|
||||
refusedSendsErr error
|
||||
// doing is every machine whose last report was not a clean apply, with since when it has said the same
|
||||
// failure (S21, novox/hq issue 233).
|
||||
doing []inventory.Doing
|
||||
doingErr error
|
||||
|
||||
loop loopFacts
|
||||
loopErr error
|
||||
@@ -373,7 +369,6 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
|
||||
}
|
||||
f.handActs, f.handActsErr = w.gatherHandActs(ctx, now)
|
||||
f.refusedSends, f.refusedSendsErr = inv.RefusedSendsSince(ctx, now.Add(-generationRefusalsSaid))
|
||||
f.doing, f.doingErr = inv.NotDoingWhatTheyWereTold(ctx)
|
||||
f.facts.taken, _, f.facts.began, f.facts.err = exportedFacts.last()
|
||||
return f
|
||||
}
|
||||
|
||||
@@ -8,7 +8,6 @@ require (
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
github.com/novox/mesh-host v0.0.0
|
||||
github.com/therootcompany/xz v1.0.1
|
||||
golang.org/x/crypto v0.57.0
|
||||
golang.org/x/net v0.58.0
|
||||
golang.org/x/sys v0.48.0
|
||||
|
||||
@@ -38,8 +38,6 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
|
||||
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||
|
||||
@@ -49,10 +49,8 @@ import (
|
||||
//
|
||||
// One check:
|
||||
//
|
||||
// 1. clones the repository at the pull request's head and **makes the merge of that head into the trunk's
|
||||
// head**, the check's trunk commit, in its own clone (novox/hq ADR 0323, issue 341; check_merge.go): what
|
||||
// is judged is what merging makes, never pushed. Beside it the repositories its check reads — the
|
||||
// controller the mesh runs, the catalogue, the host, the lab — each at the ref asked;
|
||||
// 1. clones the repository at the pull request's head, and beside it the repositories its check
|
||||
// reads — the controller the mesh runs, the catalogue, the host, the lab — each at the ref asked;
|
||||
// 2. reads the facts snapshot the controller keeps, and with it **the versions the mesh runs**: the
|
||||
// store a check's tests stand on is the store's own release, and the bus the bus's;
|
||||
// 3. raises a throwaway PostgreSQL and a throwaway bus of those versions, labelled with the ask so a
|
||||
@@ -81,8 +79,7 @@ type CheckSpec struct {
|
||||
Modules []string
|
||||
New []string
|
||||
Manifests []string
|
||||
// Base is the branch the pull request merges into: what the gate compares the change against, and the trunk
|
||||
// whose head the change is merged into before it is judged (novox/hq ADR 0323). Empty judges the head alone.
|
||||
// Base is the branch the pull request merges into: what the gate compares the change against.
|
||||
Base string
|
||||
// Judge is who judges the gate: "" the controller the mesh runs, "self" the change's own, "validator"
|
||||
// the running one with the change's validator.
|
||||
@@ -107,9 +104,6 @@ type GroupHead struct {
|
||||
Repository string
|
||||
Ref string
|
||||
Paths []string
|
||||
// Base is the branch it merges into: the head is merged into that trunk's head before it is composed
|
||||
// (novox/hq ADR 0323 §6). Empty composes the head alone, as a controller from before it asks.
|
||||
Base string
|
||||
}
|
||||
|
||||
// Gated is whether the change touches the mesh's graph, and so whether the gate runs.
|
||||
@@ -145,16 +139,6 @@ type CheckVerdict struct {
|
||||
Took time.Duration
|
||||
Gate *Layer
|
||||
Repo *Layer
|
||||
// Head and Trunk are what was judged (novox/hq ADR 0323 §3): the pull request's head merged into the trunk
|
||||
// commit, each as twelve characters. Trunk is empty for a check asked with no trunk, which judged the head
|
||||
// alone. Merge is the merge commit made for the check, whole: only checked, never pushed or registered.
|
||||
Head, Trunk, Merge string
|
||||
// Beside are the commits of the repositories the check read beside its own, by the directory each was
|
||||
// cloned under, as twelve characters: named, not judged against (ADR 0323 §3, issue 568).
|
||||
Beside map[string]string
|
||||
// Members are a delivery group's other heads as the composed check judged them, each with the trunk
|
||||
// commit it was merged into (ADR 0323 §6).
|
||||
Members []JudgedHead
|
||||
}
|
||||
|
||||
// Layer is one layer of a check, judged.
|
||||
@@ -297,34 +281,6 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
return CheckVerdict{}, err
|
||||
}
|
||||
tree := filepath.Join(root, name)
|
||||
// **What merging makes is judged, not the head** (novox/hq ADR 0323, issue 341): the head merged into the
|
||||
// trunk's head as this clone found it, in this clone alone. A head that does not merge cleanly fails here.
|
||||
judged := CheckVerdict{Head: twelve(spec.Ref)}
|
||||
paths := spec.Paths
|
||||
if spec.Base != "" {
|
||||
m, err := mergeIntoTrunk(ctx, run, tree, spec.Base, spec.Ref)
|
||||
if err != nil {
|
||||
return CheckVerdict{}, err
|
||||
}
|
||||
judged.Head, judged.Trunk = twelve(m.Head), twelve(m.Trunk)
|
||||
if len(m.Conflicts) > 0 {
|
||||
said := conflictWords(m)
|
||||
say("check", "%s", said)
|
||||
return CheckVerdict{Verdict: "fail", Summary: said, Report: said, Took: time.Since(began),
|
||||
Gate: &Layer{Verdict: "fail", Summary: said}, Repo: &Layer{Verdict: "fail", Summary: said},
|
||||
Head: judged.Head, Trunk: judged.Trunk}, nil
|
||||
}
|
||||
judged.Merge = m.Merge
|
||||
if changed := changedByMerge(ctx, run, tree, m); len(changed) > 0 {
|
||||
paths = changed
|
||||
}
|
||||
say("check", "head %s merged into trunk %s (%s): the merge is judged, and never pushed", judged.Head,
|
||||
judged.Trunk, spec.Base)
|
||||
}
|
||||
named := func(v CheckVerdict) CheckVerdict {
|
||||
v.Head, v.Trunk, v.Merge, v.Beside, v.Members = judged.Head, judged.Trunk, judged.Merge, judged.Beside, judged.Members
|
||||
return v
|
||||
}
|
||||
// **The base branch's check judges the change** (novox/hq issue 310), never the one the change brings.
|
||||
judging, err := TheCheckThatJudges(ctx, tree, spec.Base)
|
||||
if err != nil {
|
||||
@@ -341,7 +297,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
v := CheckVerdict{Verdict: "pass", Summary: noModule, Took: time.Since(began),
|
||||
Gate: &Layer{Verdict: "pass", Summary: noModule}, Repo: &Layer{Verdict: "warning", Summary: noScript}}
|
||||
say("check", "%s; %s", noModule, noScript)
|
||||
return named(v), nil
|
||||
return v, nil
|
||||
}
|
||||
for dir, b := range spec.Beside {
|
||||
if dir == name || !safeName.MatchString(dir) {
|
||||
@@ -350,13 +306,6 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
if err := clone(b.Repository, b.Ref, dir); err != nil {
|
||||
return CheckVerdict{}, fmt.Errorf("beside it, %w", err)
|
||||
}
|
||||
// The commit each one read is named in the verdict (ADR 0323 §3): its move is issue 568's question.
|
||||
if at := headAt(ctx, run, filepath.Join(root, dir)); at != "" {
|
||||
if judged.Beside == nil {
|
||||
judged.Beside = map[string]string{}
|
||||
}
|
||||
judged.Beside[dir] = at
|
||||
}
|
||||
say("check", "beside it %s at %s", dir, short(b.Ref))
|
||||
}
|
||||
|
||||
@@ -373,26 +322,6 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
return CheckVerdict{}, fmt.Errorf("a head of the group, %w", err)
|
||||
}
|
||||
say("check", "with it, of its group, %s/%s at %s", g.Owner, g.Repo, short(g.Ref))
|
||||
// Each member is composed as its merge into its own trunk's head (ADR 0323 §6).
|
||||
member := JudgedHead{Owner: g.Owner, Repo: g.Repo, Head: twelve(g.Ref)}
|
||||
if g.Base != "" {
|
||||
m, err := mergeIntoTrunk(ctx, run, filepath.Join(root, dir), g.Base, g.Ref)
|
||||
if err != nil {
|
||||
return CheckVerdict{}, fmt.Errorf("a head of the group, %s/%s: %w", g.Owner, g.Repo, err)
|
||||
}
|
||||
member.Head, member.Trunk = twelve(m.Head), twelve(m.Trunk)
|
||||
if len(m.Conflicts) > 0 {
|
||||
said := fmt.Sprintf("%s/%s: %s", g.Owner, g.Repo, conflictWords(m))
|
||||
say("check", "%s", said)
|
||||
judged.Members = append(judged.Members, member)
|
||||
return named(CheckVerdict{Verdict: "fail", Summary: said, Report: said, Took: time.Since(began),
|
||||
Gate: &Layer{Verdict: "fail", Summary: said}, Repo: &Layer{Verdict: "fail", Summary: said}}), nil
|
||||
}
|
||||
if changed := changedByMerge(ctx, run, filepath.Join(root, dir), m); len(changed) > 0 {
|
||||
g.Paths = changed
|
||||
}
|
||||
}
|
||||
judged.Members = append(judged.Members, member)
|
||||
heads = append(heads, map[string]any{"repository": g.Owner + "/" + g.Repo,
|
||||
"tree": filepath.Join(root, dir), "changed": g.Paths})
|
||||
}
|
||||
@@ -504,7 +433,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
verdictFile := filepath.Join(root, "verdict.json")
|
||||
env := []string{EnvFacts + "=" + factsFile, EnvGate + "=" + gate, EnvGateStore + "=" + storeURL,
|
||||
EnvTestStore + "=" + storeURL, EnvTestBus + "=nats://" + bus, EnvRepository + "=" + spec.Owner + "/" + spec.Repo,
|
||||
EnvChanged + "=" + strings.Join(paths, ","), EnvBeside + "=" + root,
|
||||
EnvChanged + "=" + strings.Join(spec.Paths, ","), EnvBeside + "=" + root,
|
||||
EnvModules + "=" + strings.Join(append(append([]string{}, spec.Modules...), spec.New...), ","),
|
||||
EnvGroup + "=" + groupFile,
|
||||
"GOCACHE=" + filepath.Join(workspace, "go-cache"), "GOMODCACHE=" + filepath.Join(workspace, "go-modules")}
|
||||
@@ -572,7 +501,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
v.Report, v.Took = redact.redact(withWhatFailed(out.String(), v.Repo)), time.Since(began)
|
||||
say("check", "gate %s — %s; repository %s — %s (%s)", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary,
|
||||
strings.ToUpper(v.Repo.Verdict), v.Repo.Summary, v.Took.Round(time.Second))
|
||||
return named(v), nil
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// ownCheck runs the repository's own check, part by part, each in its toolchain's own container: the
|
||||
|
||||
@@ -1,115 +0,0 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// **A merge check judges the merge, not the head** (novox/hq ADR 0323, issue 341).
|
||||
//
|
||||
// On 2026-10-09 a node-engine pull request was checked green at its head and broke the trunk when it merged:
|
||||
// a merge an hour earlier had added a field under the same JSON key on other lines, git merged the two edits
|
||||
// without a conflict, and the result did not compile. The check had judged the head alone. So a check now
|
||||
// takes the trunk the pull request merges into, its head at the moment the check clones it as the check's
|
||||
// **trunk commit**, and makes in its own clone the merge of the head into that commit, as the forge's
|
||||
// default merge would. The gate and the repository's own check judge that merge.
|
||||
//
|
||||
// **The merge is only checked** (ADR 0323 §2, ADR 0238 §1): it is a commit off the trunk, made in the
|
||||
// check's own clone, never pushed to the forge and never registered; the clone is removed when the check
|
||||
// ends. Nothing here writes to a remote.
|
||||
//
|
||||
// The forge keeps no merge reference for a pull request, only its head, so the builder makes it itself.
|
||||
|
||||
// TrunkMerge is the merge a check made: the head and the trunk commit it was merged into, whole, and the merge
|
||||
// commit; or the files in conflict when the head does not merge cleanly.
|
||||
type TrunkMerge struct {
|
||||
Head, Trunk, Merge string
|
||||
Conflicts []string
|
||||
}
|
||||
|
||||
// JudgedHead is one head a check judged, with the trunk commit it merged that head into, each as twelve
|
||||
// characters (ADR 0323 §3, §6).
|
||||
type JudgedHead struct {
|
||||
Owner, Repo string
|
||||
Head, Trunk string
|
||||
}
|
||||
|
||||
// mergeIdentity is who a check's merge is made as: it is never pushed, so it names nobody's account. No hook of the
|
||||
// clone's or the build machine's git configuration runs, and no setting there changes the merge.
|
||||
var mergeIdentity = []string{"-c", "user.name=mesh merge check", "-c", "user.email=merge-check@mesh.invalid",
|
||||
"-c", "commit.gpgsign=false", "-c", "core.hooksPath=/dev/null", "-c", "merge.verifySignatures=false"}
|
||||
|
||||
// mergeIntoTrunk makes, in the clone at tree, the merge of head into the head of origin/<base> as it stands in
|
||||
// that fresh clone: the trunk commit. A head that does not merge cleanly answers the files in conflict and leaves
|
||||
// the tree at the trunk commit; an error is that the merge could not be tried at all.
|
||||
func mergeIntoTrunk(ctx context.Context, run Runner, tree, base, head string) (TrunkMerge, error) {
|
||||
var m TrunkMerge
|
||||
revParse := func(ref string) (string, error) {
|
||||
out, err := run(ctx, tree, "git", "rev-parse", "--verify", "--quiet", ref+"^{commit}")
|
||||
return strings.TrimSpace(out), err
|
||||
}
|
||||
var err error
|
||||
if m.Head, err = revParse(head); err != nil || m.Head == "" {
|
||||
return m, fmt.Errorf("the head %s cannot be read in the check's clone: %v", head, err)
|
||||
}
|
||||
if m.Trunk, err = revParse("refs/remotes/origin/" + base); err != nil || m.Trunk == "" {
|
||||
return m, fmt.Errorf("the trunk %s cannot be read, and a check judges the head merged into it "+
|
||||
"(novox/hq ADR 0323): %v", base, err)
|
||||
}
|
||||
if _, err := run(ctx, tree, "git", "checkout", "--quiet", "--detach", m.Trunk); err != nil {
|
||||
return m, fmt.Errorf("the trunk commit %s cannot be checked out: %w", twelve(m.Trunk), err)
|
||||
}
|
||||
message := fmt.Sprintf("Check the merge of %s into %s at %s (made for a merge check, never pushed)",
|
||||
twelve(m.Head), base, twelve(m.Trunk))
|
||||
args := append(append([]string{}, mergeIdentity...), "merge", "--no-ff", "--no-edit", "--no-verify", "--quiet", "-m", message, m.Head)
|
||||
if _, err := run(ctx, tree, "git", args...); err != nil {
|
||||
out, _ := run(ctx, tree, "git", "diff", "--name-only", "--diff-filter=U")
|
||||
m.Conflicts = strings.Fields(out)
|
||||
_, _ = run(ctx, tree, "git", "merge", "--abort")
|
||||
if len(m.Conflicts) == 0 {
|
||||
return m, fmt.Errorf("the head %s could not be merged into trunk %s: %w", twelve(m.Head), twelve(m.Trunk), err)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
if m.Merge, err = revParse("HEAD"); err != nil {
|
||||
return m, fmt.Errorf("the merge made for the check cannot be read: %w", err)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// changedByMerge is every path the merge changes against the trunk commit: what the gate's delivery plan is
|
||||
// computed from (ADR 0323 §1). Nil when it cannot be read; the caller keeps the pull request's own list.
|
||||
func changedByMerge(ctx context.Context, run Runner, tree string, m TrunkMerge) []string {
|
||||
if m.Merge == "" || m.Trunk == "" {
|
||||
return nil
|
||||
}
|
||||
out, err := run(ctx, tree, "git", "diff", "--name-only", "--no-renames", m.Trunk, m.Merge)
|
||||
if err != nil {
|
||||
return nil // empty-on-error: the caller keeps the pull request's own list of changed files, the safe one
|
||||
}
|
||||
return strings.Fields(out)
|
||||
}
|
||||
|
||||
// conflictWords is a head that does not merge cleanly, said as its check's failure.
|
||||
func conflictWords(m TrunkMerge) string {
|
||||
return fmt.Sprintf("head %s does not merge cleanly into trunk %s: in conflict %s", twelve(m.Head), twelve(m.Trunk),
|
||||
strings.Join(m.Conflicts, ", "))
|
||||
}
|
||||
|
||||
// headAt is the commit a clone is checked out at, as twelve characters; empty when it cannot be read.
|
||||
func headAt(ctx context.Context, run Runner, tree string) string {
|
||||
out, err := run(ctx, tree, "git", "rev-parse", "--verify", "--quiet", "HEAD^{commit}")
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return twelve(strings.TrimSpace(out))
|
||||
}
|
||||
|
||||
// twelve is a commit as a verdict names it: its first twelve characters.
|
||||
func twelve(commit string) string {
|
||||
if len(commit) > 12 {
|
||||
return commit[:12]
|
||||
}
|
||||
return commit
|
||||
}
|
||||
@@ -1,212 +0,0 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **A merge check judges the merge, not the head** (novox/hq ADR 0323, issue 341): the head merged into the trunk
|
||||
// commit in the check's own clone, never pushed; a head that does not merge cleanly fails, naming the files; the
|
||||
// verdict names the head, the trunk commit and the commits of the repositories read beside it.
|
||||
|
||||
// aTrunkAndABranch is a repository whose main moved after the branch was cut: main holds trunkFiles over the
|
||||
// common files, the branch headFiles. It answers the repository, the branch's head and main's head.
|
||||
func aTrunkAndABranch(t *testing.T, common, trunkFiles, headFiles map[string]string) (string, string, string) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
git := func(args ...string) string {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd.Dir = dir
|
||||
cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org",
|
||||
"GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org")
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("git %v: %v\n%s", args, err, out)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
write := func(files map[string]string) {
|
||||
for name, body := range files {
|
||||
if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
git("add", "-A")
|
||||
}
|
||||
git("init", "--quiet", "-b", "main")
|
||||
write(common)
|
||||
git("commit", "--quiet", "-m", "common")
|
||||
git("checkout", "--quiet", "-b", "change")
|
||||
write(headFiles)
|
||||
git("commit", "--quiet", "-m", "the change")
|
||||
head := git("rev-parse", "HEAD")
|
||||
git("checkout", "--quiet", "main")
|
||||
write(trunkFiles)
|
||||
git("commit", "--quiet", "-m", "the trunk moved")
|
||||
return dir, head, git("rev-parse", "HEAD")
|
||||
}
|
||||
|
||||
func TestACheckJudgesTheHeadMergedIntoTheTrunk(t *testing.T) {
|
||||
// main's script passes only in a tree holding both the trunk's file and the head's: the merge.
|
||||
repo, head, trunk := aTrunkAndABranch(t,
|
||||
map[string]string{CheckScript: "test -f trunk-only && test -f head-only || { echo 'FAIL: not the merge'; exit 1; }\n"},
|
||||
map[string]string{"trunk-only": "t"}, map[string]string{"head-only": "h"})
|
||||
beside, besideHead := aCheckedRepository(t, map[string]string{"README": "the catalogue"})
|
||||
registry := aFactsRegistry(t)
|
||||
aRuntimeOnThisMachine(t)
|
||||
|
||||
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-341", Repository: repo, Ref: head, Owner: "novox",
|
||||
Repo: "mesh-lab", Number: 1, Base: "main", Toolchain: "golang",
|
||||
Beside: map[string]Beside{"mesh-catalog": {Repository: beside, Ref: besideHead}}}, t.TempDir(), registry,
|
||||
GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v.Repo == nil || v.Repo.Verdict != "pass" {
|
||||
t.Fatalf("the check did not judge the head merged into the trunk: %+v\n%s", v.Repo, v.Report)
|
||||
}
|
||||
if v.Head != head[:12] || v.Trunk != trunk[:12] {
|
||||
t.Fatalf("the verdict names head %q and trunk %q, not %s and %s", v.Head, v.Trunk, head[:12], trunk[:12])
|
||||
}
|
||||
if v.Beside["mesh-catalog"] != besideHead[:12] {
|
||||
t.Fatalf("the verdict names the catalogue read beside it as %q, not %s: %v", v.Beside["mesh-catalog"],
|
||||
besideHead[:12], v.Beside)
|
||||
}
|
||||
if v.Merge == "" || v.Merge == head || v.Merge == trunk {
|
||||
t.Fatalf("no merge of the head into the trunk was made for the check: %q", v.Merge)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHeadThatDoesNotMergeCleanlyFailsNamingTheFiles(t *testing.T) {
|
||||
repo, head, trunk := aTrunkAndABranch(t,
|
||||
map[string]string{"store.go": "package store\n\nconst Declared = 1\n", "other.go": "package store\n"},
|
||||
map[string]string{"store.go": "package store\n\nconst Declared = 2\n"},
|
||||
map[string]string{"store.go": "package store\n\nconst Declared = 3\n"})
|
||||
|
||||
// No artifact store and no container runtime: a head that does not merge is judged before either is needed.
|
||||
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-341-conflict", Repository: repo, Ref: head,
|
||||
Owner: "novox", Repo: "mesh-lab", Number: 2, Base: "main", Toolchain: "golang"}, t.TempDir(), "",
|
||||
GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("a head that does not merge cleanly was not judged: %v", err)
|
||||
}
|
||||
want := "does not merge cleanly into trunk " + trunk[:12]
|
||||
for name, l := range map[string]*Layer{"gate": v.Gate, "repository": v.Repo} {
|
||||
if l == nil || l.Verdict != "fail" || !strings.Contains(l.Summary, want) || !strings.Contains(l.Summary, "store.go") {
|
||||
t.Fatalf("its %s layer said %+v, not a failure saying %q and naming store.go", name, l, want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(v.Summary, "other.go") {
|
||||
t.Fatalf("a file not in conflict is named: %s", v.Summary)
|
||||
}
|
||||
if v.Head != head[:12] || v.Trunk != trunk[:12] {
|
||||
t.Fatalf("the failure names head %q and trunk %q", v.Head, v.Trunk)
|
||||
}
|
||||
}
|
||||
|
||||
// **The merge made for a check is only checked** (ADR 0323 §2): the forge never receives it.
|
||||
func TestTheMergeMadeForACheckIsNeverPushed(t *testing.T) {
|
||||
repo, head, trunk := aTrunkAndABranch(t, map[string]string{CheckScript: "true\n"},
|
||||
map[string]string{"trunk-only": "t"}, map[string]string{"head-only": "h"})
|
||||
registry := aFactsRegistry(t)
|
||||
aRuntimeOnThisMachine(t)
|
||||
refsBefore := gitOut(t, repo, "for-each-ref")
|
||||
|
||||
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-341-push", Repository: repo, Ref: head, Owner: "novox",
|
||||
Repo: "mesh-lab", Number: 3, Base: "main", Toolchain: "golang"}, t.TempDir(), registry, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v.Merge == "" {
|
||||
t.Fatal("the check made no merge of the head into the trunk")
|
||||
}
|
||||
if err := exec.Command("git", "-C", repo, "cat-file", "-e", v.Merge+"^{commit}").Run(); err == nil {
|
||||
t.Fatalf("the merge %s made for the check reached the forge's repository", v.Merge)
|
||||
}
|
||||
if after := gitOut(t, repo, "for-each-ref"); after != refsBefore {
|
||||
t.Fatalf("the check moved a ref of the forge's repository:\nbefore %s\nafter %s", refsBefore, after)
|
||||
}
|
||||
if got := gitOut(t, repo, "rev-parse", "main"); got != trunk {
|
||||
t.Fatalf("the trunk moved to %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A delivery group's composed check merges each member's head into its own trunk's head (ADR 0323 §6), and names
|
||||
// both for each; a member that does not merge cleanly fails the composed check, naming it.
|
||||
func TestAGroupsComposedCheckMergesEachMemberIntoItsTrunk(t *testing.T) {
|
||||
repo, head, trunk := aTrunkAndABranch(t, map[string]string{CheckScript: "true\n"},
|
||||
map[string]string{"trunk-only": "t"}, map[string]string{"head-only": "h"})
|
||||
member, memberHead, memberTrunk := aTrunkAndABranch(t, map[string]string{"a": "1"},
|
||||
map[string]string{"b": "trunk"}, map[string]string{"c": "head"})
|
||||
registry := aFactsRegistry(t)
|
||||
aRuntimeOnThisMachine(t)
|
||||
|
||||
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-341-group", Repository: repo, Ref: head, Owner: "novox",
|
||||
Repo: "mesh-controller", Number: 4, Base: "main", Toolchain: "golang", Group: []GroupHead{{Owner: "novox",
|
||||
Repo: "mesh-host", Repository: member, Ref: memberHead, Base: "main"}}}, t.TempDir(), registry,
|
||||
GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v.Head != head[:12] || v.Trunk != trunk[:12] {
|
||||
t.Fatalf("the composed check names its first head %q into %q", v.Head, v.Trunk)
|
||||
}
|
||||
if len(v.Members) != 1 || v.Members[0].Head != memberHead[:12] || v.Members[0].Trunk != memberTrunk[:12] {
|
||||
t.Fatalf("the composed check names its members %+v, not mesh-host %s into %s", v.Members, memberHead[:12],
|
||||
memberTrunk[:12])
|
||||
}
|
||||
|
||||
conflicting, conflictHead, _ := aTrunkAndABranch(t, map[string]string{"a": "1"}, map[string]string{"a": "2"},
|
||||
map[string]string{"a": "3"})
|
||||
v, err = Check(t.Context(), Command, CheckSpec{ID: "check-341-group-conflict", Repository: repo, Ref: head,
|
||||
Owner: "novox", Repo: "mesh-controller", Number: 4, Base: "main", Toolchain: "golang", Group: []GroupHead{{
|
||||
Owner: "novox", Repo: "mesh-host", Repository: conflicting, Ref: conflictHead, Base: "main"}}}, t.TempDir(),
|
||||
registry, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v.Verdict != "fail" || !strings.Contains(v.Summary, "novox/mesh-host") ||
|
||||
!strings.Contains(v.Summary, "does not merge cleanly") {
|
||||
t.Fatalf("a member that does not merge cleanly was answered %s: %s", v.Verdict, v.Summary)
|
||||
}
|
||||
}
|
||||
|
||||
func gitOut(t *testing.T, dir string, args ...string) string {
|
||||
t.Helper()
|
||||
out, err := exec.Command("git", append([]string{"-C", dir}, args...)...).CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("git %v: %v\n%s", args, err, out)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
// No hook runs when a check makes its merge: not one in the clone, whatever the build machine's git configuration says.
|
||||
func TestACheckMergeRunsNoHook(t *testing.T) {
|
||||
repo, head, _ := aTrunkAndABranch(t, map[string]string{"a": "1"}, map[string]string{"b": "trunk"},
|
||||
map[string]string{"c": "head"})
|
||||
clone := filepath.Join(t.TempDir(), "clone")
|
||||
if out, err := exec.Command("git", "clone", "--quiet", repo, clone).CombinedOutput(); err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
ran := filepath.Join(t.TempDir(), "ran")
|
||||
for _, hook := range []string{"pre-merge-commit", "commit-msg", "prepare-commit-msg", "post-merge"} {
|
||||
if err := os.WriteFile(filepath.Join(clone, ".git", "hooks", hook),
|
||||
[]byte("#!/bin/sh\necho "+hook+" >> "+ran+"\n"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
m, err := mergeIntoTrunk(t.Context(), Command, clone, "main", head)
|
||||
if err != nil || m.Merge == "" {
|
||||
t.Fatalf("the merge was not made: %+v, %v", m, err)
|
||||
}
|
||||
if said, err := os.ReadFile(ran); err == nil {
|
||||
t.Fatalf("a hook ran when the check made its merge: %s", said)
|
||||
}
|
||||
}
|
||||
@@ -1,139 +0,0 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestReplay341 is the replay of novox/hq issue 341, written with only what the build agent had before its fix,
|
||||
// so mesh-lab's prover can lay it over the commit before. On 2026-10-09 node-engine #64/#65 merged at 00:15 and
|
||||
// added Applied.Declared, under the JSON key `declared`, to the store record. #66, checked at 00:49 at its head,
|
||||
// had added its own Applied.Declared under the same key on other lines. Its check passed; git merged the two
|
||||
// edits without a conflict, and the trunk no longer compiled: "Declared redeclared". Before the fix the check
|
||||
// judged the head alone and passed. After it (novox/hq ADR 0323), the check judges the head merged into the
|
||||
// trunk commit that held the other field, and fails saying the trunk does not compile.
|
||||
//
|
||||
// It runs the check whole, with a container runtime that runs each container's command on this machine
|
||||
// (aRuntimeOnThisMachine): the repository's own check is `go build ./...`.
|
||||
func TestReplay341(t *testing.T) {
|
||||
origin, head := theNinthOfOctober(t)
|
||||
registry := aFactsRegistry(t)
|
||||
aRuntimeOnThisMachine(t)
|
||||
|
||||
v, err := Check(t.Context(), Command, CheckSpec{ID: "replay-341", Repository: origin, Ref: head, Owner: "novox",
|
||||
Repo: "mesh-host", Number: 66, Base: "main", Toolchain: "golang"}, t.TempDir(), registry, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("the check could not run: %v", err)
|
||||
}
|
||||
if v.Repo == nil || v.Repo.Verdict != "fail" {
|
||||
t.Fatalf("node-engine #66 at its head was answered %+v: the merge it makes into the trunk that holds #65's "+
|
||||
"field does not compile, and the check let it merge", v.Repo)
|
||||
}
|
||||
if said := v.Repo.Summary + "\n" + v.Repo.Failed + "\n" + v.Report; !strings.Contains(said, "Declared redeclared") {
|
||||
t.Fatalf("the check failed without saying the trunk does not compile:\n%s", said)
|
||||
}
|
||||
}
|
||||
|
||||
// theNinthOfOctober is the node-engine's repository as it stood at 00:49 on 2026-10-09, in miniature: a trunk
|
||||
// holding #65's Applied.Declared (a string, key `declared`), and #66's head, branched before it, holding its own
|
||||
// Applied.Declared (a *FoundUnit, the same key) on other lines. Each compiles alone; their merge does not.
|
||||
func theNinthOfOctober(t *testing.T) (string, string) {
|
||||
t.Helper()
|
||||
origin := t.TempDir()
|
||||
git := func(args ...string) string {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd.Dir = origin
|
||||
cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org",
|
||||
"GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org")
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("git %v: %v\n%s", args, err, out)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
write := func(name, body string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(filepath.Join(origin, name)), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(origin, name), []byte(body), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
store := func(declared65, declared66 string) string {
|
||||
return "package store\n\n// FoundUnit is a unit found on the machine.\ntype FoundUnit struct{ Name string }\n\n" +
|
||||
"// Applied is what the node-engine applied.\ntype Applied struct {\n\tGeneration int `json:\"generation\"`\n" +
|
||||
declared66 + "\tModules []string `json:\"modules\"`\n\tFiles []string `json:\"files\"`\n" +
|
||||
"\tUnits []string `json:\"units\"`\n\tAt string `json:\"at\"`\n" + declared65 + "}\n"
|
||||
}
|
||||
git("init", "--quiet", "-b", "main")
|
||||
write("go.mod", "module example.org/mesh-host\n\ngo 1.21\n")
|
||||
write("internal/store/store.go", store("", ""))
|
||||
write(CheckScript, "# mesh-check-toolchain: go\nset -e\nGOFLAGS= GOWORK=off GOTOOLCHAIN=local GOPROXY=off go build ./...\n")
|
||||
git("add", "-A")
|
||||
git("commit", "--quiet", "-m", "#63: the store record")
|
||||
git("checkout", "--quiet", "-b", "adr-0269")
|
||||
write("internal/store/store.go", store("", "\tDeclared *FoundUnit `json:\"declared\"`\n"))
|
||||
git("commit", "--quiet", "-am", "#66: the unit a declaration names")
|
||||
head := git("rev-parse", "HEAD")
|
||||
git("checkout", "--quiet", "main")
|
||||
write("internal/store/store.go", store("\tDeclared string `json:\"declared\"`\n", ""))
|
||||
git("commit", "--quiet", "-am", "#65: the declaration applied")
|
||||
return origin, head
|
||||
}
|
||||
|
||||
// aRuntimeOnThisMachine puts a container runtime on PATH that runs a container's command on this machine, in the
|
||||
// directory and with the environment the check gives it: a store and a bus that answer, a judge that is never
|
||||
// built, and the repository's own check run as written. Only for a check whose code is the test's own.
|
||||
func aRuntimeOnThisMachine(t *testing.T) {
|
||||
t.Helper()
|
||||
bus, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { bus.Close() })
|
||||
go func() {
|
||||
for {
|
||||
c, err := bus.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
c.Close()
|
||||
}
|
||||
}()
|
||||
dir := t.TempDir()
|
||||
runtime := `#!/bin/sh
|
||||
cmd="$1"; shift
|
||||
case "$cmd" in
|
||||
ps|rm|exec) exit 0 ;;
|
||||
port) echo "` + bus.Addr().String() + `"; exit 0 ;;
|
||||
run)
|
||||
workdir=""; detached=""
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
-d) detached=1; shift ;;
|
||||
--rm) shift ;;
|
||||
--workdir) workdir="$2"; shift 2 ;;
|
||||
--env) export "$2"; shift 2 ;;
|
||||
--network|--volume|--user|--label|--name|-p|-e) shift 2 ;;
|
||||
*) break ;;
|
||||
esac
|
||||
done
|
||||
if [ -n "$detached" ]; then echo replay-container; exit 0; fi
|
||||
shift
|
||||
if [ "$1" = "sh" ] && [ -n "$workdir" ]; then cd "$workdir" && exec "$@"; fi
|
||||
echo "no such image on this machine" >&2
|
||||
exit 1 ;;
|
||||
esac
|
||||
exit 1
|
||||
`
|
||||
if err := os.WriteFile(filepath.Join(dir, "docker"), []byte(runtime), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
}
|
||||
BIN
Binary file not shown.
@@ -14,13 +14,13 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/therootcompany/xz"
|
||||
)
|
||||
|
||||
// An archive whose source is somebody else's published file, pinned by URL and sha256 (novox/hq ADR 0316
|
||||
@@ -183,6 +183,7 @@ func unpackUpstream(ctx context.Context, file, into, within string) error {
|
||||
buffered := bufio.NewReader(f)
|
||||
magic, _ := buffered.Peek(6)
|
||||
var stream io.Reader
|
||||
var wait func() error
|
||||
switch {
|
||||
case bytes.HasPrefix(magic, []byte{0x1f, 0x8b}):
|
||||
zipped, err := gzip.NewReader(buffered)
|
||||
@@ -194,33 +195,53 @@ func unpackUpstream(ctx context.Context, file, into, within string) error {
|
||||
case bytes.HasPrefix(magic, []byte("BZh")):
|
||||
stream = bzip2.NewReader(buffered)
|
||||
case bytes.HasPrefix(magic, []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}):
|
||||
// Read in Go (github.com/therootcompany/xz, a port of xz-embedded), never by a tool the build seat's image
|
||||
// may not carry: the seat's image is the catalogue's build-agent recipe, which installs git and docker-cli
|
||||
// alone, and a release pinned as .tar.xz failed there on 2026-10-12 (hq issue 345). xzDictMax is a real
|
||||
// maximum: a block whose LZMA2 dictionary is larger is refused before its dictionary is allocated
|
||||
// (ErrMemlimit). The reader keeps a filter decoder for every block of a stream, so the file's framing is
|
||||
// walked first and a file of more blocks than xzBlocksMax is refused before anything is decoded
|
||||
// (xz_framing.go). What the stream unpacks to is bounded by untar (maxUpstream, maxEntries).
|
||||
if err := xzWithinLimits(file); err != nil {
|
||||
// Go's standard library reads no xz; the build seat's image carries xz (cmd/mesh-builder/Dockerfile).
|
||||
if _, err := exec.LookPath("xz"); err != nil {
|
||||
return errors.New("it is compressed with xz, and this build seat has no xz to read it")
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "xz", "-dc")
|
||||
cmd.Stdin = buffered
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
out, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
zipped, err := xz.NewReader(buffered, xzDictMax)
|
||||
if err != nil {
|
||||
return fmt.Errorf("it is not an xz stream this build seat reads (a dictionary of %d MiB at most): %v",
|
||||
xzDictMax>>20, err)
|
||||
if err := cmd.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
stream = zipped
|
||||
stream = out
|
||||
wait = func() error {
|
||||
if err := cmd.Wait(); err != nil {
|
||||
return fmt.Errorf("xz could not read it: %v %s", err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
defer func() {
|
||||
if wait != nil {
|
||||
_ = cmd.Process.Kill()
|
||||
_ = cmd.Wait()
|
||||
}
|
||||
}()
|
||||
default:
|
||||
return errors.New("it is not a tar archive compressed with gzip, bzip2 or xz; a single file is kept " +
|
||||
"whole by naming it in `file`")
|
||||
}
|
||||
return untar(stream, into, within)
|
||||
if err := untar(stream, into, within); err != nil {
|
||||
return err
|
||||
}
|
||||
if wait != nil {
|
||||
// Whatever follows the tar's end is read before xz is waited for: xz blocks on a full pipe otherwise.
|
||||
if _, err := io.Copy(io.Discard, stream); err != nil {
|
||||
return err
|
||||
}
|
||||
w := wait
|
||||
wait = nil
|
||||
return w()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// xzDictMax is the largest xz dictionary read: xz's own presets use at most 64 MiB (-9); a stream asking for
|
||||
// more is refused rather than given the memory.
|
||||
const xzDictMax = 64 << 20
|
||||
|
||||
// untar writes a tar stream's files and directories within `within` into a directory.
|
||||
func untar(stream io.Reader, into, within string) error {
|
||||
reader := tar.NewReader(stream)
|
||||
|
||||
@@ -6,10 +6,8 @@ import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
@@ -17,7 +15,6 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -386,84 +383,3 @@ func TestARedirectOffHttpsIsNotFollowed(t *testing.T) {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnXzReleaseIsReadWithoutAnyXzToolOnTheBuildSeat(t *testing.T) {
|
||||
// The build seat's image (the catalogue's build-agent recipe) carries git and docker-cli alone: Betterbird's
|
||||
// .tar.xz failed there on 2026-10-12 with "this build seat has no xz to read it" (hq issue 345). It is
|
||||
// read in Go, with nothing on PATH.
|
||||
t.Setenv("PATH", "")
|
||||
body, err := os.ReadFile(filepath.Join("testdata", "release.tar.xz"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v := &vendor{files: map[string][]byte{"/r.tar.xz": body}}
|
||||
base := serveVendor(t, v)
|
||||
packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
|
||||
URL: base + "/r.tar.xz", SHA256: sumHex(body), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e := entriesOf(t, packed)
|
||||
if len(e) != 2 || e["betterbird"] != "#!/bin/sh\n -rwxr-xr-x" || e["defaults/p.js"] != "pref(\"a\", 1);\n -rw-r--r--" {
|
||||
t.Fatalf("%v", e)
|
||||
}
|
||||
}
|
||||
|
||||
// withDictionary is an xz stream whose first block's LZMA2 dictionary property is props, its header's CRC32
|
||||
// recomputed: a stream asking for a dictionary of its own choosing.
|
||||
func withDictionary(t *testing.T, stream []byte, props byte) []byte {
|
||||
t.Helper()
|
||||
out := append([]byte{}, stream...)
|
||||
const at = 12 // the stream header
|
||||
size := (int(out[at]) + 1) * 4
|
||||
header := out[at : at+size]
|
||||
flags := header[1]
|
||||
i := 2
|
||||
readVarint := func() {
|
||||
for header[i]&0x80 != 0 {
|
||||
i++
|
||||
}
|
||||
i++
|
||||
}
|
||||
if flags&0x40 != 0 {
|
||||
readVarint()
|
||||
}
|
||||
if flags&0x80 != 0 {
|
||||
readVarint()
|
||||
}
|
||||
if header[i] != 0x21 || header[i+1] != 0x01 {
|
||||
t.Fatalf("the fixture's first filter is not LZMA2: % x", header)
|
||||
}
|
||||
header[i+2] = props
|
||||
binary.LittleEndian.PutUint32(header[size-4:], crc32.ChecksumIEEE(header[:size-4]))
|
||||
return out
|
||||
}
|
||||
|
||||
func TestAnXzStreamAskingForTooLargeADictionaryIsRefused(t *testing.T) {
|
||||
body, err := os.ReadFile(filepath.Join("testdata", "release.tar.xz"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 1 GiB (props 36) and 4 GiB less one (props 40): over the 64 MiB maximum, refused before allocation.
|
||||
for _, props := range []byte{36, 40} {
|
||||
crafted := withDictionary(t, body, props)
|
||||
v := &vendor{files: map[string][]byte{"/r.tar.xz": crafted}}
|
||||
base := serveVendor(t, v)
|
||||
var before, after runtime.MemStats
|
||||
runtime.GC()
|
||||
runtime.ReadMemStats(&before)
|
||||
_, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
|
||||
URL: base + "/r.tar.xz", SHA256: sumHex(crafted), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
|
||||
runtime.ReadMemStats(&after)
|
||||
if err == nil || !strings.Contains(err.Error(), "dictionary") {
|
||||
t.Errorf("props %d: a dictionary over the maximum was not refused: %v", props, err)
|
||||
}
|
||||
if grew := after.TotalAlloc - before.TotalAlloc; grew > 256<<20 {
|
||||
t.Errorf("props %d: %d bytes were allocated for a refused stream", props, grew)
|
||||
}
|
||||
}
|
||||
// The fixture as it is (a small dictionary) still reads.
|
||||
if crafted := withDictionary(t, body, 0); len(crafted) == 0 {
|
||||
t.Fatal("no fixture")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,290 +0,0 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
// xzBlocksMax and xzStreamsMax are the most blocks and the most concatenated streams an xz file may hold for
|
||||
// the build seat to read it.
|
||||
//
|
||||
// **Why a block count is limited at all.** The xz reader (github.com/therootcompany/xz v1.0.1) keeps one
|
||||
// filter decoder for every block that names a delta or BCJ filter, and frees them only when a stream ends:
|
||||
// its block counters are reset per stream, not per block. A stream of many tiny blocks therefore grows the
|
||||
// heap with its length: an `xz --block-size=1 --delta` file of 6 MB kept about 67 MiB live, which comes to
|
||||
// about 5.6 GiB at the 512 MiB download cap (review 68 on mesh-controller pull request 264, hq issue 345).
|
||||
//
|
||||
// **Why 1,024.** At about 340 bytes of decoder a block, 1,024 blocks keep at most about 0.35 MiB live. A
|
||||
// real release is far below it: Betterbird's archive is 1 stream with 1 block, and multi-threaded xz cuts
|
||||
// blocks of three times the dictionary (24 MiB at the default preset), so even 512 MiB unpacked in blocks
|
||||
// of 512 KiB stays within it. Streams are limited to 16 because a release is one stream; the decoders are
|
||||
// reused from one stream to the next, so the stream limit only bounds the work of this walk.
|
||||
const (
|
||||
xzBlocksMax = 1024
|
||||
xzStreamsMax = 16
|
||||
)
|
||||
|
||||
// xzWithinLimits walks an xz file's framing, without decompressing anything, and refuses one holding more
|
||||
// blocks or streams than the build seat reads.
|
||||
//
|
||||
// **The framing, not the index.** Each stream's index at its end lists its blocks, but the reader compares
|
||||
// the index with the blocks only after it has decoded them all, so a file whose index lies would still be
|
||||
// decoded block by block. This walk follows the same framing the reader follows: stream header, each block
|
||||
// header, the block's LZMA2 chunks (each chunk header carries its own compressed size, so a chunk is skipped
|
||||
// without being decoded), its padding and check, then the index and the footer. It counts the blocks the
|
||||
// reader would decode, and it also refuses an index whose count of blocks is not the count walked.
|
||||
func xzWithinLimits(file string) error {
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
_, _, err = xzFraming(bufio.NewReader(f))
|
||||
if errors.Is(err, errXzTooManyBlocks) || errors.Is(err, errXzTooManyStreams) {
|
||||
return err
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("it is not an xz file this build seat reads: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
errXzTooManyBlocks = fmt.Errorf("it holds more than %d xz blocks, the most this build seat reads: "+
|
||||
"its xz reader keeps a filter decoder for every block (hq issue 345)", xzBlocksMax)
|
||||
errXzTooManyStreams = fmt.Errorf("it holds more than %d concatenated xz streams, the most this build "+
|
||||
"seat reads", xzStreamsMax)
|
||||
)
|
||||
|
||||
// xzFraming counts the streams and blocks of an xz file, checking its framing as it goes. It stops with
|
||||
// errXzTooManyBlocks or errXzTooManyStreams at the first block or stream over the limit.
|
||||
func xzFraming(r *bufio.Reader) (streams, blocks int, err error) {
|
||||
full := func(n int) ([]byte, error) {
|
||||
b := make([]byte, n)
|
||||
if _, err := io.ReadFull(r, b); err != nil {
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
skip := func(n int) error {
|
||||
if got, err := r.Discard(n); err != nil || got != n {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for {
|
||||
header, err := full(12)
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: header: %v", streams+1, err)
|
||||
}
|
||||
if !bytes.Equal(header[:6], []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}) {
|
||||
return streams, blocks, fmt.Errorf("stream %d: no xz stream header", streams+1)
|
||||
}
|
||||
if header[6] != 0 || header[7]&0xf0 != 0 ||
|
||||
crc32.ChecksumIEEE(header[6:8]) != binary.LittleEndian.Uint32(header[8:12]) {
|
||||
return streams, blocks, fmt.Errorf("stream %d: a corrupt stream header", streams+1)
|
||||
}
|
||||
if streams++; streams > xzStreamsMax {
|
||||
return streams, blocks, errXzTooManyStreams
|
||||
}
|
||||
checkSize := 0
|
||||
if check := int(header[7]); check != 0 {
|
||||
checkSize = 4 << ((check - 1) / 3)
|
||||
}
|
||||
inStream := 0
|
||||
for {
|
||||
first, err := r.ReadByte()
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: no index: %v", streams, io.ErrUnexpectedEOF)
|
||||
}
|
||||
if first == 0 {
|
||||
break // the index indicator
|
||||
}
|
||||
rest, err := full(int(first)*4 + 3)
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: header: %v", streams, inStream+1, err)
|
||||
}
|
||||
if err := xzBlockHeader(append([]byte{first}, rest...)); err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: %v", streams, inStream+1, err)
|
||||
}
|
||||
inStream++
|
||||
if blocks++; blocks > xzBlocksMax {
|
||||
return streams, blocks, errXzTooManyBlocks
|
||||
}
|
||||
// The block's LZMA2 chunks, up to the end marker.
|
||||
size := 0
|
||||
for {
|
||||
control, err := r.ReadByte()
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: %v", streams, inStream, io.ErrUnexpectedEOF)
|
||||
}
|
||||
size++
|
||||
if control == 0 {
|
||||
break
|
||||
}
|
||||
switch {
|
||||
case control == 1 || control == 2: // stored: a 16-bit size less one, then the bytes
|
||||
b, err := full(2)
|
||||
if err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
n := int(binary.BigEndian.Uint16(b)) + 1
|
||||
if err := skip(n); err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
size += 2 + n
|
||||
case control >= 0x80: // LZMA: unpacked and packed sizes, the properties byte on a reset
|
||||
b, err := full(4)
|
||||
if err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
n := int(binary.BigEndian.Uint16(b[2:])) + 1
|
||||
size += 4
|
||||
if control >= 0xc0 {
|
||||
if err := skip(1); err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
size++
|
||||
}
|
||||
if err := skip(n); err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
size += n
|
||||
default:
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: an LZMA2 chunk of kind %#x",
|
||||
streams, inStream, control)
|
||||
}
|
||||
}
|
||||
if pad := (4 - size%4) % 4; pad > 0 {
|
||||
b, err := full(pad)
|
||||
if err != nil || !bytes.Equal(b, make([]byte, pad)) {
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: corrupt block padding", streams, inStream)
|
||||
}
|
||||
}
|
||||
if err := skip(checkSize); err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
}
|
||||
// The index: its count of records, two numbers each, padding, CRC32.
|
||||
used := 1
|
||||
count, n, err := xzVLI(r)
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err)
|
||||
}
|
||||
used += n
|
||||
if count != uint64(inStream) {
|
||||
return streams, blocks, fmt.Errorf("stream %d: its index lists %d blocks, but it holds %d",
|
||||
streams, count, inStream)
|
||||
}
|
||||
for i := 0; i < 2*inStream; i++ {
|
||||
_, n, err := xzVLI(r)
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err)
|
||||
}
|
||||
used += n
|
||||
}
|
||||
if err := skip((4-used%4)%4 + 4); err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err)
|
||||
}
|
||||
footer, err := full(12)
|
||||
if err != nil || footer[10] != 'Y' || footer[11] != 'Z' {
|
||||
return streams, blocks, fmt.Errorf("stream %d: no stream footer", streams)
|
||||
}
|
||||
// Stream padding: zero bytes, a multiple of four, then the end or another stream.
|
||||
zeros := 0
|
||||
for {
|
||||
b, err := r.ReadByte()
|
||||
if err == io.EOF {
|
||||
if zeros%4 != 0 {
|
||||
return streams, blocks, errors.New("stream padding not a multiple of four bytes")
|
||||
}
|
||||
return streams, blocks, nil
|
||||
}
|
||||
if err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
if b != 0 {
|
||||
_ = r.UnreadByte()
|
||||
break
|
||||
}
|
||||
zeros++
|
||||
}
|
||||
if zeros%4 != 0 {
|
||||
return streams, blocks, errors.New("stream padding not a multiple of four bytes")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// xzBlockHeader checks one block header: its CRC32, its flags, and that its last filter is LZMA2, the only
|
||||
// one whose chunks this walk can skip (and the only last filter the reader decodes).
|
||||
func xzBlockHeader(h []byte) error {
|
||||
size := len(h)
|
||||
if crc32.ChecksumIEEE(h[:size-4]) != binary.LittleEndian.Uint32(h[size-4:]) {
|
||||
return errors.New("a corrupt block header")
|
||||
}
|
||||
flags := h[1]
|
||||
if flags&0x3c != 0 {
|
||||
return errors.New("a block header with reserved flags")
|
||||
}
|
||||
body := bytes.NewReader(h[2 : size-4])
|
||||
if flags&0x40 != 0 {
|
||||
if _, _, err := xzVLI(body); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if flags&0x80 != 0 {
|
||||
if _, _, err := xzVLI(body); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
var last uint64
|
||||
for i := 0; i <= int(flags&3); i++ {
|
||||
id, _, err := xzVLI(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
props, _, err := xzVLI(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if props > uint64(body.Len()) {
|
||||
return errors.New("a corrupt block header")
|
||||
}
|
||||
if _, err := body.Seek(int64(props), io.SeekCurrent); err != nil {
|
||||
return err
|
||||
}
|
||||
last = id
|
||||
}
|
||||
if last != 0x21 {
|
||||
return fmt.Errorf("a block whose last filter is %#x, not LZMA2", last)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// xzVLI reads one of xz's variable-length integers (at most 9 bytes, 7 bits each), and how many bytes it took.
|
||||
func xzVLI(r io.ByteReader) (uint64, int, error) {
|
||||
var v uint64
|
||||
for i := 0; i < 9; i++ {
|
||||
b, err := r.ReadByte()
|
||||
if err != nil {
|
||||
return 0, i, io.ErrUnexpectedEOF
|
||||
}
|
||||
v |= uint64(b&0x7f) << (7 * i)
|
||||
if b&0x80 == 0 {
|
||||
if i > 0 && b == 0 {
|
||||
return 0, i + 1, errors.New("a number not in its shortest form")
|
||||
}
|
||||
return v, i + 1, nil
|
||||
}
|
||||
}
|
||||
return 0, 9, errors.New("a number longer than nine bytes")
|
||||
}
|
||||
@@ -1,207 +0,0 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"hash/crc32"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// xzStream is one xz stream (no integrity check) of data cut into blocks of blockSize bytes, each block's
|
||||
// LZMA2 data stored uncompressed, each block a delta filter before LZMA2 when delta is set: the shape of
|
||||
// `xz --block-size=N --filters=delta,lzma2`, built without any xz tool. listed is how many blocks its index
|
||||
// lists (-1: as many as it holds), so a stream whose index lies can be built too.
|
||||
func xzStream(data []byte, blockSize int, delta bool, listed int) []byte {
|
||||
var out bytes.Buffer
|
||||
flags := []byte{0, 0}
|
||||
out.Write([]byte{0xfd, '7', 'z', 'X', 'Z', 0x00})
|
||||
out.Write(flags)
|
||||
out.Write(binary.LittleEndian.AppendUint32(nil, crc32.ChecksumIEEE(flags)))
|
||||
type record struct{ unpadded, uncompressed uint64 }
|
||||
var records []record
|
||||
for at := 0; at < len(data); at += blockSize {
|
||||
block := data[at:min(at+blockSize, len(data))]
|
||||
if delta {
|
||||
// Delta-encoded with a distance of one; the decoder's history starts empty at each block.
|
||||
encoded := make([]byte, len(block))
|
||||
for i := range block {
|
||||
encoded[i] = block[i]
|
||||
if i > 0 {
|
||||
encoded[i] -= block[i-1]
|
||||
}
|
||||
}
|
||||
block = encoded
|
||||
}
|
||||
header := []byte{2, 0} // 12 bytes
|
||||
if delta {
|
||||
header[1] = 1 // two filters
|
||||
header = append(header, 0x03, 0x01, 0x00)
|
||||
}
|
||||
header = append(header, 0x21, 0x01, 0x00) // LZMA2, a 4 KiB dictionary
|
||||
for len(header) < 8 {
|
||||
header = append(header, 0)
|
||||
}
|
||||
header = binary.LittleEndian.AppendUint32(header, crc32.ChecksumIEEE(header))
|
||||
out.Write(header)
|
||||
size := 0
|
||||
for i, chunk := 0, block; len(chunk) > 0; i++ {
|
||||
n := min(len(chunk), 1<<16)
|
||||
control := byte(2)
|
||||
if i == 0 {
|
||||
control = 1 // the first chunk of a block resets the dictionary
|
||||
}
|
||||
out.WriteByte(control)
|
||||
out.Write(binary.BigEndian.AppendUint16(nil, uint16(n-1)))
|
||||
out.Write(chunk[:n])
|
||||
size += 3 + n
|
||||
chunk = chunk[n:]
|
||||
}
|
||||
out.WriteByte(0)
|
||||
size++
|
||||
out.Write(make([]byte, (4-size%4)%4))
|
||||
records = append(records, record{uint64(len(header) + size), uint64(len(block))})
|
||||
}
|
||||
if listed >= 0 {
|
||||
for len(records) < listed {
|
||||
records = append(records, records[0])
|
||||
}
|
||||
records = records[:listed]
|
||||
}
|
||||
vli := func(b []byte, v uint64) []byte {
|
||||
for v >= 0x80 {
|
||||
b = append(b, byte(v)|0x80)
|
||||
v >>= 7
|
||||
}
|
||||
return append(b, byte(v))
|
||||
}
|
||||
index := vli([]byte{0}, uint64(len(records)))
|
||||
for _, r := range records {
|
||||
index = vli(vli(index, r.unpadded), r.uncompressed)
|
||||
}
|
||||
for len(index)%4 != 0 {
|
||||
index = append(index, 0)
|
||||
}
|
||||
index = binary.LittleEndian.AppendUint32(index, crc32.ChecksumIEEE(index))
|
||||
out.Write(index)
|
||||
footer := binary.LittleEndian.AppendUint32(nil, uint32(len(index)/4-1))
|
||||
footer = append(footer, flags...)
|
||||
out.Write(binary.LittleEndian.AppendUint32(nil, crc32.ChecksumIEEE(footer)))
|
||||
out.Write(footer)
|
||||
out.WriteString("YZ")
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
// tarHolding is a tar holding betterbird/data with body.
|
||||
func tarHolding(t *testing.T, body []byte) []byte {
|
||||
t.Helper()
|
||||
var out bytes.Buffer
|
||||
w := tar.NewWriter(&out)
|
||||
if err := w.WriteHeader(&tar.Header{Name: "betterbird/data", Mode: 0o644, Size: int64(len(body)),
|
||||
Typeflag: tar.TypeReg}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.Write(body)
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
func writeTemp(t *testing.T, body []byte) string {
|
||||
t.Helper()
|
||||
file := filepath.Join(t.TempDir(), "r.tar.xz")
|
||||
if err := os.WriteFile(file, body, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return file
|
||||
}
|
||||
|
||||
func TestAnXzFileOfAFewDeltaBlocksAndTwoStreamsIsRead(t *testing.T) {
|
||||
// The streams this test builds are ones the reader decodes, so what the walk refuses below is the
|
||||
// same framing the reader follows.
|
||||
body := bytes.Repeat([]byte("betterbird "), 300)
|
||||
whole := tarHolding(t, body)
|
||||
half := len(whole) / 2
|
||||
for name, file := range map[string][]byte{
|
||||
"one stream of delta blocks": xzStream(whole, 512, true, -1),
|
||||
"two streams with padding": append(append(xzStream(whole[:half], 700, true, -1), 0, 0, 0, 0),
|
||||
xzStream(whole[half:], 100, false, -1)...),
|
||||
} {
|
||||
path := writeTemp(t, file)
|
||||
if xzPath, err := exec.LookPath("xz"); err == nil {
|
||||
if out, err := exec.Command(xzPath, "-t", path).CombinedOutput(); err != nil {
|
||||
t.Fatalf("%s: xz itself refuses the built stream: %v %s", name, err, out)
|
||||
}
|
||||
}
|
||||
into := t.TempDir()
|
||||
if err := unpackUpstream(context.Background(), path, into, "betterbird"); err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
got, err := os.ReadFile(filepath.Join(into, "data"))
|
||||
if err != nil || !bytes.Equal(got, body) {
|
||||
t.Fatalf("%s: unpacked %d bytes, want %d: %v", name, len(got), len(body), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnXzFileOfManyDeltaBlocksIsRefusedBeforeItIsDecoded(t *testing.T) {
|
||||
// Review 68 on pull request 264: `xz --block-size=1 --delta` of a 200 kB tar kept about 67 MiB live in
|
||||
// the reader, one delta decoder per block. Its index here lists one block, as a file crafted to pass a
|
||||
// check of the index alone would; the walk counts the blocks the reader would decode.
|
||||
body := make([]byte, 200_000)
|
||||
for i := range body {
|
||||
body[i] = byte(i * 7)
|
||||
}
|
||||
crafted := xzStream(tarHolding(t, body), 1, true, 1)
|
||||
path := writeTemp(t, crafted)
|
||||
var before, after runtime.MemStats
|
||||
runtime.GC()
|
||||
runtime.ReadMemStats(&before)
|
||||
err := unpackUpstream(context.Background(), path, t.TempDir(), "betterbird")
|
||||
runtime.ReadMemStats(&after)
|
||||
if err == nil || !strings.Contains(err.Error(), "more than 1024 xz blocks") {
|
||||
t.Fatalf("a file of %d bytes in blocks of one byte was not refused for its blocks: %v", len(crafted), err)
|
||||
}
|
||||
if grew := after.TotalAlloc - before.TotalAlloc; grew > 4<<20 {
|
||||
t.Fatalf("%d bytes were allocated for a refused file", grew)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheXzBlockLimitIsExact(t *testing.T) {
|
||||
data := make([]byte, xzBlocksMax+1)
|
||||
streams, blocks, err := xzFraming(bufio.NewReader(bytes.NewReader(xzStream(data[:xzBlocksMax], 1, true, -1))))
|
||||
if err != nil || streams != 1 || blocks != xzBlocksMax {
|
||||
t.Fatalf("%d blocks: %d streams, %d blocks, %v", xzBlocksMax, streams, blocks, err)
|
||||
}
|
||||
if _, _, err := xzFraming(bufio.NewReader(bytes.NewReader(xzStream(data, 1, true, -1)))); !errors.Is(err, errXzTooManyBlocks) {
|
||||
t.Fatalf("%d blocks: %v", xzBlocksMax+1, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnXzFileOfTooManyStreamsIsRefused(t *testing.T) {
|
||||
var file []byte
|
||||
for i := 0; i <= xzStreamsMax; i++ {
|
||||
file = append(file, xzStream([]byte("x"), 1, false, -1)...)
|
||||
}
|
||||
err := unpackUpstream(context.Background(), writeTemp(t, file), t.TempDir(), "")
|
||||
if err == nil || !strings.Contains(err.Error(), "more than 16 concatenated xz streams") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnXzIndexThatDoesNotListItsBlocksIsRefused(t *testing.T) {
|
||||
file := xzStream(tarHolding(t, []byte("x")), 512, false, 1)
|
||||
err := unpackUpstream(context.Background(), writeTemp(t, file), t.TempDir(), "betterbird")
|
||||
if err == nil || !strings.Contains(err.Error(), "its index lists 1 blocks, but it holds 4") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
@@ -645,11 +645,6 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
sealed = ForeseenSealed
|
||||
*foreseen = append(*foreseen, m.Module+"/"+name)
|
||||
}
|
||||
if sealed == "" && m.OwnSecrets[name].IssuedBy == IssuedOutside {
|
||||
// **Not given, and the mesh never makes it** (novox/hq issue 401, ADR 0228): no file, as for a
|
||||
// family's member (ADR 0283), so the module reads its absence as not given.
|
||||
continue
|
||||
}
|
||||
if sealed == "" {
|
||||
// Declared and not made. Refused rather than skipped: a module whose own
|
||||
// credential is silently absent starts, fails to authenticate, and the reason is
|
||||
@@ -1007,7 +1002,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
// content is where a setting lands, so a placeholder may only exist once the setting
|
||||
// has been put in — filling secrets first would look at content that is not yet what
|
||||
// the machine receives.
|
||||
if err := intoFile(copied, sealed, outsideNotGiven(m, with.Needed[m.Module]), m.Module); err != nil {
|
||||
if err := intoFile(copied, sealed, m.Module); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// **Substituted here, not on the machine.** A bound value is not secret — the mesh
|
||||
@@ -2436,9 +2431,6 @@ func secretsReadBy(resource map[string]any, m Manifest, given map[string]string)
|
||||
}
|
||||
paths := map[string]string{}
|
||||
for name, own := range m.OwnSecrets.Plain() {
|
||||
if own.IssuedBy == IssuedOutside && given[name] == "" {
|
||||
continue // not given, so not placed: nothing to restart on (novox/hq issue 401)
|
||||
}
|
||||
paths[name] = own.Path
|
||||
}
|
||||
for name, sealed := range given {
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package catalogue
|
||||
|
||||
import "path"
|
||||
|
||||
// The machine's own resolver (novox/hq ADR 0247).
|
||||
//
|
||||
// **Most machines have none.** Every machine lists the mesh's resolvers in /etc/resolv.conf and nothing
|
||||
@@ -66,43 +68,39 @@ func holdsSeat(m Manifest, seat string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// rendersResolverFile says whether a module asks the mesh to render the machine's resolver file.
|
||||
func rendersResolverFile(m Manifest) bool {
|
||||
for _, f := range m.Facts {
|
||||
if !f.Home && f.Path == ResolverFile {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ResolverFileOwner is the module that writes the machine's resolver file among the modules on one
|
||||
// machine (novox/hq ADR 0247): the holder of node-resolver when one renders it, else nobody is named here
|
||||
// and the file is whoever's it always was — the uplink holder's (ADR 0223).
|
||||
func ResolverFileOwner(modules []Manifest) string {
|
||||
for _, m := range modules {
|
||||
if holdsSeat(m, ResolverSeat) && rendersResolverFile(m) {
|
||||
return m.Module
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// stepsBack is the module as it composes on a machine whose resolver file is the node-resolver holder's:
|
||||
// **the uplink holder's rendering of that file is left out**, and nothing else of it changes. Only the
|
||||
// uplink's holder steps back — any other module rendering the file beside the resolver's is still two
|
||||
// owners of one path, and is refused as before.
|
||||
// stepsBack is the module as it composes among the modules of one machine (novox/hq ADR 0287,
|
||||
// ADR 0247):
|
||||
// **where it holds a seat that names a path stepping back for another seat, and that other seat's holder
|
||||
// is on the machine and declares the path, its own rendering of the path is left out**, and nothing else
|
||||
// of it changes. Only a file the mesh renders for it steps back, as before: a resource on the path is
|
||||
// composed as declared (the declaration copies a module's resources whole), so an uplink holder declaring
|
||||
// the resolver file as a resource beside the resolver's holder is two owners of one path, and refused. The stepping back is the seat's definition — node-uplink stepping back from the resolver
|
||||
// file for node-resolver — so any other module declaring the path beside the owner's is still two owners
|
||||
// of one path, and is refused as before.
|
||||
func stepsBack(m Manifest, modules []Manifest) Manifest {
|
||||
if !holdsSeat(m, "node-uplink") || !rendersResolverFile(m) {
|
||||
return m
|
||||
drop := map[string]bool{}
|
||||
for _, c := range m.Claims {
|
||||
seat, known := SeatNamed(c.Name)
|
||||
if !known {
|
||||
continue
|
||||
}
|
||||
for _, p := range seat.Paths {
|
||||
if p.StepsBackFor == "" {
|
||||
continue
|
||||
}
|
||||
for _, o := range modules {
|
||||
if o.Module != m.Module && holdsSeat(o, p.StepsBackFor) && declaresPath(o, p.Path) {
|
||||
drop[p.Path] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
owner := ResolverFileOwner(modules)
|
||||
if owner == "" || owner == m.Module {
|
||||
if len(drop) == 0 {
|
||||
return m
|
||||
}
|
||||
facts := make(map[string]RosterFile, len(m.Facts))
|
||||
for name, f := range m.Facts {
|
||||
if !f.Home && f.Path == ResolverFile {
|
||||
if !f.Home && drop[path.Clean(f.Path)] {
|
||||
continue
|
||||
}
|
||||
facts[name] = f
|
||||
@@ -113,9 +111,6 @@ func stepsBack(m Manifest, modules []Manifest) Manifest {
|
||||
|
||||
// steppedBack is every module of one machine as it composes there (stepsBack, each).
|
||||
func steppedBack(modules []Manifest) []Manifest {
|
||||
if ResolverFileOwner(modules) == "" {
|
||||
return modules
|
||||
}
|
||||
out := make([]Manifest, len(modules))
|
||||
for i, m := range modules {
|
||||
out[i] = stepsBack(m, modules)
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// An own secret issued outside the mesh that nobody gave (novox/hq issue 401, ADR 0228): the mesh makes no value for it,
|
||||
// so the composition places no file, a file of the module that names it says empty there, and nothing restarts on it.
|
||||
// Given, it is placed and filled like any own secret.
|
||||
|
||||
func outsideManifest(t *testing.T) Manifest {
|
||||
t.Helper()
|
||||
m, err := ParseManifest([]byte(`{"module":"assistant","version":"1",
|
||||
"own-secrets":{
|
||||
"password":{"path":"/var/lib/assistant/password.secret","taken":"at-start"},
|
||||
"vendor-key":{"path":"/var/lib/assistant/vendor-key.secret","taken":"at-start","issued-by":"outside"}},
|
||||
"resources":[
|
||||
{"id":"env","type":"file","path":"/var/lib/assistant/env","mode":"0600",
|
||||
"content":"PASSWORD=${secret:password}\nVENDOR_KEY=${secret:vendor-key}\n"},
|
||||
{"id":"server","type":"container","name":"assistant","image":"assistant:1",
|
||||
"env-file":["/var/lib/assistant/env"],
|
||||
"volumes":["/var/lib/assistant/vendor-key.secret:/run/vendor-key:ro"],
|
||||
"secrets-in-environment":"reads its settings from the environment only"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func resourceAt(out []map[string]any, path string) map[string]any {
|
||||
for _, r := range out {
|
||||
if r["path"] == path {
|
||||
return r
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resourceNamed(out []map[string]any, name string) map[string]any {
|
||||
for _, r := range out {
|
||||
if r["name"] == name {
|
||||
return r
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestAnOutsideSecretNotGivenIsNoFileAndAFileNamingItSaysEmpty(t *testing.T) {
|
||||
r := Resolution{Node: "server", Modules: []Manifest{outsideManifest(t)}}
|
||||
out, err := r.Declaration(Rendering{Needed: map[string]map[string]string{"assistant": {"password": "sealed-password"}}})
|
||||
if err != nil {
|
||||
t.Fatalf("a module whose outside secret nobody gave did not compose: %v", err)
|
||||
}
|
||||
if got := resourceAt(out, "/var/lib/assistant/vendor-key.secret"); got != nil {
|
||||
t.Fatalf("a secret nobody gave was placed: %v", got)
|
||||
}
|
||||
env := resourceAt(out, "/var/lib/assistant/env")
|
||||
if env == nil {
|
||||
t.Fatal("the module's own file is not composed")
|
||||
}
|
||||
if content := env["content"].(string); content != "PASSWORD=${secret:password}\nVENDOR_KEY=\n" {
|
||||
t.Fatalf("the file names the secret nobody gave as %q", content)
|
||||
}
|
||||
secrets, _ := env["secrets"].(map[string]any)
|
||||
if len(secrets) != 1 || secrets["password"] != "sealed-password" {
|
||||
t.Fatalf("the file is filled from %v", secrets)
|
||||
}
|
||||
server := resourceNamed(out, "assistant")
|
||||
if on := strings.Join(stringsIn(server["restart-on"]), " "); strings.Contains(on, "vendor-key") {
|
||||
t.Fatalf("the container restarts on a secret nobody gave: %q", on)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOutsideSecretGivenIsPlacedAndFilled(t *testing.T) {
|
||||
r := Resolution{Node: "server", Modules: []Manifest{outsideManifest(t)}}
|
||||
out, err := r.Declaration(Rendering{Needed: map[string]map[string]string{"assistant": {
|
||||
"password": "sealed-password", "vendor-key": "sealed-vendor-key"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
placed := resourceAt(out, "/var/lib/assistant/vendor-key.secret")
|
||||
if placed == nil || placed["sealed"] != "sealed-vendor-key" {
|
||||
t.Fatalf("the secret given is placed as %v", placed)
|
||||
}
|
||||
env := resourceAt(out, "/var/lib/assistant/env")
|
||||
if content := env["content"].(string); content != "PASSWORD=${secret:password}\nVENDOR_KEY=${secret:vendor-key}\n" {
|
||||
t.Fatalf("the file was changed: %q", content)
|
||||
}
|
||||
if secrets, _ := env["secrets"].(map[string]any); secrets["vendor-key"] != "sealed-vendor-key" {
|
||||
t.Fatalf("the file is filled from %v", secrets)
|
||||
}
|
||||
server := resourceNamed(out, "assistant")
|
||||
if on := strings.Join(stringsIn(server["restart-on"]), " "); !strings.Contains(on, "assistant."+NeedID("vendor-key")) {
|
||||
t.Fatalf("the container does not restart on the secret given: %q", on)
|
||||
}
|
||||
}
|
||||
|
||||
// A secret the mesh makes is still refused when it was not made: only an outside one is left out.
|
||||
func TestAnOwnSecretTheMeshMakesIsStillRefusedWhenNotMade(t *testing.T) {
|
||||
r := Resolution{Node: "server", Modules: []Manifest{outsideManifest(t)}}
|
||||
_, err := r.Declaration(Rendering{})
|
||||
if nm, ok := err.(*NotMadeError); !ok || nm.Name != "password" {
|
||||
t.Fatalf("a secret the mesh makes, not made: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -957,7 +957,9 @@ func checkResources(modules []Manifest) []string {
|
||||
// Judged as the machine composes them: where the node-resolver holder writes the resolver file, the
|
||||
// uplink holder's rendering of it is not composed, so it is not a second owner (novox/hq ADR 0247).
|
||||
modules = steppedBack(modules)
|
||||
var problems []string
|
||||
// And only a claimant of a seat naming a path declares it — the controller's generated resources
|
||||
// included, which are judged here at composition (novox/hq ADR 0287).
|
||||
problems := seatPathsOnMachine(modules)
|
||||
owner := map[string]string{}
|
||||
ownedPath := map[string]string{} // path → owning module, for the access check below
|
||||
packagePresent, packageAbsent := map[string][]string{}, map[string][]string{}
|
||||
|
||||
@@ -0,0 +1,315 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The paths a seat's holder owns (novox/hq ADR 0287).
|
||||
//
|
||||
// **Who owns a machine-wide file is the role's question, answered once.** Until this, which module owned
|
||||
// /etc/hosts or /etc/resolv.conf was decided record by record (ADRs 0199, 0222, 0223, 0241) and enforced
|
||||
// only after the fact, by checkResources finding two modules on one machine declaring the same path: a
|
||||
// module that did not hold node-hostname could declare /etc/hosts on a machine where nobody held it, and
|
||||
// nothing refused it. Now a seat's definition names the paths the role owns on any machine, and three
|
||||
// refusals follow from it, each where the mesh can still say no:
|
||||
//
|
||||
// - a claim whose module does not declare every path its seat names, by CanHold, at registration and at
|
||||
// handover;
|
||||
// - a module declaring a path a seat names while claiming none of the seats that name it, at
|
||||
// registration (CatalogueProblems), and what the controller generates for a machine, at composition;
|
||||
// - a module's own seat naming a path another seat already names, at registration — the second loses.
|
||||
//
|
||||
// Reading a seat's path is access, never a resource, so it stays allowed.
|
||||
|
||||
// SeatPath is one path a seat's holder owns: absolute, literal, on the machine where the holder runs.
|
||||
type SeatPath struct {
|
||||
Path string `json:"path"`
|
||||
// StepsBackFor is a node seat whose holder owns the path instead, on a machine where that seat is held
|
||||
// (novox/hq ADR 0287, ADR 0247): this seat's holder still declares the path, and its declaration is not
|
||||
// composed there. Only the mesh's own seats step back, each by a record.
|
||||
StepsBackFor string `json:"steps-back-for,omitempty"`
|
||||
}
|
||||
|
||||
// seatPathProblem is what is wrong with one path a seat names, judged alone: absolute, written literally
|
||||
// with no placeholder, and clean, so it compares with what a module declares byte for byte.
|
||||
func seatPathProblem(seat, scope, p string) string {
|
||||
switch {
|
||||
case scope != ScopeNode:
|
||||
return fmt.Sprintf("the seat %s names the path %q and is a %s seat; only a node seat names paths, "+
|
||||
"because a path is a machine's (novox/hq ADR 0287)", seat, p, scope)
|
||||
case strings.Contains(p, "${"):
|
||||
return fmt.Sprintf("the seat %s names the path %q, which has a placeholder; a seat's path is written "+
|
||||
"literally (novox/hq ADR 0287)", seat, p)
|
||||
case !strings.HasPrefix(p, "/") || path.Clean(p) != p:
|
||||
return fmt.Sprintf("the seat %s names the path %q, which is not a clean absolute path (novox/hq ADR 0287)",
|
||||
seat, p)
|
||||
case path.Dir(p) == "/":
|
||||
// The root or a top-level directory is every role's, never one role's own (ADR 0287).
|
||||
return fmt.Sprintf("the seat %s names the path %q, the root or a top-level directory, which no one "+
|
||||
"role owns (novox/hq ADR 0287)", seat, p)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// SeatPathProblems is what is wrong with the paths a set of the mesh's own seats names (novox/hq ADR
|
||||
// 0287): only a node seat names one, each absolute and literal; every path has exactly one seat naming it
|
||||
// without stepping back; and a seat steps back only for another node seat naming the same path without
|
||||
// stepping back, so the stepping back cannot cycle.
|
||||
func SeatPathProblems(set []Seat) []string {
|
||||
var problems []string
|
||||
owners := map[string][]string{} // path → the seats naming it without stepping back
|
||||
stepping := map[string][]SeatPath{} // seat → the paths it names stepping back
|
||||
byName := map[string]Seat{}
|
||||
for _, s := range set {
|
||||
byName[s.Name] = s
|
||||
for _, p := range s.Paths {
|
||||
if why := seatPathProblem(s.Name, s.Scope, p.Path); why != "" {
|
||||
problems = append(problems, why)
|
||||
}
|
||||
if p.StepsBackFor == "" {
|
||||
owners[p.Path] = append(owners[p.Path], s.Name)
|
||||
} else {
|
||||
stepping[s.Name] = append(stepping[s.Name], p)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, s := range set {
|
||||
for _, p := range stepping[s.Name] {
|
||||
other, known := byName[p.StepsBackFor]
|
||||
switch {
|
||||
case !known || other.Scope != ScopeNode:
|
||||
problems = append(problems, fmt.Sprintf("the seat %s steps back from %q for %q, which is not a "+
|
||||
"node seat of the mesh's (novox/hq ADR 0287)", s.Name, p.Path, p.StepsBackFor))
|
||||
case !namesWithoutSteppingBack(other, p.Path):
|
||||
problems = append(problems, fmt.Sprintf("the seat %s steps back from %q for %s, which does not "+
|
||||
"own that path itself; a seat steps back only for the one that owns it (novox/hq ADR 0287)",
|
||||
s.Name, p.Path, other.Name))
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, p := range sortedKeys(owners) {
|
||||
if len(owners[p]) > 1 {
|
||||
problems = append(problems, fmt.Sprintf("the path %q is named by %s, each without stepping back; "+
|
||||
"no two seats name one path (novox/hq ADR 0287)", p, strings.Join(owners[p], " and ")))
|
||||
}
|
||||
}
|
||||
for _, s := range set {
|
||||
for _, p := range stepping[s.Name] {
|
||||
if len(owners[p.Path]) == 0 {
|
||||
problems = append(problems, fmt.Sprintf("the path %q has no seat naming it without stepping back "+
|
||||
"(novox/hq ADR 0287)", p.Path))
|
||||
}
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
func namesWithoutSteppingBack(s Seat, p string) bool {
|
||||
for _, sp := range s.Paths {
|
||||
if sp.Path == p && sp.StepsBackFor == "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// declaredPaths is every machine path a module declares it owns: a resource's literal path, and a file
|
||||
// the mesh renders for it outside the operator's home. An access is not here — reading is not owning.
|
||||
//
|
||||
// **Cleaned, because the machine writes the cleaned path**: /etc//hosts, /etc/./hosts, /etc/../etc/hosts and
|
||||
// /etc/hosts/ are all /etc/hosts to it, and a seat's path is clean by its own rule, so a declaration compared
|
||||
// as written would pass for a path nobody named.
|
||||
func declaredPaths(m Manifest) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, r := range m.Resources {
|
||||
if p, ok := r["path"].(string); ok && strings.HasPrefix(p, "/") {
|
||||
out[path.Clean(p)] = true
|
||||
}
|
||||
}
|
||||
for _, f := range m.Facts {
|
||||
if !f.Home && strings.HasPrefix(f.Path, "/") {
|
||||
out[path.Clean(f.Path)] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// declaresPath says whether a module declares one path, as a resource or as a file the mesh renders.
|
||||
func declaresPath(m Manifest, p string) bool { return declaredPaths(m)[p] }
|
||||
|
||||
// missingPaths is each path a seat names that a module does not declare, in the seat's order.
|
||||
func missingPaths(m Manifest, paths []SeatPath) []string {
|
||||
if len(paths) == 0 {
|
||||
return nil
|
||||
}
|
||||
has := declaredPaths(m)
|
||||
var missing []string
|
||||
for _, p := range paths {
|
||||
if !has[p.Path] {
|
||||
missing = append(missing, p.Path)
|
||||
}
|
||||
}
|
||||
return missing
|
||||
}
|
||||
|
||||
// pathSeats is which seats name each path: the mesh's own, and then those modules declare, in the order
|
||||
// given. A module's seat naming a path a seat already names loses it, and the refusal says so (novox/hq
|
||||
// ADR 0287 — as for a seat's name, the second declaration loses).
|
||||
func pathSeats(order []string, shelf Shelf, declaredBy map[string]string) (map[string][]string, []string) {
|
||||
named := map[string][]string{}
|
||||
for _, s := range Seats() {
|
||||
for _, p := range s.Paths {
|
||||
named[p.Path] = append(named[p.Path], s.Name)
|
||||
}
|
||||
}
|
||||
var problems []string
|
||||
for _, module := range order {
|
||||
for _, s := range shelf[module].DefinesSeats {
|
||||
if declaredBy != nil && declaredBy[s.Name] != module {
|
||||
continue // a second declaration of the seat's name, refused for that already
|
||||
}
|
||||
for _, p := range s.Paths {
|
||||
if others := named[p]; len(others) > 0 {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s declares the seat %s naming the path %q, which %s already names; no two seats "+
|
||||
"name one path (novox/hq ADR 0287)", module, s.Name, p, strings.Join(others, " and ")))
|
||||
continue
|
||||
}
|
||||
named[p] = append(named[p], s.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
return named, problems
|
||||
}
|
||||
|
||||
// unclaimedSeatPaths is each path a module declares that a seat names while it claims none of the seats
|
||||
// naming it (novox/hq ADR 0287): only a claimant declares a seat's path, and a module claiming any one of
|
||||
// them may.
|
||||
func unclaimedSeatPaths(m Manifest, named map[string][]string) []string {
|
||||
claims := map[string]bool{}
|
||||
for _, c := range m.Claims {
|
||||
claims[canonicalSeat(c.Name)] = true
|
||||
}
|
||||
var problems []string
|
||||
declared := declaredPaths(m)
|
||||
for _, p := range sortedKeys(declared) {
|
||||
seats := named[p]
|
||||
if len(seats) == 0 {
|
||||
continue
|
||||
}
|
||||
claimsOne := false
|
||||
for _, s := range seats {
|
||||
claimsOne = claimsOne || claims[s]
|
||||
}
|
||||
if !claimsOne {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s declares the path %q, which the seat %s names, and claims none of them; only a holder of "+
|
||||
"a seat naming a path declares it (novox/hq ADR 0287)", m.Module, p, strings.Join(seats, " and ")))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// declaredSeatPathProblems is what one module's own seat declaration can be judged on alone: a path
|
||||
// only on a node seat, absolute, literal and clean.
|
||||
func declaredSeatPathProblems(m Manifest, s SeatDeclaration) []string {
|
||||
var problems []string
|
||||
for _, p := range s.Paths {
|
||||
if why := seatPathProblem(s.Name, s.At(), p); why != "" {
|
||||
problems = append(problems, m.Module+": "+why)
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// asSeatPaths is a module's own seat's paths in the shape the mesh's seats name theirs. A module's seat
|
||||
// never steps back: only the mesh's own seats do, each by a record.
|
||||
func asSeatPaths(paths []string) []SeatPath {
|
||||
out := make([]SeatPath, 0, len(paths))
|
||||
for _, p := range paths {
|
||||
out = append(out, SeatPath{Path: p})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// PathOwner is the module that owns one seat's path on one machine (novox/hq ADR 0287): the seat naming
|
||||
// the path, and that seat's holder there, with the stepping back applied.
|
||||
type PathOwner struct {
|
||||
Path string `json:"path"`
|
||||
Node string `json:"node"`
|
||||
Seat string `json:"seat"`
|
||||
Module string `json:"module"`
|
||||
}
|
||||
|
||||
// PathOwners is, for every path a seat in the set names and every machine where a seat naming it is held,
|
||||
// who owns the path there. A seat that steps back for another held on the same machine is passed over.
|
||||
func PathOwners(set []Seat, held []Held) []PathOwner {
|
||||
// node → seat → holding module
|
||||
holders := map[string]map[string]string{}
|
||||
for _, h := range held {
|
||||
s, ok := SeatNamed(h.Claim)
|
||||
if !ok || h.Scope != ScopeNode || s.Scope != ScopeNode {
|
||||
continue
|
||||
}
|
||||
if holders[h.Node] == nil {
|
||||
holders[h.Node] = map[string]string{}
|
||||
}
|
||||
if _, taken := holders[h.Node][s.Name]; !taken {
|
||||
holders[h.Node][s.Name] = h.Module
|
||||
}
|
||||
}
|
||||
naming := map[string][]Seat{} // path → the seats naming it
|
||||
steps := map[string]map[string]string{} // path → seat → the seat it steps back for
|
||||
for _, s := range set {
|
||||
for _, p := range s.Paths {
|
||||
naming[p.Path] = append(naming[p.Path], s)
|
||||
if p.StepsBackFor != "" {
|
||||
if steps[p.Path] == nil {
|
||||
steps[p.Path] = map[string]string{}
|
||||
}
|
||||
steps[p.Path][s.Name] = p.StepsBackFor
|
||||
}
|
||||
}
|
||||
}
|
||||
var out []PathOwner
|
||||
for _, p := range sortedKeys(naming) {
|
||||
for _, node := range sortedKeys(holders) {
|
||||
for _, s := range naming[p] {
|
||||
module, holds := holders[node][s.Name]
|
||||
if !holds {
|
||||
continue
|
||||
}
|
||||
if forSeat := steps[p][s.Name]; forSeat != "" {
|
||||
if _, there := holders[node][forSeat]; there {
|
||||
continue
|
||||
}
|
||||
}
|
||||
out = append(out, PathOwner{Path: p, Node: node, Seat: s.Name, Module: module})
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// seatPathsOnMachine is each path the modules of one machine declare that a seat names, for a module
|
||||
// claiming none of the seats naming it (novox/hq ADR 0287). Registration refuses the same of a manifest;
|
||||
// this holds what the controller generates for a machine to it too, because a computed module's resources
|
||||
// exist only once composed (checkResources is asked them there). The seats are the mesh's own and those the
|
||||
// machine's modules declare.
|
||||
func seatPathsOnMachine(modules []Manifest) []string {
|
||||
shelf := Shelf{}
|
||||
var order []string
|
||||
for _, m := range modules {
|
||||
shelf[m.Module] = m
|
||||
order = append(order, m.Module)
|
||||
}
|
||||
named, _ := pathSeats(order, shelf, nil)
|
||||
var problems []string
|
||||
for _, m := range modules {
|
||||
problems = append(problems, unclaimedSeatPaths(m, named)...)
|
||||
}
|
||||
return problems
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Defends novox/hq ADR 0287 (issue 412): a seat names the paths its holder owns, a claim without them is
|
||||
// refused, and only a claimant declares a seat's path.
|
||||
|
||||
// The starting set, exactly: node-hostname names /etc/hosts and /etc/hostname, node-uplink the resolver
|
||||
// file stepping back for node-resolver, node-resolver the resolver file, and every other seat names none.
|
||||
func TestTheSeatsThatNamePathsAreTheStartingSet(t *testing.T) {
|
||||
want := map[string]string{
|
||||
"node-hostname": "/etc/hosts /etc/hostname",
|
||||
"node-uplink": "/etc/resolv.conf>node-resolver",
|
||||
ResolverSeat: "/etc/resolv.conf",
|
||||
}
|
||||
for _, s := range DefaultSeats() {
|
||||
var got []string
|
||||
for _, p := range s.Paths {
|
||||
if p.StepsBackFor != "" {
|
||||
got = append(got, p.Path+">"+p.StepsBackFor)
|
||||
} else {
|
||||
got = append(got, p.Path)
|
||||
}
|
||||
}
|
||||
if strings.Join(got, " ") != want[s.Name] {
|
||||
t.Errorf("%s names %v; ADR 0287 gives it %q", s.Name, got, want[s.Name])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The compiled set keeps the rules: node seats only, absolute and literal, one seat per path without
|
||||
// stepping back, and a stepping back that cannot cycle.
|
||||
func TestTheCompiledSeatPathsKeepTheRules(t *testing.T) {
|
||||
if p := SeatPathProblems(DefaultSeats()); len(p) != 0 {
|
||||
t.Fatalf("the compiled set breaks ADR 0287: %v", p)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
why string
|
||||
set []Seat
|
||||
want string
|
||||
}{
|
||||
{"a mesh seat naming a path", []Seat{{Name: "mesh-x", Scope: ScopeMesh, Paths: []SeatPath{{Path: "/etc/x"}}}},
|
||||
"only a node seat names paths"},
|
||||
{"a placeholder", []Seat{{Name: "node-x", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/${setting:x}"}}}},
|
||||
"placeholder"},
|
||||
{"a relative path", []Seat{{Name: "node-x", Scope: ScopeNode, Paths: []SeatPath{{Path: "etc/x"}}}},
|
||||
"not a clean absolute path"},
|
||||
{"two owners", []Seat{
|
||||
{Name: "node-a", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/x"}}},
|
||||
{Name: "node-b", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/x"}}}}, "no two seats name one path"},
|
||||
{"a cycle", []Seat{
|
||||
{Name: "node-a", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/x", StepsBackFor: "node-b"}}},
|
||||
{Name: "node-b", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/x", StepsBackFor: "node-a"}}}},
|
||||
"does not own that path itself"},
|
||||
{"stepping back for nobody", []Seat{
|
||||
{Name: "node-a", Scope: ScopeNode, Paths: []SeatPath{{Path: "/etc/x", StepsBackFor: "node-z"}}}},
|
||||
"not a node seat"},
|
||||
} {
|
||||
if got := strings.Join(SeatPathProblems(c.set), "; "); !strings.Contains(got, c.want) {
|
||||
t.Errorf("%s: %q does not say %q", c.why, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// What the store's rows carry is never the paths: a row read back keeps the compiled ones.
|
||||
func TestTheSeatPathsSurviveTheStoresRows(t *testing.T) {
|
||||
before := seats
|
||||
t.Cleanup(func() { seats = before })
|
||||
UseSeats([]Seat{{Name: "node-hostname", Scope: ScopeNode}})
|
||||
s, _ := SeatNamed("node-hostname")
|
||||
if len(s.Paths) != 2 {
|
||||
t.Fatalf("node-hostname read back from its row names %v", s.Paths)
|
||||
}
|
||||
}
|
||||
|
||||
func hostnameSeat(t *testing.T) Seat {
|
||||
t.Helper()
|
||||
s, ok := SeatNamed("node-hostname")
|
||||
if !ok {
|
||||
t.Fatal("no node-hostname seat")
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func hostnameHolder(resources ...map[string]any) Manifest {
|
||||
return Manifest{Module: "names", Version: "1", Claims: []Claim{{Name: "node-hostname", Scope: ScopeNode,
|
||||
Serves: []string{"entries", "add", "remove"}}}, Resources: resources}
|
||||
}
|
||||
|
||||
// A holder declares every path its seat names, as a resource or as a file the mesh renders for it, or
|
||||
// CanHold refuses it naming the seat and each missing path — the judgement registration and handover share.
|
||||
func TestAClaimWithoutTheSeatsPathsIsRefused(t *testing.T) {
|
||||
seat := hostnameSeat(t)
|
||||
hosts := map[string]any{"id": "hosts", "type": "file", "path": "/etc/hosts", "content": "x"}
|
||||
name := map[string]any{"id": "name", "type": "file", "path": "/etc/hostname", "content": "x"}
|
||||
|
||||
if err := CanHold(hostnameHolder(hosts, name), seat); err != nil {
|
||||
t.Errorf("a holder declaring both paths as resources was refused: %v", err)
|
||||
}
|
||||
rendered := hostnameHolder(hosts)
|
||||
rendered.Facts = map[string]RosterFile{"name": {Path: "/etc/hostname", Template: "{{.Node}}\n"}}
|
||||
if err := CanHold(rendered, seat); err != nil {
|
||||
t.Errorf("a holder declaring a path as a file the mesh renders was refused: %v", err)
|
||||
}
|
||||
err := CanHold(hostnameHolder(), seat)
|
||||
if err == nil {
|
||||
t.Fatal("a holder declaring neither path was accepted")
|
||||
}
|
||||
for _, want := range []string{"node-hostname", "/etc/hosts", "/etc/hostname", "ADR 0287"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not name %s: %v", want, err)
|
||||
}
|
||||
}
|
||||
// A file under the operator's home is placed per account: it is not the machine's path.
|
||||
home := hostnameHolder(hosts)
|
||||
home.Facts = map[string]RosterFile{"name": {Path: "/etc/hostname", Home: true, Template: "x"}}
|
||||
if err := CanHold(home, seat); err == nil {
|
||||
t.Error("a file in the operator's home counted as the machine's /etc/hostname")
|
||||
}
|
||||
// At registration, the same refusal.
|
||||
if got := strings.Join(CatalogueProblems(Shelf{"names": hostnameHolder(hosts)}), "; "); !strings.Contains(got,
|
||||
"does not declare /etc/hostname") {
|
||||
t.Errorf("registration let a holder without /etc/hostname through: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Only a claimant declares a seat's path: a module claiming no seat that names /etc/hosts and declaring
|
||||
// it is refused at registration naming the seat; the uplink's holder and the resolver's both declaring
|
||||
// the resolver file are each a claimant of a seat naming it. Reading a seat's path is access, and allowed.
|
||||
func TestOnlyAClaimantDeclaresASeatsPath(t *testing.T) {
|
||||
stranger := Manifest{Module: "stranger", Version: "1", Resources: []map[string]any{
|
||||
{"id": "hosts", "type": "file", "path": "/etc/hosts", "into": "block", "content": "10.0.0.1 x\n"}}}
|
||||
got := strings.Join(CatalogueProblems(Shelf{"stranger": stranger}), "; ")
|
||||
if !strings.Contains(got, `stranger declares the path "/etc/hosts", which the seat node-hostname names`) {
|
||||
t.Errorf("a module declaring /etc/hosts without claiming node-hostname was not refused: %q", got)
|
||||
}
|
||||
// However the path is spelled (novox/hq issue 412 review).
|
||||
unclean := Manifest{Module: "unclean", Version: "1", Resources: []map[string]any{
|
||||
{"id": "hosts", "type": "file", "path": "/etc/../etc/hosts", "content": "10.0.0.1 x\n"}}}
|
||||
if got := strings.Join(CatalogueProblems(Shelf{"unclean": unclean}), "; "); !strings.Contains(got, "node-hostname") {
|
||||
t.Errorf("/etc/../etc/hosts from a module not claiming node-hostname was not refused: %q", got)
|
||||
}
|
||||
reader := Manifest{Module: "reader", Version: "1", Accesses: []Access{{ID: "hosts", Path: "/etc/hosts"}}}
|
||||
if got := CatalogueProblems(Shelf{"reader": reader}); len(got) != 0 {
|
||||
t.Errorf("reading /etc/hosts was refused: %v", got)
|
||||
}
|
||||
uplink := Manifest{Module: "uplink", Version: "1", Claims: []Claim{{Name: "node-uplink", Scope: ScopeNode}},
|
||||
Facts: map[string]RosterFile{"resolvers": {Path: ResolverFile, Template: "nameserver 10.42.0.1\n"}}}
|
||||
resolver := localResolver()
|
||||
resolver.Claims[0].Serves = []string{"routes", "route", "unroute"}
|
||||
if got := CatalogueProblems(Shelf{"uplink": uplink, resolver.Module: resolver}); len(got) != 0 {
|
||||
t.Errorf("the uplink's holder and the resolver's declaring the resolver file were refused: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A module's own seat names paths under the same rules: a node seat only, literal, and none another seat
|
||||
// names — the second declaration loses. Its holder declares them, as a mesh seat's does.
|
||||
func TestAModulesOwnSeatNamesPathsUnderTheSameRules(t *testing.T) {
|
||||
meshScoped := Manifest{Module: "a", Version: "1", DefinesSeats: []SeatDeclaration{
|
||||
{Name: "a-seat", Paths: []string{"/etc/a"}}}}
|
||||
if got := strings.Join(declaredSeatProblems(meshScoped), "; "); !strings.Contains(got, "only a node seat names paths") {
|
||||
t.Errorf("a module's mesh seat naming a path was accepted: %q", got)
|
||||
}
|
||||
placeholder := Manifest{Module: "a", Version: "1", DefinesSeats: []SeatDeclaration{
|
||||
{Name: "a-seat", Scope: ScopeNode, Paths: []string{"${dir:state}/a"}}}}
|
||||
if got := strings.Join(declaredSeatProblems(placeholder), "; "); !strings.Contains(got, "placeholder") {
|
||||
t.Errorf("a module's seat naming a path with a placeholder was accepted: %q", got)
|
||||
}
|
||||
|
||||
taken := Manifest{Module: "b", Version: "1", DefinesSeats: []SeatDeclaration{
|
||||
{Name: "b-seat", Scope: ScopeNode, Paths: []string{"/etc/hosts"}}}}
|
||||
if got := strings.Join(CatalogueProblems(Shelf{"b": taken}), "; "); !strings.Contains(got,
|
||||
`b declares the seat b-seat naming the path "/etc/hosts", which node-hostname already names`) {
|
||||
t.Errorf("a module's seat naming /etc/hosts was accepted: %q", got)
|
||||
}
|
||||
|
||||
own := Manifest{Module: "c", Version: "1",
|
||||
DefinesSeats: []SeatDeclaration{{Name: "c-seat", Scope: ScopeNode, Paths: []string{"/etc/c.conf"}}},
|
||||
Claims: []Claim{{Name: "c-seat", Scope: ScopeNode}}}
|
||||
if got := strings.Join(CatalogueProblems(Shelf{"c": own}), "; "); !strings.Contains(got,
|
||||
"c claims c-seat but does not declare /etc/c.conf") {
|
||||
t.Errorf("a holder of a module's seat without its path was accepted: %q", got)
|
||||
}
|
||||
own.Resources = []map[string]any{{"id": "conf", "type": "file", "path": "/etc/c.conf", "content": "x"}}
|
||||
if got := CatalogueProblems(Shelf{"c": own}); len(got) != 0 {
|
||||
t.Errorf("a holder declaring its own seat's path was refused: %v", got)
|
||||
}
|
||||
other := Manifest{Module: "d", Version: "1", Resources: []map[string]any{
|
||||
{"id": "conf", "type": "file", "path": "/etc/c.conf", "content": "x"}}}
|
||||
if got := strings.Join(CatalogueProblems(Shelf{"c": own, "d": other}), "; "); !strings.Contains(got,
|
||||
`d declares the path "/etc/c.conf", which the seat c-seat names`) {
|
||||
t.Errorf("a module not claiming c-seat declared its path: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
type writesHosts struct{}
|
||||
|
||||
func (writesHosts) Resources(string) ([]map[string]any, bool, error) {
|
||||
return []map[string]any{{"id": "hosts", "type": "file", "path": "/etc/hosts", "into": "block",
|
||||
"content": "10.42.0.1 anchor\n"}}, true, nil
|
||||
}
|
||||
|
||||
// The controller is bound too: what it generates for a machine on a seat's path is refused at composition.
|
||||
func TestAGeneratedResourceOnASeatsPathIsRefused(t *testing.T) {
|
||||
r := Resolution{Node: "workstation", Modules: []Manifest{
|
||||
{Module: "mesh-names", Computed: "mesh-names"},
|
||||
}}
|
||||
_, err := r.Declaration(Rendering{Generators: map[string]Generator{"mesh-names": writesHosts{}}})
|
||||
if err == nil {
|
||||
t.Fatal("a generated resource on /etc/hosts was composed")
|
||||
}
|
||||
for _, want := range []string{"mesh-names", "/etc/hosts", "node-hostname"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not name %s: %v", want, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Who owns a seat's path on a machine: the seat naming it and its holder there, with the stepping back
|
||||
// applied — the resolver's holder where node-resolver is held, else the uplink's.
|
||||
func TestPathOwnersApplyTheSteppingBack(t *testing.T) {
|
||||
held := []Held{
|
||||
{Claim: "node-uplink", Scope: ScopeNode, Node: "laptop", Module: "networkmanager"},
|
||||
{Claim: ResolverSeat, Scope: ScopeNode, Node: "laptop", Module: "systemd-resolved"},
|
||||
{Claim: "node-uplink", Scope: ScopeNode, Node: "server", Module: "systemd-networkd"},
|
||||
{Claim: "node-hostname", Scope: ScopeNode, Node: "server", Module: "hostname"},
|
||||
}
|
||||
var got []string
|
||||
for _, o := range PathOwners(Seats(), held) {
|
||||
got = append(got, o.Path+" "+o.Node+" "+o.Seat+" "+o.Module)
|
||||
}
|
||||
want := []string{
|
||||
"/etc/hostname server node-hostname hostname",
|
||||
"/etc/hosts server node-hostname hostname",
|
||||
"/etc/resolv.conf laptop node-resolver systemd-resolved",
|
||||
"/etc/resolv.conf server node-uplink systemd-networkd",
|
||||
}
|
||||
if strings.Join(got, "\n") != strings.Join(want, "\n") {
|
||||
t.Errorf("the owners are\n%s\nnot\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
// The stepping back comes from the seat, and is of a rendered file only: an uplink holder declaring the
|
||||
// resolver file as a resource beside a node-resolver holder is two writers of one path, refused when the
|
||||
// machine is resolved — never composed twice — as before the seat named its paths.
|
||||
func TestAnUplinkDeclaringTheResolverFileAsAResourceIsRefusedBesideTheResolver(t *testing.T) {
|
||||
shelf := resolverShelf(t)
|
||||
uplink := shelf["systemd-networkd"]
|
||||
uplink.Facts = nil
|
||||
uplink.Resources = append(append([]map[string]any(nil), uplink.Resources...),
|
||||
map[string]any{"id": "resolvers", "type": "file", "path": ResolverFile, "content": "nameserver 10.42.0.1\n"})
|
||||
shelf["systemd-networkd"] = uplink
|
||||
shelf["local-resolver"] = localResolver()
|
||||
_, err := Resolve(shelf, []string{"dnsmasq", "systemd-networkd", "local-resolver"}, splitDNSLaptop(), World{})
|
||||
if err == nil || !strings.Contains(err.Error(), `both declare the path "/etc/resolv.conf"`) {
|
||||
t.Fatalf("an uplink declaring the resolver file as a resource beside the resolver was not refused: %v", err)
|
||||
}
|
||||
// Its rendered file steps back, by the seat's definition, and nothing else of it does.
|
||||
rendered := Manifest{Module: "uplink", Version: "1", Claims: []Claim{{Name: "node-uplink"}},
|
||||
Facts: map[string]RosterFile{"resolvers": {Path: "/etc//resolv.conf", Template: "x"},
|
||||
"other": {Path: "/etc/uplink.conf", Template: "x"}}}
|
||||
got := stepsBack(rendered, []Manifest{rendered, localResolver()})
|
||||
if len(got.Facts) != 1 || got.Facts["other"].Path == "" {
|
||||
t.Errorf("the uplink's holder renders %v beside the resolver", got.Facts)
|
||||
}
|
||||
}
|
||||
|
||||
// A path is compared as the machine writes it: cleaned. Each spelling of /etc/hosts is /etc/hosts.
|
||||
func TestAnUncleanSpellingOfASeatsPathIsTheSeatsPath(t *testing.T) {
|
||||
for _, spelled := range []string{"/etc//hosts", "/etc/./hosts", "/etc/../etc/hosts", "/etc/hosts/"} {
|
||||
stranger := Manifest{Module: "stranger", Version: "1", Resources: []map[string]any{
|
||||
{"id": "hosts", "type": "file", "path": spelled, "content": "10.0.0.1 x\n"}}}
|
||||
if got := strings.Join(CatalogueProblems(Shelf{"stranger": stranger}), "; "); !strings.Contains(got,
|
||||
`stranger declares the path "/etc/hosts", which the seat node-hostname names`) {
|
||||
t.Errorf("%s from a module not claiming node-hostname was not refused: %q", spelled, got)
|
||||
}
|
||||
holder := hostnameHolder(map[string]any{"id": "hosts", "type": "file", "path": spelled, "content": "x"},
|
||||
map[string]any{"id": "name", "type": "file", "path": "/etc/hostname", "content": "x"})
|
||||
if err := CanHold(holder, hostnameSeat(t)); err != nil {
|
||||
t.Errorf("a holder declaring %s was refused: %v", spelled, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A seat names a role's own path, never the root or a top-level directory.
|
||||
func TestASeatNamesNoRootOrTopLevelDirectory(t *testing.T) {
|
||||
for _, p := range []string{"/", "/etc"} {
|
||||
m := Manifest{Module: "a", Version: "1", DefinesSeats: []SeatDeclaration{
|
||||
{Name: "a-seat", Scope: ScopeNode, Paths: []string{p}}}}
|
||||
if got := strings.Join(declaredSeatProblems(m), "; "); !strings.Contains(got, "top-level directory") {
|
||||
t.Errorf("a module's seat naming %s was accepted: %q", p, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue as it is passes: no holder of a seat naming a path is missing one, and nobody else
|
||||
// declares one.
|
||||
func TestTheCataloguePassesTheSeatPathRules(t *testing.T) {
|
||||
shelf := Shelf{}
|
||||
for _, m := range theCatalogue(t) {
|
||||
shelf[m.Module] = m
|
||||
}
|
||||
for _, p := range CatalogueProblems(shelf) {
|
||||
if strings.Contains(p, "ADR 0287") {
|
||||
t.Errorf("the catalogue breaks ADR 0287: %s", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -53,6 +53,11 @@ type Seat struct {
|
||||
// ${contribution:<seat>:<kind>}. Compiled, never stored: like the protocol, it is the mesh's
|
||||
// definition of the role, and the store's rows carry no column for it.
|
||||
Receives []Receivable
|
||||
// Paths are the paths on its machine the seat's holder owns (novox/hq ADR 0287): a holder declares
|
||||
// every one, and a module claiming no seat that names a path may not declare it. Only a node seat
|
||||
// names paths. Compiled, never stored, like Receives: adding or taking away a path is a decision,
|
||||
// recorded, and the store's rows carry no column for it.
|
||||
Paths []SeatPath
|
||||
// Decision is the record that made it a seat.
|
||||
Decision string
|
||||
}
|
||||
@@ -178,6 +183,8 @@ var defaultSeats = append([]Seat{
|
||||
// holds none of it. Named node-hosts-file until ADR 0223; the former name resolves to it as an
|
||||
// alias on a mesh that knew it.
|
||||
{Name: "node-hostname", Scope: ScopeNode, Decision: "novox/hq ADR 0199, ADR 0223",
|
||||
// The machine's names are the role's files, whatever program fills it (novox/hq ADR 0287).
|
||||
Paths: []SeatPath{{Path: "/etc/hosts"}, {Path: "/etc/hostname"}},
|
||||
Serves: []Verb{
|
||||
{Name: "entries", Description: "Every line of this machine's /etc/hosts, each marked whose it is: " +
|
||||
"the operator's, or the block of the module or tool that writes it.",
|
||||
@@ -299,12 +306,17 @@ var defaultSeats = append([]Seat{
|
||||
// resolver file as it is and who wrote it, and every link with its addresses, routes and the resolvers
|
||||
// the manager knows for it — the same for every holder, so a question about a machine's names is asked
|
||||
// the same way whatever manages its network. Optional until both holders serve them.
|
||||
{Name: "node-uplink", Scope: ScopeNode, Decision: "novox/hq ADR 0117", Serves: uplinkVerbs()},
|
||||
//
|
||||
// It owns the resolver file, stepping back for node-resolver where that is held (novox/hq ADR 0287,
|
||||
// ADR 0247).
|
||||
{Name: "node-uplink", Scope: ScopeNode, Decision: "novox/hq ADR 0117", Serves: uplinkVerbs(),
|
||||
Paths: []SeatPath{{Path: ResolverFile, StepsBackFor: ResolverSeat}}},
|
||||
// The machine's own resolver (novox/hq ADR 0247): held only where something requires `split-dns` — a
|
||||
// VPN client whose domains must go to its own servers while every other name goes to the mesh's.
|
||||
// Where it is held, its holder writes the resolver file and the uplink's holder steps back from it
|
||||
// (node_resolver.go). It knows nothing of any VPN: its verbs route domains to servers over a link.
|
||||
{Name: ResolverSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0247", Serves: resolverVerbs()},
|
||||
{Name: ResolverSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0247", Serves: resolverVerbs(),
|
||||
Paths: []SeatPath{{Path: ResolverFile}}},
|
||||
// A machine's shares and the shares it mounts (novox/hq ADR 0263): the holder of node-nfs-server
|
||||
// exports a machine's folders to the private network and provides each as `nfs-share`; the holder of
|
||||
// node-mounts writes a mount and an automount unit per share on a machine that asks (shares.go).
|
||||
@@ -364,10 +376,11 @@ func UseSeats(s []Seat) {
|
||||
row.Accepts, row.Emits, row.Serves = d.Accepts, d.Emits, d.Serves
|
||||
}
|
||||
}
|
||||
// What a seat receives and whether it is replicated are never stored (novox/hq ADR 0212, ADR
|
||||
// 0223), so they are always the compiled ones.
|
||||
// What a seat receives, whether it is replicated and the paths it names are never stored
|
||||
// (novox/hq ADR 0212, ADR 0223, ADR 0287), so they are always the compiled ones.
|
||||
if d, known := byName[row.Name]; known {
|
||||
row.Receives = d.Receives
|
||||
row.Paths = d.Paths
|
||||
row.Replicated = d.Replicated
|
||||
row.Serves = optionalAsCompiled(row.Serves, d.Serves)
|
||||
}
|
||||
@@ -541,6 +554,14 @@ func CanHold(m Manifest, seat Seat) error {
|
||||
return fmt.Errorf("%s claims %s, whose holder answers for %q, and %s does not provide %q at %s scope",
|
||||
m.Module, seat.Name, seat.Delivers, m.Module, seat.Delivers, seat.Scope)
|
||||
}
|
||||
// **A holder owns what the role owns** (novox/hq ADR 0287): every path the seat names, declared as
|
||||
// a resource or as a file the mesh renders for it, so which module owns /etc/hosts on a machine is
|
||||
// answered by the seat's definition and not by reading every manifest.
|
||||
if missing := missingPaths(m, seat.Paths); len(missing) > 0 {
|
||||
return fmt.Errorf("%s claims %s but does not declare %s, which that seat's holder owns (novox/hq "+
|
||||
"ADR 0287) — a holder declares every path its seat names, as a resource or as a file the mesh "+
|
||||
"renders for it", m.Module, seat.Name, strings.Join(missing, ", "))
|
||||
}
|
||||
// **Serving the seat's tools is a condition of holding it** (novox/hq ADR 0132). A holder that
|
||||
// does not answer what the role promises is every caller's timeout, found at registration and
|
||||
// at handover instead, naming the verbs rather than the fact that something is missing.
|
||||
|
||||
@@ -67,6 +67,10 @@ type SeatDeclaration struct {
|
||||
// Records are state buckets of the declaring module that each user reads under its own name — the
|
||||
// keys `<user>.…` and no other (ADR 0259 §3).
|
||||
Records []string `json:"records,omitempty"`
|
||||
// Paths are the paths on its machine the seat's holder owns (novox/hq ADR 0287), under the rules the
|
||||
// mesh's own seats name theirs by: a node seat only, each absolute and literal, and none another seat
|
||||
// names. A module's seat never steps back.
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
}
|
||||
|
||||
// KindedBenches are the seats that may be kinded (novox/hq ADR 0234 §2): `channel` sends to the operator,
|
||||
@@ -163,6 +167,7 @@ func declaredSeatProblems(m Manifest) []string {
|
||||
}
|
||||
}
|
||||
problems = append(problems, trafficProblems(m, s)...)
|
||||
problems = append(problems, declaredSeatPathProblems(m, s)...)
|
||||
}
|
||||
|
||||
for _, u := range m.Uses {
|
||||
@@ -271,6 +276,10 @@ func CatalogueProblems(shelf Shelf) []string {
|
||||
return ok
|
||||
}
|
||||
|
||||
// Which seats name each path (novox/hq ADR 0287): a module's seat naming one another seat names loses.
|
||||
named, pathProblems := pathSeats(shelfOrder(shelf), shelf, declaredBy)
|
||||
problems = append(problems, pathProblems...)
|
||||
|
||||
// Who claims each kind of a kinded bench, so a second claim of one kind is refused (ADR 0234 §2).
|
||||
kindsTaken := map[string]string{}
|
||||
|
||||
@@ -327,7 +336,15 @@ func CatalogueProblems(shelf Shelf) []string {
|
||||
"%s claims %s but does not serve %s, which that seat's protocol promises",
|
||||
module, c.Name, strings.Join(missing, ", ")))
|
||||
}
|
||||
// And owns what the seat's holder owns, as a holder of the mesh's own seats does (ADR 0287).
|
||||
if missing := missingPaths(m, asSeatPaths(s.Paths)); len(missing) > 0 {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s claims %s but does not declare %s, which that seat's holder owns (novox/hq ADR 0287)",
|
||||
module, c.Name, strings.Join(missing, ", ")))
|
||||
}
|
||||
}
|
||||
// Only a claimant declares a seat's path (novox/hq ADR 0287).
|
||||
problems = append(problems, unclaimedSeatPaths(m, named)...)
|
||||
}
|
||||
// **A trusted holder runs as its own account** (novox/hq ADR 0259 §8): a module saying warrants, or
|
||||
// speaking for a kind that proves its sender, is never carried by a machine's runtime.
|
||||
|
||||
@@ -156,11 +156,7 @@ func secretFilesOf(resources []map[string]any) map[string]bool {
|
||||
// A name the module never declared is refused here rather than on the machine. The host would
|
||||
// refuse it too — but it would do so having already been handed a declaration, which reads as the
|
||||
// mesh sending something broken, and the name it could not find is a manifest's typo.
|
||||
//
|
||||
// **An own secret issued outside the mesh that nobody gave is filled with nothing** (novox/hq issue 401): the mesh
|
||||
// never makes one (ADR 0228), and a file is where the module reads it, so the place says empty — the module sees it
|
||||
// was not given, where a value the mesh made looked given and wrong. notGiven names those secrets.
|
||||
func intoFile(resource map[string]any, sealed map[string]string, notGiven map[string]bool, module string) error {
|
||||
func intoFile(resource map[string]any, sealed map[string]string, module string) error {
|
||||
if fmt.Sprint(resource["type"]) != "file" {
|
||||
return nil
|
||||
}
|
||||
@@ -175,10 +171,6 @@ func intoFile(resource map[string]any, sealed map[string]string, notGiven map[st
|
||||
into := map[string]any{}
|
||||
for _, name := range used {
|
||||
value := sealed[name]
|
||||
if value == "" && notGiven[name] {
|
||||
content = strings.ReplaceAll(content, "${secret:"+name+"}", "")
|
||||
continue
|
||||
}
|
||||
if value == "" {
|
||||
return fmt.Errorf(
|
||||
"%s has a file that says ${secret:%s}, and %s has no secret of its own by that "+
|
||||
@@ -187,25 +179,10 @@ func intoFile(resource map[string]any, sealed map[string]string, notGiven map[st
|
||||
}
|
||||
into[name] = value
|
||||
}
|
||||
resource["content"] = content
|
||||
if len(into) > 0 {
|
||||
resource["secrets"] = into
|
||||
}
|
||||
resource["secrets"] = into
|
||||
return nil
|
||||
}
|
||||
|
||||
// outsideNotGiven is each own secret of a module issued outside the mesh that this composition holds no value for
|
||||
// (novox/hq issue 401): what intoFile fills with nothing, and what nothing restarts on.
|
||||
func outsideNotGiven(m Manifest, given map[string]string) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for name, own := range m.OwnSecrets.Plain() {
|
||||
if own.IssuedBy == IssuedOutside && given[name] == "" {
|
||||
out[name] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// namesOr says what a module could have written, because the answer to "that name is wrong" is
|
||||
// almost always one of two or three right ones.
|
||||
func namesOr(sealed map[string]string) string {
|
||||
|
||||
@@ -33,7 +33,9 @@ func TestTheUplinkSeatPromisesItsVerbsAndRequiresNoneYet(t *testing.T) {
|
||||
if !slices.Equal(names, []string{"resolvers", "links"}) {
|
||||
t.Fatalf("node-uplink promises %v", names)
|
||||
}
|
||||
today := Manifest{Module: "networkmanager", Version: "1", Claims: []Claim{{Name: "node-uplink", Scope: ScopeNode}}}
|
||||
// The resolver file is the seat's path (novox/hq ADR 0287), which every holder of today renders.
|
||||
today := Manifest{Module: "networkmanager", Version: "1", Claims: []Claim{{Name: "node-uplink", Scope: ScopeNode}},
|
||||
Facts: map[string]RosterFile{"resolvers": {Path: ResolverFile, Template: "x"}}}
|
||||
if err := CanHold(today, seat); err != nil {
|
||||
t.Fatalf("a holder that serves no verb yet is refused: %v", err)
|
||||
}
|
||||
@@ -55,7 +57,8 @@ func TestTheUplinkVerbsReadBackFromTheRowStayOptional(t *testing.T) {
|
||||
// The row as seeding stores it: both verbs, the mark not a column.
|
||||
UseSeats([]Seat{{Name: "node-uplink", Scope: ScopeNode, Serves: []Verb{{Name: "resolvers"}, {Name: "links"}}}})
|
||||
seat := uplinkSeat(t)
|
||||
holder := Manifest{Module: "systemd-networkd", Version: "1", Claims: []Claim{{Name: "node-uplink", Scope: ScopeNode}}}
|
||||
holder := Manifest{Module: "systemd-networkd", Version: "1", Claims: []Claim{{Name: "node-uplink", Scope: ScopeNode}},
|
||||
Facts: map[string]RosterFile{"resolvers": {Path: ResolverFile, Template: "x"}}}
|
||||
if err := CanHold(holder, seat); err != nil {
|
||||
t.Fatalf("read back from the row, the verbs are required: %v", err)
|
||||
}
|
||||
|
||||
@@ -116,15 +116,9 @@ var ControllerVerbs = []Verb{
|
||||
"limit": "how many builds to list (default 20); not with log",
|
||||
}, nil)},
|
||||
{Name: "plans", Description: "What the last merges produced and where each stands (novox/hq ADR 0162): " +
|
||||
"the tiers, the tier a plan is at, what it waits for and since when; one plan whole, given its id; " +
|
||||
"or, given wait, one plan followed until it moves.",
|
||||
"the tiers, the tier a plan is at, what it waits for and since when; one plan whole, given its id.",
|
||||
Input: schema(map[string]string{
|
||||
"id": "a plan's id (as `plans` lists them): that plan, tier by tier",
|
||||
"wait": "a plan's id: wait until it moves, then answer it as data — the plan, and what ended the wait " +
|
||||
"(moved, tier, done, or bound); ask again to follow it further (novox/hq issue 229)",
|
||||
"until": "with wait: moved (any change, the default), tier (it reaches another tier) or done (it ends)",
|
||||
"bound": "with wait: how long to wait at most, as 15s or seconds — 20s, the default, is the longest, " +
|
||||
"inside what a call is answered within",
|
||||
"id": "a plan's id (as `plans` lists them): that plan, tier by tier",
|
||||
"stop": "a plan's id: stop it — what was asked still builds, nothing further is asked",
|
||||
"close": "a plan's id: close a plan that will not move again, as failed by hand (novox/hq issue 254)",
|
||||
"retry": "a failed plan's id: ask its failed builds again under new ids, and carry the plan on from that tier (novox/hq ADR 0219)",
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **A plan's id is a value, never SQL** (novox/hq issue 561): the id reaches PlanByID from a caller of the
|
||||
// controller's plans verb, so an id holding a quote is read as a plain id — no error from the store, and no
|
||||
// other plan answered for it.
|
||||
func TestAPlanIdHoldingAQuoteIsReadAsAPlainId(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
p := Plan{ID: "plan-561", Repository: "novox/mesh-controller", Commit: "abc", Created: time.Now().UTC(),
|
||||
State: PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*PlanModule{"a": {}}}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, id := range []string{
|
||||
"x' or '1'='1", // every plan, when pasted into the query
|
||||
"plan-561' or id like '%", // the plan above, by a pattern
|
||||
"it's", // a quote alone: a syntax error, when pasted
|
||||
} {
|
||||
got, err := inv.PlanByID(ctx, id)
|
||||
if err == nil {
|
||||
t.Fatalf("PlanByID(%q) answered %s, a plan of another id", id, got.ID)
|
||||
}
|
||||
if want := "no plan " + id; err.Error() != want {
|
||||
t.Fatalf("PlanByID(%q) failed with %q, not as a plain id no plan has (%q)", id, err, want)
|
||||
}
|
||||
}
|
||||
// A plan whose id holds a quote is found by it, and by nothing else.
|
||||
quoted := p
|
||||
quoted.ID, quoted.Revision = "plan-'561'", 0
|
||||
if err := inv.SavePlan(ctx, "ed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := inv.PlanByID(ctx, quoted.ID); err != nil || got.ID != quoted.ID {
|
||||
t.Fatalf("the plan with a quote in its id was not read back: %v %+v", err, got.ID)
|
||||
}
|
||||
if recent, err := inv.RecentPlans(ctx, 1); err != nil || len(recent) != 1 || !strings.HasPrefix(recent[0].ID, "plan-") {
|
||||
t.Fatalf("RecentPlans with a parameter: %v %+v", err, recent)
|
||||
}
|
||||
}
|
||||
@@ -623,7 +623,7 @@ func (i *Inventory) PlansSince(ctx context.Context, since time.Time) ([]Plan, er
|
||||
|
||||
// RecentPlans is the last few plans, newest first, open or not — what the overview shows.
|
||||
func (i *Inventory) RecentPlans(ctx context.Context, limit int) ([]Plan, error) {
|
||||
return i.plans(ctx, `order by created desc limit $1`, limit)
|
||||
return i.plans(ctx, fmt.Sprintf(`order by created desc limit %d`, limit))
|
||||
}
|
||||
|
||||
// NewestMergeOf is the plan, in any state, of the newest merge into a repository's branch that the mesh
|
||||
@@ -638,10 +638,9 @@ func (i *Inventory) NewestMergeOf(ctx context.Context, repository, branch string
|
||||
return plans[0], true, nil
|
||||
}
|
||||
|
||||
// PlanByID is one plan. The id is a parameter, never part of the query (novox/hq issue 561): it comes from a
|
||||
// caller of the plans verb.
|
||||
// PlanByID is one plan.
|
||||
func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
|
||||
plans, err := i.plans(ctx, `where id = $1`, id)
|
||||
plans, err := i.plans(ctx, `where id = '`+id+`'`)
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
|
||||
@@ -325,8 +325,6 @@ func (i *Inventory) ModuleSecretIfIssued(
|
||||
return sealed, true, nil
|
||||
}
|
||||
|
||||
// SecretForModule makes what is not held, so it is never asked for an own secret issued outside the mesh: the
|
||||
// composition reads only a value a person gave for one (GivenOwnSecret; novox/hq issue 401, ADR 0228).
|
||||
func (i *Inventory) SecretForModule(ctx context.Context, node, module, name string) (string, error) {
|
||||
key, err := i.SealingKeyOf(ctx, node)
|
||||
if err != nil {
|
||||
@@ -657,35 +655,6 @@ func (i *Inventory) GivenMembers(ctx context.Context, node, module, family strin
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// GivenOwnSecret is an own secret issued outside the mesh as a person gave it on a machine (novox/hq issue 401, ADR
|
||||
// 0228): its value sealed to the machine and whether that is the key the machine holds now. Not given is false: a
|
||||
// value the mesh made for it is not one a person gave, and is never read as one; a machine with no sealing key
|
||||
// holds none either.
|
||||
func (i *Inventory) GivenOwnSecret(ctx context.Context, node, module, name string) (GivenMember, bool, error) {
|
||||
key, err := i.SealingKeyOf(ctx, node)
|
||||
if err != nil || key == "" {
|
||||
return GivenMember{}, false, err
|
||||
}
|
||||
record, err := i.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return GivenMember{}, false, err
|
||||
}
|
||||
g := GivenMember{Name: name}
|
||||
var against string
|
||||
err = i.store.Pool().QueryRow(ctx,
|
||||
`select sealed, node_key from module_secret
|
||||
where node = $1 and module = $2 and name = $3 and origin = 'accepted'`,
|
||||
record.ID, module, name).Scan(&g.Sealed, &against)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return GivenMember{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return GivenMember{}, false, err
|
||||
}
|
||||
g.Current = against == key
|
||||
return g, true, nil
|
||||
}
|
||||
|
||||
// GivenOwnSecrets is every own secret of a module a person gave on a machine, by name, with when (novox/hq ADR
|
||||
// 0283): what the controller checks a module's wait for a secret against. A value the mesh made is not one.
|
||||
func (i *Inventory) GivenOwnSecrets(ctx context.Context, node, module string) (map[string]time.Time, error) {
|
||||
|
||||
@@ -138,9 +138,6 @@ type GroupMember struct {
|
||||
Repository string `json:"repository"`
|
||||
Ref string `json:"ref"`
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
// Base is the branch it merges into: the build seat merges the head into that trunk's head before the
|
||||
// group is composed (novox/hq ADR 0323 §6). A build seat from before it composes the head alone.
|
||||
Base string `json:"base,omitempty"`
|
||||
}
|
||||
|
||||
// Who judges a merge check's gate.
|
||||
@@ -167,14 +164,6 @@ type CheckOutcome struct {
|
||||
// Gate and RepoCheck are its two layers; Verdict and Summary above are the gate's.
|
||||
Gate *CheckLayer `json:"gate,omitempty"`
|
||||
RepoCheck *CheckLayer `json:"repo-check,omitempty"`
|
||||
// Head and Trunk are what was judged (novox/hq ADR 0323 §3): the pull request's head merged into the trunk
|
||||
// commit, each as twelve characters. Trunk is empty from a build seat from before it, which judged the head
|
||||
// alone. Beside are the commits of the repositories read beside it, by directory, named and not judged
|
||||
// against (issue 568); Members a delivery group's other heads, each with the trunk commit it merged into.
|
||||
Head string `json:"head,omitempty"`
|
||||
Trunk string `json:"trunk,omitempty"`
|
||||
Beside map[string]string `json:"beside,omitempty"`
|
||||
Members []CheckedMember `json:"members,omitempty"`
|
||||
}
|
||||
|
||||
// SourceOnSeat names a repository by the seat whose holder serves it and its path there.
|
||||
|
||||
+10
-47
@@ -601,24 +601,6 @@ func Acknowledge(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// LongestAnswer is the longest a handler that waits on purpose may hold its caller's one answer
|
||||
// (AnswersWithin, novox/hq issue 229): inside the console's thirty seconds with room for the trip back,
|
||||
// and inside the bus's window for an answer (broker.ResponseTTL). A variable so a test need not wait.
|
||||
var LongestAnswer = 25 * time.Second
|
||||
|
||||
type answerWithinKey struct{}
|
||||
|
||||
// AnswersWithin asks that the call's caller be answered in full if the handler ends within d of the
|
||||
// call's start, rather than within AnswerWithin: for a handler that waits on purpose, with a bound —
|
||||
// `plans` waiting for a plan to move (novox/hq issue 229). Past d, or past LongestAnswer whichever is
|
||||
// shorter, its caller is told it is running, as for any call. Never shortens the window; without effect
|
||||
// outside a served call, and once the caller was answered.
|
||||
func AnswersWithin(ctx context.Context, d time.Duration) {
|
||||
if f, ok := ctx.Value(answerWithinKey{}).(func(time.Duration)); ok {
|
||||
f(d)
|
||||
}
|
||||
}
|
||||
|
||||
// running is the interim answer: what a caller reads when the call has not finished.
|
||||
func running(c *Call, within time.Duration, acknowledged bool, follow string) []byte {
|
||||
why := fmt.Sprintf("it has not finished in %s", within)
|
||||
@@ -663,13 +645,6 @@ func (l *CallLog) serveCallWithin(seat, verb string, args json.RawMessage, reply
|
||||
acknowledged := make(chan struct{})
|
||||
var once sync.Once
|
||||
ctx = context.WithValue(ctx, answerNowKey{}, func() { once.Do(func() { close(acknowledged) }) })
|
||||
longer := make(chan time.Duration, 1)
|
||||
ctx = context.WithValue(ctx, answerWithinKey{}, func(d time.Duration) {
|
||||
select {
|
||||
case longer <- d:
|
||||
default:
|
||||
}
|
||||
})
|
||||
|
||||
type outcome struct {
|
||||
body []byte
|
||||
@@ -728,29 +703,17 @@ func (l *CallLog) serveCallWithin(seat, verb string, args json.RawMessage, reply
|
||||
}
|
||||
}
|
||||
}
|
||||
within, start := AnswerWithin, time.Now()
|
||||
timer := time.NewTimer(within)
|
||||
timer := time.NewTimer(AnswerWithin)
|
||||
defer timer.Stop()
|
||||
waiting:
|
||||
for {
|
||||
select {
|
||||
case o := <-done:
|
||||
l.finish(c, o.body, o.failed, false)
|
||||
say(o.body)
|
||||
return
|
||||
case <-acknowledged:
|
||||
say(running(c, within, true, l.Follow))
|
||||
break waiting
|
||||
case d := <-longer:
|
||||
// A handler waiting on purpose (AnswersWithin): its window is stretched, never past LongestAnswer.
|
||||
if d = min(d, LongestAnswer); d > within {
|
||||
within = d
|
||||
timer.Reset(time.Until(start.Add(d)))
|
||||
}
|
||||
case <-timer.C:
|
||||
say(running(c, within, false, l.Follow))
|
||||
break waiting
|
||||
}
|
||||
select {
|
||||
case o := <-done:
|
||||
l.finish(c, o.body, o.failed, false)
|
||||
say(o.body)
|
||||
return
|
||||
case <-acknowledged:
|
||||
say(running(c, AnswerWithin, true, l.Follow))
|
||||
case <-timer.C:
|
||||
say(running(c, AnswerWithin, false, l.Follow))
|
||||
}
|
||||
o := <-done
|
||||
l.finish(c, o.body, o.failed, true)
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **A handler that waits on purpose is answered in full when it ends inside the window it asked for**
|
||||
// (novox/hq issue 229): `plans` asked to wait for a plan holds its caller's one answer until the plan
|
||||
// moves or its bound passes, rather than being answered "still running" at AnswerWithin.
|
||||
func TestAHandlerThatWaitsOnPurposeIsAnsweredInFullInsideItsWindow(t *testing.T) {
|
||||
shortWindow(t, 20*time.Millisecond)
|
||||
l, a := NewCallLog(), newAnswers(t)
|
||||
l.serveCall("mesh-controller", "plans", json.RawMessage(`{"wait":"plan-1"}`), "_INBOX.x.3",
|
||||
func(ctx context.Context, _ json.RawMessage) (any, error) {
|
||||
AnswersWithin(ctx, 200*time.Millisecond)
|
||||
time.Sleep(80 * time.Millisecond) // past AnswerWithin, inside what it asked for
|
||||
return "plan-1 moved", nil
|
||||
}, a.respond, nil)
|
||||
if got := a.only()["result"]; got != "plan-1 moved" {
|
||||
t.Fatalf("a waiting handler inside its window was answered %v, not in full", got)
|
||||
}
|
||||
}
|
||||
|
||||
// What a handler may ask for is bounded: past LongestAnswer its caller is told it is running, as any
|
||||
// call is, so no answer is ever later than the console waits.
|
||||
func TestAWaitIsNeverLongerThanTheLongestAnswer(t *testing.T) {
|
||||
shortWindow(t, 10*time.Millisecond)
|
||||
was := LongestAnswer
|
||||
LongestAnswer = 30 * time.Millisecond
|
||||
t.Cleanup(func() { LongestAnswer = was })
|
||||
l, a := NewCallLog(), newAnswers(t)
|
||||
release := make(chan struct{})
|
||||
finished := make(chan struct{})
|
||||
go func() {
|
||||
l.serveCall("mesh-controller", "plans", nil, "_INBOX.x.4", func(ctx context.Context, _ json.RawMessage) (any, error) {
|
||||
AnswersWithin(ctx, time.Hour)
|
||||
<-release
|
||||
return "late", nil
|
||||
}, a.respond, nil)
|
||||
close(finished)
|
||||
}()
|
||||
select {
|
||||
case <-a.sent:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("a handler asking for an hour held its caller past LongestAnswer")
|
||||
}
|
||||
if got, _ := a.only()["result"].(map[string]any); got["running"] != true {
|
||||
t.Fatalf("past LongestAnswer the caller was answered %v, not that it is running", a.only())
|
||||
}
|
||||
close(release)
|
||||
<-finished
|
||||
}
|
||||
+8
-25
@@ -221,17 +221,13 @@ type SourceMoved struct {
|
||||
// PullUpdated is what the forge announces when an open pull request's head moves — opened, or pushed
|
||||
// to (novox/hq to-be 45 §9): what the controller asks the build seat to check before it merges.
|
||||
type PullUpdated struct {
|
||||
Owner string `json:"owner"`
|
||||
Repo string `json:"repo"`
|
||||
Number int `json:"number"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Base string `json:"base"`
|
||||
Head string `json:"head"`
|
||||
Commit string `json:"head_sha"`
|
||||
// BaseSHA is the base branch's head when the forge announced the head (novox/hq ADR 0323): the trunk commit a
|
||||
// verdict the controller gives without the build seat — a change that runs no build — is judged against. Empty
|
||||
// from a forge module from before it.
|
||||
BaseSHA string `json:"base_sha,omitempty"`
|
||||
Owner string `json:"owner"`
|
||||
Repo string `json:"repo"`
|
||||
Number int `json:"number"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Base string `json:"base"`
|
||||
Head string `json:"head"`
|
||||
Commit string `json:"head_sha"`
|
||||
CloneURL string `json:"clone_url"`
|
||||
HTMLURL string `json:"html_url,omitempty"`
|
||||
// Paths are the files the pull request changes; PathsTruncated says there were more.
|
||||
@@ -282,24 +278,14 @@ type Checked struct {
|
||||
// status from it — the group's verdict is its owner's, mesh-delivery, to say on each member's head.
|
||||
Group string `json:"group,omitempty"`
|
||||
Members []CheckedMember `json:"members,omitempty"`
|
||||
// Trunk is the trunk commit the head was merged into and judged as, twelve characters (novox/hq ADR 0323
|
||||
// §3): the verdict holds for that merge only, and goes out of date when the trunk moves. Empty from a build
|
||||
// seat that judged the head alone. Beside are the commits of the other repositories the check read, by
|
||||
// directory, named and not judged against (issue 568). Took is how long the check ran, for the merge tool
|
||||
// to say how long the next one may take (ADR 0323 §5).
|
||||
Trunk string `json:"trunk,omitempty"`
|
||||
Beside map[string]string `json:"beside,omitempty"`
|
||||
Took string `json:"took,omitempty"`
|
||||
}
|
||||
|
||||
// CheckedMember is one head a group's composed check judged, and the trunk commit it was merged into (novox/hq
|
||||
// ADR 0323 §6), twelve characters; empty from a build seat that composed the head alone.
|
||||
// CheckedMember is one head a group's composed check judged.
|
||||
type CheckedMember struct {
|
||||
Owner string `json:"owner"`
|
||||
Repo string `json:"repo"`
|
||||
Number int `json:"number,omitempty"`
|
||||
Commit string `json:"commit"`
|
||||
Trunk string `json:"trunk,omitempty"`
|
||||
}
|
||||
|
||||
// ChangePlan is what a change does to the mesh, computed from its diffset — a repository, the branch it
|
||||
@@ -325,9 +311,6 @@ type ChangePlan struct {
|
||||
Steps []string `json:"steps,omitempty"`
|
||||
// Summary is the plan in one line, for a commit status.
|
||||
Summary string `json:"summary"`
|
||||
// Trunk is the trunk commit the head was merged into when it was judged (novox/hq ADR 0323 §3), twelve
|
||||
// characters: set on the plan a verdict carries, which mesh-delivery keeps with its other inputs.
|
||||
Trunk string `json:"trunk,omitempty"`
|
||||
}
|
||||
|
||||
// MachinePlan is one machine's part of a change plan.
|
||||
|
||||
Vendored
-8
@@ -7,14 +7,12 @@
|
||||
./cmd/mesh-controller TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt
|
||||
./cmd/mesh-controller TestAStopWithoutAMergeLeavesItOutOfABatch
|
||||
./cmd/mesh-controller TestSavingPlansAndMovingMergesIsOneAct
|
||||
./cmd/mesh-controller TestAPlanFollowedFromTheStoreAnswersWhenItMoves
|
||||
./internal/inventory TestTheLastSendSaysWhatTheMachineAnswered
|
||||
./internal/inventory TestAReportIsTheLastSendsAnswerOnlyWhenItIsAboutThatSend
|
||||
./cmd/mesh-controller TestNodeShowAndStatusReadTheLastSendFromTheStore
|
||||
./internal/inventory TestAnIssuedValueIsSealedToEveryHolderOnceAndNotAgain
|
||||
./internal/inventory TestAnIssuedValueIsSealedAgainAfterAKeyChangeOrAnAcceptance
|
||||
./internal/inventory TestAConditionsLooksAreKeptBesideTheBucket
|
||||
./internal/inventory TestAPlanIdHoldingAQuoteIsReadAsAPlainId
|
||||
./cmd/mesh-controller TestTheConditionsVerbReadsTheLooksTheBucketNoLongerHolds
|
||||
./internal/inventory TestAWalksStartIsKeptAndARefusedSaveLeavesItUnsaid
|
||||
./cmd/mesh-controller TestAWalkLetGoAndStoppedByVerbsIsSaidStartedAndEnded
|
||||
@@ -29,9 +27,3 @@
|
||||
./cmd/mesh-controller TestAnUnassignThatLeavesItsNodeResolvedSaysNothingOfIt
|
||||
./cmd/mesh-controller TestAnAssignThatLeavesItsNodeUnresolvedSaysOnlyThePlansRefusal
|
||||
./cmd/mesh-controller TestADirectoryOwnersKeyIsRefusedThroughAVerb
|
||||
./cmd/mesh-controller TestTheSendMakesNoValueForAnOutsideSecretNobodyGave
|
||||
./cmd/mesh-controller TestAQuestionComposesWithoutAnOutsideSecretNobodyGave
|
||||
./cmd/mesh-controller TestReplay201
|
||||
./cmd/mesh-controller TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend
|
||||
./cmd/mesh-controller TestAGateJudgesTheOtherMovesBesideAKeptCore
|
||||
./cmd/mesh-controller TestReplay401
|
||||
|
||||
-8
@@ -1,8 +0,0 @@
|
||||
# Package xz authors
|
||||
|
||||
Michael Cross <https://github.com/xi2>
|
||||
|
||||
# XZ Embedded authors
|
||||
|
||||
Lasse Collin <lasse.collin@tukaani.org>
|
||||
Igor Pavlov <http://7-zip.org/>
|
||||
-86
@@ -1,86 +0,0 @@
|
||||
Creative Commons CC0 1.0 Universal
|
||||
|
||||
CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE LEGAL
|
||||
SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN ATTORNEY-CLIENT
|
||||
RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS INFORMATION ON AN "AS-IS" BASIS.
|
||||
CREATIVE COMMONS MAKES NO WARRANTIES REGARDING THE USE OF THIS DOCUMENT OR THE
|
||||
INFORMATION OR WORKS PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES
|
||||
RESULTING FROM THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
|
||||
HEREUNDER.
|
||||
|
||||
Statement of Purpose
|
||||
|
||||
The laws of most jurisdictions throughout the world automatically confer
|
||||
exclusive Copyright and Related Rights (defined below) upon the creator and
|
||||
subsequent owner(s) (each and all, an "owner") of an original work of
|
||||
authorship and/or a database (each, a "Work").
|
||||
|
||||
Certain owners wish to permanently relinquish those rights to a Work for the
|
||||
purpose of contributing to a commons of creative, cultural and scientific
|
||||
works ("Commons") that the public can reliably and without fear of later
|
||||
claims of infringement build upon, modify, incorporate in other works, reuse
|
||||
and redistribute as freely as possible in any form whatsoever and for any
|
||||
purposes, including without limitation commercial purposes. These owners may
|
||||
contribute to the Commons to promote the ideal of a free culture and the
|
||||
further production of creative, cultural and scientific works, or to gain
|
||||
reputation or greater distribution for their Work in part through the use and
|
||||
efforts of others.
|
||||
|
||||
For these and/or other purposes and motivations, and without any expectation
|
||||
of additional consideration or compensation, the person associating CC0 with a
|
||||
Work (the "Affirmer"), to the extent that he or she is an owner of Copyright
|
||||
and Related Rights in the Work, voluntarily elects to apply CC0 to the Work
|
||||
and publicly distribute the Work under its terms, with knowledge of his or her
|
||||
Copyright and Related Rights in the Work and the meaning and intended legal
|
||||
effect of CC0 on those rights.
|
||||
|
||||
1. Copyright and Related Rights. A Work made available under CC0 may be protected by copyright and related or neighboring rights ("Copyright and Related Rights"). Copyright and Related Rights include, but are not limited to, the following:
|
||||
|
||||
i. the right to reproduce, adapt, distribute, perform, display, communicate,
|
||||
and translate a Work;
|
||||
|
||||
ii. moral rights retained by the original author(s) and/or performer(s);
|
||||
|
||||
iii. publicity and privacy rights pertaining to a person's image or
|
||||
likeness depicted in a Work;
|
||||
|
||||
iv. rights protecting against unfair competition in regards to a Work, subject
|
||||
to the limitations in paragraph 4(a), below;
|
||||
|
||||
v. rights protecting the extraction, dissemination, use and reuse of data in a
|
||||
Work;
|
||||
|
||||
vi. database rights (such as those arising under Directive 96/9/EC of the
|
||||
European Parliament and of the Council of 11 March 1996 on the legal
|
||||
protection of databases, and under any national implementation thereof,
|
||||
including any amended or successor version of such directive); and
|
||||
|
||||
vii. other similar, equivalent or corresponding rights throughout the world
|
||||
based on applicable law or treaty, and any national implementations thereof.
|
||||
|
||||
2. Waiver. To the greatest extent permitted by, but not in contravention of, applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and unconditionally waives, abandons, and surrenders all of Affirmer's Copyright and Related Rights and associated claims and causes of action, whether now known or unknown (including existing as well as future claims and causes of action), in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each member of the public at large and to the detriment of Affirmer's heirs and successors, fully intending that such Waiver shall not be subject to revocation, rescission, cancellation, termination, or any other legal or equitable action to disrupt the quiet enjoyment of the Work by the public as contemplated by Affirmer's express Statement of Purpose.
|
||||
|
||||
3. Public License Fallback. Should any part of the Waiver for any reason be judged legally invalid or ineffective under applicable law, then the Waiver shall be preserved to the maximum extent permitted taking into account Affirmer's express Statement of Purpose. In addition, to the extent the Waiver is so judged Affirmer hereby grants to each affected person a royalty-free, non transferable, non sublicensable, non exclusive, irrevocable and unconditional license to exercise Affirmer's Copyright and Related Rights in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "License"). The License shall be deemed effective as of the date CC0 was applied by Affirmer to the Work. Should any part of the License for any reason be judged legally invalid or ineffective under applicable law, such partial invalidity or ineffectiveness shall not invalidate the remainder of the License, and in such case Affirmer hereby affirms that he or she will not (i) exercise any of his or her remaining Copyright and Related Rights in the Work or (ii) assert any associated claims and causes of action with respect to the Work, in either case contrary to Affirmer's express Statement of Purpose.
|
||||
|
||||
4. Limitations and Disclaimers.
|
||||
|
||||
a. No trademark or patent rights held by Affirmer are waived, abandoned,
|
||||
surrendered, licensed or otherwise affected by this document.
|
||||
|
||||
b. Affirmer offers the Work as-is and makes no representations or warranties
|
||||
of any kind concerning the Work, express, implied, statutory or otherwise,
|
||||
including without limitation warranties of title, merchantability, fitness for
|
||||
a particular purpose, non infringement, or the absence of latent or other
|
||||
defects, accuracy, or the present or absence of errors, whether or not
|
||||
discoverable, all to the greatest extent permissible under applicable law.
|
||||
|
||||
c. Affirmer disclaims responsibility for clearing rights of other persons that
|
||||
may apply to the Work or any use thereof, including without limitation any
|
||||
person's Copyright and Related Rights in the Work. Further, Affirmer
|
||||
disclaims responsibility for obtaining any necessary consents, permissions or
|
||||
other rights required for any use of the Work.
|
||||
|
||||
d. Affirmer understands and acknowledges that Creative Commons is not a party
|
||||
to this document and has no duty or obligation with respect to this CC0 or use
|
||||
of the Work.
|
||||
|
||||
-19
@@ -1,19 +0,0 @@
|
||||
# Xz
|
||||
|
||||
Package xz implements XZ decompression natively in Go.
|
||||
|
||||
Documentation at <https://pkg.go.dev/github.com/therootcompany/xz>.
|
||||
|
||||
Download and install with `go get github.com/xi2/xz`.
|
||||
|
||||
If you need compression as well as decompression, you might want to
|
||||
look at <https://github.com/ulikunitz/xz>.
|
||||
|
||||
# LICENSE
|
||||
|
||||
This was originally released into the public domain by the AUTHORS.
|
||||
|
||||
Here it is licensed more explicitly as Creative Commons CC0 1.0 Universal
|
||||
(CC0-1.0) so that it can be detected by automated tooling, and satisfy the legal
|
||||
requirements for vendor integration for cases in which a "public domain"
|
||||
statement is not sufficient.
|
||||
-461
@@ -1,461 +0,0 @@
|
||||
/*
|
||||
* Branch/Call/Jump (BCJ) filter decoders
|
||||
*
|
||||
* Authors: Lasse Collin <lasse.collin@tukaani.org>
|
||||
* Igor Pavlov <http://7-zip.org/>
|
||||
*
|
||||
* Translation to Go: Michael Cross <https://github.com/xi2>
|
||||
*
|
||||
* This file has been put into the public domain.
|
||||
* You can do whatever you want with this file.
|
||||
*/
|
||||
|
||||
package xz
|
||||
|
||||
/* from linux/lib/xz/xz_dec_bcj.c *************************************/
|
||||
|
||||
type xzDecBCJ struct {
|
||||
/* Type of the BCJ filter being used */
|
||||
typ xzFilterID
|
||||
/*
|
||||
* Return value of the next filter in the chain. We need to preserve
|
||||
* this information across calls, because we must not call the next
|
||||
* filter anymore once it has returned xzStreamEnd
|
||||
*/
|
||||
ret xzRet
|
||||
/*
|
||||
* Absolute position relative to the beginning of the uncompressed
|
||||
* data (in a single .xz Block).
|
||||
*/
|
||||
pos int
|
||||
/* x86 filter state */
|
||||
x86PrevMask uint32
|
||||
/* Temporary space to hold the variables from xzBuf */
|
||||
out []byte
|
||||
outPos int
|
||||
temp struct {
|
||||
/* Amount of already filtered data in the beginning of buf */
|
||||
filtered int
|
||||
/*
|
||||
* Buffer to hold a mix of filtered and unfiltered data. This
|
||||
* needs to be big enough to hold Alignment + 2 * Look-ahead:
|
||||
*
|
||||
* Type Alignment Look-ahead
|
||||
* x86 1 4
|
||||
* PowerPC 4 0
|
||||
* IA-64 16 0
|
||||
* ARM 4 0
|
||||
* ARM-Thumb 2 2
|
||||
* SPARC 4 0
|
||||
*/
|
||||
buf []byte // slice buf will be backed by bufArray
|
||||
bufArray [16]byte
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* This is used to test the most significant byte of a memory address
|
||||
* in an x86 instruction.
|
||||
*/
|
||||
func bcjX86TestMSByte(b byte) bool {
|
||||
return b == 0x00 || b == 0xff
|
||||
}
|
||||
|
||||
func bcjX86Filter(s *xzDecBCJ, buf []byte) int {
|
||||
var maskToAllowedStatus = []bool{
|
||||
true, true, true, false, true, false, false, false,
|
||||
}
|
||||
var maskToBitNum = []byte{0, 1, 2, 2, 3, 3, 3, 3}
|
||||
var i int
|
||||
var prevPos int = -1
|
||||
var prevMask uint32 = s.x86PrevMask
|
||||
var src uint32
|
||||
var dest uint32
|
||||
var j uint32
|
||||
var b byte
|
||||
if len(buf) <= 4 {
|
||||
return 0
|
||||
}
|
||||
for i = 0; i < len(buf)-4; i++ {
|
||||
if buf[i]&0xfe != 0xe8 {
|
||||
continue
|
||||
}
|
||||
prevPos = i - prevPos
|
||||
if prevPos > 3 {
|
||||
prevMask = 0
|
||||
} else {
|
||||
prevMask = (prevMask << (uint(prevPos) - 1)) & 7
|
||||
if prevMask != 0 {
|
||||
b = buf[i+4-int(maskToBitNum[prevMask])]
|
||||
if !maskToAllowedStatus[prevMask] || bcjX86TestMSByte(b) {
|
||||
prevPos = i
|
||||
prevMask = prevMask<<1 | 1
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
prevPos = i
|
||||
if bcjX86TestMSByte(buf[i+4]) {
|
||||
src = getLE32(buf[i+1:])
|
||||
for {
|
||||
dest = src - uint32(s.pos+i+5)
|
||||
if prevMask == 0 {
|
||||
break
|
||||
}
|
||||
j = uint32(maskToBitNum[prevMask]) * 8
|
||||
b = byte(dest >> (24 - j))
|
||||
if !bcjX86TestMSByte(b) {
|
||||
break
|
||||
}
|
||||
src = dest ^ (1<<(32-j) - 1)
|
||||
}
|
||||
dest &= 0x01FFFFFF
|
||||
dest |= 0 - dest&0x01000000
|
||||
putLE32(dest, buf[i+1:])
|
||||
i += 4
|
||||
} else {
|
||||
prevMask = prevMask<<1 | 1
|
||||
}
|
||||
}
|
||||
prevPos = i - prevPos
|
||||
if prevPos > 3 {
|
||||
s.x86PrevMask = 0
|
||||
} else {
|
||||
s.x86PrevMask = prevMask << (uint(prevPos) - 1)
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
func bcjPowerPCFilter(s *xzDecBCJ, buf []byte) int {
|
||||
var i int
|
||||
var instr uint32
|
||||
for i = 0; i+4 <= len(buf); i += 4 {
|
||||
instr = getBE32(buf[i:])
|
||||
if instr&0xFC000003 == 0x48000001 {
|
||||
instr &= 0x03FFFFFC
|
||||
instr -= uint32(s.pos + i)
|
||||
instr &= 0x03FFFFFC
|
||||
instr |= 0x48000001
|
||||
putBE32(instr, buf[i:])
|
||||
}
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
var bcjIA64BranchTable = [...]byte{
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
4, 4, 6, 6, 0, 0, 7, 7,
|
||||
4, 4, 0, 0, 4, 4, 0, 0,
|
||||
}
|
||||
|
||||
func bcjIA64Filter(s *xzDecBCJ, buf []byte) int {
|
||||
var branchTable = bcjIA64BranchTable[:]
|
||||
/*
|
||||
* The local variables take a little bit stack space, but it's less
|
||||
* than what LZMA2 decoder takes, so it doesn't make sense to reduce
|
||||
* stack usage here without doing that for the LZMA2 decoder too.
|
||||
*/
|
||||
/* Loop counters */
|
||||
var i int
|
||||
var j int
|
||||
/* Instruction slot (0, 1, or 2) in the 128-bit instruction word */
|
||||
var slot uint32
|
||||
/* Bitwise offset of the instruction indicated by slot */
|
||||
var bitPos uint32
|
||||
/* bit_pos split into byte and bit parts */
|
||||
var bytePos uint32
|
||||
var bitRes uint32
|
||||
/* Address part of an instruction */
|
||||
var addr uint32
|
||||
/* Mask used to detect which instructions to convert */
|
||||
var mask uint32
|
||||
/* 41-bit instruction stored somewhere in the lowest 48 bits */
|
||||
var instr uint64
|
||||
/* Instruction normalized with bit_res for easier manipulation */
|
||||
var norm uint64
|
||||
for i = 0; i+16 <= len(buf); i += 16 {
|
||||
mask = uint32(branchTable[buf[i]&0x1f])
|
||||
for slot, bitPos = 0, 5; slot < 3; slot, bitPos = slot+1, bitPos+41 {
|
||||
if (mask>>slot)&1 == 0 {
|
||||
continue
|
||||
}
|
||||
bytePos = bitPos >> 3
|
||||
bitRes = bitPos & 7
|
||||
instr = 0
|
||||
for j = 0; j < 6; j++ {
|
||||
instr |= uint64(buf[i+j+int(bytePos)]) << (8 * uint(j))
|
||||
}
|
||||
norm = instr >> bitRes
|
||||
if (norm>>37)&0x0f == 0x05 && (norm>>9)&0x07 == 0 {
|
||||
addr = uint32((norm >> 13) & 0x0fffff)
|
||||
addr |= (uint32(norm>>36) & 1) << 20
|
||||
addr <<= 4
|
||||
addr -= uint32(s.pos + i)
|
||||
addr >>= 4
|
||||
norm &= ^(uint64(0x8fffff) << 13)
|
||||
norm |= uint64(addr&0x0fffff) << 13
|
||||
norm |= uint64(addr&0x100000) << (36 - 20)
|
||||
instr &= 1<<bitRes - 1
|
||||
instr |= norm << bitRes
|
||||
for j = 0; j < 6; j++ {
|
||||
buf[i+j+int(bytePos)] = byte(instr >> (8 * uint(j)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
func bcjARMFilter(s *xzDecBCJ, buf []byte) int {
|
||||
var i int
|
||||
var addr uint32
|
||||
for i = 0; i+4 <= len(buf); i += 4 {
|
||||
if buf[i+3] == 0xeb {
|
||||
addr = uint32(buf[i]) | uint32(buf[i+1])<<8 |
|
||||
uint32(buf[i+2])<<16
|
||||
addr <<= 2
|
||||
addr -= uint32(s.pos + i + 8)
|
||||
addr >>= 2
|
||||
buf[i] = byte(addr)
|
||||
buf[i+1] = byte(addr >> 8)
|
||||
buf[i+2] = byte(addr >> 16)
|
||||
}
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
func bcjARMThumbFilter(s *xzDecBCJ, buf []byte) int {
|
||||
var i int
|
||||
var addr uint32
|
||||
for i = 0; i+4 <= len(buf); i += 2 {
|
||||
if buf[i+1]&0xf8 == 0xf0 && buf[i+3]&0xf8 == 0xf8 {
|
||||
addr = uint32(buf[i+1]&0x07)<<19 |
|
||||
uint32(buf[i])<<11 |
|
||||
uint32(buf[i+3]&0x07)<<8 |
|
||||
uint32(buf[i+2])
|
||||
addr <<= 1
|
||||
addr -= uint32(s.pos + i + 4)
|
||||
addr >>= 1
|
||||
buf[i+1] = byte(0xf0 | (addr>>19)&0x07)
|
||||
buf[i] = byte(addr >> 11)
|
||||
buf[i+3] = byte(0xf8 | (addr>>8)&0x07)
|
||||
buf[i+2] = byte(addr)
|
||||
i += 2
|
||||
}
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
func bcjSPARCFilter(s *xzDecBCJ, buf []byte) int {
|
||||
var i int
|
||||
var instr uint32
|
||||
for i = 0; i+4 <= len(buf); i += 4 {
|
||||
instr = getBE32(buf[i:])
|
||||
if instr>>22 == 0x100 || instr>>22 == 0x1ff {
|
||||
instr <<= 2
|
||||
instr -= uint32(s.pos + i)
|
||||
instr >>= 2
|
||||
instr = (0x40000000 - instr&0x400000) |
|
||||
0x40000000 | (instr & 0x3FFFFF)
|
||||
putBE32(instr, buf[i:])
|
||||
}
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
/*
|
||||
* Apply the selected BCJ filter. Update *pos and s.pos to match the amount
|
||||
* of data that got filtered.
|
||||
*/
|
||||
func bcjApply(s *xzDecBCJ, buf []byte, pos *int) {
|
||||
var filtered int
|
||||
buf = buf[*pos:]
|
||||
switch s.typ {
|
||||
case idBCJX86:
|
||||
filtered = bcjX86Filter(s, buf)
|
||||
case idBCJPowerPC:
|
||||
filtered = bcjPowerPCFilter(s, buf)
|
||||
case idBCJIA64:
|
||||
filtered = bcjIA64Filter(s, buf)
|
||||
case idBCJARM:
|
||||
filtered = bcjARMFilter(s, buf)
|
||||
case idBCJARMThumb:
|
||||
filtered = bcjARMThumbFilter(s, buf)
|
||||
case idBCJSPARC:
|
||||
filtered = bcjSPARCFilter(s, buf)
|
||||
default:
|
||||
/* Never reached */
|
||||
}
|
||||
*pos += filtered
|
||||
s.pos += filtered
|
||||
}
|
||||
|
||||
/*
|
||||
* Flush pending filtered data from temp to the output buffer.
|
||||
* Move the remaining mixture of possibly filtered and unfiltered
|
||||
* data to the beginning of temp.
|
||||
*/
|
||||
func bcjFlush(s *xzDecBCJ, b *xzBuf) {
|
||||
var copySize int
|
||||
copySize = len(b.out) - b.outPos
|
||||
if copySize > s.temp.filtered {
|
||||
copySize = s.temp.filtered
|
||||
}
|
||||
copy(b.out[b.outPos:], s.temp.buf[:copySize])
|
||||
b.outPos += copySize
|
||||
s.temp.filtered -= copySize
|
||||
copy(s.temp.buf, s.temp.buf[copySize:])
|
||||
s.temp.buf = s.temp.buf[:len(s.temp.buf)-copySize]
|
||||
}
|
||||
|
||||
/*
|
||||
* Decode raw stream which has a BCJ filter as the first filter.
|
||||
*
|
||||
* The BCJ filter functions are primitive in sense that they process the
|
||||
* data in chunks of 1-16 bytes. To hide this issue, this function does
|
||||
* some buffering.
|
||||
*/
|
||||
func xzDecBCJRun(s *xzDecBCJ, b *xzBuf, chain func(*xzBuf) xzRet) xzRet {
|
||||
var outStart int
|
||||
/*
|
||||
* Flush pending already filtered data to the output buffer. Return
|
||||
* immediately if we couldn't flush everything, or if the next
|
||||
* filter in the chain had already returned xzStreamEnd.
|
||||
*/
|
||||
if s.temp.filtered > 0 {
|
||||
bcjFlush(s, b)
|
||||
if s.temp.filtered > 0 {
|
||||
return xzOK
|
||||
}
|
||||
if s.ret == xzStreamEnd {
|
||||
return xzStreamEnd
|
||||
}
|
||||
}
|
||||
/*
|
||||
* If we have more output space than what is currently pending in
|
||||
* temp, copy the unfiltered data from temp to the output buffer
|
||||
* and try to fill the output buffer by decoding more data from the
|
||||
* next filter in the chain. Apply the BCJ filter on the new data
|
||||
* in the output buffer. If everything cannot be filtered, copy it
|
||||
* to temp and rewind the output buffer position accordingly.
|
||||
*
|
||||
* This needs to be always run when len(temp.buf) == 0 to handle a special
|
||||
* case where the output buffer is full and the next filter has no
|
||||
* more output coming but hasn't returned xzStreamEnd yet.
|
||||
*/
|
||||
if len(s.temp.buf) < len(b.out)-b.outPos || len(s.temp.buf) == 0 {
|
||||
outStart = b.outPos
|
||||
copy(b.out[b.outPos:], s.temp.buf)
|
||||
b.outPos += len(s.temp.buf)
|
||||
s.ret = chain(b)
|
||||
if s.ret != xzStreamEnd && s.ret != xzOK {
|
||||
return s.ret
|
||||
}
|
||||
bcjApply(s, b.out[:b.outPos], &outStart)
|
||||
/*
|
||||
* As an exception, if the next filter returned xzStreamEnd,
|
||||
* we can do that too, since the last few bytes that remain
|
||||
* unfiltered are meant to remain unfiltered.
|
||||
*/
|
||||
if s.ret == xzStreamEnd {
|
||||
return xzStreamEnd
|
||||
}
|
||||
s.temp.buf = s.temp.bufArray[:b.outPos-outStart]
|
||||
b.outPos -= len(s.temp.buf)
|
||||
copy(s.temp.buf, b.out[b.outPos:])
|
||||
/*
|
||||
* If there wasn't enough input to the next filter to fill
|
||||
* the output buffer with unfiltered data, there's no point
|
||||
* to try decoding more data to temp.
|
||||
*/
|
||||
if b.outPos+len(s.temp.buf) < len(b.out) {
|
||||
return xzOK
|
||||
}
|
||||
}
|
||||
/*
|
||||
* We have unfiltered data in temp. If the output buffer isn't full
|
||||
* yet, try to fill the temp buffer by decoding more data from the
|
||||
* next filter. Apply the BCJ filter on temp. Then we hopefully can
|
||||
* fill the actual output buffer by copying filtered data from temp.
|
||||
* A mix of filtered and unfiltered data may be left in temp; it will
|
||||
* be taken care on the next call to this function.
|
||||
*/
|
||||
if b.outPos < len(b.out) {
|
||||
/* Make b.out temporarily point to s.temp. */
|
||||
s.out = b.out
|
||||
s.outPos = b.outPos
|
||||
b.out = s.temp.bufArray[:]
|
||||
b.outPos = len(s.temp.buf)
|
||||
s.ret = chain(b)
|
||||
s.temp.buf = s.temp.bufArray[:b.outPos]
|
||||
b.out = s.out
|
||||
b.outPos = s.outPos
|
||||
if s.ret != xzOK && s.ret != xzStreamEnd {
|
||||
return s.ret
|
||||
}
|
||||
bcjApply(s, s.temp.buf, &s.temp.filtered)
|
||||
/*
|
||||
* If the next filter returned xzStreamEnd, we mark that
|
||||
* everything is filtered, since the last unfiltered bytes
|
||||
* of the stream are meant to be left as is.
|
||||
*/
|
||||
if s.ret == xzStreamEnd {
|
||||
s.temp.filtered = len(s.temp.buf)
|
||||
}
|
||||
bcjFlush(s, b)
|
||||
if s.temp.filtered > 0 {
|
||||
return xzOK
|
||||
}
|
||||
}
|
||||
return s.ret
|
||||
}
|
||||
|
||||
/*
|
||||
* Allocate memory for BCJ decoders. xzDecBCJReset must be used before
|
||||
* calling xzDecBCJRun.
|
||||
*/
|
||||
func xzDecBCJCreate() *xzDecBCJ {
|
||||
return new(xzDecBCJ)
|
||||
}
|
||||
|
||||
/*
|
||||
* Decode the Filter ID of a BCJ filter and check the start offset is
|
||||
* valid. Returns xzOK if the given Filter ID and offset is
|
||||
* supported. Otherwise xzOptionsError is returned.
|
||||
*/
|
||||
func xzDecBCJReset(s *xzDecBCJ, id xzFilterID, offset int) xzRet {
|
||||
switch id {
|
||||
case idBCJX86:
|
||||
case idBCJPowerPC:
|
||||
case idBCJIA64:
|
||||
case idBCJARM:
|
||||
case idBCJARMThumb:
|
||||
case idBCJSPARC:
|
||||
default:
|
||||
/* Unsupported Filter ID */
|
||||
return xzOptionsError
|
||||
}
|
||||
// check offset is a multiple of alignment
|
||||
switch id {
|
||||
case idBCJPowerPC, idBCJARM, idBCJSPARC:
|
||||
if offset%4 != 0 {
|
||||
return xzOptionsError
|
||||
}
|
||||
case idBCJIA64:
|
||||
if offset%16 != 0 {
|
||||
return xzOptionsError
|
||||
}
|
||||
case idBCJARMThumb:
|
||||
if offset%2 != 0 {
|
||||
return xzOptionsError
|
||||
}
|
||||
}
|
||||
s.typ = id
|
||||
s.ret = xzOK
|
||||
s.pos = offset
|
||||
s.x86PrevMask = 0
|
||||
s.temp.filtered = 0
|
||||
s.temp.buf = nil
|
||||
return xzOK
|
||||
}
|
||||
-55
@@ -1,55 +0,0 @@
|
||||
/*
|
||||
* Delta decoder
|
||||
*
|
||||
* Author: Lasse Collin <lasse.collin@tukaani.org>
|
||||
*
|
||||
* Translation to Go: Michael Cross <https://github.com/xi2>
|
||||
*
|
||||
* This file has been put into the public domain.
|
||||
* You can do whatever you want with this file.
|
||||
*/
|
||||
|
||||
package xz
|
||||
|
||||
type xzDecDelta struct {
|
||||
delta [256]byte
|
||||
pos byte
|
||||
distance int // in range [1, 256]
|
||||
}
|
||||
|
||||
/*
|
||||
* Decode raw stream which has a delta filter as the first filter.
|
||||
*/
|
||||
func xzDecDeltaRun(s *xzDecDelta, b *xzBuf, chain func(*xzBuf) xzRet) xzRet {
|
||||
outStart := b.outPos
|
||||
ret := chain(b)
|
||||
for i := outStart; i < b.outPos; i++ {
|
||||
tmp := b.out[i] + s.delta[byte(s.distance+int(s.pos))]
|
||||
s.delta[s.pos] = tmp
|
||||
b.out[i] = tmp
|
||||
s.pos--
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
/*
|
||||
* Allocate memory for a delta decoder. xzDecDeltaReset must be used
|
||||
* before calling xzDecDeltaRun.
|
||||
*/
|
||||
func xzDecDeltaCreate() *xzDecDelta {
|
||||
return new(xzDecDelta)
|
||||
}
|
||||
|
||||
/*
|
||||
* Returns xzOK if the given distance is valid. Otherwise
|
||||
* xzOptionsError is returned.
|
||||
*/
|
||||
func xzDecDeltaReset(s *xzDecDelta, distance int) xzRet {
|
||||
if distance < 1 || distance > 256 {
|
||||
return xzOptionsError
|
||||
}
|
||||
s.delta = [256]byte{}
|
||||
s.pos = 0
|
||||
s.distance = distance
|
||||
return xzOK
|
||||
}
|
||||
-1235
File diff suppressed because it is too large
Load Diff
-932
@@ -1,932 +0,0 @@
|
||||
/*
|
||||
* .xz Stream decoder
|
||||
*
|
||||
* Author: Lasse Collin <lasse.collin@tukaani.org>
|
||||
*
|
||||
* Translation to Go: Michael Cross <https://github.com/xi2>
|
||||
*
|
||||
* This file has been put into the public domain.
|
||||
* You can do whatever you want with this file.
|
||||
*/
|
||||
|
||||
package xz
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"hash"
|
||||
"hash/crc32"
|
||||
"hash/crc64"
|
||||
)
|
||||
|
||||
/* from linux/lib/xz/xz_stream.h **************************************/
|
||||
|
||||
/*
|
||||
* See the .xz file format specification at
|
||||
* http://tukaani.org/xz/xz-file-format.txt
|
||||
* to understand the container format.
|
||||
*/
|
||||
const (
|
||||
streamHeaderSize = 12
|
||||
headerMagic = "\xfd7zXZ\x00"
|
||||
footerMagic = "YZ"
|
||||
)
|
||||
|
||||
/*
|
||||
* Variable-length integer can hold a 63-bit unsigned integer or a special
|
||||
* value indicating that the value is unknown.
|
||||
*/
|
||||
type vliType uint64
|
||||
|
||||
const (
|
||||
vliUnknown vliType = ^vliType(0)
|
||||
/* Maximum encoded size of a VLI */
|
||||
vliBytesMax = 8 * 8 / 7 // (Sizeof(vliType) * 8 / 7)
|
||||
)
|
||||
|
||||
/* from linux/lib/xz/xz_dec_stream.c **********************************/
|
||||
|
||||
/* Hash used to validate the Index field */
|
||||
type xzDecHash struct {
|
||||
unpadded vliType
|
||||
uncompressed vliType
|
||||
sha256 hash.Hash
|
||||
}
|
||||
|
||||
// type of xzDec.sequence
|
||||
type xzDecSeq int
|
||||
|
||||
const (
|
||||
seqStreamHeader xzDecSeq = iota
|
||||
seqBlockStart
|
||||
seqBlockHeader
|
||||
seqBlockUncompress
|
||||
seqBlockPadding
|
||||
seqBlockCheck
|
||||
seqIndex
|
||||
seqIndexPadding
|
||||
seqIndexCRC32
|
||||
seqStreamFooter
|
||||
)
|
||||
|
||||
// type of xzDec.index.sequence
|
||||
type xzDecIndexSeq int
|
||||
|
||||
const (
|
||||
seqIndexCount xzDecIndexSeq = iota
|
||||
seqIndexUnpadded
|
||||
seqIndexUncompressed
|
||||
)
|
||||
|
||||
/**
|
||||
* xzDec - Opaque type to hold the XZ decoder state
|
||||
*/
|
||||
type xzDec struct {
|
||||
/* Position in decMain */
|
||||
sequence xzDecSeq
|
||||
/* Position in variable-length integers and Check fields */
|
||||
pos int
|
||||
/* Variable-length integer decoded by decVLI */
|
||||
vli vliType
|
||||
/* Saved inPos and outPos */
|
||||
inStart int
|
||||
outStart int
|
||||
/* CRC32 checksum hash used in Index */
|
||||
crc32 hash.Hash
|
||||
/* Hashes used in Blocks */
|
||||
checkCRC32 hash.Hash
|
||||
checkCRC64 hash.Hash
|
||||
checkSHA256 hash.Hash
|
||||
/* for checkTypes CRC32/CRC64/SHA256, check is one of the above 3 hashes */
|
||||
check hash.Hash
|
||||
/* Embedded stream header struct containing CheckType */
|
||||
*Header
|
||||
/*
|
||||
* True if the next call to xzDecRun is allowed to return
|
||||
* xzBufError.
|
||||
*/
|
||||
allowBufError bool
|
||||
/* Information stored in Block Header */
|
||||
blockHeader struct {
|
||||
/*
|
||||
* Value stored in the Compressed Size field, or
|
||||
* vliUnknown if Compressed Size is not present.
|
||||
*/
|
||||
compressed vliType
|
||||
/*
|
||||
* Value stored in the Uncompressed Size field, or
|
||||
* vliUnknown if Uncompressed Size is not present.
|
||||
*/
|
||||
uncompressed vliType
|
||||
/* Size of the Block Header field */
|
||||
size int
|
||||
}
|
||||
/* Information collected when decoding Blocks */
|
||||
block struct {
|
||||
/* Observed compressed size of the current Block */
|
||||
compressed vliType
|
||||
/* Observed uncompressed size of the current Block */
|
||||
uncompressed vliType
|
||||
/* Number of Blocks decoded so far */
|
||||
count vliType
|
||||
/*
|
||||
* Hash calculated from the Block sizes. This is used to
|
||||
* validate the Index field.
|
||||
*/
|
||||
hash xzDecHash
|
||||
}
|
||||
/* Variables needed when verifying the Index field */
|
||||
index struct {
|
||||
/* Position in decIndex */
|
||||
sequence xzDecIndexSeq
|
||||
/* Size of the Index in bytes */
|
||||
size vliType
|
||||
/* Number of Records (matches block.count in valid files) */
|
||||
count vliType
|
||||
/*
|
||||
* Hash calculated from the Records (matches block.hash in
|
||||
* valid files).
|
||||
*/
|
||||
hash xzDecHash
|
||||
}
|
||||
/*
|
||||
* Temporary buffer needed to hold Stream Header, Block Header,
|
||||
* and Stream Footer. The Block Header is the biggest (1 KiB)
|
||||
* so we reserve space according to that. bufArray has to be aligned
|
||||
* to a multiple of four bytes; the variables before it
|
||||
* should guarantee this.
|
||||
*/
|
||||
temp struct {
|
||||
pos int
|
||||
buf []byte // slice buf will be backed by bufArray
|
||||
bufArray [1024]byte
|
||||
}
|
||||
// chain is the function (or to be more precise, closure) which
|
||||
// does the decompression and will call into the lzma2 and other
|
||||
// filter code as needed. It is constructed by decBlockHeader
|
||||
chain func(b *xzBuf) xzRet
|
||||
// lzma2 holds the state of the last filter (which must be LZMA2)
|
||||
lzma2 *xzDecLZMA2
|
||||
// pointers to allocated BCJ/Delta filters
|
||||
bcjs []*xzDecBCJ
|
||||
deltas []*xzDecDelta
|
||||
// number of currently in use BCJ/Delta filters from the above
|
||||
bcjsUsed int
|
||||
deltasUsed int
|
||||
}
|
||||
|
||||
/* Sizes of the Check field with different Check IDs */
|
||||
var checkSizes = [...]byte{
|
||||
0,
|
||||
4, 4, 4,
|
||||
8, 8, 8,
|
||||
16, 16, 16,
|
||||
32, 32, 32,
|
||||
64, 64, 64,
|
||||
}
|
||||
|
||||
/*
|
||||
* Fill s.temp by copying data starting from b.in[b.inPos]. Caller
|
||||
* must have set s.temp.pos to indicate how much data we are supposed
|
||||
* to copy into s.temp.buf. Return true once s.temp.pos has reached
|
||||
* len(s.temp.buf).
|
||||
*/
|
||||
func fillTemp(s *xzDec, b *xzBuf) bool {
|
||||
copySize := len(b.in) - b.inPos
|
||||
tempRemaining := len(s.temp.buf) - s.temp.pos
|
||||
if copySize > tempRemaining {
|
||||
copySize = tempRemaining
|
||||
}
|
||||
copy(s.temp.buf[s.temp.pos:], b.in[b.inPos:])
|
||||
b.inPos += copySize
|
||||
s.temp.pos += copySize
|
||||
if s.temp.pos == len(s.temp.buf) {
|
||||
s.temp.pos = 0
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/* Decode a variable-length integer (little-endian base-128 encoding) */
|
||||
func decVLI(s *xzDec, in []byte, inPos *int) xzRet {
|
||||
var byte byte
|
||||
if s.pos == 0 {
|
||||
s.vli = 0
|
||||
}
|
||||
for *inPos < len(in) {
|
||||
byte = in[*inPos]
|
||||
*inPos++
|
||||
s.vli |= vliType(byte&0x7f) << uint(s.pos)
|
||||
if byte&0x80 == 0 {
|
||||
/* Don't allow non-minimal encodings. */
|
||||
if byte == 0 && s.pos != 0 {
|
||||
return xzDataError
|
||||
}
|
||||
s.pos = 0
|
||||
return xzStreamEnd
|
||||
}
|
||||
s.pos += 7
|
||||
if s.pos == 7*vliBytesMax {
|
||||
return xzDataError
|
||||
}
|
||||
}
|
||||
return xzOK
|
||||
}
|
||||
|
||||
/*
|
||||
* Decode the Compressed Data field from a Block. Update and validate
|
||||
* the observed compressed and uncompressed sizes of the Block so that
|
||||
* they don't exceed the values possibly stored in the Block Header
|
||||
* (validation assumes that no integer overflow occurs, since vliType
|
||||
* is uint64). Update s.check if presence of the CRC32/CRC64/SHA256
|
||||
* field was indicated in Stream Header.
|
||||
*
|
||||
* Once the decoding is finished, validate that the observed sizes match
|
||||
* the sizes possibly stored in the Block Header. Update the hash and
|
||||
* Block count, which are later used to validate the Index field.
|
||||
*/
|
||||
func decBlock(s *xzDec, b *xzBuf) xzRet {
|
||||
var ret xzRet
|
||||
s.inStart = b.inPos
|
||||
s.outStart = b.outPos
|
||||
ret = s.chain(b)
|
||||
s.block.compressed += vliType(b.inPos - s.inStart)
|
||||
s.block.uncompressed += vliType(b.outPos - s.outStart)
|
||||
/*
|
||||
* There is no need to separately check for vliUnknown since
|
||||
* the observed sizes are always smaller than vliUnknown.
|
||||
*/
|
||||
if s.block.compressed > s.blockHeader.compressed ||
|
||||
s.block.uncompressed > s.blockHeader.uncompressed {
|
||||
return xzDataError
|
||||
}
|
||||
switch s.CheckType {
|
||||
case CheckCRC32, CheckCRC64, CheckSHA256:
|
||||
_, _ = s.check.Write(b.out[s.outStart:b.outPos])
|
||||
}
|
||||
if ret == xzStreamEnd {
|
||||
if s.blockHeader.compressed != vliUnknown &&
|
||||
s.blockHeader.compressed != s.block.compressed {
|
||||
return xzDataError
|
||||
}
|
||||
if s.blockHeader.uncompressed != vliUnknown &&
|
||||
s.blockHeader.uncompressed != s.block.uncompressed {
|
||||
return xzDataError
|
||||
}
|
||||
s.block.hash.unpadded +=
|
||||
vliType(s.blockHeader.size) + s.block.compressed
|
||||
s.block.hash.unpadded += vliType(checkSizes[s.CheckType])
|
||||
s.block.hash.uncompressed += s.block.uncompressed
|
||||
var buf [2 * 8]byte // 2*Sizeof(vliType)
|
||||
putLE64(uint64(s.block.hash.unpadded), buf[:])
|
||||
putLE64(uint64(s.block.hash.uncompressed), buf[8:])
|
||||
_, _ = s.block.hash.sha256.Write(buf[:])
|
||||
s.block.count++
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
/* Update the Index size and the CRC32 hash. */
|
||||
func indexUpdate(s *xzDec, b *xzBuf) {
|
||||
inUsed := b.inPos - s.inStart
|
||||
s.index.size += vliType(inUsed)
|
||||
_, _ = s.crc32.Write(b.in[s.inStart : s.inStart+inUsed])
|
||||
}
|
||||
|
||||
/*
|
||||
* Decode the Number of Records, Unpadded Size, and Uncompressed Size
|
||||
* fields from the Index field. That is, Index Padding and CRC32 are not
|
||||
* decoded by this function.
|
||||
*
|
||||
* This can return xzOK (more input needed), xzStreamEnd (everything
|
||||
* successfully decoded), or xzDataError (input is corrupt).
|
||||
*/
|
||||
func decIndex(s *xzDec, b *xzBuf) xzRet {
|
||||
var ret xzRet
|
||||
for {
|
||||
ret = decVLI(s, b.in, &b.inPos)
|
||||
if ret != xzStreamEnd {
|
||||
indexUpdate(s, b)
|
||||
return ret
|
||||
}
|
||||
switch s.index.sequence {
|
||||
case seqIndexCount:
|
||||
s.index.count = s.vli
|
||||
/*
|
||||
* Validate that the Number of Records field
|
||||
* indicates the same number of Records as
|
||||
* there were Blocks in the Stream.
|
||||
*/
|
||||
if s.index.count != s.block.count {
|
||||
return xzDataError
|
||||
}
|
||||
s.index.sequence = seqIndexUnpadded
|
||||
case seqIndexUnpadded:
|
||||
s.index.hash.unpadded += s.vli
|
||||
s.index.sequence = seqIndexUncompressed
|
||||
case seqIndexUncompressed:
|
||||
s.index.hash.uncompressed += s.vli
|
||||
var buf [2 * 8]byte // 2*Sizeof(vliType)
|
||||
putLE64(uint64(s.index.hash.unpadded), buf[:])
|
||||
putLE64(uint64(s.index.hash.uncompressed), buf[8:])
|
||||
_, _ = s.index.hash.sha256.Write(buf[:])
|
||||
s.index.count--
|
||||
s.index.sequence = seqIndexUnpadded
|
||||
}
|
||||
if !(s.index.count > 0) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return xzStreamEnd
|
||||
}
|
||||
|
||||
/*
|
||||
* Validate that the next 4 bytes match s.crc32.Sum(nil). s.pos must
|
||||
* be zero when starting to validate the first byte.
|
||||
*/
|
||||
func crcValidate(s *xzDec, b *xzBuf) xzRet {
|
||||
sum := s.crc32.Sum(nil)
|
||||
// CRC32 - reverse slice
|
||||
sum[0], sum[1], sum[2], sum[3] = sum[3], sum[2], sum[1], sum[0]
|
||||
for {
|
||||
if b.inPos == len(b.in) {
|
||||
return xzOK
|
||||
}
|
||||
if sum[s.pos] != b.in[b.inPos] {
|
||||
return xzDataError
|
||||
}
|
||||
b.inPos++
|
||||
s.pos++
|
||||
if !(s.pos < 4) {
|
||||
break
|
||||
}
|
||||
}
|
||||
s.crc32.Reset()
|
||||
s.pos = 0
|
||||
return xzStreamEnd
|
||||
}
|
||||
|
||||
/*
|
||||
* Validate that the next 4/8/32 bytes match s.check.Sum(nil). s.pos
|
||||
* must be zero when starting to validate the first byte.
|
||||
*/
|
||||
func checkValidate(s *xzDec, b *xzBuf) xzRet {
|
||||
sum := s.check.Sum(nil)
|
||||
if s.CheckType == CheckCRC32 || s.CheckType == CheckCRC64 {
|
||||
// CRC32/64 - reverse slice
|
||||
for i, j := 0, len(sum)-1; i < j; i, j = i+1, j-1 {
|
||||
sum[i], sum[j] = sum[j], sum[i]
|
||||
}
|
||||
}
|
||||
for {
|
||||
if b.inPos == len(b.in) {
|
||||
return xzOK
|
||||
}
|
||||
if sum[s.pos] != b.in[b.inPos] {
|
||||
return xzDataError
|
||||
}
|
||||
b.inPos++
|
||||
s.pos++
|
||||
if !(s.pos < len(sum)) {
|
||||
break
|
||||
}
|
||||
}
|
||||
s.check.Reset()
|
||||
s.pos = 0
|
||||
return xzStreamEnd
|
||||
}
|
||||
|
||||
/*
|
||||
* Skip over the Check field when the Check ID is not supported.
|
||||
* Returns true once the whole Check field has been skipped over.
|
||||
*/
|
||||
func checkSkip(s *xzDec, b *xzBuf) bool {
|
||||
for s.pos < int(checkSizes[s.CheckType]) {
|
||||
if b.inPos == len(b.in) {
|
||||
return false
|
||||
}
|
||||
b.inPos++
|
||||
s.pos++
|
||||
}
|
||||
s.pos = 0
|
||||
return true
|
||||
}
|
||||
|
||||
/* polynomial table used in decStreamHeader below */
|
||||
var xzCRC64Table = crc64.MakeTable(crc64.ECMA)
|
||||
|
||||
/* Decode the Stream Header field (the first 12 bytes of the .xz Stream). */
|
||||
func decStreamHeader(s *xzDec) xzRet {
|
||||
if string(s.temp.buf[:len(headerMagic)]) != headerMagic {
|
||||
return xzFormatError
|
||||
}
|
||||
if crc32.ChecksumIEEE(s.temp.buf[len(headerMagic):len(headerMagic)+2]) !=
|
||||
getLE32(s.temp.buf[len(headerMagic)+2:]) {
|
||||
return xzDataError
|
||||
}
|
||||
if s.temp.buf[len(headerMagic)] != 0 {
|
||||
return xzOptionsError
|
||||
}
|
||||
/*
|
||||
* Of integrity checks, we support none (Check ID = 0),
|
||||
* CRC32 (Check ID = 1), CRC64 (Check ID = 4) and SHA256 (Check ID = 10)
|
||||
* However, we will accept other check types too, but then the check
|
||||
* won't be verified and a warning (xzUnsupportedCheck) will be given.
|
||||
*/
|
||||
s.CheckType = CheckID(s.temp.buf[len(headerMagic)+1])
|
||||
if s.CheckType > checkMax {
|
||||
return xzOptionsError
|
||||
}
|
||||
switch s.CheckType {
|
||||
case CheckNone:
|
||||
// CheckNone: no action needed
|
||||
case CheckCRC32:
|
||||
if s.checkCRC32 == nil {
|
||||
s.checkCRC32 = crc32.NewIEEE()
|
||||
} else {
|
||||
s.checkCRC32.Reset()
|
||||
}
|
||||
s.check = s.checkCRC32
|
||||
case CheckCRC64:
|
||||
if s.checkCRC64 == nil {
|
||||
s.checkCRC64 = crc64.New(xzCRC64Table)
|
||||
} else {
|
||||
s.checkCRC64.Reset()
|
||||
}
|
||||
s.check = s.checkCRC64
|
||||
case CheckSHA256:
|
||||
if s.checkSHA256 == nil {
|
||||
s.checkSHA256 = sha256.New()
|
||||
} else {
|
||||
s.checkSHA256.Reset()
|
||||
}
|
||||
s.check = s.checkSHA256
|
||||
default:
|
||||
return xzUnsupportedCheck
|
||||
}
|
||||
return xzOK
|
||||
}
|
||||
|
||||
/* Decode the Stream Footer field (the last 12 bytes of the .xz Stream) */
|
||||
func decStreamFooter(s *xzDec) xzRet {
|
||||
if string(s.temp.buf[10:10+len(footerMagic)]) != footerMagic {
|
||||
return xzDataError
|
||||
}
|
||||
if crc32.ChecksumIEEE(s.temp.buf[4:10]) != getLE32(s.temp.buf) {
|
||||
return xzDataError
|
||||
}
|
||||
/*
|
||||
* Validate Backward Size. Note that we never added the size of the
|
||||
* Index CRC32 field to s->index.size, thus we use s->index.size / 4
|
||||
* instead of s->index.size / 4 - 1.
|
||||
*/
|
||||
if s.index.size>>2 != vliType(getLE32(s.temp.buf[4:])) {
|
||||
return xzDataError
|
||||
}
|
||||
if s.temp.buf[8] != 0 || CheckID(s.temp.buf[9]) != s.CheckType {
|
||||
return xzDataError
|
||||
}
|
||||
/*
|
||||
* Use xzStreamEnd instead of xzOK to be more convenient
|
||||
* for the caller.
|
||||
*/
|
||||
return xzStreamEnd
|
||||
}
|
||||
|
||||
/* Decode the Block Header and initialize the filter chain. */
|
||||
func decBlockHeader(s *xzDec) xzRet {
|
||||
var ret xzRet
|
||||
/*
|
||||
* Validate the CRC32. We know that the temp buffer is at least
|
||||
* eight bytes so this is safe.
|
||||
*/
|
||||
crc := getLE32(s.temp.buf[len(s.temp.buf)-4:])
|
||||
s.temp.buf = s.temp.buf[:len(s.temp.buf)-4]
|
||||
if crc32.ChecksumIEEE(s.temp.buf) != crc {
|
||||
return xzDataError
|
||||
}
|
||||
s.temp.pos = 2
|
||||
/*
|
||||
* Catch unsupported Block Flags.
|
||||
*/
|
||||
if s.temp.buf[1]&0x3C != 0 {
|
||||
return xzOptionsError
|
||||
}
|
||||
/* Compressed Size */
|
||||
if s.temp.buf[1]&0x40 != 0 {
|
||||
if decVLI(s, s.temp.buf, &s.temp.pos) != xzStreamEnd {
|
||||
return xzDataError
|
||||
}
|
||||
if s.vli >= 1<<63-8 {
|
||||
// the whole block must stay smaller than 2^63 bytes
|
||||
// the block header cannot be smaller than 8 bytes
|
||||
return xzDataError
|
||||
}
|
||||
if s.vli == 0 {
|
||||
// compressed size must be non-zero
|
||||
return xzDataError
|
||||
}
|
||||
s.blockHeader.compressed = s.vli
|
||||
} else {
|
||||
s.blockHeader.compressed = vliUnknown
|
||||
}
|
||||
/* Uncompressed Size */
|
||||
if s.temp.buf[1]&0x80 != 0 {
|
||||
if decVLI(s, s.temp.buf, &s.temp.pos) != xzStreamEnd {
|
||||
return xzDataError
|
||||
}
|
||||
s.blockHeader.uncompressed = s.vli
|
||||
} else {
|
||||
s.blockHeader.uncompressed = vliUnknown
|
||||
}
|
||||
// get total number of filters (1-4)
|
||||
filterTotal := int(s.temp.buf[1]&0x03) + 1
|
||||
// slice to hold decoded filters
|
||||
filterList := make([]struct {
|
||||
id xzFilterID
|
||||
props uint32
|
||||
}, filterTotal)
|
||||
// decode the non-last filters which cannot be LZMA2
|
||||
for i := 0; i < filterTotal-1; i++ {
|
||||
/* Valid Filter Flags always take at least two bytes. */
|
||||
if len(s.temp.buf)-s.temp.pos < 2 {
|
||||
return xzDataError
|
||||
}
|
||||
s.temp.pos += 2
|
||||
switch id := xzFilterID(s.temp.buf[s.temp.pos-2]); id {
|
||||
case idDelta:
|
||||
// delta filter
|
||||
if s.temp.buf[s.temp.pos-1] != 0x01 {
|
||||
return xzOptionsError
|
||||
}
|
||||
/* Filter Properties contains distance - 1 */
|
||||
if len(s.temp.buf)-s.temp.pos < 1 {
|
||||
return xzDataError
|
||||
}
|
||||
props := uint32(s.temp.buf[s.temp.pos])
|
||||
s.temp.pos++
|
||||
filterList[i] = struct {
|
||||
id xzFilterID
|
||||
props uint32
|
||||
}{id: id, props: props}
|
||||
case idBCJX86, idBCJPowerPC, idBCJIA64,
|
||||
idBCJARM, idBCJARMThumb, idBCJSPARC:
|
||||
// bcj filter
|
||||
var props uint32
|
||||
switch s.temp.buf[s.temp.pos-1] {
|
||||
case 0x00:
|
||||
props = 0
|
||||
case 0x04:
|
||||
if len(s.temp.buf)-s.temp.pos < 4 {
|
||||
return xzDataError
|
||||
}
|
||||
props = getLE32(s.temp.buf[s.temp.pos:])
|
||||
s.temp.pos += 4
|
||||
default:
|
||||
return xzOptionsError
|
||||
}
|
||||
filterList[i] = struct {
|
||||
id xzFilterID
|
||||
props uint32
|
||||
}{id: id, props: props}
|
||||
default:
|
||||
return xzOptionsError
|
||||
}
|
||||
}
|
||||
/*
|
||||
* decode the last filter which must be LZMA2
|
||||
*/
|
||||
if len(s.temp.buf)-s.temp.pos < 2 {
|
||||
return xzDataError
|
||||
}
|
||||
/* Filter ID = LZMA2 */
|
||||
if xzFilterID(s.temp.buf[s.temp.pos]) != idLZMA2 {
|
||||
return xzOptionsError
|
||||
}
|
||||
s.temp.pos++
|
||||
/* Size of Properties = 1-byte Filter Properties */
|
||||
if s.temp.buf[s.temp.pos] != 0x01 {
|
||||
return xzOptionsError
|
||||
}
|
||||
s.temp.pos++
|
||||
/* Filter Properties contains LZMA2 dictionary size. */
|
||||
if len(s.temp.buf)-s.temp.pos < 1 {
|
||||
return xzDataError
|
||||
}
|
||||
props := uint32(s.temp.buf[s.temp.pos])
|
||||
s.temp.pos++
|
||||
filterList[filterTotal-1] = struct {
|
||||
id xzFilterID
|
||||
props uint32
|
||||
}{id: idLZMA2, props: props}
|
||||
/*
|
||||
* Process the filter list and create s.chain, going from last
|
||||
* filter (LZMA2) to first filter
|
||||
*
|
||||
* First, LZMA2.
|
||||
*/
|
||||
ret = xzDecLZMA2Reset(s.lzma2, byte(filterList[filterTotal-1].props))
|
||||
if ret != xzOK {
|
||||
return ret
|
||||
}
|
||||
s.chain = func(b *xzBuf) xzRet {
|
||||
return xzDecLZMA2Run(s.lzma2, b)
|
||||
}
|
||||
/*
|
||||
* Now the non-last filters
|
||||
*/
|
||||
for i := filterTotal - 2; i >= 0; i-- {
|
||||
switch id := filterList[i].id; id {
|
||||
case idDelta:
|
||||
// delta filter
|
||||
var delta *xzDecDelta
|
||||
if s.deltasUsed < len(s.deltas) {
|
||||
delta = s.deltas[s.deltasUsed]
|
||||
} else {
|
||||
delta = xzDecDeltaCreate()
|
||||
s.deltas = append(s.deltas, delta)
|
||||
}
|
||||
s.deltasUsed++
|
||||
ret = xzDecDeltaReset(delta, int(filterList[i].props)+1)
|
||||
if ret != xzOK {
|
||||
return ret
|
||||
}
|
||||
chain := s.chain
|
||||
s.chain = func(b *xzBuf) xzRet {
|
||||
return xzDecDeltaRun(delta, b, chain)
|
||||
}
|
||||
case idBCJX86, idBCJPowerPC, idBCJIA64,
|
||||
idBCJARM, idBCJARMThumb, idBCJSPARC:
|
||||
// bcj filter
|
||||
var bcj *xzDecBCJ
|
||||
if s.bcjsUsed < len(s.bcjs) {
|
||||
bcj = s.bcjs[s.bcjsUsed]
|
||||
} else {
|
||||
bcj = xzDecBCJCreate()
|
||||
s.bcjs = append(s.bcjs, bcj)
|
||||
}
|
||||
s.bcjsUsed++
|
||||
ret = xzDecBCJReset(bcj, id, int(filterList[i].props))
|
||||
if ret != xzOK {
|
||||
return ret
|
||||
}
|
||||
chain := s.chain
|
||||
s.chain = func(b *xzBuf) xzRet {
|
||||
return xzDecBCJRun(bcj, b, chain)
|
||||
}
|
||||
}
|
||||
}
|
||||
/* The rest must be Header Padding. */
|
||||
for s.temp.pos < len(s.temp.buf) {
|
||||
if s.temp.buf[s.temp.pos] != 0x00 {
|
||||
return xzOptionsError
|
||||
}
|
||||
s.temp.pos++
|
||||
}
|
||||
s.temp.pos = 0
|
||||
s.block.compressed = 0
|
||||
s.block.uncompressed = 0
|
||||
return xzOK
|
||||
}
|
||||
|
||||
func decMain(s *xzDec, b *xzBuf) xzRet {
|
||||
var ret xzRet
|
||||
/*
|
||||
* Store the start position for the case when we are in the middle
|
||||
* of the Index field.
|
||||
*/
|
||||
s.inStart = b.inPos
|
||||
for {
|
||||
switch s.sequence {
|
||||
case seqStreamHeader:
|
||||
/*
|
||||
* Stream Header is copied to s.temp, and then
|
||||
* decoded from there. This way if the caller
|
||||
* gives us only little input at a time, we can
|
||||
* still keep the Stream Header decoding code
|
||||
* simple. Similar approach is used in many places
|
||||
* in this file.
|
||||
*/
|
||||
if !fillTemp(s, b) {
|
||||
return xzOK
|
||||
}
|
||||
/*
|
||||
* If decStreamHeader returns
|
||||
* xzUnsupportedCheck, it is still possible
|
||||
* to continue decoding. Thus, update s.sequence
|
||||
* before calling decStreamHeader.
|
||||
*/
|
||||
s.sequence = seqBlockStart
|
||||
ret = decStreamHeader(s)
|
||||
if ret != xzOK {
|
||||
return ret
|
||||
}
|
||||
fallthrough
|
||||
case seqBlockStart:
|
||||
/* We need one byte of input to continue. */
|
||||
if b.inPos == len(b.in) {
|
||||
return xzOK
|
||||
}
|
||||
/* See if this is the beginning of the Index field. */
|
||||
if b.in[b.inPos] == 0 {
|
||||
s.inStart = b.inPos
|
||||
b.inPos++
|
||||
s.sequence = seqIndex
|
||||
break
|
||||
}
|
||||
/*
|
||||
* Calculate the size of the Block Header and
|
||||
* prepare to decode it.
|
||||
*/
|
||||
s.blockHeader.size = (int(b.in[b.inPos]) + 1) * 4
|
||||
s.temp.buf = s.temp.bufArray[:s.blockHeader.size]
|
||||
s.temp.pos = 0
|
||||
s.sequence = seqBlockHeader
|
||||
fallthrough
|
||||
case seqBlockHeader:
|
||||
if !fillTemp(s, b) {
|
||||
return xzOK
|
||||
}
|
||||
ret = decBlockHeader(s)
|
||||
if ret != xzOK {
|
||||
return ret
|
||||
}
|
||||
s.sequence = seqBlockUncompress
|
||||
fallthrough
|
||||
case seqBlockUncompress:
|
||||
ret = decBlock(s, b)
|
||||
if ret != xzStreamEnd {
|
||||
return ret
|
||||
}
|
||||
s.sequence = seqBlockPadding
|
||||
fallthrough
|
||||
case seqBlockPadding:
|
||||
/*
|
||||
* Size of Compressed Data + Block Padding
|
||||
* must be a multiple of four. We don't need
|
||||
* s->block.compressed for anything else
|
||||
* anymore, so we use it here to test the size
|
||||
* of the Block Padding field.
|
||||
*/
|
||||
for s.block.compressed&3 != 0 {
|
||||
if b.inPos == len(b.in) {
|
||||
return xzOK
|
||||
}
|
||||
if b.in[b.inPos] != 0 {
|
||||
return xzDataError
|
||||
}
|
||||
b.inPos++
|
||||
s.block.compressed++
|
||||
}
|
||||
s.sequence = seqBlockCheck
|
||||
fallthrough
|
||||
case seqBlockCheck:
|
||||
switch s.CheckType {
|
||||
case CheckCRC32, CheckCRC64, CheckSHA256:
|
||||
ret = checkValidate(s, b)
|
||||
if ret != xzStreamEnd {
|
||||
return ret
|
||||
}
|
||||
default:
|
||||
if !checkSkip(s, b) {
|
||||
return xzOK
|
||||
}
|
||||
}
|
||||
s.sequence = seqBlockStart
|
||||
case seqIndex:
|
||||
ret = decIndex(s, b)
|
||||
if ret != xzStreamEnd {
|
||||
return ret
|
||||
}
|
||||
s.sequence = seqIndexPadding
|
||||
fallthrough
|
||||
case seqIndexPadding:
|
||||
for (s.index.size+vliType(b.inPos-s.inStart))&3 != 0 {
|
||||
if b.inPos == len(b.in) {
|
||||
indexUpdate(s, b)
|
||||
return xzOK
|
||||
}
|
||||
if b.in[b.inPos] != 0 {
|
||||
return xzDataError
|
||||
}
|
||||
b.inPos++
|
||||
}
|
||||
/* Finish the CRC32 value and Index size. */
|
||||
indexUpdate(s, b)
|
||||
/* Compare the hashes to validate the Index field. */
|
||||
if !bytes.Equal(
|
||||
s.block.hash.sha256.Sum(nil), s.index.hash.sha256.Sum(nil)) {
|
||||
return xzDataError
|
||||
}
|
||||
s.sequence = seqIndexCRC32
|
||||
fallthrough
|
||||
case seqIndexCRC32:
|
||||
ret = crcValidate(s, b)
|
||||
if ret != xzStreamEnd {
|
||||
return ret
|
||||
}
|
||||
s.temp.buf = s.temp.bufArray[:streamHeaderSize]
|
||||
s.sequence = seqStreamFooter
|
||||
fallthrough
|
||||
case seqStreamFooter:
|
||||
if !fillTemp(s, b) {
|
||||
return xzOK
|
||||
}
|
||||
return decStreamFooter(s)
|
||||
}
|
||||
}
|
||||
/* Never reached */
|
||||
}
|
||||
|
||||
/**
|
||||
* xzDecRun - Run the XZ decoder
|
||||
* @s: Decoder state allocated using xzDecInit
|
||||
* @b: Input and output buffers
|
||||
*
|
||||
* See xzRet for details of return values.
|
||||
*
|
||||
* xzDecRun is a wrapper for decMain to handle some special cases.
|
||||
*
|
||||
* We must return xzBufError when it seems clear that we are not
|
||||
* going to make any progress anymore. This is to prevent the caller
|
||||
* from calling us infinitely when the input file is truncated or
|
||||
* otherwise corrupt. Since zlib-style API allows that the caller
|
||||
* fills the input buffer only when the decoder doesn't produce any
|
||||
* new output, we have to be careful to avoid returning xzBufError
|
||||
* too easily: xzBufError is returned only after the second
|
||||
* consecutive call to xzDecRun that makes no progress.
|
||||
*/
|
||||
func xzDecRun(s *xzDec, b *xzBuf) xzRet {
|
||||
inStart := b.inPos
|
||||
outStart := b.outPos
|
||||
ret := decMain(s, b)
|
||||
if ret == xzOK && inStart == b.inPos && outStart == b.outPos {
|
||||
if s.allowBufError {
|
||||
ret = xzBufError
|
||||
}
|
||||
s.allowBufError = true
|
||||
} else {
|
||||
s.allowBufError = false
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
/**
|
||||
* xzDecInit - Allocate and initialize a XZ decoder state
|
||||
* @dictMax: Maximum size of the LZMA2 dictionary (history buffer) for
|
||||
* decoding. LZMA2 dictionary is always 2^n bytes
|
||||
* or 2^n + 2^(n-1) bytes (the latter sizes are less common
|
||||
* in practice), so other values for dictMax don't make sense.
|
||||
*
|
||||
* dictMax specifies the maximum allowed dictionary size that xzDecRun
|
||||
* may allocate once it has parsed the dictionary size from the stream
|
||||
* headers. This way excessive allocations can be avoided while still
|
||||
* limiting the maximum memory usage to a sane value to prevent running the
|
||||
* system out of memory when decompressing streams from untrusted sources.
|
||||
*
|
||||
* xzDecInit returns a pointer to an xzDec, which is ready to be used with
|
||||
* xzDecRun.
|
||||
*/
|
||||
func xzDecInit(dictMax uint32, header *Header) *xzDec {
|
||||
s := new(xzDec)
|
||||
s.crc32 = crc32.NewIEEE()
|
||||
s.Header = header
|
||||
s.block.hash.sha256 = sha256.New()
|
||||
s.index.hash.sha256 = sha256.New()
|
||||
s.lzma2 = xzDecLZMA2Create(dictMax)
|
||||
xzDecReset(s)
|
||||
return s
|
||||
}
|
||||
|
||||
/**
|
||||
* xzDecReset - Reset an already allocated decoder state
|
||||
* @s: Decoder state allocated using xzDecInit
|
||||
*
|
||||
* This function can be used to reset the decoder state without
|
||||
* reallocating memory with xzDecInit.
|
||||
*/
|
||||
func xzDecReset(s *xzDec) {
|
||||
s.sequence = seqStreamHeader
|
||||
s.allowBufError = false
|
||||
s.pos = 0
|
||||
s.crc32.Reset()
|
||||
s.check = nil
|
||||
s.CheckType = checkUnset
|
||||
s.block.compressed = 0
|
||||
s.block.uncompressed = 0
|
||||
s.block.count = 0
|
||||
s.block.hash.unpadded = 0
|
||||
s.block.hash.uncompressed = 0
|
||||
s.block.hash.sha256.Reset()
|
||||
s.index.sequence = seqIndexCount
|
||||
s.index.size = 0
|
||||
s.index.count = 0
|
||||
s.index.hash.unpadded = 0
|
||||
s.index.hash.uncompressed = 0
|
||||
s.index.hash.sha256.Reset()
|
||||
s.temp.pos = 0
|
||||
s.temp.buf = s.temp.bufArray[:streamHeaderSize]
|
||||
s.chain = nil
|
||||
s.bcjsUsed = 0
|
||||
s.deltasUsed = 0
|
||||
}
|
||||
-52
@@ -1,52 +0,0 @@
|
||||
/*
|
||||
* XZ decompressor utility functions
|
||||
*
|
||||
* Author: Michael Cross <https://github.com/xi2>
|
||||
*
|
||||
* This file has been put into the public domain.
|
||||
* You can do whatever you want with this file.
|
||||
*/
|
||||
|
||||
package xz
|
||||
|
||||
func getLE32(buf []byte) uint32 {
|
||||
return uint32(buf[0]) |
|
||||
uint32(buf[1])<<8 |
|
||||
uint32(buf[2])<<16 |
|
||||
uint32(buf[3])<<24
|
||||
}
|
||||
|
||||
func getBE32(buf []byte) uint32 {
|
||||
return uint32(buf[0])<<24 |
|
||||
uint32(buf[1])<<16 |
|
||||
uint32(buf[2])<<8 |
|
||||
uint32(buf[3])
|
||||
}
|
||||
|
||||
func putLE32(val uint32, buf []byte) {
|
||||
buf[0] = byte(val)
|
||||
buf[1] = byte(val >> 8)
|
||||
buf[2] = byte(val >> 16)
|
||||
buf[3] = byte(val >> 24)
|
||||
return
|
||||
}
|
||||
|
||||
func putBE32(val uint32, buf []byte) {
|
||||
buf[0] = byte(val >> 24)
|
||||
buf[1] = byte(val >> 16)
|
||||
buf[2] = byte(val >> 8)
|
||||
buf[3] = byte(val)
|
||||
return
|
||||
}
|
||||
|
||||
func putLE64(val uint64, buf []byte) {
|
||||
buf[0] = byte(val)
|
||||
buf[1] = byte(val >> 8)
|
||||
buf[2] = byte(val >> 16)
|
||||
buf[3] = byte(val >> 24)
|
||||
buf[4] = byte(val >> 32)
|
||||
buf[5] = byte(val >> 40)
|
||||
buf[6] = byte(val >> 48)
|
||||
buf[7] = byte(val >> 56)
|
||||
return
|
||||
}
|
||||
-124
@@ -1,124 +0,0 @@
|
||||
/*
|
||||
* XZ decompressor
|
||||
*
|
||||
* Authors: Lasse Collin <lasse.collin@tukaani.org>
|
||||
* Igor Pavlov <http://7-zip.org/>
|
||||
*
|
||||
* Translation to Go: Michael Cross <https://github.com/xi2>
|
||||
*
|
||||
* This file has been put into the public domain.
|
||||
* You can do whatever you want with this file.
|
||||
*/
|
||||
|
||||
package xz
|
||||
|
||||
/* from linux/include/linux/xz.h **************************************/
|
||||
|
||||
/**
|
||||
* xzRet - Return codes
|
||||
* @xzOK: Everything is OK so far. More input or more
|
||||
* output space is required to continue.
|
||||
* @xzStreamEnd: Operation finished successfully.
|
||||
* @xzUnSupportedCheck: Integrity check type is not supported. Decoding
|
||||
* is still possible by simply calling xzDecRun
|
||||
* again.
|
||||
* @xzMemlimitError: A bigger LZMA2 dictionary would be needed than
|
||||
* allowed by the dictMax argument given to
|
||||
* xzDecInit.
|
||||
* @xzFormatError: File format was not recognized (wrong magic
|
||||
* bytes).
|
||||
* @xzOptionsError: This implementation doesn't support the requested
|
||||
* compression options. In the decoder this means
|
||||
* that the header CRC32 matches, but the header
|
||||
* itself specifies something that we don't support.
|
||||
* @xzDataError: Compressed data is corrupt.
|
||||
* @xzBufError: Cannot make any progress.
|
||||
*
|
||||
* xzBufError is returned when two consecutive calls to XZ code cannot
|
||||
* consume any input and cannot produce any new output. This happens
|
||||
* when there is no new input available, or the output buffer is full
|
||||
* while at least one output byte is still pending. Assuming your code
|
||||
* is not buggy, you can get this error only when decoding a
|
||||
* compressed stream that is truncated or otherwise corrupt.
|
||||
*/
|
||||
type xzRet int
|
||||
|
||||
const (
|
||||
xzOK xzRet = iota
|
||||
xzStreamEnd
|
||||
xzUnsupportedCheck
|
||||
xzMemlimitError
|
||||
xzFormatError
|
||||
xzOptionsError
|
||||
xzDataError
|
||||
xzBufError
|
||||
)
|
||||
|
||||
/**
|
||||
* xzBuf - Passing input and output buffers to XZ code
|
||||
* @in: Input buffer.
|
||||
* @inPos: Current position in the input buffer. This must not exceed
|
||||
* input buffer size.
|
||||
* @out: Output buffer.
|
||||
* @outPos: Current position in the output buffer. This must not exceed
|
||||
* output buffer size.
|
||||
*
|
||||
* Only the contents of the output buffer from out[outPos] onward, and
|
||||
* the variables inPos and outPos are modified by the XZ code.
|
||||
*/
|
||||
type xzBuf struct {
|
||||
in []byte
|
||||
inPos int
|
||||
out []byte
|
||||
outPos int
|
||||
}
|
||||
|
||||
/* All XZ filter IDs */
|
||||
type xzFilterID int64
|
||||
|
||||
const (
|
||||
idDelta xzFilterID = 0x03
|
||||
idBCJX86 xzFilterID = 0x04
|
||||
idBCJPowerPC xzFilterID = 0x05
|
||||
idBCJIA64 xzFilterID = 0x06
|
||||
idBCJARM xzFilterID = 0x07
|
||||
idBCJARMThumb xzFilterID = 0x08
|
||||
idBCJSPARC xzFilterID = 0x09
|
||||
idLZMA2 xzFilterID = 0x21
|
||||
)
|
||||
|
||||
// CheckID is the type of the data integrity check in an XZ stream
|
||||
// calculated from the uncompressed data.
|
||||
type CheckID int
|
||||
|
||||
func (id CheckID) String() string {
|
||||
switch id {
|
||||
case CheckNone:
|
||||
return "None"
|
||||
case CheckCRC32:
|
||||
return "CRC32"
|
||||
case CheckCRC64:
|
||||
return "CRC64"
|
||||
case CheckSHA256:
|
||||
return "SHA256"
|
||||
default:
|
||||
return "Unknown"
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
CheckNone CheckID = 0x00
|
||||
CheckCRC32 CheckID = 0x01
|
||||
CheckCRC64 CheckID = 0x04
|
||||
CheckSHA256 CheckID = 0x0A
|
||||
checkMax CheckID = 0x0F
|
||||
checkUnset CheckID = -1
|
||||
)
|
||||
|
||||
// An XZ stream contains a stream header which holds information about
|
||||
// the stream. That information is exposed as fields of the
|
||||
// Reader. Currently it contains only the stream's data integrity
|
||||
// check type.
|
||||
type Header struct {
|
||||
CheckType CheckID // type of the stream's data integrity check
|
||||
}
|
||||
-35
@@ -1,35 +0,0 @@
|
||||
// Package xz implements XZ decompression natively in Go.
|
||||
//
|
||||
// Usage
|
||||
//
|
||||
// For ease of use, this package is designed to have a similar API to
|
||||
// compress/gzip. See the examples for further details.
|
||||
//
|
||||
// Implementation
|
||||
//
|
||||
// This package is a translation from C to Go of XZ Embedded
|
||||
// (http://tukaani.org/xz/embedded.html) with enhancements made so as
|
||||
// to implement all mandatory and optional parts of the XZ file format
|
||||
// specification v1.0.4. It supports all filters and block check
|
||||
// types, supports multiple streams, and performs index verification
|
||||
// using SHA-256 as recommended by the specification.
|
||||
//
|
||||
// Speed
|
||||
//
|
||||
// On the author's Intel Ivybridge i5, decompression speed is about
|
||||
// half that of the standard XZ Utils (tested with a recent linux
|
||||
// kernel tarball).
|
||||
//
|
||||
// Thanks
|
||||
//
|
||||
// Thanks are due to Lasse Collin and Igor Pavlov, the authors of XZ
|
||||
// Embedded, on whose code package xz is based. It would not exist
|
||||
// without their decision to allow others to modify and reuse their
|
||||
// code.
|
||||
//
|
||||
// Bug reports
|
||||
//
|
||||
// For bug reports relating to this package please contact the author
|
||||
// through https://github.com/xi2/xz/issues, and not the authors of XZ
|
||||
// Embedded.
|
||||
package xz
|
||||
-256
@@ -1,256 +0,0 @@
|
||||
/*
|
||||
* Package xz Go Reader API
|
||||
*
|
||||
* Author: Michael Cross <https://github.com/xi2>
|
||||
*
|
||||
* This file has been put into the public domain.
|
||||
* You can do whatever you want with this file.
|
||||
*/
|
||||
|
||||
package xz
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
|
||||
// Package specific errors.
|
||||
var (
|
||||
ErrUnsupportedCheck = errors.New("xz: integrity check type not supported")
|
||||
ErrMemlimit = errors.New("xz: LZMA2 dictionary size exceeds max")
|
||||
ErrFormat = errors.New("xz: file format not recognized")
|
||||
ErrOptions = errors.New("xz: compression options not supported")
|
||||
ErrData = errors.New("xz: data is corrupt")
|
||||
ErrBuf = errors.New("xz: data is truncated or corrupt")
|
||||
)
|
||||
|
||||
// DefaultDictMax is the default maximum dictionary size in bytes used
|
||||
// by the decoder. This value is sufficient to decompress files
|
||||
// created with XZ Utils "xz -9".
|
||||
const DefaultDictMax = 1 << 26 // 64 MiB
|
||||
|
||||
// inBufSize is the input buffer size used by the decoder.
|
||||
const inBufSize = 1 << 13 // 8 KiB
|
||||
|
||||
// A Reader is an io.Reader that can be used to retrieve uncompressed
|
||||
// data from an XZ file.
|
||||
//
|
||||
// In general, an XZ file can be a concatenation of other XZ
|
||||
// files. Reads from the Reader return the concatenation of the
|
||||
// uncompressed data of each.
|
||||
type Reader struct {
|
||||
Header
|
||||
r io.Reader // the wrapped io.Reader
|
||||
multistream bool // true if reader is in multistream mode
|
||||
rEOF bool // true after io.EOF received on r
|
||||
dEOF bool // true after decoder has completed
|
||||
padding int // bytes of stream padding read (or -1)
|
||||
in [inBufSize]byte // backing array for buf.in
|
||||
buf *xzBuf // decoder input/output buffers
|
||||
dec *xzDec // decoder state
|
||||
err error // the result of the last decoder call
|
||||
}
|
||||
|
||||
// NewReader creates a new Reader reading from r. The decompressor
|
||||
// will use an LZMA2 dictionary size up to dictMax bytes in
|
||||
// size. Passing a value of zero sets dictMax to DefaultDictMax. If
|
||||
// an individual XZ stream requires a dictionary size greater than
|
||||
// dictMax in order to decompress, Read will return ErrMemlimit.
|
||||
//
|
||||
// If NewReader is passed a value of nil for r then a Reader is
|
||||
// created such that all read attempts will return io.EOF. This is
|
||||
// useful if you just want to allocate memory for a Reader which will
|
||||
// later be initialized with Reset.
|
||||
//
|
||||
// Due to internal buffering, the Reader may read more data than
|
||||
// necessary from r.
|
||||
func NewReader(r io.Reader, dictMax uint32) (*Reader, error) {
|
||||
if dictMax == 0 {
|
||||
dictMax = DefaultDictMax
|
||||
}
|
||||
z := &Reader{
|
||||
r: r,
|
||||
multistream: true,
|
||||
padding: -1,
|
||||
buf: &xzBuf{},
|
||||
}
|
||||
if r == nil {
|
||||
z.rEOF, z.dEOF = true, true
|
||||
}
|
||||
z.dec = xzDecInit(dictMax, &z.Header)
|
||||
var err error
|
||||
if r != nil {
|
||||
_, err = z.Read(nil) // read stream header
|
||||
}
|
||||
return z, err
|
||||
}
|
||||
|
||||
// decode is a wrapper around xzDecRun that additionally handles
|
||||
// stream padding. It treats the padding as a kind of stream that
|
||||
// decodes to nothing.
|
||||
//
|
||||
// When decoding padding, z.padding >= 0
|
||||
// When decoding a real stream, z.padding == -1
|
||||
func (z *Reader) decode() (ret xzRet) {
|
||||
if z.padding >= 0 {
|
||||
// read all padding in input buffer
|
||||
for z.buf.inPos < len(z.buf.in) &&
|
||||
z.buf.in[z.buf.inPos] == 0 {
|
||||
z.buf.inPos++
|
||||
z.padding++
|
||||
}
|
||||
switch {
|
||||
case z.buf.inPos == len(z.buf.in) && z.rEOF:
|
||||
// case: out of padding. no more input data available
|
||||
if z.padding%4 != 0 {
|
||||
ret = xzDataError
|
||||
} else {
|
||||
ret = xzStreamEnd
|
||||
}
|
||||
case z.buf.inPos == len(z.buf.in):
|
||||
// case: read more padding next loop iteration
|
||||
ret = xzOK
|
||||
default:
|
||||
// case: out of padding. more input data available
|
||||
if z.padding%4 != 0 {
|
||||
ret = xzDataError
|
||||
} else {
|
||||
xzDecReset(z.dec)
|
||||
ret = xzStreamEnd
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ret = xzDecRun(z.dec, z.buf)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (z *Reader) Read(p []byte) (n int, err error) {
|
||||
// restore err
|
||||
err = z.err
|
||||
// set decoder output buffer to p
|
||||
z.buf.out = p
|
||||
z.buf.outPos = 0
|
||||
for {
|
||||
// update n
|
||||
n = z.buf.outPos
|
||||
// if last call to decoder ended with an error, return that error
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
// if decoder has finished, return with err == io.EOF
|
||||
if z.dEOF {
|
||||
err = io.EOF
|
||||
break
|
||||
}
|
||||
// if p full, return with err == nil, unless we have not yet
|
||||
// read the stream header with Read(nil)
|
||||
if n == len(p) && z.CheckType != checkUnset {
|
||||
break
|
||||
}
|
||||
// if needed, read more data from z.r
|
||||
if z.buf.inPos == len(z.buf.in) && !z.rEOF {
|
||||
rn, e := z.r.Read(z.in[:])
|
||||
if e != nil && e != io.EOF {
|
||||
// read error
|
||||
err = e
|
||||
break
|
||||
}
|
||||
if e == io.EOF {
|
||||
z.rEOF = true
|
||||
}
|
||||
// set new input buffer in z.buf
|
||||
z.buf.in = z.in[:rn]
|
||||
z.buf.inPos = 0
|
||||
}
|
||||
// decode more data
|
||||
ret := z.decode()
|
||||
switch ret {
|
||||
case xzOK:
|
||||
// no action needed
|
||||
case xzStreamEnd:
|
||||
if z.padding >= 0 {
|
||||
z.padding = -1
|
||||
if !z.multistream || z.rEOF {
|
||||
z.dEOF = true
|
||||
}
|
||||
} else {
|
||||
z.padding = 0
|
||||
}
|
||||
case xzUnsupportedCheck:
|
||||
err = ErrUnsupportedCheck
|
||||
case xzMemlimitError:
|
||||
err = ErrMemlimit
|
||||
case xzFormatError:
|
||||
err = ErrFormat
|
||||
case xzOptionsError:
|
||||
err = ErrOptions
|
||||
case xzDataError:
|
||||
err = ErrData
|
||||
case xzBufError:
|
||||
err = ErrBuf
|
||||
}
|
||||
// save err
|
||||
z.err = err
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Multistream controls whether the reader is operating in multistream
|
||||
// mode.
|
||||
//
|
||||
// If enabled (the default), the Reader expects the input to be a
|
||||
// sequence of XZ streams, possibly interspersed with stream padding,
|
||||
// which it reads one after another. The effect is that the
|
||||
// concatenation of a sequence of XZ streams or XZ files is
|
||||
// treated as equivalent to the compressed result of the concatenation
|
||||
// of the sequence. This is standard behaviour for XZ readers.
|
||||
//
|
||||
// Calling Multistream(false) disables this behaviour; disabling the
|
||||
// behaviour can be useful when reading file formats that distinguish
|
||||
// individual XZ streams. In this mode, when the Reader reaches the
|
||||
// end of the stream, Read returns io.EOF. To start the next stream,
|
||||
// call z.Reset(nil) followed by z.Multistream(false). If there is no
|
||||
// next stream, z.Reset(nil) will return io.EOF.
|
||||
func (z *Reader) Multistream(ok bool) {
|
||||
z.multistream = ok
|
||||
}
|
||||
|
||||
// Reset, for non-nil values of io.Reader r, discards the Reader z's
|
||||
// state and makes it equivalent to the result of its original state
|
||||
// from NewReader, but reading from r instead. This permits reusing a
|
||||
// Reader rather than allocating a new one.
|
||||
//
|
||||
// If you wish to leave r unchanged use z.Reset(nil). This keeps r
|
||||
// unchanged and ensures internal buffering is preserved. If the
|
||||
// Reader was at the end of a stream it is then ready to read any
|
||||
// follow on streams. If there are no follow on streams z.Reset(nil)
|
||||
// returns io.EOF. If the Reader was not at the end of a stream then
|
||||
// z.Reset(nil) does nothing.
|
||||
func (z *Reader) Reset(r io.Reader) error {
|
||||
switch {
|
||||
case r == nil:
|
||||
z.multistream = true
|
||||
if !z.dEOF {
|
||||
return nil
|
||||
}
|
||||
if z.rEOF {
|
||||
return io.EOF
|
||||
}
|
||||
z.dEOF = false
|
||||
_, err := z.Read(nil) // read stream header
|
||||
return err
|
||||
default:
|
||||
z.r = r
|
||||
z.multistream = true
|
||||
z.rEOF = false
|
||||
z.dEOF = false
|
||||
z.padding = -1
|
||||
z.buf.in = nil
|
||||
z.buf.inPos = 0
|
||||
xzDecReset(z.dec)
|
||||
z.err = nil
|
||||
_, err := z.Read(nil) // read stream header
|
||||
return err
|
||||
}
|
||||
}
|
||||
Vendored
-3
@@ -83,9 +83,6 @@ github.com/nats-io/nuid
|
||||
github.com/novox/mesh-host/internal/declaration
|
||||
github.com/novox/mesh-host/rootsearch
|
||||
github.com/novox/mesh-host/validate
|
||||
# github.com/therootcompany/xz v1.0.1
|
||||
## explicit; go 1.15
|
||||
github.com/therootcompany/xz
|
||||
# golang.org/x/crypto v0.57.0
|
||||
## explicit; go 1.26.0
|
||||
golang.org/x/crypto/acme
|
||||
|
||||
Reference in New Issue
Block a user