Compare commits
9
Commits
6c3cf18dfc
...
65408432af
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
65408432af | ||
|
|
d865ed6425 | ||
|
|
2843834813 | ||
|
|
27b13e4586 | ||
|
|
142a5ad71d | ||
|
|
0cd8bc0bfb | ||
|
|
7385d9c092 | ||
|
|
b0c4b849a7 | ||
|
|
b849af53dd |
@@ -16,7 +16,8 @@ RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' -o /mesh-builder ./cmd/mes
|
||||
FROM ${ALPINE_BASE}
|
||||
# git to clone what it is asked to build, and the docker client to build and push it. The daemon
|
||||
# is the machine's, reached through its socket — a build machine shares the runtime it was given
|
||||
# rather than running one inside itself.
|
||||
RUN apk add --no-cache git docker-cli
|
||||
# rather than running one inside itself. xz to read an upstream release an archive is pinned to
|
||||
# (novox/hq ADR 0316): Go's standard library reads gzip and bzip2 and not xz.
|
||||
RUN apk add --no-cache git docker-cli xz
|
||||
COPY --from=build /mesh-builder /usr/local/bin/mesh-builder
|
||||
ENTRYPOINT ["/usr/local/bin/mesh-builder"]
|
||||
|
||||
@@ -167,3 +167,54 @@ func TestASetuidSearchAsksOnlyAKnownNodeAndFailsOnARefusal(t *testing.T) {
|
||||
t.Fatalf("a start: %v, printed %q", err, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// The directory judgement's line asks only a known node, one name and nothing else, fails on a refusal, and says it
|
||||
// changed nothing (novox/hq issue 358).
|
||||
func TestADirectoryJudgementAsksOnlyAKnownNodeAndSaysItIsReadOnly(t *testing.T) {
|
||||
t.Setenv(link.CallerVar, "jo through mesh-cli on anchor")
|
||||
asked := 0
|
||||
ask := func(answer link.HandOverAnswer) func(node, by string) (link.HandOverAnswer, error) {
|
||||
return func(node, by string) (link.HandOverAnswer, error) {
|
||||
asked++
|
||||
if node != "novox" || by != "jo through mesh-cli on anchor" {
|
||||
t.Fatalf("asked %q %q", node, by)
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
}
|
||||
known := func(string) error { return nil }
|
||||
var out bytes.Buffer
|
||||
for _, args := range [][]string{nil, {"novox", "extra"}, {"-x"}} {
|
||||
if err := judgeDirectoriesAsked(args, known, ask(link.HandOverAnswer{Said: "x"}), &out); err == nil || asked != 0 {
|
||||
t.Fatalf("%v was asked: %v", args, err)
|
||||
}
|
||||
}
|
||||
if err := judgeDirectoriesAsked([]string{"novox"}, func(string) error { return errors.New("no node called novox") },
|
||||
ask(link.HandOverAnswer{Said: "x"}), &out); err == nil || asked != 0 {
|
||||
t.Fatalf("an unknown node: %v", err)
|
||||
}
|
||||
if err := judgeDirectoriesAsked([]string{"novox"}, known, ask(link.HandOverAnswer{Refused: "no; nothing was judged"}),
|
||||
&out); err == nil || !strings.Contains(err.Error(), "novox refused") || out.Len() != 0 {
|
||||
t.Fatalf("a refusal: %v, printed %q", err, out.String())
|
||||
}
|
||||
said := "2 directories declared: 1 placed, 1 would be refused\n/var/lib/a: placed\n/var/lib/b: would be refused: …"
|
||||
if err := judgeDirectoriesAsked([]string{"novox"}, known, ask(link.HandOverAnswer{Said: said}), &out); err != nil ||
|
||||
out.String() != "novox (read-only: nothing on novox was changed)\n"+said+"\n" {
|
||||
t.Fatalf("a judgement: %v, printed %q", err, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// The judgement is a seat verb an agent may call — it reads, and changes nothing on the node — and runs exactly
|
||||
// `node judge-directories <node>` (novox/hq issue 358).
|
||||
func TestTheDirectoryJudgementIsAVerbThatOnlyReads(t *testing.T) {
|
||||
argv, err := argvFor("judge-directories", map[string]any{"node": "novox"})
|
||||
if err != nil || strings.Join(argv, " ") != "node judge-directories novox" {
|
||||
t.Fatalf("the verb runs %v, %v", argv, err)
|
||||
}
|
||||
if _, err := argvFor("judge-directories", map[string]any{}); err == nil {
|
||||
t.Fatal("the verb ran with no node")
|
||||
}
|
||||
if !readingVerbs["judge-directories"] {
|
||||
t.Fatal("the judgement is not counted a reading verb")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -127,9 +127,14 @@ func nodeCommand(ctx context.Context, args []string) error {
|
||||
// and whoever may call a verb includes agents.
|
||||
return nodeSetuidSearch(ctx, open, args[1:])
|
||||
|
||||
case "judge-directories":
|
||||
// Which of a node's declared directories the walk that follows no link another account could make would
|
||||
// refuse (novox/hq issue 358). Read-only on the node and here, so a verb may run it too.
|
||||
return nodeJudgeDirectories(ctx, open, args[1:])
|
||||
|
||||
default:
|
||||
return fmt.Errorf("node has no %q; it has add, list, show, public-domain, account, agent-account, hand-over "+
|
||||
"and setuid-search", args[0])
|
||||
return fmt.Errorf("node has no %q; it has add, list, show, public-domain, account, agent-account, hand-over, "+
|
||||
"setuid-search and judge-directories", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -937,3 +942,57 @@ func printJSONTo(w io.Writer, v any) error {
|
||||
fmt.Fprintln(w, string(body))
|
||||
return nil
|
||||
}
|
||||
|
||||
const judgeDirectoriesUsage = "node judge-directories <node> — read-only: which of the directories <node> is " +
|
||||
"declared the node-engine's walk that follows no link another account could make would refuse, one line each " +
|
||||
"(novox/hq issue 358). Nothing is made, chmodded or chowned"
|
||||
|
||||
// nodeJudgeDirectories asks the node's engine to judge its declared directories, signed with the mesh's key as a
|
||||
// hand-over is. Read-only: the engine changes nothing and keeps nothing.
|
||||
func nodeJudgeDirectories(ctx context.Context, open *stores, args []string) error {
|
||||
known := func(node string) error {
|
||||
_, err := open.inventory.NodeByName(ctx, node)
|
||||
return err
|
||||
}
|
||||
ask := func(node, by string) (link.HandOverAnswer, error) {
|
||||
ident, err := open.Identity(ctx)
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, fmt.Errorf("the mesh's signing key cannot be read, so nothing was asked of %s: %w",
|
||||
node, err)
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return link.HandOverAnswer{}, fmt.Errorf("cannot reach the bus, so nothing was asked of %s: %w", node, err)
|
||||
}
|
||||
defer js.Close()
|
||||
return link.AskJudgeDirectories(ctx, js.Conn(), ident, node, by, link.HandOverWithin)
|
||||
}
|
||||
return judgeDirectoriesAsked(args, known, ask, os.Stdout)
|
||||
}
|
||||
|
||||
// judgeDirectoriesAsked is the line with its two acts given: whether the mesh knows the node, and the ask. The
|
||||
// engine's refusal is this command's failure.
|
||||
func judgeDirectoriesAsked(args []string, known func(node string) error,
|
||||
ask func(node, by string) (link.HandOverAnswer, error), out io.Writer) error {
|
||||
if len(args) != 1 || args[0] == "" || strings.HasPrefix(args[0], "-") {
|
||||
return errors.New(judgeDirectoriesUsage)
|
||||
}
|
||||
node := args[0]
|
||||
if err := known(node); err != nil {
|
||||
return fmt.Errorf("nothing was asked: %w", err)
|
||||
}
|
||||
answer, err := ask(node, handOverBy())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Refused != "" {
|
||||
return fmt.Errorf("%s refused: %s", node, answer.Refused)
|
||||
}
|
||||
fmt.Fprintf(out, "%s (read-only: nothing on %s was changed)\n", node, node)
|
||||
fmt.Fprintln(out, answer.Said)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// An own secret issued outside the mesh that nobody gave is never made (novox/hq issue 401, ADR 0228). Seen on
|
||||
// 2026-10-10: the send made a random value for the mounts module's smb-credentials and placed it, and the module read
|
||||
// it as a credential given and wrong. The send now makes none, places none, and places a value only once a person
|
||||
// gave one — and a value the mesh made for it before is not placed either.
|
||||
|
||||
// anAssistant is a module with a secret the mesh makes and one only an outside party issues.
|
||||
func anAssistant() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "assistant", Version: "1",
|
||||
OwnSecrets: catalogue.OwnSecrets{
|
||||
"password": {Path: "/var/lib/mesh/assistant/password", Taken: catalogue.TakenAtStart},
|
||||
"vendor-key": {Path: "/var/lib/mesh/assistant/vendor-key", Taken: catalogue.TakenAtStart, IssuedBy: catalogue.IssuedOutside},
|
||||
},
|
||||
Resources: []map[string]any{
|
||||
{"id": "state", "type": "directory", "path": "/var/lib/mesh/assistant", "mode": "0700"},
|
||||
}}
|
||||
}
|
||||
|
||||
// sentTo composes what a push sends to a machine, on the send path.
|
||||
func sentTo(t *testing.T, open *stores, node string) []map[string]any {
|
||||
t.Helper()
|
||||
plan, settings, err := planFor(t.Context(), open, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(t.Context(), open, node, plan, settings)
|
||||
if err != nil {
|
||||
t.Fatalf("the send to %s did not compose: %v", node, err)
|
||||
}
|
||||
return declared.Resources
|
||||
}
|
||||
|
||||
func placedAt(resources []map[string]any, path string) map[string]any {
|
||||
for _, r := range resources {
|
||||
if r["path"] == path && r["sealed"] != nil {
|
||||
return r
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestTheSendMakesNoValueForAnOutsideSecretNobodyGave(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, anAssistant())
|
||||
if _, err := assign(ctx, open, "laptop", "assistant"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sent := sentTo(t, open, "laptop")
|
||||
if placedAt(sent, "/var/lib/mesh/assistant/password") == nil {
|
||||
t.Fatal("the secret the mesh makes was not placed")
|
||||
}
|
||||
if got := placedAt(sent, "/var/lib/mesh/assistant/vendor-key"); got != nil {
|
||||
t.Fatalf("a secret nobody gave was placed: %v", got)
|
||||
}
|
||||
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "assistant", "vendor-key"); err != nil || held {
|
||||
t.Fatalf("the send made a value for a secret issued outside the mesh (held %v, %v)", held, err)
|
||||
}
|
||||
|
||||
// A value the mesh made before this rule is not one a person gave: it is not placed.
|
||||
if _, err := open.inventory.SecretForModule(ctx, "laptop", "assistant", "vendor-key"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := placedAt(sentTo(t, open, "laptop"), "/var/lib/mesh/assistant/vendor-key"); got != nil {
|
||||
t.Fatalf("a value the mesh made for a secret issued outside was placed: %v", got)
|
||||
}
|
||||
|
||||
// Given, it is placed.
|
||||
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "assistant", "vendor-key", "the vendor's key"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
given, ok, err := open.inventory.GivenOwnSecret(ctx, "laptop", "assistant", "vendor-key")
|
||||
if err != nil || !ok || !given.Current {
|
||||
t.Fatalf("the value given is not read as given: %+v %v %v", given, ok, err)
|
||||
}
|
||||
placed := placedAt(sentTo(t, open, "laptop"), "/var/lib/mesh/assistant/vendor-key")
|
||||
if placed == nil || placed["sealed"] != given.Sealed {
|
||||
t.Fatalf("the value given is placed as %v", placed)
|
||||
}
|
||||
}
|
||||
|
||||
// The read paths answer as the send does: a question about the machine composes without the secret nobody gave, and
|
||||
// asking writes nothing.
|
||||
func TestAQuestionComposesWithoutAnOutsideSecretNobodyGave(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, anAssistant())
|
||||
if _, err := assign(ctx, open, "laptop", "assistant"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pushedBy(t, open, "laptop") // the mesh's own secret is made by the send
|
||||
plan, settings, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, choosing := range []Choosing{Reading, Foreseeing} {
|
||||
got, err := declarationWith(ctx, open, "laptop", plan, settings, gens, choosing)
|
||||
if err != nil {
|
||||
t.Fatalf("%v: %v", choosing, err)
|
||||
}
|
||||
if placedAt(got.Resources, "/var/lib/mesh/assistant/vendor-key") != nil || len(got.foreseen) != 0 {
|
||||
t.Fatalf("%v: a secret nobody gave is composed (foreseen %v)", choosing, got.foreseen)
|
||||
}
|
||||
}
|
||||
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "assistant", "vendor-key"); err != nil || held {
|
||||
t.Fatalf("a question made a value (held %v, %v)", held, err)
|
||||
}
|
||||
}
|
||||
@@ -561,6 +561,13 @@ func busCredentialIssued(ctx context.Context, inv *inventory.Inventory, node, mo
|
||||
return nil
|
||||
}
|
||||
|
||||
// givenUnderAnOldKey is the refusal for a value a person gave — a family's member, or an own secret issued outside
|
||||
// the mesh — sealed to a key the machine no longer holds: the mesh cannot make another, so it is given again.
|
||||
func givenUnderAnOldKey(module, node, name string) error {
|
||||
return fmt.Errorf("%s on %s holds %q, which was given to the mesh rather than made by it, and %s has "+
|
||||
"since generated a new sealing key. The mesh cannot make another; give it again", module, node, name, node)
|
||||
}
|
||||
|
||||
// renderingFor is everything a node's declaration is composed with, and the node's record.
|
||||
func renderingFor(ctx context.Context, open *stores, node string,
|
||||
plan catalogue.Resolution, settings catalogue.SettingsBy,
|
||||
@@ -679,10 +686,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
}
|
||||
if !g.Current {
|
||||
if choosing == Allocating {
|
||||
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf(
|
||||
"%s on %s holds %q, which was given to the mesh rather than made by it, and %s has "+
|
||||
"since generated a new sealing key. The mesh cannot make another; give it again",
|
||||
m.Module, node, g.Name, node)
|
||||
return catalogue.Rendering{}, inventory.Node{}, givenUnderAnOldKey(m.Module, node, g.Name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -692,7 +696,31 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
needed[m.Module][g.Name] = g.Sealed
|
||||
}
|
||||
}
|
||||
for name := range m.OwnSecrets.Plain() {
|
||||
for name, own := range m.OwnSecrets.Plain() {
|
||||
// **An own secret issued outside the mesh is never made** (novox/hq issue 401, ADR 0228; ADR 0283 for
|
||||
// a family): only a value a person gave is placed. One not given is nothing — no file, and an empty
|
||||
// value where a file of the module names it — so the module can tell it was not given, where a value
|
||||
// the mesh made looked given and wrong. A value the mesh made for it before is not one either.
|
||||
if own.IssuedBy == catalogue.IssuedOutside {
|
||||
g, given, err := inv.GivenOwnSecret(ctx, node, m.Module, name)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
if !given {
|
||||
continue
|
||||
}
|
||||
if !g.Current {
|
||||
if choosing == Allocating {
|
||||
return catalogue.Rendering{}, inventory.Node{}, givenUnderAnOldKey(m.Module, node, name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if needed[m.Module] == nil {
|
||||
needed[m.Module] = map[string]string{}
|
||||
}
|
||||
needed[m.Module][name] = g.Sealed
|
||||
continue
|
||||
}
|
||||
// Minted on the send path and only read on every other. Making one is an insert, and
|
||||
// a question that writes is a question that can block against the machine it is about.
|
||||
var sealed string
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -67,18 +68,95 @@ func listQueue(ctx context.Context, args []string, w io.Writer) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now := time.Now()
|
||||
if *asJSON {
|
||||
body, err := json.MarshalIndent(q, "", " ")
|
||||
body, err := json.MarshalIndent(queueAnswerOf(q, now), "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintln(w, string(body))
|
||||
return nil
|
||||
}
|
||||
fmt.Fprint(w, queueText(q, time.Now()))
|
||||
fmt.Fprint(w, queueText(q, now))
|
||||
return nil
|
||||
}
|
||||
|
||||
// queueAnswer is the queue as data (novox/hq issue 455): what a caller reads to say how far back an ask
|
||||
// waits — "waiting in the build queue behind 2" — which the table alone could not tell it. Text is the
|
||||
// table, written from the same read: the verb answers it as its output and the rest as its answer, so the
|
||||
// two never disagree.
|
||||
type queueAnswer struct {
|
||||
Seat string `json:"seat"`
|
||||
Waiting int `json:"waiting"`
|
||||
InFlight int `json:"in-flight"`
|
||||
Dead int `json:"dead"`
|
||||
MaxDeliver int `json:"max-deliver"`
|
||||
Asks []queuedAnswer `json:"asks"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
// queuedAnswer is one ask as data. Never what the request carries beyond its source — not `held`, every
|
||||
// artifact the mesh has built, nor a check's checkouts beside it.
|
||||
type queuedAnswer struct {
|
||||
ID string `json:"id"`
|
||||
State string `json:"state"`
|
||||
// Kind is "check" for a pull request's merge check, "build" for anything else.
|
||||
Kind string `json:"kind"`
|
||||
Repository string `json:"repository"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
// Commit is the ref when it names a commit; empty when the ask follows a branch or the default.
|
||||
Commit string `json:"commit,omitempty"`
|
||||
// Machine is the one building an ask in flight; WasOn the one that started it and handed it back.
|
||||
Machine string `json:"machine,omitempty"`
|
||||
WasOn string `json:"was-on,omitempty"`
|
||||
AskedAt time.Time `json:"asked-at,omitzero"`
|
||||
// Since is when the ask came to its state: its start on the machine for one in flight that a
|
||||
// machine said it started, otherwise when it was asked.
|
||||
Since time.Time `json:"since,omitzero"`
|
||||
// Position is its place among the waiting, 1 being handed out next; 0 for an ask not waiting.
|
||||
Position int `json:"position,omitempty"`
|
||||
Seq uint64 `json:"seq"`
|
||||
}
|
||||
|
||||
// aCommitRef is a ref that names a commit by its full hash (SHA-1 or SHA-256).
|
||||
var aCommitRef = regexp.MustCompile(`^(?:[0-9a-f]{40}|[0-9a-f]{64})$`)
|
||||
|
||||
// queueAnswerOf is the queue as data, in the table's order: in flight, the waiting from the next one to
|
||||
// be handed out, then the dead. A pure function of one read (novox/hq issue 455).
|
||||
func queueAnswerOf(q link.Queue, now time.Time) queueAnswer {
|
||||
waiting, running, dead := q.Of(link.AskWaiting), q.Of(link.AskInFlight), q.Of(link.AskDead)
|
||||
out := queueAnswer{Seat: q.Seat, Waiting: len(waiting), InFlight: len(running), Dead: len(dead),
|
||||
MaxDeliver: q.MaxDeliver, Asks: []queuedAnswer{}, Text: queueText(q, now)}
|
||||
one := func(a link.QueuedAsk, position int) queuedAnswer {
|
||||
kind := "build"
|
||||
if a.Request.Check != nil {
|
||||
kind = "check"
|
||||
}
|
||||
since := a.AskedAt
|
||||
if a.State == link.AskInFlight && a.On != "" && !a.Started.IsZero() {
|
||||
since = a.Started
|
||||
}
|
||||
commit := ""
|
||||
if aCommitRef.MatchString(a.Ref) {
|
||||
commit = a.Ref
|
||||
}
|
||||
return queuedAnswer{ID: a.ID, State: a.State, Kind: kind, Repository: link.WithoutPassword(a.Repository), Path: a.Path,
|
||||
Ref: a.Ref, Commit: commit, Machine: a.On, WasOn: a.Was, AskedAt: a.AskedAt, Since: since,
|
||||
Position: position, Seq: a.Seq}
|
||||
}
|
||||
for _, a := range running {
|
||||
out.Asks = append(out.Asks, one(a, 0))
|
||||
}
|
||||
for i, a := range waiting {
|
||||
out.Asks = append(out.Asks, one(a, i+1))
|
||||
}
|
||||
for _, a := range dead {
|
||||
out.Asks = append(out.Asks, one(a, 0))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// queueText is the queue as a person reads it: a summary line, then each kind in queue order.
|
||||
// Never what an ask carries as `held` — that is every artifact the mesh has built.
|
||||
func queueText(q link.Queue, now time.Time) string {
|
||||
@@ -92,7 +170,7 @@ func queueText(q link.Queue, now time.Time) string {
|
||||
return "asked " + now.Sub(t).Round(time.Second).String() + " ago"
|
||||
}
|
||||
what := func(a link.QueuedAsk) string {
|
||||
s := a.Repository
|
||||
s := link.WithoutPassword(a.Repository)
|
||||
if a.Path != "" {
|
||||
s += " at " + a.Path
|
||||
}
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The queue answers as data as well as text (novox/hq issue 455): each ask's repository, commit, kind,
|
||||
// machine, since and position among the waiting, read from the same queue the table is written from.
|
||||
|
||||
const aCommit = "0123456789abcdef0123456789abcdef01234567"
|
||||
|
||||
func aQueueOfEachKind(now time.Time) link.Queue {
|
||||
asked := now.Add(-5 * time.Minute)
|
||||
started := now.Add(-2 * time.Minute)
|
||||
return link.Queue{Seat: link.TheBuildMachine, Delivered: 2, AckPending: 1, MaxDeliver: 5, Asks: []link.QueuedAsk{
|
||||
{Seq: 1, ID: "build-dead", Repository: "ssh://forge/novox/x", AskedAt: asked, State: link.AskDead},
|
||||
{Seq: 2, ID: "build-running", Repository: "ssh://forge/novox/mesh-host", Ref: aCommit, AskedAt: asked,
|
||||
State: link.AskInFlight, On: "ace", Started: started,
|
||||
Request: link.BuildRequest{ID: "build-running", Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-host"}}},
|
||||
{Seq: 3, ID: "build-first", Repository: "ssh://forge/novox/mesh-catalog", Ref: "main", AskedAt: asked,
|
||||
State: link.AskWaiting},
|
||||
{Seq: 4, ID: "build-second", Repository: "ssh://forge/novox/mesh-catalog", Path: "modules/gitea", Ref: aCommit,
|
||||
AskedAt: now.Add(-time.Minute), State: link.AskWaiting,
|
||||
Request: link.BuildRequest{ID: "build-second", Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog"}}},
|
||||
}}
|
||||
}
|
||||
|
||||
func TestTheQueueVerbAnswersEachAskAsData(t *testing.T) {
|
||||
now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
|
||||
q := aQueueOfEachKind(now)
|
||||
got := queueAnswerOf(q, now)
|
||||
if got.Waiting != 2 || got.InFlight != 1 || got.Dead != 1 || got.Seat != link.TheBuildMachine {
|
||||
t.Fatalf("counts: %+v", got)
|
||||
}
|
||||
if got.Text != queueText(q, now) {
|
||||
t.Fatalf("the data's text is not the table read from the same queue:\n%s", got.Text)
|
||||
}
|
||||
byID := map[string]queuedAnswer{}
|
||||
for _, a := range got.Asks {
|
||||
byID[a.ID] = a
|
||||
}
|
||||
running := byID["build-running"]
|
||||
if running.Kind != "check" || running.Machine != "ace" || running.Commit != aCommit || running.Position != 0 ||
|
||||
!running.Since.Equal(now.Add(-2*time.Minute)) || running.State != link.AskInFlight {
|
||||
t.Errorf("in flight: %+v", running)
|
||||
}
|
||||
first, second := byID["build-first"], byID["build-second"]
|
||||
if first.Position != 1 || first.Kind != "build" || first.Commit != "" || first.Ref != "main" || first.Machine != "" ||
|
||||
!first.Since.Equal(now.Add(-5*time.Minute)) {
|
||||
t.Errorf("first waiting: %+v", first)
|
||||
}
|
||||
if second.Position != 2 || second.Kind != "check" || second.Commit != aCommit || second.Path != "modules/gitea" ||
|
||||
second.Repository != "ssh://forge/novox/mesh-catalog" {
|
||||
t.Errorf("second waiting: %+v", second)
|
||||
}
|
||||
if dead := byID["build-dead"]; dead.Position != 0 || dead.State != link.AskDead {
|
||||
t.Errorf("dead: %+v", dead)
|
||||
}
|
||||
// In the table's order: in flight, then the waiting by position, then the dead.
|
||||
var order []string
|
||||
for _, a := range got.Asks {
|
||||
order = append(order, a.ID)
|
||||
}
|
||||
if want := []string{"build-running", "build-first", "build-second", "build-dead"}; !reflect.DeepEqual(order, want) {
|
||||
t.Errorf("order %v, want %v", order, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The verb asks its command for JSON, and its answer carries the data while its output stays the table a
|
||||
// person reads, so a text caller reads what it always did.
|
||||
func TestTheQueueVerbKeepsTheTableAndCarriesTheData(t *testing.T) {
|
||||
argv, err := argvFor("queue", nil)
|
||||
if err != nil || !reflect.DeepEqual(argv, []string{"queue", "--json"}) {
|
||||
t.Fatalf("queue's command line: %v %v", argv, err)
|
||||
}
|
||||
now := time.Now()
|
||||
q := aQueueOfEachKind(now)
|
||||
body, err := json.Marshal(queueAnswerOf(q, now))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
answer := answerOf(argv, body, "", true)
|
||||
if answer.Output != queueText(q, now) {
|
||||
t.Fatalf("the output is not the table:\n%s", answer.Output)
|
||||
}
|
||||
data, ok := answer.Answer.(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("no data: %#v", answer.Answer)
|
||||
}
|
||||
if _, doubled := data["text"]; doubled {
|
||||
t.Error("the table is said twice: once as the output and again inside the data")
|
||||
}
|
||||
asks, _ := data["asks"].([]any)
|
||||
if len(asks) != 4 || data["waiting"] != float64(2) {
|
||||
t.Fatalf("data: %v", data)
|
||||
}
|
||||
second := asks[2].(map[string]any)
|
||||
if second["position"] != float64(2) || second["kind"] != "check" || second["commit"] != aCommit {
|
||||
t.Errorf("second waiting: %v", second)
|
||||
}
|
||||
for _, a := range asks {
|
||||
if _, held := a.(map[string]any)["held"]; held {
|
||||
t.Error("an ask carries what the mesh holds")
|
||||
}
|
||||
}
|
||||
// A failure is said as text, as before.
|
||||
failed := answerOf(argv, nil, "mesh-controller: the bus cannot be reached\n", false)
|
||||
if failed.Answer != nil || !strings.Contains(failed.Output, "cannot be reached") {
|
||||
t.Errorf("a failure: %+v", failed)
|
||||
}
|
||||
}
|
||||
|
||||
// A repository named with a password never shows it, in the table or the data.
|
||||
func TestTheQueueNeverShowsARepositorysPassword(t *testing.T) {
|
||||
if got := link.WithoutPassword("https://agent:s3cret@forge.example/novox/x.git"); strings.Contains(got, "s3cret") ||
|
||||
!strings.Contains(got, "agent@forge.example/novox/x.git") {
|
||||
t.Errorf("redacted: %s", got)
|
||||
}
|
||||
now := time.Now()
|
||||
q := link.Queue{Seat: link.TheBuildMachine, Asks: []link.QueuedAsk{{Seq: 1, ID: "build-1",
|
||||
Repository: "https://agent:s3cret@forge.example/novox/x.git", State: link.AskWaiting}}}
|
||||
body, _ := json.Marshal(queueAnswerOf(q, now))
|
||||
if strings.Contains(string(body), "s3cret") {
|
||||
t.Errorf("the queue said a password: %s", body)
|
||||
}
|
||||
for _, kept := range []string{"ssh://git@forge:222/novox/x.git", "novox/x", "git@forge:novox/x.git", ""} {
|
||||
if got := link.WithoutPassword(kept); got != kept {
|
||||
t.Errorf("%q became %q", kept, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -326,7 +326,7 @@ func TestTheQueueVerbsRunTheirCommands(t *testing.T) {
|
||||
args map[string]any
|
||||
want []string
|
||||
}{
|
||||
{"queue", nil, []string{"queue"}},
|
||||
{"queue", nil, []string{"queue", "--json"}},
|
||||
{"cancel", map[string]any{"id": "build-1"}, []string{"cancel", "build-1"}},
|
||||
{"clear", nil, []string{"clear"}},
|
||||
{"clear", map[string]any{"dead": "true"}, []string{"clear", "--dead"}},
|
||||
|
||||
@@ -314,6 +314,11 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
return nil, err
|
||||
}
|
||||
return []string{"node", "show", str("node")}, nil
|
||||
case "judge-directories":
|
||||
if err := need("node"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{"node", "judge-directories", str("node")}, nil
|
||||
case "modules":
|
||||
return []string{"module", "list", "--json"}, nil
|
||||
case "seats":
|
||||
@@ -426,7 +431,8 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
return argv, nil
|
||||
// The build queue (novox/hq ADR 0219).
|
||||
case "queue":
|
||||
return []string{"queue"}, nil
|
||||
// As data, the table its output (novox/hq issue 455).
|
||||
return []string{"queue", "--json"}, nil
|
||||
case "cancel", "kill":
|
||||
if err := need("id"); err != nil {
|
||||
return nil, err
|
||||
@@ -977,7 +983,14 @@ var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "co
|
||||
// What the records say the registries may keep (novox/hq ADR 0251).
|
||||
"artifacts": true, "collect": true, "images": true, "mirrors": true,
|
||||
// The delivery's owner's verbs answer JSON where they read (plan, order, check, walks) — novox/hq ADR 0239.
|
||||
"delivery": true}
|
||||
"delivery": true,
|
||||
// The build queue, each ask with its position among the waiting (novox/hq issue 455).
|
||||
"queue": true}
|
||||
|
||||
// tableBesideVerbs are the JSON verbs whose document carries the table a person reads as its `text`: the
|
||||
// verb answers that table as its output, as it did before it answered data, and the rest as `answer` —
|
||||
// both from the one read, so they never disagree (novox/hq issue 455).
|
||||
var tableBesideVerbs = map[string]bool{"queue": true}
|
||||
|
||||
// overviewVerbs are the JSON verbs whose answer is said once, as data, and not again as the text the
|
||||
// command printed: the overviews, which grow with the mesh (novox/hq issue 314). Every reader of theirs
|
||||
@@ -1097,6 +1110,12 @@ func answerOf(argv []string, stdout []byte, stderr string, ok bool) verbAnswer {
|
||||
var parsed any
|
||||
if json.Unmarshal(bytes.TrimSpace(stdout), &parsed) == nil {
|
||||
answer.Answer = parsed
|
||||
if doc, isDoc := parsed.(map[string]any); isDoc && tableBesideVerbs[argv[0]] {
|
||||
if table, said := doc["text"].(string); said {
|
||||
answer.Output = table + stderr
|
||||
delete(doc, "text")
|
||||
}
|
||||
}
|
||||
if overviewVerbs[argv[0]] {
|
||||
// Once, as data: the same document again as text doubled an answer that already
|
||||
// outgrew one message of the bus (novox/hq issue 314).
|
||||
@@ -1479,7 +1498,9 @@ func seatAnnouncement(handlers map[string]link.ToolHandler) micro.Info {
|
||||
}
|
||||
|
||||
// nodeReads are the `node` subcommands a verb may run: the ones that only read.
|
||||
var nodeReads = map[string]bool{"list": true, "show": true}
|
||||
var nodeReads = map[string]bool{"list": true, "show": true,
|
||||
// Read-only on the node too: the engine judges its directories and changes nothing (novox/hq issue 358).
|
||||
"judge-directories": true}
|
||||
|
||||
// flagsOnly says a command line's rest names no subcommand: empty, or beginning with a flag. For a command
|
||||
// with no subcommands every word is a flag, its value or a name it reads.
|
||||
|
||||
@@ -256,7 +256,7 @@ func stringLit(e ast.Expr) (string, bool) {
|
||||
var accountedFlags = map[string]map[string]string{
|
||||
"status": {"json": "set by the verb: the answer is data"},
|
||||
"seats": {"json": "set by the verb: the answer is data"},
|
||||
"queue": {"json": "not set: the verb answers the table a person reads"},
|
||||
"queue": {"json": "set by the verb: the answer is data, its output the table (novox/hq issue 455)"},
|
||||
"plan": {"json": "set by the verb unless files is asked"},
|
||||
"push": {
|
||||
"wait": "set by the verb to 0: a tool call cannot hold a connection for a whole apply",
|
||||
|
||||
@@ -172,7 +172,7 @@ func composeStatus(open *stores) func(context.Context) ([]byte, error) {
|
||||
var readingVerbs = map[string]bool{
|
||||
"tools": true, "calls": true, "status": true, "nodes": true, "node": true, "modules": true,
|
||||
"seats": true, "builds": true, "plan": true, "queue": true, "durations": true, "hand-acts": true,
|
||||
"doctor": true, "conditions": true, "healers": true,
|
||||
"doctor": true, "conditions": true, "healers": true, "judge-directories": true,
|
||||
}
|
||||
|
||||
// nudgingListener is the enrolment, nudging the summary when a machine said something new.
|
||||
|
||||
+10
-1
@@ -336,6 +336,13 @@ func AskHandOverSubject(node string) string { return "mesh.node." + node + ".ask
|
||||
// a hand-over is (link.SetuidSearchContext), for the same reason.
|
||||
func AskSetuidSearchSubject(node string) string { return "mesh.node." + node + ".ask.setuid-search" }
|
||||
|
||||
// AskJudgeDirectoriesSubject is where the controller asks one machine's node-engine to judge, read-only, which of
|
||||
// its declared directories the walk that follows no link another account could make would refuse (novox/hq issue
|
||||
// 358): a request answered once, signed as a hand-over is (link.JudgeDirectoriesContext), for the same reason.
|
||||
func AskJudgeDirectoriesSubject(node string) string {
|
||||
return "mesh.node." + node + ".ask.judge-directories"
|
||||
}
|
||||
|
||||
// inbox is a principal's own reply space. No user is ever granted a bare `_INBOX.>` (design 25
|
||||
// §4): with one account, inbox privacy is the permission list or it is nothing, so each user's
|
||||
// inbox is derived from its own identity and its permissions name that prefix and no other.
|
||||
@@ -614,7 +621,9 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// issue 356), which it answers on the request's reply: the one request a node is asked.
|
||||
AskHandOverSubject(p.Node),
|
||||
// And asking it for a fresh search for setuid programs (novox/hq issue 361), answered the same way.
|
||||
AskSetuidSearchSubject(p.Node)}
|
||||
AskSetuidSearchSubject(p.Node),
|
||||
// And asking it to judge its declared directories, read-only (novox/hq issue 358), answered the same way.
|
||||
AskJudgeDirectoriesSubject(p.Node)}
|
||||
// The node-engine witnesses the core builds it places (novox/hq to-be 45 §8, ADR 0236; the
|
||||
// contract is lease/witness.go): it asks its own machine's node tools PING, and, where the
|
||||
// machine runs the controller, reads the lease's one key — read, never written.
|
||||
|
||||
@@ -114,7 +114,9 @@ func TestOnlyWhatCanBeAskedMayAnswer(t *testing.T) {
|
||||
node, _ := PermissionsFor(Principal{Kind: KindNode, Node: "one", PasswordHash: "x"})
|
||||
if !node.AllowResponses || !slices.Contains(node.Subscribe, AskHandOverSubject("one")) ||
|
||||
!slices.Contains(node.Subscribe, AskSetuidSearchSubject("one")) ||
|
||||
slices.Contains(node.Subscribe, AskSetuidSearchSubject("two")) {
|
||||
slices.Contains(node.Subscribe, AskSetuidSearchSubject("two")) ||
|
||||
!slices.Contains(node.Subscribe, AskJudgeDirectoriesSubject("one")) ||
|
||||
slices.Contains(node.Subscribe, AskJudgeDirectoriesSubject("two")) {
|
||||
t.Fatalf("a node cannot answer the hand-over it is asked: %v %v", node.AllowResponses, node.Subscribe)
|
||||
}
|
||||
person, _ := PermissionsFor(Principal{Kind: KindPerson, Node: "one", Module: "jo", PasswordHash: "x"})
|
||||
|
||||
+1
-1
@@ -34,7 +34,7 @@ accounts {
|
||||
} }
|
||||
{ user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: {
|
||||
publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.INFO.NODES.one", "$SRV.PING.node-tools.one", "mesh.control.one.>"] }
|
||||
subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.ask.hand-over", "mesh.node.one.ask.report", "mesh.node.one.ask.setuid-search", "mesh.node.one.declare"] }
|
||||
subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.ask.hand-over", "mesh.node.one.ask.judge-directories", "mesh.node.one.ask.report", "mesh.node.one.ask.setuid-search", "mesh.node.one.declare"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "one.nats", password: "$2a$11$bbbbbbbbbbbbbbbbbbbbbb", permissions: {
|
||||
|
||||
@@ -96,6 +96,11 @@ var WritersTable = []WriterRow{
|
||||
{State: "a fresh setuid search asked of a machine", Writer: "controller, at its terminal",
|
||||
KeptIn: "the machine, which throws its last search away", Others: "the engine verifies the mesh's signature and starts it, or refuses",
|
||||
Subjects: []string{"mesh.node.*.ask.setuid-search"}, Writes: isController},
|
||||
// A read-only judgement of a machine's declared directories asked of its engine (novox/hq issue 358): signed as a
|
||||
// hand-over is, under a signing context of its own. It changes nothing on the machine.
|
||||
{State: "a judgement of a machine's directories asked of it", Writer: "controller",
|
||||
KeptIn: "nowhere: the engine answers it and keeps nothing", Others: "the engine verifies the mesh's signature and answers, or refuses",
|
||||
Subjects: []string{"mesh.node.*.ask.judge-directories"}, Writes: isController},
|
||||
{State: "a machine's applied state and its report", Writer: "the node-engine's apply queue",
|
||||
KeptIn: "the machine; the report on the bus", Others: "the reconcile and a delivery enqueue, never apply",
|
||||
// And its health statement between reports (novox/hq ADR 0240): the same writer stating the same
|
||||
|
||||
@@ -17,6 +17,7 @@ var designRows = []string{
|
||||
"a machine's declaration",
|
||||
"a hand-over asked of a machine",
|
||||
"a fresh setuid search asked of a machine",
|
||||
"a judgement of a machine's directories asked of it",
|
||||
"a machine's applied state and its report",
|
||||
"the controller lease",
|
||||
"plans and their tiers",
|
||||
@@ -186,3 +187,21 @@ func TestASetuidSearchAskHasOnePublisher(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A judgement of a machine's directories asked of it has one publisher, the controller** (novox/hq issue 358), as a
|
||||
// hand-over has.
|
||||
func TestADirectoryJudgementAskHasOnePublisher(t *testing.T) {
|
||||
for _, p := range []Principal{
|
||||
{Kind: KindNode, Node: "laptop"},
|
||||
{Kind: KindNodeTools, Node: "laptop", Module: RuntimeModule},
|
||||
{Kind: KindModule, Node: "laptop", Module: "notes"},
|
||||
} {
|
||||
err := CheckWriters(p, []string{AskJudgeDirectoriesSubject("laptop")})
|
||||
if err == nil || !strings.Contains(err.Error(), "a judgement of a machine's directories asked of it") {
|
||||
t.Errorf("%s may ask a judgement of directories: %v", p.Username(), err)
|
||||
}
|
||||
}
|
||||
if err := CheckWriters(Principal{Kind: KindController}, []string{AskJudgeDirectoriesSubject("laptop")}); err != nil {
|
||||
t.Fatalf("the controller may not ask a judgement: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -971,8 +971,19 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil
|
||||
|
||||
case catalogue.ArtifactArchive:
|
||||
src.ownHas(cleanEntry(a.From) + "/**")
|
||||
body, err := pack(filepath.Join(tree, a.From))
|
||||
var body []byte
|
||||
var err error
|
||||
if a.URL != "" {
|
||||
// An upstream file pinned by URL and sha256 (novox/hq ADR 0316): its build source is the pin, which
|
||||
// is in module.json, always part of it.
|
||||
body, err = upstreamArchive(ctx, workspace, a, say)
|
||||
if err != nil {
|
||||
return catalogue.Built{}, fmt.Errorf("%s: %s: %w", module, a.Name, err)
|
||||
}
|
||||
} else {
|
||||
src.ownHas(cleanEntry(a.From) + "/**")
|
||||
body, err = pack(filepath.Join(tree, a.From))
|
||||
}
|
||||
if err != nil {
|
||||
return catalogue.Built{}, fmt.Errorf("%s: packing %s failed: %w", module, a.Name, err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,326 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/bzip2"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// An archive whose source is somebody else's published file, pinned by URL and sha256 (novox/hq ADR 0316
|
||||
// §2, §6, to-be 56 §7): a mail client's release, an add-on's xpi.
|
||||
//
|
||||
// **Fetched by the build seat, checked against the pin before anything is read from it, and packed as any
|
||||
// archive is packed.** What a machine receives is the mesh's own archive from the artifact store, by digest,
|
||||
// exactly as for a directory of the repository; no machine ever asks the vendor. A file whose bytes are not
|
||||
// the pin's fails the build and names both digests, because the address serving something else is the one
|
||||
// thing the pin exists to catch.
|
||||
//
|
||||
// **Kept in the build seat's workspace under its sha256**, so it is fetched once: a vendor that serves only
|
||||
// its newest version at one address (Owl) would otherwise break every later build of the same pin.
|
||||
|
||||
// maxUpstream is the largest upstream file fetched, and the largest a release may unpack to: the node-engine
|
||||
// unpacks no archive larger than this either (mesh-host internal/apply/archive.go, maxArchive).
|
||||
const maxUpstream = 512 << 20
|
||||
|
||||
// maxEntries is the most entries an upstream archive may hold: millions of empty ones would fill the build
|
||||
// seat's inodes and the node's. Betterbird's release holds 64.
|
||||
const maxEntries = 100_000
|
||||
|
||||
// noDowngrade refuses a redirect off https: the digest still guards the bytes, and a fetch in the clear says
|
||||
// to whoever is on the path what the build seat fetches.
|
||||
func noDowngrade(req *http.Request, via []*http.Request) error {
|
||||
if req.URL.Scheme != "https" {
|
||||
return fmt.Errorf("%s redirected to %s, which is not https", via[0].URL, req.URL)
|
||||
}
|
||||
if len(via) >= 10 {
|
||||
return errors.New("more than 10 redirects")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// upstreamClient fetches upstream files: a variable so a test serves its own.
|
||||
var upstreamClient = &http.Client{Timeout: 15 * time.Minute}
|
||||
|
||||
// upstreamDir is where the build seat keeps the upstream files it fetched, by sha256.
|
||||
func upstreamDir(workspace string) string { return filepath.Join(workspace, "upstream") }
|
||||
|
||||
// upstreamArchive is the packed archive of an artifact pinned by URL: the file fetched (or kept from an
|
||||
// earlier build) and checked, then laid out as the artifact says and packed.
|
||||
func upstreamArchive(ctx context.Context, workspace string, a catalogue.Artifact,
|
||||
say func(step, format string, args ...any)) ([]byte, error) {
|
||||
file, err := upstreamFile(ctx, workspace, a, say)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
laid := filepath.Join(workspace, "upstream-"+a.Name)
|
||||
if err := os.RemoveAll(laid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.RemoveAll(laid)
|
||||
if err := os.MkdirAll(laid, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if a.File != "" {
|
||||
body, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(laid, a.File), body, 0o644); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else if err := unpackUpstream(ctx, file, laid, a.Within); err != nil {
|
||||
return nil, fmt.Errorf("%s (%s): %w", a.URL, a.Version, err)
|
||||
}
|
||||
return pack(laid)
|
||||
}
|
||||
|
||||
// upstreamFile is the path of the pinned file in the workspace: kept from an earlier build when its bytes
|
||||
// are still the pin's, else fetched now and kept only when they are.
|
||||
func upstreamFile(ctx context.Context, workspace string, a catalogue.Artifact,
|
||||
say func(step, format string, args ...any)) (string, error) {
|
||||
want := strings.ToLower(a.SHA256)
|
||||
dir := upstreamDir(workspace)
|
||||
kept := filepath.Join(dir, want)
|
||||
if got, err := sumOf(kept); err == nil {
|
||||
if got == want {
|
||||
say("upstream", "%s %s: kept from an earlier build (sha256:%s)", a.Name, a.Version, short(want))
|
||||
return kept, nil
|
||||
}
|
||||
// Changed on disk since it was kept: fetched again rather than trusted.
|
||||
os.Remove(kept)
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
say("upstream", "fetching %s (%s)", a.URL, a.Version)
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, a.URL, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
client := *upstreamClient
|
||||
client.CheckRedirect = noDowngrade
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot fetch %s: %w", a.URL, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("%s answered %s", a.URL, response.Status)
|
||||
}
|
||||
part, err := os.CreateTemp(dir, want+".*.part")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer os.Remove(part.Name())
|
||||
hash := sha256.New()
|
||||
n, err := io.Copy(io.MultiWriter(part, hash), io.LimitReader(response.Body, maxUpstream+1))
|
||||
if cerr := part.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("fetching %s: %w", a.URL, err)
|
||||
}
|
||||
if n > maxUpstream {
|
||||
return "", fmt.Errorf("%s is larger than %d bytes, more than a machine unpacks; nothing was kept", a.URL, maxUpstream)
|
||||
}
|
||||
got := hex.EncodeToString(hash.Sum(nil))
|
||||
if got != want {
|
||||
return "", fmt.Errorf("%s was pinned as sha256:%s and what it served is sha256:%s (%d bytes); nothing "+
|
||||
"was kept. The address serves another file than the one reviewed: compare it, then move the pin "+
|
||||
"in a reviewed change (novox/hq ADR 0316)", a.URL, want, got, n)
|
||||
}
|
||||
if err := os.Rename(part.Name(), kept); err != nil {
|
||||
return "", err
|
||||
}
|
||||
say("upstream", "%s %s: %d bytes, sha256:%s as pinned", a.Name, a.Version, n, short(want))
|
||||
return kept, nil
|
||||
}
|
||||
|
||||
// sumOf is the sha256 of a file, in hexadecimal.
|
||||
func sumOf(file string) (string, error) {
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
hash := sha256.New()
|
||||
if _, err := io.Copy(hash, f); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// unpackUpstream unpacks a tar archive, compressed with gzip, bzip2 or xz (told by its first bytes), into a
|
||||
// directory: of its entries only those within `within` (all, when empty), with that prefix taken off.
|
||||
//
|
||||
// **Only files and directories, and nothing outside.** A link, a device or an entry naming a path outside
|
||||
// the archive, or outside `within`, refuses the build by name: the node-engine unpacks only files and
|
||||
// directories, and an archive quietly unpacked without what it held would be a release that is not the one
|
||||
// pinned.
|
||||
func unpackUpstream(ctx context.Context, file, into, within string) error {
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
buffered := bufio.NewReader(f)
|
||||
magic, _ := buffered.Peek(6)
|
||||
var stream io.Reader
|
||||
var wait func() error
|
||||
switch {
|
||||
case bytes.HasPrefix(magic, []byte{0x1f, 0x8b}):
|
||||
zipped, err := gzip.NewReader(buffered)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer zipped.Close()
|
||||
stream = zipped
|
||||
case bytes.HasPrefix(magic, []byte("BZh")):
|
||||
stream = bzip2.NewReader(buffered)
|
||||
case bytes.HasPrefix(magic, []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}):
|
||||
// Go's standard library reads no xz; the build seat's image carries xz (cmd/mesh-builder/Dockerfile).
|
||||
if _, err := exec.LookPath("xz"); err != nil {
|
||||
return errors.New("it is compressed with xz, and this build seat has no xz to read it")
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "xz", "-dc")
|
||||
cmd.Stdin = buffered
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
out, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
stream = out
|
||||
wait = func() error {
|
||||
if err := cmd.Wait(); err != nil {
|
||||
return fmt.Errorf("xz could not read it: %v %s", err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
defer func() {
|
||||
if wait != nil {
|
||||
_ = cmd.Process.Kill()
|
||||
_ = cmd.Wait()
|
||||
}
|
||||
}()
|
||||
default:
|
||||
return errors.New("it is not a tar archive compressed with gzip, bzip2 or xz; a single file is kept " +
|
||||
"whole by naming it in `file`")
|
||||
}
|
||||
if err := untar(stream, into, within); err != nil {
|
||||
return err
|
||||
}
|
||||
if wait != nil {
|
||||
// Whatever follows the tar's end is read before xz is waited for: xz blocks on a full pipe otherwise.
|
||||
if _, err := io.Copy(io.Discard, stream); err != nil {
|
||||
return err
|
||||
}
|
||||
w := wait
|
||||
wait = nil
|
||||
return w()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// untar writes a tar stream's files and directories within `within` into a directory.
|
||||
func untar(stream io.Reader, into, within string) error {
|
||||
reader := tar.NewReader(stream)
|
||||
prefix := ""
|
||||
if within != "" {
|
||||
prefix = path.Clean(within) + "/"
|
||||
}
|
||||
var total int64
|
||||
files, entries := 0, 0
|
||||
for {
|
||||
header, err := reader.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entries++; entries > maxEntries {
|
||||
return fmt.Errorf("it holds more than %d entries", maxEntries)
|
||||
}
|
||||
if header.Typeflag == tar.TypeXGlobalHeader {
|
||||
// A pax global header (git archive writes one) names no file.
|
||||
continue
|
||||
}
|
||||
name := path.Clean(strings.TrimPrefix(header.Name, "./"))
|
||||
if path.IsAbs(name) || name == ".." || strings.HasPrefix(name, "../") {
|
||||
return fmt.Errorf("%s names a path outside the archive", header.Name)
|
||||
}
|
||||
if name == "." || name+"/" == prefix ||
|
||||
(header.Typeflag == tar.TypeDir && strings.HasPrefix(prefix, name+"/")) {
|
||||
// The archive's root, the directory taken, or one of its parents.
|
||||
continue
|
||||
}
|
||||
if prefix != "" {
|
||||
if !strings.HasPrefix(name, prefix) {
|
||||
return fmt.Errorf("%s is outside %s, the directory the artifact is taken from", header.Name, within)
|
||||
}
|
||||
name = strings.TrimPrefix(name, prefix)
|
||||
}
|
||||
target := filepath.Join(into, filepath.FromSlash(name))
|
||||
switch header.Typeflag {
|
||||
case tar.TypeDir:
|
||||
if err := os.MkdirAll(target, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
case tar.TypeReg:
|
||||
total += header.Size
|
||||
if total > maxUpstream {
|
||||
return fmt.Errorf("it unpacks to more than %d bytes, more than a machine unpacks", maxUpstream)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
mode := os.FileMode(0o644)
|
||||
if header.Mode&0o111 != 0 {
|
||||
mode = 0o755
|
||||
}
|
||||
out, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(out, io.LimitReader(reader, header.Size)); err != nil {
|
||||
out.Close()
|
||||
return err
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
files++
|
||||
default:
|
||||
return fmt.Errorf("%s is a %c, and an archive carries only files and directories", header.Name,
|
||||
header.Typeflag)
|
||||
}
|
||||
}
|
||||
if files == 0 {
|
||||
if within != "" {
|
||||
return fmt.Errorf("it holds no file within %s", within)
|
||||
}
|
||||
return errors.New("it holds no file")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,385 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// An archive pinned by URL and sha256 (novox/hq ADR 0316 §2, §6; How it is checked: "an upstream artifact
|
||||
// whose download does not match its sha256 fails the build and names both digests").
|
||||
|
||||
type tarEntry struct {
|
||||
name, body string
|
||||
mode int64
|
||||
kind byte
|
||||
}
|
||||
|
||||
func tarOf(t *testing.T, entries []tarEntry) []byte {
|
||||
t.Helper()
|
||||
var raw bytes.Buffer
|
||||
w := tar.NewWriter(&raw)
|
||||
for _, e := range entries {
|
||||
kind := e.kind
|
||||
if kind == 0 {
|
||||
kind = tar.TypeReg
|
||||
}
|
||||
h := &tar.Header{Name: e.name, Mode: e.mode, Typeflag: kind, Size: int64(len(e.body))}
|
||||
if kind == tar.TypeSymlink {
|
||||
h.Linkname, h.Size = "/usr/share/hunspell", 0
|
||||
}
|
||||
if kind == tar.TypeDir {
|
||||
h.Size = 0
|
||||
}
|
||||
if err := w.WriteHeader(h); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if kind == tar.TypeReg {
|
||||
if _, err := w.Write([]byte(e.body)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return raw.Bytes()
|
||||
}
|
||||
|
||||
func gzipped(t *testing.T, body []byte) []byte {
|
||||
t.Helper()
|
||||
var out bytes.Buffer
|
||||
z := gzip.NewWriter(&out)
|
||||
if _, err := z.Write(body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := z.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
func sumHex(b []byte) string {
|
||||
s := sha256.Sum256(b)
|
||||
return hex.EncodeToString(s[:])
|
||||
}
|
||||
|
||||
// vendor serves files by path over https and counts what it was asked.
|
||||
type vendor struct {
|
||||
files map[string][]byte
|
||||
asked int
|
||||
}
|
||||
|
||||
func serveVendor(t *testing.T, v *vendor) string {
|
||||
t.Helper()
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
v.asked++
|
||||
body, ok := v.files[r.URL.Path]
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Write(body)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
before := upstreamClient
|
||||
// Under a public name, as a manifest must say it: example.com, which the test server's certificate
|
||||
// names, dialled at the server.
|
||||
transport := server.Client().Transport.(*http.Transport).Clone()
|
||||
addr := server.Listener.Addr().String()
|
||||
transport.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) {
|
||||
return (&net.Dialer{}).DialContext(ctx, network, addr)
|
||||
}
|
||||
upstreamClient = &http.Client{Transport: transport}
|
||||
t.Cleanup(func() { upstreamClient = before })
|
||||
return "https://example.com"
|
||||
}
|
||||
|
||||
// entriesOf lists a packed archive: each file with its mode and content.
|
||||
func entriesOf(t *testing.T, packed []byte) map[string]string {
|
||||
t.Helper()
|
||||
z, err := gzip.NewReader(bytes.NewReader(packed))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := tar.NewReader(z)
|
||||
out := map[string]string{}
|
||||
for {
|
||||
h, err := r.Next()
|
||||
if err == io.EOF {
|
||||
return out
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, _ := io.ReadAll(r)
|
||||
out[h.Name] = string(body) + " " + os.FileMode(h.Mode).String()
|
||||
}
|
||||
}
|
||||
|
||||
func release(t *testing.T) []byte {
|
||||
return gzipped(t, tarOf(t, []tarEntry{
|
||||
{name: "betterbird/", kind: tar.TypeDir, mode: 0o755},
|
||||
{name: "betterbird/application.ini", body: "[App]\nVersion=153.4.0\n", mode: 0o644},
|
||||
{name: "betterbird/betterbird", body: "#!/bin/sh\n", mode: 0o755},
|
||||
{name: "betterbird/defaults/pref/channel-prefs.js", body: "pref(\"app.update.channel\", \"default\");\n", mode: 0o644},
|
||||
}))
|
||||
}
|
||||
|
||||
func pinnedManifest(base, sum string) string {
|
||||
return `{"module":"betterbird","version":"1",
|
||||
"build":{"artifacts":[
|
||||
{"name":"archive","kind":"archive","url":"` + base + `/release.tar.gz","sha256":"` + sum + `","version":"153.4.0esr-bb10","within":"betterbird"}]},
|
||||
"resources":[{"id":"installation","type":"archive","path":"/opt/betterbird","artifact":"archive"}]}`
|
||||
}
|
||||
|
||||
func TestAnArchivePinnedByURLIsFetchedCheckedAndPackedWithoutItsOwnDirectory(t *testing.T) {
|
||||
body := release(t)
|
||||
v := &vendor{files: map[string][]byte{"/release.tar.gz": body}}
|
||||
base := serveVendor(t, v)
|
||||
r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil)
|
||||
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
|
||||
Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest, _ := got.Manifest.Resources[0]["digest"].(string)
|
||||
if !strings.HasPrefix(digest, "sha256:") || r.archives["betterbird/archive"] != digest {
|
||||
t.Fatalf("the release was not published and pinned as the module's archive: %v, published %v",
|
||||
got.Manifest.Resources[0], r.archives)
|
||||
}
|
||||
|
||||
packed, err := upstreamArchive(context.Background(), workspace, catalogue.Artifact{Name: "archive", Kind: "archive",
|
||||
URL: base + "/release.tar.gz", SHA256: sumHex(body), Version: "153.4.0esr-bb10", Within: "betterbird"},
|
||||
func(string, string, ...any) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if "sha256:"+sumHex(packed) != digest {
|
||||
t.Fatalf("a second packing of one pin gave another digest: %s and %s", digest, sumHex(packed))
|
||||
}
|
||||
entries := entriesOf(t, packed)
|
||||
want := map[string]string{
|
||||
"application.ini": "[App]\nVersion=153.4.0\n -rw-r--r--",
|
||||
"betterbird": "#!/bin/sh\n -rwxr-xr-x",
|
||||
"defaults/pref/channel-prefs.js": "pref(\"app.update.channel\", \"default\");\n -rw-r--r--",
|
||||
}
|
||||
if len(entries) != len(want) {
|
||||
t.Fatalf("the archive holds %v, want %v", entries, want)
|
||||
}
|
||||
for name, w := range want {
|
||||
if entries[name] != w {
|
||||
t.Errorf("%s is %q in the archive, want %q", name, entries[name], w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpstreamFileIsFetchedOnceAndKeptByItsDigest(t *testing.T) {
|
||||
// The vendor of an add-on serves only its newest version at one address: a pin that was fetched once
|
||||
// must still build after the address moved on.
|
||||
body := release(t)
|
||||
v := &vendor{files: map[string][]byte{"/release.tar.gz": body}}
|
||||
base := serveVendor(t, v)
|
||||
r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil)
|
||||
first, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
|
||||
Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v.files["/release.tar.gz"] = []byte("the vendor's next release")
|
||||
second, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
|
||||
Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("a pin fetched before failed to build once the address served something else: %v", err)
|
||||
}
|
||||
if v.asked != 1 {
|
||||
t.Fatalf("the vendor was asked %d times for one pin, want once", v.asked)
|
||||
}
|
||||
if first.Manifest.Resources[0]["digest"] != second.Manifest.Resources[0]["digest"] {
|
||||
t.Fatal("two builds of one pin gave two digests")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpstreamFileWhoseDownloadDoesNotMatchItsPinFailsAndNamesBothDigests(t *testing.T) {
|
||||
pinned := release(t)
|
||||
served := gzipped(t, tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "not what was reviewed", mode: 0o755}}))
|
||||
v := &vendor{files: map[string][]byte{"/release.tar.gz": served}}
|
||||
base := serveVendor(t, v)
|
||||
r, workspace := aRepository(t, pinnedManifest(base, sumHex(pinned)), nil)
|
||||
_, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
|
||||
Npmrc{}, GitCredential{}, nil)
|
||||
if err == nil {
|
||||
t.Fatal("a download that is not the pinned file was built")
|
||||
}
|
||||
for _, digest := range []string{sumHex(pinned), sumHex(served)} {
|
||||
if !strings.Contains(err.Error(), "sha256:"+digest) {
|
||||
t.Errorf("the refusal does not name sha256:%s: %v", digest, err)
|
||||
}
|
||||
}
|
||||
if len(r.archives) != 0 {
|
||||
t.Fatalf("something was published: %v", r.archives)
|
||||
}
|
||||
kept, _ := filepath.Glob(filepath.Join(upstreamDir(workspace), "*"))
|
||||
if len(kept) != 0 {
|
||||
t.Fatalf("the refused download was kept: %v", kept)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpstreamFileThatIsNotAnArchiveIsKeptWholeUnderItsName(t *testing.T) {
|
||||
xpi := []byte("PK\x03\x04 an add-on")
|
||||
v := &vendor{files: map[string][]byte{"/owl.xpi": xpi}}
|
||||
base := serveVendor(t, v)
|
||||
packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "owl", Kind: "archive",
|
||||
URL: base + "/owl.xpi", SHA256: sumHex(xpi), Version: "1.5.6.1", File: "owl.xpi"}, func(string, string, ...any) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entries := entriesOf(t, packed)
|
||||
if len(entries) != 1 || entries["owl.xpi"] != string(xpi)+" -rw-r--r--" {
|
||||
t.Fatalf("the add-on was not kept whole as owl.xpi: %v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpstreamArchiveHoldingALinkOrAnEntryOutsideItsDirectoryIsRefused(t *testing.T) {
|
||||
cases := map[string][]tarEntry{
|
||||
// The community package links the dictionaries; a release carrying a link is not one a machine unpacks.
|
||||
"is a 2": {{name: "betterbird/betterbird", body: "x", mode: 0o755},
|
||||
{name: "betterbird/dictionaries", kind: tar.TypeSymlink}},
|
||||
"outside betterbird": {{name: "betterbird/betterbird", body: "x", mode: 0o755},
|
||||
{name: "README", body: "x", mode: 0o644}},
|
||||
"outside the archive": {{name: "../etc/passwd", body: "x", mode: 0o644}},
|
||||
"no file within": {{name: "betterbird/", kind: tar.TypeDir, mode: 0o755}},
|
||||
}
|
||||
for want, entries := range cases {
|
||||
body := gzipped(t, tarOf(t, entries))
|
||||
v := &vendor{files: map[string][]byte{"/r.tar.gz": body}}
|
||||
base := serveVendor(t, v)
|
||||
_, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
|
||||
URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
|
||||
if err == nil || !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("an archive whose entry is %q was not refused for it: %v", want, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpstreamArchiveCompressedWithXzIsRead(t *testing.T) {
|
||||
// Betterbird publishes its Linux release as .tar.xz; the build seat's image carries xz for it.
|
||||
if _, err := exec.LookPath("xz"); err != nil {
|
||||
t.Skip("NOT TESTED here: no xz on this machine (the build seat's image carries it)")
|
||||
}
|
||||
plain := tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "#!/bin/sh\n", mode: 0o755}})
|
||||
cmd := exec.Command("xz", "-zc")
|
||||
cmd.Stdin = bytes.NewReader(plain)
|
||||
compressed, err := cmd.Output()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v := &vendor{files: map[string][]byte{"/r.tar.xz": compressed}}
|
||||
base := serveVendor(t, v)
|
||||
packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
|
||||
URL: base + "/r.tar.xz", SHA256: sumHex(compressed), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var names []string
|
||||
for n := range entriesOf(t, packed) {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
if strings.Join(names, ",") != "betterbird" {
|
||||
t.Fatalf("the xz release unpacked to %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpstreamTarWithAGlobalHeaderNestedWithinAndTrailingDataIsRead(t *testing.T) {
|
||||
// git archive writes a pax global header; a release may unpack under a/b with a/ listed first; xz may be
|
||||
// followed by bytes after the tar's end, which must be read before xz is waited for.
|
||||
var raw bytes.Buffer
|
||||
w := tar.NewWriter(&raw)
|
||||
w.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, Name: "pax_global_header", PAXRecords: map[string]string{"comment": "abc"}})
|
||||
w.WriteHeader(&tar.Header{Name: "a/", Typeflag: tar.TypeDir, Mode: 0o755})
|
||||
w.WriteHeader(&tar.Header{Name: "a/b/", Typeflag: tar.TypeDir, Mode: 0o755})
|
||||
w.WriteHeader(&tar.Header{Name: "a/b/f", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1})
|
||||
w.Write([]byte("x"))
|
||||
w.Close()
|
||||
body := gzipped(t, raw.Bytes())
|
||||
v := &vendor{files: map[string][]byte{"/r.tar.gz": body}}
|
||||
base := serveVendor(t, v)
|
||||
packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive",
|
||||
URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1", Within: "a/b"}, func(string, string, ...any) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e := entriesOf(t, packed); len(e) != 1 || e["f"] == "" {
|
||||
t.Fatalf("%v", e)
|
||||
}
|
||||
if _, err := exec.LookPath("xz"); err != nil {
|
||||
t.Skip("NOT TESTED here: no xz")
|
||||
}
|
||||
// A megabyte after the tar's end: more than a pipe holds.
|
||||
trailing := append(append([]byte{}, raw.Bytes()...), bytes.Repeat([]byte{0}, 1<<20)...)
|
||||
cmd := exec.Command("xz", "-zc")
|
||||
cmd.Stdin = bytes.NewReader(trailing)
|
||||
compressed, err := cmd.Output()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v.files["/r.tar.xz"] = compressed
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := upstreamArchive(ctx, t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive", URL: base + "/r.tar.xz",
|
||||
SHA256: sumHex(compressed), Version: "1", Within: "a/b"}, func(string, string, ...any) {}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpstreamArchiveWithTooManyEntriesIsRefused(t *testing.T) {
|
||||
var raw bytes.Buffer
|
||||
w := tar.NewWriter(&raw)
|
||||
for i := 0; i <= maxEntries; i++ {
|
||||
w.WriteHeader(&tar.Header{Name: fmt.Sprintf("d%d/", i), Typeflag: tar.TypeDir, Mode: 0o755})
|
||||
}
|
||||
w.Close()
|
||||
body := gzipped(t, raw.Bytes())
|
||||
v := &vendor{files: map[string][]byte{"/r.tar.gz": body}}
|
||||
base := serveVendor(t, v)
|
||||
_, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive",
|
||||
URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1"}, func(string, string, ...any) {})
|
||||
if err == nil || !strings.Contains(err.Error(), "entries") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARedirectOffHttpsIsNotFollowed(t *testing.T) {
|
||||
plain := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("x")) }))
|
||||
t.Cleanup(plain.Close)
|
||||
secure := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, plain.URL+"/r", http.StatusFound)
|
||||
}))
|
||||
t.Cleanup(secure.Close)
|
||||
before := upstreamClient
|
||||
upstreamClient = secure.Client()
|
||||
t.Cleanup(func() { upstreamClient = before })
|
||||
_, err := upstreamFile(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", URL: secure.URL + "/r",
|
||||
SHA256: sumHex([]byte("x")), Version: "1", File: "a"}, func(string, string, ...any) {})
|
||||
if err == nil || !strings.Contains(err.Error(), "not https") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,10 @@ package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"path"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
@@ -312,7 +315,7 @@ func (b *Build) problems(module string) []string {
|
||||
module, a.Name, a.System, spokenSystems()))
|
||||
}
|
||||
} else {
|
||||
if a.From == "" {
|
||||
if a.From == "" && a.URL == "" {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q says nothing about what it is built from", module, a.Name))
|
||||
}
|
||||
@@ -350,6 +353,85 @@ func (b *Build) problems(module string) []string {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q mirrors %q, which names no tag or digest", module, a.Name, a.From))
|
||||
}
|
||||
problems = append(problems, upstreamFileProblems(module, a)...)
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// reservedSuffixes are names no public vendor publishes under: an installation's own and documentation's.
|
||||
var reservedSuffixes = []string{".internal", ".local", ".lan", ".home", ".corp", ".localhost", ".home.arpa",
|
||||
".test", ".invalid", ".example", ".intranet", ".private"}
|
||||
|
||||
// notAPublicHost says why a host is not a public vendor's name, or "".
|
||||
func notAPublicHost(host string) string {
|
||||
h := strings.ToLower(strings.TrimSuffix(host, "."))
|
||||
if net.ParseIP(h) != nil {
|
||||
return "an address rather than a name"
|
||||
}
|
||||
if !strings.Contains(h, ".") {
|
||||
return "a name with no domain"
|
||||
}
|
||||
for _, s := range reservedSuffixes {
|
||||
if strings.HasSuffix(h, s) {
|
||||
return "a name under " + s + ", which is nobody's public domain"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// sha256Hex is a pinned sha256 as a manifest writes it: 64 lower-case hexadecimal characters.
|
||||
var sha256Hex = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
||||
|
||||
// upstreamFileProblems is what is wrong with an archive pinned by URL (novox/hq ADR 0316 §2, §6): it is an
|
||||
// archive, fetched over https, pinned by a sha256 and a version, and it names no directory of the repository
|
||||
// besides; a file name or a directory within is a plain path that stays where it is put.
|
||||
func upstreamFileProblems(module string, a Artifact) []string {
|
||||
pinned := a.URL != "" || a.SHA256 != "" || a.Version != "" || a.File != "" || a.Within != ""
|
||||
if !pinned {
|
||||
return nil
|
||||
}
|
||||
var problems []string
|
||||
say := func(format string, args ...any) {
|
||||
problems = append(problems, fmt.Sprintf("%s: %q ", module, a.Name)+fmt.Sprintf(format, args...))
|
||||
}
|
||||
if a.Kind != ArtifactArchive {
|
||||
say("is a %q and names an upstream file (url, sha256, version, file, within): only an archive is "+
|
||||
"mirrored from one (novox/hq ADR 0316)", a.Kind)
|
||||
return problems
|
||||
}
|
||||
if a.URL == "" {
|
||||
say("names a sha256, a version, a file or a directory within, and no url to fetch it from")
|
||||
return problems
|
||||
}
|
||||
if a.From != "" {
|
||||
say("is built from %q and fetched from %s: an archive is one or the other", a.From, a.URL)
|
||||
}
|
||||
if u, err := url.Parse(a.URL); err != nil || u.Scheme != "https" || u.Host == "" {
|
||||
say("is fetched from %q, and an upstream file is fetched over https from a named host", a.URL)
|
||||
} else if why := notAPublicHost(u.Hostname()); why != "" {
|
||||
say("is fetched from %s, %s: an upstream file is somebody else's release, published under a public name", a.URL, why)
|
||||
}
|
||||
if !sha256Hex.MatchString(a.SHA256) {
|
||||
say("is fetched from %s without a pinned sha256 (64 lower-case hexadecimal characters): what that "+
|
||||
"address serves changes under the mesh, and only the digest says it is what was reviewed", a.URL)
|
||||
}
|
||||
if strings.TrimSpace(a.Version) == "" {
|
||||
say("pins %s and says no version, so a person reading the pin cannot tell which release it is", a.URL)
|
||||
}
|
||||
if a.File != "" && a.Within != "" {
|
||||
say("names a file (%q) and a directory within (%q): a file is kept whole, a directory is taken out "+
|
||||
"of an archive", a.File, a.Within)
|
||||
}
|
||||
for field, p := range map[string]string{"file": a.File, "within": a.Within} {
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
clean := path.Clean(p)
|
||||
if clean != p || strings.HasPrefix(p, "/") || clean == "." || clean == ".." || strings.HasPrefix(clean, "../") ||
|
||||
(field == "file" && strings.Contains(p, "/")) {
|
||||
say("names %s %q, which is not a plain relative %s", field, p, map[string]string{
|
||||
"file": "file name", "within": "directory"}[field])
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
@@ -645,6 +645,11 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
sealed = ForeseenSealed
|
||||
*foreseen = append(*foreseen, m.Module+"/"+name)
|
||||
}
|
||||
if sealed == "" && m.OwnSecrets[name].IssuedBy == IssuedOutside {
|
||||
// **Not given, and the mesh never makes it** (novox/hq issue 401, ADR 0228): no file, as for a
|
||||
// family's member (ADR 0283), so the module reads its absence as not given.
|
||||
continue
|
||||
}
|
||||
if sealed == "" {
|
||||
// Declared and not made. Refused rather than skipped: a module whose own
|
||||
// credential is silently absent starts, fails to authenticate, and the reason is
|
||||
@@ -1002,7 +1007,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
// content is where a setting lands, so a placeholder may only exist once the setting
|
||||
// has been put in — filling secrets first would look at content that is not yet what
|
||||
// the machine receives.
|
||||
if err := intoFile(copied, sealed, m.Module); err != nil {
|
||||
if err := intoFile(copied, sealed, outsideNotGiven(m, with.Needed[m.Module]), m.Module); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// **Substituted here, not on the machine.** A bound value is not secret — the mesh
|
||||
@@ -2431,6 +2436,9 @@ func secretsReadBy(resource map[string]any, m Manifest, given map[string]string)
|
||||
}
|
||||
paths := map[string]string{}
|
||||
for name, own := range m.OwnSecrets.Plain() {
|
||||
if own.IssuedBy == IssuedOutside && given[name] == "" {
|
||||
continue // not given, so not placed: nothing to restart on (novox/hq issue 401)
|
||||
}
|
||||
paths[name] = own.Path
|
||||
}
|
||||
for name, sealed := range given {
|
||||
|
||||
@@ -71,6 +71,9 @@ var hostname = regexp.MustCompile(
|
||||
`(?:be|nl|de|fr|uk|eu|com|net|org|io|dev|app|cloud|site|online|me|co|ch|at|lu|` +
|
||||
`internal|example|tld|test|invalid)`)
|
||||
|
||||
// upstreamVersion is where an artifact pinned by URL says which release it is (Artifact.Version).
|
||||
var upstreamVersion = regexp.MustCompile(`^build\.artifacts\[[0-9]+\]\.version$`)
|
||||
|
||||
// address is a dotted quad.
|
||||
var address = regexp.MustCompile(`(?:[0-9]{1,3}\.){3}[0-9]{1,3}`)
|
||||
|
||||
@@ -120,7 +123,10 @@ func InstallationProblems(m Manifest) []string {
|
||||
problems = append(problems, fmt.Sprintf("%s names %s at %s", m.Module, name, at))
|
||||
}
|
||||
for _, ip := range addressesIn(value) {
|
||||
if meant[ip] {
|
||||
// An upstream pin's version may have four dotted numbers (Owl 1.5.6.1), which is no address; a name
|
||||
// in it is still judged above. Its URL is judged whole: the vendor's host goes through the artifact's
|
||||
// names-on-purpose like any name a definition means (novox/hq ADR 0316).
|
||||
if meant[ip] || upstreamVersion.MatchString(at) {
|
||||
continue
|
||||
}
|
||||
problems = append(problems, fmt.Sprintf("%s names the public address %s at %s", m.Module, ip, at))
|
||||
|
||||
@@ -976,6 +976,35 @@ type Artifact struct {
|
||||
// success, and was invisible to the launcher (novox/hq 04-ISSUES/142).
|
||||
Binary string `json:"binary,omitempty"`
|
||||
|
||||
// URL is where an archive's upstream file is fetched from, for an archive that is a release
|
||||
// somebody else published rather than a directory of this repository (novox/hq ADR 0316 §2, §6).
|
||||
// With it come SHA256 and Version, and From stays empty.
|
||||
//
|
||||
// **Pinned by the digest of what the vendor published, mirrored by the build seat.** A release of
|
||||
// about 80 MB copied into a repository is kept for ever in its history at every upgrade, and an
|
||||
// add-on whose licence forbids copying it to third parties may not be in a repository at all. So the
|
||||
// manifest names the file, the build seat fetches it, refuses it when its sha256 is not the pin's and
|
||||
// names both digests, and packs it as any archive is packed: what reaches a machine is the mesh's own
|
||||
// archive in the artifact store, fetched by digest like every other, and a machine never asks the
|
||||
// vendor. Upgrading is changing the pin, reviewed like any other change.
|
||||
URL string `json:"url,omitempty"`
|
||||
// SHA256 is the sha256 of the file at URL, as 64 lower-case hexadecimal characters.
|
||||
SHA256 string `json:"sha256,omitempty"`
|
||||
// Version is what the vendor calls the release pinned, said where a person reads the pin; the build
|
||||
// does not read it.
|
||||
Version string `json:"version,omitempty"`
|
||||
// File is the name an upstream file that is not an archive is kept under: the archive built from it
|
||||
// holds that one file (an add-on's xpi). Empty means the file is a tar archive (gzip, bzip2 or xz)
|
||||
// whose files are the artifact.
|
||||
File string `json:"file,omitempty"`
|
||||
// Within is the directory inside an upstream tar archive whose content is the artifact, for a
|
||||
// release that unpacks into a directory of its own name (`betterbird/`). An entry outside it refuses
|
||||
// the build. Empty means the archive's root.
|
||||
Within string `json:"within,omitempty"`
|
||||
// NamesOnPurpose are the names this artifact means to name, each with why: the vendor's host an upstream
|
||||
// file is fetched from, which the installation check judges like any name (InstallationProblems).
|
||||
NamesOnPurpose map[string]string `json:"names-on-purpose,omitempty"`
|
||||
|
||||
// Entrypoints are the compiled files a tool host should load from this module, relative to the
|
||||
// bundle's root.
|
||||
//
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// An own secret issued outside the mesh that nobody gave (novox/hq issue 401, ADR 0228): the mesh makes no value for it,
|
||||
// so the composition places no file, a file of the module that names it says empty there, and nothing restarts on it.
|
||||
// Given, it is placed and filled like any own secret.
|
||||
|
||||
func outsideManifest(t *testing.T) Manifest {
|
||||
t.Helper()
|
||||
m, err := ParseManifest([]byte(`{"module":"assistant","version":"1",
|
||||
"own-secrets":{
|
||||
"password":{"path":"/var/lib/assistant/password.secret","taken":"at-start"},
|
||||
"vendor-key":{"path":"/var/lib/assistant/vendor-key.secret","taken":"at-start","issued-by":"outside"}},
|
||||
"resources":[
|
||||
{"id":"env","type":"file","path":"/var/lib/assistant/env","mode":"0600",
|
||||
"content":"PASSWORD=${secret:password}\nVENDOR_KEY=${secret:vendor-key}\n"},
|
||||
{"id":"server","type":"container","name":"assistant","image":"assistant:1",
|
||||
"env-file":["/var/lib/assistant/env"],
|
||||
"volumes":["/var/lib/assistant/vendor-key.secret:/run/vendor-key:ro"],
|
||||
"secrets-in-environment":"reads its settings from the environment only"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func resourceAt(out []map[string]any, path string) map[string]any {
|
||||
for _, r := range out {
|
||||
if r["path"] == path {
|
||||
return r
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resourceNamed(out []map[string]any, name string) map[string]any {
|
||||
for _, r := range out {
|
||||
if r["name"] == name {
|
||||
return r
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestAnOutsideSecretNotGivenIsNoFileAndAFileNamingItSaysEmpty(t *testing.T) {
|
||||
r := Resolution{Node: "server", Modules: []Manifest{outsideManifest(t)}}
|
||||
out, err := r.Declaration(Rendering{Needed: map[string]map[string]string{"assistant": {"password": "sealed-password"}}})
|
||||
if err != nil {
|
||||
t.Fatalf("a module whose outside secret nobody gave did not compose: %v", err)
|
||||
}
|
||||
if got := resourceAt(out, "/var/lib/assistant/vendor-key.secret"); got != nil {
|
||||
t.Fatalf("a secret nobody gave was placed: %v", got)
|
||||
}
|
||||
env := resourceAt(out, "/var/lib/assistant/env")
|
||||
if env == nil {
|
||||
t.Fatal("the module's own file is not composed")
|
||||
}
|
||||
if content := env["content"].(string); content != "PASSWORD=${secret:password}\nVENDOR_KEY=\n" {
|
||||
t.Fatalf("the file names the secret nobody gave as %q", content)
|
||||
}
|
||||
secrets, _ := env["secrets"].(map[string]any)
|
||||
if len(secrets) != 1 || secrets["password"] != "sealed-password" {
|
||||
t.Fatalf("the file is filled from %v", secrets)
|
||||
}
|
||||
server := resourceNamed(out, "assistant")
|
||||
if on := strings.Join(stringsIn(server["restart-on"]), " "); strings.Contains(on, "vendor-key") {
|
||||
t.Fatalf("the container restarts on a secret nobody gave: %q", on)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOutsideSecretGivenIsPlacedAndFilled(t *testing.T) {
|
||||
r := Resolution{Node: "server", Modules: []Manifest{outsideManifest(t)}}
|
||||
out, err := r.Declaration(Rendering{Needed: map[string]map[string]string{"assistant": {
|
||||
"password": "sealed-password", "vendor-key": "sealed-vendor-key"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
placed := resourceAt(out, "/var/lib/assistant/vendor-key.secret")
|
||||
if placed == nil || placed["sealed"] != "sealed-vendor-key" {
|
||||
t.Fatalf("the secret given is placed as %v", placed)
|
||||
}
|
||||
env := resourceAt(out, "/var/lib/assistant/env")
|
||||
if content := env["content"].(string); content != "PASSWORD=${secret:password}\nVENDOR_KEY=${secret:vendor-key}\n" {
|
||||
t.Fatalf("the file was changed: %q", content)
|
||||
}
|
||||
if secrets, _ := env["secrets"].(map[string]any); secrets["vendor-key"] != "sealed-vendor-key" {
|
||||
t.Fatalf("the file is filled from %v", secrets)
|
||||
}
|
||||
server := resourceNamed(out, "assistant")
|
||||
if on := strings.Join(stringsIn(server["restart-on"]), " "); !strings.Contains(on, "assistant."+NeedID("vendor-key")) {
|
||||
t.Fatalf("the container does not restart on the secret given: %q", on)
|
||||
}
|
||||
}
|
||||
|
||||
// A secret the mesh makes is still refused when it was not made: only an outside one is left out.
|
||||
func TestAnOwnSecretTheMeshMakesIsStillRefusedWhenNotMade(t *testing.T) {
|
||||
r := Resolution{Node: "server", Modules: []Manifest{outsideManifest(t)}}
|
||||
_, err := r.Declaration(Rendering{})
|
||||
if nm, ok := err.(*NotMadeError); !ok || nm.Name != "password" {
|
||||
t.Fatalf("a secret the mesh makes, not made: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -156,7 +156,11 @@ func secretFilesOf(resources []map[string]any) map[string]bool {
|
||||
// A name the module never declared is refused here rather than on the machine. The host would
|
||||
// refuse it too — but it would do so having already been handed a declaration, which reads as the
|
||||
// mesh sending something broken, and the name it could not find is a manifest's typo.
|
||||
func intoFile(resource map[string]any, sealed map[string]string, module string) error {
|
||||
//
|
||||
// **An own secret issued outside the mesh that nobody gave is filled with nothing** (novox/hq issue 401): the mesh
|
||||
// never makes one (ADR 0228), and a file is where the module reads it, so the place says empty — the module sees it
|
||||
// was not given, where a value the mesh made looked given and wrong. notGiven names those secrets.
|
||||
func intoFile(resource map[string]any, sealed map[string]string, notGiven map[string]bool, module string) error {
|
||||
if fmt.Sprint(resource["type"]) != "file" {
|
||||
return nil
|
||||
}
|
||||
@@ -171,6 +175,10 @@ func intoFile(resource map[string]any, sealed map[string]string, module string)
|
||||
into := map[string]any{}
|
||||
for _, name := range used {
|
||||
value := sealed[name]
|
||||
if value == "" && notGiven[name] {
|
||||
content = strings.ReplaceAll(content, "${secret:"+name+"}", "")
|
||||
continue
|
||||
}
|
||||
if value == "" {
|
||||
return fmt.Errorf(
|
||||
"%s has a file that says ${secret:%s}, and %s has no secret of its own by that "+
|
||||
@@ -179,10 +187,25 @@ func intoFile(resource map[string]any, sealed map[string]string, module string)
|
||||
}
|
||||
into[name] = value
|
||||
}
|
||||
resource["secrets"] = into
|
||||
resource["content"] = content
|
||||
if len(into) > 0 {
|
||||
resource["secrets"] = into
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// outsideNotGiven is each own secret of a module issued outside the mesh that this composition holds no value for
|
||||
// (novox/hq issue 401): what intoFile fills with nothing, and what nothing restarts on.
|
||||
func outsideNotGiven(m Manifest, given map[string]string) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for name, own := range m.OwnSecrets.Plain() {
|
||||
if own.IssuedBy == IssuedOutside && given[name] == "" {
|
||||
out[name] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// namesOr says what a module could have written, because the answer to "that name is wrong" is
|
||||
// almost always one of two or three right ones.
|
||||
func namesOr(sealed map[string]string) string {
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// An archive pinned by URL and sha256 (novox/hq ADR 0316 §2, §6): the vendor's file, mirrored by the build
|
||||
// seat, never a copy in the repository.
|
||||
|
||||
const pinnedSum = "38dd24890a66b4cf9a45bc24e15192e00c26b929e1bb8df7974c16a4539129e2"
|
||||
|
||||
func upstreamManifest(artifact string) string {
|
||||
return `{"module":"a","version":"1","build":{"artifacts":[` + artifact + `]}}`
|
||||
}
|
||||
|
||||
func TestAnArchivePinnedByURLAndSha256IsAccepted(t *testing.T) {
|
||||
for _, a := range []string{
|
||||
`{"name":"archive","kind":"archive","url":"https://vendor.example.org/release-1.tar.xz","sha256":"` + pinnedSum +
|
||||
`","version":"1.0","within":"release"}`,
|
||||
`{"name":"addon","kind":"archive","url":"https://vendor.example.org/addon.xpi","sha256":"` + pinnedSum +
|
||||
`","version":"1.5.6.1","file":"addon.xpi"}`,
|
||||
} {
|
||||
if _, err := ParseManifest([]byte(upstreamManifest(a))); err != nil {
|
||||
t.Errorf("%s was refused: %v", a, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpstreamFileThatIsNotPinnedIsRefused(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
// No digest: what the address serves changes under the mesh.
|
||||
`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","version":"1"}`: "without a pinned sha256",
|
||||
`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"ABC","version":"1"}`: "without a pinned sha256",
|
||||
// No version: a person reading the pin cannot tell which release.
|
||||
`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum + `"}`: "says no version",
|
||||
// Not https.
|
||||
`{"name":"x","kind":"archive","url":"http://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum + `","version":"1"}`: "over https",
|
||||
// Both a directory of the repository and a URL.
|
||||
`{"name":"x","kind":"archive","from":"files","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum +
|
||||
`","version":"1"}`: "one or the other",
|
||||
// A pin without a URL.
|
||||
`{"name":"x","kind":"archive","from":"files","sha256":"` + pinnedSum + `"}`: "no url",
|
||||
// Only an archive is mirrored from a file.
|
||||
`{"name":"x","kind":"image","from":"Dockerfile","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum +
|
||||
`","version":"1"}`: "only an archive",
|
||||
// A file name that is a path, a directory within that leaves the archive, and both at once.
|
||||
`{"name":"x","kind":"archive","url":"https://vendor.invalid/a.xpi","sha256":"` + pinnedSum +
|
||||
`","version":"1","file":"../a.xpi"}`: "not a plain relative file name",
|
||||
`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum +
|
||||
`","version":"1","within":"../up"}`: "not a plain relative directory",
|
||||
`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum +
|
||||
`","version":"1","within":"r","file":"r.tar.xz"}`: "a file is kept whole",
|
||||
}
|
||||
for artifact, want := range cases {
|
||||
_, err := ParseManifest([]byte(upstreamManifest(artifact)))
|
||||
if err == nil {
|
||||
t.Errorf("%s was accepted", artifact)
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("%s was refused for another reason than %q: %v", artifact, want, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpstreamFilesVendorIsNamedOnPurposeAndAnInstallationsIsRefused(t *testing.T) {
|
||||
// The vendor's host is judged like any name, and passes through names-on-purpose; a version of four dotted
|
||||
// numbers is not an address, and a name in it is still judged.
|
||||
m, err := ParseManifest([]byte(upstreamManifest(`{"name":"owl","kind":"archive","url":"https://vendor.be/owl/owl.xpi","sha256":"` +
|
||||
pinnedSum + `","version":"1.5.6.1","file":"owl.xpi","names-on-purpose":{"vendor.be":"Owl's vendor publishes it there"}}`)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if problems := InstallationProblems(m); len(problems) != 0 {
|
||||
t.Fatalf("the declared vendor and the version were taken for an installation: %v", problems)
|
||||
}
|
||||
m.Build.Artifacts[0].NamesOnPurpose = nil
|
||||
if problems := InstallationProblems(m); len(problems) != 1 || !strings.Contains(problems[0], "vendor.be") {
|
||||
t.Fatalf("an undeclared vendor host is judged: %v", problems)
|
||||
}
|
||||
m.Build.Artifacts[0].Version = "git.novox.be 10.0.0.1"
|
||||
if problems := InstallationProblems(m); len(problems) != 2 {
|
||||
t.Fatalf("a name in the version is judged: %v", problems)
|
||||
}
|
||||
m.Build.Artifacts[0].URL, m.Build.Artifacts[0].Version = "https://mail.novox.be/owl.xpi", "1"
|
||||
if problems := InstallationProblems(m); len(problems) != 1 || !strings.Contains(problems[0], "mail.novox.be") {
|
||||
t.Fatalf("an installation's host in the url is judged: %v", problems)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpstreamFileFromAnAddressOrAReservedNameIsRefused(t *testing.T) {
|
||||
for _, u := range []string{"https://git.novox.internal/r.tar.xz", "https://10.1.2.3/r.tar.xz",
|
||||
"https://203.0.113.9/r.tar.xz", "https://[2001:db8::1]/r.tar.xz", "https://nas.lan/r.tar.xz", "https://vendor/r.tar.xz"} {
|
||||
_, err := ParseManifest([]byte(upstreamManifest(`{"name":"x","kind":"archive","url":"` + u + `","sha256":"` +
|
||||
pinnedSum + `","version":"1"}`)))
|
||||
if err == nil || !strings.Contains(err.Error(), "published under a public name") {
|
||||
t.Errorf("%s: %v", u, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -333,7 +333,9 @@ var ControllerVerbs = []Verb{
|
||||
// The build queue, controlled by hand (novox/hq ADR 0219). Every verb that drops an ask leaves a
|
||||
// failed outcome for it, so a plan waiting on it fails visibly instead of hanging.
|
||||
{Name: "queue", Description: "Every ask in the build queue: waiting, in flight (on which machine, for how long), " +
|
||||
"and dead (handed out as often as allowed and never settled) — each with its id, repository, path, ref and when it was asked.",
|
||||
"and dead (handed out as often as allowed and never settled) — each with its id, repository, path, ref and when it was asked. " +
|
||||
"Answers the table as text and, as data, each ask's state, kind (check or build), repository, commit, machine, since " +
|
||||
"and position among the waiting (novox/hq issue 455).",
|
||||
Input: schema(nil, nil)},
|
||||
{Name: "cancel", Description: "Drop one waiting or dead ask from the build queue; its outcome is recorded failed, " +
|
||||
"cancelled by hand, and a plan that asked for it fails. One in flight is refused: `kill` ends it where it runs.",
|
||||
@@ -490,6 +492,13 @@ var ControllerVerbs = []Verb{
|
||||
"why": "with consumer or older-than: why — required, and recorded in the hand-act log",
|
||||
"cause": "with consumer or older-than: the cause in a word (cleanup-waiting when absent)",
|
||||
}, nil, "confirm")},
|
||||
{Name: "judge-directories", Description: "Read-only: which of the directories one machine is declared its " +
|
||||
"node-engine's walk that follows no link another account could make would refuse (novox/hq issue 358), one line " +
|
||||
"each: placed, would be refused and why (the link and who could replace it), refused already by the placement " +
|
||||
"guard, or below a home. The engine judges the declaration it keeps and makes, chmods and chowns nothing.",
|
||||
Input: schema(map[string]string{
|
||||
"node": "the machine's name",
|
||||
}, []string{"node"})},
|
||||
// What a consumer gave up on (novox/hq issue 330): kept in DEAD_LETTERS until a person acts on it.
|
||||
{Name: "root-free", Description: "Whether each machine named is root-free now (novox/hq ADR 0259 §8): no agent " +
|
||||
"there can become root without a person. Judged when asked, never from a condition: free only when the machine " +
|
||||
|
||||
@@ -325,6 +325,8 @@ func (i *Inventory) ModuleSecretIfIssued(
|
||||
return sealed, true, nil
|
||||
}
|
||||
|
||||
// SecretForModule makes what is not held, so it is never asked for an own secret issued outside the mesh: the
|
||||
// composition reads only a value a person gave for one (GivenOwnSecret; novox/hq issue 401, ADR 0228).
|
||||
func (i *Inventory) SecretForModule(ctx context.Context, node, module, name string) (string, error) {
|
||||
key, err := i.SealingKeyOf(ctx, node)
|
||||
if err != nil {
|
||||
@@ -655,6 +657,35 @@ func (i *Inventory) GivenMembers(ctx context.Context, node, module, family strin
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// GivenOwnSecret is an own secret issued outside the mesh as a person gave it on a machine (novox/hq issue 401, ADR
|
||||
// 0228): its value sealed to the machine and whether that is the key the machine holds now. Not given is false: a
|
||||
// value the mesh made for it is not one a person gave, and is never read as one; a machine with no sealing key
|
||||
// holds none either.
|
||||
func (i *Inventory) GivenOwnSecret(ctx context.Context, node, module, name string) (GivenMember, bool, error) {
|
||||
key, err := i.SealingKeyOf(ctx, node)
|
||||
if err != nil || key == "" {
|
||||
return GivenMember{}, false, err
|
||||
}
|
||||
record, err := i.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return GivenMember{}, false, err
|
||||
}
|
||||
g := GivenMember{Name: name}
|
||||
var against string
|
||||
err = i.store.Pool().QueryRow(ctx,
|
||||
`select sealed, node_key from module_secret
|
||||
where node = $1 and module = $2 and name = $3 and origin = 'accepted'`,
|
||||
record.ID, module, name).Scan(&g.Sealed, &against)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return GivenMember{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return GivenMember{}, false, err
|
||||
}
|
||||
g.Current = against == key
|
||||
return g, true, nil
|
||||
}
|
||||
|
||||
// GivenOwnSecrets is every own secret of a module a person gave on a machine, by name, with when (novox/hq ADR
|
||||
// 0283): what the controller checks a module's wait for a secret against. A value the mesh made is not one.
|
||||
func (i *Inventory) GivenOwnSecrets(ctx context.Context, node, module string) (map[string]time.Time, error) {
|
||||
|
||||
@@ -148,6 +148,28 @@ func AskSetuidSearch(ctx context.Context, conn *nats.Conn, signer Signer, node,
|
||||
"there", record: "a start"})
|
||||
}
|
||||
|
||||
// JudgeDirectoriesContext is prefixed to a directory judgement ask's bytes before signing (novox/hq issue 358): never
|
||||
// another ask's signature, nor a declaration's. The engine holds the same words.
|
||||
const JudgeDirectoriesContext = "novox-mesh judge-directories v1\n"
|
||||
|
||||
// AskJudgeDirectories asks one machine's node-engine to judge, read-only, which of the directories it is declared
|
||||
// the walk that follows no link another account could make would refuse (novox/hq issue 358): the ask a hand-over
|
||||
// is, naming no path, signed under JudgeDirectoriesContext. The engine changes nothing and answers one line per
|
||||
// directory.
|
||||
func AskJudgeDirectories(ctx context.Context, conn *nats.Conn, signer Signer, node, by string,
|
||||
timeout time.Duration) (HandOverAnswer, error) {
|
||||
if conn == nil {
|
||||
return HandOverAnswer{}, errors.New("this controller is not on the bus")
|
||||
}
|
||||
body, err := signAsk(ctx, signer, JudgeDirectoriesContext, HandOverAsk{Node: node, By: by})
|
||||
if err != nil {
|
||||
return HandOverAnswer{}, err
|
||||
}
|
||||
return askSigned(ctx, conn, broker.AskJudgeDirectoriesSubject(node), body, node, timeout, askWords{
|
||||
what: "a judgement of its directories", nothing: "nothing was judged, and nothing is changed by one",
|
||||
issue: "issue 358", unknown: "ask again: a judgement changes nothing", record: "a judgement"})
|
||||
}
|
||||
|
||||
// SignHandOver fills the ask's expiry and nonce and signs it with the mesh's key, over HandOverContext and the
|
||||
// ask's bytes exactly as they travel.
|
||||
func SignHandOver(ctx context.Context, signer Signer, ask HandOverAsk) ([]byte, error) {
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// A judgement of a machine's directories is asked of the machine on its own subject, signed under its own context
|
||||
// (never a hand-over's), naming no path, and the engine's answer comes back (novox/hq issue 358). The engine holds the
|
||||
// same subject and context (mesh-host internal/link, TestTheDirectoryJudgementAskKeepsItsSubjectAndContext).
|
||||
func TestADirectoryJudgementIsAskedOfTheMachineSignedUnderItsOwnContext(t *testing.T) {
|
||||
if broker.AskJudgeDirectoriesSubject("laptop") != "mesh.node.laptop.ask.judge-directories" ||
|
||||
JudgeDirectoriesContext != "novox-mesh judge-directories v1\n" {
|
||||
t.Fatalf("the subject %q, the context %q", broker.AskJudgeDirectoriesSubject("laptop"), JudgeDirectoriesContext)
|
||||
}
|
||||
conn, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
signer, public := testSigner(t)
|
||||
var heard HandOverAsk
|
||||
engine, err := conn.Subscribe(broker.AskJudgeDirectoriesSubject("laptop"), func(msg *nats.Msg) {
|
||||
var signed SignedHandOver
|
||||
_ = json.Unmarshal(msg.Data, &signed)
|
||||
if ed25519.Verify(public, append([]byte(HandOverContext), signed.Ask...), signed.Signature) ||
|
||||
ed25519.Verify(public, append([]byte(SetuidSearchContext), signed.Ask...), signed.Signature) ||
|
||||
!ed25519.Verify(public, append([]byte(JudgeDirectoriesContext), signed.Ask...), signed.Signature) {
|
||||
_ = msg.Respond([]byte(`{"refused":"not signed as a judgement"}`))
|
||||
return
|
||||
}
|
||||
_ = json.Unmarshal(signed.Ask, &heard)
|
||||
body, _ := json.Marshal(HandOverAnswer{Said: "3 directories declared, asked by " + heard.By})
|
||||
_ = msg.Respond(body)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = engine.Unsubscribe() })
|
||||
a, err := AskJudgeDirectories(context.Background(), conn, signer, "laptop", "jo", 5*time.Second)
|
||||
if err != nil || a.Said != "3 directories declared, asked by jo" {
|
||||
t.Fatalf("answered %+v, %v", a, err)
|
||||
}
|
||||
if heard.Node != "laptop" || heard.Path != "" || heard.Nonce == "" || heard.Expires.IsZero() {
|
||||
t.Fatalf("the engine heard %+v", heard)
|
||||
}
|
||||
if _, err := AskJudgeDirectories(context.Background(), conn, signer, "anchor", "jo", 5*time.Second); err == nil ||
|
||||
!strings.Contains(err.Error(), "nothing was judged") {
|
||||
t.Fatalf("a machine with no engine listening: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
@@ -268,6 +269,30 @@ func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, e
|
||||
return q, nil
|
||||
}
|
||||
|
||||
// WithoutPassword is a repository as a listing may say it: a URL's password, should one ever be written
|
||||
// into it, left out (novox/hq issue 455). Only for what is shown: the ask itself keeps what it was given,
|
||||
// which a cancel's outcome and a rebuild's match are read from.
|
||||
func WithoutPassword(repository string) string {
|
||||
scheme, rest, found := strings.Cut(repository, "://")
|
||||
if !found {
|
||||
return repository
|
||||
}
|
||||
authority, path, _ := strings.Cut(rest, "/")
|
||||
at := strings.LastIndex(authority, "@")
|
||||
if at < 0 {
|
||||
return repository
|
||||
}
|
||||
user, _, password := strings.Cut(authority[:at], ":")
|
||||
if !password {
|
||||
return repository
|
||||
}
|
||||
out := scheme + "://" + user + authority[at:]
|
||||
if len(rest) > len(authority) {
|
||||
out += "/" + path
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ReadBuildEvents is every start and outcome the seat announced since a moment, from the events
|
||||
// stream, with a consumer of its own that is gone when this returns.
|
||||
func ReadBuildEvents(ctx context.Context, js *broker.JetStream, seat string, since time.Time) (BuildHeard, error) {
|
||||
|
||||
@@ -89,6 +89,7 @@
|
||||
"cleanup",
|
||||
"dead-letters",
|
||||
"root-free",
|
||||
"judge-directories",
|
||||
"data",
|
||||
"build",
|
||||
"artifacts",
|
||||
|
||||
Vendored
+2
@@ -22,3 +22,5 @@
|
||||
./cmd/mesh-controller TestAWalkEndedByCancelOrClearIsSaidEnded
|
||||
./cmd/mesh-controller TestAWalkRestartedByARebuildIsSaidStarted
|
||||
./cmd/mesh-controller TestAWalkStoppedByHandAndRetriedEndsFailedWhenItFailsAgain
|
||||
./cmd/mesh-controller TestTheSendMakesNoValueForAnOutsideSecretNobodyGave
|
||||
./cmd/mesh-controller TestAQuestionComposesWithoutAnOutsideSecretNobodyGave
|
||||
|
||||
Reference in New Issue
Block a user