Merge origin/main into feat/335, and pin mesh-host at #98's new head
This commit is contained in:
@@ -0,0 +1,91 @@
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Listing and letting go of tagged documents: the check builds the scratch namespace keeps (novox/hq ADR 0350
|
||||
// §5). A check build is found by its tag, and the build seat's holder lets each go when its pull request closes
|
||||
// or it is seven days old, so the store keeps a week of check builds and no more.
|
||||
|
||||
// Tags is every tag repository holds, in the store's order. None for a repository the store does not hold.
|
||||
func (s Store) Tags(ctx context.Context, repository string) ([]string, error) {
|
||||
if s.Address == "" {
|
||||
return nil, fmt.Errorf("this mesh has no artifact store on its network to list %s in", repository)
|
||||
}
|
||||
var tags []string
|
||||
next := "http://" + s.Address + "/v2/" + repository + "/tags/list?n=1000"
|
||||
for next != "" {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, next, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
|
||||
}
|
||||
switch response.StatusCode {
|
||||
case http.StatusOK:
|
||||
case http.StatusNotFound:
|
||||
response.Body.Close()
|
||||
return tags, nil
|
||||
default:
|
||||
response.Body.Close()
|
||||
return nil, fmt.Errorf("the artifact store answered %s for the tags of %s", response.Status, repository)
|
||||
}
|
||||
var page struct {
|
||||
Tags []string `json:"tags"`
|
||||
}
|
||||
err = json.NewDecoder(io.LimitReader(response.Body, 8<<20)).Decode(&page)
|
||||
link := response.Header.Get("Link")
|
||||
response.Body.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the tags of %s could not be read: %w", repository, err)
|
||||
}
|
||||
tags = append(tags, page.Tags...)
|
||||
next = nextPage(link, s.Address)
|
||||
}
|
||||
return tags, nil
|
||||
}
|
||||
|
||||
// nextPage is the registry's `Link: </v2/…>; rel="next"`, made whole; empty when there is no next page.
|
||||
func nextPage(link, address string) string {
|
||||
if !strings.Contains(link, `rel="next"`) {
|
||||
return ""
|
||||
}
|
||||
start, end := strings.Index(link, "<"), strings.Index(link, ">")
|
||||
if start < 0 || end <= start+1 {
|
||||
return ""
|
||||
}
|
||||
target := link[start+1 : end]
|
||||
if strings.HasPrefix(target, "/") {
|
||||
return "http://" + address + target
|
||||
}
|
||||
return target
|
||||
}
|
||||
|
||||
// DeleteTagged lets go of the document tag names in repository: its manifest, by digest, so the next
|
||||
// collection takes its layer. Gone when the tag names nothing.
|
||||
func (s Store) DeleteTagged(ctx context.Context, repository, tag string) error {
|
||||
if s.Address == "" {
|
||||
return fmt.Errorf("this mesh has no artifact store on its network to let go of %s:%s in", repository, tag)
|
||||
}
|
||||
digest, err := s.manifestDigest(ctx, repository, tag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if digest == "" {
|
||||
return Gone
|
||||
}
|
||||
err = s.remove(ctx, s.url(repository, "manifests", digest), repository+":"+tag)
|
||||
if errors.Is(err, Gone) {
|
||||
return Gone
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -82,7 +82,7 @@ func theCarriedAccounts(t *testing.T) string {
|
||||
// the commit the mesh runs, or as captured in testdata/beside (internal/beside, novox/hq issue 432).
|
||||
func theTemplate(t *testing.T) []map[string]any {
|
||||
t.Helper()
|
||||
path := filepath.Join(beside.Dir(t, "mesh-host"), "examples", "foundation-first-node-nats.lock")
|
||||
path := filepath.Join(beside.Dir(t, "mesh-host"), "examples", "foundation-first-node.lock")
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
+222
-4
@@ -91,6 +91,16 @@ type Result struct {
|
||||
// (novox/hq issue 280).
|
||||
Source string
|
||||
|
||||
// ReuseKey is the build's reuse key (novox/hq ADR 0350 §1): its source fingerprint's parts with the builder's
|
||||
// version, and ReuseParts those parts; empty, with NoKey saying why, where the source does not pin the build.
|
||||
ReuseKey string
|
||||
ReuseParts map[string]string
|
||||
NoKey string
|
||||
// Reused is the check build a walk took instead of building (ADR 0350 §2), and Why, when it built, why it did
|
||||
// not take one (§3). Both empty for a build not asked to reuse.
|
||||
Reused *Reused
|
||||
Why string
|
||||
|
||||
// Sources are what this build was made from, as files (novox/hq ADR 0267 rule 1): per repository, the
|
||||
// entries a changed file is tested against (SourceHolds). The module's own repository has an empty
|
||||
// Repository. Said only for a build of a commit on the trunk, and only for a repository whose every
|
||||
@@ -136,14 +146,44 @@ func Build(ctx context.Context, run Runner, publish Publisher,
|
||||
// one record that it is there.
|
||||
var mirrored []string
|
||||
result, err := build(ctx, run, publish, repository, path, ref, workspace, held, npmrc, forge, log,
|
||||
&mirrored, seats...)
|
||||
&mirrored, nil, seats...)
|
||||
result.Mirrored = mirrored
|
||||
return result, err
|
||||
}
|
||||
|
||||
// Reuse is what a walk's build may take instead of building (novox/hq ADR 0350 §2): the scratch namespace's check
|
||||
// builds, and the pull requests the walk answers, whose checks name a differing key.
|
||||
type Reuse struct {
|
||||
Scratch Scratch
|
||||
Pulls []PullRef
|
||||
// Named are the builds the verdicts of the walk's own pull requests name (novox/hq ADR 0350 §2): a scratch build
|
||||
// is taken only when one of them names its key and every digest. The record in the store alone is never enough.
|
||||
Named []NamedBuild
|
||||
// keyFirst is a check's build (§1): its key is taken before anything is built, and a module with none is not
|
||||
// built at all — nothing could ever take it.
|
||||
keyFirst bool
|
||||
}
|
||||
|
||||
// errNoKey is a check's build of a module whose source does not pin it: not built (ADR 0350 §1).
|
||||
var errNoKey = errors.New("it has no reuse key")
|
||||
|
||||
// BuildReusing is Build for the walk of a trunk commit (novox/hq ADR 0350 §2): before building, it takes the reuse
|
||||
// key of the commit's tree, and where a check build holds the same key it copies that build into the module's
|
||||
// namespace by digest and answers with it, building nothing. Otherwise it builds, and Result.Why says why (§3).
|
||||
// Only a commit on its trunk takes a check build: anything else builds, as Build does.
|
||||
func BuildReusing(ctx context.Context, run Runner, publish Publisher, reuse Reuse,
|
||||
repository, path, ref, workspace string, held map[string]string, npmrc Npmrc,
|
||||
forge GitCredential, log Log, seats ...map[string]string) (Result, error) {
|
||||
var mirrored []string
|
||||
result, err := build(ctx, run, publish, repository, path, ref, workspace, held, npmrc, forge, log,
|
||||
&mirrored, &reuse, seats...)
|
||||
result.Mirrored = mirrored
|
||||
return result, err
|
||||
}
|
||||
|
||||
func build(ctx context.Context, run Runner, publish Publisher,
|
||||
repository, path, ref, workspace string, held map[string]string, npmrc Npmrc,
|
||||
forge GitCredential, log Log, mirrored *[]string, seats ...map[string]string) (Result, error) {
|
||||
forge GitCredential, log Log, mirrored *[]string, reuse *Reuse, seats ...map[string]string) (Result, error) {
|
||||
// The clone base of each seat a context may name (novox/hq ADR 0155); variadic so the callers
|
||||
// that hand none — tests of everything but contexts — read as they did.
|
||||
var seatBases map[string]string
|
||||
@@ -226,6 +266,13 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
||||
say("manifest", "INVALID: %v", err)
|
||||
return Result{}, err
|
||||
}
|
||||
// What is judged only where a definition is written (novox/hq ADR 0326 §3, issue 574): a build is
|
||||
// where a changed definition is first read, so it is refused here before anything is made of it.
|
||||
if problems := catalogue.WrittenSettingsProblems(manifest); len(problems) > 0 {
|
||||
err := fmt.Errorf("%s", strings.Join(problems, "; "))
|
||||
say("manifest", "INVALID: %v", err)
|
||||
return Result{}, err
|
||||
}
|
||||
say("manifest", "%s v%s — %d artifact(s)", manifest.Module, manifest.Version, artifactCount(manifest))
|
||||
|
||||
// What it is made from, for its source fingerprint: the module's own tree first.
|
||||
@@ -244,6 +291,20 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
||||
// never resolve a mesh package — making their image non-deterministic (a needless rollout every
|
||||
// build) and leaking the credential into a build stage. Absent entirely with no registry, which
|
||||
// is the bootstrap case (novox/hq ADR 0076).
|
||||
// **A build that needs the registry and has none is refused before anything is built or
|
||||
// published** (novox/hq ADR 0336 §6) — before a base is copied into the store, and never built
|
||||
// without the credential or against the public registry.
|
||||
if !npmrc.Enabled() {
|
||||
why, err := needsRegistry(manifest, within)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if why != "" {
|
||||
err := npmrc.unbound(manifest.Module, why)
|
||||
say("packages", "REFUSED: %v", err)
|
||||
return Result{}, err
|
||||
}
|
||||
}
|
||||
var npmrcPath string
|
||||
if npmrc.Enabled() && manifest.Build != nil && wantsPackages(manifest, within) {
|
||||
content, err := npmrc.File()
|
||||
@@ -256,13 +317,16 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
||||
}
|
||||
// Only for as long as the build: a later check's container mounts this workspace (novox/hq issue 462).
|
||||
defer os.Remove(npmrcPath)
|
||||
say("packages", "resolving %s from the mesh's package registry", npmrc.Scope)
|
||||
say("packages", "resolving %s from the mesh's package registry, from %s", npmrc.Scope, fromOr(npmrc.From))
|
||||
src.notPinned("it resolves packages from the mesh's registry at build time")
|
||||
}
|
||||
|
||||
var built []catalogue.Built
|
||||
// stoodOn is every base the build was handed, as resolved — the edges the catalogue derives.
|
||||
var stoodOn []string
|
||||
// reused is the check build a walk took, and builtWhy why it built instead (novox/hq ADR 0350).
|
||||
var reused *Reused
|
||||
builtWhy := ""
|
||||
if manifest.Build != nil {
|
||||
// What this module said it stands on, answered with what this mesh actually holds. Done
|
||||
// before anything is built, so a missing base is refused in front of the person who can
|
||||
@@ -306,7 +370,35 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
||||
// Ordered, so two builds of one commit do the same work in the same sequence and their
|
||||
// logs can be compared.
|
||||
sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name })
|
||||
// **A walk takes the check's build of the same key** (novox/hq ADR 0350 §2): only a commit on its trunk, and
|
||||
// only when asked — a check, a dry run, `build --ref` and `rebuild` never are. Its key is taken by the same
|
||||
// steps a build takes, stopped before anything is built; what it takes is copied by digest.
|
||||
if reuse != nil && reuse.keyFirst {
|
||||
_, key, err := keyFor(ctx, run, publish, manifest, within, workspace, commit, credentials, artifacts, args,
|
||||
held, npmrcPath, npmrc, seatBases, src, say)
|
||||
if err == nil && key.Key == "" {
|
||||
err = fmt.Errorf("%w: %s", errNoKey, key.Why)
|
||||
}
|
||||
if err != nil {
|
||||
say("reuse", "not built for the merge: %v", err)
|
||||
return Result{Commit: commit, NoKey: err.Error()}, err
|
||||
}
|
||||
} else if reuse != nil && reuse.Scratch != nil {
|
||||
taken, keyed, why := takeACheckBuild(ctx, run, publish, *reuse, manifest, within, workspace, commit,
|
||||
credentials, artifacts, args, held, npmrcPath, npmrc, seatBases, src, onTrunk, say)
|
||||
if taken != nil {
|
||||
built = taken.Made
|
||||
src = keyed
|
||||
reused = taken
|
||||
} else {
|
||||
builtWhy = why
|
||||
say("reuse", "built: %s", why)
|
||||
}
|
||||
}
|
||||
for _, a := range artifacts {
|
||||
if reused != nil {
|
||||
break
|
||||
}
|
||||
say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a))
|
||||
made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, npmrc, seatBases, src, say)
|
||||
if err != nil {
|
||||
@@ -342,9 +434,107 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
||||
say("source", "%d path(s) of %s", len(s.Paths), where)
|
||||
}
|
||||
}
|
||||
// The reuse key (novox/hq ADR 0350 §1): what a check keeps its build under, and what a walk takes one by.
|
||||
key := keyOf(src, manifest, Version)
|
||||
return Result{Manifest: resolved, Commit: commit, Built: built,
|
||||
Against: against(within, manifest, stoodOn), Read: readBy(manifest), Source: fingerprint,
|
||||
Trunk: trunk, OnTrunk: onTrunk, Branches: branches, Sources: sources}, nil
|
||||
Trunk: trunk, OnTrunk: onTrunk, Branches: branches, Sources: sources,
|
||||
ReuseKey: key.Key, ReuseParts: key.Parts, NoKey: key.Why, Reused: reused, Why: builtWhy}, nil
|
||||
}
|
||||
|
||||
// takeACheckBuild is the walk's look for a check build of its key (novox/hq ADR 0350 §2, §3): the key taken by
|
||||
// the steps a build takes, each artifact stopped before it is built; the build kept under it copied by digest into
|
||||
// the module's namespace. Answers what it took and the source inputs its key was taken from, or why it builds. A
|
||||
// copy that fails is said, and the walk builds (§3): it never fails the walk on its own.
|
||||
func takeACheckBuild(ctx context.Context, run Runner, publish Publisher, reuse Reuse, manifest catalogue.Manifest,
|
||||
within, workspace, commit, credentials string, artifacts []catalogue.Artifact, args []string,
|
||||
held map[string]string, npmrcPath string, npmrc Npmrc, seats map[string]string, src *sourceInputs, onTrunk bool,
|
||||
say func(step, format string, args ...any)) (*Reused, *sourceInputs, string) {
|
||||
if !onTrunk {
|
||||
return nil, nil, "the commit is not on its trunk, and only a trunk commit's walk takes a check's build"
|
||||
}
|
||||
keyed, key, err := keyFor(ctx, run, publish, manifest, within, workspace, commit, credentials, artifacts, args,
|
||||
held, npmrcPath, npmrc, seats, src, say)
|
||||
if err != nil {
|
||||
return nil, nil, err.Error()
|
||||
}
|
||||
if key.Key == "" {
|
||||
return nil, nil, "it has no reuse key: " + key.Why
|
||||
}
|
||||
found, ok, err := reuse.Scratch.Find(ctx, manifest.Module, key.Key)
|
||||
if err != nil {
|
||||
return nil, nil, "the scratch namespace could not be read: " + err.Error()
|
||||
}
|
||||
if !ok {
|
||||
return nil, nil, notKeptWhy(ctx, reuse, manifest.Module, key)
|
||||
}
|
||||
// **Only a build a verdict names** (ADR 0350 §2): the store takes writes from anything on the network, so a
|
||||
// record is believed only when a check verdict of the walk's own pull requests names its key and its digests.
|
||||
if why := unnamed(reuse.Named, manifest.Module, key.Key, found); why != "" {
|
||||
return nil, nil, why
|
||||
}
|
||||
taken := &Reused{From: found}
|
||||
for _, made := range found.Made {
|
||||
copied, err := reuse.Scratch.Take(ctx, manifest.Module, made)
|
||||
if err != nil {
|
||||
why := fmt.Sprintf("copying %s's %s from %s failed (%v)", manifest.Module, made.Name, found.Said(), err)
|
||||
say("reuse", "%s; built instead", why)
|
||||
return nil, nil, why
|
||||
}
|
||||
taken.Made = append(taken.Made, copied)
|
||||
}
|
||||
// Every artifact the manifest declares, or none: a check build missing one is not this build.
|
||||
if len(taken.Made) != len(artifacts) {
|
||||
return nil, nil, fmt.Sprintf("%s holds %d artifact(s) of %s, which declares %d", found.Said(), len(taken.Made),
|
||||
manifest.Module, len(artifacts))
|
||||
}
|
||||
say("reuse", "%s", taken.Words())
|
||||
return taken, keyed, ""
|
||||
}
|
||||
|
||||
// keyFor takes a build's reuse key by the steps a build takes, each artifact stopped before anything is built or
|
||||
// published (ADR 0350 §2): its own copy of the source inputs, so the build's are untouched.
|
||||
func keyFor(ctx context.Context, run Runner, publish Publisher, manifest catalogue.Manifest,
|
||||
within, workspace, commit, credentials string, artifacts []catalogue.Artifact, args []string,
|
||||
held map[string]string, npmrcPath string, npmrc Npmrc, seats map[string]string, src *sourceInputs,
|
||||
say func(step, format string, args ...any)) (*sourceInputs, reuseKey, error) {
|
||||
keyed := src.copy()
|
||||
keyed.onlyKey = true
|
||||
for _, a := range artifacts {
|
||||
if _, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held,
|
||||
npmrcPath, npmrc, seats, keyed, say); err != nil {
|
||||
return nil, reuseKey{}, fmt.Errorf("its reuse key could not be taken: %w", err)
|
||||
}
|
||||
}
|
||||
keyed.onlyKey = false
|
||||
return keyed, keyOf(keyed, manifest, Version), nil
|
||||
}
|
||||
|
||||
// notKeptWhy is why no check build has the key (§3): no check of the walk's pull requests built the module, the
|
||||
// key differs — naming the part — or the build it made is gone.
|
||||
func notKeptWhy(ctx context.Context, reuse Reuse, module string, key reuseKey) string {
|
||||
if len(reuse.Pulls) == 0 {
|
||||
return "no check built it with this key"
|
||||
}
|
||||
builds, err := reuse.Scratch.OfPulls(ctx, reuse.Pulls)
|
||||
if err != nil {
|
||||
return "no check build has this key, and which checks built it could not be read: " + err.Error()
|
||||
}
|
||||
for _, b := range builds {
|
||||
if b.Module != module {
|
||||
continue
|
||||
}
|
||||
if b.Key == key.Key {
|
||||
return b.Said() + " built it with this key, and it was removed"
|
||||
}
|
||||
return "the key differs from " + b.Said() + "'s in " + strings.Join(differing(b.Parts, key.Parts), ", ")
|
||||
}
|
||||
var pulls []string
|
||||
for _, p := range reuse.Pulls {
|
||||
pulls = append(pulls, p.String())
|
||||
}
|
||||
return "no check of " + strings.Join(pulls, ", ") + " built it (a module its check did not move, or a check " +
|
||||
"that did not finish)"
|
||||
}
|
||||
|
||||
// branchesHolding is every branch of a fresh clone's origin the commit is on, without `origin/`.
|
||||
@@ -694,6 +884,14 @@ func against(within string, manifest catalogue.Manifest, resolved []string) []st
|
||||
return out
|
||||
}
|
||||
|
||||
// fromOr is where a registry was resolved from, as a build's log says it.
|
||||
func fromOr(from string) string {
|
||||
if from == "" {
|
||||
return "what this build was handed"
|
||||
}
|
||||
return from
|
||||
}
|
||||
|
||||
// ManifestName is the one file a module repository must have.
|
||||
//
|
||||
// At the root, and named the same in every repository. A convention somebody can look for beats a
|
||||
@@ -725,6 +923,10 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
|
||||
switch a.Kind {
|
||||
case catalogue.ArtifactUpstream:
|
||||
// Its key is the image it copies, read from the manifest (keyOf): nothing to take here.
|
||||
if src.keyOnly() {
|
||||
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
|
||||
}
|
||||
// Mirrored, not built: copied under a name of the mesh's own, so what a machine fetches is
|
||||
// pinned by a digest this registry assigned rather than by a tag somebody else can move.
|
||||
//
|
||||
@@ -853,6 +1055,10 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
} else {
|
||||
src.ownWhole()
|
||||
}
|
||||
// Taken for a reuse key (novox/hq ADR 0350 §2): everything its fingerprint reads is read; nothing is built.
|
||||
if src.keyOnly() {
|
||||
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
|
||||
}
|
||||
invocation := append([]string{"build", "-f", recipePath, "-t", local}, args...)
|
||||
if a.Target != "" {
|
||||
invocation = append(invocation, "--target", a.Target)
|
||||
@@ -920,6 +1126,9 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
src.notPinned(a.Name + " resolves packages of its own at build time")
|
||||
}
|
||||
}
|
||||
if src.keyOnly() {
|
||||
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
|
||||
}
|
||||
// The module's own packages first, where the compiler and the bundler resolve them from
|
||||
// (dependencies.go); nothing at all for a module whose package.json names only the SDK.
|
||||
if err := installOwn(ctx, run, tree, chain, base, registry, say); err != nil {
|
||||
@@ -963,6 +1172,9 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
say("package", "building and publishing %s (%s)", a.Name, a.Language)
|
||||
src.ownWhole()
|
||||
src.notPinned(a.Name + " is a package, built from what the registry holds when it is built")
|
||||
if src.keyOnly() {
|
||||
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
|
||||
}
|
||||
reference, err := publishPackage(ctx, run, module, tree, a, npmrc, say)
|
||||
if err != nil {
|
||||
return catalogue.Built{}, fmt.Errorf("%s: publishing %s failed: %w", module, a.Name, err)
|
||||
@@ -971,6 +1183,12 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil
|
||||
|
||||
case catalogue.ArtifactArchive:
|
||||
if src.keyOnly() {
|
||||
if a.URL == "" {
|
||||
src.ownHas(cleanEntry(a.From) + "/**")
|
||||
}
|
||||
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
|
||||
}
|
||||
var body []byte
|
||||
var err error
|
||||
if a.URL != "" {
|
||||
|
||||
@@ -98,6 +98,11 @@ type CheckSpec struct {
|
||||
// and composed with it by the gate as one future state. The repository's own check is not run for a
|
||||
// group: each member's pull request runs its own.
|
||||
Group []GroupHead
|
||||
// Builds are the modules the delivery plan moves, in the walk's order, and BuildWith what they are built with
|
||||
// (novox/hq ADR 0350 §1): built from the merge the check judged into the scratch namespace once it has judged,
|
||||
// for the walk to take. None for a group's composed check, whose members' own checks build.
|
||||
Builds []CheckBuildSpec
|
||||
BuildWith *CheckBuilding
|
||||
}
|
||||
|
||||
// GroupHead is one other head of a delivery group's composed check.
|
||||
@@ -155,6 +160,8 @@ type CheckVerdict struct {
|
||||
// Members are a delivery group's other heads as the composed check judged them, each with the trunk
|
||||
// commit it was merged into (ADR 0323 §6).
|
||||
Members []JudgedHead
|
||||
// Built are the builds it made for its merge (novox/hq ADR 0350 §1), each with its key, or why not.
|
||||
Built []CheckBuilt
|
||||
}
|
||||
|
||||
// Layer is one layer of a check, judged.
|
||||
@@ -572,9 +579,38 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
v.Report, v.Took = redact.redact(withWhatFailed(out.String(), v.Repo)), time.Since(began)
|
||||
say("check", "gate %s — %s; repository %s — %s (%s)", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary,
|
||||
strings.ToUpper(v.Repo.Verdict), v.Repo.Summary, v.Took.Round(time.Second))
|
||||
|
||||
// **What the merge would move, built once, here** (novox/hq ADR 0350 §1): after the judging, from the merge it
|
||||
// judged, into the scratch namespace — so the walk of the merge commit takes it rather than building again.
|
||||
if len(spec.Builds) > 0 && spec.BuildWith != nil && len(spec.Group) == 0 {
|
||||
if why := skipsTheBuilds(v); why != "" {
|
||||
say("check", "%s", why)
|
||||
} else {
|
||||
began := time.Now()
|
||||
v.Built = buildForTheMerge(ctx, run, spec, *spec.BuildWith, workspace, tree, judged.Merge, judged, forge,
|
||||
func(step, message string) {
|
||||
if log != nil {
|
||||
log(step, redact.redact(message))
|
||||
}
|
||||
}, say)
|
||||
say("check", "built for the merge: %d of %d module(s) (%s)", builtCount(v.Built), len(v.Built),
|
||||
time.Since(began).Round(time.Second))
|
||||
}
|
||||
}
|
||||
return named(v), nil
|
||||
}
|
||||
|
||||
// builtCount is how many of a check's builds for its merge were built.
|
||||
func builtCount(built []CheckBuilt) int {
|
||||
n := 0
|
||||
for _, b := range built {
|
||||
if b.NotBuilt == "" {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// ownCheck runs the repository's own check, part by part, each in its toolchain's own container: the
|
||||
// layer's verdict. Nil when the check was ended from outside (not past its bound), which the caller
|
||||
// answers as the context's error.
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// The builds a merge check makes for its merge (novox/hq ADR 0350 §1): every artifact of every module the delivery
|
||||
// plan moves, built from the merged tree the check judged into the scratch namespace, kept under its reuse key, so
|
||||
// the walk of the merge commit takes it instead of building again. Built after the check has judged, and only when
|
||||
// it did not fail: a change that fails is not merged, and nothing it built would be taken.
|
||||
|
||||
// CheckBuildSpec is one module a check builds for its merge, in the walk's order (Tier).
|
||||
type CheckBuildSpec struct {
|
||||
Module string
|
||||
// Repository, Path and Ref are where it is built from; Merged says from the merge the check made instead — the
|
||||
// module is built from the repository checked, at the branch merged into.
|
||||
Repository string
|
||||
Path string
|
||||
Ref string
|
||||
Merged bool
|
||||
Tier int
|
||||
// Skip says why it cannot be built as the walk would build it: not built, and nothing after it.
|
||||
Skip string
|
||||
}
|
||||
|
||||
// CheckBuilding is what a check builds with: where it publishes and keeps its builds, and what a build is handed.
|
||||
type CheckBuilding struct {
|
||||
Publish Publisher
|
||||
Scratch Scratch
|
||||
Held map[string]string
|
||||
Npmrc Npmrc
|
||||
Seats map[string]string
|
||||
}
|
||||
|
||||
// CheckBuilt is one module a check built for its merge, as its verdict lists it (§1): its key and its artifacts, or
|
||||
// why it was not built.
|
||||
type CheckBuilt struct {
|
||||
Module string
|
||||
Key string
|
||||
Made []catalogue.Built
|
||||
// NotBuilt says why it was not: no key, a build that failed, or a module after one of those.
|
||||
NotBuilt string
|
||||
Took time.Duration
|
||||
}
|
||||
|
||||
// skipsTheBuilds is why a check builds nothing for its merge; empty when it builds.
|
||||
func skipsTheBuilds(v CheckVerdict) string {
|
||||
for _, l := range []*Layer{v.Gate, v.Repo} {
|
||||
if l != nil && (l.Verdict == "fail" || l.Verdict == "error") {
|
||||
return "the check did not pass, so nothing is built for its merge"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// buildForTheMerge builds each module the plan moves, tier by tier, into the scratch namespace, and keeps what it
|
||||
// built under its keys. A module standing after one not built is not built either: what it stands on in the walk
|
||||
// is not what it would stand on here. tree is the check's clone and merge the merge commit it made there.
|
||||
func buildForTheMerge(ctx context.Context, run Runner, spec CheckSpec, with CheckBuilding, workspace, tree, merge string,
|
||||
judged CheckVerdict, forge GitCredential, log Log, say func(step, format string, args ...any)) []CheckBuilt {
|
||||
if len(spec.Builds) == 0 || with.Publish == nil || with.Scratch == nil {
|
||||
return nil
|
||||
}
|
||||
ref := merge
|
||||
if ref == "" {
|
||||
ref = spec.Ref
|
||||
}
|
||||
held := map[string]string{}
|
||||
for k, v := range with.Held {
|
||||
held[k] = v
|
||||
}
|
||||
var out []CheckBuilt
|
||||
var kept []CheckBuild
|
||||
stopAt, stopWhy := -1, ""
|
||||
for _, b := range spec.Builds {
|
||||
began := time.Now()
|
||||
if stopAt >= 0 && b.Tier > stopAt {
|
||||
out = append(out, CheckBuilt{Module: b.Module, NotBuilt: stopWhy})
|
||||
continue
|
||||
}
|
||||
if b.Skip != "" {
|
||||
say("build", "%s not built for the merge: %s", b.Module, b.Skip)
|
||||
out = append(out, CheckBuilt{Module: b.Module, NotBuilt: b.Skip})
|
||||
if stopAt < 0 || b.Tier < stopAt {
|
||||
stopAt, stopWhy = b.Tier, "it stands after "+b.Module+", which was not built for the merge"
|
||||
}
|
||||
continue
|
||||
}
|
||||
repository, at := b.Repository, b.Ref
|
||||
if b.Merged {
|
||||
repository, at = tree, ref
|
||||
}
|
||||
say("build", "for the merge: %s (tier %d)", b.Module, b.Tier)
|
||||
var mirrored []string
|
||||
result, err := build(ctx, run, scratchPublisher{to: with.Publish}, repository, b.Path, at,
|
||||
filepath.Join(workspace, "for-the-merge"), held, with.Npmrc, forge, log, &mirrored, &Reuse{keyFirst: true},
|
||||
with.Seats)
|
||||
built := CheckBuilt{Module: b.Module, Key: result.ReuseKey, Took: time.Since(began)}
|
||||
switch {
|
||||
case errors.Is(err, errNoKey):
|
||||
built.NotBuilt = err.Error()
|
||||
case err != nil:
|
||||
built.NotBuilt = "its build failed: " + err.Error()
|
||||
case result.ReuseKey == "":
|
||||
built.NotBuilt = "it has no reuse key: " + result.NoKey
|
||||
}
|
||||
if built.NotBuilt != "" {
|
||||
say("build", "%s not built for the merge: %s", b.Module, built.NotBuilt)
|
||||
out = append(out, built)
|
||||
if stopAt < 0 || b.Tier < stopAt {
|
||||
stopAt, stopWhy = b.Tier, "it stands after "+b.Module+", which was not built for the merge"
|
||||
}
|
||||
continue
|
||||
}
|
||||
built.Made = result.Built
|
||||
for _, m := range result.Built {
|
||||
held[b.Module+"/"+m.Name] = m.Reference
|
||||
}
|
||||
say("build", "%s built for the merge under %s (%s)", b.Module, twelve(result.ReuseKey[3:]), built.Took.Round(time.Second))
|
||||
out = append(out, built)
|
||||
if len(result.Built) > 0 {
|
||||
kept = append(kept, CheckBuild{Key: result.ReuseKey, Module: b.Module, Parts: result.ReuseParts,
|
||||
Made: result.Built, Pull: PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number},
|
||||
Head: judged.Head, Trunk: judged.Trunk, Check: spec.ID, At: time.Now().UTC()})
|
||||
}
|
||||
}
|
||||
_ = os.RemoveAll(filepath.Join(workspace, "for-the-merge"))
|
||||
if len(kept) > 0 {
|
||||
if err := with.Scratch.Keep(ctx, PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number}, spec.ID,
|
||||
time.Now().UTC(), kept); err != nil {
|
||||
// Built and not findable: the walk builds, and says no check built it. Said, never the check's verdict.
|
||||
say("build", "the builds for the merge could not be kept, and the walk will build: %v", err)
|
||||
if err := with.Scratch.Discard(ctx, kept); err != nil {
|
||||
say("build", "and what they made could not all be let go of: %v", err)
|
||||
}
|
||||
for i := range out {
|
||||
if out[i].NotBuilt == "" {
|
||||
out[i].NotBuilt = fmt.Sprintf("built, and could not be kept: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -2,9 +2,16 @@ package builder
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// Npmrc is what a build needs to resolve the mesh's own shared library — and any other package the
|
||||
@@ -37,8 +44,36 @@ type Npmrc struct {
|
||||
// username is the consumer's mesh identity. Ignored when Token is set.
|
||||
Username string
|
||||
Password string
|
||||
|
||||
// From says where the build agent resolved this registry from (novox/hq ADR 0336 §4): its
|
||||
// npm-package-registry binding, the genesis document the installer sealed to it, or the
|
||||
// environment of a builder run by a person. Empty when it resolved none.
|
||||
From string
|
||||
// Machine is the machine the build agent runs on, named when a build that needs the registry is
|
||||
// refused for having none (ADR 0336 §6).
|
||||
Machine string
|
||||
}
|
||||
|
||||
// Where a build agent resolves the mesh's package registry from (novox/hq ADR 0336 §3, §4).
|
||||
const (
|
||||
// FromBinding is the npm-package-registry binding the mesh wrote, with the credential it sealed.
|
||||
FromBinding = "the npm-package-registry binding"
|
||||
// FromGenesis is the genesis document the installer sealed at genesis's packages step, read only
|
||||
// while no binding is present.
|
||||
FromGenesis = "the genesis document"
|
||||
// FromEnvironment is the environment of a builder run by a person.
|
||||
FromEnvironment = "the environment"
|
||||
)
|
||||
|
||||
// RegistryRequirement is the build agent's optional requirement the registry is bound through, and
|
||||
// RegistryProvider the catalogue module that provides it (novox/hq ADR 0336 §1, §6): named when a
|
||||
// build that needs the registry is refused, so the outcome says what to assign. The controller's
|
||||
// catalogue test holds the provider to the catalogue's.
|
||||
const (
|
||||
RegistryRequirement = "npm-package-registry"
|
||||
RegistryProvider = "gitea"
|
||||
)
|
||||
|
||||
// Enabled reports whether there is a registry to resolve against at all. A bootstrap build that
|
||||
// runs before any package registry exists has none, and must still build whatever needs no
|
||||
// mesh-published dependency.
|
||||
@@ -90,6 +125,77 @@ func (n Npmrc) File() (string, error) {
|
||||
return fmt.Sprintf("%s:registry=%s\n%s", scope, reg, auth), nil
|
||||
}
|
||||
|
||||
// needsRegistry says why this module's build needs the mesh's package registry, or "" when it needs
|
||||
// none (novox/hq ADR 0336 §5): a `package` artifact, an image whose Dockerfile names .npmrc, or a
|
||||
// TypeScript artifact with an `@novox/` dependency. Every Go artifact, every image that does not name
|
||||
// .npmrc, and every image pulled upstream builds without it.
|
||||
func needsRegistry(manifest catalogue.Manifest, within string) (string, error) {
|
||||
if manifest.Build == nil {
|
||||
return "", nil
|
||||
}
|
||||
for _, a := range manifest.Build.Artifacts {
|
||||
switch {
|
||||
case a.Kind == catalogue.ArtifactPackage:
|
||||
return fmt.Sprintf("%s is a package, published to the registry", a.Name), nil
|
||||
case a.Kind == catalogue.ArtifactImage:
|
||||
raw, err := os.ReadFile(filepath.Join(within, a.From))
|
||||
if err == nil && strings.Contains(string(raw), ".npmrc") {
|
||||
return fmt.Sprintf("%s's Dockerfile names .npmrc", a.Name), nil
|
||||
}
|
||||
case a.Language == "typescript":
|
||||
scoped, err := scopedDependencies(within)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(scoped) > 0 {
|
||||
return fmt.Sprintf("%s is TypeScript and depends on %s", a.Name, strings.Join(scoped, ", ")), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// scopedDependencies is every `@novox/` package the module's package.json depends on, the SDK
|
||||
// included.
|
||||
func scopedDependencies(tree string) ([]string, error) {
|
||||
raw, err := os.ReadFile(filepath.Join(tree, "package.json"))
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var p struct {
|
||||
Dependencies map[string]string `json:"dependencies"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &p); err != nil {
|
||||
return nil, fmt.Errorf("the module's package.json is not JSON: %w", err)
|
||||
}
|
||||
var names []string
|
||||
for name := range p.Dependencies {
|
||||
if strings.HasPrefix(name, "@novox/") {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// unbound is the refusal of a build that needs the registry on a build agent that has none (novox/hq
|
||||
// ADR 0336 §6): it names the machine, the requirement left unbound there and the module that provides
|
||||
// it. Nothing is built, nothing is published, and the `@novox` scope is never resolved from the
|
||||
// public registry instead.
|
||||
func (n Npmrc) unbound(module, why string) error {
|
||||
machine := n.Machine
|
||||
if machine == "" {
|
||||
machine = "this machine"
|
||||
}
|
||||
return fmt.Errorf("%s needs the mesh's package registry (%s), and %s is unbound on %s: the build "+
|
||||
"agent there has neither its binding nor the genesis document. It is provided by the %s module; "+
|
||||
"once %s is assigned the requirement binds (novox/hq ADR 0336 §6)",
|
||||
module, why, RegistryRequirement, machine, RegistryProvider, RegistryProvider)
|
||||
}
|
||||
|
||||
// packageRecipe is how a `package` artifact is built and published: on a PUBLIC base image, never
|
||||
// the mesh toolchain, because the toolchain is built from the package this produces (the SDK). The
|
||||
// script builds the module, then publishes it to the mesh's package registry unless that exact
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/beside"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// **A build that needs the registry fails loudly while the build agent has none** (novox/hq ADR 0336
|
||||
// §5, §6, issue 121). On a holder with neither the npm-package-registry binding nor the genesis
|
||||
// document, a `package` artifact, an image whose Dockerfile names .npmrc and a TypeScript artifact
|
||||
// with an `@novox/` dependency each fail before anything is built or published, naming the machine,
|
||||
// the unbound requirement and the module that provides it. A Go artifact, and an image that does not
|
||||
// name .npmrc, build there as before.
|
||||
func TestABuildNeedingTheRegistryIsRefusedOnAnAgentWithoutOne(t *testing.T) {
|
||||
none := Npmrc{Machine: "workstation"}
|
||||
cases := map[string]struct {
|
||||
manifest string
|
||||
files map[string]string
|
||||
held map[string]string
|
||||
}{
|
||||
"a package": {aPackage, map[string]string{"package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`}, nil},
|
||||
"an image naming .npmrc": {withBoth, map[string]string{
|
||||
"Dockerfile": "FROM scratch\nCOPY .npmrc /root/.npmrc\n", "files/x": "y"}, nil},
|
||||
"a TypeScript bundle depending on the SDK": {aBundle, map[string]string{
|
||||
"index.ts": "console.log(1)", "package.json": `{"dependencies":{"@novox/mesh-sdk":"^0.1.0"}}`},
|
||||
map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)}},
|
||||
}
|
||||
for name, c := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
r, workspace := aRepository(t, c.manifest, c.files)
|
||||
var said []string
|
||||
_, err := Build(context.Background(), compiling{r}.run, r,
|
||||
"https://forge.invalid/m.git", "", "", workspace, c.held, none, GitCredential{},
|
||||
func(step, message string) { said = append(said, step+": "+message) })
|
||||
if err == nil {
|
||||
t.Fatal("built with no package registry")
|
||||
}
|
||||
for _, want := range []string{"workstation", "npm-package-registry", "unbound", "gitea"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
for _, line := range r.ran {
|
||||
if strings.HasPrefix(line, "docker build") || strings.HasPrefix(line, "docker run") ||
|
||||
strings.HasPrefix(line, "docker pull") {
|
||||
t.Errorf("something was built after the refusal: %s", line)
|
||||
}
|
||||
}
|
||||
if len(r.images) != 0 || len(r.archives) != 0 {
|
||||
t.Errorf("something was published after the refusal: %v %v", r.images, r.archives)
|
||||
}
|
||||
if !strings.Contains(strings.Join(said, "\n"), "REFUSED") {
|
||||
t.Errorf("the build's log does not say it was refused:\n%s", strings.Join(said, "\n"))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// A Go artifact builds on the same agent.
|
||||
files := aSharedRepository("one", "1")
|
||||
manifest := `{"module":"ctl","version":"1","build":{"artifacts":[
|
||||
{"name":"controller","kind":"bundle","language":"go","system":"arch","from":"cmd/ctl","binary":"ctl"}]},
|
||||
"resources":[{"id":"controller","type":"process","name":"ctl","artifact":"controller","run":["./ctl"]}]}`
|
||||
r := &recorded{contents: map[string]string{ManifestName: manifest}}
|
||||
for k, v := range files {
|
||||
r.contents[k] = v
|
||||
}
|
||||
held := map[string]string{"mesh-tools-go/build": "registry.invalid/mesh-tools-go/build@sha256:" + strings.Repeat("b", 64)}
|
||||
if _, err := Build(context.Background(), onTrunk{recorded: r}.run, r,
|
||||
"https://forge.invalid/ctl.git", "", "", t.TempDir(), held, none, GitCredential{}, nil); err != nil {
|
||||
t.Fatalf("a Go artifact did not build on an agent without the registry: %v", err)
|
||||
}
|
||||
// And an image that does not name .npmrc.
|
||||
r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch\nCOPY . .", "files/x": "y"})
|
||||
if _, err := Build(context.Background(), r.run, r,
|
||||
"https://forge.invalid/meshboard.git", "", "", workspace, nil, none, GitCredential{}, nil); err != nil {
|
||||
t.Fatalf("an image not naming .npmrc did not build on an agent without the registry: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **The build's log says which registry it resolved from** (novox/hq ADR 0336 §4).
|
||||
func TestABuildSaysWhichRegistryItResolvedFrom(t *testing.T) {
|
||||
for _, from := range []string{FromBinding, FromGenesis} {
|
||||
r, workspace := aRepository(t, aPackage, map[string]string{
|
||||
"package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`,
|
||||
})
|
||||
n := Npmrc{Scope: "@novox", Registry: "http://127.0.0.1:3000/api/packages/novox/npm/",
|
||||
Username: "mesh-builder", Password: "p", From: from, Machine: "anchor"}
|
||||
var said []string
|
||||
if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/mesh-sdk.git", "", "",
|
||||
workspace, nil, n, GitCredential{}, func(step, message string) { said = append(said, step+": "+message) }); err != nil {
|
||||
t.Fatalf("%s: %v", from, err)
|
||||
}
|
||||
if !strings.Contains(strings.Join(said, "\n"), "from "+from) {
|
||||
t.Errorf("the log does not say the registry came from %s:\n%s", from, strings.Join(said, "\n"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The refusal names the module that provides the registry; it is the catalogue's provider.
|
||||
func TestTheRegistryABuildIsRefusedForNamesItsProvider(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join(beside.Catalogue(t), RegistryProvider, "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
forge, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
provides := false
|
||||
for _, p := range forge.Provides {
|
||||
provides = provides || p.Name == RegistryRequirement
|
||||
}
|
||||
if !provides {
|
||||
t.Fatalf("%s does not provide %s, which a refused build names it for", RegistryProvider, RegistryRequirement)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestReplay363 is the replay of novox/hq issue 363's built part (ADR 0267 rule 6), written with only what the
|
||||
// builder had before its fix (mesh-controller #292), so mesh-lab's prover can lay it over the commit before.
|
||||
//
|
||||
// A Go module built from its repository's root — the controller, the node-engine — is compiled with the clone's
|
||||
// `.git` in view, and Go stamped the commit, its time and whether the tree was modified into the binary. On
|
||||
// 2026-10-13 the node-engine running on the laptop carried `vcs.revision=7a41e120…` in its build info, so a merge
|
||||
// that changed only its README made a new digest of an unchanged program. Before the fix the two builds below give
|
||||
// two digests; after it (`-buildvcs=false` in the Go toolchain) they give one.
|
||||
//
|
||||
// It runs the builder whole, Build to the published bundle, with a container runtime that runs the toolchain's
|
||||
// command on this machine: the module's directory mounted where the toolchain would see it, the real git and the
|
||||
// real Go compiler. Two builds of one module at two commits that differ only in README.md, outside its build source.
|
||||
func TestReplay363(t *testing.T) {
|
||||
for _, tool := range []string{"go", "git"} {
|
||||
if _, err := exec.LookPath(tool); err != nil {
|
||||
t.Skipf("%s is not on this machine: %v", tool, err)
|
||||
}
|
||||
}
|
||||
root := t.TempDir()
|
||||
origin := filepath.Join(root, "origin")
|
||||
files := map[string]string{
|
||||
"module.json": `{"module":"ctl","version":"1","build":{"artifacts":[` +
|
||||
`{"name":"runtime","kind":"bundle","language":"go","system":"arch","from":"cmd/ctl","binary":"ctl"}]},` +
|
||||
`"resources":[{"id":"runtime","type":"process","name":"ctl","artifact":"runtime","run":["./ctl"]}]}`,
|
||||
"go.mod": "module example.com/ctl\n\ngo 1.22\n",
|
||||
"cmd/ctl/main.go": "package main\n\nfunc main() { println(\"ctl\") }\n",
|
||||
"README.md": "one\n",
|
||||
}
|
||||
for name, body := range files {
|
||||
if err := os.MkdirAll(filepath.Dir(filepath.Join(origin, name)), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(origin, name), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
git := func(dir string, args ...string) string {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", append([]string{"-c", "user.name=t", "-c", "user.email=t@example.invalid",
|
||||
"-c", "commit.gpgsign=false", "-c", "init.defaultBranch=main"}, args...)...)
|
||||
cmd.Dir = dir
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
git(origin, "init", "-q")
|
||||
git(origin, "add", ".")
|
||||
git(origin, "commit", "-q", "-m", "one")
|
||||
first := git(origin, "rev-parse", "HEAD")
|
||||
if err := os.WriteFile(filepath.Join(origin, "README.md"), []byte("two\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
git(origin, "commit", "-q", "-am", "only the README")
|
||||
second := git(origin, "rev-parse", "HEAD")
|
||||
|
||||
held := map[string]string{"mesh-tools-go/build": "registry.invalid/mesh-tools-go/build@sha256:" + strings.Repeat("b", 64)}
|
||||
build := func(commit, at string) string {
|
||||
t.Helper()
|
||||
got, err := Build(context.Background(), replay363Runtime(t), replay363Store{}, origin, "", commit,
|
||||
filepath.Join(root, at), held, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("the build at %s: %v", commit[:12], err)
|
||||
}
|
||||
if len(got.Built) != 1 || got.Built[0].Digest == "" {
|
||||
t.Fatalf("the build at %s made %+v", commit[:12], got.Built)
|
||||
}
|
||||
return got.Built[0].Digest
|
||||
}
|
||||
a, b := build(first, "one"), build(second, "elsewhere/two")
|
||||
if a != b {
|
||||
t.Fatalf("one build source, two digests: %s at %s, %s at %s — a README-only merge is a new program",
|
||||
a, first[:12], b, second[:12])
|
||||
}
|
||||
}
|
||||
|
||||
// replay363Store publishes nothing: the digest is the builder's own, over the packed bundle.
|
||||
type replay363Store struct{}
|
||||
|
||||
func (replay363Store) PublishImage(_ context.Context, local, repository string) (string, error) {
|
||||
return "registry.invalid/" + repository + "@sha256:" + strings.Repeat("c", 64), nil
|
||||
}
|
||||
|
||||
func (replay363Store) PublishArchive(_ context.Context, repository string, _ []byte, digest string) (string, error) {
|
||||
return "artifact-store://" + repository + "@" + digest, nil
|
||||
}
|
||||
|
||||
// replay363Runtime runs every command on this machine, and a toolchain container's command in the directory
|
||||
// mounted for it, as the container would see it: nothing above the mounted directory.
|
||||
func replay363Runtime(t *testing.T) Runner {
|
||||
t.Helper()
|
||||
return func(ctx context.Context, dir, name string, args ...string) (string, error) {
|
||||
if name == "docker" && len(args) > 0 && args[0] == "run" {
|
||||
mounts := map[string]string{}
|
||||
workdir := ""
|
||||
i := 1
|
||||
for i < len(args) {
|
||||
switch args[i] {
|
||||
case "--rm":
|
||||
i++
|
||||
continue
|
||||
case "--volume":
|
||||
host, inside, _ := strings.Cut(args[i+1], ":")
|
||||
mounts[inside] = host
|
||||
i += 2
|
||||
continue
|
||||
case "--workdir":
|
||||
workdir = args[i+1]
|
||||
i += 2
|
||||
continue
|
||||
case "--network", "--user", "--env", "-e", "--label", "--name":
|
||||
i += 2
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
command := args[i+1:] // the image is args[i]
|
||||
where := dir
|
||||
for inside, host := range mounts {
|
||||
if workdir == inside {
|
||||
where = host
|
||||
} else if rest, ok := strings.CutPrefix(workdir, inside+"/"); ok {
|
||||
where = filepath.Join(host, rest)
|
||||
}
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, command[0], command[1:]...)
|
||||
cmd.Dir = where
|
||||
cmd.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOPROXY=off", "GOWORK=off", "GOFLAGS=")
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return string(out), &exec.ExitError{ProcessState: cmd.ProcessState, Stderr: out}
|
||||
}
|
||||
return string(out), nil
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
cmd.Dir = dir
|
||||
out, err := cmd.Output()
|
||||
return string(out), err
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **A Go bundle does not vary with the commit it was built at** (novox/hq ADR 0267 rule 6, issue 363). A
|
||||
// module built from its repository's root — the controller — is compiled with the clone's `.git` in what
|
||||
// the toolchain sees, and Go stamps the commit, its time and whether the tree was modified into every
|
||||
// binary built there unless told not to. Two builds of one program at commits that differ only in a
|
||||
// README then made two binaries, so an unchanged build source was a new digest.
|
||||
//
|
||||
// The toolchain's own compile line is run here, as the build seat runs it inside the toolchain image: in
|
||||
// two clones at two paths, of two commits that differ outside the program. One digest is wanted.
|
||||
func TestAGoBundleBuiltTwiceFromOneBuildSourceIsOneDigest(t *testing.T) {
|
||||
for _, tool := range []string{"go", "git"} {
|
||||
if _, err := exec.LookPath(tool); err != nil {
|
||||
t.Skipf("%s is not on this machine: %v", tool, err)
|
||||
}
|
||||
}
|
||||
chain, err := ToolchainFor("go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
root := t.TempDir()
|
||||
repo := filepath.Join(root, "origin")
|
||||
files := map[string]string{
|
||||
"go.mod": "module example.com/ctl\n\ngo 1.22\n",
|
||||
"cmd/ctl/main.go": "package main\n\nfunc main() { println(\"ctl\") }\n",
|
||||
"README.md": "one\n",
|
||||
}
|
||||
for name, body := range files {
|
||||
if err := os.MkdirAll(filepath.Dir(filepath.Join(repo, name)), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(repo, name), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
git := func(dir string, args ...string) string {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", append([]string{"-c", "user.name=t", "-c", "user.email=t@example.invalid",
|
||||
"-c", "commit.gpgsign=false", "-c", "init.defaultBranch=main"}, args...)...)
|
||||
cmd.Dir = dir
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
git(repo, "init", "-q")
|
||||
git(repo, "add", ".")
|
||||
git(repo, "commit", "-q", "-m", "one")
|
||||
first := git(repo, "rev-parse", "HEAD")
|
||||
if err := os.WriteFile(filepath.Join(repo, "README.md"), []byte("two\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
git(repo, "commit", "-q", "-am", "only the README")
|
||||
second := git(repo, "rev-parse", "HEAD")
|
||||
|
||||
build := func(commit, at string) string {
|
||||
t.Helper()
|
||||
clone := filepath.Join(root, at)
|
||||
git(root, "clone", "-q", repo, clone)
|
||||
git(clone, "checkout", "-q", commit)
|
||||
out := filepath.Join(clone, Out("server"), "ctl")
|
||||
line := append([]string(nil), chain.Compile...)
|
||||
line = append(line, "-ldflags", strings.Join(chain.LinkerFlags, " "), chain.OutputFlag, out, "./cmd/ctl")
|
||||
cmd := exec.Command(line[0], line[1:]...)
|
||||
cmd.Dir = clone
|
||||
cmd.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOPROXY=off", "GOWORK=off")
|
||||
if b, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("%s: %v\n%s", strings.Join(line, " "), err, b)
|
||||
}
|
||||
body, err := os.ReadFile(out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if bytes.Contains(body, []byte(commit)) {
|
||||
t.Errorf("the binary built at %s carries the commit it was built at", commit[:12])
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
a, b := build(first, "a"), build(second, "elsewhere/b")
|
||||
if a != b {
|
||||
t.Fatalf("one build source, two digests: %s at %s, %s at %s (compiled with %q)",
|
||||
a[:12], first[:12], b[:12], second[:12], strings.Join(chain.Compile, " "))
|
||||
}
|
||||
}
|
||||
|
||||
// The toolchain says so itself, so a reader of the compile line sees it without running a build.
|
||||
func TestTheGoToolchainStampsNoVersionControl(t *testing.T) {
|
||||
chain, err := ToolchainFor("go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if joined := strings.Join(chain.Compile, " "); !strings.Contains(joined, "-buildvcs=false") {
|
||||
t.Fatalf("the go toolchain stamps the commit into what it compiles: %q", joined)
|
||||
}
|
||||
}
|
||||
|
||||
// **And so does every recipe in this repository that compiles Go** (review of issue 363's fix): the
|
||||
// controller's genesis image copies the whole clone, `.git` with it, and the reference images beside it
|
||||
// are built the same way by the Makefile.
|
||||
func TestEveryGoRecipeInThisRepositoryStampsNoVersionControl(t *testing.T) {
|
||||
root := filepath.Join("..", "..")
|
||||
var recipes []string
|
||||
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() && (d.Name() == "vendor" || d.Name() == "testdata" || d.Name() == ".git") {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if !d.IsDir() && d.Name() == "Dockerfile" {
|
||||
recipes = append(recipes, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(recipes) == 0 {
|
||||
t.Fatal("no recipe found in this repository")
|
||||
}
|
||||
for _, path := range recipes {
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, line := range strings.Split(string(body), "\n") {
|
||||
if strings.Contains(line, "go build") && !strings.Contains(line, "-buildvcs=false") {
|
||||
t.Errorf("%s compiles Go and stamps the commit into it: %s", path, strings.TrimSpace(line))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,608 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// One build per change (novox/hq ADR 0350).
|
||||
//
|
||||
// **The merge check builds what the merge would move** (§1): every artifact of every module the delivery plan
|
||||
// moves, from the merged tree it checks, into the artifact store's scratch namespace — never a module's own —
|
||||
// kept under a **reuse key**: the source fingerprint as the build takes it (the module's tree, its contexts' trees,
|
||||
// its bases and toolchains by digest; issue 280) with the builder's own version and, for an upstream artifact, the
|
||||
// image it copies. **The walk of a trunk commit takes that build** (§2): before building, it takes the reuse key
|
||||
// of the trunk commit's tree the same way, and where a check build holds the same key it copies it by digest into
|
||||
// the module's namespace and answers with that, building nothing. Otherwise it builds and says why (§3): no check
|
||||
// built it, the key differs (naming the part), or the check build is gone; a copy that fails is said and built.
|
||||
//
|
||||
// **What the key names is what a fresh build could differ by.** The tree is every file the recipe and the
|
||||
// compiler see; the contexts are the other repositories it reads, by tree; the bases and toolchains are what it
|
||||
// stands on, by digest; the builder's version is how it is built. A build whose source does not pin it — one that
|
||||
// resolves packages from the mesh's registry when it is built, or copies an upstream image named by tag — has no
|
||||
// key, and is always built.
|
||||
|
||||
// Version is this builder's own version, as the reuse key takes it: the digest of the running executable, so any
|
||||
// change to how it builds is a change of key (§1). Set once by the build seat's holder (SelfVersion); empty — no
|
||||
// key, every build its own — when it cannot be said.
|
||||
var Version string
|
||||
|
||||
// SelfVersion is the version the reuse key takes: the running executable's digest, or the version stamped at build
|
||||
// when the executable cannot be read. Empty when neither says anything ("development" says nothing: every unstamped
|
||||
// builder would read as one).
|
||||
func SelfVersion(stamped string) string {
|
||||
if exe, err := os.Executable(); err == nil {
|
||||
if f, err := os.Open(exe); err == nil {
|
||||
defer f.Close()
|
||||
sum := sha256.New()
|
||||
if _, err := io.Copy(sum, f); err == nil {
|
||||
return "exe:sha256:" + hex.EncodeToString(sum.Sum(nil))
|
||||
}
|
||||
}
|
||||
}
|
||||
if stamped == "" || stamped == "development" {
|
||||
return ""
|
||||
}
|
||||
return stamped
|
||||
}
|
||||
|
||||
// ScratchPrefix is the namespace a check builds into (novox/hq ADR 0239 §9, ADR 0350 §1): `scratch/<module>/<artifact>`.
|
||||
// Registration refuses a reference into it; only the walk's copy, in the module's own namespace, is registered.
|
||||
const ScratchPrefix = "scratch/"
|
||||
|
||||
// checksRepository holds one document per check that built: which builds it made, of which pull request, when.
|
||||
// What the holder lets go of by (§5).
|
||||
const checksRepository = ScratchPrefix + "checks"
|
||||
|
||||
// reuseTagPrefix names a check build's record under its module's scratch repository: `r1-<hex>`.
|
||||
const reuseTagPrefix = "r1-"
|
||||
|
||||
// CheckBuildsKept is how long a check build is kept in the scratch namespace (§5).
|
||||
const CheckBuildsKept = 7 * 24 * time.Hour
|
||||
|
||||
// The parts of a reuse key, as the walk names the one that differs (§3).
|
||||
const (
|
||||
partModule = "the module"
|
||||
partTree = "the build source"
|
||||
partBases = "a base image"
|
||||
partBuilder = "the builder's version"
|
||||
partContext = "a context"
|
||||
partTool = "a toolchain"
|
||||
partUpstream = "an upstream image"
|
||||
)
|
||||
|
||||
// reuseKey is a build's reuse key and the parts it was made from; Key empty, with why, when it has none.
|
||||
type reuseKey struct {
|
||||
Key string
|
||||
Parts map[string]string
|
||||
Why string
|
||||
}
|
||||
|
||||
// keyOf is the reuse key of what src says the build was made from, built by version (ADR 0350 §1).
|
||||
func keyOf(src *sourceInputs, manifest catalogue.Manifest, version string) reuseKey {
|
||||
if src == nil || src.unpinned != "" || src.tree == "" {
|
||||
why := "its source does not pin the build"
|
||||
if src != nil && src.unpinned != "" {
|
||||
why = src.unpinned
|
||||
} else if src != nil && src.tree == "" {
|
||||
why = "its tree could not be named"
|
||||
}
|
||||
return reuseKey{Why: why}
|
||||
}
|
||||
if version == "" {
|
||||
return reuseKey{Why: "the builder's version cannot be said"}
|
||||
}
|
||||
parts := map[string]string{partModule: src.module, partTree: src.tree, partBuilder: version}
|
||||
var bases []string
|
||||
seen := map[string]bool{}
|
||||
for _, b := range src.bases {
|
||||
if d := referenceDigest(b); !seen[d] {
|
||||
seen[d] = true
|
||||
bases = append(bases, d)
|
||||
}
|
||||
}
|
||||
sort.Strings(bases)
|
||||
parts[partBases] = strings.Join(bases, " ")
|
||||
for a, t := range src.contexts {
|
||||
parts[partContext+" "+a] = t
|
||||
}
|
||||
for a, t := range src.toolchains {
|
||||
parts[partTool+" "+a] = t
|
||||
}
|
||||
// An upstream artifact is copied, not built: what it copies is in module.json, and so in the tree — but a tag
|
||||
// names whatever its registry says today, so only an image pinned by digest is the same image tomorrow.
|
||||
if manifest.Build != nil {
|
||||
for _, a := range manifest.Build.Artifacts {
|
||||
if a.Kind != catalogue.ArtifactUpstream {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(a.From, "@sha256:") {
|
||||
return reuseKey{Why: fmt.Sprintf("%s copies %s, named by tag, which can move", a.Name, a.From)}
|
||||
}
|
||||
parts[partUpstream+" "+a.Name] = a.From
|
||||
}
|
||||
}
|
||||
lines := make([]string, 0, len(parts))
|
||||
for k, v := range parts {
|
||||
lines = append(lines, k+"="+v)
|
||||
}
|
||||
sort.Strings(lines)
|
||||
sum := sha256.Sum256([]byte(strings.Join(lines, "\n")))
|
||||
return reuseKey{Key: "r1:" + hex.EncodeToString(sum[:]), Parts: parts}
|
||||
}
|
||||
|
||||
// tagOf is the tag a key's record is kept under.
|
||||
func tagOf(key string) string { return reuseTagPrefix + strings.TrimPrefix(key, "r1:") }
|
||||
|
||||
// differing names the parts of two keys that differ, in a fixed order: what the walk says when it builds (§3).
|
||||
func differing(was, now map[string]string) []string {
|
||||
names := map[string]bool{}
|
||||
for k := range was {
|
||||
names[k] = true
|
||||
}
|
||||
for k := range now {
|
||||
names[k] = true
|
||||
}
|
||||
var out []string
|
||||
for k := range names {
|
||||
if was[k] != now[k] {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// PullRef names a pull request.
|
||||
type PullRef struct {
|
||||
Owner string `json:"owner"`
|
||||
Repo string `json:"repo"`
|
||||
Number int `json:"number"`
|
||||
}
|
||||
|
||||
func (p PullRef) String() string { return fmt.Sprintf("%s/%s#%d", p.Owner, p.Repo, p.Number) }
|
||||
|
||||
// slug is the pull request as a tag's first part: owner.repo.number, every other character a dash.
|
||||
func (p PullRef) slug() string {
|
||||
clean := func(s string) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_' || r == '-' {
|
||||
return r
|
||||
}
|
||||
return '-'
|
||||
}, s)
|
||||
}
|
||||
return clean(p.Owner) + "." + clean(p.Repo) + "." + strconv.Itoa(p.Number)
|
||||
}
|
||||
|
||||
// CheckBuild is one module a check built into the scratch namespace, kept under its reuse key (§1).
|
||||
type CheckBuild struct {
|
||||
Key string `json:"key"`
|
||||
Module string `json:"module"`
|
||||
Parts map[string]string `json:"parts"`
|
||||
// Made are its artifacts, in the scratch namespace.
|
||||
Made []catalogue.Built `json:"made"`
|
||||
// Pull is the pull request checked, Head its head and Trunk the trunk commit it was merged into; Check the
|
||||
// check's ask; At when it was built.
|
||||
Pull PullRef `json:"pull"`
|
||||
Head string `json:"head,omitempty"`
|
||||
Trunk string `json:"trunk,omitempty"`
|
||||
Check string `json:"check"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// Said is the check build as the walk names it: "the check of owner/repo#n at head H (trunk T)".
|
||||
func (c CheckBuild) Said() string {
|
||||
s := "the check of " + c.Pull.String()
|
||||
if c.Head != "" {
|
||||
s += " at head " + twelve(c.Head)
|
||||
}
|
||||
if c.Trunk != "" {
|
||||
s += " (merged into trunk " + twelve(c.Trunk) + ")"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// checkIndex is one check's document under checksRepository: every build it made.
|
||||
type checkIndex struct {
|
||||
Pull PullRef `json:"pull"`
|
||||
Check string `json:"check"`
|
||||
At time.Time `json:"at"`
|
||||
Builds []CheckBuild `json:"builds"`
|
||||
}
|
||||
|
||||
// Scratch is where check builds are kept and taken from: the artifact store's scratch namespace (ADR 0350). The
|
||||
// Registry is one; a test's is a map.
|
||||
type Scratch interface {
|
||||
// Keep records a check's builds: each under its module and key, and the check's own document.
|
||||
Keep(ctx context.Context, pull PullRef, check string, at time.Time, builds []CheckBuild) error
|
||||
// Find is the check build kept under a module's key; false when none is.
|
||||
Find(ctx context.Context, module, key string) (CheckBuild, bool, error)
|
||||
// OfPulls are the check builds kept for those pull requests, newest first: what the walk names a differing
|
||||
// key from.
|
||||
OfPulls(ctx context.Context, pulls []PullRef) ([]CheckBuild, error)
|
||||
// Take copies one artifact of a check build by digest into the module's own namespace, and answers it there.
|
||||
Take(ctx context.Context, module string, made catalogue.Built) (catalogue.Built, error)
|
||||
// Forget lets go of every check build of a pull request, and Expire of every check older than the bound;
|
||||
// each answers how many checks it let go of.
|
||||
Forget(ctx context.Context, pull PullRef) (int, error)
|
||||
Expire(ctx context.Context, older time.Duration, now time.Time) (int, error)
|
||||
// Discard lets go of what builds a check could not keep made: nothing would ever find them.
|
||||
Discard(ctx context.Context, builds []CheckBuild) error
|
||||
}
|
||||
|
||||
// scratchPublisher is a Publisher into the scratch namespace: what a check builds is never under a module's
|
||||
// own name (ADR 0239 §9). A base the build stands on is copied where every build copies it.
|
||||
type scratchPublisher struct{ to Publisher }
|
||||
|
||||
func (s scratchPublisher) PublishImage(ctx context.Context, localTag, repository string) (string, error) {
|
||||
return s.to.PublishImage(ctx, localTag, ScratchPrefix+repository)
|
||||
}
|
||||
|
||||
func (s scratchPublisher) PublishArchive(ctx context.Context, repository string, body []byte, digest string) (string, error) {
|
||||
return s.to.PublishArchive(ctx, ScratchPrefix+repository, body, digest)
|
||||
}
|
||||
|
||||
func (s scratchPublisher) MirrorImage(ctx context.Context, from, repository string) (string, error) {
|
||||
m, can := s.to.(Mirrorer)
|
||||
if !can {
|
||||
return "", errors.New("the artifact store here cannot copy an upstream image")
|
||||
}
|
||||
return m.MirrorImage(ctx, from, ScratchPrefix+repository)
|
||||
}
|
||||
|
||||
func (s scratchPublisher) MirrorBase(ctx context.Context, from, repository, former string) (string, error) {
|
||||
if m, can := s.to.(BaseMirrorer); can {
|
||||
return m.MirrorBase(ctx, from, repository, former)
|
||||
}
|
||||
if m, can := s.to.(Mirrorer); can {
|
||||
return m.MirrorImage(ctx, from, repository)
|
||||
}
|
||||
return "", errors.New("the artifact store here cannot copy a base")
|
||||
}
|
||||
|
||||
// The registry as the scratch namespace.
|
||||
|
||||
func (r Registry) store() artifacts.Store {
|
||||
return artifacts.Store{Address: r.Address, HTTP: r.client()}
|
||||
}
|
||||
|
||||
const recordMedia = "application/vnd.novox.mesh.check-build.v1+json"
|
||||
|
||||
// Keep puts each build's record under its module's scratch repository, by key, and the check's document.
|
||||
func (r Registry) Keep(ctx context.Context, pull PullRef, check string, at time.Time, builds []CheckBuild) error {
|
||||
store := r.store()
|
||||
for _, b := range builds {
|
||||
body, err := json.Marshal(b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := store.PutTagged(ctx, ScratchPrefix+b.Module, tagOf(b.Key), recordMedia, body); err != nil {
|
||||
return fmt.Errorf("keeping %s's check build: %w", b.Module, err)
|
||||
}
|
||||
}
|
||||
body, err := json.Marshal(checkIndex{Pull: pull, Check: check, At: at, Builds: builds})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := store.PutTagged(ctx, checksRepository, pull.slug()+"."+check, recordMedia, body); err != nil {
|
||||
return fmt.Errorf("keeping which builds the check of %s made: %w", pull, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Find reads the record kept under a module's key.
|
||||
func (r Registry) Find(ctx context.Context, module, key string) (CheckBuild, bool, error) {
|
||||
body, _, err := r.store().GetTagged(ctx, ScratchPrefix+module, tagOf(key))
|
||||
if errors.Is(err, artifacts.ErrNoTag) {
|
||||
return CheckBuild{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return CheckBuild{}, false, err
|
||||
}
|
||||
var b CheckBuild
|
||||
if err := json.Unmarshal(body, &b); err != nil {
|
||||
return CheckBuild{}, false, fmt.Errorf("%s's check build under %s is not one: %w", module, tagOf(key), err)
|
||||
}
|
||||
if b.Key != key || b.Module != module {
|
||||
return CheckBuild{}, false, nil
|
||||
}
|
||||
return b, true, nil
|
||||
}
|
||||
|
||||
// checks are the check documents kept, by tag; only those of the pull requests named when any are.
|
||||
func (r Registry) checks(ctx context.Context, pulls []PullRef) (map[string]checkIndex, error) {
|
||||
store := r.store()
|
||||
tags, err := store.Tags(ctx, checksRepository)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
wanted := map[string]PullRef{}
|
||||
for _, p := range pulls {
|
||||
wanted[p.slug()+"."] = p
|
||||
}
|
||||
out := map[string]checkIndex{}
|
||||
for _, tag := range tags {
|
||||
if len(pulls) > 0 {
|
||||
match := false
|
||||
for prefix := range wanted {
|
||||
match = match || strings.HasPrefix(tag, prefix)
|
||||
}
|
||||
if !match {
|
||||
continue
|
||||
}
|
||||
}
|
||||
body, _, err := store.GetTagged(ctx, checksRepository, tag)
|
||||
if errors.Is(err, artifacts.ErrNoTag) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var c checkIndex
|
||||
if err := json.Unmarshal(body, &c); err != nil {
|
||||
continue // not a document this wrote: left to nobody, said by nothing — it names no build
|
||||
}
|
||||
if len(pulls) > 0 {
|
||||
if p := wanted[c.Pull.slug()+"."]; p != c.Pull {
|
||||
continue // a slug two pull requests share: only the one named
|
||||
}
|
||||
}
|
||||
out[tag] = c
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// OfPulls are the check builds of those pull requests, newest first.
|
||||
func (r Registry) OfPulls(ctx context.Context, pulls []PullRef) ([]CheckBuild, error) {
|
||||
if len(pulls) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
checks, err := r.checks(ctx, pulls)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []CheckBuild
|
||||
for _, c := range checks {
|
||||
out = append(out, c.Builds...)
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].At.After(out[j].At) })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Take copies one artifact of a check build into the module's namespace by digest: an image's manifests as they
|
||||
// are, so the digest registered is the digest the check built; an archive's blob mounted across, and held.
|
||||
func (r Registry) Take(ctx context.Context, module string, made catalogue.Built) (catalogue.Built, error) {
|
||||
repository := module + "/" + made.Name
|
||||
from := ScratchPrefix + repository
|
||||
if made.Digest != "" || strings.Contains(made.Reference, "/blobs/") {
|
||||
digest := made.Digest
|
||||
if digest == "" {
|
||||
_, digest, _ = strings.Cut(made.Reference, "/blobs/")
|
||||
}
|
||||
where := upstream{base: "http://" + r.Address, repository: from, reference: digest}
|
||||
if err := r.copyBlob(ctx, &source{client: r.client(), mountFrom: from}, where, digest, repository); err != nil {
|
||||
return catalogue.Built{}, err
|
||||
}
|
||||
size, err := r.blobSize(ctx, repository, digest)
|
||||
if err != nil {
|
||||
return catalogue.Built{}, err
|
||||
}
|
||||
if err := r.hold(ctx, repository, digest, int(size)); err != nil {
|
||||
return catalogue.Built{}, err
|
||||
}
|
||||
return catalogue.Built{Name: made.Name, Kind: made.Kind, Digest: digest, Launchers: made.Launchers,
|
||||
Reference: "http://" + r.Address + "/v2/" + repository + "/blobs/" + digest}, nil
|
||||
}
|
||||
_, digest, pinned := strings.Cut(made.Reference, "@")
|
||||
if !pinned || !strings.HasPrefix(digest, "sha256:") {
|
||||
return catalogue.Built{}, fmt.Errorf("%s's check build %s is not pinned by digest", module, made.Reference)
|
||||
}
|
||||
reference, err := r.MirrorBase(ctx, r.Address+"/"+from+"@"+digest, repository, from)
|
||||
if err != nil {
|
||||
return catalogue.Built{}, err
|
||||
}
|
||||
if !strings.HasSuffix(reference, "@"+digest) {
|
||||
return catalogue.Built{}, fmt.Errorf("%s's check build was copied as %s, not as %s", module, reference, digest)
|
||||
}
|
||||
return catalogue.Built{Name: made.Name, Kind: made.Kind, Reference: reference, Launchers: made.Launchers}, nil
|
||||
}
|
||||
|
||||
// blobSize is the size the store says a blob has.
|
||||
func (r Registry) blobSize(ctx context.Context, repository, digest string) (int64, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodHead, "http://"+r.Address+"/v2/"+repository+"/blobs/"+digest, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
response, err := r.client().Do(request)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK || response.ContentLength < 0 {
|
||||
return 0, fmt.Errorf("%s/blobs/%s: %s", repository, digest, response.Status)
|
||||
}
|
||||
return response.ContentLength, nil
|
||||
}
|
||||
|
||||
// Forget lets go of every check build of a pull request (§5): what each check built in the scratch namespace, the
|
||||
// record under its key when it is still that check's, and the check's document.
|
||||
func (r Registry) Forget(ctx context.Context, pull PullRef) (int, error) {
|
||||
all, err := r.checks(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
mine := map[string]checkIndex{}
|
||||
for tag, c := range all {
|
||||
if c.Pull == pull {
|
||||
mine[tag] = c
|
||||
}
|
||||
}
|
||||
return r.letGo(ctx, mine, all)
|
||||
}
|
||||
|
||||
// Expire lets go of every check older than the bound (§5).
|
||||
func (r Registry) Expire(ctx context.Context, older time.Duration, now time.Time) (int, error) {
|
||||
checks, err := r.checks(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
old := map[string]checkIndex{}
|
||||
for tag, c := range checks {
|
||||
if now.Sub(c.At) > older {
|
||||
old[tag] = c
|
||||
}
|
||||
}
|
||||
return r.letGo(ctx, old, checks)
|
||||
}
|
||||
|
||||
// Discard lets go of builds no check document names.
|
||||
func (r Registry) Discard(ctx context.Context, builds []CheckBuild) error {
|
||||
all, err := r.checks(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = r.letGo(ctx, map[string]checkIndex{"": {Builds: builds}}, all)
|
||||
return err
|
||||
}
|
||||
|
||||
// letGo lets go of each check's builds, then its document; a check whose builds could not all be let go of keeps
|
||||
// its document, so the next sweep asks again. **A build another check still names stays**: deletion is by digest,
|
||||
// and two checks that built one digest in one repository — a check asked again, a dependent two pull requests move —
|
||||
// share it; it goes with the last check that names it. all are every check kept, the ones going among them.
|
||||
func (r Registry) letGo(ctx context.Context, checks, all map[string]checkIndex) (int, error) {
|
||||
store := r.store()
|
||||
stays := map[string]bool{}
|
||||
for tag, c := range all {
|
||||
if _, going := checks[tag]; going {
|
||||
continue
|
||||
}
|
||||
for _, b := range c.Builds {
|
||||
for _, m := range b.Made {
|
||||
stays[catalogue.Recorded(m.Reference)] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
gone := 0
|
||||
var failed []string
|
||||
for tag, c := range checks {
|
||||
ok := true
|
||||
for _, b := range c.Builds {
|
||||
for _, m := range b.Made {
|
||||
reference := catalogue.Recorded(m.Reference)
|
||||
if path, kept := catalogue.InArtifactStore(reference); !kept || !strings.HasPrefix(path, ScratchPrefix) {
|
||||
continue // never anything outside the scratch namespace
|
||||
}
|
||||
if stays[reference] {
|
||||
continue // another check still names it
|
||||
}
|
||||
if err := store.LetGo(ctx, reference); err != nil && !errors.Is(err, artifacts.Gone) {
|
||||
ok = false
|
||||
failed = append(failed, fmt.Sprintf("%s: %v", m.Reference, err))
|
||||
}
|
||||
}
|
||||
// The record under the key, only while it is still this check's: a later check of the same key
|
||||
// replaced it, and that one is its own check's to let go of.
|
||||
if kept, found, err := r.Find(ctx, b.Module, b.Key); err == nil && found && kept.Check == c.Check {
|
||||
if err := store.DeleteTagged(ctx, ScratchPrefix+b.Module, tagOf(b.Key)); err != nil && !errors.Is(err, artifacts.Gone) {
|
||||
ok = false
|
||||
failed = append(failed, fmt.Sprintf("%s's record: %v", b.Module, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if tag == "" {
|
||||
gone++
|
||||
continue
|
||||
}
|
||||
if err := store.DeleteTagged(ctx, checksRepository, tag); err != nil && !errors.Is(err, artifacts.Gone) {
|
||||
failed = append(failed, fmt.Sprintf("%s: %v", tag, err))
|
||||
continue
|
||||
}
|
||||
gone++
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
sort.Strings(failed)
|
||||
return gone, fmt.Errorf("not let go of: %s", strings.Join(failed, "; "))
|
||||
}
|
||||
return gone, nil
|
||||
}
|
||||
|
||||
// Reused is what the walk took from a check instead of building (§2, §4).
|
||||
type Reused struct {
|
||||
From CheckBuild
|
||||
// Made are the copies in the module's namespace: what is registered.
|
||||
Made []catalogue.Built
|
||||
}
|
||||
|
||||
// Words is the walk's word for a module it took: "reused from <check build>", with each digest (§4).
|
||||
func (r Reused) Words() string {
|
||||
var digests []string
|
||||
for _, m := range r.Made {
|
||||
d := m.Digest
|
||||
if d == "" {
|
||||
_, d, _ = strings.Cut(m.Reference, "@")
|
||||
}
|
||||
digests = append(digests, m.Name+" "+d)
|
||||
}
|
||||
return "reused from " + r.From.Said() + ": " + strings.Join(digests, ", ")
|
||||
}
|
||||
|
||||
// NamedBuild is a build a check's verdict names (ADR 0350 §2): its module, key and each artifact's digest.
|
||||
type NamedBuild struct {
|
||||
Pull PullRef
|
||||
Check string
|
||||
Module string
|
||||
Key string
|
||||
Digests map[string]string
|
||||
}
|
||||
|
||||
// DigestOf is the digest a reference names: an image's after `@`, an archive's after `/blobs/`.
|
||||
func DigestOf(reference string) string {
|
||||
if _, d, ok := strings.Cut(reference, "/blobs/"); ok {
|
||||
return d
|
||||
}
|
||||
if _, d, ok := strings.Cut(reference, "@"); ok {
|
||||
return d
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// unnamed is why a record found under a key is not taken: no verdict of the walk's pull requests names that module,
|
||||
// key and every one of its digests. Empty when one does.
|
||||
func unnamed(named []NamedBuild, module, key string, found CheckBuild) string {
|
||||
for _, n := range named {
|
||||
if n.Module != module || n.Key != key || n.Check != found.Check || len(n.Digests) != len(found.Made) {
|
||||
continue
|
||||
}
|
||||
same := true
|
||||
for _, m := range found.Made {
|
||||
d := m.Digest
|
||||
if d == "" {
|
||||
d = DigestOf(m.Reference)
|
||||
}
|
||||
same = same && d != "" && n.Digests[m.Name] == d
|
||||
}
|
||||
if same {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return "a build is kept under this key, and no check verdict of the walk's pull requests names it with its " +
|
||||
"digests: not taken"
|
||||
}
|
||||
@@ -0,0 +1,627 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// One build per change (novox/hq ADR 0350): the merge check builds what the merge would move into the scratch
|
||||
// namespace under its reuse key, and the walk of the trunk commit takes that build by digest instead of building.
|
||||
|
||||
// aScratch is the scratch namespace in memory: check builds by module and key, and what the walk took.
|
||||
type aScratch struct {
|
||||
builds map[string]CheckBuild
|
||||
took []string
|
||||
failTake bool
|
||||
}
|
||||
|
||||
func newScratch() *aScratch { return &aScratch{builds: map[string]CheckBuild{}} }
|
||||
|
||||
func (s *aScratch) Keep(_ context.Context, _ PullRef, _ string, _ time.Time, builds []CheckBuild) error {
|
||||
for _, b := range builds {
|
||||
s.builds[b.Module+" "+b.Key] = b
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *aScratch) Find(_ context.Context, module, key string) (CheckBuild, bool, error) {
|
||||
b, ok := s.builds[module+" "+key]
|
||||
return b, ok, nil
|
||||
}
|
||||
|
||||
func (s *aScratch) OfPulls(_ context.Context, pulls []PullRef) ([]CheckBuild, error) {
|
||||
var out []CheckBuild
|
||||
for _, b := range s.builds {
|
||||
if slices.Contains(pulls, b.Pull) {
|
||||
out = append(out, b)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *aScratch) Take(_ context.Context, module string, made catalogue.Built) (catalogue.Built, error) {
|
||||
if s.failTake {
|
||||
return catalogue.Built{}, errors.New("the store refused the mount")
|
||||
}
|
||||
s.took = append(s.took, made.Reference)
|
||||
made.Reference = strings.Replace(made.Reference, ScratchPrefix, "", 1)
|
||||
return made, nil
|
||||
}
|
||||
|
||||
func (s *aScratch) Forget(context.Context, PullRef) (int, error) { return 0, nil }
|
||||
func (s *aScratch) Expire(context.Context, time.Duration, time.Time) (int, error) { return 0, nil }
|
||||
func (s *aScratch) Discard(context.Context, []CheckBuild) error { return nil }
|
||||
|
||||
const reusedModule = `{"module":"meshboard","version":"1",
|
||||
"build":{"artifacts":[
|
||||
{"name":"server","kind":"image","from":"Dockerfile"},
|
||||
{"name":"look","kind":"archive","from":"files"}]},
|
||||
"resources":[
|
||||
{"id":"svc","type":"container","name":"meshboard","artifact":"server"},
|
||||
{"id":"theme","type":"archive","path":"/opt/meshboard","artifact":"look"}]}`
|
||||
|
||||
// aModuleAt is a repository holding the module under mod/, whose directory git names as tree, on its trunk.
|
||||
func aModuleAt(t *testing.T, tree string) (*recorded, Runner, string) {
|
||||
t.Helper()
|
||||
r := &recorded{tree: tree, contents: map[string]string{"mod/" + ManifestName: reusedModule,
|
||||
"mod/Dockerfile": "FROM scratch", "mod/files/theme.conf": "dark"}}
|
||||
run := func(ctx context.Context, dir, name string, args ...string) (string, error) {
|
||||
if name == "git" && len(args) > 0 && args[0] == "symbolic-ref" {
|
||||
r.ran = append(r.ran, name+" "+strings.Join(args, " "))
|
||||
return "origin/main\n", nil
|
||||
}
|
||||
return r.run(ctx, dir, name, args...)
|
||||
}
|
||||
return r, run, t.TempDir()
|
||||
}
|
||||
|
||||
func dockerBuilt(r *recorded) bool {
|
||||
for _, line := range r.ran {
|
||||
if strings.HasPrefix(line, "docker build") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var aPull = PullRef{Owner: "novox", Repo: "meshboard", Number: 7}
|
||||
|
||||
// aCheckBuilt is the check of aPull building the module from a tree, into the scratch namespace.
|
||||
func aCheckBuilt(t *testing.T, scratch *aScratch, tree string) CheckBuilt {
|
||||
t.Helper()
|
||||
r, run, workspace := aModuleAt(t, tree)
|
||||
spec := CheckSpec{ID: "build-1", Owner: aPull.Owner, Repo: aPull.Repo, Number: aPull.Number,
|
||||
Builds: []CheckBuildSpec{{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod",
|
||||
Ref: "main"}}}
|
||||
built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: scratch}, workspace, "", "",
|
||||
CheckVerdict{Head: "abcdefabcdef", Trunk: "123456123456"}, GitCredential{}, nil, logging(nil))
|
||||
if len(built) != 1 || built[0].NotBuilt != "" || built[0].Key == "" {
|
||||
t.Fatalf("the check did not build the module for its merge: %+v", built)
|
||||
}
|
||||
if !dockerBuilt(r) {
|
||||
t.Fatal("the check built no image")
|
||||
}
|
||||
for repository := range r.images {
|
||||
if !strings.HasPrefix(repository, ScratchPrefix) {
|
||||
t.Fatalf("the check published %s outside the scratch namespace", repository)
|
||||
}
|
||||
}
|
||||
for repository := range r.archives {
|
||||
if !strings.HasPrefix(repository, ScratchPrefix) {
|
||||
t.Fatalf("the check published %s outside the scratch namespace", repository)
|
||||
}
|
||||
}
|
||||
return built[0]
|
||||
}
|
||||
|
||||
// named is the walk's leave naming what a check's verdict says it built, as the controller reads it.
|
||||
func named(c CheckBuilt) []NamedBuild {
|
||||
n := NamedBuild{Pull: aPull, Check: "build-1", Module: c.Module, Key: c.Key, Digests: map[string]string{}}
|
||||
for _, m := range c.Made {
|
||||
d := m.Digest
|
||||
if d == "" {
|
||||
d = DigestOf(m.Reference)
|
||||
}
|
||||
n.Digests[m.Name] = d
|
||||
}
|
||||
return []NamedBuild{n}
|
||||
}
|
||||
|
||||
func withVersion(t *testing.T, v string) {
|
||||
t.Helper()
|
||||
was := Version
|
||||
Version = v
|
||||
t.Cleanup(func() { Version = was })
|
||||
}
|
||||
|
||||
// A merge whose reuse key equals its check's builds nothing again: the walk copies the check's build by digest
|
||||
// and registers that digest.
|
||||
func TestAWalkWhoseKeyIsItsChecksBuildsNothingAndRegistersTheChecksDigest(t *testing.T) {
|
||||
withVersion(t, "builder-1")
|
||||
scratch := newScratch()
|
||||
checked := aCheckBuilt(t, scratch, "tree-one")
|
||||
|
||||
r, run, workspace := aModuleAt(t, "tree-one")
|
||||
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)},
|
||||
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if dockerBuilt(r) || len(r.images) > 0 || len(r.archives) > 0 {
|
||||
t.Fatalf("the walk built again: ran %v, published %v %v", r.ran, r.images, r.archives)
|
||||
}
|
||||
if got.Reused == nil || got.Why != "" {
|
||||
t.Fatalf("the walk did not say it reused the check's build: reused %+v, why %q", got.Reused, got.Why)
|
||||
}
|
||||
if got.ReuseKey != checked.Key {
|
||||
t.Fatalf("the walk's key %s is not the check's %s", got.ReuseKey, checked.Key)
|
||||
}
|
||||
if len(scratch.took) != 2 {
|
||||
t.Fatalf("the walk copied %v, not the check's two artifacts", scratch.took)
|
||||
}
|
||||
for _, made := range checked.Made {
|
||||
registered := strings.Replace(made.Reference, ScratchPrefix, "", 1)
|
||||
found := false
|
||||
for _, b := range got.Built {
|
||||
found = found || (b.Reference == registered && b.Digest == made.Digest)
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("the walk registers %+v, not the check's %s copied", got.Built, made.Reference)
|
||||
}
|
||||
}
|
||||
if image, _ := got.Manifest.Resources[0]["image"].(string); image != "registry.invalid/meshboard/server@sha256:"+strings.Repeat("a", 64) {
|
||||
t.Fatalf("the manifest pins %q, not the check's digest in the module's namespace", image)
|
||||
}
|
||||
words := got.Reused.Words()
|
||||
if !strings.HasPrefix(words, "reused from the check of novox/meshboard#7 at head abcdefabcdef (merged into trunk 123456123456)") ||
|
||||
!strings.Contains(words, "sha256:") {
|
||||
t.Fatalf("the walk says %q", words)
|
||||
}
|
||||
}
|
||||
|
||||
// A record no verdict names is refused and built instead, and the walk says so: the store takes writes from anything
|
||||
// on its network, so a record planted under another merge's key is never registered.
|
||||
func TestARecordNoVerdictNamesIsRefusedAndBuilt(t *testing.T) {
|
||||
withVersion(t, "builder-1")
|
||||
scratch := newScratch()
|
||||
checked := aCheckBuilt(t, scratch, "tree-one")
|
||||
for name, leave := range map[string][]NamedBuild{
|
||||
"no verdict": nil,
|
||||
"another digest": func() []NamedBuild {
|
||||
n := named(checked)
|
||||
n[0].Digests["server"] = "sha256:" + strings.Repeat("e", 64)
|
||||
return n
|
||||
}(),
|
||||
"another check": func() []NamedBuild { n := named(checked); n[0].Check = "build-9"; return n }(),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
scratch.took = nil
|
||||
r, run, workspace := aModuleAt(t, "tree-one")
|
||||
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: leave},
|
||||
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Reused != nil || len(scratch.took) > 0 || !dockerBuilt(r) {
|
||||
t.Fatalf("a record no verdict names was taken: %+v, copied %v", got.Reused, scratch.took)
|
||||
}
|
||||
if !strings.Contains(got.Why, "no check verdict of the walk's pull requests names it") {
|
||||
t.Fatalf("the walk says %q", got.Why)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A differing key builds, and says which part differs.
|
||||
func TestAWalkWhoseKeyDiffersBuildsAndSaysWhy(t *testing.T) {
|
||||
withVersion(t, "builder-1")
|
||||
scratch := newScratch()
|
||||
checked := aCheckBuilt(t, scratch, "tree-one")
|
||||
|
||||
r, run, workspace := aModuleAt(t, "tree-two")
|
||||
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)},
|
||||
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !dockerBuilt(r) || got.Reused != nil {
|
||||
t.Fatalf("a walk whose source differs did not build: reused %+v", got.Reused)
|
||||
}
|
||||
if !strings.Contains(got.Why, "the key differs") || !strings.Contains(got.Why, partTree) {
|
||||
t.Fatalf("the walk says %q, not which part of the key differs", got.Why)
|
||||
}
|
||||
|
||||
// And a builder of another version builds too, saying so.
|
||||
withVersion(t, "builder-2")
|
||||
r, run, workspace = aModuleAt(t, "tree-one")
|
||||
got, err = BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)},
|
||||
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !dockerBuilt(r) || !strings.Contains(got.Why, partBuilder) {
|
||||
t.Fatalf("a newer builder took the older one's build, or did not say why not: %q", got.Why)
|
||||
}
|
||||
|
||||
// No check of the walk's pull requests built it: said.
|
||||
r, run, workspace = aModuleAt(t, "tree-one")
|
||||
got, _ = BuildReusing(t.Context(), run, r, Reuse{Scratch: newScratch(), Pulls: []PullRef{aPull}},
|
||||
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if !dockerBuilt(r) || !strings.Contains(got.Why, "no check of novox/meshboard#7 built it") {
|
||||
t.Fatalf("the walk says %q", got.Why)
|
||||
}
|
||||
}
|
||||
|
||||
// A copy that fails is said and falls back to a build; it never fails the walk.
|
||||
func TestAFailedCopyOfAChecksBuildFallsBackToBuilding(t *testing.T) {
|
||||
withVersion(t, "builder-1")
|
||||
scratch := newScratch()
|
||||
checked := aCheckBuilt(t, scratch, "tree-one")
|
||||
scratch.failTake = true
|
||||
|
||||
r, run, workspace := aModuleAt(t, "tree-one")
|
||||
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)},
|
||||
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("a failed copy failed the walk: %v", err)
|
||||
}
|
||||
if !dockerBuilt(r) || got.Reused != nil || len(r.images) == 0 {
|
||||
t.Fatal("a failed copy did not fall back to a build")
|
||||
}
|
||||
if !strings.Contains(got.Why, "copying meshboard's") || !strings.Contains(got.Why, "refused the mount") {
|
||||
t.Fatalf("the walk says %q", got.Why)
|
||||
}
|
||||
for repository := range r.images {
|
||||
if strings.HasPrefix(repository, ScratchPrefix) {
|
||||
t.Fatalf("the walk's own build went into the scratch namespace: %s", repository)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Only a walk takes a check's build: a plain build — a check's, a dry run's, `build --ref`'s — builds, and a commit
|
||||
// off the trunk never takes one.
|
||||
func TestOnlyAWalkOfATrunkCommitTakesAChecksBuild(t *testing.T) {
|
||||
withVersion(t, "builder-1")
|
||||
scratch := newScratch()
|
||||
checked := aCheckBuilt(t, scratch, "tree-one")
|
||||
|
||||
r, run, workspace := aModuleAt(t, "tree-one")
|
||||
if _, err := Build(t.Context(), run, r, "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil,
|
||||
Npmrc{}, GitCredential{}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !dockerBuilt(r) || len(scratch.took) > 0 {
|
||||
t.Fatal("a build not asked to reuse took a check's build")
|
||||
}
|
||||
|
||||
r, _, workspace = aModuleAt(t, "tree-one") // no trunk said: off it
|
||||
got, err := BuildReusing(t.Context(), r.run, r, Reuse{Scratch: scratch, Named: named(checked)}, "https://forge.invalid/meshboard.git",
|
||||
"mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !dockerBuilt(r) || got.Reused != nil || !strings.Contains(got.Why, "not on its trunk") {
|
||||
t.Fatalf("a commit off the trunk took a check's build: %q", got.Why)
|
||||
}
|
||||
}
|
||||
|
||||
// A check that did not pass builds nothing for its merge, and one asked to build nothing builds nothing.
|
||||
func TestACheckThatFailedBuildsNothingForItsMerge(t *testing.T) {
|
||||
if skipsTheBuilds(CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "fail"}}) == "" {
|
||||
t.Fatal("a check whose repository layer failed builds for its merge")
|
||||
}
|
||||
if skipsTheBuilds(CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "warning"}}) != "" {
|
||||
t.Fatal("a check that passed with a warning builds nothing for its merge")
|
||||
}
|
||||
r, run, workspace := aModuleAt(t, "tree-one")
|
||||
if built := buildForTheMerge(t.Context(), run, CheckSpec{ID: "build-1"}, CheckBuilding{Publish: r, Scratch: newScratch()},
|
||||
workspace, "", "", CheckVerdict{}, GitCredential{}, nil, logging(nil)); built != nil || len(r.ran) > 0 {
|
||||
t.Fatalf("a check moving no module built %v, ran %v", built, r.ran)
|
||||
}
|
||||
}
|
||||
|
||||
// A module whose source does not pin its build has no key and is not built by the check; what stands after it
|
||||
// is not built either.
|
||||
func TestACheckBuildsNoModuleWithoutAKeyNorWhatStandsAfterIt(t *testing.T) {
|
||||
withVersion(t, "builder-1")
|
||||
r, run, workspace := aModuleAt(t, "tree-one")
|
||||
r.contents["mod/"+ManifestName] = `{"module":"meshboard","version":"1","build":{"artifacts":[
|
||||
{"name":"mirror","kind":"upstream","from":"docker.io/library/alpine:3"}]}}`
|
||||
spec := CheckSpec{ID: "build-1", Owner: "novox", Repo: "meshboard", Number: 7, Builds: []CheckBuildSpec{
|
||||
{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod", Tier: 0},
|
||||
{Module: "after", Repository: "https://forge.invalid/after.git", Tier: 1}}}
|
||||
built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: newScratch()}, workspace, "", "",
|
||||
CheckVerdict{}, GitCredential{}, nil, logging(nil))
|
||||
if len(built) != 2 || !strings.Contains(built[0].NotBuilt, "named by tag") ||
|
||||
!strings.Contains(built[1].NotBuilt, "stands after meshboard") {
|
||||
t.Fatalf("built %+v", built)
|
||||
}
|
||||
if len(r.images) > 0 {
|
||||
t.Fatalf("a module without a key was published: %v", r.images)
|
||||
}
|
||||
}
|
||||
|
||||
// What the key names: every input a fresh build could differ by.
|
||||
func TestTheReuseKeyNamesEveryInputOfTheBuild(t *testing.T) {
|
||||
src := newSourceInputs("m")
|
||||
src.tree = "t1"
|
||||
src.bases = []string{"store:5000/b/x@sha256:" + strings.Repeat("b", 64)}
|
||||
src.contexts["server"] = "ctx1"
|
||||
src.toolchains["tools"] = "go sha256:c 1234"
|
||||
manifest := catalogue.Manifest{Module: "m"}
|
||||
base := keyOf(src, manifest, "v1")
|
||||
if base.Key == "" {
|
||||
t.Fatalf("no key: %s", base.Why)
|
||||
}
|
||||
for name, change := range map[string]func(*sourceInputs) string{
|
||||
partTree: func(s *sourceInputs) string { s.tree = "t2"; return "v1" },
|
||||
partBases: func(s *sourceInputs) string {
|
||||
s.bases = []string{"other@sha256:" + strings.Repeat("c", 64)}
|
||||
return "v1"
|
||||
},
|
||||
partContext: func(s *sourceInputs) string { s.contexts["server"] = "ctx2"; return "v1" },
|
||||
partTool: func(s *sourceInputs) string { s.toolchains["tools"] = "go sha256:d 1234"; return "v1" },
|
||||
partBuilder: func(*sourceInputs) string { return "v2" },
|
||||
} {
|
||||
s := src.copy()
|
||||
version := change(s)
|
||||
if k := keyOf(s, manifest, version); k.Key == base.Key {
|
||||
t.Errorf("a change of %s left the key the same", name)
|
||||
}
|
||||
}
|
||||
// A base copied to another address is the same base: the key names it by digest.
|
||||
moved := src.copy()
|
||||
moved.bases = []string{"elsewhere:5000/scratch/b/x@sha256:" + strings.Repeat("b", 64)}
|
||||
if keyOf(moved, manifest, "v1").Key != base.Key {
|
||||
t.Error("the same base at another address changed the key")
|
||||
}
|
||||
if k := keyOf(src, manifest, ""); k.Key != "" {
|
||||
t.Error("a builder whose version cannot be said keyed its build")
|
||||
}
|
||||
unpinned := src.copy()
|
||||
unpinned.notPinned("it resolves packages from the mesh's registry at build time")
|
||||
if k := keyOf(unpinned, manifest, "v1"); k.Key != "" || !strings.Contains(k.Why, "registry") {
|
||||
t.Errorf("a build its source does not pin was keyed: %+v", k)
|
||||
}
|
||||
}
|
||||
|
||||
// aTaggingRegistry is the in-memory registry with tags and their listing, as the store's scratch namespace needs.
|
||||
type aTaggingRegistry struct {
|
||||
*aRegistryOfRepositories
|
||||
tags map[string]map[string]string // repository → tag → manifest digest
|
||||
}
|
||||
|
||||
func (m *aTaggingRegistry) serve() http.Handler {
|
||||
under := m.handler()
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if repository, ok := strings.CutSuffix(strings.TrimPrefix(r.URL.Path, "/v2/"), "/tags/list"); ok {
|
||||
m.mu.Lock()
|
||||
var tags []string
|
||||
for tag := range m.tags[repository] {
|
||||
tags = append(tags, tag)
|
||||
}
|
||||
m.mu.Unlock()
|
||||
if tags == nil {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"name": repository, "tags": tags})
|
||||
return
|
||||
}
|
||||
repository, kind, rest := splitPath(r.URL.Path)
|
||||
if kind == "blobs" && r.Method == http.MethodHead {
|
||||
m.mu.Lock()
|
||||
body, linked := m.blobs[rest], m.links[repository][rest]
|
||||
m.mu.Unlock()
|
||||
if !linked {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
if kind != "manifests" {
|
||||
under.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
if !strings.HasPrefix(rest, "sha256:") {
|
||||
m.mu.Lock()
|
||||
switch r.Method {
|
||||
case http.MethodPut:
|
||||
body, _ := readAll(r)
|
||||
if m.tags[repository] == nil {
|
||||
m.tags[repository] = map[string]string{}
|
||||
}
|
||||
m.tags[repository][rest] = digestOf(body)
|
||||
m.manifests[repository+"@"+digestOf(body)] = body
|
||||
m.mu.Unlock()
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
return
|
||||
default:
|
||||
digest, ok := m.tags[repository][rest]
|
||||
body, held := m.manifests[repository+"@"+digest]
|
||||
m.mu.Unlock()
|
||||
if !ok || !held {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Docker-Content-Digest", digest)
|
||||
w.Header().Set("Content-Type", mediaTypeOf(body))
|
||||
if r.Method == http.MethodGet {
|
||||
_, _ = w.Write(body)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
}
|
||||
if r.Method == http.MethodDelete {
|
||||
m.mu.Lock()
|
||||
for _, tags := range m.tags {
|
||||
for tag, digest := range tags {
|
||||
if digest == rest {
|
||||
delete(tags, tag)
|
||||
}
|
||||
}
|
||||
}
|
||||
m.mu.Unlock()
|
||||
}
|
||||
under.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// A check's builds go after seven days, and when its pull request closes or is merged; the walk's copy in the
|
||||
// module's own namespace stays.
|
||||
func TestCheckBuildsExpireAfterSevenDaysAndGoWhenTheirPullRequestCloses(t *testing.T) {
|
||||
m := &aTaggingRegistry{aRegistryOfRepositories: newRegistryOfRepositories(), tags: map[string]map[string]string{}}
|
||||
server := httptest.NewServer(m.serve())
|
||||
defer server.Close()
|
||||
address := strings.TrimPrefix(server.URL, "http://")
|
||||
registry := Registry{Address: address}
|
||||
ctx := t.Context()
|
||||
now := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC)
|
||||
|
||||
image := func(repository, body string) string {
|
||||
digest := digestOf([]byte(body))
|
||||
m.manifests[repository+"@"+digest] = []byte(body)
|
||||
return address + "/" + repository + "@" + digest
|
||||
}
|
||||
check := func(pull PullRef, id string, at time.Time, key string) string {
|
||||
reference := image("scratch/meshboard/server", `{"mediaType":"`+mediaManifestOCI+`","layers":[],"n":"`+id+`"}`)
|
||||
build := CheckBuild{Key: key, Module: "meshboard", Made: []catalogue.Built{{Name: "server", Kind: "image",
|
||||
Reference: reference}}, Pull: pull, Check: id, At: at}
|
||||
if err := registry.Keep(ctx, pull, id, at, []CheckBuild{build}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return reference
|
||||
}
|
||||
held := func(reference string) bool {
|
||||
_, digest, _ := strings.Cut(reference, "@")
|
||||
_, ok := m.manifests["scratch/meshboard/server@"+digest]
|
||||
return ok
|
||||
}
|
||||
other := PullRef{Owner: "novox", Repo: "meshboard", Number: 8}
|
||||
old := check(aPull, "build-1", now.Add(-8*24*time.Hour), "r1:"+strings.Repeat("1", 64))
|
||||
fresh := check(aPull, "build-2", now.Add(-time.Hour), "r1:"+strings.Repeat("2", 64))
|
||||
others := check(other, "build-3", now.Add(-time.Hour), "r1:"+strings.Repeat("3", 64))
|
||||
walked := image("meshboard/server", `{"mediaType":"`+mediaManifestOCI+`","layers":[],"n":"walk"}`)
|
||||
|
||||
if _, found, err := registry.Find(ctx, "meshboard", "r1:"+strings.Repeat("2", 64)); err != nil || !found {
|
||||
t.Fatalf("a check build kept is not found by its key: %v %v", found, err)
|
||||
}
|
||||
n, err := registry.Expire(ctx, CheckBuildsKept, now)
|
||||
if err != nil || n != 1 {
|
||||
t.Fatalf("expired %d checks (%v), want the one eight days old", n, err)
|
||||
}
|
||||
if held(old) || !held(fresh) || !held(others) {
|
||||
t.Fatalf("after the week's sweep: old %v, fresh %v, another's %v", held(old), held(fresh), held(others))
|
||||
}
|
||||
if _, found, _ := registry.Find(ctx, "meshboard", "r1:"+strings.Repeat("1", 64)); found {
|
||||
t.Fatal("the expired check build's record is still found")
|
||||
}
|
||||
|
||||
n, err = registry.Forget(ctx, aPull)
|
||||
if err != nil || n != 1 {
|
||||
t.Fatalf("forgot %d checks of %s (%v)", n, aPull, err)
|
||||
}
|
||||
if held(fresh) || !held(others) {
|
||||
t.Fatalf("after %s closed: its build held %v, another pull request's %v", aPull, held(fresh), held(others))
|
||||
}
|
||||
if _, ok := m.manifests["meshboard/server@"+strings.SplitN(walked, "@", 2)[1]]; !ok {
|
||||
t.Fatal("letting go of check builds took the walk's copy in the module's namespace")
|
||||
}
|
||||
if left, _ := registry.OfPulls(ctx, []PullRef{aPull}); len(left) != 0 {
|
||||
t.Fatalf("%s still has check builds: %+v", aPull, left)
|
||||
}
|
||||
}
|
||||
|
||||
// The walk copies a check build by digest: the digest registered is the digest the check built, and the copy is
|
||||
// in the module's own namespace.
|
||||
func TestAChecksBuildIsCopiedByDigestIntoTheModulesNamespace(t *testing.T) {
|
||||
m := &aTaggingRegistry{aRegistryOfRepositories: newRegistryOfRepositories(), tags: map[string]map[string]string{}}
|
||||
server := httptest.NewServer(m.serve())
|
||||
defer server.Close()
|
||||
address := strings.TrimPrefix(server.URL, "http://")
|
||||
registry := Registry{Address: address}
|
||||
|
||||
layer := []byte("layer")
|
||||
m.blobs[digestOf(layer)] = layer
|
||||
m.link("scratch/meshboard/server", digestOf(layer))
|
||||
manifest := []byte(`{"mediaType":"` + mediaManifestOCI + `","layers":[{"digest":"` + digestOf(layer) + `","size":5}]}`)
|
||||
m.manifests["scratch/meshboard/server@"+digestOf(manifest)] = manifest
|
||||
archive := []byte("archive")
|
||||
m.blobs[digestOf(archive)] = archive
|
||||
m.link("scratch/meshboard/look", digestOf(archive))
|
||||
|
||||
got, err := registry.Take(t.Context(), "meshboard", catalogue.Built{Name: "server", Kind: "image",
|
||||
Reference: address + "/scratch/meshboard/server@" + digestOf(manifest)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Reference != address+"/meshboard/server@"+digestOf(manifest) {
|
||||
t.Fatalf("copied as %s", got.Reference)
|
||||
}
|
||||
if m.uploads != 0 || m.mounts != 1 {
|
||||
t.Fatalf("the image's layer was moved (%d uploads, %d mounts), not mounted", m.uploads, m.mounts)
|
||||
}
|
||||
got, err = registry.Take(t.Context(), "meshboard", catalogue.Built{Name: "look", Kind: "archive",
|
||||
Digest: digestOf(archive), Reference: "http://" + address + "/v2/scratch/meshboard/look/blobs/" + digestOf(archive)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Digest != digestOf(archive) || got.Reference != "http://"+address+"/v2/meshboard/look/blobs/"+digestOf(archive) ||
|
||||
!m.links["meshboard/look"][digestOf(archive)] {
|
||||
t.Fatalf("the archive was copied as %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Two checks that built one digest in one repository share it: letting go of one check's builds keeps it while the
|
||||
// other still names it, and the last to go takes it.
|
||||
func TestABuildTwoChecksShareGoesWithTheLastOfThem(t *testing.T) {
|
||||
m := &aTaggingRegistry{aRegistryOfRepositories: newRegistryOfRepositories(), tags: map[string]map[string]string{}}
|
||||
server := httptest.NewServer(m.serve())
|
||||
defer server.Close()
|
||||
address := strings.TrimPrefix(server.URL, "http://")
|
||||
registry := Registry{Address: address}
|
||||
ctx := t.Context()
|
||||
now := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC)
|
||||
|
||||
body := `{"mediaType":"` + mediaManifestOCI + `","layers":[]}`
|
||||
digest := digestOf([]byte(body))
|
||||
m.manifests["scratch/meshboard/server@"+digest] = []byte(body)
|
||||
reference := address + "/scratch/meshboard/server@" + digest
|
||||
other := PullRef{Owner: "novox", Repo: "meshboard", Number: 8}
|
||||
for i, pull := range []PullRef{aPull, other} {
|
||||
id := "build-" + strconv.Itoa(i+1)
|
||||
b := CheckBuild{Key: "r1:" + strings.Repeat(strconv.Itoa(i+1), 64), Module: "meshboard", Pull: pull, Check: id,
|
||||
At: now, Made: []catalogue.Built{{Name: "server", Kind: "image", Reference: reference}}}
|
||||
if err := registry.Keep(ctx, pull, id, now, []CheckBuild{b}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
held := func() bool { _, ok := m.manifests["scratch/meshboard/server@"+digest]; return ok }
|
||||
|
||||
if n, err := registry.Forget(ctx, aPull); err != nil || n != 1 {
|
||||
t.Fatalf("forgot %d (%v)", n, err)
|
||||
}
|
||||
if !held() {
|
||||
t.Fatal("letting go of one check's builds took a build another check still names")
|
||||
}
|
||||
if n, err := registry.Forget(ctx, other); err != nil || n != 1 {
|
||||
t.Fatalf("forgot %d (%v)", n, err)
|
||||
}
|
||||
if held() {
|
||||
t.Fatal("the build stayed after the last check naming it went")
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,43 @@ type sourceInputs struct {
|
||||
// contextsAtHead is false once a context was cloned at something other than its trunk's head: its
|
||||
// closure is not the one the next merge there meets, and the build says no build source.
|
||||
contextsAtHead bool
|
||||
// onlyKey says the inputs are being taken for a reuse key (novox/hq ADR 0350 §2): each artifact goes as far
|
||||
// as what its source fingerprint reads, and nothing is built or published.
|
||||
onlyKey bool
|
||||
}
|
||||
|
||||
// keyOnly is whether these inputs are only being taken for a reuse key.
|
||||
func (s *sourceInputs) keyOnly() bool { return s != nil && s.onlyKey }
|
||||
|
||||
// copy is the inputs taken so far, apart: a walk's key is taken from them without touching the build's own.
|
||||
func (s *sourceInputs) copy() *sourceInputs {
|
||||
c := newSourceInputs(s.module)
|
||||
c.tree, c.unpinned, c.prefix, c.ownIsAll, c.contextsAtHead = s.tree, s.unpinned, s.prefix, s.ownIsAll, s.contextsAtHead
|
||||
c.bases = append([]string{}, s.bases...)
|
||||
for k, v := range s.contexts {
|
||||
c.contexts[k] = v
|
||||
}
|
||||
for k, v := range s.toolchains {
|
||||
c.toolchains[k] = v
|
||||
}
|
||||
for k, v := range s.own {
|
||||
c.own[k] = v
|
||||
}
|
||||
for k, v := range s.read {
|
||||
if v == nil {
|
||||
c.read[k] = nil
|
||||
continue
|
||||
}
|
||||
set := map[string]bool{}
|
||||
for e := range v {
|
||||
set[e] = true
|
||||
}
|
||||
c.read[k] = set
|
||||
}
|
||||
for k, v := range s.readAs {
|
||||
c.readAs[k] = v
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// contextOffHead says a context was not its trunk's head.
|
||||
|
||||
@@ -167,8 +167,13 @@ var toolchains = []Toolchain{
|
||||
// binary that needs a libc it did not bring is a delivery that works until a machine
|
||||
// differs. Trimmed of its own paths for the same reason a version comes from where it sits
|
||||
// rather than from the linker: two builds of one commit produce the same bytes.
|
||||
//
|
||||
// **And of one build source at two commits** (novox/hq ADR 0267 rule 6, issue 363): a module built
|
||||
// from its repository's root is compiled with the clone's `.git` in view, and Go stamps the commit,
|
||||
// its time and whether the tree was modified into the binary unless told not to — so a merge that
|
||||
// changed only a README made a new digest of an unchanged program.
|
||||
Compile: []string{
|
||||
"env", "CGO_ENABLED=0", "GOFLAGS=-trimpath",
|
||||
"env", "CGO_ENABLED=0", "GOFLAGS=-trimpath -buildvcs=false",
|
||||
"go", "build",
|
||||
},
|
||||
// Stripped of symbols and debug info: what a machine holds is a file it runs, not one it
|
||||
|
||||
@@ -87,6 +87,11 @@ func Openings(rules []Rule, foundation []int, published map[string]map[int]int)
|
||||
widen(key{rule.Protocol, rule.Port}, OpeningFromEverywhere)
|
||||
case FromMesh:
|
||||
widen(key{rule.Protocol, rule.Port}, OpeningFromMesh)
|
||||
case FromLocalNetwork:
|
||||
// The private network's part only (novox/hq ADR 0340). An adopted node's found firewall
|
||||
// stays, and it is what admits the machine's local network; an opening has no word for a
|
||||
// local network, and widening it to everywhere would open the port to the internet.
|
||||
widen(key{rule.Protocol, rule.Port}, OpeningFromMesh)
|
||||
}
|
||||
}
|
||||
for _, port := range foundation {
|
||||
|
||||
@@ -127,6 +127,7 @@ func boundNeed(n Needed, bound Chosen, catalogue map[string]Manifest, node Node,
|
||||
identity = pm.IdentityBoundOf(n.Name)
|
||||
}
|
||||
held.From, held.At, held.Module, held.Serves, held.Identity = p.Node, p.At, p.Module, p.Serves, identity
|
||||
held.LeftOut = p.LeftOut
|
||||
return held, true, ""
|
||||
}
|
||||
|
||||
|
||||
@@ -26,21 +26,10 @@ import (
|
||||
// serves — whose keys are agreed by the requirement's name, not by this file.
|
||||
|
||||
// bound is where a module says a value from one of its bindings belongs:
|
||||
// ${bound:<provision>.<key>}.
|
||||
var bound = regexp.MustCompile(`\$\{bound:([a-z0-9][a-z0-9.-]*[a-z0-9]):([a-z0-9][a-z0-9_-]*)\}`)
|
||||
|
||||
// boundUsed are the (provision, key) pairs a file's content asks for, first appearance first.
|
||||
func boundUsed(content string) [][2]string {
|
||||
var used [][2]string
|
||||
seen := map[string]bool{}
|
||||
for _, m := range bound.FindAllStringSubmatch(content, -1) {
|
||||
if key := m[1] + ":" + m[2]; !seen[key] {
|
||||
seen[key] = true
|
||||
used = append(used, [2]string{m[1], m[2]})
|
||||
}
|
||||
}
|
||||
return used
|
||||
}
|
||||
// ${bound:<provision>:<key>}, and for an optional requirement the value it takes while absent,
|
||||
// ${bound:<provision>:<key>|default:<value>} (novox/hq ADR 0330 §4). A default is never empty: an
|
||||
// absent requirement is never rendered as an empty string.
|
||||
var bound = regexp.MustCompile(`\$\{bound:([a-z0-9][a-z0-9.-]*[a-z0-9]):([a-z0-9][a-z0-9_-]*)(?:\|default:([^}\n]+))?\}`)
|
||||
|
||||
// knownFor is everything a module may name from one of its bindings.
|
||||
//
|
||||
@@ -125,7 +114,11 @@ func plainly(value any) string {
|
||||
// serve, is refused. Left as it was, the literal `${bound:x:y}` would be written into a
|
||||
// configuration file and read as a value — a connection to a host called `${bound:x:y}`, failing
|
||||
// somewhere that names neither the module nor the mesh.
|
||||
func boundInto(resource map[string]any, known map[string]map[string]string, module string) error {
|
||||
//
|
||||
// **An optional requirement left absent takes the default its placeholder names** (novox/hq ADR
|
||||
// 0330 §4), and one that names none is refused rather than rendered empty. The module check refuses
|
||||
// that already; this is the same rule where the file is made, for a manifest registered before it.
|
||||
func boundInto(resource map[string]any, known map[string]map[string]string, module string, absent map[string]bool) error {
|
||||
if fmt.Sprint(resource["type"]) != "file" {
|
||||
return nil
|
||||
}
|
||||
@@ -133,24 +126,43 @@ func boundInto(resource map[string]any, known map[string]map[string]string, modu
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
for _, pair := range boundUsed(content) {
|
||||
provision, key := pair[0], pair[1]
|
||||
var refused error
|
||||
filled := bound.ReplaceAllStringFunc(content, func(token string) string {
|
||||
if refused != nil {
|
||||
return token
|
||||
}
|
||||
m := bound.FindStringSubmatch(token)
|
||||
provision, key, fallback := m[1], m[2], m[3]
|
||||
values, has := known[provision]
|
||||
if !has {
|
||||
return fmt.Errorf(
|
||||
if absent[provision] {
|
||||
if fallback != "" {
|
||||
return fallback
|
||||
}
|
||||
refused = fmt.Errorf(
|
||||
"%s has a file that says ${bound:%s:%s}, and %q is optional and absent here: it names no "+
|
||||
"value for then (${bound:%s:%s|default:<value>}), so it would be written empty "+
|
||||
"(novox/hq ADR 0330 §4)", module, provision, key, provision, provision, key)
|
||||
return token
|
||||
}
|
||||
refused = fmt.Errorf(
|
||||
"%s has a file that says ${bound:%s:%s}, and %s does not require %q. It may name %s",
|
||||
module, provision, key, module, provision, orNothing(namesOfBindings(known)))
|
||||
return token
|
||||
}
|
||||
value, said := values[key]
|
||||
if !said {
|
||||
return fmt.Errorf(
|
||||
refused = fmt.Errorf(
|
||||
"%s asks its %s binding for %q, and what answers it says %s",
|
||||
module, provision, key, orNothing(namesOfKeys(values)))
|
||||
return token
|
||||
}
|
||||
resource["content"] = strings.ReplaceAll(
|
||||
content, fmt.Sprintf("${bound:%s:%s}", provision, key), value)
|
||||
content = resource["content"].(string)
|
||||
return value
|
||||
})
|
||||
if refused != nil {
|
||||
return refused
|
||||
}
|
||||
resource["content"] = filled
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -135,6 +135,9 @@ func CheckServes(m Manifest) []string {
|
||||
for _, key := range sortedAnyKeys(m.Serves[provision]) {
|
||||
text, ok := m.Serves[provision][key].(string)
|
||||
if !ok {
|
||||
// A provider's own name is filled only in a value of its own (novox/hq ADR 0346): one
|
||||
// nested in a list or an object would be served as the placeholder, unsaid.
|
||||
problems = append(problems, nestedOwnNames(m, provision, key, m.Serves[provision][key])...)
|
||||
continue
|
||||
}
|
||||
// A probe identity, because what is checked is the shape of the statement and not
|
||||
@@ -143,6 +146,8 @@ func CheckServes(m Manifest) []string {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s serves %s, and the value it serves as %q %s", m.Module, provision, key, err))
|
||||
}
|
||||
// And the provider's own names it may serve (novox/hq ADR 0346).
|
||||
problems = append(problems, checkServedOwnNames(m, provision, key, text)...)
|
||||
// A label longer than DNS keeps is not truncated here (asDNSLabel), so the offer's own
|
||||
// bound has to keep the identity inside one (ADR 0225).
|
||||
if strings.Contains(text, "${consumer:as:dns}") {
|
||||
|
||||
@@ -135,6 +135,12 @@ type Rendering struct {
|
||||
// composes no filter for it rather than writing a rule around a link with no name.
|
||||
OutwardLinks []string
|
||||
|
||||
// LocalNetworks is this machine's local networks (novox/hq ADR 0340): the networks it reported on a
|
||||
// link facing outside it that lie wholly in a private range (JudgeLocalNetworks), which a port
|
||||
// reached from `local-network` is admitted from beside the private network. Empty when it reported
|
||||
// none, and the rule set then says so rather than widening the port or dropping it.
|
||||
LocalNetworks []string
|
||||
|
||||
// TunnelInterface is the interface the mesh's private network runs on, named here rather than
|
||||
// imported because the overlay package rests on this one. Traffic arriving on it is the mesh's,
|
||||
// not this machine's own guest, so the filter admits it only by a rule.
|
||||
@@ -532,8 +538,8 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
"`node show %s` says whether it has. Until then %s is not sent, and the machine "+
|
||||
"keeps the filter it has", r.Node, r.Node, filters)
|
||||
}
|
||||
filtering := AsNftables(rules, with.Mesh, r.PublicDomain != "", with.Foundation,
|
||||
with.OutwardLinks, with.TunnelInterface)
|
||||
filtering := AsNftablesWithLocal(rules, with.Mesh, r.PublicDomain != "", with.Foundation,
|
||||
with.OutwardLinks, with.TunnelInterface, with.LocalNetworks)
|
||||
|
||||
// **A variable two modules set is refused whether or not anything places it** (novox/hq ADR
|
||||
// 0203 §5): the account has one environment, and a machine whose holder arrives later should not
|
||||
@@ -855,6 +861,11 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
// An account depends on nothing the mesh computes.
|
||||
accounts, rest := accountsFirst(resources)
|
||||
resources = append(append(accounts, first...), rest...)
|
||||
// **A file delivered only while its optional requirement is bound is left out while it is
|
||||
// absent** (novox/hq ADR 0330 §5), and the field is dropped from the rest: the node-engine
|
||||
// parses strictly and knows no such field.
|
||||
absent := r.AbsentFor(m.Module)
|
||||
resources = withoutAbsent(resources, absent)
|
||||
|
||||
// No container is given the mesh's names (novox/hq ADR 0148). It used to be: every
|
||||
// container got the whole roster as `--add-host` entries at creation, and a name that
|
||||
@@ -1018,7 +1029,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
// **Substituted here, not on the machine.** A bound value is not secret — the mesh
|
||||
// holds it in the clear — so there is nothing for the host to be the only witness of,
|
||||
// and sending it already filled in means the host learns no new field.
|
||||
if err := boundInto(copied, known, m.Module); err != nil {
|
||||
if err := boundInto(copied, known, m.Module, absent); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And what the module could not have written: the machine it turned out to be
|
||||
@@ -2035,7 +2046,13 @@ func (r Resolution) servedOnThisMachine(provision string, with Rendering) (map[s
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("%s serving %s: %w", m.Module, provision, err)
|
||||
}
|
||||
return settled, true, nil
|
||||
// And the provider's own names, as this machine composes them (novox/hq ADR 0346); one it does
|
||||
// not compose leaves its key out, said by the plan on this machine (ServedLeftOut).
|
||||
named, _, err := r.ServedOwnNames(m, provision, settled, WithDefaults(m, with.Settings[m.Module]))
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("%s serving %s: %w", m.Module, provision, err)
|
||||
}
|
||||
return named, true, nil
|
||||
}
|
||||
return nil, false, nil
|
||||
}
|
||||
@@ -2553,8 +2570,8 @@ func onlyGroups(r map[string]any) bool {
|
||||
|
||||
// generatedHere is what each computed module's generator answers for this machine, by module —
|
||||
// absent for a module whose machine is not yet part of what it generates — held to the rule every
|
||||
// module is held to: no two modules on a machine declare one path, unit, name or package (novox/hq
|
||||
// issue 190, step 5; ADR 0222).
|
||||
// module is held to: no two modules on a machine declare one path, unit or name, nor one package present
|
||||
// and absent (novox/hq issue 190, step 5; ADR 0222, ADR 0303).
|
||||
//
|
||||
// Resolution checks the catalogue's manifests, and a computed module's manifest has none of the
|
||||
// resources it will declare — they exist only once its generator has answered for this machine,
|
||||
|
||||
@@ -152,9 +152,10 @@ func Exposure(m Manifest, layers []Layer) (map[int]string, error) {
|
||||
m.Module, port)
|
||||
}
|
||||
source, ok := value.(string)
|
||||
if !ok || (source != FromMesh && source != FromEverywhere && source != FromMachine) {
|
||||
return nil, fmt.Errorf("%s exposes port %d as %v; it is %q, %q or %q",
|
||||
m.Module, port, value, FromMesh, FromEverywhere, FromMachine)
|
||||
if !ok || (source != FromMesh && source != FromEverywhere && source != FromMachine &&
|
||||
source != FromLocalNetwork) {
|
||||
return nil, fmt.Errorf("%s exposes port %d as %v; it is %q, %q, %q or %q",
|
||||
m.Module, port, value, FromMesh, FromEverywhere, FromMachine, FromLocalNetwork)
|
||||
}
|
||||
out[port] = source
|
||||
}
|
||||
@@ -172,7 +173,9 @@ func Exposure(m Manifest, layers []Layer) (map[int]string, error) {
|
||||
// the appearance-of-restriction this mechanism exists to remove. The narrower rule's sources are
|
||||
// carried onto the wider one, so the module that wanted it is still named.
|
||||
func widest(rules []Rule) []Rule {
|
||||
reach := map[string]int{FromMachine: 0, FromMesh: 1, FromEverywhere: 2}
|
||||
// The local network is the private network and the machine's own local networks (novox/hq ADR
|
||||
// 0340 §4), so it covers `mesh` and is covered by `anywhere`.
|
||||
reach := map[string]int{FromMachine: 0, FromMesh: 1, FromLocalNetwork: 2, FromEverywhere: 3}
|
||||
var out []Rule
|
||||
for _, rule := range rules {
|
||||
covered := false
|
||||
@@ -232,6 +235,15 @@ const SSHPort = 22
|
||||
// any module asks for, and neither may be derived away.
|
||||
func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int,
|
||||
outwardLinks []string, tunnel string) string {
|
||||
return AsNftablesWithLocal(rules, mesh, outward, foundation, outwardLinks, tunnel, nil)
|
||||
}
|
||||
|
||||
// AsNftablesWithLocal is AsNftables with the machine's local networks: the networks it reported on a
|
||||
// link facing outside that count (novox/hq ADR 0340, JudgeLocalNetworks), which a rule from
|
||||
// `local-network` admits on its own port alone, beside the private network. None is not widened to
|
||||
// anywhere and not dropped silently: the rule set says the port wanted the local network and found none.
|
||||
func AsNftablesWithLocal(rules []Rule, mesh []string, outward bool, foundation []int,
|
||||
outwardLinks []string, tunnel string, local []string) string {
|
||||
// The links that are not this machine's own: the ones facing outside, and the mesh's tunnel.
|
||||
// Traffic arriving on any of them is admitted only by a rule below; traffic arriving anywhere
|
||||
// else is this machine's own guest and is not something the mesh has a position on.
|
||||
@@ -394,6 +406,31 @@ func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int,
|
||||
b.WriteString(fmt.Sprintf("\t\tip6 saddr { %s } %s dport %d accept\n",
|
||||
strings.Join(six, ", "), rule.Protocol, rule.Port))
|
||||
}
|
||||
case FromLocalNetwork:
|
||||
// The private network and the machine's own local networks, on this port alone (novox/hq
|
||||
// ADR 0340 §5).
|
||||
four, six := byFamily(mesh)
|
||||
lfour, lsix := byFamily(local)
|
||||
four, six = append(four, lfour...), append(six, lsix...)
|
||||
if len(lfour) == 0 && len(lsix) == 0 {
|
||||
b.WriteString(fmt.Sprintf(
|
||||
"\t\t# %s/%d wanted the local network, and this machine reported no private network "+
|
||||
"on a link facing outside; admitted from no local network\n", rule.Protocol, rule.Port))
|
||||
}
|
||||
if len(four) == 0 && len(six) == 0 {
|
||||
b.WriteString(fmt.Sprintf(
|
||||
"\t\t# %s/%d wanted the local network, and this node knows neither mesh addresses nor a "+
|
||||
"local network; closed\n", rule.Protocol, rule.Port))
|
||||
break
|
||||
}
|
||||
if len(four) > 0 {
|
||||
b.WriteString(fmt.Sprintf("\t\tip saddr { %s } %s dport %d accept\n",
|
||||
strings.Join(four, ", "), rule.Protocol, rule.Port))
|
||||
}
|
||||
if len(six) > 0 {
|
||||
b.WriteString(fmt.Sprintf("\t\tip6 saddr { %s } %s dport %d accept\n",
|
||||
strings.Join(six, ", "), rule.Protocol, rule.Port))
|
||||
}
|
||||
case FromEverywhere:
|
||||
b.WriteString(fmt.Sprintf("\t\t%s dport %d accept\n", rule.Protocol, rule.Port))
|
||||
}
|
||||
@@ -487,6 +524,18 @@ func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int,
|
||||
b.WriteString(fmt.Sprintf("\t\tip6 saddr { %s } ct original proto-dst %d accept\n",
|
||||
strings.Join(six, ", "), rule.Port))
|
||||
}
|
||||
case FromLocalNetwork:
|
||||
four, six := byFamily(mesh)
|
||||
lfour, lsix := byFamily(local)
|
||||
four, six = append(four, lfour...), append(six, lsix...)
|
||||
if len(four) > 0 {
|
||||
b.WriteString(fmt.Sprintf("\t\tip saddr { %s } ct original proto-dst %d accept\n",
|
||||
strings.Join(four, ", "), rule.Port))
|
||||
}
|
||||
if len(six) > 0 {
|
||||
b.WriteString(fmt.Sprintf("\t\tip6 saddr { %s } ct original proto-dst %d accept\n",
|
||||
strings.Join(six, ", "), rule.Port))
|
||||
}
|
||||
case FromEverywhere:
|
||||
b.WriteString(fmt.Sprintf("\t\tct original proto-dst %d accept\n", rule.Port))
|
||||
}
|
||||
@@ -793,10 +842,14 @@ const (
|
||||
// are a subset of anywhere. What differs is the names, which is the whole reason reach is not
|
||||
// simply the filter's vocabulary with nicer words.
|
||||
ReachBoth = "both"
|
||||
// ReachLocalNetwork is `internal` with the machine's own local networks added to what the filter
|
||||
// opens (novox/hq ADR 0340 §4): the internal name and not the public one, the mesh's own authority,
|
||||
// and the port opened to the private network and to the private networks the machine is attached to.
|
||||
ReachLocalNetwork = "local-network"
|
||||
)
|
||||
|
||||
// reaches is every value, in the order a refusal lists them.
|
||||
var reaches = []string{ReachMachine, ReachInternal, ReachPublic, ReachBoth}
|
||||
var reaches = []string{ReachMachine, ReachInternal, ReachPublic, ReachBoth, ReachLocalNetwork}
|
||||
|
||||
// RoutedPorts are the ports a module serves through a proxy, taken from its route contributions.
|
||||
//
|
||||
@@ -845,6 +898,8 @@ func FilterSource(reach string) (string, bool) {
|
||||
return FromMesh, true
|
||||
case ReachPublic, ReachBoth:
|
||||
return FromEverywhere, true
|
||||
case ReachLocalNetwork:
|
||||
return FromLocalNetwork, true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
@@ -854,7 +909,9 @@ func FilterSource(reach string) (string, bool) {
|
||||
func WantsPublicName(reach string) bool { return reach == ReachPublic || reach == ReachBoth }
|
||||
|
||||
// WantsInternalName is whether a reach asks for the route's internal name to be composed.
|
||||
func WantsInternalName(reach string) bool { return reach == ReachInternal || reach == ReachBoth }
|
||||
func WantsInternalName(reach string) bool {
|
||||
return reach == ReachInternal || reach == ReachBoth || reach == ReachLocalNetwork
|
||||
}
|
||||
|
||||
// Reaches reads a module's per-node reach settings: declared port → how far it reaches.
|
||||
//
|
||||
|
||||
@@ -6,7 +6,8 @@ import (
|
||||
)
|
||||
|
||||
// novox/hq issue 190, step 5 (ADR 0222): what the mesh computes for a module is held to the rule
|
||||
// every module is — no two modules on a machine declare one path, unit, name or package. The
|
||||
// every module is — no two modules on a machine declare one path, unit or name (a package may be declared by
|
||||
// several, novox/hq ADR 0303). The
|
||||
// private network once declared the container runtime's file and service beside the runtime's own
|
||||
// module, and nothing refused it, because the collision check only ever saw catalogue manifests.
|
||||
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Genesis assigns the build agent before the package registry exists (novox/hq ADR 0336, issue 121),
|
||||
// judged over the real catalogue. Genesis's step 13 was refused with "nothing provides
|
||||
// \"npm-package-registry\", wanted by build-agent": the forge genesis raises at its packages step is a
|
||||
// container, not the gitea module, and provides nothing.
|
||||
|
||||
// agentBuilt is the build agent as the mesh holds it once built: its image pinned.
|
||||
func agentBuilt(t *testing.T) Manifest {
|
||||
t.Helper()
|
||||
agent := catalogueManifest(t, "build-agent")
|
||||
built, err := agent.Resolve([]Built{{Name: "server", Kind: ArtifactImage,
|
||||
Reference: "mesh-builder@sha256:" + strings.Repeat("a", 64)}})
|
||||
if err != nil {
|
||||
t.Fatalf("the build agent does not resolve against its own build: %v", err)
|
||||
}
|
||||
return built
|
||||
}
|
||||
|
||||
func TestGenesisAssignsTheBuildAgentWithNothingProvidingTheRegistry(t *testing.T) {
|
||||
agent := agentBuilt(t)
|
||||
if !agent.IsOptional("npm-package-registry") {
|
||||
t.Fatalf("the build agent's npm-package-registry is hard (ADR 0336 §1): %v", agent.Requires)
|
||||
}
|
||||
if agent.IsOptional("artifact-store") {
|
||||
t.Fatal("the build agent's artifact-store is optional; it stays hard")
|
||||
}
|
||||
store := map[string][]Provider{"artifact-store": {
|
||||
{Node: "anchor", At: "anchor.internal", Module: "distribution", Serves: map[string]any{"port": float64(5000)}}}}
|
||||
books := shelf(agent, catalogueManifest(t, "gitea"), catalogueManifest(t, "distribution"))
|
||||
|
||||
// Genesis's node at step 13: the store answers, nothing provides the registry.
|
||||
got, err := Resolve(books, []string{"build-agent"}, reachable(), World{Offered: store})
|
||||
if err != nil {
|
||||
t.Fatalf("assigning the build agent with no registry was refused: %v", err)
|
||||
}
|
||||
if have := strings.Join(names(got), " "); have != "build-agent" {
|
||||
t.Fatalf("assigning the build agent put %s on the machine; nothing is installed for the registry", have)
|
||||
}
|
||||
if len(got.Absent) != 1 || got.Absent[0] != (Absent{Consumer: "build-agent", Requirement: "npm-package-registry"}) {
|
||||
t.Fatalf("the registry is not listed unbound for the build agent: %+v", got.Absent)
|
||||
}
|
||||
out, err := got.Declaration(Rendering{ArtifactStore: "anchor.internal:5000",
|
||||
Needed: map[string]map[string]string{"build-agent": {"broker": "sealed-broker"}}})
|
||||
if err != nil {
|
||||
t.Fatalf("the build agent does not compose with the registry unbound: %v", err)
|
||||
}
|
||||
for _, id := range []string{"build-agent." + BoundID("npm-package-registry"), "build-agent." + SecretID("npm-package-registry"),
|
||||
"build-agent." + NeedID("genesis-package-registry")} {
|
||||
if fileNamed(out, id) != nil {
|
||||
t.Errorf("%s was delivered with the registry unbound and no genesis document given", id)
|
||||
}
|
||||
}
|
||||
|
||||
// The genesis document, once the installer seals it at the packages step (ADR 0336 §3).
|
||||
out, err = got.Declaration(Rendering{ArtifactStore: "anchor.internal:5000",
|
||||
Needed: map[string]map[string]string{"build-agent": {"broker": "sealed-broker", "genesis-package-registry": "sealed-genesis"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
doc := fileNamed(out, "build-agent."+NeedID("genesis-package-registry"))
|
||||
if doc == nil || doc["sealed"] != "sealed-genesis" || !strings.HasSuffix(doc["path"].(string), "/genesis-package-registry.json") {
|
||||
t.Fatalf("the genesis document is not delivered where the agent reads it: %v", doc)
|
||||
}
|
||||
|
||||
// The gitea module assigned: the requirement binds, with no act on the build agent (ADR 0336 §2).
|
||||
registry := map[string][]Provider{"artifact-store": store["artifact-store"], "npm-package-registry": {
|
||||
{Node: "anchor", At: "anchor.internal", Module: "gitea",
|
||||
Serves: map[string]any{"port": float64(3000), "scheme": "http", "npm-path": "/api/packages/novox/npm/"}}}}
|
||||
bound, err := Resolve(books, []string{"build-agent"}, reachable(), World{Offered: registry})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := boundFrom(t, bound, "build-agent", "npm-package-registry"); n.Module != "gitea" {
|
||||
t.Fatalf("bound to %s", n.Module)
|
||||
}
|
||||
if len(bound.Absent) != 0 {
|
||||
t.Fatalf("the registry is still listed unbound once gitea is assigned: %+v", bound.Absent)
|
||||
}
|
||||
}
|
||||
@@ -25,13 +25,13 @@ func TestAnOfferMaySayItsSoftwareIssuesTheCredential(t *testing.T) {
|
||||
}
|
||||
|
||||
for want, raw := range map[string]string{
|
||||
"the one issuer an offer may name": `{"module":"d","version":"1",
|
||||
"the issuers an offer may name": `{"module":"d","version":"1",
|
||||
"provides":[{"name":"d-api","credential":{"issued-by":"vendor","tool":"d_token"}}]}`,
|
||||
"names no tool that answers it": `{"module":"d","version":"1",
|
||||
"provides":[{"name":"d-api","credential":{"issued-by":"software"}}]}`,
|
||||
"names its own secret": `{"module":"d","version":"1","own-secrets":{"token":{"path":"/p","taken":"at-start"}},
|
||||
"provides":[{"name":"d-api","credential":{"issued-by":"software","tool":"d_token","own":"token"}}]}`,
|
||||
"the one issuer an offer may name ": `{"module":"d","version":"1",
|
||||
"the issuers an offer may name ": `{"module":"d","version":"1",
|
||||
"provides":[{"name":"d-api","credential":{"tool":"d_token"}}]}`,
|
||||
} {
|
||||
want = strings.TrimSpace(want)
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A provision's credential may be one only a person outside the mesh can give (novox/hq ADR 0337 §1, issue 164):
|
||||
// the offer says `issued-by: outside` and names neither an own secret nor a tool, and the parser refuses it beside
|
||||
// either. An offer that says `software` keeps ADR 0305's shape.
|
||||
func TestAnOfferMaySayAPersonOutsideTheMeshIssuesItsCredential(t *testing.T) {
|
||||
m, err := ParseManifest([]byte(`{"module":"tv","version":"1",
|
||||
"provides":[{"name":"tv-api","scope":"mesh","credential":{"issued-by":"outside"}}],
|
||||
"serves":{"tv-api":{"port":8989}}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !m.IssuedOutsideOf("tv-api") {
|
||||
t.Fatal("the offer's credential issued outside was not read")
|
||||
}
|
||||
if m.IssuedOutsideOf("other") {
|
||||
t.Fatal("a provision the module does not offer reads as issued outside")
|
||||
}
|
||||
if _, issued := m.IssuedCredentialOf("tv-api"); issued {
|
||||
t.Fatal("a credential issued outside was read as issued by the software")
|
||||
}
|
||||
if _, shared := m.SharedCredentialOf("tv-api"); shared {
|
||||
t.Fatal("a credential issued outside was read as an own secret shared")
|
||||
}
|
||||
if who, offered := OffersIssuedOutside(map[string]Manifest{"tv": m}, "tv-api"); !offered || who != "tv" {
|
||||
t.Fatalf("the catalogue does not say tv offers tv-api issued outside: %q %v", who, offered)
|
||||
}
|
||||
|
||||
for want, raw := range map[string]string{
|
||||
"names its own secret": `{"module":"d","version":"1","own-secrets":{"key":{"path":"/p","taken":"at-start"}},
|
||||
"provides":[{"name":"d-api","credential":{"issued-by":"outside","own":"key"}}]}`,
|
||||
"names the tool": `{"module":"d","version":"1",
|
||||
"provides":[{"name":"d-api","credential":{"issued-by":"outside","tool":"d_key"}}]}`,
|
||||
"the issuers an offer may name": `{"module":"d","version":"1",
|
||||
"provides":[{"name":"d-api","credential":{"issued-by":"a-person"}}]}`,
|
||||
} {
|
||||
if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("expected a refusal saying %q, got %v", want, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// AttachedNetwork is one network a machine said it is directly attached to: the link it is on and the
|
||||
// network as address and prefix (novox/hq ADR 0340 §7), as the node-engine reports it among its facts.
|
||||
type AttachedNetwork struct {
|
||||
Link string `json:"link"`
|
||||
Network string `json:"network"`
|
||||
}
|
||||
|
||||
// LocalNetwork is one attached network judged: whether it counts as this machine's local network for a
|
||||
// port reached from `local-network`, and why — said for one that counts as well as for one that does
|
||||
// not, so `node` can show which counted and why the others did not (ADR 0340 §8).
|
||||
type LocalNetwork struct {
|
||||
Link string
|
||||
Network string
|
||||
Counts bool
|
||||
Why string
|
||||
}
|
||||
|
||||
// privateRanges are the only ranges a local network may lie in (ADR 0340 §2): RFC 1918 for IPv4 and the
|
||||
// unique local addresses for IPv6. A network counts only when it lies wholly inside one of them.
|
||||
var privateRanges = []struct {
|
||||
prefix netip.Prefix
|
||||
name string
|
||||
}{
|
||||
{netip.MustParsePrefix("10.0.0.0/8"), "a private range (RFC 1918, 10.0.0.0/8)"},
|
||||
{netip.MustParsePrefix("172.16.0.0/12"), "a private range (RFC 1918, 172.16.0.0/12)"},
|
||||
{netip.MustParsePrefix("192.168.0.0/16"), "a private range (RFC 1918, 192.168.0.0/16)"},
|
||||
{netip.MustParsePrefix("fc00::/7"), "the IPv6 unique local range (fc00::/7)"},
|
||||
}
|
||||
|
||||
// JudgeLocalNetworks says, for each network a machine reported, whether it counts as its local network.
|
||||
//
|
||||
// **A public subnet never counts** (ADR 0340 §2): a network counts only when it lies wholly in a private
|
||||
// range, so `local-network` can never open a port to the internet — a machine sitting on a public /24 has
|
||||
// that /24 judged and refused, never rendered. A link-local network does not count either. And only a
|
||||
// link the machine reported as facing outside it counts (§3): the private network's own link and a
|
||||
// container bridge are this machine's own, not its local network, whatever their addresses.
|
||||
//
|
||||
// Sorted by link and then network, one entry per (link, network), so what `node` shows and what the
|
||||
// filter is rendered from do not depend on the order a machine happened to say them in.
|
||||
func JudgeLocalNetworks(attached []AttachedNetwork, outward []string, tunnel string) []LocalNetwork {
|
||||
seen := map[[2]string]bool{}
|
||||
var out []LocalNetwork
|
||||
for _, a := range attached {
|
||||
key := [2]string{a.Link, a.Network}
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, judgeOne(a, outward, tunnel))
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Link != out[j].Link {
|
||||
return out[i].Link < out[j].Link
|
||||
}
|
||||
return out[i].Network < out[j].Network
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func judgeOne(a AttachedNetwork, outward []string, tunnel string) LocalNetwork {
|
||||
j := LocalNetwork{Link: a.Link, Network: a.Network}
|
||||
prefix, err := netip.ParsePrefix(strings.TrimSpace(a.Network))
|
||||
if err != nil {
|
||||
j.Why = "not a network the mesh can read (" + err.Error() + ")"
|
||||
return j
|
||||
}
|
||||
prefix = prefix.Masked()
|
||||
j.Network = prefix.String()
|
||||
switch {
|
||||
case tunnel != "" && a.Link == tunnel:
|
||||
j.Why = "the private network's own link, not the local network"
|
||||
return j
|
||||
case !slices.Contains(outward, a.Link):
|
||||
j.Why = fmt.Sprintf("%s does not face outside this machine (a bridge or a link of its own)", a.Link)
|
||||
return j
|
||||
case prefix.Addr().IsLoopback():
|
||||
j.Why = "loopback"
|
||||
return j
|
||||
case prefix.Addr().IsLinkLocalUnicast():
|
||||
j.Why = "link-local, never counted"
|
||||
return j
|
||||
}
|
||||
for _, r := range privateRanges {
|
||||
if r.prefix.Contains(prefix.Addr()) && prefix.Bits() >= r.prefix.Bits() {
|
||||
j.Counts = true
|
||||
j.Why = "in " + r.name
|
||||
return j
|
||||
}
|
||||
}
|
||||
if prefix.Addr().IsPrivate() {
|
||||
// Overlaps a private range without lying wholly inside it (a /7 around 10.0.0.0/8, say).
|
||||
j.Why = "not wholly in a private range, so never counted"
|
||||
return j
|
||||
}
|
||||
j.Why = "public, never counted: local-network may not open a port to the internet"
|
||||
return j
|
||||
}
|
||||
|
||||
// CountedLocalNetworks is the networks that count, as the filter renders them: sorted, without repeats.
|
||||
func CountedLocalNetworks(judged []LocalNetwork) []string {
|
||||
var out []string
|
||||
for _, j := range judged {
|
||||
if j.Counts && !slices.Contains(out, j.Network) {
|
||||
out = append(out, j.Network)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// LocalNetworkPort is one port the filter admits from `local-network`, and what that resolved to on this
|
||||
// machine (ADR 0340 §8): what `plan` shows for it.
|
||||
type LocalNetworkPort struct {
|
||||
Port int
|
||||
Protocol string
|
||||
Because []string
|
||||
// Networks is the local networks it is admitted from; empty when the machine reported none (§6).
|
||||
Networks []string
|
||||
}
|
||||
|
||||
// LocalNetworkPorts is every rule admitted from `local-network`, with the networks it resolved to.
|
||||
func LocalNetworkPorts(rules []Rule, networks []string) []LocalNetworkPort {
|
||||
var out []LocalNetworkPort
|
||||
for _, rule := range rules {
|
||||
if rule.From != FromLocalNetwork {
|
||||
continue
|
||||
}
|
||||
out = append(out, LocalNetworkPort{Port: rule.Port, Protocol: rule.Protocol,
|
||||
Because: rule.Because, Networks: networks})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Said is how `plan` says one such port: the networks, or that there are none and the port is admitted
|
||||
// from the private network only.
|
||||
func (p LocalNetworkPort) Said() string {
|
||||
by := strings.Join(p.Because, ", ")
|
||||
if len(p.Networks) == 0 {
|
||||
return fmt.Sprintf("%s/%d (%s) from the local network: none — this machine reported no private "+
|
||||
"network on a link facing outside, so it is admitted from the private network only", p.Protocol, p.Port, by)
|
||||
}
|
||||
return fmt.Sprintf("%s/%d (%s) from the local network: %s, and the private network", p.Protocol, p.Port, by,
|
||||
strings.Join(p.Networks, ", "))
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The home server's case (novox/hq issue 154): it sits on 192.168.1.0/24, and also on a public /24 a
|
||||
// provider handed it, beside the private network's own link and a container bridge.
|
||||
var homeServer = []AttachedNetwork{
|
||||
{Link: "eth0", Network: "192.168.1.10/24"},
|
||||
{Link: "eth1", Network: "203.0.113.7/24"},
|
||||
{Link: "eth0", Network: "fe80::1/64"},
|
||||
{Link: "eth0", Network: "fd12:3456:789a::5/64"},
|
||||
{Link: "wg-mesh", Network: "10.10.0.4/24"},
|
||||
{Link: "docker0", Network: "172.17.0.1/16"},
|
||||
}
|
||||
|
||||
func judged(t *testing.T) []LocalNetwork {
|
||||
t.Helper()
|
||||
return JudgeLocalNetworks(homeServer, []string{"eth0", "eth1"}, "wg-mesh")
|
||||
}
|
||||
|
||||
// **A public subnet never counts** (ADR 0340 §2), and only a private one on a link facing outside does
|
||||
// (§3): the tunnel's and a bridge's are this machine's own, whatever their addresses.
|
||||
func TestOnlyAPrivateNetworkOnALinkFacingOutsideCounts(t *testing.T) {
|
||||
got := CountedLocalNetworks(judged(t))
|
||||
want := []string{"192.168.1.0/24", "fd12:3456:789a::/64"}
|
||||
if strings.Join(got, " ") != strings.Join(want, " ") {
|
||||
t.Fatalf("counted %v, want %v", got, want)
|
||||
}
|
||||
for _, j := range judged(t) {
|
||||
if j.Why == "" {
|
||||
t.Errorf("%s on %s is judged without a why", j.Network, j.Link)
|
||||
}
|
||||
switch j.Network {
|
||||
case "203.0.113.0/24":
|
||||
if j.Counts || !strings.Contains(j.Why, "public") {
|
||||
t.Errorf("the public subnet is judged %v (%s), want never counted as public", j.Counts, j.Why)
|
||||
}
|
||||
case "fe80::/64":
|
||||
if j.Counts || !strings.Contains(j.Why, "link-local") {
|
||||
t.Errorf("a link-local network is judged %v (%s)", j.Counts, j.Why)
|
||||
}
|
||||
case "10.10.0.0/24", "172.17.0.0/16":
|
||||
if j.Counts {
|
||||
t.Errorf("%s on %s counts, and it is not the local network", j.Network, j.Link)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A network overlapping a private range without lying wholly inside it does not count either, so a
|
||||
// wide prefix around a private range cannot carry the internet with it.
|
||||
func TestANetworkNotWhollyInAPrivateRangeDoesNotCount(t *testing.T) {
|
||||
for _, n := range []string{"10.0.0.0/7", "172.0.0.0/8", "100.64.0.0/10", "0.0.0.0/0", "::/0"} {
|
||||
got := JudgeLocalNetworks([]AttachedNetwork{{Link: "eth0", Network: n}}, []string{"eth0"}, "")
|
||||
if len(got) != 1 || got[0].Counts {
|
||||
t.Errorf("%s counts as a local network: %+v", n, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **The filter admits the local network on that port's rule alone** (§5), beside the private network,
|
||||
// in both chains, and never the public subnet.
|
||||
func TestALocalNetworkPortIsAdmittedFromItsNetworksAndTheMesh(t *testing.T) {
|
||||
rules := []Rule{
|
||||
{Port: 1883, Protocol: "tcp", From: FromLocalNetwork, Because: []string{"mosquitto"}},
|
||||
{Port: 5432, Protocol: "tcp", From: FromMesh, Because: []string{"postgres"}},
|
||||
}
|
||||
local := CountedLocalNetworks(judged(t))
|
||||
out := AsNftablesWithLocal(rules, []string{"10.10.0.1", "10.10.0.4"}, false, nil,
|
||||
[]string{"eth0", "eth1"}, "wg-mesh", local)
|
||||
|
||||
for _, want := range []string{
|
||||
"ip saddr { 10.10.0.1, 10.10.0.4, 192.168.1.0/24 } tcp dport 1883 accept",
|
||||
"ip6 saddr { fd12:3456:789a::/64 } tcp dport 1883 accept",
|
||||
"ip saddr { 10.10.0.1, 10.10.0.4, 192.168.1.0/24 } ct original proto-dst 1883 accept",
|
||||
// The mesh-only port is not widened by the machine having a local network.
|
||||
"ip saddr { 10.10.0.1, 10.10.0.4 } tcp dport 5432 accept",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("the rule set lacks %q:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
if strings.Contains(out, "203.0.113") {
|
||||
t.Errorf("the public subnet reached the rule set:\n%s", out)
|
||||
}
|
||||
if strings.Contains(out, "tcp dport 1883 accept\n") && !strings.Contains(out, "saddr") {
|
||||
t.Errorf("1883 is open to anywhere:\n%s", out)
|
||||
}
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
if strings.TrimSpace(line) == "tcp dport 1883 accept" {
|
||||
t.Fatalf("1883 is open to anywhere:\n%s", out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Nothing resolved is closed and said** (§6): a machine that reported no private network admits the
|
||||
// port from the private network only, and the rule set says it found none — never widened to anywhere.
|
||||
func TestNoLocalNetworkIsSaidAndNotWidened(t *testing.T) {
|
||||
rules := []Rule{{Port: 1883, Protocol: "tcp", From: FromLocalNetwork, Because: []string{"mosquitto"}}}
|
||||
out := AsNftablesWithLocal(rules, []string{"10.10.0.1"}, false, nil, []string{"eth0"}, "wg-mesh", nil)
|
||||
if !strings.Contains(out, "1883 wanted the local network, and this machine reported no private network") {
|
||||
t.Errorf("the rule set does not say the local network resolved to nothing:\n%s", out)
|
||||
}
|
||||
if !strings.Contains(out, "ip saddr { 10.10.0.1 } tcp dport 1883 accept") {
|
||||
t.Errorf("the port is not admitted from the private network:\n%s", out)
|
||||
}
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
if strings.TrimSpace(line) == "tcp dport 1883 accept" {
|
||||
t.Fatalf("an unresolved local network was widened to anywhere:\n%s", out)
|
||||
}
|
||||
}
|
||||
said := LocalNetworkPorts(rules, nil)[0].Said()
|
||||
if !strings.Contains(said, "none") {
|
||||
t.Errorf("plan says %q, want it to say no local network was found", said)
|
||||
}
|
||||
}
|
||||
|
||||
// `local-network` is a source a manifest may name (§1, changing ADR 0045 §3), and an unknown source is
|
||||
// still refused.
|
||||
func TestAManifestMayListenFromTheLocalNetwork(t *testing.T) {
|
||||
if _, err := ParseManifest([]byte(`{"module":"mosquitto","version":"1",
|
||||
"listens":[{"port":1883,"from":"local-network"}]}`)); err != nil {
|
||||
t.Fatalf("a listen from the local network is refused: %v", err)
|
||||
}
|
||||
_, err := ParseManifest([]byte(`{"module":"mosquitto","version":"1",
|
||||
"listens":[{"port":1883,"from":"lan"}]}`))
|
||||
if err == nil || !strings.Contains(err.Error(), `from "lan"`) {
|
||||
t.Fatalf("a source the mesh cannot render is not refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A fourth reach** (§4, changing ADR 0138 §4): `internal` with the local networks added to what the
|
||||
// filter opens — the internal name, not the public one.
|
||||
func TestTheLocalNetworkReachIsInternalWithTheLocalNetworkAdded(t *testing.T) {
|
||||
public, internal := namesFor(t, aRoutedWeb(), reachSet(ReachLocalNetwork))
|
||||
if public != "" || internal != "app.anchor.internal" {
|
||||
t.Fatalf("names are %q and %q, want the internal name only", public, internal)
|
||||
}
|
||||
bare := aRoutedWeb()
|
||||
bare.Contributes = nil
|
||||
r := Resolution{Node: "anchor", Modules: []Manifest{bare}}
|
||||
rules, err := r.Rules(Rendering{Settings: reachSet(ReachLocalNetwork)})
|
||||
if err != nil {
|
||||
t.Fatalf("rules: %v", err)
|
||||
}
|
||||
if len(rules) != 1 || rules[0].From != FromLocalNetwork {
|
||||
t.Fatalf("reach local-network made the filter say %+v", rules)
|
||||
}
|
||||
}
|
||||
|
||||
// Two modules wanting one port, one from the mesh and one from the local network, render one rule from
|
||||
// the local network: it covers the mesh, and anywhere covers it.
|
||||
func TestTheLocalNetworkSitsBetweenTheMeshAndAnywhere(t *testing.T) {
|
||||
got := widest([]Rule{
|
||||
{Port: 53, Protocol: "udp", From: FromMesh, Because: []string{"a"}},
|
||||
{Port: 53, Protocol: "udp", From: FromLocalNetwork, Because: []string{"b"}},
|
||||
})
|
||||
if len(got) != 1 || got[0].From != FromLocalNetwork || len(got[0].Because) != 2 {
|
||||
t.Fatalf("got %+v", got)
|
||||
}
|
||||
got = widest(append(got, Rule{Port: 53, Protocol: "udp", From: FromEverywhere, Because: []string{"c"}}))
|
||||
if len(got) != 1 || got[0].From != FromEverywhere {
|
||||
t.Fatalf("got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// On an adopted node the opening is the private network's part only: the found firewall admits the
|
||||
// local network, and an opening that widened to everywhere would open the port to the internet.
|
||||
func TestAnAdoptedNodeOpensTheLocalNetworkPortToTheMeshOnly(t *testing.T) {
|
||||
out := Openings([]Rule{{Port: 1883, Protocol: "tcp", From: FromLocalNetwork}}, nil, nil)
|
||||
if len(out) != 1 || out[0]["from"] != OpeningFromMesh {
|
||||
t.Fatalf("openings %v", out)
|
||||
}
|
||||
}
|
||||
|
||||
// A global unicast IPv6 subnet on a link facing outside is public, and never counts: the IPv6 side of the
|
||||
// rule that `local-network` cannot open a port to the internet (ADR 0340 §2).
|
||||
func TestAGlobalIPv6SubnetIsPublicAndNeverCounts(t *testing.T) {
|
||||
got := JudgeLocalNetworks([]AttachedNetwork{{Link: "eth0", Network: "2001:db8:1234::7/48"}}, []string{"eth0"}, "wg-mesh")
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("judged %+v", got)
|
||||
}
|
||||
if got[0].Counts {
|
||||
t.Fatalf("the global IPv6 subnet %s counts as a local network", got[0].Network)
|
||||
}
|
||||
if got[0].Network != "2001:db8:1234::/48" || !strings.Contains(got[0].Why, "public, never counted") {
|
||||
t.Fatalf("judged %s with why %q, want it named public and never counted", got[0].Network, got[0].Why)
|
||||
}
|
||||
if n := CountedLocalNetworks(got); len(n) != 0 {
|
||||
t.Fatalf("counted %v", n)
|
||||
}
|
||||
}
|
||||
+104
-10
@@ -269,7 +269,8 @@ func KeepsConsumerData(catalogue map[string]Manifest, provision string) bool {
|
||||
func (o Offer) MachineReach() bool { return o.Reach == ReachMachine }
|
||||
|
||||
// OfferCredential names which of the provider's own secrets a provision's consumers receive — or, with
|
||||
// IssuedBy, that the provider's software issues the provision's one credential itself (novox/hq ADR 0305,
|
||||
// IssuedBy "outside", that only a person can give the provision's credential (novox/hq ADR 0337, issue 164) — or,
|
||||
// with IssuedBy "software", that the provider's software issues the provision's one credential itself (novox/hq ADR 0305,
|
||||
// issue 526): a media server's sign-in token from its vendor, which the mesh can neither make nor rotate.
|
||||
// Tool is then the provider's own tool that answers that value, sealed to a key made for the controller's
|
||||
// one call, and to the controller alone.
|
||||
@@ -293,6 +294,34 @@ func (m Manifest) IssuedCredentialOf(provision string) (string, bool) {
|
||||
return "", false
|
||||
}
|
||||
|
||||
// IssuedOutsideOf is whether this module's offer of a provision says only a person outside the mesh can give its
|
||||
// credential (novox/hq ADR 0337): the mesh never makes a value for it, and a consumer holds one only once a person
|
||||
// gave it with `secret accept`.
|
||||
func (m Manifest) IssuedOutsideOf(provision string) bool {
|
||||
for _, o := range m.Provides {
|
||||
if o.Name == provision && o.Credential != nil && o.Credential.IssuedBy == IssuedOutside {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// OffersIssuedOutside is whether any module of a catalogue offers the provision with a credential issued outside
|
||||
// the mesh (novox/hq ADR 0337), and the first such module by name.
|
||||
func OffersIssuedOutside(shelf map[string]Manifest, provision string) (string, bool) {
|
||||
var names []string
|
||||
for name, m := range shelf {
|
||||
if m.IssuedOutsideOf(provision) {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
if len(names) == 0 {
|
||||
return "", false
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names[0], true
|
||||
}
|
||||
|
||||
// SharedCredentialOf is the own secret an offer of this module names as the provision's credential,
|
||||
// and whether it names one.
|
||||
func (m Manifest) SharedCredentialOf(provision string) (string, bool) {
|
||||
@@ -385,6 +414,10 @@ type Manifest struct {
|
||||
|
||||
// Requires are names that must be provided by something assigned to the same node.
|
||||
Requires []string `json:"requires,omitempty"`
|
||||
// OptionalRequires is which of Requires the module can run without (novox/hq ADR 0330): filled
|
||||
// by UnmarshalJSON from the long form of a `requires` entry, {name, optional}, or from the stored
|
||||
// form's `optional-requires` list; never written by hand.
|
||||
OptionalRequires map[string]bool `json:"-"`
|
||||
|
||||
// Emits are the events this module publishes, named **locally**: `order.placed`, not a subject
|
||||
// and not a routing key. The mesh derives where it lands (design 29 §1), so reorganising the
|
||||
@@ -1132,6 +1165,12 @@ const (
|
||||
FromEverywhere = "anywhere"
|
||||
// FromMachine is this machine only — a port bound for something else on the same host.
|
||||
FromMachine = "machine"
|
||||
// FromLocalNetwork is the private network and the networks this machine is directly attached to
|
||||
// through its uplink, as it reports them, and only those wholly in a private range (novox/hq ADR
|
||||
// 0340): a local service reached by devices that will never be nodes, said as exactly that rather
|
||||
// than as `anywhere`. A public subnet the machine sits on never counts, so it cannot open a port to
|
||||
// the internet.
|
||||
FromLocalNetwork = "local-network"
|
||||
)
|
||||
|
||||
// At is this port's protocol, with the default applied.
|
||||
@@ -1354,6 +1393,35 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
|
||||
}
|
||||
delete(keys, "contributes")
|
||||
}
|
||||
var optionalRequires map[string]bool
|
||||
if requires, ok := keys["requires"]; ok && string(requires) != "null" {
|
||||
names, optional, err := readRequires(requires)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
plainNames, err := json.Marshal(names)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
keys["requires"] = plainNames
|
||||
optionalRequires = optional
|
||||
}
|
||||
// The stored form: `requires` plain, and which are optional beside it (OptionalRequiresKey).
|
||||
if stored, ok := keys[OptionalRequiresKey]; ok {
|
||||
if string(stored) != "null" {
|
||||
optional, err := readOptionalRequires(stored)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for name := range optional {
|
||||
if optionalRequires == nil {
|
||||
optionalRequires = map[string]bool{}
|
||||
}
|
||||
optionalRequires[name] = true
|
||||
}
|
||||
}
|
||||
delete(keys, OptionalRequiresKey)
|
||||
}
|
||||
rest, err := json.Marshal(keys)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1385,6 +1453,7 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
|
||||
}
|
||||
*m = Manifest(fields)
|
||||
m.unknown = unknown
|
||||
m.OptionalRequires = optionalRequires
|
||||
if len(plain) > 0 {
|
||||
m.Secrets = plain
|
||||
}
|
||||
@@ -1407,13 +1476,22 @@ func (m Manifest) MarshalJSON() ([]byte, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(m.SecretsMany) == 0 && len(m.ContributesMany) == 0 {
|
||||
if len(m.SecretsMany) == 0 && len(m.ContributesMany) == 0 && len(m.OptionalRequires) == 0 {
|
||||
return raw, nil
|
||||
}
|
||||
var keys map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &keys); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if optional := optionalRequiresList(m); len(optional) > 0 {
|
||||
// Beside `requires`, never inside it: an older controller reads `requires` as plain names, and one
|
||||
// manifest it cannot read stops the whole catalogue (OptionalRequiresKey).
|
||||
listed, err := json.Marshal(optional)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys[OptionalRequiresKey] = listed
|
||||
}
|
||||
if len(m.ContributesMany) > 0 {
|
||||
mergedContributes := map[string]any{}
|
||||
for to, values := range m.Contributes {
|
||||
@@ -1604,14 +1682,28 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
"makes is %d", m.Module, p, id.Max, shortest))
|
||||
}
|
||||
}
|
||||
if c := offer.Credential; c != nil && (c.IssuedBy != "" || c.Tool != "") {
|
||||
if c := offer.Credential; c != nil && c.IssuedBy == IssuedOutside {
|
||||
// Only a person can give it (novox/hq ADR 0337 §1, issue 164): the software holds a value the mesh can
|
||||
// neither make nor fetch, so the offer names nothing that would answer it — no own secret, no tool.
|
||||
switch {
|
||||
case c.Own != "":
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s provides %q with a credential issued outside the mesh and names its own secret %q too; a "+
|
||||
"person gives the value with `secret accept`, so the offer names no own secret (ADR 0337)",
|
||||
m.Module, p, c.Own))
|
||||
case c.Tool != "":
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s provides %q with a credential issued outside the mesh and names the tool %q; only a "+
|
||||
"person gives it, so the offer names no tool (ADR 0337)", m.Module, p, c.Tool))
|
||||
}
|
||||
} else if c := offer.Credential; c != nil && (c.IssuedBy != "" || c.Tool != "") {
|
||||
// The software issues it (novox/hq ADR 0305): named by the tool that answers it, never by an own
|
||||
// secret, which would be a second copy for the mesh to keep in step with the software's.
|
||||
switch {
|
||||
case c.IssuedBy != IssuedBySoftware:
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s provides %q with a credential issued by %q; the one issuer an offer may name is %q (ADR 0305)",
|
||||
m.Module, p, c.IssuedBy, IssuedBySoftware))
|
||||
"%s provides %q with a credential issued by %q; the issuers an offer may name are %q and %q "+
|
||||
"(ADR 0305, ADR 0337)", m.Module, p, c.IssuedBy, IssuedBySoftware, IssuedOutside))
|
||||
case c.Own != "":
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s provides %q with a credential its software issues and names its own secret %q too; the "+
|
||||
@@ -1693,6 +1785,8 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
problems = append(problems, LogProblems(m)...)
|
||||
// And the defaults it gives its settings (setting_defaults.go, novox/hq ADR 0262).
|
||||
problems = append(problems, SettingProblems(m)...)
|
||||
// And what it requires optionally, and the files it delivers only while bound (novox/hq ADR 0330).
|
||||
problems = append(problems, OptionalProblems(m)...)
|
||||
// And which setting keys each mergeable file takes (setting_takes.go, novox/hq ADR 0326).
|
||||
problems = append(problems, TakenSettingsProblems(m)...)
|
||||
wellFormed := true
|
||||
@@ -1841,17 +1935,17 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
"%s listens on port %d, which is not a port", m.Module, l.Port))
|
||||
}
|
||||
switch l.From {
|
||||
case FromMesh, FromEverywhere, FromMachine:
|
||||
case FromMesh, FromEverywhere, FromMachine, FromLocalNetwork:
|
||||
case "":
|
||||
// The fault this field exists to prevent. A rule with no source is open, and a
|
||||
// manifest that omitted it would read as a restriction and be none.
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s listens on %d and does not say from where; it is %q, %q or %q",
|
||||
m.Module, l.Port, FromMesh, FromEverywhere, FromMachine))
|
||||
"%s listens on %d and does not say from where; it is %q, %q, %q or %q",
|
||||
m.Module, l.Port, FromMesh, FromEverywhere, FromMachine, FromLocalNetwork))
|
||||
default:
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s listens on %d from %q; it is %q, %q or %q",
|
||||
m.Module, l.Port, l.From, FromMesh, FromEverywhere, FromMachine))
|
||||
"%s listens on %d from %q; it is %q, %q, %q or %q",
|
||||
m.Module, l.Port, l.From, FromMesh, FromEverywhere, FromMachine, FromLocalNetwork))
|
||||
}
|
||||
if p := l.At(); p != "tcp" && p != "udp" {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
|
||||
@@ -0,0 +1,282 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A requirement a module can run without (novox/hq ADR 0330, issue 166).
|
||||
//
|
||||
// **Hard stays the default.** A requirement written as a bare name is hard, exactly as before: it
|
||||
// resolves, or the assignment is refused, and a single catalogue candidate is installed for it. One
|
||||
// written `{"name": "influxdb-api", "optional": true}` resolves the same way when a provider is
|
||||
// already assigned in the mesh or reachable from the consumer (§2), and is otherwise **absent**: not
|
||||
// refused, and nothing is installed for it, on this machine or elsewhere (§3).
|
||||
//
|
||||
// **An absent requirement is never rendered as an empty value** (§4–§6). Its binding, its pair
|
||||
// credential and its grant are not composed, so a file that names one of them is delivered only while
|
||||
// the requirement is bound (`only-when-bound`), and a `${bound:…}` placeholder of it outside such a
|
||||
// file names the value it takes while absent: `${bound:<provision>:<key>|default:<value>}`. The
|
||||
// module check refuses anything else, so the gap is found when the manifest is written, not on a
|
||||
// machine where it reads as a blank.
|
||||
|
||||
// OnlyWhenBound is a resource's field naming the optional requirement it is delivered only while
|
||||
// bound (ADR 0330 §5). Said in the catalogue and dropped before the declaration is sent: the
|
||||
// node-engine parses strictly and knows no such field.
|
||||
const OnlyWhenBound = "only-when-bound"
|
||||
|
||||
// requirementForm is one entry of `requires` in its long form.
|
||||
type requirementForm struct {
|
||||
Name string `json:"name"`
|
||||
Optional bool `json:"optional,omitempty"`
|
||||
}
|
||||
|
||||
// readRequires reads `requires` in both of its forms — a bare name, or {name, optional} — into the
|
||||
// names and which of them are optional.
|
||||
func readRequires(raw json.RawMessage) ([]string, map[string]bool, error) {
|
||||
var entries []json.RawMessage
|
||||
if err := json.Unmarshal(raw, &entries); err != nil {
|
||||
return nil, nil, fmt.Errorf("requires: a list of names, or of {name, optional}: %w", err)
|
||||
}
|
||||
var names []string
|
||||
optional := map[string]bool{}
|
||||
for i, e := range entries {
|
||||
var plain string
|
||||
if err := json.Unmarshal(e, &plain); err == nil {
|
||||
names = append(names, plain)
|
||||
continue
|
||||
}
|
||||
var full requirementForm
|
||||
dec := json.NewDecoder(bytes.NewReader(e))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&full); err != nil {
|
||||
return nil, nil, fmt.Errorf("requires[%d]: a name, or {name, optional} (novox/hq ADR 0330): %w", i, err)
|
||||
}
|
||||
if strings.TrimSpace(full.Name) == "" {
|
||||
return nil, nil, fmt.Errorf("requires[%d] names no requirement: {name, optional}", i)
|
||||
}
|
||||
names = append(names, full.Name)
|
||||
if full.Optional {
|
||||
optional[full.Name] = true
|
||||
}
|
||||
}
|
||||
if len(optional) == 0 {
|
||||
optional = nil
|
||||
}
|
||||
return names, optional, nil
|
||||
}
|
||||
|
||||
// OptionalRequiresKey is where a stored manifest keeps which of its requirements are optional: beside
|
||||
// `requires`, which stays a plain list of names (mesh-controller #281's review). A controller older than
|
||||
// optional requirements reads `requires` as names; the long form inside it failed that read, and with it
|
||||
// the whole catalogue, so a rollback below this controller would have composed no machine. An older
|
||||
// controller sets this unknown key aside instead and leaves only the module that carries it out. A
|
||||
// manifest may be written either way; it is stored this way.
|
||||
const OptionalRequiresKey = "optional-requires"
|
||||
|
||||
// readOptionalRequires reads the stored form's list of optional requirements.
|
||||
func readOptionalRequires(raw json.RawMessage) (map[string]bool, error) {
|
||||
var names []string
|
||||
if err := json.Unmarshal(raw, &names); err != nil {
|
||||
return nil, fmt.Errorf("%s: a list of the names in `requires` that are optional: %w", OptionalRequiresKey, err)
|
||||
}
|
||||
out := map[string]bool{}
|
||||
for _, n := range names {
|
||||
out[n] = true
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// optionalRequiresList is the stored form's list: the optional requirements, sorted.
|
||||
func optionalRequiresList(m Manifest) []string {
|
||||
var out []string
|
||||
for name, optional := range m.OptionalRequires {
|
||||
if optional {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// IsOptional is whether this module requires the name optionally (ADR 0330 §1). A name it contributes
|
||||
// to without requiring, or requires as a bare name, is hard.
|
||||
func (m Manifest) IsOptional(requirement string) bool { return m.OptionalRequires[requirement] }
|
||||
|
||||
// wantsHard is whether this module wants the name as a hard requirement: required without
|
||||
// `optional`, or contributed to.
|
||||
func (m Manifest) wantsHard(name string) bool {
|
||||
for _, w := range m.Wants() {
|
||||
if w == name {
|
||||
return !m.IsOptional(name)
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Absent is one optional requirement of one consumer that nothing assigned answers (ADR 0330 §3):
|
||||
// left unbound rather than refused, and listed by `plan` and `node` (§7).
|
||||
type Absent struct {
|
||||
// Consumer is the module that requires it optionally.
|
||||
Consumer string `json:"consumer"`
|
||||
// Requirement is what it requires.
|
||||
Requirement string `json:"requirement"`
|
||||
}
|
||||
|
||||
// AbsentFor is the optional requirements of one module this resolution leaves unbound.
|
||||
func (r Resolution) AbsentFor(module string) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, a := range r.Absent {
|
||||
if a.Consumer == module {
|
||||
out[a.Requirement] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// absentOf is, for each requirement left absent, every module of the set that requires it
|
||||
// optionally, sorted by consumer then requirement.
|
||||
func absentOf(names map[string]bool, order []string, catalogue map[string]Manifest) []Absent {
|
||||
var out []Absent
|
||||
for _, module := range order {
|
||||
m := catalogue[module]
|
||||
for _, r := range m.Requires {
|
||||
if names[r] && m.IsOptional(r) {
|
||||
out = append(out, Absent{Consumer: module, Requirement: r})
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Consumer != out[j].Consumer {
|
||||
return out[i].Consumer < out[j].Consumer
|
||||
}
|
||||
return out[i].Requirement < out[j].Requirement
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// withoutAbsent drops the resources delivered only while a requirement is bound whose requirement is
|
||||
// absent here (ADR 0330 §5), and the field itself from every resource that keeps it.
|
||||
func withoutAbsent(resources []map[string]any, absent map[string]bool) []map[string]any {
|
||||
out := make([]map[string]any, 0, len(resources))
|
||||
for _, r := range resources {
|
||||
when, has := r[OnlyWhenBound]
|
||||
if !has {
|
||||
out = append(out, r)
|
||||
continue
|
||||
}
|
||||
if absent[fmt.Sprint(when)] {
|
||||
continue
|
||||
}
|
||||
kept := make(map[string]any, len(r))
|
||||
for k, v := range r {
|
||||
if k != OnlyWhenBound {
|
||||
kept[k] = v
|
||||
}
|
||||
}
|
||||
out = append(out, kept)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// secretNamesOf is every `${secret:…}` name that carries a credential of this requirement: its own
|
||||
// name, and each local name it keeps several under (ADR 0094).
|
||||
func secretNamesOf(m Manifest, requirement string) []string {
|
||||
names := []string{requirement}
|
||||
for local := range m.SecretsMany[requirement] {
|
||||
names = append(names, local)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
// OptionalProblems is what the module check refuses of a manifest's optional requirements and its
|
||||
// only-when-bound resources (ADR 0330 §4–§6).
|
||||
func OptionalProblems(m Manifest) []string {
|
||||
var problems []string
|
||||
required := map[string]bool{}
|
||||
for _, r := range m.Requires {
|
||||
if required[r] && m.IsOptional(r) {
|
||||
problems = append(problems, fmt.Sprintf("%s requires %q twice; say it once, optional or not", m.Module, r))
|
||||
}
|
||||
required[r] = true
|
||||
}
|
||||
for _, r := range optionalRequiresList(m) {
|
||||
if !required[r] {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s says %q is optional in %s and does not require it", m.Module, r, OptionalRequiresKey))
|
||||
}
|
||||
}
|
||||
// A contribution is to whoever answers the requirement, so it goes with it: contributing to one
|
||||
// that is optional is allowed, and composed only while it is bound.
|
||||
for _, r := range m.Resources {
|
||||
id := fmt.Sprint(r["id"])
|
||||
when, marked := r[OnlyWhenBound]
|
||||
on := ""
|
||||
if marked {
|
||||
name, ok := when.(string)
|
||||
switch {
|
||||
case !ok || name == "":
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s's resource %s says %s %v; it names one requirement the module requires optionally",
|
||||
m.Module, id, OnlyWhenBound, when))
|
||||
continue
|
||||
case fmt.Sprint(r["type"]) != "file":
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s's resource %s is a %v and says %s: only a file is left out while a requirement is "+
|
||||
"absent (novox/hq ADR 0330 §5)", m.Module, id, r["type"], OnlyWhenBound))
|
||||
continue
|
||||
case !required[name]:
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s's resource %s is delivered only when %q is bound, and %s does not require it",
|
||||
m.Module, id, name, m.Module))
|
||||
continue
|
||||
case !m.IsOptional(name):
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s's resource %s is delivered only when %q is bound, which is a hard requirement and "+
|
||||
"always bound: mark the requirement optional, or drop %s", m.Module, id, name, OnlyWhenBound))
|
||||
continue
|
||||
}
|
||||
on = name
|
||||
}
|
||||
if fmt.Sprint(r["type"]) != "file" {
|
||||
continue
|
||||
}
|
||||
content, _ := r["content"].(string)
|
||||
for _, use := range bound.FindAllStringSubmatch(content, -1) {
|
||||
provision, key, hasDefault := use[1], use[2], use[3] != ""
|
||||
if !m.IsOptional(provision) {
|
||||
if hasDefault {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s's resource %s gives ${bound:%s:%s} a default, and %q is a hard requirement, always "+
|
||||
"bound: a default it never renders only misleads a reader", m.Module, id, provision, key, provision))
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !hasDefault && on != provision {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s's resource %s says ${bound:%s:%s}, and %q is optional: while it is absent this would "+
|
||||
"be empty. Name its value then — ${bound:%s:%s|default:<value>} — or deliver the file "+
|
||||
"only while it is bound, with \"%s\": %q (novox/hq ADR 0330 §4)",
|
||||
m.Module, id, provision, key, provision, provision, key, OnlyWhenBound, provision))
|
||||
}
|
||||
}
|
||||
for _, provision := range m.Requires {
|
||||
if !m.IsOptional(provision) || on == provision {
|
||||
continue
|
||||
}
|
||||
for _, name := range secretNamesOf(m, provision) {
|
||||
if strings.Contains(content, "${secret:"+name+"}") {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s's resource %s says ${secret:%s}, the credential of %q, which is optional and not "+
|
||||
"delivered while absent: deliver the file only while it is bound, with \"%s\": %q "+
|
||||
"(novox/hq ADR 0330 §6)", m.Module, id, name, provision, OnlyWhenBound, provision))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A requirement a module can run without (novox/hq ADR 0330, issue 166). The shape of the issue:
|
||||
// grafana runs without a data source and requires `influxdb-api` softly; a mesh without influxdb
|
||||
// could not assign grafana at all, and one with influxdb in its catalogue would have had it installed
|
||||
// for grafana.
|
||||
|
||||
func optionalShelf() map[string]Manifest {
|
||||
grafana := Manifest{Module: "grafana", Version: "1", Requires: []string{"influxdb-api", "shell"},
|
||||
OptionalRequires: map[string]bool{"influxdb-api": true, "shell": true}}
|
||||
return shelf(
|
||||
Manifest{Module: "influxdb", Version: "1", Provides: FromAnywhere("influxdb-api")},
|
||||
Manifest{Module: "zsh", Version: "1", Provides: Offers("shell")},
|
||||
grafana,
|
||||
// What wants the same names hard, for the cases where one consumer's optional is another's hard.
|
||||
Manifest{Module: "telegraf", Version: "1", Requires: []string{"influxdb-api"}},
|
||||
Manifest{Module: "editor", Version: "1", Requires: []string{"shell"}},
|
||||
Manifest{Module: "suite", Version: "1", Requires: []string{"editor"}},
|
||||
)
|
||||
}
|
||||
|
||||
func TestARequirementIsHardUnlessMarkedOptional(t *testing.T) {
|
||||
m := mustParse(t, `{"module":"grafana","version":"1",
|
||||
"requires":["route",{"name":"influxdb-api","optional":true},{"name":"oidc-client"}]}`)
|
||||
if strings.Join(m.Requires, " ") != "route influxdb-api oidc-client" {
|
||||
t.Fatalf("the requirements read as %v", m.Requires)
|
||||
}
|
||||
if m.IsOptional("route") || m.IsOptional("oidc-client") {
|
||||
t.Error("a requirement without `optional` read as optional; hard is the default (ADR 0330 §1)")
|
||||
}
|
||||
if !m.IsOptional("influxdb-api") {
|
||||
t.Error("a requirement marked optional read as hard")
|
||||
}
|
||||
// And written back as it was read, so a manifest through the store keeps its form.
|
||||
raw, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var again Manifest
|
||||
if err := json.Unmarshal(raw, &again); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !again.IsOptional("influxdb-api") || again.IsOptional("route") || len(again.Requires) != 3 {
|
||||
t.Fatalf("written and read back, it says %v optional %v (%s)", again.Requires, again.OptionalRequires, raw)
|
||||
}
|
||||
if strings.Contains(string(raw), `"name":`) {
|
||||
t.Errorf("a requirement was stored in the long form: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// The stored form reads on a controller older than optional requirements (review of mesh-controller
|
||||
// #281). Inventory.Catalogue stops at the first manifest it cannot read, so a long-form entry inside
|
||||
// `requires` would have stopped every plan after a rollback. Stored, `requires` is plain names, as main's
|
||||
// manifest type reads it, and which are optional sits beside it in a key such a controller sets aside.
|
||||
func TestTheStoredFormReadsWhereRequiresIsAListOfNames(t *testing.T) {
|
||||
m := mustParse(t, `{"module":"grafana","version":"1",
|
||||
"requires":["route",{"name":"influxdb-api","optional":true}]}`)
|
||||
raw, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// main's field, exactly: `Requires []string`.
|
||||
var older struct {
|
||||
Requires []string `json:"requires"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &older); err != nil {
|
||||
t.Fatalf("a controller reading requires as names cannot read the stored form: %v\n%s", err, raw)
|
||||
}
|
||||
if strings.Join(older.Requires, " ") != "route influxdb-api" {
|
||||
t.Fatalf("it reads the requirements as %v", older.Requires)
|
||||
}
|
||||
var keys map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &keys); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(keys[OptionalRequiresKey]) != `["influxdb-api"]` {
|
||||
t.Fatalf("which are optional is stored as %s", keys[OptionalRequiresKey])
|
||||
}
|
||||
// And this controller reads the stored form back the same, leniently and strictly.
|
||||
var back Manifest
|
||||
if err := json.Unmarshal(raw, &back); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
strict, err := ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, got := range []Manifest{back, strict} {
|
||||
if !got.IsOptional("influxdb-api") || got.IsOptional("route") || len(got.Requires) != 2 {
|
||||
t.Fatalf("read back as %v optional %v", got.Requires, got.OptionalRequires)
|
||||
}
|
||||
}
|
||||
// A stored list naming something the module does not require is refused.
|
||||
if _, err := ParseManifest([]byte(`{"module":"x","version":"1","requires":["route"],"optional-requires":["mqtt-topic"]}`)); err == nil {
|
||||
t.Error("an optional requirement the module does not require passed the check")
|
||||
}
|
||||
}
|
||||
|
||||
func TestARequirementsLongFormRefusesWhatItDoesNotKnow(t *testing.T) {
|
||||
for _, raw := range []string{
|
||||
`{"module":"x","version":"1","requires":[{"name":"influxdb-api","optinal":true}]}`,
|
||||
`{"module":"x","version":"1","requires":[{"optional":true}]}`,
|
||||
} {
|
||||
if _, err := ParseManifest([]byte(raw)); err == nil {
|
||||
t.Errorf("accepted %s", raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOptionalRequirementWithNoProviderIsAbsentAndNothingIsInstalled(t *testing.T) {
|
||||
// Exactly one catalogue module could answer each — the case where a hard requirement installs it.
|
||||
// Brokered (influxdb-api, from anywhere) and on this machine (shell); both passes.
|
||||
for _, world := range []World{{}, {Unchecked: true}} {
|
||||
got, err := Resolve(optionalShelf(), []string{"grafana"}, reachable(), world)
|
||||
if err != nil {
|
||||
t.Fatalf("an optional requirement with no provider refused the assignment: %v", err)
|
||||
}
|
||||
if have := strings.Join(names(got), " "); have != "grafana" {
|
||||
t.Fatalf("assigning grafana put %s on the machine; nothing is installed for an optional requirement (ADR 0330 §3)", have)
|
||||
}
|
||||
if world.Unchecked {
|
||||
continue
|
||||
}
|
||||
want := []Absent{{Consumer: "grafana", Requirement: "influxdb-api"}, {Consumer: "grafana", Requirement: "shell"}}
|
||||
if len(got.Absent) != 2 || got.Absent[0] != want[0] || got.Absent[1] != want[1] {
|
||||
t.Fatalf("absent: %+v, want %+v", got.Absent, want)
|
||||
}
|
||||
if len(got.Needs) != 0 {
|
||||
t.Fatalf("an absent requirement was given a binding: %+v", got.Needs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOptionalRequirementWithAProviderBindsAsAHardOne(t *testing.T) {
|
||||
world := World{Offered: map[string][]Provider{"influxdb-api": {
|
||||
{Node: "anchor", At: "anchor.internal", Module: "influxdb", Serves: map[string]any{"port": 8086}}}}}
|
||||
got, err := Resolve(optionalShelf(), []string{"grafana", "zsh"}, reachable(), world)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := boundFrom(t, got, "grafana", "influxdb-api"); n.From != "anchor" || n.Module != "influxdb" {
|
||||
t.Fatalf("bound to %s/%s", n.From, n.Module)
|
||||
}
|
||||
if len(got.Absent) != 0 {
|
||||
t.Fatalf("a requirement with a provider was left absent: %+v", got.Absent)
|
||||
}
|
||||
// And the same assignment without the provider is absent: what binds it later is only the
|
||||
// provider being assigned, with no act on grafana (ADR 0330 §7).
|
||||
before, err := Resolve(optionalShelf(), []string{"grafana", "zsh"}, reachable(), World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(before.Absent) != 1 || before.Absent[0].Requirement != "influxdb-api" {
|
||||
t.Fatalf("absent before the provider was assigned: %+v", before.Absent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOneConsumersOptionalRequirementIsAnothersHardOne(t *testing.T) {
|
||||
// telegraf wants influxdb-api hard: with nothing offering it, the assignment is refused as
|
||||
// before, naming what to assign — grafana's optional does not soften telegraf's.
|
||||
_, err := Resolve(optionalShelf(), []string{"grafana", "telegraf"}, reachable(), World{})
|
||||
if err == nil || !strings.Contains(err.Error(), "assign influxdb") {
|
||||
t.Fatalf("a hard requirement with no provider was not refused naming the candidate: %v", err)
|
||||
}
|
||||
// And the refusal names telegraf, whose remedy it is, not grafana, which met the name first.
|
||||
if !strings.Contains(err.Error(), "wanted by telegraf") {
|
||||
t.Errorf("the refusal does not name the module that wants it hard: %v", err)
|
||||
}
|
||||
// And a module pulled in later (suite → editor) that wants the shell hard installs it, even
|
||||
// though grafana, met first, wanted it only optionally — and grafana is then bound to it.
|
||||
got, err := Resolve(optionalShelf(), []string{"grafana", "suite"}, workstation(), World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if have := strings.Join(names(got), " "); !strings.Contains(have, "zsh") {
|
||||
t.Fatalf("a hard requirement met after an optional one was not installed: %s", have)
|
||||
}
|
||||
for _, a := range got.Absent {
|
||||
if a.Requirement == "shell" {
|
||||
t.Fatalf("the shell is installed and still listed absent: %+v", got.Absent)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHardRequirementWithNoProviderIsStillRefused(t *testing.T) {
|
||||
_, err := Resolve(optionalShelf(), []string{"telegraf"}, reachable(), World{})
|
||||
if err == nil || !strings.Contains(err.Error(), "influxdb") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func optionalModule(resources string) string {
|
||||
return `{"module":"grafana","version":"1",
|
||||
"requires":["route",{"name":"influxdb-api","optional":true}],
|
||||
"secrets":{"influxdb-api":"/var/lib/grafana/influxdb-api"},
|
||||
"resources":[` + resources + `]}`
|
||||
}
|
||||
|
||||
func TestTheModuleCheckRefusesAnOptionalPlaceholderThatCouldRenderEmpty(t *testing.T) {
|
||||
refused := map[string]string{
|
||||
"a placeholder without a default": `{"id":"ds","type":"file","path":"/a","mode":"0644",
|
||||
"content":"url: ${bound:influxdb-api:at}"}`,
|
||||
"a secret outside an only-when-bound file": `{"id":"s","type":"file","path":"/a","mode":"0400",
|
||||
"content":"${secret:influxdb-api}"}`,
|
||||
"only-when-bound on a hard requirement": `{"id":"r","type":"file","path":"/a","mode":"0644",
|
||||
"content":"x","only-when-bound":"route"}`,
|
||||
"only-when-bound on something not required": `{"id":"r","type":"file","path":"/a","mode":"0644",
|
||||
"content":"x","only-when-bound":"mqtt-topic"}`,
|
||||
"only-when-bound on a directory": `{"id":"d","type":"directory","mode":"0700",
|
||||
"only-when-bound":"influxdb-api"}`,
|
||||
"a default on a hard requirement": `{"id":"r","type":"file","path":"/a","mode":"0644",
|
||||
"content":"${bound:route:name|default:x}"}`,
|
||||
}
|
||||
for what, resource := range refused {
|
||||
if _, err := ParseManifest([]byte(optionalModule(resource))); err == nil {
|
||||
t.Errorf("%s passed the module check", what)
|
||||
} else if what == "a placeholder without a default" && !strings.Contains(err.Error(), "|default:") {
|
||||
t.Errorf("the refusal does not say how to write it: %v", err)
|
||||
}
|
||||
}
|
||||
accepted := map[string]string{
|
||||
"a placeholder with a default": `{"id":"ds","type":"file","path":"/a","mode":"0644",
|
||||
"content":"url: ${bound:influxdb-api:at|default:none}"}`,
|
||||
"a placeholder and a secret inside an only-when-bound file": `{"id":"ds","type":"file","path":"/a",
|
||||
"mode":"0400","content":"url: ${bound:influxdb-api:at}\npassword: ${secret:influxdb-api}",
|
||||
"only-when-bound":"influxdb-api"}`,
|
||||
}
|
||||
for what, resource := range accepted {
|
||||
if _, err := ParseManifest([]byte(optionalModule(resource))); err != nil {
|
||||
t.Errorf("%s was refused: %v", what, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func absentGrafana(t *testing.T, absent bool) Resolution {
|
||||
t.Helper()
|
||||
m := mustParse(t, optionalModule(`
|
||||
{"id":"env","type":"file","path":"/var/lib/grafana/env","mode":"0644",
|
||||
"content":"INFLUX=${bound:influxdb-api:at|default:none}:${bound:influxdb-api:port|default:0}\n"},
|
||||
{"id":"ds","type":"file","path":"/var/lib/grafana/ds.yaml","mode":"0400",
|
||||
"content":"url: ${bound:influxdb-api:at}\npassword: ${secret:influxdb-api}\n","only-when-bound":"influxdb-api"}`))
|
||||
m.Requires = []string{"influxdb-api"}
|
||||
m.Binds = map[string]string{"influxdb-api": "/var/lib/grafana/influxdb.json"}
|
||||
r := Resolution{Node: "workstation", Modules: []Manifest{m}}
|
||||
if absent {
|
||||
r.Absent = []Absent{{Consumer: "grafana", Requirement: "influxdb-api"}}
|
||||
} else {
|
||||
r.Needs = []Needed{{Name: "influxdb-api", From: "anchor", At: "anchor.internal", For: "grafana",
|
||||
Module: "influxdb", Sealed: "sealed-influx", Serves: map[string]any{"port": float64(8086)}}}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func TestAnAbsentRequirementIsComposedWithItsDefaultsAndWithoutItsFiles(t *testing.T) {
|
||||
out, err := absentGrafana(t, true).Declaration(Rendering{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := fileNamed(out, "grafana.env")
|
||||
if env == nil || env["content"] != "INFLUX=none:0\n" {
|
||||
t.Fatalf("the defaults were not rendered: %v", env)
|
||||
}
|
||||
for _, id := range []string{"grafana.ds", "grafana." + SecretID("influxdb-api"), "grafana." + BoundID("influxdb-api")} {
|
||||
if fileNamed(out, id) != nil {
|
||||
t.Errorf("%s was delivered while influxdb-api is absent (ADR 0330 §5, §6)", id)
|
||||
}
|
||||
}
|
||||
for _, r := range out {
|
||||
if _, has := r[OnlyWhenBound]; has {
|
||||
t.Errorf("%v carries %s to the node-engine, which knows no such field", r["id"], OnlyWhenBound)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestABoundOptionalRequirementIsComposedAsAHardOne(t *testing.T) {
|
||||
out, err := absentGrafana(t, false).Declaration(Rendering{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if env := fileNamed(out, "grafana.env"); env == nil || env["content"] != "INFLUX=anchor.internal:8086\n" {
|
||||
t.Fatalf("the bound values were not rendered: %v", env)
|
||||
}
|
||||
ds := fileNamed(out, "grafana.ds")
|
||||
if ds == nil {
|
||||
t.Fatalf("the only-when-bound file was left out while bound: %v", out)
|
||||
}
|
||||
if _, has := ds[OnlyWhenBound]; has {
|
||||
t.Errorf("the file carries %s to the node-engine", OnlyWhenBound)
|
||||
}
|
||||
for _, id := range []string{"grafana." + SecretID("influxdb-api"), "grafana." + BoundID("influxdb-api")} {
|
||||
if fileNamed(out, id) == nil {
|
||||
t.Errorf("%s was not delivered while bound", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Replay of novox/hq issue 166: every soft dependency had become a hard requirement. grafana runs without a
|
||||
// data source, yet a manifest could not say so: a mesh without influxdb could not register a grafana that
|
||||
// said its influxdb-api was optional, and with influxdb only in the catalogue, assigning grafana installed
|
||||
// influxdb for it. Written only with what the catalogue had before its fix (mesh-controller #281):
|
||||
// ParseManifest, Resolve, and the resolve tests' shelf, reachable machine and names, so it is laid over the
|
||||
// commit before.
|
||||
func TestReplay166(t *testing.T) {
|
||||
grafana, err := ParseManifest([]byte(`{"module":"grafana","version":"1",
|
||||
"requires":[{"name":"influxdb-api","optional":true}]}`))
|
||||
if err != nil {
|
||||
t.Fatalf("a manifest cannot say grafana runs without influxdb-api: %v", err)
|
||||
}
|
||||
catalogue := shelf(
|
||||
Manifest{Module: "influxdb", Version: "1", Provides: FromAnywhere("influxdb-api")},
|
||||
grafana,
|
||||
)
|
||||
// Nothing in the mesh runs influxdb; it is only in the catalogue.
|
||||
got, err := Resolve(catalogue, []string{"grafana"}, reachable(), World{})
|
||||
if err != nil {
|
||||
t.Fatalf("grafana could not be assigned on a mesh without influxdb: %v", err)
|
||||
}
|
||||
if have := strings.Join(names(got), " "); have != "grafana" {
|
||||
t.Fatalf("assigning grafana put %s on the machine; nothing is installed for an optional requirement", have)
|
||||
}
|
||||
if len(got.Needs) != 0 {
|
||||
t.Fatalf("grafana was given a binding with nothing providing it: %+v", got.Needs)
|
||||
}
|
||||
// And once influxdb runs somewhere, grafana binds to it as before.
|
||||
got, err = Resolve(catalogue, []string{"grafana"}, reachable(), World{Offered: map[string][]Provider{
|
||||
"influxdb-api": {{Node: "anchor", At: "anchor.internal", Module: "influxdb"}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.Needs) != 1 || got.Needs[0].From != "anchor" || got.Needs[0].For != "grafana" {
|
||||
t.Fatalf("grafana did not bind to influxdb on anchor: %+v", got.Needs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package catalogue
|
||||
|
||||
import "testing"
|
||||
|
||||
// Replay of novox/hq issue 279 (2026-10-06): the media managers' recycle-bin folders were declared owned by
|
||||
// a literal user and group, and on a machine whose puid and pgid settings named another account the apps
|
||||
// refused them as not writable. A directory's owner could not follow the account its module runs as. Written
|
||||
// only with what the catalogue had before its fix (mesh-controller #260): Compose, LeftOut, a setting layer,
|
||||
// and the adoption tests' anchor, so it is laid over the commit before.
|
||||
func TestReplay279(t *testing.T) {
|
||||
r := anAdoptedAnchor()
|
||||
r.Modules = append(r.Modules, Manifest{Module: "sonarr",
|
||||
Resources: []map[string]any{
|
||||
{"id": "recycle-bin", "type": "directory", "path": "/srv/sonarr/recycle", "mode": "0755",
|
||||
"owner": "${setting:puid}:${setting:pgid}"},
|
||||
{"id": "identity", "type": "file", "path": "/srv/sonarr/identity.env", "mode": "0644",
|
||||
"content": "PUID=${setting:puid}\nPGID=${setting:pgid}\n"},
|
||||
},
|
||||
})
|
||||
with := anchorRendering(false)
|
||||
with.Settings["sonarr"] = []Layer{{From: "node anchor", Values: map[string]any{"puid": "1001", "pgid": "2000"}}}
|
||||
composed, err := r.Compose(with)
|
||||
if err != nil {
|
||||
t.Fatalf("the machine did not compose: %v", err)
|
||||
}
|
||||
if why, left := composed.LeftOut["sonarr"]; left {
|
||||
t.Fatalf("the module was left out: %s", why)
|
||||
}
|
||||
got, ok := byID(composed.Resources)["sonarr.recycle-bin"]
|
||||
if !ok {
|
||||
t.Fatal("the recycle bin was not declared")
|
||||
}
|
||||
if got["owner"] != "1001:2000" {
|
||||
t.Fatalf("the recycle bin reaches the node-engine owned by %v, not the account the module runs as (1001:2000)", got["owner"])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Replay of novox/hq issue 412 (2026-10-10): which module owns /etc/hosts was enforced only after the fact,
|
||||
// when two modules on one machine declared it; a module that did not claim node-hostname could declare
|
||||
// /etc/hosts, and a module could claim node-hostname declaring none of its paths, and nothing refused either.
|
||||
// Written only with what the catalogue had before its fix (mesh-controller #267): CatalogueProblems, Shelf,
|
||||
// SeatNamed and CanHold, so it is laid over the commit before.
|
||||
func TestReplay412(t *testing.T) {
|
||||
stranger := Manifest{Module: "stranger", Version: "1", Resources: []map[string]any{
|
||||
{"id": "hosts", "type": "file", "path": "/etc/hosts", "into": "block", "content": "10.0.0.1 x\n"}}}
|
||||
got := strings.Join(CatalogueProblems(Shelf{"stranger": stranger}), "; ")
|
||||
if !strings.Contains(got, "node-hostname") {
|
||||
t.Errorf("a module declaring /etc/hosts without claiming node-hostname was not refused: %q", got)
|
||||
}
|
||||
|
||||
seat, ok := SeatNamed("node-hostname")
|
||||
if !ok {
|
||||
t.Fatal("no node-hostname seat")
|
||||
}
|
||||
holder := Manifest{Module: "names", Version: "1", Claims: []Claim{{Name: "node-hostname", Scope: ScopeNode,
|
||||
Serves: []string{"entries", "add", "remove"}}}}
|
||||
err := CanHold(holder, seat)
|
||||
if err == nil {
|
||||
t.Fatal("a holder of node-hostname declaring neither /etc/hosts nor /etc/hostname was accepted")
|
||||
}
|
||||
for _, want := range []string{"/etc/hosts", "/etc/hostname"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not name %s: %v", want, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -111,6 +111,9 @@ type Provider struct {
|
||||
// (novox/hq to-be 23), and the pair is what tells the holder of a seat apart from another module
|
||||
// providing the same thing (ADR 0110).
|
||||
Module string
|
||||
// LeftOut says each served key left out of Serves because that node composes no name it asks
|
||||
// for (novox/hq ADR 0346 §3): said under each consumer's need.
|
||||
LeftOut []string
|
||||
}
|
||||
|
||||
// Held is a claim somebody already has, used for the scopes wider than one node.
|
||||
@@ -183,6 +186,10 @@ type Resolution struct {
|
||||
// that keeps its data, and did not (novox/hq ADR 0232): it stays bound where its data is, and the
|
||||
// controller raises an urgent condition naming the move until a person pins one or the other.
|
||||
Kept []KeptBinding
|
||||
// Absent is every optional requirement of this node's modules that nothing assigned answers
|
||||
// (novox/hq ADR 0330 §3): left unbound rather than refused, with nothing installed for it, and
|
||||
// listed by `plan` and `node` (§7). It binds on the first composition after a provider is assigned.
|
||||
Absent []Absent
|
||||
}
|
||||
|
||||
// Unhostable is one directly-assigned module the machine cannot run.
|
||||
@@ -226,6 +233,9 @@ type Needed struct {
|
||||
// state, not a consumer missing its key. Set by the plan, which is the only layer that knows a
|
||||
// licence's manager; empty for every consumer.
|
||||
Manager bool
|
||||
// LeftOut says each served key the provider left out, because its machine composes no name the
|
||||
// value asks for (novox/hq ADR 0346 §3).
|
||||
LeftOut []string
|
||||
// Module is the providing module on From, empty for a need answered by a record or by nothing
|
||||
// that serves it. A provider is a (node, module) pair (novox/hq to-be 23), and a binding to data
|
||||
// is a binding to the pair (ADR 0232).
|
||||
@@ -374,12 +384,80 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
||||
// identical lines before realising there is none.
|
||||
reported := map[string]bool{}
|
||||
|
||||
for len(queue) > 0 {
|
||||
// **An optional requirement is settled last** (novox/hq ADR 0330). Whether a name is optional is
|
||||
// a question about every module that wants it, and a module pulled in later may want it hard; so
|
||||
// a name only optional modules want so far is put aside until the walk has nothing else, and only
|
||||
// then, if still nobody wants it hard, settled by the optional rule: bound to a provider that is
|
||||
// already assigned or reachable, exactly as a hard one, and otherwise absent — never refused,
|
||||
// and never answered by installing anything.
|
||||
var deferred []string
|
||||
settling := map[string]bool{}
|
||||
absent := map[string]bool{}
|
||||
wantedHard := func(want string) bool {
|
||||
for _, name := range order {
|
||||
if catalogue[name].wantsHard(want) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
wantedOptionally := func(want string) bool {
|
||||
for _, name := range order {
|
||||
if catalogue[name].IsOptional(want) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
leaveAbsent := func(want string) {
|
||||
absent[want] = true
|
||||
reported[want] = true
|
||||
}
|
||||
|
||||
for {
|
||||
if len(queue) == 0 {
|
||||
if len(deferred) == 0 {
|
||||
break
|
||||
}
|
||||
// What a module chosen since wants hard goes back to the walk, which may install for
|
||||
// it; only when none does is the rest settled, since settling installs nothing and so
|
||||
// can change no other name's answer.
|
||||
var hard, optional []string
|
||||
for _, d := range deferred {
|
||||
if wantedHard(d) {
|
||||
hard = append(hard, d)
|
||||
} else {
|
||||
optional = append(optional, d)
|
||||
}
|
||||
}
|
||||
if len(hard) > 0 {
|
||||
queue, deferred = hard, optional
|
||||
continue
|
||||
}
|
||||
for _, d := range optional {
|
||||
settling[d] = true
|
||||
}
|
||||
queue, deferred = optional, nil
|
||||
}
|
||||
want := queue[0]
|
||||
queue = queue[1:]
|
||||
if chosen[want] || reported[want] {
|
||||
continue
|
||||
}
|
||||
if !settling[want] && because[want] != "assigned" && !wantedHard(want) && wantedOptionally(want) {
|
||||
deferred = append(deferred, want)
|
||||
continue
|
||||
}
|
||||
// Walked as a hard requirement: a refusal names a module that wants it hard, never one that
|
||||
// wanted it only optionally and met it first — whose remedy it is not.
|
||||
if first, known := catalogue[because[want]]; !settling[want] && known && !first.wantsHard(want) {
|
||||
for _, name := range order {
|
||||
if catalogue[name].wantsHard(want) {
|
||||
because[want] = name
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// Already answered by something in the set. This is the case that makes assigning zsh do
|
||||
// what a person meant by it.
|
||||
//
|
||||
@@ -483,6 +561,12 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
||||
if brokered[want] {
|
||||
reported[want] = true
|
||||
where := world.Offered[want]
|
||||
if settling[want] && len(where) == 0 {
|
||||
// Nothing assigned anywhere answers it: absent, and nothing is put on a machine
|
||||
// for it (ADR 0330 §3), however few modules in the catalogue could.
|
||||
leaveAbsent(want)
|
||||
continue
|
||||
}
|
||||
names := providerNames(where)
|
||||
take := func(p Provider) {
|
||||
if node.At != "" && p.At == "" || node.At == "" && p.At != "" || node.At == "" && p.At == "" {
|
||||
@@ -504,7 +588,7 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
||||
bound = pm.IdentityBoundOf(want)
|
||||
}
|
||||
needs = append(needs, Needed{Name: want, From: p.Node, At: p.At, Module: p.Module,
|
||||
Serves: p.Serves, For: because[want], SharedOwn: shared, Identity: bound})
|
||||
Serves: p.Serves, For: because[want], SharedOwn: shared, Identity: bound, LeftOut: p.LeftOut})
|
||||
}
|
||||
switch {
|
||||
case world.Unchecked:
|
||||
@@ -579,6 +663,10 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
||||
// research 026 found. Another node's provider never counts — node scope is never brokered.
|
||||
if machineReach[want] && !isModule(catalogue, want) {
|
||||
reported[want] = true
|
||||
if settling[want] {
|
||||
leaveAbsent(want)
|
||||
continue
|
||||
}
|
||||
if world.Unchecked {
|
||||
// The first pass's refusals take a machine off the network; the second says it.
|
||||
continue
|
||||
@@ -591,6 +679,13 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
||||
}
|
||||
|
||||
candidates := offers[want]
|
||||
if settling[want] && !(len(candidates) == 0 && len(world.Licences[want]) > 0) {
|
||||
// Not answered by anything in this node's set, and only installing could answer it
|
||||
// here — which is never done for an optional requirement (ADR 0330 §3), not even
|
||||
// for the only candidate. A licence record still answers it, in the post-pass.
|
||||
leaveAbsent(want)
|
||||
continue
|
||||
}
|
||||
switch len(candidates) {
|
||||
case 0:
|
||||
if len(world.Licences[want]) > 0 {
|
||||
@@ -724,7 +819,8 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
||||
resolution := Resolution{Node: node.Name, At: node.At, PublicDomain: node.PublicDomain,
|
||||
Account: node.Account, AccountHome: node.AccountHome, AgentAccount: node.AgentAccount,
|
||||
AgentAccountHome: node.AgentAccountHome, Capabilities: node.Capabilities,
|
||||
Because: because, Needs: needs, Unhostable: unhostable, Kept: kept}
|
||||
Because: because, Needs: needs, Unhostable: unhostable, Kept: kept,
|
||||
Absent: absentOf(absent, order, catalogue)}
|
||||
for _, n := range providersFirst(order, catalogue) {
|
||||
resolution.Modules = append(resolution.Modules, catalogue[n])
|
||||
}
|
||||
|
||||
@@ -63,13 +63,17 @@ func TestTheResolverForwardsToFixedUpstreamsAndNeverReadsResolvConf(t *testing.T
|
||||
t.Errorf("the resolver's configuration lacks %q:\n%s", strings.TrimSpace(want), config)
|
||||
}
|
||||
}
|
||||
// Bound to its addresses, one way or the other. mesh-catalog PR 161 (novox/hq issue 348) moves it from
|
||||
// bind-dynamic, which closed the private address's listener when a bridge teardown failed its re-read
|
||||
// of the machine's addresses, to bind-interfaces. This test reads the catalogue beside it, which may be
|
||||
// on either side of that merge, so it takes both; once the catalogue's main has it, bind-dynamic is
|
||||
// refused here.
|
||||
if !strings.Contains(config, "\nbind-interfaces\n") && !strings.Contains(config, "\nbind-dynamic\n") {
|
||||
t.Errorf("the resolver's configuration binds neither by bind-interfaces nor by bind-dynamic:\n%s", config)
|
||||
// Bound to each address once, at start, and never by bind-dynamic (novox/hq issue 348): bind-dynamic
|
||||
// re-reads the machine's addresses on every change of any link and closes each listener that reading
|
||||
// did not find, and a reading made while the container runtime tears down a bridge's virtual links
|
||||
// fails, so the private address was left unbound for twenty minutes, the daemon running and silent.
|
||||
// The catalogue's main binds once since mesh-catalog PR 161 (105cc7c0).
|
||||
if !strings.Contains(config, "\nbind-interfaces\n") {
|
||||
t.Errorf("the resolver's configuration does not bind its addresses once (bind-interfaces):\n%s", config)
|
||||
}
|
||||
if strings.Contains(config, "\nbind-dynamic\n") {
|
||||
t.Errorf("the resolver's configuration says bind-dynamic, which closes its listeners when a bridge is "+
|
||||
"torn down (novox/hq issue 348):\n%s", config)
|
||||
}
|
||||
// By address and never by interface: dnsmasq admits a query by the interface it arrives on
|
||||
// when told one, and a container's query to the private address arrives on the runtime's
|
||||
|
||||
@@ -331,12 +331,13 @@ func UnassignRefusal(catalogue map[string]Manifest, node string, assigned, remov
|
||||
}
|
||||
|
||||
// CollisionRefusal is why assigning `adding` beside `assigned` is refused for what two modules
|
||||
// would both declare, or nothing (novox/hq ADR 0210 §1, 04-ISSUES/235).
|
||||
// would both declare, or nothing (novox/hq ADR 0210 §1, 04-ISSUES/235): one path, unit or name, or a package one
|
||||
// declares present and another absent. A package several modules declare present is no collision (ADR 0303).
|
||||
//
|
||||
// **Refused, not kept like an unresolved provision.** An assignment is otherwise kept when the
|
||||
// node does not resolve, because assignment is not an ordering: a consumer's provider can follow.
|
||||
// A collision is not an order anything can complete — no further assignment makes two owners of one
|
||||
// package one owner — and kept, it leaves the node unresolvable, so the next push of anything drops
|
||||
// file one owner — and kept, it leaves the node unresolvable, so the next push of anything drops
|
||||
// it from the mesh. Only collisions involving a module being added are refused; one already on the
|
||||
// node is `status`'s, and refusing an unrelated assignment for it would block its own remedy.
|
||||
func CollisionRefusal(catalogue map[string]Manifest, node string, assigned, adding []string) error {
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A provider may tell its consumers its own composed names (novox/hq ADR 0346, issue 624).
|
||||
//
|
||||
// **The name is already the mesh's.** A module contributes a label to a requirement — a route — and
|
||||
// the mesh joins it with the providing machine's domains; the module reads the result in its own files
|
||||
// as `${bound:<requirement>:name}` (novox/hq 04-ISSUES/122). Its consumers could not be told it: a
|
||||
// served value named only the consumer (ADR 0202) and settings. The forge's package registry writes
|
||||
// every tarball at its public name, and a consumer that must trust that name had no source for it but a
|
||||
// second copy kept by hand. So a served value may say `${bound:<requirement>:<name>}` for a requirement
|
||||
// the provider contributes to, and the mesh fills it from the same composition the provider's own files
|
||||
// and its proxy receive.
|
||||
//
|
||||
// **A name the providing machine does not compose leaves the key out** (ADR 0346 §3): a machine with
|
||||
// no public domain has no public name to give, and an empty value or a refusal would be worse than a
|
||||
// consumer told nothing — a refused provider machine takes every provision it offers down with it. The
|
||||
// plan says which key was left out, and why, on the provider's machine and under each consumer's need.
|
||||
|
||||
// ownNameFacts are what a served value may name of the provider's own composition: the names, as its
|
||||
// own files read them (withOwnNames).
|
||||
var ownNameFacts = []string{"name", "internal-name"}
|
||||
|
||||
// requirementsContributedTo is every requirement m contributes to, sorted.
|
||||
func requirementsContributedTo(m Manifest) []string {
|
||||
seen := map[string]bool{}
|
||||
for to := range m.Contributes {
|
||||
seen[to] = true
|
||||
}
|
||||
for to := range m.ContributesMany {
|
||||
seen[to] = true
|
||||
}
|
||||
out := make([]string, 0, len(seen))
|
||||
for to := range seen {
|
||||
out = append(out, to)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// ownNameKeys is every name a served value of m may say for one requirement it contributes to:
|
||||
// `name` and `internal-name`, and `name-<local>` and `internal-name-<local>` for each local name it
|
||||
// contributes under.
|
||||
func ownNameKeys(m Manifest, to string) []string {
|
||||
keys := append([]string{}, ownNameFacts...)
|
||||
for _, local := range sortedKeys(m.ContributesMany[to]) {
|
||||
for _, fact := range ownNameFacts {
|
||||
keys = append(keys, fact+"-"+local)
|
||||
}
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// ServedOwnNames fills the provider's own composed names into what it serves one provision's
|
||||
// consumers, from this resolution's composition of the provider's machine (ADR 0346).
|
||||
//
|
||||
// A value naming a name this machine does not compose is left out, with its key, and said. The map is
|
||||
// copied only when something changes, as ServedTo does: the caller's may be the manifest's.
|
||||
func (r Resolution) ServedOwnNames(m Manifest, provision string, serves map[string]any, layers []Layer) (
|
||||
map[string]any, []string, error) {
|
||||
var asking []string
|
||||
for _, key := range sortedAnyKeys(serves) {
|
||||
if text, ok := serves[key].(string); ok && bound.MatchString(text) {
|
||||
asking = append(asking, key)
|
||||
}
|
||||
}
|
||||
if len(asking) == 0 {
|
||||
return serves, nil, nil
|
||||
}
|
||||
own := map[string]map[string]string{}
|
||||
for _, to := range requirementsContributedTo(m) {
|
||||
names, err := r.ownNames(m, to, layers)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
values := map[string]string{}
|
||||
withOwnNames(values, names)
|
||||
own[to] = values
|
||||
}
|
||||
out := make(map[string]any, len(serves))
|
||||
for k, v := range serves {
|
||||
out[k] = v
|
||||
}
|
||||
var said []string
|
||||
for _, key := range asking {
|
||||
text := serves[key].(string)
|
||||
var missing string
|
||||
filled := bound.ReplaceAllStringFunc(text, func(token string) string {
|
||||
parts := bound.FindStringSubmatch(token)
|
||||
if value := own[parts[1]][parts[2]]; value != "" && parts[3] == "" {
|
||||
return value
|
||||
}
|
||||
if missing == "" {
|
||||
missing = token
|
||||
}
|
||||
return token
|
||||
})
|
||||
if missing != "" {
|
||||
delete(out, key)
|
||||
said = append(said, fmt.Sprintf(
|
||||
"%s serves %s without %q: it names %s, and %s composes no such name for %s (it has no "+
|
||||
"public domain, or the endpoint is not reached publicly) (novox/hq ADR 0346)",
|
||||
m.Module, provision, key, missing, orHere(r.Node), m.Module))
|
||||
continue
|
||||
}
|
||||
out[key] = filled
|
||||
}
|
||||
return out, said, nil
|
||||
}
|
||||
|
||||
func orHere(node string) string {
|
||||
if node == "" {
|
||||
return "this machine"
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
// ServedLeftOut is what the providers in this node's own set leave out of what they serve, because
|
||||
// this machine composes no name a served value asks for (ADR 0346 §3): said on the provider's machine,
|
||||
// where the remedy is.
|
||||
func (r Resolution) ServedLeftOut(settings SettingsBy) []string {
|
||||
var said []string
|
||||
for _, m := range r.Modules {
|
||||
for _, provision := range sortedServes(m.Serves) {
|
||||
_, lines, err := r.ServedOwnNames(m, provision, m.Serves[provision], WithDefaults(m, settings[m.Module]))
|
||||
if err != nil {
|
||||
said = append(said, fmt.Sprintf("%s serves %s, and its own names could not be composed: %v",
|
||||
m.Module, provision, err))
|
||||
continue
|
||||
}
|
||||
said = append(said, lines...)
|
||||
}
|
||||
}
|
||||
return said
|
||||
}
|
||||
|
||||
// checkServedOwnNames refuses a served `${bound:…}` that names a requirement the module does not
|
||||
// contribute to, a fact other than its own names, or a local name it does not contribute under; and
|
||||
// one with a default, since a served value is left out rather than given one (ADR 0346 §3, §4).
|
||||
func checkServedOwnNames(m Manifest, provision, key, text string) []string {
|
||||
var problems []string
|
||||
contributes := requirementsContributedTo(m)
|
||||
for _, parts := range bound.FindAllStringSubmatch(text, -1) {
|
||||
to, fact, fallback := parts[1], parts[2], parts[3]
|
||||
says := fmt.Sprintf("%s serves %s, and the value it serves as %q says %s", m.Module, provision, key, parts[0])
|
||||
switch {
|
||||
case fallback != "":
|
||||
problems = append(problems, says+": a served name takes no default; one the machine does not "+
|
||||
"compose leaves the key out (novox/hq ADR 0346)")
|
||||
case !oneOf(contributes, to):
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s, and %s contributes to %s: a served value names only the provider's own names "+
|
||||
"(novox/hq ADR 0346)", says, m.Module, orNothing(quoted(contributes))))
|
||||
case !oneOf(ownNameKeys(m, to), fact):
|
||||
problems = append(problems, fmt.Sprintf("%s, and what %s is called through %s is %s (novox/hq ADR 0346)",
|
||||
says, m.Module, to, strings.Join(ownNameKeys(m, to), ", ")))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
func quoted(values []string) []string {
|
||||
out := make([]string, len(values))
|
||||
for i, v := range values {
|
||||
out[i] = fmt.Sprintf("%q", v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// nestedOwnNames refuses a `${bound:…}` inside a served list or object: only a string value of its own is
|
||||
// filled (ServedOwnNames), so a nested one would reach the consumer as the placeholder (ADR 0346 §5).
|
||||
func nestedOwnNames(m Manifest, provision, key string, value any) []string {
|
||||
var found []string
|
||||
var walk func(v any)
|
||||
walk = func(v any) {
|
||||
switch x := v.(type) {
|
||||
case string:
|
||||
found = append(found, bound.FindAllString(x, -1)...)
|
||||
case []any:
|
||||
for _, e := range x {
|
||||
walk(e)
|
||||
}
|
||||
case map[string]any:
|
||||
for _, k := range sortedAnyKeys(x) {
|
||||
walk(x[k])
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(value)
|
||||
if len(found) == 0 {
|
||||
return nil
|
||||
}
|
||||
return []string{fmt.Sprintf(
|
||||
"%s serves %s, and the value it serves as %q holds %s inside a list or an object: a provider's own "+
|
||||
"name is filled only in a string value of its own (novox/hq ADR 0346)",
|
||||
m.Module, provision, key, strings.Join(found, ", "))}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0346, issue 624: the forge's package registry writes every tarball at its public name,
|
||||
// and the lab, its consumer, must trust that name and no other. The name is the mesh's own composition
|
||||
// of the forge's route, which the forge's files read as ${bound:route:name-web}; a consumer can now be
|
||||
// served it.
|
||||
|
||||
const aRegistryForge = `{"module":"forge","version":"1","requires":["route"],
|
||||
"provides":[{"name":"npm-package-registry","scope":"mesh"}],
|
||||
"contributes":{"route":{"web":{"label":"git","port":3000}}},
|
||||
"serves":{"npm-package-registry":{"scheme":"http","port":3000,"npm-path":"/api/packages/novox/npm/",
|
||||
"public-root":"https://${bound:route:name-web}/api/packages/novox/npm/"}}}`
|
||||
|
||||
func aForgeManifest(t *testing.T) Manifest {
|
||||
t.Helper()
|
||||
m, err := ParseManifest([]byte(aRegistryForge))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func TestAProviderServesItsOwnComposedName(t *testing.T) {
|
||||
m := aForgeManifest(t)
|
||||
r := Resolution{Node: "home", PublicDomain: "example.org"}
|
||||
got, said, err := r.ServedOwnNames(m, "npm-package-registry", m.Serves["npm-package-registry"], nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got["public-root"] != "https://git.example.org/api/packages/novox/npm/" || len(said) != 0 {
|
||||
t.Fatalf("the public root is the route's composed name: %v, said %v", got, said)
|
||||
}
|
||||
if got["npm-path"] != "/api/packages/novox/npm/" || got["port"] != float64(3000) {
|
||||
t.Fatalf("the rest is served as it was: %v", got)
|
||||
}
|
||||
if m.Serves["npm-package-registry"]["public-root"] != "https://${bound:route:name-web}/api/packages/novox/npm/" {
|
||||
t.Fatal("the manifest's own map was rewritten")
|
||||
}
|
||||
}
|
||||
|
||||
func TestANameTheMachineDoesNotComposeLeavesTheKeyOutAndSaysSo(t *testing.T) {
|
||||
m := aForgeManifest(t)
|
||||
r := Resolution{Node: "home"} // no public domain
|
||||
got, said, err := r.ServedOwnNames(m, "npm-package-registry", m.Serves["npm-package-registry"], nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, there := got["public-root"]; there {
|
||||
t.Fatalf("a name not composed is left out, never written as the placeholder or empty: %v", got)
|
||||
}
|
||||
if got["npm-path"] == nil {
|
||||
t.Fatalf("only the key that names it is left out: %v", got)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], `forge serves npm-package-registry without "public-root"`) ||
|
||||
!strings.Contains(said[0], "home composes no such name") {
|
||||
t.Fatalf("said: %v", said)
|
||||
}
|
||||
if lines := (Resolution{Node: "home", Modules: []Manifest{m}}).ServedLeftOut(nil); len(lines) != 1 {
|
||||
t.Fatalf("the plan on the provider's machine says it: %v", lines)
|
||||
}
|
||||
if lines := (Resolution{Node: "home", PublicDomain: "example.org", Modules: []Manifest{m}}).ServedLeftOut(nil); len(lines) != 0 {
|
||||
t.Fatalf("nothing to say when the name is composed: %v", lines)
|
||||
}
|
||||
}
|
||||
|
||||
// On the provider's own machine the consumer is served the same name (ADR 0346 §2).
|
||||
func TestAConsumerBesideTheProviderIsServedItsName(t *testing.T) {
|
||||
m := aForgeManifest(t)
|
||||
consumer := Manifest{Module: "lab", Version: "1", Requires: []string{"npm-package-registry"},
|
||||
Binds: map[string]string{"npm-package-registry": "/var/lib/lab/package-registry.json"}}
|
||||
node := workstation()
|
||||
node.PublicDomain = "example.org"
|
||||
got, err := Resolve(shelf(minimalRouteProxy(), m, consumer), []string{"route-proxy", "forge", "lab"}, node, World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
served, answered, err := got.servedOnThisMachine("npm-package-registry", Rendering{})
|
||||
if err != nil || !answered {
|
||||
t.Fatalf("answered %v: %v", answered, err)
|
||||
}
|
||||
if served["public-root"] != "https://git.example.org/api/packages/novox/npm/" {
|
||||
t.Fatalf("served beside the provider: %v", served)
|
||||
}
|
||||
out, err := got.Declaration(Rendering{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var binding string
|
||||
for _, r := range out {
|
||||
if r["path"] == "/var/lib/lab/package-registry.json" {
|
||||
binding, _ = r["content"].(string)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(binding, `"public-root": "https://git.example.org/api/packages/novox/npm/"`) &&
|
||||
!strings.Contains(binding, `"public-root":"https://git.example.org/api/packages/novox/npm/"`) {
|
||||
t.Fatalf("the consumer's binding carries the name:\n%s", binding)
|
||||
}
|
||||
if strings.Contains(binding, "${bound:") {
|
||||
t.Fatalf("a placeholder reached the consumer:\n%s", binding)
|
||||
}
|
||||
}
|
||||
|
||||
// From another machine, the provider's served values carry what that machine composed, and a key it
|
||||
// left out is said on the consumer's need (ADR 0346 §2, §3).
|
||||
func TestAConsumerElsewhereIsToldWhatTheProviderLeftOut(t *testing.T) {
|
||||
consumer := Manifest{Module: "lab", Version: "1", Requires: []string{"npm-package-registry"},
|
||||
Binds: map[string]string{"npm-package-registry": "/var/lib/lab/package-registry.json"}}
|
||||
node := workstation()
|
||||
node.At = "workstation.internal"
|
||||
world := World{Offered: map[string][]Provider{"npm-package-registry": {{Node: "home", At: "home.internal",
|
||||
Module: "forge", Serves: map[string]any{"port": 3000},
|
||||
LeftOut: []string{`forge serves npm-package-registry without "public-root"`}}}}}
|
||||
got, err := Resolve(shelf(aForgeManifest(t), consumer), []string{"lab"}, node, world)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, n := range got.Needs {
|
||||
if n.Name == "npm-package-registry" {
|
||||
if len(n.LeftOut) != 1 {
|
||||
t.Fatalf("the need carries what was left out: %+v", n)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("no need: %+v", got.Needs)
|
||||
}
|
||||
|
||||
func TestAServedNameIsCheckedWhenTheDefinitionIsRead(t *testing.T) {
|
||||
for bad, want := range map[string]string{
|
||||
`"public-root":"https://${bound:route:name-api}/x"`: "what forge is called through route is name, internal-name, name-web, internal-name-web",
|
||||
`"public-root":"https://${bound:postgres-database:name}/x"`: `forge contributes to "route"`,
|
||||
`"public-root":"https://${bound:route:name-web|default:x}/x"`: "takes no default",
|
||||
`"public-root":"https://${bound:route:at}/x"`: "what forge is called through route",
|
||||
} {
|
||||
raw := strings.Replace(aRegistryForge, `"public-root":"https://${bound:route:name-web}/api/packages/novox/npm/"`, bad, 1)
|
||||
_, err := ParseManifest([]byte(raw))
|
||||
if err == nil || !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("%s: want a refusal saying %q, got %v", bad, want, err)
|
||||
}
|
||||
}
|
||||
nested := strings.Replace(aRegistryForge, `"public-root":"https://${bound:route:name-web}/api/packages/novox/npm/"`,
|
||||
`"roots":["https://${bound:route:name-web}/x"]`, 1)
|
||||
if _, err := ParseManifest([]byte(nested)); err == nil || !strings.Contains(err.Error(), "inside a list or an object") {
|
||||
t.Errorf("a nested own name is refused, not served as the placeholder: %v", err)
|
||||
}
|
||||
if _, err := ParseManifest([]byte(aRegistryForge)); err != nil {
|
||||
t.Fatalf("its own name is accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The provider's contributions file carries no own name on a consumer's entry (ADR 0346 §4): only what
|
||||
// is derived for that consumer is there (ADR 0202 §5), and the provider reads its own names in its files.
|
||||
func TestAProvidersOwnNameIsNotOnItsConsumersEntry(t *testing.T) {
|
||||
m := aForgeManifest(t)
|
||||
r := Resolution{Node: "home", PublicDomain: "example.org", Modules: []Manifest{m}}
|
||||
derived, err := r.derivedFor("npm-package-registry", "lab_home", "lab", "", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if derived != nil {
|
||||
t.Fatalf("a consumer's entry carries only what was derived for it: %v", derived)
|
||||
}
|
||||
}
|
||||
@@ -70,12 +70,42 @@ func namesItsKeys(m Manifest) bool {
|
||||
// reads itself, one the file keeps for itself, or one mergeable file naming its keys beside another
|
||||
// that names none.
|
||||
func TakenSettingsProblems(m Manifest) []string {
|
||||
// Not yet here: refusing a module that declares a setting and owns a mergeable file naming no keys
|
||||
// (ADR 0326 §3). betterbird is one, and the catalogue cannot name its keys before this controller
|
||||
// runs, so that refusal follows once the catalogue does (novox/hq issue 574).
|
||||
return takenSettingsProblems(m)
|
||||
}
|
||||
|
||||
// WrittenSettingsProblems is ADR 0326 §3, judged only where a definition is written — the module
|
||||
// check, `module add`, a build taken in, the builder and the merge gate's registration — and never
|
||||
// where a stored one is read again (ParseManifest, a build's manifest at a commit, a module put back):
|
||||
// a module that declares a setting names the keys of every mergeable file it owns (novox/hq issue
|
||||
// 574, ADR 0164 §11 and §15). A build stored before the rule, such as betterbird's before its accounts
|
||||
// file named its keys, must still compose and still be put back; only its next change is held to it.
|
||||
//
|
||||
// Said only when none of the module's mergeable files names its keys: when one does, §2 already
|
||||
// names every file that does not, and a file is not reported twice.
|
||||
func WrittenSettingsProblems(m Manifest) []string {
|
||||
if len(m.Settings) == 0 {
|
||||
return nil
|
||||
}
|
||||
var silent []string
|
||||
for _, r := range m.Resources {
|
||||
how, _ := r["merge"].(string)
|
||||
if how == "" {
|
||||
continue
|
||||
}
|
||||
if _, said := r[TakesSettingsField]; said {
|
||||
return nil
|
||||
}
|
||||
silent = append(silent, fmt.Sprint(r["id"]))
|
||||
}
|
||||
var problems []string
|
||||
for _, id := range silent {
|
||||
problems = append(problems, fmt.Sprintf("%s declares settings, and its mergeable %s names no %s: "+
|
||||
"a module that declares a setting names the keys each mergeable file takes, an empty list "+
|
||||
"for none (novox/hq ADR 0326 §3, ADR 0164 §11)", m.Module, id, TakesSettingsField))
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// takenSettingsProblems is every rule of ADR 0326 §2 and §4, judged where a definition is checked and
|
||||
// where a stored one is composed.
|
||||
func takenSettingsProblems(m Manifest) []string {
|
||||
|
||||
@@ -194,3 +194,50 @@ func containsText(list []string, text string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestAModuleThatDeclaresASettingNamesItsFilesKeys(t *testing.T) {
|
||||
// ADR 0326 §3, ADR 0164 §15, novox/hq issue 574: only a module that declares no settings keeps
|
||||
// today's form. One that declares a preference and owns a silent mergeable file is refused where
|
||||
// a definition is written — and read and composed where a stored one is read again, so a build kept
|
||||
// from before the rule is still sent and still put back.
|
||||
declaring := func(app, sidecar []any) Manifest {
|
||||
m := twoFiles(app, sidecar)
|
||||
m.Resources[2]["content"] = "WORKERS=${setting:workers}\n"
|
||||
m.Settings = map[string]SettingDeclaration{"workers": {Kind: KindPreference, Default: 4.0, Why: "a number"}}
|
||||
return m
|
||||
}
|
||||
m := declaring(nil, nil)
|
||||
problems := WrittenSettingsProblems(m)
|
||||
if len(problems) != 2 || !containsText(problems, "mergeable app names no takes-settings") ||
|
||||
!containsText(problems, "mergeable sidecar names no takes-settings") {
|
||||
t.Errorf("where it is written: %v", problems)
|
||||
}
|
||||
if problems := TakenSettingsProblems(m); len(problems) != 0 {
|
||||
t.Errorf("what a stored definition is read by refuses it: %v", problems)
|
||||
}
|
||||
raw, err := json.Marshal(map[string]any{"module": m.Module, "version": "1", "resources": m.Resources,
|
||||
"settings": map[string]any{"workers": map[string]any{"kind": KindPreference, "default": 4, "why": "a number"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
read, err := ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("a stored build from before the rule is not read again: %v", err)
|
||||
}
|
||||
if err := JudgeSettings(read, nil, false); err != nil {
|
||||
t.Errorf("a stored definition is composed as before, and was refused: %v", err)
|
||||
}
|
||||
|
||||
if problems := WrittenSettingsProblems(declaring([]any{"server"}, []any{})); len(problems) != 0 {
|
||||
t.Errorf("every mergeable file names its keys, and was refused: %v", problems)
|
||||
}
|
||||
if problems := WrittenSettingsProblems(twoFiles(nil, nil)); len(problems) != 0 {
|
||||
t.Errorf("a module declaring no setting was refused: %v", problems)
|
||||
}
|
||||
// One naming file and one silent: §2 names the silent one, and §3 does not name it again.
|
||||
mixed := declaring([]any{"server"}, nil)
|
||||
if written, taken := WrittenSettingsProblems(mixed), TakenSettingsProblems(mixed); len(written) != 0 || len(taken) != 1 {
|
||||
t.Errorf("the silent file is reported %d times by §3 and %d by §2; want once, by §2: %v %v",
|
||||
len(written), len(taken), written, taken)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -192,6 +192,16 @@ func settle(base map[string]any, layers []Layer, protected map[string]bool, what
|
||||
"%s sets %q on %v, and that module keeps %q for itself — it is not settable",
|
||||
layer.From, key, what, key)
|
||||
}
|
||||
// **No layer replaces a secret the vault fills, at any depth** (novox/hq issue 610): a
|
||||
// mergeable file's `${secret:<name>}` is the vault's value, and a layer that reached it —
|
||||
// `server.secret_key` under a `server` the file takes — would put a value the operator or an
|
||||
// agent chose where the vault's belongs. Judged against the file's own content, so it holds
|
||||
// whatever a module's `protected` list says.
|
||||
if at := secretReplacedBy(base[key], value, key); at != "" {
|
||||
return nil, fmt.Errorf(
|
||||
"%s sets %q on %v, which the vault fills (${secret:…}); a secret comes from the vault, "+
|
||||
"never from a setting (novox/hq issues 342, 610)", layer.From, at, what)
|
||||
}
|
||||
merged[key] = mergeValue(merged[key], value)
|
||||
}
|
||||
}
|
||||
@@ -576,3 +586,54 @@ func JudgeSettings(m Manifest, layers []Layer, adopted bool) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// secretReplacedBy is the path in a mergeable file's own content that a layer's value would replace while the
|
||||
// vault fills it — a string holding `${secret:` — or "" when the value leaves every such string as it is
|
||||
// (novox/hq issue 610). Maps merge key by key (mergeValue), so only the keys the value names are judged; any
|
||||
// other value replaces the whole of what it lands on.
|
||||
func secretReplacedBy(base, value any, at string) string {
|
||||
if baseMap, isMap := base.(map[string]any); isMap {
|
||||
if valueMap, also := value.(map[string]any); also {
|
||||
keys := make([]string, 0, len(valueMap))
|
||||
for k := range valueMap {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if found := secretReplacedBy(baseMap[k], valueMap[k], at+"."+k); found != "" {
|
||||
return found
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return secretIn(base, at)
|
||||
}
|
||||
|
||||
// secretIn is the first path under v holding a vault placeholder, or "".
|
||||
func secretIn(v any, at string) string {
|
||||
switch t := v.(type) {
|
||||
case string:
|
||||
if strings.Contains(t, "${secret:") {
|
||||
return at
|
||||
}
|
||||
case map[string]any:
|
||||
keys := make([]string, 0, len(t))
|
||||
for k := range t {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if found := secretIn(t[k], at+"."+k); found != "" {
|
||||
return found
|
||||
}
|
||||
}
|
||||
case []any:
|
||||
for i, item := range t {
|
||||
if found := secretIn(item, fmt.Sprintf("%s[%d]", at, i)); found != "" {
|
||||
return found
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -234,3 +234,29 @@ func TestAServedValueMayBeTheOperators(t *testing.T) {
|
||||
t.Errorf("a setting a served fact asks for is not stray: %v", unused)
|
||||
}
|
||||
}
|
||||
|
||||
// No layer replaces a value the vault fills, at any depth (novox/hq issue 610): searxng's server.secret_key
|
||||
// sits under `server`, a key its file takes, and a layer reaching it was let through by a top-level-only check.
|
||||
func TestNoLayerReplacesAVaultSecretAtAnyDepth(t *testing.T) {
|
||||
file := map[string]any{"id": "settings", "type": "file", "merge": MergeJSON,
|
||||
"content": `{"server": {"secret_key": "${secret:secret}", "port": 8080}, "ui": {"theme": "simple"}, "token": "${secret:token}"}`}
|
||||
layer := func(values map[string]any) []Layer { return []Layer{{From: "anchor", Values: values}} }
|
||||
|
||||
merged, err := ApplySettings(file, layer(map[string]any{"server": map[string]any{"port": float64(9000)}, "ui": map[string]any{"theme": "dark"}}))
|
||||
if err != nil {
|
||||
t.Fatalf("a key beside the secret was refused: %v", err)
|
||||
}
|
||||
if c := merged["content"].(string); !strings.Contains(c, `"secret_key": "${secret:secret}"`) || !strings.Contains(c, `"port": 9000`) {
|
||||
t.Fatalf("the secret or the setting did not land as meant:\n%s", c)
|
||||
}
|
||||
for name, values := range map[string]map[string]any{
|
||||
"the nested secret": {"server": map[string]any{"secret_key": "chosen"}},
|
||||
"the whole map holding it": {"server": "elsewhere"},
|
||||
"a top-level secret": {"token": "chosen"},
|
||||
"the secret, emptied": {"server": map[string]any{"secret_key": ""}},
|
||||
} {
|
||||
if _, err := ApplySettings(file, layer(values)); err == nil || !strings.Contains(err.Error(), "vault") {
|
||||
t.Errorf("%s: %v, want a refusal naming the vault", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,6 +33,10 @@ import (
|
||||
// 4. **A module's whole layer, when it has a mergeable file not marked `"trusted": false`** (novox/hq issue 340,
|
||||
// TrustedMergeable). A mergeable file takes any key a layer sets, not only those its content names, so no list
|
||||
// of keys covers it: an empty runtime configuration a provider reads would take a `url` of the caller's.
|
||||
// **Unless the module's files name the keys they take** (novox/hq ADR 0326, ADR 0329 §5): then a mergeable
|
||||
// file takes exactly its `takes-settings` and no other key, so those keys of a trusted file are terminal keys
|
||||
// and the rest of the layer is not. A key no file names is stray and refused where it is set. So betterbird's
|
||||
// `tags`, in a file nothing trusts, is set through the verb, while its `accounts` and `identities` are not.
|
||||
// 5. **Every setting a directory's owner asks for** (novox/hq issue 279): root chowns the directory to the number
|
||||
// at the next push, so through a verb any caller could hand itself — or uid 0 — a module's folder. Whatever
|
||||
// else asks for the key, a file marked `"trusted": false` included, the owner makes it the terminal's.
|
||||
@@ -109,8 +113,13 @@ func UnsaidTrust(m Manifest) []string {
|
||||
// TrustedMergeable is every mergeable file of a module not marked `"trusted": false`, by id (novox/hq issue 340).
|
||||
// A mergeable file takes any key a layer sets, not only those its content names: an empty runtime configuration a
|
||||
// provider reads takes a `url` of a caller's as surely as a declared one. So a module holding one has its whole
|
||||
// layer set at the controller's terminal; a verb may read it and change nothing.
|
||||
// layer set at the controller's terminal; a verb may read it and change nothing. A module whose mergeable files
|
||||
// name the keys they take (ADR 0326) has none here: each such file takes only its named keys, which TerminalKeys
|
||||
// holds for a trusted one (ADR 0329 §5).
|
||||
func TrustedMergeable(m Manifest) []string {
|
||||
if namesItsKeys(m) {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, r := range m.Resources {
|
||||
if fmt.Sprint(r["type"]) != "file" {
|
||||
@@ -135,6 +144,16 @@ func TrustedMergeable(m Manifest) []string {
|
||||
func asksFor(r map[string]any) []string {
|
||||
content, _ := r["content"].(string)
|
||||
asked := settingsUsed(content)
|
||||
if how, _ := r["merge"].(string); how != "" {
|
||||
if taken, names := takenKeys(r); names {
|
||||
// The file takes exactly the keys it names (ADR 0326), whatever its content holds.
|
||||
for key := range taken {
|
||||
asked = append(asked, key)
|
||||
}
|
||||
sort.Strings(asked)
|
||||
return asked
|
||||
}
|
||||
}
|
||||
if how, _ := r["merge"].(string); how != "" && strings.TrimSpace(content) != "" {
|
||||
var base map[string]any
|
||||
if json.Unmarshal([]byte(content), &base) == nil {
|
||||
|
||||
@@ -166,6 +166,33 @@ func TestTerminalKeysAreDerived(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A module whose mergeable files name the keys they take (ADR 0326) has no whole layer at the terminal: a
|
||||
// trusted file's named keys are terminal keys, and a file nothing trusts leaves its keys to the verb (novox/hq
|
||||
// ADR 0329 §5, issue 586: betterbird's tags beside its trusted accounts, identities, calendars, address books
|
||||
// and filters). A module whose files name no keys keeps the whole layer at the terminal (issue 340).
|
||||
func TestOnlyANamedKeyOfATrustedFileIsTheTerminals(t *testing.T) {
|
||||
betterbird := Manifest{Module: "betterbird", Resources: []map[string]any{
|
||||
{"id": "prefs", "type": "file", "path": "/home/p/prefs.json", "merge": MergeJSON, "trusted": true,
|
||||
TakesSettingsField: []any{"prefs"}, "content": `{"prefs": {}}`},
|
||||
{"id": "accounts", "type": "file", "path": "/home/p/accounts.json", "merge": MergeJSON, "trusted": true,
|
||||
TakesSettingsField: []any{"accounts"}, "content": `{"accounts": {}}`},
|
||||
{"id": "configuration", "type": "file", "path": "/home/p/configuration.json", "merge": MergeJSON,
|
||||
TakesSettingsField: []any{"identities", "calendars", "address-books", "filters"}, "content": `{}`},
|
||||
{"id": "tags", "type": "file", "path": "/home/p/tags.json", "merge": MergeJSON, "trusted": false,
|
||||
TakesSettingsField: []any{"tags"}, "content": `{"tags": {}}`}}}
|
||||
if got := TrustedMergeable(betterbird); len(got) != 0 {
|
||||
t.Fatalf("files that name their keys put no whole layer at the terminal: %v", got)
|
||||
}
|
||||
if got := strings.Join(TerminalKeys(betterbird), ","); got != "places,accesses,accounts,address-books,calendars,filters,identities,prefs" {
|
||||
t.Fatalf("the named keys of the trusted files, unmarked counting as trusted, and not tags: %s", got)
|
||||
}
|
||||
silent := Manifest{Module: "keycloak", Resources: []map[string]any{{"id": "runtime", "type": "file",
|
||||
"path": "/var/lib/kc/runtime.json", "merge": MergeJSON, "content": `{}`}}}
|
||||
if got := TrustedMergeable(silent); len(got) != 1 {
|
||||
t.Fatalf("a file that names no keys still takes any key, so its module's layer stays the terminal's: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A file that asks for a setting says whether what it asks is trusted (novox/hq issue 339): `trusted` is a
|
||||
// boolean, on a file alone, and a file asking for a setting without it is named.
|
||||
func TestAFileSaysWhetherItsSettingsAreTrusted(t *testing.T) {
|
||||
|
||||
@@ -217,14 +217,16 @@ var ControllerVerbs = []Verb{
|
||||
Input: schema(map[string]string{"node": "the machine's name", "provision": "the provision"}, []string{"node", "provision"})},
|
||||
{Name: "push", Description: "Send one machine everything it should be. With no machine named it is a push of the " +
|
||||
"WHOLE mesh — every machine that is behind — and the answer says so first; behind says that outright. " +
|
||||
"Answers at once that it is running, with a call id: `calls` with that id says what it sent " +
|
||||
"(a push can reload the bus, which then refuses any answer still to come). A push by hand is a repair, " +
|
||||
"Answers at once that it is running, with a call id: `calls` with that id says what it sent and, for a " +
|
||||
"machine named, how that machine applied it — applied, refused with its reason, or each failed resource — " +
|
||||
"once it reports within wait (a push can reload the bus, which then refuses any answer still to come). A push by hand is a repair, " +
|
||||
"and says why: recorded in the hand-act log (novox/hq to-be 45 §7). A machine whose push would give a " +
|
||||
"running module's data another directory is held and named, the others sent (novox/hq ADR 0217): read " +
|
||||
"`plan` with diff, and name the machine with move to send it.",
|
||||
Input: schema(map[string]string{
|
||||
"node": "the machine's name; without it, every machine that is behind",
|
||||
"move": "with node: the modules whose data may move with this push, comma-separated (optional)",
|
||||
"wait": "with node: how long the call waits for the machine to report how it applied what it was sent, as \"90s\" or \"2m\"; \"0\" waits for nothing; without it, the controller's setting push-wait (optional)",
|
||||
"behind": "\"true\": every machine that is behind, the whole mesh — the same as naming none, said outright; not with node",
|
||||
"why": "why this is pushed by hand: recorded in the hand-act log",
|
||||
"cause": "the cause in a word, or a condition's kind — the word a second push for the same reason uses (optional)",
|
||||
@@ -360,6 +362,12 @@ var ControllerVerbs = []Verb{
|
||||
"register": "\"true\" to register what it builds",
|
||||
"older": "\"true\", with register: even though a newer build of the module is registered",
|
||||
}, []string{"id"}, "register", "older")},
|
||||
// novox/hq ADR 0350 §5: asked by the delivery's owner when the forge says a pull request closed unmerged.
|
||||
{Name: "forget-checks", Description: "Let go of every build a pull request's merge checks made for its merge in " +
|
||||
"the artifact store's scratch namespace (novox/hq ADR 0350 §5): asked when it closed unmerged; a merged one's go " +
|
||||
"when the walk answering its merge ends, and any check build goes after a week.",
|
||||
Input: schema(map[string]string{"owner": "the repository's owner", "repo": "the repository",
|
||||
"number": "the pull request's number"}, []string{"owner", "repo", "number"})},
|
||||
{Name: "kill", Description: "End a build where it runs: the machine that took it stops its commands and containers and " +
|
||||
"announces it failed, killed by hand — settled, never handed to another machine.",
|
||||
Input: schema(map[string]string{"id": "the build's id"}, []string{"id"})},
|
||||
|
||||
@@ -58,7 +58,42 @@ func (i *Inventory) Dependencies(ctx context.Context) ([]Edge, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return dependenciesOf(entries, against, nil), nil
|
||||
held, err := i.Held(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return dependenciesWith(entries, against, held), nil
|
||||
}
|
||||
|
||||
// packagesHeld is, by package name, the module whose artifact the mesh holds as that package: a
|
||||
// `package` artifact's reference is `<name>@<version>` (the builder's publishPackage), never an
|
||||
// artifact-store reference, so a build that stood on it recorded the package's name and version.
|
||||
func packagesHeld(held map[string]string) map[string]string {
|
||||
out := map[string]string{}
|
||||
for key, reference := range held {
|
||||
module, _, ok := strings.Cut(key, "/")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if name := packageName(reference); name != "" {
|
||||
out[name] = module
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// packageName is the name of a `<name>@<version>` package reference, or "" for anything else: a
|
||||
// kept or pinned image (`…@sha256:…`), a URL, or a reference with no version.
|
||||
func packageName(reference string) string {
|
||||
if strings.HasPrefix(reference, catalogue.ArtifactStoreScheme) || strings.Contains(reference, "@sha256:") ||
|
||||
strings.Contains(reference, "://") {
|
||||
return ""
|
||||
}
|
||||
at := strings.LastIndex(reference, "@")
|
||||
if at <= 0 || at == len(reference)-1 {
|
||||
return ""
|
||||
}
|
||||
return reference[:at]
|
||||
}
|
||||
|
||||
// dependenciesOf is Dependencies over what was read, so a test can hand it a catalogue.
|
||||
@@ -66,6 +101,20 @@ func (i *Inventory) Dependencies(ctx context.Context) ([]Edge, error) {
|
||||
// `read` draws no edge (novox/hq ADR 0267 rule 4): what a build read moves it through its build source, in
|
||||
// the planner, never through the relation.
|
||||
func dependenciesOf(entries []Entry, against map[string][]string, _ map[string][]ReadRepository) []Edge {
|
||||
return dependenciesWith(entries, against, nil)
|
||||
}
|
||||
|
||||
// dependenciesWith is the relation with the artifacts the mesh holds, "<module>/<artifact>" to its
|
||||
// reference (Held), so a package a build stood on is read as its module's.
|
||||
//
|
||||
// **A built manifest carries no `build`** (catalogue.Resolve drops it), so what a module declared it
|
||||
// is built on is read from what is kept of it once it is built (novox/hq issue 614): the bases its
|
||||
// newest build was handed (`against`) — an artifact-store reference, or a package's `<name>@<version>`
|
||||
// — and the language of every bundle it compiled (`Bundles`). Read only from records the mesh already
|
||||
// keeps, so it holds for every module built before it, and a controller rolled back to one without it
|
||||
// reads the same records as before.
|
||||
func dependenciesWith(entries []Entry, against map[string][]string, held map[string]string) []Edge {
|
||||
packages := packagesHeld(held)
|
||||
known := map[string]bool{}
|
||||
var builders []string
|
||||
for _, e := range entries {
|
||||
@@ -111,6 +160,13 @@ func dependenciesOf(entries []Entry, against map[string][]string, _ map[string][
|
||||
}
|
||||
}
|
||||
}
|
||||
// **And once it is built**, from the bundles its resolved manifest keeps (novox/hq issue 614): the
|
||||
// `build` it was declared with is gone by then, and a toolchain that moved rebuilt none of them.
|
||||
for _, b := range e.Manifest.Bundles {
|
||||
if chain, err := builder.ToolchainFor(b.Language); err == nil {
|
||||
add(name, chain.Base, EdgeStandsOn)
|
||||
}
|
||||
}
|
||||
for _, ref := range against[name] {
|
||||
if rest, ok := strings.CutPrefix(ref, catalogue.ArtifactStoreScheme); ok {
|
||||
// A copy of an upstream image is no module's artifact (novox/hq ADR 0257): its
|
||||
@@ -121,6 +177,13 @@ func dependenciesOf(entries []Entry, against map[string][]string, _ map[string][
|
||||
if base, _, found := strings.Cut(rest, "/"); found {
|
||||
add(name, base, EdgeStandsOn)
|
||||
}
|
||||
continue
|
||||
}
|
||||
// A package base (`build.on` naming a module's `package` artifact) is recorded as the
|
||||
// package's name and version (novox/hq issue 614): the module that publishes that package
|
||||
// is what it stood on, whichever version the mesh holds now.
|
||||
if base, ok := packages[packageName(ref)]; ok {
|
||||
add(name, base, EdgeStandsOn)
|
||||
}
|
||||
}
|
||||
if e.Source.Repository != "" {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
@@ -174,3 +175,74 @@ func TestACopyOfAnUpstreamImageIsNoEdge(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A built module keeps its build edges** (novox/hq issue 614). A manifest the mesh registers from a
|
||||
// build carries no `build` (catalogue.Resolve drops it), so mesh-tools' `build.on` naming mesh-sdk's
|
||||
// `lib` package and a provider's TypeScript bundle drew no edge once built, and a new SDK rebuilt
|
||||
// neither. The package base is read from what the build was handed (`@novox/mesh-sdk@0.1.13`, the
|
||||
// package's name whichever version the mesh holds now), the toolchain from the bundles it keeps.
|
||||
func TestABuiltModuleKeepsItsBuildEdges(t *testing.T) {
|
||||
built := func(name string, bundles ...catalogue.Bundle) Entry {
|
||||
return Entry{Manifest: catalogue.Manifest{Module: name, Bundles: bundles},
|
||||
Source: Source{Repository: "http://forge/novox/" + name + ".git"}}
|
||||
}
|
||||
entries := []Entry{
|
||||
built("mesh-sdk"),
|
||||
built("mesh-tools"),
|
||||
built("mesh-tools-go"),
|
||||
built("minio", catalogue.Bundle{Name: "code", Language: "typescript"}),
|
||||
built("umami", catalogue.Bundle{Name: "code", Language: "TypeScript"}),
|
||||
built("dnsmasq", catalogue.Bundle{Name: "tools", Language: "go"}),
|
||||
built("plain"),
|
||||
}
|
||||
against := map[string][]string{
|
||||
"mesh-tools": {"@novox/mesh-sdk@0.1.13",
|
||||
catalogue.ArtifactStoreScheme + "upstream/docker.io/library/node@sha256:" + strings.Repeat("0", 64)},
|
||||
// A package nobody in the mesh publishes is no edge.
|
||||
"plain": {"left-pad@1.3.0"},
|
||||
}
|
||||
held := map[string]string{
|
||||
"mesh-sdk/lib": "@novox/mesh-sdk@0.1.14",
|
||||
"mesh-tools/runtime": catalogue.ArtifactStoreScheme + "mesh-tools/runtime@sha256:" + strings.Repeat("1", 64),
|
||||
}
|
||||
got := dependenciesWith(entries, against, held)
|
||||
has := func(from, to string) bool {
|
||||
for _, e := range got {
|
||||
if e.From == from && e.To == to && e.Kind == EdgeStandsOn {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
for _, want := range [][2]string{
|
||||
{"mesh-tools", "mesh-sdk"},
|
||||
{"minio", "mesh-tools"},
|
||||
{"umami", "mesh-tools"},
|
||||
{"dnsmasq", "mesh-tools-go"},
|
||||
} {
|
||||
if !has(want[0], want[1]) {
|
||||
t.Errorf("no %s stands-on %s: %+v", want[0], want[1], got)
|
||||
}
|
||||
}
|
||||
for _, e := range got {
|
||||
if e.From == "plain" && e.Kind != EdgeBuiltBy {
|
||||
t.Errorf("an unknown package drew an edge: %+v", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPackageNameIsReadOnlyFromAPackageReference(t *testing.T) {
|
||||
for ref, want := range map[string]string{
|
||||
"@novox/mesh-sdk@0.1.14": "@novox/mesh-sdk",
|
||||
"left-pad@1.3.0": "left-pad",
|
||||
"@novox/mesh-sdk": "",
|
||||
"novox.internal:5100/mesh-tools/build@sha256:" + strings.Repeat("0", 64): "",
|
||||
catalogue.ArtifactStoreScheme + "mesh-tools/build@sha256:0": "",
|
||||
"http://store/v2/x/blobs/sha256:0": "",
|
||||
"": "",
|
||||
} {
|
||||
if got := packageName(ref); got != want {
|
||||
t.Errorf("packageName(%q) = %q, want %q", ref, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,7 +44,13 @@ type Wait struct {
|
||||
Part string `json:"part"`
|
||||
Secret string `json:"secret,omitempty"`
|
||||
Setting string `json:"setting,omitempty"`
|
||||
What string `json:"what"`
|
||||
// Provision is a provision the module requires whose credential only a person can give and nobody gave yet
|
||||
// (novox/hq ADR 0337 §5): a wait names exactly one of Secret, Setting or Provision.
|
||||
Provision string `json:"provision,omitempty"`
|
||||
What string `json:"what"`
|
||||
// Act is the line that gives a provision's credential, filled by the controller when it excuses the wait
|
||||
// (ADR 0337 §7); never said by a machine.
|
||||
Act string `json:"act,omitempty"`
|
||||
}
|
||||
|
||||
// NodeHealth is a machine's newest statement, as kept.
|
||||
|
||||
@@ -36,7 +36,16 @@ type MadeOf struct {
|
||||
BootedAt *time.Time `json:"booted_at"`
|
||||
Timezone string `json:"timezone"`
|
||||
Filesystems []MountedFilesystem `json:"filesystems"`
|
||||
Unknown map[string]string `json:"unknown"`
|
||||
// Networks is every network the machine is directly attached to on a link facing outside it, as
|
||||
// link and network (novox/hq ADR 0340 §7): what a port reached from `local-network` resolves from.
|
||||
Networks []AttachedNetwork `json:"networks"`
|
||||
Unknown map[string]string `json:"unknown"`
|
||||
}
|
||||
|
||||
// AttachedNetwork is one network a machine said it is attached to, and the link it is on.
|
||||
type AttachedNetwork struct {
|
||||
Link string `json:"link"`
|
||||
Network string `json:"network"`
|
||||
}
|
||||
|
||||
// MountedFilesystem is one local filesystem a machine has mounted.
|
||||
|
||||
@@ -40,3 +40,18 @@ func TestAProfileWithoutTheFactsIsAnOlderNodeEngineNotNothing(t *testing.T) {
|
||||
t.Error("a node that never reported read as having reported")
|
||||
}
|
||||
}
|
||||
|
||||
// The networks a machine is attached to (novox/hq ADR 0340 §7), in the node-engine's words: what a port
|
||||
// reached from `local-network` resolves from.
|
||||
func TestAMachinesAttachedNetworksAreRead(t *testing.T) {
|
||||
a, had, err := machineFrom([]byte(`{"architecture":"amd64","kernel":"linux",
|
||||
"machine":{"observed_at":"2026-10-13T09:00:00Z",
|
||||
"networks":[{"link":"eth0","network":"192.168.1.0/24"},{"link":"eth1","network":"203.0.113.0/24"}]}}`))
|
||||
if err != nil || !had || a.Machine == nil {
|
||||
t.Fatalf("%+v had %v err %v", a, had, err)
|
||||
}
|
||||
n := a.Machine.Networks
|
||||
if len(n) != 2 || n[0].Link != "eth0" || n[0].Network != "192.168.1.0/24" || n[1].Network != "203.0.113.0/24" {
|
||||
t.Fatalf("networks read as %+v", n)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/secrets"
|
||||
)
|
||||
|
||||
// A provision whose credential only a person outside the mesh can give (novox/hq ADR 0337, issue 164): a Servarr
|
||||
// app's API key, an indexer's key, a downloader's password. The provider's software holds the value; the mesh can
|
||||
// neither make one that would work nor fetch it. So the mesh never makes one (§2), a person gives it once per
|
||||
// provider and it is sealed to every consumer of that provider (§3), and until then a consumer holds no credential
|
||||
// for it and is told so (§4), and the rest of it runs.
|
||||
|
||||
// NotGivenError is a pair credential only a person can give, which no person gave for this pair under the keys its
|
||||
// two ends have now: the consumer receives no credential for it, and waits (ADR 0337 §4, §8). It is not a fault of
|
||||
// the machine, so the plan does not refuse the machine for it.
|
||||
type NotGivenError struct {
|
||||
Provision, Consumer, ConsumerModule, Provider, ProviderModule, Local string
|
||||
// Own is set when the credential is the provider's own secret issued outside (ADR 0158, ADR 0228), and names it.
|
||||
Own string
|
||||
// Why is why no value is held, in words that never carry one.
|
||||
Why string
|
||||
}
|
||||
|
||||
func (e *NotGivenError) Error() string {
|
||||
if e.Own != "" {
|
||||
return fmt.Sprintf("%s on %s holds no %s from %s: %s; give it with `secret accept %s %s %s`",
|
||||
e.ConsumerModule, e.Consumer, e.Provision, e.Provider, e.Why, e.Provider, e.ProviderModule, e.Own)
|
||||
}
|
||||
return fmt.Sprintf("%s on %s holds no %s from %s: %s; it is issued outside the mesh and given with `%s`",
|
||||
e.ConsumerModule, e.Consumer, e.Provision, e.Provider, e.Why, acceptLine(e.Provider, e.ProviderModule, e.Provision))
|
||||
}
|
||||
|
||||
// IsNotGiven is whether an error is a credential only a person gives, not given yet.
|
||||
func IsNotGiven(err error) bool {
|
||||
var ng *NotGivenError
|
||||
return errors.As(err, &ng)
|
||||
}
|
||||
|
||||
// acceptLine is the one line that gives a provision's credential issued outside, for every consumer of a provider.
|
||||
func acceptLine(provider, providerModule, provision string) string {
|
||||
if providerModule == "" {
|
||||
providerModule = "<provider module>"
|
||||
}
|
||||
return fmt.Sprintf("secret accept %s %s %s", provider, providerModule, provision)
|
||||
}
|
||||
|
||||
// AcceptLine is acceptLine, for the controller's words.
|
||||
func AcceptLine(provider, providerModule, provision string) string {
|
||||
return acceptLine(provider, providerModule, provision)
|
||||
}
|
||||
|
||||
// issuedOutside is whether a module in the catalogue offers the provision with a credential issued outside the
|
||||
// mesh, and which.
|
||||
func (i *Inventory) issuedOutside(ctx context.Context, provision string) (string, bool, error) {
|
||||
shelf, err := i.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
module, outside := catalogue.OffersIssuedOutside(shelf, provision)
|
||||
return module, outside, nil
|
||||
}
|
||||
|
||||
// ownIssuedOutside is whether a provider's own secret says only an outside party issues it (ADR 0228).
|
||||
func (i *Inventory) ownIssuedOutside(ctx context.Context, providerModule, own string) (string, bool, error) {
|
||||
if providerModule == "" {
|
||||
return own, false, nil
|
||||
}
|
||||
m, err := i.declared(ctx, providerModule)
|
||||
if err != nil {
|
||||
return own, false, err
|
||||
}
|
||||
return own, m.OwnSecrets[own].IssuedBy == catalogue.IssuedOutside, nil
|
||||
}
|
||||
|
||||
// GivenPairSecret is the credential of a pair whose provision is issued outside the mesh, as a person gave it, and
|
||||
// never one the mesh makes (ADR 0337 §2, §4). It writes nothing. A value is answered only when a person gave it
|
||||
// (`accepted`) and it is sealed to the keys both ends have now; otherwise a NotGivenError says why — never given, or
|
||||
// given under a key one end no longer has (§8), or a value the mesh made before the rule, which is not given and
|
||||
// never delivered again (§2).
|
||||
func (i *Inventory) GivenPairSecret(ctx context.Context, name, consumer, consumerModule, provider, providerModule,
|
||||
local string) (Secret, error) {
|
||||
consumerKey, err := i.SealingKeyOf(ctx, consumer)
|
||||
if err != nil {
|
||||
return Secret{}, err
|
||||
}
|
||||
providerKey, err := i.SealingKeyOf(ctx, provider)
|
||||
if err != nil {
|
||||
return Secret{}, err
|
||||
}
|
||||
consumerNode, err := i.NodeByName(ctx, consumer)
|
||||
if err != nil {
|
||||
return Secret{}, err
|
||||
}
|
||||
providerNode, err := i.NodeByName(ctx, provider)
|
||||
if err != nil {
|
||||
return Secret{}, err
|
||||
}
|
||||
notGiven := func(why string) (Secret, error) {
|
||||
return Secret{}, &NotGivenError{Provision: name, Consumer: consumer, ConsumerModule: consumerModule,
|
||||
Provider: provider, ProviderModule: providerModule, Local: local, Why: why}
|
||||
}
|
||||
var held Secret
|
||||
err = i.store.Pool().QueryRow(ctx,
|
||||
`select for_consumer, for_provider, consumer_key, provider_key, origin from secret
|
||||
where name = $1 and consumer = $2 and consumer_module = $3 and provider = $4 and local = $5`,
|
||||
name, consumerNode.ID, consumerModule, providerNode.ID, local).
|
||||
Scan(&held.ForConsumer, &held.ForProvider, &held.ConsumerKey, &held.ProviderKey, &held.Origin)
|
||||
switch {
|
||||
case errors.Is(err, pgx.ErrNoRows):
|
||||
return notGiven("nobody gave it")
|
||||
case err != nil:
|
||||
return Secret{}, err
|
||||
case held.Origin != OriginAccepted:
|
||||
return notGiven("the value held was made by the mesh, which " + providerModule + "'s software would refuse")
|
||||
case consumerKey == "" || providerKey == "" || held.ConsumerKey != consumerKey || held.ProviderKey != providerKey:
|
||||
return notGiven("it was given under a sealing key one end no longer has")
|
||||
}
|
||||
held.Name, held.Consumer, held.Provider = name, consumer, provider
|
||||
held.ConsumerModule, held.Local = consumerModule, local
|
||||
return held, nil
|
||||
}
|
||||
|
||||
// PairEnd is one consumer of a provision from one provider: the machine, the module there and its local name.
|
||||
type PairEnd struct {
|
||||
Consumer, ConsumerModule, Local string
|
||||
}
|
||||
|
||||
// ProvisionAccepted is what one accept for a provider sealed (ADR 0337 §3): the consumers it reached, those it could
|
||||
// not (no sealing key on their machine), and the machines to send.
|
||||
type ProvisionAccepted struct {
|
||||
Sealed, Unsealable []PairEnd
|
||||
Machines []string
|
||||
}
|
||||
|
||||
// AcceptSecretForProvision seals one value a person gave for a provision issued outside the mesh to the provider's
|
||||
// machine, to every consumer named, and to the operator, in one act under one generation, each row `accepted`
|
||||
// (ADR 0337 §3, issue 165); the plaintext is not kept. The consumers are every one that binds the provision from
|
||||
// this provider now, which the caller finds by resolving every machine. A consumer whose machine has no sealing key
|
||||
// is not sealed to, and said.
|
||||
func (i *Inventory) AcceptSecretForProvision(ctx context.Context, provision, provider, providerModule, value string,
|
||||
consumers []PairEnd) (ProvisionAccepted, error) {
|
||||
m, err := i.declared(ctx, providerModule)
|
||||
if err != nil {
|
||||
return ProvisionAccepted{}, err
|
||||
}
|
||||
if !m.IssuedOutsideOf(provision) {
|
||||
return ProvisionAccepted{}, fmt.Errorf("%s does not offer %q with a credential issued outside the mesh "+
|
||||
"(\"credential\": {\"issued-by\": \"outside\"}, ADR 0337), so its consumers' credentials are not given "+
|
||||
"once for the provider", providerModule, provision)
|
||||
}
|
||||
if value == "" {
|
||||
return ProvisionAccepted{}, errors.New("there is nothing to seal")
|
||||
}
|
||||
providerNode, err := i.NodeByName(ctx, provider)
|
||||
if err != nil {
|
||||
return ProvisionAccepted{}, err
|
||||
}
|
||||
providerKey, err := i.SealingKeyOf(ctx, provider)
|
||||
if err != nil {
|
||||
return ProvisionAccepted{}, err
|
||||
}
|
||||
if providerKey == "" {
|
||||
return ProvisionAccepted{}, fmt.Errorf("%s has no sealing key, so nothing can be sealed to it; a node joins to get one", provider)
|
||||
}
|
||||
forOperator, operatorKey, err := i.operatorSeal(ctx, value)
|
||||
if err != nil {
|
||||
return ProvisionAccepted{}, err
|
||||
}
|
||||
generation := secrets.Stamp()
|
||||
var out ProvisionAccepted
|
||||
machines := map[string]bool{provider: true}
|
||||
tx, err := i.store.Pool().Begin(ctx)
|
||||
if err != nil {
|
||||
return ProvisionAccepted{}, err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
for _, c := range consumers {
|
||||
consumerNode, err := i.NodeByName(ctx, c.Consumer)
|
||||
if err != nil {
|
||||
return ProvisionAccepted{}, err
|
||||
}
|
||||
consumerKey, err := i.SealingKeyOf(ctx, c.Consumer)
|
||||
if err != nil {
|
||||
return ProvisionAccepted{}, err
|
||||
}
|
||||
if consumerKey == "" {
|
||||
out.Unsealable = append(out.Unsealable, c)
|
||||
continue
|
||||
}
|
||||
sealed, err := secrets.Accept(value, consumerKey, providerKey)
|
||||
if err != nil {
|
||||
return ProvisionAccepted{}, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx,
|
||||
`insert into secret (name, consumer, consumer_module, provider, for_consumer, for_provider,
|
||||
consumer_key, provider_key, operator_sealed, operator_key, origin, local, generation)
|
||||
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
|
||||
on conflict (name, local, consumer, consumer_module, provider) do update set
|
||||
for_consumer = excluded.for_consumer, for_provider = excluded.for_provider,
|
||||
consumer_key = excluded.consumer_key, provider_key = excluded.provider_key,
|
||||
created_at = now(), origin = excluded.origin, generation = excluded.generation,
|
||||
operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`,
|
||||
provision, consumerNode.ID, c.ConsumerModule, providerNode.ID,
|
||||
sealed.ForConsumer, sealed.ForProvider, sealed.ConsumerKey, sealed.ProviderKey,
|
||||
forOperator, operatorKey, OriginAccepted, c.Local, generation); err != nil {
|
||||
return ProvisionAccepted{}, err
|
||||
}
|
||||
out.Sealed = append(out.Sealed, c)
|
||||
machines[c.Consumer] = true
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return ProvisionAccepted{}, err
|
||||
}
|
||||
for n := range machines {
|
||||
out.Machines = append(out.Machines, n)
|
||||
}
|
||||
sort.Strings(out.Machines)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// GivenPairsOf is every provision a module on a machine holds a value a person gave for, sealed to the keys both
|
||||
// ends have now: what a wait for a provision's credential is checked against (ADR 0337 §6).
|
||||
func (i *Inventory) GivenPairsOf(ctx context.Context, machine, module string) (map[string]bool, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select s.name
|
||||
from secret s
|
||||
join node c on c.id = s.consumer
|
||||
join node p on p.id = s.provider
|
||||
where c.name = $1 and s.consumer_module = $2 and s.origin = 'accepted'
|
||||
and s.consumer_key = c.sealing_key and s.provider_key = p.sealing_key`, machine, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]bool{}
|
||||
for rows.Next() {
|
||||
var name string
|
||||
if err := rows.Scan(&name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[name] = true
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// outsideProvision registers a provider offering tv-api with a credential issued outside the mesh (novox/hq ADR
|
||||
// 0337 §1) and two consumers requiring it, on twoNodesWithKeys' machines.
|
||||
func outsideProvision(t *testing.T, inv *Inventory) {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "tv", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "tv-api", Scope: catalogue.ScopeMesh,
|
||||
Credential: &catalogue.OfferCredential{IssuedBy: catalogue.IssuedOutside}}}}, Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range []string{"requests", "indexer"} {
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1", Requires: []string{"tv-api"},
|
||||
Secrets: map[string]string{"tv-api": "/run/tv-api.secret"}}, Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pairRows is how many pair rows the store holds for tv-api.
|
||||
func pairRows(t *testing.T, inv *Inventory) int {
|
||||
t.Helper()
|
||||
var n int
|
||||
if err := inv.store.Pool().QueryRow(t.Context(), `select count(*) from secret where name = 'tv-api'`).Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// A credential only a person can give is never made (ADR 0337 §2, issue 164): not for a pair with no row, which is
|
||||
// what a newly bound consumer has, and not after a key change; a value the mesh made before the rule is not given.
|
||||
func TestACredentialIssuedOutsideIsNeverMade(t *testing.T) {
|
||||
inv, ctx := twoNodesWithKeys(t)
|
||||
outsideProvision(t, inv)
|
||||
|
||||
_, err := inv.SecretFor(ctx, "tv-api", "consumer", "requests", "provider", "")
|
||||
if !IsNotGiven(err) {
|
||||
t.Fatalf("SecretFor made or refused a credential only a person gives, rather than saying it was not given: %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "secret accept provider tv tv-api") {
|
||||
t.Fatalf("not given does not name the one line that gives it: %v", err)
|
||||
}
|
||||
if n := pairRows(t, inv); n != 0 {
|
||||
t.Fatalf("the store holds %d rows for tv-api, which nobody gave", n)
|
||||
}
|
||||
if _, err := inv.GivenPairSecret(ctx, "tv-api", "consumer", "requests", "provider", "tv", ""); !IsNotGiven(err) {
|
||||
t.Fatalf("a pair nobody gave read as given: %v", err)
|
||||
}
|
||||
if n := pairRows(t, inv); n != 0 {
|
||||
t.Fatalf("reading the pair wrote %d rows", n)
|
||||
}
|
||||
|
||||
// A value the mesh made before the rule (the minted value of issue 164) is not given.
|
||||
if _, err := inv.store.Pool().Exec(ctx, `insert into secret (name, consumer, consumer_module, provider,
|
||||
for_consumer, for_provider, consumer_key, provider_key, local)
|
||||
select 'tv-api', c.id, 'indexer', p.id, 'x', 'y', c.sealing_key, p.sealing_key, ''
|
||||
from node c, node p where c.name = 'consumer' and p.name = 'provider'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.GivenPairSecret(ctx, "tv-api", "consumer", "indexer", "provider", "tv", ""); !IsNotGiven(err) ||
|
||||
!strings.Contains(err.Error(), "made by the mesh") {
|
||||
t.Fatalf("a value the mesh made before the rule read as given: %v", err)
|
||||
}
|
||||
|
||||
// Given, then a key changes: not given, and nothing is made in its place (ADR 0337 §8).
|
||||
if _, err := inv.AcceptSecretForProvision(ctx, "tv-api", "provider", "tv", "the app's key",
|
||||
[]PairEnd{{Consumer: "consumer", ConsumerModule: "requests"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.GivenPairSecret(ctx, "tv-api", "consumer", "requests", "provider", "tv", ""); err != nil {
|
||||
t.Fatalf("the value given does not read as given: %v", err)
|
||||
}
|
||||
cn, _ := inv.NodeByName(ctx, "consumer")
|
||||
key, _ := aSealingKey(t)
|
||||
if err := inv.RecordSealingKey(ctx, cn.ID, key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.GivenPairSecret(ctx, "tv-api", "consumer", "requests", "provider", "tv", ""); !IsNotGiven(err) ||
|
||||
!strings.Contains(err.Error(), "sealing key") {
|
||||
t.Fatalf("a value given under a key the consumer no longer has read as given: %v", err)
|
||||
}
|
||||
var origin string
|
||||
if err := inv.store.Pool().QueryRow(ctx, `select origin from secret where name = 'tv-api' and consumer_module = 'requests'`).
|
||||
Scan(&origin); err != nil || origin != OriginAccepted {
|
||||
t.Fatalf("the given row was replaced (origin %q, %v)", origin, err)
|
||||
}
|
||||
|
||||
// Rotate refuses it, naming the accept (ADR 0337 §9).
|
||||
err = inv.RotateSecret(ctx, "tv-api", "consumer", "requests", "provider", "")
|
||||
if err == nil || !strings.Contains(err.Error(), "secret accept provider tv tv-api") {
|
||||
t.Fatalf("rotate of a credential issued outside was not refused naming the accept: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// One accept for a provider is sealed to every consumer named, to the provider and to the operator, under one
|
||||
// generation, each row accepted (ADR 0337 §3, issue 165); a consumer with no sealing key is said, not sealed to.
|
||||
func TestOneAcceptForAProviderIsSealedToEveryConsumer(t *testing.T) {
|
||||
inv, ctx := twoNodesWithKeys(t)
|
||||
outsideProvision(t, inv)
|
||||
operatorKey, operatorOpens := aSealingKey(t)
|
||||
if _, err := inv.SetOperatorKey(ctx, operatorKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumerKey, consumerOpens := aSealingKey(t)
|
||||
cn, _ := inv.NodeByName(ctx, "consumer")
|
||||
if err := inv.RecordSealingKey(ctx, cn.ID, consumerKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.AddNode(ctx, "keyless"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
accepted, err := inv.AcceptSecretForProvision(ctx, "tv-api", "provider", "tv", "the app's key", []PairEnd{
|
||||
{Consumer: "consumer", ConsumerModule: "requests"}, {Consumer: "consumer", ConsumerModule: "indexer"},
|
||||
{Consumer: "keyless", ConsumerModule: "requests"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(accepted.Sealed) != 2 || len(accepted.Unsealable) != 1 || accepted.Unsealable[0].Consumer != "keyless" {
|
||||
t.Fatalf("the accept did not seal to both consumers with keys and say the keyless one: %+v", accepted)
|
||||
}
|
||||
if strings.Join(accepted.Machines, ",") != "consumer,provider" {
|
||||
t.Fatalf("the machines to send are the provider's and every consumer's sealed to: %v", accepted.Machines)
|
||||
}
|
||||
rows, err := inv.store.Pool().Query(ctx, `select consumer_module, origin, coalesce(generation, ''), for_consumer,
|
||||
coalesce(operator_sealed, '') from secret where name = 'tv-api'`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
generations := map[string]bool{}
|
||||
seen := 0
|
||||
for rows.Next() {
|
||||
var module, origin, generation, forConsumer, forOperator string
|
||||
if err := rows.Scan(&module, &origin, &generation, &forConsumer, &forOperator); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seen++
|
||||
if origin != OriginAccepted || generation == "" {
|
||||
t.Fatalf("%s's row is not accepted under a generation (origin %q)", module, origin)
|
||||
}
|
||||
generations[generation] = true
|
||||
if opened, ok := consumerOpens(forConsumer); !ok || string(opened) != "the app's key" {
|
||||
t.Fatalf("%s's copy does not open to the value given", module)
|
||||
}
|
||||
if opened, ok := operatorOpens(forOperator); !ok || string(opened) != "the app's key" {
|
||||
t.Fatalf("%s's operator copy does not open to the value given", module)
|
||||
}
|
||||
}
|
||||
if seen != 2 || len(generations) != 1 {
|
||||
t.Fatalf("%d rows under %d generations; want 2 under one", seen, len(generations))
|
||||
}
|
||||
given, err := inv.GivenPairsOf(ctx, "consumer", "indexer")
|
||||
if err != nil || !given["tv-api"] {
|
||||
t.Fatalf("the provision given does not read as given for indexer (%v)", err)
|
||||
}
|
||||
|
||||
// Only an offer that says so is given this way.
|
||||
if _, err := inv.AcceptSecretForProvision(ctx, "secret", "provider", "gitea", "x", nil); err == nil {
|
||||
t.Fatal("a provision not issued outside was given once for its provider")
|
||||
}
|
||||
}
|
||||
|
||||
// The provider's own secret issued outside the mesh, named as a provision's shared credential (ADR 0158, ADR 0228),
|
||||
// is never made for a consumer that binds: the consumer is told it was not given (ADR 0337 §2, §4).
|
||||
func TestASharedOwnSecretIssuedOutsideIsNeverMade(t *testing.T) {
|
||||
inv, ctx := twoNodesWithKeys(t)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "downloader", Version: "1",
|
||||
OwnSecrets: catalogue.OwnSecrets{"password": {Path: "/run/password", Taken: catalogue.TakenAtStart,
|
||||
IssuedBy: catalogue.IssuedOutside}},
|
||||
Provides: []catalogue.Offer{{Name: "downloader-api", Scope: catalogue.ScopeMesh,
|
||||
Credential: &catalogue.OfferCredential{Own: "password"}}}}, Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := inv.SharedSecretFor(ctx, "downloader-api", "consumer", "gitea", "provider", "downloader", "", "password")
|
||||
if !IsNotGiven(err) {
|
||||
t.Fatalf("a shared own secret issued outside was made or refused rather than not given: %v", err)
|
||||
}
|
||||
var n int
|
||||
if err := inv.store.Pool().QueryRow(ctx, `select count(*) from module_secret where module = 'downloader'`).Scan(&n); err != nil || n != 0 {
|
||||
t.Fatalf("the mesh made %d values of an own secret issued outside (%v)", n, err)
|
||||
}
|
||||
if err := inv.remakeShared(ctx, nil, "", "downloader", "password", "", nil, "", "", ""); err == nil ||
|
||||
!strings.Contains(err.Error(), "issued outside") {
|
||||
t.Fatalf("remakeShared made a value of an own secret issued outside: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -220,6 +220,11 @@ type PlanDelivery struct {
|
||||
// Withheld are, on a walk cut from a batch a stop walked merges again in, the modules it did not build: a
|
||||
// stopped merge moves them, and built only as dependents at the branch they would deliver it (ADR 0299).
|
||||
Withheld []string `json:"withheld,omitempty"`
|
||||
// Unchanged are the modules a merge it answers changed files in the directory of, whose recorded build source
|
||||
// holds none of them, each with why (novox/hq issue 611, ADR 0267 §5): not built, because the build the mesh
|
||||
// holds is their build. A delivery plan that read them whole named them; this says they matched what runs,
|
||||
// rather than that the walk left them unbuilt. Modules keeps meaning what the walk builds.
|
||||
Unchanged map[string]string `json:"unchanged,omitempty"`
|
||||
// Batch is the window of a batch; nil once it is cut into a walk.
|
||||
Batch *PlanBatch `json:"batch,omitempty"`
|
||||
// Alone says the walk walks one merge on its own commit, after a failed walk carried it in a later one:
|
||||
@@ -281,6 +286,9 @@ type PlanModule struct {
|
||||
FirstAt *time.Time `json:"first_at,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
// Made is what the walk's build of it did (novox/hq ADR 0350 §4): "reused from <check build>", naming the pull
|
||||
// request's head, the trunk commit and the digests, or "built: <why>". Said in `plan-moved` and kept here.
|
||||
Made string `json:"made,omitempty"`
|
||||
// Build is the id of the build the plan asked for this module (novox/hq ADR 0219), so the plan
|
||||
// matches its outcome by id — the one thing every outcome echoes, a failed one that never learnt
|
||||
// its module's name included. Empty in a plan from before it was kept, which is matched by
|
||||
@@ -318,13 +326,25 @@ type PlanGate struct {
|
||||
// machine's report is held against. Absent on a gate kept before it was, which reads the report
|
||||
// against the send made last, as before.
|
||||
Sent map[string]SentDeclaration `json:"sent,omitempty"`
|
||||
// Applied is, per machine, the time of the first report the gate read of its send applied: the
|
||||
// node-engine started the build before it, so a witness window counted from it ends no earlier than the
|
||||
// node-engine's own (novox/hq ADR 0351 decisions 1 and 4).
|
||||
Applied map[string]time.Time `json:"applied,omitempty"`
|
||||
// Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that
|
||||
// does not resets them.
|
||||
Passes int `json:"passes,omitempty"`
|
||||
// Wants is how many passes the send needs: a core module's three, a leaf module's one (novox/hq ADR 0282
|
||||
// decision 4 as ADR 0304 decision 3 changed it). Zero in a gate judged before it was said: three.
|
||||
// Wants is how many passes the send needs: three for the node-engine or the mesh's resolver, or fewer at two
|
||||
// minutes when healthy throughout, one for any other module (novox/hq ADR 0351 and ADR 0352, changing ADR
|
||||
// 0282 decision 4 and ADR 0304 decision 3). Zero in a gate
|
||||
// judged before it was said: three.
|
||||
Wants int `json:"wants,omitempty"`
|
||||
LastPass *time.Time `json:"last_pass,omitempty"`
|
||||
// Faulted says a judging found the send broken, or not healthy after a healthy reading: a module under the three-reading
|
||||
// rule then passes on its readings alone, never at its two minutes (novox/hq ADR 0352).
|
||||
Faulted bool `json:"faulted,omitempty"`
|
||||
// HealthySeen says a judging found the send healthy, counted or not: one taken before a module's floor
|
||||
// counts too, so a dip after it is a fault (novox/hq ADR 0352 decision 3).
|
||||
HealthySeen bool `json:"healthy_seen,omitempty"`
|
||||
// Last is what the newest judging found wanting, while it still may pass.
|
||||
Last string `json:"last,omitempty"`
|
||||
// Verdict is empty while judging, then passed or failed, with Why, at JudgedAt, Took after Since.
|
||||
@@ -461,6 +481,20 @@ type PlanRelease struct {
|
||||
Skipped []string `json:"skipped,omitempty"`
|
||||
// By is the person who released it, empty when the mesh did.
|
||||
By string `json:"by,omitempty"`
|
||||
// Settings is set on a settings walk (novox/hq ADR 0344): not the builds waiting for a gate, but a settings
|
||||
// layer the operator approved, sent to the machines it reaches one at a time, its module judged on each.
|
||||
Settings *PlanSettings `json:"settings,omitempty"`
|
||||
}
|
||||
|
||||
// PlanSettings is what a settings walk sends (novox/hq ADR 0344): the module whose layer the operator approved, the
|
||||
// layer's scope, the ask that approved it and what changed, in words.
|
||||
type PlanSettings struct {
|
||||
Module string `json:"module"`
|
||||
// Node is the machine the layer is for, empty for the whole mesh.
|
||||
Node string `json:"node,omitempty"`
|
||||
// Ask is the proposal the operator approved, and Change what its layer changed ("~ prefs").
|
||||
Ask string `json:"ask"`
|
||||
Change string `json:"change,omitempty"`
|
||||
}
|
||||
|
||||
// The states a plan passes through.
|
||||
|
||||
@@ -104,6 +104,21 @@ func (i *Inventory) SecretFor(ctx context.Context, name, consumer, consumerModul
|
||||
consumerModule, name, provider, consumer, consumerModule, name, provider, localFlag(local))
|
||||
}
|
||||
|
||||
// **Never made where only a person can give it** (novox/hq ADR 0337 §2, issue 164): a provision whose offer
|
||||
// says `issued-by: outside` gets no value of the mesh's, which its provider's software would refuse. Asked
|
||||
// here, on the one path that makes a value, so no caller can have one made by not knowing; the plan reads
|
||||
// such a pair with GivenPairSecret, which never makes one and delivers only a value a person gave.
|
||||
why := "nobody gave it"
|
||||
if err == nil {
|
||||
why = "the value held was sealed to a key one end no longer has"
|
||||
}
|
||||
if providerModule, outside, oerr := i.issuedOutside(ctx, name); oerr != nil {
|
||||
return Secret{}, oerr
|
||||
} else if outside {
|
||||
return Secret{}, &NotGivenError{Provision: name, Consumer: consumer, ConsumerModule: consumerModule,
|
||||
Provider: provider, ProviderModule: providerModule, Local: local, Why: why}
|
||||
}
|
||||
|
||||
// And to the operator, when the mesh has one (novox/hq ADR 0085, amended): the third copy that
|
||||
// makes a vault-provided secret recoverable, and nothing the mesh can open.
|
||||
operator, err := i.OperatorKey(ctx)
|
||||
@@ -239,6 +254,16 @@ func (i *Inventory) RotateSecret(ctx context.Context, name, consumer, consumerMo
|
||||
`select origin from secret where name = $1 and consumer = $2 and consumer_module = $3
|
||||
and provider = $4 and local = $5`,
|
||||
name, consumerNode.ID, consumerModule, providerNode.ID, local).Scan(&origin)
|
||||
if providerModule, outside, oerr := i.issuedOutside(ctx, name); oerr != nil {
|
||||
return oerr
|
||||
} else if outside {
|
||||
// Only a person can give it, so only a person can give its next value (novox/hq ADR 0337 §9, ADR 0092 §7):
|
||||
// deleting the row would leave the consumer waiting for nothing new.
|
||||
return fmt.Errorf(
|
||||
"%s is issued outside the mesh: %s's software holds it and the mesh cannot make its replacement. "+
|
||||
"Give the new value with `%s`, which seals it to every consumer of %s on %s",
|
||||
name, providerModule, acceptLine(provider, providerModule, name), providerModule, provider)
|
||||
}
|
||||
if err == nil && origin == OriginAccepted {
|
||||
return fmt.Errorf(
|
||||
"%s's %q credential from %s was accepted from a person, and the mesh cannot make "+
|
||||
@@ -886,6 +911,15 @@ func (i *Inventory) SharedSecretFor(ctx context.Context, provision, consumer, co
|
||||
held.ConsumerModule, held.Local = consumerModule, local
|
||||
return held, nil
|
||||
}
|
||||
if own, outside, err := i.ownIssuedOutside(ctx, providerModule, own); err != nil {
|
||||
return Secret{}, err
|
||||
} else if outside && (ownOrigin == nil || *ownOrigin != OriginAccepted) {
|
||||
// The provider's own secret only an outside party issues, which nobody gave (ADR 0228, issue 401): the
|
||||
// mesh makes no value for it here either, and the consumer is told it was not given (ADR 0337 §4).
|
||||
return Secret{}, &NotGivenError{Provision: provision, Consumer: consumer, ConsumerModule: consumerModule,
|
||||
Provider: provider, ProviderModule: providerModule, Local: local, Own: own,
|
||||
Why: "nobody gave " + providerModule + "'s own secret " + own + ", which is issued outside the mesh"}
|
||||
}
|
||||
if ownOrigin != nil && *ownOrigin == OriginAccepted {
|
||||
return Secret{}, fmt.Errorf(
|
||||
"%s on %s needs %s from %s, whose credential is %s's own secret %q — a value given to the "+
|
||||
@@ -912,6 +946,11 @@ func (i *Inventory) remakeShared(ctx context.Context, providerID any, providerKe
|
||||
if len(provisions) == 0 {
|
||||
return fmt.Errorf("%s names no provision whose credential is its own secret %q", providerModule, own)
|
||||
}
|
||||
if m.OwnSecrets[own].IssuedBy == catalogue.IssuedOutside {
|
||||
// Never made (novox/hq ADR 0228, ADR 0337 §2): only a person can give it, with `secret accept`.
|
||||
return fmt.Errorf("%s's own secret %q is issued outside the mesh, so the mesh makes no value for it; a "+
|
||||
"person gives it with `secret accept <machine> %s %s`", providerModule, own, providerModule, own)
|
||||
}
|
||||
operator, err := i.OperatorKey(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// A build kept from before ADR 0326 §3 is still read and composed (novox/hq issue 574). betterbird's
|
||||
// builds from before mesh-catalog 8f80cd4 declare a preference and own a mergeable file that names no
|
||||
// keys: refused where a definition is written, but a machine still runs that build and a rollback may
|
||||
// put it back, so reading it again — ManifestAt, which a walk, the gate and a release compose from —
|
||||
// must not refuse it.
|
||||
func TestAStoredBuildWithASilentMergeableFileIsStillReadAndComposed(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
ctx := context.Background()
|
||||
manifest, err := json.Marshal(map[string]any{
|
||||
"module": "mail-client", "version": "1",
|
||||
"settings": map[string]any{"prefs": map[string]any{
|
||||
"kind": catalogue.KindPreference, "default": "// none", "why": "its own defaults until a machine names one"}},
|
||||
"resources": []any{
|
||||
map[string]any{"id": "user-js", "type": "file", "path": "/var/lib/mail-client/user.js", "mode": "0644",
|
||||
"content": "${setting:prefs}\n"},
|
||||
map[string]any{"id": "accounts", "type": "file", "path": "/var/lib/mail-client/accounts.json",
|
||||
"mode": "0600", "merge": "json", "content": "{\"accounts\": {}}\n"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := aBuild("before-574", "mail-client", "")
|
||||
b.Manifest = manifest
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
m, found, err := inv.ManifestAt(ctx, "mail-client", b.Commit)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("the stored build is not read again: found %v, %v", found, err)
|
||||
}
|
||||
if written := catalogue.WrittenSettingsProblems(m); len(written) != 1 {
|
||||
t.Fatalf("the rule this build predates should name its accounts file where it is written: %v", written)
|
||||
}
|
||||
|
||||
// And composed, in today's form: the accounts file still takes its layer's keys.
|
||||
with := catalogue.Rendering{Settings: catalogue.SettingsBy{"mail-client": {{From: "workstation",
|
||||
Values: map[string]any{"accounts": map[string]any{"work": map[string]any{"email": "a@example.invalid"}}}}}}}
|
||||
if err := catalogue.JudgeSettings(m, with.Settings["mail-client"], false); err != nil {
|
||||
t.Fatalf("the stored build's settings do not compose: %v", err)
|
||||
}
|
||||
out, err := catalogue.Resolution{Node: "workstation", Modules: []catalogue.Manifest{m}}.Declaration(with)
|
||||
if err != nil {
|
||||
t.Fatalf("the stored build is not composed: %v", err)
|
||||
}
|
||||
for _, r := range out {
|
||||
if r["id"] == "mail-client.accounts" {
|
||||
if content, _ := r["content"].(string); !strings.Contains(content, "a@example.invalid") {
|
||||
t.Fatalf("the accounts file did not take its layer: %v", content)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("the accounts file was not composed: %v", out)
|
||||
}
|
||||
@@ -92,6 +92,38 @@ type BuildRequest struct {
|
||||
// facts snapshot, raises the throwaway stores the check needs, runs the repository's own
|
||||
// merge-check.sh and answers its verdict. Nothing is built, published or registered.
|
||||
Check *CheckRequest `json:"check,omitempty"`
|
||||
// Reuse makes this ask a walk's build of a trunk commit (novox/hq ADR 0350 §2): the builder takes the reuse key
|
||||
// of the commit's tree and, where a check build holds the same key, copies it by digest into the module's
|
||||
// namespace instead of building. Pulls are the pull requests the walk answers, whose checks name the part of a
|
||||
// differing key. Nil for every other ask — a check, a dry run, `build --ref`, `rebuild`, `replay` — which never
|
||||
// takes a check's build. A builder from before it ignores it and builds.
|
||||
Reuse *ReuseAsk `json:"reuse,omitempty"`
|
||||
}
|
||||
|
||||
// ReuseAsk is a walk's leave to take a check's build (novox/hq ADR 0350 §2).
|
||||
//
|
||||
// **Only a build a verdict names is taken.** Builds are the builds the build seat said, in the outcomes of its checks
|
||||
// of the walk's own pull requests, that it made for their merges: each key with its artifacts' digests. The record
|
||||
// in the scratch namespace is not trusted alone — the artifact store takes writes from anything on the network, a
|
||||
// check runs a pull request's own script on that network, and a record planted under another merge's key would
|
||||
// otherwise be registered. The builder takes a scratch build only when a build here names its key and every digest.
|
||||
type ReuseAsk struct {
|
||||
Pulls []PullRef `json:"pulls,omitempty"`
|
||||
Builds []CheckedBuild `json:"builds,omitempty"`
|
||||
}
|
||||
|
||||
// CheckedBuild is one build a check's verdict names: the pull request it checked, its ask, and the build.
|
||||
type CheckedBuild struct {
|
||||
Pull PullRef `json:"pull"`
|
||||
Check string `json:"check"`
|
||||
Built CheckBuilt `json:"built"`
|
||||
}
|
||||
|
||||
// PullRef names a pull request.
|
||||
type PullRef struct {
|
||||
Owner string `json:"owner"`
|
||||
Repo string `json:"repo"`
|
||||
Number int `json:"number"`
|
||||
}
|
||||
|
||||
// CheckRequest is what a merge check needs beyond the repository and its head.
|
||||
@@ -128,6 +160,34 @@ type CheckRequest struct {
|
||||
// repository's own check is not run for a group: each member's pull request runs its own.
|
||||
Group string `json:"group,omitempty"`
|
||||
Members []GroupMember `json:"members,omitempty"`
|
||||
// Builds are the modules the delivery plan moves, in the walk's order (novox/hq ADR 0350 §1): the check builds
|
||||
// each, from the merge it judged, into the scratch namespace once it has judged and not failed, for the walk to
|
||||
// take. A build seat from before it builds nothing, and the walk builds as before.
|
||||
Builds []CheckBuild `json:"builds,omitempty"`
|
||||
}
|
||||
|
||||
// CheckBuild is one module a check builds for its merge: from the merge it made when Merged — a module built from
|
||||
// the repository checked, at the branch merged into — else from Repository at Ref, as the walk would.
|
||||
type CheckBuild struct {
|
||||
Module string `json:"module"`
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Merged bool `json:"merged,omitempty"`
|
||||
Tier int `json:"tier"`
|
||||
// Skip says why the check cannot build it as the walk would — a module that reads the repository checked
|
||||
// through a context, which a check cannot hand the merge — and nothing after it is built either.
|
||||
Skip string `json:"skip,omitempty"`
|
||||
}
|
||||
|
||||
// CheckBuilt is one build a check made for its merge, as its verdict lists it (novox/hq ADR 0350 §1): the reuse key
|
||||
// and the artifacts, in the scratch namespace — or why it was not built.
|
||||
type CheckBuilt struct {
|
||||
Module string `json:"module"`
|
||||
Key string `json:"key,omitempty"`
|
||||
Made []MadeArtifact `json:"made,omitempty"`
|
||||
NotBuilt string `json:"not-built,omitempty"`
|
||||
Took string `json:"took,omitempty"`
|
||||
}
|
||||
|
||||
// GroupMember is one other head of a delivery group's composed check.
|
||||
@@ -175,6 +235,8 @@ type CheckOutcome struct {
|
||||
Trunk string `json:"trunk,omitempty"`
|
||||
Beside map[string]string `json:"beside,omitempty"`
|
||||
Members []CheckedMember `json:"members,omitempty"`
|
||||
// Built are the builds it made for its merge (novox/hq ADR 0350 §1).
|
||||
Built []CheckBuilt `json:"built,omitempty"`
|
||||
}
|
||||
|
||||
// SourceOnSeat names a repository by the seat whose holder serves it and its path there.
|
||||
@@ -257,9 +319,21 @@ type BuildResult struct {
|
||||
// build; then only identical artifacts make a rebuild no move.
|
||||
SourceFingerprint string `json:"source-fingerprint,omitempty"`
|
||||
|
||||
// Delivery is the how-manyth time the queue handed this ask out (BuildStart.Delivery): more than 1
|
||||
// says the outcome is from a run started again after an earlier taker left it (novox/hq issue 594).
|
||||
Delivery int `json:"delivery,omitempty"`
|
||||
|
||||
// Failed is why, when it did.
|
||||
Failed string `json:"failed,omitempty"`
|
||||
|
||||
// ReuseKey is the build's reuse key (novox/hq ADR 0350 §1). Reused, on a walk's build that took a check's build
|
||||
// instead of building, is "reused from <check build>" with its digests, and ReusedFrom which; BuiltWhy, on one
|
||||
// that built, why it took none (§3, §4). Both empty from a builder that predates it.
|
||||
ReuseKey string `json:"reuse-key,omitempty"`
|
||||
Reused string `json:"reused,omitempty"`
|
||||
ReusedFrom *ReusedFrom `json:"reused-from,omitempty"`
|
||||
BuiltWhy string `json:"built-why,omitempty"`
|
||||
|
||||
// Source is the request's, echoed: the seat form of the repository, for whoever registers.
|
||||
Source *SourceOnSeat `json:"source,omitempty"`
|
||||
|
||||
@@ -296,3 +370,25 @@ type MadeArtifact struct {
|
||||
Kind string `json:"kind"`
|
||||
Reference string `json:"reference"`
|
||||
}
|
||||
|
||||
// ReusedFrom is the check build a walk's build took (novox/hq ADR 0350 §4): its pull request, the head and the trunk
|
||||
// commit the check merged it into, the check's ask, and its key.
|
||||
type ReusedFrom struct {
|
||||
Pull PullRef `json:"pull"`
|
||||
Head string `json:"head,omitempty"`
|
||||
Trunk string `json:"trunk,omitempty"`
|
||||
Check string `json:"check,omitempty"`
|
||||
Key string `json:"key"`
|
||||
}
|
||||
|
||||
// MadeWords is what a walk says a module's build did (novox/hq ADR 0350 §4): "reused from <check build>", or "built:
|
||||
// <why>". A build seat from before it says neither, and that is said.
|
||||
func (r BuildResult) MadeWords() string {
|
||||
switch {
|
||||
case r.Reused != "":
|
||||
return r.Reused
|
||||
case r.BuiltWhy != "":
|
||||
return "built: " + r.BuiltWhy
|
||||
}
|
||||
return "built: the build seat did not say whether a check's build could be taken (it predates ADR 0350)"
|
||||
}
|
||||
|
||||
@@ -92,6 +92,31 @@ type BuildStart struct {
|
||||
Ref string `json:"ref,omitempty"`
|
||||
On string `json:"on"`
|
||||
At string `json:"at"`
|
||||
// Delivery is the how-manyth time the queue handed this ask out: 1 for a first start, more for one
|
||||
// an earlier taker left unfinished — most often a build agent replaced mid-build — and started
|
||||
// again from the beginning (novox/hq issue 594). 0 from a machine that predates it.
|
||||
Delivery int `json:"delivery,omitempty"`
|
||||
}
|
||||
|
||||
// Restarted is how a build handed out again says so (novox/hq issue 594), in its log and in its check's
|
||||
// report, so a second run never reads as the first. Neutral on why: a holder replaced mid-build, a paused
|
||||
// holder handing an ask back as it arrived, and a bus that held its acknowledgements past the wait all
|
||||
// count as a hand-out. Empty for a first start.
|
||||
func Restarted(delivery int) string {
|
||||
if delivery < 2 {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("restarted: handed out %d times; an earlier hand-out did not finish here, so this run "+
|
||||
"started from the beginning", delivery)
|
||||
}
|
||||
|
||||
// RunMarker is the short form of Restarted, for a line clipped to a few words — a check's status on the
|
||||
// forge (novox/hq issue 594): "(run 2)". Empty for a first start.
|
||||
func RunMarker(delivery int) string {
|
||||
if delivery < 2 {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("(run %d)", delivery)
|
||||
}
|
||||
|
||||
// BuildLine is one thing a build said while building.
|
||||
|
||||
@@ -315,7 +315,7 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
// built. A set that cannot be read is said and the ask built: a cancel is a person's
|
||||
// exception, and a holder that refused every build while its grant was missing would
|
||||
// stop the mesh building for it.
|
||||
build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat}
|
||||
build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat, delivery: deliveryOf(msg)}
|
||||
if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err)
|
||||
} else if cancelled {
|
||||
@@ -347,6 +347,18 @@ type natsBuild struct {
|
||||
// seat is the role this build was taken from; what the machine says about it is that role's.
|
||||
seat string
|
||||
seq int
|
||||
// delivery is the how-manyth time the queue handed this ask out (novox/hq issue 594); 0 when the
|
||||
// server did not say.
|
||||
delivery int
|
||||
}
|
||||
|
||||
// deliveryOf is how many times the queue has handed this message out, this time included.
|
||||
func deliveryOf(msg *nats.Msg) int {
|
||||
meta, err := msg.Metadata()
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return int(meta.NumDelivered)
|
||||
}
|
||||
|
||||
func (b *natsBuild) Request() BuildRequest { return b.request }
|
||||
@@ -367,6 +379,11 @@ func (b *natsBuild) Announce(ctx context.Context, result BuildResult) error {
|
||||
if result.Failed == "" && result.Commit != "" {
|
||||
result.Module = ModuleOf(result.Manifest)
|
||||
}
|
||||
// **Which run this is** (novox/hq issue 594): stamped by the transport, which alone knows, so every
|
||||
// outcome carries it whatever the machine's code filled in.
|
||||
if result.Delivery == 0 {
|
||||
result.Delivery = b.delivery
|
||||
}
|
||||
body, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -386,11 +403,16 @@ func (b *natsBuild) Done() error { return b.msg.Ack() }
|
||||
func (b *natsBuild) Began(ctx context.Context) error {
|
||||
body, err := json.Marshal(BuildStart{
|
||||
ID: b.request.ID, Repository: b.request.Repository, Path: b.request.Path,
|
||||
Ref: b.request.Ref, On: b.on, At: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
Ref: b.request.Ref, On: b.on, At: time.Now().UTC().Format(time.RFC3339Nano), Delivery: b.delivery,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// **A start that is a restart says so in the build's log** (novox/hq issue 594): a reader following
|
||||
// the build sees why it began again, never a silent second run.
|
||||
if said := Restarted(b.delivery); said != "" {
|
||||
b.Say("restarted", said)
|
||||
}
|
||||
publish, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
if _, err := b.js.Context().Publish(BuildStartedOf(b.seat), body, nats.Context(publish)); err != nil {
|
||||
|
||||
@@ -401,3 +401,97 @@ func TestNatsAHolderSaysItTakesWorkOnceItIsBound(t *testing.T) {
|
||||
t.Fatal("a machine that could not bind said it takes work")
|
||||
}
|
||||
}
|
||||
|
||||
// **An ask handed out again says so** (novox/hq issue 594). A build agent replaced mid-build leaves its
|
||||
// ask unsettled, and the queue hands it out again to start over from the beginning. That second start
|
||||
// must not look like a first one: the machine that takes it says it restarted — in its started event,
|
||||
// in the build's log and in its outcome — so the queue and the pull request can say it too.
|
||||
func TestNatsAnAskHandedOutAgainSaysItRestarted(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
|
||||
started, err := js.Conn().SubscribeSync(BuildStarted())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = started.Unsubscribe() }()
|
||||
lines, err := js.Conn().SubscribeSync(BuildLog("b-594"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = lines.Unsubscribe() }()
|
||||
outcomes, err := js.Conn().SubscribeSync(BuildOutcome())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = outcomes.Unsubscribe() }()
|
||||
_ = js.Conn().Flush()
|
||||
|
||||
body, _ := json.Marshal(BuildRequest{ID: "b-594", Repository: "/r"})
|
||||
if _, err := js.Context().Publish(BuildWork(), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
machine := MachineOverNATS(js, "anchor")
|
||||
defer machine.Close()
|
||||
var first sync.Once
|
||||
go func() {
|
||||
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
||||
_ = work.Began(ctx)
|
||||
handedBack := false
|
||||
// The first time it is left unfinished — an agent replaced mid-build — and comes back.
|
||||
first.Do(func() { _ = work.Hold(100 * time.Millisecond); handedBack = true })
|
||||
if handedBack {
|
||||
return
|
||||
}
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
|
||||
var starts []BuildStart
|
||||
for len(starts) < 2 {
|
||||
msg, err := started.NextMsg(10 * time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("heard %d start(s) of an ask handed out twice: %v", len(starts), err)
|
||||
}
|
||||
var s BuildStart
|
||||
if err := json.Unmarshal(msg.Data, &s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
starts = append(starts, s)
|
||||
}
|
||||
if starts[0].Delivery != 1 || starts[1].Delivery != 2 {
|
||||
t.Fatalf("the starts say deliveries %d and %d, want 1 and 2: a restart reads as a first start",
|
||||
starts[0].Delivery, starts[1].Delivery)
|
||||
}
|
||||
|
||||
msg, err := outcomes.NextMsg(10 * time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var result BuildResult
|
||||
if err := json.Unmarshal(msg.Data, &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Delivery != 2 {
|
||||
t.Fatalf("the outcome says delivery %d, want 2: the pull request cannot say the check restarted",
|
||||
result.Delivery)
|
||||
}
|
||||
|
||||
sawRestarted := false
|
||||
for {
|
||||
msg, err := lines.NextMsg(time.Second)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
var line BuildLine
|
||||
if json.Unmarshal(msg.Data, &line) == nil && line.Step == "restarted" {
|
||||
sawRestarted = true
|
||||
}
|
||||
}
|
||||
if !sawRestarted {
|
||||
t.Fatal("the build's log never said it restarted")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The builds a walk may take are read from the build seat's own outcomes of its checks (novox/hq ADR 0350 §2): only
|
||||
// checks of the walk's pull requests, only builds made, newest first.
|
||||
func TestNatsTheBuildsAWalkMayTakeAreTheChecksOutcomesOfItsPullRequests(t *testing.T) {
|
||||
js := aBus(t)
|
||||
since := time.Now().Add(-time.Minute)
|
||||
say := func(r BuildResult) {
|
||||
body, _ := json.Marshal(r)
|
||||
if _, err := js.Context().Publish(BuildOutcomeOf(TheBuildMachine), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
mine := &CheckRequest{Owner: "novox", Repo: "shop", Number: 7}
|
||||
built := CheckBuilt{Module: "app", Key: "r1:1", Made: []MadeArtifact{{Name: "server", Reference: "s/app/server@sha256:a"}}}
|
||||
say(BuildResult{ID: "build-1", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{built,
|
||||
{Module: "web", NotBuilt: "it has no reuse key"}}}})
|
||||
say(BuildResult{ID: "build-2", Checked: &CheckRequest{Owner: "novox", Repo: "shop", Number: 8},
|
||||
Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{built}}})
|
||||
say(BuildResult{ID: "build-3", Repository: "novox/shop", Commit: "c0ffee"}) // a build, not a check
|
||||
newer := built
|
||||
newer.Key = "r1:2"
|
||||
say(BuildResult{ID: "build-4", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{newer}}})
|
||||
|
||||
got, err := ReadCheckBuilds(t.Context(), js, TheBuildMachine, []PullRef{{Owner: "novox", Repo: "shop", Number: 7}}, since)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 2 || got[0].Check != "build-4" || got[0].Built.Key != "r1:2" || got[1].Check != "build-1" ||
|
||||
got[1].Pull.Number != 7 {
|
||||
t.Fatalf("read %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -290,6 +290,9 @@ type Checked struct {
|
||||
Trunk string `json:"trunk,omitempty"`
|
||||
Beside map[string]string `json:"beside,omitempty"`
|
||||
Took string `json:"took,omitempty"`
|
||||
// Built are the builds the check made for its merge, each with its reuse key and digests, or why not
|
||||
// (novox/hq ADR 0350 §1): what the walk of the merge takes instead of building.
|
||||
Built []CheckBuilt `json:"built,omitempty"`
|
||||
}
|
||||
|
||||
// CheckedMember is one head a group's composed check judged, and the trunk commit it was merged into (novox/hq
|
||||
|
||||
@@ -608,7 +608,10 @@ type Wait struct {
|
||||
Part string `json:"part"`
|
||||
Secret string `json:"secret,omitempty"`
|
||||
Setting string `json:"setting,omitempty"`
|
||||
What string `json:"what"`
|
||||
// Provision is a provision the module requires whose credential only a person can give and nobody gave yet
|
||||
// (novox/hq ADR 0337 §5): a wait names exactly one of Secret, Setting or Provision.
|
||||
Provision string `json:"provision,omitempty"`
|
||||
What string `json:"what"`
|
||||
}
|
||||
|
||||
// ResourceHealth is one long-running resource's state.
|
||||
|
||||
+63
-2
@@ -73,6 +73,9 @@ type QueuedAsk struct {
|
||||
On string `json:"on,omitempty"`
|
||||
Was string `json:"was-on,omitempty"`
|
||||
Started time.Time `json:"started,omitempty"`
|
||||
// Delivery is the how-manyth time the ask was handed out, as its latest start said: more than 1 is
|
||||
// a build started again after an earlier taker left it (novox/hq issue 594).
|
||||
Delivery int `json:"delivery,omitempty"`
|
||||
}
|
||||
|
||||
// Queue is the seat's work queue as it stands.
|
||||
@@ -162,7 +165,7 @@ func ClassifyQueue(asks []QueuedAsk, delivered uint64, ackPending int, heard Bui
|
||||
}
|
||||
a.State = AskDead
|
||||
if s, ok := current[a.ID]; ok {
|
||||
a.On, a.Started = s.On, startedAt(s)
|
||||
a.On, a.Started, a.Delivery = s.On, startedAt(s), s.Delivery
|
||||
running = append(running, i)
|
||||
continue
|
||||
}
|
||||
@@ -183,7 +186,7 @@ func ClassifyQueue(asks []QueuedAsk, delivered uint64, ackPending int, heard Bui
|
||||
for _, i := range group {
|
||||
if slots == 0 {
|
||||
// Past what the worker counts: handed out as often as it may be.
|
||||
out[i].On, out[i].Started = "", time.Time{}
|
||||
out[i].On, out[i].Started, out[i].Delivery = "", time.Time{}, 0
|
||||
continue
|
||||
}
|
||||
out[i].State = AskInFlight
|
||||
@@ -361,6 +364,64 @@ func ReadBuildEvents(ctx context.Context, js *broker.JetStream, seat string, sin
|
||||
return heard, nil
|
||||
}
|
||||
|
||||
// ReadCheckBuilds is every build the seat's checks of those pull requests said they made for their merges since a
|
||||
// moment (novox/hq ADR 0350 §2): read from the seat's own outcomes on the events stream, which only its holders may
|
||||
// publish, never from the artifact store. Newest first.
|
||||
func ReadCheckBuilds(ctx context.Context, js *broker.JetStream, seat string, pulls []PullRef, since time.Time) ([]CheckedBuild, error) {
|
||||
if len(pulls) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
wanted := map[PullRef]bool{}
|
||||
for _, p := range pulls {
|
||||
wanted[p] = true
|
||||
}
|
||||
subject := BuildOutcomeOf(seat)
|
||||
sub, err := js.Context().PullSubscribe(subject, "",
|
||||
nats.BindStream(broker.EventsStream), nats.StartTime(since), nats.AckNone())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot read %s from the bus: %w", subject, err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
info, err := sub.ConsumerInfo()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot read %s from the bus: %w", subject, err)
|
||||
}
|
||||
var out []CheckedBuild
|
||||
for remaining := info.NumPending; remaining > 0; {
|
||||
batch, err := sub.Fetch(int(min(remaining, 500)), nats.MaxWait(2*time.Second))
|
||||
if err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.DeadlineExceeded) {
|
||||
return nil, fmt.Errorf("reading %s: %w", subject, err)
|
||||
}
|
||||
if len(batch) == 0 {
|
||||
break
|
||||
}
|
||||
remaining -= uint64(len(batch))
|
||||
for _, msg := range batch {
|
||||
var r BuildResult
|
||||
if json.Unmarshal(msg.Data, &r) != nil || r.Checked == nil || r.Check == nil || r.Failed != "" {
|
||||
continue
|
||||
}
|
||||
pull := PullRef{Owner: r.Checked.Owner, Repo: r.Checked.Repo, Number: r.Checked.Number}
|
||||
if !wanted[pull] {
|
||||
continue
|
||||
}
|
||||
for _, b := range r.Check.Built {
|
||||
if b.NotBuilt == "" && b.Key != "" {
|
||||
out = append(out, CheckedBuild{Pull: pull, Check: r.ID, Built: b})
|
||||
}
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
// Newest first: the stream's order is the order they were said.
|
||||
for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 {
|
||||
out[i], out[j] = out[j], out[i]
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// --- the cancelled set ----------------------------------------------------------------------
|
||||
|
||||
// safeKey is an id that can be a key, and a subject token, as it is: a build id, never a pattern.
|
||||
|
||||
+56
-11
@@ -80,6 +80,57 @@ func Check(text string, machines ...string) (Refusal, bool) {
|
||||
return Refusal{"address", "a hardware address"}, false
|
||||
case reIPv6.MatchString(text):
|
||||
return Refusal{"address", "an IPv6 address"}, false
|
||||
}
|
||||
if refusal, ok := secretShape(text); !ok {
|
||||
return refusal, false
|
||||
}
|
||||
if reWinPath.MatchString(text) {
|
||||
return Refusal{"path", "a drive path"}, false
|
||||
}
|
||||
if refusal, ok := randomRun(text); !ok {
|
||||
return refusal, false
|
||||
}
|
||||
for _, word := range strings.FieldsFunc(text, isSeparator) {
|
||||
w := strings.TrimRight(word, ".:!?")
|
||||
if isPath(w) {
|
||||
return Refusal{"path", "a file path"}, false
|
||||
}
|
||||
if isHostName(w) {
|
||||
return Refusal{"address", "a host name"}, false
|
||||
}
|
||||
}
|
||||
return Refusal{}, true
|
||||
}
|
||||
|
||||
// Secret says whether a text carries a secret's shape, the one class that leaves the mesh nowhere — the
|
||||
// messenger's CheckSecret, one for one. It is what an ask's whole words (asks.Ask.Whole, novox/hq issue 383)
|
||||
// are held to: on a channel that proves who answers, a path or an address is what the operator approves and
|
||||
// is shown; a secret never is. A path is read as one: a run with a slash in it is judged piece by piece
|
||||
// between the slashes, so "/mnt/Photos_2024/Jochen" is a path and not a random string (the review of
|
||||
// 2026-10-10); the named shapes hold whatever the run holds.
|
||||
func Secret(text string, machines ...string) (Refusal, bool) {
|
||||
text = withoutMachines(text, machines)
|
||||
if refusal, ok := secretShape(text); !ok {
|
||||
return refusal, false
|
||||
}
|
||||
return randomRunOutsidePaths(text)
|
||||
}
|
||||
|
||||
// randomRunOutsidePaths is randomRun with each run that holds a slash judged by its pieces between the slashes.
|
||||
func randomRunOutsidePaths(text string) (Refusal, bool) {
|
||||
for _, run := range reRun.FindAllString(text, -1) {
|
||||
for _, piece := range strings.Split(run, "/") {
|
||||
if len(piece) >= 20 && looksRandom(piece) {
|
||||
return Refusal{"secret", "a long random-looking string"}, false
|
||||
}
|
||||
}
|
||||
}
|
||||
return Refusal{}, true
|
||||
}
|
||||
|
||||
// secretShape is the secret shapes a pattern names.
|
||||
func secretShape(text string) (Refusal, bool) {
|
||||
switch {
|
||||
case rePEM.MatchString(text):
|
||||
return Refusal{"secret", "a key block"}, false
|
||||
case reJWT.MatchString(text):
|
||||
@@ -92,23 +143,17 @@ func Check(text string, machines ...string) (Refusal, bool) {
|
||||
return Refusal{"secret", "a value given to a secret's name"}, false
|
||||
case reHex.MatchString(text):
|
||||
return Refusal{"secret", "a long hexadecimal string"}, false
|
||||
case reWinPath.MatchString(text):
|
||||
return Refusal{"path", "a drive path"}, false
|
||||
}
|
||||
return Refusal{}, true
|
||||
}
|
||||
|
||||
// randomRun is a long unbroken run of characters spread as a random string's are.
|
||||
func randomRun(text string) (Refusal, bool) {
|
||||
for _, run := range reRun.FindAllString(text, -1) {
|
||||
if looksRandom(run) {
|
||||
return Refusal{"secret", "a long random-looking string"}, false
|
||||
}
|
||||
}
|
||||
for _, word := range strings.FieldsFunc(text, isSeparator) {
|
||||
w := strings.TrimRight(word, ".:!?")
|
||||
if isPath(w) {
|
||||
return Refusal{"path", "a file path"}, false
|
||||
}
|
||||
if isHostName(w) {
|
||||
return Refusal{"address", "a host name"}, false
|
||||
}
|
||||
}
|
||||
return Refusal{}, true
|
||||
}
|
||||
|
||||
|
||||
@@ -86,3 +86,40 @@ func TestScrubAlwaysGivesWhatMayLeave(t *testing.T) {
|
||||
t.Errorf("a text that passes was changed: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Secret is the one class that leaves the mesh nowhere (novox/hq issue 383): it refuses every secret's shape
|
||||
// Check refuses, and nothing Check refuses as an address or a path — those an ask's whole words may carry.
|
||||
func TestSecretRefusesOnlyASecretsShape(t *testing.T) {
|
||||
for _, text := range []string{
|
||||
"-----BEGIN RSA PRIVATE KEY-----",
|
||||
"eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.abc",
|
||||
"123456789:ABCdefGHIjklMNOpqrSTUvwxYZ0123456789ab",
|
||||
"ghp_abcdefghijklmnopqrstuvwxyz0123456789",
|
||||
"password=hunter2",
|
||||
"0123456789abcdef0123456789abcdef",
|
||||
"a key xK9mQ2vL8pR4tW7yB3nF6hJ1dG5sA0zC",
|
||||
} {
|
||||
r, ok := Secret(text)
|
||||
if ok || r.Class != "secret" {
|
||||
t.Errorf("not refused as a secret: %q (%s)", text, r)
|
||||
}
|
||||
}
|
||||
for _, text := range []string{
|
||||
"recalbox=smb://nas.lan/recalbox@/mnt/recalbox:ro",
|
||||
"library=/mnt/library",
|
||||
"photos=/mnt/Photos_2024/Jochen",
|
||||
"games=smb://nas.lan/Recalbox_Games2024",
|
||||
"/srv/media/Series_Archive/Season01",
|
||||
"10.77.0.9:53",
|
||||
"jochen@example.com",
|
||||
"C:\\Users\\jo",
|
||||
"anchor and the laptop",
|
||||
} {
|
||||
if r, ok := Secret(text, "anchor"); !ok {
|
||||
t.Errorf("refused as %s: %q", r, text)
|
||||
}
|
||||
if _, ok := Check(text, "anchor"); ok && text != "anchor and the laptop" {
|
||||
t.Errorf("Check lets %q leave, so Secret is not the narrower rule", text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user